mirror of
https://github.com/ruby/ruby.git
synced 2026-10-28 16:36:20 +08:00
Hiroshi SHIBATAandClaude Opus 5
84a30ecc87
The parser calls back into Ruby for every event, so a handler can call #parse again on the same object. That reinitialised the parser and re-pointed it at new input while the outer loop was still driving it, which left the outer loop reading the inner document's freed buffer, or resuming on a parser with no input at all and tripping libyaml's read_handler assertion. Psych::Exception is used so the caller can rescue it, and the in-use flag is cleared with rb_ensure so a parser stays reusable afterwards. https://github.com/ruby/psych/commit/5f2e37dc1c Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
591 lines
17 KiB
Ruby
591 lines
17 KiB
Ruby
# coding: utf-8
|
|
# frozen_string_literal: true
|
|
|
|
require_relative 'helper'
|
|
|
|
module Psych
|
|
class TestParser < TestCase
|
|
class EventCatcher < Handler
|
|
attr_accessor :parser
|
|
attr_reader :calls, :marks
|
|
def initialize
|
|
@parser = nil
|
|
@calls = []
|
|
@marks = []
|
|
end
|
|
|
|
(Handler.instance_methods(true) -
|
|
Object.instance_methods).each do |m|
|
|
class_eval <<~RUBY, __FILE__, __LINE__ + 1
|
|
def #{m} *args
|
|
super
|
|
@marks << @parser.mark if @parser
|
|
@calls << [:#{m}, args]
|
|
end
|
|
RUBY
|
|
end
|
|
end
|
|
|
|
# Calls Parser#parse again, once, from inside a callback of the parse it is
|
|
# already handling.
|
|
class ReentrantHandler < Handler
|
|
attr_accessor :parser, :inner_yaml
|
|
attr_reader :inner_error, :scalars, :empty_calls
|
|
|
|
def initialize
|
|
@parser = nil
|
|
@inner_yaml = nil
|
|
@inner_error = nil
|
|
@scalars = []
|
|
@empty_calls = 0
|
|
end
|
|
|
|
def empty
|
|
@empty_calls += 1
|
|
raise "handler#empty keeps being called, the parse loop is not terminating" if @empty_calls > 1000
|
|
end
|
|
|
|
def scalar value, anchor, tag, plain, quoted, style
|
|
@scalars << value
|
|
|
|
inner, @inner_yaml = @inner_yaml, nil
|
|
return unless inner
|
|
|
|
begin
|
|
@parser.parse inner
|
|
rescue => e
|
|
@inner_error = e
|
|
end
|
|
end
|
|
end
|
|
|
|
def setup
|
|
super
|
|
@handler = EventCatcher.new
|
|
@parser = Psych::Parser.new @handler
|
|
@handler.parser = @parser
|
|
end
|
|
|
|
def test_ast_roundtrip
|
|
parser = Psych.parser
|
|
parser.parse('null')
|
|
ast = parser.handler.root
|
|
assert_match(/^null/, ast.yaml)
|
|
end
|
|
|
|
def test_exception_memory_leak
|
|
yaml = <<-eoyaml
|
|
%YAML 1.1
|
|
%TAG ! tag:tenderlovemaking.com,2009:
|
|
--- &ponies
|
|
- first element
|
|
- *ponies
|
|
- foo: bar
|
|
...
|
|
eoyaml
|
|
|
|
[:start_stream, :start_document, :end_document, :alias, :scalar,
|
|
:start_sequence, :end_sequence, :start_mapping, :end_mapping,
|
|
:end_stream].each do |method|
|
|
|
|
klass = Class.new(Psych::Handler) do
|
|
define_method(method) do |*args|
|
|
raise
|
|
end
|
|
end
|
|
|
|
parser = Psych::Parser.new klass.new
|
|
2.times {
|
|
assert_raise(RuntimeError, method.to_s) do
|
|
parser.parse yaml
|
|
end
|
|
}
|
|
end
|
|
end
|
|
|
|
def test_event_location_exception_is_propagated
|
|
klass = Class.new(Psych::Handler) do
|
|
def event_location start_line, start_column, end_line, end_column
|
|
raise "from event_location"
|
|
end
|
|
end
|
|
|
|
parser = Psych::Parser.new klass.new
|
|
2.times do
|
|
ex = assert_raise(RuntimeError) { parser.parse "--- hello\n" }
|
|
assert_equal "from event_location", ex.message
|
|
end
|
|
end
|
|
|
|
def test_parse_is_not_reentrant
|
|
pend "Failing on JRuby" if RUBY_PLATFORM =~ /java/
|
|
|
|
handler = ReentrantHandler.new
|
|
handler.inner_yaml = "--- inner\n"
|
|
parser = Psych::Parser.new handler
|
|
handler.parser = parser
|
|
|
|
parser.parse "--- outer\n"
|
|
|
|
assert_kind_of Psych::Exception, handler.inner_error
|
|
assert_equal ['outer'], handler.scalars
|
|
assert_equal 0, handler.empty_calls
|
|
|
|
# The in-use flag is cleared when the parse finishes, so the same parser
|
|
# can be used again afterwards.
|
|
handler.scalars.clear
|
|
parser.parse "--- second\n"
|
|
assert_equal ['second'], handler.scalars
|
|
end
|
|
|
|
def test_parse_is_not_reentrant_with_invalid_inner_document
|
|
pend "Failing on JRuby" if RUBY_PLATFORM =~ /java/
|
|
|
|
handler = ReentrantHandler.new
|
|
handler.inner_yaml = "--- \x00bad\n"
|
|
parser = Psych::Parser.new handler
|
|
handler.parser = parser
|
|
|
|
parser.parse "--- outer\n"
|
|
|
|
assert_kind_of Psych::Exception, handler.inner_error
|
|
assert_equal ['outer'], handler.scalars
|
|
assert_equal 0, handler.empty_calls
|
|
end
|
|
|
|
def test_multiparse
|
|
3.times do
|
|
@parser.parse '--- foo'
|
|
end
|
|
end
|
|
|
|
def test_filename
|
|
ex = assert_raise(Psych::SyntaxError) do
|
|
@parser.parse '--- `', 'omg!'
|
|
end
|
|
assert_match 'omg!', ex.message
|
|
end
|
|
|
|
def test_line_numbers
|
|
omit 'libfyaml reports event marks differently from libyaml' if libfyaml?
|
|
assert_equal 0, @parser.mark.line
|
|
pend "Failing on JRuby" if RUBY_PLATFORM =~ /java/
|
|
|
|
@parser.parse "---\n- hello\n- world"
|
|
line_calls = @handler.marks.map(&:line).zip(@handler.calls.map(&:first))
|
|
assert_equal [
|
|
[0, :event_location],
|
|
[0, :start_stream],
|
|
[0, :event_location],
|
|
[0, :start_document],
|
|
[1, :event_location],
|
|
[1, :start_sequence],
|
|
[2, :event_location],
|
|
[2, :scalar],
|
|
[3, :event_location],
|
|
[3, :scalar],
|
|
[3, :event_location],
|
|
[3, :end_sequence],
|
|
[3, :event_location],
|
|
[3, :end_document],
|
|
[3, :event_location],
|
|
[3, :end_stream]], line_calls
|
|
|
|
assert_equal 3, @parser.mark.line
|
|
end
|
|
|
|
def test_column_numbers
|
|
omit 'libfyaml reports event marks differently from libyaml' if libfyaml?
|
|
assert_equal 0, @parser.mark.column
|
|
pend "Failing on JRuby" if RUBY_PLATFORM =~ /java/
|
|
|
|
@parser.parse "---\n- hello\n- world"
|
|
col_calls = @handler.marks.map(&:column).zip(@handler.calls.map(&:first))
|
|
assert_equal [
|
|
[0, :event_location],
|
|
[0, :start_stream],
|
|
[3, :event_location],
|
|
[3, :start_document],
|
|
[1, :event_location],
|
|
[1, :start_sequence],
|
|
[0, :event_location],
|
|
[0, :scalar],
|
|
[0, :event_location],
|
|
[0, :scalar],
|
|
[0, :event_location],
|
|
[0, :end_sequence],
|
|
[0, :event_location],
|
|
[0, :end_document],
|
|
[0, :event_location],
|
|
[0, :end_stream]], col_calls
|
|
|
|
assert_equal 0, @parser.mark.column
|
|
end
|
|
|
|
def test_index_numbers
|
|
omit 'libfyaml reports event marks differently from libyaml' if libfyaml?
|
|
assert_equal 0, @parser.mark.index
|
|
pend "Failing on JRuby" if RUBY_PLATFORM =~ /java/
|
|
|
|
@parser.parse "---\n- hello\n- world"
|
|
idx_calls = @handler.marks.map(&:index).zip(@handler.calls.map(&:first))
|
|
assert_equal [
|
|
[0, :event_location],
|
|
[0, :start_stream],
|
|
[3, :event_location],
|
|
[3, :start_document],
|
|
[5, :event_location],
|
|
[5, :start_sequence],
|
|
[12, :event_location],
|
|
[12, :scalar],
|
|
[19, :event_location],
|
|
[19, :scalar],
|
|
[19, :event_location],
|
|
[19, :end_sequence],
|
|
[19, :event_location],
|
|
[19, :end_document],
|
|
[19, :event_location],
|
|
[19, :end_stream]], idx_calls
|
|
|
|
assert_equal 19, @parser.mark.index
|
|
end
|
|
|
|
def test_bom
|
|
tadpole = 'おたまじゃくし'
|
|
|
|
# BOM + text
|
|
yml = "\uFEFF#{tadpole}".encode('UTF-16LE')
|
|
@parser.parse yml
|
|
assert_equal tadpole, @parser.handler.calls.find { |method, args| method == :scalar }[1].first
|
|
end
|
|
|
|
# BOM + multi-line mapping used to lose every line after the first one
|
|
# https://github.com/ruby/psych/issues/331
|
|
def test_bom_multiline_utf8
|
|
@parser.parse "\uFEFFa: b\nc: d\n"
|
|
assert_equal %w[a b c d], scalars(@parser.handler)
|
|
end
|
|
|
|
def test_bom_multiline_utf16
|
|
%w[UTF-16LE UTF-16BE].each do |enc|
|
|
handler = EventCatcher.new
|
|
Psych::Parser.new(handler).parse "\uFEFFa: b\nc: d\n".encode(enc)
|
|
assert_equal %w[a b c d], scalars(handler), enc
|
|
end
|
|
end
|
|
|
|
def test_bom_multiline_utf32
|
|
%w[UTF-32LE UTF-32BE].each do |enc|
|
|
handler = EventCatcher.new
|
|
Psych::Parser.new(handler).parse "\uFEFFa: b\nc: d\n".encode(enc)
|
|
assert_equal %w[a b c d], scalars(handler), enc
|
|
end
|
|
end
|
|
|
|
def test_bom_multiline_io
|
|
@parser.parse StringIO.new("\uFEFFa: b\nc: d\n")
|
|
assert_equal %w[a b c d], scalars(@parser.handler)
|
|
end
|
|
|
|
def test_bom_only
|
|
@parser.parse "\uFEFF"
|
|
assert_equal [], scalars(@parser.handler)
|
|
end
|
|
|
|
def test_io_without_bom_is_not_modified
|
|
io = StringIO.new "a: b\nc: d\n".freeze
|
|
@parser.parse io
|
|
assert_equal %w[a b c d], scalars(@parser.handler)
|
|
end
|
|
|
|
def test_external_encoding
|
|
tadpole = 'おたまじゃくし'
|
|
|
|
@parser.external_encoding = Psych::Parser::UTF16LE
|
|
@parser.parse tadpole.encode 'UTF-16LE'
|
|
assert_equal tadpole, @parser.handler.calls.find { |method, args| method == :scalar }[1].first
|
|
end
|
|
|
|
def test_bogus_io
|
|
o = Object.new
|
|
def o.external_encoding; nil end
|
|
def o.read len; self end
|
|
|
|
assert_raise(TypeError) do
|
|
@parser.parse o
|
|
end
|
|
end
|
|
|
|
def test_parse_io
|
|
@parser.parse StringIO.new("--- a")
|
|
assert_called :start_stream
|
|
assert_called :scalar
|
|
assert_called :end_stream
|
|
end
|
|
|
|
def test_parse_io_returns_more_bytes_than_requested
|
|
# An IO-like source whose #read returns more bytes than the size it was
|
|
# asked for must not overflow libyaml's read buffer.
|
|
io = Object.new
|
|
def io.external_encoding; Encoding::UTF_8 end
|
|
def io.read len
|
|
return nil if @done
|
|
@done = true
|
|
"--- a\n" + ("#" * (len + (1 << 20)))
|
|
end
|
|
|
|
# CRuby clamps the over-read and parses; JRuby's parser rejects the
|
|
# over-reading IO with an IOError. Either way there is no overflow.
|
|
begin
|
|
@parser.parse io
|
|
rescue IOError
|
|
return
|
|
end
|
|
assert_called :start_stream
|
|
assert_called :scalar
|
|
assert_called :end_stream
|
|
end
|
|
|
|
def test_parse_io_returns_more_bytes_than_requested_multibyte
|
|
# The over-read is rounded down to a character boundary so a multibyte
|
|
# character is never split when the copy is clamped.
|
|
io = Object.new
|
|
def io.external_encoding; Encoding::UTF_8 end
|
|
def io.read len
|
|
return nil if @done
|
|
@done = true
|
|
"--- a\n#" + ("あ" * (len + (1 << 20)))
|
|
end
|
|
|
|
begin
|
|
@parser.parse io
|
|
rescue IOError
|
|
return
|
|
end
|
|
assert_called :scalar
|
|
end
|
|
|
|
def test_syntax_error
|
|
assert_raise(Psych::SyntaxError) do
|
|
@parser.parse("---\n\"foo\"\n\"bar\"\n")
|
|
end
|
|
end
|
|
|
|
def test_syntax_error_twice
|
|
assert_raise(Psych::SyntaxError) do
|
|
@parser.parse("---\n\"foo\"\n\"bar\"\n")
|
|
end
|
|
|
|
assert_raise(Psych::SyntaxError) do
|
|
@parser.parse("---\n\"foo\"\n\"bar\"\n")
|
|
end
|
|
end
|
|
|
|
def test_syntax_error_has_path_for_string
|
|
e = assert_raise(Psych::SyntaxError) do
|
|
@parser.parse("---\n\"foo\"\n\"bar\"\n")
|
|
end
|
|
assert_match '(<unknown>):', e.message
|
|
end
|
|
|
|
def test_syntax_error_has_path_for_io
|
|
io = StringIO.new "---\n\"foo\"\n\"bar\"\n"
|
|
def io.path; "hello!"; end
|
|
|
|
e = assert_raise(Psych::SyntaxError) do
|
|
@parser.parse(io)
|
|
end
|
|
assert_match "(#{io.path}):", e.message
|
|
end
|
|
|
|
def test_mapping_end
|
|
@parser.parse("---\n!!map { key: value }")
|
|
assert_called :end_mapping
|
|
end
|
|
|
|
def test_mapping_tag
|
|
@parser.parse("---\n!!map { key: value }")
|
|
assert_called :start_mapping, ["tag:yaml.org,2002:map", false, Nodes::Mapping::FLOW]
|
|
end
|
|
|
|
def test_mapping_anchor
|
|
@parser.parse("---\n&A { key: value }")
|
|
assert_called :start_mapping, ['A', true, Nodes::Mapping::FLOW]
|
|
end
|
|
|
|
def test_mapping_block
|
|
@parser.parse("---\n key: value")
|
|
assert_called :start_mapping, [true, Nodes::Mapping::BLOCK]
|
|
end
|
|
|
|
def test_mapping_start
|
|
@parser.parse("---\n{ key: value }")
|
|
assert_called :start_mapping
|
|
assert_called :start_mapping, [true, Nodes::Mapping::FLOW]
|
|
end
|
|
|
|
def test_sequence_end
|
|
@parser.parse("---\n&A [1, 2]")
|
|
assert_called :end_sequence
|
|
end
|
|
|
|
def test_sequence_start_anchor
|
|
@parser.parse("---\n&A [1, 2]")
|
|
assert_called :start_sequence, ["A", true, Nodes::Sequence::FLOW]
|
|
end
|
|
|
|
def test_sequence_start_tag
|
|
@parser.parse("---\n!!seq [1, 2]")
|
|
assert_called :start_sequence, ["tag:yaml.org,2002:seq", false, Nodes::Sequence::FLOW]
|
|
end
|
|
|
|
def test_sequence_start_flow
|
|
@parser.parse("---\n[1, 2]")
|
|
assert_called :start_sequence, [true, Nodes::Sequence::FLOW]
|
|
end
|
|
|
|
def test_sequence_start_block
|
|
@parser.parse("---\n - 1\n - 2")
|
|
assert_called :start_sequence, [true, Nodes::Sequence::BLOCK]
|
|
end
|
|
|
|
def test_literal_scalar
|
|
@parser.parse(<<-eoyml)
|
|
%YAML 1.1
|
|
---
|
|
"literal\n\
|
|
\ttext\n"
|
|
eoyml
|
|
assert_called :scalar, ['literal text ', false, true, Nodes::Scalar::DOUBLE_QUOTED]
|
|
end
|
|
|
|
def test_scalar
|
|
@parser.parse("--- foo\n")
|
|
assert_called :scalar, ['foo', true, false, Nodes::Scalar::PLAIN]
|
|
end
|
|
|
|
def test_scalar_with_tag
|
|
@parser.parse("---\n!!str foo\n")
|
|
assert_called :scalar, ['foo', 'tag:yaml.org,2002:str', false, false, Nodes::Scalar::PLAIN]
|
|
end
|
|
|
|
def test_scalar_with_anchor
|
|
@parser.parse("---\n&A foo\n")
|
|
assert_called :scalar, ['foo', 'A', true, false, Nodes::Scalar::PLAIN]
|
|
end
|
|
|
|
def test_scalar_plain_implicit
|
|
@parser.parse("---\n&A foo\n")
|
|
assert_called :scalar, ['foo', 'A', true, false, Nodes::Scalar::PLAIN]
|
|
end
|
|
|
|
def test_alias
|
|
@parser.parse(<<-eoyml)
|
|
%YAML 1.1
|
|
---
|
|
!!seq [
|
|
!!str "Without properties",
|
|
&A !!str "Anchored",
|
|
!!str "Tagged",
|
|
*A,
|
|
!!str "",
|
|
]
|
|
eoyml
|
|
assert_called :alias, ['A']
|
|
end
|
|
|
|
def test_end_stream
|
|
@parser.parse("--- foo\n")
|
|
assert_called :end_stream
|
|
end
|
|
|
|
def test_start_stream
|
|
@parser.parse("--- foo\n")
|
|
assert_called :start_stream
|
|
end
|
|
|
|
def test_end_document_implicit
|
|
@parser.parse("\"foo\"\n")
|
|
assert_called :end_document, [true]
|
|
end
|
|
|
|
def test_end_document_explicit
|
|
@parser.parse("\"foo\"\n...")
|
|
assert_called :end_document, [false]
|
|
end
|
|
|
|
def test_start_document_version
|
|
@parser.parse("%YAML 1.1\n---\n\"foo\"\n")
|
|
assert_called :start_document, [[1,1], [], false]
|
|
end
|
|
|
|
def test_start_document_tag
|
|
@parser.parse("%TAG !yaml! tag:yaml.org,2002\n---\n!yaml!str \"foo\"\n")
|
|
assert_called :start_document, [[], [['!yaml!', 'tag:yaml.org,2002']], false]
|
|
end
|
|
|
|
def test_event_location
|
|
pend "Failing on JRuby" if RUBY_PLATFORM =~ /java/
|
|
|
|
@parser.parse "foo:\n" \
|
|
" barbaz: [1, 2]"
|
|
|
|
events = @handler.calls.each_slice(2).map do |location, event|
|
|
[event[0], location[1]]
|
|
end
|
|
|
|
assert_equal [
|
|
[:start_stream, [0, 0, 0, 0]],
|
|
[:start_document, [0, 0, 0, 0]],
|
|
[:start_mapping, [0, 0, 0, 0]],
|
|
[:scalar, [0, 0, 0, 3]],
|
|
[:start_mapping, [1, 2, 1, 2]],
|
|
[:scalar, [1, 2, 1, 8]],
|
|
[:start_sequence, [1, 10, 1, 11]],
|
|
[:scalar, [1, 11, 1, 12]],
|
|
[:scalar, [1, 14, 1, 15]],
|
|
[:end_sequence, [1, 15, 1, 16]],
|
|
[:end_mapping, [2, 0, 2, 0]],
|
|
[:end_mapping, [2, 0, 2, 0]],
|
|
[:end_document, [2, 0, 2, 0]],
|
|
[:end_stream, [2, 0, 2, 0]]], events
|
|
end
|
|
|
|
if Psych::Parser.method_defined?(:code_point_limit)
|
|
def test_code_point_limit
|
|
yaml = "foo: bar\n" * 500_000
|
|
assert_raise(org.snakeyaml.engine.v2.exceptions.YamlEngineException) do
|
|
Psych.load(yaml)
|
|
end
|
|
|
|
assert_nothing_raised do
|
|
begin
|
|
old_code_point_limit, Psych::Parser.code_point_limit = Psych::Parser::code_point_limit, 5_000_000
|
|
|
|
Psych.load(yaml)
|
|
ensure
|
|
Psych::Parser.code_point_limit = old_code_point_limit
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
def scalars handler
|
|
handler.calls.select { |method, _| method == :scalar }.map { |_, args| args.first }
|
|
end
|
|
|
|
def assert_called call, with = nil, parser = @parser
|
|
if with
|
|
call = parser.handler.calls.find { |x|
|
|
x.first == call && x.last.compact == with
|
|
}
|
|
assert(call,
|
|
"#{[call,with].inspect} not in #{parser.handler.calls.inspect}"
|
|
)
|
|
else
|
|
assert parser.handler.calls.any? { |x| x.first == call }
|
|
end
|
|
end
|
|
end
|
|
end
|