From be5995286d2de27e6242cdacb3a36f4e2e12a378 Mon Sep 17 00:00:00 2001
From: Hiroshi SHIBATA <hsbt@ruby-lang.org>
Date: Fri, 12 Jun 2026 10:24:26 +0900
Subject: [PATCH] [ruby/psych] Strip a leading byte order mark before parsing

libyaml only discounts the BOM when it detects the stream encoding by
itself. Psych passes the encoding explicitly whenever it is known, and
on that path libyaml counts the BOM as a first-line character, shifting
every token on the first line one column right and silently terminating
a block mapping at the second line.

https://github.com/ruby/psych/issues/331

https://github.com/ruby/psych/commit/924c15ed1f

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 ext/psych/lib/psych/parser.rb | 36 ++++++++++++++++++++++++++++++++++-
 test/psych/test_parser.rb     | 35 ++++++++++++++++++++++++++++++++++
 test/psych/test_psych.rb      | 10 ++++++++++
 3 files changed, 80 insertions(+), 1 deletion(-)

diff --git a/ext/psych/lib/psych/parser.rb b/ext/psych/lib/psych/parser.rb
index 2181c730e59..43bf9938e2b 100644
--- a/ext/psych/lib/psych/parser.rb
+++ b/ext/psych/lib/psych/parser.rb
@@ -59,7 +59,41 @@ module Psych
     # See Psych::Parser and Psych::Parser#handler
 
     def parse yaml, path = yaml.respond_to?(:path) ? yaml.path : "<unknown>"
-      _native_parse @handler, yaml, path
+      _native_parse @handler, strip_bom(yaml), path
+    end
+
+    private
+
+    BOM = {
+      Encoding::UTF_8 => "\u{FEFF}".freeze,
+      Encoding::UTF_16LE => "\u{FEFF}".encode(Encoding::UTF_16LE).freeze,
+      Encoding::UTF_16BE => "\u{FEFF}".encode(Encoding::UTF_16BE).freeze,
+    }.freeze
+    private_constant :BOM
+
+    # libyaml only skips a leading byte order mark when it detects the stream
+    # encoding by itself.  Psych passes the encoding explicitly whenever it is
+    # known, and on that path libyaml counts the BOM as a first-line character,
+    # which shifts the column of every token on the first line and silently
+    # terminates a block mapping at the second line [Bug #13615].
+    def strip_bom yaml
+      if String === yaml
+        bom = BOM[yaml.encoding]
+        return yaml[1..-1] if bom && yaml.start_with?(bom)
+      elsif yaml.respond_to?(:read) && yaml.respond_to?(:external_encoding) &&
+            yaml.respond_to?(:pos) && yaml.respond_to?(:seek)
+        bom = BOM[yaml.external_encoding]
+        skip_io_bom yaml, bom.b if bom
+      end
+      yaml
+    end
+
+    def skip_io_bom io, bom
+      pos = io.pos
+      head = io.read(bom.bytesize)
+      io.seek(pos, IO::SEEK_SET) if head && head.b != bom
+    rescue SystemCallError, IOError
+      # Not seekable; pos raises before anything is consumed.
     end
   end
 end
diff --git a/test/psych/test_parser.rb b/test/psych/test_parser.rb
index 4ca4d63d80c..de8c4fa0aea 100644
--- a/test/psych/test_parser.rb
+++ b/test/psych/test_parser.rb
@@ -173,6 +173,37 @@ module Psych
       assert_equal tadpole, @parser.handler.calls.find { |method, args| method == :scalar }[1].first
     end
 
+    # BOM + multi-line mapping used to lose every line after the first one
+    # https://github.com/ruby/psych/issues/331
+    def test_bom_multiline_utf8
+      @parser.parse "\uFEFFa: b\nc: d\n"
+      assert_equal %w[a b c d], scalars(@parser.handler)
+    end
+
+    def test_bom_multiline_utf16
+      %w[UTF-16LE UTF-16BE].each do |enc|
+        handler = EventCatcher.new
+        Psych::Parser.new(handler).parse "\uFEFFa: b\nc: d\n".encode(enc)
+        assert_equal %w[a b c d], scalars(handler), enc
+      end
+    end
+
+    def test_bom_multiline_io
+      @parser.parse StringIO.new("\uFEFFa: b\nc: d\n")
+      assert_equal %w[a b c d], scalars(@parser.handler)
+    end
+
+    def test_bom_only
+      @parser.parse "\uFEFF"
+      assert_equal [], scalars(@parser.handler)
+    end
+
+    def test_io_without_bom_is_not_modified
+      io = StringIO.new "a: b\nc: d\n".freeze
+      @parser.parse io
+      assert_equal %w[a b c d], scalars(@parser.handler)
+    end
+
     def test_external_encoding
       tadpole = 'おたまじゃくし'
 
@@ -445,6 +476,10 @@ module Psych
       end
     end
 
+    def scalars handler
+      handler.calls.select { |method, _| method == :scalar }.map { |_, args| args.first }
+    end
+
     def assert_called call, with = nil, parser = @parser
       if with
         call = parser.handler.calls.find { |x|
diff --git a/test/psych/test_psych.rb b/test/psych/test_psych.rb
index 4455c471e71..8e5ec9419e4 100644
--- a/test/psych/test_psych.rb
+++ b/test/psych/test_psych.rb
@@ -158,6 +158,16 @@ class TestPsych < Psych::TestCase
     assert_equal(%w[foo bar], docs.children.map(&:transform))
   end
 
+  # https://github.com/ruby/psych/issues/331
+  def test_load_with_leading_bom
+    assert_equal({ "a" => "b", "c" => "d" }, Psych.load("\uFEFFa: b\nc: d"))
+  end
+
+  def test_parse_stream_with_leading_bom
+    docs = Psych.parse_stream("\uFEFFa: b\nc: d")
+    assert_equal [{ "a" => "b", "c" => "d" }], docs.children.map(&:to_ruby)
+  end
+
   def test_parse_stream_with_block
     docs = []
     Psych.parse_stream("--- foo\n...\n--- bar\n...") do |node|