mirror of
https://github.com/ruby/ruby.git
synced 2026-10-28 16:36:20 +08:00
Koichi SasadaandClaude Opus 5
6ca4a12032
UnicodeNormalize.normalize and .normalized? read three constants that are
not shareable, so any call from a non-main Ractor raised
Ractor::IsolationError:
* NF_HASH_C / NF_HASH_D, the memoization caches. They cannot be made
shareable because a shareable Hash cannot be written to. Keep them per
Ractor instead: normalization is deterministic, so recomputing an entry
in another Ractor always gives the same string, and the cache is only a
cache. The DoS bound of MAX_HASH_LENGTH now applies per Ractor.
* UNICODE_ENCODINGS, which is splatted in a `when` clause on the path for
UTF-16, UTF-32, GB18030 and UCS-2/4BE. Freeze it.
The other constants are already shareable on master: tables.rb is
frozen_string_literal with frozen tables, and Regexp instances are frozen
since cb03bf43eb.
Keeping the caches keeps main Ractor performance: dropping them instead
costs 2.7x to 11x on real text, and asking the main Ractor for an entry
costs 19.4us per round trip against 3.8us to just compute it. The per
Ractor lookup adds about 60ns per call; normalized? fetches its cache
lazily so a string without a match does not pay it.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
258 lines
10 KiB
Ruby
258 lines
10 KiB
Ruby
# coding: utf-8
|
|
# frozen_string_literal: false
|
|
|
|
# Copyright Ayumu Nojima (野島 歩) and Martin J. Dürst (duerst@it.aoyama.ac.jp)
|
|
|
|
require 'test/unit'
|
|
require 'unicode_normalize/normalize'
|
|
|
|
class TestUnicodeNormalize < Test::Unit::TestCase
|
|
|
|
UNICODE_VERSION = RbConfig::CONFIG['UNICODE_VERSION']
|
|
path = File.expand_path("../enc/unicode/data/#{UNICODE_VERSION}", __dir__)
|
|
UNICODE_DATA_PATH = File.directory?("#{path}/ucd") ? "#{path}/ucd" : path
|
|
|
|
def self.expand_filename(basename)
|
|
File.expand_path("#{basename}.txt", UNICODE_DATA_PATH)
|
|
end
|
|
end
|
|
|
|
%w[NormalizationTest].all? {|f|
|
|
File.exist?(TestUnicodeNormalize.expand_filename(f))
|
|
} and
|
|
class TestUnicodeNormalize
|
|
NormTest = Struct.new :source, :NFC, :NFD, :NFKC, :NFKD, :line
|
|
|
|
def self.read_tests
|
|
lines = File.readlines(expand_filename('NormalizationTest'), encoding: 'utf-8')
|
|
firstline = lines.shift
|
|
define_method "test_0_normalizationtest_firstline" do
|
|
assert_include(firstline, "NormalizationTest-#{UNICODE_VERSION}.txt")
|
|
end
|
|
lines
|
|
.collect.with_index { |linedata, linenumber| [linedata, linenumber]}
|
|
.reject { |line| line[0] =~ /^[\#@]/ }
|
|
.collect do |line|
|
|
NormTest.new(*(line[0].split(';').take(5).collect do |code_string|
|
|
code_string.split(/\s/).collect { |cp| cp.to_i(16) }.pack('U*')
|
|
end + [line[1]+1]))
|
|
end
|
|
end
|
|
|
|
def to_codepoints(string)
|
|
string.codepoints.collect { |cp| cp.to_s(16).upcase.rjust(4, '0') }
|
|
end
|
|
|
|
begin
|
|
@@tests ||= read_tests
|
|
rescue Errno::ENOENT
|
|
@@tests ||= []
|
|
end
|
|
|
|
def self.generate_test_normalize(target, normalization, source, prechecked)
|
|
define_method "test_normalize_to_#{target}_from_#{source}_with_#{normalization}" do
|
|
expected = actual = test = nil
|
|
mesg = proc {"#{to_codepoints(expected)} expected but was #{to_codepoints(actual)} on line #{test[:line]} (#{normalization})"}
|
|
@@tests.each do |t|
|
|
test = t
|
|
if prechecked.nil? or test[prechecked]==test[source]
|
|
expected = test[target]
|
|
actual = test[source].unicode_normalize(normalization)
|
|
assert_equal expected, actual, mesg
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
# source; NFC; NFD; NFKC; NFKD
|
|
# NFC
|
|
# :NFC == toNFC(:source) == toNFC(:NFC) == toNFC(:NFD)
|
|
generate_test_normalize :NFC, :nfc, :source, nil
|
|
generate_test_normalize :NFC, :nfc, :NFC, :source
|
|
generate_test_normalize :NFC, :nfc, :NFD, :source
|
|
# :NFKC == toNFC(:NFKC) == toNFC(:NFKD)
|
|
generate_test_normalize :NFKC, :nfc, :NFKC, nil
|
|
generate_test_normalize :NFKC, :nfc, :NFKD, :NFKC
|
|
#
|
|
# NFD
|
|
# :NFD == toNFD(:source) == toNFD(:NFC) == toNFD(:NFD)
|
|
generate_test_normalize :NFD, :nfd, :source, nil
|
|
generate_test_normalize :NFD, :nfd, :NFC, :source
|
|
generate_test_normalize :NFD, :nfd, :NFD, :source
|
|
# :NFKD == toNFD(:NFKC) == toNFD(:NFKD)
|
|
generate_test_normalize :NFKD, :nfd, :NFKC, nil
|
|
generate_test_normalize :NFKD, :nfd, :NFKD, :NFKC
|
|
#
|
|
# NFKC
|
|
# :NFKC == toNFKC(:source) == toNFKC(:NFC) == toNFKC(:NFD) == toNFKC(:NFKC) == toNFKC(:NFKD)
|
|
generate_test_normalize :NFKC, :nfkc, :source, nil
|
|
generate_test_normalize :NFKC, :nfkc, :NFC, :source
|
|
generate_test_normalize :NFKC, :nfkc, :NFD, :source
|
|
generate_test_normalize :NFKC, :nfkc, :NFKC, :NFC
|
|
generate_test_normalize :NFKC, :nfkc, :NFKD, :NFD
|
|
#
|
|
# NFKD
|
|
# :NFKD == toNFKD(:source) == toNFKD(:NFC) == toNFKD(:NFD) == toNFKD(:NFKC) == toNFKD(:NFKD)
|
|
generate_test_normalize :NFKD, :nfkd, :source, nil
|
|
generate_test_normalize :NFKD, :nfkd, :NFC, :source
|
|
generate_test_normalize :NFKD, :nfkd, :NFD, :source
|
|
generate_test_normalize :NFKD, :nfkd, :NFKC, :NFC
|
|
generate_test_normalize :NFKD, :nfkd, :NFKD, :NFD
|
|
|
|
def self.generate_test_check_true(source, normalization)
|
|
define_method "test_check_true_#{source}_as_#{normalization}" do
|
|
test = nil
|
|
mesg = proc {"#{to_codepoints(test[source])} should check as #{normalization} but does not on line #{test[:line]}"}
|
|
@@tests.each do |t|
|
|
test = t
|
|
actual = test[source].unicode_normalized?(normalization)
|
|
assert_equal true, actual, mesg
|
|
end
|
|
end
|
|
end
|
|
|
|
def self.generate_test_check_false(source, compare, normalization)
|
|
define_method "test_check_false_#{source}_as_#{normalization}" do
|
|
test = nil
|
|
mesg = proc {"#{to_codepoints(test[source])} should not check as #{normalization} but does on line #{test[:line]}"}
|
|
@@tests.each do |t|
|
|
test = t
|
|
if test[source] != test[compare]
|
|
actual = test[source].unicode_normalized?(normalization)
|
|
assert_equal false, actual, mesg
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
generate_test_check_true :NFC, :nfc
|
|
generate_test_check_true :NFD, :nfd
|
|
generate_test_check_true :NFKC, :nfc
|
|
generate_test_check_true :NFKC, :nfkc
|
|
generate_test_check_true :NFKD, :nfd
|
|
generate_test_check_true :NFKD, :nfkd
|
|
|
|
generate_test_check_false :source, :NFD, :nfd
|
|
generate_test_check_false :NFC, :NFD, :nfd
|
|
generate_test_check_false :NFKC, :NFKD, :nfd
|
|
generate_test_check_false :source, :NFC, :nfc
|
|
generate_test_check_false :NFD, :NFC, :nfc
|
|
generate_test_check_false :NFKD, :NFKC, :nfc
|
|
generate_test_check_false :source, :NFKD, :nfkd
|
|
generate_test_check_false :NFC, :NFKD, :nfkd
|
|
generate_test_check_false :NFD, :NFKD, :nfkd
|
|
generate_test_check_false :NFKC, :NFKD, :nfkd
|
|
generate_test_check_false :source, :NFKC, :nfkc
|
|
generate_test_check_false :NFC, :NFKC, :nfkc
|
|
generate_test_check_false :NFD, :NFKC, :nfkc
|
|
generate_test_check_false :NFKD, :NFKC, :nfkc
|
|
end
|
|
|
|
class TestUnicodeNormalize
|
|
def test_non_UTF_8
|
|
assert_equal "\u1E0A".encode('UTF-16BE'), "D\u0307".encode('UTF-16BE').unicode_normalize(:nfc)
|
|
assert_equal true, "\u1E0A".encode('UTF-16BE').unicode_normalized?(:nfc)
|
|
assert_equal false, "D\u0307".encode('UTF-16BE').unicode_normalized?(:nfc)
|
|
end
|
|
|
|
def test_singleton_with_accents
|
|
assert_equal "\u0136", "\u212A\u0327".unicode_normalize(:nfc)
|
|
end
|
|
|
|
def test_partial_jamo_compose
|
|
assert_equal "\uAC01", "\uAC00\u11A8".unicode_normalize(:nfc)
|
|
end
|
|
|
|
def test_partial_jamo_decompose
|
|
assert_equal "\u1100\u1161\u11A8", "\uAC00\u11A8".unicode_normalize(:nfd)
|
|
end
|
|
|
|
# preventive tests for (non-)bug #14934
|
|
def test_no_trailing_jamo
|
|
assert_equal "\u1100\u1176\u11a8", "\u1100\u1176\u11a8".unicode_normalize(:nfc)
|
|
assert_equal "\uae30\u11a7", "\u1100\u1175\u11a7".unicode_normalize(:nfc)
|
|
assert_equal "\uae30\u11c3", "\u1100\u1175\u11c3".unicode_normalize(:nfc)
|
|
end
|
|
|
|
def test_hangul_plus_accents
|
|
assert_equal "\uAC00\u0323\u0300", "\uAC00\u0300\u0323".unicode_normalize(:nfc)
|
|
assert_equal "\uAC00\u0323\u0300", "\u1100\u1161\u0300\u0323".unicode_normalize(:nfc)
|
|
assert_equal "\u1100\u1161\u0323\u0300", "\uAC00\u0300\u0323".unicode_normalize(:nfd)
|
|
assert_equal "\u1100\u1161\u0323\u0300", "\u1100\u1161\u0300\u0323".unicode_normalize(:nfd)
|
|
end
|
|
|
|
def test_raise_exception_for_non_unicode_encoding
|
|
assert_raise(Encoding::CompatibilityError) { "abc".force_encoding('ISO-8859-1').unicode_normalize }
|
|
assert_raise(Encoding::CompatibilityError) { "abc".force_encoding('ISO-8859-1').unicode_normalize! }
|
|
assert_raise(Encoding::CompatibilityError) { "abc".force_encoding('ISO-8859-1').unicode_normalized? }
|
|
end
|
|
|
|
def test_reiwa
|
|
assert_equal "\u4EE4\u548C", "\u32FF".unicode_normalize(:nfkc)
|
|
end
|
|
|
|
def test_us_ascii
|
|
ascii_string = 'abc'.encode('US-ASCII')
|
|
|
|
assert_equal ascii_string, ascii_string.unicode_normalize
|
|
assert_equal ascii_string, ascii_string.unicode_normalize(:nfd)
|
|
assert_equal ascii_string, ascii_string.unicode_normalize(:nfkc)
|
|
assert_equal ascii_string, ascii_string.unicode_normalize(:nfkd)
|
|
|
|
assert_equal ascii_string, ascii_string.dup.unicode_normalize!
|
|
assert_equal ascii_string, ascii_string.dup.unicode_normalize!(:nfd)
|
|
assert_equal ascii_string, ascii_string.dup.unicode_normalize!(:nfkc)
|
|
assert_equal ascii_string, ascii_string.dup.unicode_normalize!(:nfkd)
|
|
|
|
assert_equal true, ascii_string.unicode_normalized?
|
|
assert_equal true, ascii_string.unicode_normalized?(:nfd)
|
|
assert_equal true, ascii_string.unicode_normalized?(:nfkc)
|
|
assert_equal true, ascii_string.unicode_normalized?(:nfkd)
|
|
end
|
|
|
|
def test_bug_21559
|
|
str = "s\u{1611e}\u{323}\u{1611e}\u{307}\u{1611f}"
|
|
assert_equal str.unicode_normalize(:nfd), str.unicode_normalize(:nfc).unicode_normalize(:nfd)
|
|
end
|
|
|
|
def test_gurung_khema
|
|
assert_equal "\u{16121 16121 16121 16121 16121 1611E}", "\u{1611E 16121 16121 16121 16121 16121}".unicode_normalize
|
|
end
|
|
|
|
def test_ractor
|
|
assert_ractor(<<~'RUBY')
|
|
s = "D\u0307\u0323"
|
|
nfc = "\u1E0C\u0307"
|
|
nfd = "D\u0323\u0307"
|
|
# the first call fills the per-Ractor cache, the second hits it
|
|
assert_equal([nfc, nfc], Ractor.new(s) {|s| 2.times.map {s.unicode_normalize(:nfc)}}.value)
|
|
assert_equal([nfd, nfd], Ractor.new(s) {|s| 2.times.map {s.unicode_normalize(:nfd)}}.value)
|
|
assert_equal([nfc, nfc], Ractor.new(s) {|s| 2.times.map {s.unicode_normalize(:nfkc)}}.value)
|
|
assert_equal([nfd, nfd], Ractor.new(s) {|s| 2.times.map {s.unicode_normalize(:nfkd)}}.value)
|
|
assert_equal([false] * 4, Ractor.new(s) {|s| %i[nfc nfd nfkc nfkd].map {s.unicode_normalized?(_1)}}.value)
|
|
# non-UTF-8 Unicode encodings take the UNICODE_ENCODINGS branch
|
|
s16 = s.encode("UTF-16BE")
|
|
assert_equal(nfc.encode("UTF-16BE"), Ractor.new(s16) {|s| s.unicode_normalize(:nfc)}.value)
|
|
RUBY
|
|
end
|
|
|
|
def test_canonical_ordering
|
|
a = "\u03B1\u0313\u0300\u0345"
|
|
a_unordered1 = "\u03B1\u0345\u0313\u0300"
|
|
a_unordered2 = "\u03B1\u0313\u0345\u0300"
|
|
u1 = "U\u0308\u0304"
|
|
u2 = "U\u0304\u0308"
|
|
s = "s\u0323\u0307"
|
|
s_unordered = "s\u0307\u0323"
|
|
o = "\u{1611e}\u{1611e}\u{1611f}"
|
|
# Actual cases called through String#unicode_normalize
|
|
assert_equal(s + o, UnicodeNormalize.canonical_ordering_one(s_unordered + o))
|
|
assert_equal(a[1..], UnicodeNormalize.canonical_ordering_one(a_unordered1[1..]))
|
|
assert_equal(a[1..] + o, UnicodeNormalize.canonical_ordering_one(a_unordered2[1..] + o))
|
|
# Artificial cases
|
|
assert_equal(a + u1 + o + u2 + s, UnicodeNormalize.canonical_ordering_one(a + u1 + o + u2 + s))
|
|
assert_equal(s[1..] + a + a, UnicodeNormalize.canonical_ordering_one(s_unordered[1..] + a_unordered1 + a_unordered2))
|
|
assert_equal(o + s + u1 + a + o + a + u2 + o, UnicodeNormalize.canonical_ordering_one(o + s_unordered + u1 + a_unordered1 + o + a_unordered2 + u2 + o))
|
|
end
|
|
end
|