mirrors/RustPython
0
0 Fork 0
mirror of https://github.com/RustPython/RustPython.git synced 2026-10-28 10:44:36 +08:00
Code Activity

common: add VM-independent codec and text engines (#8622)

Browse Source 
* common: add PyPy CJK codec engines

Assisted-by: Codex:GPT-5

* common: mark CJK mapping tables generated

Assisted-by: Codex:GPT-5

* common: add reusable byte and text engines

Assisted-by: Codex:GPT-5

* common: record opt-in engine dependencies

Assisted-by: Codex:GPT-5

* stdlib: add `_multibytecodec` and the six `_codecs_*` modules

Wire the CJK codec engines in `rustpython-common` up to the VM. `_multibytecodec`
holds `MultibyteCodec`, `MultibyteIncrementalEncoder`, `MultibyteIncrementalDecoder`,
`MultibyteStreamReader` and `MultibyteStreamWriter`, the encode/decode drivers and
the error-handler protocol; each `_codecs_*` module exposes `getcodec` for the
encodings of its region.

`rustpython-stdlib` now enables the `cjk-codecs` feature of `rustpython-common`.

Add `cjk::decode_reset`, which the decoder `reset()` methods need. It differs from
building a fresh decoder state: ISO 2022 keeps its G1..G3 designations and the
escape-throughout flag, and resets only G0 and the shifted flag.

Drop the markers from the CJK codec tests and from the `test_codecs`, `test_io`,
`test_email`, `test_zipfile` and `test_traceback` cases that needed a CJK codec.
`test_codecs.BasicUnicodeTest.{test_basics,test_decoder_state}` stay expected
failures for an unrelated reason: `codecs.charmap_encode` rejects a `None` mapping.

Assisted-by: Claude Code:claude-opus-5

* stdlib: use the byte and text engines from `rustpython-common`

`binascii`, `_json` and `_socket`'s address converters now call the engines in
`rustpython-common` instead of carrying their own copies, and `rustpython-stdlib`
enables the `binascii`, `inet` and `json` features for them. That leaves
`base64` and `crc32fast` with no caller in `rustpython-stdlib`, and
`crates/stdlib/src/json/machinery.rs`, the last of the json_in_type-derived code,
goes away with its last caller.

The address converters change behaviour. `inet_aton` reads the lenient forms
(`127.1`, `16909060`, `0x7f.1`, `0177.0.0.1`), `inet_pton` rejects an octet with a
redundant leading zero and a group of five hex digits, `inet_ntop` writes the
IPv4-compatible form as `::c000:201`, and an address family with no converter
raises `EAFNOSUPPORT` instead of a bare `OSError`.

Give the JSON engine the closing-quote scan and the allocation-free `\uXXXX`
escape it needs to carry `_json`'s workload. Against the code it replaces,
`json.dumps` runs 2.3x faster with `ensure_ascii=True` and 1.7x without it, and
`json.loads` 1.2x.

`binascii.a2b_base64` now reports the messages `binascii.c` does: no
`error decoding base64: ` prefix, and the data-character count in parentheses.

Assisted-by: Claude Code:claude-opus-5

* test: enable the Windows cp932 codec case

The shared CJK engine now satisfies the Windows code-page adapter, so keeping this test marked as an expected failure turns its success into a CI failure.

Assisted-by: Codex:GPT-5
This commit is contained in:
Jeong, YunWon authored and GitHub committed 2026-08-31 15:44:55 +09:00
1 parent d1bbf4bc1a
commit 322f294a21
50 files changed
+47283 -1262
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.cspell.dict/cpython.txt
+5
View File
@@ -37,6 +37,7 @@ cfws
CFWS
chans
cidx
cjkcodecs
CLASSDEREF
classdict
cmpop
@@ -65,6 +66,7 @@ elts
eooh
eofs
EOOH
esize
evalloop
excepthandler
exceptiontable
@@ -138,6 +140,9 @@ lslpp
lsprof
MAXBLOCKS
maxdepth
mbiencoder
mbstreamreader
mbstreamwriter
metavars
miscompiles
mult
.cspell.json
+16
View File
@@ -52,6 +52,22 @@
"Lib/**",
"crates/host_env/**"
],
"overrides": [
{
"filename": "crates/common/src/encodings/cjk/**",
"words": [
"CHOSEONG",
"EUCKR",
"famille",
"GBCOMMON",
"jong",
"JONGSEONG",
"JUNGSEONG",
"PHINT",
"Xabc"
]
}
],
// words - list of words to be always considered correct
// (compound words like pyarg, baseclass, microbenchmark are handled by allowCompoundWords)
"words": [
.gitattributes
+1
View File
@@ -6,6 +6,7 @@ Lib/*.py text working-tree-encoding=UTF-8 eol=LF
crates/vm/src/stdlib/ast/gen.rs linguist-generated -merge
crates/doc/src/data.inc.rs generated
crates/compiler-core/src/bytecode/opcode_metadata.rs generated
crates/common/src/encodings/cjk/mappings_*.rs generated
.github/workflows/*.lock.yml linguist-generated=true merge=ours
Cargo.lock
Generated
+3 -2
View File
@@ -3360,7 +3360,9 @@ name = "rustpython-common"
version = "0.5.0"
dependencies = [
"ascii",
"base64",
"bitflags 2.13.1",
"crc32fast",
"getrandom 0.4.3",
"itertools 0.15.0",
"libc",
@@ -3368,6 +3370,7 @@ dependencies = [
"malachite-base",
"malachite-bigint",
"malachite-q",
"memchr",
"num-complex",
"num-traits",
"parking_lot",
@@ -3599,11 +3602,9 @@ version = "0.5.0"
dependencies = [
"adler32",
"ascii",
"base64",
"blake2",
"bzip2",
"constant_time_eq",
"crc32fast",
"crossbeam-utils",
"csv-core",
"der",
Lib/test/test_codecencodings_cn.py
Vendored
-4
View File
@@ -6,7 +6,6 @@
from test import multibytecodec_support
import unittest
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gb2312")
class Test_GB2312(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'gb2312'
tstring = multibytecodec_support.load_teststring('gb2312')
@@ -20,7 +19,6 @@ class Test_GB2312(multibytecodec_support.TestBase, unittest.TestCase):
(b"\xc1\x64", "strict", None),
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gbk")
class Test_GBK(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'gbk'
tstring = multibytecodec_support.load_teststring('gbk')
@@ -35,7 +33,6 @@ class Test_GBK(multibytecodec_support.TestBase, unittest.TestCase):
("\u30fb", "strict", None),
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gb18030")
class Test_GB18030(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'gb18030'
tstring = multibytecodec_support.load_teststring('gb18030')
@@ -60,7 +57,6 @@ class Test_GB18030(multibytecodec_support.TestBase, unittest.TestCase):
)
has_iso10646 = True
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: hz")
class Test_HZ(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'hz'
tstring = multibytecodec_support.load_teststring('hz')
Lib/test/test_codecencodings_hk.py
Vendored
-1
View File
@@ -6,7 +6,6 @@
from test import multibytecodec_support
import unittest
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: big5hkscs")
class Test_Big5HKSCS(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'big5hkscs'
tstring = multibytecodec_support.load_teststring('big5hkscs')
Lib/test/test_codecencodings_iso2022.py
Vendored
-5
View File
@@ -10,7 +10,6 @@ COMMON_CODEC_TESTS = (
(b'ab\x1B$def', 'replace', 'ab\uFFFD'),
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp")
class Test_ISO2022_JP(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'iso2022_jp'
tstring = multibytecodec_support.load_teststring('iso2022_jp')
@@ -18,7 +17,6 @@ class Test_ISO2022_JP(multibytecodec_support.TestBase, unittest.TestCase):
(b'ab\x1BNdef', 'replace', 'ab\x1BNdef'),
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_2")
class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'iso2022_jp_2'
tstring = multibytecodec_support.load_teststring('iso2022_jp')
@@ -26,7 +24,6 @@ class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase):
(b'ab\x1BNdef', 'replace', 'abdef'),
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_3")
class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'iso2022_jp_3'
tstring = multibytecodec_support.load_teststring('iso2022_jp')
@@ -50,7 +47,6 @@ class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase):
b'\x1B$(O\x29\x28\x1B(Bℜ\x1B$(O\x29\x32\x1B(B = ⟨ሴ⟩'
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_2004")
class Test_ISO2022_JP2004(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'iso2022_jp_2004'
tstring = multibytecodec_support.load_teststring('iso2022_jp')
@@ -74,7 +70,6 @@ class Test_ISO2022_JP2004(multibytecodec_support.TestBase, unittest.TestCase):
b'\x1B$(Q\x29\x28\x1B(Bℜ\x1B$(Q\x29\x32\x1B(B = ⟨ሴ⟩'
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_kr")
class Test_ISO2022_KR(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'iso2022_kr'
tstring = multibytecodec_support.load_teststring('iso2022_kr')
Lib/test/test_codecencodings_jp.py
Vendored
-7
View File
@@ -6,7 +6,6 @@
from test import multibytecodec_support
import unittest
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: cp932")
class Test_CP932(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'cp932'
tstring = multibytecodec_support.load_teststring('shift_jis')
@@ -40,7 +39,6 @@ euc_commontests = (
(b"\x8eXY", "replace", "\ufffdXY"),
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jis_2004")
class Test_EUC_JIS_2004(multibytecodec_support.TestBase,
unittest.TestCase):
encoding = 'euc_jis_2004'
@@ -51,7 +49,6 @@ class Test_EUC_JIS_2004(multibytecodec_support.TestBase,
b"\xa9\xa8ℜ\xa9\xb2 = ⟨ሴ⟩"
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jisx0213")
class Test_EUC_JISX0213(multibytecodec_support.TestBase,
unittest.TestCase):
encoding = 'euc_jisx0213'
@@ -62,7 +59,6 @@ class Test_EUC_JISX0213(multibytecodec_support.TestBase,
b"\xa9\xa8ℜ\xa9\xb2 = ⟨ሴ⟩"
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jp")
class Test_EUC_JP_COMPAT(multibytecodec_support.TestBase,
unittest.TestCase):
encoding = 'euc_jp'
@@ -78,7 +74,6 @@ shiftjis_commonenctests = (
(b"abc\x80\x80\x82\x84def", "ignore", "abc\uff44def"),
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jis")
class Test_SJIS_COMPAT(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'shift_jis'
tstring = multibytecodec_support.load_teststring('shift_jis')
@@ -93,7 +88,6 @@ class Test_SJIS_COMPAT(multibytecodec_support.TestBase, unittest.TestCase):
(b"abc\xFF\x58", "replace", "abc\ufffdX"),
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jis_2004")
class Test_SJIS_2004(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'shift_jis_2004'
tstring = multibytecodec_support.load_teststring('shift_jis')
@@ -112,7 +106,6 @@ class Test_SJIS_2004(multibytecodec_support.TestBase, unittest.TestCase):
b"\x85Gℜ\x85Q = ⟨ሴ⟩"
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jisx0213")
class Test_SJISX0213(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'shift_jisx0213'
tstring = multibytecodec_support.load_teststring('shift_jisx0213')
Lib/test/test_codecencodings_kr.py
Vendored
-3
View File
@@ -6,7 +6,6 @@
from test import multibytecodec_support
import unittest
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: cp949")
class Test_CP949(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'cp949'
tstring = multibytecodec_support.load_teststring('cp949')
@@ -19,7 +18,6 @@ class Test_CP949(multibytecodec_support.TestBase, unittest.TestCase):
(b"abc\x80\x80\xc1\xc4", "ignore", "abc\uc894"),
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_kr")
class Test_EUCKR(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'euc_kr'
tstring = multibytecodec_support.load_teststring('euc_kr')
@@ -51,7 +49,6 @@ class Test_EUCKR(multibytecodec_support.TestBase, unittest.TestCase):
(b"\xc1\xc4", "strict", "\uc894"),
)
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: johab")
class Test_JOHAB(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'johab'
tstring = multibytecodec_support.load_teststring('johab')
Lib/test/test_codecencodings_tw.py
Vendored
-1
View File
@@ -6,7 +6,6 @@
from test import multibytecodec_support
import unittest
@unittest.skip("TODO: RUSTPYTHON; unknown encoding: big5")
class Test_Big5(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'big5'
tstring = multibytecodec_support.load_teststring('big5')
Lib/test/test_codecmaps_cn.py
Vendored
-12
View File
@@ -11,28 +11,16 @@ class TestGB2312Map(multibytecodec_support.TestBase_Mapping,
encoding = 'gb2312'
mapfileurl = 'http://www.pythontest.net/unicode/EUC-CN.TXT'
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gb2312
def test_mapping_file(self):
return super().test_mapping_file()
class TestGBKMap(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
encoding = 'gbk'
mapfileurl = 'http://www.pythontest.net/unicode/CP936.TXT'
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gbk
def test_mapping_file(self):
return super().test_mapping_file()
class TestGB18030Map(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
encoding = 'gb18030'
mapfileurl = 'http://www.pythontest.net/unicode/gb-18030-2000.xml'
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gb18030
def test_mapping_file(self):
return super().test_mapping_file()
if __name__ == "__main__":
unittest.main()
Lib/test/test_codecmaps_hk.py
Vendored
-4
View File
@@ -11,9 +11,5 @@ class TestBig5HKSCSMap(multibytecodec_support.TestBase_Mapping,
encoding = 'big5hkscs'
mapfileurl = 'http://www.pythontest.net/unicode/BIG5HKSCS-2004.TXT'
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5hkscs
def test_mapping_file(self):
return super().test_mapping_file()
if __name__ == "__main__":
unittest.main()
Lib/test/test_codecmaps_jp.py
Vendored
-24
View File
@@ -20,14 +20,6 @@ class TestCP932Map(multibytecodec_support.TestBase_Mapping,
for i in range(0xa1, 0xe0):
supmaps.append((bytes([i]), chr(i+0xfec0)))
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp932
def test_mapping_file(self):
return super().test_mapping_file()
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp932
def test_mapping_supplemental(self):
return super().test_mapping_supplemental()
class TestEUCJPCOMPATMap(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
@@ -35,10 +27,6 @@ class TestEUCJPCOMPATMap(multibytecodec_support.TestBase_Mapping,
mapfilename = 'EUC-JP.TXT'
mapfileurl = 'http://www.pythontest.net/unicode/EUC-JP.TXT'
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jp
def test_mapping_file(self):
return super().test_mapping_file()
class TestSJISCOMPATMap(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
@@ -54,20 +42,12 @@ class TestSJISCOMPATMap(multibytecodec_support.TestBase_Mapping,
(b'\x81_', '\\'),
]
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: shift_jis
def test_mapping_file(self):
return super().test_mapping_file()
class TestEUCJISX0213Map(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
encoding = 'euc_jisx0213'
mapfilename = 'EUC-JISX0213.TXT'
mapfileurl = 'http://www.pythontest.net/unicode/EUC-JISX0213.TXT'
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jisx0213
def test_mapping_file(self):
return super().test_mapping_file()
class TestSJISX0213Map(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
@@ -75,10 +55,6 @@ class TestSJISX0213Map(multibytecodec_support.TestBase_Mapping,
mapfilename = 'SHIFT_JISX0213.TXT'
mapfileurl = 'http://www.pythontest.net/unicode/SHIFT_JISX0213.TXT'
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: shift_jisx0213
def test_mapping_file(self):
return super().test_mapping_file()
if __name__ == "__main__":
unittest.main()
Lib/test/test_codecmaps_kr.py
Vendored
-12
View File
@@ -11,10 +11,6 @@ class TestCP949Map(multibytecodec_support.TestBase_Mapping,
encoding = 'cp949'
mapfileurl = 'http://www.pythontest.net/unicode/CP949.TXT'
@unittest.expectedFailureIf(__import__("sys").platform in ("android", "darwin", "linux"), "TODO: RUSTPYTHON; LookupError: unknown encoding: cp949")
def test_mapping_file(self):
return super().test_mapping_file()
class TestEUCKRMap(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
@@ -25,10 +21,6 @@ class TestEUCKRMap(multibytecodec_support.TestBase_Mapping,
pass_enctest = [(b'\xa4\xd4', '\u3164')]
pass_dectest = [(b'\xa4\xd4', '\u3164')]
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_kr
def test_mapping_file(self):
return super().test_mapping_file()
class TestJOHABMap(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
@@ -41,9 +33,5 @@ class TestJOHABMap(multibytecodec_support.TestBase_Mapping,
pass_enctest = [(b'\\', '\u20a9')]
pass_dectest = [(b'\\', '\u20a9')]
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: johab
def test_mapping_file(self):
return super().test_mapping_file()
if __name__ == "__main__":
unittest.main()
Lib/test/test_codecmaps_tw.py
Vendored
-12
View File
@@ -11,10 +11,6 @@ class TestBIG5Map(multibytecodec_support.TestBase_Mapping,
encoding = 'big5'
mapfileurl = 'http://www.pythontest.net/unicode/BIG5.TXT'
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5
def test_mapping_file(self):
return super().test_mapping_file()
class TestCP950Map(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
encoding = 'cp950'
@@ -27,13 +23,5 @@ class TestCP950Map(multibytecodec_support.TestBase_Mapping,
(b"\xFFxy", "replace", "\ufffdxy"),
)
@unittest.expectedFailureIf(__import__("sys").platform in ("android", "darwin", "linux"), "TODO: RUSTPYTHON; LookupError: unknown encoding: cp950")
def test_errorhandle(self):
return super().test_errorhandle()
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp950
def test_mapping_file(self):
return super().test_mapping_file()
if __name__ == "__main__":
unittest.main()
Lib/test/test_codecs.py
Vendored
+2 -6
View File
@@ -2155,7 +2155,7 @@ broken_unicode_with_stateful = [
class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling):
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5
@unittest.expectedFailure # TODO: RUSTPYTHON; charmap_encode rejects a None mapping
def test_basics(self):
s = "abc123" # all codecs should be able to encode these
for encoding in all_unicode_encodings:
@@ -2275,7 +2275,6 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling):
self.assertEqual(decodedresult, s,
"encoding=%r" % encoding)
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5
def test_seek(self):
# all codecs should be able to encode these
s = "%s\n%s\n" % (100*"abc123", 100*"def456")
@@ -2291,7 +2290,6 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling):
data = reader.read()
self.assertEqual(s, data)
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5
def test_bad_decode_args(self):
for encoding in all_unicode_encodings:
decoder = codecs.getdecoder(encoding)
@@ -2299,7 +2297,6 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling):
if encoding not in ("idna", "punycode"):
self.assertRaises(TypeError, decoder, 42)
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5
def test_bad_encode_args(self):
for encoding in all_unicode_encodings:
encoder = codecs.getencoder(encoding)
@@ -2311,7 +2308,7 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling):
table_type = type(cp1140.encoding_table)
self.assertEqual(table_type, table_type)
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5
@unittest.expectedFailure # TODO: RUSTPYTHON; charmap_encode rejects a None mapping
def test_decoder_state(self):
# Check that getstate() and setstate() handle the state properly
u = "abc123"
@@ -3363,7 +3360,6 @@ class CodePageTest(unittest.TestCase):
self.assertRaises(UnicodeEncodeError,
text.encode, f'cp{cp}', errors)
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_cp932(self):
self.check_encode(932, (
('abc', 'strict', b'abc'),
Lib/test/test_email/test_email.py
Vendored
-5
View File
@@ -916,7 +916,6 @@ class TestEncoders(unittest.TestCase):
msg = MIMEText('hello \xf8 world', _charset='iso-8859-1')
eq(msg['content-transfer-encoding'], 'quoted-printable')
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: iso-2022-jp
def test_encode7or8bit(self):
# Make sure a charset whose input character set is 8bit but
# whose output character set is 7bit gets a transfer-encoding
@@ -1079,12 +1078,10 @@ wasnipoop; giraffes="very-long-necked-animals";
wasnipoop; giraffes="very-long-necked-animals";
\tspooge="yummy"; hippos="gargantuan"; marshmallows="gooey"''')
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: iso-2022-jp
def test_header_encode_with_different_output_charset(self):
h = Header('文', 'euc-jp')
self.assertEqual(h.encode(), "=?iso-2022-jp?b?GyRCSjgbKEI=?=")
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc-jp
def test_long_header_encode_with_different_output_charset(self):
h = Header(b'test-ja \xa4\xd8\xc5\xea\xb9\xc6\xa4\xb5\xa4\xec\xa4'
b'\xbf\xa5\xe1\xa1\xbc\xa5\xeb\xa4\xcf\xbb\xca\xb2\xf1\xbc\xd4'
@@ -4984,7 +4981,6 @@ class TestCharset(unittest.TestCase):
c = Charset('utf-8')
eq(c.header_encode(s), '=?utf-8?b?wqTCosKkwqTCpMKmwqTCqMKkwqo=?=')
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: iso_2022_jp
def test_body_encode(self):
eq = self.assertEqual
# Try a charset with QP body encoding
@@ -5353,7 +5349,6 @@ A very long line that must get split to something other than at the
s = 'Subject: =?EUC-KR?B?CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3I ?='
raises(errors.HeaderParseError, decode_header, s)
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: iso-2022-jp
def test_shift_jis_charset(self):
h = Header('文', charset='shift_jis')
self.assertEqual(h.encode(), '=?iso-2022-jp?b?GyRCSjgbKEI=?=')
Lib/test/test_email/test_headerregistry.py
Vendored
-4
View File
@@ -165,10 +165,6 @@ class TestUnstructuredHeader(TestHeaderBase):
}
TestUnstructuredHeader.test_value_rfc2047_gb2312_base64 = unittest.expectedFailure( # TODO: RUSTPYTHON
TestUnstructuredHeader.test_value_rfc2047_gb2312_base64
)
@parameterize
class TestDateHeader(TestHeaderBase):
Lib/test/test_io.py
Vendored
-9
View File
@@ -3326,7 +3326,6 @@ class TextIOWrapperTest(unittest.TestCase):
finally:
StatefulIncrementalDecoder.codecEnabled = 0
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jp
def test_multibyte_seek_and_tell(self):
f = self.open(os_helper.TESTFN, "w", encoding="euc_jp")
f.write("AB\n\u3046\u3048\n")
@@ -4216,10 +4215,6 @@ class CTextIOWrapperTest(TextIOWrapperTest):
wrapper = self.TextIOWrapper(raw)
wrapper.close() # should not crash
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jis_2004
def test_seek_with_encoder_state(self):
return super().test_seek_with_encoder_state()
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'NoneType'
def test_read_non_blocking(self):
return super().test_read_non_blocking()
@@ -4229,10 +4224,6 @@ class PyTextIOWrapperTest(TextIOWrapperTest):
io = pyio
shutdown_error = "LookupError: unknown encoding: ascii"
@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jis_2004
def test_seek_with_encoder_state(self):
return super().test_seek_with_encoder_state()
@unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.set_blocking not available on Windows")
def test_read_non_blocking(self):
return super().test_read_non_blocking()
Lib/test/test_multibytecodec.py
Vendored
+1 -5
View File
@@ -3,16 +3,12 @@
# Unit test for multibytecodec itself
#
import _multibytecodec
import codecs
import io
import sys
import textwrap
import unittest
try:
import _multibytecodec
except ImportError:
# TODO: RUSTPYTHON; _multibytecodec not implemented
raise unittest.SkipTest('_multibytecodec not available')
from test import support
from test.support import os_helper
from test.support.os_helper import TESTFN
Lib/test/test_traceback.py
Vendored
-1
View File
@@ -406,7 +406,6 @@ class TracebackCases(unittest.TestCase):
self.assertEqual(len(err), 1)
self.assertEqual(err[-1], 'SyntaxError: error\n')
@unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: index out of range
@requires_subprocess()
@force_not_colorized
def test_encoded_file(self):
Lib/test/test_zipfile/test_core.py
Vendored
+1 -3
View File
@@ -3570,7 +3570,6 @@ class TestExecutablePrependedZip(unittest.TestCase):
self.assertIn(b'number in executable: 5', output)
@unittest.skip("TODO: RUSTPYTHON; LookupError: unknown encoding: shift_jis")
class EncodedMetadataTests(unittest.TestCase):
file_names = ['\u4e00', '\u4e8c', '\u4e09'] # Han 'one', 'two', 'three'
file_content = [
@@ -3578,8 +3577,7 @@ class EncodedMetadataTests(unittest.TestCase):
# This is modern Japanese. (UTF-8)
"\u3053\u308c\u306f\u73fe\u4ee3\u7684\u65e5\u672c\u8a9e\u3067\u3059\u3002\n".encode('utf-8'),
# This is obsolete Japanese. (Shift JIS)
# TODO: RUSTPYTHON; LookupError: unknown encoding: shift_jis
# "\u3053\u308c\u306f\u53e4\u3044\u65e5\u672c\u8a9e\u3067\u3059\u3002\n".encode('shift_jis'),
"\u3053\u308c\u306f\u53e4\u3044\u65e5\u672c\u8a9e\u3067\u3059\u3002\n".encode('shift_jis'),
]
def setUp(self):
crates/common/Cargo.toml
+7
View File
@@ -10,6 +10,10 @@ license.workspace = true
[features]
default = ["std"]
binascii = ["dep:base64", "dep:crc32fast"]
cjk-codecs = []
inet = ["std"]
json = ["dep:memchr", "std"]
std = []
threading = ["parking_lot", "std"]
wasm_js = ["getrandom/wasm_js"]
@@ -20,13 +24,16 @@ rustpython-unicode = { workspace = true }
rustpython-wtf8 = { workspace = true }
ascii = { workspace = true }
base64 = { workspace = true, optional = true }
bitflags = { workspace = true }
crc32fast = { workspace = true, optional = true }
getrandom = { workspace = true }
itertools = { workspace = true }
libc = { workspace = true }
malachite-bigint = { workspace = true }
malachite-q = { workspace = true }
malachite-base = { workspace = true }
memchr = { workspace = true, optional = true }
num-traits = { workspace = true }
parking_lot = { workspace = true, optional = true }
radium = { workspace = true }
crates/common/src/binascii.rs
+769
View File
@@ -0,0 +1,769 @@
// spell-checker:ignore hexlify unhexlify uuencodes CRCTAB rlecode rledecode
//! Runtime-independent byte transforms used by Python's `binascii` module.
//!
//! Every function here operates purely on byte slices and returns plain Rust
//! values or a [`Error`]. Mapping to Python exceptions and argument handling is
//! the caller's responsibility (see `mod.rs`).
use alloc::{vec, vec::Vec};
use base64::Engine;
/// The base64 pad character `=`.
pub const PAD: u8 = 61u8;
const MAXLINESIZE: usize = 76; // Excluding the CRLF
/// A transform error carrying enough information to build the matching
/// `binascii.Error` message.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Error {
/// `unhexlify`: input length is odd.
OddLengthString,
/// `unhexlify`: a byte outside `[0-9a-fA-F]` was found.
NonHexadecimalDigit,
/// `a2b_uu`: the input has no length byte.
MissingLengthByte,
/// `a2b_uu`: an illegal character was found.
IllegalChar,
/// `a2b_uu`: trailing garbage after the decoded length.
TrailingGarbage,
/// `b2a_uu`: more than 45 bytes were passed.
TooLong,
/// `a2b_base64`: base64 decoding failed.
Base64(Base64DecodeError),
}
/// Mirrors CPython's `binascii_a2b_base64_impl` (`Modules/binascii.c`) error
/// paths one-for-one, so the caller can build the exact Python error message.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Base64DecodeError {
/// A `=` at `quad_pos == 0, i == 0`, strict mode.
LeadingPaddingNotAllowed,
/// A `=` outside the padding-completion window, strict mode.
ExcessPaddingNotAllowed,
/// A byte outside the base64 alphabet, strict mode.
OnlyBase64DataAllowed,
/// A valid base64 char after a completed pad sequence, strict mode.
ExcessDataAfterPadding,
/// A valid base64 char after an incomplete pad sequence, strict mode.
DiscontinuousPaddingNotAllowed,
/// Exactly one dangling data character at end of input.
InvalidLastSymbol { index: usize },
/// `quad_pos != 0` and the trailing padding does not complete the quad.
IncorrectPadding,
}
const fn hex_nibble(n: u8) -> u8 {
match n {
0..=9 => b'0' + n,
10..=15 => b'a' + (n - 10),
_ => unreachable!(),
}
}
const fn unhex_nibble(c: u8) -> Option<u8> {
match c {
b'0'..=b'9' => Some(c - b'0'),
b'a'..=b'f' => Some(c - b'a' + 10),
b'A'..=b'F' => Some(c - b'A' + 10),
_ => None,
}
}
/// `b2a_hex` / `hexlify`. `sep` is the (already validated, length-1 ASCII)
/// separator byte, or `None` for no separator.
#[must_use]
pub fn hexlify(bytes: &[u8], sep: Option<u8>, bytes_per_sep: isize) -> Vec<u8> {
// If no separator or bytes_per_sep is 0, use simple hexlify
if sep.is_none() || bytes_per_sep == 0 || bytes.is_empty() {
let mut hex = Vec::<u8>::with_capacity(bytes.len() * 2);
for b in bytes {
hex.push(hex_nibble(b >> 4));
hex.push(hex_nibble(b & 0xf));
}
return hex;
}
let sep_char = sep.unwrap();
let abs_bytes_per_sep = bytes_per_sep.unsigned_abs();
// If separator interval is >= data length, no separators needed
if abs_bytes_per_sep >= bytes.len() {
let mut hex = Vec::<u8>::with_capacity(bytes.len() * 2);
for b in bytes {
hex.push(hex_nibble(b >> 4));
hex.push(hex_nibble(b & 0xf));
}
return hex;
}
// Calculate result length
let num_separators = (bytes.len() - 1) / abs_bytes_per_sep;
let result_len = bytes.len() * 2 + num_separators;
let mut hex = vec![0u8; result_len];
if bytes_per_sep < 0 {
// Left-to-right processing (negative bytes_per_sep)
let mut i = 0; // input index
let mut j = 0; // output index
let chunks = bytes.len() / abs_bytes_per_sep;
// Process complete chunks
for _ in 0..chunks {
for _ in 0..abs_bytes_per_sep {
let b = bytes[i];
hex[j] = hex_nibble(b >> 4);
hex[j + 1] = hex_nibble(b & 0xf);
i += 1;
j += 2;
}
if i < bytes.len() {
hex[j] = sep_char;
j += 1;
}
}
// Process remaining bytes
while i < bytes.len() {
let b = bytes[i];
hex[j] = hex_nibble(b >> 4);
hex[j + 1] = hex_nibble(b & 0xf);
i += 1;
j += 2;
}
} else {
// Right-to-left processing (positive bytes_per_sep)
let mut i = bytes.len() as isize - 1; // input index
let mut j = result_len as isize - 1; // output index
let chunks = bytes.len() / abs_bytes_per_sep;
// Process complete chunks from right
for _ in 0..chunks {
for _ in 0..abs_bytes_per_sep {
let b = bytes[i as usize];
hex[j as usize] = hex_nibble(b & 0xf);
hex[(j - 1) as usize] = hex_nibble(b >> 4);
i -= 1;
j -= 2;
}
if i >= 0 {
hex[j as usize] = sep_char;
j -= 1;
}
}
// Process remaining bytes
while i >= 0 {
let b = bytes[i as usize];
hex[j as usize] = hex_nibble(b & 0xf);
hex[(j - 1) as usize] = hex_nibble(b >> 4);
i -= 1;
j -= 2;
}
}
hex
}
/// `a2b_hex` / `unhexlify`.
pub fn unhexlify(hex_bytes: &[u8]) -> Result<Vec<u8>, Error> {
if !hex_bytes.len().is_multiple_of(2) {
return Err(Error::OddLengthString);
}
let mut unhex = Vec::<u8>::with_capacity(hex_bytes.len() / 2);
for pair in hex_bytes.as_chunks::<2>().0 {
if let (Some(n1), Some(n2)) = (unhex_nibble(pair[0]), unhex_nibble(pair[1])) {
unhex.push((n1 << 4) | n2);
} else {
return Err(Error::NonHexadecimalDigit);
}
}
Ok(unhex)
}
/// `crc32`. `init` is the (masked to u32) starting value.
#[must_use]
pub fn crc32(bytes: &[u8], init: u32) -> u32 {
let mut hasher = crc32fast::Hasher::new_with_initial(init);
hasher.update(bytes);
hasher.finalize()
}
/// `crc_hqx`. `init` is the (masked to u32) starting value.
#[must_use]
pub fn crc_hqx(bytes: &[u8], init: u32) -> u32 {
const CRCTAB_HQX: [u16; 256] = [
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129, 0xa14a,
0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294,
0x72f7, 0x62d6, 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 0x2462,
0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, 0xa56a, 0xb54b, 0x8528, 0x9509,
0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695,
0x46b4, 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, 0x48c4, 0x58e5,
0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948,
0x9969, 0xa90a, 0xb92b, 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 0x6ca6, 0x7c87, 0x4ce4,
0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b,
0x8d68, 0x9d49, 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, 0xff9f,
0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, 0x9188, 0x81a9, 0xb1ca, 0xa1eb,
0xd10c, 0xc12d, 0xf14e, 0xe16f, 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046,
0x6067, 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, 0x02b1, 0x1290,
0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e,
0xe54f, 0xd52c, 0xc50d, 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, 0x26d3, 0x36f2, 0x0691,
0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9,
0xb98a, 0xa9ab, 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, 0xcb7d,
0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, 0x4a75, 0x5a54, 0x6a37, 0x7a16,
0x0af1, 0x1ad0, 0x2ab3, 0x3a92, 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8,
0x8dc9, 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, 0xef1f, 0xff3e,
0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93,
0x3eb2, 0x0ed1, 0x1ef0,
];
let mut crc = init & 0xffff;
for byte in bytes {
crc = ((crc << 8) & 0xFF00) ^ CRCTAB_HQX[((crc >> 8) as u8 ^ (byte)) as usize] as u32;
}
crc
}
/// `a2b_base64`. A line-by-line port of `binascii_a2b_base64_impl`
/// (CPython `Modules/binascii.c`): padding characters within the completion
/// window (`quad_pos >= 2 && quad_pos + pads <= 4`) are always skipped —
/// even in strict mode — and non-strict mode additionally skips *every*
/// pad and every non-alphabet byte outside that window. Errors therefore
/// only fire in strict mode (mid-loop) or from the two post-loop length
/// checks, which apply unconditionally.
pub fn a2b_base64(b: &[u8], strict_mode: bool) -> Result<Vec<u8>, Error> {
// Converts between ASCII and base-64 characters. The index of a given number yields the
// number in ASCII while the value of said index yields the number in base-64. For example
// "=" is 61 in ASCII but 0 (since it's the pad character) in base-64, so BASE64_TABLE[61] == 0
#[rustfmt::skip]
const BASE64_TABLE: [i8; 256] = [
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63,
52,53,54,55, 56,57,58,59, 60,61,-1,-1, -1, 0,-1,-1, /* Note PAD->0 */
-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14,
15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1,
-1,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40,
41,42,43,44, 45,46,47,48, 49,50,51,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
];
let mut decoded: Vec<u8> = Vec::with_capacity(b.len().div_ceil(4) * 3);
let mut quad_pos: u32 = 0; // position in the quad, 0..=3
let mut pads: u32 = 0;
let mut left_char: u8 = 0;
for (i, &el) in b.iter().enumerate() {
if el == PAD {
pads += 1;
if quad_pos >= 2 && quad_pos + pads <= 4 {
continue;
}
// RFC 4648 section-3.3: implementations MAY ignore a pad
// character present before the end of the encoded data, and
// MAY ignore excess pad characters. Non-strict mode does.
if !strict_mode {
continue;
}
if quad_pos == 1 {
// One dangling data character: handled by the post-loop
// `quad_pos == 1` check below.
break;
}
return Err(Error::Base64(if quad_pos == 0 && i == 0 {
Base64DecodeError::LeadingPaddingNotAllowed
} else {
Base64DecodeError::ExcessPaddingNotAllowed
}));
}
let binary_char = BASE64_TABLE[el as usize];
if binary_char < 0 {
if strict_mode {
return Err(Error::Base64(Base64DecodeError::OnlyBase64DataAllowed));
}
continue;
}
if pads > 0 && strict_mode {
return Err(Error::Base64(if quad_pos + pads == 4 {
Base64DecodeError::ExcessDataAfterPadding
} else {
Base64DecodeError::DiscontinuousPaddingNotAllowed
}));
}
pads = 0;
// Decode individual ASCII character
match quad_pos {
0 => {
quad_pos = 1;
left_char = binary_char as u8;
}
1 => {
quad_pos = 2;
decoded.push((left_char << 2) | (binary_char >> 4) as u8);
left_char = (binary_char & 0x0f) as u8;
}
2 => {
quad_pos = 3;
decoded.push((left_char << 4) | (binary_char >> 2) as u8);
left_char = (binary_char & 0x03) as u8;
}
3 => {
quad_pos = 0;
decoded.push((left_char << 6) | binary_char as u8);
left_char = 0;
}
_ => unsafe {
// quad_pos is only assigned in this match statement to constants
core::hint::unreachable_unchecked()
},
}
}
if quad_pos == 1 {
// Exactly one extra valid, non-padding character: no possible
// input encodes to this length.
return Err(Error::Base64(Base64DecodeError::InvalidLastSymbol {
index: decoded.len() / 3 * 4 + 1,
}));
}
if quad_pos != 0 && quad_pos + pads < 4 {
return Err(Error::Base64(Base64DecodeError::IncorrectPadding));
}
Ok(decoded)
}
/// `b2a_base64`.
#[must_use]
pub fn b2a_base64(data: &[u8], newline: bool) -> Vec<u8> {
// https://stackoverflow.com/questions/63916821
let mut encoded = base64::engine::general_purpose::STANDARD
.encode(data)
.into_bytes();
if newline {
encoded.push(b'\n');
}
encoded
}
#[inline]
fn uu_a2b_read(c: u8) -> Result<u8, Error> {
// Check the character for legality
// The 64 instead of the expected 63 is because
// there are a few uuencodes out there that use
// '`' as zero instead of space.
if !(b' '..=(b' ' + 64)).contains(&c) {
if b"\r\n".contains(&c) {
return Ok(0);
}
return Err(Error::IllegalChar);
}
Ok((c - b' ') & 0x3f)
}
/// `a2b_qp`.
#[must_use]
pub fn a2b_qp(buffer: &[u8], header: bool) -> Vec<u8> {
let len = buffer.len();
let mut out_data = Vec::with_capacity(len);
let mut idx = 0;
while idx < len {
if buffer[idx] == b'=' {
idx += 1;
if idx >= len {
break;
}
// Soft line breaks
if (buffer[idx] == b'\n') || (buffer[idx] == b'\r') {
if buffer[idx] != b'\n' {
while idx < len && buffer[idx] != b'\n' {
idx += 1;
}
}
if idx < len {
idx += 1;
}
} else if buffer[idx] == b'=' {
// broken case from broken python qp
out_data.push(b'=');
idx += 1;
} else if idx + 1 < len
&& ((buffer[idx] >= b'A' && buffer[idx] <= b'F')
|| (buffer[idx] >= b'a' && buffer[idx] <= b'f')
|| (buffer[idx] >= b'0' && buffer[idx] <= b'9'))
&& ((buffer[idx + 1] >= b'A' && buffer[idx + 1] <= b'F')
|| (buffer[idx + 1] >= b'a' && buffer[idx + 1] <= b'f')
|| (buffer[idx + 1] >= b'0' && buffer[idx + 1] <= b'9'))
{
// hex val
if let (Some(ch1), Some(ch2)) =
(unhex_nibble(buffer[idx]), unhex_nibble(buffer[idx + 1]))
{
out_data.push((ch1 << 4) | ch2);
}
idx += 2;
} else {
out_data.push(b'=');
}
} else if header && buffer[idx] == b'_' {
out_data.push(b' ');
idx += 1;
} else {
out_data.push(buffer[idx]);
idx += 1;
}
}
out_data
}
/// `b2a_qp`.
#[must_use]
pub fn b2a_qp(buf: &[u8], quotetabs: bool, istext: bool, header: bool) -> Vec<u8> {
let buflen = buf.len();
let mut line_len = 0;
let mut out_data_len = 0;
let mut crlf = false;
let mut ch;
let mut in_idx;
let mut out_idx;
in_idx = 0;
while in_idx < buflen {
if buf[in_idx] == b'\n' {
break;
}
in_idx += 1;
}
if in_idx > 0 && in_idx < buflen && buf[in_idx - 1] == b'\r' {
crlf = true;
}
in_idx = 0;
while in_idx < buflen {
let mut delta = 0;
if (buf[in_idx] > 126)
|| (buf[in_idx] == b'=')
|| (header && buf[in_idx] == b'_')
|| (buf[in_idx] == b'.'
&& line_len == 0
&& (in_idx + 1 == buflen
|| buf[in_idx + 1] == b'\n'
|| buf[in_idx + 1] == b'\r'
|| buf[in_idx + 1] == 0))
|| (!istext && ((buf[in_idx] == b'\r') || (buf[in_idx] == b'\n')))
|| ((buf[in_idx] == b'\t' || buf[in_idx] == b' ') && (in_idx + 1 == buflen))
|| ((buf[in_idx] < 33)
&& (buf[in_idx] != b'\r')
&& (buf[in_idx] != b'\n')
&& (quotetabs || ((buf[in_idx] != b'\t') && (buf[in_idx] != b' '))))
{
if (line_len + 3) >= MAXLINESIZE {
line_len = 0;
delta += if crlf { 3 } else { 2 };
}
line_len += 3;
delta += 3;
in_idx += 1;
} else if istext
&& ((buf[in_idx] == b'\n')
|| ((in_idx + 1 < buflen) && (buf[in_idx] == b'\r') && (buf[in_idx + 1] == b'\n')))
{
line_len = 0;
// Protect against whitespace on end of line
if (in_idx != 0) && ((buf[in_idx - 1] == b' ') || (buf[in_idx - 1] == b'\t')) {
delta += 2;
}
delta += if crlf { 2 } else { 1 };
in_idx += if buf[in_idx] == b'\r' { 2 } else { 1 };
} else {
if (in_idx + 1 != buflen) && (buf[in_idx + 1] != b'\n') && (line_len + 1) >= MAXLINESIZE
{
line_len = 0;
delta += if crlf { 3 } else { 2 };
}
line_len += 1;
delta += 1;
in_idx += 1;
}
out_data_len += delta;
}
let mut out_data = Vec::with_capacity(out_data_len);
in_idx = 0;
out_idx = 0;
line_len = 0;
while in_idx < buflen {
if (buf[in_idx] > 126)
|| (buf[in_idx] == b'=')
|| (header && buf[in_idx] == b'_')
|| ((buf[in_idx] == b'.')
&& (line_len == 0)
&& (in_idx + 1 == buflen
|| buf[in_idx + 1] == b'\n'
|| buf[in_idx + 1] == b'\r'
|| buf[in_idx + 1] == 0))
|| (!istext && ((buf[in_idx] == b'\r') || (buf[in_idx] == b'\n')))
|| ((buf[in_idx] == b'\t' || buf[in_idx] == b' ') && (in_idx + 1 == buflen))
|| ((buf[in_idx] < 33)
&& (buf[in_idx] != b'\r')
&& (buf[in_idx] != b'\n')
&& (quotetabs || ((buf[in_idx] != b'\t') && (buf[in_idx] != b' '))))
{
if (line_len + 3) >= MAXLINESIZE {
// MAXLINESIZE = 76
out_data.push(b'=');
out_idx += 1;
if crlf {
out_data.push(b'\r');
out_idx += 1;
}
out_data.push(b'\n');
out_idx += 1;
line_len = 0;
}
out_data.push(b'=');
out_idx += 1;
ch = hex_nibble(buf[in_idx] >> 4);
if (b'a'..=b'f').contains(&ch) {
ch -= b' ';
}
out_data.push(ch);
ch = hex_nibble(buf[in_idx] & 0xf);
if (b'a'..=b'f').contains(&ch) {
ch -= b' ';
}
out_data.push(ch);
out_idx += 2;
in_idx += 1;
line_len += 3;
} else if istext
&& ((buf[in_idx] == b'\n')
|| ((in_idx + 1 < buflen) && (buf[in_idx] == b'\r') && (buf[in_idx + 1] == b'\n')))
{
line_len = 0;
if (out_idx != 0)
&& ((out_data[out_idx - 1] == b' ') || (out_data[out_idx - 1] == b'\t'))
{
ch = hex_nibble(out_data[out_idx - 1] >> 4);
if (b'a'..=b'f').contains(&ch) {
ch -= b' ';
}
out_data.push(ch);
ch = hex_nibble(out_data[out_idx - 1] & 0xf);
if (b'a'..=b'f').contains(&ch) {
ch -= b' ';
}
out_data.push(ch);
out_data[out_idx - 1] = b'=';
out_idx += 2;
}
if crlf {
out_data.push(b'\r');
out_idx += 1;
}
out_data.push(b'\n');
out_idx += 1;
in_idx += if buf[in_idx] == b'\r' { 2 } else { 1 };
} else {
if (in_idx + 1 != buflen) && (buf[in_idx + 1] != b'\n') && (line_len + 1) >= 76 {
// MAXLINESIZE = 76
out_data.push(b'=');
out_idx += 1;
if crlf {
out_data.push(b'\r');
out_idx += 1;
}
out_data.push(b'\n');
out_idx += 1;
line_len = 0;
}
line_len += 1;
if header && buf[in_idx] == b' ' {
out_data.push(b'_');
out_idx += 1;
in_idx += 1;
} else {
out_data.push(buf[in_idx]);
out_idx += 1;
in_idx += 1;
}
}
}
out_data
}
/// `rlecode_hqx`.
#[must_use]
pub fn rlecode_hqx(buffer: &[u8]) -> Vec<u8> {
const RUN_CHAR: u8 = 0x90; // b'\x90'
let len = buffer.len();
let mut out_data = Vec::<u8>::with_capacity((len * 2) + 2);
let mut idx = 0;
while idx < len {
let ch = buffer[idx];
if ch == RUN_CHAR {
out_data.push(RUN_CHAR);
out_data.push(0);
return out_data;
}
let mut in_end = idx + 1;
while in_end < len && buffer[in_end] == ch && in_end < idx + 255 {
in_end += 1;
}
if in_end - idx > 3 {
out_data.push(ch);
out_data.push(RUN_CHAR);
out_data.push(((in_end - idx) % 256) as u8);
idx = in_end - 1;
} else {
out_data.push(ch);
}
idx += 1;
}
out_data
}
/// `rledecode_hqx`.
#[must_use]
pub fn rledecode_hqx(buffer: &[u8]) -> Vec<u8> {
const RUN_CHAR: u8 = 0x90; //b'\x90'
let len = buffer.len();
let mut out_data = Vec::<u8>::with_capacity(len);
let mut idx = 0;
out_data.push(buffer[idx]);
idx += 1;
while idx < len {
if buffer[idx] == RUN_CHAR {
if buffer[idx + 1] == 0 {
out_data.push(RUN_CHAR);
} else {
let ch = buffer[idx - 1];
let range = buffer[idx + 1];
idx += 1;
for _ in 1..range {
out_data.push(ch);
}
}
} else {
out_data.push(buffer[idx]);
}
idx += 1;
}
out_data
}
/// `a2b_uu`.
pub fn a2b_uu(b: &[u8]) -> Result<Vec<u8>, Error> {
// First byte: binary data length (in bytes)
if b.is_empty() {
return Err(Error::MissingLengthByte);
}
let length = (b[0].wrapping_sub(b' ') & 0x3f) as usize;
// Allocate the buffer
let mut res = Vec::<u8>::with_capacity(length);
let trailing_garbage_error = || Err(Error::TrailingGarbage);
for chunk in b.get(1..).unwrap_or_default().chunks(4) {
let (char_a, char_b, char_c, char_d) = {
let mut chunk = chunk
.iter()
.map(|&x| uu_a2b_read(x))
.collect::<Result<Vec<_>, _>>()?;
while chunk.len() < 4 {
chunk.push(0);
}
(chunk[0], chunk[1], chunk[2], chunk[3])
};
if res.len() < length {
res.push((char_a << 2) | (char_b >> 4));
} else if char_a != 0 || char_b != 0 {
return trailing_garbage_error();
}
if res.len() < length {
res.push(((char_b & 0xf) << 4) | (char_c >> 2));
} else if char_c != 0 {
return trailing_garbage_error();
}
if res.len() < length {
res.push(((char_c & 0x3) << 6) | char_d);
} else if char_d != 0 {
return trailing_garbage_error();
}
}
let remaining_length = length - res.len();
if remaining_length > 0 {
res.extend(vec![0; remaining_length]);
}
Ok(res)
}
/// `b2a_uu`.
pub fn b2a_uu(b: &[u8], backtick: bool) -> Result<Vec<u8>, Error> {
#[inline]
const fn uu_b2a(num: u8, backtick: bool) -> u8 {
if backtick && num == 0 {
0x60
} else {
b' ' + num
}
}
let length = b.len();
if length > 45 {
return Err(Error::TooLong);
}
let mut res = Vec::<u8>::with_capacity(2 + length.div_ceil(3) * 4);
res.push(uu_b2a(length as u8, backtick));
for chunk in b.chunks(3) {
let char_a = *chunk.first().unwrap();
let char_b = *chunk.get(1).unwrap_or(&0);
let char_c = *chunk.get(2).unwrap_or(&0);
res.push(uu_b2a(char_a >> 2, backtick));
res.push(uu_b2a(((char_a & 0x3) << 4) | (char_b >> 4), backtick));
res.push(uu_b2a(((char_b & 0xf) << 2) | (char_c >> 6), backtick));
res.push(uu_b2a(char_c & 0x3f, backtick));
}
res.push(0xau8);
Ok(res)
}
crates/common/src/encodings.rs
+3
View File
@@ -5,6 +5,9 @@ use num_traits::ToPrimitive;
use crate::str::StrKind;
use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf};
#[cfg(feature = "cjk-codecs")]
pub mod cjk;
pub trait StrBuffer: AsRef<Wtf8> {
fn is_compatible_with(&self, kind: StrKind) -> bool {
let s = self.as_ref();
crates/common/src/encodings/cjk/cn.rs
+356
View File
@@ -0,0 +1,356 @@
//! Line-by-line Rust port of PyPy cjkcodecs `_codecs_cn.c` stateless codecs.
use super::mappings_cn::{
GB2312_DECMAP, GB2312_DECMAP_DATA, GB18030_TO_UNIBMP_RANGES, GB18030EXT_DECMAP,
GB18030EXT_DECMAP_DATA, GB18030EXT_ENCMAP, GB18030EXT_ENCMAP_DATA, GBCOMMON_ENCMAP,
GBCOMMON_ENCMAP_DATA, GBKEXT_DECMAP, GBKEXT_DECMAP_DATA, MapIndex, NOCHAR, UNIINV,
};
use super::{DecodeOne, EncodeOne};
fn try_map_decode(index: &[MapIndex; 256], data: &[u16], c1: u8, c2: u8) -> Option<u32> {
let page = index[c1 as usize];
if !page.present || c2 < page.bottom || c2 > page.top {
return None;
}
let value = data[page.offset + usize::from(c2 - page.bottom)];
(value != UNIINV).then_some(u32::from(value))
}
fn try_map_encode(index: &[MapIndex; 256], data: &[u16], c: u32) -> Option<u16> {
if c > 0xffff {
return None;
}
let page = index[(c >> 8) as usize];
let low = c as u8;
if !page.present || low < page.bottom || low > page.top {
return None;
}
let value = data[page.offset + usize::from(low - page.bottom)];
(value != NOCHAR).then_some(value)
}
fn gbk_decode(c1: u8, c2: u8) -> Option<u32> {
match (c1, c2) {
(0xa1, 0xaa) => Some(0x2014),
(0xa8, 0x44) => Some(0x2015),
(0xa1, 0xa4) => Some(0x00b7),
_ => try_map_decode(&GB2312_DECMAP, &GB2312_DECMAP_DATA, c1 ^ 0x80, c2 ^ 0x80)
.or_else(|| try_map_decode(&GBKEXT_DECMAP, &GBKEXT_DECMAP_DATA, c1, c2)),
}
}
fn gbk_encode(c: u32) -> Option<u16> {
match c {
0x2014 => Some(0xa1aa),
0x2015 => Some(0xa844),
0x00b7 => Some(0xa1a4),
0x30fb => None,
_ => try_map_encode(&GBCOMMON_ENCMAP, &GBCOMMON_ENCMAP_DATA, c),
}
}
fn ascii(c: u32) -> Option<EncodeOne> {
if c >= 0x80 {
return None;
}
let mut output = [0; 8];
output[0] = c as u8;
Some(EncodeOne::Bytes(output, 1, 1))
}
fn dbcs(code: u16) -> EncodeOne {
let mut output = [0; 8];
output[0] = (code >> 8) as u8 | 0x80;
output[1] = if code & 0x8000 != 0 {
code as u8
} else {
code as u8 | 0x80
};
EncodeOne::Bytes(output, 2, 1)
}
pub(super) fn encode_gb2312(c: u32) -> EncodeOne {
if let Some(output) = ascii(c) {
return output;
}
let Some(code) = try_map_encode(&GBCOMMON_ENCMAP, &GBCOMMON_ENCMAP_DATA, c) else {
return EncodeOne::Illegal(1);
};
if code & 0x8000 != 0 {
return EncodeOne::Illegal(1);
}
dbcs(code)
}
pub(super) fn decode_gb2312(input: &[u8]) -> DecodeOne {
let c = input[0];
if c < 0x80 {
return DecodeOne::Char(u32::from(c), 1);
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
try_map_decode(
&GB2312_DECMAP,
&GB2312_DECMAP_DATA,
c ^ 0x80,
input[1] ^ 0x80,
)
.map_or(DecodeOne::Illegal(1), |value| DecodeOne::Char(value, 2))
}
pub(super) fn encode_gbk(c: u32) -> EncodeOne {
if let Some(output) = ascii(c) {
return output;
}
gbk_encode(c).map_or(EncodeOne::Illegal(1), dbcs)
}
pub(super) fn decode_gbk(input: &[u8]) -> DecodeOne {
let c = input[0];
if c < 0x80 {
return DecodeOne::Char(u32::from(c), 1);
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
gbk_decode(c, input[1]).map_or(DecodeOne::Illegal(1), |value| DecodeOne::Char(value, 2))
}
fn four_bytes(mut value: u32, first_base: u8) -> EncodeOne {
let mut output = [0; 8];
output[3] = (value % 10) as u8 + 0x30;
value /= 10;
output[2] = (value % 126) as u8 + 0x81;
value /= 126;
output[1] = (value % 10) as u8 + 0x30;
value /= 10;
output[0] = value as u8 + first_base;
EncodeOne::Bytes(output, 4, 1)
}
pub(super) fn encode_gb18030(c: u32) -> EncodeOne {
if let Some(output) = ascii(c) {
return output;
}
if c > 0x10ffff || (0xd800..=0xdfff).contains(&c) {
return EncodeOne::Illegal(1);
}
if c >= 0x10000 {
return four_bytes(c - 0x10000, 0x90);
}
if let Some(code) =
gbk_encode(c).or_else(|| try_map_encode(&GB18030EXT_ENCMAP, &GB18030EXT_ENCMAP_DATA, c))
{
return dbcs(code);
}
for range in &GB18030_TO_UNIBMP_RANGES {
if range.first == 0 {
break;
}
if range.first <= c && c <= range.last {
return four_bytes(c - range.first + range.base, 0x81);
}
}
EncodeOne::Illegal(1)
}
pub(super) fn decode_gb18030(input: &[u8]) -> DecodeOne {
let c = input[0];
if c < 0x80 {
return DecodeOne::Char(u32::from(c), 1);
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
let c2 = input[1];
if (0x30..=0x39).contains(&c2) {
if input.len() < 4 {
return DecodeOne::Incomplete;
}
let c3 = input[2];
let c4 = input[3];
if !(0x81..=0xfe).contains(&c)
|| !(0x81..=0xfe).contains(&c3)
|| !(0x30..=0x39).contains(&c4)
{
return DecodeOne::Illegal(1);
}
let lead = c - 0x81;
let second = c2 - 0x30;
let third = c3 - 0x81;
let fourth = c4 - 0x30;
if lead < 4 {
let sequence = (u32::from(lead) * 10 + u32::from(second)) * 1260
+ u32::from(third) * 10
+ u32::from(fourth);
if sequence < 39420 {
let mut range = &GB18030_TO_UNIBMP_RANGES[0];
for next in &GB18030_TO_UNIBMP_RANGES[1..] {
if sequence < next.base {
break;
}
range = next;
}
return DecodeOne::Char(range.first - range.base + sequence, 4);
}
} else if lead >= 15 {
let value = 0x10000
+ ((u32::from(lead) - 15) * 10 + u32::from(second)) * 1260
+ u32::from(third) * 10
+ u32::from(fourth);
if value <= 0x10ffff {
return DecodeOne::Char(value, 4);
}
}
return DecodeOne::Illegal(1);
}
gbk_decode(c, c2)
.or_else(|| try_map_decode(&GB18030EXT_DECMAP, &GB18030EXT_DECMAP_DATA, c, c2))
.map_or(DecodeOne::Illegal(1), |value| DecodeOne::Char(value, 2))
}
pub(super) fn encode_hz(c: u32, state: &mut [u8; 8]) -> EncodeOne {
let mut output = [0; 8];
if c < 0x80 {
let mut length = 0;
if state[0] != 0 {
output[0] = b'~';
output[1] = b'}';
length = 2;
state[0] = 0;
}
output[length] = c as u8;
length += 1;
if c == u32::from(b'~') {
output[length] = b'~';
length += 1;
}
return EncodeOne::Bytes(output, length, 1);
}
let Some(code) = try_map_encode(&GBCOMMON_ENCMAP, &GBCOMMON_ENCMAP_DATA, c) else {
return EncodeOne::Illegal(1);
};
if code & 0x8000 != 0 {
return EncodeOne::Illegal(1);
}
if state[0] == 0 {
output[0] = b'~';
output[1] = b'{';
output[2] = (code >> 8) as u8;
output[3] = code as u8;
state[0] = 1;
EncodeOne::Bytes(output, 4, 1)
} else {
output[0] = (code >> 8) as u8;
output[1] = code as u8;
EncodeOne::Bytes(output, 2, 1)
}
}
pub(super) fn reset_hz(state: &mut [u8; 8]) -> Option<([u8; 8], usize)> {
if state[0] == 0 {
return None;
}
let mut output = [0; 8];
output[0] = b'~';
output[1] = b'}';
state[0] = 0;
Some((output, 2))
}
pub(super) fn reset_decode_hz(state: &mut [u8; 8]) {
state[0] = 0;
}
pub(super) fn decode_hz(input: &[u8], state: &mut [u8; 8]) -> DecodeOne {
let c = input[0];
if c == b'~' {
if input.len() < 2 {
return DecodeOne::Incomplete;
}
let c2 = input[1];
if c2 == b'~' && state[0] == 0 {
return DecodeOne::Char(u32::from(b'~'), 2);
}
if c2 == b'{' && state[0] == 0 {
state[0] = 1;
} else if c2 == b'\n' && state[0] == 0 {
// line continuation
} else if c2 == b'}' && state[0] == 1 {
state[0] = 0;
} else {
return DecodeOne::Illegal(1);
}
return DecodeOne::Skip(2);
}
if c & 0x80 != 0 {
return DecodeOne::Illegal(1);
}
if state[0] == 0 {
return DecodeOne::Char(u32::from(c), 1);
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
try_map_decode(&GB2312_DECMAP, &GB2312_DECMAP_DATA, c, input[1])
.map_or(DecodeOne::Illegal(1), |value| DecodeOne::Char(value, 2))
}
#[cfg(test)]
mod tests {
use super::super::{Codec, DecodeOne, EncodeOne, decode_one, encode_one};
fn assert_vector(codec: Codec, value: u32, expected: &[u8]) {
let EncodeOne::Bytes(bytes, len, 1) = encode_one(codec, &[value], true, &mut [0; 8]) else {
panic!("U+{value:04X} is not encodable");
};
assert_eq!(&bytes[..len], expected);
assert!(matches!(
decode_one(codec, expected, &mut [0; 8]),
DecodeOne::Char(decoded, consumed)
if decoded == value && consumed == expected.len()
));
}
#[test]
fn pypy_cn_oracle_vectors() {
assert_vector(Codec::Gb2312, 0x30fb, &[0xa1, 0xa4]);
assert_vector(Codec::Gbk, 0x00b7, &[0xa1, 0xa4]);
assert_vector(Codec::Gbk, 0x2014, &[0xa1, 0xaa]);
assert_vector(Codec::Gbk, 0x2015, &[0xa8, 0x44]);
assert_vector(Codec::Gb18030, 0x0080, &[0x81, 0x30, 0x81, 0x30]);
assert_vector(Codec::Gb18030, 0x30fb, &[0x81, 0x39, 0xa7, 0x39]);
assert_vector(Codec::Gb18030, 0x10000, &[0x90, 0x30, 0x81, 0x30]);
assert_vector(Codec::Gb18030, 0x1f600, &[0x94, 0x39, 0xfc, 0x36]);
}
#[test]
fn pypy_hz_shift_state_and_reset() {
let mut state = [0; 8];
assert!(matches!(
encode_one(Codec::Hz, &['聊' as u32], false, &mut state),
EncodeOne::Bytes(bytes, 4, 1) if bytes[..4] == *b"~{AD"
));
assert_eq!(state[0], 1);
assert!(matches!(
encode_one(Codec::Hz, &['聊' as u32], false, &mut state),
EncodeOne::Bytes(bytes, 2, 1) if bytes[..2] == *b"AD"
));
let (reset, length) = super::super::encode_reset(Codec::Hz, &mut state).unwrap();
assert_eq!(&reset[..length], b"~}");
assert_eq!(state[0], 0);
assert!(matches!(
decode_one(Codec::Hz, b"~{", &mut state),
DecodeOne::Skip(2)
));
assert_eq!(state[0], 1);
assert!(matches!(
decode_one(Codec::Hz, b"AD", &mut state),
DecodeOne::Char(value, 2) if value == '聊' as u32
));
}
}
crates/common/src/encodings/cjk/hk.rs
+207
View File
@@ -0,0 +1,207 @@
//! Line-by-line Rust port of PyPy cjkcodecs `_codecs_hk.c`.
use super::mappings_hk::{
BIG5HKSCS_BMP_ENCMAP, BIG5HKSCS_BMP_ENCMAP_DATA, BIG5HKSCS_DECMAP, BIG5HKSCS_DECMAP_DATA,
BIG5HKSCS_NONBMP_ENCMAP, BIG5HKSCS_NONBMP_ENCMAP_DATA, BIG5HKSCS_PHINT_0_DATA,
BIG5HKSCS_PHINT_12130_DATA, BIG5HKSCS_PHINT_21924_DATA, MULTIC,
};
use super::mappings_tw::{BIG5_DECMAP, BIG5_DECMAP_DATA, BIG5_ENCMAP, BIG5_ENCMAP_DATA};
use super::{DecodeOne, EncodeOne};
const PAIR_ENCODE_TABLE: [u16; 4] = [0x8862, 0x8864, 0x88a3, 0x88a5];
fn try_map_decode(
index: &[super::mappings_hk::MapIndex; 256],
data: &[u16],
c1: u8,
c2: u8,
) -> Option<u32> {
let page = index[c1 as usize];
if !page.present || c2 < page.bottom || c2 > page.top {
return None;
}
let value = data[page.offset + usize::from(c2 - page.bottom)];
(value != super::mappings_hk::UNIINV).then_some(u32::from(value))
}
fn try_map_big5_decode(c1: u8, c2: u8) -> Option<u32> {
let page = BIG5_DECMAP[c1 as usize];
if !page.present || c2 < page.bottom || c2 > page.top {
return None;
}
let value = BIG5_DECMAP_DATA[page.offset + usize::from(c2 - page.bottom)];
(value != super::mappings_tw::UNIINV).then_some(u32::from(value))
}
fn try_map_encode(
index: &[super::mappings_hk::MapIndex; 256],
data: &[u16],
c: u32,
) -> Option<u16> {
let page = index[(c >> 8) as usize];
let low = c as u8;
if !page.present || low < page.bottom || low > page.top {
return None;
}
let value = data[page.offset + usize::from(low - page.bottom)];
(value != super::mappings_hk::NOCHAR).then_some(value)
}
fn try_map_big5_encode(c: u32) -> Option<u16> {
if c > 0xffff {
return None;
}
let page = BIG5_ENCMAP[(c >> 8) as usize];
let low = c as u8;
if !page.present || low < page.bottom || low > page.top {
return None;
}
let value = BIG5_ENCMAP_DATA[page.offset + usize::from(low - page.bottom)];
(value != super::mappings_tw::NOCHAR).then_some(value)
}
fn encoded(code: u16, consumed: usize) -> EncodeOne {
let mut output = [0; 8];
output[0] = (code >> 8) as u8;
output[1] = code as u8;
EncodeOne::Bytes(output, 2, consumed)
}
pub(super) fn encode_big5hkscs(input: &[u32], final_input: bool) -> EncodeOne {
let c = input[0];
if c < 0x80 {
let mut output = [0; 8];
output[0] = c as u8;
return EncodeOne::Bytes(output, 1, 1);
}
let code = if c < 0x10000 {
if let Some(mut code) = try_map_encode(&BIG5HKSCS_BMP_ENCMAP, &BIG5HKSCS_BMP_ENCMAP_DATA, c)
{
if code == MULTIC {
if input.len() >= 2 && c & 0xffdf == 0x00ca && input[1] & 0xfff7 == 0x0304 {
code = PAIR_ENCODE_TABLE[((c >> 4 | input[1] >> 3) & 3) as usize];
return encoded(code, 2);
}
if input.len() < 2 && !final_input {
return EncodeOne::Incomplete;
}
code = if c == 0x00ca { 0x8866 } else { 0x88a7 };
}
code
} else if let Some(code) = try_map_big5_encode(c) {
code
} else {
return EncodeOne::Illegal(1);
}
} else if c < 0x20000 {
return EncodeOne::Illegal(1);
} else if c < 0x30000 {
let Some(code) = try_map_encode(
&BIG5HKSCS_NONBMP_ENCMAP,
&BIG5HKSCS_NONBMP_ENCMAP_DATA,
c & 0xffff,
) else {
return EncodeOne::Illegal(1);
};
code
} else {
return EncodeOne::Illegal(1);
};
encoded(code, 1)
}
fn bh2s(c1: u8, c2: u8) -> usize {
usize::from(c1 - 0x87) * (0xfe - 0x40 + 1) + usize::from(c2 - 0x40)
}
pub(super) fn decode_big5hkscs(input: &[u8]) -> DecodeOne {
let c = input[0];
if c < 0x80 {
return DecodeOne::Char(u32::from(c), 1);
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
let c2 = input[1];
let force_hkscs = (0xc6..=0xc8).contains(&c) && (c >= 0xc7 || c2 >= 0xa1);
if !force_hkscs {
if let Some(value) = try_map_big5_decode(c, c2) {
return DecodeOne::Char(value, 2);
}
}
if let Some(decoded) = try_map_decode(&BIG5HKSCS_DECMAP, &BIG5HKSCS_DECMAP_DATA, c, c2) {
// `_codecs_hk.c::DECODER(big5hkscs)` uses the three bit-vector hints
// to distinguish BMP values from table entries whose high plane is 2.
if c < 0x87 || c2 < 0x40 {
debug_assert!(false, "mapping violates PyPy BIG5HKSCS bounds");
return DecodeOne::Illegal(1);
}
let s = bh2s(c, c2);
let (hints, relative) = if s <= bh2s(0xa0, 0xfe) {
(&BIG5HKSCS_PHINT_0_DATA[..], s)
} else if (bh2s(0xc6, 0xa1)..=bh2s(0xc8, 0xfe)).contains(&s) {
(&BIG5HKSCS_PHINT_12130_DATA[..], s - bh2s(0xc6, 0xa1))
} else if (bh2s(0xf9, 0xd6)..=bh2s(0xfe, 0xfe)).contains(&s) {
(&BIG5HKSCS_PHINT_21924_DATA[..], s - bh2s(0xf9, 0xd6))
} else {
debug_assert!(false, "mapping violates PyPy BIG5HKSCS hint ranges");
return DecodeOne::Illegal(1);
};
let value = if hints[relative >> 3] & (1 << (relative & 7)) != 0 {
decoded | 0x20000
} else {
decoded
};
return DecodeOne::Char(value, 2);
}
match (u16::from(c) << 8) | u16::from(c2) {
0x8862 => DecodeOne::Pair(0x00ca, 0x0304, 2),
0x8864 => DecodeOne::Pair(0x00ca, 0x030c, 2),
0x88a3 => DecodeOne::Pair(0x00ea, 0x0304, 2),
0x88a5 => DecodeOne::Pair(0x00ea, 0x030c, 2),
_ => DecodeOne::Illegal(1),
}
}
#[cfg(test)]
mod tests {
use super::super::{Codec, DecodeOne, EncodeOne, decode_one, encode_one};
#[test]
fn pypy_hk_pair_oracle_vectors() {
for (points, expected) in [
(&[0x00ca, 0x0304][..], [0x88, 0x62]),
(&[0x00ca, 0x030c][..], [0x88, 0x64]),
(&[0x00ea, 0x0304][..], [0x88, 0xa3]),
(&[0x00ea, 0x030c][..], [0x88, 0xa5]),
] {
let EncodeOne::Bytes(bytes, 2, 2) =
encode_one(Codec::Big5Hkscs, points, true, &mut [0; 8])
else {
panic!("pair is not encodable");
};
assert_eq!(&bytes[..2], expected);
assert!(matches!(
decode_one(Codec::Big5Hkscs, &expected, &mut [0; 8]),
DecodeOne::Pair(first, second, 2)
if first == points[0] && second == points[1]
));
}
}
#[test]
fn pair_prefix_is_incrementally_incomplete() {
assert!(matches!(
encode_one(Codec::Big5Hkscs, &[0x00ca], false, &mut [0; 8]),
EncodeOne::Incomplete
));
assert!(matches!(
encode_one(Codec::Big5Hkscs, &[0x00ca], true, &mut [0; 8]),
EncodeOne::Bytes(bytes, 2, 1) if bytes[..2] == [0x88, 0x66]
));
}
}
crates/common/src/encodings/cjk/iso2022.rs
+785
View File
@@ -0,0 +1,785 @@
//! Line-by-line Rust port of PyPy cjkcodecs `_codecs_iso2022.c`.
use super::jp::{
decode_jisx0213_plane1, decode_jisx0213_plane2, emulate_2000_encode_bmp, encode_jisx0213_code,
find_pair_encode, jisx0201_decode, try_map_decode, try_map_encode,
};
use super::mappings_cn::{
GB2312_DECMAP, GB2312_DECMAP_DATA, GBCOMMON_ENCMAP, GBCOMMON_ENCMAP_DATA,
};
use super::mappings_jp::{
JISX0208_DECMAP, JISX0208_DECMAP_DATA, JISX0212_DECMAP, JISX0212_DECMAP_DATA,
JISX0213_BMP_ENCMAP, JISX0213_BMP_ENCMAP_DATA, JISXCOMMON_ENCMAP, JISXCOMMON_ENCMAP_DATA,
MULTIC,
};
use super::mappings_kr::{CP949_ENCMAP, CP949_ENCMAP_DATA, KSX1001_DECMAP, KSX1001_DECMAP_DATA};
use super::{Codec, DecodeOne, EncodeOne};
const ESC: u8 = 0x1b;
const SO: u8 = 0x0e;
const SI: u8 = 0x0f;
const LF: u8 = 0x0a;
const MAX_ESCSEQLEN: usize = 16;
const CHARSET_ISO8859_1: u8 = b'A';
const CHARSET_ASCII: u8 = b'B';
const CHARSET_ISO8859_7: u8 = b'F';
const CHARSET_JISX0201_K: u8 = b'I';
const CHARSET_JISX0201_R: u8 = b'J';
const CHARSET_DBCS: u8 = 0x80;
const CHARSET_GB2312: u8 = b'A' | CHARSET_DBCS;
const CHARSET_JISX0208: u8 = b'B' | CHARSET_DBCS;
const CHARSET_KSX1001: u8 = b'C' | CHARSET_DBCS;
const CHARSET_JISX0212: u8 = b'D' | CHARSET_DBCS;
const CHARSET_JISX0213_2000_1: u8 = b'O' | CHARSET_DBCS;
const CHARSET_JISX0213_2: u8 = b'P' | CHARSET_DBCS;
const CHARSET_JISX0213_2004_1: u8 = b'Q' | CHARSET_DBCS;
const CHARSET_JISX0208_O: u8 = b'@' | CHARSET_DBCS;
const F_SHIFTED: u8 = 0x01;
const F_ESCTHROUGHOUT: u8 = 0x02;
const NO_SHIFT: u8 = 0x01;
const USE_G2: u8 = 0x02;
const USE_JISX0208_EXT: u8 = 0x04;
#[derive(Clone, Copy)]
struct Designation {
mark: u8,
plane: u8,
width: u8,
mapping: Mapping,
}
#[derive(Clone, Copy)]
enum Mapping {
Ksx1001,
JisX0201R,
JisX0201K,
JisX0208,
JisX0212,
JisX0213_2000_1,
JisX0213_2000_1PairOnly,
JisX0213_2000_2,
JisX0213_2004_1,
JisX0213_2004_1PairOnly,
JisX0213_2004_2,
Gb2312,
Iso8859_1,
Iso8859_7,
}
const KSX1001_G0: Designation = Designation {
mark: CHARSET_KSX1001,
plane: 0,
width: 2,
mapping: Mapping::Ksx1001,
};
const KSX1001_G1: Designation = Designation {
plane: 1,
..KSX1001_G0
};
const JISX0201_R: Designation = Designation {
mark: CHARSET_JISX0201_R,
plane: 0,
width: 1,
mapping: Mapping::JisX0201R,
};
const JISX0201_K: Designation = Designation {
mark: CHARSET_JISX0201_K,
plane: 0,
width: 1,
mapping: Mapping::JisX0201K,
};
const JISX0208: Designation = Designation {
mark: CHARSET_JISX0208,
plane: 0,
width: 2,
mapping: Mapping::JisX0208,
};
const JISX0208_O: Designation = Designation {
mark: CHARSET_JISX0208_O,
..JISX0208
};
const JISX0212: Designation = Designation {
mark: CHARSET_JISX0212,
plane: 0,
width: 2,
mapping: Mapping::JisX0212,
};
const JISX0213_2000_1: Designation = Designation {
mark: CHARSET_JISX0213_2000_1,
plane: 0,
width: 2,
mapping: Mapping::JisX0213_2000_1,
};
const JISX0213_2000_1_PAIRONLY: Designation = Designation {
mapping: Mapping::JisX0213_2000_1PairOnly,
..JISX0213_2000_1
};
const JISX0213_2000_2: Designation = Designation {
mark: CHARSET_JISX0213_2,
plane: 0,
width: 2,
mapping: Mapping::JisX0213_2000_2,
};
const JISX0213_2004_1: Designation = Designation {
mark: CHARSET_JISX0213_2004_1,
plane: 0,
width: 2,
mapping: Mapping::JisX0213_2004_1,
};
const JISX0213_2004_1_PAIRONLY: Designation = Designation {
mapping: Mapping::JisX0213_2004_1PairOnly,
..JISX0213_2004_1
};
const JISX0213_2004_2: Designation = Designation {
mark: CHARSET_JISX0213_2,
plane: 0,
width: 2,
mapping: Mapping::JisX0213_2004_2,
};
const GB2312: Designation = Designation {
mark: CHARSET_GB2312,
plane: 0,
width: 2,
mapping: Mapping::Gb2312,
};
const ISO8859_1: Designation = Designation {
mark: CHARSET_ISO8859_1,
plane: 2,
width: 1,
mapping: Mapping::Iso8859_1,
};
const ISO8859_7: Designation = Designation {
mark: CHARSET_ISO8859_7,
plane: 2,
width: 1,
mapping: Mapping::Iso8859_7,
};
const KR: &[Designation] = &[KSX1001_G1];
const JP: &[Designation] = &[JISX0208, JISX0201_R, JISX0208_O];
const JP_1: &[Designation] = &[JISX0208, JISX0212, JISX0201_R, JISX0208_O];
const JP_2: &[Designation] = &[
JISX0208, JISX0212, KSX1001_G0, GB2312, JISX0201_R, JISX0208_O, ISO8859_1, ISO8859_7,
];
const JP_2004: &[Designation] = &[
JISX0213_2004_1_PAIRONLY,
JISX0208,
JISX0213_2004_1,
JISX0213_2004_2,
];
const JP_3: &[Designation] = &[
JISX0213_2000_1_PAIRONLY,
JISX0208,
JISX0213_2000_1,
JISX0213_2000_2,
];
const JP_EXT: &[Designation] = &[JISX0208, JISX0212, JISX0201_R, JISX0201_K, JISX0208_O];
fn config(codec: Codec) -> (u8, &'static [Designation]) {
match codec {
Codec::Iso2022Kr => (0, KR),
Codec::Iso2022Jp => (NO_SHIFT | USE_JISX0208_EXT, JP),
Codec::Iso2022Jp1 => (NO_SHIFT | USE_JISX0208_EXT, JP_1),
Codec::Iso2022Jp2 => (NO_SHIFT | USE_G2 | USE_JISX0208_EXT, JP_2),
Codec::Iso2022Jp2004 => (NO_SHIFT | USE_JISX0208_EXT, JP_2004),
Codec::Iso2022Jp3 => (NO_SHIFT | USE_JISX0208_EXT, JP_3),
Codec::Iso2022JpExt => (NO_SHIFT | USE_JISX0208_EXT, JP_EXT),
_ => unreachable!("ISO-2022 dispatcher received another codec"),
}
}
pub(super) fn prepare_decode_state(state: &mut [u8; 8]) {
if state.iter().all(|&byte| byte == 0) {
state[0] = CHARSET_ASCII;
state[1] = CHARSET_ASCII;
state[2] = CHARSET_ASCII;
}
}
pub(super) fn prepare_encode_state(state: &mut [u8; 8]) {
if state.iter().all(|&byte| byte == 0) {
state[0] = CHARSET_ASCII;
state[1] = CHARSET_ASCII;
}
}
fn raw_map(index: &[super::mappings_kr::MapIndex; 256], data: &[u16], c: u32) -> Option<u16> {
if c > 0xffff {
return None;
}
let page = index[(c >> 8) as usize];
let low = c as u8;
if !page.present || low < page.bottom || low > page.top {
return None;
}
let value = data[page.offset + usize::from(low - page.bottom)];
(value != super::mappings_kr::NOCHAR).then_some(value)
}
fn raw_map_cn(index: &[super::mappings_cn::MapIndex; 256], data: &[u16], c: u32) -> Option<u16> {
if c > 0xffff {
return None;
}
let page = index[(c >> 8) as usize];
let low = c as u8;
if !page.present || low < page.bottom || low > page.top {
return None;
}
let value = data[page.offset + usize::from(low - page.bottom)];
(value != super::mappings_cn::NOCHAR).then_some(value)
}
fn raw_decode_kr(
index: &[super::mappings_kr::MapIndex; 256],
data: &[u16],
c1: u8,
c2: u8,
) -> Option<u32> {
let page = index[c1 as usize];
if !page.present || c2 < page.bottom || c2 > page.top {
return None;
}
let value = data[page.offset + usize::from(c2 - page.bottom)];
(value != super::mappings_kr::UNIINV).then_some(u32::from(value))
}
fn raw_decode_cn(
index: &[super::mappings_cn::MapIndex; 256],
data: &[u16],
c1: u8,
c2: u8,
) -> Option<u32> {
let page = index[c1 as usize];
if !page.present || c2 < page.bottom || c2 > page.top {
return None;
}
let value = data[page.offset + usize::from(c2 - page.bottom)];
(value != super::mappings_cn::UNIINV).then_some(u32::from(value))
}
fn encode_pair_only(
input: &[u32],
final_input: bool,
config_2000: bool,
) -> Result<(u16, usize), EncodeOne> {
let c = input[0];
if c > 0xffff {
return Err(EncodeOne::Illegal(1));
}
if config_2000 && emulate_2000_encode_bmp(c).is_err() {
return Err(EncodeOne::Illegal(1));
}
if try_map_encode(&JISX0213_BMP_ENCMAP, &JISX0213_BMP_ENCMAP_DATA, c) != Some(MULTIC) {
return Err(EncodeOne::Illegal(1));
}
if input.len() < 2 {
return if final_input {
Err(EncodeOne::Illegal(1))
} else {
Err(EncodeOne::Incomplete)
};
}
if input[1] == 0 {
return Err(EncodeOne::Illegal(1));
}
find_pair_encode(c, input[1])
.map(|code| (code, 2))
.ok_or(EncodeOne::Illegal(1))
}
fn encode_mapping(
mapping: Mapping,
input: &[u32],
final_input: bool,
) -> Result<(u16, usize), EncodeOne> {
let c = input[0];
let mapped = match mapping {
Mapping::Ksx1001 => {
raw_map(&CP949_ENCMAP, &CP949_ENCMAP_DATA, c).filter(|code| code & 0x8000 == 0)
}
Mapping::JisX0201R => {
if c < 0x80 && c != 0x5c && c != 0x7e {
Some(c as u16)
} else if c == 0x00a5 {
Some(0x5c)
} else if c == 0x203e {
Some(0x7e)
} else {
None
}
}
Mapping::JisX0201K => (0xff61..=0xff9f).contains(&c).then(|| (c - 0xff40) as u16),
Mapping::JisX0208 => {
if c == 0xff3c {
Some(0x2140)
} else {
try_map_encode(&JISXCOMMON_ENCMAP, &JISXCOMMON_ENCMAP_DATA, c)
.filter(|code| code & 0x8000 == 0)
}
}
Mapping::JisX0212 => try_map_encode(&JISXCOMMON_ENCMAP, &JISXCOMMON_ENCMAP_DATA, c)
.filter(|code| code & 0x8000 != 0)
.map(|code| code & 0x7fff),
Mapping::JisX0213_2000_1PairOnly => return encode_pair_only(input, final_input, true),
Mapping::JisX0213_2004_1PairOnly => return encode_pair_only(input, final_input, false),
Mapping::JisX0213_2000_1
| Mapping::JisX0213_2000_2
| Mapping::JisX0213_2004_1
| Mapping::JisX0213_2004_2 => {
let config_2000 =
matches!(mapping, Mapping::JisX0213_2000_1 | Mapping::JisX0213_2000_2);
let (code, consumed) = encode_jisx0213_code(input, final_input, config_2000, false)?;
let plane2 = matches!(mapping, Mapping::JisX0213_2000_2 | Mapping::JisX0213_2004_2);
if plane2 == (code & 0x8000 != 0) {
return Ok((code & 0x7fff, consumed));
}
None
}
Mapping::Gb2312 => {
raw_map_cn(&GBCOMMON_ENCMAP, &GBCOMMON_ENCMAP_DATA, c).filter(|code| code & 0x8000 == 0)
}
Mapping::Iso8859_1 | Mapping::Iso8859_7 => None,
};
mapped.map(|code| (code, 1)).ok_or(EncodeOne::Illegal(1))
}
fn append(output: &mut [u8; 8], length: &mut usize, bytes: &[u8]) {
output[*length..*length + bytes.len()].copy_from_slice(bytes);
*length += bytes.len();
}
pub(super) fn encode_one(
codec: Codec,
input: &[u32],
final_input: bool,
state: &mut [u8; 8],
) -> EncodeOne {
prepare_encode_state(state);
let (_, designations) = config(codec);
let c = input[0];
let mut output = [0; 8];
let mut length = 0;
if c < 0x80 {
if state[0] != CHARSET_ASCII {
append(&mut output, &mut length, &[ESC, b'(', b'B']);
state[0] = CHARSET_ASCII;
}
if state[4] & F_SHIFTED != 0 {
append(&mut output, &mut length, &[SI]);
state[4] &= !F_SHIFTED;
}
append(&mut output, &mut length, &[c as u8]);
return EncodeOne::Bytes(output, length, 1);
}
let mut selected = None;
for designation in designations {
match encode_mapping(designation.mapping, input, final_input) {
Ok((code, consumed)) => {
selected = Some((*designation, code, consumed));
break;
}
Err(EncodeOne::Incomplete) => return EncodeOne::Incomplete,
Err(_) => {}
}
}
let Some((designation, encoded, consumed)) = selected else {
return EncodeOne::Illegal(1);
};
match designation.plane {
0 => {
if state[4] & F_SHIFTED != 0 {
append(&mut output, &mut length, &[SI]);
state[4] &= !F_SHIFTED;
}
if state[0] != designation.mark {
if designation.width == 1 {
append(
&mut output,
&mut length,
&[ESC, b'(', designation.mark & 0x7f],
);
} else if designation.mark == CHARSET_JISX0208 {
append(
&mut output,
&mut length,
&[ESC, b'$', designation.mark & 0x7f],
);
} else {
append(
&mut output,
&mut length,
&[ESC, b'$', b'(', designation.mark & 0x7f],
);
}
state[0] = designation.mark;
}
}
1 => {
if state[1] != designation.mark {
if designation.width == 1 {
append(
&mut output,
&mut length,
&[ESC, b')', designation.mark & 0x7f],
);
} else {
append(
&mut output,
&mut length,
&[ESC, b'$', b')', designation.mark & 0x7f],
);
}
state[1] = designation.mark;
}
if state[4] & F_SHIFTED == 0 {
append(&mut output, &mut length, &[SO]);
state[4] |= F_SHIFTED;
}
}
_ => return EncodeOne::Illegal(1),
}
if designation.width == 1 {
append(&mut output, &mut length, &[encoded as u8]);
} else {
append(
&mut output,
&mut length,
&[(encoded >> 8) as u8, encoded as u8],
);
}
EncodeOne::Bytes(output, length, consumed)
}
pub(super) fn encode_reset(state: &mut [u8; 8]) -> Option<([u8; 8], usize)> {
prepare_encode_state(state);
let mut output = [0; 8];
let mut length = 0;
if state[4] & F_SHIFTED != 0 {
append(&mut output, &mut length, &[SI]);
state[4] &= !F_SHIFTED;
}
if state[0] != CHARSET_ASCII {
append(&mut output, &mut length, &[ESC, b'(', b'B']);
state[0] = CHARSET_ASCII;
}
(length != 0).then_some((output, length))
}
pub(super) fn decode_reset(state: &mut [u8; 8]) {
prepare_decode_state(state);
state[0] = CHARSET_ASCII;
state[4] &= !F_SHIFTED;
}
fn is_esc_end(c: u8) -> bool {
c.is_ascii_uppercase() || c == b'@'
}
fn is_iso2022_esc(c: u8) -> bool {
matches!(c, b'(' | b')' | b'$' | b'.' | b'&')
}
fn process_escape(codec: Codec, input: &[u8], state: &mut [u8; 8]) -> DecodeOne {
let (flags, designations) = config(codec);
let mut i = 1;
let escape_length = loop {
if i >= MAX_ESCSEQLEN {
return DecodeOne::Illegal(1);
}
if i >= input.len() {
return DecodeOne::Incomplete;
}
if is_esc_end(input[i]) {
break i + 1;
}
if flags & USE_JISX0208_EXT != 0
&& i + 1 < input.len()
&& input[i] == b'&'
&& input[i + 1] == b'@'
{
i += 2;
}
i += 1;
};
let (charset, plane) = match escape_length {
3 if input[1] == b'$' => (input[2] | CHARSET_DBCS, 0),
3 => {
let plane = match input[1] {
b'(' => 0,
b')' => 1,
b'.' if flags & USE_G2 != 0 => 2,
_ => return DecodeOne::Illegal(3),
};
(input[2], plane)
}
4 if input[1] == b'$' => {
let plane = match input[2] {
b'(' => 0,
b')' => 1,
_ => return DecodeOne::Illegal(4),
};
(input[3] | CHARSET_DBCS, plane)
}
4 => return DecodeOne::Illegal(4),
6 if flags & USE_JISX0208_EXT != 0
&& input[3] == ESC
&& input[4] == b'$'
&& input[5] == b'B' =>
{
(CHARSET_JISX0208, 0)
}
length => return DecodeOne::Illegal(length),
};
if charset != CHARSET_ASCII && !designations.iter().any(|dsg| dsg.mark == charset) {
return DecodeOne::Illegal(escape_length);
}
state[plane] = charset;
DecodeOne::Skip(escape_length)
}
fn decode_iso8859_7(c: u8) -> Option<u32> {
if c < 0xa0 || (c < 0xc0 && 0x288f3bc9_u32 & (1 << (c - 0xa0)) != 0) {
Some(u32::from(c))
} else if (0xb4..=0xfe).contains(&c) && (c >= 0xd4 || 0xbffffd77_u64 & (1 << (c - 0xb4)) != 0) {
Some(0x02d0 + u32::from(c))
} else {
match c {
0xa1 => Some(0x2018),
0xa2 => Some(0x2019),
0xaf => Some(0x2015),
_ => None,
}
}
}
fn process_g2(input: &[u8], state: &[u8; 8]) -> DecodeOne {
if input.len() < 3 {
return DecodeOne::Incomplete;
}
let c = input[2];
let decoded = match state[2] {
CHARSET_ISO8859_1 if c < 0x80 => Some(u32::from(c) + 0x80),
CHARSET_ISO8859_7 => decode_iso8859_7(c ^ 0x80),
CHARSET_ASCII if c & 0x80 == 0 => Some(u32::from(c)),
_ => None,
};
decoded.map_or(DecodeOne::Illegal(3), |value| DecodeOne::Char(value, 3))
}
fn decode_mapping(mapping: Mapping, input: &[u8], width: usize) -> DecodeOne {
let c1 = input[0];
let c2 = input.get(1).copied().unwrap_or(0);
let mapped = match mapping {
Mapping::Ksx1001 => raw_decode_kr(&KSX1001_DECMAP, &KSX1001_DECMAP_DATA, c1, c2),
Mapping::JisX0201R => jisx0201_decode(c1),
Mapping::JisX0201K => {
jisx0201_decode(c1 ^ 0x80).filter(|value| (0xff61..=0xff9f).contains(value))
}
Mapping::JisX0208 => {
if (c1, c2) == (0x21, 0x40) {
Some(0xff3c)
} else {
try_map_decode(&JISX0208_DECMAP, &JISX0208_DECMAP_DATA, c1, c2)
}
}
Mapping::JisX0212 => try_map_decode(&JISX0212_DECMAP, &JISX0212_DECMAP_DATA, c1, c2),
Mapping::JisX0213_2000_1 | Mapping::JisX0213_2000_1PairOnly => {
return match decode_jisx0213_plane1(c1, c2, true, width, true, false) {
DecodeOne::Illegal(_) => DecodeOne::Illegal(width),
decoded => decoded,
};
}
Mapping::JisX0213_2000_2 => {
// PyPy `_codecs_iso2022.c::jisx0213_2000_2_decoder` expands
// `EMULATE_JISX0213_2000_DECODE_PLANE2` and then, deliberately
// without `else`, runs the 2004 table lookup. Thus row 0x7d,
// cell 0x3b is first assigned U+9B1D and then overwritten by the
// table's U+9B1C. The EUC/Shift-JIS decoders put `else` there and
// must keep U+9B1D, so preserve the ISO-only ordering here.
let table = decode_jisx0213_plane2(c1, c2, false, width);
if !matches!(table, DecodeOne::Illegal(_)) {
return table;
}
return match decode_jisx0213_plane2(c1, c2, true, width) {
DecodeOne::Illegal(_) => DecodeOne::Illegal(width),
decoded => decoded,
};
}
Mapping::JisX0213_2004_1 | Mapping::JisX0213_2004_1PairOnly => {
return match decode_jisx0213_plane1(c1, c2, false, width, true, false) {
DecodeOne::Illegal(_) => DecodeOne::Illegal(width),
decoded => decoded,
};
}
Mapping::JisX0213_2004_2 => {
return match decode_jisx0213_plane2(c1, c2, false, width) {
DecodeOne::Illegal(_) => DecodeOne::Illegal(width),
decoded => decoded,
};
}
Mapping::Gb2312 => raw_decode_cn(&GB2312_DECMAP, &GB2312_DECMAP_DATA, c1, c2),
Mapping::Iso8859_1 | Mapping::Iso8859_7 => None,
};
mapped.map_or(DecodeOne::Illegal(width), |value| {
DecodeOne::Char(value, width)
})
}
pub(super) fn decode_one(codec: Codec, input: &[u8], state: &mut [u8; 8]) -> DecodeOne {
prepare_decode_state(state);
let (flags, designations) = config(codec);
let c = input[0];
if state[4] & F_ESCTHROUGHOUT != 0 {
if is_esc_end(c) {
state[4] &= !F_ESCTHROUGHOUT;
}
return DecodeOne::Char(u32::from(c), 1);
}
match c {
ESC => {
if input.len() < 2 {
DecodeOne::Incomplete
} else if is_iso2022_esc(input[1]) {
process_escape(codec, input, state)
} else if flags & USE_G2 != 0 && input[1] == b'N' {
process_g2(input, state)
} else {
state[4] |= F_ESCTHROUGHOUT;
DecodeOne::Char(u32::from(ESC), 1)
}
}
SI if flags & NO_SHIFT == 0 => {
state[4] &= !F_SHIFTED;
DecodeOne::Skip(1)
}
SO if flags & NO_SHIFT == 0 => {
state[4] |= F_SHIFTED;
DecodeOne::Skip(1)
}
LF => {
state[4] &= !F_SHIFTED;
DecodeOne::Char(u32::from(LF), 1)
}
_ if c < 0x20 => DecodeOne::Char(u32::from(c), 1),
_ if c >= 0x80 => DecodeOne::Illegal(1),
_ => {
let charset = if state[4] & F_SHIFTED != 0 {
state[1]
} else {
state[0]
};
if charset == CHARSET_ASCII {
return DecodeOne::Char(u32::from(c), 1);
}
let Some(designation) = designations.iter().find(|dsg| dsg.mark == charset) else {
return DecodeOne::Illegal(1);
};
let width = usize::from(designation.width);
if input.len() < width {
return DecodeOne::Incomplete;
}
decode_mapping(designation.mapping, input, width)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn encode(codec: Codec, text: &str) -> Vec<u8> {
let input: Vec<u32> = text.chars().map(u32::from).collect();
let mut state = [0; 8];
let mut output = Vec::new();
let mut position = 0;
while position < input.len() {
let EncodeOne::Bytes(bytes, length, consumed) =
encode_one(codec, &input[position..], true, &mut state)
else {
panic!("U+{:04X} is not encodable", input[position]);
};
output.extend_from_slice(&bytes[..length]);
position += consumed;
}
if let Some((bytes, length)) = encode_reset(&mut state) {
output.extend_from_slice(&bytes[..length]);
}
output
}
fn decode(codec: Codec, input: &[u8]) -> String {
let mut state = [0; 8];
let mut output = String::new();
let mut position = 0;
while position < input.len() {
match decode_one(codec, &input[position..], &mut state) {
DecodeOne::Char(value, consumed) => {
output.push(char::from_u32(value).unwrap());
position += consumed;
}
DecodeOne::Pair(first, second, consumed) => {
output.push(char::from_u32(first).unwrap());
output.push(char::from_u32(second).unwrap());
position += consumed;
}
DecodeOne::Skip(consumed) => position += consumed,
DecodeOne::Incomplete | DecodeOne::Illegal(_) => {
panic!("input is not decodable at byte {position}")
}
}
}
output
}
#[test]
fn pypy_iso2022_oracle_vectors() {
let vectors = [
(Codec::Iso2022Kr, "한국어", &b"\x1b$)C\x0eGQ19>n\x0f"[..]),
(Codec::Iso2022Jp, "日本語", &b"\x1b$BF|K\\8l\x1b(B"[..]),
(Codec::Iso2022Jp1, "日本語", &b"\x1b$BF|K\\8l\x1b(B"[..]),
(Codec::Iso2022Jp2, "한국어", &b"\x1b$(CGQ19>n\x1b(B"[..]),
(
Codec::Iso2022Jp2004,
"か\u{309a}\u{2000b}",
&b"\x1b$(Q$w.\x22\x1b(B"[..],
),
(
Codec::Iso2022Jp3,
"か\u{309a}\u{2000b}",
&b"\x1b$(O$w.\x22\x1b(B"[..],
),
(Codec::Iso2022JpExt, "日本語", &b"\x1b$BF|K\\8l\x1b(B"[..]),
];
for (codec, text, expected) in vectors {
assert_eq!(encode(codec, text), expected, "{codec:?}");
assert_eq!(decode(codec, expected), text, "{codec:?}");
}
assert_eq!(
encode(Codec::Iso2022Jp2, "¥‾\\～"),
b"\x1b(J\\~\x1b(B\\\x1b$(C\x22&\x1b(B"
);
assert_eq!(
decode(Codec::Iso2022Jp2, b"\x1b(B:hu4:unit\x1b.A\x1bNi de famille"),
":hu4:unit\u{e9} de famille"
);
assert_eq!(decode(Codec::Iso2022Jp, b"\x1bXabcZ"), "\x1bXabcZ");
assert_eq!(decode(Codec::Iso2022Jp2004, b"\x1b$(Q\x22\x32\x1b(B"), "~");
assert_eq!(decode(Codec::Iso2022Jp3, b"\x1b$(O\x22\x32\x1b(B"), "~");
assert_eq!(
decode(Codec::Iso2022Jp3, b"\x1b$(P\x7d\x3b\x1b(B"),
"\u{9b1c}"
);
assert!(encode(Codec::Iso2022Jp2004, "か\0").ends_with(&[0]));
assert!(encode(Codec::Iso2022Jp3, "か\0").ends_with(&[0]));
}
}
crates/common/src/encodings/cjk/jp.rs
+777
View File
@@ -0,0 +1,777 @@
//! Line-by-line Rust port of PyPy cjkcodecs `_codecs_jp.c`.
use super::mappings_jisx0213_pair::{
JISX0213_PAIR_DECMAP, JISX0213_PAIR_DECMAP_DATA, JISX0213_PAIR_ENCMAP,
};
use super::mappings_jp::{
CP932EXT_DECMAP, CP932EXT_DECMAP_DATA, CP932EXT_ENCMAP, CP932EXT_ENCMAP_DATA, JISX0208_DECMAP,
JISX0208_DECMAP_DATA, JISX0212_DECMAP, JISX0212_DECMAP_DATA, JISX0213_1_BMP_DECMAP,
JISX0213_1_BMP_DECMAP_DATA, JISX0213_1_EMP_DECMAP, JISX0213_1_EMP_DECMAP_DATA,
JISX0213_2_BMP_DECMAP, JISX0213_2_BMP_DECMAP_DATA, JISX0213_2_EMP_DECMAP,
JISX0213_2_EMP_DECMAP_DATA, JISX0213_BMP_ENCMAP, JISX0213_BMP_ENCMAP_DATA, JISX0213_EMP_ENCMAP,
JISX0213_EMP_ENCMAP_DATA, JISXCOMMON_ENCMAP, JISXCOMMON_ENCMAP_DATA, MULTIC, MapIndex, NOCHAR,
UNIINV,
};
use super::{DecodeOne, EncodeOne};
const EMPBASE: u32 = 0x20000;
pub(super) fn try_map_decode(index: &[MapIndex; 256], data: &[u16], c1: u8, c2: u8) -> Option<u32> {
let page = index[c1 as usize];
if !page.present || c2 < page.bottom || c2 > page.top {
return None;
}
let value = data[page.offset + usize::from(c2 - page.bottom)];
(value != UNIINV).then_some(u32::from(value))
}
pub(super) fn try_map_encode(index: &[MapIndex; 256], data: &[u16], c: u32) -> Option<u16> {
if c > 0xffff {
return None;
}
let page = index[(c >> 8) as usize];
let low = c as u8;
if !page.present || low < page.bottom || low > page.top {
return None;
}
let value = data[page.offset + usize::from(low - page.bottom)];
(value != NOCHAR).then_some(value)
}
fn try_pair_decode(c1: u8, c2: u8) -> Option<u32> {
let page = JISX0213_PAIR_DECMAP[c1 as usize];
if !page.present || c2 < page.bottom || c2 > page.top {
return None;
}
let value = JISX0213_PAIR_DECMAP_DATA[page.offset + usize::from(c2 - page.bottom)];
(value != u32::from(super::mappings_jisx0213_pair::UNIINV)).then_some(value)
}
pub(super) fn find_pair_encode(first: u32, second: u32) -> Option<u16> {
if first > 0xffff || second > 0xffff {
return None;
}
let key = first << 16 | second;
JISX0213_PAIR_ENCMAP
.binary_search_by_key(&key, |entry| entry.uniseq)
.ok()
.map(|index| JISX0213_PAIR_ENCMAP[index].code)
}
fn bytes1(c: u8) -> EncodeOne {
let mut output = [0; 8];
output[0] = c;
EncodeOne::Bytes(output, 1, 1)
}
fn bytes2(c1: u8, c2: u8, consumed: usize) -> EncodeOne {
let mut output = [0; 8];
output[0] = c1;
output[1] = c2;
EncodeOne::Bytes(output, 2, consumed)
}
fn bytes3(c1: u8, c2: u8, c3: u8, consumed: usize) -> EncodeOne {
let mut output = [0; 8];
output[0] = c1;
output[1] = c2;
output[2] = c3;
EncodeOne::Bytes(output, 3, consumed)
}
fn illegal_decode_lead() -> DecodeOne {
// [3.14-spec] `DecodeOne::Illegal(1)` ↔ PyPy `_codecs_jp.c`'s decoder
// bodies return the whole candidate width to `multibytecodec_decerror`.
// The directly observable span from pinned CPython 3.14.2 (and the real
// pypy3 oracle) ends after the lead byte, so this combined Rust engine /
// error-boundary carries that one-byte span explicitly.
DecodeOne::Illegal(1)
}
pub(super) fn jisx0201_k_encode(c: u32) -> Option<u8> {
if (0xff61..=0xff9f).contains(&c) {
Some((c - 0xfec0) as u8)
} else {
None
}
}
fn jisx0201_encode(c: u32) -> Option<u8> {
if c < 0x80 && c != 0x5c && c != 0x7e {
Some(c as u8)
} else if c == 0x00a5 {
Some(0x5c)
} else if c == 0x203e {
Some(0x7e)
} else {
jisx0201_k_encode(c)
}
}
pub(super) fn jisx0201_decode(c: u8) -> Option<u32> {
if c < 0x5c {
Some(u32::from(c))
} else if c == 0x5c {
Some(0x00a5)
} else if c < 0x7e {
Some(u32::from(c))
} else if c == 0x7e {
Some(0x203e)
} else if c == 0x7f {
Some(0x7f)
} else if (0xa1..=0xdf).contains(&c) {
Some(0xfec0 + u32::from(c))
} else {
None
}
}
fn jis_to_shift(code: u16) -> (u8, u8) {
let c1 = (code >> 8) as u8;
let c2 = code as u8;
let c2 = (if (c1 - 0x21) & 1 != 0 { 0x5e } else { 0 }) + (c2 - 0x21);
let c1 = (c1 - 0x21) >> 1;
(
if c1 < 0x1f { c1 + 0x81 } else { c1 + 0xc1 },
if c2 < 0x3f { c2 + 0x40 } else { c2 + 0x41 },
)
}
fn shift_to_jis(c: u8, c2: u8) -> Option<(u8, u8)> {
if c2 < 0x40 || c2 == 0x7f || c2 > 0xfc {
return None;
}
let c1 = if c < 0xe0 { c - 0x81 } else { c - 0xc1 };
let trail = if c2 < 0x80 { c2 - 0x40 } else { c2 - 0x41 };
Some((
2 * c1 + if trail < 0x5e { 0 } else { 1 } + 0x21,
(if trail < 0x5e { trail } else { trail - 0x5e }) + 0x21,
))
}
pub(super) fn encode_cp932(c: u32) -> EncodeOne {
if c <= 0x80 {
return bytes1(c as u8);
}
if let Some(code) = jisx0201_k_encode(c) {
return bytes1(code);
}
if (0xf8f0..=0xf8f3).contains(&c) {
return bytes1(if c == 0xf8f0 {
0xa0
} else {
(c - 0xf8f1) as u8 + 0xfd
});
}
let code = if let Some(code) = try_map_encode(&CP932EXT_ENCMAP, &CP932EXT_ENCMAP_DATA, c) {
return bytes2((code >> 8) as u8, code as u8, 1);
} else if let Some(code) = try_map_encode(&JISXCOMMON_ENCMAP, &JISXCOMMON_ENCMAP_DATA, c) {
if code & 0x8000 != 0 {
return EncodeOne::Illegal(1);
}
code
} else if (0xe000..0xe758).contains(&c) {
let offset = c - 0xe000;
let c1 = (offset / 188) as u8;
let c2 = (offset % 188) as u8;
return bytes2(c1 + 0xf0, if c2 < 0x3f { c2 + 0x40 } else { c2 + 0x41 }, 1);
} else {
return EncodeOne::Illegal(1);
};
let (c1, c2) = jis_to_shift(code);
bytes2(c1, c2, 1)
}
pub(super) fn decode_cp932(input: &[u8]) -> DecodeOne {
let c = input[0];
if c <= 0x80 {
return DecodeOne::Char(u32::from(c), 1);
}
if (0xa0..=0xdf).contains(&c) {
return DecodeOne::Char(
if c == 0xa0 {
0xf8f0
} else {
0xfec0 + u32::from(c)
},
1,
);
}
if c >= 0xfd {
return DecodeOne::Char(0xf8f1 - 0xfd + u32::from(c), 1);
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
let c2 = input[1];
if let Some(value) = try_map_decode(&CP932EXT_DECMAP, &CP932EXT_DECMAP_DATA, c, c2) {
return DecodeOne::Char(value, 2);
}
if ((0x81..=0x9f).contains(&c) || (0xe0..=0xea).contains(&c))
&& let Some((c1, c2)) = shift_to_jis(c, c2)
&& let Some(value) = try_map_decode(&JISX0208_DECMAP, &JISX0208_DECMAP_DATA, c1, c2)
{
return DecodeOne::Char(value, 2);
}
if (0xf0..=0xf9).contains(&c) && ((0x40..=0x7e).contains(&c2) || (0x80..=0xfc).contains(&c2)) {
return DecodeOne::Char(
0xe000
+ 188 * u32::from(c - 0xf0)
+ u32::from(if c2 < 0x80 { c2 - 0x40 } else { c2 - 0x41 }),
2,
);
}
illegal_decode_lead()
}
pub(super) fn encode_euc_jp(c: u32) -> EncodeOne {
if c < 0x80 {
return bytes1(c as u8);
}
let code = if let Some(code) = try_map_encode(&JISXCOMMON_ENCMAP, &JISXCOMMON_ENCMAP_DATA, c) {
code
} else if let Some(code) = jisx0201_k_encode(c) {
return bytes2(0x8e, code, 1);
} else if c == 0xff3c {
0x2140
} else if c == 0x00a5 {
return bytes1(0x5c);
} else if c == 0x203e {
return bytes1(0x7e);
} else {
return EncodeOne::Illegal(1);
};
if code & 0x8000 != 0 {
bytes3(0x8f, (code >> 8) as u8, code as u8 | 0x80, 1)
} else {
bytes2((code >> 8) as u8 | 0x80, code as u8 | 0x80, 1)
}
}
pub(super) fn decode_euc_jp(input: &[u8]) -> DecodeOne {
let c = input[0];
if c < 0x80 {
return DecodeOne::Char(u32::from(c), 1);
}
if c == 0x8e {
if input.len() < 2 {
return DecodeOne::Incomplete;
}
let c2 = input[1];
return if (0xa1..=0xdf).contains(&c2) {
DecodeOne::Char(0xfec0 + u32::from(c2), 2)
} else {
illegal_decode_lead()
};
}
if c == 0x8f {
if input.len() < 3 {
return DecodeOne::Incomplete;
}
return try_map_decode(
&JISX0212_DECMAP,
&JISX0212_DECMAP_DATA,
input[1] ^ 0x80,
input[2] ^ 0x80,
)
.map_or_else(illegal_decode_lead, |value| DecodeOne::Char(value, 3));
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
let c2 = input[1];
if c == 0xa1 && c2 == 0xc0 {
return DecodeOne::Char(0xff3c, 2);
}
try_map_decode(&JISX0208_DECMAP, &JISX0208_DECMAP_DATA, c ^ 0x80, c2 ^ 0x80)
.map_or_else(illegal_decode_lead, |value| DecodeOne::Char(value, 2))
}
pub(super) fn encode_shift_jis(c: u32) -> EncodeOne {
let code = if c < 0x80 {
c as u16
} else if c == 0x00a5 {
0x5c
} else if c == 0x203e {
0x7e
} else if let Some(code) = jisx0201_k_encode(c) {
u16::from(code)
} else {
NOCHAR
};
if code < 0x80 || (0xa1..=0xdf).contains(&code) {
return bytes1(code as u8);
}
let code = if code == NOCHAR {
if let Some(code) = try_map_encode(&JISXCOMMON_ENCMAP, &JISXCOMMON_ENCMAP_DATA, c) {
if code & 0x8000 != 0 {
return EncodeOne::Illegal(1);
}
code
} else if c == 0xff3c {
0x2140
} else {
return EncodeOne::Illegal(1);
}
} else {
code
};
let (c1, c2) = jis_to_shift(code);
bytes2(c1, c2, 1)
}
pub(super) fn decode_shift_jis(input: &[u8]) -> DecodeOne {
let c = input[0];
if c < 0x80 {
return DecodeOne::Char(u32::from(c), 1);
}
if (0xa1..=0xdf).contains(&c) {
return DecodeOne::Char(0xfec0 + u32::from(c), 1);
}
if !((0x81..=0x9f).contains(&c) || (0xe0..=0xea).contains(&c)) {
return illegal_decode_lead();
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
let Some((c1, c2)) = shift_to_jis(c, input[1]) else {
return illegal_decode_lead();
};
if c1 == 0x21 && c2 == 0x40 {
return DecodeOne::Char(0xff3c, 2);
}
try_map_decode(&JISX0208_DECMAP, &JISX0208_DECMAP_DATA, c1, c2)
.map_or_else(illegal_decode_lead, |value| DecodeOne::Char(value, 2))
}
pub(super) fn emulate_2000_encode_bmp(c: u32) -> Result<Option<u16>, ()> {
if matches!(
c,
0x9b1c | 0x4ff1 | 0x525d | 0x541e | 0x5653 | 0x59f8 | 0x5c5b | 0x5e77 | 0x7626 | 0x7e6b
) {
Err(())
} else if c == 0x9b1d {
Ok(Some(0x8000 | 0x7d3b))
} else {
Ok(None)
}
}
fn emulate_2000_plane1(c1: u8, c2: u8) -> bool {
matches!(
(c1, c2),
(0x2e, 0x21)
| (0x2f, 0x7e)
| (0x4f, 0x54)
| (0x4f, 0x7e)
| (0x74, 0x27)
| (0x7e, 0x7a..=0x7e)
)
}
pub(super) fn encode_jisx0213_code(
input: &[u32],
final_input: bool,
config_2000: bool,
allow_jisx0212_common: bool,
) -> Result<(u16, usize), EncodeOne> {
let c = input[0];
if c <= 0xffff {
let mut code = if config_2000 {
emulate_2000_encode_bmp(c).map_err(|()| EncodeOne::Illegal(1))?
} else {
None
};
if code.is_none() {
code = try_map_encode(&JISX0213_BMP_ENCMAP, &JISX0213_BMP_ENCMAP_DATA, c);
}
if let Some(mut code) = code {
if code == MULTIC {
if input.len() < 2 {
if !final_input {
return Err(EncodeOne::Incomplete);
}
code = find_pair_encode(c, 0).ok_or(EncodeOne::Illegal(1))?;
// [3.14-spec] PyPy `_codecs_jp.c::jisx0213_encoder` treats
// the zero-second pair-map entry as an actual pair and eats
// the caller's trailing NUL. CPython 3.14's
// `multibytecodec_support.TestBase.test_null_terminator`
// requires that observable NUL to remain a separate byte.
} else if input[1] != 0
&& let Some(pair) = find_pair_encode(c, input[1])
{
return Ok((pair, 2));
} else {
code = find_pair_encode(c, 0).ok_or(EncodeOne::Illegal(1))?;
}
}
return Ok((code, 1));
}
if let Some(code) = try_map_encode(&JISXCOMMON_ENCMAP, &JISXCOMMON_ENCMAP_DATA, c) {
// PyPy `_codecs_jp.c`'s `euc_jis_2004_encoder` accepts JIS X 0212
// entries from `jisxcommon`, while `shift_jis_2004_encoder` and
// `_codecs_iso2022.c`'s `jisx0213_encoder` reject them. The high
// bit is a codeset-2 marker only on the EUC path.
if allow_jisx0212_common || code & 0x8000 == 0 {
return Ok((code, 1));
}
}
Err(EncodeOne::Illegal(1))
} else if c >> 16 == EMPBASE >> 16 {
if config_2000 && c == 0x20b9f {
return Err(EncodeOne::Illegal(1));
}
try_map_encode(&JISX0213_EMP_ENCMAP, &JISX0213_EMP_ENCMAP_DATA, c & 0xffff)
.map(|code| (code, 1))
.ok_or(EncodeOne::Illegal(1))
} else {
Err(EncodeOne::Illegal(1))
}
}
pub(super) fn encode_euc_jis_2004(
input: &[u32],
final_input: bool,
config_2000: bool,
) -> EncodeOne {
let c = input[0];
if c < 0x80 {
return bytes1(c as u8);
}
let (code, consumed) = match encode_jisx0213_code(input, final_input, config_2000, true) {
Ok(value) => value,
Err(EncodeOne::Illegal(_)) if (0xff61..=0xff9f).contains(&c) => {
return bytes2(0x8e, (c - 0xfec0) as u8, 1);
}
Err(EncodeOne::Illegal(_)) if c == 0xff3c => (0x2140, 1),
Err(EncodeOne::Illegal(_)) if c == 0xff5e => (0x2232, 1),
Err(error) => return error,
};
if code & 0x8000 != 0 {
bytes3(0x8f, (code >> 8) as u8, code as u8 | 0x80, consumed)
} else {
bytes2((code >> 8) as u8 | 0x80, code as u8 | 0x80, consumed)
}
}
pub(super) fn decode_jisx0213_plane1(
c1: u8,
c2: u8,
config_2000: bool,
consumed: usize,
compat_fullwidth_reverse_solidus: bool,
compat_fullwidth_tilde: bool,
) -> DecodeOne {
if config_2000 && emulate_2000_plane1(c1, c2) {
return DecodeOne::Illegal(consumed);
}
if compat_fullwidth_reverse_solidus && c1 == 0x21 && c2 == 0x40 {
return DecodeOne::Char(0xff3c, consumed);
}
if compat_fullwidth_tilde && c1 == 0x22 && c2 == 0x32 {
return DecodeOne::Char(0xff5e, consumed);
}
if let Some(value) = try_map_decode(&JISX0208_DECMAP, &JISX0208_DECMAP_DATA, c1, c2)
.or_else(|| try_map_decode(&JISX0213_1_BMP_DECMAP, &JISX0213_1_BMP_DECMAP_DATA, c1, c2))
{
return DecodeOne::Char(value, consumed);
}
if let Some(value) = try_map_decode(&JISX0213_1_EMP_DECMAP, &JISX0213_1_EMP_DECMAP_DATA, c1, c2)
{
return DecodeOne::Char(EMPBASE | value, consumed);
}
if let Some(value) = try_pair_decode(c1, c2) {
return DecodeOne::Pair(value >> 16, value & 0xffff, consumed);
}
illegal_decode_lead()
}
pub(super) fn decode_jisx0213_plane2(
c1: u8,
c2: u8,
config_2000: bool,
consumed: usize,
) -> DecodeOne {
if config_2000 && c1 == 0x7d && c2 == 0x3b {
return DecodeOne::Char(0x9b1d, consumed);
}
if let Some(value) = try_map_decode(&JISX0213_2_BMP_DECMAP, &JISX0213_2_BMP_DECMAP_DATA, c1, c2)
{
return DecodeOne::Char(value, consumed);
}
if let Some(value) = try_map_decode(&JISX0213_2_EMP_DECMAP, &JISX0213_2_EMP_DECMAP_DATA, c1, c2)
{
return DecodeOne::Char(EMPBASE | value, consumed);
}
illegal_decode_lead()
}
pub(super) fn decode_euc_jis_2004(input: &[u8], config_2000: bool) -> DecodeOne {
let c = input[0];
if c < 0x80 {
return DecodeOne::Char(u32::from(c), 1);
}
if c == 0x8e {
if input.len() < 2 {
return DecodeOne::Incomplete;
}
return if (0xa1..=0xdf).contains(&input[1]) {
DecodeOne::Char(0xfec0 + u32::from(input[1]), 2)
} else {
illegal_decode_lead()
};
}
if c == 0x8f {
if input.len() < 3 {
return DecodeOne::Incomplete;
}
let c2 = input[1] ^ 0x80;
let c3 = input[2] ^ 0x80;
let decoded = decode_jisx0213_plane2(c2, c3, config_2000, 3);
if !matches!(decoded, DecodeOne::Illegal(_)) {
return decoded;
}
return try_map_decode(&JISX0212_DECMAP, &JISX0212_DECMAP_DATA, c2, c3)
.map_or_else(illegal_decode_lead, |value| DecodeOne::Char(value, 3));
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
decode_jisx0213_plane1(c ^ 0x80, input[1] ^ 0x80, config_2000, 2, true, true)
}
pub(super) fn encode_shift_jis_2004(
input: &[u32],
final_input: bool,
config_2000: bool,
) -> EncodeOne {
let c = input[0];
if let Some(code) = jisx0201_encode(c) {
return bytes1(code);
}
let (code, consumed) = match encode_jisx0213_code(input, final_input, config_2000, false) {
Ok((code, consumed)) if code & 0x8000 == 0 || c > 0xffff => (code, consumed),
Ok((code, consumed)) => {
if (config_2000 && c == 0x9b1d)
|| try_map_encode(&JISX0213_BMP_ENCMAP, &JISX0213_BMP_ENCMAP_DATA, c).is_some()
{
(code, consumed)
} else {
return EncodeOne::Illegal(1);
}
}
Err(error) => return error,
};
let mut c1 = i32::from(code >> 8);
let mut c2 = i32::from(code & 0xff) - 0x21;
if c1 & 0x80 != 0 {
if c1 >= 0xee {
c1 -= 0x87;
} else if c1 >= 0xac || c1 == 0xa8 {
c1 -= 0x49;
} else {
c1 -= 0x43;
}
} else {
c1 -= 0x21;
}
if c1 & 1 != 0 {
c2 += 0x5e;
}
c1 >>= 1;
bytes2(
(c1 + if c1 < 0x1f { 0x81 } else { 0xc1 }) as u8,
(c2 + if c2 < 0x3f { 0x40 } else { 0x41 }) as u8,
consumed,
)
}
pub(super) fn decode_shift_jis_2004(input: &[u8], config_2000: bool) -> DecodeOne {
let c = input[0];
if let Some(value) = jisx0201_decode(c) {
return DecodeOne::Char(value, 1);
}
if !((0x81..=0x9f).contains(&c) || (0xe0..=0xfc).contains(&c)) {
return illegal_decode_lead();
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
let c2 = input[1];
if c2 < 0x40 || c2 == 0x7f || c2 > 0xfc {
return illegal_decode_lead();
}
let lead = if c < 0xe0 { c - 0x81 } else { c - 0xc1 };
let trail = if c2 < 0x80 { c2 - 0x40 } else { c2 - 0x41 };
let mut c1 = 2 * lead + if trail < 0x5e { 0 } else { 1 };
let c2 = (if trail < 0x5e { trail } else { trail - 0x5e }) + 0x21;
if c1 < 0x5e {
c1 += 0x21;
return decode_jisx0213_plane1(c1, c2, config_2000, 2, false, false);
}
c1 = if c1 >= 0x67 {
c1 + 0x07
} else if c1 >= 0x63 || c1 == 0x5f {
c1 - 0x37
} else {
c1 - 0x3d
};
decode_jisx0213_plane2(c1, c2, config_2000, 2)
}
#[cfg(test)]
mod tests {
use super::super::{Codec, DecodeOne, EncodeOne, decode_one, encode_one};
fn encode(codec: Codec, text: &str) -> Vec<u8> {
let points: Vec<u32> = text.chars().map(u32::from).collect();
let mut output = Vec::new();
let mut position = 0;
while position < points.len() {
let EncodeOne::Bytes(bytes, len, consumed) =
encode_one(codec, &points[position..], true, &mut [0; 8])
else {
panic!("U+{:04X} is not encodable", points[position]);
};
output.extend_from_slice(&bytes[..len]);
position += consumed;
}
output
}
fn decode(codec: Codec, input: &[u8]) -> String {
let mut output = String::new();
let mut position = 0;
while position < input.len() {
match decode_one(codec, &input[position..], &mut [0; 8]) {
DecodeOne::Char(value, consumed) => {
output.push(char::from_u32(value).unwrap());
position += consumed;
}
DecodeOne::Pair(first, second, consumed) => {
output.push(char::from_u32(first).unwrap());
output.push(char::from_u32(second).unwrap());
position += consumed;
}
_ => panic!("{:x?} is not decodable", &input[position..]),
}
}
output
}
#[test]
fn pypy_jp_oracle_vectors() {
for (codec, text, expected, decoded) in [
(
Codec::ShiftJis,
"日本語¥‾＼",
&b"\x93\xfa\x96\x7b\x8c\xea\\~\x81_"[..],
"日本語\\~＼",
),
(
Codec::Cp932,
"日本語\u{f8f0}\u{e000}",
&b"\x93\xfa\x96\x7b\x8c\xea\xa0\xf0@"[..],
"日本語\u{f8f0}\u{e000}",
),
(
Codec::EucJp,
"日本語¥‾＼",
&b"\xc6\xfc\xcb\xdc\xb8\xec\\~\xa1\xc0"[..],
"日本語\\~＼",
),
(
Codec::ShiftJis2004,
"日本語か\u{309a}",
&b"\x93\xfa\x96\x7b\x8c\xea\x82\xf5"[..],
"日本語か\u{309a}",
),
(
Codec::EucJis2004,
"日本語か\u{309a}",
&b"\xc6\xfc\xcb\xdc\xb8\xec\xa4\xf7"[..],
"日本語か\u{309a}",
),
] {
assert_eq!(encode(codec, text), expected, "{codec:?}");
assert_eq!(decode(codec, expected), decoded, "{codec:?}");
}
for codec in [
Codec::ShiftJis2004,
Codec::EucJis2004,
Codec::EucJisX0213,
Codec::ShiftJisX0213,
] {
assert!(encode(codec, "フルーツ\0").ends_with(&[0]), "{codec:?}");
}
assert_eq!(decode(Codec::ShiftJis2004, b"\x81\x5f"), "\\");
assert_eq!(decode(Codec::ShiftJisX0213, b"\x81\x5f"), "\\");
// PyPy `_codecs_jp.c::shift_jis_2004_decoder` consults
// `jisx0208` before the JIS X 0213 tables and therefore keeps the
// ASCII tilde mapping at row 0x22, cell 0x32.
assert_eq!(decode(Codec::ShiftJis2004, b"\x81\xb0"), "~");
assert_eq!(decode(Codec::ShiftJisX0213, b"\x81\xb0"), "~");
// `_codecs_jp.c::euc_jis_2004_encoder` accepts the codeset-2 bit
// from `jisxcommon`; only `shift_jis_2004_encoder` rejects it.
assert!(matches!(
encode_one(Codec::EucJis2004, &[0x010a], true, &mut [0; 8]),
EncodeOne::Bytes(bytes, 3, 1) if bytes[..3] == [0x8f, 0xaa, 0xaf]
));
}
#[test]
fn jisx0213_2000_emulation_matches_pypy_tables() {
assert!(matches!(
encode_one(Codec::EucJisX0213, &[0x9b1c], true, &mut [0; 8]),
EncodeOne::Illegal(1)
));
assert!(matches!(
encode_one(Codec::EucJisX0213, &[0x9b1d], true, &mut [0; 8]),
EncodeOne::Bytes(bytes, 3, 1) if bytes[..3] == [0x8f, 0xfd, 0xbb]
));
assert!(matches!(
encode_one(Codec::ShiftJisX0213, &[0x9b1d], true, &mut [0; 8]),
EncodeOne::Bytes(bytes, 2, 1) if bytes[..2] == [0xfc, 0x5a]
));
assert_eq!(decode(Codec::EucJisX0213, b"\x8f\xfd\xbb"), "\u{9b1d}");
assert_eq!(decode(Codec::ShiftJisX0213, b"\xfc\x5a"), "\u{9b1d}");
// `EMULATE_JISX0213_2000_DECODE_PLANE1` returns the complete
// malformed sequence width. Ordinary final map misses retain the
// one-byte error span selected by `illegal_decode_lead`.
assert!(matches!(
decode_one(Codec::EucJisX0213, b"\xae\xa1", &mut [0; 8]),
DecodeOne::Illegal(2)
));
assert!(matches!(
decode_one(Codec::ShiftJisX0213, b"\x87\x9f", &mut [0; 8]),
DecodeOne::Illegal(2)
));
}
#[test]
fn every_japanese_three_byte_candidate_and_scalar_is_total() {
for codec in [Codec::EucJp, Codec::EucJis2004, Codec::EucJisX0213] {
for second in 0..=u8::MAX {
for third in 0..=u8::MAX {
let _ = decode_one(codec, &[0x8f, second, third], &mut [0; 8]);
}
}
}
for codec in [
Codec::ShiftJis,
Codec::Cp932,
Codec::EucJp,
Codec::ShiftJis2004,
Codec::EucJis2004,
Codec::EucJisX0213,
Codec::ShiftJisX0213,
] {
for value in 0..=0x10ffff {
if !(0xd800..=0xdfff).contains(&value) {
let _ = encode_one(codec, &[value, 0x309a], true, &mut [0; 8]);
}
}
}
}
}
crates/common/src/encodings/cjk/kr.rs
+418
View File
@@ -0,0 +1,418 @@
//! Line-by-line Rust port of PyPy cjkcodecs `_codecs_kr.c`.
use super::mappings_kr::{
CP949_ENCMAP, CP949EXT_DECMAP, CP949EXT_DECMAP_DATA, KSX1001_DECMAP, KSX1001_DECMAP_DATA,
MapIndex, NOCHAR, UNIINV,
};
use super::{DecodeOne, EncodeOne};
const EUCKR_JAMO_FIRSTBYTE: u8 = 0xa4;
const EUCKR_JAMO_FILLER: u8 = 0xd4;
const U2CGK_CHOSEONG: [u8; 19] = [
0xa1, 0xa2, 0xa4, 0xa7, 0xa8, 0xa9, 0xb1, 0xb2, 0xb3, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb,
0xbc, 0xbd, 0xbe,
];
const U2CGK_JUNGSEONG: [u8; 21] = [
0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce,
0xcf, 0xd0, 0xd1, 0xd2, 0xd3,
];
const U2CGK_JONGSEONG: [u8; 28] = [
0xd4, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0,
0xb1, 0xb2, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xba, 0xbb, 0xbc, 0xbd, 0xbe,
];
const NONE_CGK: u8 = 127;
const CGK2U_CHOSEONG: [u8; 30] = [
0, 1, NONE_CGK, 2, NONE_CGK, NONE_CGK, 3, 4, 5, NONE_CGK, NONE_CGK, NONE_CGK, NONE_CGK,
NONE_CGK, NONE_CGK, NONE_CGK, 6, 7, 8, NONE_CGK, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
];
const CGK2U_JONGSEONG: [u8; 30] = [
1, 2, 3, 4, 5, 6, 7, NONE_CGK, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, NONE_CGK, 18, 19, 20, 21,
22, NONE_CGK, 23, 24, 25, 26, 27,
];
const U2JOHABIDX_CHOSEONG: [u8; 32] = [
0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11,
0x12, 0x13, 0x14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
];
const U2JOHABIDX_JUNGSEONG: [u8; 32] = [
0x03, 0x04, 0x05, 0x06, 0x07, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x12, 0x13, 0x14, 0x15, 0x16,
0x17, 0x1a, 0x1b, 0x1c, 0x1d, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
];
const U2JOHABIDX_JONGSEONG: [u8; 32] = [
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10,
0x11, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0, 0, 0, 0,
];
const U2JOHABJAMO: [u16; 51] = [
0x8841, 0x8c41, 0x8444, 0x9041, 0x8446, 0x8447, 0x9441, 0x9841, 0x9c41, 0x844a, 0x844b, 0x844c,
0x844d, 0x844e, 0x844f, 0x8450, 0xa041, 0xa441, 0xa841, 0x8454, 0xac41, 0xb041, 0xb441, 0xb841,
0xbc41, 0xc041, 0xc441, 0xc841, 0xcc41, 0xd041, 0x8461, 0x8481, 0x84a1, 0x84c1, 0x84e1, 0x8541,
0x8561, 0x8581, 0x85a1, 0x85c1, 0x85e1, 0x8641, 0x8661, 0x8681, 0x86a1, 0x86c1, 0x86e1, 0x8741,
0x8761, 0x8781, 0x87a1,
];
const FILL: u8 = 0xfd;
const NONE: u8 = 0xff;
const JOHABIDX_CHOSEONG: [u8; 32] = [
NONE, FILL, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d,
0x0e, 0x0f, 0x10, 0x11, 0x12, NONE, NONE, NONE, NONE, NONE, NONE, NONE, NONE, NONE, NONE, NONE,
];
const JOHABIDX_JUNGSEONG: [u8; 32] = [
NONE, NONE, FILL, 0x00, 0x01, 0x02, 0x03, 0x04, NONE, NONE, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a,
NONE, NONE, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, NONE, NONE, 0x11, 0x12, 0x13, 0x14, NONE, NONE,
];
const JOHABIDX_JONGSEONG: [u8; 32] = [
NONE, FILL, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
0x0f, 0x10, NONE, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, NONE, NONE,
];
const JOHABJAMO_CHOSEONG: [u8; 32] = [
NONE, FILL, 0x31, 0x32, 0x34, 0x37, 0x38, 0x39, 0x41, 0x42, 0x43, 0x45, 0x46, 0x47, 0x48, 0x49,
0x4a, 0x4b, 0x4c, 0x4d, 0x4e, NONE, NONE, NONE, NONE, NONE, NONE, NONE, NONE, NONE, NONE, NONE,
];
const JOHABJAMO_JUNGSEONG: [u8; 32] = [
NONE, NONE, FILL, 0x4f, 0x50, 0x51, 0x52, 0x53, NONE, NONE, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59,
NONE, NONE, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, NONE, NONE, 0x60, 0x61, 0x62, 0x63, NONE, NONE,
];
const JOHABJAMO_JONGSEONG: [u8; 32] = [
NONE, FILL, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
0x40, 0x41, NONE, 0x42, 0x44, 0x45, 0x46, 0x47, 0x48, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, NONE, NONE,
];
fn try_map_decode(index: &[MapIndex; 256], data: &[u16], c1: u8, c2: u8) -> Option<u32> {
let page = index[c1 as usize];
if !page.present || c2 < page.bottom || c2 > page.top {
return None;
}
let value = data[page.offset + usize::from(c2 - page.bottom)];
(value != UNIINV).then_some(u32::from(value))
}
fn try_map_encode(c: u32) -> Option<u16> {
if c > 0xffff {
return None;
}
let page = CP949_ENCMAP[(c >> 8) as usize];
let low = c as u8;
if !page.present || low < page.bottom || low > page.top {
return None;
}
let value = super::mappings_kr::CP949_ENCMAP_DATA[page.offset + usize::from(low - page.bottom)];
(value != NOCHAR).then_some(value)
}
fn bytes2(c1: u8, c2: u8) -> EncodeOne {
let mut output = [0; 8];
output[0] = c1;
output[1] = c2;
EncodeOne::Bytes(output, 2, 1)
}
pub(super) fn encode_euc_kr(c: u32) -> EncodeOne {
if c < 0x80 {
let mut output = [0; 8];
output[0] = c as u8;
return EncodeOne::Bytes(output, 1, 1);
}
let Some(code) = try_map_encode(c) else {
return EncodeOne::Illegal(1);
};
if code & 0x8000 == 0 {
return bytes2((code >> 8) as u8 | 0x80, code as u8 | 0x80);
}
// `_codecs_kr.c::ENCODER(euc_kr)`: KS X 1001:1998 Annex 3 make-up
// sequence, retaining PyPy's syllable-composition order.
debug_assert!((0xac00..=0xd7a3).contains(&c));
let syllable = c - 0xac00;
let output = [
EUCKR_JAMO_FIRSTBYTE,
EUCKR_JAMO_FILLER,
EUCKR_JAMO_FIRSTBYTE,
U2CGK_CHOSEONG[(syllable / 588) as usize],
EUCKR_JAMO_FIRSTBYTE,
U2CGK_JUNGSEONG[((syllable / 28) % 21) as usize],
EUCKR_JAMO_FIRSTBYTE,
U2CGK_JONGSEONG[(syllable % 28) as usize],
];
EncodeOne::Bytes(output, 8, 1)
}
pub(super) fn decode_euc_kr(input: &[u8]) -> DecodeOne {
let c = input[0];
if c < 0x80 {
return DecodeOne::Char(u32::from(c), 1);
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
if c == EUCKR_JAMO_FIRSTBYTE && input[1] == EUCKR_JAMO_FILLER {
if input.len() < 8 {
return DecodeOne::Incomplete;
}
if input[2] != EUCKR_JAMO_FIRSTBYTE
|| input[4] != EUCKR_JAMO_FIRSTBYTE
|| input[6] != EUCKR_JAMO_FIRSTBYTE
{
return DecodeOne::Illegal(1);
}
let cho = if (0xa1..=0xbe).contains(&input[3]) {
CGK2U_CHOSEONG[(input[3] - 0xa1) as usize]
} else {
NONE_CGK
};
let jung = if (0xbf..=0xd3).contains(&input[5]) {
input[5] - 0xbf
} else {
NONE_CGK
};
let jong = if input[7] == EUCKR_JAMO_FILLER {
0
} else if (0xa1..=0xbe).contains(&input[7]) {
CGK2U_JONGSEONG[(input[7] - 0xa1) as usize]
} else {
NONE_CGK
};
if cho == NONE_CGK || jung == NONE_CGK || jong == NONE_CGK {
return DecodeOne::Illegal(1);
}
return DecodeOne::Char(
0xac00 + u32::from(cho) * 588 + u32::from(jung) * 28 + u32::from(jong),
8,
);
}
try_map_decode(
&KSX1001_DECMAP,
&KSX1001_DECMAP_DATA,
c ^ 0x80,
input[1] ^ 0x80,
)
.map_or(DecodeOne::Illegal(1), |value| DecodeOne::Char(value, 2))
}
pub(super) fn encode_cp949(c: u32) -> EncodeOne {
if c < 0x80 {
let mut output = [0; 8];
output[0] = c as u8;
return EncodeOne::Bytes(output, 1, 1);
}
let Some(code) = try_map_encode(c) else {
return EncodeOne::Illegal(1);
};
bytes2(
(code >> 8) as u8 | 0x80,
if code & 0x8000 != 0 {
code as u8
} else {
code as u8 | 0x80
},
)
}
pub(super) fn decode_cp949(input: &[u8]) -> DecodeOne {
let c = input[0];
if c < 0x80 {
return DecodeOne::Char(u32::from(c), 1);
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
let value = try_map_decode(
&KSX1001_DECMAP,
&KSX1001_DECMAP_DATA,
c ^ 0x80,
input[1] ^ 0x80,
)
.or_else(|| try_map_decode(&CP949EXT_DECMAP, &CP949EXT_DECMAP_DATA, c, input[1]));
value.map_or(DecodeOne::Illegal(1), |value| DecodeOne::Char(value, 2))
}
pub(super) fn encode_johab(mut c: u32) -> EncodeOne {
if c < 0x80 {
let mut output = [0; 8];
output[0] = c as u8;
return EncodeOne::Bytes(output, 1, 1);
}
let code = if (0xac00..=0xd7a3).contains(&c) {
c -= 0xac00;
0x8000
| u16::from(U2JOHABIDX_CHOSEONG[(c / 588) as usize]) << 10
| u16::from(U2JOHABIDX_JUNGSEONG[((c / 28) % 21) as usize]) << 5
| u16::from(U2JOHABIDX_JONGSEONG[(c % 28) as usize])
} else if (0x3131..=0x3163).contains(&c) {
U2JOHABJAMO[(c - 0x3131) as usize]
} else {
let Some(code) = try_map_encode(c) else {
return EncodeOne::Illegal(1);
};
debug_assert_eq!(code & 0x8000, 0);
let c1 = (code >> 8) as u8;
let c2 = code as u8;
if !(((0x21..=0x2c).contains(&c1) || (0x4a..=0x7d).contains(&c1))
&& (0x21..=0x7e).contains(&c2))
{
return EncodeOne::Illegal(1);
}
let t1 = if c1 < 0x4a {
u16::from(c1 - 0x21) + 0x1b2
} else {
u16::from(c1 - 0x21) + 0x197
};
let t2 = (if t1 & 1 != 0 { 0x5e } else { 0 }) + (c2 - 0x21);
return bytes2(
(t1 >> 1) as u8,
if t2 < 0x4e { t2 + 0x31 } else { t2 + 0x43 },
);
};
bytes2((code >> 8) as u8, code as u8)
}
pub(super) fn decode_johab(input: &[u8]) -> DecodeOne {
let c = input[0];
if c < 0x80 {
return DecodeOne::Char(u32::from(c), 1);
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
let c2 = input[1];
if c < 0xd8 {
let c_cho = (c >> 2) & 0x1f;
let c_jung = ((c << 3) | c2 >> 5) & 0x1f;
let c_jong = c2 & 0x1f;
let i_cho = JOHABIDX_CHOSEONG[c_cho as usize];
let i_jung = JOHABIDX_JUNGSEONG[c_jung as usize];
let i_jong = JOHABIDX_JONGSEONG[c_jong as usize];
if i_cho == NONE || i_jung == NONE || i_jong == NONE {
return DecodeOne::Illegal(1);
}
let value = if i_cho == FILL {
if i_jung == FILL {
if i_jong == FILL {
0x3000
} else {
0x3100 | u32::from(JOHABJAMO_JONGSEONG[c_jong as usize])
}
} else if i_jong == FILL {
0x3100 | u32::from(JOHABJAMO_JUNGSEONG[c_jung as usize])
} else {
return DecodeOne::Illegal(1);
}
} else if i_jung == FILL {
if i_jong == FILL {
0x3100 | u32::from(JOHABJAMO_CHOSEONG[c_cho as usize])
} else {
return DecodeOne::Illegal(1);
}
} else {
0xac00
+ u32::from(i_cho) * 588
+ u32::from(i_jung) * 28
+ if i_jong == FILL { 0 } else { u32::from(i_jong) }
};
return DecodeOne::Char(value, 2);
}
if c == 0xdf
|| c > 0xf9
|| c2 < 0x31
|| (0x80..0x91).contains(&c2)
|| c2 & 0x7f == 0x7f
|| (c == 0xda && (0xa1..=0xd3).contains(&c2))
{
return DecodeOne::Illegal(1);
}
// `_codecs_kr.c::DECODER(johab)` performs this in promoted signed `int`
// and assigns it to an `unsigned char`; c == 0xd8 therefore wraps -2 to
// 254 before the page lookup.
let mut t1 = if c < 0xe0 {
(2 * (i16::from(c) - 0xd9)) as u8
} else {
(2 * u16::from(c) - 0x197) as u8
};
let mut t2 = if c2 < 0x91 { c2 - 0x31 } else { c2 - 0x43 };
t1 = t1
.wrapping_add(if t2 < 0x5e { 0 } else { 1 })
.wrapping_add(0x21);
t2 = (if t2 < 0x5e { t2 } else { t2 - 0x5e }) + 0x21;
try_map_decode(&KSX1001_DECMAP, &KSX1001_DECMAP_DATA, t1, t2)
.map_or(DecodeOne::Illegal(1), |value| DecodeOne::Char(value, 2))
}
#[cfg(test)]
mod tests {
use super::super::{Codec, decode_one, encode_one};
use super::*;
fn encode(codec: Codec, text: &str) -> Vec<u8> {
let mut output = Vec::new();
for c in text.chars() {
let points = [c as u32];
let EncodeOne::Bytes(bytes, len, 1) = encode_one(codec, &points, true, &mut [0; 8])
else {
panic!("{c:?} is not encodable");
};
output.extend_from_slice(&bytes[..len]);
}
output
}
fn decode(codec: Codec, input: &[u8]) -> String {
let mut output = String::new();
let mut position = 0;
while position < input.len() {
let DecodeOne::Char(value, len) = decode_one(codec, &input[position..], &mut [0; 8])
else {
panic!("{:x?} is not decodable", &input[position..]);
};
output.push(char::from_u32(value).unwrap());
position += len;
}
output
}
#[test]
fn pypy_kr_oracle_vectors() {
let text = "가힣똠ㄱ漢字";
let cases = [
(
Codec::EucKr,
&[
0xb0, 0xa1, 0xa4, 0xd4, 0xa4, 0xbe, 0xa4, 0xd3, 0xa4, 0xbe, 0xa4, 0xd4, 0xa4,
0xa8, 0xa4, 0xc7, 0xa4, 0xb1, 0xa4, 0xa1, 0xf9, 0xd3, 0xed, 0xae,
][..],
),
(
Codec::Cp949,
&[
0xb0, 0xa1, 0xc6, 0x52, 0x8c, 0x63, 0xa4, 0xa1, 0xf9, 0xd3, 0xed, 0xae,
][..],
),
(
Codec::Johab,
&[
0x88, 0x61, 0xd3, 0xbd, 0x99, 0xb1, 0x88, 0x41, 0xf7, 0xd3, 0xf1, 0xae,
][..],
),
];
for (codec, expected) in cases {
assert_eq!(encode(codec, text), expected);
assert_eq!(decode(codec, expected), text);
}
}
#[test]
fn malformed_candidates_reject_the_lead_byte() {
for (codec, input) in [
(Codec::EucKr, &[0xff, 0xff][..]),
(Codec::Cp949, &[0x81, 0x00][..]),
(Codec::Johab, &[0x84, 0x00][..]),
(Codec::Johab, &[0xd8, 0x31][..]),
] {
assert!(matches!(
decode_one(codec, input, &mut [0; 8]),
DecodeOne::Illegal(1)
));
}
}
}
crates/common/src/encodings/cjk/mappings_cn.rs
Generated
+8120
View File
File diff suppressed because it is too large. Load diff
crates/common/src/encodings/cjk/mappings_hk.rs
Generated
+7600
View File
File diff suppressed because it is too large. Load diff
crates/common/src/encodings/cjk/mappings_jisx0213_pair.rs
Generated
+550
View File
@@ -0,0 +1,550 @@
// Generated mechanically from PyPy's cjkcodecs mapping header by
// `scripts/port_cjk_mappings.py`. Do not change the page layout:
// `cjkcodecs.h::_TRYMAP_DEC` / `_TRYMAP_ENC` index these exact arrays.
pub(super) const UNIINV: u16 = 0xfffe;
#[allow(dead_code)]
pub(super) const NOCHAR: u16 = 0xffff;
#[allow(dead_code)]
pub(super) const MULTIC: u16 = 0xfffe;
#[allow(dead_code)]
pub(super) const DBCINV: u16 = 0xfffd;
#[derive(Clone, Copy)]
pub(super) struct MapIndex {
pub(super) offset: usize,
pub(super) bottom: u8,
pub(super) top: u8,
pub(super) present: bool,
}
const EMPTY_INDEX: MapIndex = MapIndex {
offset: 0,
bottom: 0,
top: 0,
present: false,
};
pub(super) static JISX0213_PAIR_DECMAP_DATA: [u32; 49] = [
810234010,
810365082,
810496154,
810627226,
810758298,
816525466,
816656538,
816787610,
816918682,
817049754,
817574042,
818163866,
818426010,
838283418,
15074048,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
39060224,
39060225,
42730240,
42730241,
39387904,
39387905,
39453440,
39453441,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
UNIINV as u32,
48825061,
48562921,
];
pub(super) static JISX0213_PAIR_DECMAP: [MapIndex; 256] = [
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
MapIndex {
offset: 0,
bottom: 119,
top: 123,
present: true,
},
MapIndex {
offset: 5,
bottom: 119,
top: 126,
present: true,
},
MapIndex {
offset: 13,
bottom: 120,
top: 120,
present: true,
},
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
MapIndex {
offset: 14,
bottom: 68,
top: 102,
present: true,
},
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
EMPTY_INDEX,
];
#[derive(Clone, Copy)]
pub(super) struct PairEncodeMap {
pub(super) uniseq: u32,
pub(super) code: u16,
}
pub(super) static JISX0213_PAIR_ENCMAP: [PairEncodeMap; 46] = [
PairEncodeMap {
uniseq: 0x00e60000,
code: 0x295c,
},
PairEncodeMap {
uniseq: 0x00e60300,
code: 0x2b44,
},
PairEncodeMap {
uniseq: 0x02540000,
code: 0x2b38,
},
PairEncodeMap {
uniseq: 0x02540300,
code: 0x2b48,
},
PairEncodeMap {
uniseq: 0x02540301,
code: 0x2b49,
},
PairEncodeMap {
uniseq: 0x02590000,
code: 0x2b30,
},
PairEncodeMap {
uniseq: 0x02590300,
code: 0x2b4c,
},
PairEncodeMap {
uniseq: 0x02590301,
code: 0x2b4d,
},
PairEncodeMap {
uniseq: 0x025a0000,
code: 0x2b43,
},
PairEncodeMap {
uniseq: 0x025a0300,
code: 0x2b4e,
},
PairEncodeMap {
uniseq: 0x025a0301,
code: 0x2b4f,
},
PairEncodeMap {
uniseq: 0x028c0000,
code: 0x2b37,
},
PairEncodeMap {
uniseq: 0x028c0300,
code: 0x2b4a,
},
PairEncodeMap {
uniseq: 0x028c0301,
code: 0x2b4b,
},
PairEncodeMap {
uniseq: 0x02e50000,
code: 0x2b60,
},
PairEncodeMap {
uniseq: 0x02e502e9,
code: 0x2b66,
},
PairEncodeMap {
uniseq: 0x02e90000,
code: 0x2b64,
},
PairEncodeMap {
uniseq: 0x02e902e5,
code: 0x2b65,
},
PairEncodeMap {
uniseq: 0x304b0000,
code: 0x242b,
},
PairEncodeMap {
uniseq: 0x304b309a,
code: 0x2477,
},
PairEncodeMap {
uniseq: 0x304d0000,
code: 0x242d,
},
PairEncodeMap {
uniseq: 0x304d309a,
code: 0x2478,
},
PairEncodeMap {
uniseq: 0x304f0000,
code: 0x242f,
},
PairEncodeMap {
uniseq: 0x304f309a,
code: 0x2479,
},
PairEncodeMap {
uniseq: 0x30510000,
code: 0x2431,
},
PairEncodeMap {
uniseq: 0x3051309a,
code: 0x247a,
},
PairEncodeMap {
uniseq: 0x30530000,
code: 0x2433,
},
PairEncodeMap {
uniseq: 0x3053309a,
code: 0x247b,
},
PairEncodeMap {
uniseq: 0x30ab0000,
code: 0x252b,
},
PairEncodeMap {
uniseq: 0x30ab309a,
code: 0x2577,
},
PairEncodeMap {
uniseq: 0x30ad0000,
code: 0x252d,
},
PairEncodeMap {
uniseq: 0x30ad309a,
code: 0x2578,
},
PairEncodeMap {
uniseq: 0x30af0000,
code: 0x252f,
},
PairEncodeMap {
uniseq: 0x30af309a,
code: 0x2579,
},
PairEncodeMap {
uniseq: 0x30b10000,
code: 0x2531,
},
PairEncodeMap {
uniseq: 0x30b1309a,
code: 0x257a,
},
PairEncodeMap {
uniseq: 0x30b30000,
code: 0x2533,
},
PairEncodeMap {
uniseq: 0x30b3309a,
code: 0x257b,
},
PairEncodeMap {
uniseq: 0x30bb0000,
code: 0x253b,
},
PairEncodeMap {
uniseq: 0x30bb309a,
code: 0x257c,
},
PairEncodeMap {
uniseq: 0x30c40000,
code: 0x2544,
},
PairEncodeMap {
uniseq: 0x30c4309a,
code: 0x257d,
},
PairEncodeMap {
uniseq: 0x30c80000,
code: 0x2548,
},
PairEncodeMap {
uniseq: 0x30c8309a,
code: 0x257e,
},
PairEncodeMap {
uniseq: 0x31f70000,
code: 0x2675,
},
PairEncodeMap {
uniseq: 0x31f7309a,
code: 0x2678,
},
];
crates/common/src/encodings/cjk/mappings_jp.rs
Generated
+13454
View File
File diff suppressed because it is too large. Load diff
crates/common/src/encodings/cjk/mappings_kr.rs
Generated
+6383
View File
File diff suppressed because it is too large. Load diff
crates/common/src/encodings/cjk/mappings_tw.rs
Generated
+5181
View File
File diff suppressed because it is too large. Load diff
crates/common/src/encodings/cjk/mod.rs
+297
View File
@@ -0,0 +1,297 @@
//! Rust ports of PyPy's `pypy/module/_multibytecodec/src/cjkcodecs` units.
//!
//! This module contains only the VM-independent codec engines and mapping
//! tables. Interpreter-specific objects, buffers, and error handlers belong in
//! the VM adapter that calls this API.
// Keep the control flow and arithmetic visibly aligned with PyPy's C engines.
#![allow(
clippy::bool_to_int_with_if,
clippy::collapsible_if,
clippy::trivially_copy_pass_by_ref,
clippy::unnested_or_patterns
)]
mod cn;
mod hk;
mod iso2022;
mod jp;
mod kr;
mod mappings_cn;
mod mappings_hk;
mod mappings_jisx0213_pair;
mod mappings_jp;
mod mappings_kr;
mod mappings_tw;
mod tw;
/// A VM-independent CJK codec implemented by this module.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Codec {
EucKr,
Cp949,
Johab,
Big5,
Cp950,
Big5Hkscs,
ShiftJis,
Cp932,
EucJp,
ShiftJis2004,
EucJis2004,
EucJisX0213,
ShiftJisX0213,
Iso2022Kr,
Iso2022Jp,
Iso2022Jp1,
Iso2022Jp2,
Iso2022Jp2004,
Iso2022Jp3,
Iso2022JpExt,
Gb2312,
Gbk,
Gb18030,
Hz,
}
impl Codec {
/// Look up the canonical codec name used by Python's CJK codec modules.
#[must_use]
pub fn from_name(name: &str) -> Option<Self> {
match name {
"euc_kr" => Some(Self::EucKr),
"cp949" => Some(Self::Cp949),
"johab" => Some(Self::Johab),
"big5" => Some(Self::Big5),
"cp950" => Some(Self::Cp950),
"big5hkscs" => Some(Self::Big5Hkscs),
"shift_jis" => Some(Self::ShiftJis),
"cp932" => Some(Self::Cp932),
"euc_jp" => Some(Self::EucJp),
"shift_jis_2004" => Some(Self::ShiftJis2004),
"euc_jis_2004" => Some(Self::EucJis2004),
"euc_jisx0213" => Some(Self::EucJisX0213),
"shift_jisx0213" => Some(Self::ShiftJisX0213),
"iso2022_kr" => Some(Self::Iso2022Kr),
"iso2022_jp" => Some(Self::Iso2022Jp),
"iso2022_jp_1" => Some(Self::Iso2022Jp1),
"iso2022_jp_2" => Some(Self::Iso2022Jp2),
"iso2022_jp_2004" => Some(Self::Iso2022Jp2004),
"iso2022_jp_3" => Some(Self::Iso2022Jp3),
"iso2022_jp_ext" => Some(Self::Iso2022JpExt),
"gb2312" => Some(Self::Gb2312),
"gbk" => Some(Self::Gbk),
"gb18030" => Some(Self::Gb18030),
"hz" => Some(Self::Hz),
_ => None,
}
}
}
fn is_iso2022(codec: Codec) -> bool {
matches!(
codec,
Codec::Iso2022Kr
| Codec::Iso2022Jp
| Codec::Iso2022Jp1
| Codec::Iso2022Jp2
| Codec::Iso2022Jp2004
| Codec::Iso2022Jp3
| Codec::Iso2022JpExt
)
}
/// Construct the initial opaque state for an encoder or decoder.
#[must_use]
pub fn initial_state(codec: Codec, decoder: bool) -> [u8; 8] {
let mut state = [0; 8];
if is_iso2022(codec) {
if decoder {
iso2022::prepare_decode_state(&mut state);
} else {
iso2022::prepare_encode_state(&mut state);
}
}
state
}
/// The result of decoding one engine step.
pub enum DecodeOne {
Char(u32, usize),
Pair(u32, u32, usize),
Skip(usize),
Incomplete,
Illegal(usize),
}
/// The result of encoding one engine step.
pub enum EncodeOne {
Bytes([u8; 8], usize, usize),
Incomplete,
Illegal(usize),
}
/// Decode one engine step from `input`, updating the opaque codec `state`.
pub fn decode_one(codec: Codec, input: &[u8], state: &mut [u8; 8]) -> DecodeOne {
if input.is_empty() {
return DecodeOne::Incomplete;
}
if is_iso2022(codec) {
return iso2022::decode_one(codec, input, state);
}
match codec {
Codec::EucKr => kr::decode_euc_kr(input),
Codec::Cp949 => kr::decode_cp949(input),
Codec::Johab => kr::decode_johab(input),
Codec::Big5 => tw::decode_big5(input),
Codec::Cp950 => tw::decode_cp950(input),
Codec::Big5Hkscs => hk::decode_big5hkscs(input),
Codec::ShiftJis => jp::decode_shift_jis(input),
Codec::Cp932 => jp::decode_cp932(input),
Codec::EucJp => jp::decode_euc_jp(input),
Codec::ShiftJis2004 => jp::decode_shift_jis_2004(input, false),
Codec::EucJis2004 => jp::decode_euc_jis_2004(input, false),
Codec::EucJisX0213 => jp::decode_euc_jis_2004(input, true),
Codec::ShiftJisX0213 => jp::decode_shift_jis_2004(input, true),
Codec::Gb2312 => cn::decode_gb2312(input),
Codec::Gbk => cn::decode_gbk(input),
Codec::Gb18030 => cn::decode_gb18030(input),
Codec::Hz => cn::decode_hz(input, state),
Codec::Iso2022Kr
| Codec::Iso2022Jp
| Codec::Iso2022Jp1
| Codec::Iso2022Jp2
| Codec::Iso2022Jp2004
| Codec::Iso2022Jp3
| Codec::Iso2022JpExt => unreachable!(),
}
}
/// Encode one engine step from Unicode scalar values.
pub fn encode_one(
codec: Codec,
input: &[u32],
final_input: bool,
state: &mut [u8; 8],
) -> EncodeOne {
if input.is_empty() {
return EncodeOne::Incomplete;
}
if is_iso2022(codec) {
return iso2022::encode_one(codec, input, final_input, state);
}
match codec {
Codec::EucKr => kr::encode_euc_kr(input[0]),
Codec::Cp949 => kr::encode_cp949(input[0]),
Codec::Johab => kr::encode_johab(input[0]),
Codec::Big5 => tw::encode_big5(input[0]),
Codec::Cp950 => tw::encode_cp950(input[0]),
Codec::Big5Hkscs => hk::encode_big5hkscs(input, final_input),
Codec::ShiftJis => jp::encode_shift_jis(input[0]),
Codec::Cp932 => jp::encode_cp932(input[0]),
Codec::EucJp => jp::encode_euc_jp(input[0]),
Codec::ShiftJis2004 => jp::encode_shift_jis_2004(input, final_input, false),
Codec::EucJis2004 => jp::encode_euc_jis_2004(input, final_input, false),
Codec::EucJisX0213 => jp::encode_euc_jis_2004(input, final_input, true),
Codec::ShiftJisX0213 => jp::encode_shift_jis_2004(input, final_input, true),
Codec::Gb2312 => cn::encode_gb2312(input[0]),
Codec::Gbk => cn::encode_gbk(input[0]),
Codec::Gb18030 => cn::encode_gb18030(input[0]),
Codec::Hz => cn::encode_hz(input[0], state),
Codec::Iso2022Kr
| Codec::Iso2022Jp
| Codec::Iso2022Jp1
| Codec::Iso2022Jp2
| Codec::Iso2022Jp2004
| Codec::Iso2022Jp3
| Codec::Iso2022JpExt => unreachable!(),
}
}
/// Flush an encoder's shift state, if the selected codec has one.
pub fn encode_reset(codec: Codec, state: &mut [u8; 8]) -> Option<([u8; 8], usize)> {
match codec {
Codec::Hz => cn::reset_hz(state),
Codec::Iso2022Kr
| Codec::Iso2022Jp
| Codec::Iso2022Jp1
| Codec::Iso2022Jp2
| Codec::Iso2022Jp2004
| Codec::Iso2022Jp3
| Codec::Iso2022JpExt => iso2022::encode_reset(state),
_ => None,
}
}
/// Reset a decoder's shift state, if the selected codec has one.
///
/// ISO 2022 keeps the G1..G3 designations and the escape-throughout flag; only
/// G0 and the shifted flag go back to their defaults.
pub fn decode_reset(codec: Codec, state: &mut [u8; 8]) {
match codec {
Codec::Hz => cn::reset_decode_hz(state),
Codec::Iso2022Kr
| Codec::Iso2022Jp
| Codec::Iso2022Jp1
| Codec::Iso2022Jp2
| Codec::Iso2022Jp2004
| Codec::Iso2022Jp3
| Codec::Iso2022JpExt => iso2022::decode_reset(state),
_ => {}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn empty_input_is_incomplete() {
assert!(matches!(
decode_one(Codec::EucKr, &[], &mut [0; 8]),
DecodeOne::Incomplete
));
assert!(matches!(
encode_one(Codec::EucKr, &[], true, &mut [0; 8]),
EncodeOne::Incomplete
));
}
#[test]
fn every_one_and_two_byte_candidate_is_total() {
let codecs = [
Codec::EucKr,
Codec::Cp949,
Codec::Johab,
Codec::Big5,
Codec::Cp950,
Codec::Big5Hkscs,
Codec::ShiftJis,
Codec::Cp932,
Codec::EucJp,
Codec::ShiftJis2004,
Codec::EucJis2004,
Codec::EucJisX0213,
Codec::ShiftJisX0213,
Codec::Iso2022Kr,
Codec::Iso2022Jp,
Codec::Iso2022Jp1,
Codec::Iso2022Jp2,
Codec::Iso2022Jp2004,
Codec::Iso2022Jp3,
Codec::Iso2022JpExt,
Codec::Gb2312,
Codec::Gbk,
Codec::Gb18030,
Codec::Hz,
];
for codec in codecs {
for first in 0..=u8::MAX {
let _ = decode_one(codec, &[first], &mut [0; 8]);
for second in 0..=u8::MAX {
let _ = decode_one(codec, &[first, second], &mut [0; 8]);
}
}
}
}
}
crates/common/src/encodings/cjk/tw.rs
+127
View File
@@ -0,0 +1,127 @@
//! Line-by-line Rust port of PyPy cjkcodecs `_codecs_tw.c`.
use super::mappings_tw::{
BIG5_DECMAP, BIG5_DECMAP_DATA, BIG5_ENCMAP, BIG5_ENCMAP_DATA, CP950EXT_DECMAP,
CP950EXT_DECMAP_DATA, CP950EXT_ENCMAP, CP950EXT_ENCMAP_DATA, MapIndex, NOCHAR, UNIINV,
};
use super::{DecodeOne, EncodeOne};
fn try_map_decode(index: &[MapIndex; 256], data: &[u16], c1: u8, c2: u8) -> Option<u32> {
let page = index[c1 as usize];
if !page.present || c2 < page.bottom || c2 > page.top {
return None;
}
let value = data[page.offset + usize::from(c2 - page.bottom)];
(value != UNIINV).then_some(u32::from(value))
}
fn try_map_encode(index: &[MapIndex; 256], data: &[u16], c: u32) -> Option<u16> {
if c > 0xffff {
return None;
}
let page = index[(c >> 8) as usize];
let low = c as u8;
if !page.present || low < page.bottom || low > page.top {
return None;
}
let value = data[page.offset + usize::from(low - page.bottom)];
(value != NOCHAR).then_some(value)
}
fn encode_ascii(c: u32) -> Option<EncodeOne> {
if c >= 0x80 {
return None;
}
let mut output = [0; 8];
output[0] = c as u8;
Some(EncodeOne::Bytes(output, 1, 1))
}
fn encode_dbcs(code: u16) -> EncodeOne {
let mut output = [0; 8];
output[0] = (code >> 8) as u8;
output[1] = code as u8;
EncodeOne::Bytes(output, 2, 1)
}
pub(super) fn encode_big5(c: u32) -> EncodeOne {
if let Some(output) = encode_ascii(c) {
return output;
}
try_map_encode(&BIG5_ENCMAP, &BIG5_ENCMAP_DATA, c).map_or(EncodeOne::Illegal(1), encode_dbcs)
}
pub(super) fn decode_big5(input: &[u8]) -> DecodeOne {
let c = input[0];
if c < 0x80 {
return DecodeOne::Char(u32::from(c), 1);
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
try_map_decode(&BIG5_DECMAP, &BIG5_DECMAP_DATA, c, input[1])
// The checked-in `_codecs_tw.c::DECODER(big5)` returns two here, but
// the real PyPy codec oracle exposes a one-byte error span (as does
// 3.14). Preserve that observable PyPy result at the Rust boundary.
.map_or(DecodeOne::Illegal(1), |value| DecodeOne::Char(value, 2))
}
pub(super) fn encode_cp950(c: u32) -> EncodeOne {
if let Some(output) = encode_ascii(c) {
return output;
}
try_map_encode(&CP950EXT_ENCMAP, &CP950EXT_ENCMAP_DATA, c)
.or_else(|| try_map_encode(&BIG5_ENCMAP, &BIG5_ENCMAP_DATA, c))
.map_or(EncodeOne::Illegal(1), encode_dbcs)
}
pub(super) fn decode_cp950(input: &[u8]) -> DecodeOne {
let c = input[0];
if c < 0x80 {
return DecodeOne::Char(u32::from(c), 1);
}
if input.len() < 2 {
return DecodeOne::Incomplete;
}
try_map_decode(&CP950EXT_DECMAP, &CP950EXT_DECMAP_DATA, c, input[1])
.or_else(|| try_map_decode(&BIG5_DECMAP, &BIG5_DECMAP_DATA, c, input[1]))
.map_or(DecodeOne::Illegal(1), |value| DecodeOne::Char(value, 2))
}
#[cfg(test)]
mod tests {
use super::super::{Codec, decode_one, encode_one};
use super::{DecodeOne, EncodeOne};
#[test]
fn pypy_tw_oracle_vectors() {
for (codec, c, expected) in [
(Codec::Big5, '漢', [0xba, 0x7e]),
(Codec::Cp950, '漢', [0xba, 0x7e]),
(Codec::Cp950, '€', [0xa3, 0xe1]),
] {
let points = [c as u32];
let EncodeOne::Bytes(bytes, 2, 1) = encode_one(codec, &points, true, &mut [0; 8])
else {
panic!("{c:?} is not encodable");
};
assert_eq!(&bytes[..2], expected);
assert!(matches!(
decode_one(codec, &expected, &mut [0; 8]),
DecodeOne::Char(value, 2) if value == c as u32
));
}
}
#[test]
fn malformed_candidates_reject_the_lead_byte() {
assert!(matches!(
decode_one(Codec::Big5, &[0x81, 0x00], &mut [0; 8]),
DecodeOne::Illegal(1)
));
assert!(matches!(
decode_one(Codec::Cp950, &[0x81, 0x00], &mut [0; 8]),
DecodeOne::Illegal(1)
));
}
}
crates/common/src/inet.rs
+359
View File
@@ -0,0 +1,359 @@
//! The address converters' text half, which is arithmetic over a spelling
//! rather than a call into a socket layer.
//!
//! A target whose C library carries no `inet_pton` still has to answer for
//! these four names, so the conversions are written out here. They follow
//! musl, which is wasi-libc: the strict parser reads exactly four octets with
//! no redundant leading zero and the IPv6 grammar without a scope suffix, and
//! the writer compresses the longest run of zero groups.
//!
//! Nothing here touches a descriptor or a host, so the module compiles under
//! `cfg(test)` on every target and its corpus runs with the rest of the unit
//! tests, on hosts where the entry points themselves reach libc instead.
/// `inet_aton` — the lenient parser, which reads one to four parts and gives
/// the last one every byte the earlier ones did not claim. A part is decimal,
/// octal behind a `0`, or hexadecimal behind a `0x`.
#[must_use]
pub fn aton(text: &[u8]) -> Option<[u8; 4]> {
let mut parts = Vec::new();
for field in core::str::from_utf8(text).ok()?.split('.') {
let (digits, radix) = match field.as_bytes() {
[b'0', b'x' | b'X', rest @ ..] => (rest, 16),
[b'0', rest @ ..] if !rest.is_empty() => (rest, 8),
other => (other, 10),
};
let digits = core::str::from_utf8(digits).ok()?;
parts.push(u32::from_str_radix(digits, radix).ok()?);
if parts.len() > 4 {
return None;
}
}
// The last part carries the bytes the leading ones left: `127.1` is
// `127.0.0.1`, and a bare number is the whole address.
let leading = parts.len().checked_sub(1)?;
let mut address: u32 = 0;
for (index, part) in parts.iter().enumerate() {
let width = if index == leading {
32 - 8 * leading
} else {
8
};
if width < 32 && *part >= 1 << width {
return None;
}
address |= part << (32 - 8 * index - width);
}
Some(address.to_be_bytes())
}
/// `inet_ntoa` — the dotted-quad spelling of four address bytes.
#[must_use]
pub fn ntoa(packed: [u8; 4]) -> String {
format!("{}.{}.{}.{}", packed[0], packed[1], packed[2], packed[3])
}
/// One to three decimal digits with no redundant leading zero, which is the
/// only octet spelling the strict parser reads.
fn strict_octet(field: &[u8]) -> Option<u8> {
if field.is_empty() || field.len() > 3 || (field.len() > 1 && field[0] == b'0') {
return None;
}
let mut value: u32 = 0;
for digit in field {
value = value * 10 + u32::from(digit.checked_sub(b'0').filter(|d| *d < 10)?);
}
u8::try_from(value).ok()
}
/// `inet_pton(AF_INET, ...)` — exactly four strict octets.
#[must_use]
pub fn pton_v4(text: &[u8]) -> Option<[u8; 4]> {
let mut address = [0u8; 4];
let mut fields = text.split(|b| *b == b'.');
for slot in &mut address {
*slot = strict_octet(fields.next()?)?;
}
fields.next().is_none().then_some(address)
}
/// `inet_pton(AF_INET6, ...)`.
#[must_use]
pub fn pton_v6(text: &[u8]) -> Option<[u8; 16]> {
// A trailing dotted quad occupies the last two groups, so it is taken off
// first and the hexadecimal grammar reads what is left. The colon before
// it is a separator, except when the quad follows a compressed run: there
// it is that run's second colon, and cutting it away would leave a `::`
// the grammar can no longer see.
let (head, tail) = match text.iter().position(|b| *b == b'.') {
None => (text, None),
Some(_) => {
let colon = text.iter().rposition(|b| *b == b':')?;
let head_end = colon + usize::from(text[..colon].ends_with(b":"));
(&text[..head_end], Some(pton_v4(&text[colon + 1..])?))
}
};
let groups_wanted = if tail.is_some() { 6 } else { 8 };
let (before, after) = match find_double_colon(head)? {
None => (head, None),
Some(at) => (&head[..at], Some(&head[at + 2..])),
};
let leading = hex_groups(before, groups_wanted)?;
let trailing = match after {
None => Vec::new(),
Some(rest) => hex_groups(rest, groups_wanted - leading.len())?,
};
// Without `::` every group must be spelled; with it at least one must not.
let elided = groups_wanted - leading.len() - trailing.len();
if (after.is_none() && elided != 0) || (after.is_some() && elided == 0) {
return None;
}
let mut address = [0u8; 16];
let mut out = address.iter_mut();
for group in leading
.iter()
.chain(core::iter::repeat_n(&0u16, elided))
.chain(trailing.iter())
{
*out.next()? = (group >> 8) as u8;
*out.next()? = *group as u8;
}
if let Some(quad) = tail {
address[12..].copy_from_slice(&quad);
}
Some(address)
}
/// The offset of the one `::` a spelling may carry, or nothing when a second
/// one makes the address ambiguous.
fn find_double_colon(text: &[u8]) -> Option<Option<usize>> {
let mut found = None;
let mut index = 0;
while index + 1 < text.len() {
if text[index] == b':' && text[index + 1] == b':' {
if found.is_some() {
return None;
}
found = Some(index);
index += 1;
}
index += 1;
}
Some(found)
}
/// Colon-separated groups of one to four hexadecimal digits. An empty run is
/// no groups at all, which is what either side of a leading or trailing `::`
/// reads as.
fn hex_groups(text: &[u8], limit: usize) -> Option<Vec<u16>> {
if text.is_empty() {
return Some(Vec::new());
}
let mut groups = Vec::new();
for field in text.split(|b| *b == b':') {
if field.is_empty() || field.len() > 4 || groups.len() == limit {
return None;
}
let mut value: u16 = 0;
for digit in field {
value = value * 16 + u16::from((*digit as char).to_digit(16)? as u8);
}
groups.push(value);
}
Some(groups)
}
/// `inet_ntop(AF_INET6, ...)`.
///
/// The address is written out group by group and then has its longest run of
/// zero groups replaced by `::`, which is the rewrite musl performs and the
/// reason a run of one group is left spelled out.
#[must_use]
pub fn ntop_v6(packed: &[u8]) -> String {
let group = |i: usize| (u16::from(packed[2 * i]) << 8) | u16::from(packed[2 * i + 1]);
// An IPv4-mapped address keeps its last four bytes in dotted-quad form.
let text = if packed[..12] == [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff] {
format!(
"{:x}:{:x}:{:x}:{:x}:{:x}:{:x}:{}.{}.{}.{}",
group(0),
group(1),
group(2),
group(3),
group(4),
group(5),
packed[12],
packed[13],
packed[14],
packed[15]
)
} else {
(0..8)
.map(|i| format!("{:x}", group(i)))
.collect::<Vec<_>>()
.join(":")
};
// The longest run of `:` and `0` that starts the string or starts at a
// colon, taken only when it spans more than one zero group.
let bytes = text.as_bytes();
let (mut best, mut longest) = (0, 2);
for start in 0..bytes.len() {
if start != 0 && bytes[start] != b':' {
continue;
}
let run = bytes[start..]
.iter()
.take_while(|b| **b == b':' || **b == b'0')
.count();
// An interior run includes the colon before its first zero while a
// leading run does not. Once the leading run is best, make an
// interior candidate beat it by that extra byte as well; equal-size
// zero runs therefore keep the first one, as musl's `inet_ntop` does.
if run > longest + usize::from(best == 0) {
(best, longest) = (start, run);
}
}
if longest <= 3 {
return text;
}
format!("{}::{}", &text[..best], &text[best + longest..])
}
#[cfg(test)]
mod tests {
use super::*;
use core::fmt::Write;
/// Spellings every one of glibc, musl and the BSDs reads the same way, so
/// the expectations here are the ones a host `inet_pton` would also meet.
#[test]
fn pton_v6_reads_the_shared_grammar() {
let cases: &[(&str, Option<&str>)] = &[
("::", Some("00000000000000000000000000000000")),
("::1", Some("00000000000000000000000000000001")),
("1::", Some("00010000000000000000000000000000")),
// A dotted quad directly after a compressed run: the colon the
// quad is cut from is that run's own second colon.
("::192.0.2.1", Some("000000000000000000000000c0000201")),
(
"2001:db8::192.0.2.1",
Some("20010db80000000000000000c0000201"),
),
("::ffff:192.0.2.1", Some("00000000000000000000ffffc0000201")),
(
"0:0:0:0:0:ffff:192.0.2.1",
Some("00000000000000000000ffffc0000201"),
),
("1::2:192.0.2.1", Some("000100000000000000000002c0000201")),
(
"1:2:3:4:5:6:1.2.3.4",
Some("00010002000300040005000601020304"),
),
("1:2:3:4:5:6:7:8", Some("00010002000300040005000600070008")),
("fe80::1", Some("fe800000000000000000000000000001")),
("1:2:3:4:5:6:7::", Some("00010002000300040005000600070000")),
("::1:2:3:4:5:6:7", Some("00000001000200030004000500060007")),
// Rejected everywhere: two compressed runs, a quad with a fifth
// part, one group too many, one too few, a lone dotted quad, and
// an octet with a redundant leading zero.
("1::2::3", None),
(":::192.0.2.1", None),
("::192.0.2.1.5", None),
("::1.2.3", None),
("1:2:3:4:5:6:7:8:9", None),
("1:2:3:4:5:6:7", None),
("1.2.3.4", None),
("::01.2.3.4", None),
("::00001", None),
("::%eth0", None),
("", None),
(":", None),
];
for (text, want) in cases {
let got = pton_v6(text.as_bytes()).map(|bytes| {
bytes.iter().fold(String::new(), |mut output, byte| {
write!(output, "{byte:02x}").unwrap();
output
})
});
assert_eq!(got.as_deref(), *want, "inet_pton(AF_INET6, {text:?})");
}
}
#[test]
fn pton_v4_reads_four_strict_octets() {
for (text, want) in [
("1.2.3.4", Some([1, 2, 3, 4])),
("255.255.255.255", Some([255, 255, 255, 255])),
("0.0.0.0", Some([0, 0, 0, 0])),
("01.2.3.4", None),
("1.2.3", None),
("1.2.3.4.5", None),
("1.2.3.256", None),
("1.2.3.", None),
] {
assert_eq!(
pton_v4(text.as_bytes()),
want,
"inet_pton(AF_INET, {text:?})"
);
}
}
/// The lenient parser gives the last part every byte the earlier ones did
/// not claim, and reads octal and hexadecimal.
#[test]
fn aton_is_the_lenient_parser() {
for (text, want) in [
("127.1", Some([127, 0, 0, 1])),
("1.2.3.4", Some([1, 2, 3, 4])),
("16909060", Some([1, 2, 3, 4])),
("0x7f.1", Some([127, 0, 0, 1])),
("0177.0.0.1", Some([127, 0, 0, 1])),
("1.2.3.4.5", None),
("256.1.1.1", None),
("1.2.3.4.", None),
("", None),
] {
assert_eq!(aton(text.as_bytes()), want, "inet_aton({text:?})");
}
assert_eq!(ntoa([1, 2, 3, 4]), "1.2.3.4");
}
/// Every spelling the writer produces has to read back to the bytes it was
/// given, and the compressed run is the longest one.
#[test]
fn ntop_v6_round_trips_and_compresses_the_longest_run() {
for (packed, want) in [
("00000000000000000000000000000000", "::"),
("00000000000000000000000000000001", "::1"),
("20010db8000000000000000000000001", "2001:db8::1"),
// One zero group is left spelled out; the longer run wins.
("20010000000100000000000200030004", "2001:0:1::2:3:4"),
// Equal runs keep the first. In the uncompressed text the later
// run appears one byte longer only because it owns a leading
// colon; that byte is not another zero group.
("00000000000100000000000200030004", "::1:0:0:2:3:4"),
// musl uses dotted-quad output for mapped addresses, but not the
// older IPv4-compatible form.
("000000000000000000000000c0000201", "::c000:201"),
("00000000000000000000ffffc0000201", "::ffff:192.0.2.1"),
("fe800000000000000000000000000001", "fe80::1"),
] {
let bytes: Vec<u8> = (0..16)
.map(|i| u8::from_str_radix(&packed[2 * i..2 * i + 2], 16).unwrap())
.collect();
let text = ntop_v6(&bytes);
assert_eq!(text, want, "inet_ntop({packed})");
assert_eq!(
pton_v6(text.as_bytes())
.as_ref()
.map(|address| &address[..]),
Some(&bytes[..]),
"round trip of {text}"
);
}
}
}
crates/common/src/json.rs
+163
View File
@@ -0,0 +1,163 @@
//! VM-independent JSON string escaping and scanning over WTF-8.
use memchr::memchr2;
use rustpython_wtf8::{CodePoint, Wtf8, Wtf8Buf};
pub static ESCAPE_CHARS: [&str; 0x20] = [
"\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", "\\u0007", "\\b",
"\\t", "\\n", "\\u000b", "\\f", "\\r", "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012",
"\\u0013", "\\u0014", "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a",
"\\u001b", "\\u001c", "\\u001d", "\\u001e", "\\u001f",
];
#[must_use]
pub fn encode_string(value: &Wtf8, ascii_only: bool) -> Wtf8Buf {
let mut out = Wtf8Buf::with_capacity(value.len() + 2);
out.push_char('"');
for cp in value.code_points() {
let n = cp.to_u32();
match n {
0x08 => out.push_str("\\b"),
0x09 => out.push_str("\\t"),
0x0a => out.push_str("\\n"),
0x0c => out.push_str("\\f"),
0x0d => out.push_str("\\r"),
0x00..=0x1f => out.push_str(ESCAPE_CHARS[n as usize]),
0x22 => out.push_str("\\\""),
0x5c => out.push_str("\\\\"),
0x7f if ascii_only => out.push_str("\\u007f"),
0x80.. if ascii_only => {
if n <= 0xffff {
push_hex_escape(&mut out, n as u16);
} else {
let m = n - 0x10000;
push_hex_escape(&mut out, (0xd800 | ((m >> 10) & 0x3ff)) as u16);
push_hex_escape(&mut out, (0xdc00 | (m & 0x3ff)) as u16);
}
}
_ => out.push(cp),
}
}
out.push_char('"');
out
}
/// The `\uXXXX` spelling of one UTF-16 unit, written without allocating.
fn push_hex_escape(out: &mut Wtf8Buf, unit: u16) {
const DIGITS: &[u8; 16] = b"0123456789abcdef";
let mut escape = *b"\\u0000";
for (digit, shift) in escape[2..].iter_mut().zip([12, 8, 4, 0]) {
*digit = DIGITS[((unit >> shift) & 0xf) as usize];
}
out.push_str(core::str::from_utf8(&escape).expect("the escape is ASCII"));
}
#[derive(Debug)]
pub struct DecodeError {
pub msg: String,
pub pos: usize,
}
fn decode_hex<I>(chars: &mut I, pos: usize) -> Result<CodePoint, DecodeError>
where
I: Iterator<Item = (usize, (usize, CodePoint))>,
{
let mut value = 0u16;
for _ in 0..4 {
let (_, (_, cp)) = chars.next().ok_or_else(|| DecodeError {
msg: "Invalid \\uXXXX escape".to_owned(),
pos,
})?;
let digit = cp
.to_char()
.and_then(|c| c.to_digit(16))
.ok_or_else(|| DecodeError {
msg: "Invalid \\uXXXX escape".to_owned(),
pos,
})?;
value = (value << 4) | digit as u16;
}
Ok(value.into())
}
/// CPython `_json.scanstring`, expressed over WTF-8 so lone surrogates retain
/// their Python `str` identity rather than being lossily converted to UTF-8.
pub fn scan_string(
value: &Wtf8,
char_offset: usize,
strict: bool,
) -> Result<(Wtf8Buf, usize, usize), DecodeError> {
let unterminated = || DecodeError {
msg: "Unterminated string starting at".to_owned(),
pos: char_offset.saturating_sub(1),
};
// A string with no escape and no closing-quote surprise is the common one,
// so find that quote and take everything before it in one copy.
let bytes = value.as_bytes();
if let Some(end) = memchr2(b'"', b'\\', bytes)
&& bytes[end] == b'"'
&& !(strict && bytes[..end].iter().any(|b| *b < 0x20))
{
let content = &value[..end];
let mut out = Wtf8Buf::with_capacity(end);
out.push_wtf8(content);
return Ok((
out,
char_offset + content.code_points().count() + 1,
end + 1,
));
}
let mut out = Wtf8Buf::new();
let mut chars = value.code_point_indices().enumerate().peekable();
while let Some((char_i, (byte_i, cp))) = chars.next() {
match cp.to_char_lossy() {
'"' => return Ok((out, char_offset + char_i + 1, byte_i + 1)),
'\\' => {
let (escape_i, (_, escaped)) = chars.next().ok_or_else(unterminated)?;
match escaped.to_char_lossy() {
'"' => out.push_char('"'),
'\\' => out.push_char('\\'),
'/' => out.push_char('/'),
'b' => out.push_char('\x08'),
'f' => out.push_char('\x0c'),
'n' => out.push_char('\n'),
'r' => out.push_char('\r'),
't' => out.push_char('\t'),
'u' => {
let mut decoded = decode_hex(&mut chars, char_offset + escape_i)?;
if let Some(lead) = decoded.to_lead_surrogate() {
let mut lookahead = chars.clone();
if let (Some((_, (_, slash))), Some((u_i, (_, u)))) =
(lookahead.next(), lookahead.next())
&& slash == '\\'
&& u == 'u'
{
let second = decode_hex(&mut lookahead, char_offset + u_i)?;
if let Some(trail) = second.to_trail_surrogate() {
decoded = lead.merge(trail).into();
chars = lookahead;
}
}
}
out.push(decoded);
}
_ => {
return Err(DecodeError {
msg: format!("Invalid \\escape: {escaped:?}"),
pos: char_offset + char_i,
});
}
}
}
'\x00'..='\x1f' if strict => {
return Err(DecodeError {
msg: format!("Invalid control character {cp:?} at"),
pos: char_offset + char_i,
});
}
_ => out.push(cp),
}
}
Err(unterminated())
}
crates/common/src/lib.rs
+6
View File
@@ -6,6 +6,8 @@
extern crate alloc;
pub mod atomic;
#[cfg(feature = "binascii")]
pub mod binascii;
pub mod borrow;
pub mod boxvec;
pub mod cformat;
@@ -13,7 +15,11 @@ pub mod encodings;
pub mod float_ops;
pub mod format;
pub mod hash;
#[cfg(feature = "inet")]
pub mod inet;
pub mod int;
#[cfg(feature = "json")]
pub mod json;
pub mod linked_list;
pub mod lock;
pub mod rand;
crates/stdlib/Cargo.toml
+1 -3
View File
@@ -30,7 +30,7 @@ __ssl-rustls = ["ssl", "rustls", "rustls-native-certs", "rustls-pemfile", "rustl
# rustpython crates
rustpython-derive = { workspace = true }
rustpython-vm = { workspace = true, default-features = false, features = ["compiler"]}
rustpython-common = { workspace = true }
rustpython-common = { workspace = true, features = ["binascii", "cjk-codecs", "inet", "json"] }
rustpython-host_env = { workspace = true }
rustpython-unicode = { workspace = true }
@@ -56,7 +56,6 @@ rapidhash = { workspace = true }
scopeguard = { workspace = true }
memchr = { workspace = true }
base64 = { workspace = true }
csv-core = { workspace = true }
dyn-clone = { workspace = true }
pymath = { workspace = true }
@@ -80,7 +79,6 @@ constant_time_eq = { workspace = true }
# compression
adler32 = { workspace = true }
crc32fast = { workspace = true }
flate2 = { workspace = true, features = ["zlib-rs"] }
libz-rs-sys = { workspace = true }
bzip2 = { workspace = true }
crates/stdlib/src/binascii.rs
+73 -762
View File
@@ -1,26 +1,20 @@
// spell-checker:ignore hexlify unhexlify uuencodes CRCTAB rlecode rledecode
// spell-checker:ignore hexlify unhexlify uuencodes rlecode rledecode
pub(super) use decl::crc32;
pub(crate) use decl::module_def;
use rustpython_common::wtf8::Wtf8Buf;
use rustpython_vm::{VirtualMachine, builtins::PyBaseExceptionRef, convert::ToPyException};
const PAD: u8 = 61u8;
const MAXLINESIZE: usize = 76; // Excluding the CRLF
use rustpython_common::binascii::{Base64DecodeError, Error};
use rustpython_vm::{VirtualMachine, builtins::PyBaseExceptionRef};
#[pymodule(name = "binascii")]
mod decl {
use super::{MAXLINESIZE, PAD};
use super::new_binascii_error;
use crate::vm::{
PyResult, VirtualMachine,
builtins::{PyIntRef, PyTypeRef},
convert::ToPyException,
function::{ArgAsciiBuffer, ArgBytesLike, OptionalArg},
};
use base64::Engine;
use itertools::Itertools;
use rustpython_common::binascii;
#[pyattr(name = "Error", once)]
pub(super) fn error_type(vm: &VirtualMachine) -> PyTypeRef {
@@ -36,14 +30,6 @@ mod decl {
vm.ctx.new_exception_type("binascii", "Incomplete", None)
}
const fn hex_nibble(n: u8) -> u8 {
match n {
0..=9 => b'0' + n,
10..=15 => b'a' + (n - 10),
_ => unreachable!(),
}
}
#[pyfunction(name = "b2a_hex")]
#[pyfunction]
fn hexlify(
@@ -52,197 +38,38 @@ mod decl {
bytes_per_sep: OptionalArg<isize>,
vm: &VirtualMachine,
) -> PyResult<Vec<u8>> {
let sep = match sep {
OptionalArg::Present(sep) => sep.with_ref(|sep| {
let [sep] = sep else {
return Err(vm.new_value_error("sep must be length 1."));
};
if !sep.is_ascii() {
return Err(vm.new_value_error("sep must be ASCII."));
}
Ok(Some(*sep))
})?,
OptionalArg::Missing => None,
};
let bytes_per_sep = bytes_per_sep.unwrap_or(1);
data.with_ref(|bytes| {
// Get separator character if provided
let sep_char = if let OptionalArg::Present(sep_buf) = sep {
sep_buf.with_ref(|sep_bytes| {
if sep_bytes.len() != 1 {
return Err(vm.new_value_error("sep must be length 1."));
}
let sep_char = sep_bytes[0];
if !sep_char.is_ascii() {
return Err(vm.new_value_error("sep must be ASCII."));
}
Ok(Some(sep_char))
})?
} else {
None
};
// If no separator or bytes_per_sep is 0, use simple hexlify
if sep_char.is_none() || bytes_per_sep == 0 || bytes.is_empty() {
let mut hex = Vec::<u8>::with_capacity(bytes.len() * 2);
for b in bytes {
hex.push(hex_nibble(b >> 4));
hex.push(hex_nibble(b & 0xf));
}
return Ok(hex);
}
let sep_char = sep_char.unwrap();
let abs_bytes_per_sep = bytes_per_sep.unsigned_abs();
// If separator interval is >= data length, no separators needed
if abs_bytes_per_sep >= bytes.len() {
let mut hex = Vec::<u8>::with_capacity(bytes.len() * 2);
for b in bytes {
hex.push(hex_nibble(b >> 4));
hex.push(hex_nibble(b & 0xf));
}
return Ok(hex);
}
// Calculate result length
let num_separators = (bytes.len() - 1) / abs_bytes_per_sep;
let result_len = bytes.len() * 2 + num_separators;
let mut hex = vec![0u8; result_len];
if bytes_per_sep < 0 {
// Left-to-right processing (negative bytes_per_sep)
let mut i = 0; // input index
let mut j = 0; // output index
let chunks = bytes.len() / abs_bytes_per_sep;
// Process complete chunks
for _ in 0..chunks {
for _ in 0..abs_bytes_per_sep {
let b = bytes[i];
hex[j] = hex_nibble(b >> 4);
hex[j + 1] = hex_nibble(b & 0xf);
i += 1;
j += 2;
}
if i < bytes.len() {
hex[j] = sep_char;
j += 1;
}
}
// Process remaining bytes
while i < bytes.len() {
let b = bytes[i];
hex[j] = hex_nibble(b >> 4);
hex[j + 1] = hex_nibble(b & 0xf);
i += 1;
j += 2;
}
} else {
// Right-to-left processing (positive bytes_per_sep)
let mut i = bytes.len() as isize - 1; // input index
let mut j = result_len as isize - 1; // output index
let chunks = bytes.len() / abs_bytes_per_sep;
// Process complete chunks from right
for _ in 0..chunks {
for _ in 0..abs_bytes_per_sep {
let b = bytes[i as usize];
hex[j as usize] = hex_nibble(b & 0xf);
hex[(j - 1) as usize] = hex_nibble(b >> 4);
i -= 1;
j -= 2;
}
if i >= 0 {
hex[j as usize] = sep_char;
j -= 1;
}
}
// Process remaining bytes
while i >= 0 {
let b = bytes[i as usize];
hex[j as usize] = hex_nibble(b & 0xf);
hex[(j - 1) as usize] = hex_nibble(b >> 4);
i -= 1;
j -= 2;
}
}
Ok(hex)
})
}
const fn unhex_nibble(c: u8) -> Option<u8> {
match c {
b'0'..=b'9' => Some(c - b'0'),
b'a'..=b'f' => Some(c - b'a' + 10),
b'A'..=b'F' => Some(c - b'A' + 10),
_ => None,
}
Ok(data.with_ref(|bytes| binascii::hexlify(bytes, sep, bytes_per_sep)))
}
#[pyfunction(name = "a2b_hex")]
#[pyfunction]
fn unhexlify(data: ArgAsciiBuffer, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
data.with_ref(|hex_bytes| {
if hex_bytes.len() % 2 != 0 {
return Err(super::new_binascii_error("Odd-length string", vm));
}
let mut unhex = Vec::<u8>::with_capacity(hex_bytes.len() / 2);
for (n1, n2) in hex_bytes.iter().tuples() {
if let (Some(n1), Some(n2)) = (unhex_nibble(*n1), unhex_nibble(*n2)) {
unhex.push((n1 << 4) | n2);
} else {
return Err(super::new_binascii_error("Non-hexadecimal digit found", vm));
}
}
Ok(unhex)
})
data.with_ref(binascii::unhexlify)
.map_err(|e| new_binascii_error(e, vm))
}
#[pyfunction]
pub(crate) fn crc32(data: ArgBytesLike, init: OptionalArg<PyIntRef>) -> u32 {
let init = init.map_or(0, |i| i.as_u32_mask());
let mut hasher = crc32fast::Hasher::new_with_initial(init);
data.with_ref(|bytes| {
hasher.update(bytes);
hasher.finalize()
})
data.with_ref(|bytes| binascii::crc32(bytes, init))
}
#[pyfunction]
pub(crate) fn crc_hqx(data: ArgBytesLike, init: PyIntRef) -> u32 {
const CRCTAB_HQX: [u16; 256] = [
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129, 0xa14a,
0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294,
0x72f7, 0x62d6, 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 0x2462,
0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, 0xa56a, 0xb54b, 0x8528, 0x9509,
0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695,
0x46b4, 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, 0x48c4, 0x58e5,
0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948,
0x9969, 0xa90a, 0xb92b, 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12,
0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 0x6ca6, 0x7c87, 0x4ce4,
0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b,
0x8d68, 0x9d49, 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, 0xff9f,
0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, 0x9188, 0x81a9, 0xb1ca, 0xa1eb,
0xd10c, 0xc12d, 0xf14e, 0xe16f, 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046,
0x6067, 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, 0x02b1, 0x1290,
0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e,
0xe54f, 0xd52c, 0xc50d, 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, 0x26d3, 0x36f2, 0x0691,
0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9,
0xb98a, 0xa9ab, 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, 0xcb7d,
0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, 0x4a75, 0x5a54, 0x6a37, 0x7a16,
0x0af1, 0x1ad0, 0x2ab3, 0x3a92, 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8,
0x8dc9, 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, 0xef1f, 0xff3e,
0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93,
0x3eb2, 0x0ed1, 0x1ef0,
];
let mut crc = init.as_u32_mask() & 0xffff;
data.with_ref(|buf| {
for byte in buf {
crc =
((crc << 8) & 0xFF00) ^ CRCTAB_HQX[((crc >> 8) as u8 ^ (byte)) as usize] as u32;
}
});
crc
data.with_ref(|bytes| binascii::crc_hqx(bytes, init.as_u32_mask()))
}
#[derive(FromArgs)]
@@ -261,160 +88,14 @@ mod decl {
#[pyfunction]
fn a2b_base64(args: A2bBase64Args, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
// Converts between ASCII and base-64 characters. The index of a given number yields the
// number in ASCII while the value of said index yields the number in base-64. For example
// "=" is 61 in ASCII but 0 (since it's the pad character) in base-64, so BASE64_TABLE[61] == 0
#[rustfmt::skip]
const BASE64_TABLE: [i8; 256] = [
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63,
52,53,54,55, 56,57,58,59, 60,61,-1,-1, -1, 0,-1,-1, /* Note PAD->0 */
-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14,
15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1,
-1,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40,
41,42,43,44, 45,46,47,48, 49,50,51,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
];
let A2bBase64Args { s, strict_mode } = args;
s.with_ref(|b| {
if b.is_empty() {
return Ok(vec![]);
}
let mut decoded: Vec<u8> = vec![];
let mut quad_pos = 0; // position in the nibble
let mut pads = 0;
let mut left_char: u8 = 0;
for (i, &el) in b.iter().enumerate() {
if el == PAD {
pads += 1;
// A pad that finishes the quad it belongs to is the expected one.
if quad_pos >= 2 && quad_pos + pads <= 4 {
continue;
}
// RFC 4648 section 3.3 allows a decoder to ignore a pad that
// shows up anywhere else, so only strict mode complains.
if !strict_mode {
continue;
}
if quad_pos == 1 {
// A single data character cannot be padded into a quad,
// and the check after the loop already reports that.
break;
}
return Err(if quad_pos == 0 && i == 0 {
// Represents leading padding error
base64::DecodeError::InvalidByte(0, PAD)
} else {
// Represents excess padding error
base64::DecodeError::InvalidPadding
});
}
let binary_char = BASE64_TABLE[el as usize];
if binary_char >= 64 || binary_char == -1 {
if strict_mode {
// Represents non-base64 data error
return Err(base64::DecodeError::InvalidByte(i, el));
}
continue;
}
if pads > 0 && strict_mode {
return Err(if quad_pos + pads == 4 {
// Represents excess data after padding error
base64::DecodeError::InvalidLastSymbol(i, PAD)
} else {
// Represents discontinuous padding error
base64::DecodeError::InvalidByte(i, PAD)
});
}
pads = 0;
// Decode individual ASCII character
match quad_pos {
0 => {
quad_pos = 1;
left_char = binary_char as u8;
}
1 => {
quad_pos = 2;
decoded.push((left_char << 2) | (binary_char >> 4) as u8);
left_char = (binary_char & 0x0f) as u8;
}
2 => {
quad_pos = 3;
decoded.push((left_char << 4) | (binary_char >> 2) as u8);
left_char = (binary_char & 0x03) as u8;
}
3 => {
quad_pos = 0;
decoded.push((left_char << 6) | binary_char as u8);
left_char = 0;
}
_ => unsafe {
// quad_pos is only assigned in this match statement to constants
core::hint::unreachable_unchecked()
},
}
}
if quad_pos == 1 {
// One data character too many: no input encodes to that length.
return Err(base64::DecodeError::InvalidLastSymbol(
decoded.len() / 3 * 4 + 1,
0,
));
}
if quad_pos != 0 && quad_pos + pads < 4 {
return Err(base64::DecodeError::InvalidLength(quad_pos));
}
Ok(decoded)
})
.map_err(|err| super::Base64DecodeError(err).to_pyexception(vm))
s.with_ref(|b| binascii::a2b_base64(b, strict_mode))
.map_err(|e| new_binascii_error(e, vm))
}
#[pyfunction]
fn b2a_base64(data: ArgBytesLike, NewlineArg { newline }: NewlineArg) -> Vec<u8> {
// https://stackoverflow.com/questions/63916821
let mut encoded = data
.with_ref(|b| base64::engine::general_purpose::STANDARD.encode(b))
.into_bytes();
if newline {
encoded.push(b'\n');
}
encoded
}
#[inline]
fn uu_a2b_read(c: u8, vm: &VirtualMachine) -> PyResult<u8> {
// Check the character for legality
// The 64 instead of the expected 63 is because
// there are a few uuencodes out there that use
// '`' as zero instead of space.
if !(b' '..=(b' ' + 64)).contains(&c) {
if b"\r\n".contains(&c) {
return Ok(0);
}
return Err(super::new_binascii_error("Illegal char", vm));
}
Ok((c - b' ') & 0x3f)
data.with_ref(|bytes| binascii::b2a_base64(bytes, newline))
}
#[derive(FromArgs)]
@@ -426,64 +107,9 @@ mod decl {
}
#[pyfunction]
fn a2b_qp(args: A2bQpArgs) -> PyResult<Vec<u8>> {
let s = args.data;
let header = args.header;
s.with_ref(|buffer| {
let len = buffer.len();
let mut out_data = Vec::with_capacity(len);
let mut idx = 0;
while idx < len {
if buffer[idx] == b'=' {
idx += 1;
if idx >= len {
break;
}
// Soft line breaks
if (buffer[idx] == b'\n') || (buffer[idx] == b'\r') {
if buffer[idx] != b'\n' {
while idx < len && buffer[idx] != b'\n' {
idx += 1;
}
}
if idx < len {
idx += 1;
}
} else if buffer[idx] == b'=' {
// broken case from broken python qp
out_data.push(b'=');
idx += 1;
} else if idx + 1 < len
&& ((buffer[idx] >= b'A' && buffer[idx] <= b'F')
|| (buffer[idx] >= b'a' && buffer[idx] <= b'f')
|| (buffer[idx] >= b'0' && buffer[idx] <= b'9'))
&& ((buffer[idx + 1] >= b'A' && buffer[idx + 1] <= b'F')
|| (buffer[idx + 1] >= b'a' && buffer[idx + 1] <= b'f')
|| (buffer[idx + 1] >= b'0' && buffer[idx + 1] <= b'9'))
{
// hex val
if let (Some(ch1), Some(ch2)) =
(unhex_nibble(buffer[idx]), unhex_nibble(buffer[idx + 1]))
{
out_data.push((ch1 << 4) | ch2);
}
idx += 2;
} else {
out_data.push(b'=');
}
} else if header && buffer[idx] == b'_' {
out_data.push(b' ');
idx += 1;
} else {
out_data.push(buffer[idx]);
idx += 1;
}
}
Ok(out_data)
})
fn a2b_qp(args: A2bQpArgs) -> Vec<u8> {
let A2bQpArgs { data, header } = args;
data.with_ref(|buffer| binascii::a2b_qp(buffer, header))
}
#[derive(FromArgs)]
@@ -499,318 +125,30 @@ mod decl {
}
#[pyfunction]
fn b2a_qp(args: B2aQpArgs) -> PyResult<Vec<u8>> {
let s = args.data;
let quotetabs = args.quotetabs;
let istext = args.istext;
let header = args.header;
s.with_ref(|buf| {
let buflen = buf.len();
let mut line_len = 0;
let mut out_data_len = 0;
let mut crlf = false;
let mut ch;
let mut in_idx;
let mut out_idx;
in_idx = 0;
while in_idx < buflen {
if buf[in_idx] == b'\n' {
break;
}
in_idx += 1;
}
if in_idx > 0 && in_idx < buflen && buf[in_idx - 1] == b'\r' {
crlf = true;
}
in_idx = 0;
while in_idx < buflen {
let mut delta = 0;
if (buf[in_idx] > 126)
|| (buf[in_idx] == b'=')
|| (header && buf[in_idx] == b'_')
|| (buf[in_idx] == b'.'
&& line_len == 0
&& (in_idx + 1 == buflen
|| buf[in_idx + 1] == b'\n'
|| buf[in_idx + 1] == b'\r'
|| buf[in_idx + 1] == 0))
|| (!istext && ((buf[in_idx] == b'\r') || (buf[in_idx] == b'\n')))
|| ((buf[in_idx] == b'\t' || buf[in_idx] == b' ') && (in_idx + 1 == buflen))
|| ((buf[in_idx] < 33)
&& (buf[in_idx] != b'\r')
&& (buf[in_idx] != b'\n')
&& (quotetabs || ((buf[in_idx] != b'\t') && (buf[in_idx] != b' '))))
{
if (line_len + 3) >= MAXLINESIZE {
line_len = 0;
delta += if crlf { 3 } else { 2 };
}
line_len += 3;
delta += 3;
in_idx += 1;
} else if istext
&& ((buf[in_idx] == b'\n')
|| ((in_idx + 1 < buflen)
&& (buf[in_idx] == b'\r')
&& (buf[in_idx + 1] == b'\n')))
{
line_len = 0;
// Protect against whitespace on end of line
if (in_idx != 0) && ((buf[in_idx - 1] == b' ') || (buf[in_idx - 1] == b'\t')) {
delta += 2;
}
delta += if crlf { 2 } else { 1 };
in_idx += if buf[in_idx] == b'\r' { 2 } else { 1 };
} else {
if (in_idx + 1 != buflen)
&& (buf[in_idx + 1] != b'\n')
&& (line_len + 1) >= MAXLINESIZE
{
line_len = 0;
delta += if crlf { 3 } else { 2 };
}
line_len += 1;
delta += 1;
in_idx += 1;
}
out_data_len += delta;
}
let mut out_data = Vec::with_capacity(out_data_len);
in_idx = 0;
out_idx = 0;
line_len = 0;
while in_idx < buflen {
if (buf[in_idx] > 126)
|| (buf[in_idx] == b'=')
|| (header && buf[in_idx] == b'_')
|| ((buf[in_idx] == b'.')
&& (line_len == 0)
&& (in_idx + 1 == buflen
|| buf[in_idx + 1] == b'\n'
|| buf[in_idx + 1] == b'\r'
|| buf[in_idx + 1] == 0))
|| (!istext && ((buf[in_idx] == b'\r') || (buf[in_idx] == b'\n')))
|| ((buf[in_idx] == b'\t' || buf[in_idx] == b' ') && (in_idx + 1 == buflen))
|| ((buf[in_idx] < 33)
&& (buf[in_idx] != b'\r')
&& (buf[in_idx] != b'\n')
&& (quotetabs || ((buf[in_idx] != b'\t') && (buf[in_idx] != b' '))))
{
if (line_len + 3) >= MAXLINESIZE {
// MAXLINESIZE = 76
out_data.push(b'=');
out_idx += 1;
if crlf {
out_data.push(b'\r');
out_idx += 1;
}
out_data.push(b'\n');
out_idx += 1;
line_len = 0;
}
out_data.push(b'=');
out_idx += 1;
ch = hex_nibble(buf[in_idx] >> 4);
if (b'a'..=b'f').contains(&ch) {
ch -= b' ';
}
out_data.push(ch);
ch = hex_nibble(buf[in_idx] & 0xf);
if (b'a'..=b'f').contains(&ch) {
ch -= b' ';
}
out_data.push(ch);
out_idx += 2;
in_idx += 1;
line_len += 3;
} else if istext
&& ((buf[in_idx] == b'\n')
|| ((in_idx + 1 < buflen)
&& (buf[in_idx] == b'\r')
&& (buf[in_idx + 1] == b'\n')))
{
line_len = 0;
if (out_idx != 0)
&& ((out_data[out_idx - 1] == b' ') || (out_data[out_idx - 1] == b'\t'))
{
ch = hex_nibble(out_data[out_idx - 1] >> 4);
if (b'a'..=b'f').contains(&ch) {
ch -= b' ';
}
out_data.push(ch);
ch = hex_nibble(out_data[out_idx - 1] & 0xf);
if (b'a'..=b'f').contains(&ch) {
ch -= b' ';
}
out_data.push(ch);
out_data[out_idx - 1] = b'=';
out_idx += 2;
}
if crlf {
out_data.push(b'\r');
out_idx += 1;
}
out_data.push(b'\n');
out_idx += 1;
in_idx += if buf[in_idx] == b'\r' { 2 } else { 1 };
} else {
if (in_idx + 1 != buflen) && (buf[in_idx + 1] != b'\n') && (line_len + 1) >= 76
{
// MAXLINESIZE = 76
out_data.push(b'=');
out_idx += 1;
if crlf {
out_data.push(b'\r');
out_idx += 1;
}
out_data.push(b'\n');
out_idx += 1;
line_len = 0;
}
line_len += 1;
if header && buf[in_idx] == b' ' {
out_data.push(b'_');
out_idx += 1;
in_idx += 1;
} else {
out_data.push(buf[in_idx]);
out_idx += 1;
in_idx += 1;
}
}
}
Ok(out_data)
})
fn b2a_qp(args: B2aQpArgs) -> Vec<u8> {
let B2aQpArgs {
data,
quotetabs,
istext,
header,
} = args;
data.with_ref(|buf| binascii::b2a_qp(buf, quotetabs, istext, header))
}
#[pyfunction]
fn rlecode_hqx(s: ArgAsciiBuffer) -> PyResult<Vec<u8>> {
const RUN_CHAR: u8 = 0x90; // b'\x90'
s.with_ref(|buffer| {
let len = buffer.len();
let mut out_data = Vec::<u8>::with_capacity((len * 2) + 2);
let mut idx = 0;
while idx < len {
let ch = buffer[idx];
if ch == RUN_CHAR {
out_data.push(RUN_CHAR);
out_data.push(0);
return Ok(out_data);
}
let mut in_end = idx + 1;
while in_end < len && buffer[in_end] == ch && in_end < idx + 255 {
in_end += 1;
}
if in_end - idx > 3 {
out_data.push(ch);
out_data.push(RUN_CHAR);
out_data.push(((in_end - idx) % 256) as u8);
idx = in_end - 1;
} else {
out_data.push(ch);
}
idx += 1;
}
Ok(out_data)
})
fn rlecode_hqx(s: ArgAsciiBuffer) -> Vec<u8> {
s.with_ref(binascii::rlecode_hqx)
}
#[pyfunction]
fn rledecode_hqx(s: ArgAsciiBuffer) -> PyResult<Vec<u8>> {
const RUN_CHAR: u8 = 0x90; //b'\x90'
s.with_ref(|buffer| {
let len = buffer.len();
let mut out_data = Vec::<u8>::with_capacity(len);
let mut idx = 0;
out_data.push(buffer[idx]);
idx += 1;
while idx < len {
if buffer[idx] == RUN_CHAR {
if buffer[idx + 1] == 0 {
out_data.push(RUN_CHAR);
} else {
let ch = buffer[idx - 1];
let range = buffer[idx + 1];
idx += 1;
for _ in 1..range {
out_data.push(ch);
}
}
} else {
out_data.push(buffer[idx]);
}
idx += 1;
}
Ok(out_data)
})
fn rledecode_hqx(s: ArgAsciiBuffer) -> Vec<u8> {
s.with_ref(binascii::rledecode_hqx)
}
#[pyfunction]
fn a2b_uu(s: ArgAsciiBuffer, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
s.with_ref(|b| {
if b.is_empty() {
return Err(super::new_binascii_error("Missing length byte", vm));
}
// First byte: binary data length (in bytes)
let length = ((b[0] - b' ') & 0x3f) as usize;
// Allocate the buffer
let mut res = Vec::<u8>::with_capacity(length);
let trailing_garbage_error = || Err(super::new_binascii_error("Trailing garbage", vm));
for chunk in b.get(1..).unwrap_or_default().chunks(4) {
let (char_a, char_b, char_c, char_d) = {
let mut chunk = chunk
.iter()
.map(|&x| uu_a2b_read(x, vm))
.collect::<Result<Vec<_>, _>>()?;
while chunk.len() < 4 {
chunk.push(0);
}
(chunk[0], chunk[1], chunk[2], chunk[3])
};
if res.len() < length {
res.push((char_a << 2) | (char_b >> 4));
} else if char_a != 0 || char_b != 0 {
return trailing_garbage_error();
}
if res.len() < length {
res.push(((char_b & 0xf) << 4) | (char_c >> 2));
} else if char_c != 0 {
return trailing_garbage_error();
}
if res.len() < length {
res.push(((char_c & 0x3) << 6) | char_d);
} else if char_d != 0 {
return trailing_garbage_error();
}
}
let remaining_length = length - res.len();
if remaining_length > 0 {
res.extend(vec![0; remaining_length]);
}
Ok(res)
})
s.with_ref(binascii::a2b_uu)
.map_err(|e| new_binascii_error(e, vm))
}
#[derive(FromArgs)]
@@ -825,64 +163,37 @@ mod decl {
BacktickArg { backtick }: BacktickArg,
vm: &VirtualMachine,
) -> PyResult<Vec<u8>> {
#[inline]
const fn uu_b2a(num: u8, backtick: bool) -> u8 {
if backtick && num == 0 {
0x60
} else {
b' ' + num
}
data.with_ref(|b| binascii::b2a_uu(b, backtick))
.map_err(|e| new_binascii_error(e, vm))
}
}
/// Builds the `binascii.Error` a transform failure maps to.
fn new_binascii_error(error: Error, vm: &VirtualMachine) -> PyBaseExceptionRef {
let message = match error {
Error::OddLengthString => "Odd-length string".to_owned(),
Error::NonHexadecimalDigit => "Non-hexadecimal digit found".to_owned(),
Error::MissingLengthByte => "Missing length byte".to_owned(),
Error::IllegalChar => "Illegal char".to_owned(),
Error::TrailingGarbage => "Trailing garbage".to_owned(),
Error::TooLong => "At most 45 bytes at once".to_owned(),
Error::Base64(e) => base64_message(e),
};
vm.new_exception_msg(decl::error_type(vm), message.into())
}
fn base64_message(error: Base64DecodeError) -> String {
match error {
Base64DecodeError::LeadingPaddingNotAllowed => "Leading padding not allowed".to_owned(),
Base64DecodeError::ExcessPaddingNotAllowed => "Excess padding not allowed".to_owned(),
Base64DecodeError::OnlyBase64DataAllowed => "Only base64 data is allowed".to_owned(),
Base64DecodeError::ExcessDataAfterPadding => "Excess data after padding".to_owned(),
Base64DecodeError::DiscontinuousPaddingNotAllowed => {
"Discontinuous padding not allowed".to_owned()
}
data.with_ref(|b| {
let length = b.len();
if length > 45 {
return Err(super::new_binascii_error("At most 45 bytes at once", vm));
}
let mut res = Vec::<u8>::with_capacity(2 + length.div_ceil(3) * 4);
res.push(uu_b2a(length as u8, backtick));
for chunk in b.chunks(3) {
let char_a = *chunk.first().unwrap();
let char_b = *chunk.get(1).unwrap_or(&0);
let char_c = *chunk.get(2).unwrap_or(&0);
res.push(uu_b2a(char_a >> 2, backtick));
res.push(uu_b2a(((char_a & 0x3) << 4) | (char_b >> 4), backtick));
res.push(uu_b2a(((char_b & 0xf) << 2) | (char_c >> 6), backtick));
res.push(uu_b2a(char_c & 0x3f, backtick));
}
res.push(0xau8);
Ok(res)
})
}
}
struct Base64DecodeError(base64::DecodeError);
fn new_binascii_error<T: Into<Wtf8Buf>>(msg: T, vm: &VirtualMachine) -> PyBaseExceptionRef {
vm.new_exception_msg(decl::error_type(vm), msg.into())
}
impl ToPyException for Base64DecodeError {
fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef {
use base64::DecodeError;
let message = match &self.0 {
DecodeError::InvalidByte(0, PAD) => "Leading padding not allowed".to_owned(),
DecodeError::InvalidByte(_, PAD) => "Discontinuous padding not allowed".to_owned(),
DecodeError::InvalidByte(_, _) => "Only base64 data is allowed".to_owned(),
DecodeError::InvalidLastSymbol(_, PAD) => "Excess data after padding".to_owned(),
DecodeError::InvalidLastSymbol(length, _) => {
format!(
"Invalid base64-encoded string: number of data characters {length} cannot be 1 more than a multiple of 4"
)
}
// TODO: clean up errors
DecodeError::InvalidLength(_) => "Incorrect padding".to_owned(),
DecodeError::InvalidPadding => "Excess padding not allowed".to_owned(),
};
new_binascii_error(format!("error decoding base64: {message}"), vm)
Base64DecodeError::InvalidLastSymbol { index } => format!(
"Invalid base64-encoded string: number of data characters ({index}) cannot be 1 more than a multiple of 4"
),
Base64DecodeError::IncorrectPadding => "Incorrect padding".to_owned(),
}
}
crates/stdlib/src/cjkcodecs.rs
+55
View File
@@ -0,0 +1,55 @@
//! The VM adapter for the CJK codec engines in `rustpython-common`.
//!
//! Laid out like `Modules/cjkcodecs/`: `multibytecodec` holds the codec objects
//! and the encode/decode drivers, and one `_codecs_*` module per region hands
//! out the codecs it registers.
pub(crate) mod multibytecodec;
/// Declares a `_codecs_*` module exposing `getcodec`, after
/// `cjkcodecs.h::I_AM_A_MODULE_FOR`.
macro_rules! codecs_module {
($module:ident, [$($name:literal),+ $(,)?]) => {
#[pymodule]
pub(crate) mod $module {
use crate::cjkcodecs::multibytecodec;
use crate::vm::{PyObjectRef, PyResult, VirtualMachine};
const CODECS: &[&'static str] = &[$($name),+];
#[pyfunction]
fn getcodec(encoding: PyObjectRef, vm: &VirtualMachine) -> PyResult {
multibytecodec::get_codec(CODECS, &encoding, vm)
}
}
};
}
codecs_module!(_codecs_cn, ["gb2312", "gbk", "gb18030", "hz"]);
codecs_module!(_codecs_hk, ["big5hkscs"]);
codecs_module!(
_codecs_iso2022,
[
"iso2022_kr",
"iso2022_jp",
"iso2022_jp_1",
"iso2022_jp_2",
"iso2022_jp_2004",
"iso2022_jp_3",
"iso2022_jp_ext",
]
);
codecs_module!(
_codecs_jp,
[
"shift_jis",
"cp932",
"euc_jp",
"shift_jis_2004",
"euc_jis_2004",
"euc_jisx0213",
"shift_jisx0213",
]
);
codecs_module!(_codecs_kr, ["euc_kr", "cp949", "johab"]);
codecs_module!(_codecs_tw, ["big5", "cp950"]);
crates/stdlib/src/cjkcodecs/multibytecodec.rs
+1284
View File
File diff suppressed because it is too large. Load diff
crates/stdlib/src/json.rs
+23 -40
View File
@@ -1,9 +1,7 @@
pub(crate) use _json::module_def;
mod machinery;
#[pymodule]
mod _json {
use super::machinery;
use crate::vm::{
AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine,
builtins::{PyBaseExceptionRef, PyStrRef, PyType},
@@ -14,7 +12,7 @@ mod _json {
};
use core::str::FromStr;
use malachite_bigint::BigInt;
use rustpython_common::wtf8::Wtf8Buf;
use rustpython_common::{json, wtf8::Wtf8Buf};
use std::collections::HashMap;
/// Skip JSON whitespace characters (space, tab, newline, carriage return).
@@ -113,10 +111,9 @@ mod _json {
match first_byte {
b'"' => {
// Parse string - pass slice starting after the quote
let (wtf8_result, chars_consumed, _bytes_consumed) =
machinery::scanstring(&wtf8[byte_idx + 1..], char_idx + 1, self.strict)
let (wtf8_result, end_char_idx, _bytes_consumed) =
json::scan_string(&wtf8[byte_idx + 1..], char_idx + 1, self.strict)
.map_err(|e| py_decode_error(e, pystr.clone(), vm))?;
let end_char_idx = char_idx + 1 + chars_consumed;
return Ok(PyIterReturn::Return(
vm.new_tuple((wtf8_result, end_char_idx)).into(),
));
@@ -297,11 +294,11 @@ mod _json {
byte_idx += 1;
// Parse key string using scanstring with byte slice
let (key_wtf8, chars_consumed, bytes_consumed) =
machinery::scanstring(&wtf8[byte_idx..], char_idx, self.strict)
let (key_wtf8, end_char_idx, bytes_consumed) =
json::scan_string(&wtf8[byte_idx..], char_idx, self.strict)
.map_err(|e| py_decode_error(e, pystr.clone(), vm))?;
char_idx += chars_consumed;
char_idx = end_char_idx;
byte_idx += bytes_consumed;
// Key memoization - reuse existing key strings.
@@ -559,15 +556,11 @@ mod _json {
// String - pass slice starting after the quote.
// Feed the Wtf8Buf directly to new_str; routing through
// .to_string() here would lossy-collapse surrogates to U+FFFD.
let (wtf8_result, chars_consumed, bytes_consumed) =
machinery::scanstring(&wtf8[byte_idx + 1..], char_idx + 1, self.strict)
let (wtf8_result, end_char_idx, bytes_consumed) =
json::scan_string(&wtf8[byte_idx + 1..], char_idx + 1, self.strict)
.map_err(|e| py_decode_error(e, pystr.clone(), vm))?;
let py_str = vm.ctx.new_str(wtf8_result);
Ok((
py_str.into(),
char_idx + 1 + chars_consumed,
byte_idx + 1 + bytes_consumed,
))
Ok((py_str.into(), end_char_idx, byte_idx + 1 + bytes_consumed))
}
b'{' => {
// Object
@@ -658,7 +651,10 @@ mod _json {
pos: usize,
vm: &VirtualMachine,
) -> PyBaseExceptionRef {
let err = machinery::DecodeError::new(msg, pos);
let err = json::DecodeError {
msg: msg.to_owned(),
pos,
};
py_decode_error(err, s, vm)
}
}
@@ -691,34 +687,18 @@ mod _json {
}
}
fn encode_string(wtf8: &rustpython_common::wtf8::Wtf8, ascii_only: bool) -> Wtf8Buf {
flame_guard!("_json::encode_string");
let mut buf = Vec::<u8>::with_capacity(wtf8.len() + 2);
machinery::write_json_string(wtf8, ascii_only, &mut buf)
// SAFETY: writing to a vec can't fail
.unwrap_or_else(|_| unsafe { core::hint::unreachable_unchecked() });
// write_json_string is designed to produce valid WTF-8 bytes:
// - ASCII control characters and JSON-specials are written as ASCII escapes
// - Valid Unicode scalars are written as UTF-8 (a subset of WTF-8)
// - Lone surrogates (ascii_only=false branch only) pass through as the
// input's WTF-8 byte sequences unchanged
// Use the checked constructor so any violation of that invariant
// surfaces as a panic during testing instead of undefined behavior.
Wtf8Buf::from_bytes(buf).expect("write_json_string produced invalid WTF-8")
}
#[pyfunction]
fn encode_basestring(s: PyStrRef) -> Wtf8Buf {
encode_string(s.as_wtf8(), false)
json::encode_string(s.as_wtf8(), false)
}
#[pyfunction]
fn encode_basestring_ascii(s: PyStrRef) -> Wtf8Buf {
encode_string(s.as_wtf8(), true)
json::encode_string(s.as_wtf8(), true)
}
fn py_decode_error(
e: machinery::DecodeError,
e: json::DecodeError,
s: PyStrRef,
vm: &VirtualMachine,
) -> PyBaseExceptionRef {
@@ -751,17 +731,20 @@ mod _json {
.map(|(i, _)| i)
.ok_or_else(|| {
py_decode_error(
machinery::DecodeError::new("Unterminated string starting at", end - 1),
json::DecodeError {
msg: "Unterminated string starting at".to_owned(),
pos: end - 1,
},
s.clone(),
vm,
)
})?
};
let (result, chars_consumed, _bytes_consumed) =
machinery::scanstring(&wtf8[byte_idx..], end, strict.unwrap_or(true))
let (result, end_char_idx, _bytes_consumed) =
json::scan_string(&wtf8[byte_idx..], end, strict.unwrap_or(true))
.map_err(|e| py_decode_error(e, s, vm))?;
Ok((result, end + chars_consumed))
Ok((result, end_char_idx))
}
}
crates/stdlib/src/json/machinery.rs
-295
View File
@@ -1,295 +0,0 @@
// spell-checker:ignore LOJKINE
// derived from https://github.com/lovasoa/json_in_type
// BSD 2-Clause License
//
// Copyright (c) 2018, Ophir LOJKINE
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
use std::io;
use itertools::Itertools;
use memchr::memchr2;
use rustpython_common::wtf8::{CodePoint, Wtf8, Wtf8Buf};
static ESCAPE_CHARS: [&str; 0x20] = [
"\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", "\\u0007", "\\b",
"\\t", "\\n", "\\u000b", "\\f", "\\r", "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012",
"\\u0013", "\\u0014", "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a",
"\\u001b", "\\u001c", "\\u001d", "\\u001e", "\\u001f",
];
// This bitset represents which bytes can be copied as-is to a JSON string (0)
// And which one need to be escaped (1)
// The characters that need escaping are 0x00 to 0x1F, 0x22 ("), 0x5C (\), 0x7F (DEL)
// Non-ASCII unicode characters can be safely included in a JSON string
static NEEDS_ESCAPING_BITSET: [u64; 4] = [
//fedcba9876543210_fedcba9876543210_fedcba9876543210_fedcba9876543210
0b0000000000000000_0000000000000100_1111111111111111_1111111111111111, // 3_2_1_0
0b1000000000000000_0000000000000000_0001000000000000_0000000000000000, // 7_6_5_4
0b0000000000000000_0000000000000000_0000000000000000_0000000000000000, // B_A_9_8
0b0000000000000000_0000000000000000_0000000000000000_0000000000000000, // F_E_D_C
];
#[inline(always)]
fn json_escaped_char(c: u8) -> Option<&'static str> {
let bitset_value = NEEDS_ESCAPING_BITSET[(c / 64) as usize] & (1 << (c % 64));
if bitset_value == 0 {
None
} else {
Some(match c {
x if x < 0x20 => ESCAPE_CHARS[c as usize],
b'\\' => "\\\\",
b'\"' => "\\\"",
0x7F => "\\u007f",
_ => unreachable!(),
})
}
}
pub(super) fn write_json_string<W: io::Write>(
wtf8: &Wtf8,
ascii_only: bool,
w: &mut W,
) -> io::Result<()> {
w.write_all(b"\"")?;
let mut write_start_idx = 0;
let bytes = wtf8.as_bytes();
if ascii_only {
for (idx, cp) in wtf8.code_point_indices() {
if let Some(c) = cp.to_char() {
// Valid Unicode scalar.
if c.is_ascii() {
if let Some(escaped) = json_escaped_char(c as u8) {
w.write_all(&bytes[write_start_idx..idx])?;
w.write_all(escaped.as_bytes())?;
write_start_idx = idx + 1;
}
} else {
w.write_all(&bytes[write_start_idx..idx])?;
write_start_idx = idx + c.len_utf8();
// codepoints outside the BMP get 2 '\uxxxx' sequences to represent them
for point in c.encode_utf16(&mut [0; 2]) {
write!(w, "\\u{point:04x}")?;
}
}
} else {
// Lone surrogate code point (U+D800..U+DFFF).
// WTF-8 encodes these as 3-byte sequences; skip those raw bytes
// and emit a \uXXXX escape with the surrogate value.
w.write_all(&bytes[write_start_idx..idx])?;
write_start_idx = idx + 3;
write!(w, "\\u{:04x}", cp.to_u32())?;
}
}
} else {
// ensure_ascii is false: only JSON-required escapes (< 0x20, \, ")
// are applied. 0x7F (DEL) is NOT escaped here, matching CPython's
// py_encode_basestring. Multi-byte UTF-8 characters and WTF-8
// surrogate sequences flow through unchanged via the trailing flush,
// so surrogates round-trip as-is (matching CPython behavior).
for (idx, c) in wtf8.as_bytes().iter().enumerate() {
let escaped_opt: Option<&'static str> = match *c {
x if x < 0x20 => Some(ESCAPE_CHARS[x as usize]),
b'\\' => Some("\\\\"),
b'\"' => Some("\\\""),
_ => None,
};
if let Some(escaped) = escaped_opt {
w.write_all(&bytes[write_start_idx..idx])?;
w.write_all(escaped.as_bytes())?;
write_start_idx = idx + 1;
}
}
}
w.write_all(&bytes[write_start_idx..])?;
w.write_all(b"\"")
}
#[derive(Debug)]
pub(super) struct DecodeError {
pub msg: String,
pub pos: usize,
}
impl DecodeError {
pub(super) fn new(msg: impl Into<String>, pos: usize) -> Self {
let msg = msg.into();
Self { msg, pos }
}
}
enum StrOrChar<'a> {
Str(&'a Wtf8),
Char(CodePoint),
}
impl StrOrChar<'_> {
const fn len(&self) -> usize {
match self {
StrOrChar::Str(s) => s.len(),
StrOrChar::Char(c) => c.len_wtf8(),
}
}
}
/// Scan a JSON string starting right after the opening quote.
///
/// # Arguments
/// * `s` - The string slice starting at the first character after the opening `"`
/// * `char_offset` - The character index where this slice starts (for error messages)
/// * `strict` - Whether to reject control characters
///
/// # Returns
/// * `Ok((result, chars_consumed, bytes_consumed))` - The decoded string and how much was consumed
/// * `Err(DecodeError)` - If the string is malformed
pub(super) fn scanstring<'a>(
s: &'a Wtf8,
char_offset: usize,
strict: bool,
) -> Result<(Wtf8Buf, usize, usize), DecodeError> {
flame_guard!("machinery::scanstring");
let unterminated_err = || DecodeError::new("Unterminated string starting at", char_offset - 1);
let bytes = s.as_bytes();
// Fast path: use memchr to find " or \ quickly
if let Some(pos) = memchr2(b'"', b'\\', bytes)
&& bytes[pos] == b'"'
{
let content_bytes = &bytes[..pos];
// In strict mode, check for control characters (0x00-0x1F)
let has_control_char = strict && content_bytes.iter().any(|&b| b < 0x20);
if !has_control_char {
flame_guard!("machinery::scanstring::fast_path");
let result_slice = &s[..pos];
let char_count = result_slice.code_points().count();
let mut out = Wtf8Buf::with_capacity(pos);
out.push_wtf8(result_slice);
// +1 for the closing quote
return Ok((out, char_count + 1, pos + 1));
}
}
// Slow path: chunk-based parsing for strings with escapes or control chars
flame_guard!("machinery::scanstring::slow_path");
let mut chunks: Vec<StrOrChar<'a>> = Vec::new();
let mut output_len = 0usize;
let mut push_chunk = |chunk: StrOrChar<'a>| {
output_len += chunk.len();
chunks.push(chunk);
};
let mut chars = s.code_point_indices().enumerate().peekable();
let mut chunk_start: usize = 0;
while let Some((char_i, (byte_i, c))) = chars.next() {
match c.to_char_lossy() {
'"' => {
push_chunk(StrOrChar::Str(&s[chunk_start..byte_i]));
flame_guard!("machinery::scanstring::assemble_chunks");
let mut out = Wtf8Buf::with_capacity(output_len);
for x in chunks {
match x {
StrOrChar::Str(s) => out.push_wtf8(s),
StrOrChar::Char(c) => out.push(c),
}
}
// +1 for the closing quote
return Ok((out, char_i + 1, byte_i + 1));
}
'\\' => {
push_chunk(StrOrChar::Str(&s[chunk_start..byte_i]));
let (next_char_i, (_, c)) = chars.next().ok_or_else(unterminated_err)?;
let esc = match c.to_char_lossy() {
'"' => "\"",
'\\' => "\\",
'/' => "/",
'b' => "\x08",
'f' => "\x0c",
'n' => "\n",
'r' => "\r",
't' => "\t",
'u' => {
// Error position for an invalid \uXXXX escape points at the
// `u`, not the `\` -- matches CPython's json.decoder.
let mut uni = decode_unicode(&mut chars, char_offset + next_char_i)?;
chunk_start = byte_i + 6;
if let Some(lead) = uni.to_lead_surrogate() {
// uni is a surrogate -- try to find its pair
let mut chars2 = chars.clone();
if let Some(((_, (byte_pos2, _)), (u2_char_i, (_, _)))) = chars2
.next_tuple()
.filter(|((_, (_, c1)), (_, (_, c2)))| *c1 == '\\' && *c2 == 'u')
{
// u2_char_i is the `u` of the second \uXXXX; same
// position convention as the primary call above.
let uni2 = decode_unicode(&mut chars2, char_offset + u2_char_i)?;
if let Some(trail) = uni2.to_trail_surrogate() {
// ok, we found what we were looking for -- \uXXXX\uXXXX, both surrogates
uni = lead.merge(trail).into();
chunk_start = byte_pos2 + 6;
chars = chars2;
}
}
}
push_chunk(StrOrChar::Char(uni));
continue;
}
_ => {
return Err(DecodeError::new(
format!("Invalid \\escape: {c:?}"),
char_offset + char_i,
));
}
};
chunk_start = byte_i + 2;
push_chunk(StrOrChar::Str(esc.as_ref()));
}
'\x00'..='\x1f' if strict => {
return Err(DecodeError::new(
format!("Invalid control character {c:?} at"),
char_offset + char_i,
));
}
_ => {}
}
}
Err(unterminated_err())
}
#[inline]
fn decode_unicode<I>(it: &mut I, pos: usize) -> Result<CodePoint, DecodeError>
where
I: Iterator<Item = (usize, (usize, CodePoint))>,
{
flame_guard!("machinery::decode_unicode");
let err = || DecodeError::new("Invalid \\uXXXX escape", pos);
let mut uni = 0u16;
for _ in 0..4 {
let (_, (_, c)) = it.next().ok_or_else(err)?;
let d = c.to_char().and_then(|c| c.to_digit(16)).ok_or_else(err)? as u16;
uni = (uni << 4) | d;
}
Ok(uni.into())
}
crates/stdlib/src/lib.rs
+8
View File
@@ -17,6 +17,7 @@ pub mod array;
mod binascii;
mod bisect;
mod bz2;
mod cjkcodecs;
mod cmath;
mod compression; // internal module
mod contextvars;
@@ -171,6 +172,13 @@ use crate::vm::{Context, builtins};
pub fn stdlib_module_defs(ctx: &Context) -> Vec<&'static builtins::PyModuleDef> {
vec![
_asyncio::module_def(ctx),
cjkcodecs::_codecs_cn::module_def(ctx),
cjkcodecs::_codecs_hk::module_def(ctx),
cjkcodecs::_codecs_iso2022::module_def(ctx),
cjkcodecs::_codecs_jp::module_def(ctx),
cjkcodecs::_codecs_kr::module_def(ctx),
cjkcodecs::_codecs_tw::module_def(ctx),
cjkcodecs::multibytecodec::module_def(ctx),
_opcode::module_def(ctx),
_remote_debugging::module_def(ctx),
array::module_def(ctx),
crates/stdlib/src/socket.rs
+27 -42
View File
@@ -7,7 +7,10 @@ pub(super) use _socket::{PySocket, SockWaitKind, sock_wait, timeout_error_msg};
#[pymodule]
mod _socket {
use crate::common::lock::{PyMappedRwLockReadGuard, PyRwLock, PyRwLockReadGuard};
use crate::common::{
inet,
lock::{PyMappedRwLockReadGuard, PyRwLock, PyRwLockReadGuard},
};
#[cfg(all(unix, not(target_os = "redox")))]
use crate::vm::convert::ToPyException;
use crate::vm::{
@@ -2329,19 +2332,17 @@ mod _socket {
#[pyfunction]
fn inet_aton(ip_string: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
ip_string
.as_str()
.parse::<Ipv4Addr>()
.map(|ip_addr| Vec::<u8>::from(ip_addr.octets()))
.map_err(|_| vm.new_os_error("illegal IP address string passed to inet_aton"))
inet::aton(ip_string.as_str().as_bytes())
.map(Vec::from)
.ok_or_else(|| vm.new_os_error("illegal IP address string passed to inet_aton"))
}
#[pyfunction]
fn inet_ntoa(packed_ip: ArgBytesLike, vm: &VirtualMachine) -> PyResult<PyStrRef> {
let packed_ip = packed_ip.borrow_buf();
let packed_ip = <&[u8; 4]>::try_from(&*packed_ip)
let packed_ip = <[u8; 4]>::try_from(&*packed_ip)
.map_err(|_| vm.new_os_error("packed IP wrong length for inet_ntoa"))?;
Ok(vm.ctx.new_str(Ipv4Addr::from(*packed_ip).to_string()))
Ok(vm.ctx.new_str(inet::ntoa(packed_ip)))
}
fn cstr_opt_as_ptr(x: &OptionalArg<ffi::CString>) -> *const core::ffi::c_char {
@@ -2747,40 +2748,34 @@ mod _socket {
#[pyfunction]
fn inet_pton(af_inet: i32, ip_string: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
static ERROR_MSG: &str = "illegal IP address string passed to inet_pton";
let ip_addr = match af_inet {
c::AF_INET => ip_string
.as_str()
.parse::<Ipv4Addr>()
.map_err(|_| vm.new_os_error(ERROR_MSG.to_owned()))?
.octets()
.to_vec(),
c::AF_INET6 => ip_string
.as_str()
.parse::<Ipv6Addr>()
.map_err(|_| vm.new_os_error(ERROR_MSG.to_owned()))?
.octets()
.to_vec(),
_ => return Err(vm.new_os_error("Address family not supported by protocol")),
let text = ip_string.as_str().as_bytes();
let packed = match af_inet {
c::AF_INET => inet::pton_v4(text).map(Vec::from),
c::AF_INET6 => inet::pton_v6(text).map(Vec::from),
// What the host would report for an address family it has no
// converter for.
_ => {
let unsupported = rustpython_host_env::errno::errors::EAFNOSUPPORT;
return Err(io::Error::from_raw_os_error(unsupported).into_pyexception(vm));
}
};
Ok(ip_addr)
packed.ok_or_else(|| vm.new_os_error("illegal IP address string passed to inet_pton"))
}
#[pyfunction]
fn inet_ntop(af_inet: i32, packed_ip: ArgBytesLike, vm: &VirtualMachine) -> PyResult<String> {
let packed_ip = packed_ip.borrow_buf();
let wrong_length = || vm.new_value_error("invalid length of packed IP address string");
match af_inet {
c::AF_INET => {
let packed_ip = <&[u8; 4]>::try_from(&*packed_ip).map_err(|_| {
vm.new_value_error("invalid length of packed IP address string")
})?;
Ok(Ipv4Addr::from(*packed_ip).to_string())
let packed_ip = <[u8; 4]>::try_from(&*packed_ip).map_err(|_| wrong_length())?;
Ok(inet::ntoa(packed_ip))
}
c::AF_INET6 => {
let packed_ip = <&[u8; 16]>::try_from(&*packed_ip).map_err(|_| {
vm.new_value_error("invalid length of packed IP address string")
})?;
Ok(get_ipv6_addr_str(Ipv6Addr::from(*packed_ip)))
if packed_ip.len() != 16 {
return Err(wrong_length());
}
Ok(inet::ntop_v6(&packed_ip))
}
_ => Err(vm.new_value_error(format!("unknown address family {af_inet}"))),
}
@@ -3099,16 +3094,6 @@ mod _socket {
vm.new_os_subtype_error(timeout(vm), None, msg)
}
fn get_ipv6_addr_str(ipv6: Ipv6Addr) -> String {
match ipv6.to_ipv4() {
// instead of "::0.0.ddd.ddd" it's "::xxxx"
Some(v4) if !ipv6.is_unspecified() && matches!(v4.octets(), [0, 0, _, _]) => {
format!("::{:x}", u32::from(v4))
}
_ => ipv6.to_string(),
}
}
pub(crate) struct Deadline {
deadline: Instant,
}
scripts/port_cjk_mappings.py
+221
View File
@@ -0,0 +1,221 @@
#!/usr/bin/env python3
"""Mechanically translate PyPy cjkcodecs mapping headers to Rust arrays.
The codec state machines stay hand-ported line by line. This script only
changes the spelling of the generated static data and its page indexes, so the
Rust port keeps PyPy's ``decode_map`` / ``encode_map`` storage shape.
"""
from __future__ import annotations
import argparse
import re
from pathlib import Path
ARRAY_RE = re.compile(
r"static const (?P<type>ucs2_t|ucs4_t|DBCHAR|unsigned char) "
r"(?P<name>[A-Za-z0-9_]+)\[(?P<size>\d*)\] = \{(?P<body>.*?)\n\};",
re.S,
)
INDEX_RE = re.compile(
r"static const struct (?P<type>dbcs_index|widedbcs_index|unim_index) "
r"(?P<name>[A-Za-z0-9_]+)\[256\] = \{(?P<body>.*?)\n\};",
re.S,
)
GB18030_RANGE_RE = re.compile(
r"static const struct _gb18030_to_unibmp_ranges \{.*?\} "
r"gb18030_to_unibmp_ranges\[\] = \{(?P<body>.*?)\};",
re.S,
)
PAIR_ENCODE_RE = re.compile(
r"static const struct pair_encodemap (?P<name>[A-Za-z0-9_]+)"
r"\[(?P<size>[A-Za-z0-9_]+)\] = \{(?P<body>.*?)\n\};",
re.S,
)
RUST_TYPE = {
"ucs2_t": "u16",
"ucs4_t": "u32",
"DBCHAR": "u16",
"unsigned char": "u8",
}
SENTINEL = {"U": "UNIINV", "N": "NOCHAR", "M": "MULTIC", "D": "DBCINV"}
def rust_name(name: str) -> str:
return name.lstrip("_").upper()
def wrap(items: list[str], width: int = 100) -> list[str]:
lines: list[str] = []
line = " "
for item in items:
token = item + ","
if len(line) > 4 and len(line) + len(token) + 1 > width:
lines.append(line.rstrip())
line = " "
line += token + " "
if len(line) > 4:
lines.append(line.rstrip())
return lines
def translate(source: Path) -> str:
text = source.read_text()
output = [
"// Generated mechanically from PyPy's cjkcodecs mapping header by",
"// `scripts/port_cjk_mappings.py`. Do not change the page layout:",
"// `cjkcodecs.h::_TRYMAP_DEC` / `_TRYMAP_ENC` index these exact arrays.",
"",
"pub(super) const UNIINV: u16 = 0xfffe;",
"#[allow(dead_code)]",
"pub(super) const NOCHAR: u16 = 0xffff;",
"#[allow(dead_code)]",
"pub(super) const MULTIC: u16 = 0xfffe;",
"#[allow(dead_code)]",
"pub(super) const DBCINV: u16 = 0xfffd;",
"",
"#[derive(Clone, Copy)]",
"pub(super) struct MapIndex {",
" pub(super) offset: usize,",
" pub(super) bottom: u8,",
" pub(super) top: u8,",
" pub(super) present: bool,",
"}",
"",
"const EMPTY_INDEX: MapIndex = MapIndex {",
" offset: 0,",
" bottom: 0,",
" top: 0,",
" present: false,",
"};",
"",
]
arrays: dict[str, int] = {}
for match in ARRAY_RE.finditer(text):
c_name = match.group("name")
name = rust_name(c_name) + "_DATA"
values = [
part.strip() for part in match.group("body").replace("\n", "").split(",")
]
rust_type = RUST_TYPE[match.group("type")]
values = [
(
f"{SENTINEL[value]} as {rust_type}"
if value in SENTINEL and rust_type != "u16"
else SENTINEL.get(value, value)
)
for value in values
if value
]
size = int(match.group("size")) if match.group("size") else len(values)
if len(values) != size:
raise ValueError(f"{c_name}: declared {size}, parsed {len(values)}")
arrays[c_name] = size
output.append(f"pub(super) static {name}: [{rust_type}; {size}] = [")
output.extend(wrap(values))
output.extend(["];", ""])
for match in INDEX_RE.finditer(text):
c_name = match.group("name")
name = c_name.upper()
entries = re.findall(r"\{([^{}]*)\}", match.group("body"))
if len(entries) != 256:
raise ValueError(f"{c_name}: expected 256 indexes, parsed {len(entries)}")
translated: list[str] = []
for entry in entries:
compact = re.sub(r"\s+", "", entry)
pointer, bottom, top = compact.split(",")
if pointer == "0":
translated.append("EMPTY_INDEX")
continue
array_name, offset = pointer.split("+")
if array_name not in arrays:
raise ValueError(f"{c_name}: unknown array {array_name}")
translated.append(
"MapIndex { "
f"offset: {offset}, bottom: {bottom}, top: {top}, present: true "
"}"
)
output.append(f"pub(super) static {name}: [MapIndex; 256] = [")
output.extend(wrap(translated, 120))
output.extend(["];", ""])
if match := GB18030_RANGE_RE.search(text):
entries = re.findall(
r"\{\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*\}", match.group("body")
)
output.extend(
[
"#[derive(Clone, Copy)]",
"pub(super) struct Gb18030Range {",
" pub(super) first: u32,",
" pub(super) last: u32,",
" pub(super) base: u32,",
"}",
"",
f"pub(super) static GB18030_TO_UNIBMP_RANGES: [Gb18030Range; {len(entries)}] = [",
]
)
output.extend(
wrap(
[
f"Gb18030Range {{ first: {first}, last: {last}, base: {base} }}"
for first, last, base in entries
],
120,
)
)
output.extend(["];", ""])
if match := PAIR_ENCODE_RE.search(text):
entries = re.findall(
r"\{\s*(0x[0-9a-fA-F]+)\s*,\s*(0x[0-9a-fA-F]+)\s*\}", match.group("body")
)
size_name = match.group("size")
size_match = re.search(rf"#define\s+{re.escape(size_name)}\s+(\d+)", text)
if size_match is None:
raise ValueError(f"{match.group('name')}: unknown size {size_name}")
size = int(size_match.group(1))
if len(entries) != size:
raise ValueError(
f"{match.group('name')}: declared {size}, parsed {len(entries)}"
)
output.extend(
[
"#[derive(Clone, Copy)]",
"pub(super) struct PairEncodeMap {",
" pub(super) uniseq: u32,",
" pub(super) code: u16,",
"}",
"",
f"pub(super) static {match.group('name').upper()}: [PairEncodeMap; {size}] = [",
]
)
output.extend(
wrap(
[
f"PairEncodeMap {{ uniseq: {uniseq}, code: {code} }}"
for uniseq, code in entries
],
120,
)
)
output.extend(["];", ""])
if not arrays:
raise ValueError(f"{source}: no mapping arrays found")
return "\n".join(output)
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("source", type=Path)
parser.add_argument("output", type=Path)
args = parser.parse_args()
args.output.write_text(translate(args.source))
if __name__ == "__main__":
main()