From 65f30fd151189def30b2ee7bafdcdaef0217372f Mon Sep 17 00:00:00 2001 From: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:28:10 +0300 Subject: [PATCH] Return character counts from the UTF-8 encoder fast path (#9205) Assisted-by: Codex:GPT-6 --- crates/vm/src/stdlib/_codecs.rs | 2 +- extra_tests/snippets/stdlib_codecs_wide.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/stdlib/_codecs.rs b/crates/vm/src/stdlib/_codecs.rs index 23e92f847..371a91295 100644 --- a/crates/vm/src/stdlib/_codecs.rs +++ b/crates/vm/src/stdlib/_codecs.rs @@ -241,7 +241,7 @@ mod _codecs { .as_ref() .is_some_and(|s| s.is(identifier!(vm, surrogatepass))) { - return Ok((args.str.as_bytes().to_vec(), args.str.byte_len())); + return Ok((args.str.as_bytes().to_vec(), args.str.char_len())); } do_codec!(utf8::encode, args, vm) } diff --git a/extra_tests/snippets/stdlib_codecs_wide.py b/extra_tests/snippets/stdlib_codecs_wide.py index 00f8a780b..817cecb79 100644 --- a/extra_tests/snippets/stdlib_codecs_wide.py +++ b/extra_tests/snippets/stdlib_codecs_wide.py @@ -21,6 +21,10 @@ assert b"+-".decode("utf-7") == "+" import _codecs +# Encoder counts are characters consumed, not the size of the encoded bytes. +assert _codecs.utf_8_encode("é😀") == (b"\xc3\xa9\xf0\x9f\x98\x80", 2) +assert _codecs.utf_8_encode("a\ud800b", "surrogatepass") == (b"a\xed\xa0\x80b", 3) + assert _codecs.escape_encode(b"a\tb") == (b"a\\tb", 3) assert _codecs.escape_decode(rb"a\tb") == (b"a\tb", 4) assert _codecs.utf_16_ex_decode(b"\xff\xfeh\x00i\x00", "strict", 0, True)[0] == "hi"