mirror of
https://github.com/RustPython/RustPython.git
synced 2026-10-28 18:59:04 +08:00
str: count a stepped slice's characters with a ceiling division (#8525)
`do_stepped_slice` and `do_stepped_slice_reverse` took the character count of
a non-ASCII result as `(range.len() / step) + 1`, which overshoots by one
whenever the span is an exact multiple of the step: `"aéc"[::3]` is one
character and reported two, `"가나다라"[::2]` two and reported three. ASCII
subjects were unaffected, since that arm collects into an `AsciiString` whose
length comes from the data.
The count is stored as the string's character length, so the result then
claimed a character its buffer does not hold, and `reversed()` on it indexed
past the end and panicked:
s = "".join(["a", "é", "c"])
list(reversed(s[::3])) # index out of bounds: the len is 1 but the index is 1
Use `div_ceil`, which is the number of elements the underlying range yields.
A differential over 44331 subscript and slice shapes -- every combination of
start, stop and step over strings straddling the ASCII split, the surrogate
range and the astral plane -- now matches CPython 3.14 exactly, where it
differed on 3100 lines before.
Assisted-by: Claude
This commit is contained in:
1 parent
d04318efac
commit
9f90967b0a
2 files changed
+33
-4
No files matched your search
@@ -1870,14 +1870,14 @@ impl SliceableSequenceOp for PyStr {
|
||||
.collect::<AsciiString>()
|
||||
.into(),
|
||||
PyKindStr::Utf8(s) => {
|
||||
let char_len = (range.len() / step) + 1;
|
||||
let char_len = range.len().div_ceil(step);
|
||||
let mut out = String::with_capacity(2 * char_len);
|
||||
out.extend(s.chars().skip(range.start).take(range.len()).step_by(step));
|
||||
// SAFETY: char_len is accurate
|
||||
unsafe { Self::new_with_char_len(out, char_len) }
|
||||
}
|
||||
PyKindStr::Wtf8(w) => {
|
||||
let char_len = (range.len() / step) + 1;
|
||||
let char_len = range.len().div_ceil(step);
|
||||
let mut out = Wtf8Buf::with_capacity(2 * char_len);
|
||||
out.extend(
|
||||
w.code_points()
|
||||
@@ -1900,7 +1900,7 @@ impl SliceableSequenceOp for PyStr {
|
||||
.collect::<AsciiString>()
|
||||
.into(),
|
||||
PyKindStr::Utf8(s) => {
|
||||
let char_len = (range.len() / step) + 1;
|
||||
let char_len = range.len().div_ceil(step);
|
||||
// not ascii, so the codepoints have to be at least 2 bytes each
|
||||
let mut out = String::with_capacity(2 * char_len);
|
||||
out.extend(
|
||||
@@ -1914,7 +1914,7 @@ impl SliceableSequenceOp for PyStr {
|
||||
unsafe { Self::new_with_char_len(out, char_len) }
|
||||
}
|
||||
PyKindStr::Wtf8(w) => {
|
||||
let char_len = (range.len() / step) + 1;
|
||||
let char_len = range.len().div_ceil(step);
|
||||
// not ascii, so the codepoints have to be at least 2 bytes each
|
||||
let mut out = Wtf8Buf::with_capacity(2 * char_len);
|
||||
out.extend(
|
||||
|
||||
@@ -59,3 +59,32 @@ assert hebrew_text[30:10:-3] == "אםהֱאּ "
|
||||
assert len(hebrew_text[30:10:-3]) == 7
|
||||
assert hebrew_text[30:10:-1] == "א ,םיִהֹלֱא אָרָּב ,"
|
||||
assert len(hebrew_text[30:10:-1]) == 20
|
||||
|
||||
|
||||
# A stepped slice whose span is an exact multiple of the step ends on the last
|
||||
# character it collects rather than one past it, so the character count is the
|
||||
# span divided by the step and not one more. The subject goes through a
|
||||
# variable because a constant subscript is folded at compile time and would
|
||||
# never reach the runtime slice at all.
|
||||
def stepped(s, step):
|
||||
return s[::step]
|
||||
|
||||
|
||||
for subject, step, expected in [
|
||||
("a\u00e9c", 3, "a"),
|
||||
("가나다라", 2, "가다"),
|
||||
("가나다라마바", 3, "가라"),
|
||||
("가나다라", -2, "라나"),
|
||||
("가나다라마바", -3, "바다"),
|
||||
("\U0001f600\U0001f601\U0001f602\U0001f603", 2, "\U0001f600\U0001f602"),
|
||||
]:
|
||||
sliced = stepped(subject, step)
|
||||
assert sliced == expected, (subject, step, sliced)
|
||||
assert len(sliced) == len(expected), (subject, step, len(sliced))
|
||||
# An overstated count makes the string claim characters its buffer does not
|
||||
# hold, which reversed() then reads past.
|
||||
assert list(reversed(sliced)) == list(expected)[::-1]
|
||||
|
||||
assert len(stepped(hebrew_text, 2)) == 30
|
||||
assert len(stepped(hebrew_text, 4)) == 15
|
||||
assert len(stepped(hebrew_text, -2)) == 30
|
||||
Reference in new issue
Block a user