mirror of
https://github.com/RustPython/RustPython.git
synced 2026-10-29 03:04:38 +08:00
Jeong, YunWon
ee94b5b723
`search_info_literal`'s len>1 arm reset the tail-match cursor by advancing one character past the matched prefix. That position equals the INFO block's prefix_skip boundary only when the prefix ends where the skip does; for a pattern like `ab(cd)`, `_get_literal_prefix` reports prefix_len=4 with prefix_skip=2, so the tail resumed two characters too far. `state.cursor` already holds `req.start + skip`, which is what `sre_lib.h` SRE(search) computes as `ptr - (prefix_len - prefix_skip - 1)`, and which the len==1 arm above already uses. Take it unconditionally. Searching "xabcdcd" for `ab(cd)` returned span (1, 7) instead of (1, 5); over a 600-case corpus stamped with CPython's own answers, 94 cases disagreed before this change and none after. The wrong resume also dropped matches outright, not only widened them. Assisted-by: Claude
271 lines
12 KiB
Rust
271 lines
12 KiB
Rust
// spell-checker:disable
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use rustpython_sre_engine::{Request, State, StrDrive};
|
|
use rustpython_wtf8::Wtf8Buf;
|
|
|
|
struct Pattern {
|
|
#[expect(unused, reason = "Needed for automated script")]
|
|
pattern: &'static str,
|
|
code: &'static [u32],
|
|
}
|
|
|
|
impl Pattern {
|
|
fn state<'a, S: StrDrive>(&self, string: S) -> (Request<'a, S>, State) {
|
|
let req = Request::new(string, 0, usize::MAX, self.code, false);
|
|
let state = State::default();
|
|
(req, state)
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn gh_2427() {
|
|
// pattern lookbehind = re.compile(r'(?<!\.)x\b')
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let lookbehind = Pattern { pattern: "(?<!\\.)x\\b", code: &[14, 4, 0, 1, 1, 5, 5, 1, 16, 46, 1, 16, 120, 6, 10, 1] };
|
|
// END GENERATED
|
|
let (req, mut state) = lookbehind.state("x");
|
|
assert!(state.py_match(&req));
|
|
}
|
|
|
|
#[test]
|
|
fn basic_assert() {
|
|
// pattern positive_lookbehind = re.compile(r'(?<=abc)def')
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let positive_lookbehind = Pattern { pattern: "(?<=abc)def", code: &[14, 4, 0, 3, 3, 4, 9, 3, 16, 97, 16, 98, 16, 99, 1, 16, 100, 16, 101, 16, 102, 1] };
|
|
// END GENERATED
|
|
let (req, mut state) = positive_lookbehind.state("abcdef");
|
|
assert!(state.search(req));
|
|
}
|
|
|
|
#[test]
|
|
fn string_boundaries() {
|
|
// pattern big_b = re.compile(r'\B')
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let big_b = Pattern { pattern: "\\B", code: &[14, 4, 0, 0, 0, 6, 11, 1] };
|
|
// END GENERATED
|
|
let (req, mut state) = big_b.state("");
|
|
assert!(state.search(req));
|
|
}
|
|
|
|
#[test]
|
|
fn zerowidth() {
|
|
// pattern p = re.compile(r'\b|:+')
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "\\b|:+", code: &[14, 4, 0, 0, 4294967295, 7, 5, 6, 10, 15, 12, 10, 24, 6, 1, 4294967295, 16, 58, 1, 15, 2, 0, 1] };
|
|
// END GENERATED
|
|
let (mut req, mut state) = p.state("a:");
|
|
req.must_advance = true;
|
|
assert!(state.search(req));
|
|
assert_eq!(state.cursor.position, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn repeat_context_panic() {
|
|
use optional::Optioned;
|
|
// pattern p = re.compile(r'(?:a*?(xx)??z)*')
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "(?:a*?(xx)??z)*", code: &[14, 4, 0, 0, 4294967295, 23, 25, 0, 4294967295, 26, 6, 0, 4294967295, 16, 97, 1, 23, 11, 0, 1, 17, 0, 16, 120, 16, 120, 17, 1, 19, 16, 122, 18, 1] };
|
|
// END GENERATED
|
|
let (req, mut state) = p.state("axxzaz");
|
|
assert!(state.py_match(&req));
|
|
assert_eq!(
|
|
*state.marks.raw(),
|
|
vec![Optioned::some(1), Optioned::some(3)]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn double_max_until() {
|
|
// pattern p = re.compile(r'((1)?)*')
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "((1)?)*", code: &[14, 4, 0, 0, 4294967295, 23, 18, 0, 4294967295, 17, 0, 23, 9, 0, 1, 17, 2, 16, 49, 17, 3, 18, 17, 1, 18, 1] };
|
|
// END GENERATED
|
|
let (req, mut state) = p.state("1111");
|
|
assert!(state.py_match(&req));
|
|
assert_eq!(state.cursor.position, 4);
|
|
}
|
|
|
|
#[test]
|
|
fn info_single() {
|
|
// pattern p = re.compile(r'aa*')
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "aa*", code: &[14, 8, 1, 1, 4294967295, 1, 1, 97, 0, 16, 97, 24, 6, 0, 4294967295, 16, 97, 1, 1] };
|
|
// END GENERATED
|
|
let (req, mut state) = p.state("baaaa");
|
|
assert!(state.search(req));
|
|
assert_eq!(state.start, 1);
|
|
assert_eq!(state.cursor.position, 5);
|
|
}
|
|
|
|
#[test]
|
|
fn info_single2() {
|
|
// pattern p = re.compile(r'Python|Perl')
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "Python|Perl", code: &[14, 8, 1, 4, 6, 1, 1, 80, 0, 16, 80, 7, 13, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 15, 11, 9, 16, 101, 16, 114, 16, 108, 15, 2, 0, 1] };
|
|
// END GENERATED
|
|
let (req, mut state) = p.state("Perl");
|
|
assert!(state.search(req));
|
|
}
|
|
|
|
#[test]
|
|
fn info_literal() {
|
|
// pattern p = re.compile(r'ababc+')
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "ababc+", code: &[14, 14, 1, 5, 4294967295, 4, 4, 97, 98, 97, 98, 0, 0, 1, 2, 16, 97, 16, 98, 16, 97, 16, 98, 24, 6, 1, 4294967295, 16, 99, 1, 1] };
|
|
// END GENERATED
|
|
let (req, mut state) = p.state("!ababc");
|
|
assert!(state.search(req));
|
|
}
|
|
|
|
#[test]
|
|
fn info_literal2() {
|
|
// pattern p = re.compile(r'(python)\1')
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "(python)\\1", code: &[14, 18, 1, 12, 12, 6, 0, 112, 121, 116, 104, 111, 110, 0, 0, 0, 0, 0, 0, 17, 0, 16, 112, 16, 121, 16, 116, 16, 104, 16, 111, 16, 110, 17, 1, 11, 0, 1] };
|
|
// END GENERATED
|
|
let (req, mut state) = p.state("pythonpython");
|
|
assert!(state.search(req));
|
|
}
|
|
|
|
#[test]
|
|
fn repeat_in_assertions() {
|
|
// pattern p = re.compile('^([ab]*?)(?=(b)?)c', re.IGNORECASE)
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "^([ab]*?)(?=(b)?)c", code: &[14, 4, 0, 1, 4294967295, 6, 0, 17, 0, 26, 10, 0, 4294967295, 39, 5, 22, 97, 98, 0, 1, 17, 1, 4, 14, 0, 23, 9, 0, 1, 17, 2, 40, 98, 17, 3, 18, 1, 40, 99, 1] };
|
|
// END GENERATED
|
|
let (req, mut state) = p.state("abc");
|
|
assert!(state.search(req));
|
|
}
|
|
|
|
#[test]
|
|
fn possessive_quantifier() {
|
|
// pattern p = re.compile('e++a')
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "e++a", code: &[14, 4, 0, 2, 4294967295, 29, 6, 1, 4294967295, 16, 101, 1, 16, 97, 1] };
|
|
// END GENERATED
|
|
let (req, mut state) = p.state("eeea");
|
|
assert!(state.py_match(&req));
|
|
}
|
|
|
|
#[test]
|
|
fn possessive_repeat_fullmatch() {
|
|
// pattern p = re.compile("([0-9]++(?:\.[0-9]+)*+)", re.I )
|
|
// [INFO, 4, 0, 1, 4294967295, MARK, 0, POSSESSIVE_REPEAT_ONE, 10, 1, MAXREPEAT, IN, 5, RANGE, 48, 57, FAILURE, SUCCESS, POSSESSIVE_REPEAT, 16, 0, MAXREPEAT, LITERAL, 46, REPEAT_ONE, 10, 1, MAXREPEAT, IN, 5, RANGE, 48, 57, FAILURE, SUCCESS, SUCCESS, MARK, 1, SUCCESS]
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "([0-9]++(?:\\.[0-9]+)*+)", code: &[14, 4, 0, 1, 4294967295, 17, 0, 29, 10, 1, 4294967295, 13, 5, 22, 48, 57, 0, 1, 28, 16, 0, 4294967295, 16, 46, 24, 10, 1, 4294967295, 13, 5, 22, 48, 57, 0, 1, 1, 17, 1, 1] };
|
|
// END GENERATED
|
|
let (mut req, mut state) = p.state("1.25.38");
|
|
req.match_all = true;
|
|
assert!(state.py_match(&req), "should match");
|
|
}
|
|
|
|
#[test]
|
|
fn possessive_atomic_group() {
|
|
// pattern p = re.compile('(?>x)++x')
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "(?>x)++x", code: &[14, 4, 0, 2, 4294967295, 28, 8, 1, 4294967295, 27, 4, 16, 120, 1, 1, 16, 120, 1] };
|
|
// END GENERATED
|
|
let (req, mut state) = p.state("xxx");
|
|
assert!(!state.py_match(&req));
|
|
}
|
|
|
|
#[test]
|
|
fn possessive_repeat_keeps_last_capture() {
|
|
use optional::Optioned;
|
|
|
|
let single_code = &[17, 0, 24, 6, 0, 1, 16, 101, 1, 17, 1, 1];
|
|
let req = Request::new("eeea", 3, usize::MAX, single_code, false);
|
|
let mut single_state = State::default();
|
|
assert!(single_state.py_match(&req));
|
|
assert_eq!(
|
|
single_state.marks.get(0),
|
|
(Optioned::some(3), Optioned::some(3))
|
|
);
|
|
|
|
// (e?){2,4}+a: the fourth successful iteration is empty, so group 1
|
|
// must retain its final empty span rather than the previous "e".
|
|
#[rustfmt::skip] let optional = Pattern {
|
|
pattern: "(e?){2,4}+a",
|
|
code: &[14, 4, 0, 1, 5, 28, 14, 2, 4, 17, 0, 24, 6, 0, 1, 16, 101, 1, 17, 1, 1, 16, 97, 1],
|
|
};
|
|
let (req, mut state) = optional.state("eeea");
|
|
assert!(state.py_match(&req));
|
|
assert_eq!(state.marks.get(0), (Optioned::some(3), Optioned::some(3)));
|
|
|
|
// ((x)|y|z){3}+: group 1 is the final "z"; group 2 retains "x".
|
|
#[rustfmt::skip] let alternation = Pattern {
|
|
pattern: "((x)|y|z){3}+",
|
|
code: &[14, 4, 0, 3, 3, 28, 28, 3, 3, 17, 0, 7, 9, 17, 2, 16, 120, 17, 3, 15, 12, 5, 16, 121, 15, 7, 5, 16, 122, 15, 2, 0, 17, 1, 1, 1],
|
|
};
|
|
let (req, mut state) = alternation.state("xyz");
|
|
assert!(state.py_match(&req));
|
|
assert_eq!(state.marks.get(0), (Optioned::some(2), Optioned::some(3)));
|
|
assert_eq!(state.marks.get(1), (Optioned::some(0), Optioned::some(1)));
|
|
}
|
|
|
|
#[test]
|
|
fn bug_20998() {
|
|
// pattern p = re.compile('[a-c]+', re.I)
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "[a-c]+", code: &[14, 4, 0, 1, 4294967295, 24, 10, 1, 4294967295, 39, 5, 22, 97, 99, 0, 1, 1] };
|
|
// END GENERATED
|
|
let (mut req, mut state) = p.state("ABC");
|
|
req.match_all = true;
|
|
assert!(state.py_match(&req));
|
|
assert_eq!(state.cursor.position, 3);
|
|
}
|
|
|
|
#[test]
|
|
fn ascii_ignore_keeps_nonascii_range_literal() {
|
|
// pattern p = re.compile(r'[\u0430-\u045f]', re.I | re.A)
|
|
//
|
|
// ASCII-only case folding must not discard an exact non-ASCII range:
|
|
// U+0450 lies in the compiled U+0430..U+045F interval.
|
|
#[rustfmt::skip] let p = Pattern {
|
|
pattern: "[\\u0430-\\u045f]",
|
|
code: &[14, 8, 4, 1, 1, 22, 1072, 1119, 0, 13, 5, 22, 1072, 1119, 0, 1],
|
|
};
|
|
let (req, mut state) = p.state("\u{0450}");
|
|
assert!(state.py_match(&req));
|
|
let subject = Wtf8Buf::from("\u{0450}");
|
|
let (req, mut state) = p.state(subject.as_ref());
|
|
assert!(state.py_match(&req));
|
|
}
|
|
|
|
#[test]
|
|
fn bigcharset() {
|
|
// pattern p = re.compile('[a-z]*', re.I)
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "[a-z]*", code: &[14, 4, 0, 0, 4294967295, 24, 97, 0, 4294967295, 39, 92, 10, 3, 33685760, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 33686018, 0, 0, 0, 134217726, 0, 0, 0, 0, 0, 131072, 0, 2147483648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1] };
|
|
// END GENERATED
|
|
let (req, mut state) = p.state("x ");
|
|
assert!(state.py_match(&req));
|
|
assert_eq!(state.cursor.position, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn search_nonascii() {
|
|
#[allow(unused)]
|
|
// pattern p = re.compile('\xe0+')
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "\u{e0}+", code: &[14, 4, 0, 1, 4294967295, 24, 6, 1, 4294967295, 16, 224, 1, 1] };
|
|
// END GENERATED
|
|
}
|
|
|
|
#[test]
|
|
fn search_literal_prefix_longer_than_skip() {
|
|
// The INFO block carries prefix_len=4 and prefix_skip=2, so the tail
|
|
// match has to resume at the skip boundary rather than past the whole
|
|
// prefix.
|
|
// pattern p = re.compile('ab(cd)')
|
|
// START GENERATED by generate_tests.py
|
|
#[rustfmt::skip] let p = Pattern { pattern: "ab(cd)", code: &[14, 14, 1, 4, 4, 4, 2, 97, 98, 99, 100, 0, 0, 0, 0, 16, 97, 16, 98, 17, 0, 16, 99, 16, 100, 17, 1, 1] };
|
|
// END GENERATED
|
|
let (req, mut state) = p.state("xabcdcd");
|
|
assert!(state.search(req));
|
|
assert_eq!(state.start, 1);
|
|
assert_eq!(state.cursor.position, 5);
|
|
}
|
|
}
|