impl re.template(),

template_compile template_expand subx
This commit is contained in:
Kangzhi Shi authored and Jeong YunWon committed 2024-03-22 14:59:19 +09:00
1 parent 1e3d57817c
commit d9375b9fe1
5 files changed
+191 -97

No files matched your search

Generated
+1
View File
@@ -2532,6 +2532,7 @@ dependencies = [
[[package]]
name = "sre-engine"
version = "0.6.0"
source = "git+https://github.com/RustPython/sre-engine?rev=refs/pull/17/head#9725808c302b13e873ff7f955dd4c1632f1137fd"
dependencies = [
"bitflags 2.4.0",
"num_enum",
+30 -1
View File
@@ -45,7 +45,7 @@ def capwords(s, sep=None):
sep is used to split and join the words.
"""
return (sep or ' ').join(x.capitalize() for x in s.split(sep))
return (sep or ' ').join(map(str.capitalize, s.split(sep)))
####################################################################
@@ -141,6 +141,35 @@ class Template:
self.pattern)
return self.pattern.sub(convert, self.template)
def is_valid(self):
for mo in self.pattern.finditer(self.template):
if mo.group('invalid') is not None:
return False
if (mo.group('named') is None
and mo.group('braced') is None
and mo.group('escaped') is None):
# If all the groups are None, there must be
# another group we're not expecting
raise ValueError('Unrecognized named group in pattern',
self.pattern)
return True
def get_identifiers(self):
ids = []
for mo in self.pattern.finditer(self.template):
named = mo.group('named') or mo.group('braced')
if named is not None and named not in ids:
# add a named group only the first time it appears
ids.append(named)
elif (named is None
and mo.group('invalid') is None
and mo.group('escaped') is None):
# If all the groups are None, there must be
# another group we're not expecting
raise ValueError('Unrecognized named group in pattern',
self.pattern)
return ids
# Initialize Template.pattern. __init_subclass__() is automatically called
# only for subclasses, not for the Template class itself.
Template.__init_subclass__()
+4 -34
View File
@@ -116,8 +116,6 @@ class ReTests(unittest.TestCase):
int_value = int(matchobj.group(0))
return str(int_value + 1)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_basic_re_sub(self):
self.assertTypedEqual(re.sub('y', 'a', 'xyz'), 'xaz')
self.assertTypedEqual(re.sub('y', S('a'), S('xyz')), 'xaz')
@@ -161,15 +159,11 @@ class ReTests(unittest.TestCase):
self.assertEqual(re.sub(r'^\s*', 'X', 'test'), 'Xtest')
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bug_449964(self):
# fails for group followed by other escape
self.assertEqual(re.sub(r'(?P<unk>x)', r'\g<1>\g<1>\b', 'xx'),
'xx\bxx\b')
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bug_449000(self):
# Test for sub() on escaped characters
self.assertEqual(re.sub(r'\r\n', r'\n', 'abc\r\ndef\r\n'),
@@ -193,8 +187,6 @@ class ReTests(unittest.TestCase):
# A regex that triggered a bug in the sre-code validator
re.compile("(?P<quote>)(?(quote))")
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_sub_template_numeric_escape(self):
# bug 776311 and friends
self.assertEqual(re.sub('x', r'\0', 'x'), '\0')
@@ -248,8 +240,6 @@ class ReTests(unittest.TestCase):
self.assertEqual(re.sub('a', 'b', 'aaaaa', 1), 'baaaa')
self.assertEqual(re.sub('a', 'b', 'aaaaa', count=1), 'baaaa')
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bug_114660(self):
self.assertEqual(re.sub(r'(\S)\s+(\S)', r'\1 \2', 'hello there'),
'hello there')
@@ -303,8 +293,6 @@ class ReTests(unittest.TestCase):
self.checkPatternError(b'(?(\xc2\xb5)y)',
r"bad character in group name '\xc2\xb5'", 3)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_symbolic_refs(self):
self.assertEqual(re.sub('(?P<a>x)|(?P<b>y)', r'\g<b>', 'xx'), '')
self.assertEqual(re.sub('(?P<a>x)|(?P<b>y)', r'\2', 'xx'), '')
@@ -316,8 +304,6 @@ class ReTests(unittest.TestCase):
pat = '|'.join('x(?P<a%d>%x)y' % (i, i) for i in range(1, 200 + 1))
self.assertEqual(re.sub(pat, r'\g<200>', 'xc8yzxc8y'), 'c8zc8')
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_symbolic_refs_errors(self):
self.checkTemplateError('(?P<a>x)', r'\g<a', 'xx',
'missing >, unterminated name', 3)
@@ -651,8 +637,6 @@ class ReTests(unittest.TestCase):
with self.subTest(code=i):
re.compile(r'()(?(1)\x%02x?)' % i)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_re_groupref_overflow(self):
from re._constants import MAXGROUPS
self.checkTemplateError('()', r'\g<%s>' % MAXGROUPS, 'xx',
@@ -1754,8 +1738,6 @@ class ReTests(unittest.TestCase):
self.assertTrue(re.fullmatch('(?x)#x\na|#y\nb', 'a'))
self.assertTrue(re.fullmatch('(?x)#x\na|#y\nb', 'b'))
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bug_6509(self):
# Replacement strings of both types must parse properly.
# all strings
@@ -1902,8 +1884,6 @@ class ReTests(unittest.TestCase):
)
self.assertRegex(repr(second), pattern)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_zerowidth(self):
# Issues 852532, 1647489, 3262, 25054.
self.assertEqual(re.split(r"\b", "a::bc"), ['', 'a', '::', 'bc', ''])
@@ -2235,8 +2215,6 @@ class ReTests(unittest.TestCase):
p = r'(?:a*?(xx)??z)*'
self.assertEqual(re.match(p, s).groups(), ('xx',))
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_ASSERT_NOT_mark_bug(self):
# Fixed in issue35859, reported in issue725149.
# JUMP_ASSERT_NOT should LASTMARK_SAVE()
@@ -2249,16 +2227,12 @@ class ReTests(unittest.TestCase):
self.assertEqual(m.span(3), (3, 4))
self.assertEqual(m.groups(), ('b', None, 'b'))
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bug_40736(self):
with self.assertRaisesRegex(TypeError, "got 'int'"):
re.search("x*", 5)
with self.assertRaisesRegex(TypeError, "got 'type'"):
re.search("x*", type)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_search_anchor_at_beginning(self):
s = 'x'*10**7
start = time.perf_counter()
@@ -2273,7 +2247,8 @@ class ReTests(unittest.TestCase):
# With optimization -- 0.0003 seconds.
self.assertLess(t, 0.1)
@unittest.skip('dead lock')
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_possessive_quantifiers(self):
"""Test Possessive Quantifiers
Test quantifiers of the form @+ for some repetition operator @,
@@ -2342,7 +2317,6 @@ class ReTests(unittest.TestCase):
self.assertTrue(re.fullmatch(r'(?:ab)?+c', 'abc'))
self.assertTrue(re.fullmatch(r'(?:ab){1,3}+c', 'abc'))
@unittest.skip("dead lock")
def test_findall_possessive_quantifiers(self):
self.assertEqual(re.findall(r'a++', 'aab'), ['aa'])
self.assertEqual(re.findall(r'a*+', 'aab'), ['aa', '', ''])
@@ -2354,8 +2328,6 @@ class ReTests(unittest.TestCase):
self.assertEqual(re.findall(r'(?:ab)?+', 'ababc'), ['ab', 'ab', '', ''])
self.assertEqual(re.findall(r'(?:ab){1,3}+', 'ababc'), ['abab'])
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_atomic_grouping(self):
"""Test Atomic Grouping
Test non-capturing groups of the form (?>...), which does
@@ -2399,8 +2371,6 @@ class ReTests(unittest.TestCase):
self.assertTrue(re.fullmatch(r'(?>(?:ab)?)c', 'abc'))
self.assertTrue(re.fullmatch(r'(?>(?:ab){1,3})c', 'abc'))
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_findall_atomic_grouping(self):
self.assertEqual(re.findall(r'(?>a+)', 'aab'), ['aa'])
self.assertEqual(re.findall(r'(?>a*)', 'aab'), ['aa', '', ''])
@@ -2412,6 +2382,8 @@ class ReTests(unittest.TestCase):
self.assertEqual(re.findall(r'(?>(?:ab)?)', 'ababc'), ['ab', 'ab', '', ''])
self.assertEqual(re.findall(r'(?>(?:ab){1,3})', 'ababc'), ['abab'])
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bug_gh91616(self):
self.assertTrue(re.fullmatch(r'(?s:(?>.*?\.).*)\Z', "a.txt")) # reproducer
self.assertTrue(re.fullmatch(r'(?s:(?=(?P<g0>.*?\.))(?P=g0).*)\Z', "a.txt"))
@@ -2442,8 +2414,6 @@ class ReTests(unittest.TestCase):
self.assertTrue(template_re1.match('ahoy'))
self.assertFalse(template_re1.match('nope'))
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bug_gh106052(self):
# gh-100061
self.assertEqual(re.match('(?>(?:.(?!D))+)', 'ABCDE').span(), (0, 2))
+1 -2
View File
@@ -80,9 +80,8 @@ timsort = "0.1.2"
# RustPython crates implementing functionality based on CPython
# sre-engine = "0.4.1"
# to work on sre-engine locally or git version
# sre-engine = { git = "https://github.com/RustPython/sre-engine", rev = "refs/pull/14/head" }
sre-engine = { git = "https://github.com/RustPython/sre-engine", rev = "refs/pull/17/head" }
# sre-engine = { git = "https://github.com/RustPython/sre-engine" }
sre-engine = { path = "../../sre-engine" }
## unicode stuff
unicode_names2 = { workspace = true }
+155 -60
View File
@@ -5,13 +5,13 @@ mod _sre {
use crate::{
atomic_func,
builtins::{
PyCallableIterator, PyDictRef, PyGenericAlias, PyInt, PyList, PyStr, PyStrRef, PyTuple,
PyTupleRef, PyTypeRef,
PyCallableIterator, PyDictRef, PyGenericAlias, PyInt, PyList, PyListRef, PyStr,
PyStrRef, PyTuple, PyTupleRef, PyTypeRef,
},
common::{ascii, hash::PyHash},
convert::ToPyObject,
function::{ArgCallable, OptionalArg, PosArgs, PyComparisonValue},
protocol::{PyBuffer, PyMappingMethods},
protocol::{PyBuffer, PyCallable, PyMappingMethods},
stdlib::sys,
types::{AsMapping, Comparable, Hashable, Representable},
Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject,
@@ -22,12 +22,12 @@ mod _sre {
use itertools::Itertools;
use num_traits::ToPrimitive;
use sre_engine::{
constants::SreFlag,
engine::{lower_ascii, lower_unicode, upper_unicode, Request, SearchIter, State, StrDrive},
string::{lower_ascii, lower_unicode, upper_unicode},
Request, SearchIter, SreFlag, State, StrDrive,
};
#[pyattr]
pub use sre_engine::{constants::SRE_MAGIC as MAGIC, CODESIZE, MAXGROUPS, MAXREPEAT};
pub use sre_engine::{CODESIZE, MAXGROUPS, MAXREPEAT, SRE_MAGIC as MAGIC};
#[pyfunction]
fn getcodesize() -> usize {
@@ -103,6 +103,58 @@ mod _sre {
})
}
#[pyattr]
#[pyclass(name = "SRE_Template")]
#[derive(Debug, PyPayload)]
struct Template {
literal: PyObjectRef,
items: Vec<(usize, PyObjectRef)>,
}
#[pyclass]
impl Template {
fn compile(
pattern: PyRef<Pattern>,
repl: PyObjectRef,
vm: &VirtualMachine,
) -> PyResult<PyRef<Self>> {
let re = vm.import("re", None, 0)?;
let func = re.get_attr("_compile_template", vm)?;
let result = func.call((pattern, repl.clone()), vm)?;
result
.downcast::<Self>()
.map_err(|_| vm.new_runtime_error("expected SRE_Template".to_owned()))
}
}
#[pyfunction]
fn template(
_pattern: PyObjectRef,
template: PyListRef,
vm: &VirtualMachine,
) -> PyResult<Template> {
let err = || vm.new_type_error("invalid template".to_owned());
let mut items = Vec::with_capacity(1);
let v = template.borrow_vec();
let literal = v.first().ok_or_else(err)?.clone();
let trunks = v[1..].chunks_exact(2);
if !trunks.remainder().is_empty() {
return Err(err());
}
for trunk in trunks {
let index: usize = trunk[0]
.payload::<PyInt>()
.ok_or_else(|| vm.new_type_error("expected usize".to_owned()))?
.try_to_primitive(vm)?;
items.push((index, trunk[1].clone()));
}
Ok(Template { literal, items })
}
#[derive(FromArgs)]
struct StringArgs {
string: PyObjectRef,
@@ -157,9 +209,9 @@ mod _sre {
where
F: FnOnce(&str) -> PyResult<R>,
{
let string = string
.payload::<PyStr>()
.ok_or_else(|| vm.new_type_error("expected string".to_owned()))?;
let string = string.payload::<PyStr>().ok_or_else(|| {
vm.new_type_error(format!("expected string got '{}'", string.class()))
})?;
f(string.as_str())
}
@@ -184,10 +236,9 @@ mod _sre {
with_sre_str!(zelf, &string.clone(), vm, |x| {
let req = x.create_request(&zelf, pos, endpos);
let mut state = State::default();
state.pymatch(req);
Ok(state
.has_matched
.then(|| Match::new(&state, zelf.clone(), string).into_ref(&vm.ctx)))
.pymatch(&req)
.then(|| Match::new(&mut state, zelf.clone(), string).into_ref(&vm.ctx)))
})
}
@@ -201,9 +252,8 @@ mod _sre {
let mut req = x.create_request(&zelf, string_args.pos, string_args.endpos);
req.match_all = true;
let mut state = State::default();
state.pymatch(req);
Ok(state.has_matched.then(|| {
Match::new(&state, zelf.clone(), string_args.string).into_ref(&vm.ctx)
Ok(state.pymatch(&req).then(|| {
Match::new(&mut state, zelf.clone(), string_args.string).into_ref(&vm.ctx)
}))
})
}
@@ -217,9 +267,8 @@ mod _sre {
with_sre_str!(zelf, &string_args.string.clone(), vm, |x| {
let req = x.create_request(&zelf, string_args.pos, string_args.endpos);
let mut state = State::default();
state.search(req);
Ok(state.has_matched.then(|| {
Match::new(&state, zelf.clone(), string_args.string).into_ref(&vm.ctx)
Ok(state.search(req).then(|| {
Match::new(&mut state, zelf.clone(), string_args.string).into_ref(&vm.ctx)
}))
})
}
@@ -237,7 +286,7 @@ mod _sre {
let mut iter = SearchIter { req, state };
while iter.next().is_some() {
let m = Match::new(&iter.state, zelf.clone(), string_args.string.clone());
let m = Match::new(&mut iter.state, zelf.clone(), string_args.string.clone());
let item = if zelf.groups == 0 || zelf.groups == 1 {
m.get_slice(zelf.groups, s, vm)
@@ -318,7 +367,7 @@ mod _sre {
/* get segment before this match */
splitlist.push(s.slice(last, iter.state.start, vm));
let m = Match::new(&iter.state, zelf.clone(), split_args.string.clone());
let m = Match::new(&mut iter.state, zelf.clone(), split_args.string.clone());
// add groups (if any)
for i in 1..=zelf.groups {
@@ -326,7 +375,7 @@ mod _sre {
}
n += 1;
last = iter.state.string_position;
last = iter.state.cursor.position;
}
// get segment following last match (even if empty)
@@ -365,23 +414,48 @@ mod _sre {
count,
} = sub_args;
let (is_callable, filter) = if repl.is_callable() {
(true, repl)
enum FilterType<'a> {
Literal(PyObjectRef),
Callable(PyCallable<'a>),
Template(PyRef<Template>),
}
let filter = if let Some(callable) = repl.to_callable() {
FilterType::Callable(callable)
} else {
let is_template = if zelf.isbytes {
Self::with_bytes(&repl, vm, |x| Ok(x.contains(&b'\\')))
Self::with_bytes(&repl, vm, |x| Ok(x.contains(&b'\\')))?
} else {
Self::with_str(&repl, vm, |x| Ok(x.contains('\\')))
}?;
Self::with_str(&repl, vm, |x| Ok(x.contains('\\')))?
};
if is_template {
let re = vm.import("re", None, 0)?;
let func = re.get_attr("_subx", vm)?;
let filter = func.call((zelf.clone(), repl), vm)?;
(filter.is_callable(), filter)
FilterType::Template(Template::compile(zelf.clone(), repl, vm)?)
} else {
(false, repl)
FilterType::Literal(repl)
}
};
// };
// Self::with_bytes(&repl, vm, |x| Ok(x.contains(&b'\\')))?
// || Self::with_str(&repl, vm, |x| Ok(x.contains('\\')))?
// {
// FilterType::Template(Template::compile(zelf.clone(), repl, vm)?)
// } else {
// FilterType::Literal(repl)
// };
// let (is_callable, filter) = if repl.is_callable() {
// (true, repl)
// } else {
// if is_template {
// let re = vm.import("re", None, 0)?;
// let func = re.get_attr("_compile_template", vm)?;
// let filter = func.call((zelf.clone(), repl), vm)?;
// (filter.is_callable(), filter)
// } else {
// (false, repl)
// }
// };
with_sre_str!(zelf, &string, vm, |s| {
let req = s.create_request(&zelf, 0, usize::MAX);
@@ -397,15 +471,36 @@ mod _sre {
sublist.push(s.slice(last_pos, iter.state.start, vm));
}
if is_callable {
let m = Match::new(&iter.state, zelf.clone(), string.clone());
let ret = filter.call((m.into_ref(&vm.ctx),), vm)?;
sublist.push(ret);
} else {
sublist.push(filter.clone());
}
match &filter {
FilterType::Literal(literal) => sublist.push(literal.clone()),
FilterType::Callable(callable) => {
let m = Match::new(&mut iter.state, zelf.clone(), string.clone())
.into_ref(&vm.ctx);
sublist.push(callable.invoke((m,), vm)?);
}
FilterType::Template(template) => {
let m = Match::new(&mut iter.state, zelf.clone(), string.clone());
// template.expand(m)?
// let mut list = vec![template.literal.clone()];
sublist.push(template.literal.clone());
for (index, literal) in template.items.iter().cloned() {
if let Some(item) = m.get_slice(index, s, vm) {
sublist.push(item);
}
sublist.push(literal);
}
}
};
last_pos = iter.state.string_position;
// if is_callable {
// let m = Match::new(&iter.state, zelf.clone(), string.clone());
// let ret = filter.call((m.into_ref(&vm.ctx),), vm)?;
// sublist.push(ret);
// } else {
// sublist.push(filter.clone());
// }
last_pos = iter.state.cursor.position;
n += 1;
}
@@ -528,17 +623,15 @@ mod _sre {
#[pyclass(with(AsMapping, Representable))]
impl Match {
pub(crate) fn new<S: StrDrive>(
state: &State<S>,
pattern: PyRef<Pattern>,
string: PyObjectRef,
) -> Self {
let mut regs = vec![(state.start as isize, state.string_position as isize)];
pub(crate) fn new(state: &mut State, pattern: PyRef<Pattern>, string: PyObjectRef) -> Self {
let string_position = state.cursor.position;
let marks = &state.marks;
let mut regs = vec![(state.start as isize, string_position as isize)];
for group in 0..pattern.groups {
let mark_index = 2 * group;
if mark_index + 1 < state.marks.len() {
let start = state.marks[mark_index];
let end = state.marks[mark_index + 1];
if mark_index + 1 < marks.raw().len() {
let start = marks.raw()[mark_index];
let end = marks.raw()[mark_index + 1];
if start.is_some() && end.is_some() {
regs.push((start.unpack() as isize, end.unpack() as isize));
continue;
@@ -550,8 +643,8 @@ mod _sre {
string,
pattern,
pos: state.start,
endpos: state.string_position,
lastindex: state.marks.last_index(),
endpos: string_position,
lastindex: marks.last_index(),
regs,
}
}
@@ -796,14 +889,15 @@ mod _sre {
let mut req = s.create_request(&self.pattern, self.start.load(), self.end);
let mut state = State::default();
req.must_advance = self.must_advance.load();
state.pymatch(req);
let has_matched = state.pymatch(&req);
self.must_advance
.store(state.string_position == state.start);
self.start.store(state.string_position);
.store(state.cursor.position == state.start);
self.start.store(state.cursor.position);
Ok(state.has_matched.then(|| {
Match::new(&state, self.pattern.clone(), self.string.clone()).into_ref(&vm.ctx)
Ok(has_matched.then(|| {
Match::new(&mut state, self.pattern.clone(), self.string.clone())
.into_ref(&vm.ctx)
}))
})
}
@@ -818,14 +912,15 @@ mod _sre {
let mut state = State::default();
req.must_advance = self.must_advance.load();
state.search(req);
let has_matched = state.search(req);
self.must_advance
.store(state.string_position == state.start);
self.start.store(state.string_position);
.store(state.cursor.position == state.start);
self.start.store(state.cursor.position);
Ok(state.has_matched.then(|| {
Match::new(&state, self.pattern.clone(), self.string.clone()).into_ref(&vm.ctx)
Ok(has_matched.then(|| {
Match::new(&mut state, self.pattern.clone(), self.string.clone())
.into_ref(&vm.ctx)
}))
})
}