Extract the sre interpreter into a separate repo

Cargo.lock

@@ -1278,9 +1278,9 @@ dependencies = [
 
 [[package]]
 name = "num_enum"
-version = "0.4.3"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca565a7df06f3d4b485494f25ba05da1435950f4dc263440eda7a6fa9b8e36e4"
+checksum = "226b45a5c2ac4dd696ed30fa6b94b057ad909c7b7fc2e0d0808192bced894066"
 dependencies = [
  "derivative",
  "num_enum_derive",
@@ -1288,9 +1288,9 @@ dependencies = [
 
 [[package]]
 name = "num_enum_derive"
-version = "0.4.3"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffa5a33ddddfee04c0283a7653987d634e880347e96b5b2ed64de07efb59db9d"
+checksum = "1c0fd9eba1d5db0994a239e09c1be402d35622277e35468ba891aa5e3188ce7e"
 dependencies = [
  "proc-macro-crate",
  "proc-macro2",
@@ -2020,6 +2020,7 @@ dependencies = [
  "sha2",
  "sha3",
  "socket2",
+ "sre-engine",
  "static_assertions",
  "thiserror",
  "thread_local",
@@ -2246,6 +2247,14 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "sre-engine"
+version = "0.1.0"
+dependencies = [
+ "bitflags",
+ "num_enum",
+]
+
 [[package]]
 name = "static_assertions"
 version = "1.1.0"

vm/Cargo.toml

@@ -40,7 +40,6 @@ num-iter = "0.1.39"
 rand = "0.8"
 rand_core = "0.6"
 getrandom = { version = "0.2", features = ["js"] }
-mt19937 = "2.0"
 log = "0.4"
 rustpython-derive = { path = "../derive", version = "0.1.2" }
 rustpython-ast = { path = "../ast", optional = true, version = "0.1" }
@@ -73,7 +72,7 @@ base64 = "0.13"
 is-macro = "0.1"
 result-like = "^0.3"
 foreign-types-shared = "0.1"
-num_enum = "0.4"
+num_enum = "0.5"
 bstr = "0.2.12"
 crossbeam-utils = "0.8"
 parking_lot = "0.11"
@@ -85,6 +84,12 @@ atty = "0.2"
 static_assertions = "1.1"
 half = "1.6"
 
+# RustPython crates implementing functionality based on CPython
+mt19937 = "2.0"
+sre-engine = { git = "https://github.com/RustPython/sre-engine" }
+# to work on sre-engine locally
+# sre-engine = { path = "../../sre-engine" }
+
 ## unicode stuff
 unicode_names2 = "0.4"
 # TODO: use unic for this; needed for title case:

vm/src/stdlib/sre.rs

@@ -1,6 +1,3 @@
-mod constants;
-mod interp;
-
 pub(crate) use _sre::make_module;
 
 #[pymodule]
@@ -10,8 +7,6 @@ mod _sre {
     use num_traits::ToPrimitive;
     use rustpython_common::borrow::BorrowValue;
 
-    use super::constants::SreFlag;
-    use super::interp::{lower_ascii, lower_unicode, upper_unicode, State, StrDrive};
     use crate::builtins::list::PyListRef;
     use crate::builtins::memory::try_buffer_from_object;
     use crate::builtins::tuple::PyTupleRef;
@@ -25,23 +20,11 @@ mod _sre {
     };
     use crate::VirtualMachine;
     use core::str;
+    use sre_engine::constants::SreFlag;
+    use sre_engine::engine::{lower_ascii, lower_unicode, upper_unicode, State, StrDrive};
 
     #[pyattr]
-    pub const CODESIZE: usize = 4;
-    #[pyattr]
-    pub use super::constants::SRE_MAGIC as MAGIC;
-    #[cfg(target_pointer_width = "32")]
-    #[pyattr]
-    pub const MAXREPEAT: usize = usize::MAX;
-    #[cfg(target_pointer_width = "64")]
-    #[pyattr]
-    pub const MAXREPEAT: usize = u32::MAX as usize;
-    #[cfg(target_pointer_width = "32")]
-    #[pyattr]
-    pub const MAXGROUPS: usize = MAXREPEAT / 4 / 2;
-    #[cfg(target_pointer_width = "64")]
-    #[pyattr]
-    pub const MAXGROUPS: usize = MAXREPEAT / 2;
+    pub use sre_engine::{constants::SRE_MAGIC as MAGIC, CODESIZE, MAXGROUPS, MAXREPEAT};
 
     #[pyfunction]
     fn getcodesize() -> usize {
@@ -65,6 +48,22 @@ mod _sre {
         lower_unicode(ch as u32) as i32
     }
 
+    fn slice_drive(
+        this: &StrDrive<'_>,
+        start: usize,
+        end: usize,
+        vm: &VirtualMachine,
+    ) -> PyObjectRef {
+        match this {
+            StrDrive::Str(s) => vm
+                .ctx
+                .new_str(s.chars().take(end).skip(start).collect::<String>()),
+            StrDrive::Bytes(b) => vm
+                .ctx
+                .new_bytes(b.iter().take(end).skip(start).cloned().collect()),
+        }
+    }
+
     #[pyfunction]
     fn compile(
         pattern: PyObjectRef,
@@ -376,7 +375,7 @@ mod _sre {
                         }
 
                         /* get segment before this match */
-                        splitlist.push(state.string.slice_to_pyobject(last, state.start, vm));
+                        splitlist.push(slice_drive(&state.string, last, state.start, vm));
 
                         let m = Match::new(&state, zelf.clone(), split_args.string.clone());
 
@@ -399,11 +398,7 @@ mod _sre {
                     }
 
                     // get segment following last match (even if empty)
-                    splitlist.push(
-                        state
-                            .string
-                            .slice_to_pyobject(last, state.string.count(), vm),
-                    );
+                    splitlist.push(slice_drive(&state.string, last, state.string.count(), vm));
 
                     Ok(PyList::from(splitlist).into_ref(vm))
                 },
@@ -472,7 +467,7 @@ mod _sre {
 
                     if last_pos < state.start {
                         /* get segment before this match */
-                        sublist.push(state.string.slice_to_pyobject(last_pos, state.start, vm));
+                        sublist.push(slice_drive(&state.string, last_pos, state.start, vm));
                     }
 
                     if !(last_pos == state.start && last_pos == state.string_position && n > 1) {
@@ -497,7 +492,7 @@ mod _sre {
                 }
 
                 /* get segment following last match */
-                sublist.push(state.string.slice_to_pyobject(last_pos, state.end, vm));
+                sublist.push(slice_drive(&state.string, last_pos, state.end, vm));
 
                 let list = PyList::from(sublist).into_object(vm);
 
@@ -758,7 +753,7 @@ mod _sre {
             if start < 0 || end < 0 {
                 return None;
             }
-            Some(str_drive.slice_to_pyobject(start as usize, end as usize, vm))
+            Some(slice_drive(&str_drive, start as usize, end as usize, vm))
         }
     }
 

vm/src/stdlib/sre/constants.rs

@@ -1,122 +0,0 @@
-/*
- * Secret Labs' Regular Expression Engine
- *
- * regular expression matching engine
- *
- * NOTE: This file is generated by sre_constants.py.  If you need
- * to change anything in here, edit sre_constants.py and run it.
- *
- * Copyright (c) 1997-2001 by Secret Labs AB.  All rights reserved.
- *
- * See the _sre.c file for information on usage and redistribution.
- */
-
-use bitflags::bitflags;
-
-pub const SRE_MAGIC: usize = 20171005;
-#[derive(num_enum::TryFromPrimitive, Debug)]
-#[repr(u32)]
-#[allow(non_camel_case_types)]
-pub enum SreOpcode {
-    FAILURE = 0,
-    SUCCESS = 1,
-    ANY = 2,
-    ANY_ALL = 3,
-    ASSERT = 4,
-    ASSERT_NOT = 5,
-    AT = 6,
-    BRANCH = 7,
-    CALL = 8,
-    CATEGORY = 9,
-    CHARSET = 10,
-    BIGCHARSET = 11,
-    GROUPREF = 12,
-    GROUPREF_EXISTS = 13,
-    IN = 14,
-    INFO = 15,
-    JUMP = 16,
-    LITERAL = 17,
-    MARK = 18,
-    MAX_UNTIL = 19,
-    MIN_UNTIL = 20,
-    NOT_LITERAL = 21,
-    NEGATE = 22,
-    RANGE = 23,
-    REPEAT = 24,
-    REPEAT_ONE = 25,
-    SUBPATTERN = 26,
-    MIN_REPEAT_ONE = 27,
-    GROUPREF_IGNORE = 28,
-    IN_IGNORE = 29,
-    LITERAL_IGNORE = 30,
-    NOT_LITERAL_IGNORE = 31,
-    GROUPREF_LOC_IGNORE = 32,
-    IN_LOC_IGNORE = 33,
-    LITERAL_LOC_IGNORE = 34,
-    NOT_LITERAL_LOC_IGNORE = 35,
-    GROUPREF_UNI_IGNORE = 36,
-    IN_UNI_IGNORE = 37,
-    LITERAL_UNI_IGNORE = 38,
-    NOT_LITERAL_UNI_IGNORE = 39,
-    RANGE_UNI_IGNORE = 40,
-}
-#[derive(num_enum::TryFromPrimitive, Debug)]
-#[repr(u32)]
-#[allow(non_camel_case_types)]
-pub enum SreAtCode {
-    BEGINNING = 0,
-    BEGINNING_LINE = 1,
-    BEGINNING_STRING = 2,
-    BOUNDARY = 3,
-    NON_BOUNDARY = 4,
-    END = 5,
-    END_LINE = 6,
-    END_STRING = 7,
-    LOC_BOUNDARY = 8,
-    LOC_NON_BOUNDARY = 9,
-    UNI_BOUNDARY = 10,
-    UNI_NON_BOUNDARY = 11,
-}
-#[derive(num_enum::TryFromPrimitive, Debug)]
-#[repr(u32)]
-#[allow(non_camel_case_types)]
-pub enum SreCatCode {
-    DIGIT = 0,
-    NOT_DIGIT = 1,
-    SPACE = 2,
-    NOT_SPACE = 3,
-    WORD = 4,
-    NOT_WORD = 5,
-    LINEBREAK = 6,
-    NOT_LINEBREAK = 7,
-    LOC_WORD = 8,
-    LOC_NOT_WORD = 9,
-    UNI_DIGIT = 10,
-    UNI_NOT_DIGIT = 11,
-    UNI_SPACE = 12,
-    UNI_NOT_SPACE = 13,
-    UNI_WORD = 14,
-    UNI_NOT_WORD = 15,
-    UNI_LINEBREAK = 16,
-    UNI_NOT_LINEBREAK = 17,
-}
-bitflags! {
-    pub struct SreFlag: u16 {
-        const TEMPLATE = 1;
-        const IGNORECASE = 2;
-        const LOCALE = 4;
-        const MULTILINE = 8;
-        const DOTALL = 16;
-        const UNICODE = 32;
-        const VERBOSE = 64;
-        const DEBUG = 128;
-        const ASCII = 256;
-    }
-}
-bitflags! {
-    pub struct SreInfo: u32 {
-        const PREFIX = 1;
-        const LITERAL = 2;
-        const CHARSET = 4;
-    }
-}