Add recursion guard to the c-api (#8796)

* Add recursion guard to the c-api

* Do not change with_recursion as this has side effects

crates/capi/src/ceval.rs

@@ -1,7 +1,7 @@
 use crate::pyframe::PyFrameObject;
 use crate::pystate::with_vm;
 use crate::unicodeobject::decode_fsdefault_and_size;
-use crate::util::FfiPtrExt;
+use crate::util::{CStrExt, FfiPtrExt};
 use core::ffi::{CStr, c_char, c_int};
 use rustpython_vm::builtins::{PyCode, PyDict};
 use rustpython_vm::function::ArgMapping;
@@ -157,10 +157,21 @@ pub unsafe extern "C" fn PyEval_GetFuncDesc(func: *mut PyObject) -> *const c_cha
     })
 }
 
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn Py_EnterRecursiveCall(where_: *const c_char) -> c_int {
+    with_vm(|vm| vm.enter_recursive_call(unsafe { where_.try_as_str(vm)? }))
+}
+
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn Py_LeaveRecursiveCall() {
+    with_vm(|vm| vm.leave_recursive_call())
+}
+
 #[cfg(test)]
 mod tests {
+    use super::{Py_EnterRecursiveCall, Py_LeaveRecursiveCall};
     use alloc::ffi::CString;
-    use pyo3::exceptions::PyException;
+    use pyo3::exceptions::{PyException, PyRecursionError};
     use pyo3::prelude::*;
 
     #[pyfunction]
@@ -268,4 +279,23 @@ assert not hidden_leaked
             .unwrap();
         })
     }
+
+    #[test]
+    fn recursion_error() {
+        fn recurse(py: Python<'_>) -> PyResult<()> {
+            unsafe {
+                if Py_EnterRecursiveCall(c"".as_ptr().cast()) != 0 {
+                    return Err(PyErr::fetch(py));
+                };
+                let result = recurse(py);
+                Py_LeaveRecursiveCall();
+                result
+            }
+        }
+
+        Python::attach(|py| {
+            let err = recurse(py).unwrap_err();
+            assert!(err.is_instance_of::<PyRecursionError>(py));
+        })
+    }
 }

crates/vm/src/vm/mod.rs

@@ -2585,6 +2585,33 @@ impl VirtualMachine {
         false
     }
 
+    /// Enter a native-recursion section equivalent to `Py_EnterRecursiveCall`.
+    pub fn enter_recursive_call(&self, _where: &str) -> PyResult<()> {
+        #[cfg(any(miri, target_env = "musl"))]
+        let counted_too_deep =
+            self.native_recursion_depth.get() >= Self::NATIVE_RECURSION_LIMIT_UNMEASURED;
+        #[cfg(not(any(miri, target_env = "musl")))]
+        let counted_too_deep = false;
+
+        if counted_too_deep || self.check_c_stack_overflow() {
+            return Err(
+                self.new_recursion_error(format!("maximum recursion depth exceeded {_where}"))
+            );
+        }
+
+        #[cfg(any(miri, target_env = "musl"))]
+        self.native_recursion_depth.update(|d| d + 1);
+
+        Ok(())
+    }
+
+    /// Leave a native-recursion section equivalent to
+    /// `Py_LeaveRecursiveCall`.
+    pub fn leave_recursive_call(&self) {
+        #[cfg(any(miri, target_env = "musl"))]
+        self.native_recursion_depth.update(|d| d.saturating_sub(1));
+    }
+
     /// Used to run the body of a (possibly) recursive function. It will raise a
     /// RecursionError if recursive functions are nested far too many times,
     /// preventing a stack overflow.