Add minimal capi lifecycle support (#7648)

* Add minimal capi lifecycle support

* Force enable `threading` on `stdlib`

---------

Co-authored-by: Jeong, YunWon <jeong@youknowone.org>

.cspell.dict/cpython.txt

@@ -161,6 +161,7 @@ Pyfunc
 pylifecycle
 pymain
 pyrepl
+pystate
 PYTHONTRACEMALLOC
 PYTHONUTF8
 pythonw

.cspell.dict/python-more.txt

@@ -189,6 +189,7 @@ pycodecs
 pycs
 pydatetime
 pyexpat
+PYGILSTATE
 pyio
 pymain
 PYTHONAPI

Cargo.lock

@@ -3142,6 +3142,7 @@ dependencies = [
  "libc",
  "log",
  "pyo3",
+ "rustpython-capi",
  "rustpython-compiler",
  "rustpython-pylib",
  "rustpython-ruff_python_parser",
@@ -3151,6 +3152,15 @@ dependencies = [
  "winresource",
 ]
 
+[[package]]
+name = "rustpython-capi"
+version = "0.5.0"
+dependencies = [
+ "pyo3",
+ "rustpython-stdlib",
+ "rustpython-vm",
+]
+
 [[package]]
 name = "rustpython-codegen"
 version = "0.5.0"
@@ -3551,7 +3561,6 @@ dependencies = [
  "rustpython-ruff_text_size",
  "rustpython-sre_engine",
  "rustyline",
- "scoped-tls",
  "scopeguard",
  "serde_core",
  "static_assertions",
@@ -3663,12 +3672,6 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
-[[package]]
-name = "scoped-tls"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
-
 [[package]]
 name = "scopeguard"
 version = "1.2.0"

Cargo.toml

@@ -10,6 +10,7 @@ repository.workspace = true
 license.workspace = true
 
 [features]
+capi = ["dep:rustpython-capi", "threading"]
 default = ["threading", "stdlib", "stdio", "importlib", "ssl-rustls", "host_env"]
 host_env = ["rustpython-vm/host_env", "rustpython-stdlib?/host_env"]
 importlib = ["rustpython-vm/importlib"]
@@ -31,6 +32,7 @@ tkinter = ["rustpython-stdlib/tkinter"]
 winresource = "0.1"
 
 [dependencies]
+rustpython-capi = { workspace = true, optional = true }
 rustpython-compiler = { workspace = true }
 rustpython-pylib = { workspace = true, optional = true }
 rustpython-stdlib = { workspace = true, optional = true, features = ["compiler"] }
@@ -140,6 +142,7 @@ repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
 [workspace.dependencies]
+rustpython-capi = { path = "crates/capi", version = "0.5.0" }
 rustpython-compiler-core = { path = "crates/compiler-core", version = "0.5.0" }
 rustpython-compiler = { path = "crates/compiler", version = "0.5.0" }
 rustpython-codegen = { path = "crates/codegen", version = "0.5.0" }
@@ -257,7 +260,6 @@ rustls-platform-verifier = "0.7"
 rustyline = "18"
 serde = { package = "serde_core", version = "1.0.225", default-features = false, features = ["alloc"] }
 schannel = "0.1.29"
-scoped-tls = "1"
 scopeguard = "1"
 sha-1 = "0.10.0"
 sha2 = "0.10.2"

build.rs

@@ -1,17 +1,29 @@
 fn main() {
-    if std::env::var("CARGO_CFG_TARGET_OS").unwrap() == "windows" {
-        println!("cargo:rerun-if-changed=logo.ico");
-        let mut res = winresource::WindowsResource::new();
-        if std::path::Path::new("logo.ico").exists() {
-            res.set_icon("logo.ico");
-        } else {
-            println!("cargo:warning=logo.ico not found, skipping icon embedding");
-            return;
+    let target = std::env::var("CARGO_CFG_TARGET_OS").unwrap();
+    let capi_enabled = std::env::var_os("CARGO_FEATURE_CAPI").is_some();
+
+    match target.as_str() {
+        "linux" if capi_enabled => {
+            println!("cargo:rustc-link-arg-bin=rustpython=-Wl,--export-dynamic");
         }
-        res.compile()
-            .map_err(|e| {
-                println!("cargo:warning=Failed to compile Windows resources: {e}");
-            })
-            .ok();
+        "macos" if capi_enabled => {
+            println!("cargo:rustc-link-arg-bin=rustpython=-Wl,-export_dynamic");
+        }
+        "windows" => {
+            println!("cargo:rerun-if-changed=logo.ico");
+            let mut res = winresource::WindowsResource::new();
+            if std::path::Path::new("logo.ico").exists() {
+                res.set_icon("logo.ico");
+            } else {
+                println!("cargo:warning=logo.ico not found, skipping icon embedding");
+                return;
+            }
+            res.compile()
+                .map_err(|e| {
+                    println!("cargo:warning=Failed to compile Windows resources: {e}");
+                })
+                .ok();
+        }
+        _ => {}
     }
 }

crates/capi/.cargo/config.toml

@@ -0,0 +1,3 @@
+[env]
+PYO3_CONFIG_FILE = { value = "pyo3-rustpython.config", relative = true }
+PYO3_NO_PYTHON = { value = "1" }

crates/capi/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "rustpython-capi"
+description = "Minimal CPython C-API compatibility exports for RustPython"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+repository.workspace = true
+license.workspace = true
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+rustpython-vm = { workspace = true, features = ["threading"] }
+rustpython-stdlib = {workspace = true, features = ["threading"] }
+
+[dev-dependencies]
+pyo3 = { version = "0.28", features = ["auto-initialize", "abi3"] }
+
+[lints]
+workspace = true
+
+[package.metadata.cargo-shear]
+# Not a direct dependency (yet), but we need to enable threading support in the stdlib.
+ignored = ["rustpython-stdlib"]

crates/capi/pyo3-rustpython.config

@@ -0,0 +1,5 @@
+implementation=CPython
+version=3.14
+shared=true
+abi3=true
+suppress_build_script_link_lines=true

crates/capi/src/lib.rs

@@ -0,0 +1,19 @@
+#![allow(clippy::missing_safety_doc)]
+
+use crate::pylifecycle::MAIN_INTERP;
+use rustpython_vm::Interpreter;
+pub use rustpython_vm::PyObject;
+use std::sync::MutexGuard;
+
+extern crate alloc;
+
+pub mod pylifecycle;
+pub mod pystate;
+pub mod refcount;
+
+/// Get main interpreter of this process. Will be None if it has not been initialized yet.
+pub fn get_main_interpreter() -> MutexGuard<'static, Option<Interpreter>> {
+    MAIN_INTERP
+        .lock()
+        .expect("Failed to lock interpreter mutex")
+}

crates/capi/src/pylifecycle.rs

@@ -0,0 +1,51 @@
+use crate::get_main_interpreter;
+use crate::pystate::ensure_thread_has_vm_attached;
+use core::ffi::c_int;
+use rustpython_vm::Interpreter;
+use rustpython_vm::vm::thread::ThreadedVirtualMachine;
+use std::sync::Mutex;
+
+pub(crate) static MAIN_INTERP: Mutex<Option<Interpreter>> = Mutex::new(None);
+
+/// Request a thread local vm from the main interpreter
+pub(crate) fn request_vm_from_interpreter() -> ThreadedVirtualMachine {
+    get_main_interpreter()
+        .as_ref()
+        .expect("Interpreter not initialized")
+        .enter(|vm| vm.new_thread())
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_IsInitialized() -> c_int {
+    get_main_interpreter().is_some() as c_int
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_Initialize() {
+    Py_InitializeEx(0);
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_InitializeEx(_initsigs: c_int) {
+    let mut interp = get_main_interpreter();
+    if interp.is_none() {
+        *interp = Interpreter::with_init(Default::default(), |_vm| {}).into();
+        drop(interp);
+        ensure_thread_has_vm_attached();
+    }
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_Finalize() {
+    let _ = Py_FinalizeEx();
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_FinalizeEx() -> c_int {
+    0
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn Py_IsFinalizing() -> c_int {
+    0
+}

crates/capi/src/pystate.rs

@@ -0,0 +1,111 @@
+use crate::pylifecycle::request_vm_from_interpreter;
+use core::ffi::c_int;
+use core::ptr;
+use rustpython_vm::vm::thread::{
+    CurrentVmAttachState, attach_current_thread, release_current_thread,
+};
+
+#[allow(non_camel_case_types)]
+type PyGILState_STATE = c_int;
+const PYGILSTATE_LOCKED: PyGILState_STATE = 0;
+const PYGILSTATE_UNLOCKED: PyGILState_STATE = 1;
+
+#[repr(C)]
+pub struct PyThreadState {
+    _interp: *mut core::ffi::c_void,
+}
+
+/// Make sure this thread has a running vm attached. This only creates a new vm if we don't already
+/// have one. So this will only create a new vm when we are in a new thread created outside RustPython.
+pub(crate) fn ensure_thread_has_vm_attached() -> CurrentVmAttachState {
+    attach_current_thread(request_vm_from_interpreter)
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn PyGILState_Ensure() -> PyGILState_STATE {
+    match ensure_thread_has_vm_attached() {
+        CurrentVmAttachState::AlreadyAttached => PYGILSTATE_LOCKED,
+        CurrentVmAttachState::Attached => PYGILSTATE_UNLOCKED,
+    }
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn PyGILState_Release(state: PyGILState_STATE) {
+    if state == PYGILSTATE_UNLOCKED {
+        release_current_thread(CurrentVmAttachState::Attached);
+    }
+}
+
+#[unsafe(no_mangle)]
+pub extern "C" fn PyEval_SaveThread() -> *mut PyThreadState {
+    ptr::null_mut()
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::get_main_interpreter;
+    use crate::pystate::{PyGILState_Ensure, PyGILState_Release};
+    use pyo3::prelude::*;
+    use rustpython_vm::vm::thread::{current_vm_is_set, with_current_vm};
+
+    #[test]
+    fn test_new_thread() {
+        Python::attach(|_py| {
+            with_current_vm(|_vm| {
+                assert!(
+                    current_vm_is_set(),
+                    "This thread did not have a vm attached"
+                )
+            });
+
+            std::thread::spawn(move || {
+                Python::attach(|_py| {
+                    with_current_vm(|_vm| {
+                        assert!(
+                            current_vm_is_set(),
+                            "This thread did not have a vm attached"
+                        )
+                    });
+                });
+            })
+            .join()
+            .unwrap();
+        })
+    }
+
+    #[test]
+    fn test_current_vm_main_thread() {
+        Python::initialize();
+
+        // let RustPython create a vm for this thread.
+        let vm = get_main_interpreter()
+            .as_ref()
+            .unwrap()
+            .enter(|vm| vm.new_thread());
+
+        // Attach the vm using RustPython
+        vm.run(|_vm| {
+            assert!(current_vm_is_set(), "This thread should have a vm attached");
+
+            Python::attach(|_py| {
+                with_current_vm(|_vm| {
+                    assert!(current_vm_is_set());
+                })
+            })
+        });
+    }
+
+    #[test]
+    fn test_gilstate_release_detaches_external_thread() {
+        Python::initialize();
+
+        std::thread::spawn(|| {
+            let state = PyGILState_Ensure();
+            assert!(current_vm_is_set());
+            PyGILState_Release(state);
+            assert!(!current_vm_is_set());
+        })
+        .join()
+        .unwrap();
+    }
+}

crates/capi/src/refcount.rs

@@ -0,0 +1,15 @@
+use crate::PyObject;
+use core::ptr::NonNull;
+use rustpython_vm::PyObjectRef;
+
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn _Py_DecRef(op: *mut PyObject) {
+    // By dropping PyObjectRef, we will decrement the reference count.
+    unsafe { drop(PyObjectRef::from_raw(NonNull::new_unchecked(op))) };
+}
+
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn _Py_IncRef(op: *mut PyObject) {
+    // Don't drop the owned value, as we just want to increment the refcount.
+    core::mem::forget(unsafe { (*op).to_owned() });
+}

crates/vm/Cargo.toml

@@ -65,7 +65,6 @@ num-traits = { workspace = true }
 num_enum = { workspace = true }
 parking_lot = { workspace = true }
 paste = { workspace = true }
-scoped-tls = { workspace = true }
 scopeguard = { workspace = true }
 serde = { workspace = true, optional = true }
 static_assertions = { workspace = true }

crates/vm/src/codecs.rs

@@ -158,7 +158,7 @@ impl CodecsRegistry {
     /// # Safety
     /// Must only be called after fork() in the child process when no other
     /// threads exist.
-    #[cfg(all(unix, feature = "threading"))]
+    #[cfg(all(unix, feature = "threading", feature = "host_env"))]
     pub(crate) unsafe fn reinit_after_fork(&self) {
         unsafe { crate::common::lock::reinit_rwlock_after_fork(&self.inner) };
     }

crates/vm/src/intern.rs

@@ -36,7 +36,7 @@ impl StringPool {
     /// # Safety
     /// Must only be called after fork() in the child process when no other
     /// threads exist.
-    #[cfg(all(unix, feature = "threading"))]
+    #[cfg(all(unix, feature = "threading", feature = "host_env"))]
     pub(crate) unsafe fn reinit_after_fork(&self) {
         unsafe { crate::common::lock::reinit_rwlock_after_fork(&self.inner) };
     }

crates/vm/src/object/core.rs

@@ -474,7 +474,7 @@ mod weakref_lock {
 
     /// Reset all weakref stripe locks after fork in child process.
     /// Locks held by parent threads would cause infinite spin in the child.
-    #[cfg(unix)]
+    #[cfg(all(unix, feature = "host_env"))]
     pub(crate) fn reset_all_after_fork() {
         for lock in &LOCKS {
             lock.store(0, Ordering::Release);
@@ -497,7 +497,7 @@ mod weakref_lock {
 
 /// Reset weakref stripe locks after fork. Must be called before any
 /// Python code runs in the child process.
-#[cfg(all(unix, feature = "threading"))]
+#[cfg(all(unix, feature = "threading", feature = "host_env"))]
 pub(crate) fn reset_weakref_locks_after_fork() {
     weakref_lock::reset_all_after_fork();
 }

crates/vm/src/stdlib/_imp.rs

@@ -38,6 +38,7 @@ mod lock {
         IMP_LOCK.lock();
     }
 
+    #[cfg(all(unix, feature = "host_env"))]
     pub(super) fn release_lock_after_fork_parent() {
         if IMP_LOCK.is_locked() && IMP_LOCK.is_owned_by_current_thread() {
             unsafe { IMP_LOCK.unlock() };
@@ -53,7 +54,7 @@ mod lock {
     /// # Safety
     ///
     /// Must only be called from single-threaded child after fork().
-    #[cfg(unix)]
+    #[cfg(all(unix, feature = "host_env"))]
     pub(crate) unsafe fn reinit_after_fork() {
         if IMP_LOCK.is_locked() && !IMP_LOCK.is_owned_by_current_thread() {
             // Held by a dead thread — reset to unlocked.
@@ -65,7 +66,7 @@ mod lock {
     /// behavior in the post-fork child:
     /// 1) if ownership metadata is stale (dead owner / changed tid), reset;
     /// 2) if current thread owns the lock, release it.
-    #[cfg(unix)]
+    #[cfg(all(unix, feature = "host_env"))]
     pub(super) unsafe fn after_fork_child_reinit_and_release() {
         unsafe { reinit_after_fork() };
         if IMP_LOCK.is_locked() && IMP_LOCK.is_owned_by_current_thread() {
@@ -75,22 +76,22 @@ mod lock {
 }
 
 /// Re-export for fork safety code in posix.rs
-#[cfg(feature = "threading")]
+#[cfg(all(unix, feature = "threading", feature = "host_env"))]
 pub(crate) fn acquire_imp_lock_for_fork() {
     lock::acquire_lock_for_fork();
 }
 
-#[cfg(feature = "threading")]
+#[cfg(all(unix, feature = "threading", feature = "host_env"))]
 pub(crate) fn release_imp_lock_after_fork_parent() {
     lock::release_lock_after_fork_parent();
 }
 
-#[cfg(all(unix, feature = "threading"))]
+#[cfg(all(unix, feature = "threading", feature = "host_env"))]
 pub(crate) unsafe fn reinit_imp_lock_after_fork() {
     unsafe { lock::reinit_after_fork() }
 }
 
-#[cfg(all(unix, feature = "threading"))]
+#[cfg(all(unix, feature = "threading", feature = "host_env"))]
 pub(crate) unsafe fn after_fork_child_imp_lock_release() {
     unsafe { lock::after_fork_child_reinit_and_release() }
 }

crates/vm/src/stdlib/_io.rs

@@ -2,7 +2,7 @@
  * I/O core tools.
  */
 pub(crate) use _io::module_def;
-#[cfg(all(unix, feature = "threading"))]
+#[cfg(all(unix, feature = "threading", feature = "host_env"))]
 pub(crate) use _io::reinit_std_streams_after_fork;
 
 cfg_select! {
@@ -5003,7 +5003,7 @@ mod _io {
     ///
     /// Must only be called from the single-threaded child process immediately
     /// after `fork()`, before any other thread is created.
-    #[cfg(all(unix, feature = "threading"))]
+    #[cfg(all(unix, feature = "threading", feature = "host_env"))]
     pub(crate) unsafe fn reinit_std_streams_after_fork(vm: &VirtualMachine) {
         for name in ["stdin", "stdout", "stderr"] {
             let Ok(stream) = vm.sys_module.get_attr(name, vm) else {
@@ -5013,7 +5013,7 @@ mod _io {
         }
     }
 
-    #[cfg(all(unix, feature = "threading"))]
+    #[cfg(all(unix, feature = "threading", feature = "host_env"))]
     fn reinit_io_locks(obj: &PyObject) {
         use crate::common::lock::reinit_thread_mutex_after_fork;
 

crates/vm/src/stdlib/_thread.rs

@@ -1,5 +1,5 @@
 //! Implementation of the _thread module
-#[cfg(unix)]
+#[cfg(all(unix, feature = "threading", feature = "host_env"))]
 pub(crate) use _thread::after_fork_child;
 pub use _thread::get_ident;
 #[cfg_attr(target_arch = "wasm32", allow(unused_imports))]
@@ -457,6 +457,7 @@ pub(crate) mod _thread {
     /// Get thread ID for a given thread handle (used by start_new_thread)
     fn thread_to_id(handle: &thread::JoinHandle<()>) -> u64 {
         #[cfg(unix)]
+        #[allow(clippy::unnecessary_cast)]
         {
             // On Unix, use pthread ID from the handle
             use std::os::unix::thread::JoinHandleExt;
@@ -1056,7 +1057,7 @@ pub(crate) mod _thread {
     ///
     /// Precondition: `reinit_locks_after_fork()` has already been called, so all
     /// parking_lot-based locks in VmState are in unlocked state.
-    #[cfg(unix)]
+    #[cfg(all(unix, feature = "threading", feature = "host_env"))]
     pub(crate) fn after_fork_child(vm: &VirtualMachine) {
         let current_ident = get_ident();
 
@@ -1140,7 +1141,7 @@ pub(crate) mod _thread {
     }
 
     /// Reset a parking_lot::Mutex to unlocked state after fork.
-    #[cfg(unix)]
+    #[cfg(all(unix, feature = "host_env"))]
     fn reinit_parking_lot_mutex<T: ?Sized>(mutex: &parking_lot::Mutex<T>) {
         unsafe { rustpython_common::lock::zero_reinit_after_fork(mutex.raw()) };
     }

crates/vm/src/vm/thread.rs

@@ -50,6 +50,20 @@ pub type CurrentFrameSlot = Arc<ThreadSlot>;
 thread_local! {
     pub(super) static VM_STACK: RefCell<Vec<NonNull<VirtualMachine>>> = Vec::with_capacity(1).into();
 
+    /// Thread state created through the GILState-style C API.
+    ///
+    /// This is separate from the current VM stack: it only means "attached now",
+    /// while this owns the per-thread VM that may be detached and re-attached.
+    /// Despite the historical CPython "GILState" name, this does not model a
+    /// GIL; it stores the VM used by that compatibility API.
+    ///
+    /// The Box keeps the VM address stable while VM_STACK holds a raw pointer to it.
+    /// This matters when release_current_thread() moves the owner out of TLS and
+    /// drops it while the VM is still current, so object destructors can still find
+    /// their VM.
+    #[cfg(feature = "threading")]
+    static GILSTATE_VM: RefCell<Option<Box<ThreadedVirtualMachine>>> = const { RefCell::new(None) };
+
     pub(crate) static COROUTINE_ORIGIN_TRACKING_DEPTH: Cell<u32> = const { Cell::new(0) };
 
     /// Current thread's slot for sys._current_frames() and sys._current_exceptions()
@@ -66,42 +80,114 @@ thread_local! {
 
 }
 
-scoped_tls::scoped_thread_local!(static VM_CURRENT: VirtualMachine);
+#[must_use]
+pub fn current_vm_is_set() -> bool {
+    VM_STACK.with(|vms| !vms.borrow().is_empty())
+}
 
 pub fn with_current_vm<R>(f: impl FnOnce(&VirtualMachine) -> R) -> R {
-    if !VM_CURRENT.is_set() {
-        panic!("call with_current_vm() but VM_CURRENT is null");
-    }
-    VM_CURRENT.with(f)
+    VM_STACK.with(|vms| {
+        let vm = vms
+            .borrow()
+            .last()
+            .copied()
+            .expect("call with_current_vm() but no current VM is attached");
+        // SAFETY: entries in VM_STACK either borrow a VM for the dynamic
+        // scope of a set_current_vm()/enter_vm() call or point at GILSTATE_VM.
+        f(unsafe { vm.as_ref() })
+    })
+}
+
+fn set_current_vm<R>(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R {
+    VM_STACK.with(|vms| {
+        vms.borrow_mut().push(vm.into());
+        scopeguard::defer! {
+            vms.borrow_mut().pop();
+        }
+        f()
+    })
 }
 
 pub fn enter_vm<R>(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R {
-    VM_STACK.with(|vms| {
-        // Outermost enter_vm: transition DETACHED → ATTACHED
-        #[cfg(all(unix, feature = "threading"))]
-        let was_outermost = vms.borrow().is_empty();
+    // Outermost enter_vm: transition DETACHED → ATTACHED
+    #[cfg(all(unix, feature = "threading"))]
+    let was_outermost = !current_vm_is_set();
 
-        vms.borrow_mut().push(vm.into());
+    // Initialize thread slot for this thread if not already done
+    #[cfg(feature = "threading")]
+    init_thread_slot_if_needed(vm);
 
-        // Initialize thread slot for this thread if not already done
-        #[cfg(feature = "threading")]
-        init_thread_slot_if_needed(vm);
+    #[cfg(all(unix, feature = "threading"))]
+    if was_outermost {
+        attach_thread(vm);
+    }
 
+    scopeguard::defer! {
+        // Outermost exit: transition ATTACHED → DETACHED
         #[cfg(all(unix, feature = "threading"))]
         if was_outermost {
-            attach_thread(vm);
+            detach_thread();
         }
+    }
+    set_current_vm(vm, f)
+}
 
-        scopeguard::defer! {
-            // Outermost exit: transition ATTACHED → DETACHED
-            #[cfg(all(unix, feature = "threading"))]
-            if vms.borrow().len() == 1 {
-                detach_thread();
-            }
-            vms.borrow_mut().pop();
-        }
-        VM_CURRENT.set(vm, f)
-    })
+#[cfg(feature = "threading")]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum CurrentVmAttachState {
+    AlreadyAttached,
+    Attached,
+}
+
+/// Attach the current native thread to a RustPython VM until
+/// `release_current_thread()` is called.
+#[cfg(feature = "threading")]
+pub fn attach_current_thread(
+    make_vm: impl FnOnce() -> ThreadedVirtualMachine,
+) -> CurrentVmAttachState {
+    if current_vm_is_set() {
+        return CurrentVmAttachState::AlreadyAttached;
+    }
+
+    GILSTATE_VM.with(|gilstate_vm| {
+        let mut gilstate_vm = gilstate_vm.borrow_mut();
+        let threaded_vm = gilstate_vm.get_or_insert_with(|| Box::new(make_vm()));
+        let vm = &threaded_vm.vm;
+
+        vm.c_stack_soft_limit
+            .set(VirtualMachine::calculate_c_stack_soft_limit());
+
+        init_thread_slot_if_needed(vm);
+
+        #[cfg(unix)]
+        attach_thread(vm);
+
+        VM_STACK.with(|vms| {
+            debug_assert!(vms.borrow().is_empty());
+            vms.borrow_mut().push(vm.into());
+        });
+    });
+
+    CurrentVmAttachState::Attached
+}
+
+#[cfg(feature = "threading")]
+pub fn release_current_thread(state: CurrentVmAttachState) {
+    if state == CurrentVmAttachState::AlreadyAttached {
+        return;
+    }
+
+    let gilstate_vm = GILSTATE_VM.with(|gilstate_vm| gilstate_vm.borrow_mut().take());
+    drop(gilstate_vm);
+
+    VM_STACK.with(|vms| {
+        vms.borrow_mut()
+            .pop()
+            .expect("release_current_thread() called without an attached VM");
+    });
+
+    #[cfg(unix)]
+    detach_thread();
 }
 
 /// Initialize thread slot for current thread if not already initialized.
@@ -443,7 +529,7 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) {
 
     // Guard against OS thread-id reuse races: only remove the registry entry
     // if it still points at this thread's own slot.
-    let removed = if let Some(slot) = &current_slot {
+    let _removed = if let Some(slot) = &current_slot {
         let mut registry = vm.state.thread_frames.lock();
         match registry.get(&thread_id) {
             Some(registered) if Arc::ptr_eq(registered, slot) => registry.remove(&thread_id),
@@ -453,7 +539,7 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) {
         None
     };
     #[cfg(all(unix, feature = "threading"))]
-    if let Some(slot) = &removed
+    if let Some(slot) = &_removed
         && vm.state.stop_the_world.requested.load(Ordering::Acquire)
         && thread_id != vm.state.stop_the_world.requester_ident()
         && slot.state.load(Ordering::Relaxed) != THREAD_SUSPENDED
@@ -509,17 +595,20 @@ where
         obj.fast_isinstance(vm.ctx.types.object_type)
     };
     VM_STACK.with(|vms| {
-        let interp = match vms.borrow().iter().copied().exactly_one() {
-            Ok(x) => {
-                debug_assert!(vm_owns_obj(x));
-                x
+        let interp = {
+            let vms = vms.borrow();
+            match vms.iter().copied().exactly_one() {
+                Ok(x) => {
+                    debug_assert!(vm_owns_obj(x));
+                    x
+                }
+                Err(mut others) => others.find(|x| vm_owns_obj(*x))?,
             }
-            Err(mut others) => others.find(|x| vm_owns_obj(*x))?,
         };
         // SAFETY: all references in VM_STACK should be valid, and should not be changed or moved
         // at least until this function returns and the stack unwinds to an enter_vm() call
         let vm = unsafe { interp.as_ref() };
-        Some(VM_CURRENT.set(vm, || f(vm)))
+        Some(set_current_vm(vm, || f(vm)))
     })
 }
 

src/lib.rs

@@ -118,7 +118,15 @@ pub fn run(mut builder: InterpreterBuilder) -> ExitCode {
     builder = builder.settings(settings);
 
     let interp = builder.interpreter();
-    let exitcode = interp.run(move |vm| run_rustpython(vm, run_mode));
+    let exitcode = cfg_select! {
+        feature = "capi" => {{
+            let local_vm = interp.enter(|vm| vm.new_thread());
+            *(rustpython_capi::get_main_interpreter()) = Some(interp);
+            let result = local_vm.run(|vm| run_rustpython(vm, run_mode));
+            rustpython_capi::get_main_interpreter().take().unwrap().finalize(result.err())
+        }},
+        _ => interp.run(move |vm| run_rustpython(vm, run_mode)),
+    };
 
     rustpython_vm::host_env::os::exit_code(exitcode)
 }