Fix /dev/fd, rlimit, spawn scheduler, and context cache (#8651)

* Fix /dev/fd, rlimit, spawn scheduler, and context cache

Honor PYTHONEXECUTABLE when computing sys.executable.
Rewind the file offset after a pyc magic probe.
Stop masking rlimit values with RLIM_INFINITY.
Print the pyrepl-unavailable warning before the rustyline shell.
Parse posix_spawn scheduler and apply it where the host supports it.
Update ContextVar cache fields under one lock.
Remove expectedFailure and skip markers for tests that now pass.

Assisted-by: Grok:4.6

* address review: check rlim_t overflow on convert

Replace the u64-to-rlim_t as-cast in py2rlim_limit with
a checked try_from, matching py2rlim.

Assisted-by: Grok:4.6

Lib/test/test_cmd_line.py

@@ -985,7 +985,6 @@ class CmdLineTest(unittest.TestCase):
                 contents = file.read()
                 self.assertIn('Remaining objects', contents)
 
-    @unittest.expectedFailureIf(sys.platform == "darwin", "TODO: RUSTPYTHON")
     @unittest.skipUnless(sys.platform == 'darwin', 'PYTHONEXECUTABLE only works on macOS')
     def test_python_executable(self):
         code = 'import sys; print(sys.executable)'

Lib/test/test_cmd_line_script.py

@@ -806,7 +806,6 @@ class CmdLineTest(unittest.TestCase):
         self.assertIn(": can't open file ", err)
         self.assertNotEqual(proc.returncode, 0)
 
-    @unittest.skipIf(sys.platform.startswith("darwin"), "TODO: RUSTPYTHON; Problems with Mac os descriptor")
     @unittest.skipUnless(os.path.exists('/dev/fd/0'), 'requires /dev/fd platform')
     @unittest.skipIf(sys.platform.startswith("freebsd") and
                      os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev,

Lib/test/test_context.py

@@ -380,7 +380,6 @@ class ContextTest(unittest.TestCase):
             with self.subTest(name=name):
                 self.assertTrue(callable(getattr(ctx, name)))
 
-    @unittest.skipIf(sys.platform == "darwin", "TODO: RUSTPYTHON; Flaky on Mac, self.assertEqual(cvar.get(), num + i) AssertionError: 8 != 12")
     @isolated_context
     @threading_helper.requires_working_threading()
     def test_context_threads_1(self):

Lib/test/test_posix.py

@@ -2044,7 +2044,6 @@ class _PosixSpawnMixin:
         pid = self.spawn_func(path, args, os.environ, scheduler=None)
         support.wait_process(pid, exitcode=0)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; Wrong error message
     @support.subTests("scheduler", [object(), 1, [1, 2]])
     def test_scheduler_wrong_type(self, scheduler):
         path, args = self.NOOP_PROGRAM[0], self.NOOP_PROGRAM
@@ -2054,7 +2053,6 @@ class _PosixSpawnMixin:
         ):
             self.spawn_func(path, args, os.environ, scheduler=scheduler)
 
-    @unittest.expectedFailureIf(sys.platform in ("darwin", "linux"), "TODO: RUSTPYTHON; NotImplementedError: scheduler parameter is not yet implemented")
     @requires_sched
     @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')),
                      "bpo-34685: test can fail on BSD")
@@ -2075,7 +2073,6 @@ class _PosixSpawnMixin:
         )
         support.wait_process(pid, exitcode=0)
 
-    @unittest.expectedFailureIf(sys.platform in ("darwin", "linux"), "TODO: RUSTPYTHON; NotImplementedError: scheduler parameter is not yet implemented")
     @requires_sched
     @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')),
                      "bpo-34685: test can fail on BSD")

Lib/test/test_pyrepl/test_pyrepl.py

@@ -1521,7 +1521,6 @@ class TestPasteEvent(TestCase):
 
 @skipUnless(pty, "requires pty")
 class TestDumbTerminal(ReplTestCase):
-    @unittest.expectedFailureIf(sys.platform in ("darwin", "linux"), "TODO: RUSTPYTHON")
     def test_dumb_terminal_exits_cleanly(self):
         env = os.environ.copy()
         env.pop('PYTHON_BASIC_REPL', None)

Lib/test/test_resource.py

@@ -106,7 +106,6 @@ class ResourceTest(unittest.TestCase):
         except (OverflowError, ValueError):
             pass
 
-    @unittest.skipIf(sys.platform == "darwin", "TODO: RUSTPYTHON; crash")
     @unittest.skipIf(sys.platform == "vxworks",
                      "setting RLIMIT_FSIZE is not supported on VxWorks")
     @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE')

crates/host_env/src/posix.rs

@@ -82,6 +82,15 @@ pub enum PosixSpawnFileAction {
     },
 }
 
+#[cfg(all(
+    any(target_os = "linux", target_os = "freebsd", target_os = "android"),
+    not(target_env = "musl")
+))]
+pub struct PosixSpawnScheduler {
+    pub policy: Option<i32>,
+    pub param: libc::sched_param,
+}
+
 #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))]
 pub struct PosixSpawnConfig<'a> {
     pub path: &'a CStr,
@@ -94,6 +103,11 @@ pub struct PosixSpawnConfig<'a> {
     pub setsid: bool,
     pub setsigmask: Option<&'a [i32]>,
     pub spawnp: bool,
+    #[cfg(all(
+        any(target_os = "linux", target_os = "freebsd", target_os = "android"),
+        not(target_env = "musl")
+    ))]
+    pub scheduler: Option<PosixSpawnScheduler>,
 }
 
 pub fn set_inheritable(fd: BorrowedFd<'_>, inheritable: bool) -> std::io::Result<()> {
@@ -1428,6 +1442,32 @@ fn build_posix_spawn_attrs(
         flags.insert(nix::spawn::PosixSpawnFlags::POSIX_SPAWN_SETSIGMASK);
     }
 
+    #[cfg(all(
+        any(target_os = "linux", target_os = "freebsd", target_os = "android"),
+        not(target_env = "musl")
+    ))]
+    if let Some(scheduler) = &config.scheduler {
+        // nix does not wrap these yet; the attr type is transparent over
+        // posix_spawnattr_t.
+        let attr_ptr = (&raw mut attrp).cast::<libc::posix_spawnattr_t>();
+        if let Some(policy) = scheduler.policy {
+            let err = unsafe { libc::posix_spawnattr_setschedpolicy(attr_ptr, policy) };
+            if err != 0 {
+                return Err(std::io::Error::from_raw_os_error(err));
+            }
+            flags.insert(nix::spawn::PosixSpawnFlags::from_bits_retain(
+                libc::POSIX_SPAWN_SETSCHEDULER,
+            ));
+        }
+        let err = unsafe { libc::posix_spawnattr_setschedparam(attr_ptr, &scheduler.param) };
+        if err != 0 {
+            return Err(std::io::Error::from_raw_os_error(err));
+        }
+        flags.insert(nix::spawn::PosixSpawnFlags::from_bits_retain(
+            libc::POSIX_SPAWN_SETSCHEDPARAM,
+        ));
+    }
+
     if !flags.is_empty() {
         attrp.set_flags(flags).map_err(std::io::Error::from)?;
     }

crates/stdlib/src/contextvars.rs

@@ -335,20 +335,23 @@ mod _contextvars {
             Ok(())
         }
 
-        // contextvar_set in CPython
+        // contextvar_set
         fn set_inner(zelf: &Py<Self>, value: PyObjectRef, vm: &VirtualMachine) {
             let ctx = PyContext::current(vm);
 
             let replaced = ctx.borrow_vars_mut().insert(zelf.to_owned(), value.clone());
             drop(replaced);
 
-            zelf.cached_id.store(ctx.get_id(), Ordering::SeqCst);
-
+            // cached and cached_id are one snapshot: another thread's get()
+            // must not observe a new id with an old value, or the reverse.
             let cache = ContextVarCache {
                 object: value,
                 idx: ctx.inner.idx.load(Ordering::Relaxed),
             };
-            let replaced = zelf.cached.lock().replace(cache);
+            let mut cached = zelf.cached.lock();
+            zelf.cached_id.store(ctx.get_id(), Ordering::SeqCst);
+            let replaced = cached.replace(cache);
+            drop(cached);
             drop(replaced);
         }
 

crates/stdlib/src/resource.rs

@@ -10,6 +10,7 @@ mod resource {
         convert::{ToPyException, ToPyObject},
         types::PyStructSequence,
     };
+    use num_traits::Signed;
     use rustpython_host_env::resource as host_resource;
     use std::io;
 
@@ -145,11 +146,11 @@ mod resource {
 
     impl<'a> TryFromBorrowedObject<'a> for Limits {
         fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult<Self> {
-            let seq: Vec<host_resource::rlim_t> = obj.try_to_value(vm)?;
-            match *seq {
+            let seq: Vec<PyObjectRef> = obj.try_to_value(vm)?;
+            match seq.as_slice() {
                 [cur, max] => Ok(Self(host_resource::rlimit {
-                    rlim_cur: cur & RLIM_INFINITY,
-                    rlim_max: max & RLIM_INFINITY,
+                    rlim_cur: py2rlim_limit(cur, vm)?,
+                    rlim_max: py2rlim_limit(max, vm)?,
                 })),
                 _ => Err(vm.new_value_error("expected a tuple of 2 integers")),
             }
@@ -173,6 +174,18 @@ mod resource {
             .map_err(|_| vm.new_overflow_error("Python int too large to convert to C rlim_t"))
     }
 
+    fn py2rlim_limit(obj: &PyObject, vm: &VirtualMachine) -> PyResult<host_resource::rlim_t> {
+        let int = obj.try_to_ref::<crate::vm::builtins::PyInt>(vm)?;
+        if int.as_bigint().is_negative() {
+            return Err(vm.new_value_error("Cannot convert negative int"));
+        }
+        let value = int
+            .try_to_primitive::<u64>(vm)
+            .map_err(|_| vm.new_overflow_error("Python int too large to convert to C rlim_t"))?;
+        host_resource::rlim_t::try_from(value)
+            .map_err(|_| vm.new_overflow_error("Python int too large to convert to C rlim_t"))
+    }
+
     #[pyfunction]
     fn getrlimit(resource: PyIntRef, vm: &VirtualMachine) -> PyResult<Limits> {
         let resource = py2rlim(resource, vm)?;

crates/vm/src/getpath.rs

@@ -115,15 +115,13 @@ pub fn init_path_config(settings: &Settings) -> Paths {
         .map(|p| p.to_string_lossy().into_owned())
         .unwrap_or_default();
 
-    // Step 1: Check for __PYVENV_LAUNCHER__ environment variable
-    // When launched from a venv launcher, __PYVENV_LAUNCHER__ contains the venv's python.exe path
-    // In this case:
-    //   - sys.executable should be the launcher path (where user invoked Python)
-    //   - sys._base_executable should be the real Python executable
-    let exe_dir = if let Ok(launcher) = crate::host_env::os::var("__PYVENV_LAUNCHER__") {
-        paths.executable.clone_from(&launcher);
+    // Step 1: Check for PYTHONEXECUTABLE / __PYVENV_LAUNCHER__
+    // When set, these are used as sys.executable and when searching for venvs.
+    // The argv0 path is kept as the base executable for prefix calculation.
+    let exe_dir = if let Some(override_exe) = env_executable_override() {
+        paths.executable.clone_from(&override_exe);
         paths.base_executable = real_executable;
-        PathBuf::from(&launcher).parent().map(PathBuf::from)
+        PathBuf::from(&override_exe).parent().map(PathBuf::from)
     } else {
         paths.executable = real_executable;
         executable
@@ -165,7 +163,7 @@ pub fn init_path_config(settings: &Settings) -> Paths {
     };
     paths.base_exec_prefix.clone_from(&paths.base_prefix);
 
-    // Step 7: Calculate base_executable (if not already set by __PYVENV_LAUNCHER__)
+    // Step 7: Calculate base_executable (if not already set by an env override)
     if paths.base_executable.is_empty() {
         paths.base_executable = calculate_base_executable(executable.as_ref(), home_dir.as_ref());
     }
@@ -367,6 +365,19 @@ fn build_module_search_paths(settings: &Settings, prefix: &str, exec_prefix: &st
     paths
 }
 
+/// `PYTHONEXECUTABLE` takes precedence over `__PYVENV_LAUNCHER__`.
+/// An empty value is ignored.
+fn env_executable_override() -> Option<String> {
+    for name in ["PYTHONEXECUTABLE", "__PYVENV_LAUNCHER__"] {
+        if let Ok(value) = crate::host_env::os::var(name)
+            && !value.is_empty()
+        {
+            return Some(value);
+        }
+    }
+    None
+}
+
 /// Get the current executable path
 fn get_executable_path() -> Option<PathBuf> {
     #[cfg(not(target_arch = "wasm32"))]

crates/vm/src/stdlib/posix.rs

@@ -18,7 +18,7 @@ pub use rustpython_host_env::posix::set_inheritable;
 pub mod module {
     use crate::{
         AsObject, Py, PyObjectRef, PyResult, VirtualMachine,
-        builtins::{PyDictRef, PyInt, PyListRef, PyTupleRef, PyUtf8Str},
+        builtins::{PyDictRef, PyInt, PyListRef, PyTuple, PyTupleRef, PyUtf8Str},
         convert::{IntoPyException, ToPyException, ToPyObject, TryFromObject},
         exceptions::OSErrorBuilder,
         function::{ArgMapping, Either, KwArgs, OptionalArg},
@@ -1464,7 +1464,7 @@ pub mod module {
         #[pyarg(named, default)]
         setsigmask: Option<crate::function::ArgIterable<i32>>,
         #[pyarg(named, default)]
-        scheduler: Option<PyTupleRef>,
+        scheduler: Option<PyObjectRef>,
     }
 
     #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))]
@@ -1476,6 +1476,58 @@ pub mod module {
         Dup2,
     }
 
+    #[cfg(all(
+        any(target_os = "linux", target_os = "freebsd", target_os = "android"),
+        not(target_env = "musl")
+    ))]
+    fn parse_posix_spawn_scheduler(
+        scheduler: Option<PyObjectRef>,
+        vm: &VirtualMachine,
+    ) -> PyResult<Option<rustpython_host_env::posix::PosixSpawnScheduler>> {
+        let Some(obj) = scheduler else {
+            return Ok(None);
+        };
+        if obj.is(&vm.ctx.none()) {
+            return Ok(None);
+        }
+        let Some(tuple) = obj.downcast_ref::<PyTuple>() else {
+            return Err(vm.new_type_error("scheduler must be a tuple or None"));
+        };
+        if tuple.len() != 2 {
+            return Err(vm.new_type_error("A scheduler tuple must have two elements"));
+        }
+        let policy = if tuple[0].is(&vm.ctx.none()) {
+            None
+        } else {
+            Some(i32::try_from_object(vm, tuple[0].clone())?)
+        };
+        let param = super::posix_sched::convert_sched_param(&tuple[1], vm)?;
+        Ok(Some(rustpython_host_env::posix::PosixSpawnScheduler {
+            policy,
+            param,
+        }))
+    }
+
+    #[cfg(any(target_os = "macos", target_env = "musl"))]
+    fn parse_posix_spawn_scheduler(
+        scheduler: Option<PyObjectRef>,
+        vm: &VirtualMachine,
+    ) -> PyResult<()> {
+        let Some(obj) = scheduler else {
+            return Ok(());
+        };
+        if obj.is(&vm.ctx.none()) {
+            return Ok(());
+        }
+        let Some(tuple) = obj.downcast_ref::<PyTuple>() else {
+            return Err(vm.new_type_error("scheduler must be a tuple or None"));
+        };
+        if tuple.len() != 2 {
+            return Err(vm.new_type_error("A scheduler tuple must have two elements"));
+        }
+        Err(vm.new_not_implemented_error("The scheduler option is not supported in this system."))
+    }
+
     #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))]
     impl PosixSpawnArgs {
         fn spawn(self, spawnp: bool, vm: &VirtualMachine) -> PyResult<libc::pid_t> {
@@ -1555,13 +1607,13 @@ pub mod module {
 
             let setsigdef = self.setsigdef.map(&collect_signals).transpose()?;
 
-            if let Some(_scheduler) = self.scheduler {
-                // TODO: Implement scheduler parameter handling
-                // This requires platform-specific sched_param struct handling
-                return Err(
-                    vm.new_not_implemented_error("scheduler parameter is not yet implemented")
-                );
-            }
+            #[cfg(all(
+                any(target_os = "linux", target_os = "freebsd", target_os = "android"),
+                not(target_env = "musl")
+            ))]
+            let scheduler = parse_posix_spawn_scheduler(self.scheduler, vm)?;
+            #[cfg(any(target_os = "macos", target_env = "musl"))]
+            parse_posix_spawn_scheduler(self.scheduler, vm)?;
 
             if self.setsid && !rustpython_host_env::posix::supports_posix_spawn_setsid() {
                 return Err(vm.new_not_implemented_error(
@@ -1603,6 +1655,11 @@ pub mod module {
                 setsid: self.setsid,
                 setsigmask: setsigmask.as_deref(),
                 spawnp,
+                #[cfg(all(
+                    any(target_os = "linux", target_os = "freebsd", target_os = "android"),
+                    not(target_env = "musl")
+                ))]
+                scheduler,
             })
             .map_err(|err| OSErrorBuilder::with_filename(&err, self.path, vm))
         }
@@ -2460,7 +2517,10 @@ mod posix_sched {
     }
 
     #[cfg(not(target_env = "musl"))]
-    fn convert_sched_param(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<libc::sched_param> {
+    pub(super) fn convert_sched_param(
+        obj: &PyObjectRef,
+        vm: &VirtualMachine,
+    ) -> PyResult<libc::sched_param> {
         use crate::{
             builtins::{PyInt, PyTuple},
             class::StaticType,

crates/vm/src/vm/python_run.rs

@@ -175,8 +175,12 @@ mod file_run {
         let mut file = crate::host_env::fs::open(path)?;
         let mut buf = [0u8; 2];
 
-        use std::io::Read;
-        if file.read(&mut buf)? != 2 {
+        use std::io::{Read, Seek, SeekFrom};
+        let n = file.read(&mut buf)?;
+        // /dev/fd/N shares the file offset across every open of that path.
+        // Restore it so a later read of the same script still starts at 0.
+        let _ = file.seek(SeekFrom::Start(0));
+        if n != 2 {
             return Ok(false);
         }
 

src/lib.rs

@@ -361,6 +361,7 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> {
         RunMode::Repl => Ok(()),
     };
     let result = if is_repl || vm.state.config.settings.inspect {
+        warn_if_pyrepl_unavailable(vm);
         shell::run_shell(vm, scope)
     } else {
         res
@@ -377,6 +378,23 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> {
     result
 }
 
+/// When the fancy REPL cannot start (for example `TERM=dumb`), print the same
+/// warning `_pyrepl.main` would have printed, then keep the rustyline shell.
+fn warn_if_pyrepl_unavailable(vm: &VirtualMachine) {
+    let _ = vm.run_simple_string(
+        r"
+import os, sys
+if not os.getenv('PYTHON_BASIC_REPL'):
+    try:
+        from _pyrepl.main import CAN_USE_PYREPL, FAIL_REASON
+        if not CAN_USE_PYREPL and FAIL_REASON:
+            print(FAIL_REASON, file=sys.stderr)
+    except Exception:
+        pass
+",
+    );
+}
+
 #[cfg(feature = "flame-it")]
 fn write_profile(settings: &Settings) -> Result<(), Box<dyn core::error::Error>> {
     use std::{fs, io};