From 25b5dfe5cab57996128dcbcebb7fd5be796be415 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:57:29 +0900 Subject: [PATCH] Fix /dev/fd, rlimit, spawn scheduler, and context cache (#8651) * Fix /dev/fd, rlimit, spawn scheduler, and context cache Honor PYTHONEXECUTABLE when computing sys.executable. Rewind the file offset after a pyc magic probe. Stop masking rlimit values with RLIM_INFINITY. Print the pyrepl-unavailable warning before the rustyline shell. Parse posix_spawn scheduler and apply it where the host supports it. Update ContextVar cache fields under one lock. Remove expectedFailure and skip markers for tests that now pass. Assisted-by: Grok:4.6 * address review: check rlim_t overflow on convert Replace the u64-to-rlim_t as-cast in py2rlim_limit with a checked try_from, matching py2rlim. Assisted-by: Grok:4.6 --- Lib/test/test_cmd_line.py | 1 - Lib/test/test_cmd_line_script.py | 1 - Lib/test/test_context.py | 1 - Lib/test/test_posix.py | 3 -- Lib/test/test_pyrepl/test_pyrepl.py | 1 - Lib/test/test_resource.py | 1 - crates/host_env/src/posix.rs | 40 +++++++++++++++ crates/stdlib/src/contextvars.rs | 11 ++-- crates/stdlib/src/resource.rs | 21 ++++++-- crates/vm/src/getpath.rs | 29 +++++++---- crates/vm/src/stdlib/posix.rs | 80 +++++++++++++++++++++++++---- crates/vm/src/vm/python_run.rs | 8 ++- src/lib.rs | 18 +++++++ 13 files changed, 178 insertions(+), 37 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 018ca41e2..d6fcfa3b4 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -985,7 +985,6 @@ class CmdLineTest(unittest.TestCase): contents = file.read() self.assertIn('Remaining objects', contents) - @unittest.expectedFailureIf(sys.platform == "darwin", "TODO: RUSTPYTHON") @unittest.skipUnless(sys.platform == 'darwin', 'PYTHONEXECUTABLE only works on macOS') def test_python_executable(self): code = 'import sys; print(sys.executable)' diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index fc894fda6..2c8a80b70 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -806,7 +806,6 @@ class CmdLineTest(unittest.TestCase): self.assertIn(": can't open file ", err) self.assertNotEqual(proc.returncode, 0) - @unittest.skipIf(sys.platform.startswith("darwin"), "TODO: RUSTPYTHON; Problems with Mac os descriptor") @unittest.skipUnless(os.path.exists('/dev/fd/0'), 'requires /dev/fd platform') @unittest.skipIf(sys.platform.startswith("freebsd") and os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, diff --git a/Lib/test/test_context.py b/Lib/test/test_context.py index 104f2f4da..97089815d 100644 --- a/Lib/test/test_context.py +++ b/Lib/test/test_context.py @@ -380,7 +380,6 @@ class ContextTest(unittest.TestCase): with self.subTest(name=name): self.assertTrue(callable(getattr(ctx, name))) - @unittest.skipIf(sys.platform == "darwin", "TODO: RUSTPYTHON; Flaky on Mac, self.assertEqual(cvar.get(), num + i) AssertionError: 8 != 12") @isolated_context @threading_helper.requires_working_threading() def test_context_threads_1(self): diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index d7e1c00f1..4ced0b966 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -2044,7 +2044,6 @@ class _PosixSpawnMixin: pid = self.spawn_func(path, args, os.environ, scheduler=None) support.wait_process(pid, exitcode=0) - @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message @support.subTests("scheduler", [object(), 1, [1, 2]]) def test_scheduler_wrong_type(self, scheduler): path, args = self.NOOP_PROGRAM[0], self.NOOP_PROGRAM @@ -2054,7 +2053,6 @@ class _PosixSpawnMixin: ): self.spawn_func(path, args, os.environ, scheduler=scheduler) - @unittest.expectedFailureIf(sys.platform in ("darwin", "linux"), "TODO: RUSTPYTHON; NotImplementedError: scheduler parameter is not yet implemented") @requires_sched @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')), "bpo-34685: test can fail on BSD") @@ -2075,7 +2073,6 @@ class _PosixSpawnMixin: ) support.wait_process(pid, exitcode=0) - @unittest.expectedFailureIf(sys.platform in ("darwin", "linux"), "TODO: RUSTPYTHON; NotImplementedError: scheduler parameter is not yet implemented") @requires_sched @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')), "bpo-34685: test can fail on BSD") diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 2d6d42d8c..887d11acd 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1521,7 +1521,6 @@ class TestPasteEvent(TestCase): @skipUnless(pty, "requires pty") class TestDumbTerminal(ReplTestCase): - @unittest.expectedFailureIf(sys.platform in ("darwin", "linux"), "TODO: RUSTPYTHON") def test_dumb_terminal_exits_cleanly(self): env = os.environ.copy() env.pop('PYTHON_BASIC_REPL', None) diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py index 41ab28d1a..2302716a1 100644 --- a/Lib/test/test_resource.py +++ b/Lib/test/test_resource.py @@ -106,7 +106,6 @@ class ResourceTest(unittest.TestCase): except (OverflowError, ValueError): pass - @unittest.skipIf(sys.platform == "darwin", "TODO: RUSTPYTHON; crash") @unittest.skipIf(sys.platform == "vxworks", "setting RLIMIT_FSIZE is not supported on VxWorks") @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') diff --git a/crates/host_env/src/posix.rs b/crates/host_env/src/posix.rs index 557da77b4..3320731f7 100644 --- a/crates/host_env/src/posix.rs +++ b/crates/host_env/src/posix.rs @@ -82,6 +82,15 @@ pub enum PosixSpawnFileAction { }, } +#[cfg(all( + any(target_os = "linux", target_os = "freebsd", target_os = "android"), + not(target_env = "musl") +))] +pub struct PosixSpawnScheduler { + pub policy: Option, + pub param: libc::sched_param, +} + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] pub struct PosixSpawnConfig<'a> { pub path: &'a CStr, @@ -94,6 +103,11 @@ pub struct PosixSpawnConfig<'a> { pub setsid: bool, pub setsigmask: Option<&'a [i32]>, pub spawnp: bool, + #[cfg(all( + any(target_os = "linux", target_os = "freebsd", target_os = "android"), + not(target_env = "musl") + ))] + pub scheduler: Option, } pub fn set_inheritable(fd: BorrowedFd<'_>, inheritable: bool) -> std::io::Result<()> { @@ -1428,6 +1442,32 @@ fn build_posix_spawn_attrs( flags.insert(nix::spawn::PosixSpawnFlags::POSIX_SPAWN_SETSIGMASK); } + #[cfg(all( + any(target_os = "linux", target_os = "freebsd", target_os = "android"), + not(target_env = "musl") + ))] + if let Some(scheduler) = &config.scheduler { + // nix does not wrap these yet; the attr type is transparent over + // posix_spawnattr_t. + let attr_ptr = (&raw mut attrp).cast::(); + if let Some(policy) = scheduler.policy { + let err = unsafe { libc::posix_spawnattr_setschedpolicy(attr_ptr, policy) }; + if err != 0 { + return Err(std::io::Error::from_raw_os_error(err)); + } + flags.insert(nix::spawn::PosixSpawnFlags::from_bits_retain( + libc::POSIX_SPAWN_SETSCHEDULER, + )); + } + let err = unsafe { libc::posix_spawnattr_setschedparam(attr_ptr, &scheduler.param) }; + if err != 0 { + return Err(std::io::Error::from_raw_os_error(err)); + } + flags.insert(nix::spawn::PosixSpawnFlags::from_bits_retain( + libc::POSIX_SPAWN_SETSCHEDPARAM, + )); + } + if !flags.is_empty() { attrp.set_flags(flags).map_err(std::io::Error::from)?; } diff --git a/crates/stdlib/src/contextvars.rs b/crates/stdlib/src/contextvars.rs index 2377f45ec..529de1d94 100644 --- a/crates/stdlib/src/contextvars.rs +++ b/crates/stdlib/src/contextvars.rs @@ -335,20 +335,23 @@ mod _contextvars { Ok(()) } - // contextvar_set in CPython + // contextvar_set fn set_inner(zelf: &Py, value: PyObjectRef, vm: &VirtualMachine) { let ctx = PyContext::current(vm); let replaced = ctx.borrow_vars_mut().insert(zelf.to_owned(), value.clone()); drop(replaced); - zelf.cached_id.store(ctx.get_id(), Ordering::SeqCst); - + // cached and cached_id are one snapshot: another thread's get() + // must not observe a new id with an old value, or the reverse. let cache = ContextVarCache { object: value, idx: ctx.inner.idx.load(Ordering::Relaxed), }; - let replaced = zelf.cached.lock().replace(cache); + let mut cached = zelf.cached.lock(); + zelf.cached_id.store(ctx.get_id(), Ordering::SeqCst); + let replaced = cached.replace(cache); + drop(cached); drop(replaced); } diff --git a/crates/stdlib/src/resource.rs b/crates/stdlib/src/resource.rs index 5da06632d..7e0523b78 100644 --- a/crates/stdlib/src/resource.rs +++ b/crates/stdlib/src/resource.rs @@ -10,6 +10,7 @@ mod resource { convert::{ToPyException, ToPyObject}, types::PyStructSequence, }; + use num_traits::Signed; use rustpython_host_env::resource as host_resource; use std::io; @@ -145,11 +146,11 @@ mod resource { impl<'a> TryFromBorrowedObject<'a> for Limits { fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { - let seq: Vec = obj.try_to_value(vm)?; - match *seq { + let seq: Vec = obj.try_to_value(vm)?; + match seq.as_slice() { [cur, max] => Ok(Self(host_resource::rlimit { - rlim_cur: cur & RLIM_INFINITY, - rlim_max: max & RLIM_INFINITY, + rlim_cur: py2rlim_limit(cur, vm)?, + rlim_max: py2rlim_limit(max, vm)?, })), _ => Err(vm.new_value_error("expected a tuple of 2 integers")), } @@ -173,6 +174,18 @@ mod resource { .map_err(|_| vm.new_overflow_error("Python int too large to convert to C rlim_t")) } + fn py2rlim_limit(obj: &PyObject, vm: &VirtualMachine) -> PyResult { + let int = obj.try_to_ref::(vm)?; + if int.as_bigint().is_negative() { + return Err(vm.new_value_error("Cannot convert negative int")); + } + let value = int + .try_to_primitive::(vm) + .map_err(|_| vm.new_overflow_error("Python int too large to convert to C rlim_t"))?; + host_resource::rlim_t::try_from(value) + .map_err(|_| vm.new_overflow_error("Python int too large to convert to C rlim_t")) + } + #[pyfunction] fn getrlimit(resource: PyIntRef, vm: &VirtualMachine) -> PyResult { let resource = py2rlim(resource, vm)?; diff --git a/crates/vm/src/getpath.rs b/crates/vm/src/getpath.rs index bd90a3d27..eef1417ab 100644 --- a/crates/vm/src/getpath.rs +++ b/crates/vm/src/getpath.rs @@ -115,15 +115,13 @@ pub fn init_path_config(settings: &Settings) -> Paths { .map(|p| p.to_string_lossy().into_owned()) .unwrap_or_default(); - // Step 1: Check for __PYVENV_LAUNCHER__ environment variable - // When launched from a venv launcher, __PYVENV_LAUNCHER__ contains the venv's python.exe path - // In this case: - // - sys.executable should be the launcher path (where user invoked Python) - // - sys._base_executable should be the real Python executable - let exe_dir = if let Ok(launcher) = crate::host_env::os::var("__PYVENV_LAUNCHER__") { - paths.executable.clone_from(&launcher); + // Step 1: Check for PYTHONEXECUTABLE / __PYVENV_LAUNCHER__ + // When set, these are used as sys.executable and when searching for venvs. + // The argv0 path is kept as the base executable for prefix calculation. + let exe_dir = if let Some(override_exe) = env_executable_override() { + paths.executable.clone_from(&override_exe); paths.base_executable = real_executable; - PathBuf::from(&launcher).parent().map(PathBuf::from) + PathBuf::from(&override_exe).parent().map(PathBuf::from) } else { paths.executable = real_executable; executable @@ -165,7 +163,7 @@ pub fn init_path_config(settings: &Settings) -> Paths { }; paths.base_exec_prefix.clone_from(&paths.base_prefix); - // Step 7: Calculate base_executable (if not already set by __PYVENV_LAUNCHER__) + // Step 7: Calculate base_executable (if not already set by an env override) if paths.base_executable.is_empty() { paths.base_executable = calculate_base_executable(executable.as_ref(), home_dir.as_ref()); } @@ -367,6 +365,19 @@ fn build_module_search_paths(settings: &Settings, prefix: &str, exec_prefix: &st paths } +/// `PYTHONEXECUTABLE` takes precedence over `__PYVENV_LAUNCHER__`. +/// An empty value is ignored. +fn env_executable_override() -> Option { + for name in ["PYTHONEXECUTABLE", "__PYVENV_LAUNCHER__"] { + if let Ok(value) = crate::host_env::os::var(name) + && !value.is_empty() + { + return Some(value); + } + } + None +} + /// Get the current executable path fn get_executable_path() -> Option { #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 30e7333d8..d45f927a2 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -18,7 +18,7 @@ pub use rustpython_host_env::posix::set_inheritable; pub mod module { use crate::{ AsObject, Py, PyObjectRef, PyResult, VirtualMachine, - builtins::{PyDictRef, PyInt, PyListRef, PyTupleRef, PyUtf8Str}, + builtins::{PyDictRef, PyInt, PyListRef, PyTuple, PyTupleRef, PyUtf8Str}, convert::{IntoPyException, ToPyException, ToPyObject, TryFromObject}, exceptions::OSErrorBuilder, function::{ArgMapping, Either, KwArgs, OptionalArg}, @@ -1464,7 +1464,7 @@ pub mod module { #[pyarg(named, default)] setsigmask: Option>, #[pyarg(named, default)] - scheduler: Option, + scheduler: Option, } #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] @@ -1476,6 +1476,58 @@ pub mod module { Dup2, } + #[cfg(all( + any(target_os = "linux", target_os = "freebsd", target_os = "android"), + not(target_env = "musl") + ))] + fn parse_posix_spawn_scheduler( + scheduler: Option, + vm: &VirtualMachine, + ) -> PyResult> { + let Some(obj) = scheduler else { + return Ok(None); + }; + if obj.is(&vm.ctx.none()) { + return Ok(None); + } + let Some(tuple) = obj.downcast_ref::() else { + return Err(vm.new_type_error("scheduler must be a tuple or None")); + }; + if tuple.len() != 2 { + return Err(vm.new_type_error("A scheduler tuple must have two elements")); + } + let policy = if tuple[0].is(&vm.ctx.none()) { + None + } else { + Some(i32::try_from_object(vm, tuple[0].clone())?) + }; + let param = super::posix_sched::convert_sched_param(&tuple[1], vm)?; + Ok(Some(rustpython_host_env::posix::PosixSpawnScheduler { + policy, + param, + })) + } + + #[cfg(any(target_os = "macos", target_env = "musl"))] + fn parse_posix_spawn_scheduler( + scheduler: Option, + vm: &VirtualMachine, + ) -> PyResult<()> { + let Some(obj) = scheduler else { + return Ok(()); + }; + if obj.is(&vm.ctx.none()) { + return Ok(()); + } + let Some(tuple) = obj.downcast_ref::() else { + return Err(vm.new_type_error("scheduler must be a tuple or None")); + }; + if tuple.len() != 2 { + return Err(vm.new_type_error("A scheduler tuple must have two elements")); + } + Err(vm.new_not_implemented_error("The scheduler option is not supported in this system.")) + } + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "macos"))] impl PosixSpawnArgs { fn spawn(self, spawnp: bool, vm: &VirtualMachine) -> PyResult { @@ -1555,13 +1607,13 @@ pub mod module { let setsigdef = self.setsigdef.map(&collect_signals).transpose()?; - if let Some(_scheduler) = self.scheduler { - // TODO: Implement scheduler parameter handling - // This requires platform-specific sched_param struct handling - return Err( - vm.new_not_implemented_error("scheduler parameter is not yet implemented") - ); - } + #[cfg(all( + any(target_os = "linux", target_os = "freebsd", target_os = "android"), + not(target_env = "musl") + ))] + let scheduler = parse_posix_spawn_scheduler(self.scheduler, vm)?; + #[cfg(any(target_os = "macos", target_env = "musl"))] + parse_posix_spawn_scheduler(self.scheduler, vm)?; if self.setsid && !rustpython_host_env::posix::supports_posix_spawn_setsid() { return Err(vm.new_not_implemented_error( @@ -1603,6 +1655,11 @@ pub mod module { setsid: self.setsid, setsigmask: setsigmask.as_deref(), spawnp, + #[cfg(all( + any(target_os = "linux", target_os = "freebsd", target_os = "android"), + not(target_env = "musl") + ))] + scheduler, }) .map_err(|err| OSErrorBuilder::with_filename(&err, self.path, vm)) } @@ -2460,7 +2517,10 @@ mod posix_sched { } #[cfg(not(target_env = "musl"))] - fn convert_sched_param(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + pub(super) fn convert_sched_param( + obj: &PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { use crate::{ builtins::{PyInt, PyTuple}, class::StaticType, diff --git a/crates/vm/src/vm/python_run.rs b/crates/vm/src/vm/python_run.rs index a1c2552ce..11db4b78b 100644 --- a/crates/vm/src/vm/python_run.rs +++ b/crates/vm/src/vm/python_run.rs @@ -175,8 +175,12 @@ mod file_run { let mut file = crate::host_env::fs::open(path)?; let mut buf = [0u8; 2]; - use std::io::Read; - if file.read(&mut buf)? != 2 { + use std::io::{Read, Seek, SeekFrom}; + let n = file.read(&mut buf)?; + // /dev/fd/N shares the file offset across every open of that path. + // Restore it so a later read of the same script still starts at 0. + let _ = file.seek(SeekFrom::Start(0)); + if n != 2 { return Ok(false); } diff --git a/src/lib.rs b/src/lib.rs index 33dd905bd..62ba14c2b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -361,6 +361,7 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { RunMode::Repl => Ok(()), }; let result = if is_repl || vm.state.config.settings.inspect { + warn_if_pyrepl_unavailable(vm); shell::run_shell(vm, scope) } else { res @@ -377,6 +378,23 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { result } +/// When the fancy REPL cannot start (for example `TERM=dumb`), print the same +/// warning `_pyrepl.main` would have printed, then keep the rustyline shell. +fn warn_if_pyrepl_unavailable(vm: &VirtualMachine) { + let _ = vm.run_simple_string( + r" +import os, sys +if not os.getenv('PYTHON_BASIC_REPL'): + try: + from _pyrepl.main import CAN_USE_PYREPL, FAIL_REASON + if not CAN_USE_PYREPL and FAIL_REASON: + print(FAIL_REASON, file=sys.stderr) + except Exception: + pass +", + ); +} + #[cfg(feature = "flame-it")] fn write_profile(settings: &Settings) -> Result<(), Box> { use std::{fs, io};