Align contextvars, CLI startup, and stat validation with CPython (#8657)

* Align contextvars, CLI startup, and stat validation with CPython

Validate Context and ContextVar arguments and mapping keys, hash names through the Python protocol, and preserve string contents without invoking __str__. Set the main module loader for command and stdin execution, report missing scripts with exit status 2, and reject file descriptors combined with follow_symlinks=False.

Enable the nine corresponding stdlib regression tests.

Assisted-by: Codex:gpt-6

* Move set_main_builtin_importer off the compiler feature

Define the method in vm_new.rs so builds without rustpython-compiler,
such as example_projects/barebone, can compile. Also set BuiltinImporter
on an existing __main__ when its loader is missing or None.

Assisted-by: Grok:4.6

Lib/test/test_cmd_line_script.py

@@ -153,13 +153,11 @@ class CmdLineTest(unittest.TestCase):
             print('Expected output: %r' % expected_msg)
         self.assertIn(expected_msg.encode('utf-8'), err)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_dash_c_loader(self):
         rc, out, err = assert_python_ok("-c", "print(__loader__)")
         expected = repr(importlib.machinery.BuiltinImporter).encode("utf-8")
         self.assertIn(expected, out)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_stdin_loader(self):
         # Unfortunately, there's no way to automatically test the fully
         # interactive REPL, since that code path only gets executed when
@@ -790,7 +788,6 @@ class CmdLineTest(unittest.TestCase):
             traceback_lines = stderr.decode().splitlines()
             self.assertIn("No module named script_pkg", traceback_lines[-1])
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_nonexisting_script(self):
         # bpo-34783: "./python script.py" must not crash
         # if the script file doesn't exist.

Lib/test/test_context.py

@@ -27,7 +27,6 @@ def isolated_context(func):
 
 
 class ContextTest(unittest.TestCase):
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_context_var_new_1(self):
         with self.assertRaisesRegex(TypeError, 'takes exactly 1'):
             contextvars.ContextVar()
@@ -85,7 +84,6 @@ class ContextTest(unittest.TestCase):
             class MyToken(contextvars.Token):
                 pass
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_context_new_1(self):
         with self.assertRaisesRegex(TypeError, 'any arguments'):
             contextvars.Context(1)
@@ -95,7 +93,6 @@ class ContextTest(unittest.TestCase):
             contextvars.Context(a=1)
         contextvars.Context(**{})
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON; AssertionError: TypeError not raised
     def test_context_new_unhashable_str_subclass(self):
         # gh-132002: it used to crash on unhashable str subtypes.
         class weird_str(str):
@@ -105,7 +102,6 @@ class ContextTest(unittest.TestCase):
         with self.assertRaisesRegex(TypeError, 'unhashable type'):
             contextvars.ContextVar(weird_str())
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_context_typerrors_1(self):
         ctx = contextvars.Context()
 
@@ -120,7 +116,6 @@ class ContextTest(unittest.TestCase):
         ctx = contextvars.copy_context()
         self.assertIsInstance(ctx, contextvars.Context)
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     def test_context_run_1(self):
         ctx = contextvars.Context()
 

Lib/test/test_posix.py

@@ -668,7 +668,6 @@ class PosixTester(unittest.TestCase):
         finally:
             fp.close()
 
-    @unittest.expectedFailure  # TODO: RUSTPYTHON
     @unittest.skipUnless(hasattr(posix, 'stat'),
                          'test needs posix.stat()')
     @unittest.skipUnless(os.stat in os.supports_follow_symlinks,

crates/stdlib/src/contextvars.rs

@@ -13,15 +13,14 @@ thread_local! {
 mod _contextvars {
     use crate::vm::{
         AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func,
-        builtins::{PyGenericAlias, PyList, PyStrRef, PyType, PyTypeRef},
-        class::PyClassDef,
+        builtins::{PyGenericAlias, PyList, PyStr, PyType, PyTypeRef},
         class::StaticType,
         common::{
             hash::PyHash,
             lock::{LazyLock, PyMutex},
             wtf8::Wtf8Buf,
         },
-        function::{ArgCallable, FuncArgs, OptionalArg},
+        function::{FuncArgs, OptionalArg},
         protocol::{PyMappingMethods, PySequenceMethods},
         types::{AsMapping, AsSequence, Constructor, Hashable, Iterable, Representable},
     };
@@ -154,17 +153,33 @@ mod _contextvars {
         }
     }
 
+    fn context_check_key_type<'a>(
+        key: &'a crate::vm::PyObject,
+        vm: &VirtualMachine,
+    ) -> PyResult<&'a Py<ContextVar>> {
+        match key.downcast_ref::<ContextVar>() {
+            Some(var) => Ok(var),
+            None => Err(vm.new_type_error(format!(
+                "a ContextVar key was expected, got {}",
+                key.repr(vm)?
+            ))),
+        }
+    }
+
     #[pyclass(with(Constructor, AsMapping, AsSequence, Iterable))]
     impl PyContext {
         #[pymethod]
-        fn run(
-            zelf: &Py<Self>,
-            callable: ArgCallable,
-            args: FuncArgs,
-            vm: &VirtualMachine,
-        ) -> PyResult {
+        fn run(zelf: &Py<Self>, args: FuncArgs, vm: &VirtualMachine) -> PyResult {
+            let (callable, rest) = args
+                .args
+                .split_first()
+                .ok_or_else(|| vm.new_type_error("run() missing 1 required positional argument"))?;
+            let rest = FuncArgs {
+                args: rest.to_vec(),
+                kwargs: args.kwargs,
+            };
             Self::enter(zelf, vm)?;
-            let result = callable.invoke(args, vm);
+            let result = callable.call(rest, vm);
             Self::exit(zelf, vm)?;
             result
         }
@@ -200,14 +215,16 @@ mod _contextvars {
         #[pymethod]
         fn get(
             &self,
-            key: PyRef<ContextVar>,
+            key: PyObjectRef,
             default: OptionalArg<PyObjectRef>,
-        ) -> Option<PyObjectRef> {
-            let found = self.get_inner(&key);
+            vm: &VirtualMachine,
+        ) -> PyResult<Option<PyObjectRef>> {
+            let key = context_check_key_type(&key, vm)?;
+            let found = self.get_inner(key);
             if found.is_some() {
-                found
+                Ok(found)
             } else {
-                default.into_option()
+                Ok(default.into_option())
             }
         }
 
@@ -236,9 +253,17 @@ mod _contextvars {
     }
 
     impl Constructor for PyContext {
-        type Args = ();
-        fn py_new(_cls: &Py<PyType>, _args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> {
-            Ok(Self::empty(vm))
+        type Args = FuncArgs;
+
+        fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult {
+            if !args.args.is_empty() || !args.kwargs.is_empty() {
+                return Err(vm.new_type_error("Context() does not accept any arguments"));
+            }
+            Self::empty(vm).into_ref_with_type(vm, cls).map(Into::into)
+        }
+
+        fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> {
+            unreachable!("use slot_new")
         }
     }
 
@@ -249,7 +274,7 @@ mod _contextvars {
                     PyContext::mapping_downcast(mapping).__len__()
                 )),
                 subscript: atomic_func!(|mapping, needle, vm| {
-                    let needle = needle.try_to_value(vm)?;
+                    let needle = context_check_key_type(needle, vm)?;
                     PyContext::mapping_downcast(mapping)
                         .get_inner(needle)
                         .ok_or_else(|| vm.new_key_error(needle.to_owned().into()))
@@ -264,7 +289,7 @@ mod _contextvars {
         fn as_sequence() -> &'static PySequenceMethods {
             static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods {
                 contains: atomic_func!(|seq, target, vm| {
-                    let target = target.try_to_value(vm)?;
+                    let target = context_check_key_type(target, vm)?;
                     Ok(PyContext::sequence_downcast(seq).contains(target))
                 }),
                 ..PySequenceMethods::NOT_IMPLEMENTED
@@ -355,10 +380,10 @@ mod _contextvars {
             drop(replaced);
         }
 
-        fn generate_hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyHash {
-            let name_hash = vm.state.hash_secret.hash_str(&zelf.name);
+        fn generate_hash(zelf: &Py<Self>, name_hash: PyHash) -> PyHash {
             let pointer_hash = crate::common::hash::hash_pointer(zelf.as_object().get_id());
-            pointer_hash ^ name_hash
+            let hash = pointer_hash ^ name_hash;
+            if hash == -1 { -2 } else { hash }
         }
     }
 
@@ -476,35 +501,46 @@ mod _contextvars {
         }
     }
 
-    #[derive(FromArgs)]
-    struct ContextVarOptions {
-        #[pyarg(positional)]
-        name: PyStrRef,
-        #[pyarg(any, optional)]
-        default: OptionalArg<PyObjectRef>,
-    }
-
     impl Constructor for ContextVar {
-        type Args = ContextVarOptions;
+        type Args = FuncArgs;
 
         fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult {
-            let args: Self::Args = args.bind_for(vm, Self::NAME)?;
+            let mut args = args;
+            if args.args.len() != 1 {
+                return Err(vm.new_type_error(format!(
+                    "ContextVar() takes exactly 1 argument ({} given)",
+                    args.args.len()
+                )));
+            }
+            let default = args.take_keyword("default");
+            if let Some((name, _)) = args.kwargs.first() {
+                return Err(vm.new_type_error(format!(
+                    "ContextVar() got an unexpected keyword argument '{name}'"
+                )));
+            }
+
+            let name = args.args.swap_remove(0);
+            let name = name
+                .downcast::<PyStr>()
+                .map_err(|_| vm.new_type_error("context variable name must be a str"))?;
+            let name_hash = name.as_object().hash(vm)?;
+            let name = name.to_string();
+
             let var = Self {
-                name: args.name.to_string(),
-                default: args.default.into_option(),
+                name,
+                default,
                 cached_id: 0.into(),
                 cached: PyMutex::new(None),
                 hash: AtomicI64::new(0),
             };
             let py_var = var.into_ref_with_type(vm, cls)?;
-
-            let hash = Self::generate_hash(&py_var, vm);
+            let hash = Self::generate_hash(&py_var, name_hash);
             py_var.hash.store(hash, Ordering::Relaxed);
             Ok(py_var.into())
         }
 
         fn py_new(_cls: &Py<PyType>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<Self> {
-            unimplemented!("use slot_new")
+            unreachable!("use slot_new")
         }
     }
 

crates/vm/src/stdlib/os.rs

@@ -1421,6 +1421,9 @@ pub(super) mod _os {
         follow_symlinks: FollowSymlinks,
         vm: &VirtualMachine,
     ) -> PyResult {
+        if matches!(file, OsPathOrFd::Fd(_)) && !follow_symlinks.0 {
+            return Err(vm.new_value_error("stat: cannot use fd and follow_symlinks together"));
+        }
         let stat = stat_inner(file.clone(), dir_fd, follow_symlinks)
             .map_err(|err| OSErrorBuilder::with_filename(&err, file, vm))?
             .ok_or_else(|| crate::exceptions::nul_char_error(vm))?;

crates/vm/src/vm/vm_new.rs

@@ -342,6 +342,7 @@ impl VirtualMachine {
             main_module.into(),
             self,
         )?;
+        self.set_main_builtin_importer(&scope.globals)?;
 
         Ok(scope)
     }
@@ -349,15 +350,37 @@ impl VirtualMachine {
     /// Create `__main__` if it is missing and return it.
     pub fn ensure_main_module(&self) -> PyResult<PyRef<PyModule>> {
         let sys_modules = self.sys_module.get_attr("modules", self)?;
-        if let Ok(existing) = sys_modules.get_item("__main__", self)
+        let module = if let Ok(existing) = sys_modules.get_item("__main__", self)
             && let Ok(module) = existing.downcast::<PyModule>()
         {
-            return Ok(module);
+            module
+        } else {
+            let dict = self.ctx.new_dict();
+            let main_module = self.new_module("__main__", dict, None);
+            sys_modules.set_item("__main__", main_module.clone().into(), self)?;
+            main_module
+        };
+        self.set_main_builtin_importer(&module.dict())?;
+        Ok(module)
+    }
+
+    /// Set `__main__.__loader__` to BuiltinImporter when it is missing or None.
+    pub fn set_main_builtin_importer(
+        &self,
+        module_dict: &Py<crate::builtins::PyDict>,
+    ) -> PyResult<()> {
+        if let Ok(loader) = module_dict.get_item("__loader__", self)
+            && !self.is_none(&loader)
+        {
+            return Ok(());
         }
-        let dict = self.ctx.new_dict();
-        let main_module = self.new_module("__main__", dict, None);
-        sys_modules.set_item("__main__", main_module.clone().into(), self)?;
-        Ok(main_module)
+        let sys_modules = self.sys_module.get_attr("modules", self)?;
+        let Ok(importlib) = sys_modules.get_item("_frozen_importlib", self) else {
+            return Ok(());
+        };
+        let loader = importlib.get_attr("BuiltinImporter", self)?;
+        module_dict.set_item("__loader__", loader, self)?;
+        Ok(())
     }
 
     /// `__dict__` of this interpreter's `__main__` module.

src/lib.rs

@@ -236,19 +236,32 @@ fn run_file(vm: &VirtualMachine, scope: Scope, argv0: &str) -> PyResult<()> {
     }
 
     cfg_select! {
-        feature = "host_env" => vm.run_any_file(scope, path),
+        feature = "host_env" => {
+            match rustpython_vm::host_env::fs::metadata(path) {
+                Ok(_) => vm.run_any_file(scope, path),
+                Err(err) => cant_open_file(vm, path, &err),
+            }
+        }
         _ => {
             // In sandbox mode, the binary reads the file and feeds source to the VM.
             // The VM itself has no filesystem access.
             let path = if path.is_empty() { "???" } else { path };
             match std::fs::read_to_string(path) {
                 Ok(source) => vm.run_string(scope, &source, path).map(drop),
-                Err(err) => Err(vm.new_os_error(err.to_string())),
+                Err(err) => cant_open_file(vm, path, &err),
             }
         }
     }
 }
 
+fn cant_open_file(vm: &VirtualMachine, path: &str, err: &std::io::Error) -> PyResult<()> {
+    let program = &vm.state.config.paths.executable;
+    let filename_repr = vm.ctx.new_str(path).as_object().repr(vm)?;
+    let errno = err.raw_os_error().unwrap_or(2);
+    eprintln!("{program}: can't open file {filename_repr}: [Errno {errno}] {err}");
+    Err(vm.new_system_exit(vec![vm.ctx.new_int(2).into()].into()))
+}
+
 fn get_importer(path: &str, vm: &VirtualMachine) -> PyResult<Option<PyObjectRef>> {
     use rustpython_vm::builtins::PyDictRef;
     use rustpython_vm::convert::TryFromObject;