impl path_converter and os functions (#6484)

* os.setpgrp

* tcgetpgrp

* impl more os functions

* impl PathConverter

Lib/test/test_os.py

@@ -1725,15 +1725,15 @@ class BytesWalkTests(WalkTests):
             bdirs[:] = list(map(os.fsencode, dirs))
             bfiles[:] = list(map(os.fsencode, files))
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON; (TypeError: Can't mix strings and bytes in path components)
+    @unittest.expectedFailure # TODO: RUSTPYTHON; WalkTests doesn't have these methods
     def test_compare_to_walk(self):
         return super().test_compare_to_walk()
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON; (TypeError: Can't mix strings and bytes in path components)
+    @unittest.expectedFailure # TODO: RUSTPYTHON; WalkTests doesn't have these methods
     def test_dir_fd(self):
         return super().test_dir_fd()
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON; (TypeError: Can't mix strings and bytes in path components)
+    @unittest.expectedFailure # TODO: RUSTPYTHON; WalkTests doesn't have these methods
     def test_yields_correct_dir_fd(self):
         return super().test_yields_correct_dir_fd()
 
@@ -4502,7 +4502,6 @@ class OSErrorTests(unittest.TestCase):
 
         self.filenames = self.bytes_filenames + self.unicode_filenames
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON; (AssertionError: b'@test_22106_tmp\xe7w\xf0' is not b'@test_22106_tmp\xe7w\xf0' : <built-in function chdir>)
     def test_oserror_filename(self):
         funcs = [
             (self.filenames, os.chdir,),
@@ -4906,7 +4905,6 @@ class TestDirEntry(unittest.TestCase):
     def test_uninstantiable(self):
         self.assertRaises(TypeError, os.DirEntry)
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON; (pickle.PicklingError: Can't pickle <class '_os.DirEntry'>: it's not found as _os.DirEntry)
     def test_unpickable(self):
         filename = create_file(os.path.join(self.path, "file.txt"), b'python')
         entry = [entry for entry in os.scandir(self.path)].pop()
@@ -5337,7 +5335,6 @@ class TestPEP519(unittest.TestCase):
                 return ''
         self.assertFalse(hasattr(A(), '__dict__'))
 
-    @unittest.expectedFailure # TODO: RUSTPYTHON
     def test_fspath_set_to_None(self):
         class Foo:
             __fspath__ = None

crates/vm/src/function/fspath.rs

@@ -7,6 +7,7 @@ use crate::{
 };
 use std::{borrow::Cow, ffi::OsStr, path::PathBuf};
 
+/// Helper to implement os.fspath()
 #[derive(Clone)]
 pub enum FsPath {
     Str(PyStrRef),
@@ -27,7 +28,7 @@ impl FsPath {
         )
     }
 
-    // PyOS_FSPath in CPython
+    // PyOS_FSPath
     pub fn try_from(
         obj: PyObjectRef,
         check_for_nul: bool,

crates/vm/src/ospath.rs

@@ -2,20 +2,181 @@ use rustpython_common::crt_fd;
 
 use crate::{
     PyObjectRef, PyResult, VirtualMachine,
+    builtins::{PyBytes, PyStr},
     convert::{IntoPyException, ToPyException, ToPyObject, TryFromObject},
     function::FsPath,
 };
 use std::path::{Path, PathBuf};
 
-// path_ without allow_fd in CPython
+/// path_converter
+#[derive(Clone, Copy, Default)]
+pub struct PathConverter {
+    /// Function name for error messages (e.g., "rename")
+    pub function_name: Option<&'static str>,
+    /// Argument name for error messages (e.g., "src", "dst")
+    pub argument_name: Option<&'static str>,
+    /// If true, embedded null characters are allowed
+    pub non_strict: bool,
+}
+
+impl PathConverter {
+    pub const fn new() -> Self {
+        Self {
+            function_name: None,
+            argument_name: None,
+            non_strict: false,
+        }
+    }
+
+    pub const fn function(mut self, name: &'static str) -> Self {
+        self.function_name = Some(name);
+        self
+    }
+
+    pub const fn argument(mut self, name: &'static str) -> Self {
+        self.argument_name = Some(name);
+        self
+    }
+
+    pub const fn non_strict(mut self) -> Self {
+        self.non_strict = true;
+        self
+    }
+
+    /// Generate error message prefix like "rename: "
+    fn error_prefix(&self) -> String {
+        match self.function_name {
+            Some(func) => format!("{}: ", func),
+            None => String::new(),
+        }
+    }
+
+    /// Get argument name for error messages, defaults to "path"
+    fn arg_name(&self) -> &'static str {
+        self.argument_name.unwrap_or("path")
+    }
+
+    /// Format a type error message
+    fn type_error_msg(&self, type_name: &str, allow_fd: bool) -> String {
+        let expected = if allow_fd {
+            "string, bytes, os.PathLike or integer"
+        } else {
+            "string, bytes or os.PathLike"
+        };
+        format!(
+            "{}{} should be {}, not {}",
+            self.error_prefix(),
+            self.arg_name(),
+            expected,
+            type_name
+        )
+    }
+
+    /// Convert to OsPathOrFd (path or file descriptor)
+    pub(crate) fn try_path_or_fd<'fd>(
+        &self,
+        obj: PyObjectRef,
+        vm: &VirtualMachine,
+    ) -> PyResult<OsPathOrFd<'fd>> {
+        // Handle fd (before __fspath__ check, like CPython)
+        if let Some(int) = obj.try_index_opt(vm) {
+            let fd = int?.try_to_primitive(vm)?;
+            return unsafe { crt_fd::Borrowed::try_borrow_raw(fd) }
+                .map(OsPathOrFd::Fd)
+                .map_err(|e| e.into_pyexception(vm));
+        }
+
+        self.try_path_inner(obj, true, vm).map(OsPathOrFd::Path)
+    }
+
+    /// Convert to OsPath only (no fd support)
+    fn try_path_inner(
+        &self,
+        obj: PyObjectRef,
+        allow_fd: bool,
+        vm: &VirtualMachine,
+    ) -> PyResult<OsPath> {
+        // Try direct str/bytes match
+        let obj = match self.try_match_str_bytes(obj.clone(), vm)? {
+            Ok(path) => return Ok(path),
+            Err(obj) => obj,
+        };
+
+        // Call __fspath__
+        let type_error_msg = || self.type_error_msg(&obj.class().name(), allow_fd);
+        let method =
+            vm.get_method_or_type_error(obj.clone(), identifier!(vm, __fspath__), type_error_msg)?;
+        if vm.is_none(&method) {
+            return Err(vm.new_type_error(type_error_msg()));
+        }
+        let result = method.call((), vm)?;
+
+        // Match __fspath__ result
+        self.try_match_str_bytes(result.clone(), vm)?.map_err(|_| {
+            vm.new_type_error(format!(
+                "{}expected {}.__fspath__() to return str or bytes, not {}",
+                self.error_prefix(),
+                obj.class().name(),
+                result.class().name(),
+            ))
+        })
+    }
+
+    /// Try to match str or bytes, returns Err(obj) if neither
+    fn try_match_str_bytes(
+        &self,
+        obj: PyObjectRef,
+        vm: &VirtualMachine,
+    ) -> PyResult<Result<OsPath, PyObjectRef>> {
+        let check_nul = |b: &[u8]| {
+            if self.non_strict || memchr::memchr(b'\0', b).is_none() {
+                Ok(())
+            } else {
+                Err(vm.new_value_error(format!(
+                    "{}embedded null character in {}",
+                    self.error_prefix(),
+                    self.arg_name()
+                )))
+            }
+        };
+
+        match_class!(match obj {
+            s @ PyStr => {
+                check_nul(s.as_bytes())?;
+                let path = vm.fsencode(&s)?.into_owned();
+                Ok(Ok(OsPath {
+                    path,
+                    origin: Some(s.into()),
+                }))
+            }
+            b @ PyBytes => {
+                check_nul(&b)?;
+                let path = FsPath::bytes_as_os_str(&b, vm)?.to_owned();
+                Ok(Ok(OsPath {
+                    path,
+                    origin: Some(b.into()),
+                }))
+            }
+            obj => Ok(Err(obj)),
+        })
+    }
+
+    /// Convert to OsPath directly
+    pub fn try_path(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<OsPath> {
+        self.try_path_inner(obj, false, vm)
+    }
+}
+
+/// path_t output - the converted path
 #[derive(Clone)]
 pub struct OsPath {
     pub path: std::ffi::OsString,
-    pub(super) mode: OutputMode,
+    /// Original Python object for identity preservation in OSError
+    pub(super) origin: Option<PyObjectRef>,
 }
 
 #[derive(Debug, Copy, Clone)]
-pub(super) enum OutputMode {
+pub enum OutputMode {
     String,
     Bytes,
 }
@@ -38,19 +199,19 @@ impl OutputMode {
 impl OsPath {
     pub fn new_str(path: impl Into<std::ffi::OsString>) -> Self {
         let path = path.into();
-        Self {
-            path,
-            mode: OutputMode::String,
-        }
+        Self { path, origin: None }
     }
 
     pub(crate) fn from_fspath(fspath: FsPath, vm: &VirtualMachine) -> PyResult<Self> {
         let path = fspath.as_os_str(vm)?.into_owned();
-        let mode = match fspath {
-            FsPath::Str(_) => OutputMode::String,
-            FsPath::Bytes(_) => OutputMode::Bytes,
+        let origin = match fspath {
+            FsPath::Str(s) => s.into(),
+            FsPath::Bytes(b) => b.into(),
         };
-        Ok(Self { path, mode })
+        Ok(Self {
+            path,
+            origin: Some(origin),
+        })
     }
 
     /// Convert an object to OsPath using the os.fspath-style error message.
@@ -83,7 +244,20 @@ impl OsPath {
     }
 
     pub fn filename(&self, vm: &VirtualMachine) -> PyObjectRef {
-        self.mode.process_path(self.path.clone(), vm)
+        if let Some(ref origin) = self.origin {
+            origin.clone()
+        } else {
+            // Default to string when no origin (e.g., from new_str)
+            OutputMode::String.process_path(self.path.clone(), vm)
+        }
+    }
+
+    /// Get the output mode based on origin type (bytes -> Bytes, otherwise -> String)
+    pub fn mode(&self) -> OutputMode {
+        match &self.origin {
+            Some(obj) if obj.downcast_ref::<PyBytes>().is_some() => OutputMode::Bytes,
+            _ => OutputMode::String,
+        }
     }
 }
 
@@ -94,15 +268,8 @@ impl AsRef<Path> for OsPath {
 }
 
 impl TryFromObject for OsPath {
-    // TODO: path_converter with allow_fd=0 in CPython
     fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
-        let fspath = FsPath::try_from(
-            obj,
-            true,
-            "should be string, bytes, os.PathLike or integer",
-            vm,
-        )?;
-        Self::from_fspath(fspath, vm)
+        PathConverter::new().try_path(obj, vm)
     }
 }
 
@@ -115,15 +282,7 @@ pub(crate) enum OsPathOrFd<'fd> {
 
 impl TryFromObject for OsPathOrFd<'_> {
     fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
-        match obj.try_index_opt(vm) {
-            Some(int) => {
-                let fd = int?.try_to_primitive(vm)?;
-                unsafe { crt_fd::Borrowed::try_borrow_raw(fd) }
-                    .map(Self::Fd)
-                    .map_err(|e| e.into_pyexception(vm))
-            }
-            None => obj.try_into_value(vm).map(Self::Path),
-        }
+        PathConverter::new().try_path_or_fd(obj, vm)
     }
 }
 

crates/vm/src/stdlib/nt.rs

@@ -17,6 +17,7 @@ pub(crate) mod module {
         builtins::{PyBaseExceptionRef, PyDictRef, PyListRef, PyStrRef, PyTupleRef},
         common::{crt_fd, suppress_iph, windows::ToWideString},
         convert::ToPyException,
+        exceptions::OSErrorBuilder,
         function::{Either, OptionalArg},
         ospath::{OsPath, OsPathOrFd},
         stdlib::os::{_os, DirFd, SupportFunc, TargetIsDirectory},
@@ -193,10 +194,12 @@ pub(crate) mod module {
         };
 
         // Use symlink_metadata to avoid following dangling symlinks
-        let meta = fs::symlink_metadata(&actual_path).map_err(|err| err.to_pyexception(vm))?;
+        let meta = fs::symlink_metadata(&actual_path)
+            .map_err(|err| OSErrorBuilder::with_filename(&err, path.clone(), vm))?;
         let mut permissions = meta.permissions();
         permissions.set_readonly(mode & S_IWRITE == 0);
-        fs::set_permissions(&*actual_path, permissions).map_err(|err| err.to_pyexception(vm))
+        fs::set_permissions(&*actual_path, permissions)
+            .map_err(|err| OSErrorBuilder::with_filename(&err, path, vm))
     }
 
     /// Get the real file name (with correct case) without accessing the file.
@@ -925,7 +928,7 @@ pub(crate) mod module {
             .as_ref()
             .canonicalize()
             .map_err(|e| e.to_pyexception(vm))?;
-        Ok(path.mode.process_path(real, vm))
+        Ok(path.mode().process_path(real, vm))
     }
 
     #[pyfunction]
@@ -958,7 +961,7 @@ pub(crate) mod module {
             }
         }
         let buffer = widestring::WideCString::from_vec_truncate(buffer);
-        Ok(path.mode.process_path(buffer.to_os_string(), vm))
+        Ok(path.mode().process_path(buffer.to_os_string(), vm))
     }
 
     #[pyfunction]
@@ -973,7 +976,7 @@ pub(crate) mod module {
             return Err(vm.new_last_os_error());
         }
         let buffer = widestring::WideCString::from_vec_truncate(buffer);
-        Ok(path.mode.process_path(buffer.to_os_string(), vm))
+        Ok(path.mode().process_path(buffer.to_os_string(), vm))
     }
 
     /// Implements _Py_skiproot logic for Windows paths
@@ -1053,7 +1056,7 @@ pub(crate) mod module {
         use crate::builtins::{PyBytes, PyStr};
         use rustpython_common::wtf8::Wtf8Buf;
 
-        // Handle path-like objects via os.fspath, but without null check (nonstrict=True)
+        // Handle path-like objects via os.fspath, but without null check (non_strict=True)
         let path = if let Some(fspath) = vm.get_method(path.clone(), identifier!(vm, __fspath__)) {
             fspath?.call((), vm)?
         } else {
@@ -1585,7 +1588,7 @@ pub(crate) mod module {
         use windows_sys::Win32::System::IO::DeviceIoControl;
         use windows_sys::Win32::System::Ioctl::FSCTL_GET_REPARSE_POINT;
 
-        let mode = path.mode;
+        let mode = path.mode();
         let wide_path = path.as_ref().to_wide_with_nul();
 
         // Open the file/directory with reparse point flag
@@ -1602,7 +1605,11 @@ pub(crate) mod module {
         };
 
         if handle == INVALID_HANDLE_VALUE {
-            return Err(io::Error::last_os_error().to_pyexception(vm));
+            return Err(OSErrorBuilder::with_filename(
+                &io::Error::last_os_error(),
+                path.clone(),
+                vm,
+            ));
         }
 
         // Buffer for reparse data - MAXIMUM_REPARSE_DATA_BUFFER_SIZE is 16384
@@ -1626,7 +1633,11 @@ pub(crate) mod module {
         unsafe { CloseHandle(handle) };
 
         if result == 0 {
-            return Err(io::Error::last_os_error().to_pyexception(vm));
+            return Err(OSErrorBuilder::with_filename(
+                &io::Error::last_os_error(),
+                path.clone(),
+                vm,
+            ));
         }
 
         // Parse the reparse data buffer

crates/vm/src/stdlib/os.rs

@@ -164,7 +164,7 @@ pub(super) mod _os {
         convert::{IntoPyException, ToPyObject},
         exceptions::OSErrorBuilder,
         function::{ArgBytesLike, FsPath, FuncArgs, OptionalArg},
-        ospath::{OsPath, OsPathOrFd, OutputMode},
+        ospath::{OsPath, OsPathOrFd, OutputMode, PathConverter},
         protocol::PyIterReturn,
         recursion::ReprGuard,
         types::{IterNext, Iterable, PyStructSequence, Representable, SelfIter},
@@ -366,10 +366,12 @@ pub(super) mod _os {
 
     #[pyfunction]
     fn listdir(
-        path: OptionalArg<OsPathOrFd<'_>>,
+        path: OptionalArg<Option<OsPathOrFd<'_>>>,
         vm: &VirtualMachine,
     ) -> PyResult<Vec<PyObjectRef>> {
-        let path = path.unwrap_or_else(|| OsPathOrFd::Path(OsPath::new_str(".")));
+        let path = path
+            .flatten()
+            .unwrap_or_else(|| OsPathOrFd::Path(OsPath::new_str(".")));
         let list = match path {
             OsPathOrFd::Path(path) => {
                 let dir_iter = match fs::read_dir(&path) {
@@ -378,9 +380,10 @@ pub(super) mod _os {
                         return Err(OSErrorBuilder::with_filename(&err, path, vm));
                     }
                 };
+                let mode = path.mode();
                 dir_iter
                     .map(|entry| match entry {
-                        Ok(entry_path) => Ok(path.mode.process_path(entry_path.file_name(), vm)),
+                        Ok(entry_path) => Ok(mode.process_path(entry_path.file_name(), vm)),
                         Err(err) => Err(OSErrorBuilder::with_filename(&err, path.clone(), vm)),
                     })
                     .collect::<PyResult<_>>()?
@@ -545,7 +548,7 @@ pub(super) mod _os {
 
     #[pyfunction]
     fn readlink(path: OsPath, dir_fd: DirFd<'_, 0>, vm: &VirtualMachine) -> PyResult {
-        let mode = path.mode;
+        let mode = path.mode();
         let [] = dir_fd.0;
         let path =
             fs::read_link(&path).map_err(|err| OSErrorBuilder::with_filename(&err, path, vm))?;
@@ -640,7 +643,7 @@ pub(super) mod _os {
                 stat(
                     OsPath {
                         path: self.pathval.as_os_str().to_owned(),
-                        mode: OutputMode::String,
+                        origin: None,
                     }
                     .into(),
                     dir_fd,
@@ -671,11 +674,7 @@ pub(super) mod _os {
                 Some(ino) => Ok(ino),
                 None => {
                     let stat = stat_inner(
-                        OsPath {
-                            path: self.pathval.as_os_str().to_owned(),
-                            mode: OutputMode::String,
-                        }
-                        .into(),
+                        OsPath::new_str(self.pathval.as_os_str()).into(),
                         DirFd::default(),
                         FollowSymlinks(false),
                     )
@@ -730,6 +729,11 @@ pub(super) mod _os {
         ) -> PyGenericAlias {
             PyGenericAlias::from_args(cls, args, vm)
         }
+
+        #[pymethod]
+        fn __reduce__(&self, vm: &VirtualMachine) -> PyResult {
+            Err(vm.new_type_error("cannot pickle 'DirEntry' object".to_owned()))
+        }
     }
 
     impl Representable for DirEntry {
@@ -785,6 +789,11 @@ pub(super) mod _os {
         fn __exit__(zelf: PyRef<Self>, _args: FuncArgs) {
             zelf.close()
         }
+
+        #[pymethod]
+        fn __reduce__(&self, vm: &VirtualMachine) -> PyResult {
+            Err(vm.new_type_error("cannot pickle 'ScandirIterator' object".to_owned()))
+        }
     }
     impl SelfIter for ScandirIterator {}
     impl IterNext for ScandirIterator {
@@ -861,7 +870,7 @@ pub(super) mod _os {
             .map_err(|err| OSErrorBuilder::with_filename(&err, path.clone(), vm))?;
         Ok(ScandirIterator {
             entries: PyRwLock::new(Some(entries)),
-            mode: path.mode,
+            mode: path.mode(),
         }
         .into_ref(&vm.ctx)
         .into())
@@ -1124,7 +1133,16 @@ pub(super) mod _os {
 
     #[pyfunction]
     #[pyfunction(name = "replace")]
-    fn rename(src: OsPath, dst: OsPath, vm: &VirtualMachine) -> PyResult<()> {
+    fn rename(src: PyObjectRef, dst: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        let src = PathConverter::new()
+            .function("rename")
+            .argument("src")
+            .try_path(src, vm)?;
+        let dst = PathConverter::new()
+            .function("rename")
+            .argument("dst")
+            .try_path(dst, vm)?;
+
         fs::rename(&src.path, &dst.path).map_err(|err| {
             let builder = err.to_os_error_builder(vm);
             let builder = builder.filename(src.filename(vm));
@@ -1708,6 +1726,8 @@ pub(super) mod _os {
             SupportFunc::new("fstat", Some(true), Some(STAT_DIR_FD), Some(true)),
             SupportFunc::new("symlink", Some(false), Some(SYMLINK_DIR_FD), Some(false)),
             SupportFunc::new("truncate", Some(true), Some(false), Some(false)),
+            SupportFunc::new("ftruncate", Some(true), Some(false), Some(false)),
+            SupportFunc::new("fsync", Some(true), Some(false), Some(false)),
             SupportFunc::new(
                 "utime",
                 Some(false),

crates/vm/src/stdlib/posix.rs

@@ -1240,6 +1240,12 @@ pub mod module {
             .map_err(|err| err.into_pyexception(vm))
     }
 
+    #[pyfunction]
+    fn setpgrp(vm: &VirtualMachine) -> PyResult<()> {
+        // setpgrp() is equivalent to setpgid(0, 0)
+        unistd::setpgid(Pid::from_raw(0), Pid::from_raw(0)).map_err(|err| err.into_pyexception(vm))
+    }
+
     #[cfg(not(any(target_os = "wasi", target_os = "redox")))]
     #[pyfunction]
     fn setsid(vm: &VirtualMachine) -> PyResult<()> {
@@ -1248,6 +1254,24 @@ pub mod module {
             .map_err(|err| err.into_pyexception(vm))
     }
 
+    #[cfg(not(any(target_os = "wasi", target_os = "redox")))]
+    #[pyfunction]
+    fn tcgetpgrp(fd: i32, vm: &VirtualMachine) -> PyResult<libc::pid_t> {
+        use std::os::fd::BorrowedFd;
+        let fd = unsafe { BorrowedFd::borrow_raw(fd) };
+        unistd::tcgetpgrp(fd)
+            .map(|pid| pid.as_raw())
+            .map_err(|err| err.into_pyexception(vm))
+    }
+
+    #[cfg(not(any(target_os = "wasi", target_os = "redox")))]
+    #[pyfunction]
+    fn tcsetpgrp(fd: i32, pgid: libc::pid_t, vm: &VirtualMachine) -> PyResult<()> {
+        use std::os::fd::BorrowedFd;
+        let fd = unsafe { BorrowedFd::borrow_raw(fd) };
+        unistd::tcsetpgrp(fd, Pid::from_raw(pgid)).map_err(|err| err.into_pyexception(vm))
+    }
+
     fn try_from_id(vm: &VirtualMachine, obj: PyObjectRef, typ_name: &str) -> PyResult<u32> {
         use std::cmp::Ordering;
         let i = obj
@@ -1833,6 +1857,8 @@ pub mod module {
             SupportFunc::new("umask", Some(false), Some(false), Some(false)),
             SupportFunc::new("execv", None, None, None),
             SupportFunc::new("pathconf", Some(true), None, None),
+            SupportFunc::new("fpathconf", Some(true), None, None),
+            SupportFunc::new("fchdir", Some(true), None, None),
         ]
     }