Merge branch 'main' into update-types-310

Cargo.lock

@@ -993,9 +993,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.107"
+version = "0.2.112"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbe5e23404da5b4f555ef85ebed98fb4083e55a00c317800bc2a50ede9f3d219"
+checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125"
 
 [[package]]
 name = "libffi"

Lib/os.py

@@ -28,13 +28,15 @@ import stat as st
 
 from _collections_abc import _check_methods
 
+GenericAlias = type(list[int])
+
 _names = sys.builtin_module_names
 
 # Note:  more names are added to __all__ later.
 __all__ = ["altsep", "curdir", "pardir", "sep", "pathsep", "linesep",
            "defpath", "name", "path", "devnull", "SEEK_SET", "SEEK_CUR",
            "SEEK_END", "fsencode", "fsdecode", "get_exec_path", "fdopen",
-           "popen", "extsep"]
+           "extsep"]
 
 def _exists(name):
     return name in globals()
@@ -336,7 +338,10 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
             dirs.remove('CVS')  # don't visit CVS directories
 
     """
-    top = fspath(top)
+    sys.audit("os.walk", top, topdown, onerror, followlinks)
+    return _walk(fspath(top), topdown, onerror, followlinks)
+
+def _walk(top, topdown, onerror, followlinks):
     dirs = []
     nondirs = []
     walk_dirs = []
@@ -410,11 +415,11 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
             # the caller can replace the directory entry during the "yield"
             # above.
             if followlinks or not islink(new_path):
-                yield from walk(new_path, topdown, onerror, followlinks)
+                yield from _walk(new_path, topdown, onerror, followlinks)
     else:
         # Recurse into sub-directories
         for new_path in walk_dirs:
-            yield from walk(new_path, topdown, onerror, followlinks)
+            yield from _walk(new_path, topdown, onerror, followlinks)
         # Yield after recursion if going bottom up
         yield top, dirs, nondirs
 
@@ -455,8 +460,8 @@ if {open, stat} <= supports_dir_fd and {scandir, stat} <= supports_fd:
             if 'CVS' in dirs:
                 dirs.remove('CVS')  # don't visit CVS directories
         """
-        if not isinstance(top, int) or not hasattr(top, '__index__'):
-            top = fspath(top)
+        sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd)
+        top = fspath(top)
         # Note: To guard against symlink races, we use the standard
         # lstat()/open()/fstat() trick.
         if not follow_symlinks:
@@ -654,17 +659,15 @@ def get_exec_path(env=None):
     return path_list.split(pathsep)
 
 
-# Change environ to automatically call putenv(), unsetenv if they exist.
-from _collections_abc import MutableMapping
+# Change environ to automatically call putenv() and unsetenv()
+from _collections_abc import MutableMapping, Mapping
 
 class _Environ(MutableMapping):
-    def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue, putenv, unsetenv):
+    def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue):
         self.encodekey = encodekey
         self.decodekey = decodekey
         self.encodevalue = encodevalue
         self.decodevalue = decodevalue
-        self.putenv = putenv
-        self.unsetenv = unsetenv
         self._data = data
 
     def __getitem__(self, key):
@@ -678,12 +681,12 @@ class _Environ(MutableMapping):
     def __setitem__(self, key, value):
         key = self.encodekey(key)
         value = self.encodevalue(value)
-        self.putenv(key, value)
+        putenv(key, value)
         self._data[key] = value
 
     def __delitem__(self, key):
         encodedkey = self.encodekey(key)
-        self.unsetenv(encodedkey)
+        unsetenv(encodedkey)
         try:
             del self._data[encodedkey]
         except KeyError:
@@ -700,9 +703,11 @@ class _Environ(MutableMapping):
         return len(self._data)
 
     def __repr__(self):
-        return 'environ({{{}}})'.format(', '.join(
-            ('{!r}: {!r}'.format(self.decodekey(key), self.decodevalue(value))
-            for key, value in self._data.items())))
+        formatted_items = ", ".join(
+            f"{self.decodekey(key)!r}: {self.decodevalue(value)!r}"
+            for key, value in self._data.items()
+        )
+        return f"environ({{{formatted_items}}})"
 
     def copy(self):
         return dict(self)
@@ -712,21 +717,23 @@ class _Environ(MutableMapping):
             self[key] = value
         return self[key]
 
-try:
-    _putenv = putenv
-except NameError:
-    _putenv = lambda key, value: None
-else:
-    if "putenv" not in __all__:
-        __all__.append("putenv")
+    def __ior__(self, other):
+        self.update(other)
+        return self
 
-try:
-    _unsetenv = unsetenv
-except NameError:
-    _unsetenv = lambda key: _putenv(key, "")
-else:
-    if "unsetenv" not in __all__:
-        __all__.append("unsetenv")
+    def __or__(self, other):
+        if not isinstance(other, Mapping):
+            return NotImplemented
+        new = dict(self)
+        new.update(other)
+        return new
+
+    def __ror__(self, other):
+        if not isinstance(other, Mapping):
+            return NotImplemented
+        new = dict(other)
+        new.update(self)
+        return new
 
 def _createenviron():
     if name == 'nt':
@@ -755,8 +762,7 @@ def _createenviron():
         data = environ
     return _Environ(data,
         encodekey, decode,
-        encode, decode,
-        _putenv, _unsetenv)
+        encode, decode)
 
 # unicode environ
 environ = _createenviron()
@@ -781,8 +787,7 @@ if supports_bytes_environ:
     # bytes environ
     environb = _Environ(environ._data,
         _check_bytes, bytes,
-        _check_bytes, bytes,
-        _putenv, _unsetenv)
+        _check_bytes, bytes)
     del _check_bytes
 
     def getenvb(key, default=None):
@@ -862,12 +867,8 @@ if _exists("fork") and not _exists("spawnv") and _exists("execv"):
                 wpid, sts = waitpid(pid, 0)
                 if WIFSTOPPED(sts):
                     continue
-                elif WIFSIGNALED(sts):
-                    return -WTERMSIG(sts)
-                elif WIFEXITED(sts):
-                    return WEXITSTATUS(sts)
-                else:
-                    raise OSError("Not stopped, signaled or exited???")
+
+                return waitstatus_to_exitcode(sts)
 
     def spawnv(mode, file, args):
         """spawnv(mode, file, args) -> integer
@@ -969,28 +970,30 @@ otherwise return -SIG, where SIG is the signal that killed it. """
 
     __all__.extend(["spawnlp", "spawnlpe"])
 
-
+# VxWorks has no user space shell provided. As a result, running
+# command in a shell can't be supported.
+if sys.platform != 'vxworks':
 # Supply os.popen()
-def popen(cmd, mode="r", buffering=-1):
-    if not isinstance(cmd, str):
-        raise TypeError("invalid cmd type (%s, expected string)" % type(cmd))
-    if mode not in ("r", "w"):
-        raise ValueError("invalid mode %r" % mode)
-    if buffering == 0 or buffering is None:
-        raise ValueError("popen() does not support unbuffered streams")
-    import subprocess, io
-    if mode == "r":
-        proc = subprocess.Popen(cmd,
-                                shell=True,
-                                stdout=subprocess.PIPE,
-                                bufsize=buffering)
-        return _wrap_close(io.TextIOWrapper(proc.stdout), proc)
-    else:
-        proc = subprocess.Popen(cmd,
-                                shell=True,
-                                stdin=subprocess.PIPE,
-                                bufsize=buffering)
-        return _wrap_close(io.TextIOWrapper(proc.stdin), proc)
+    def popen(cmd, mode="r", buffering=-1):
+        if not isinstance(cmd, str):
+            raise TypeError("invalid cmd type (%s, expected string)" % type(cmd))
+        if mode not in ("r", "w"):
+            raise ValueError("invalid mode %r" % mode)
+        if buffering == 0 or buffering is None:
+            raise ValueError("popen() does not support unbuffered streams")
+        import subprocess, io
+        if mode == "r":
+            proc = subprocess.Popen(cmd,
+                                    shell=True, text=True,
+                                    stdout=subprocess.PIPE,
+                                    bufsize=buffering)
+            return _wrap_close(proc.stdout, proc)
+        else:
+            proc = subprocess.Popen(cmd,
+                                    shell=True, text=True,
+                                    stdin=subprocess.PIPE,
+                                    bufsize=buffering)
+            return _wrap_close(proc.stdin, proc)
 
 # Helper for popen() -- a proxy for a file whose close waits for the process
 class _wrap_close:
@@ -1015,12 +1018,18 @@ class _wrap_close:
     def __iter__(self):
         return iter(self._stream)
 
+    __all__.append("popen")
+
 # Supply os.fdopen()
-def fdopen(fd, *args, **kwargs):
+def fdopen(fd, mode="r", buffering=-1, encoding=None, *args, **kwargs):
     if not isinstance(fd, int):
         raise TypeError("invalid fd type (%s, expected integer)" % type(fd))
     import io
-    return io.open(fd, *args, **kwargs)
+    if "b" not in mode:
+        # TODO: RUSTPYTHON (module 'io' has no attribute 'text_encoding')
+        # encoding = io.text_encoding(encoding)
+        pass
+    return io.open(fd, mode, buffering, encoding, *args, **kwargs)
 
 
 # For testing purposes, make sure the function is available when the C
@@ -1076,6 +1085,8 @@ class PathLike(abc.ABC):
             return _check_methods(subclass, '__fspath__')
         return NotImplemented
 
+    __class_getitem__ = classmethod(GenericAlias)
+
 
 if name == 'nt':
     class _AddedDllDirectory:

Lib/test/test_os.py

@@ -1080,7 +1080,8 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol):
 
     @unittest.skipUnless(os.supports_bytes_environ,
                          "os.environb required for this test.")
-    # TODO: RUSTPYTHON (KeyError: 'surrogateescape')
+    # TODO: RUSTPYTHON (UnicodeDecodeError: can't decode bytes for utf-8)
+    # Need to fix 'surrogateescape'
     @unittest.expectedFailure
     def test_environb(self):
         # os.environ -> os.environb
@@ -1185,8 +1186,6 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol):
 
         self.assertEqual(expected, value)
 
-    # TODO: RUSTPYTHON (KeyError: 'surrogateescape')
-    @unittest.expectedFailure
     def test_or_operator(self):
         overridden_key = '_TEST_VAR_'
         original_value = 'original_value'
@@ -1206,8 +1205,6 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol):
         self._test_underlying_process_env('_A_', '')
         self._test_underlying_process_env(overridden_key, original_value)
 
-    # TODO: RUSTPYTHON (KeyError: 'surrogateescape')
-    @unittest.expectedFailure
     def test_ior_operator(self):
         overridden_key = '_TEST_VAR_'
         os.environ[overridden_key] = 'original_value'
@@ -1236,8 +1233,6 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol):
         # Check nothing was added.
         self.assertEqual(os_environ_copy, os.environ)
 
-    # TODO: RUSTPYTHON (KeyError: 'surrogateescape')
-    @unittest.expectedFailure
     def test_ior_operator_key_value_iterable(self):
         overridden_key = '_TEST_VAR_'
         os.environ[overridden_key] = 'original_value'
@@ -1253,8 +1248,6 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol):
         self._test_underlying_process_env('_A_', '1')
         self._test_underlying_process_env(overridden_key, '3')
 
-    # TODO: RUSTPYTHON (KeyError: 'surrogateescape')
-    @unittest.expectedFailure
     def test_ror_operator(self):
         overridden_key = '_TEST_VAR_'
         original_value = 'original_value'
@@ -4544,8 +4537,6 @@ class TestPEP519(unittest.TestCase):
         self.assertFalse(issubclass(FakePath, A))
         self.assertTrue(issubclass(FakePath, os.PathLike))
 
-    # TODO: RUSTPYTHON (TypeError: 'ABCMeta' object is not subscriptable)
-    @unittest.expectedFailure
     def test_pathlike_class_getitem(self):
         self.assertIsInstance(os.PathLike[bytes], types.GenericAlias)
 

Lib/test/test_typing.py

@@ -4360,8 +4360,6 @@ class IOTests(BaseTestCase):
         a = stuff.__annotations__['a']
         self.assertEqual(a.__parameters__, ())
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_io_submodule(self):
         from typing.io import IO, TextIO, BinaryIO, __all__, __name__
         self.assertIs(IO, typing.IO)

Lib/typing.py

@@ -2628,10 +2628,8 @@ class io:
     TextIO = TextIO
     BinaryIO = BinaryIO
 
-
-# TODO: RUSTPYTHON
-# io.__name__ = __name__ + '.io'
-# sys.modules[io.__name__] = io
+io.__name__ = __name__ + '.io'
+sys.modules[io.__name__] = io
 
 Pattern = _alias(stdlib_re.Pattern, 1)
 Match = _alias(stdlib_re.Match, 1)

stdlib/src/fcntl.rs

@@ -21,6 +21,14 @@ mod fcntl {
     #[pyattr]
     use libc::{F_GETOWN, F_RDLCK, F_SETOWN, F_UNLCK, F_WRLCK};
 
+    #[cfg(target_vendor = "apple")]
+    #[pyattr]
+    use libc::F_NOCACHE;
+
+    #[cfg(target_os = "freebsd")]
+    #[pyattr]
+    use libc::{F_DUP2FD, F_DUP2FD_CLOEXEC};
+
     #[pyfunction]
     fn fcntl(
         io::Fildes(fd): io::Fildes,

vm/Cargo.toml

@@ -48,7 +48,7 @@ hexf-parse = "0.2.1"
 indexmap = "1.0.2"
 ahash = "0.7.2"
 bitflags = "1.3"
-libc = "0.2.99"
+libc = "0.2.107"
 nix = "0.23"
 paste = "1.0.5"
 is-macro = "0.1"

vm/src/builtins/pytype.rs

@@ -286,7 +286,12 @@ impl PyType {
 
     pub fn name(&self) -> BorrowedValue<str> {
         PyRwLockReadGuard::map(self.slots.name.read(), |slot_name| {
-            slot_name.as_ref().unwrap().rsplit('.').next().unwrap()
+            let name = slot_name.as_ref().unwrap();
+            if self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) {
+                name.as_str()
+            } else {
+                name.rsplit('.').next().unwrap()
+            }
         })
         .into()
     }
@@ -543,6 +548,26 @@ impl PyType {
         ))
     }
 
+    #[pyproperty(magic, setter)]
+    fn set_name(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {
+        if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) {
+            return Err(vm.new_type_error(format!(
+                "cannot set '{}' attribute of immutable type '{}'",
+                "__name__",
+                self.name()
+            )));
+        }
+        let name = value.downcast_ref::<PyStr>().ok_or_else(|| {
+            vm.new_type_error(format!(
+                "can only assign string to {}.__name__, not '{}'",
+                self.name(),
+                value.class().name()
+            ))
+        })?;
+        *self.slots.name.write() = Some(name.as_str().to_string());
+        Ok(())
+    }
+
     #[pyproperty(magic)]
     fn text_signature(&self) -> Option<String> {
         self.slots

vm/src/stdlib/io.rs

@@ -3602,6 +3602,18 @@ mod _io {
         .unwrap()
     }
 
+    #[pyfunction]
+    fn text_encoding(
+        encoding: PyObjectRef,
+        _stacklevel: OptionalArg<i32>,
+        vm: &VirtualMachine,
+    ) -> PyResult<PyStrRef> {
+        if vm.is_none(&encoding) {
+            return Ok(vm.ctx.new_str("locale"));
+        }
+        encoding.try_into_value(vm)
+    }
+
     #[cfg(test)]
     mod tests {
         use super::*;

vm/src/stdlib/posix.rs

@@ -1980,6 +1980,11 @@ pub mod module {
         Ok(vm.ctx.new_int(written as u64).into())
     }
 
+    #[cfg(target_os = "linux")]
+    unsafe fn sys_getrandom(buf: *mut libc::c_void, buflen: usize, flags: u32) -> isize {
+        libc::syscall(libc::SYS_getrandom, buf, buflen, flags as usize) as _
+    }
+
     #[cfg(target_os = "linux")]
     #[pyfunction]
     fn getrandom(size: isize, flags: OptionalArg<u32>, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
@@ -1987,7 +1992,7 @@ pub mod module {
             .map_err(|_| vm.new_os_error(format!("Invalid argument for size: {}", size)))?;
         let mut buf = Vec::with_capacity(size);
         unsafe {
-            let len = libc::getrandom(
+            let len = sys_getrandom(
                 buf.as_mut_ptr() as *mut libc::c_void,
                 size,
                 flags.unwrap_or(0),

vm/src/stdlib/time.rs

@@ -28,6 +28,7 @@ mod time {
         naive::{NaiveDate, NaiveDateTime, NaiveTime},
         Datelike, Timelike,
     };
+    use std::time::Duration;
 
     #[allow(dead_code)]
     pub(super) const SEC_TO_MS: i64 = 1000;
@@ -46,7 +47,7 @@ mod time {
     #[allow(dead_code)]
     pub(super) const NS_TO_US: i64 = 1000;
 
-    fn duration_since_system_now(vm: &VirtualMachine) -> PyResult<std::time::Duration> {
+    fn duration_since_system_now(vm: &VirtualMachine) -> PyResult<Duration> {
         use std::time::{SystemTime, UNIX_EPOCH};
 
         SystemTime::now()
@@ -56,19 +57,19 @@ mod time {
 
     // TODO: implement proper monotonic time for wasm/wasi.
     #[cfg(not(any(unix, windows)))]
-    fn get_monotonic_time(vm: &VirtualMachine) -> PyResult<std::time::Duration> {
+    fn get_monotonic_time(vm: &VirtualMachine) -> PyResult<Duration> {
         duration_since_system_now(vm)
     }
 
     // TODO: implement proper perf time for wasm/wasi.
     #[cfg(not(any(unix, windows)))]
-    fn get_perf_time(vm: &VirtualMachine) -> PyResult<std::time::Duration> {
+    fn get_perf_time(vm: &VirtualMachine) -> PyResult<Duration> {
         duration_since_system_now(vm)
     }
 
     #[cfg(not(unix))]
     #[pyfunction]
-    fn sleep(dur: std::time::Duration) {
+    fn sleep(dur: Duration) {
         std::thread::sleep(dur);
     }
 
@@ -233,7 +234,7 @@ mod time {
         target_os = "fuchsia",
         target_os = "emscripten",
     )))]
-    fn get_thread_time(vm: &VirtualMachine) -> PyResult<std::time::Duration> {
+    fn get_thread_time(vm: &VirtualMachine) -> PyResult<Duration> {
         Err(vm.new_not_implemented_error("thread time unsupported in this system".to_owned()))
     }
 
@@ -247,7 +248,7 @@ mod time {
         Ok(get_thread_time(vm)?.as_nanos() as u64)
     }
 
-    #[cfg(any(windows, all(target_arch = "wasm32", not(target_os = "unknown"))))]
+    #[cfg(any(windows, all(target_arch = "wasm32", target_arch = "emscripten")))]
     pub(super) fn time_muldiv(ticks: i64, mul: i64, div: i64) -> u64 {
         let intpart = ticks / div;
         let ticks = ticks % div;
@@ -255,8 +256,8 @@ mod time {
         (intpart * mul + remaining) as u64
     }
 
-    #[cfg(all(target_arch = "wasm32", not(target_os = "unknown")))]
-    fn get_process_time(vm: &VirtualMachine) -> PyResult<std::time::Duration> {
+    #[cfg(all(target_arch = "wasm32", target_os = "emscripten"))]
+    fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> {
         let t: libc::tms = unsafe {
             let mut t = std::mem::MaybeUninit::uninit();
             if libc::times(t.as_mut_ptr()) == -1 {
@@ -264,17 +265,26 @@ mod time {
             }
             t.assume_init()
         };
-
-        #[cfg(target_os = "wasi")]
-        let freq = 60;
-        #[cfg(not(target_os = "wasi"))]
         let freq = unsafe { libc::sysconf(libc::_SC_CLK_TCK) };
 
-        Ok(std::time::Duration::from_nanos(
+        Ok(Duration::from_nanos(
             time_muldiv(t.tms_utime, SEC_TO_NS, freq) + time_muldiv(t.tms_stime, SEC_TO_NS, freq),
         ))
     }
 
+    // same as the get_process_time impl for most unixes
+    #[cfg(all(target_arch = "wasm32", target_os = "wasi"))]
+    pub(super) fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> {
+        let time: libc::timespec = unsafe {
+            let mut time = std::mem::MaybeUninit::uninit();
+            if libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, time.as_mut_ptr()) == -1 {
+                return Err(vm.new_os_error("Failed to get clock time".to_owned()));
+            }
+            time.assume_init()
+        };
+        Ok(Duration::new(time.tv_sec as u64, time.tv_nsec as u32))
+    }
+
     #[cfg(not(any(
         windows,
         target_os = "macos",
@@ -289,7 +299,7 @@ mod time {
         target_os = "redox",
         all(target_arch = "wasm32", not(target_os = "unknown"))
     )))]
-    fn get_process_time(vm: &VirtualMachine) -> PyResult<std::time::Duration> {
+    fn get_process_time(vm: &VirtualMachine) -> PyResult<Duration> {
         Err(vm.new_not_implemented_error("process time unsupported in this system".to_owned()))
     }