diff --git a/Cargo.lock b/Cargo.lock index 4ee04733e..1d1fd5fb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -993,9 +993,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.107" +version = "0.2.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbe5e23404da5b4f555ef85ebed98fb4083e55a00c317800bc2a50ede9f3d219" +checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" [[package]] name = "libffi" diff --git a/Lib/os.py b/Lib/os.py index 253cad1a5..2bc0979b3 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -28,13 +28,15 @@ import stat as st from _collections_abc import _check_methods +GenericAlias = type(list[int]) + _names = sys.builtin_module_names # Note: more names are added to __all__ later. __all__ = ["altsep", "curdir", "pardir", "sep", "pathsep", "linesep", "defpath", "name", "path", "devnull", "SEEK_SET", "SEEK_CUR", "SEEK_END", "fsencode", "fsdecode", "get_exec_path", "fdopen", - "popen", "extsep"] + "extsep"] def _exists(name): return name in globals() @@ -336,7 +338,10 @@ def walk(top, topdown=True, onerror=None, followlinks=False): dirs.remove('CVS') # don't visit CVS directories """ - top = fspath(top) + sys.audit("os.walk", top, topdown, onerror, followlinks) + return _walk(fspath(top), topdown, onerror, followlinks) + +def _walk(top, topdown, onerror, followlinks): dirs = [] nondirs = [] walk_dirs = [] @@ -410,11 +415,11 @@ def walk(top, topdown=True, onerror=None, followlinks=False): # the caller can replace the directory entry during the "yield" # above. if followlinks or not islink(new_path): - yield from walk(new_path, topdown, onerror, followlinks) + yield from _walk(new_path, topdown, onerror, followlinks) else: # Recurse into sub-directories for new_path in walk_dirs: - yield from walk(new_path, topdown, onerror, followlinks) + yield from _walk(new_path, topdown, onerror, followlinks) # Yield after recursion if going bottom up yield top, dirs, nondirs @@ -455,8 +460,8 @@ if {open, stat} <= supports_dir_fd and {scandir, stat} <= supports_fd: if 'CVS' in dirs: dirs.remove('CVS') # don't visit CVS directories """ - if not isinstance(top, int) or not hasattr(top, '__index__'): - top = fspath(top) + sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd) + top = fspath(top) # Note: To guard against symlink races, we use the standard # lstat()/open()/fstat() trick. if not follow_symlinks: @@ -654,17 +659,15 @@ def get_exec_path(env=None): return path_list.split(pathsep) -# Change environ to automatically call putenv(), unsetenv if they exist. -from _collections_abc import MutableMapping +# Change environ to automatically call putenv() and unsetenv() +from _collections_abc import MutableMapping, Mapping class _Environ(MutableMapping): - def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue, putenv, unsetenv): + def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue): self.encodekey = encodekey self.decodekey = decodekey self.encodevalue = encodevalue self.decodevalue = decodevalue - self.putenv = putenv - self.unsetenv = unsetenv self._data = data def __getitem__(self, key): @@ -678,12 +681,12 @@ class _Environ(MutableMapping): def __setitem__(self, key, value): key = self.encodekey(key) value = self.encodevalue(value) - self.putenv(key, value) + putenv(key, value) self._data[key] = value def __delitem__(self, key): encodedkey = self.encodekey(key) - self.unsetenv(encodedkey) + unsetenv(encodedkey) try: del self._data[encodedkey] except KeyError: @@ -700,9 +703,11 @@ class _Environ(MutableMapping): return len(self._data) def __repr__(self): - return 'environ({{{}}})'.format(', '.join( - ('{!r}: {!r}'.format(self.decodekey(key), self.decodevalue(value)) - for key, value in self._data.items()))) + formatted_items = ", ".join( + f"{self.decodekey(key)!r}: {self.decodevalue(value)!r}" + for key, value in self._data.items() + ) + return f"environ({{{formatted_items}}})" def copy(self): return dict(self) @@ -712,21 +717,23 @@ class _Environ(MutableMapping): self[key] = value return self[key] -try: - _putenv = putenv -except NameError: - _putenv = lambda key, value: None -else: - if "putenv" not in __all__: - __all__.append("putenv") + def __ior__(self, other): + self.update(other) + return self -try: - _unsetenv = unsetenv -except NameError: - _unsetenv = lambda key: _putenv(key, "") -else: - if "unsetenv" not in __all__: - __all__.append("unsetenv") + def __or__(self, other): + if not isinstance(other, Mapping): + return NotImplemented + new = dict(self) + new.update(other) + return new + + def __ror__(self, other): + if not isinstance(other, Mapping): + return NotImplemented + new = dict(other) + new.update(self) + return new def _createenviron(): if name == 'nt': @@ -755,8 +762,7 @@ def _createenviron(): data = environ return _Environ(data, encodekey, decode, - encode, decode, - _putenv, _unsetenv) + encode, decode) # unicode environ environ = _createenviron() @@ -781,8 +787,7 @@ if supports_bytes_environ: # bytes environ environb = _Environ(environ._data, _check_bytes, bytes, - _check_bytes, bytes, - _putenv, _unsetenv) + _check_bytes, bytes) del _check_bytes def getenvb(key, default=None): @@ -862,12 +867,8 @@ if _exists("fork") and not _exists("spawnv") and _exists("execv"): wpid, sts = waitpid(pid, 0) if WIFSTOPPED(sts): continue - elif WIFSIGNALED(sts): - return -WTERMSIG(sts) - elif WIFEXITED(sts): - return WEXITSTATUS(sts) - else: - raise OSError("Not stopped, signaled or exited???") + + return waitstatus_to_exitcode(sts) def spawnv(mode, file, args): """spawnv(mode, file, args) -> integer @@ -969,28 +970,30 @@ otherwise return -SIG, where SIG is the signal that killed it. """ __all__.extend(["spawnlp", "spawnlpe"]) - +# VxWorks has no user space shell provided. As a result, running +# command in a shell can't be supported. +if sys.platform != 'vxworks': # Supply os.popen() -def popen(cmd, mode="r", buffering=-1): - if not isinstance(cmd, str): - raise TypeError("invalid cmd type (%s, expected string)" % type(cmd)) - if mode not in ("r", "w"): - raise ValueError("invalid mode %r" % mode) - if buffering == 0 or buffering is None: - raise ValueError("popen() does not support unbuffered streams") - import subprocess, io - if mode == "r": - proc = subprocess.Popen(cmd, - shell=True, - stdout=subprocess.PIPE, - bufsize=buffering) - return _wrap_close(io.TextIOWrapper(proc.stdout), proc) - else: - proc = subprocess.Popen(cmd, - shell=True, - stdin=subprocess.PIPE, - bufsize=buffering) - return _wrap_close(io.TextIOWrapper(proc.stdin), proc) + def popen(cmd, mode="r", buffering=-1): + if not isinstance(cmd, str): + raise TypeError("invalid cmd type (%s, expected string)" % type(cmd)) + if mode not in ("r", "w"): + raise ValueError("invalid mode %r" % mode) + if buffering == 0 or buffering is None: + raise ValueError("popen() does not support unbuffered streams") + import subprocess, io + if mode == "r": + proc = subprocess.Popen(cmd, + shell=True, text=True, + stdout=subprocess.PIPE, + bufsize=buffering) + return _wrap_close(proc.stdout, proc) + else: + proc = subprocess.Popen(cmd, + shell=True, text=True, + stdin=subprocess.PIPE, + bufsize=buffering) + return _wrap_close(proc.stdin, proc) # Helper for popen() -- a proxy for a file whose close waits for the process class _wrap_close: @@ -1015,12 +1018,18 @@ class _wrap_close: def __iter__(self): return iter(self._stream) + __all__.append("popen") + # Supply os.fdopen() -def fdopen(fd, *args, **kwargs): +def fdopen(fd, mode="r", buffering=-1, encoding=None, *args, **kwargs): if not isinstance(fd, int): raise TypeError("invalid fd type (%s, expected integer)" % type(fd)) import io - return io.open(fd, *args, **kwargs) + if "b" not in mode: + # TODO: RUSTPYTHON (module 'io' has no attribute 'text_encoding') + # encoding = io.text_encoding(encoding) + pass + return io.open(fd, mode, buffering, encoding, *args, **kwargs) # For testing purposes, make sure the function is available when the C @@ -1076,6 +1085,8 @@ class PathLike(abc.ABC): return _check_methods(subclass, '__fspath__') return NotImplemented + __class_getitem__ = classmethod(GenericAlias) + if name == 'nt': class _AddedDllDirectory: diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index fbdb078f8..5efcd77db 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1080,7 +1080,8 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol): @unittest.skipUnless(os.supports_bytes_environ, "os.environb required for this test.") - # TODO: RUSTPYTHON (KeyError: 'surrogateescape') + # TODO: RUSTPYTHON (UnicodeDecodeError: can't decode bytes for utf-8) + # Need to fix 'surrogateescape' @unittest.expectedFailure def test_environb(self): # os.environ -> os.environb @@ -1185,8 +1186,6 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol): self.assertEqual(expected, value) - # TODO: RUSTPYTHON (KeyError: 'surrogateescape') - @unittest.expectedFailure def test_or_operator(self): overridden_key = '_TEST_VAR_' original_value = 'original_value' @@ -1206,8 +1205,6 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol): self._test_underlying_process_env('_A_', '') self._test_underlying_process_env(overridden_key, original_value) - # TODO: RUSTPYTHON (KeyError: 'surrogateescape') - @unittest.expectedFailure def test_ior_operator(self): overridden_key = '_TEST_VAR_' os.environ[overridden_key] = 'original_value' @@ -1236,8 +1233,6 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol): # Check nothing was added. self.assertEqual(os_environ_copy, os.environ) - # TODO: RUSTPYTHON (KeyError: 'surrogateescape') - @unittest.expectedFailure def test_ior_operator_key_value_iterable(self): overridden_key = '_TEST_VAR_' os.environ[overridden_key] = 'original_value' @@ -1253,8 +1248,6 @@ class EnvironTests(mapping_tests.BasicTestMappingProtocol): self._test_underlying_process_env('_A_', '1') self._test_underlying_process_env(overridden_key, '3') - # TODO: RUSTPYTHON (KeyError: 'surrogateescape') - @unittest.expectedFailure def test_ror_operator(self): overridden_key = '_TEST_VAR_' original_value = 'original_value' @@ -4544,8 +4537,6 @@ class TestPEP519(unittest.TestCase): self.assertFalse(issubclass(FakePath, A)) self.assertTrue(issubclass(FakePath, os.PathLike)) - # TODO: RUSTPYTHON (TypeError: 'ABCMeta' object is not subscriptable) - @unittest.expectedFailure def test_pathlike_class_getitem(self): self.assertIsInstance(os.PathLike[bytes], types.GenericAlias) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index c93ff8547..58f5e51ef 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4360,8 +4360,6 @@ class IOTests(BaseTestCase): a = stuff.__annotations__['a'] self.assertEqual(a.__parameters__, ()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_io_submodule(self): from typing.io import IO, TextIO, BinaryIO, __all__, __name__ self.assertIs(IO, typing.IO) diff --git a/Lib/typing.py b/Lib/typing.py index c74be9a51..842eaeaa6 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2628,10 +2628,8 @@ class io: TextIO = TextIO BinaryIO = BinaryIO - -# TODO: RUSTPYTHON -# io.__name__ = __name__ + '.io' -# sys.modules[io.__name__] = io +io.__name__ = __name__ + '.io' +sys.modules[io.__name__] = io Pattern = _alias(stdlib_re.Pattern, 1) Match = _alias(stdlib_re.Match, 1) diff --git a/stdlib/src/fcntl.rs b/stdlib/src/fcntl.rs index 5a5e89502..1aec065f2 100644 --- a/stdlib/src/fcntl.rs +++ b/stdlib/src/fcntl.rs @@ -21,6 +21,14 @@ mod fcntl { #[pyattr] use libc::{F_GETOWN, F_RDLCK, F_SETOWN, F_UNLCK, F_WRLCK}; + #[cfg(target_vendor = "apple")] + #[pyattr] + use libc::F_NOCACHE; + + #[cfg(target_os = "freebsd")] + #[pyattr] + use libc::{F_DUP2FD, F_DUP2FD_CLOEXEC}; + #[pyfunction] fn fcntl( io::Fildes(fd): io::Fildes, diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 3989b5648..22a6ad985 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -48,7 +48,7 @@ hexf-parse = "0.2.1" indexmap = "1.0.2" ahash = "0.7.2" bitflags = "1.3" -libc = "0.2.99" +libc = "0.2.107" nix = "0.23" paste = "1.0.5" is-macro = "0.1" diff --git a/vm/src/builtins/pytype.rs b/vm/src/builtins/pytype.rs index f1b4e536c..3b1580601 100644 --- a/vm/src/builtins/pytype.rs +++ b/vm/src/builtins/pytype.rs @@ -286,7 +286,12 @@ impl PyType { pub fn name(&self) -> BorrowedValue { PyRwLockReadGuard::map(self.slots.name.read(), |slot_name| { - slot_name.as_ref().unwrap().rsplit('.').next().unwrap() + let name = slot_name.as_ref().unwrap(); + if self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + name.as_str() + } else { + name.rsplit('.').next().unwrap() + } }) .into() } @@ -543,6 +548,26 @@ impl PyType { )) } + #[pyproperty(magic, setter)] + fn set_name(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + if !self.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) { + return Err(vm.new_type_error(format!( + "cannot set '{}' attribute of immutable type '{}'", + "__name__", + self.name() + ))); + } + let name = value.downcast_ref::().ok_or_else(|| { + vm.new_type_error(format!( + "can only assign string to {}.__name__, not '{}'", + self.name(), + value.class().name() + )) + })?; + *self.slots.name.write() = Some(name.as_str().to_string()); + Ok(()) + } + #[pyproperty(magic)] fn text_signature(&self) -> Option { self.slots diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index a2174f114..c312a7459 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -3602,6 +3602,18 @@ mod _io { .unwrap() } + #[pyfunction] + fn text_encoding( + encoding: PyObjectRef, + _stacklevel: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + if vm.is_none(&encoding) { + return Ok(vm.ctx.new_str("locale")); + } + encoding.try_into_value(vm) + } + #[cfg(test)] mod tests { use super::*; diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index 7e973a480..81fae3272 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -1980,6 +1980,11 @@ pub mod module { Ok(vm.ctx.new_int(written as u64).into()) } + #[cfg(target_os = "linux")] + unsafe fn sys_getrandom(buf: *mut libc::c_void, buflen: usize, flags: u32) -> isize { + libc::syscall(libc::SYS_getrandom, buf, buflen, flags as usize) as _ + } + #[cfg(target_os = "linux")] #[pyfunction] fn getrandom(size: isize, flags: OptionalArg, vm: &VirtualMachine) -> PyResult> { @@ -1987,7 +1992,7 @@ pub mod module { .map_err(|_| vm.new_os_error(format!("Invalid argument for size: {}", size)))?; let mut buf = Vec::with_capacity(size); unsafe { - let len = libc::getrandom( + let len = sys_getrandom( buf.as_mut_ptr() as *mut libc::c_void, size, flags.unwrap_or(0), diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index f097d3221..940f2197c 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -28,6 +28,7 @@ mod time { naive::{NaiveDate, NaiveDateTime, NaiveTime}, Datelike, Timelike, }; + use std::time::Duration; #[allow(dead_code)] pub(super) const SEC_TO_MS: i64 = 1000; @@ -46,7 +47,7 @@ mod time { #[allow(dead_code)] pub(super) const NS_TO_US: i64 = 1000; - fn duration_since_system_now(vm: &VirtualMachine) -> PyResult { + fn duration_since_system_now(vm: &VirtualMachine) -> PyResult { use std::time::{SystemTime, UNIX_EPOCH}; SystemTime::now() @@ -56,19 +57,19 @@ mod time { // TODO: implement proper monotonic time for wasm/wasi. #[cfg(not(any(unix, windows)))] - fn get_monotonic_time(vm: &VirtualMachine) -> PyResult { + fn get_monotonic_time(vm: &VirtualMachine) -> PyResult { duration_since_system_now(vm) } // TODO: implement proper perf time for wasm/wasi. #[cfg(not(any(unix, windows)))] - fn get_perf_time(vm: &VirtualMachine) -> PyResult { + fn get_perf_time(vm: &VirtualMachine) -> PyResult { duration_since_system_now(vm) } #[cfg(not(unix))] #[pyfunction] - fn sleep(dur: std::time::Duration) { + fn sleep(dur: Duration) { std::thread::sleep(dur); } @@ -233,7 +234,7 @@ mod time { target_os = "fuchsia", target_os = "emscripten", )))] - fn get_thread_time(vm: &VirtualMachine) -> PyResult { + fn get_thread_time(vm: &VirtualMachine) -> PyResult { Err(vm.new_not_implemented_error("thread time unsupported in this system".to_owned())) } @@ -247,7 +248,7 @@ mod time { Ok(get_thread_time(vm)?.as_nanos() as u64) } - #[cfg(any(windows, all(target_arch = "wasm32", not(target_os = "unknown"))))] + #[cfg(any(windows, all(target_arch = "wasm32", target_arch = "emscripten")))] pub(super) fn time_muldiv(ticks: i64, mul: i64, div: i64) -> u64 { let intpart = ticks / div; let ticks = ticks % div; @@ -255,8 +256,8 @@ mod time { (intpart * mul + remaining) as u64 } - #[cfg(all(target_arch = "wasm32", not(target_os = "unknown")))] - fn get_process_time(vm: &VirtualMachine) -> PyResult { + #[cfg(all(target_arch = "wasm32", target_os = "emscripten"))] + fn get_process_time(vm: &VirtualMachine) -> PyResult { let t: libc::tms = unsafe { let mut t = std::mem::MaybeUninit::uninit(); if libc::times(t.as_mut_ptr()) == -1 { @@ -264,17 +265,26 @@ mod time { } t.assume_init() }; - - #[cfg(target_os = "wasi")] - let freq = 60; - #[cfg(not(target_os = "wasi"))] let freq = unsafe { libc::sysconf(libc::_SC_CLK_TCK) }; - Ok(std::time::Duration::from_nanos( + Ok(Duration::from_nanos( time_muldiv(t.tms_utime, SEC_TO_NS, freq) + time_muldiv(t.tms_stime, SEC_TO_NS, freq), )) } + // same as the get_process_time impl for most unixes + #[cfg(all(target_arch = "wasm32", target_os = "wasi"))] + pub(super) fn get_process_time(vm: &VirtualMachine) -> PyResult { + let time: libc::timespec = unsafe { + let mut time = std::mem::MaybeUninit::uninit(); + if libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, time.as_mut_ptr()) == -1 { + return Err(vm.new_os_error("Failed to get clock time".to_owned())); + } + time.assume_init() + }; + Ok(Duration::new(time.tv_sec as u64, time.tv_nsec as u32)) + } + #[cfg(not(any( windows, target_os = "macos", @@ -289,7 +299,7 @@ mod time { target_os = "redox", all(target_arch = "wasm32", not(target_os = "unknown")) )))] - fn get_process_time(vm: &VirtualMachine) -> PyResult { + fn get_process_time(vm: &VirtualMachine) -> PyResult { Err(vm.new_not_implemented_error("process time unsupported in this system".to_owned())) }