diff --git a/Cargo.lock b/Cargo.lock index 4f5a39109..fed711f0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3876,6 +3876,7 @@ dependencies = [ "rustls", "rustls-native-certs", "rustls-pemfile", + "rustls-pki-types", "rustls-webpki 0.103.13", "rustpython-wtf8", "rustyline", @@ -4028,6 +4029,7 @@ dependencies = [ "csv-core", "flame", "foreign-types-shared", + "getrandom 0.4.3", "hex", "indexmap", "insta", diff --git a/Cargo.toml b/Cargo.toml index a9ffbb892..529643ae5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/fre jit = ["rustpython-vm/jit"] threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"] sqlite = ["rustpython-stdlib/sqlite"] -ssl = ["host_env"] +ssl = ["host_env", "rustpython-stdlib?/ssl"] ssl-rustls = ["ssl", "rustpython-stdlib/ssl-rustls"] ssl-rustls-aws-lc = ["ssl-rustls", "dep:rustls", "rustls/aws_lc_rs"] ssl-rustls-aws-lc-fips = ["ssl-rustls-aws-lc", "rustls/fips"] diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 5bee24cc7..4bebd6e83 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1183,7 +1183,6 @@ class ThreadTests(BaseTestCase): self.assertEqual(out.strip(), b"OK") self.assertIn(b"can't create new thread at interpreter shutdown", err) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_join_daemon_thread_in_finalization(self): # gh-123940: Py_Finalize() prevents other threads from running Python # code, so join() can not succeed unless the thread is already done. diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 3e220539d..e310d3d30 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -208,6 +208,7 @@ impl NormalizedParseDiagnostic { /// column. These are reconstructed by re-scanning after ruff's parse has already failed, so they /// carry CPython's wording rather than a translation of ruff's own error, and they never reach /// ruff — `NormalizedParseDiagnostic` and `CompileError` are the only things that consume one. +#[derive(Clone)] struct CpythonDiagnostic { message: String, range: ruff_text_size::TextRange, @@ -243,6 +244,52 @@ impl CpythonDiagnostic { } } +/// Lexer-class failures outrank print hints. Decode and f-string diagnostics +/// compete with lexer failures by offset, and an unterminated quote is only +/// a fallback when no decode/f-string diagnostic exists. Print is considered +/// against the final winner: a later 0x still beats print, but an f-string +/// that replaced that 0x must not hide an earlier print. A decode diagnostic +/// also suppresses an EOF unclosed opener. +#[derive(Clone, Copy, PartialEq, Eq)] +enum OverrideClass { + Lexer, + Decode, + Print, +} + +struct RankedOverride { + diagnostic: CpythonDiagnostic, + unclosed_bracket: bool, + class: OverrideClass, +} + +fn consider_override( + best: &mut Option, + diagnostic: CpythonDiagnostic, + class: OverrideClass, +) { + let unclosed_bracket = diagnostic.is_unclosed_bracket; + consider_ranked(best, diagnostic, unclosed_bracket, class); +} + +fn consider_ranked( + best: &mut Option, + diagnostic: CpythonDiagnostic, + unclosed_bracket: bool, + class: OverrideClass, +) { + if best + .as_ref() + .is_none_or(|current| diagnostic.range.start() < current.diagnostic.range.start()) + { + *best = Some(RankedOverride { + diagnostic, + unclosed_bracket, + class, + }); + } +} + fn cpython_parse_diagnostic_override( error: &parser::ParseError, source_file: &SourceFile, @@ -258,18 +305,80 @@ fn cpython_parse_diagnostic_override( }; } - source_error!(invalid_number_literal_error(source_text)); - source_error!(invalid_legacy_statement_error(source_text)); - source_error!(incompatible_string_prefix_error(source_text)); - source_error!(malformed_unicode_n_escape_error(source_text)); - source_error!(non_printable_character_error(source_text)); - source_error!(invalid_interpolated_string_error(source_text)); - source_error!(mixed_tstring_literal_error(error, source_text)); - - if let Some(bracket) = bracket_syntax_error(source_text) { + let mut earliest: Option = None; + if let Some(diagnostic) = invalid_number_literal_error(source_text) { + consider_override(&mut earliest, diagnostic, OverrideClass::Lexer); + } + if let Some(diagnostic) = incompatible_string_prefix_error(source_text) { + consider_override(&mut earliest, diagnostic, OverrideClass::Lexer); + } + if let Some(diagnostic) = non_printable_character_error(source_text) { + consider_override(&mut earliest, diagnostic, OverrideClass::Lexer); + } + let bracket = bracket_syntax_error(source_text); + if let Some(bracket) = bracket.as_ref() { + // Unclosed openers are reported at the opener and only become errors + // at EOF. A later token-time diagnostic (invalid number, prefix, …) + // must keep winning. Mismatched closers stay in the lexer-class + // positional ranking. + if !bracket.unclosed { + consider_ranked( + &mut earliest, + bracket.diagnostic.clone(), + false, + OverrideClass::Lexer, + ); + } + } + let mut saw_decode = false; + if let Some(diagnostic) = malformed_unicode_n_escape_error(source_text) { + saw_decode = true; + consider_override(&mut earliest, diagnostic, OverrideClass::Decode); + } + if let Some(diagnostic) = invalid_interpolated_string_error(source_text) { + saw_decode = true; + consider_override(&mut earliest, diagnostic, OverrideClass::Decode); + } + if let Some(diagnostic) = mixed_tstring_literal_error(error, source_text) { + saw_decode = true; + consider_override(&mut earliest, diagnostic, OverrideClass::Decode); + } + // A later ordinary unterminated quote is only a fallback. Format-spec + // newlines and empty fields are decode diagnostics and must keep winning. + let line_continuation = matches!( + &error.error, + parser::ParseErrorType::Lexical(parser::LexicalErrorType::LineContinuationError) + ); + if !saw_decode + && !line_continuation + && let Some(diagnostic) = unterminated_string_error(source_text, mode) + { + consider_override(&mut earliest, diagnostic, OverrideClass::Lexer); + } + if earliest + .as_ref() + .is_none_or(|current| current.class != OverrideClass::Lexer) + && let Some(diagnostic) = invalid_legacy_statement_error(source_text) + { + consider_override(&mut earliest, diagnostic, OverrideClass::Print); + } + if !saw_decode + && earliest + .as_ref() + .is_none_or(|current| current.class == OverrideClass::Print) + && let Some(bracket) = bracket.filter(|bracket| bracket.unclosed) + { + consider_ranked( + &mut earliest, + bracket.diagnostic, + true, + OverrideClass::Lexer, + ); + } + if let Some(override_diag) = earliest { return Some( - NormalizedParseDiagnostic::other(source_file, bracket.diagnostic) - .with_unclosed_bracket(bracket.unclosed), + NormalizedParseDiagnostic::other(source_file, override_diag.diagnostic) + .with_unclosed_bracket(override_diag.unclosed_bracket), ); } @@ -293,7 +402,9 @@ fn cpython_parse_diagnostic_override( } let loc = source_location(source_file, error.location.start() + TextSize::from(1)); return Some(NormalizedParseDiagnostic::new( - error.error.clone(), + parser::ParseErrorType::OtherError( + "unexpected character after line continuation character".to_owned(), + ), loc, loc, )); @@ -591,7 +702,11 @@ fn invalid_radix_literal_error( let mut has_digit = false; loop { let Some(&byte) = bytes.get(index) else { - return Some((format!("invalid {kind} literal"), start + 1)); + return if has_digit { + None + } else { + Some((format!("invalid {kind} literal"), start + 1)) + }; }; if byte == b'_' { let Some(&next) = bytes.get(index + 1) else { @@ -5876,6 +5991,7 @@ fn expected_opening_bracket(closing: char) -> char { /// A bracket diagnostic, and whether it is an opener that was never closed. The caller needs /// that apart from the message because ruff reports the unclosed case as an EOF error. +#[derive(Clone)] struct BracketError { diagnostic: CpythonDiagnostic, unclosed: bool, @@ -5932,6 +6048,24 @@ fn bracket_syntax_error(source: &str) -> Option { continue; } + if ch == '\\' { + match chars.get(index + 1).map(|(_, next)| *next) { + Some('\n' | '\r') => { + escape_next = true; + index += 1; + continue; + } + Some(_) => { + index += 2; + continue; + } + None => { + index += 1; + continue; + } + } + } + if ch == '\'' || ch == '"' { is_raw_string = false; for look_back in 1..=2.min(index) { @@ -7496,6 +7630,49 @@ mod tests { ("fu''", "'u' and 'f' prefixes are incompatible"), ("fb''", "'b' and 'f' prefixes are incompatible"), ("ufr''", "'u' and 'r' prefixes are incompatible"), + ( + "(]\nbu'x'", + "closing parenthesis ']' does not match opening parenthesis '('", + ), + ("0x\nbu'x'", "invalid hexadecimal literal"), + ("(0x", "invalid hexadecimal literal"), + ("print x; 0x", "invalid hexadecimal literal"), + ("exec x; 0x", "invalid hexadecimal literal"), + ( + "print x; 0x1", + "Missing parentheses in call to 'print'. Did you mean print(...)?", + ), + ( + "print x; (", + "Missing parentheses in call to 'print'. Did you mean print(...)?", + ), + ("print x; )", "unmatched ')'"), + ( + "( '\\N'", + "(unicode error) 'unicodeescape' codec can't decode bytes in position 0-1: malformed \\N character escape", + ), + ("(print x", "'(' was never closed"), + ( + "(print x; '\\N'", + "Missing parentheses in call to 'print'. Did you mean print(...)?", + ), + ("f'{x'; '", "f-string: expecting '}'"), + ("f'{x'; 0x", "f-string: expecting '}'"), + ( + "print x; f'{x'; 0x", + "Missing parentheses in call to 'print'. Did you mean print(...)?", + ), + ( + concat!("f'{1:", "d\n}'"), + "f-string: newlines are not allowed in format specifiers", + ), + ("f'{\n}'", "f-string: valid expression required before '}'"), + ( + "f'''\n{\n# only a comment\n}'''", + "f-string: valid expression required before '}'", + ), + ("{\\'a\\'}", "unexpected character after line continuation"), + ("\"\\\n\"(1 for c in I,\\\n\\", "'(' was never closed"), ( r"'\N'", "(unicode error) 'unicodeescape' codec can't decode bytes in position 0-1: malformed \\N character escape", @@ -7514,6 +7691,23 @@ mod tests { } } + #[test] + fn unclosed_fstring_field_keeps_the_unclosed_bracket_flag() { + let err = compile("f'{", Mode::Eval, "", CompileOpts::default()) + .expect_err("should not compile"); + let crate::CompileError::Parse(parse) = err else { + panic!("expected a parse error, got {err}"); + }; + assert!( + parse.is_unclosed_bracket, + "unclosed f-string field must stay incomplete, got {parse}" + ); + assert!( + parse.to_string().contains("'{' was never closed"), + "got {parse}" + ); + } + #[test] fn interpolated_literals_do_not_take_the_escaped_quote_hint() { // Parser/lexer/lexer.c offers "perhaps you escaped the end quote?" from its plain-string diff --git a/crates/host_env/Cargo.toml b/crates/host_env/Cargo.toml index f0a1e1e0a..1ef5cb0c8 100644 --- a/crates/host_env/Cargo.toml +++ b/crates/host_env/Cargo.toml @@ -16,19 +16,19 @@ ssl = ["dep:rustls", "dep:rustls-pemfile", "dep:rustls-webpki", "dep:x509-parser rustpython-wtf8 = { workspace = true } bitflags = { workspace = true } +der = { workspace = true, optional = true } getrandom = { workspace = true } libc = { workspace = true } num-traits = { workspace = true } parking_lot = { workspace = true } paste = { workspace = true } -widestring = { workspace = true } +pkcs8 = { workspace = true, features = ["encryption", "pkcs5", "pem"], optional = true } rustls = { workspace = true, default-features = false, features = ["std", "tls12"], optional = true } rustls-pemfile = { workspace = true, optional = true } rustls-webpki = { workspace = true, optional = true } -x509-parser = { workspace = true, optional = true } -der = { workspace = true, optional = true } -pkcs8 = { workspace = true, features = ["encryption", "pkcs5", "pem"], optional = true } sha2 = { workspace = true, optional = true } +widestring = { workspace = true } +x509-parser = { workspace = true, optional = true } [target.'cfg(unix)'.dependencies] nix = { workspace = true } @@ -39,6 +39,11 @@ rustix = { workspace = true } [target.'cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))'.dependencies] num_cpus = "1.17.0" +# rustls-pki-types::UnixTime::now is gated off wasm32-unknown-unknown unless +# the `web` feature is on. Browser rustls and Charon/LLBC both need it. +[target.'cfg(all(target_arch = "wasm32", not(target_os = "wasi")))'.dependencies] +rustls-pki-types = { version = "1", default-features = false, features = ["alloc", "std", "web"] } + [target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "windows", target_arch = "wasm32", target_os = "redox")))'.dependencies] mac_address = { workspace = true } @@ -76,9 +81,11 @@ windows-sys = { workspace = true, features = [ "Win32_Security", "Win32_Security_Authorization", "Win32_Storage_FileSystem", + "Win32_System_Com", "Win32_System_Console", "Win32_System_Diagnostics_Debug", "Win32_System_Environment", + "Win32_System_Ole", "Win32_System_IO", "Win32_System_Ioctl", "Win32_System_JobObjects", @@ -106,3 +113,8 @@ cc = "1" [lints] workspace = true + +[package.metadata.cargo-shear] +# Feature-unification only: enables rustls-pki-types `web` so UnixTime::now +# exists on wasm32-unknown-unknown. No rust source imports this crate. +ignored = ["rustls-pki-types"] diff --git a/crates/host_env/src/ctypes.rs b/crates/host_env/src/ctypes.rs index 9b999d7c5..db30a22ab 100644 --- a/crates/host_env/src/ctypes.rs +++ b/crates/host_env/src/ctypes.rs @@ -2749,6 +2749,100 @@ pub fn open_library(name: impl AsRef) -> Result libcache().write().open_library(name) } +/// `LoadLibraryExW` — the raw module handle `_ctypes.LoadLibrary` stores. +#[cfg(windows)] +pub fn load_library_ex_w( + name: &widestring::WideCStr, + flags: u32, +) -> std::io::Result { + let module = unsafe { + windows_sys::Win32::System::LibraryLoader::LoadLibraryExW( + name.as_ptr(), + core::ptr::null_mut(), + flags, + ) + }; + if module.is_null() { + Err(std::io::Error::last_os_error()) + } else { + Ok(module) + } +} + +#[cfg(windows)] +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub fn get_proc_address( + module: windows_sys::Win32::Foundation::HMODULE, + name: &CStr, +) -> Option { + unsafe { + windows_sys::Win32::System::LibraryLoader::GetProcAddress(module, name.as_ptr().cast()) + } + .map(|addr| addr as usize) + .filter(|&addr| addr != 0) +} + +#[cfg(windows)] +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub fn free_library(module: windows_sys::Win32::Foundation::HMODULE) -> std::io::Result<()> { + if unsafe { windows_sys::Win32::Foundation::FreeLibrary(module) } == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +#[cfg(windows)] +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub fn sys_string_len(bstr: *const u16) -> usize { + unsafe { windows_sys::Win32::Foundation::SysStringLen(bstr) as usize } +} + +#[cfg(windows)] +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub fn sys_free_string(bstr: *const u16) { + unsafe { windows_sys::Win32::Foundation::SysFreeString(bstr) }; +} + +#[cfg(windows)] +pub fn sys_alloc_string_len(units: &[u16]) -> Option<*mut u16> { + let len = u32::try_from(units.len()).ok()?; + let bstr = unsafe { windows_sys::Win32::Foundation::SysAllocStringLen(units.as_ptr(), len) }; + (!bstr.is_null()).then_some(bstr as *mut u16) +} + +#[cfg(windows)] +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub fn co_task_mem_free(ptr: *mut core::ffi::c_void) { + unsafe { windows_sys::Win32::System::Com::CoTaskMemFree(ptr) }; +} + +/// `GetErrorInfo(0)`. `None` when the thread has no error object (`S_FALSE`). +#[cfg(windows)] +pub fn get_error_info() -> Option<*mut core::ffi::c_void> { + let mut pei = core::ptr::null_mut(); + if unsafe { windows_sys::Win32::System::Com::GetErrorInfo(0, &mut pei) } != 0 { + None + } else { + Some(pei.cast()) + } +} + +/// `ProgIDFromCLSID`. The returned pointer is `CoTaskMemFree`'d by the caller. +#[cfg(windows)] +pub fn prog_id_from_clsid(guid: &[u8; 16]) -> Option<*mut u16> { + let mut progid = core::ptr::null_mut(); + if unsafe { + windows_sys::Win32::System::Com::ProgIDFromCLSID(guid.as_ptr().cast(), &mut progid) + } != 0 + || progid.is_null() + { + None + } else { + Some(progid) + } +} + #[cfg(unix)] pub fn open_library_with_mode( name: impl AsRef, @@ -2789,10 +2883,16 @@ pub fn lookup_data_symbol_addr( symbol_name: &[u8], ) -> Result { let cache = libcache().read(); - cache - .get_lib(handle) - .ok_or(LookupSymbolError::LibraryNotFound)? - .lookup_data_symbol_addr(symbol_name) + if let Some(lib) = cache.get_lib(handle) { + return lib.lookup_data_symbol_addr(symbol_name); + } + #[cfg(windows)] + { + drop(cache); + lookup_raw_windows_symbol(handle, symbol_name) + } + #[cfg(not(windows))] + Err(LookupSymbolError::LibraryNotFound) } #[cfg(any(unix, windows))] @@ -2801,10 +2901,47 @@ pub fn lookup_function_symbol_addr( symbol_name: &[u8], ) -> Result { let cache = libcache().read(); - cache - .get_lib(handle) - .ok_or(LookupSymbolError::LibraryNotFound)? - .lookup_function_symbol_addr(symbol_name) + if let Some(lib) = cache.get_lib(handle) { + return lib.lookup_function_symbol_addr(symbol_name); + } + #[cfg(windows)] + { + drop(cache); + lookup_raw_windows_symbol(handle, symbol_name) + } + #[cfg(not(windows))] + Err(LookupSymbolError::LibraryNotFound) +} + +#[cfg(windows)] +fn lookup_raw_windows_symbol( + handle: usize, + symbol_name: &[u8], +) -> Result { + let owned; + let name = if let Ok(name) = CStr::from_bytes_with_nul(symbol_name) { + name + } else { + owned = alloc::ffi::CString::new(symbol_name) + .map_err(|err| LookupSymbolError::Load(err.to_string()))?; + owned.as_c_str() + }; + match get_proc_address(handle as _, name) { + Some(addr) => Ok(addr), + None => { + // A valid LoadLibraryExW handle with no such export is a missing + // symbol, not a missing library. GetProcAddress sets + // ERROR_PROC_NOT_FOUND (127); invalid/unloaded modules typically + // set ERROR_INVALID_HANDLE (6) or ERROR_MOD_NOT_FOUND (126). + match std::io::Error::last_os_error().raw_os_error() { + Some(6 | 126) => Err(LookupSymbolError::LibraryNotFound), + _ => Err(LookupSymbolError::Load(format!( + "function '{}' not found", + name.to_string_lossy() + ))), + } + } + } } #[cfg(all(unix, not(target_os = "wasi")))] diff --git a/crates/host_env/src/locale.rs b/crates/host_env/src/locale.rs index fc35e9c2a..e9df052c2 100644 --- a/crates/host_env/src/locale.rs +++ b/crates/host_env/src/locale.rs @@ -190,6 +190,64 @@ pub fn acp() -> u32 { unsafe { windows_sys::Win32::Globalization::GetACP() } } +#[cfg(windows)] +pub fn user_default_lcid() -> u32 { + unsafe { windows_sys::Win32::Globalization::GetUserDefaultLCID() } +} + +#[cfg(windows)] +pub const LOCALE_SISO639LANGNAME: u32 = 0x0000_0059; +#[cfg(windows)] +pub const LOCALE_SISO3166CTRYNAME: u32 = 0x0000_005A; + +/// `GetLocaleInfoW` into a 16-unit buffer, the size `_getdefaultlocale` uses. +#[cfg(windows)] +pub fn locale_info(lcid: u32, lctype: u32) -> Option { + use windows_sys::Win32::Globalization::GetLocaleInfoW; + + let mut buffer = [0u16; 16]; + let len = unsafe { GetLocaleInfoW(lcid, lctype, buffer.as_mut_ptr(), buffer.len() as i32) }; + if len <= 1 { + None + } else { + Some(String::from_utf16_lossy(&buffer[..len as usize - 1])) + } +} + +#[cfg(windows)] +fn wide_cstr(units: &[u16]) -> alloc::borrow::Cow<'_, [u16]> { + if units.last() == Some(&0) { + alloc::borrow::Cow::Borrowed(units) + } else { + let mut owned = units.to_vec(); + owned.push(0); + alloc::borrow::Cow::Owned(owned) + } +} + +#[cfg(windows)] +pub fn wcscoll(s1: &[u16], s2: &[u16]) -> i32 { + unsafe extern "C" { + fn wcscoll(s1: *const u16, s2: *const u16) -> i32; + } + let s1 = wide_cstr(s1); + let s2 = wide_cstr(s2); + unsafe { wcscoll(s1.as_ptr(), s2.as_ptr()) } +} + +#[cfg(windows)] +pub fn wcsxfrm(src: &[u16]) -> Vec { + unsafe extern "C" { + fn wcsxfrm(dst: *mut u16, src: *const u16, count: usize) -> usize; + } + let src = wide_cstr(src); + let needed = unsafe { wcsxfrm(core::ptr::null_mut(), src.as_ptr(), 0) }; + let mut dst = vec![0u16; needed + 1]; + let written = unsafe { wcsxfrm(dst.as_mut_ptr(), src.as_ptr(), dst.len()) }; + dst.truncate(written); + dst +} + #[cfg(windows)] pub fn decode_ansi_bytes(bytes: &[u8]) -> Option { use core::ptr; diff --git a/crates/host_env/src/nt.rs b/crates/host_env/src/nt.rs index b0bf5f57a..94d3bfe74 100644 --- a/crates/host_env/src/nt.rs +++ b/crates/host_env/src/nt.rs @@ -2141,6 +2141,7 @@ pub fn spawnve( argv: &[&widestring::WideCStr], envp: &[&widestring::WideCStr], ) -> io::Result { + crate::os::ensure_drive_current_directory(); let argv_ptrs = null_terminated_ptrs(argv); let envp_ptrs = null_terminated_ptrs(envp); let result = unsafe { @@ -2176,6 +2177,7 @@ pub fn execve( argv: &[&widestring::WideCStr], envp: &[&widestring::WideCStr], ) -> io::Result<()> { + crate::os::ensure_drive_current_directory(); let argv_ptrs = null_terminated_ptrs(argv); let envp_ptrs = null_terminated_ptrs(envp); let result = unsafe { diff --git a/crates/host_env/src/socket.rs b/crates/host_env/src/socket.rs index 3206d97b6..aee53261c 100644 --- a/crates/host_env/src/socket.rs +++ b/crates/host_env/src/socket.rs @@ -915,3 +915,41 @@ pub fn if_nameindex() -> io::Result> { .map(|entry| Ok((entry.InterfaceIndex, get_name(&entry.InterfaceLuid)?))) .collect() } + +/// `UuidFromStringW`. The status is the RPC code, not `GetLastError`. +#[cfg(windows)] +pub fn uuid_from_string_w(wide: &widestring::WideCStr) -> Result { + let mut guid = windows_sys::core::GUID { + data1: 0, + data2: 0, + data3: 0, + data4: [0; 8], + }; + let status = + unsafe { windows_sys::Win32::System::Rpc::UuidFromStringW(wide.as_ptr(), &mut guid) }; + if status == windows_sys::Win32::System::Rpc::RPC_S_OK { + Ok(guid) + } else { + Err(status as u32) + } +} + +/// `UuidToStringW` / `RpcStringFreeW`. +#[cfg(windows)] +pub fn uuid_to_string_w(guid: &windows_sys::core::GUID) -> Result { + use windows_sys::Win32::System::Rpc::{RPC_S_OK, RpcStringFreeW, UuidToStringW}; + let mut raw = core::ptr::null_mut(); + let status = unsafe { UuidToStringW(guid, &mut raw) }; + if status != RPC_S_OK { + return Err(status as u32); + } + let mut len = 0usize; + unsafe { + while *raw.add(len) != 0 { + len += 1; + } + } + let text = String::from_utf16_lossy(unsafe { core::slice::from_raw_parts(raw, len) }); + unsafe { RpcStringFreeW(&mut raw) }; + Ok(text) +} diff --git a/crates/host_env/src/ssl/constants.rs b/crates/host_env/src/ssl/constants.rs index d06254eae..73c9b5523 100644 --- a/crates/host_env/src/ssl/constants.rs +++ b/crates/host_env/src/ssl/constants.rs @@ -67,6 +67,21 @@ pub const HOSTFLAG_NEVER_CHECK_SUBJECT: i32 = 0x20; pub const OP_NO_TLSV1_2: i32 = 0x0800_0000; pub const OP_NO_TLSV1_3: i32 = 0x2000_0000; +pub const SSL3_RT_CHANGE_CIPHER_SPEC: i32 = 20; +pub const SSL3_RT_ALERT: i32 = 21; +pub const SSL3_RT_HANDSHAKE: i32 = 22; +pub const SSL3_RT_APPLICATION_DATA: i32 = 23; +pub const SSL3_RT_HEADER: i32 = 256; +pub const SSL3_MT_CHANGE_CIPHER_SPEC: i32 = 0x0101; + +pub const TLS_ERROR_SSL: i32 = 1; +pub const TLS_ERROR_WANT_READ: i32 = 2; +pub const TLS_ERROR_WANT_WRITE: i32 = 3; +pub const TLS_ERROR_ZERO_RETURN: i32 = 6; +pub const TLS_ERROR_EOF: i32 = 8; +pub const TLS_ERROR_NO_MEMORY: i32 = 9; +pub const TLS_ERROR_CERT_VERIFY_BASE: i32 = 1_000; + #[cfg(test)] mod tests { use super::*; diff --git a/crates/host_env/src/ssl/mod.rs b/crates/host_env/src/ssl/mod.rs index 433b295e1..64d53d5b2 100644 --- a/crates/host_env/src/ssl/mod.rs +++ b/crates/host_env/src/ssl/mod.rs @@ -1,9 +1,8 @@ -//! Host rustls `_ssl` engine. +//! Shared `_ssl` surface. //! -//! TLS helpers live here because they share the host trust store, default -//! verify paths, and certificate files with the rest of `host_env`. The -//! engine reports plain Rust results so interpreter layers can provide their -//! own object and exception adapters. +//! The `ssl` feature compiles this whole module. MemoryBIO, constants, OID, +//! and ALPN do not use rustls. The rustls engine compiles wherever `rustls` +//! is available, including browser wasm via rustls-rustcrypto. pub mod bio; pub mod cert; diff --git a/crates/host_env/src/ssl/msg.rs b/crates/host_env/src/ssl/msg.rs index bb0d6369b..fb6802ea8 100644 --- a/crates/host_env/src/ssl/msg.rs +++ b/crates/host_env/src/ssl/msg.rs @@ -2,12 +2,11 @@ use rustls::SupportedCipherSuite; -pub const SSL3_RT_CHANGE_CIPHER_SPEC: i32 = 20; -pub const SSL3_RT_ALERT: i32 = 21; -pub const SSL3_RT_HANDSHAKE: i32 = 22; -pub const SSL3_RT_APPLICATION_DATA: u8 = 23; -pub const SSL3_RT_HEADER: i32 = 256; -pub const SSL3_MT_CHANGE_CIPHER_SPEC: i32 = 0x0101; +pub use super::constants::{ + SSL3_MT_CHANGE_CIPHER_SPEC, SSL3_RT_ALERT, SSL3_RT_CHANGE_CIPHER_SPEC, SSL3_RT_HANDSHAKE, + SSL3_RT_HEADER, +}; +pub const SSL3_RT_APPLICATION_DATA: u8 = super::constants::SSL3_RT_APPLICATION_DATA as u8; const TLS_RECORD_HEADER_SIZE: usize = 5; /// Per-connection assembler for TLS records and handshake messages. diff --git a/crates/host_env/src/ssl/protocol.rs b/crates/host_env/src/ssl/protocol.rs index 62ea97068..25bab6755 100644 --- a/crates/host_env/src/ssl/protocol.rs +++ b/crates/host_env/src/ssl/protocol.rs @@ -1,9 +1,5 @@ //! Protocol version, ALPN, and hostname helpers. -use rustls::version::{TLS12, TLS13}; - -use super::constants::{OP_NO_TLSV1_2, OP_NO_TLSV1_3, PROTO_TLSV1_2, PROTO_TLSV1_3, ProtoVersion}; - /// Why `server_hostname` was rejected. #[derive(Debug, Clone, PartialEq, Eq)] pub enum HostnameError { @@ -49,6 +45,12 @@ pub fn rustls_versions( maximum: i32, options: i32, ) -> &'static [&'static rustls::SupportedProtocolVersion] { + use rustls::version::{TLS12, TLS13}; + + use super::constants::{ + OP_NO_TLSV1_2, OP_NO_TLSV1_3, PROTO_TLSV1_2, PROTO_TLSV1_3, ProtoVersion, + }; + static TLS12_ONLY: &[&rustls::SupportedProtocolVersion] = &[&TLS12]; static TLS13_ONLY: &[&rustls::SupportedProtocolVersion] = &[&TLS13]; diff --git a/crates/host_env/src/thread.rs b/crates/host_env/src/thread.rs index 5b0d42477..dd6af5ee9 100644 --- a/crates/host_env/src/thread.rs +++ b/crates/host_env/src/thread.rs @@ -1,6 +1,25 @@ #[cfg(any(target_os = "linux", target_os = "macos"))] use alloc::ffi::CString; +/// OS thread-name cap, matching CPython `_thread._NAME_MAXLEN`. +pub const NAME_MAXLEN: usize = { + if cfg!(windows) { + 100 + } else if cfg!(target_os = "linux") { + 15 + } else if cfg!(target_os = "macos") { + 63 + } else { + 16 + } +}; + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn truncate_thread_name_bytes(name: &[u8]) -> &[u8] { + let name = name.split(|&b| b == 0).next().unwrap_or(b""); + name.get(..NAME_MAXLEN.min(name.len())).unwrap_or(b"") +} + #[cfg(unix)] pub fn current_thread_id() -> u64 { unsafe { libc::pthread_self() as u64 } @@ -11,32 +30,102 @@ pub fn current_thread_id() -> u64 { unsafe { windows_sys::Win32::System::Threading::GetCurrentThreadId() as u64 } } -#[cfg(target_os = "linux")] +#[cfg(windows)] +#[allow(clippy::not_unsafe_ptr_arg_deref)] +pub fn thread_id_from_handle(handle: *mut core::ffi::c_void) -> u64 { + unsafe { windows_sys::Win32::System::Threading::GetThreadId(handle) as u64 } +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] pub fn set_current_thread_name(name: &str) { - if CString::new(name).is_ok() { - let truncated = if name.len() > 15 { - let mut end = 15; - while !name.is_char_boundary(end) { - end -= 1; - } - CString::new(&name[..end]).expect("slice of null-free string is null-free") - } else { - CString::new(name).expect("name was already checked for nul bytes") - }; - unsafe { - libc::pthread_setname_np(libc::pthread_self(), truncated.as_ptr()); - } + set_current_thread_name_bytes(name.as_bytes()); +} + +/// Set the OS thread name from filesystem-encoded bytes (NUL-truncated). +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub fn set_current_thread_name_bytes(name: &[u8]) { + let truncated = truncate_thread_name_bytes(name); + let Ok(c_name) = CString::new(truncated) else { + return; + }; + unsafe { + #[cfg(target_os = "linux")] + libc::pthread_setname_np(libc::pthread_self(), c_name.as_ptr()); + #[cfg(target_os = "macos")] + libc::pthread_setname_np(c_name.as_ptr()); } } -#[cfg(target_os = "macos")] +#[cfg(windows)] pub fn set_current_thread_name(name: &str) { - if let Ok(c_name) = CString::new(name) { - unsafe { - libc::pthread_setname_np(c_name.as_ptr()); - } - } + let wide: Vec = name.encode_utf16().chain(core::iter::once(0)).collect(); + let _ = set_current_thread_name_wide(&wide); } -#[cfg(not(any(target_os = "linux", target_os = "macos")))] +#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] pub fn set_current_thread_name(_name: &str) {} + +/// The current thread's description as UTF-16 code units, without the terminator. +#[cfg(windows)] +pub fn current_thread_name_wide() -> std::io::Result> { + use windows_sys::Win32::{ + Foundation::LocalFree, + System::Threading::{GetCurrentThread, GetThreadDescription}, + }; + + let mut raw = core::ptr::null_mut(); + let status = unsafe { GetThreadDescription(GetCurrentThread(), &mut raw) }; + if status < 0 { + return Err(std::io::Error::from_raw_os_error(status)); + } + if raw.is_null() { + return Ok(Vec::new()); + } + let mut len = 0usize; + unsafe { + while *raw.add(len) != 0 { + len += 1; + } + } + let name = unsafe { core::slice::from_raw_parts(raw, len) }.to_vec(); + unsafe { LocalFree(raw.cast()) }; + Ok(name) +} + +/// `SetThreadDescription` on the calling thread. `name` must be NUL-terminated. +#[cfg(windows)] +pub fn set_current_thread_name_wide(name: &[u16]) -> std::io::Result<()> { + use windows_sys::Win32::System::Threading::{GetCurrentThread, SetThreadDescription}; + + if name.last() != Some(&0) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "thread name must be NUL-terminated", + )); + } + let status = unsafe { SetThreadDescription(GetCurrentThread(), name.as_ptr()) }; + if status < 0 { + Err(std::io::Error::from_raw_os_error(status)) + } else { + Ok(()) + } +} + +/// The current thread name as bytes, without the terminator. +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub fn current_thread_name(buf_len: usize) -> std::io::Result> { + let mut buffer = vec![0u8; buf_len]; + let status = unsafe { + libc::pthread_getname_np( + libc::pthread_self(), + buffer.as_mut_ptr().cast(), + buffer.len(), + ) + }; + if status != 0 { + return Err(std::io::Error::from_raw_os_error(status)); + } + let len = buffer.iter().position(|&b| b == 0).unwrap_or(buffer.len()); + buffer.truncate(len); + Ok(buffer) +} diff --git a/crates/host_env/src/winapi.rs b/crates/host_env/src/winapi.rs index d281b1733..3e13aa681 100644 --- a/crates/host_env/src/winapi.rs +++ b/crates/host_env/src/winapi.rs @@ -1426,3 +1426,18 @@ pub fn shell_execute_w( Ok(()) } } + +/// `CompareStringOrdinal`. Returns the CSTR_* result, or 0 on failure. +pub fn compare_string_ordinal(left: &[u16], right: &[u16], ignore_case: bool) -> i32 { + unsafe { + windows_sys::Win32::Globalization::CompareStringOrdinal( + left.as_ptr(), + left.len() as i32, + right.as_ptr(), + right.len() as i32, + i32::from(ignore_case), + ) + } +} + +pub const CSTR_EQUAL: i32 = windows_sys::Win32::Globalization::CSTR_EQUAL; diff --git a/crates/host_env/src/wmi.rs b/crates/host_env/src/wmi.rs index 746200484..ceccf4d79 100644 --- a/crates/host_env/src/wmi.rs +++ b/crates/host_env/src/wmi.rs @@ -114,11 +114,32 @@ unsafe extern "system" { #[link(name = "oleaut32")] unsafe extern "system" { - fn SysAllocString(psz: *const u16) -> *mut u16; - fn SysFreeString(bstrString: *mut u16); fn VariantClear(pvarg: *mut VARIANT) -> HRESULT; } +fn alloc_bstr(wide: &[u16]) -> *mut u16 { + let len = wide + .iter() + .position(|&unit| unit == 0) + .unwrap_or(wide.len()); + crate::ctypes::sys_alloc_string_len(&wide[..len]).unwrap_or(null_mut()) +} + +fn alloc_bstr_ptr(ptr: *const u16) -> *mut u16 { + let Some(ptr) = NonNull::new(ptr.cast_mut()) else { + return null_mut(); + }; + let len = unsafe { crate::ctypes::wcslen(ptr) }; + crate::ctypes::sys_alloc_string_len(unsafe { core::slice::from_raw_parts(ptr.as_ptr(), len) }) + .unwrap_or(null_mut()) +} + +fn free_bstr(bstr: *mut u16) { + if !bstr.is_null() { + crate::ctypes::sys_free_string(bstr); + } +} + #[link(name = "propsys")] unsafe extern "system" { fn VariantToString(varIn: *const VARIANT, pszBuf: *mut u16, cchBuf: u32) -> HRESULT; @@ -290,7 +311,7 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { let mut enumerator: *mut c_void = null_mut(); let mut hr: HRESULT = 0; - let bstr_query = unsafe { SysAllocString(data.query.as_ptr()) }; + let bstr_query = alloc_bstr(data.query.as_slice()); if bstr_query.is_null() { hr = hresult_from_win32(ERROR_NOT_ENOUGH_MEMORY); } @@ -304,9 +325,7 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { if failed(hr) { unsafe { CloseHandle(write_pipe); - if !bstr_query.is_null() { - SysFreeString(bstr_query); - } + free_bstr(bstr_query); } return hr as u32; } @@ -345,7 +364,7 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { if succeeded(hr) { let root_cimv2 = w!("ROOT\\CIMV2"); - let bstr_root = unsafe { SysAllocString(root_cimv2) }; + let bstr_root = alloc_bstr_ptr(root_cimv2); hr = unsafe { locator_connect_server( locator, @@ -359,9 +378,7 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { &mut services, ) }; - if !bstr_root.is_null() { - unsafe { SysFreeString(bstr_root) }; - } + free_bstr(bstr_root); } if succeeded(hr) && unsafe { SetEvent(connect_event) } == 0 { hr = hresult_from_win32(unsafe { GetLastError() }); @@ -383,7 +400,7 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { } if succeeded(hr) { let wql = w!("WQL"); - let bstr_wql = unsafe { SysAllocString(wql) }; + let bstr_wql = alloc_bstr_ptr(wql); hr = unsafe { services_exec_query( services, @@ -394,9 +411,7 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { &mut enumerator, ) }; - if !bstr_wql.is_null() { - unsafe { SysFreeString(bstr_wql) }; - } + free_bstr(bstr_wql); } let mut value: *mut c_void; @@ -466,9 +481,7 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { let Some(cb_str1) = NonNull::new(prop_name) .map(|prop_name| (unsafe { wcslen(prop_name) } * 2) as u32) else { - unsafe { - SysFreeString(prop_name); - } + free_bstr(prop_name); break; }; @@ -523,8 +536,8 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { unsafe { VariantClear(&mut prop_value); - SysFreeString(prop_name); } + free_bstr(prop_name); } } @@ -535,9 +548,7 @@ unsafe fn query_thread_impl(param: *mut c_void) -> u32 { } unsafe { - if !bstr_query.is_null() { - SysFreeString(bstr_query); - } + free_bstr(bstr_query); if !enumerator.is_null() { com_release(enumerator); } diff --git a/crates/stdlib/Cargo.toml b/crates/stdlib/Cargo.toml index 9950bb4c8..4955b99a0 100644 --- a/crates/stdlib/Cargo.toml +++ b/crates/stdlib/Cargo.toml @@ -17,9 +17,9 @@ compiler = ["rustpython-vm/compiler"] threading = ["rustpython-common/threading", "rustpython-vm/threading"] sqlite = ["dep:libsqlite3-sys"] # SSL backends -ssl = ["host_env"] -ssl-rustls = ["__ssl-rustls", "rustls/custom-provider"] -ssl-openssl = ["ssl", "openssl", "openssl-sys", "foreign-types-shared", "rustpython-host_env/native-certs"] +ssl = ["rustpython-host_env/ssl"] +ssl-rustls = ["__ssl-rustls", "host_env", "rustls/custom-provider"] +ssl-openssl = ["ssl", "host_env", "openssl", "openssl-sys", "foreign-types-shared", "rustpython-host_env/native-certs"] ssl-openssl-vendor = ["ssl-openssl", "openssl/vendored"] tkinter = ["dep:tk-sys", "dep:tcl-sys", "dep:widestring"] flame-it = ["flame"] @@ -43,6 +43,7 @@ ascii = { workspace = true } crossbeam-utils = { workspace = true } flame = { workspace = true, optional = true } hex = { workspace = true } +getrandom = { workspace = true } itertools = { workspace = true } indexmap = { workspace = true } libc = { workspace = true } diff --git a/crates/stdlib/src/lib.rs b/crates/stdlib/src/lib.rs index 08a3c805b..f4c6df884 100644 --- a/crates/stdlib/src/lib.rs +++ b/crates/stdlib/src/lib.rs @@ -66,6 +66,10 @@ mod suggestions; #[cfg(all(feature = "host_env", not(target_arch = "wasm32")))] pub mod socket; +#[cfg(target_arch = "wasm32")] +#[path = "socket_wasm.rs"] +pub mod socket; + #[cfg(all(feature = "host_env", unix, not(target_os = "redox")))] mod syslog; @@ -127,6 +131,11 @@ mod select; ))] mod openssl; +// Full rustls `_ssl` stays native. All wasm targets (browser unknown and +// WASI) bind the rustls-free `rustpython_host_env::ssl` surface (MemoryBIO, +// constants, OID, ALPN). WASI still compiles the rustls engine inside +// host_env. `_socket` on wasm is the rustls-free-style shim in +// `socket_wasm.rs` so `Lib/ssl.py` can import. #[cfg(all( feature = "host_env", feature = "__ssl-rustls", @@ -134,6 +143,14 @@ mod openssl; ))] pub mod ssl; +#[cfg(all( + target_arch = "wasm32", + feature = "ssl", + not(all(feature = "host_env", feature = "__ssl-rustls", target_os = "wasi")), +))] +#[path = "ssl_wasm.rs"] +pub mod ssl; + #[cfg(all(feature = "ssl-openssl", feature = "__ssl-rustls", not(clippy)))] compile_error!(r#"features "ssl-openssl" and "ssl-rustls" are mutually exclusive"#); @@ -248,6 +265,8 @@ pub fn stdlib_module_defs(ctx: &Context) -> Vec<&'static builtins::PyModuleDef> sha512::module_def(ctx), #[cfg(all(feature = "host_env", not(target_arch = "wasm32")))] socket::module_def(ctx), + #[cfg(target_arch = "wasm32")] + socket::module_def(ctx), #[cfg(all( feature = "sqlite", not(any(target_os = "android", target_arch = "wasm32")) @@ -259,6 +278,12 @@ pub fn stdlib_module_defs(ctx: &Context) -> Vec<&'static builtins::PyModuleDef> any(not(target_arch = "wasm32"), target_os = "wasi"), ))] ssl::module_def(ctx), + #[cfg(all( + target_arch = "wasm32", + feature = "ssl", + not(all(feature = "host_env", feature = "__ssl-rustls", target_os = "wasi")), + ))] + ssl::module_def(ctx), statistics::module_def(ctx), suggestions::module_def(ctx), _tokenize::module_def(ctx), diff --git a/crates/stdlib/src/locale.rs b/crates/stdlib/src/locale.rs index 2f929e8b5..490f4394a 100644 --- a/crates/stdlib/src/locale.rs +++ b/crates/stdlib/src/locale.rs @@ -80,6 +80,18 @@ mod _locale { ) } + #[cfg(windows)] + #[pyfunction] + fn strcoll(string1: PyUtf8StrRef, string2: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { + if string1.as_str().contains('\0') || string2.as_str().contains('\0') { + return Err(vm.new_value_error("embedded null character")); + } + let w1: Vec = string1.as_str().encode_utf16().chain([0]).collect(); + let w2: Vec = string2.as_str().encode_utf16().chain([0]).collect(); + Ok(vm.new_pyobj(host_locale::wcscoll(&w1, &w2))) + } + + #[cfg(not(windows))] #[pyfunction] fn strcoll(string1: PyUtf8StrRef, string2: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { let cstr1 = CString::new(string1.as_str()).map_err(|e| e.to_pyexception(vm))?; @@ -87,6 +99,18 @@ mod _locale { Ok(vm.new_pyobj(host_locale::strcoll(&cstr1, &cstr2))) } + #[cfg(windows)] + #[pyfunction] + fn strxfrm(string: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { + if string.as_str().contains('\0') { + return Err(vm.new_value_error("embedded null character")); + } + let wide: Vec = string.as_str().encode_utf16().chain([0]).collect(); + let transformed = host_locale::wcsxfrm(&wide); + Ok(vm.new_pyobj(String::from_utf16_lossy(&transformed))) + } + + #[cfg(not(windows))] #[pyfunction] fn strxfrm(string: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { // https://github.com/python/cpython/blob/eaae563b6878aa050b4ad406b67728b6b066220e/Modules/_localemodule.c#L390-L442 @@ -96,6 +120,20 @@ mod _locale { Ok(vm.new_pyobj(String::from_utf8(buff).expect("strxfrm returned invalid utf-8 string"))) } + #[cfg(windows)] + #[pyfunction] + fn _getdefaultlocale(vm: &VirtualMachine) -> PyObjectRef { + let lcid = host_locale::user_default_lcid(); + let language = host_locale::locale_info(lcid, host_locale::LOCALE_SISO639LANGNAME); + let territory = host_locale::locale_info(lcid, host_locale::LOCALE_SISO3166CTRYNAME); + let locale = match (language, territory) { + (Some(language), Some(territory)) => vm.new_pyobj(format!("{language}_{territory}")), + _ => vm.ctx.none(), + }; + let encoding = vm.new_pyobj(format!("cp{}", host_locale::acp())); + vm.ctx.new_tuple(vec![locale, encoding]).into() + } + #[pyfunction] fn localeconv(vm: &VirtualMachine) -> PyResult { let result = vm.ctx.new_dict(); @@ -211,33 +249,28 @@ mod _locale { Ok(pystr_from_bytes(vm, &result)) } - /// Get the current locale encoding. + #[cfg(windows)] #[pyfunction] fn getencoding() -> String { - #[cfg(windows)] + let acp = host_locale::acp(); + format!("cp{acp}") + } + + #[cfg(not(windows))] + #[pyfunction] + fn getencoding() -> String { + #[cfg(all( + unix, + not(any(target_os = "ios", target_os = "android", target_os = "redox")) + ))] { - let acp = host_locale::acp(); - format!("cp{acp}") - } - #[cfg(not(windows))] - { - #[cfg(all( - unix, - not(any(target_os = "ios", target_os = "android", target_os = "redox")) - ))] + if let Some(codeset) = host_locale::nl_langinfo_codeset() + && let Ok(s) = core::str::from_utf8(&codeset) + && !s.is_empty() { - if let Some(codeset) = host_locale::nl_langinfo_codeset() - && let Ok(s) = core::str::from_utf8(&codeset) - && !s.is_empty() - { - return s.to_string(); - } - "UTF-8".to_string() - } - #[cfg(any(target_os = "ios", target_os = "android", target_os = "redox"))] - { - "UTF-8".to_string() + return s.to_string(); } } + "UTF-8".to_string() } } diff --git a/crates/wasm/src/socket.rs b/crates/stdlib/src/socket_wasm.rs similarity index 87% rename from crates/wasm/src/socket.rs rename to crates/stdlib/src/socket_wasm.rs index f1c43bcfd..df9bc75f9 100644 --- a/crates/wasm/src/socket.rs +++ b/crates/stdlib/src/socket_wasm.rs @@ -1,12 +1,19 @@ +//! Wasm `_socket` surface so `Lib/socket.py` and `Lib/ssl.py` can import. +//! +//! There are no BSD sockets on wasm32-unknown-unknown, and WASI does not +//! expose the host_env socket engine yet. Constants, address conversion, +//! timeouts, and a constructible `socket` type are provided; connect-side +//! operations raise `OSError`. + pub(crate) use _socket::module_def; #[pymodule] mod _socket { use rustpython_vm::{ Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - builtins::{PyType, PyTypeRef}, + builtins::{PyType, PyTypeRef, PyUtf8StrRef}, common::lock::PyMutex, - function::{ArgIntoFloat, OptionalArg}, + function::{ArgBytesLike, ArgIntoFloat, OptionalArg}, types::{Constructor, Initializer}, }; use std::net::{Ipv4Addr, Ipv6Addr}; @@ -63,6 +70,13 @@ mod _socket { #[pyattr] const SHUT_RDWR: i32 = 2; + #[pyattr] + const MSG_OOB: i32 = 1; + #[pyattr] + const MSG_PEEK: i32 = 2; + #[pyattr] + const MSG_DONTROUTE: i32 = 4; + #[pyattr] const AI_PASSIVE: i32 = 1; #[pyattr] @@ -160,10 +174,7 @@ mod _socket { } #[pyfunction] - fn inet_aton( - ip: rustpython_vm::builtins::PyUtf8StrRef, - vm: &VirtualMachine, - ) -> PyResult> { + fn inet_aton(ip: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult> { ip.as_str() .parse::() .map(|addr| addr.octets().to_vec()) @@ -171,10 +182,7 @@ mod _socket { } #[pyfunction] - fn inet_ntoa( - packed: rustpython_vm::function::ArgBytesLike, - vm: &VirtualMachine, - ) -> PyResult { + fn inet_ntoa(packed: ArgBytesLike, vm: &VirtualMachine) -> PyResult { let buf = packed.borrow_buf(); let octets: [u8; 4] = (&*buf) .try_into() @@ -183,11 +191,7 @@ mod _socket { } #[pyfunction] - fn inet_pton( - af: i32, - ip: rustpython_vm::builtins::PyUtf8StrRef, - vm: &VirtualMachine, - ) -> PyResult> { + fn inet_pton(af: i32, ip: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult> { match af { AF_INET => ip .as_str() @@ -204,11 +208,7 @@ mod _socket { } #[pyfunction] - fn inet_ntop( - af: i32, - packed: rustpython_vm::function::ArgBytesLike, - vm: &VirtualMachine, - ) -> PyResult { + fn inet_ntop(af: i32, packed: ArgBytesLike, vm: &VirtualMachine) -> PyResult { let buf = packed.borrow_buf(); match af { AF_INET => { @@ -251,6 +251,29 @@ mod _socket { Ok(()) } + #[pyfunction] + fn gethostname(vm: &VirtualMachine) -> PyResult { + Err(unsupported(vm, "gethostname").into()) + } + + #[pyfunction] + fn gethostbyname(_name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { + Err(unsupported(vm, "gethostbyname").into()) + } + + #[pyfunction] + fn getaddrinfo( + _host: OptionalArg, + _port: OptionalArg, + _family: OptionalArg, + _type: OptionalArg, + _proto: OptionalArg, + _flags: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult> { + Err(unsupported(vm, "getaddrinfo").into()) + } + #[derive(FromArgs)] struct SocketInitArgs { #[pyarg(any, optional)] @@ -403,17 +426,13 @@ mod _socket { } #[pymethod] - fn accept(&self, vm: &VirtualMachine) -> PyResult<(PyObjectRef, PyObjectRef)> { + fn _accept(&self, vm: &VirtualMachine) -> PyResult<(PyObjectRef, PyObjectRef)> { self.ensure_open(vm)?; Err(unsupported(vm, "accept").into()) } #[pymethod] - fn send( - &self, - _data: rustpython_vm::function::ArgBytesLike, - vm: &VirtualMachine, - ) -> PyResult { + fn send(&self, _data: ArgBytesLike, vm: &VirtualMachine) -> PyResult { self.ensure_open(vm)?; Err(unsupported(vm, "send").into()) } diff --git a/crates/stdlib/src/ssl.rs b/crates/stdlib/src/ssl.rs index 5171764a8..8d32cb507 100644 --- a/crates/stdlib/src/ssl.rs +++ b/crates/stdlib/src/ssl.rs @@ -4003,18 +4003,15 @@ mod _ssl { #[pyclass(with(Constructor), flags(BASETYPE))] impl PyMemoryBIO { #[pymethod] - fn read(&self, len: OptionalArg, vm: &VirtualMachine) -> PyResult { + fn read(&self, len: OptionalArg, vm: &VirtualMachine) -> PyBytesRef { let mut bio = self.inner.lock(); let read_len = match len { OptionalArg::Present(n) if n >= 0 => n as usize, - OptionalArg::Present(n) => { - return Err(vm.new_value_error(format!("negative read length: {n}"))); - } - OptionalArg::Missing => bio.pending(), + OptionalArg::Present(_) | OptionalArg::Missing => bio.pending(), }; - Ok(vm.ctx.new_bytes(bio.read(read_len))) + vm.ctx.new_bytes(bio.read(read_len)) } #[pymethod] diff --git a/crates/stdlib/src/ssl/error.rs b/crates/stdlib/src/ssl/error.rs index 8b8758708..8ac88de73 100644 --- a/crates/stdlib/src/ssl/error.rs +++ b/crates/stdlib/src/ssl/error.rs @@ -101,6 +101,7 @@ pub(crate) mod ssl_error { pub(crate) struct PySSLCertVerificationError(PySSLError); // Helper functions to create SSL exceptions with proper errno attribute + #[cfg_attr(target_arch = "wasm32", expect(dead_code))] pub(crate) fn create_ssl_want_read_error(vm: &VirtualMachine) -> PyRef { vm.new_os_subtype_error( PySSLWantReadError::class(&vm.ctx).to_owned(), @@ -109,6 +110,7 @@ pub(crate) mod ssl_error { ) } + #[cfg_attr(target_arch = "wasm32", expect(dead_code))] pub(crate) fn create_ssl_want_write_error(vm: &VirtualMachine) -> PyRef { vm.new_os_subtype_error( PySSLWantWriteError::class(&vm.ctx).to_owned(), @@ -117,6 +119,7 @@ pub(crate) mod ssl_error { ) } + #[cfg_attr(target_arch = "wasm32", expect(dead_code))] pub(crate) fn create_ssl_eof_error(vm: &VirtualMachine) -> PyRef { vm.new_os_subtype_error( PySSLEOFError::class(&vm.ctx).to_owned(), @@ -126,7 +129,10 @@ pub(crate) mod ssl_error { } #[cfg_attr( - all(feature = "ssl-openssl", not(feature = "ssl-rustls")), + any( + target_arch = "wasm32", + all(feature = "ssl-openssl", not(feature = "ssl-rustls")) + ), expect(dead_code) )] pub(crate) fn create_ssl_zero_return_error(vm: &VirtualMachine) -> PyRef { @@ -138,7 +144,10 @@ pub(crate) mod ssl_error { } #[cfg_attr( - all(feature = "ssl-openssl", not(feature = "ssl-rustls")), + any( + target_arch = "wasm32", + all(feature = "ssl-openssl", not(feature = "ssl-rustls")) + ), expect(dead_code) )] pub(crate) fn create_ssl_syscall_error( diff --git a/crates/stdlib/src/ssl_wasm.rs b/crates/stdlib/src/ssl_wasm.rs new file mode 100644 index 000000000..799ead44f --- /dev/null +++ b/crates/stdlib/src/ssl_wasm.rs @@ -0,0 +1,960 @@ +//! Wasm `_ssl` on the rustls-free `rustpython_host_env::ssl` surface. +//! +//! MemoryBIO, protocol constants, OID lookup, hostname checks, and ALPN +//! parsing work. TLS connections are not available until a wasm TLS engine +//! is wired to the same types. + +#[path = "ssl/error.rs"] +mod error; + +pub(crate) use _ssl::module_def; + +#[allow(non_snake_case)] +#[allow(non_upper_case_globals)] +#[pymodule(with(error::ssl_error))] +mod _ssl { + use super::error::{PySSLError, SSL_ERROR_SSL}; + use crate::{ + common::lock::{PyMutex, PyRwLock}, + vm::{ + AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, + VirtualMachine, + builtins::{PyBytesRef, PyType, PyUtf8StrRef}, + function::{ArgBytesLike, FuncArgs, OptionalArg, PyComparisonValue, PySetterValue}, + stdlib::_warnings, + types::{Comparable, Constructor, PyComparisonOp, Representable}, + }, + }; + use rustpython_host_env::ssl::oid; + use rustpython_vm::exceptions; + + const UNAVAILABLE: &str = "TLS is not available on this platform"; + + #[pyattr] + const PROTOCOL_TLS: i32 = rustpython_host_env::ssl::PROTOCOL_TLS; + #[pyattr] + const PROTOCOL_SSLv23: i32 = PROTOCOL_TLS; + #[pyattr] + const PROTOCOL_TLS_CLIENT: i32 = rustpython_host_env::ssl::PROTOCOL_TLS_CLIENT; + #[pyattr] + const PROTOCOL_TLS_SERVER: i32 = rustpython_host_env::ssl::PROTOCOL_TLS_SERVER; + #[pyattr] + const PROTOCOL_TLSv1: i32 = rustpython_host_env::ssl::PROTOCOL_TLSV1; + #[pyattr] + const PROTOCOL_TLSv1_1: i32 = rustpython_host_env::ssl::PROTOCOL_TLSV1_1; + #[pyattr] + const PROTOCOL_TLSv1_2: i32 = rustpython_host_env::ssl::PROTOCOL_TLSV1_2; + #[pyattr] + const PROTOCOL_TLSv1_3: i32 = rustpython_host_env::ssl::PROTOCOL_TLSV1_3; + + #[pyattr] + const PROTO_SSLv3: i32 = 0x0300; + #[pyattr] + const PROTO_TLSv1: i32 = 0x0301; + #[pyattr] + const PROTO_TLSv1_1: i32 = 0x0302; + #[pyattr] + const PROTO_TLSv1_2: i32 = rustpython_host_env::ssl::PROTO_TLSV1_2; + #[pyattr] + const PROTO_TLSv1_3: i32 = rustpython_host_env::ssl::PROTO_TLSV1_3; + #[pyattr] + const PROTO_MINIMUM_SUPPORTED: i32 = -2; + #[pyattr] + const PROTO_MAXIMUM_SUPPORTED: i32 = -1; + + #[pyattr] + const CERT_NONE: i32 = rustpython_host_env::ssl::CERT_NONE; + #[pyattr] + const CERT_OPTIONAL: i32 = rustpython_host_env::ssl::CERT_OPTIONAL; + #[pyattr] + const CERT_REQUIRED: i32 = rustpython_host_env::ssl::CERT_REQUIRED; + + #[pyattr] + const VERIFY_DEFAULT: i32 = rustpython_host_env::ssl::VERIFY_DEFAULT; + #[pyattr] + const VERIFY_CRL_CHECK_LEAF: i32 = rustpython_host_env::ssl::VERIFY_CRL_CHECK_LEAF; + #[pyattr] + const VERIFY_CRL_CHECK_CHAIN: i32 = rustpython_host_env::ssl::VERIFY_CRL_CHECK_CHAIN; + #[pyattr] + const VERIFY_X509_STRICT: i32 = rustpython_host_env::ssl::VERIFY_X509_STRICT; + #[pyattr] + const VERIFY_ALLOW_PROXY_CERTS: i32 = rustpython_host_env::ssl::VERIFY_ALLOW_PROXY_CERTS; + #[pyattr] + const VERIFY_X509_TRUSTED_FIRST: i32 = rustpython_host_env::ssl::VERIFY_X509_TRUSTED_FIRST; + #[pyattr] + const VERIFY_X509_PARTIAL_CHAIN: i32 = rustpython_host_env::ssl::VERIFY_X509_PARTIAL_CHAIN; + #[pyattr] + const HOSTFLAG_NEVER_CHECK_SUBJECT: i32 = + rustpython_host_env::ssl::HOSTFLAG_NEVER_CHECK_SUBJECT; + + #[pyattr] + const OP_NO_SSLv2: i32 = 0x00000000; + #[pyattr] + const OP_NO_SSLv3: i32 = 0x02000000; + #[pyattr] + const OP_NO_TLSv1: i32 = 0x04000000; + #[pyattr] + const OP_NO_TLSv1_1: i32 = 0x10000000; + #[pyattr] + const OP_NO_TLSv1_2: i32 = rustpython_host_env::ssl::OP_NO_TLSV1_2; + #[pyattr] + const OP_NO_TLSv1_3: i32 = rustpython_host_env::ssl::OP_NO_TLSV1_3; + #[pyattr] + const OP_NO_COMPRESSION: i32 = 0x00020000; + #[pyattr] + const OP_CIPHER_SERVER_PREFERENCE: i32 = 0x00400000; + #[pyattr] + const OP_SINGLE_DH_USE: i32 = 0x00000000; + #[pyattr] + const OP_SINGLE_ECDH_USE: i32 = 0x00000000; + #[pyattr] + const OP_NO_TICKET: i32 = 0x00004000; + #[pyattr] + const OP_LEGACY_SERVER_CONNECT: i32 = 0x00000004; + #[pyattr] + const OP_NO_RENEGOTIATION: i32 = 0x40000000; + #[pyattr] + const OP_IGNORE_UNEXPECTED_EOF: i32 = 0x00000080; + #[pyattr] + const OP_ENABLE_MIDDLEBOX_COMPAT: i32 = 0x00100000; + #[pyattr] + const OP_ALL: i32 = 0x00000BFB; + + #[pyattr] + const ALERT_DESCRIPTION_CLOSE_NOTIFY: i32 = 0; + #[pyattr] + const ALERT_DESCRIPTION_UNEXPECTED_MESSAGE: i32 = 10; + #[pyattr] + const ALERT_DESCRIPTION_BAD_RECORD_MAC: i32 = 20; + #[pyattr] + const ALERT_DESCRIPTION_DECRYPTION_FAILED: i32 = 21; + #[pyattr] + const ALERT_DESCRIPTION_RECORD_OVERFLOW: i32 = 22; + #[pyattr] + const ALERT_DESCRIPTION_DECOMPRESSION_FAILURE: i32 = 30; + #[pyattr] + const ALERT_DESCRIPTION_HANDSHAKE_FAILURE: i32 = 40; + #[pyattr] + const ALERT_DESCRIPTION_NO_CERTIFICATE: i32 = 41; + #[pyattr] + const ALERT_DESCRIPTION_BAD_CERTIFICATE: i32 = 42; + #[pyattr] + const ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE: i32 = 43; + #[pyattr] + const ALERT_DESCRIPTION_CERTIFICATE_REVOKED: i32 = 44; + #[pyattr] + const ALERT_DESCRIPTION_CERTIFICATE_EXPIRED: i32 = 45; + #[pyattr] + const ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN: i32 = 46; + #[pyattr] + const ALERT_DESCRIPTION_ILLEGAL_PARAMETER: i32 = 47; + #[pyattr] + const ALERT_DESCRIPTION_UNKNOWN_CA: i32 = 48; + #[pyattr] + const ALERT_DESCRIPTION_ACCESS_DENIED: i32 = 49; + #[pyattr] + const ALERT_DESCRIPTION_DECODE_ERROR: i32 = 50; + #[pyattr] + const ALERT_DESCRIPTION_DECRYPT_ERROR: i32 = 51; + #[pyattr] + const ALERT_DESCRIPTION_EXPORT_RESTRICTION: i32 = 60; + #[pyattr] + const ALERT_DESCRIPTION_PROTOCOL_VERSION: i32 = 70; + #[pyattr] + const ALERT_DESCRIPTION_INSUFFICIENT_SECURITY: i32 = 71; + #[pyattr] + const ALERT_DESCRIPTION_INTERNAL_ERROR: i32 = 80; + #[pyattr] + const ALERT_DESCRIPTION_INAPPROPRIATE_FALLBACK: i32 = 86; + #[pyattr] + const ALERT_DESCRIPTION_USER_CANCELLED: i32 = 90; + #[pyattr] + const ALERT_DESCRIPTION_NO_RENEGOTIATION: i32 = 100; + #[pyattr] + const ALERT_DESCRIPTION_MISSING_EXTENSION: i32 = 109; + #[pyattr] + const ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION: i32 = 110; + #[pyattr] + const ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE: i32 = 111; + #[pyattr] + const ALERT_DESCRIPTION_UNRECOGNIZED_NAME: i32 = 112; + #[pyattr] + const ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE: i32 = 113; + #[pyattr] + const ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE: i32 = 114; + #[pyattr] + const ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY: i32 = 115; + #[pyattr] + const ALERT_DESCRIPTION_CERTIFICATE_REQUIRED: i32 = 116; + #[pyattr] + const ALERT_DESCRIPTION_NO_APPLICATION_PROTOCOL: i32 = 120; + + #[pyattr] + const OPENSSL_VERSION_NUMBER: i32 = 0x30300000; + #[pyattr] + const OPENSSL_VERSION: &str = "OpenSSL 3.3.0-compatible (rustls-free wasm)"; + #[pyattr] + const OPENSSL_VERSION_INFO: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); + #[pyattr] + const _OPENSSL_API_VERSION: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); + + #[pyattr(once)] + fn _DEFAULT_CIPHERS(_vm: &VirtualMachine) -> String { + "DEFAULT".to_owned() + } + + #[pyattr] + const HAS_SNI: bool = true; + #[pyattr] + const HAS_TLS_UNIQUE: bool = false; + #[pyattr] + const HAS_ECDH: bool = false; + #[pyattr] + const HAS_NPN: bool = false; + #[pyattr] + const HAS_ALPN: bool = true; + #[pyattr] + const HAS_PSK: bool = false; + #[pyattr] + const HAS_SSLv2: bool = false; + #[pyattr] + const HAS_SSLv3: bool = false; + #[pyattr] + const HAS_TLSv1: bool = false; + #[pyattr] + const HAS_TLSv1_1: bool = false; + #[pyattr] + const HAS_TLSv1_2: bool = false; + #[pyattr] + const HAS_TLSv1_3: bool = false; + #[pyattr] + const HAS_PHA: bool = false; + + #[pyattr] + const ENCODING_PEM: i32 = 1; + #[pyattr] + const ENCODING_DER: i32 = 2; + #[pyattr] + const ENCODING_PEM_AUX: i32 = 0x101; + + fn ssl_error(vm: &VirtualMachine, message: impl Into) -> PyResult { + Err(vm + .new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + Some(SSL_ERROR_SSL), + message.into(), + ) + .upcast()) + } + + fn validate_hostname(hostname: &str, vm: &VirtualMachine) -> PyResult<()> { + use rustpython_host_env::ssl::HostnameError; + match rustpython_host_env::ssl::validate_hostname(hostname) { + Ok(()) => Ok(()), + Err(HostnameError::EmbeddedNul) => Err(exceptions::nul_char_type_error(vm)), + Err(error) => Err(vm.new_value_error(error.message().to_owned())), + } + } + + fn parse_length_prefixed_alpn(bytes: &[u8], vm: &VirtualMachine) -> PyResult>> { + rustpython_host_env::ssl::parse_length_prefixed_alpn(bytes) + .map_err(|error| vm.new_value_error(error.0)) + } + + #[pyattr] + #[pyclass(name = "_SSLContext", module = "ssl", traverse)] + #[derive(Debug, PyPayload)] + struct PySSLContext { + #[pytraverse(skip)] + protocol: i32, + #[pytraverse(skip)] + check_hostname: PyRwLock, + #[pytraverse(skip)] + host_flags: PyRwLock, + #[pytraverse(skip)] + verify_mode: PyRwLock, + #[pytraverse(skip)] + verify_flags: PyRwLock, + #[pytraverse(skip)] + options: PyRwLock, + #[pytraverse(skip)] + alpn_protocols: PyRwLock>>, + #[pytraverse(skip)] + post_handshake_auth: PyRwLock, + #[pytraverse(skip)] + num_tickets: PyRwLock, + #[pytraverse(skip)] + minimum_version: PyRwLock, + #[pytraverse(skip)] + maximum_version: PyRwLock, + sni_callback: PyRwLock>, + msg_callback: PyRwLock>, + keylog_filename: PyRwLock>, + } + + #[pyclass(with(Constructor, Representable), flags(BASETYPE))] + impl PySSLContext { + #[pygetset] + fn check_hostname(&self) -> bool { + *self.check_hostname.read() + } + + #[pygetset(setter)] + fn set_check_hostname(&self, value: bool) { + *self.check_hostname.write() = value; + if value && *self.verify_mode.read() == CERT_NONE { + *self.verify_mode.write() = CERT_REQUIRED; + } + } + + #[pygetset] + fn _host_flags(&self) -> i32 { + *self.host_flags.read() + } + + #[pygetset(setter)] + fn set__host_flags(&self, value: i32) { + *self.host_flags.write() = value; + } + + #[pygetset] + fn verify_mode(&self) -> i32 { + *self.verify_mode.read() + } + + #[pygetset(setter)] + fn set_verify_mode(&self, mode: i32, vm: &VirtualMachine) -> PyResult<()> { + if !(CERT_NONE..=CERT_REQUIRED).contains(&mode) { + return Err(vm.new_value_error("invalid verify mode")); + } + if mode == CERT_NONE && *self.check_hostname.read() { + return Err(vm.new_value_error( + "Cannot set verify_mode to CERT_NONE when check_hostname is enabled", + )); + } + *self.verify_mode.write() = mode; + Ok(()) + } + + #[pygetset] + fn protocol(&self) -> i32 { + self.protocol + } + + #[pygetset] + fn verify_flags(&self) -> i32 { + *self.verify_flags.read() + } + + #[pygetset(setter)] + fn set_verify_flags(&self, value: i32) { + *self.verify_flags.write() = value; + } + + #[pygetset] + fn post_handshake_auth(&self) -> bool { + *self.post_handshake_auth.read() + } + + #[pygetset(setter)] + fn set_post_handshake_auth(&self, value: bool) { + *self.post_handshake_auth.write() = value; + } + + #[pygetset] + fn num_tickets(&self) -> i32 { + *self.num_tickets.read() + } + + #[pygetset(setter)] + fn set_num_tickets(&self, value: i32, vm: &VirtualMachine) -> PyResult<()> { + if value < 0 { + return Err(vm.new_value_error("num_tickets must be a non-negative integer")); + } + if self.protocol != PROTOCOL_TLS_SERVER { + return Err( + vm.new_value_error("num_tickets can only be set on server-side contexts") + ); + } + *self.num_tickets.write() = value; + Ok(()) + } + + #[pygetset] + fn options(&self) -> i32 { + *self.options.read() + } + + #[pygetset(setter)] + fn set_options(&self, value: i32, vm: &VirtualMachine) -> PyResult<()> { + if value < 0 { + return Err(vm.new_value_error("options must be non-negative")); + } + let opt_no = OP_NO_SSLv2 + | OP_NO_SSLv3 + | OP_NO_TLSv1 + | OP_NO_TLSv1_1 + | OP_NO_TLSv1_2 + | OP_NO_TLSv1_3; + let old_opts = *self.options.read(); + let set = !old_opts & value; + if (set & opt_no) != 0 { + _warnings::warn( + vm.ctx.exceptions.deprecation_warning, + "ssl.OP_NO_SSL*/ssl.OP_NO_TLS* options are deprecated".to_owned(), + 2, + vm, + )?; + } + *self.options.write() = value; + Ok(()) + } + + #[pygetset] + fn minimum_version(&self) -> i32 { + let v = *self.minimum_version.read(); + if v == 0 { PROTO_MINIMUM_SUPPORTED } else { v } + } + + #[pygetset(setter)] + fn set_minimum_version(&self, value: i32, vm: &VirtualMachine) -> PyResult<()> { + if value != 0 + && value != -2 + && value != -1 + && !(PROTO_SSLv3..=PROTO_TLSv1_3).contains(&value) + { + return Err(vm.new_value_error(format!("invalid protocol version: {value}"))); + } + *self.minimum_version.write() = value; + Ok(()) + } + + #[pygetset] + fn maximum_version(&self) -> i32 { + let v = *self.maximum_version.read(); + if v == 0 { PROTO_MAXIMUM_SUPPORTED } else { v } + } + + #[pygetset(setter)] + fn set_maximum_version(&self, value: i32, vm: &VirtualMachine) -> PyResult<()> { + if value != 0 + && value != -2 + && value != -1 + && !(PROTO_SSLv3..=PROTO_TLSv1_3).contains(&value) + { + return Err(vm.new_value_error(format!("invalid protocol version: {value}"))); + } + *self.maximum_version.write() = value; + Ok(()) + } + + #[pygetset] + fn sni_callback(&self) -> Option { + self.sni_callback.read().clone() + } + + #[pygetset(setter)] + fn set_sni_callback( + &self, + callback: Option, + vm: &VirtualMachine, + ) -> PyResult<()> { + if let Some(ref cb) = callback + && !cb.is(vm.ctx.types.none_type) + && !cb.is_callable() + { + return Err(vm.new_type_error("sni_callback must be callable or None")); + } + *self.sni_callback.write() = callback; + Ok(()) + } + + #[pymethod] + fn set_servername_callback( + &self, + callback: Option, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.set_sni_callback(callback, vm) + } + + #[pygetset] + fn security_level(&self) -> i32 { + 2 + } + + #[pygetset] + fn _msg_callback(&self) -> Option { + self.msg_callback.read().clone() + } + + #[pygetset(setter)] + fn set__msg_callback( + &self, + callback: Option, + vm: &VirtualMachine, + ) -> PyResult<()> { + if let Some(ref cb) = callback + && !cb.is(vm.ctx.types.none_type) + && !cb.is_callable() + { + return Err(vm.new_type_error("msg_callback must be callable or None")); + } + *self.msg_callback.write() = callback; + Ok(()) + } + + #[pygetset] + fn keylog_filename(&self) -> Option { + self.keylog_filename.read().clone() + } + + #[pygetset(setter)] + fn set_keylog_filename(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let PySetterValue::Assign(value) = value else { + return Err(vm.new_attribute_error("attribute 'keylog_filename' cannot be deleted")); + }; + if vm.is_none(&value) { + *self.keylog_filename.write() = None; + return Ok(()); + } + *self.keylog_filename.write() = Some(value); + Ok(()) + } + + #[pymethod] + fn _set_alpn_protocols(&self, protos: ArgBytesLike, vm: &VirtualMachine) -> PyResult<()> { + let bytes = protos.borrow_buf(); + let alpn_list = parse_length_prefixed_alpn(&bytes, vm)?; + *self.alpn_protocols.write() = alpn_list; + Ok(()) + } + + #[pymethod] + fn set_default_verify_paths(&self, _vm: &VirtualMachine) -> PyResult<()> { + Ok(()) + } + + #[pymethod] + fn load_default_certs(&self, _vm: &VirtualMachine) -> PyResult<()> { + Ok(()) + } + + #[pymethod] + fn load_verify_locations( + &self, + args: LoadVerifyLocationsArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + let has_cafile = matches!(&args.cafile, OptionalArg::Present(Some(_))); + let has_capath = matches!(&args.capath, OptionalArg::Present(Some(_))); + let has_cadata = matches!(&args.cadata, OptionalArg::Present(Some(_))); + if !has_cafile && !has_capath && !has_cadata { + return Err(vm.new_type_error("cafile, capath and cadata cannot be all omitted")); + } + Ok(()) + } + + #[pymethod] + fn load_cert_chain(&self, _args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + ssl_error(vm, UNAVAILABLE) + } + + #[pymethod] + fn set_ciphers(&self, _ciphers: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<()> { + ssl_error(vm, UNAVAILABLE) + } + + #[pymethod] + fn set_ecdh_curve(&self, _name: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult<()> { + ssl_error(vm, UNAVAILABLE) + } + + #[pymethod] + fn load_dh_params(&self, _filepath: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + ssl_error(vm, UNAVAILABLE) + } + + #[pymethod] + fn get_ciphers(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_list(Vec::new()).into()) + } + + #[pymethod] + fn get_ca_certs(&self, args: GetCertArgs, vm: &VirtualMachine) -> PyResult { + let _ = args.binary_form; + Ok(vm.ctx.new_list(Vec::new()).into()) + } + + #[pymethod] + fn cert_store_stats(&self, vm: &VirtualMachine) -> PyResult { + let dict = vm.ctx.new_dict(); + dict.set_item("x509", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("crl", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("x509_ca", vm.ctx.new_int(0).into(), vm)?; + Ok(dict.into()) + } + + #[pymethod] + fn session_stats(&self, vm: &VirtualMachine) -> PyResult { + let dict = vm.ctx.new_dict(); + dict.set_item("number", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("connect", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("connect_good", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("connect_renegotiate", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("accept", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("accept_good", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("accept_renegotiate", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("hits", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("misses", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("timeouts", vm.ctx.new_int(0).into(), vm)?; + dict.set_item("cache_full", vm.ctx.new_int(0).into(), vm)?; + Ok(dict.into()) + } + + #[pymethod] + fn _wrap_socket(&self, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { + ssl_error(vm, UNAVAILABLE) + } + + #[pymethod] + fn _wrap_bio(&self, args: WrapBioArgs, vm: &VirtualMachine) -> PyResult { + if let Some(hostname) = args.server_hostname.into_option().flatten() { + validate_hostname(hostname.as_str(), vm)?; + } + if args.server_side.unwrap_or(false) && self.protocol == PROTOCOL_TLS_CLIENT { + return ssl_error( + vm, + "Cannot create a server socket with a PROTOCOL_TLS_CLIENT context", + ); + } + if !args.server_side.unwrap_or(false) && self.protocol == PROTOCOL_TLS_SERVER { + return ssl_error( + vm, + "Cannot create a client socket with a PROTOCOL_TLS_SERVER context", + ); + } + ssl_error(vm, UNAVAILABLE) + } + } + + impl Representable for PySSLContext { + #[inline] + fn repr_str(zelf: &Py, _vm: &VirtualMachine) -> PyResult { + Ok(format!("", zelf.protocol)) + } + } + + impl Constructor for PySSLContext { + type Args = (i32,); + + fn py_new( + _cls: &Py, + (protocol,): Self::Args, + vm: &VirtualMachine, + ) -> PyResult { + let deprecated_protocol = match protocol { + PROTOCOL_TLS => Some("PROTOCOL_TLS"), + PROTOCOL_TLSv1_2 => Some("PROTOCOL_TLSv1_2"), + PROTOCOL_TLS_CLIENT | PROTOCOL_TLS_SERVER | PROTOCOL_TLSv1_3 => None, + PROTOCOL_TLSv1 | PROTOCOL_TLSv1_1 => { + return Err(vm.new_value_error( + "TLS 1.0 and 1.1 are not supported by rustls for security reasons", + )); + } + _ => { + return Err(vm.new_value_error(format!("invalid protocol version: {protocol}"))); + } + }; + if let Some(protocol_name) = deprecated_protocol { + _warnings::warn( + vm.ctx.exceptions.deprecation_warning, + format!("ssl.{protocol_name} is deprecated"), + 2, + vm, + )?; + } + + let default_options = OP_ALL + | OP_NO_SSLv2 + | OP_NO_SSLv3 + | OP_NO_COMPRESSION + | OP_CIPHER_SERVER_PREFERENCE + | OP_SINGLE_DH_USE + | OP_SINGLE_ECDH_USE + | OP_ENABLE_MIDDLEBOX_COMPAT; + let default_verify_mode = if protocol == PROTOCOL_TLS_CLIENT { + CERT_REQUIRED + } else { + CERT_NONE + }; + let (min_version, max_version) = match protocol { + PROTOCOL_TLSv1_2 => (PROTO_TLSv1_2, PROTO_TLSv1_2), + PROTOCOL_TLSv1_3 => (PROTO_TLSv1_3, PROTO_TLSv1_3), + _ => (PROTO_MINIMUM_SUPPORTED, PROTO_MAXIMUM_SUPPORTED), + }; + + Ok(Self { + protocol, + check_hostname: PyRwLock::new(protocol == PROTOCOL_TLS_CLIENT), + host_flags: PyRwLock::new(0), + verify_mode: PyRwLock::new(default_verify_mode), + verify_flags: PyRwLock::new(VERIFY_DEFAULT | VERIFY_X509_TRUSTED_FIRST), + options: PyRwLock::new(default_options), + alpn_protocols: PyRwLock::new(Vec::new()), + post_handshake_auth: PyRwLock::new(false), + num_tickets: PyRwLock::new(2), + minimum_version: PyRwLock::new(min_version), + maximum_version: PyRwLock::new(max_version), + sni_callback: PyRwLock::new(None), + msg_callback: PyRwLock::new(None), + keylog_filename: PyRwLock::new(None), + }) + } + } + + #[derive(FromArgs)] + struct GetCertArgs { + #[pyarg(any, optional)] + binary_form: OptionalArg, + } + + #[derive(FromArgs)] + struct LoadVerifyLocationsArgs { + #[pyarg(any, optional)] + cafile: OptionalArg>, + #[pyarg(any, optional)] + capath: OptionalArg>, + #[pyarg(any, optional)] + cadata: OptionalArg>, + } + + #[derive(FromArgs)] + struct WrapBioArgs { + #[allow(dead_code)] + incoming: PyRef, + #[allow(dead_code)] + outgoing: PyRef, + #[pyarg(named, optional)] + server_side: OptionalArg, + #[pyarg(named, optional)] + server_hostname: OptionalArg>, + #[pyarg(named, optional)] + #[allow(dead_code)] + owner: OptionalArg, + #[pyarg(named, optional)] + #[allow(dead_code)] + session: OptionalArg, + } + + #[pyattr] + #[pyclass(name = "MemoryBIO", module = "ssl")] + #[derive(Debug, PyPayload)] + struct PyMemoryBIO { + inner: PyMutex, + } + + #[pyclass(with(Constructor, Representable), flags(BASETYPE))] + impl PyMemoryBIO { + #[pymethod] + fn read(&self, len: OptionalArg, vm: &VirtualMachine) -> PyResult { + let mut bio = self.inner.lock(); + let read_len = match len { + OptionalArg::Present(n) if n >= 0 => n as usize, + OptionalArg::Present(_) | OptionalArg::Missing => bio.pending(), + }; + Ok(vm.ctx.new_bytes(bio.read(read_len))) + } + + #[pymethod] + fn write(&self, buf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(mem_view) = buf.get_attr("c_contiguous", vm) { + let is_contiguous: bool = mem_view.try_to_bool(vm)?; + if !is_contiguous { + return Err(vm.new_buffer_error("non-contiguous buffer is not supported")); + } + } + let bytes_like = ArgBytesLike::try_from_object(vm, buf)?; + let data = bytes_like.borrow_buf(); + self.inner.lock().write(&data).map_err(|err| { + vm.new_os_subtype_error( + PySSLError::class(&vm.ctx).to_owned(), + None, + err.to_string(), + ) + .upcast() + }) + } + + #[pymethod] + fn write_eof(&self, _vm: &VirtualMachine) { + self.inner.lock().write_eof(); + } + + #[pygetset] + fn pending(&self) -> i32 { + self.inner.lock().pending() as i32 + } + + #[pygetset] + fn eof(&self) -> bool { + self.inner.lock().eof() + } + } + + impl Representable for PyMemoryBIO { + #[inline] + fn repr_str(_zelf: &Py, _vm: &VirtualMachine) -> PyResult { + Ok("".to_owned()) + } + } + + impl Constructor for PyMemoryBIO { + type Args = (); + + fn py_new(_cls: &Py, _args: Self::Args, _vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: PyMutex::new(rustpython_host_env::ssl::MemoryBio::new()), + }) + } + } + + #[pyattr] + #[pyclass(name = "SSLSession", module = "ssl")] + #[derive(Debug, PyPayload)] + struct PySSLSession { + session_id: Vec, + lifetime: u64, + has_ticket: bool, + } + + #[pyclass(flags(BASETYPE), with(Comparable, Representable))] + impl PySSLSession { + #[pygetset] + fn time(&self) -> i64 { + 0 + } + + #[pygetset] + fn timeout(&self) -> i64 { + self.lifetime as i64 + } + + #[pygetset] + fn ticket_lifetime_hint(&self) -> i64 { + self.lifetime as i64 + } + + #[pygetset] + fn id(&self, vm: &VirtualMachine) -> PyBytesRef { + vm.ctx.new_bytes(self.session_id.clone()) + } + + #[pygetset] + fn has_ticket(&self) -> bool { + self.has_ticket + } + } + + impl Comparable for PySSLSession { + fn cmp( + zelf: &Py, + other: &PyObject, + op: PyComparisonOp, + _vm: &VirtualMachine, + ) -> PyResult { + op.eq_only(|| { + if let Some(other_session) = other.downcast_ref::() { + Ok((zelf.session_id == other_session.session_id).into()) + } else { + Ok(PyComparisonValue::NotImplemented) + } + }) + } + } + + impl Representable for PySSLSession { + #[inline] + fn repr_str(_zelf: &Py, _vm: &VirtualMachine) -> PyResult { + Ok("".to_owned()) + } + } + + #[derive(FromArgs)] + struct Txt2ObjArgs { + txt: PyUtf8StrRef, + #[pyarg(named, optional)] + name: OptionalArg, + } + + fn oid_tuple(entry: &oid::OidEntry, vm: &VirtualMachine) -> PyObjectRef { + vm.new_tuple(( + vm.ctx.new_int(entry.nid), + vm.ctx.new_str(entry.short_name), + vm.ctx.new_str(entry.long_name), + entry + .oid_string() + .map_or_else(|| vm.ctx.none(), |oid| vm.ctx.new_str(oid).into()), + )) + .into() + } + + #[pyfunction] + fn txt2obj(args: Txt2ObjArgs, vm: &VirtualMachine) -> PyResult { + let txt = args.txt.as_str(); + let name = args.name.unwrap_or(false); + let entry = if txt.chars().next().is_some_and(|c| c.is_ascii_digit()) { + oid::find_by_oid_string(txt) + } else if name { + oid::find_by_name(txt) + } else { + None + }; + let entry = entry.ok_or_else(|| vm.new_value_error(format!("unknown object '{txt}'")))?; + Ok(oid_tuple(entry, vm)) + } + + #[pyfunction] + fn nid2obj(nid: i32, vm: &VirtualMachine) -> PyResult { + let entry = oid::find_by_nid(nid) + .ok_or_else(|| vm.new_value_error(format!("unknown NID {nid}")))?; + Ok(oid_tuple(entry, vm)) + } + + #[pyfunction] + fn get_default_verify_paths(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx + .new_tuple(vec![ + vm.ctx.new_str("SSL_CERT_FILE").into(), + vm.ctx.new_str("").into(), + vm.ctx.new_str("SSL_CERT_DIR").into(), + vm.ctx.new_str("").into(), + ]) + .into() + } + + #[pyfunction] + fn RAND_status() -> i32 { + 1 + } + + #[pyfunction] + fn RAND_add(_string: PyObjectRef, _entropy: f64) {} + + #[pyfunction] + fn RAND_bytes(n: i32, vm: &VirtualMachine) -> PyResult { + if n < 0 { + return Err(vm.new_value_error("num must be positive")); + } + let mut buf = vm.new_zeroed_bytes(n as usize)?; + getrandom::fill(&mut buf) + .map_err(|_| vm.new_os_error("Failed to generate random bytes"))?; + Ok(PyBytesRef::from(vm.ctx.new_bytes(buf))) + } + + #[pyfunction] + fn RAND_pseudo_bytes(n: i32, vm: &VirtualMachine) -> PyResult<(PyBytesRef, bool)> { + let bytes = RAND_bytes(n, vm)?; + Ok((bytes, true)) + } +} diff --git a/crates/vm/src/stdlib/_ctypes.rs b/crates/vm/src/stdlib/_ctypes.rs index e4857d0ee..0cab54e6e 100644 --- a/crates/vm/src/stdlib/_ctypes.rs +++ b/crates/vm/src/stdlib/_ctypes.rs @@ -94,6 +94,8 @@ pub(crate) mod _ctypes { use super::{PyCArray, PyCData, PyCPointer, PyCSimple, PyCStructure, PyCUnion}; use crate::builtins::{PyType, PyTypeRef}; use crate::class::StaticType; + #[cfg(windows)] + use crate::convert::ToPyException; use crate::convert::ToPyObject; use crate::function::{Either, OptionalArg}; use crate::types::Representable; @@ -419,12 +421,15 @@ pub(crate) mod _ctypes { #[pyfunction(name = "LoadLibrary")] fn load_library_windows( name: String, - _load_flags: OptionalArg, - _vm: &VirtualMachine, - ) -> usize { - // TODO: audit functions first - // TODO: load_flags - rustpython_host_env::ctypes::open_library(&name).unwrap() + load_flags: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let wide = widestring::WideCString::from_str(&name) + .map_err(|_| vm.new_value_error("embedded null character"))?; + let flags = load_flags.unwrap_or(0) as u32; + rustpython_host_env::ctypes::load_library_ex_w(&wide, flags) + .map(|module| module as usize) + .map_err(|error| error.to_pyexception(vm)) } #[cfg(not(windows))] @@ -456,6 +461,14 @@ pub(crate) mod _ctypes { } } + #[cfg(windows)] + #[pyfunction(name = "FreeLibrary")] + fn free_library(handle: usize, vm: &VirtualMachine) -> PyResult<()> { + rustpython_host_env::ctypes::free_library(handle as _) + .map_err(|error| error.to_pyexception(vm)) + } + + #[cfg(not(windows))] #[pyfunction(name = "FreeLibrary")] fn free_library(handle: usize) { rustpython_host_env::ctypes::drop_library(handle); diff --git a/crates/vm/src/stdlib/_thread.rs b/crates/vm/src/stdlib/_thread.rs index 6864a9320..d08d6a6f8 100644 --- a/crates/vm/src/stdlib/_thread.rs +++ b/crates/vm/src/stdlib/_thread.rs @@ -21,10 +21,10 @@ pub(crate) mod _thread { use crate::{ AsObject, Py, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{ - PyBaseExceptionRef, PyDictRef, PyIntRef, PyStr, PyTupleRef, PyType, PyTypeRef, - PyUtf8StrRef, + PyBaseExceptionRef, PyDictRef, PyIntRef, PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, }, common::{lock::PyMutex, wtf8::Wtf8Buf}, + convert::ToPyException, frame::FrameObjectRef, function::{ArgCallable, FuncArgs, KwArgs, OptionalArg, PySetterValue, TimeoutSeconds}, object::{Traverse, TraverseFn}, @@ -410,26 +410,86 @@ pub(crate) mod _thread { vm.state.stop_the_world.reset_stats(); } - /// Set the name of the current thread - #[pyfunction] - fn set_name(name: PyUtf8StrRef) { - #[cfg(any(unix, windows))] - host_thread::set_current_thread_name(name.as_str()); - #[cfg(not(any(unix, windows)))] - let _ = name; + #[cfg(any(windows, target_os = "linux", target_os = "macos"))] + #[pyattr] + const _NAME_MAXLEN: usize = host_thread::NAME_MAXLEN; + + /// Truncate a Windows thread name to `_NAME_MAXLEN` UTF-16 code units, + /// dropping a trailing non-BMP character that would not fit as a pair. + #[cfg(windows)] + fn truncate_thread_name_wide(name: &crate::common::wtf8::Wtf8) -> Vec { + let encoded: Vec = name.encode_wide().collect(); + let mut units = Vec::new(); + let mut i = 0; + while i < encoded.len() { + let unit = encoded[i]; + if unit == 0 { + break; + } + let width = match encoded.get(i + 1) { + Some(&lo) + if (0xD800..=0xDBFF).contains(&unit) && (0xDC00..=0xDFFF).contains(&lo) => + { + 2 + } + _ => 1, + }; + if units.len() + width > host_thread::NAME_MAXLEN { + break; + } + units.extend_from_slice(&encoded[i..i + width]); + i += width; + } + units.push(0); + units } - /// Get OS-level thread ID (pthread_self on Unix) + #[cfg(any(windows, target_os = "linux", target_os = "macos"))] + #[pyfunction] + fn set_name(name: PyStrRef, vm: &VirtualMachine) -> PyResult<()> { + #[cfg(windows)] + { + let units = truncate_thread_name_wide(name.as_wtf8()); + host_thread::set_current_thread_name_wide(&units).map_err(|e| e.to_pyexception(vm))?; + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + let os_name = vm.fsencode(&name)?; + host_thread::set_current_thread_name_bytes(os_name.as_encoded_bytes()); + } + Ok(()) + } + + #[cfg(any(windows, target_os = "linux", target_os = "macos"))] + #[pyfunction(name = "_get_name")] + fn get_name(vm: &VirtualMachine) -> PyResult { + #[cfg(windows)] + { + let units = + host_thread::current_thread_name_wide().map_err(|e| e.to_pyexception(vm))?; + Ok(vm.ctx.new_str(Wtf8Buf::from_wide(&units)).into()) + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + let bytes = + host_thread::current_thread_name(host_thread::NAME_MAXLEN.saturating_add(1)) + .map_err(|e| e.to_pyexception(vm))?; + let os = unsafe { std::ffi::OsString::from_encoded_bytes_unchecked(bytes) }; + Ok(vm.fsdecode(os).into()) + } + } + + /// Get OS-level thread ID (pthread_self on Unix, GetCurrentThreadId on Windows) /// This is important for fork compatibility - the ID must remain stable after fork fn current_thread_id() -> u64 { cfg_select! { - unix => host_thread::current_thread_id(), + any(unix, windows) => host_thread::current_thread_id(), _ => thread_to_rust_id(&thread::current()), } } - /// Convert Rust thread to ID (used for non-unix platforms) - #[cfg(not(unix))] + /// Convert Rust thread to ID (used when no native thread id exists) + #[cfg(not(any(unix, windows)))] fn thread_to_rust_id(t: &thread::Thread) -> u64 { use core::hash::{Hash, Hasher}; @@ -466,7 +526,12 @@ pub(crate) mod _thread { handle.as_pthread_t() as _ } - #[cfg(not(unix))] + #[cfg(windows)] + { + use std::os::windows::io::AsRawHandle; + host_thread::thread_id_from_handle(handle.as_raw_handle()) + } + #[cfg(not(any(unix, windows)))] { thread_to_rust_id(handle.thread()) } diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index db806e067..3b1b20116 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -648,7 +648,7 @@ impl Interpreter { /// 1. Handle exit exception and turn it to exit code. /// 1. Call threading._shutdown() to join non-daemon threads. /// 1. Run atexit exit functions. - /// 1. Set finalizing flag (suppresses unraisable exceptions from __del__). + /// 1. Set finalizing flag and hang remaining daemon threads. /// 1. Forced GC collection pass (collect cycles while builtins are available). /// 1. Module finalization (finalize_modules). /// 1. Clear interpreter-owned cross-interpreter data. @@ -690,9 +690,18 @@ impl Interpreter { #[cfg(feature = "threading")] finalize_subinterpreters(vm); - // Now suppress unraisable exceptions from daemon threads and __del__ - // methods during the rest of shutdown. + // Remaining daemon threads hang in `_PyThreadState_HangThread` when + // they next attach (or at the eval-breaker if they are already + // running). Their `_ThreadHandle` stays not-done so `is_alive()` + // is still true for `join()` during the GC that follows. + #[cfg(feature = "threading")] + vm.state.stop_the_world.stop_the_world(&vm.state); vm.state.finalizing.store(true, Ordering::Release); + #[cfg(feature = "threading")] + { + thread::set_other_threads_shutting_down(&vm.state); + vm.state.stop_the_world.start_the_world(&vm.state); + } // GC pass - collect cycles before module cleanup vm.state.gc.collect_force(2); diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 7168789bd..ac6465e28 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -280,8 +280,11 @@ impl StopTheWorldState { // HEAD_LOCK-guarded stop-the-world bookkeeping. self.requested.store(true, Ordering::Release); let count = registry - .keys() - .filter(|&&thread_id| thread_id != requester) + .iter() + .filter(|(thread_id, slot)| { + **thread_id != requester + && slot.state.load(Ordering::Relaxed) != thread::THREAD_SHUTTING_DOWN + }) .count(); let count = (count.min(i64::MAX as usize)) as i64; self.thread_countdown.store(count, Ordering::Release); @@ -304,7 +307,7 @@ impl StopTheWorldState { /// Try to CAS detached threads directly to SUSPENDED and check whether /// stop countdown reached zero after parking detached threads. fn park_detached_threads(&self, state: &PyGlobalState) -> bool { - use thread::{THREAD_ATTACHED, THREAD_DETACHED, THREAD_SUSPENDED}; + use thread::{THREAD_ATTACHED, THREAD_DETACHED, THREAD_SHUTTING_DOWN, THREAD_SUSPENDED}; let requester = self.requester.load(Ordering::Relaxed); let registry = state.thread_frames.lock(); let mut attached_seen = 0u64; @@ -345,6 +348,9 @@ impl StopTheWorldState { slot.stop_requested.store(false, Ordering::Release); // Another path parked it first. } + Err(THREAD_SHUTTING_DOWN) => { + slot.stop_requested.store(false, Ordering::Release); + } Err(other) => { debug_assert!( false, @@ -358,7 +364,7 @@ impl StopTheWorldState { // Thread is in bytecode — it will see `requested` and self-suspend attached_seen = attached_seen.saturating_add(1); } - // THREAD_SUSPENDED → already parked + // THREAD_SUSPENDED / THREAD_SHUTTING_DOWN → already parked } if attached_seen != 0 { self.stats_attached_seen @@ -501,7 +507,7 @@ impl StopTheWorldState { /// Resume all suspended threads (`start_the_world`). pub fn start_the_world(&self, state: &PyGlobalState) { - use thread::{THREAD_DETACHED, THREAD_SUSPENDED}; + use thread::{THREAD_DETACHED, THREAD_SHUTTING_DOWN, THREAD_SUSPENDED}; let requester = self.requester.load(Ordering::Relaxed); stw_trace(format_args!("start begin requester={requester}")); let registry = state.thread_frames.lock(); @@ -525,6 +531,11 @@ impl StopTheWorldState { slot.stop_requested.store(false, Ordering::Release); let state = slot.state.load(Ordering::Relaxed); + if state == THREAD_SHUTTING_DOWN { + // `_PyThreadState_RemoveExcept` + SetShuttingDown already + // took this thread off the resume path. Leave it hanging. + continue; + } debug_assert!( state == THREAD_SUSPENDED, "non-requester thread not suspended at start-the-world: id={id} state={state}" @@ -624,7 +635,7 @@ impl StopTheWorldState { /// already SUSPENDED when a new stop counts it neither notifies nor is /// force-parked again, so an edge-based countdown could never reach zero. fn all_non_requester_suspended(&self, state: &PyGlobalState) -> bool { - use thread::THREAD_SUSPENDED; + use thread::{THREAD_SHUTTING_DOWN, THREAD_SUSPENDED}; let requester = self.requester.load(Ordering::Relaxed); let registry = state.thread_frames.lock(); @@ -636,7 +647,8 @@ impl StopTheWorldState { if id == requester { continue; } - if slot.state.load(Ordering::Acquire) != THREAD_SUSPENDED { + let slot_state = slot.state.load(Ordering::Acquire); + if slot_state != THREAD_SUSPENDED && slot_state != THREAD_SHUTTING_DOWN { return false; } } @@ -645,7 +657,7 @@ impl StopTheWorldState { #[cfg(debug_assertions)] fn debug_assert_all_non_requester_suspended(&self, state: &PyGlobalState) { - use thread::THREAD_SUSPENDED; + use thread::{THREAD_SHUTTING_DOWN, THREAD_SUSPENDED}; let requester = self.requester.load(Ordering::Relaxed); let registry = state.thread_frames.lock(); @@ -660,7 +672,7 @@ impl StopTheWorldState { let state = slot.state.load(Ordering::Relaxed); debug_assert!( - state == THREAD_SUSPENDED, + state == THREAD_SUSPENDED || state == THREAD_SHUTTING_DOWN, "non-requester thread not suspended during stop-the-world: id={id} state={state}" ); } @@ -3014,9 +3026,11 @@ impl VirtualMachine { pub fn check_signals(&self) -> PyResult<()> { #[cfg(feature = "threading")] if self.state.finalizing.load(Ordering::Acquire) && !self.is_main_thread() { - // once finalization starts, - // non-main Python threads should stop running bytecode. - return Err(self.new_system_exit(vec![].into())); + // `_PyThreadState_MustExit` → `_PyThreadState_HangThread`. + // Do not return SystemExit: that would mark the handle done and + // make `Thread.is_alive()` false for a daemon still forced off + // during finalize. + thread::hang_current_thread(&self.state); } // Suspend this thread if stop-the-world is in progress diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index 7b9f5102c..7ec387f29 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -23,16 +23,20 @@ use itertools::Itertools; use std::collections::HashMap; use std::thread_local; -// Thread states for stop-the-world support. +// Thread states for stop-the-world support (`_Py_THREAD_*`). // DETACHED: not executing Python bytecode (in native code, or idle) // ATTACHED: actively executing Python bytecode // SUSPENDED: parked by a stop-the-world request +// SHUTTING_DOWN: interpreter is finalizing; the OS thread must hang +// (`_PyThreadState_HangThread`) and must not look done to `_ThreadHandle`. #[cfg(feature = "threading")] pub const THREAD_DETACHED: i32 = 0; #[cfg(feature = "threading")] pub const THREAD_ATTACHED: i32 = 1; #[cfg(feature = "threading")] pub const THREAD_SUSPENDED: i32 = 2; +#[cfg(feature = "threading")] +pub const THREAD_SHUTTING_DOWN: i32 = 3; /// Per-thread shared state for sys._current_frames() and sys._current_exceptions(). /// The exception field uses atomic operations for lock-free cross-thread reads. @@ -58,7 +62,7 @@ pub struct ThreadSlot { #[cfg(not(unix))] pub frames: parking_lot::Mutex>, pub exception: crate::PyAtomicRef>, - /// Thread state for stop-the-world: DETACHED / ATTACHED / SUSPENDED + /// Thread state for stop-the-world: DETACHED / ATTACHED / SUSPENDED / SHUTTING_DOWN pub state: core::sync::atomic::AtomicI32, /// Per-thread stop request bit (eval breaker equivalent). pub stop_requested: core::sync::atomic::AtomicBool, @@ -502,6 +506,59 @@ fn wait_while_suspended(slot: &ThreadSlot) -> u64 { wait_yields } +/// `PyThread_hang_thread`: park this OS thread forever. +#[cfg(feature = "threading")] +fn hang_thread() -> ! { + loop { + std::thread::park(); + } +} + +/// `_PyThreadState_HangThread`: this thread may no longer run Python. +/// +/// Mark the slot shutting-down (so later stop-the-world requests do not wait +/// for it) and never return. The matching `_ThreadHandle` stays not-done, so +/// `Thread.is_alive()` remains true for a daemon forced off during finalize. +#[cfg(feature = "threading")] +pub fn hang_current_thread(state: &PyGlobalState) -> ! { + CURRENT_THREAD_SLOT.with(|slot| { + if let Some(s) = slot.borrow().as_ref() { + let prev = s.state.swap(THREAD_SHUTTING_DOWN, Ordering::AcqRel); + if prev == THREAD_ATTACHED { + crate::object::qsbr::QSBR.offline(&s.qsbr); + } + s.stop_requested.store(false, Ordering::Release); + } + }); + if state.stop_the_world.requested.load(Ordering::Acquire) { + state.stop_the_world.notify_thread_gone(); + } + hang_thread(); +} + +/// `_PyThreadState_SetShuttingDown` on every non-current thread. +/// +/// Call while the world is stopped. Wake parked threads so they observe +/// `SHUTTING_DOWN` and hang on the next attach, instead of resuming Python. +#[cfg(feature = "threading")] +pub fn set_other_threads_shutting_down(state: &PyGlobalState) { + let current = crate::stdlib::_thread::get_ident(); + let registry = state.thread_frames.lock(); + + #[expect( + clippy::iter_over_hash_type, + reason = "Iteration order doesn't matter here" + )] + for (&id, slot) in registry.iter() { + if id == current { + continue; + } + slot.stop_requested.store(false, Ordering::Release); + slot.state.store(THREAD_SHUTTING_DOWN, Ordering::Release); + slot.thread.unpark(); + } +} + #[cfg(feature = "threading")] fn attach_thread(vm: &VirtualMachine) { CURRENT_THREAD_SLOT.with(|slot| { @@ -526,6 +583,10 @@ fn attach_thread(vm: &VirtualMachine) { vm.state.stop_the_world.add_attach_wait_yields(wait_yields); // Retry CAS } + Err(THREAD_SHUTTING_DOWN) => { + super::stw_trace(format_args!("attach hang shutting-down")); + hang_thread(); + } Err(state) => { debug_assert!(false, "unexpected thread state in attach: {state}"); break; @@ -750,6 +811,11 @@ fn do_suspend(state: &PyGlobalState) { super::stw_trace(format_args!("suspend skip already-suspended")); return; } + Some(Err(THREAD_SHUTTING_DOWN)) => { + s.stop_requested.store(false, Ordering::Release); + super::stw_trace(format_args!("suspend hang shutting-down")); + hang_thread(); + } Some(Err(state)) => { debug_assert!(false, "unexpected thread state in suspend: {state}"); return; @@ -782,6 +848,10 @@ fn do_suspend(state: &PyGlobalState) { stw.add_suspend_wait_yields(extra_wait); } Err(THREAD_ATTACHED) => break, + Err(THREAD_SHUTTING_DOWN) => { + super::stw_trace(format_args!("suspend resume hang shutting-down")); + hang_thread(); + } Err(state) => { debug_assert!(false, "unexpected post-suspend state: {state}"); break; diff --git a/crates/wasm/Cargo.toml b/crates/wasm/Cargo.toml index 547f84d24..ab34320af 100644 --- a/crates/wasm/Cargo.toml +++ b/crates/wasm/Cargo.toml @@ -13,7 +13,7 @@ crate-type = ["cdylib", "rlib"] [features] default = ["freeze-stdlib"] -freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib/freeze-stdlib", "rustpython-stdlib"] +freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib/freeze-stdlib", "rustpython-stdlib", "rustpython-stdlib/ssl"] no-start-func = [] [dependencies] diff --git a/crates/wasm/src/lib.rs b/crates/wasm/src/lib.rs index 2461d9a9d..990d1bcc1 100644 --- a/crates/wasm/src/lib.rs +++ b/crates/wasm/src/lib.rs @@ -4,8 +4,6 @@ pub mod browser_module; pub mod convert; pub mod js_module; #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] -mod socket; -#[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] mod ssl; pub mod vm_class; pub mod wasm_builtins; diff --git a/crates/wasm/src/vm_class.rs b/crates/wasm/src/vm_class.rs index 0b1e5575f..4b3d6f4cb 100644 --- a/crates/wasm/src/vm_class.rs +++ b/crates/wasm/src/vm_class.rs @@ -86,9 +86,8 @@ impl StoredVirtualMachine { .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB); } - // Add wasm-specific modules. Browser `_ssl` / `_socket` live here - // rather than rustpython-stdlib: there are no BSD sockets, and rustls - // is driven through MemoryBIO / wrap_bio. + // Browser rustls `_ssl` overrides the rustls-free stdlib `_ssl`. + // `_socket` is rustpython-stdlib's wasm shim (`socket_wasm.rs`). let js_def = js_module::module_def(&builder.ctx); builder = builder.add_native_module(js_def); @@ -96,8 +95,7 @@ impl StoredVirtualMachine { { install_browser_tls_provider(); let ssl_def = crate::ssl::module_def(&builder.ctx); - let socket_def = crate::socket::module_def(&builder.ctx); - builder = builder.add_native_modules(&[ssl_def, socket_def]); + builder = builder.add_native_module(ssl_def); } if inject_browser_module { diff --git a/src/lib.rs b/src/lib.rs index b3560ad94..0e3d0b588 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -71,7 +71,8 @@ pub use shell::run_shell; #[cfg(all( feature = "ssl", - not(any(feature = "ssl-rustls", feature = "ssl-openssl")) + not(any(feature = "ssl-rustls", feature = "ssl-openssl")), + not(target_arch = "wasm32"), ))] compile_error!( "Feature \"ssl\" is now enabled by either \"ssl-rustls\" or \"ssl-openssl\". Do not manually pass \"ssl\" feature. To enable ssl-openssl, use --no-default-features to disable ssl-rustls*"