From 38d54fa0cdfd36823f2bf31e2597f986e20c97c0 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 16 Feb 2026 01:23:26 +0900 Subject: [PATCH] impl more ctypes - Add __pointer_type__ getter/setter and StgInfo cache - Add kept_refs (HashMap-keyed) for c_char_p lifetime - Simplify ensure_z_null_terminated to 2-value return - Add RawMemoryBuffer for zero-copy memoryview_at_addr - Add bitfield support in PyCField with validation - Add CArgObject offset handling in extract_ptr_from_arg - Add deprecation warning for _pack_ without _layout_ - Fix comparison TypeError message format - Fix unsupported binop error message format - Mark failing CI tests as expectedFailure/skip --- Lib/test/test_ctypes/test_dlerror.py | 2 + Lib/test/test_ctypes/test_dllist.py | 4 + Lib/test/test_ctypes/test_python_api.py | 4 + Lib/test/test_ctypes/test_values.py | 4 + Lib/test/test_ctypes/test_win32.py | 2 + crates/vm/src/protocol/object.rs | 7 +- crates/vm/src/stdlib/ctypes.rs | 18 +- crates/vm/src/stdlib/ctypes/array.rs | 15 +- crates/vm/src/stdlib/ctypes/base.rs | 396 +++++++++++++++++++---- crates/vm/src/stdlib/ctypes/function.rs | 106 +++++- crates/vm/src/stdlib/ctypes/pointer.rs | 45 ++- crates/vm/src/stdlib/ctypes/simple.rs | 32 +- crates/vm/src/stdlib/ctypes/structure.rs | 184 ++++++++++- crates/vm/src/stdlib/ctypes/union.rs | 142 +++++++- crates/vm/src/vm/vm_new.rs | 2 +- 15 files changed, 859 insertions(+), 104 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 5658234f9..ea2d97d90 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -55,6 +55,8 @@ class TestNullDlsym(unittest.TestCase): this 'dlsym returned NULL -> throw Error' rule. """ + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_null_dlsym(self): import subprocess import tempfile diff --git a/Lib/test/test_ctypes/test_dllist.py b/Lib/test/test_ctypes/test_dllist.py index 15603dc3d..0e7c65127 100644 --- a/Lib/test/test_ctypes/test_dllist.py +++ b/Lib/test/test_ctypes/test_dllist.py @@ -26,6 +26,8 @@ else: ) class ListSharedLibraries(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.skipIf(not APPLE, "TODO: RUSTPYTHON") def test_lists_system(self): dlls = ctypes.util.dllist() @@ -34,6 +36,8 @@ class ListSharedLibraries(unittest.TestCase): any(lib in dll for dll in dlls for lib in KNOWN_LIBRARIES), f"loaded={dlls}" ) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_lists_updates(self): dlls = ctypes.util.dllist() diff --git a/Lib/test/test_ctypes/test_python_api.py b/Lib/test/test_ctypes/test_python_api.py index 1072a1098..a1ee8a0de 100644 --- a/Lib/test/test_ctypes/test_python_api.py +++ b/Lib/test/test_ctypes/test_python_api.py @@ -7,6 +7,8 @@ from ctypes import (pythonapi, POINTER, create_string_buffer, sizeof, class PythonAPITestCase(unittest.TestCase): + # TODO: RUSTPYTHON - requires pythonapi (Python C API) + @unittest.expectedFailure def test_PyBytes_FromStringAndSize(self): PyBytes_FromStringAndSize = pythonapi.PyBytes_FromStringAndSize @@ -57,6 +59,8 @@ class PythonAPITestCase(unittest.TestCase): del pyobj self.assertEqual(sys.getrefcount(s), ref) + # TODO: RUSTPYTHON - requires pythonapi (Python C API) + @unittest.expectedFailure def test_PyOS_snprintf(self): PyOS_snprintf = pythonapi.PyOS_snprintf PyOS_snprintf.argtypes = POINTER(c_char), c_size_t, c_char_p diff --git a/Lib/test/test_ctypes/test_values.py b/Lib/test/test_ctypes/test_values.py index 8d1ee25ac..18554e193 100644 --- a/Lib/test/test_ctypes/test_values.py +++ b/Lib/test/test_ctypes/test_values.py @@ -39,6 +39,8 @@ class ValuesTestCase(unittest.TestCase): class PythonValuesTestCase(unittest.TestCase): """This test only works when python itself is a dll/shared library""" + # TODO: RUSTPYTHON - requires pythonapi (Python C API) + @unittest.expectedFailure def test_optimizeflag(self): # This test accesses the Py_OptimizeFlag integer, which is # exported by the Python dll and should match the sys.flags value @@ -46,6 +48,8 @@ class PythonValuesTestCase(unittest.TestCase): opt = c_int.in_dll(pythonapi, "Py_OptimizeFlag").value self.assertEqual(opt, sys.flags.optimize) + # TODO: RUSTPYTHON - requires pythonapi (Python C API) + @unittest.expectedFailure @thread_unsafe('overrides frozen modules') def test_frozentable(self): # Python exports a PyImport_FrozenModules symbol. This is a diff --git a/Lib/test/test_ctypes/test_win32.py b/Lib/test/test_ctypes/test_win32.py index 7d5133221..bb2fc0ca2 100644 --- a/Lib/test/test_ctypes/test_win32.py +++ b/Lib/test/test_ctypes/test_win32.py @@ -13,6 +13,8 @@ from ._support import Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYP @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class FunctionCallTestCase(unittest.TestCase): + # TODO: RUSTPYTHON: SEH not implemented, crashes with STATUS_ACCESS_VIOLATION + @unittest.skip("TODO: RUSTPYTHON") @unittest.skipUnless('MSC' in sys.version, "SEH only supported by MSC") @unittest.skipIf(sys.executable.lower().endswith('_d.exe'), "SEH not enabled in debug builds") diff --git a/crates/vm/src/protocol/object.rs b/crates/vm/src/protocol/object.rs index d0e068e50..d2e4b31aa 100644 --- a/crates/vm/src/protocol/object.rs +++ b/crates/vm/src/protocol/object.rs @@ -322,7 +322,12 @@ impl PyObject { match op { PyComparisonOp::Eq => Ok(Either::B(self.is(&other))), PyComparisonOp::Ne => Ok(Either::B(!self.is(&other))), - _ => Err(vm.new_unsupported_bin_op_error(self, other, op.operator_token())), + _ => Err(vm.new_type_error(format!( + "'{}' not supported between instances of '{}' and '{}'", + op.operator_token(), + self.class().name(), + other.class().name() + ))), } } #[inline(always)] diff --git a/crates/vm/src/stdlib/ctypes.rs b/crates/vm/src/stdlib/ctypes.rs index dd6a6e390..441a5ce37 100644 --- a/crates/vm/src/stdlib/ctypes.rs +++ b/crates/vm/src/stdlib/ctypes.rs @@ -70,18 +70,26 @@ impl Py { } impl PyType { - /// Check if StgInfo is already initialized - prevent double initialization + /// Check if StgInfo is already initialized. + /// Raises SystemError if already initialized. pub(crate) fn check_not_initialized(&self, vm: &VirtualMachine) -> PyResult<()> { if let Some(stg_info) = self.get_type_data::() && stg_info.initialized { return Err(vm.new_exception_msg( vm.ctx.exceptions.system_error.to_owned(), - format!("StgInfo of '{}' is already initialized.", self.name()), + format!("class \"{}\" already initialized", self.name()), )); } Ok(()) } + + /// Check if StgInfo is already initialized, returning true if so. + /// Unlike check_not_initialized, does not raise an error. + pub(crate) fn is_initialized(&self) -> bool { + self.get_type_data::() + .is_some_and(|stg_info| stg_info.initialized) + } } // Dynamic type check helpers for PyCData @@ -988,6 +996,11 @@ pub(crate) mod _ctypes { super::function::INTERNAL_CAST_ADDR } + #[pyattr] + fn _memoryview_at_addr(_vm: &VirtualMachine) -> usize { + super::function::INTERNAL_MEMORYVIEW_AT_ADDR + } + #[pyfunction] fn _cast( obj: PyObjectRef, @@ -1293,6 +1306,7 @@ pub(crate) mod _ctypes { structure::PyCStructType::make_class(ctx); union::PyCUnionType::make_class(ctx); function::PyCFuncPtrType::make_class(ctx); + function::RawMemoryBuffer::make_class(ctx); extend_module!(vm, module, { "_CData" => PyCData::make_class(ctx), diff --git a/crates/vm/src/stdlib/ctypes/array.rs b/crates/vm/src/stdlib/ctypes/array.rs index ecb9c0ecc..843dd7d5b 100644 --- a/crates/vm/src/stdlib/ctypes/array.rs +++ b/crates/vm/src/stdlib/ctypes/array.rs @@ -301,6 +301,16 @@ impl Initializer for PyCArrayType { #[pyclass(flags(IMMUTABLETYPE), with(Initializer, AsNumber))] impl PyCArrayType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + #[pymethod] fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { // zelf is the array type class that from_param was called on @@ -777,8 +787,9 @@ impl PyCArray { let (ptr_val, converted) = if value.is(&vm.ctx.none) { (0usize, None) } else if let Some(bytes) = value.downcast_ref::() { - let (c, ptr) = super::base::ensure_z_null_terminated(bytes, vm); - (ptr, Some(c)) + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + zelf.0.keep_alive(index, kept_alive); + (ptr, Some(value.to_owned())) } else if let Ok(int_val) = value.try_index(vm) { (int_val.as_bigint().to_usize().unwrap_or(0), None) } else { diff --git a/crates/vm/src/stdlib/ctypes/base.rs b/crates/vm/src/stdlib/ctypes/base.rs index 0ef10ea6c..ba2b98733 100644 --- a/crates/vm/src/stdlib/ctypes/base.rs +++ b/crates/vm/src/stdlib/ctypes/base.rs @@ -1,9 +1,9 @@ use super::array::{WCHAR_SIZE, wchar_from_bytes, wchar_to_bytes}; -use crate::builtins::{PyBytes, PyDict, PyMemoryView, PyStr, PyType, PyTypeRef}; +use crate::builtins::{PyBytes, PyDict, PyMemoryView, PyStr, PyTuple, PyType, PyTypeRef}; use crate::class::StaticType; use crate::function::{ArgBytesLike, OptionalArg, PySetterValue}; use crate::protocol::{BufferMethods, PyBuffer}; -use crate::types::{GetDescriptor, Representable}; +use crate::types::{Constructor, GetDescriptor, Representable}; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, }; @@ -97,6 +97,9 @@ pub struct StgInfo { // FFI field types for structure/union passing (inherited from base class) pub ffi_field_types: Vec, + + // Cached pointer type (non-inheritable via descriptor) + pub pointer_type: Option, } // StgInfo is stored in type_data which requires Send + Sync. @@ -141,6 +144,7 @@ impl Default for StgInfo { paramfunc: ParamFunc::None, big_endian: cfg!(target_endian = "big"), // native endian by default ffi_field_types: Vec::new(), + pointer_type: None, } } } @@ -161,6 +165,7 @@ impl StgInfo { paramfunc: ParamFunc::None, big_endian: cfg!(target_endian = "big"), // native endian by default ffi_field_types: Vec::new(), + pointer_type: None, } } @@ -212,6 +217,7 @@ impl StgInfo { paramfunc: ParamFunc::Array, big_endian: cfg!(target_endian = "big"), // native endian by default ffi_field_types: Vec::new(), + pointer_type: None, } } @@ -293,6 +299,34 @@ impl StgInfo { } } +/// __pointer_type__ getter for ctypes metaclasses. +/// Reads from StgInfo.pointer_type (non-inheritable). +pub(super) fn pointer_type_get(zelf: &Py, vm: &VirtualMachine) -> PyResult { + zelf.stg_info_opt() + .and_then(|info| info.pointer_type.clone()) + .ok_or_else(|| { + vm.new_attribute_error(format!( + "type {} has no attribute '__pointer_type__'", + zelf.name() + )) + }) +} + +/// __pointer_type__ setter for ctypes metaclasses. +/// Writes to StgInfo.pointer_type (non-inheritable). +pub(super) fn pointer_type_set( + zelf: &Py, + value: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult<()> { + if let Some(mut info) = zelf.get_type_data_mut::() { + info.pointer_type = Some(value); + Ok(()) + } else { + Err(vm.new_attribute_error(format!("cannot set __pointer_type__ on {}", zelf.name()))) + } +} + /// Get PEP3118 format string for a field type /// Returns the format string considering byte order pub(super) fn get_field_format( @@ -380,25 +414,20 @@ fn vec_to_bytes(vec: Vec) -> Vec { unsafe { Vec::from_raw_parts(ptr, len, cap) } } -/// Ensure PyBytes is null-terminated. Returns (PyBytes to keep, pointer). -/// If already contains null, returns original. Otherwise creates new with null appended. +/// Ensure PyBytes data is null-terminated. Returns (kept_alive_obj, pointer). +/// The caller must keep the returned object alive to keep the pointer valid. pub(super) fn ensure_z_null_terminated( bytes: &PyBytes, vm: &VirtualMachine, ) -> (PyObjectRef, usize) { let data = bytes.as_bytes(); - if data.contains(&0) { - // Already has null, use original - let original: PyObjectRef = vm.ctx.new_bytes(data.to_vec()).into(); - (original, data.as_ptr() as usize) - } else { - // Create new with null appended - let mut buffer = data.to_vec(); + let mut buffer = data.to_vec(); + if !buffer.ends_with(&[0]) { buffer.push(0); - let ptr = buffer.as_ptr() as usize; - let new_bytes: PyObjectRef = vm.ctx.new_bytes(buffer).into(); - (new_bytes, ptr) } + let ptr = buffer.as_ptr() as usize; + let kept_alive: PyObjectRef = vm.ctx.new_bytes(buffer).into(); + (kept_alive, ptr) } /// Convert str to null-terminated wchar_t buffer. Returns (PyBytes holder, pointer). @@ -434,6 +463,11 @@ pub struct PyCData { pub objects: PyRwLock>, /// number of references we need (b_length) pub length: AtomicCell, + /// References kept alive but not visible in _objects. + /// Used for null-terminated c_char_p buffer copies, since + /// RustPython's PyBytes lacks CPython's internal trailing null. + /// Keyed by unique_key (hierarchical) so nested fields don't collide. + pub(super) kept_refs: PyRwLock>, } impl PyCData { @@ -446,6 +480,7 @@ impl PyCData { index: AtomicCell::new(0), objects: PyRwLock::new(None), length: AtomicCell::new(stg_info.length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } @@ -458,6 +493,7 @@ impl PyCData { index: AtomicCell::new(0), objects: PyRwLock::new(objects), length: AtomicCell::new(0), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } @@ -474,6 +510,7 @@ impl PyCData { index: AtomicCell::new(0), objects: PyRwLock::new(objects), length: AtomicCell::new(length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } @@ -494,6 +531,7 @@ impl PyCData { index: AtomicCell::new(0), objects: PyRwLock::new(None), length: AtomicCell::new(0), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } @@ -516,6 +554,7 @@ impl PyCData { index: AtomicCell::new(idx), objects: PyRwLock::new(None), length: AtomicCell::new(length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } @@ -542,6 +581,7 @@ impl PyCData { index: AtomicCell::new(idx), objects: PyRwLock::new(None), length: AtomicCell::new(0), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } @@ -576,6 +616,7 @@ impl PyCData { index: AtomicCell::new(0), objects: PyRwLock::new(Some(objects_dict.into())), length: AtomicCell::new(length), + kept_refs: PyRwLock::new(std::collections::HashMap::new()), } } @@ -797,6 +838,22 @@ impl PyCData { Ok(()) } + /// Keep a reference alive without exposing it in _objects. + /// Walks up to root object (same as keep_ref) so the reference + /// lives as long as the owning ctypes object. + /// Uses unique_key (hierarchical) so nested fields don't collide. + pub fn keep_alive(&self, index: usize, obj: PyObjectRef) { + let key = self.unique_key(index); + if let Some(base_obj) = self.base.read().clone() { + let root = Self::find_root_object(&base_obj); + if let Some(cdata) = root.downcast_ref::() { + cdata.kept_refs.write().insert(key, obj); + return; + } + } + self.kept_refs.write().insert(key, obj); + } + /// Find the root object (one without a base) by walking up the base chain fn find_root_object(obj: &PyObject) -> PyObjectRef { // Try to get base from different ctypes types @@ -942,12 +999,68 @@ impl PyCData { return Ok(()); } + // For array fields with tuple/list input, instantiate the array type + // and unpack elements as positional args (Array_init expects *args) + if let Some(proto_type) = proto.downcast_ref::() + && let Some(stg) = proto_type.stg_info_opt() + && stg.element_type.is_some() + { + let items: Option> = + if let Some(tuple) = value.downcast_ref::() { + Some(tuple.to_vec()) + } else { + value + .downcast_ref::() + .map(|list| list.borrow_vec().to_vec()) + }; + if let Some(items) = items { + let array_obj = proto_type.as_object().call(items, vm).map_err(|e| { + // Wrap errors in RuntimeError with type name prefix + let type_name = proto_type.name().to_string(); + let exc_name = e.class().name().to_string(); + let exc_args = e.args(); + let exc_msg = exc_args + .first() + .and_then(|a| a.downcast_ref::().map(|s| s.to_string())) + .unwrap_or_default(); + vm.new_runtime_error(format!("({type_name}) {exc_name}: {exc_msg}")) + })?; + if let Some(arr) = array_obj.downcast_ref::() { + let arr_buffer = arr.0.buffer.read(); + let len = core::cmp::min(arr_buffer.len(), size); + self.write_bytes_at_offset(offset, &arr_buffer[..len]); + drop(arr_buffer); + self.keep_ref(index, array_obj, vm)?; + return Ok(()); + } + } + } + // Get field type code for special handling let field_type_code = proto .get_attr("_type_", vm) .ok() .and_then(|attr| attr.downcast_ref::().map(|s| s.to_string())); + // c_char_p (z type) with bytes: store original in _objects, keep + // null-terminated copy alive separately for the pointer. + if field_type_code.as_deref() == Some("z") + && let Some(bytes_val) = value.downcast_ref::() + { + let (kept_alive, ptr) = ensure_z_null_terminated(bytes_val, vm); + let mut result = vec![0u8; size]; + let addr_bytes = ptr.to_ne_bytes(); + let len = core::cmp::min(addr_bytes.len(), size); + result[..len].copy_from_slice(&addr_bytes[..len]); + if needs_swap { + result.reverse(); + } + self.write_bytes_at_offset(offset, &result); + self.keep_ref(index, value, vm)?; + self.keep_alive(index, kept_alive); + return Ok(()); + } + let (mut bytes, converted_value) = if let Some(type_code) = &field_type_code { PyCField::value_to_bytes_for_type(type_code, &value, size, vm)? } else { @@ -1111,7 +1224,7 @@ impl PyCData { // CDataType_methods - shared across all ctypes types #[pyclassmethod] - fn from_buffer( + pub(super) fn from_buffer( cls: PyTypeRef, source: PyObjectRef, offset: OptionalArg, @@ -1122,7 +1235,7 @@ impl PyCData { } #[pyclassmethod] - fn from_buffer_copy( + pub(super) fn from_buffer_copy( cls: PyTypeRef, source: ArgBytesLike, offset: OptionalArg, @@ -1134,7 +1247,7 @@ impl PyCData { } #[pyclassmethod] - fn from_address(cls: PyTypeRef, address: isize, vm: &VirtualMachine) -> PyResult { + pub(super) fn from_address(cls: PyTypeRef, address: isize, vm: &VirtualMachine) -> PyResult { let size = { let stg_info = cls.stg_info(vm)?; stg_info.size @@ -1150,7 +1263,7 @@ impl PyCData { } #[pyclassmethod] - fn in_dll( + pub(super) fn in_dll( cls: PyTypeRef, dll: PyObjectRef, name: crate::builtins::PyStrRef, @@ -1217,78 +1330,84 @@ impl PyCData { #[pyclass(name = "CField", module = "_ctypes")] #[derive(Debug, PyPayload)] pub struct PyCField { + /// Field name + pub(crate) name: String, /// Byte offset of the field within the structure/union pub(crate) offset: isize, - /// Encoded size: for bitfields (bit_size << 16) | bit_offset, otherwise byte size - pub(crate) size: isize, + /// Byte size of the underlying type + pub(crate) byte_size_val: isize, /// Index into PyCData's object array pub(crate) index: usize, /// The ctypes type for this field pub(crate) proto: PyTypeRef, /// Flag indicating if the field is anonymous (MakeAnonFields sets this) pub(crate) anonymous: bool, -} - -#[inline(always)] -const fn num_bits(size: isize) -> isize { - size >> 16 -} - -#[inline(always)] -const fn field_size(size: isize) -> isize { - size & 0xFFFF -} - -#[inline(always)] -const fn is_bitfield(size: isize) -> bool { - (size >> 16) != 0 + /// Bitfield size in bits (0 for non-bitfield) + pub(crate) bitfield_size: u16, + /// Bit offset within the storage unit (only meaningful for bitfields) + pub(crate) bit_offset_val: u16, } impl PyCField { /// Create a new CField descriptor (non-bitfield) - pub fn new(proto: PyTypeRef, offset: isize, size: isize, index: usize) -> Self { + pub fn new( + name: String, + proto: PyTypeRef, + offset: isize, + byte_size: isize, + index: usize, + ) -> Self { Self { + name, offset, - size, + byte_size_val: byte_size, index, proto, anonymous: false, + bitfield_size: 0, + bit_offset_val: 0, } } /// Create a new CField descriptor for a bitfield - #[allow(dead_code)] pub fn new_bitfield( + name: String, proto: PyTypeRef, offset: isize, - bit_size: u16, + byte_size: isize, + bitfield_size: u16, bit_offset: u16, index: usize, ) -> Self { - let encoded_size = ((bit_size as isize) << 16) | (bit_offset as isize); Self { + name, offset, - size: encoded_size, + byte_size_val: byte_size, index, proto, anonymous: false, + bitfield_size, + bit_offset_val: bit_offset, } } - /// Get the actual byte size (for non-bitfields) or bit storage size (for bitfields) - pub fn byte_size(&self) -> usize { - field_size(self.size) as usize + /// Get the byte size of the field's underlying type + pub fn get_byte_size(&self) -> usize { + self.byte_size_val as usize } /// Create a new CField from an existing field with adjusted offset and index /// Used by MakeFields to promote anonymous fields pub fn new_from_field(fdescr: &PyCField, index_offset: usize, offset_delta: isize) -> Self { Self { + name: fdescr.name.clone(), offset: fdescr.offset + offset_delta, - size: fdescr.size, + byte_size_val: fdescr.byte_size_val, index: fdescr.index + index_offset, proto: fdescr.proto.clone(), anonymous: false, // promoted fields are not anonymous themselves + bitfield_size: fdescr.bitfield_size, + bit_offset_val: fdescr.bit_offset_val, } } @@ -1298,6 +1417,107 @@ impl PyCField { } } +impl Constructor for PyCField { + type Args = crate::function::FuncArgs; + + fn py_new(_cls: &Py, args: Self::Args, vm: &VirtualMachine) -> PyResult { + // PyCField_new_impl: requires _internal_use=True + let internal_use = if let Some(v) = args.kwargs.get("_internal_use") { + v.clone().try_to_bool(vm)? + } else { + false + }; + + if !internal_use { + return Err(vm.new_type_error( + "CField is not intended to be used directly; use it via Structure or Union fields" + .to_string(), + )); + } + + let name: String = args + .kwargs + .get("name") + .ok_or_else(|| vm.new_type_error("missing required argument: 'name'"))? + .try_to_value(vm)?; + + let field_type: PyTypeRef = args + .kwargs + .get("type") + .ok_or_else(|| vm.new_type_error("missing required argument: 'type'"))? + .clone() + .downcast() + .map_err(|_| vm.new_type_error("'type' must be a ctypes type"))?; + + let byte_size: isize = args + .kwargs + .get("byte_size") + .ok_or_else(|| vm.new_type_error("missing required argument: 'byte_size'"))? + .try_to_value(vm)?; + + let byte_offset: isize = args + .kwargs + .get("byte_offset") + .ok_or_else(|| vm.new_type_error("missing required argument: 'byte_offset'"))? + .try_to_value(vm)?; + + let index: usize = args + .kwargs + .get("index") + .ok_or_else(|| vm.new_type_error("missing required argument: 'index'"))? + .try_to_value(vm)?; + + // Validate byte_size matches the type + let type_size = super::base::get_field_size(field_type.as_object(), vm)? as isize; + if byte_size != type_size { + return Err(vm.new_value_error(format!( + "byte_size {} does not match type size {}", + byte_size, type_size + ))); + } + + let bit_size_val: Option = args + .kwargs + .get("bit_size") + .map(|v| v.try_to_value(vm)) + .transpose()?; + + let bit_offset_val: Option = args + .kwargs + .get("bit_offset") + .map(|v| v.try_to_value(vm)) + .transpose()?; + + if let Some(bs) = bit_size_val { + if bs < 0 { + return Err(vm.new_value_error("number of bits invalid for bit field".to_string())); + } + let bo = bit_offset_val.unwrap_or(0); + if bo < 0 { + return Err(vm.new_value_error("bit_offset must be >= 0".to_string())); + } + let type_bits = byte_size * 8; + if bo + bs > type_bits { + return Err(vm.new_value_error(format!( + "bit field '{}' overflows its type ({} + {} > {})", + name, bo, bs, type_bits + ))); + } + Ok(Self::new_bitfield( + name, + field_type, + byte_offset, + byte_size, + bs as u16, + bo as u16, + index, + )) + } else { + Ok(Self::new(name, field_type, byte_offset, byte_size, index)) + } + } +} + impl Representable for PyCField { fn repr_str(zelf: &Py, _vm: &VirtualMachine) -> PyResult { // Get type name from proto (which is always PyTypeRef) @@ -1305,17 +1525,15 @@ impl Representable for PyCField { // Bitfield: // Regular: - if is_bitfield(zelf.size) { - let bit_offset = field_size(zelf.size); - let bits = num_bits(zelf.size); + if zelf.bitfield_size > 0 { Ok(format!( "", - tp_name, zelf.offset, bit_offset, bits + tp_name, zelf.offset, zelf.bit_offset_val, zelf.bitfield_size )) } else { Ok(format!( "", - tp_name, zelf.offset, zelf.size + tp_name, zelf.offset, zelf.byte_size_val )) } } @@ -1340,7 +1558,7 @@ impl GetDescriptor for PyCField { }; let offset = zelf.offset as usize; - let size = zelf.byte_size(); + let size = zelf.get_byte_size(); // Get PyCData from obj (works for both Structure and Union) let cdata = PyCField::get_cdata_from_obj(&obj, vm)?; @@ -1468,15 +1686,8 @@ impl PyCField { Ok((f.to_ne_bytes().to_vec(), None)) } "z" => { - // c_char_p: store pointer to null-terminated bytes - if let Some(bytes) = value.downcast_ref::() { - let (converted, ptr) = ensure_z_null_terminated(bytes, vm); - let mut result = vec![0u8; size]; - let addr_bytes = ptr.to_ne_bytes(); - let len = core::cmp::min(addr_bytes.len(), size); - result[..len].copy_from_slice(&addr_bytes[..len]); - return Ok((result, Some(converted))); - } + // c_char_p with bytes is handled in set_field before this call. + // This handles integer address and None cases. // Integer address if let Ok(int_val) = value.try_index(vm) { let v = int_val.as_bigint().to_usize().unwrap_or(0); @@ -1583,10 +1794,7 @@ impl PyCField { } } -#[pyclass( - flags(DISALLOW_INSTANTIATION, IMMUTABLETYPE), - with(Representable, GetDescriptor) -)] +#[pyclass(flags(IMMUTABLETYPE), with(Representable, GetDescriptor, Constructor))] impl PyCField { /// Get PyCData from object (works for both Structure and Union) fn get_cdata_from_obj<'a>(obj: &'a PyObjectRef, vm: &VirtualMachine) -> PyResult<&'a PyCData> { @@ -1615,7 +1823,7 @@ impl PyCField { .ok_or_else(|| vm.new_type_error("expected CField"))?; let offset = zelf.offset as usize; - let size = zelf.byte_size(); + let size = zelf.get_byte_size(); // Get PyCData from obj (works for both Structure and Union) let cdata = Self::get_cdata_from_obj(&obj, vm)?; @@ -1650,14 +1858,65 @@ impl PyCField { } } + #[pygetset] + fn name(&self) -> String { + self.name.clone() + } + + #[pygetset(name = "type")] + fn type_(&self) -> PyTypeRef { + self.proto.clone() + } + #[pygetset] fn offset(&self) -> isize { self.offset } + #[pygetset] + fn byte_offset(&self) -> isize { + self.offset + } + #[pygetset] fn size(&self) -> isize { - self.size + // Legacy: encode as (bitfield_size << 16) | bit_offset for bitfields + if self.bitfield_size > 0 { + ((self.bitfield_size as isize) << 16) | (self.bit_offset_val as isize) + } else { + self.byte_size_val + } + } + + #[pygetset] + fn byte_size(&self) -> isize { + self.byte_size_val + } + + #[pygetset] + fn bit_offset(&self) -> isize { + self.bit_offset_val as isize + } + + #[pygetset] + fn bit_size(&self, vm: &VirtualMachine) -> PyObjectRef { + if self.bitfield_size > 0 { + vm.ctx.new_int(self.bitfield_size).into() + } else { + // Non-bitfield: bit_size = byte_size * 8 + let byte_size = self.byte_size_val as i128; + vm.ctx.new_int(byte_size * 8).into() + } + } + + #[pygetset] + fn is_bitfield(&self) -> bool { + self.bitfield_size > 0 + } + + #[pygetset] + fn is_anonymous(&self) -> bool { + self.anonymous } } @@ -2139,7 +2398,12 @@ pub(super) fn set_or_init_stginfo(type_ref: &PyType, stg_info: StgInfo) { if type_ref.init_type_data(stg_info.clone()).is_err() && let Some(mut existing) = type_ref.get_type_data_mut::() { + // Preserve pointer_type cache across StgInfo replacement + let old_pointer_type = existing.pointer_type.take(); *existing = stg_info; + if existing.pointer_type.is_none() { + existing.pointer_type = old_pointer_type; + } } } diff --git a/crates/vm/src/stdlib/ctypes/function.rs b/crates/vm/src/stdlib/ctypes/function.rs index e690ffbaf..d24102ed6 100644 --- a/crates/vm/src/stdlib/ctypes/function.rs +++ b/crates/vm/src/stdlib/ctypes/function.rs @@ -31,6 +31,7 @@ use rustpython_common::lock::PyRwLock; pub(super) const INTERNAL_CAST_ADDR: usize = 1; pub(super) const INTERNAL_STRING_AT_ADDR: usize = 2; pub(super) const INTERNAL_WSTRING_AT_ADDR: usize = 3; +pub(super) const INTERNAL_MEMORYVIEW_AT_ADDR: usize = 4; // Thread-local errno storage for ctypes std::thread_local! { @@ -512,7 +513,17 @@ impl Initializer for PyCFuncPtrType { } #[pyclass(flags(IMMUTABLETYPE), with(Initializer))] -impl PyCFuncPtrType {} +impl PyCFuncPtrType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } +} /// PyCFuncPtr - Function pointer instance /// Saved in _base.buffer @@ -561,6 +572,16 @@ impl Debug for PyCFuncPtr { /// Extract pointer value from a ctypes argument (c_void_p conversion) fn extract_ptr_from_arg(arg: &PyObject, vm: &VirtualMachine) -> PyResult { + // Try CArgObject first - extract the wrapped pointer value, applying offset + if let Some(carg) = arg.downcast_ref::() { + if carg.offset != 0 + && let Some(cdata) = carg.obj.downcast_ref::() + { + let base = cdata.buffer.read().as_ptr() as isize; + return Ok((base + carg.offset) as usize); + } + return extract_ptr_from_arg(&carg.obj, vm); + } // Try to get pointer value from various ctypes types if let Some(ptr) = arg.downcast_ref::() { return Ok(ptr.get_ptr_value()); @@ -655,6 +676,68 @@ fn wstring_at_impl(ptr: usize, size: isize, vm: &VirtualMachine) -> PyResult { } } +/// A buffer wrapping raw memory at a given pointer, for zero-copy memoryview. +#[pyclass(name = "_RawMemoryBuffer", module = "_ctypes")] +#[derive(Debug, PyPayload)] +pub(super) struct RawMemoryBuffer { + ptr: *const u8, + size: usize, + readonly: bool, +} + +// SAFETY: The caller ensures the pointer remains valid +unsafe impl Send for RawMemoryBuffer {} +unsafe impl Sync for RawMemoryBuffer {} + +static RAW_MEMORY_BUFFER_METHODS: crate::protocol::BufferMethods = crate::protocol::BufferMethods { + obj_bytes: |buffer| { + let raw = buffer.obj_as::(); + let slice = unsafe { core::slice::from_raw_parts(raw.ptr, raw.size) }; + rustpython_common::borrow::BorrowedValue::Ref(slice) + }, + obj_bytes_mut: |buffer| { + let raw = buffer.obj_as::(); + let slice = unsafe { core::slice::from_raw_parts_mut(raw.ptr as *mut u8, raw.size) }; + rustpython_common::borrow::BorrowedValueMut::RefMut(slice) + }, + release: |_| {}, + retain: |_| {}, +}; + +#[pyclass(with(AsBuffer))] +impl RawMemoryBuffer {} + +impl AsBuffer for RawMemoryBuffer { + fn as_buffer(zelf: &Py, _vm: &VirtualMachine) -> PyResult { + Ok(PyBuffer::new( + zelf.to_owned().into(), + BufferDescriptor::simple(zelf.size, zelf.readonly), + &RAW_MEMORY_BUFFER_METHODS, + )) + } +} + +/// memoryview_at implementation - create a memoryview from memory at ptr +fn memoryview_at_impl(ptr: usize, size: isize, readonly: bool, vm: &VirtualMachine) -> PyResult { + use crate::builtins::PyMemoryView; + + if ptr == 0 { + return Err(vm.new_value_error("NULL pointer access")); + } + if size < 0 { + return Err(vm.new_value_error("negative size")); + } + let len = size as usize; + let raw_buf = RawMemoryBuffer { + ptr: ptr as *const u8, + size: len, + readonly, + } + .into_pyobject(vm); + let mv = PyMemoryView::from_object(&raw_buf, vm)?; + Ok(mv.into_pyobject(vm)) +} + // cast_check_pointertype fn cast_check_pointertype(ctype: &PyObject, vm: &VirtualMachine) -> bool { use super::pointer::PyCPointerType; @@ -1064,6 +1147,25 @@ fn handle_internal_func(addr: usize, args: &FuncArgs, vm: &VirtualMachine) -> Op })); } + if addr == INTERNAL_MEMORYVIEW_AT_ADDR { + let result: PyResult<(PyObjectRef, PyObjectRef, Option)> = + args.clone().bind(vm); + return Some(result.and_then(|(ptr_arg, size_arg, readonly_arg)| { + let ptr = extract_ptr_from_arg(&ptr_arg, vm)?; + let size_int = size_arg.try_int(vm)?; + let size = size_int + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_value_error("size too large"))?; + let readonly = readonly_arg + .and_then(|r| r.try_int(vm).ok()) + .and_then(|i| i.as_bigint().to_i32()) + .unwrap_or(0) + != 0; + memoryview_at_impl(ptr, size, readonly, vm) + })); + } + None } @@ -2091,7 +2193,7 @@ unsafe extern "C" fn thunk_callback( .map(|s| s.to_string()) .unwrap_or_else(|_| "".to_string()); let msg = format!( - "Exception ignored on calling ctypes callback function {}", + "Exception ignored while calling ctypes callback function {}", repr ); vm.run_unraisable(exc.clone(), Some(msg), vm.ctx.none()); diff --git a/crates/vm/src/stdlib/ctypes/pointer.rs b/crates/vm/src/stdlib/ctypes/pointer.rs index 087a7bfd3..6000bb57a 100644 --- a/crates/vm/src/stdlib/ctypes/pointer.rs +++ b/crates/vm/src/stdlib/ctypes/pointer.rs @@ -27,16 +27,24 @@ impl Initializer for PyCPointerType { .downcast() .map_err(|_| vm.new_type_error("expected type"))?; - new_type.check_not_initialized(vm)?; + if new_type.is_initialized() { + return Ok(()); + } // Get the _type_ attribute (element type) - // PyCPointerType_init gets the element type from _type_ attribute let proto = new_type .as_object() .get_attr("_type_", vm) .ok() .and_then(|obj| obj.downcast::().ok()); + // Validate that _type_ has storage info (is a ctypes type) + if let Some(ref proto_type) = proto + && proto_type.stg_info_opt().is_none() + { + return Err(vm.new_type_error(format!("{} must have storage info", proto_type.name()))); + } + // Initialize StgInfo for pointer type let pointer_size = core::mem::size_of::(); let mut stg_info = StgInfo::new(pointer_size, pointer_size); @@ -62,12 +70,31 @@ impl Initializer for PyCPointerType { let _ = new_type.init_type_data(stg_info); + // Cache: set target_type.__pointer_type__ = self (via StgInfo, not as inheritable attr) + if let Ok(type_attr) = new_type.as_object().get_attr("_type_", vm) + && let Ok(target_type) = type_attr.downcast::() + && let Some(mut target_info) = target_type.get_type_data_mut::() + { + let zelf_obj: PyObjectRef = zelf.into(); + target_info.pointer_type = Some(zelf_obj); + } + Ok(()) } } -#[pyclass(flags(IMMUTABLETYPE), with(AsNumber, Initializer))] +#[pyclass(flags(BASETYPE, IMMUTABLETYPE), with(AsNumber, Initializer))] impl PyCPointerType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + #[pymethod] fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { // zelf is the pointer type class that from_param was called on @@ -182,7 +209,12 @@ impl PyCPointerType { } // 4. Set _type_ attribute on the pointer type - zelf.as_object().set_attr("_type_", typ_type, vm)?; + zelf.as_object().set_attr("_type_", typ_type.clone(), vm)?; + + // 5. Cache: set target_type.__pointer_type__ = self (via StgInfo) + if let Some(mut target_info) = typ_type.get_type_data_mut::() { + target_info.pointer_type = Some(zelf.into()); + } Ok(()) } @@ -598,11 +630,12 @@ impl PyCPointer { if type_code.as_deref() == Some("z") && let Some(bytes) = value.downcast_ref::() { - let (converted, ptr_val) = super::base::ensure_z_null_terminated(bytes, vm); + let (kept_alive, ptr_val) = super::base::ensure_z_null_terminated(bytes, vm); unsafe { *(addr as *mut usize) = ptr_val; } - return zelf.0.keep_ref(index as usize, converted, vm); + zelf.0.keep_alive(index as usize, kept_alive); + return zelf.0.keep_ref(index as usize, value.clone(), vm); } else if type_code.as_deref() == Some("Z") && let Some(s) = value.downcast_ref::() { diff --git a/crates/vm/src/stdlib/ctypes/simple.rs b/crates/vm/src/stdlib/ctypes/simple.rs index 48ff52e9c..67c07dcb7 100644 --- a/crates/vm/src/stdlib/ctypes/simple.rs +++ b/crates/vm/src/stdlib/ctypes/simple.rs @@ -18,8 +18,12 @@ use core::fmt::Debug; use num_traits::ToPrimitive; /// Valid type codes for ctypes simple types +#[cfg(windows)] // spell-checker: disable-next-line pub(super) const SIMPLE_TYPE_CHARS: &str = "cbBhHiIlLdfuzZqQPXOv?g"; +#[cfg(not(windows))] +// spell-checker: disable-next-line +pub(super) const SIMPLE_TYPE_CHARS: &str = "cbBhHiIlLdfuzZqQPOv?g"; /// Convert ctypes type code to PEP 3118 format code. /// Some ctypes codes need to be mapped to standard-size codes based on platform. @@ -235,6 +239,16 @@ pub struct PyCSimpleType(PyType); #[pyclass(flags(BASETYPE), with(AsNumber, Initializer))] impl PyCSimpleType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + #[allow(clippy::new_ret_no_self)] #[pymethod] fn new(cls: PyTypeRef, _: OptionalArg, vm: &VirtualMachine) -> PyResult { @@ -327,10 +341,10 @@ impl PyCSimpleType { Some("z") => { // 1. bytes → create CArgObject with null-terminated buffer if let Some(bytes) = value.downcast_ref::() { - let (holder, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); return Ok(CArgObject { tag: b'z', - value: FfiArgValue::OwnedPointer(ptr, holder), + value: FfiArgValue::OwnedPointer(ptr, kept_alive), obj: value.clone(), size: 0, offset: 0, @@ -381,10 +395,10 @@ impl PyCSimpleType { } // 2. bytes → create CArgObject with null-terminated buffer if let Some(bytes) = value.downcast_ref::() { - let (holder, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); return Ok(CArgObject { tag: b'z', - value: FfiArgValue::OwnedPointer(ptr, holder), + value: FfiArgValue::OwnedPointer(ptr, kept_alive), obj: value.clone(), size: 0, offset: 0, @@ -1027,9 +1041,10 @@ impl Constructor for PyCSimple { if let Some(ref v) = init_arg { if _type_ == "z" { if let Some(bytes) = v.downcast_ref::() { - let (converted, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); let buffer = ptr.to_ne_bytes().to_vec(); - let cdata = PyCData::from_bytes(buffer, Some(converted)); + let cdata = PyCData::from_bytes(buffer, Some(v.clone())); + *cdata.base.write() = Some(kept_alive); return PyCSimple(cdata).into_ref_with_type(vm, cls).map(Into::into); } } else if _type_ == "Z" @@ -1258,9 +1273,10 @@ impl PyCSimple { // Handle z/Z types with PyBytes/PyStr separately to avoid memory leak if type_code == "z" { if let Some(bytes) = value.downcast_ref::() { - let (converted, ptr) = super::base::ensure_z_null_terminated(bytes, vm); + let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); *zelf.0.buffer.write() = alloc::borrow::Cow::Owned(ptr.to_ne_bytes().to_vec()); - *zelf.0.objects.write() = Some(converted); + *zelf.0.objects.write() = Some(value); + *zelf.0.base.write() = Some(kept_alive); return Ok(()); } } else if type_code == "Z" diff --git a/crates/vm/src/stdlib/ctypes/structure.rs b/crates/vm/src/stdlib/ctypes/structure.rs index cd36ce855..c0116d9d7 100644 --- a/crates/vm/src/stdlib/ctypes/structure.rs +++ b/crates/vm/src/stdlib/ctypes/structure.rs @@ -1,9 +1,9 @@ use super::base::{CDATA_BUFFER_METHODS, PyCData, PyCField, StgInfo, StgInfoFlags}; use crate::builtins::{PyList, PyStr, PyTuple, PyType, PyTypeRef}; use crate::convert::ToPyObject; -use crate::function::FuncArgs; -use crate::function::PySetterValue; +use crate::function::{FuncArgs, OptionalArg, PySetterValue}; use crate::protocol::{BufferDescriptor, PyBuffer, PyNumberMethods}; +use crate::stdlib::warnings; use crate::types::{AsBuffer, AsNumber, Constructor, Initializer, SetAttr}; use crate::{AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; use alloc::borrow::Cow; @@ -103,6 +103,7 @@ impl Initializer for PyCStructType { let mut stg_info = baseinfo.clone(); stg_info.flags &= !StgInfoFlags::DICTFLAG_FINAL; // Clear FINAL in subclass stg_info.initialized = true; + stg_info.pointer_type = None; // Non-inheritable stg_info }); @@ -130,6 +131,16 @@ impl Initializer for PyCStructType { #[pyclass(flags(BASETYPE), with(AsNumber, Constructor, Initializer, SetAttr))] impl PyCStructType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + #[pymethod] fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { // zelf is the structure type class that from_param was called on @@ -154,6 +165,55 @@ impl PyCStructType { ))) } + // CDataType methods - delegated to PyCData implementations + + #[pymethod] + fn from_address(zelf: PyObjectRef, address: isize, vm: &VirtualMachine) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_address(cls, address, vm) + } + + #[pymethod] + fn from_buffer( + zelf: PyObjectRef, + source: PyObjectRef, + offset: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer(cls, source, offset, vm) + } + + #[pymethod] + fn from_buffer_copy( + zelf: PyObjectRef, + source: crate::function::ArgBytesLike, + offset: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer_copy(cls, source, offset, vm) + } + + #[pymethod] + fn in_dll( + zelf: PyObjectRef, + dll: PyObjectRef, + name: crate::builtins::PyStrRef, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::in_dll(cls, dll, name, vm) + } + /// Called when a new Structure subclass is created #[pyclassmethod] fn __init_subclass__(cls: PyTypeRef, vm: &VirtualMachine) -> PyResult<()> { @@ -185,6 +245,24 @@ impl PyCStructType { }; let pack = super::base::get_usize_attr(cls.as_object(), "_pack_", 0, vm)?; + + // Emit DeprecationWarning on non-Windows when _pack_ is set without _layout_ + if pack > 0 && !cfg!(windows) { + let has_layout = cls.as_object().get_attr("_layout_", vm).is_ok(); + if !has_layout { + let base_type_name = "Structure"; + let msg = format!( + "Due to '_pack_', the '{}' {} will use memory layout compatible with \ + MSVC (Windows). If this is intended, set _layout_ to 'ms'. \ + The implicit default is deprecated and slated to become an error in \ + Python 3.19.", + cls.name(), + base_type_name, + ); + warnings::warn(vm.ctx.exceptions.deprecation_warning, msg, 1, vm)?; + } + } + let forced_alignment = super::base::get_usize_attr(cls.as_object(), "_align_", 1, vm)?.max(1); @@ -221,6 +299,11 @@ impl PyCStructType { let mut format = String::from("T{"); let mut last_end = 0usize; // Track end of last field for padding calculation + // Bitfield layout tracking + let mut bitfield_bit_offset: u16 = 0; // Current bit position within bitfield group + let mut last_field_bit_size: u16 = 0; // For MSVC: bit size of previous storage unit + let use_msvc_bitfields = pack > 0; // MSVC layout when _pack_ is set + for (index, field) in fields.iter().enumerate() { let field_tuple = field .downcast_ref::() @@ -336,14 +419,93 @@ impl PyCStructType { .clone() .downcast::() .map_err(|_| vm.new_type_error("_fields_ type must be a ctypes type"))?; - let c_field = PyCField::new(field_type_ref, offset as isize, size as isize, index); + + // Check for bitfield size (optional 3rd element in tuple) + let (c_field, field_advances_offset) = if field_tuple.len() > 2 { + let bit_size_obj = field_tuple.get(2).expect("len checked"); + let bit_size = bit_size_obj + .try_int(vm)? + .as_bigint() + .to_u16() + .ok_or_else(|| { + vm.new_value_error("number of bits invalid for bit field".to_string()) + })?; + has_bitfield = true; + + let type_bits = (size * 8) as u16; + let (advances, bit_offset); + + if use_msvc_bitfields { + // MSVC layout: different types start new storage unit + if bitfield_bit_offset + bit_size > type_bits + || type_bits != last_field_bit_size + { + // Close previous bitfield, start new allocation unit + bitfield_bit_offset = 0; + advances = true; + } else { + advances = false; + } + bit_offset = bitfield_bit_offset; + bitfield_bit_offset += bit_size; + last_field_bit_size = type_bits; + } else { + // GCC System V layout: pack within same type + let fits_in_current = bitfield_bit_offset + bit_size <= type_bits; + advances = if fits_in_current && bitfield_bit_offset > 0 { + false + } else if !fits_in_current { + bitfield_bit_offset = 0; + true + } else { + true + }; + bit_offset = bitfield_bit_offset; + bitfield_bit_offset += bit_size; + } + + // For packed bitfields that share offset, use the same offset as previous + let field_offset = if !advances { + offset - size // Reuse the previous field's offset + } else { + offset + }; + + ( + PyCField::new_bitfield( + name.clone(), + field_type_ref, + field_offset as isize, + size as isize, + bit_size, + bit_offset, + index, + ), + advances, + ) + } else { + bitfield_bit_offset = 0; // Reset on non-bitfield + last_field_bit_size = 0; + ( + PyCField::new( + name.clone(), + field_type_ref, + offset as isize, + size as isize, + index, + ), + true, + ) + }; // Set the CField as a class attribute cls.set_attr(vm.ctx.intern_str(name.clone()), c_field.to_pyobject(vm)); - // Update tracking - last_end = offset + size; - offset += size; + // Update tracking - don't advance offset for packed bitfields + if field_advances_offset { + last_end = offset + size; + offset += size; + } } // Calculate total_align = max(max_align, forced_alignment) @@ -366,6 +528,16 @@ impl PyCStructType { } format.push('}'); + // Check for circular self-reference: if a field of the same type as this + // structure was encountered, it would have marked this type's stginfo as FINAL. + if let Some(stg_info) = cls.get_type_data::() + && stg_info.is_final() + { + return Err( + vm.new_attribute_error("Structure or union cannot contain itself".to_string()) + ); + } + // Store StgInfo with aligned size and total alignment let mut stg_info = StgInfo::new(aligned_size, total_align); stg_info.length = fields.len(); diff --git a/crates/vm/src/stdlib/ctypes/union.rs b/crates/vm/src/stdlib/ctypes/union.rs index aecef18ec..fba9a75e9 100644 --- a/crates/vm/src/stdlib/ctypes/union.rs +++ b/crates/vm/src/stdlib/ctypes/union.rs @@ -2,12 +2,13 @@ use super::base::{CDATA_BUFFER_METHODS, StgInfoFlags}; use super::{PyCData, PyCField, StgInfo}; use crate::builtins::{PyList, PyStr, PyTuple, PyType, PyTypeRef}; use crate::convert::ToPyObject; -use crate::function::FuncArgs; -use crate::function::PySetterValue; +use crate::function::{ArgBytesLike, FuncArgs, OptionalArg, PySetterValue}; use crate::protocol::{BufferDescriptor, PyBuffer}; +use crate::stdlib::warnings; use crate::types::{AsBuffer, Constructor, Initializer, SetAttr}; use crate::{AsObject, Py, PyObjectRef, PyPayload, PyResult, VirtualMachine}; use alloc::borrow::Cow; +use num_traits::ToPrimitive; /// Calculate Union type size from _fields_ (max field size) pub(super) fn calculate_union_size(cls: &Py, vm: &VirtualMachine) -> PyResult { @@ -106,6 +107,7 @@ impl Initializer for PyCUnionType { let mut stg_info = baseinfo.clone(); stg_info.flags &= !StgInfoFlags::DICTFLAG_FINAL; // Clear FINAL flag in subclass stg_info.initialized = true; + stg_info.pointer_type = None; // Non-inheritable stg_info }); @@ -163,6 +165,22 @@ impl PyCUnionType { }; let pack = super::base::get_usize_attr(cls.as_object(), "_pack_", 0, vm)?; + + // Emit DeprecationWarning on non-Windows when _pack_ is set without _layout_ + if pack > 0 && !cfg!(windows) { + let has_layout = cls.as_object().get_attr("_layout_", vm).is_ok(); + if !has_layout { + let msg = format!( + "Due to '_pack_', the '{}' Union will use memory layout compatible with \ + MSVC (Windows). If this is intended, set _layout_ to 'ms'. \ + The implicit default is deprecated and slated to become an error in \ + Python 3.19.", + cls.name(), + ); + warnings::warn(vm.ctx.exceptions.deprecation_warning, msg, 1, vm)?; + } + } + let forced_alignment = super::base::get_usize_attr(cls.as_object(), "_align_", 1, vm)?.max(1); @@ -258,7 +276,42 @@ impl PyCUnionType { .clone() .downcast::() .map_err(|_| vm.new_type_error("_fields_ type must be a ctypes type"))?; - let c_field = PyCField::new(field_type_ref, 0, size as isize, index); + + // Check for bitfield size (optional 3rd element in tuple) + // For unions, each field starts fresh (CPython: _layout.py) + let c_field = if field_tuple.len() > 2 { + let bit_size_obj = field_tuple.get(2).expect("len checked"); + let bit_size = bit_size_obj + .try_int(vm)? + .as_bigint() + .to_u16() + .ok_or_else(|| { + vm.new_value_error("number of bits invalid for bit field".to_string()) + })?; + has_bitfield = true; + + // Union fields all start at offset 0, so bit_offset = 0 + let mut bit_offset: u16 = 0; + let type_bits = (size * 8) as u16; + + // Big-endian: bit_offset = type_bits - bit_size + let big_endian = is_swapped != cfg!(target_endian = "big"); + if big_endian && type_bits >= bit_size { + bit_offset = type_bits - bit_size; + } + + PyCField::new_bitfield( + name.clone(), + field_type_ref, + 0, // Union fields always at offset 0 + size as isize, + bit_size, + bit_offset, + index, + ) + } else { + PyCField::new(name.clone(), field_type_ref, 0, size as isize, index) + }; cls.set_attr(vm.ctx.intern_str(name), c_field.to_pyobject(vm)); } @@ -271,6 +324,15 @@ impl PyCUnionType { max_size }; + // Check for circular self-reference + if let Some(stg_info) = cls.get_type_data::() + && stg_info.is_final() + { + return Err( + vm.new_attribute_error("Structure or union cannot contain itself".to_string()) + ); + } + // Store StgInfo with aligned size let mut stg_info = StgInfo::new(aligned_size, total_align); stg_info.length = fields.len(); @@ -299,6 +361,16 @@ impl PyCUnionType { #[pyclass(flags(BASETYPE), with(Constructor, Initializer, SetAttr))] impl PyCUnionType { + #[pygetset(name = "__pointer_type__")] + fn pointer_type(zelf: PyTypeRef, vm: &VirtualMachine) -> PyResult { + super::base::pointer_type_get(&zelf, vm) + } + + #[pygetset(name = "__pointer_type__", setter)] + fn set_pointer_type(zelf: PyTypeRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + super::base::pointer_type_set(&zelf, value, vm) + } + #[pymethod] fn from_param(zelf: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { // zelf is the union type class that from_param was called on @@ -344,6 +416,55 @@ impl PyCUnionType { ))) } + // CDataType methods - delegated to PyCData implementations + + #[pymethod] + fn from_address(zelf: PyObjectRef, address: isize, vm: &VirtualMachine) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_address(cls, address, vm) + } + + #[pymethod] + fn from_buffer( + zelf: PyObjectRef, + source: PyObjectRef, + offset: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer(cls, source, offset, vm) + } + + #[pymethod] + fn from_buffer_copy( + zelf: PyObjectRef, + source: ArgBytesLike, + offset: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::from_buffer_copy(cls, source, offset, vm) + } + + #[pymethod] + fn in_dll( + zelf: PyObjectRef, + dll: PyObjectRef, + name: crate::builtins::PyStrRef, + vm: &VirtualMachine, + ) -> PyResult { + let cls: PyTypeRef = zelf + .downcast() + .map_err(|_| vm.new_type_error("expected a type"))?; + PyCData::in_dll(cls, dll, name, vm) + } + /// Called when a new Union subclass is created #[pyclassmethod] fn __init_subclass__(cls: PyTypeRef, vm: &VirtualMachine) -> PyResult<()> { @@ -383,6 +504,14 @@ impl SetAttr for PyCUnionType { } } + // 2. If _fields_, call process_fields (which checks FINAL internally) + // Check BEFORE writing to dict to avoid storing _fields_ when FINAL + if attr_name.as_str() == "_fields_" + && let PySetterValue::Assign(ref fields_value) = value + { + PyCUnionType::process_fields(pytype, fields_value.clone(), vm)?; + } + // Store in type's attributes dict match &value { PySetterValue::Assign(v) => { @@ -403,13 +532,6 @@ impl SetAttr for PyCUnionType { } } - // 2. If _fields_, call process_fields (which checks FINAL internally) - if attr_name.as_str() == "_fields_" - && let PySetterValue::Assign(fields_value) = value - { - PyCUnionType::process_fields(pytype, fields_value, vm)?; - } - Ok(()) } } diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index a67c06366..9e33f4309 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -211,7 +211,7 @@ impl VirtualMachine { op: &str, ) -> PyBaseExceptionRef { self.new_type_error(format!( - "'{}' not supported between instances of '{}' and '{}'", + "unsupported operand type(s) for {}: '{}' and '{}'", op, a.class().name(), b.class().name()