From 980edd8b575289cc23b61cdb82a6d98c4c77eb4c Mon Sep 17 00:00:00 2001 From: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:37:33 +0300 Subject: [PATCH] Preserve ctypes element views and their references (#9229) * Preserve ctypes element views and their references Assisted-by: Codex:GPT-6 * Fix ctypes reentrant ownership and scalar view handling Assisted-by: Codex:GPT-6 * Fix ctypes pointer view writes and indexed object lookup Assisted-by: Codex:GPT-6 * Fix ctypes field byte order and encoded instance copies Assisted-by: Codex:GPT-6 * Fix ctypes effective field types and overlapping pointer copies Assisted-by: Codex:GPT-6 * Reject arrays for incomplete ctypes pointer fields Assisted-by: Codex:GPT-6 --- crates/vm/src/stdlib/_ctypes/array.rs | 249 +++++++++--------- crates/vm/src/stdlib/_ctypes/base.rs | 296 ++++++++++++++++++---- crates/vm/src/stdlib/_ctypes/function.rs | 44 ++-- crates/vm/src/stdlib/_ctypes/pointer.rs | 116 ++++++--- crates/vm/src/stdlib/_ctypes/simple.rs | 131 +++++----- crates/vm/src/stdlib/_ctypes/structure.rs | 5 +- extra_tests/snippets/stdlib_ctypes.py | 251 ++++++++++++++++++ 7 files changed, 807 insertions(+), 285 deletions(-) diff --git a/crates/vm/src/stdlib/_ctypes/array.rs b/crates/vm/src/stdlib/_ctypes/array.rs index 827b6fa42..124020bd6 100644 --- a/crates/vm/src/stdlib/_ctypes/array.rs +++ b/crates/vm/src/stdlib/_ctypes/array.rs @@ -545,23 +545,51 @@ impl PyCArray { let index = index as usize; let element_size = stg.as_ref().map_or(0, |i| i.element_size); let offset = index * element_size; + if let Some(element_type) = stg.as_ref().and_then(|info| info.element_type.as_ref()) + && !super::simple::returns_scalar(element_type) + { + let length = element_type.stg_info(vm)?.length; + let buffer = zelf.0.buffer.read(); + let Some(data) = buffer.get(offset..offset + element_size) else { + drop(buffer); + return Err(vm.new_index_error("invalid index")); + }; + // SAFETY: this element lies within the array's live buffer. The view + // retains the array; ctypes users must still avoid reallocating or + // concurrently accessing the same native memory through aliases. + let view = unsafe { + PyCData::from_base_obj( + data.as_ptr().cast_mut(), + element_size, + zelf.to_owned().into(), + index, + ) + }; + view.base_offset.store(offset); + view.length.store(length); + drop(buffer); + return view.into_view(element_type.clone(), vm); + } let type_code = Self::get_element_type_code(zelf, vm); - // Get target buffer and offset (base's buffer if available, otherwise own) - let base_obj = zelf.0.base.read().clone(); - let (buffer_lock, final_offset) = if let Some(cdata) = base_obj - .as_ref() - .and_then(|b| b.downcast_ref::()) + let buffer = zelf.0.buffer.read(); + if type_code.as_deref() == Some("O") { + let address = rustpython_host_env::ctypes::read_pointer_from_buffer( + &buffer[offset..offset + element_size], + ); + drop(buffer); + return zelf.0.object_at(address, Some(index), vm); + } + if let Some(element_type) = stg.as_ref().and_then(|info| info.element_type.as_ref()) + && super::simple::is_swapped(element_type) { - (&cdata.buffer, zelf.0.base_offset.load() + offset) - } else { - (&zelf.0.buffer, offset) - }; - - let buffer = buffer_lock.read(); + let mut bytes = buffer[offset..offset + element_size].to_vec(); + bytes.reverse(); + return Ok(super::base::bytes_to_pyobject(element_type, &bytes, vm)); + } Ok(Self::read_element_from_buffer( &buffer, - final_offset, + offset, element_size, type_code.as_deref(), vm, @@ -589,10 +617,7 @@ impl PyCArray { } /// Helper to write an element value to a buffer at given offset - /// This is extracted to share code between direct write and base-buffer write - #[allow(clippy::too_many_arguments)] fn write_element_to_buffer( - buffer: &mut [u8], offset: usize, element_size: usize, type_code: Option<&str>, @@ -601,20 +626,22 @@ impl PyCArray { index: usize, vm: &VirtualMachine, ) -> PyResult<()> { + // Conversions and reference replacement may call Python. Only the raw + // storage write runs under the buffer lock. + let write = |value| { + let mut buffer = zelf.0.buffer.write(); + write_array_element(cow_bytes_as_mut(&mut buffer), offset, value); + }; match type_code { Some("c") => { if let Some(b) = value.downcast_ref::() { - write_array_element( - buffer, - offset, - ArrayElementWriteValue::Byte(b.as_bytes().first().copied().unwrap_or(0)), - ); + write(ArrayElementWriteValue::Byte( + b.as_bytes().first().copied().unwrap_or(0), + )); } else if let Ok(int_val) = value.try_int(vm) { - write_array_element( - buffer, - offset, - ArrayElementWriteValue::Byte(int_val.as_bigint().to_u8().unwrap_or(0)), - ); + write(ArrayElementWriteValue::Byte( + int_val.as_bigint().to_u8().unwrap_or(0), + )); } else { return Err(vm.new_type_error("an integer or bytes of length 1 is required")); } @@ -622,17 +649,18 @@ impl PyCArray { Some("u") => { if let Some(s) = value.downcast_ref::() { let code = s.as_wtf8().code_points().next().map_or(0, |c| c.to_u32()); - write_array_element(buffer, offset, ArrayElementWriteValue::Wchar(code)); + write(ArrayElementWriteValue::Wchar(code)); } else { return Err(vm.new_type_error("unicode string expected")); } } Some("z") => { + let mut hidden = None; let (ptr_val, converted) = if value.is(&vm.ctx.none) { (0usize, None) } else if let Some(bytes) = value.downcast_ref::() { let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); - zelf.0.keep_alive(index, kept_alive); + hidden = Some(kept_alive); (ptr, Some(value.to_owned())) } else if let Some(simple) = value.downcast_ref::() && value.class().type_code(vm).as_deref() == Some("z") @@ -650,16 +678,12 @@ impl PyCArray { value.class().name() ))); }; - write_array_element( - buffer, - offset, - ArrayElementWriteValue::Pointer { - value: ptr_val, - size: element_size, - }, - ); + write(ArrayElementWriteValue::Pointer { + value: ptr_val, + size: element_size, + }); if let Some(c) = converted { - return zelf.0.keep_ref(index, c, vm); + return zelf.0.keep_ref_with_hidden(index, c, hidden, vm); } } Some("Z") => { @@ -681,18 +705,40 @@ impl PyCArray { } else { return Err(vm.new_type_error("unicode string or integer address expected")); }; - write_array_element( - buffer, - offset, - ArrayElementWriteValue::Pointer { - value: ptr_val, - size: element_size, - }, - ); + write(ArrayElementWriteValue::Pointer { + value: ptr_val, + size: element_size, + }); if let Some(c) = converted { return zelf.0.keep_ref(index, c, vm); } } + Some("O") => { + // A py_object instance contributes its stored reference, while + // other Python objects are the value itself. + if value.downcast_ref::().is_none() { + write(ArrayElementWriteValue::Pointer { + value: value.get_id(), + size: element_size, + }); + return zelf.0.keep_ref(index, value.to_owned(), vm); + } + let stg = zelf.class().stg_info(vm)?; + let element_type = stg.element_type.as_ref().unwrap(); + if !value.fast_isinstance(element_type) { + return Err(vm.new_type_error(format!( + "expected {} instance, not {}", + element_type.name(), + value.class().name() + ))); + } + let cdata = value.downcast_ref::().unwrap(); + let bytes = cdata.buffer.read().to_vec(); + write(ArrayElementWriteValue::Bytes { + bytes: &bytes, + size: element_size, + }); + } Some("f") => { // c_float: convert int/float to f32 bytes let f32_val = if let Ok(float_val) = value.try_float(vm) { @@ -702,14 +748,10 @@ impl PyCArray { } else { return Err(vm.new_type_error("a float is required")); }; - write_array_element( - buffer, - offset, - ArrayElementWriteValue::Float { - value: f32_val.into(), - size: 4, - }, - ); + write(ArrayElementWriteValue::Float { + value: f32_val.into(), + size: 4, + }); } Some("d" | "g") => { // c_double / c_longdouble: convert int/float to f64 bytes @@ -720,39 +762,27 @@ impl PyCArray { } else { return Err(vm.new_type_error("a float is required")); }; - write_array_element( - buffer, - offset, - ArrayElementWriteValue::Float { - value: f64_val, - size: 8, - }, - ); + write(ArrayElementWriteValue::Float { + value: f64_val, + size: 8, + }); // For "g" type, remaining bytes stay zero } _ => { // Handle ctypes instances (copy their buffer) if let Some(cdata) = value.downcast_ref::() { - let src_buffer = cdata.buffer.read(); - write_array_element( - buffer, - offset, - ArrayElementWriteValue::Bytes { - bytes: &src_buffer, - size: element_size, - }, - ); + let src_buffer = cdata.buffer.read().to_vec(); + write(ArrayElementWriteValue::Bytes { + bytes: &src_buffer, + size: element_size, + }); // Other types: use int_to_bytes } else if let Ok(int_val) = value.try_int(vm) { let bytes = Self::int_to_bytes(int_val.as_bigint(), element_size); - write_array_element( - buffer, - offset, - ArrayElementWriteValue::Bytes { - bytes: &bytes, - size: element_size, - }, - ); + write(ArrayElementWriteValue::Bytes { + bytes: &bytes, + size: element_size, + }); } else { return Err(vm.new_type_error(format!( "expected {} instance, not {}", @@ -765,8 +795,7 @@ impl PyCArray { // KeepRef if super::base::PyCData::should_keep_ref(value) { - let to_keep = super::base::PyCData::get_kept_objects(value, vm); - zelf.0.keep_ref(index, to_keep, vm)?; + zelf.0.keep_cdata(index, value, vm)?; } Ok(()) @@ -789,49 +818,29 @@ impl PyCArray { let offset = index * element_size; let type_code = Self::get_element_type_code(zelf, vm); - // Get target buffer and offset (base's buffer if available, otherwise own) - let base_obj = zelf.0.base.read().clone(); - let (buffer_lock, final_offset) = if let Some(cdata) = base_obj - .as_ref() - .and_then(|b| b.downcast_ref::()) + if let Some(element_type) = stg.as_ref().and_then(|info| info.element_type.as_ref()) + && super::simple::is_swapped(element_type) + && value.downcast_ref::().is_none() { - (&cdata.buffer, zelf.0.base_offset.load() + offset) - } else { - (&zelf.0.buffer, offset) - }; - - let mut buffer = buffer_lock.write(); - - // For shared memory (Cow::Borrowed), we need to write directly to the memory - // For owned memory (Cow::Owned), we can write to the owned buffer - match &mut *buffer { - Cow::Borrowed(slice) => { - // SAFETY: For from_buffer, the slice points to writable shared memory. - // Python's from_buffer requires writable buffer, so this is safe. - let owned_slice = - unsafe { rustpython_host_env::ctypes::borrowed_slice_as_mut(slice) }; - Self::write_element_to_buffer( - owned_slice, - final_offset, - element_size, - type_code.as_deref(), - value, - zelf, - index, - vm, - ) - } - Cow::Owned(vec) => Self::write_element_to_buffer( - vec, - final_offset, - element_size, - type_code.as_deref(), - value, - zelf, - index, - vm, - ), + let type_code = type_code + .as_deref() + .ok_or_else(|| vm.new_type_error("no _type_ attribute"))?; + let bytes = super::simple::swapped_value_bytes(type_code, value, vm)?; + zelf.0.write_bytes_at_offset(offset, &bytes); + return Ok(()); } + + // A view's buffer already points to its element. Its base can be a + // pointer object, whose own buffer stores an address rather than data. + Self::write_element_to_buffer( + offset, + element_size, + type_code.as_deref(), + value, + zelf, + index, + vm, + ) } // Array_subscript diff --git a/crates/vm/src/stdlib/_ctypes/base.rs b/crates/vm/src/stdlib/_ctypes/base.rs index 916f621a1..6375705c6 100644 --- a/crates/vm/src/stdlib/_ctypes/base.rs +++ b/crates/vm/src/stdlib/_ctypes/base.rs @@ -5,6 +5,7 @@ use crate::builtins::{ }; use crate::class::StaticType; use crate::convert::ToPyObject; +use crate::dict_inner::DictKey; use crate::function::{ArgBytesLike, ArgStrictInt, OptionalArg, PySetterValue}; use crate::protocol::{BufferMethods, PyBuffer}; use crate::types::{Constructor, GetDescriptor, Representable}; @@ -439,6 +440,36 @@ pub struct PyCData { pub(super) kept_refs: PyRwLock>, } +// Retention metadata is user-mutable. Its fast lookup must not call Python +// equality on a colliding key; non-exact string keys use the graph fallback. +struct ObjectKey<'a>(&'a str); + +impl DictKey for ObjectKey<'_> { + type Owned = String; + + fn _to_owned(&self, _vm: &VirtualMachine) -> String { + self.0.to_owned() + } + + fn key_hash(&self, vm: &VirtualMachine) -> PyResult { + self.0.key_hash(vm) + } + + fn key_is(&self, _other: &PyObject) -> bool { + false + } + + fn key_eq(&self, vm: &VirtualMachine, other: &PyObject) -> PyResult { + Ok(other + .downcast_ref_if_exact::(vm) + .is_some_and(|key| key.as_wtf8() == self.0)) + } + + fn key_as_isize(&self, vm: &VirtualMachine) -> PyResult { + self.0.key_as_isize(vm) + } +} + impl PyCData { /// Create from StgInfo (PyCData_MallocBuffer pattern) pub(crate) fn from_stg_info(stg_info: &StgInfo) -> Self { @@ -696,9 +727,18 @@ impl PyCData { pub fn unique_key(&self, index: usize) -> String { let mut key = format!("{index:x}"); // Walk up the base chain to build hierarchical key - if self.base.read().is_some() { - let parent_index = self.index.load(); - key.push_str(&format!(":{parent_index:x}")); + let mut base = self.base.read().clone(); + if base.is_some() { + key.push_str(&format!(":{:x}", self.index.load())); + } + while let Some(parent) = base.take() { + let Some(cdata) = parent.downcast_ref::() else { + break; + }; + base.clone_from(&cdata.base.read()); + if base.is_some() { + key.push_str(&format!(":{:x}", cdata.index.load())); + } } key } @@ -758,8 +798,11 @@ impl PyCData { *objects = Some(dict.into()); } + // Replacing a reference can run a finalizer that accesses this object. + let dict_obj = objects.clone(); + drop(objects); // Store in dict with unique key - if let Some(dict_obj) = objects.as_ref() + if let Some(dict_obj) = dict_obj && let Some(dict) = dict_obj.downcast_ref::() { let key_obj: PyObjectRef = vm.ctx.new_str(key).into(); @@ -774,33 +817,49 @@ impl PyCData { /// lives as long as the owning ctypes object. /// Uses unique_key (hierarchical) so nested fields don't collide. pub fn keep_alive(&self, index: usize, obj: PyObjectRef) { + drop(self.replace_hidden(index, Some(obj))); + } + + fn replace_hidden(&self, index: usize, obj: Option) -> Option { let key = self.unique_key(index); let base = self.base.read().clone(); - if let Some(base_obj) = base { - let root = Self::find_root_object(&base_obj); - if let Some(cdata) = root.downcast_ref::() { - cdata.kept_refs.write().insert(key, obj); - return; - } + let root = base.as_ref().map(|base| Self::find_root_object(base)); + let cdata = root + .as_ref() + .and_then(|root| root.downcast_ref::()) + .map_or(self, |cdata| cdata); + let mut refs = cdata.kept_refs.write(); + match obj { + Some(obj) => refs.insert(key, obj), + None => refs.remove(&key), } - self.kept_refs.write().insert(key, obj); + } + + pub(super) fn keep_ref_with_hidden( + &self, + index: usize, + keep: PyObjectRef, + hidden: Option, + vm: &VirtualMachine, + ) -> PyResult<()> { + // Publish all owners before dropping the old visible reference: its + // finalizer may overwrite this same slot. Keep old storage alive until + // then, and do not overwrite the finalizer's owners afterwards. + let _old_hidden = self.replace_hidden(index, hidden); + self.keep_ref(index, keep, vm) } /// Find the root object (one without a base) by walking up the base chain fn find_root_object(obj: &PyObject) -> PyObjectRef { - // Try to get base from different ctypes types - let base = if let Some(cdata) = obj.downcast_ref::() { - cdata.base.read().clone() - } else { - None - }; - - // Recurse if there's a base, otherwise this is the root - if let Some(base_obj) = base { - Self::find_root_object(&base_obj) - } else { - obj.to_owned() + let mut root = obj.to_owned(); + while let Some(cdata) = root.downcast_ref::() { + let base = cdata.base.read().clone(); + match base { + Some(base) if base.downcast_ref::().is_some() => root = base, + _ => break, + } } + root } /// Store a value in an object's _objects dict with the given key @@ -834,8 +893,10 @@ impl PyCData { *objects = Some(dict.into()); } + let dict_obj = objects.clone(); + drop(objects); // Store in dict - if let Some(dict_obj) = objects.as_ref() + if let Some(dict_obj) = dict_obj && let Some(dict) = dict_obj.downcast_ref::() { let key_obj: PyObjectRef = vm.ctx.new_str(key).into(); @@ -846,14 +907,121 @@ impl PyCData { } /// Get kept objects from a CData instance - /// Returns the _objects of the CData, or an empty dict if None. + /// Returns the _objects of the root CData, or an empty dict if None. pub fn get_kept_objects(value: &PyObject, vm: &VirtualMachine) -> PyObjectRef { - value + Self::find_root_object(value) .downcast_ref::() .and_then(|cdata| cdata.objects.read().clone()) .unwrap_or_else(|| vm.ctx.new_dict().into()) } + // Copying a view must retain its root's references, including private + // backing allocations used for null-terminated strings. + pub(super) fn keep_cdata( + &self, + index: usize, + value: &PyObject, + vm: &VirtualMachine, + ) -> PyResult<()> { + let root = Self::find_root_object(value); + let hidden: Vec<_> = root + .downcast_ref::() + .map(|cdata| cdata.kept_refs.read().values().cloned().collect()) + .unwrap_or_default(); + let hidden = (!hidden.is_empty()).then(|| vm.ctx.new_tuple(hidden).into()); + self.keep_ref_with_hidden(index, Self::get_kept_objects(&root, vm), hidden, vm) + } + + // A scalar owns one reference; a scalar view stores it in its root instead. + pub(super) fn keep_value( + &self, + value: PyObjectRef, + hidden: Option, + vm: &VirtualMachine, + ) -> PyResult<()> { + let _old_hidden = self.replace_hidden(0, hidden); + if self.base.read().is_some() { + self.keep_ref(0, value, vm) + } else { + let old = self.objects.write().replace(value); + drop(old); + Ok(()) + } + } + + // Resolve a py_object through owned references, never by turning an + // arbitrary integer from native memory into a Rust reference. Views can + // reach the object through nested _objects dictionaries or cast cycles. + pub(super) fn object_at( + &self, + address: usize, + index: Option, + vm: &VirtualMachine, + ) -> PyResult { + if address == 0 { + return Err(vm.new_value_error("PyObject is NULL")); + } + if address == vm.ctx.none.get_id() { + return Ok(vm.ctx.none()); + } + let objects = self.objects.read().clone(); + if let Some(obj) = &objects + && obj.get_id() == address + { + return Ok(obj.clone()); + } + let base = self.base.read().clone(); + let root = base.as_ref().map(|base| Self::find_root_object(base)); + let owners = root + .as_ref() + .and_then(|root| root.downcast_ref::()) + .and_then(|root| root.objects.read().clone()) + .or_else(|| objects.clone()); + if let Some(dict) = owners.as_ref().and_then(|obj| obj.downcast_ref::()) { + let key = self.unique_key(index.unwrap_or(0)); + // A scalar view's setter stores under "0:", whereas + // assignment through its parent stores under "". + let parent_key = if index.is_none() { + key.strip_prefix("0:") + } else { + None + }; + for key in core::iter::once(key.as_str()).chain(parent_key) { + if let Some(obj) = dict.inner_getitem_opt(&ObjectKey(key), vm)? + && obj.get_id() == address + { + return Ok(obj); + } + } + } + // Casts, copied aggregates and buffer views can retain the referent + // through other keys or containers. Search those only on a direct miss. + let mut pending: Vec<_> = objects.into_iter().collect(); + pending.extend(base); + pending.extend(self.kept_refs.read().values().cloned()); + let mut visited = std::collections::HashSet::new(); + while let Some(obj) = pending.pop() { + if obj.get_id() == address { + return Ok(obj); + } + if !visited.insert(obj.get_id()) { + continue; + } + if let Some(cdata) = obj.downcast_ref::() { + pending.extend(cdata.objects.read().iter().cloned()); + pending.extend(cdata.base.read().iter().cloned()); + pending.extend(cdata.kept_refs.read().values().cloned()); + } else if let Some(dict) = obj.downcast_ref::() { + pending.extend(dict.values_vec()); + } else if let Some(tuple) = obj.downcast_ref::() { + pending.extend(tuple.as_slice().iter().cloned()); + } else if let Some(view) = obj.downcast_ref::() { + pending.push(view.viewed_object().to_owned()); + } + } + Err(vm.new_value_error("PyObject is NULL")) + } + /// Check if a value should be stored in _objects /// Returns true for ctypes objects and bytes (for c_char_p) pub(crate) fn should_keep_ref(value: &PyObject) -> bool { @@ -905,12 +1073,35 @@ impl PyCData { } } + // A compatible ctypes instance already contains the encoded field bytes. + // Snapshot them before writing: source and destination may share storage. + if let Some(proto_type) = proto.downcast_ref::() + && value.class().fast_issubclass(proto_type) + && let Some(source) = value.downcast_ref::() + { + let bytes = { + let buffer = source.buffer.read(); + rustpython_host_env::ctypes::copy_to_sized_bytes(&buffer, size) + }; + self.write_bytes_at_offset(offset, &bytes); + return self.keep_cdata(index, &value, vm); + } + // Special handling for Pointer fields with Array values if let Some(proto_type) = proto.downcast_ref::() && proto_type .class() .fast_issubclass(super::pointer::PyCPointerType::static_type()) && let Some(array) = value.downcast_ref::() + && { + let target_info = proto_type.stg_info(vm)?; + let array_info = array.class().stg_info(vm)?; + target_info + .proto + .as_deref() + .zip(array_info.element_type.as_deref()) + .is_some_and(|(target, element)| target.is(element)) + } { let buffer_addr = { let array_buffer = array.0.buffer.read(); @@ -922,6 +1113,17 @@ impl PyCData { return Ok(()); } + if value.downcast_ref::().is_some() { + return Err(vm.new_type_error(format!( + "incompatible types, {} instance instead of {} instance", + value.class().name(), + proto + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("field proto is not a type"))? + .name() + ))); + } + // For array fields with tuple/list input, instantiate the array type // and unpack elements as positional args (Array_init expects *args) if let Some(proto_type) = proto.downcast_ref::() @@ -975,9 +1177,7 @@ impl PyCData { let result = rustpython_host_env::ctypes::pointer_to_sized_bytes_endian(ptr, size, needs_swap); self.write_bytes_at_offset(offset, &result); - self.keep_ref(index, value, vm)?; - self.keep_alive(index, kept_alive); - return Ok(()); + return self.keep_ref_with_hidden(index, value, Some(kept_alive), vm); } let (mut bytes, converted_value) = if let Some(type_code) = &field_type_code { @@ -997,13 +1197,22 @@ impl PyCData { if let Some(converted) = converted_value { self.keep_ref(index, converted, vm)?; } else if Self::should_keep_ref(&value) { - let to_keep = Self::get_kept_objects(&value, vm); - self.keep_ref(index, to_keep, vm)?; + self.keep_cdata(index, &value, vm)?; } Ok(()) } + pub(super) fn into_view(self, cls: PyTypeRef, vm: &VirtualMachine) -> PyResult { + if cls.fast_issubclass(super::function::PyCFuncPtr::static_type()) { + super::function::PyCFuncPtr::from_cdata(self) + .into_ref_with_type(vm, cls) + .map(Into::into) + } else { + self.into_ref_with_type(vm, cls).map(Into::into) + } + } + /// PyCData_get /// Gets a field value at the given offset pub fn get_field( @@ -1080,15 +1289,7 @@ impl PyCData { // Simple types: return primitive value if proto_metaclass.fast_issubclass(super::simple::PyCSimpleType::static_type()) { // Check for byte swapping - let needs_swap = base_obj - .class() - .as_object() - .get_attr("_swappedbytes_", vm) - .is_ok() - || proto_type - .as_object() - .get_attr("_swappedbytes_", vm) - .is_ok(); + let needs_swap = super::simple::is_swapped(&proto_type); let data = if needs_swap && size > 1 { let mut swapped = buffer_data; @@ -1108,8 +1309,9 @@ impl PyCData { if proto_metaclass.fast_issubclass(super::structure::PyCStructType::static_type()) || proto_metaclass.fast_issubclass(super::union::PyCUnionType::static_type()) || proto_metaclass.fast_issubclass(super::pointer::PyCPointerType::static_type()) + || proto_type.fast_issubclass(super::function::PyCFuncPtr::static_type()) { - cdata_obj.into_ref_with_type(vm, proto_type).map(Into::into) + cdata_obj.into_view(proto_type, vm) } else { // Fallback Ok(vm.ctx.new_int(0).into()) @@ -1735,17 +1937,7 @@ impl Py { match value { PySetterValue::Assign(value) => { // Check if needs byte swapping - let needs_swap = (obj - .class() - .as_object() - .get_attr("_swappedbytes_", vm) - .is_ok() - || zelf - .proto - .as_object() - .get_attr("_swappedbytes_", vm) - .is_ok()) - && size > 1; + let needs_swap = super::simple::is_swapped(&zelf.proto) && size > 1; // PyCData_set cdata.set_field( diff --git a/crates/vm/src/stdlib/_ctypes/function.rs b/crates/vm/src/stdlib/_ctypes/function.rs index 5ee048888..e8537b412 100644 --- a/crates/vm/src/stdlib/_ctypes/function.rs +++ b/crates/vm/src/stdlib/_ctypes/function.rs @@ -645,6 +645,24 @@ pub(super) fn cast_impl( } impl PyCFuncPtr { + pub(super) fn from_cdata(data: PyCData) -> Self { + Self { + _base: data, + thunk: PyRwLock::new(None), + callable: PyRwLock::new(None), + converters: PyRwLock::new(None), + argtypes: PyRwLock::new(None), + restype: PyRwLock::new(None), + checker: PyRwLock::new(None), + errcheck: PyRwLock::new(None), + #[cfg(windows)] + index: PyRwLock::new(None), + #[cfg(windows)] + iid: PyRwLock::new(None), + paramflags: PyRwLock::new(None), + } + } + /// Get function pointer address from buffer fn get_func_ptr(&self) -> usize { let buffer = self._base.buffer.read(); @@ -670,23 +688,9 @@ impl Constructor for PyCFuncPtr { let ptr_size = pointer_size(); if args.args.is_empty() { - return Self { - _base: PyCData::from_bytes(vec![0u8; ptr_size], None), - thunk: PyRwLock::new(None), - callable: PyRwLock::new(None), - converters: PyRwLock::new(None), - argtypes: PyRwLock::new(None), - restype: PyRwLock::new(None), - checker: PyRwLock::new(None), - errcheck: PyRwLock::new(None), - #[cfg(windows)] - index: PyRwLock::new(None), - #[cfg(windows)] - iid: PyRwLock::new(None), - paramflags: PyRwLock::new(None), - } - .into_ref_with_type(vm, cls) - .map(Into::into); + return Self::from_cdata(PyCData::from_bytes(vec![0u8; ptr_size], None)) + .into_ref_with_type(vm, cls) + .map(Into::into); } let first_arg = &args.args[0]; @@ -847,9 +851,13 @@ impl Constructor for PyCFuncPtr { // Store the thunk as a Python object to keep it alive let thunk_ref: PyRef = thunk.into_ref(&vm.ctx); + let base = PyCData::from_bytes(Self::make_ptr_buffer(ptr_val), None); + let objects = vm.ctx.new_dict(); + objects.set_item("0", thunk_ref.clone().into(), vm)?; + *base.objects.write() = Some(objects.into()); return Self { - _base: PyCData::from_bytes(Self::make_ptr_buffer(ptr_val), None), + _base: base, thunk: PyRwLock::new(Some(thunk_ref)), callable: PyRwLock::new(Some(first_arg.clone())), converters: PyRwLock::new(None), diff --git a/crates/vm/src/stdlib/_ctypes/pointer.rs b/crates/vm/src/stdlib/_ctypes/pointer.rs index 2bad8e227..3e9b6058a 100644 --- a/crates/vm/src/stdlib/_ctypes/pointer.rs +++ b/crates/vm/src/stdlib/_ctypes/pointer.rs @@ -288,13 +288,7 @@ impl PyCPointer { /// Set the pointer value in buffer pub(crate) fn set_ptr_value(&self, value: usize) { - let mut buffer = self.0.buffer.write(); - rustpython_host_env::ctypes::write_pointer_to_buffer_at( - buffer.to_mut(), - 0, - rustpython_host_env::ctypes::pointer_size(), - value, - ); + self.0.write_bytes_at_offset(0, &value.to_ne_bytes()); } // contents getter - reads address from b_ptr and creates an instance of the pointed-to type @@ -313,12 +307,13 @@ impl PyCPointer { .stg_info_opt() .map_or_else(rustpython_host_env::ctypes::pointer_size, |info| info.size); - // Create instance that references the memory directly - // PyCData.into_ref_with_type works for all ctypes (simple, structure, union, array, pointer) - let cdata = unsafe { super::base::PyCData::at_address(ptr_val as *const u8, element_size) }; - cdata - .into_ref_with_type(vm, proto_type.to_owned()) - .map(Into::into) + // SAFETY: the pointer must designate valid native memory. Retaining it + // keeps Python-owned referents alive, but cannot validate arbitrary addresses. + let cdata = unsafe { + PyCData::from_base_obj(ptr_val as *mut u8, element_size, zelf.to_owned().into(), 0) + }; + cdata.length.store(proto_type.stg_info(vm)?.length); + cdata.into_view(proto_type.to_owned(), vm) } // contents setter - stores address in b_ptr and keeps reference @@ -400,11 +395,30 @@ impl PyCPointer { // offset = index * iteminfo->size let addr = pointer_item_address(ptr_value, index, element_size); - // Check if it's a simple type (has _type_ attribute) - if let Ok(type_attr) = proto_type.as_object().get_attr("_type_", vm) - && let Ok(type_str) = type_attr.str(vm) + // Direct simple types and their generated endian variants return a + // scalar; user subclasses and aggregates return a view. + if super::simple::returns_scalar(proto_type) + && let Ok(type_attr) = proto_type.as_object().get_attr("_type_", vm) + && let Some(type_str) = type_attr.downcast_ref::() { let type_code = type_str.to_string(); + if type_code == "O" || super::simple::is_swapped(proto_type) { + // SAFETY: as for ordinary pointer indexing below, the caller + // must supply a valid address covering the element's storage. + let bytes = unsafe { + rustpython_host_env::ctypes::borrow_memory(addr as *const u8, element_size) + }; + if type_code == "O" { + return zelf.0.object_at( + rustpython_host_env::ctypes::read_pointer_from_buffer(bytes), + Some(index as usize), + vm, + ); + } + let mut bytes = bytes.to_vec(); + bytes.reverse(); + return Ok(super::base::bytes_to_pyobject(proto_type, &bytes, vm)); + } return Ok(Self::read_value_at_address( addr, element_size, @@ -415,11 +429,19 @@ impl PyCPointer { // Complex type: create instance that references the memory directly (not a copy) // This allows p[i].val = x to modify the original memory - // PyCData.into_ref_with_type works for all ctypes (array, structure, union, pointer) - let cdata = unsafe { super::base::PyCData::at_address(addr as *const u8, element_size) }; - cdata - .into_ref_with_type(vm, proto_type.to_owned()) - .map(Into::into) + // SAFETY: pointer indexing relies on the caller providing valid native + // memory. Retaining the pointer keeps its Python-owned referents alive, + // but does not make arbitrary addresses or concurrent native access safe. + let cdata = unsafe { + PyCData::from_base_obj( + addr as *mut u8, + element_size, + zelf.to_owned().into(), + index as usize, + ) + }; + cdata.length.store(proto_type.stg_info(vm)?.length); + cdata.into_view(proto_type.to_owned(), vm) } // Pointer_subscript slice handling (manual parsing, not PySlice_Unpack) @@ -575,15 +597,30 @@ impl PyCPointer { // Write value at address // Handle Structure/Array types by copying their buffer + if type_code.as_deref() == Some("O") + && value.downcast_ref::().is_some() + && !value.fast_isinstance(proto_type) + { + return Err(vm.new_type_error(format!( + "expected {} instance, not {}", + proto_type.name(), + value.class().name() + ))); + } if let Some(cdata) = value.downcast_ref::() && (cdata.fast_isinstance(PyCStructure::static_type()) || cdata.fast_isinstance(PyCArray::static_type()) || cdata.fast_isinstance(PyCSimple::static_type())) { - let src_buffer = cdata.buffer.read(); + // The value may be a view of this element or overlapping storage. + let bytes = { + let buffer = cdata.buffer.read(); + rustpython_host_env::ctypes::copy_to_sized_bytes(&buffer, element_size) + }; unsafe { - rustpython_host_env::ctypes::copy_bytes_to_address(addr, &src_buffer, element_size); + rustpython_host_env::ctypes::copy_bytes_to_address(addr, &bytes, element_size); } + zelf.0.keep_cdata(index as usize, &value, vm) } else { // Handle z/Z specially to store converted value if type_code.as_deref() == Some("z") @@ -597,8 +634,12 @@ impl PyCPointer { AddressWriteValue::Pointer(ptr_val), ); } - zelf.0.keep_alive(index as usize, kept_alive); - return zelf.0.keep_ref(index as usize, value.clone(), vm); + return zelf.0.keep_ref_with_hidden( + index as usize, + value.clone(), + Some(kept_alive), + vm, + ); } else if type_code.as_deref() == Some("Z") && let Some(s) = value.downcast_ref::() { @@ -612,11 +653,28 @@ impl PyCPointer { } return zelf.0.keep_ref(index as usize, holder, vm); } - Self::write_value_at_address(addr, element_size, &value, type_code.as_deref(), vm)?; + if type_code.as_deref() == Some("O") { + unsafe { + rustpython_host_env::ctypes::write_value_to_address( + addr, + element_size, + AddressWriteValue::Pointer(value.get_id()), + ); + } + } else if super::simple::is_swapped(proto_type) { + let type_code = type_code + .as_deref() + .ok_or_else(|| vm.new_type_error("no _type_ attribute"))?; + let bytes = super::simple::swapped_value_bytes(type_code, &value, vm)?; + unsafe { + rustpython_host_env::ctypes::copy_bytes_to_address(addr, &bytes, element_size); + } + } else { + Self::write_value_at_address(addr, element_size, &value, type_code.as_deref(), vm)?; + } + // KeepRef: store reference to keep value alive using actual index + zelf.0.keep_ref(index as usize, value, vm) } - - // KeepRef: store reference to keep value alive using actual index - zelf.0.keep_ref(index as usize, value, vm) } /// Read a value from memory address diff --git a/crates/vm/src/stdlib/_ctypes/simple.rs b/crates/vm/src/stdlib/_ctypes/simple.rs index 221a54d7f..831d8cc3b 100644 --- a/crates/vm/src/stdlib/_ctypes/simple.rs +++ b/crates/vm/src/stdlib/_ctypes/simple.rs @@ -1,11 +1,12 @@ use super::_ctypes::CArgObject; use super::array::PyCArray; use super::base::{ - CArgValue, CDATA_BUFFER_METHODS, PyCData, StgInfo, StgInfoFlags, bytes_to_pyobject, + CArgValue, CDATA_BUFFER_METHODS, ParamFunc, PyCData, StgInfo, StgInfoFlags, bytes_to_pyobject, }; use super::function::PyCFuncPtr; use super::pointer::PyCPointer; use crate::builtins::{PyByteArray, PyBytes, PyInt, PyNone, PyStr, PyType, PyTypeRef}; +use crate::class::StaticType; use crate::convert::ToPyObject; use crate::function::{Either, FuncArgs, OptionalArg}; use crate::protocol::{BufferDescriptor, PyBuffer, PyNumberMethods, PySequenceMethods}; @@ -569,7 +570,9 @@ impl Initializer for PyCSimpleType { super::base::set_or_init_stginfo(&type_ref, stg_info); // Create __ctype_le__ and __ctype_be__ swapped types - create_swapped_types(&type_ref, &type_str, vm)?; + if type_ref.class().is(Self::static_type()) { + create_swapped_types(&type_ref, &type_str, args, vm)?; + } Ok(()) } @@ -585,15 +588,9 @@ impl Initializer for PyCSimpleType { fn create_swapped_types( type_ref: &Py, type_str: &str, + mut args: FuncArgs, vm: &VirtualMachine, ) -> PyResult<()> { - use crate::builtins::PyDict; - - // Avoid infinite recursion - if __ctype_le__ already exists, skip - if type_ref.as_object().get_attr("__ctype_le__", vm).is_ok() { - return Ok(()); - } - // Types that don't support byte order swapping - no __ctype_le__/__ctype_be__ // c_void_p (P), c_char_p (z), c_wchar_p (Z), c_wchar (u), py_object (O) let unsupported_types = ["P", "z", "Z", "u", "O"]; @@ -617,31 +614,15 @@ fn create_swapped_types( // Check system byte order at compile time let is_little_endian = cfg!(target_endian = "little"); - // Create dict for the swapped (non-native) type - let swapped_dict: crate::PyRef = PyDict::default().into_ref(&vm.ctx); - swapped_dict.set_item("_type_", vm.ctx.new_str(type_str).into(), vm)?; - - // Create the swapped type using the same metaclass - let metaclass = type_ref.class(); - let bases = vm.ctx.new_tuple(vec![type_ref.as_object().to_owned()]); - - // Set placeholder first to prevent recursion - set_attrs!( - type_ref.as_object(), vm, - "__ctype_le__" => vm.ctx.none(), - "__ctype_be__" => vm.ctx.none(), - ); - // Create only the non-native endian type let suffix = if is_little_endian { "_be" } else { "_le" }; - let swapped_type = metaclass.as_object().call( - ( - vm.ctx.new_str(format!("{}{}", type_ref.name(), suffix)), - bases, - swapped_dict.as_object().to_owned(), - ), - vm, - )?; + args.args[0] = vm + .ctx + .new_str(format!("{}{}", type_ref.name(), suffix)) + .into(); + // Endian variants share bases and class attributes; neither subclasses + // the other. Bypass the initializer to avoid generating another pair. + let swapped_type = PyType::slot_new(type_ref.class().to_owned(), args, vm)?; // Set _swappedbytes_ on the swapped type to indicate byte swapping is needed set_attrs!( @@ -650,14 +631,15 @@ fn create_swapped_types( ); // Update swapped type's StgInfo format to use opposite endian prefix - if let Ok(swapped_type_ref) = swapped_type.clone().downcast::() - && let Some(mut sw_stg) = swapped_type_ref.get_type_data_mut::() - { + if let Ok(swapped_type_ref) = swapped_type.clone().downcast::() { + let mut sw_stg = type_ref.stg_info(vm)?.clone(); // Swapped: little-endian system uses big-endian prefix and vice versa + sw_stg.big_endian = is_little_endian; sw_stg.format = Some(alloc_format_string_for_type( type_str.chars().next().unwrap_or('?'), is_little_endian, )); + super::base::set_or_init_stginfo(&swapped_type_ref, sw_stg); } // Set attributes based on system byte order @@ -704,6 +686,32 @@ impl Debug for PyCSimple { } } +pub(super) fn is_swapped(cls: &Py) -> bool { + cls.stg_info_opt().is_some_and(|info| { + info.paramfunc == ParamFunc::Simple && info.big_endian != cfg!(target_endian = "big") + }) +} + +pub(super) fn returns_scalar(cls: &Py) -> bool { + cls.base + .deref() + .is_some_and(|base| base.is(PyCSimple::static_type())) +} + +pub(super) fn swapped_value_bytes( + type_code: &str, + value: &PyObject, + vm: &VirtualMachine, +) -> PyResult> { + // Convert once before writing: numeric conversion may reenter ctypes. + let value = match type_code { + "f" | "d" | "g" => SimpleStorageValue::Float(value.try_float(vm)?.to_f64()), + "?" | "v" => SimpleStorageValue::Bool(value.to_owned().try_to_bool(vm)?), + _ => SimpleStorageValue::Signed(bigint_to_i128_wrapping(value.try_index(vm)?.as_bigint())), + }; + Ok(simple_storage_value_to_bytes_endian(type_code, value, true)) +} + fn value_to_bytes_endian( _type_: &str, value: &PyObject, @@ -979,7 +987,7 @@ impl Constructor for PyCSimple { let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); let buffer = rustpython_host_env::ctypes::pointer_bytes(ptr); let cdata = PyCData::from_bytes(buffer, Some(v.clone())); - *cdata.base.write() = Some(kept_alive); + cdata.keep_alive(0, kept_alive); return Self(cdata).into_ref_with_type(vm, cls).map(Into::into); } } else if _type_ == "Z" @@ -1006,10 +1014,15 @@ impl Constructor for PyCSimple { } }; - // Check if this is a swapped endian type (presence of attribute indicates swapping) - let swapped = cls.as_object().get_attr("_swappedbytes_", vm).is_ok(); + // Storage metadata belongs to this exact type, unlike inherited attributes. + let swapped = is_swapped(&cls); - let buffer = value_to_bytes_endian(&_type_, &value, swapped, vm); + let buffer = if _type_ == "O" && init_arg.is_none() { + // An omitted py_object value is NULL, not the None singleton. + rustpython_host_env::ctypes::pointer_bytes(0) + } else { + value_to_bytes_endian(&_type_, &value, swapped, vm) + }; // For c_char_p (type "z"), c_wchar_p (type "Z"), and py_object (type "O"), // store the initial value in _objects @@ -1117,20 +1130,12 @@ impl Py { // O_get: py_object - read PyObject pointer from buffer if type_code == "O" { - let buffer = zelf.0.buffer.read(); - let ptr = rustpython_host_env::ctypes::read_pointer_from_buffer(&buffer); - if ptr == 0 { - return Err(vm.new_value_error("PyObject is NULL")); - } - // Non-NULL: return stored object from _objects if available - if let Some(obj) = zelf.0.objects.read().as_ref() { - return Ok(obj.clone()); - } - return Err(vm.new_value_error("PyObject is NULL")); + let ptr = rustpython_host_env::ctypes::read_pointer_from_buffer(&zelf.0.buffer.read()); + return zelf.0.object_at(ptr, None, vm); } - // Check if this is a swapped endian type (presence of attribute indicates swapping) - let swapped = cls.as_object().get_attr("_swappedbytes_", vm).is_ok(); + // Storage metadata belongs to this exact type, unlike inherited attributes. + let swapped = is_swapped(cls); // Read value from buffer, swap bytes if needed let buffer = zelf.0.buffer.read(); @@ -1166,30 +1171,25 @@ impl Py { if type_code == "z" { if let Some(bytes) = value.downcast_ref::() { let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm); - *zelf.0.buffer.write() = - alloc::borrow::Cow::Owned(rustpython_host_env::ctypes::pointer_bytes(ptr)); - *zelf.0.objects.write() = Some(value); - *zelf.0.base.write() = Some(kept_alive); + zelf.0 + .write_bytes_at_offset(0, &rustpython_host_env::ctypes::pointer_bytes(ptr)); + zelf.0.keep_value(value, Some(kept_alive), vm)?; return Ok(()); } } else if type_code == "Z" && let Some(s) = value.downcast_ref::() { let (holder, ptr) = super::base::str_to_wchar_bytes(s.as_wtf8(), vm); - *zelf.0.buffer.write() = - alloc::borrow::Cow::Owned(rustpython_host_env::ctypes::pointer_bytes(ptr)); - *zelf.0.objects.write() = Some(holder); + zelf.0 + .write_bytes_at_offset(0, &rustpython_host_env::ctypes::pointer_bytes(ptr)); + zelf.0.keep_value(holder, None, vm)?; return Ok(()); } let content = set_primitive(&type_code, &value, vm)?; - // Check if this is a swapped endian type (presence of attribute indicates swapping) - let swapped = instance - .class() - .as_object() - .get_attr("_swappedbytes_", vm) - .is_ok(); + // Storage metadata belongs to this exact type, unlike inherited attributes. + let swapped = is_swapped(instance.class()); // Update buffer when value changes let buffer_bytes = value_to_bytes_endian(&type_code, &content, swapped, vm); @@ -1198,11 +1198,12 @@ impl Py { // Otherwise replace with new owned buffer let mut buffer = zelf.0.buffer.write(); write_simple_storage_buffer(&mut buffer, &buffer_bytes); + drop(buffer); // For c_char_p (type "z"), c_wchar_p (type "Z"), and py_object (type "O"), // keep the reference in _objects if type_code == "z" || type_code == "Z" || type_code == "O" { - *zelf.0.objects.write() = Some(value); + zelf.0.keep_value(value, None, vm)?; } Ok(()) } diff --git a/crates/vm/src/stdlib/_ctypes/structure.rs b/crates/vm/src/stdlib/_ctypes/structure.rs index 16d24cbac..34d1ef584 100644 --- a/crates/vm/src/stdlib/_ctypes/structure.rs +++ b/crates/vm/src/stdlib/_ctypes/structure.rs @@ -85,7 +85,10 @@ impl Initializer for PyCStructType { // Process _fields_ if defined directly on this class (not inherited) if let Some(fields_attr) = new_type.get_direct_attr(vm.ctx.intern_str("_fields_")) { - Self::process_fields(&new_type, &fields_attr, vm)?; + // Let the metaclass transform field types, as for later assignment. + new_type + .as_object() + .set_attr(vm.ctx.intern_str("_fields_"), fields_attr, vm)?; } else { // No _fields_ defined - try to copy from base class (PyCStgInfo_clone) let (has_base_info, base_clone) = { diff --git a/extra_tests/snippets/stdlib_ctypes.py b/extra_tests/snippets/stdlib_ctypes.py index bd306159f..a61db9323 100644 --- a/extra_tests/snippets/stdlib_ctypes.py +++ b/extra_tests/snippets/stdlib_ctypes.py @@ -4,6 +4,7 @@ import itertools import os as _os import sys as _sys import types as _types +import weakref from _ctypes import RTLD_GLOBAL, RTLD_LOCAL, Array, _SimpleCData, sizeof from _ctypes import CFuncPtr as _CFuncPtr from struct import calcsize as _calcsize @@ -604,3 +605,253 @@ for read in ( raise AssertionError("non-pointer argument wrapper accepted") print("done") + + +# Indexed compound objects are views of the element, not decoded scalars. +class IndexedPoint(ctypes.Structure): + _fields_ = [("x", ctypes.c_int)] + + +class IndexedInt(ctypes.c_int): + pass + + +for _element in (IndexedPoint, IndexedInt, ctypes.c_int * 2): + _owner = (_element * 2)() + _view = _owner[1] + assert type(_view) is _element + assert _view._b_base_ is _owner + assert ctypes.addressof(_view) == ctypes.addressof(_owner) + ctypes.sizeof(_element) + _pointer = ctypes.cast(_owner, ctypes.POINTER(_element)) + _pointer[1] = _pointer[1] + assert bytes(_pointer[1]) == bytes(_view) + +_array = (ctypes.c_int * 2)(1, 2) +_pointer = ctypes.pointer(_array) +for _value, _view in enumerate((_pointer[0], _pointer.contents), 42): + assert _view._b_base_ is _pointer + assert len(_view) == 2 + _view[1] = _value + assert _array[1] == _value + +_callback_type = ctypes.CFUNCTYPE(ctypes.c_int) +_callbacks = (_callback_type * 1)(_callback_type(lambda: 42)) +_callback = _callbacks[0] +assert _callback._b_base_ is _callbacks +_copied_callbacks = (_callback_type * 1)(_callback) +del _callback, _callbacks +gc.collect() +assert _copied_callbacks[0]() == 42 + + +class CallbackField(ctypes.Structure): + _fields_ = [("callback", _callback_type)] + + +assert CallbackField(_copied_callbacks[0]).callback() == 42 + +# Nested element keys and copies must retain both pointees after the source dies. +_nested = (((ctypes.POINTER(ctypes.c_int) * 1) * 1) * 2)() +_refs = [] +for _index in range(2): + _item = ctypes.c_int(_index) + _refs.append(weakref.ref(_item)) + _nested[_index][0][0] = ctypes.pointer(_item) +_copy = type(_nested)(*_nested) +del _nested, _item +gc.collect() +assert [ref().value for ref in _refs] == [0, 1] + +# Replacing a pointer view's contents updates its slot and retains the pointee. +_replacement = ctypes.c_int(42) +_replacement_ref = weakref.ref(_replacement) +_view = _copy[1][0][0] +_view.contents = _replacement +del _replacement, _view +gc.collect() +assert _replacement_ref() is not None +assert _copy[1][0][0].contents.value == 42 + +# Replacing a callback may reenter the same array from its finalizer. +_finalized = [] + + +class ReenteringCallback: + def __call__(self): + return 1 + + def __del__(self): + _finalized.append(_copied_callbacks[0]()) + _copied_callbacks[0] = _callback_type(lambda: 3) + + +_copied_callbacks[0] = _callback_type(ReenteringCallback()) +_copied_callbacks[0] = _callback_type(lambda: 2) +assert _finalized == [2] +assert _copied_callbacks[0]() == 3 + +# Pointer-valued scalar views retain their owner and write through to it. +for _type, _old, _new in ( + (ctypes.c_char_p, b"old", b"new"), + (ctypes.c_wchar_p, "old", "new"), +): + + class IndexedStringPointer(_type): + pass + + _owner = (IndexedStringPointer * 1)(_old) + _view = _owner[0] + _view.value = _new + assert _view._b_base_ is _owner + assert _owner[0].value == _new + _copy = type(_owner)(_view) + del _owner, _view + gc.collect() + assert _copy[0].value == _new + + +# Reentry must see matching pointer bytes and owners, including private copies. +class StringField(ctypes.Structure): + _fields_ = [("text", ctypes.c_char_p)] + + +class ReenteringBytes(bytes): + def __del__(self): + if _replace_record: + _strings[0] = StringField(_inner) + _events.append(_strings[0].text) + else: + _strings.text = _inner + _events.append(_strings.text) + + +_inner = b"inner replacement" * 20 +for _replace_record in (False, True): + _events = [] + if _replace_record: + _strings = (StringField * 1)(StringField(ReenteringBytes(b"old"))) + _strings[0] = StringField(b"outer replacement") + assert _strings[0].text == _inner + else: + _strings = StringField(ReenteringBytes(b"old")) + _strings.text = b"outer replacement" + assert _strings.text == _inner + assert _events == [_inner] + + +# py_object views locate the referent in their owner, including after a copy. +class IndexedObject(ctypes.py_object): + pass + + +_object = {} +_object_scalar = IndexedObject(_object) +_objects = (IndexedObject * 1)(_object_scalar) +assert type(_objects).from_buffer(memoryview(_objects))[0].value is _object +_object_pointer = ctypes.cast(_objects, ctypes.POINTER(IndexedObject)) +_object_pointer[0] = _object_scalar +_object_scalar.value = object() +assert _object_pointer[0].value is _object +_copy = type(_objects)(_objects[0]) +del _objects, _object_pointer +gc.collect() +assert _copy[0].value is _object +_copy[0].value = None +assert _copy[0].value is None + + +class RetentionKey(str): + __hash__ = str.__hash__ + + def __eq__(self, other): + raise AssertionError("py_object lookup called key equality") + + +_copy[0].value = _object +_copy._objects.clear() +_copy._objects[RetentionKey("0:0")] = _object +assert _copy[0].value is _object + +# Generated endian types return scalars, but their user subclasses return views. +_swapped = ( + ctypes.c_uint32.__ctype_be__ + if _sys.byteorder == "little" + else ctypes.c_uint32.__ctype_le__ +) +_words = (_swapped * 1).from_buffer_copy(b"\x01\x02\x03\x04") +_word_pointer = ctypes.cast(_words, ctypes.POINTER(_swapped)) +_expected = int.from_bytes( + bytes(_words), "big" if _sys.byteorder == "little" else "little" +) +assert type(_words[0]) is type(_word_pointer[0]) is int +assert _words[0] == _word_pointer[0] == _expected +_words[0] = 0x11223344 +assert bytes(_words) == bytes(_swapped(0x11223344)) +_word_pointer[0] = 0x55667788 +assert bytes(_words) == bytes(_swapped(0x55667788)) + + +class IndexedSwapped(_swapped): + pass + + +_words = (IndexedSwapped * 1).from_buffer_copy(b"\x01\x02\x03\x04") +assert type(_words[0]) is IndexedSwapped +assert _words[0].value == int.from_bytes(bytes(_words), _sys.byteorder) + +# Field assignment copies encoded instances; only numeric inputs are converted. +for _base in (ctypes.Structure, ctypes.Union): + for _field_type in (_swapped, IndexedSwapped): + + class EndianField(_base): + _fields_ = [("word", _field_type)] + + _value = _field_type(0x01020304) + _aggregate = EndianField(_value) + assert bytes(_aggregate) == bytes(_value) + _field = EndianField.from_buffer_copy(bytes(_value)).word + assert getattr(_field, "value", _field) == _value.value + if _field_type is IndexedSwapped: + _aggregate.word = 0x55667788 + assert bytes(_aggregate) == bytes(IndexedSwapped(0x55667788)) + +# Endian fields reject the native sibling rather than converting its bytes. +assert not issubclass(_swapped, ctypes.c_uint32) +for _base in ( + (ctypes.BigEndianStructure, ctypes.BigEndianUnion) + if _sys.byteorder == "little" + else (ctypes.LittleEndianStructure, ctypes.LittleEndianUnion) +): + + class ConvertedEndianField(_base): + _fields_ = [("word", ctypes.c_uint32)] + + assert ConvertedEndianField._fields_[0][1] is _swapped + try: + ConvertedEndianField(ctypes.c_uint32(0x01020304)) + except TypeError: + pass + else: + raise AssertionError("incompatible native field value accepted") + _value = _swapped(0x01020304) + assert bytes(ConvertedEndianField(_value)) == bytes(_value) + + +# Array assignment cannot complete a pointer field's missing target type. +class IncompletePointer(ctypes._Pointer): + pass + + +class IncompletePointerField(ctypes.Structure): + _fields_ = [("value", IncompletePointer)] + + +_field = IncompletePointerField() +_before = bytes(_field) +try: + _field.value = (ctypes.c_int * 1)(42) +except TypeError: + pass +else: + raise AssertionError("array accepted for an incomplete pointer field") +assert bytes(_field) == _before