From 1150f507e2c3ca564279cef1759c6ebc4ed09293 Mon Sep 17 00:00:00 2001 From: Kangzhi Shi Date: Mon, 31 Aug 2020 11:48:50 +0200 Subject: [PATCH 01/12] Fix: array with float type should accept integer parameter --- tests/snippets/stdlib_array.py | 14 +++++- vm/src/stdlib/array.rs | 80 ++++++++++++++++++++++++++-------- 2 files changed, 75 insertions(+), 19 deletions(-) diff --git a/tests/snippets/stdlib_array.py b/tests/snippets/stdlib_array.py index 0ef0969ff..8d4b9d85e 100644 --- a/tests/snippets/stdlib_array.py +++ b/tests/snippets/stdlib_array.py @@ -19,4 +19,16 @@ a = array("b", [0, 1, 2, 3]) b = a assert a.__ne__(b) is False b = array("B", [3, 2, 1, 0]) -assert a.__ne__(b) is True \ No newline at end of file +assert a.__ne__(b) is True + +def test_float_with_integer_input(): + f = array("f", [0, 1, 2.0, 3.0]) + f.append(4) + f.insert(0, -1) + assert f.count(4) == 1 + f.remove(1) + assert f.index(0) == 1 + f[0] = -2 + assert f == array("f", [-2, 0, 2, 3, 4]) + +test_float_with_integer_input() diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index d01773e9d..c5a6e0a64 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -4,6 +4,7 @@ use crate::common::cell::{ }; use crate::function::OptionalArg; use crate::obj::objbytes::PyBytesRef; +use crate::obj::objfloat::try_float; use crate::obj::objsequence::PySliceableSequence; use crate::obj::objslice::PySliceRef; use crate::obj::objstr::PyStringRef; @@ -11,7 +12,7 @@ use crate::obj::objtype::PyClassRef; use crate::obj::{objbool, objiter}; use crate::pyobject::{ BorrowValue, Either, IntoPyObject, PyArithmaticValue, PyClassImpl, PyComparisonValue, - PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::VirtualMachine; use crossbeam_utils::atomic::AtomicCell; @@ -74,7 +75,7 @@ macro_rules! def_array_enum { fn push(&mut self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { match self { $(ArrayContentType::$n(v) => { - let val = $t::try_from_object(vm, obj)?; + let val = $t::try_into_from_object(vm, obj)?; v.push(val); })* } @@ -92,7 +93,7 @@ macro_rules! def_array_enum { fn insert(&mut self, i: usize, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { match self { $(ArrayContentType::$n(v) => { - let val = $t::try_from_object(vm, obj)?; + let val = $t::try_into_from_object(vm, obj)?; v.insert(i, val); })* } @@ -102,9 +103,11 @@ macro_rules! def_array_enum { fn count(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { match self { $(ArrayContentType::$n(v) => { - Ok(>::try_from_object(vm, obj)?.map_or(0, |val| { + Ok(if let Ok(val) = $t::try_into_from_object(vm, obj) { v.iter().filter(|&&a| a == val).count() - })) + } else { + 0 + }) })* } } @@ -112,13 +115,15 @@ macro_rules! def_array_enum { fn remove(&mut self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()>{ match self { $(ArrayContentType::$n(v) => { - let pos = >::try_from_object(vm, obj)?.map_or(None, |val| { - v.iter().position(|&a| a == val) - }).ok_or_else(||vm.new_value_error("array.remove(x): x not in array".to_owned()) )?; - v.remove(pos); + if let Ok(val) = $t::try_into_from_object(vm, obj) { + if let Some(pos) = v.iter().position(|&a| a == val) { + v.remove(pos); + return Ok(()); + } + } + Err(vm.new_value_error("array.remove(x): x not in array".to_owned())) })* } - Ok(()) } fn frombytes(&mut self, b: &[u8]) { @@ -156,12 +161,15 @@ macro_rules! def_array_enum { } } - fn index(&self, x: PyObjectRef, vm: &VirtualMachine) -> PyResult> { + fn index(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { match self { $(ArrayContentType::$n(v) => { - Ok(>::try_from_object(vm, x)?.map_or(None, |val| { - v.iter().position(|&a| a == val) - })) + if let Ok(val) = $t::try_into_from_object(vm, obj) { + if let Some(pos) = v.iter().position(|&a| a == val) { + return Ok(pos); + } + } + Err(vm.new_value_error("array.index(x): x not in array".to_owned())) })* } } @@ -228,7 +236,7 @@ macro_rules! def_array_enum { Either::A(i) => { let i = self.idx(i, "array assignment", vm)?; match self { - $(ArrayContentType::$n(v) => { v[i] = TryFromObject::try_from_object(vm, value)? },)* + $(ArrayContentType::$n(v) => { v[i] = $t::try_into_from_object(vm, value)? },)* } Ok(()) } @@ -264,6 +272,44 @@ def_array_enum!( (Double, f64, 'd'), ); +trait TryIntoFromObject: Sized { + fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult; +} + +macro_rules! adapt_try_into_from_object { + ($(($t:ty, $f: path),)*) => {$( + impl TryIntoFromObject for $t { + fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + $f(vm, obj) + } + } + )*}; +} + +adapt_try_into_from_object!( + (i8, i8::try_from_object), + (u8, u8::try_from_object), + (i16, i16::try_from_object), + (u16, u16::try_from_object), + (i32, i32::try_from_object), + (u32, u32::try_from_object), + (i64, i64::try_from_object), + (u64, u64::try_from_object), + (f32, f32_try_into_from_object), + (f64, f64_try_into_from_object), +); + +fn f32_try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + try_float(&obj, vm)? + .map(|x| x as f32) + .ok_or_else(|| vm.new_type_error(format!("must be real number, not {}", obj.class().name))) +} + +fn f64_try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + try_float(&obj, vm)? + .ok_or_else(|| vm.new_type_error(format!("must be real number, not {}", obj.class().name))) +} + #[pyclass(module = "array", name = "array")] #[derive(Debug)] pub struct PyArray { @@ -364,9 +410,7 @@ impl PyArray { #[pymethod] fn index(&self, x: PyObjectRef, vm: &VirtualMachine) -> PyResult { - self.borrow_value() - .index(x, vm)? - .ok_or_else(|| vm.new_value_error("x not in array".to_owned())) + self.borrow_value().index(x, vm) } #[pymethod] From 0d014935f052e20e2d4adf24f1b0dd3a99a2974e Mon Sep 17 00:00:00 2001 From: Kangzhi Shi Date: Mon, 31 Aug 2020 18:14:21 +0200 Subject: [PATCH 02/12] Implement array __add__ --- Lib/test/test_array.py | 2 -- vm/src/stdlib/array.rs | 28 ++++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index c82a35a76..a82938828 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -549,8 +549,6 @@ class BaseTest: self.assertIs(a > ab, False) self.assertIs(a >= ab, False) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_add(self): a = array.array(self.typecode, self.example) \ + array.array(self.typecode, self.example[::-1]) diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index af5dd5ad7..f47e6a202 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -335,6 +335,25 @@ macro_rules! def_array_enum { Ok(()) } + fn add(&self, other: &ArrayContentType, vm: &VirtualMachine) -> PyResult { + match self { + $(ArrayContentType::$n(v) => if let ArrayContentType::$n(other) = other { + let elements = v.iter().chain(other.iter()).cloned().collect(); + let sliced = ArrayContentType::$n(elements); + let obj = PyObject::new( + PyArray { + array: PyRwLock::new(sliced) + }, + PyArray::class(vm), + None + ); + Ok(obj) + } else { + Err(vm.new_type_error("bad argument type for built-in operation".to_owned())) + },)* + } + } + fn repr(&self, _vm: &VirtualMachine) -> PyResult { // we don't need ReprGuard here let s = match self { @@ -580,6 +599,15 @@ impl PyArray { } } + #[pymethod(name = "__add__")] + fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(other) = other.payload::() { + self.borrow_value().add(&*other.borrow_value(), vm) + } else { + Err(vm.new_type_error(format!("can only append array (not \"{}\") to array", other.class().name))) + } + } + #[pymethod(name = "__repr__")] fn repr(zelf: PyRef, vm: &VirtualMachine) -> PyResult { zelf.borrow_value().repr(vm) From 22be6c89c1ec4bab3209cda63444b1db73c4a560 Mon Sep 17 00:00:00 2001 From: Kangzhi Shi Date: Mon, 31 Aug 2020 19:30:22 +0200 Subject: [PATCH 03/12] Implement array __iadd__ --- Lib/test/test_array.py | 2 -- vm/src/stdlib/array.rs | 30 +++++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index a82938828..b0179059e 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -562,8 +562,6 @@ class BaseTest: self.assertRaises(TypeError, a.__add__, "bad") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_iadd(self): a = array.array(self.typecode, self.example[::-1]) b = a diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index f47e6a202..2fd74a2e0 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -354,6 +354,17 @@ macro_rules! def_array_enum { } } + fn iadd(&mut self, other: ArrayContentType, vm: &VirtualMachine) -> PyResult<()> { + match self { + $(ArrayContentType::$n(v) => if let ArrayContentType::$n(mut other) = other { + v.append(&mut other); + Ok(()) + } else { + Err(vm.new_type_error("can only extend with array of same kind".to_owned())) + },)* + } + } + fn repr(&self, _vm: &VirtualMachine) -> PyResult { // we don't need ReprGuard here let s = match self { @@ -604,7 +615,24 @@ impl PyArray { if let Some(other) = other.payload::() { self.borrow_value().add(&*other.borrow_value(), vm) } else { - Err(vm.new_type_error(format!("can only append array (not \"{}\") to array", other.class().name))) + Err(vm.new_type_error(format!( + "can only append array (not \"{}\") to array", + other.class().name + ))) + } + } + + #[pymethod(name = "__iadd__")] + fn iadd(zelf: PyRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(other) = other.payload::() { + let other = other.borrow_value().clone(); + let result = zelf.borrow_value_mut().iadd(other, vm); + result.map(|_| zelf.into_object()) + } else { + Err(vm.new_type_error(format!( + "can only extend array with array (not \"{}\")", + other.class().name + ))) } } From d26321212f5a11a371a762a2a3dd428a61c78f05 Mon Sep 17 00:00:00 2001 From: Kangzhi Shi Date: Mon, 31 Aug 2020 20:11:48 +0200 Subject: [PATCH 04/12] Fix conflict --- vm/src/stdlib/array.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index ff674a241..269ad6252 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -233,7 +233,7 @@ macro_rules! def_array_enum { Either::B(slice) => self.getitem_by_slice(slice, vm), } } - + fn setitem_by_slice(&mut self, slice: PySliceRef, items: &ArrayContentType, vm: &VirtualMachine) -> PyResult<()> { let start = slice.start_index(vm)?; let stop = slice.stop_index(vm)?; @@ -338,7 +338,7 @@ macro_rules! def_array_enum { fn setitem_by_idx(&mut self, i: isize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let i = self.idx(i, "array assignment", vm)?; match self { - $(ArrayContentType::$n(v) => { v[i] = TryFromObject::try_from_object(vm, value)? },)* + $(ArrayContentType::$n(v) => { v[i] = $t::try_into_from_object(vm, value)? },)* } Ok(()) } From d332c1f5a29ae6c58ffbdc9d888334af54850c1c Mon Sep 17 00:00:00 2001 From: Kangzhi Shi Date: Mon, 31 Aug 2020 20:40:03 +0200 Subject: [PATCH 05/12] Implement array __mul__ and __rmul__ --- Lib/test/test_array.py | 4 ---- vm/src/stdlib/array.rs | 27 +++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index b0179059e..c85962fa4 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -38,8 +38,6 @@ class MiscTest(unittest.TestCase): self.assertRaises(TypeError, array.array, 'xx') self.assertRaises(ValueError, array.array, 'x') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_empty(self): # Exercise code for handling zero-length arrays a = array.array('B') @@ -583,8 +581,6 @@ class BaseTest: self.assertRaises(TypeError, a.__iadd__, "bad") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_mul(self): a = 5*array.array(self.typecode, self.example) self.assertEqual( diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index 2fd74a2e0..119894f39 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -365,6 +365,23 @@ macro_rules! def_array_enum { } } + fn mul(&self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { + let counter = if counter < 0 { 0 } else { counter as usize }; + match self { + $(ArrayContentType::$n(v) => { + let elements = v.iter().cycle().take(v.len() * counter).cloned().collect(); + let sliced = ArrayContentType::$n(elements); + PyObject::new( + PyArray { + array: PyRwLock::new(sliced) + }, + PyArray::class(vm), + None + ) + })* + } + } + fn repr(&self, _vm: &VirtualMachine) -> PyResult { // we don't need ReprGuard here let s = match self { @@ -636,6 +653,16 @@ impl PyArray { } } + #[pymethod(name = "__mul__")] + fn mul(&self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { + self.borrow_value().mul(counter, vm) + } + + #[pymethod(name = "__rmul__")] + fn rmul(&self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { + self.mul(counter, &vm) + } + #[pymethod(name = "__repr__")] fn repr(zelf: PyRef, vm: &VirtualMachine) -> PyResult { zelf.borrow_value().repr(vm) From cbaa7e1c00215a30db7cf25620a7df44f56d921a Mon Sep 17 00:00:00 2001 From: Kangzhi Shi Date: Mon, 31 Aug 2020 21:04:00 +0200 Subject: [PATCH 06/12] Implement array __imul__ --- Lib/test/test_array.py | 2 -- vm/src/stdlib/array.rs | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index c85962fa4..8d68176ff 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -614,8 +614,6 @@ class BaseTest: self.assertRaises(TypeError, a.__mul__, "bad") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_imul(self): a = array.array(self.typecode, self.example) b = a diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index 119894f39..597602516 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -382,6 +382,16 @@ macro_rules! def_array_enum { } } + fn imul(&mut self, counter: isize) { + let counter = if counter < 0 { 0 } else { counter as usize }; + match self { + $(ArrayContentType::$n(v) => { + let mut elements = v.iter().cycle().take(v.len() * counter).cloned().collect(); + std::mem::swap(v, &mut elements); + })* + } + } + fn repr(&self, _vm: &VirtualMachine) -> PyResult { // we don't need ReprGuard here let s = match self { @@ -663,6 +673,12 @@ impl PyArray { self.mul(counter, &vm) } + #[pymethod(name = "__imul__")] + fn imul(zelf: PyRef, counter: isize) -> PyRef { + zelf.borrow_value_mut().imul(counter); + zelf + } + #[pymethod(name = "__repr__")] fn repr(zelf: PyRef, vm: &VirtualMachine) -> PyResult { zelf.borrow_value().repr(vm) From 717784d38ae64ce356f34cd8f313780db8a5fb14 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 1 Sep 2020 12:28:45 -0500 Subject: [PATCH 07/12] Fix test_calendar tests --- Cargo.lock | 1 + Cargo.toml | 1 + Lib/_codecs.py | 4 ++-- Lib/test/test_calendar.py | 7 ------- src/main.rs | 13 +++++++++++++ 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b4507ae3b..21bd128a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1585,6 +1585,7 @@ dependencies = [ "env_logger", "flame", "flamescope", + "libc", "log", "num-traits", "rustpython-compiler", diff --git a/Cargo.toml b/Cargo.toml index 2eaf31095..38f080585 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ pylib = { package = "rustpython-pylib", path = "vm/pylib-crate", version = "0.1. dirs = { package = "dirs-next", version = "1.0" } num-traits = "0.2.8" cfg-if = "0.1" +libc = "0.2" flame = { version = "0.2", optional = true } flamescope = { version = "0.1", optional = true } diff --git a/Lib/_codecs.py b/Lib/_codecs.py index 2c86f4932..eca7a38c2 100644 --- a/Lib/_codecs.py +++ b/Lib/_codecs.py @@ -1006,8 +1006,8 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru # have any nested variables. def STORECHAR(CH, byteorder): - hi = chr(((CH) >> 8) & 0xff) - lo = chr((CH) & 0xff) + hi = (CH >> 8) & 0xff + lo = CH & 0xff if byteorder == 'little': return [lo, hi] else: diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index aa0dd6d87..6241d114d 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -817,18 +817,14 @@ class CommandLineTestCase(unittest.TestCase): self.assertIn(b'January', stdout) self.assertIn(b'Mo Tu We Th Fr Sa Su', stdout) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_output_year(self): stdout = self.run_ok('2004') self.assertEqual(stdout, conv(result_2004_text)) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_output_month(self): stdout = self.run_ok('2004', '1') self.assertEqual(stdout, conv(result_2004_01_text)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_option_encoding(self): self.assertFailure('-e') self.assertFailure('--encoding') @@ -860,7 +856,6 @@ class CommandLineTestCase(unittest.TestCase): stdout = self.run_ok('--width', '3', '2004') self.assertIn(b'Mon Tue Wed Thu Fri Sat Sun', stdout) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_option_lines(self): self.assertFailure('-l') self.assertFailure('--lines') @@ -875,7 +870,6 @@ class CommandLineTestCase(unittest.TestCase): stdout = self.run_ok('--spacing', '8', '2004') self.assertIn(b'Su Mo', stdout) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_option_months(self): self.assertFailure('-m') self.assertFailure('--month') @@ -883,7 +877,6 @@ class CommandLineTestCase(unittest.TestCase): stdout = self.run_ok('--months', '1', '2004') self.assertIn(conv('\nMo Tu We Th Fr Sa Su\n'), stdout) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_option_type(self): self.assertFailure('-t') self.assertFailure('--type') diff --git a/src/main.rs b/src/main.rs index 9c4e075be..dda81e9e2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -36,6 +36,19 @@ fn main() { settings.initialization_parameter = InitParameter::InitializeInternal; } + // don't translate newlines (\r\n <=> \n) + #[cfg(windows)] + { + extern "C" { + fn _setmode(fd: i32, flags: i32) -> i32; + } + unsafe { + _setmode(0, libc::O_BINARY); + _setmode(1, libc::O_BINARY); + _setmode(2, libc::O_BINARY); + } + } + let vm = VirtualMachine::new(settings); let res = run_rustpython(&vm, &matches); From 5dcc8146fe8125b8ca2dc5e8799cd01a446e9552 Mon Sep 17 00:00:00 2001 From: Kangzhi Shi Date: Wed, 2 Sep 2020 07:46:11 +0200 Subject: [PATCH 08/12] Replace PyObject::new by into_simple_object --- vm/src/stdlib/array.rs | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index 597602516..9ca633da7 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -11,7 +11,7 @@ use crate::obj::objtype::PyClassRef; use crate::obj::{objbool, objiter}; use crate::pyobject::{ BorrowValue, Either, IdProtocol, IntoPyObject, PyArithmaticValue, PyClassImpl, - PyComparisonValue, PyIterable, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + PyComparisonValue, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::VirtualMachine; @@ -203,13 +203,10 @@ macro_rules! def_array_enum { $(ArrayContentType::$n(v) => { let elements = v.get_slice_items(vm, &slice)?; let sliced = ArrayContentType::$n(elements); - let obj = PyObject::new( - PyArray { - array: PyRwLock::new(sliced) - }, - PyArray::class(vm), - None - ); + let obj = PyArray { + array: PyRwLock::new(sliced) + } + .into_simple_object(vm); Ok(obj) })* } @@ -340,13 +337,10 @@ macro_rules! def_array_enum { $(ArrayContentType::$n(v) => if let ArrayContentType::$n(other) = other { let elements = v.iter().chain(other.iter()).cloned().collect(); let sliced = ArrayContentType::$n(elements); - let obj = PyObject::new( - PyArray { - array: PyRwLock::new(sliced) - }, - PyArray::class(vm), - None - ); + let obj = PyArray { + array: PyRwLock::new(sliced) + } + .into_simple_object(vm); Ok(obj) } else { Err(vm.new_type_error("bad argument type for built-in operation".to_owned())) @@ -371,13 +365,10 @@ macro_rules! def_array_enum { $(ArrayContentType::$n(v) => { let elements = v.iter().cycle().take(v.len() * counter).cloned().collect(); let sliced = ArrayContentType::$n(elements); - PyObject::new( - PyArray { - array: PyRwLock::new(sliced) - }, - PyArray::class(vm), - None - ) + PyArray { + array: PyRwLock::new(sliced) + } + .into_simple_object(vm) })* } } From 2bc343d45fa8e2b618923d0c228f26ba2df4fd35 Mon Sep 17 00:00:00 2001 From: Kangzhi Shi Date: Wed, 2 Sep 2020 07:56:17 +0200 Subject: [PATCH 09/12] Remove PyResult wrap, array.count() never fail. --- vm/src/stdlib/array.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index 269ad6252..2367d31a0 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -103,14 +103,14 @@ macro_rules! def_array_enum { Ok(()) } - fn count(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn count(&self, obj: PyObjectRef, vm: &VirtualMachine) -> usize { match self { $(ArrayContentType::$n(v) => { - Ok(if let Ok(val) = $t::try_into_from_object(vm, obj) { + if let Ok(val) = $t::try_into_from_object(vm, obj) { v.iter().filter(|&&a| a == val).count() } else { 0 - }) + } })* } } @@ -385,13 +385,13 @@ def_array_enum!( (Double, f64, 'd'), ); -trait TryIntoFromObject: Sized { +trait ArrayElement: Sized { fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult; } macro_rules! adapt_try_into_from_object { ($(($t:ty, $f: path),)*) => {$( - impl TryIntoFromObject for $t { + impl ArrayElement for $t { fn try_into_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { $f(vm, obj) } @@ -490,7 +490,7 @@ impl PyArray { } #[pymethod] - fn count(&self, x: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn count(&self, x: PyObjectRef, vm: &VirtualMachine) -> usize { self.borrow_value().count(x, vm) } From 51d6785ff11c8bf6ec7f866758736ddd4e17e79c Mon Sep 17 00:00:00 2001 From: hyperbora Date: Wed, 2 Sep 2020 21:17:15 +0900 Subject: [PATCH 10/12] implement socket.inet_pton, socket.inet_ntop --- vm/src/stdlib/socket.rs | 68 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 7eea1d507..84783ca96 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -1,6 +1,6 @@ use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use std::io::{self, prelude::*}; -use std::net::{Ipv4Addr, Shutdown, SocketAddr, ToSocketAddrs}; +use std::net::{Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr, ToSocketAddrs}; use std::time::Duration; use byteorder::{BigEndian, ByteOrder}; @@ -596,6 +596,56 @@ fn socket_gethostbyname(name: PyStringRef, vm: &VirtualMachine) -> PyResult PyResult { + match af_inet { + c::AF_INET => ip_string + .borrow_value() + .parse::() + .map(|ip_addr| vm.ctx.new_bytes(ip_addr.octets().to_vec())) + .map_err(|_| { + vm.new_os_error("illegal IP address string passed to inet_pton".to_owned()) + }), + c::AF_INET6 => ip_string + .borrow_value() + .parse::() + .map(|ip_addr| vm.ctx.new_bytes(ip_addr.octets().to_vec())) + .map_err(|_| { + vm.new_os_error("illegal IP address string passed to inet_pton".to_owned()) + }), + _ => Err(vm.new_os_error("Address family not supported by protocol".to_owned())), + } +} + +fn socket_inet_ntop(af_inet: i32, packed_ip: PyBytesRef, vm: &VirtualMachine) -> PyResult { + match af_inet { + c::AF_INET => { + if packed_ip.len() != 4 { + return Err( + vm.new_value_error("invalid length of packed IP address string".to_owned()) + ); + } + let ip_num = BigEndian::read_u32(&packed_ip); + Ok(vm.ctx.new_str(Ipv4Addr::from(ip_num).to_string())) + } + c::AF_INET6 => { + if packed_ip.len() != 16 { + return Err( + vm.new_value_error("invalid length of packed IP address string".to_owned()) + ); + } + let mut iter = packed_ip.iter(); + let mut ip_num: u128 = *iter.next().unwrap() as u128; + for item in iter { + ip_num <<= 8; + ip_num |= *item as u128 + } + let ipv6 = Ipv6Addr::from(ip_num); + Ok(vm.ctx.new_str(get_ipv6_addr_str(ipv6))) + } + _ => Err(vm.new_value_error(format!("unknown address family {}", af_inet))), + } +} + fn get_addr(vm: &VirtualMachine, addr: T) -> PyResult where T: ToSocketAddrs, @@ -667,6 +717,20 @@ fn convert_sock_error(vm: &VirtualMachine, err: io::Error) -> PyBaseExceptionRef } } +fn get_ipv6_addr_str(ipv6: Ipv6Addr) -> String { + match ipv6.to_ipv4() { + Some(_) => { + let segments = ipv6.segments(); + if segments[5] == 0 && segments[6] == 0 { + format!("::{:x}", segments[7] as u32) + } else { + ipv6.to_string() + } + } + None => ipv6.to_string(), + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; let socket_timeout = ctx.new_class( @@ -694,6 +758,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "ntohs" => ctx.new_function(u16::from_be), "getdefaulttimeout" => ctx.new_function(|vm: &VirtualMachine| vm.get_none()), "has_ipv6" => ctx.new_bool(false), + "inet_pton" => ctx.new_function(socket_inet_pton), + "inet_ntop" => ctx.new_function(socket_inet_ntop), // constants "AF_UNSPEC" => ctx.new_int(0), "AF_INET" => ctx.new_int(c::AF_INET), From c5116b70dd41fd8a48f91046cc4e4dcd5bce08fe Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 1 Sep 2020 21:55:36 -0500 Subject: [PATCH 11/12] quickfix for conv() in test_calendar --- Lib/test/test_calendar.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 6241d114d..fd39c4093 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -787,7 +787,9 @@ class LeapdaysTestCase(unittest.TestCase): def conv(s): - return s.replace('\n', os.linesep).encode() + # XXX RUSTPYTHON TODO: TextIOWrapper newline translation + return s.encode() + # return s.replace('\n', os.linesep).encode() class CommandLineTestCase(unittest.TestCase): def run_ok(self, *args): From 55946604593f222969392082037a49cef9facca7 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 1 Sep 2020 22:08:46 -0500 Subject: [PATCH 12/12] Fix bytes decode/encoding tests --- Lib/_codecs.py | 13 ++++++------- Lib/test/test_bytes.py | 4 ---- vm/src/bytesinner.rs | 26 ++++++++++++++++++++++++++ vm/src/obj/objbytearray.rs | 27 ++++++--------------------- vm/src/obj/objbytes.rs | 29 +++++++---------------------- 5 files changed, 45 insertions(+), 54 deletions(-) diff --git a/Lib/_codecs.py b/Lib/_codecs.py index eca7a38c2..160e51fb8 100644 --- a/Lib/_codecs.py +++ b/Lib/_codecs.py @@ -915,7 +915,7 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru p = [] if byteorder == 'native': if (size >= 2): - bom = (ord(s[ihi]) << 8) | ord(s[ilo]) + bom = (s[ihi] << 8) | s[ilo] #ifdef BYTEORDER_IS_LITTLE_ENDIAN if sys.byteorder == 'little': if (bom == 0xFEFF): @@ -962,11 +962,11 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru # /* The remaining input chars are ignored if the callback ## chooses to skip the input */ - ch = (ord(s[q+ihi]) << 8) | ord(s[q+ilo]) + ch = (s[q+ihi] << 8) | s[q+ilo] q += 2 if (ch < 0xD800 or ch > 0xDFFF): - p += chr(ch) + p.append(chr(ch)) continue #/* UTF-16 code pair: */ @@ -977,15 +977,14 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) if (0xD800 <= ch and ch <= 0xDBFF): - ch2 = (ord(s[q+ihi]) << 8) | ord(s[q+ilo]) + ch2 = (s[q+ihi] << 8) | s[q+ilo] q += 2 if (0xDC00 <= ch2 and ch2 <= 0xDFFF): #ifndef Py_UNICODE_WIDE if sys.maxunicode < 65536: - p += chr(ch) - p += chr(ch2) + p += [chr(ch), chr(ch2)] else: - p += chr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000) + p.append(chr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000)) #endif continue diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index c1fbb225d..cbb4bfd4d 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -300,8 +300,6 @@ class BaseBytesTest: for step in indices[1:]: self.assertEqual(b[start:stop:step], self.type2test(L[start:stop:step])) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encoding(self): sample = "Hello world\n\u1234\u5678\u9abc" for enc in ("utf-8", "utf-16"): @@ -311,8 +309,6 @@ class BaseBytesTest: b = self.type2test(sample, "latin-1", "ignore") self.assertEqual(b, self.type2test(sample[:-3], "utf-8")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decode(self): sample = "Hello world\n\u1234\u5678\u9abc" for enc in ("utf-8", "utf-16"): diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index dc42c900b..f9650fa94 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -1290,3 +1290,29 @@ impl<'s> PyCommonString<'s, u8> for [u8] { splited } } + +#[derive(FromArgs)] +pub struct DecodeArgs { + #[pyarg(positional_or_keyword, default = "None")] + encoding: Option, + #[pyarg(positional_or_keyword, default = "None")] + errors: Option, +} + +pub fn bytes_decode( + zelf: PyObjectRef, + args: DecodeArgs, + vm: &VirtualMachine, +) -> PyResult { + let DecodeArgs { encoding, errors } = args; + vm.decode(zelf, encoding.clone(), errors)? + .downcast::() + .map_err(|obj| { + vm.new_type_error(format!( + "'{}' decoder returned '{}' instead of 'str'; use codecs.encode() to \ + encode arbitrary types", + encoding.as_ref().map_or("utf-8", |s| s.borrow_value()), + obj.lease_class().name, + )) + }) +} diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 858f562ec..6fff56797 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -7,17 +7,17 @@ use std::mem::size_of; use super::objint::PyIntRef; use super::objiter; use super::objsequence::SequenceIndex; -use super::objstr::{PyString, PyStringRef}; +use super::objstr::PyStringRef; use super::objtype::PyClassRef; use crate::bytesinner::{ - ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, - ByteInnerTranslateOptions, ByteOr, PyBytesInner, + bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, + ByteInnerSplitOptions, ByteInnerTranslateOptions, ByteOr, DecodeArgs, PyBytesInner, }; use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use crate::function::{OptionalArg, OptionalOption}; use crate::pyobject::{ BorrowValue, Either, PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef, - PyResult, PyValue, TryFromObject, TypeProtocol, + PyResult, PyValue, TryFromObject, }; use crate::pystr::{self, PyCommonString}; use crate::vm::VirtualMachine; @@ -582,23 +582,8 @@ impl PyByteArray { } #[pymethod] - fn decode( - zelf: PyRef, - encoding: OptionalArg, - errors: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - let encoding = encoding.into_option(); - vm.decode(zelf.into_object(), encoding.clone(), errors.into_option())? - .downcast::() - .map_err(|obj| { - vm.new_type_error(format!( - "'{}' decoder returned '{}' instead of 'str'; use codecs.encode() to \ - encode arbitrary types", - encoding.as_ref().map_or("utf-8", |s| s.borrow_value()), - obj.lease_class().name, - )) - }) + fn decode(zelf: PyRef, args: DecodeArgs, vm: &VirtualMachine) -> PyResult { + bytes_decode(zelf.into_object(), args, vm) } } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 6cb056946..0443b22bd 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -6,18 +6,18 @@ use std::ops::Deref; use super::objint::PyIntRef; use super::objiter; use super::objsequence::SequenceIndex; -use super::objstr::{PyString, PyStringRef}; +use super::objstr::PyStringRef; use super::objtype::PyClassRef; use crate::bytesinner::{ - ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, - ByteInnerTranslateOptions, PyBytesInner, + bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, + ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, }; use crate::function::{OptionalArg, OptionalOption}; use crate::pyobject::{ BorrowValue, Either, IntoPyObject, PyArithmaticValue::{self, *}, PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, - TryFromObject, TypeProtocol, + TryFromObject, }; use crate::pystr::{self, PyCommonString}; use crate::vm::VirtualMachine; @@ -467,24 +467,9 @@ impl PyBytes { /// For a list of possible encodings, /// see https://docs.python.org/3/library/codecs.html#standard-encodings /// currently, only 'utf-8' and 'ascii' emplemented - #[pymethod(name = "decode")] - fn decode( - zelf: PyRef, - encoding: OptionalArg, - errors: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - let encoding = encoding.into_option(); - vm.decode(zelf.into_object(), encoding.clone(), errors.into_option())? - .downcast::() - .map_err(|obj| { - vm.new_type_error(format!( - "'{}' decoder returned '{}' instead of 'str'; use codecs.encode() to \ - encode arbitrary types", - encoding.as_ref().map_or("utf-8", |s| s.borrow_value()), - obj.lease_class().name, - )) - }) + #[pymethod] + fn decode(zelf: PyRef, args: DecodeArgs, vm: &VirtualMachine) -> PyResult { + bytes_decode(zelf.into_object(), args, vm) } }