From 717784d38ae64ce356f34cd8f313780db8a5fb14 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 1 Sep 2020 12:28:45 -0500 Subject: [PATCH 1/3] Fix test_calendar tests --- Cargo.lock | 1 + Cargo.toml | 1 + Lib/_codecs.py | 4 ++-- Lib/test/test_calendar.py | 7 ------- src/main.rs | 13 +++++++++++++ 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b4507ae3b..21bd128a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1585,6 +1585,7 @@ dependencies = [ "env_logger", "flame", "flamescope", + "libc", "log", "num-traits", "rustpython-compiler", diff --git a/Cargo.toml b/Cargo.toml index 2eaf31095..38f080585 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ pylib = { package = "rustpython-pylib", path = "vm/pylib-crate", version = "0.1. dirs = { package = "dirs-next", version = "1.0" } num-traits = "0.2.8" cfg-if = "0.1" +libc = "0.2" flame = { version = "0.2", optional = true } flamescope = { version = "0.1", optional = true } diff --git a/Lib/_codecs.py b/Lib/_codecs.py index 2c86f4932..eca7a38c2 100644 --- a/Lib/_codecs.py +++ b/Lib/_codecs.py @@ -1006,8 +1006,8 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru # have any nested variables. def STORECHAR(CH, byteorder): - hi = chr(((CH) >> 8) & 0xff) - lo = chr((CH) & 0xff) + hi = (CH >> 8) & 0xff + lo = CH & 0xff if byteorder == 'little': return [lo, hi] else: diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index aa0dd6d87..6241d114d 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -817,18 +817,14 @@ class CommandLineTestCase(unittest.TestCase): self.assertIn(b'January', stdout) self.assertIn(b'Mo Tu We Th Fr Sa Su', stdout) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_output_year(self): stdout = self.run_ok('2004') self.assertEqual(stdout, conv(result_2004_text)) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_output_month(self): stdout = self.run_ok('2004', '1') self.assertEqual(stdout, conv(result_2004_01_text)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_option_encoding(self): self.assertFailure('-e') self.assertFailure('--encoding') @@ -860,7 +856,6 @@ class CommandLineTestCase(unittest.TestCase): stdout = self.run_ok('--width', '3', '2004') self.assertIn(b'Mon Tue Wed Thu Fri Sat Sun', stdout) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_option_lines(self): self.assertFailure('-l') self.assertFailure('--lines') @@ -875,7 +870,6 @@ class CommandLineTestCase(unittest.TestCase): stdout = self.run_ok('--spacing', '8', '2004') self.assertIn(b'Su Mo', stdout) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_option_months(self): self.assertFailure('-m') self.assertFailure('--month') @@ -883,7 +877,6 @@ class CommandLineTestCase(unittest.TestCase): stdout = self.run_ok('--months', '1', '2004') self.assertIn(conv('\nMo Tu We Th Fr Sa Su\n'), stdout) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_option_type(self): self.assertFailure('-t') self.assertFailure('--type') diff --git a/src/main.rs b/src/main.rs index 9c4e075be..dda81e9e2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -36,6 +36,19 @@ fn main() { settings.initialization_parameter = InitParameter::InitializeInternal; } + // don't translate newlines (\r\n <=> \n) + #[cfg(windows)] + { + extern "C" { + fn _setmode(fd: i32, flags: i32) -> i32; + } + unsafe { + _setmode(0, libc::O_BINARY); + _setmode(1, libc::O_BINARY); + _setmode(2, libc::O_BINARY); + } + } + let vm = VirtualMachine::new(settings); let res = run_rustpython(&vm, &matches); From c5116b70dd41fd8a48f91046cc4e4dcd5bce08fe Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 1 Sep 2020 21:55:36 -0500 Subject: [PATCH 2/3] quickfix for conv() in test_calendar --- Lib/test/test_calendar.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 6241d114d..fd39c4093 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -787,7 +787,9 @@ class LeapdaysTestCase(unittest.TestCase): def conv(s): - return s.replace('\n', os.linesep).encode() + # XXX RUSTPYTHON TODO: TextIOWrapper newline translation + return s.encode() + # return s.replace('\n', os.linesep).encode() class CommandLineTestCase(unittest.TestCase): def run_ok(self, *args): From 55946604593f222969392082037a49cef9facca7 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 1 Sep 2020 22:08:46 -0500 Subject: [PATCH 3/3] Fix bytes decode/encoding tests --- Lib/_codecs.py | 13 ++++++------- Lib/test/test_bytes.py | 4 ---- vm/src/bytesinner.rs | 26 ++++++++++++++++++++++++++ vm/src/obj/objbytearray.rs | 27 ++++++--------------------- vm/src/obj/objbytes.rs | 29 +++++++---------------------- 5 files changed, 45 insertions(+), 54 deletions(-) diff --git a/Lib/_codecs.py b/Lib/_codecs.py index eca7a38c2..160e51fb8 100644 --- a/Lib/_codecs.py +++ b/Lib/_codecs.py @@ -915,7 +915,7 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru p = [] if byteorder == 'native': if (size >= 2): - bom = (ord(s[ihi]) << 8) | ord(s[ilo]) + bom = (s[ihi] << 8) | s[ilo] #ifdef BYTEORDER_IS_LITTLE_ENDIAN if sys.byteorder == 'little': if (bom == 0xFEFF): @@ -962,11 +962,11 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru # /* The remaining input chars are ignored if the callback ## chooses to skip the input */ - ch = (ord(s[q+ihi]) << 8) | ord(s[q+ilo]) + ch = (s[q+ihi] << 8) | s[q+ilo] q += 2 if (ch < 0xD800 or ch > 0xDFFF): - p += chr(ch) + p.append(chr(ch)) continue #/* UTF-16 code pair: */ @@ -977,15 +977,14 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True) if (0xD800 <= ch and ch <= 0xDBFF): - ch2 = (ord(s[q+ihi]) << 8) | ord(s[q+ilo]) + ch2 = (s[q+ihi] << 8) | s[q+ilo] q += 2 if (0xDC00 <= ch2 and ch2 <= 0xDFFF): #ifndef Py_UNICODE_WIDE if sys.maxunicode < 65536: - p += chr(ch) - p += chr(ch2) + p += [chr(ch), chr(ch2)] else: - p += chr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000) + p.append(chr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000)) #endif continue diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index c1fbb225d..cbb4bfd4d 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -300,8 +300,6 @@ class BaseBytesTest: for step in indices[1:]: self.assertEqual(b[start:stop:step], self.type2test(L[start:stop:step])) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encoding(self): sample = "Hello world\n\u1234\u5678\u9abc" for enc in ("utf-8", "utf-16"): @@ -311,8 +309,6 @@ class BaseBytesTest: b = self.type2test(sample, "latin-1", "ignore") self.assertEqual(b, self.type2test(sample[:-3], "utf-8")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_decode(self): sample = "Hello world\n\u1234\u5678\u9abc" for enc in ("utf-8", "utf-16"): diff --git a/vm/src/bytesinner.rs b/vm/src/bytesinner.rs index dc42c900b..f9650fa94 100644 --- a/vm/src/bytesinner.rs +++ b/vm/src/bytesinner.rs @@ -1290,3 +1290,29 @@ impl<'s> PyCommonString<'s, u8> for [u8] { splited } } + +#[derive(FromArgs)] +pub struct DecodeArgs { + #[pyarg(positional_or_keyword, default = "None")] + encoding: Option, + #[pyarg(positional_or_keyword, default = "None")] + errors: Option, +} + +pub fn bytes_decode( + zelf: PyObjectRef, + args: DecodeArgs, + vm: &VirtualMachine, +) -> PyResult { + let DecodeArgs { encoding, errors } = args; + vm.decode(zelf, encoding.clone(), errors)? + .downcast::() + .map_err(|obj| { + vm.new_type_error(format!( + "'{}' decoder returned '{}' instead of 'str'; use codecs.encode() to \ + encode arbitrary types", + encoding.as_ref().map_or("utf-8", |s| s.borrow_value()), + obj.lease_class().name, + )) + }) +} diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index 858f562ec..6fff56797 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -7,17 +7,17 @@ use std::mem::size_of; use super::objint::PyIntRef; use super::objiter; use super::objsequence::SequenceIndex; -use super::objstr::{PyString, PyStringRef}; +use super::objstr::PyStringRef; use super::objtype::PyClassRef; use crate::bytesinner::{ - ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, - ByteInnerTranslateOptions, ByteOr, PyBytesInner, + bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, + ByteInnerSplitOptions, ByteInnerTranslateOptions, ByteOr, DecodeArgs, PyBytesInner, }; use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; use crate::function::{OptionalArg, OptionalOption}; use crate::pyobject::{ BorrowValue, Either, PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef, - PyResult, PyValue, TryFromObject, TypeProtocol, + PyResult, PyValue, TryFromObject, }; use crate::pystr::{self, PyCommonString}; use crate::vm::VirtualMachine; @@ -582,23 +582,8 @@ impl PyByteArray { } #[pymethod] - fn decode( - zelf: PyRef, - encoding: OptionalArg, - errors: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - let encoding = encoding.into_option(); - vm.decode(zelf.into_object(), encoding.clone(), errors.into_option())? - .downcast::() - .map_err(|obj| { - vm.new_type_error(format!( - "'{}' decoder returned '{}' instead of 'str'; use codecs.encode() to \ - encode arbitrary types", - encoding.as_ref().map_or("utf-8", |s| s.borrow_value()), - obj.lease_class().name, - )) - }) + fn decode(zelf: PyRef, args: DecodeArgs, vm: &VirtualMachine) -> PyResult { + bytes_decode(zelf.into_object(), args, vm) } } diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index 6cb056946..0443b22bd 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -6,18 +6,18 @@ use std::ops::Deref; use super::objint::PyIntRef; use super::objiter; use super::objsequence::SequenceIndex; -use super::objstr::{PyString, PyStringRef}; +use super::objstr::PyStringRef; use super::objtype::PyClassRef; use crate::bytesinner::{ - ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, - ByteInnerTranslateOptions, PyBytesInner, + bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, + ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, }; use crate::function::{OptionalArg, OptionalOption}; use crate::pyobject::{ BorrowValue, Either, IntoPyObject, PyArithmaticValue::{self, *}, PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, - TryFromObject, TypeProtocol, + TryFromObject, }; use crate::pystr::{self, PyCommonString}; use crate::vm::VirtualMachine; @@ -467,24 +467,9 @@ impl PyBytes { /// For a list of possible encodings, /// see https://docs.python.org/3/library/codecs.html#standard-encodings /// currently, only 'utf-8' and 'ascii' emplemented - #[pymethod(name = "decode")] - fn decode( - zelf: PyRef, - encoding: OptionalArg, - errors: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - let encoding = encoding.into_option(); - vm.decode(zelf.into_object(), encoding.clone(), errors.into_option())? - .downcast::() - .map_err(|obj| { - vm.new_type_error(format!( - "'{}' decoder returned '{}' instead of 'str'; use codecs.encode() to \ - encode arbitrary types", - encoding.as_ref().map_or("utf-8", |s| s.borrow_value()), - obj.lease_class().name, - )) - }) + #[pymethod] + fn decode(zelf: PyRef, args: DecodeArgs, vm: &VirtualMachine) -> PyResult { + bytes_decode(zelf.into_object(), args, vm) } }