Merge pull request #2177 from RustPython/coolreader18/windows-binaryio

Fix test_calendar tests

Cargo.lock

@@ -1585,6 +1585,7 @@ dependencies = [
  "env_logger",
  "flame",
  "flamescope",
+ "libc",
  "log",
  "num-traits",
  "rustpython-compiler",

Cargo.toml

@@ -35,6 +35,7 @@ pylib = { package = "rustpython-pylib", path = "vm/pylib-crate", version = "0.1.
 dirs = { package = "dirs-next", version = "1.0" }
 num-traits = "0.2.8"
 cfg-if = "0.1"
+libc = "0.2"
 
 flame = { version = "0.2", optional = true }
 flamescope = { version = "0.1", optional = true }

Lib/_codecs.py

@@ -915,7 +915,7 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru
     p = []
     if byteorder == 'native':
         if (size >= 2):
-            bom = (ord(s[ihi]) << 8) | ord(s[ilo])
+            bom = (s[ihi] << 8) | s[ilo]
 #ifdef BYTEORDER_IS_LITTLE_ENDIAN
             if sys.byteorder == 'little':
                 if (bom == 0xFEFF):
@@ -962,11 +962,11 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru
 #           /* The remaining input chars are ignored if the callback
 ##             chooses to skip the input */
     
-        ch = (ord(s[q+ihi]) << 8) | ord(s[q+ilo])
+        ch = (s[q+ihi] << 8) | s[q+ilo]
         q += 2
     
         if (ch < 0xD800 or ch > 0xDFFF):
-            p += chr(ch)
+            p.append(chr(ch))
             continue
     
         #/* UTF-16 code pair: */
@@ -977,15 +977,14 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru
             unicode_call_errorhandler(errors, 'utf-16', errmsg, s, startinpos, endinpos, True)
 
         if (0xD800 <= ch and ch <= 0xDBFF):
-            ch2 = (ord(s[q+ihi]) << 8) | ord(s[q+ilo])
+            ch2 = (s[q+ihi] << 8) | s[q+ilo]
             q += 2
             if (0xDC00 <= ch2 and ch2 <= 0xDFFF):
     #ifndef Py_UNICODE_WIDE
                 if sys.maxunicode < 65536:
-                    p += chr(ch)
-                    p += chr(ch2)
+                    p += [chr(ch), chr(ch2)]
                 else:
-                    p += chr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000)
+                    p.append(chr((((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000))
     #endif
                 continue
 
@@ -1006,8 +1005,8 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=Tru
 # have any nested variables.
 
 def STORECHAR(CH, byteorder):
-    hi = chr(((CH) >> 8) & 0xff)
-    lo = chr((CH) & 0xff)
+    hi = (CH >> 8) & 0xff
+    lo = CH & 0xff
     if byteorder == 'little':
         return [lo, hi]
     else:

Lib/test/test_bytes.py

@@ -300,8 +300,6 @@ class BaseBytesTest:
                 for step in indices[1:]:
                     self.assertEqual(b[start:stop:step], self.type2test(L[start:stop:step]))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_encoding(self):
         sample = "Hello world\n\u1234\u5678\u9abc"
         for enc in ("utf-8", "utf-16"):
@@ -311,8 +309,6 @@ class BaseBytesTest:
         b = self.type2test(sample, "latin-1", "ignore")
         self.assertEqual(b, self.type2test(sample[:-3], "utf-8"))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_decode(self):
         sample = "Hello world\n\u1234\u5678\u9abc"
         for enc in ("utf-8", "utf-16"):

Lib/test/test_calendar.py

@@ -787,7 +787,9 @@ class LeapdaysTestCase(unittest.TestCase):
 
 
 def conv(s):
-    return s.replace('\n', os.linesep).encode()
+    # XXX RUSTPYTHON TODO: TextIOWrapper newline translation
+    return s.encode()
+    # return s.replace('\n', os.linesep).encode()
 
 class CommandLineTestCase(unittest.TestCase):
     def run_ok(self, *args):
@@ -817,18 +819,14 @@ class CommandLineTestCase(unittest.TestCase):
         self.assertIn(b'January', stdout)
         self.assertIn(b'Mo Tu We Th Fr Sa Su', stdout)
 
-    @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows')
     def test_output_year(self):
         stdout = self.run_ok('2004')
         self.assertEqual(stdout, conv(result_2004_text))
 
-    @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows')
     def test_output_month(self):
         stdout = self.run_ok('2004', '1')
         self.assertEqual(stdout, conv(result_2004_01_text))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_option_encoding(self):
         self.assertFailure('-e')
         self.assertFailure('--encoding')
@@ -860,7 +858,6 @@ class CommandLineTestCase(unittest.TestCase):
         stdout = self.run_ok('--width', '3', '2004')
         self.assertIn(b'Mon Tue Wed Thu Fri Sat Sun', stdout)
 
-    @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows')
     def test_option_lines(self):
         self.assertFailure('-l')
         self.assertFailure('--lines')
@@ -875,7 +872,6 @@ class CommandLineTestCase(unittest.TestCase):
         stdout = self.run_ok('--spacing', '8', '2004')
         self.assertIn(b'Su        Mo', stdout)
 
-    @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows')
     def test_option_months(self):
         self.assertFailure('-m')
         self.assertFailure('--month')
@@ -883,7 +879,6 @@ class CommandLineTestCase(unittest.TestCase):
         stdout = self.run_ok('--months', '1', '2004')
         self.assertIn(conv('\nMo Tu We Th Fr Sa Su\n'), stdout)
 
-    @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows')
     def test_option_type(self):
         self.assertFailure('-t')
         self.assertFailure('--type')

src/main.rs

@@ -36,6 +36,19 @@ fn main() {
         settings.initialization_parameter = InitParameter::InitializeInternal;
     }
 
+    // don't translate newlines (\r\n <=> \n)
+    #[cfg(windows)]
+    {
+        extern "C" {
+            fn _setmode(fd: i32, flags: i32) -> i32;
+        }
+        unsafe {
+            _setmode(0, libc::O_BINARY);
+            _setmode(1, libc::O_BINARY);
+            _setmode(2, libc::O_BINARY);
+        }
+    }
+
     let vm = VirtualMachine::new(settings);
 
     let res = run_rustpython(&vm, &matches);

vm/src/bytesinner.rs

@@ -1290,3 +1290,29 @@ impl<'s> PyCommonString<'s, u8> for [u8] {
         splited
     }
 }
+
+#[derive(FromArgs)]
+pub struct DecodeArgs {
+    #[pyarg(positional_or_keyword, default = "None")]
+    encoding: Option<PyStringRef>,
+    #[pyarg(positional_or_keyword, default = "None")]
+    errors: Option<PyStringRef>,
+}
+
+pub fn bytes_decode(
+    zelf: PyObjectRef,
+    args: DecodeArgs,
+    vm: &VirtualMachine,
+) -> PyResult<PyStringRef> {
+    let DecodeArgs { encoding, errors } = args;
+    vm.decode(zelf, encoding.clone(), errors)?
+        .downcast::<PyString>()
+        .map_err(|obj| {
+            vm.new_type_error(format!(
+                "'{}' decoder returned '{}' instead of 'str'; use codecs.encode() to \
+                     encode arbitrary types",
+                encoding.as_ref().map_or("utf-8", |s| s.borrow_value()),
+                obj.lease_class().name,
+            ))
+        })
+}

vm/src/obj/objbytearray.rs

@@ -7,17 +7,17 @@ use std::mem::size_of;
 use super::objint::PyIntRef;
 use super::objiter;
 use super::objsequence::SequenceIndex;
-use super::objstr::{PyString, PyStringRef};
+use super::objstr::PyStringRef;
 use super::objtype::PyClassRef;
 use crate::bytesinner::{
-    ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions,
-    ByteInnerTranslateOptions, ByteOr, PyBytesInner,
+    bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions,
+    ByteInnerSplitOptions, ByteInnerTranslateOptions, ByteOr, DecodeArgs, PyBytesInner,
 };
 use crate::common::cell::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard};
 use crate::function::{OptionalArg, OptionalOption};
 use crate::pyobject::{
     BorrowValue, Either, PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef,
-    PyResult, PyValue, TryFromObject, TypeProtocol,
+    PyResult, PyValue, TryFromObject,
 };
 use crate::pystr::{self, PyCommonString};
 use crate::vm::VirtualMachine;
@@ -582,23 +582,8 @@ impl PyByteArray {
     }
 
     #[pymethod]
-    fn decode(
-        zelf: PyRef<Self>,
-        encoding: OptionalArg<PyStringRef>,
-        errors: OptionalArg<PyStringRef>,
-        vm: &VirtualMachine,
-    ) -> PyResult<PyStringRef> {
-        let encoding = encoding.into_option();
-        vm.decode(zelf.into_object(), encoding.clone(), errors.into_option())?
-            .downcast::<PyString>()
-            .map_err(|obj| {
-                vm.new_type_error(format!(
-                    "'{}' decoder returned '{}' instead of 'str'; use codecs.encode() to \
-                     encode arbitrary types",
-                    encoding.as_ref().map_or("utf-8", |s| s.borrow_value()),
-                    obj.lease_class().name,
-                ))
-            })
+    fn decode(zelf: PyRef<Self>, args: DecodeArgs, vm: &VirtualMachine) -> PyResult<PyStringRef> {
+        bytes_decode(zelf.into_object(), args, vm)
     }
 }
 

vm/src/obj/objbytes.rs

@@ -6,18 +6,18 @@ use std::ops::Deref;
 use super::objint::PyIntRef;
 use super::objiter;
 use super::objsequence::SequenceIndex;
-use super::objstr::{PyString, PyStringRef};
+use super::objstr::PyStringRef;
 use super::objtype::PyClassRef;
 use crate::bytesinner::{
-    ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions,
-    ByteInnerTranslateOptions, PyBytesInner,
+    bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions,
+    ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner,
 };
 use crate::function::{OptionalArg, OptionalOption};
 use crate::pyobject::{
     BorrowValue, Either, IntoPyObject,
     PyArithmaticValue::{self, *},
     PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue,
-    TryFromObject, TypeProtocol,
+    TryFromObject,
 };
 use crate::pystr::{self, PyCommonString};
 use crate::vm::VirtualMachine;
@@ -467,24 +467,9 @@ impl PyBytes {
     /// For a list of possible encodings,
     /// see https://docs.python.org/3/library/codecs.html#standard-encodings
     /// currently, only 'utf-8' and 'ascii' emplemented
-    #[pymethod(name = "decode")]
-    fn decode(
-        zelf: PyRef<Self>,
-        encoding: OptionalArg<PyStringRef>,
-        errors: OptionalArg<PyStringRef>,
-        vm: &VirtualMachine,
-    ) -> PyResult<PyStringRef> {
-        let encoding = encoding.into_option();
-        vm.decode(zelf.into_object(), encoding.clone(), errors.into_option())?
-            .downcast::<PyString>()
-            .map_err(|obj| {
-                vm.new_type_error(format!(
-                    "'{}' decoder returned '{}' instead of 'str'; use codecs.encode() to \
-                     encode arbitrary types",
-                    encoding.as_ref().map_or("utf-8", |s| s.borrow_value()),
-                    obj.lease_class().name,
-                ))
-            })
+    #[pymethod]
+    fn decode(zelf: PyRef<Self>, args: DecodeArgs, vm: &VirtualMachine) -> PyResult<PyStringRef> {
+        bytes_decode(zelf.into_object(), args, vm)
     }
 }