diff --git a/.travis.yml b/.travis.yml index aff28cd6a..be1a47aa2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -111,6 +111,8 @@ matrix: - target script: - tests/.travis-runner.sh + # Only do code coverage on master via a cron job. + if: branch = master AND type = cron env: - TRAVIS_RUST_VERSION=nightly - REGULAR_TEST=false diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 4545f7f58..c453cff2b 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -52,6 +52,10 @@ assert ('d', 3) == next(it) with assertRaises(StopIteration): next(it) +with assertRaises(KeyError) as cm: + del x[10] +assert cm.exception.args[0] == 10 + # Iterating a dictionary is just its keys: assert ['a', 'b', 'd'] == list(x) @@ -95,9 +99,6 @@ x[7] = 7 x[2] = 2 x[(5, 6)] = 5 -with assertRaises(KeyError): - x["not here"] - with assertRaises(TypeError): x[[]] # Unhashable type. @@ -167,19 +168,42 @@ assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} y.update(y) assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} # hasn't changed +# KeyError has object that used as key as an .args[0] +with assertRaises(KeyError) as cm: + x['not here'] +assert cm.exception.args[0] == "not here" +with assertRaises(KeyError) as cm: + x.pop('not here') +assert cm.exception.args[0] == "not here" + +with assertRaises(KeyError) as cm: + x[10] +assert cm.exception.args[0] == 10 +with assertRaises(KeyError) as cm: + x.pop(10) +assert cm.exception.args[0] == 10 + +class MyClass: pass +obj = MyClass() + +with assertRaises(KeyError) as cm: + x[obj] +assert cm.exception.args[0] == obj +with assertRaises(KeyError) as cm: + x.pop(obj) +assert cm.exception.args[0] == obj + x = {1: 'a', '1': None} assert x.pop(1) == 'a' assert x.pop('1') is None assert x == {} -with assertRaises(KeyError): - x.pop("not here") - x = {1: 'a'} assert (1, 'a') == x.popitem() -with assertRaises(KeyError): - x.popitem() assert x == {} +with assertRaises(KeyError) as cm: + x.popitem() +assert cm.exception.args == ('popitem(): dictionary is empty',) x = {'a': 4} assert 4 == x.setdefault('a', 0) diff --git a/tests/snippets/mappingproxy.py b/tests/snippets/mappingproxy.py index 624bf6d5f..254f2a925 100644 --- a/tests/snippets/mappingproxy.py +++ b/tests/snippets/mappingproxy.py @@ -9,8 +9,10 @@ class A(dict): assert A.__dict__['a'] == A.a -with assertRaises(KeyError): - A.__dict__['not_here'] +with assertRaises(KeyError) as cm: + A.__dict__['not here'] + +assert cm.exception.args[0] == "not here" assert 'b' in A.__dict__ assert 'c' not in A.__dict__ diff --git a/tests/snippets/memoryview.py b/tests/snippets/memoryview.py new file mode 100644 index 000000000..ebf27a9a6 --- /dev/null +++ b/tests/snippets/memoryview.py @@ -0,0 +1,6 @@ + +obj = b"abcde" +a = memoryview(obj) +assert a.obj == obj + +assert a[2:3] == b"c" diff --git a/tests/snippets/stdlib_io.py b/tests/snippets/stdlib_io.py index e6da760d2..08786c219 100644 --- a/tests/snippets/stdlib_io.py +++ b/tests/snippets/stdlib_io.py @@ -1,4 +1,5 @@ from io import BufferedReader, FileIO +import os fi = FileIO('README.md') bb = BufferedReader(fi) @@ -14,3 +15,9 @@ with FileIO('README.md') as fio: assert len(result) <= 8*1024 assert len(result) >= 0 assert isinstance(result, bytes) + +fd = os.open('README.md', os.O_RDONLY) + +with FileIO(fd) as fio: + res2 = fio.read() + assert res == res2 diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index ef9b9111d..d2e39c1ce 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -244,7 +244,7 @@ with TestWithTempDir() as tmpdir: with TestWithTempCurrentDir(): os.chdir(tmpdir) - assert os.getcwd() == tmpdir + assert os.getcwd() == os.path.realpath(tmpdir) os.path.exists(FILE_NAME) # supports diff --git a/tests/snippets/sysmod.py b/tests/snippets/sysmod.py index 23dd70f9b..adabb57d2 100644 --- a/tests/snippets/sysmod.py +++ b/tests/snippets/sysmod.py @@ -14,6 +14,8 @@ assert isinstance(sys.implementation.cache_tag, str) assert sys.getfilesystemencoding() == 'utf-8' assert sys.getfilesystemencodeerrors().startswith('surrogate') +assert sys.byteorder == "little" or sys.byteorder == "big" + assert isinstance(sys.flags, tuple) assert type(sys.flags).__name__ == "flags" assert type(sys.flags.optimize) is int diff --git a/tests/snippets/test_struct.py b/tests/snippets/test_struct.py index c13e713a1..8f535f63c 100644 --- a/tests/snippets/test_struct.py +++ b/tests/snippets/test_struct.py @@ -8,3 +8,17 @@ v1, v2 = struct.unpack('IH', data) assert v1 == 14 assert v2 == 12 +data = struct.pack('IH', 14, 12) +assert data == bytes([0, 0, 0, 14, 0, 12]) + +v1, v2 = struct.unpack('>IH', data) +assert v1 == 14 +assert v2 == 12 + diff --git a/tests/snippets/testutils.py b/tests/snippets/testutils.py index ea4dd9138..21662c461 100644 --- a/tests/snippets/testutils.py +++ b/tests/snippets/testutils.py @@ -23,6 +23,7 @@ def assert_raises(exc_type, expr, msg=None): class assertRaises: def __init__(self, expected): self.expected = expected + self.exception = None def __enter__(self): return self @@ -33,6 +34,8 @@ class assertRaises: assert False, failmsg if not issubclass(exc_type, self.expected): return False + + self.exception = exc_val return True diff --git a/tests/snippets/tuple.py b/tests/snippets/tuple.py index b7393fe00..86f12fe94 100644 --- a/tests/snippets/tuple.py +++ b/tests/snippets/tuple.py @@ -9,7 +9,7 @@ assert y[0] == 1 assert x + y == (1, 2, 1) assert x * 3 == (1, 2, 1, 2, 1, 2) -# assert 3 * x == (1, 2, 1, 2, 1, 2) +assert 3 * x == (1, 2, 1, 2, 1, 2) assert x * 0 == () assert x * -1 == () # integers less than zero treated as 0 diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index ca20f2012..e52e83b8c 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -129,8 +129,7 @@ impl Dict { if self.delete_if_exists(vm, key)? { Ok(()) } else { - let key_repr = vm.to_pystr(key)?; - Err(vm.new_key_error(format!("Key not found: {}", key_repr))) + Err(vm.new_key_error(key.clone())) } } @@ -247,8 +246,7 @@ impl Dict { self.unchecked_delete(index); Ok(value) } else { - let key_repr = vm.to_pystr(key)?; - Err(vm.new_key_error(format!("Key not found: {}", key_repr))) + Err(vm.new_key_error(key.clone())) } } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index 8d2e0fdff..7b708611e 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -210,7 +210,7 @@ impl PyDictRef { let method = method_or_err?; return vm.invoke(method, vec![key]); } - Err(vm.new_key_error(format!("Key not found: {}", vm.to_pystr(&key)?))) + Err(vm.new_key_error(key.clone())) } fn get( @@ -266,7 +266,8 @@ impl PyDictRef { if let Some((key, value)) = entries.pop_front() { Ok(vm.ctx.new_tuple(vec![key, value])) } else { - Err(vm.new_key_error("popitem(): dictionary is empty".to_string())) + let err_msg = vm.new_str("popitem(): dictionary is empty".to_string()); + Err(vm.new_key_error(err_msg)) } } diff --git a/vm/src/obj/objmappingproxy.rs b/vm/src/obj/objmappingproxy.rs index c46591105..243a8c89c 100644 --- a/vm/src/obj/objmappingproxy.rs +++ b/vm/src/obj/objmappingproxy.rs @@ -28,7 +28,7 @@ impl PyMappingProxy { if let Some(value) = objtype::class_get_attr(&self.class, key.as_str()) { return Ok(value); } - Err(vm.new_key_error(format!("Key not found: {}", key))) + Err(vm.new_key_error(key.into_object())) } #[pymethod(name = "__contains__")] diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index 8c2af26f0..7e1628d13 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -33,6 +33,11 @@ impl PyMemoryView { fn obj(&self, __vm: &VirtualMachine) -> PyObjectRef { self.obj_ref.clone() } + + #[pymethod(name = "__getitem__")] + fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm.call_method(&self.obj_ref, "__getitem__", vec![needle]) + } } impl PyValue for PyMemoryView { diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index b7f0276ce..ede6029f3 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -269,7 +269,8 @@ impl PySetInner { if let Some((key, _)) = self.content.pop_front() { Ok(key) } else { - Err(vm.new_key_error("pop from an empty set".to_string())) + let err_msg = vm.new_str("pop from an empty set".to_string()); + Err(vm.new_key_error(err_msg)) } } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 1b6870e14..fafd23a65 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -1335,7 +1335,7 @@ fn perform_format( let result = match arguments.get_optional_kwarg(&keyword) { Some(argument) => call_object_format(vm, argument.clone(), &format_spec)?, None => { - return Err(vm.new_key_error(format!("'{}'", keyword))); + return Err(vm.new_key_error(vm.new_str(keyword.to_string()))); } }; get_value(&result) diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index b4f1699dd..a6e181f3b 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -168,6 +168,11 @@ impl PyTupleRef { vm.ctx.new_tuple(new_elements) } + fn rmul(self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { + let new_elements = seq_mul(&self.elements, counter); + vm.ctx.new_tuple(new_elements) + } + fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { get_item(vm, self.as_object(), &self.elements, needle.clone()) } @@ -263,6 +268,7 @@ If the argument is a tuple, the return value is the same object."; "__len__" => context.new_rustfunc(PyTupleRef::len), "__new__" => context.new_rustfunc(tuple_new), "__mul__" => context.new_rustfunc(PyTupleRef::mul), + "__rmul__" => context.new_rustfunc(PyTupleRef::rmul), "__repr__" => context.new_rustfunc(PyTupleRef::repr), "count" => context.new_rustfunc(PyTupleRef::count), "__lt__" => context.new_rustfunc(PyTupleRef::lt), diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index 184fc5003..4c0f023d9 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -77,6 +77,10 @@ fn imp_is_frozen_package(_name: PyStringRef, _vm: &VirtualMachine) -> bool { false } +fn imp_fix_co_filename(_code: PyObjectRef, _path: PyStringRef, _vm: &VirtualMachine) { + // TODO: +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; let module = py_module!(vm, "_imp", { @@ -91,6 +95,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "get_frozen_object" => ctx.new_rustfunc(imp_get_frozen_object), "init_frozen" => ctx.new_rustfunc(imp_init_frozen), "is_frozen_package" => ctx.new_rustfunc(imp_is_frozen_package), + "_fix_co_filename" => ctx.new_rustfunc(imp_fix_co_filename), }); module diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 783e9b96f..2d0b2286c 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -2,9 +2,7 @@ * I/O core tools. */ use std::cell::RefCell; -use std::fs::File; use std::io::prelude::*; -use std::io::BufReader; use std::io::Cursor; use std::io::SeekFrom; @@ -18,9 +16,12 @@ use crate::function::{OptionalArg, PyFuncArgs}; use crate::import; use crate::obj::objbytearray::PyByteArray; use crate::obj::objbytes; +use crate::obj::objbytes::PyBytes; use crate::obj::objint; use crate::obj::objstr; +use crate::obj::objtype; use crate::obj::objtype::PyClassRef; +use crate::pyobject::TypeProtocol; use crate::pyobject::{BufferProtocol, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; @@ -284,16 +285,19 @@ fn buffered_reader_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } fn compute_c_flag(mode: &str) -> u32 { - let flags = match mode { - "w" => os::FileCreationFlags::O_WRONLY | os::FileCreationFlags::O_CREAT, - "x" => { - os::FileCreationFlags::O_WRONLY - | os::FileCreationFlags::O_CREAT - | os::FileCreationFlags::O_EXCL - } - "a" => os::FileCreationFlags::O_APPEND, - "+" => os::FileCreationFlags::O_RDWR, - _ => os::FileCreationFlags::O_RDONLY, + let flags = match mode.chars().next() { + Some(mode) => match mode { + 'w' => os::FileCreationFlags::O_WRONLY | os::FileCreationFlags::O_CREAT, + 'x' => { + os::FileCreationFlags::O_WRONLY + | os::FileCreationFlags::O_CREAT + | os::FileCreationFlags::O_EXCL + } + 'a' => os::FileCreationFlags::O_APPEND, + '+' => os::FileCreationFlags::O_RDWR, + _ => os::FileCreationFlags::O_RDONLY, + }, + None => os::FileCreationFlags::O_RDONLY, }; flags.bits() } @@ -302,47 +306,43 @@ fn file_io_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, args, - required = [(file_io, None), (name, Some(vm.ctx.str_type()))], + required = [(file_io, None), (name, None)], optional = [(mode, Some(vm.ctx.str_type()))] ); - let rust_mode = mode.map_or("r".to_string(), objstr::get_value); + let file_no = if objtype::isinstance(&name, &vm.ctx.str_type()) { + let rust_mode = mode.map_or("r".to_string(), objstr::get_value); + let args = vec![ + name.clone(), + vm.ctx + .new_int(compute_c_flag(&rust_mode).to_bigint().unwrap()), + ]; + os::os_open(vm, PyFuncArgs::new(args, vec![]))? + } else if objtype::isinstance(&name, &vm.ctx.int_type()) { + name.clone() + } else { + return Err(vm.new_type_error("name parameter must be string or int".to_string())); + }; - match compute_c_flag(&rust_mode).to_bigint() { - Some(os_mode) => { - let args = vec![name.clone(), vm.ctx.new_int(os_mode)]; - let file_no = os::os_open(vm, PyFuncArgs::new(args, vec![]))?; - - vm.set_attr(file_io, "name", name.clone())?; - vm.set_attr(file_io, "fileno", file_no)?; - vm.set_attr(file_io, "closefd", vm.new_bool(false))?; - vm.set_attr(file_io, "closed", vm.new_bool(false))?; - - Ok(vm.get_none()) - } - None => Err(vm.new_type_error(format!("invalid mode {}", rust_mode))), - } + vm.set_attr(file_io, "name", name.clone())?; + vm.set_attr(file_io, "fileno", file_no)?; + vm.set_attr(file_io, "closefd", vm.new_bool(false))?; + vm.set_attr(file_io, "closed", vm.new_bool(false))?; + Ok(vm.get_none()) } fn file_io_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(file_io, None)]); - let py_name = vm.get_attribute(file_io.clone(), "name")?; - let f = match File::open(objstr::get_value(&py_name)) { - Ok(v) => Ok(v), - Err(_) => Err(vm.new_type_error("Error opening file".to_string())), - }; - let buffer = match f { - Ok(v) => Ok(BufReader::new(v)), - Err(_) => Err(vm.new_type_error("Error reading from file".to_string())), - }; + let file_no = vm.get_attribute(file_io.clone(), "fileno")?; + let raw_fd = objint::get_value(&file_no).to_i64().unwrap(); + + let mut handle = os::rust_file(raw_fd); let mut bytes = vec![]; - if let Ok(mut buff) = buffer { - match buff.read_to_end(&mut bytes) { - Ok(_) => {} - Err(_) => return Err(vm.new_value_error("Error reading from Buffer".to_string())), - } + match handle.read_to_end(&mut bytes) { + Ok(_) => {} + Err(_) => return Err(vm.new_value_error("Error reading from Buffer".to_string())), } Ok(vm.ctx.new_bytes(bytes)) @@ -385,11 +385,7 @@ fn file_io_readinto(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } fn file_io_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(file_io, None), (obj, Some(vm.ctx.bytes_type()))] - ); + arg_check!(vm, args, required = [(file_io, None), (obj, None)]); let file_no = vm.get_attribute(file_io.clone(), "fileno")?; let raw_fd = objint::get_value(&file_no).to_i64().unwrap(); @@ -399,22 +395,25 @@ fn file_io_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { //to support windows - i.e. raw file_handles let mut handle = os::rust_file(raw_fd); - match obj.payload::() { - Some(bytes) => { - let value_mut = &mut bytes.inner.borrow_mut().elements; - match handle.write(&value_mut[..]) { - Ok(len) => { - //reset raw fd on the FileIO object - let updated = os::raw_file_number(handle); - vm.set_attr(file_io, "fileno", vm.ctx.new_int(updated))?; + let bytes = match_class!(obj.clone(), + i @ PyBytes => Ok(i.get_value().to_vec()), + j @ PyByteArray => Ok(j.inner.borrow().elements.to_vec()), + obj => Err(vm.new_type_error(format!( + "a bytes-like object is required, not {}", + obj.class() + ))) + ); - //return number of bytes written - Ok(vm.ctx.new_int(len)) - } - Err(_) => Err(vm.new_value_error("Error Writing Bytes to Handle".to_string())), - } + match handle.write(&bytes?) { + Ok(len) => { + //reset raw fd on the FileIO object + let updated = os::raw_file_number(handle); + vm.set_attr(file_io, "fileno", vm.ctx.new_int(updated))?; + + //return number of bytes written + Ok(vm.ctx.new_int(len)) } - None => Err(vm.new_value_error("Expected Bytes Object".to_string())), + Err(_) => Err(vm.new_value_error("Error Writing Bytes to Handle".to_string())), } } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 24a7a6741..11f6c1676 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -605,6 +605,17 @@ fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { unimplemented!(); } +fn os_lstat(path: PyStringRef, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult { + os_stat( + path, + dir_fd, + FollowSymlinks { + follow_symlinks: false, + }, + vm, + ) +} + #[cfg(unix)] fn os_symlink( src: PyStringRef, @@ -788,6 +799,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "ScandirIter" => scandir_iter, "DirEntry" => dir_entry, "stat_result" => stat_result, + "lstat" => ctx.new_rustfunc(os_lstat), "getcwd" => ctx.new_rustfunc(os_getcwd), "chdir" => ctx.new_rustfunc(os_chdir), "fspath" => ctx.new_rustfunc(os_fspath), diff --git a/vm/src/stdlib/pwd.rs b/vm/src/stdlib/pwd.rs index 1f5092540..85cb2d3eb 100644 --- a/vm/src/stdlib/pwd.rs +++ b/vm/src/stdlib/pwd.rs @@ -48,7 +48,7 @@ fn pwd_getpwnam(name: PyStringRef, vm: &VirtualMachine) -> PyResult { Ok(Some(passwd)) => Ok(passwd), _ => { let name_repr = vm.to_repr(name.as_object())?; - let message = format!("getpwnam(): name not found: {}", name_repr); + let message = vm.new_str(format!("getpwnam(): name not found: {}", name_repr)); Err(vm.new_key_error(message)) } } @@ -58,7 +58,7 @@ fn pwd_getpwuid(uid: u32, vm: &VirtualMachine) -> PyResult { match Passwd::from_uid(uid) { Some(passwd) => Ok(passwd), _ => { - let message = format!("getpwuid(): uid not found: {}", uid); + let message = vm.new_str(format!("getpwuid(): uid not found: {}", uid)); Err(vm.new_key_error(message)) } } diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index 4a454b0c9..09f007e60 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -1,6 +1,8 @@ /* * Python struct module. * + * Docs: https://docs.python.org/3/library/struct.html + * * renamed to pystruct since struct is a rust keyword. * * Use this rust module to do byte packing: @@ -8,8 +10,9 @@ */ use std::io::{Cursor, Read, Write}; +use std::iter::Peekable; -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use byteorder::{ReadBytesExt, WriteBytesExt}; use num_bigint::BigInt; use num_traits::ToPrimitive; @@ -19,28 +22,86 @@ use crate::pyobject::{PyObjectRef, PyResult}; use crate::VirtualMachine; #[derive(Debug)] -struct FormatCode { - code: char, - repeat: i32, +struct FormatSpec { + endianness: Endianness, + codes: Vec, } -fn parse_format_string(fmt: String) -> Vec { +#[derive(Debug)] +enum Endianness { + Native, + Little, + Big, + Network, +} + +#[derive(Debug)] +struct FormatCode { + code: char, +} + +fn parse_format_string(fmt: String) -> Result { + let mut chars = fmt.chars().peekable(); + // First determine "<", ">","!" or "=" - // TODO + let endianness = parse_endiannes(&mut chars); // Now, analyze struct string furter: + let codes = parse_format_codes(&mut chars)?; + + Ok(FormatSpec { endianness, codes }) +} + +/// Parse endianness +/// See also: https://docs.python.org/3/library/struct.html?highlight=struct#byte-order-size-and-alignment +fn parse_endiannes(chars: &mut Peekable) -> Endianness +where + I: Sized, + I: Iterator, +{ + match chars.peek() { + Some('@') => { + chars.next().unwrap(); + Endianness::Native + } + Some('=') => { + chars.next().unwrap(); + Endianness::Native + } + Some('<') => { + chars.next().unwrap(); + Endianness::Little + } + Some('>') => { + chars.next().unwrap(); + Endianness::Big + } + Some('!') => { + chars.next().unwrap(); + Endianness::Network + } + _ => Endianness::Native, + } +} + +fn parse_format_codes(chars: &mut Peekable) -> Result, String> +where + I: Sized, + I: Iterator, +{ let mut codes = vec![]; - for c in fmt.chars() { + for c in chars { match c { - 'b' | 'B' | 'h' | 'H' | 'i' | 'I' | 'q' | 'Q' | 'f' | 'd' => { - codes.push(FormatCode { code: c, repeat: 1 }) + 'b' | 'B' | 'h' | 'H' | 'i' | 'I' | 'l' | 'L' | 'q' | 'Q' | 'f' | 'd' => { + codes.push(FormatCode { code: c }) } c => { - panic!("Illegal format code {:?}", c); + return Err(format!("Illegal format code {:?}", c)); } } } - codes + + Ok(codes) } fn get_int(vm: &VirtualMachine, arg: &PyObjectRef) -> PyResult { @@ -69,60 +130,112 @@ fn pack_bool(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResu } } -fn pack_i16(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { +fn pack_i16(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ let v = get_int(vm, arg)?.to_i16().unwrap(); - data.write_i16::(v).unwrap(); + data.write_i16::(v).unwrap(); Ok(()) } -fn pack_u16(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { +fn pack_u16(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ let v = get_int(vm, arg)?.to_u16().unwrap(); - data.write_u16::(v).unwrap(); + data.write_u16::(v).unwrap(); Ok(()) } -fn pack_i32(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { +fn pack_i32(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ let v = get_int(vm, arg)?.to_i32().unwrap(); - data.write_i32::(v).unwrap(); + data.write_i32::(v).unwrap(); Ok(()) } -fn pack_u32(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { +fn pack_u32(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ let v = get_int(vm, arg)?.to_u32().unwrap(); - data.write_u32::(v).unwrap(); + data.write_u32::(v).unwrap(); Ok(()) } -fn pack_i64(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { +fn pack_i64(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ let v = get_int(vm, arg)?.to_i64().unwrap(); - data.write_i64::(v).unwrap(); + data.write_i64::(v).unwrap(); Ok(()) } -fn pack_u64(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { +fn pack_u64(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ let v = get_int(vm, arg)?.to_u64().unwrap(); - data.write_u64::(v).unwrap(); + data.write_u64::(v).unwrap(); Ok(()) } -fn pack_f32(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { +fn pack_f32(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ + let v = get_float(vm, arg)? as f32; + data.write_f32::(v).unwrap(); + Ok(()) +} + +fn pack_f64(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ + let v = get_float(vm, arg)?; + data.write_f64::(v).unwrap(); + Ok(()) +} + +fn get_float(vm: &VirtualMachine, arg: &PyObjectRef) -> PyResult { if objtype::isinstance(&arg, &vm.ctx.float_type()) { - let v = objfloat::get_value(arg) as f32; - data.write_f32::(v).unwrap(); - Ok(()) + Ok(objfloat::get_value(arg)) } else { Err(vm.new_type_error("Expected float".to_string())) } } -fn pack_f64(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { - if objtype::isinstance(&arg, &vm.ctx.float_type()) { - let v = objfloat::get_value(arg) as f64; - data.write_f64::(v).unwrap(); - Ok(()) - } else { - Err(vm.new_type_error("Expected float".to_string())) +fn pack_item( + vm: &VirtualMachine, + code: &FormatCode, + arg: &PyObjectRef, + data: &mut Write, +) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ + match code.code { + 'b' => pack_i8(vm, arg, data)?, + 'B' => pack_u8(vm, arg, data)?, + '?' => pack_bool(vm, arg, data)?, + 'h' => pack_i16::(vm, arg, data)?, + 'H' => pack_u16::(vm, arg, data)?, + 'i' | 'l' => pack_i32::(vm, arg, data)?, + 'I' | 'L' => pack_u32::(vm, arg, data)?, + 'q' => pack_i64::(vm, arg, data)?, + 'Q' => pack_u64::(vm, arg, data)?, + 'f' => pack_f32::(vm, arg, data)?, + 'd' => pack_f64::(vm, arg, data)?, + c => { + panic!("Unsupported format code {:?}", c); + } } + Ok(()) } fn struct_pack(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -136,30 +249,26 @@ fn struct_pack(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if objtype::isinstance(&fmt_arg, &vm.ctx.str_type()) { let fmt_str = objstr::get_value(&fmt_arg); - let codes = parse_format_string(fmt_str); + let format_spec = parse_format_string(fmt_str).map_err(|e| vm.new_value_error(e))?;; - if codes.len() + 1 == args.args.len() { + if format_spec.codes.len() + 1 == args.args.len() { // Create data vector: let mut data = Vec::::new(); // Loop over all opcodes: - for (code, arg) in codes.iter().zip(args.args.iter().skip(1)) { + for (code, arg) in format_spec.codes.iter().zip(args.args.iter().skip(1)) { debug!("code: {:?}", code); - match code.code { - 'b' => pack_i8(vm, arg, &mut data)?, - 'B' => pack_u8(vm, arg, &mut data)?, - '?' => pack_bool(vm, arg, &mut data)?, - 'h' => pack_i16(vm, arg, &mut data)?, - 'H' => pack_u16(vm, arg, &mut data)?, - 'i' => pack_i32(vm, arg, &mut data)?, - 'I' => pack_u32(vm, arg, &mut data)?, - 'l' => pack_i32(vm, arg, &mut data)?, - 'L' => pack_u32(vm, arg, &mut data)?, - 'q' => pack_i64(vm, arg, &mut data)?, - 'Q' => pack_u64(vm, arg, &mut data)?, - 'f' => pack_f32(vm, arg, &mut data)?, - 'd' => pack_f64(vm, arg, &mut data)?, - c => { - panic!("Unsupported format code {:?}", c); + match format_spec.endianness { + Endianness::Little => { + pack_item::(vm, code, arg, &mut data)? + } + Endianness::Big => { + pack_item::(vm, code, arg, &mut data)? + } + Endianness::Network => { + pack_item::(vm, code, arg, &mut data)? + } + Endianness::Native => { + pack_item::(vm, code, arg, &mut data)? } } } @@ -168,7 +277,7 @@ fn struct_pack(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } else { Err(vm.new_type_error(format!( "Expected {} arguments (got: {})", - codes.len() + 1, + format_spec.codes.len() + 1, args.args.len() ))) } @@ -199,57 +308,81 @@ fn unpack_bool(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { } } -fn unpack_i16(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_i16::() { +fn unpack_i16(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_i16::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_int(v)), } } -fn unpack_u16(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_u16::() { +fn unpack_u16(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_u16::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_int(v)), } } -fn unpack_i32(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_i32::() { +fn unpack_i32(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_i32::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_int(v)), } } -fn unpack_u32(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_u32::() { +fn unpack_u32(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_u32::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_int(v)), } } -fn unpack_i64(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_i64::() { +fn unpack_i64(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_i64::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_int(v)), } } -fn unpack_u64(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_u64::() { +fn unpack_u64(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_u64::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_int(v)), } } -fn unpack_f32(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_f32::() { +fn unpack_f32(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_f32::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_float(f64::from(v))), } } -fn unpack_f64(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_f64::() { +fn unpack_f64(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_f64::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_float(v)), } @@ -267,41 +400,55 @@ fn struct_unpack(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let fmt_str = objstr::get_value(&fmt); - let codes = parse_format_string(fmt_str); + let format_spec = parse_format_string(fmt_str).map_err(|e| vm.new_value_error(e))?; let data = objbytes::get_value(buffer).to_vec(); let mut rdr = Cursor::new(data); let mut items = vec![]; - for code in codes { + for code in format_spec.codes { debug!("unpack code: {:?}", code); - match code.code { - 'b' => items.push(unpack_i8(vm, &mut rdr)?), - 'B' => items.push(unpack_u8(vm, &mut rdr)?), - '?' => items.push(unpack_bool(vm, &mut rdr)?), - 'h' => items.push(unpack_i16(vm, &mut rdr)?), - 'H' => items.push(unpack_u16(vm, &mut rdr)?), - 'i' => items.push(unpack_i32(vm, &mut rdr)?), - 'I' => items.push(unpack_u32(vm, &mut rdr)?), - 'l' => items.push(unpack_i32(vm, &mut rdr)?), - 'L' => items.push(unpack_u32(vm, &mut rdr)?), - 'q' => items.push(unpack_i64(vm, &mut rdr)?), - 'Q' => items.push(unpack_u64(vm, &mut rdr)?), - 'f' => items.push(unpack_f32(vm, &mut rdr)?), - 'd' => items.push(unpack_f64(vm, &mut rdr)?), - c => { - panic!("Unsupported format code {:?}", c); - } - } + let item = match format_spec.endianness { + Endianness::Little => unpack_code::(vm, &code, &mut rdr)?, + Endianness::Big => unpack_code::(vm, &code, &mut rdr)?, + Endianness::Network => unpack_code::(vm, &code, &mut rdr)?, + Endianness::Native => unpack_code::(vm, &code, &mut rdr)?, + }; + items.push(item); } Ok(vm.ctx.new_tuple(items)) } +fn unpack_code(vm: &VirtualMachine, code: &FormatCode, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match code.code { + 'b' => unpack_i8(vm, rdr), + 'B' => unpack_u8(vm, rdr), + '?' => unpack_bool(vm, rdr), + 'h' => unpack_i16::(vm, rdr), + 'H' => unpack_u16::(vm, rdr), + 'i' | 'l' => unpack_i32::(vm, rdr), + 'I' | 'L' => unpack_u32::(vm, rdr), + 'q' => unpack_i64::(vm, rdr), + 'Q' => unpack_u64::(vm, rdr), + 'f' => unpack_f32::(vm, rdr), + 'd' => unpack_f64::(vm, rdr), + c => { + panic!("Unsupported format code {:?}", c); + } + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; + let struct_error = ctx.new_class("struct.error", ctx.object()); + py_module!(vm, "struct", { "pack" => ctx.new_rustfunc(struct_pack), - "unpack" => ctx.new_rustfunc(struct_unpack) + "unpack" => ctx.new_rustfunc(struct_unpack), + "error" => struct_error, }) } diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index b81b8db52..c20f7b167 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -151,6 +151,15 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectR "unknown".to_string() }; + // https://doc.rust-lang.org/reference/conditional-compilation.html#target_endian + let bytorder = if cfg!(target_endian = "little") { + "little".to_string() + } else if cfg!(target_endian = "big") { + "big".to_string() + } else { + "unknown".to_string() + }; + let sys_doc = "This module provides access to some objects used or maintained by the interpreter and to functions that interact strongly with the interpreter. @@ -229,6 +238,7 @@ settrace() -- set the global debug tracing function "__name__" => ctx.new_str(String::from("sys")), "argv" => argv(ctx), "builtin_module_names" => ctx.new_tuple(module_names.iter().map(|v| v.into_pyobject(vm).unwrap()).collect()), + "byteorder" => ctx.new_str(bytorder), "flags" => flags, "getrefcount" => ctx.new_rustfunc(sys_getrefcount), "getsizeof" => ctx.new_rustfunc(sys_getsizeof), diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 8656c7dd4..a1c7e5868 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -167,20 +167,20 @@ impl VirtualMachine { self.ctx.new_bool(b) } - pub fn new_empty_exception(&self, exc_type: PyClassRef) -> PyResult { + fn new_exception_obj(&self, exc_type: PyClassRef, args: Vec) -> PyResult { + // TODO: add repr of args into logging? info!("New exception created: {}", exc_type.name); - let args = PyFuncArgs::default(); self.invoke(exc_type.into_object(), args) } + pub fn new_empty_exception(&self, exc_type: PyClassRef) -> PyResult { + self.new_exception_obj(exc_type, vec![]) + } + /// Create Python instance of `exc_type` with message as first element of `args` tuple pub fn new_exception(&self, exc_type: PyClassRef, msg: String) -> PyObjectRef { - // TODO: exc_type may be user-defined exception, so we should return PyResult - // TODO: maybe there is a clearer way to create an instance: - info!("New exception created: {}('{}')", exc_type.name, msg); - let pymsg = self.new_str(msg); - let args: Vec = vec![pymsg]; - self.invoke(exc_type.into_object(), args).unwrap() + let pystr_msg = self.new_str(msg); + self.new_exception_obj(exc_type, vec![pystr_msg]).unwrap() } pub fn new_attribute_error(&self, msg: String) -> PyObjectRef { @@ -224,9 +224,9 @@ impl VirtualMachine { self.new_exception(value_error, msg) } - pub fn new_key_error(&self, msg: String) -> PyObjectRef { + pub fn new_key_error(&self, obj: PyObjectRef) -> PyObjectRef { let key_error = self.ctx.exceptions.key_error.clone(); - self.new_exception(key_error, msg) + self.new_exception_obj(key_error, vec![obj]).unwrap() } pub fn new_index_error(&self, msg: String) -> PyObjectRef {