diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 10783cf33..937a0b6a1 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -655,7 +655,6 @@ class BuiltinTest(ComplexesAreIdenticalMixin, unittest.TestCase): msg = r"^attribute name must be string, not 'int'$" self.assertRaisesRegex(TypeError, msg, delattr, sys, 1) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '__repr__' unexpectedly found in ['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__firstlineno__', '__format__', '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__slots__', '__str__', '__subclasshook__', 'bar'] def test_dir(self): # dir(wrong number of arguments) self.assertRaises(TypeError, dir, 42, 42) @@ -989,7 +988,6 @@ class BuiltinTest(ComplexesAreIdenticalMixin, unittest.TestCase): self.assertRaisesRegex(NameError, "name 'superglobal' is not defined", eval, code, ns) - @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message def test_exec_builtins_mapping_import(self): code = compile("import foo.bar", "test", "exec") ns = {'__builtins__': types.MappingProxyType({})} @@ -998,7 +996,6 @@ class BuiltinTest(ComplexesAreIdenticalMixin, unittest.TestCase): exec(code, ns) self.assertEqual(ns['foo'], ('foo.bar', ns, ns, None, 0)) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: AttributeError not raised by eval def test_eval_builtins_mapping_reduce(self): # list_iterator.__reduce__() calls _PyEval_GetBuiltin("iter") code = compile("x.__reduce__()", "test", "eval") @@ -2970,7 +2967,6 @@ class TestType(unittest.TestCase): A.__firstlineno__ = 43 self.assertEqual(A.__dict__['__firstlineno__'], 43) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'tuple' but 'str' found. def test_type_typeparams(self): class A[T]: pass diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 298bbfb79..8fc4d08dd 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -2788,7 +2788,6 @@ class ClassPropertiesAndMethods(unittest.TestCase): with self.assertRaises(TypeError): super(Base, kw=1) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_basic_inheritance(self): # Testing inheritance from basic types... diff --git a/Lib/test/test_except_star.py b/Lib/test/test_except_star.py index 807e7c5a5..47006c6d3 100644 --- a/Lib/test/test_except_star.py +++ b/Lib/test/test_except_star.py @@ -891,7 +891,6 @@ class TestExceptStarRaiseFrom(ExceptStarTest): class TestExceptStarExceptionGroupSubclass(ExceptStarTest): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_except_star_EG_subclass(self): class EG(ExceptionGroup): def __new__(cls, message, excs, code): diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 5d78a4d69..db6ff9d33 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -20,7 +20,6 @@ class TestExceptionGroupTypeHierarchy(unittest.TestCase): class BadConstructorArgs(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_EG_construction__too_many_args(self): MSG = r'BaseExceptionGroup.__new__\(\) takes exactly 2 arguments' with self.assertRaisesRegex(TypeError, MSG): @@ -194,7 +193,6 @@ class StrAndReprTests(unittest.TestCase): "MyEG('flat', [ValueError(1), TypeError(2)]), " "TypeError(2)])")) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Tuples differ: ('test', (ValueError(1), TypeError(2))) != ('test', []) def test_exceptions_mutation(self): class MyEG(ExceptionGroup): pass @@ -236,7 +234,6 @@ class StrAndReprTests(unittest.TestCase): "ExceptionGroup('test', deque([ValueError(1), TypeError(2)]))" ) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised def test_repr_raises(self): class MySeq(collections.abc.Sequence): def __init__(self, raises): diff --git a/Lib/test/test_exception_hierarchy.py b/Lib/test/test_exception_hierarchy.py index 3472019ea..3318fa8e7 100644 --- a/Lib/test/test_exception_hierarchy.py +++ b/Lib/test/test_exception_hierarchy.py @@ -146,7 +146,6 @@ class AttributesTest(unittest.TestCase): self.assertEqual(e.strerror, "File already exists") self.assertEqual(e.filename, "foo.txt") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_blockingioerror(self): args = ("a", "b", "c", "d", "e") for n in range(6): @@ -181,7 +180,6 @@ class ExplicitSubclassingTest(unittest.TestCase): self.assertEqual(e.bar, "baz") self.assertEqual(e.args, ("some message",)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_new_overridden(self): e = SubOSErrorWithNew("some message", "baz") self.assertEqual(e.baz, "baz") diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index d881c5941..edea5bc05 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -60,7 +60,6 @@ class ExceptionTests(unittest.TestCase): self.assertEqual(buf1, buf2) self.assertEqual(exc.__name__, excname) - @unittest.expectedFailure # TODO: RUSTPYTHON def testRaising(self): self.raise_catch(AttributeError, "AttributeError") self.assertRaises(AttributeError, getattr, sys, "undefined_attribute") @@ -145,7 +144,6 @@ class ExceptionTests(unittest.TestCase): self.raise_catch(StopAsyncIteration, "StopAsyncIteration") - @unittest.expectedFailure # TODO: RUSTPYTHON def testSyntaxErrorMessage(self): # make sure the right exception message is raised for each of # these code fragments @@ -170,7 +168,6 @@ class ExceptionTests(unittest.TestCase): ckmsg("continue\n", "'continue' not properly in loop") ckmsg("f'{6 0}'", "invalid syntax. Perhaps you forgot a comma?") - @unittest.expectedFailure # TODO: RUSTPYTHON def testSyntaxErrorMissingParens(self): def ckmsg(src, msg, exception=SyntaxError): try: @@ -235,7 +232,6 @@ class ExceptionTests(unittest.TestCase): check = self.check check('"\\\n"(1 for c in I,\\\n\\', 2, 2) - @unittest.expectedFailure # TODO: RUSTPYTHON def testSyntaxErrorOffset(self): check = self.check check('def fact(x):\n\treturn x!\n', 2, 10) @@ -442,7 +438,6 @@ class ExceptionTests(unittest.TestCase): with self.assertRaisesRegex(OSError, 'Windows Error 0x%x' % code): ctypes.pythonapi.PyErr_SetFromWindowsErr(code) - @unittest.expectedFailure # TODO: RUSTPYTHON def testAttributes(self): # test that exception attributes are happy @@ -661,7 +656,6 @@ class ExceptionTests(unittest.TestCase): else: self.fail("No exception raised") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_setattr(self): TE = TypeError exc = Exception() @@ -674,7 +668,6 @@ class ExceptionTests(unittest.TestCase): msg = "exception context must be None or derive from BaseException" self.assertRaisesRegex(TE, msg, setattr, exc, '__context__', 1) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_delattr(self): TE = TypeError try: @@ -1306,7 +1299,6 @@ class ExceptionTests(unittest.TestCase): self.assertIs(exc, oe) self.assertIs(exc.__context__, ve) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unicode_change_attributes(self): # See issue 7309. This was a crasher. @@ -1350,7 +1342,6 @@ class ExceptionTests(unittest.TestCase): for klass in klasses: self.assertEqual(str(klass.__new__(klass)), "") - @unittest.expectedFailure # TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust usize def test_unicode_error_str_does_not_crash(self): # Test that str(UnicodeError(...)) does not crash. # See https://github.com/python/cpython/issues/123378. @@ -1374,13 +1365,11 @@ class ExceptionTests(unittest.TestCase): exc = UnicodeDecodeError('utf-8', encoded, start, end, '') self.assertIsInstance(str(exc), str) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unicode_error_evil_str_set_none_object(self): def side_effect(exc): exc.object = None self.do_test_unicode_error_mutate(side_effect) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_unicode_error_evil_str_del_self_object(self): def side_effect(exc): del exc.object @@ -2054,7 +2043,6 @@ class AttributeErrorTests(unittest.TestCase): class ImportErrorTests(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_attributes(self): # Setting 'name' and 'path' should not be a problem. exc = ImportError('test') @@ -2402,14 +2390,12 @@ class SyntaxErrorTests(unittest.TestCase): self.assertEqual(err[-3], ' (') self.assertEqual(err[-2], ' ^') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_non_utf8(self): # Check non utf-8 characters self.addCleanup(unlink, TESTFN) err = run_script(b"\x89") self.assertIn("SyntaxError: Non-UTF-8 code starting with '\\x89' in file", err[-1]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_source(self): def try_compile(source): with self.assertRaises(SyntaxError) as cm: @@ -2524,7 +2510,6 @@ class SyntaxErrorTests(unittest.TestCase): args = ("bad.py", 1, 2, "abcdefg", 1) self.assertRaises(TypeError, SyntaxError, "bad bad", args) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 2 is not None def test_syntax_error_memory_leak(self): # gh-146250: memory leak with re-initialization of SyntaxError e = SyntaxError("msg", ("file.py", 1, 2, "txt", 2, 3)) @@ -2600,7 +2585,6 @@ class PEP626Tests(unittest.TestCase): pass self.lineno_after_raise(in_except, 4) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_lineno_after_other_except(self): def other_except(): try: @@ -2618,7 +2602,6 @@ class PEP626Tests(unittest.TestCase): pass self.lineno_after_raise(in_named_except, 4) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_lineno_in_try(self): def in_try(): try: @@ -2657,7 +2640,6 @@ class PEP626Tests(unittest.TestCase): pass self.lineno_after_raise(after_with, 2) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_missing_lineno_shows_as_none(self): def f(): 1/0 diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 5eea014cc..25fbf3a9b 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1688,7 +1688,7 @@ def test_post_mortem(): ... test_function_2() ... print('Not reached.') - >>> with PdbTestInput([ # TODO: RUSTPYTHON # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE +EXPECTED_FAILURE + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE ... 'step', # step to test_function_2() line ... 'next', # step over exception-raising call ... 'bt', # get a backtrace @@ -2806,7 +2806,7 @@ def test_pdb_closure(): ... g = 3 ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - >>> with PdbTestInput([ # TODO: RUSTPYTHON # doctest: +NORMALIZE_WHITESPACE +EXPECTED_FAILURE + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'k', ... 'g', ... 'y = y', @@ -3224,7 +3224,7 @@ def test_pdb_issue_gh_127321(): ... import pdb; pdb_instance = pdb.Pdb(nosigint=True, readrc=False) ... [1, 2] and pdb_instance.set_trace() ... a = 1 - >>> with PdbTestInput([ # TODO: RUSTPYTHON # doctest: +NORMALIZE_WHITESPACE +EXPECTED_FAILURE + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'continue' ... ]): ... test_function() @@ -4575,7 +4575,6 @@ def bœr(): self.assertIn('4 B', stdout, "breakpoint not found") self.assertIn('-> def func():', stdout, "stack entry not found") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_empty_file(self): script = '' commands = 'q\n' diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 14657dd2e..d06b9a4b6 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -727,7 +727,6 @@ class TestTranforms(BytecodeTestCase): self.assertEqual(format('x = %s!', '%% %s'), 'x = %% %s!') self.assertEqual(format('x = %s, y = %d', 12, 34), 'x = 12, y = 34') - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: unsupported format character 'z' (0x7a) at index 3 def test_format_errors(self): with self.assertRaisesRegex(TypeError, 'not enough arguments for format string'): @@ -1006,7 +1005,6 @@ class TestMarkingVariablesAsUnKnown(BytecodeTestCase): self.assertNotInBytecode(f, "LOAD_FAST_CHECK") self.assertEqual(f.__code__.co_code, co_code) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered def test_setting_lineno_one_undefined(self): code = textwrap.dedent("""\ def f(): @@ -1041,7 +1039,6 @@ class TestMarkingVariablesAsUnKnown(BytecodeTestCase): self.assertNotInBytecode(f, "LOAD_FAST_CHECK") self.assertEqual(f.__code__.co_code, co_code) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered def test_setting_lineno_two_undefined(self): code = textwrap.dedent("""\ def f(): diff --git a/Lib/test/test_print.py b/Lib/test/test_print.py index 7c71c5eef..12256b3b5 100644 --- a/Lib/test/test_print.py +++ b/Lib/test/test_print.py @@ -146,7 +146,6 @@ class TestPy2MigrationHint(unittest.TestCase): if print statement is executed as in Python 2. """ - @unittest.expectedFailure # TODO: RUSTPYTHON def test_normal_string(self): python2_print_str = 'print "Hello World"' with self.assertRaises(SyntaxError) as context: @@ -155,7 +154,6 @@ class TestPy2MigrationHint(unittest.TestCase): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_soft_space(self): python2_print_str = 'print "Hello World",' with self.assertRaises(SyntaxError) as context: @@ -164,7 +162,6 @@ class TestPy2MigrationHint(unittest.TestCase): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_excessive_whitespace(self): python2_print_str = 'print "Hello World", ' with self.assertRaises(SyntaxError) as context: @@ -173,7 +170,6 @@ class TestPy2MigrationHint(unittest.TestCase): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_leading_whitespace(self): python2_print_str = '''if 1: print "Hello World" @@ -187,7 +183,6 @@ class TestPy2MigrationHint(unittest.TestCase): # bpo-32685: Suggestions for print statement should be proper when # it is in the same line as the header of a compound statement # and/or followed by a semicolon - @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_semicolon(self): python2_print_str = 'print p;' with self.assertRaises(SyntaxError) as context: @@ -196,7 +191,6 @@ class TestPy2MigrationHint(unittest.TestCase): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_in_loop_on_same_line(self): python2_print_str = 'for i in s: print i' with self.assertRaises(SyntaxError) as context: diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 6c492e96e..d3cb80bbb 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1013,7 +1013,7 @@ Misuse of the nonlocal and global statement can lead to a few unique syntax erro ... SyntaxError: name 'x' is parameter and nonlocal - >>> def f(): # TODO: RUSTPYTHON; Wrong error message # doctest: +EXPECTED_FAILURE + >>> def f(): ... global x ... nonlocal x Traceback (most recent call last): diff --git a/crates/capi/src/pyerrors.rs b/crates/capi/src/pyerrors.rs index a4ead856b..714cf549e 100644 --- a/crates/capi/src/pyerrors.rs +++ b/crates/capi/src/pyerrors.rs @@ -5,7 +5,7 @@ use core::convert::Infallible; use core::ffi::{c_char, c_int}; use core::ptr::NonNull; use core::slice; -use rustpython_vm::builtins::{PyBaseException, PyTuple, PyType}; +use rustpython_vm::builtins::{PyBaseException, PyTraceback, PyTuple, PyType}; use rustpython_vm::convert::IntoObject; use rustpython_vm::exceptions::ExceptionZoo; use rustpython_vm::{AsObject, PyObjectRef, PyResult}; @@ -298,7 +298,7 @@ pub unsafe extern "C" fn PyException_SetCause(exc: *mut PyObject, cause: *mut Py let exc = unsafe { &*exc }.try_downcast_ref::(vm)?; let cause = NonNull::new(cause) .map(|obj| unsafe { PyObjectRef::from_raw(obj).downcast_unchecked() }); - exc.set___cause__(cause); + exc.set_cause(cause); Ok(()) }) } @@ -309,7 +309,7 @@ pub unsafe extern "C" fn PyException_SetContext(exc: *mut PyObject, context: *mu let exc = unsafe { &*exc }.try_downcast_ref::(vm)?; let context = NonNull::new(context) .map(|obj| unsafe { PyObjectRef::from_raw(obj).downcast_unchecked() }); - exc.set___context__(context); + exc.set_context(context); Ok(()) }) } @@ -362,8 +362,16 @@ pub unsafe extern "C" fn PyUnicodeDecodeError_Create( pub unsafe extern "C" fn PyException_SetTraceback(exc: *mut PyObject, tb: *mut PyObject) -> c_int { with_vm(|vm| { let exc = unsafe { &*exc }.try_downcast_ref::(vm)?; - let traceback = unsafe { tb.as_ref() }.map(|obj| obj.to_owned()); - exc.set___traceback__(vm.unwrap_or_none(traceback), vm) + let traceback = match unsafe { tb.as_ref() }.map(|obj| obj.to_owned()) { + None => None, + Some(obj) if vm.is_none(&obj) => None, + Some(obj) => Some( + obj.downcast::() + .map_err(|_| vm.new_type_error("__traceback__ must be a traceback or None"))?, + ), + }; + exc.set_traceback(traceback); + Ok(()) }) } diff --git a/crates/capi/src/pyframe.rs b/crates/capi/src/pyframe.rs index 611cf79b0..082bb1411 100644 --- a/crates/capi/src/pyframe.rs +++ b/crates/capi/src/pyframe.rs @@ -14,8 +14,5 @@ pub unsafe extern "C" fn PyFrame_GetCode(frame: *mut PyFrameObject) -> *mut PyCo #[unsafe(no_mangle)] pub unsafe extern "C" fn PyFrame_GetLineNumber(frame: *mut PyFrameObject) -> c_int { - with_vm(|_vm| { - let lineno = unsafe { &*frame }.f_lineno(); - Ok(lineno.try_into().unwrap_or(c_int::MAX)) - }) + with_vm(|_vm| Ok(unsafe { &*frame }.lineno())) } diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index d936bfc1a..bff326087 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -808,6 +808,16 @@ impl SymbolTableAnalyzer { sub_tables: &[SymbolTable], class_entry: Option<&SymbolMap>, ) -> SymbolTableResult { + if symbol + .flags + .contains(SymbolFlags::DEF_GLOBAL | SymbolFlags::DEF_NONLOCAL) + { + return Err(SymbolTableError { + error: format!("name '{}' is nonlocal and global", symbol.name), + location: symbol.location, + end_location: symbol.end_location, + }); + } match symbol.scope { // Only an explicit `nonlocal` has to name a binding. Every other free // variable was already resolved by the scope it travelled up from. @@ -3285,7 +3295,8 @@ impl SymbolTableBuilder { .or_insert(symbol) }; - if matches!(role, SymbolUsage::Global | SymbolUsage::Nonlocal) { + if matches!(role, SymbolUsage::Global | SymbolUsage::Nonlocal) && symbol.location.is_none() + { symbol.location = location; symbol.end_location = end_location; } diff --git a/crates/common/src/cformat.rs b/crates/common/src/cformat.rs index 6a37718cb..de437f2ce 100644 --- a/crates/common/src/cformat.rs +++ b/crates/common/src/cformat.rs @@ -105,17 +105,24 @@ pub enum CFormatType { Character(CCharacterType), String(CFormatConversion), Bytes, + /// Parsed conversion that is not one of the supported specifiers. + /// The argument is still consumed, then formatting raises ValueError. + Unsupported { + ch: CodePoint, + index: usize, + }, } impl CFormatType { #[must_use] - pub const fn to_char(self) -> char { + pub fn to_char(self) -> char { match self { Self::Number(x) => x as u8 as char, Self::Float(x) => x as u8 as char, Self::Character(x) => x as u8 as char, Self::String(x) => x as u8 as char, Self::Bytes => 'b', + Self::Unsupported { ch, .. } => ch.to_char_lossy(), } } } @@ -660,7 +667,10 @@ where 's' => CFormatType::String(CFormatConversion::Str), 'a' => CFormatType::String(CFormatConversion::Ascii), 'b' if context == CFormatContext::Bytes => CFormatType::Bytes, - _ => return Err((CFormatErrorType::UnsupportedFormatChar(c.into()), index)), + _ => CFormatType::Unsupported { + ch: c.into(), + index, + }, }) } @@ -962,12 +972,17 @@ mod tests { #[test] fn format_parse_type_fail() { - assert_eq!( - "Hello %n".parse::(), - Err(CFormatError { - typ: CFormatErrorType::UnsupportedFormatChar('n'.into()), - index: 7 - }) + let parsed = "Hello %n".parse::().unwrap(); + let spec = parsed.iter().find_map(|(_, part)| match part { + CFormatPart::Spec(spec) => Some(&spec.spec), + CFormatPart::Literal(_) => None, + }); + assert!( + matches!( + spec.map(|s| &s.format_type), + Some(CFormatType::Unsupported { ch, index: 7 }) if *ch == 'n' + ), + "{spec:?}" ); } diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index e310d3d30..320c37101 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -355,9 +355,16 @@ fn cpython_parse_diagnostic_override( { consider_override(&mut earliest, diagnostic, OverrideClass::Lexer); } - if earliest - .as_ref() - .is_none_or(|current| current.class != OverrideClass::Lexer) + // Indentation errors outrank a print/exec missing-parentheses rewrite. + let indent_error = matches!( + &error.error, + parser::ParseErrorType::Lexical(parser::LexicalErrorType::IndentationError) + | parser::ParseErrorType::UnexpectedIndentation + ) || expected_indented_block_error(error, source_text).is_some(); + if !indent_error + && earliest + .as_ref() + .is_none_or(|current| current.class != OverrideClass::Lexer) && let Some(diagnostic) = invalid_legacy_statement_error(source_text) { consider_override(&mut earliest, diagnostic, OverrideClass::Print); @@ -471,6 +478,24 @@ fn cpython_parse_diagnostic_override( source_error!(mixed_except_handlers_error(source_text)); } + if matches!( + &error.error, + parser::ParseErrorType::Lexical(parser::LexicalErrorType::InvalidByteLiteral) + ) && let Some((start, end)) = + bytes_literal_span(source_text, error.location.start().to_usize()) + { + let (loc, end_loc) = source_locations( + source_file, + TextSize::new(start as u32), + TextSize::new(end as u32), + ); + return Some(NormalizedParseDiagnostic::new( + error.error.clone(), + loc, + end_loc, + )); + } + if matches!( &error.error, parser::ParseErrorType::Lexical(parser::LexicalErrorType::IndentationError) @@ -956,6 +981,32 @@ fn quoted_string_is_closed(bytes: &[u8], start: usize) -> bool { } } +fn bytes_literal_span(source: &str, error_at: usize) -> Option<(usize, usize)> { + let bytes = source.as_bytes(); + if error_at > bytes.len() || bytes.is_empty() { + return None; + } + let mut quote_idx = error_at.min(bytes.len().saturating_sub(1)); + loop { + if matches!(bytes[quote_idx], b'\'' | b'"') { + break; + } + if quote_idx == 0 { + return None; + } + quote_idx -= 1; + } + let mut start = quote_idx; + while start > 0 && matches!(bytes[start - 1], b'b' | b'B' | b'r' | b'R') { + start -= 1; + } + if !matches!(bytes.get(start), Some(b'b' | b'B' | b'r' | b'R')) { + return None; + } + let end = skip_quoted_string(bytes, quote_idx); + Some((start, end)) +} + fn skip_quoted_string(bytes: &[u8], mut index: usize) -> usize { let quote = bytes[index]; let triple = bytes.get(index + 1) == Some("e) && bytes.get(index + 2) == Some("e); @@ -5015,6 +5066,28 @@ fn replacement_field_error( )); } + // Adjacent atoms in a replacement field are a missing comma, not an + // f-string "expecting '}'" at the opening brace. Prefix `yield`/`await`/ + // `not` and continuation keywords (`and`, `for`, ...) are not a second atom. + if !starts_identifier(bytes, expr_start, b"yield") + && !starts_identifier(bytes, expr_start, b"await") + && !starts_identifier(bytes, expr_start, b"not") + && let Some(atom_end) = adjacent_atom_end(bytes, expr_start) + { + let next = skip_ascii_whitespace(bytes, atom_end, expr_end); + if next > atom_end + && expression_atom_start(bytes, next) + && !expression_continuation_keyword(bytes, next) + { + let second_end = adjacent_atom_end(bytes, next).unwrap_or(next + 1); + return Some(CpythonDiagnostic::new( + "invalid syntax. Perhaps you forgot a comma?".to_owned(), + expr_start, + second_end, + )); + } + } + // The expression started fine but ran into a character that cannot continue it. CPython // points at that character and lists the separators it wanted instead. if let Some(stray) = replacement_expression_stray_character(bytes, expr_start, expr_end) { @@ -7691,6 +7764,27 @@ mod tests { } } + #[test] + fn missing_indent_outranks_print_missing_parentheses() { + for (source, expected) in [ + ( + "if True:\nprint \"No indent\"", + "expected an indented block after 'if' statement on line 1", + ), + ( + "print \"old style\"", + "Missing parentheses in call to 'print'. Did you mean print(...)?", + ), + ] { + let err = compile(source, Mode::Exec, "", CompileOpts::default()) + .expect_err("should not compile"); + assert!( + err.to_string().contains(expected), + "{source:?}: expected {expected:?}, got {err}" + ); + } + } + #[test] fn unclosed_fstring_field_keeps_the_unclosed_bracket_flag() { let err = compile("f'{", Mode::Eval, "", CompileOpts::default()) @@ -7918,6 +8012,37 @@ mod tests { assert_eq!(err.python_location(), (2, 3)); } + #[test] + fn fstring_adjacent_atoms_are_a_missing_comma() { + let err = compile("f'{6 0}'", Mode::Exec, "", CompileOpts::default()) + .expect_err("adjacent atoms in an f-string field are a syntax error"); + assert_eq!( + err.to_string(), + "invalid syntax. Perhaps you forgot a comma?" + ); + assert_eq!(err.python_location(), (1, 4)); + assert_eq!(err.python_end_location(), Some((1, 7))); + + compile( + "f'{not x}'", + Mode::Exec, + "", + CompileOpts::default(), + ) + .expect("unary not is a prefix, not two atoms"); + let err = compile( + "f'{a and}'", + Mode::Exec, + "", + CompileOpts::default(), + ) + .expect_err("a dangling 'and' is an f-string separator error"); + assert_eq!( + err.to_string(), + "f-string: expecting '=', or '!', or ':', or '}'" + ); + } + #[test] fn missing_comma_diagnostic_spans_the_whole_second_atom() { // `(start, end)` reported as one-based character columns, matching diff --git a/crates/stdlib/src/_asyncio.rs b/crates/stdlib/src/_asyncio.rs index f419bc413..9055e9d6a 100644 --- a/crates/stdlib/src/_asyncio.rs +++ b/crates/stdlib/src/_asyncio.rs @@ -14,7 +14,7 @@ pub(crate) mod _asyncio { AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{ PyBaseException, PyBaseExceptionRef, PyDict, PyGenericAlias, PyList, PyListRef, - PyModule, PySet, PyTuple, PyType, PyTypeRef, + PyModule, PySet, PyTraceback, PyTuple, PyType, PyTypeRef, }, extend_module, function::{FuncArgs, KwArgs, OptionalArg, OptionalOption, PySetterValue}, @@ -230,8 +230,10 @@ pub(crate) mod _asyncio { let exc: PyBaseExceptionRef = exc.downcast().unwrap(); // Restore the original traceback to prevent traceback accumulation let fut_exception_tb = self.fut_exception_tb.read().clone(); - if let Some(tb) = fut_exception_tb { - let _ = exc.set___traceback__(tb, vm); + if let Some(tb) = fut_exception_tb + && let Ok(tb) = tb.downcast::() + { + exc.set_traceback(Some(tb)); } Err(exc) } else { @@ -311,8 +313,8 @@ pub(crate) mod _asyncio { let runtime_err = vm.new_runtime_error(msg.to_string()); // Set cause and context to the original StopIteration let stop_iter: PyRef = exc.downcast().unwrap(); - runtime_err.set___cause__(Some(stop_iter.clone())); - runtime_err.set___context__(Some(stop_iter)); + runtime_err.set_cause(Some(stop_iter.clone())); + runtime_err.set_context(Some(stop_iter)); runtime_err.into() } else { exc @@ -1276,8 +1278,10 @@ pub(crate) mod _asyncio { let exc: PyBaseExceptionRef = exc.downcast().unwrap(); // Restore the original traceback to prevent traceback accumulation let fut_exception_tb = self.base.fut_exception_tb.read().clone(); - if let Some(tb) = fut_exception_tb { - let _ = exc.set___traceback__(tb, vm); + if let Some(tb) = fut_exception_tb + && let Ok(tb) = tb.downcast::() + { + exc.set_traceback(Some(tb)); } Err(exc) diff --git a/crates/stdlib/src/_sqlite3.rs b/crates/stdlib/src/_sqlite3.rs index 6a20a95d9..c6fe0516a 100644 --- a/crates/stdlib/src/_sqlite3.rs +++ b/crates/stdlib/src/_sqlite3.rs @@ -1515,7 +1515,7 @@ mod _sqlite3 { #[pymethod] fn iterdump(zelf: PyRef, args: IterDumpArgs, vm: &VirtualMachine) -> PyResult { let from_list = PyTuple::new_ref_typed(vec![vm.ctx.new_str("_iterdump")], &vm.ctx); - let module = vm.import_from("sqlite3.dump", &from_list, 0)?; + let module = vm.import_from("sqlite3.dump", from_list, 0)?; let func = module.get_attr("_iterdump", vm)?; let filter: PyObjectRef = args .filter diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs index 662557497..8a972ac00 100644 --- a/crates/stdlib/src/array.rs +++ b/crates/stdlib/src/array.rs @@ -1502,14 +1502,14 @@ pub mod array { } #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let func = builtins_iter(vm); - self.internal.lock().reduce( + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let func = builtins_iter(vm)?; + Ok(self.internal.lock().reduce( func, |x| x.clone().into(), |vm| vm.ctx.empty_tuple.clone().into(), vm, - ) + )) } } diff --git a/crates/stdlib/src/cjkcodecs/multibytecodec.rs b/crates/stdlib/src/cjkcodecs/multibytecodec.rs index 87fa6fecb..8bb374051 100644 --- a/crates/stdlib/src/cjkcodecs/multibytecodec.rs +++ b/crates/stdlib/src/cjkcodecs/multibytecodec.rs @@ -147,7 +147,7 @@ mod _multibytecodec { obj.set_attr("reason", vm.ctx.new_str(reason), vm)?; return Ok(exc.clone()); } - let exc = vm.new_unicode_encode_error_real( + let exc = vm.new_unicode_encode_error( vm.ctx.new_str(encoding), self.input.clone(), start, @@ -487,7 +487,7 @@ mod _multibytecodec { if pos < chars { if chars - pos > MAXENCPENDING { // Normal codecs can't reach here. - return Err(vm.new_unicode_encode_error_real( + return Err(vm.new_unicode_encode_error( vm.ctx.new_str(self.encoding), input, pos, @@ -778,7 +778,7 @@ mod _multibytecodec { Some(pending) => { let bytes = pending.as_wtf8().as_bytes(); if bytes.len() > MAXENCPENDING * 4 { - return Err(vm.new_unicode_encode_error_real( + return Err(vm.new_unicode_encode_error( vm.ctx.new_str(self.codec.encoding), pending.clone(), 0, diff --git a/crates/stdlib/src/pickle.rs b/crates/stdlib/src/pickle.rs index 44153f69d..d38ba9426 100644 --- a/crates/stdlib/src/pickle.rs +++ b/crates/stdlib/src/pickle.rs @@ -727,7 +727,7 @@ mod _pickle { let err = vm.new_attribute_error(format!( "Can't resolve path {name_repr} on module {module_repr}" )); - err.set___context__(Some(e)); + err.set_context(Some(e)); err }) } else { @@ -3528,7 +3528,7 @@ mod _pickle { safe_repr(module.as_object(), vm) ), ); - err.set___context__(Some(e)); + err.set_context(Some(e)); err })?; let name_bytes = vm @@ -3543,7 +3543,7 @@ mod _pickle { safe_repr(global_name.as_object(), vm) ), ); - err.set___context__(Some(e)); + err.set_context(Some(e)); err })?; let mut buf = vec![GLOBAL as u8]; @@ -3647,7 +3647,7 @@ mod _pickle { .map_or_else(|_| String::new(), |s| s.to_string_lossy().into_owned()); let err = new_pickling_error(vm, format!("Can't pickle {}: {msg}", safe_repr(obj, vm))); - err.set___context__(Some(e)); + err.set_context(Some(e)); return Err(err); } }; @@ -3670,7 +3670,7 @@ mod _pickle { module_name.to_string_lossy() ), ); - err.set___context__(Some(e)); + err.set_context(Some(e)); Err(err) } } diff --git a/crates/stdlib/src/pystruct.rs b/crates/stdlib/src/pystruct.rs index a70976f45..0c85e99b8 100644 --- a/crates/stdlib/src/pystruct.rs +++ b/crates/stdlib/src/pystruct.rs @@ -41,7 +41,7 @@ pub(crate) mod _struct { .code_points() .position(|cp| !cp.to_char().is_some_and(|c| c.is_ascii())) .unwrap_or(0); - return Err(vm.new_unicode_encode_error_real( + return Err(vm.new_unicode_encode_error( vm.ctx.new_str("ascii"), s, start, diff --git a/crates/stdlib/src/socket.rs b/crates/stdlib/src/socket.rs index b37f77e80..7405dbdae 100644 --- a/crates/stdlib/src/socket.rs +++ b/crates/stdlib/src/socket.rs @@ -2741,7 +2741,7 @@ mod _socket { .code_points() .position(|c| c.to_char().is_none()) .unwrap(); - vm.new_unicode_encode_error_real( + vm.new_unicode_encode_error( vm.ctx.new_str("utf-8"), (*s).clone(), start, diff --git a/crates/stdlib/src/syslog.rs b/crates/stdlib/src/syslog.rs index ab4f78bdb..e9989e8aa 100644 --- a/crates/stdlib/src/syslog.rs +++ b/crates/stdlib/src/syslog.rs @@ -39,7 +39,7 @@ mod syslog { .code_points() .position(|c| c.to_char().is_none()) .unwrap_or(0); - vm.new_unicode_encode_error_real( + vm.new_unicode_encode_error( vm.ctx.new_str("utf-8"), ident.clone(), start, diff --git a/crates/vm/src/builtins/bytearray.rs b/crates/vm/src/builtins/bytearray.rs index ccf7728e4..a71be4650 100644 --- a/crates/vm/src/builtins/bytearray.rs +++ b/crates/vm/src/builtins/bytearray.rs @@ -914,14 +914,14 @@ impl PyByteArrayIterator { self.internal.lock().length_hint(|obj| obj.__len__()) } #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let func = builtins_iter(vm); - self.internal.lock().reduce( + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let func = builtins_iter(vm)?; + Ok(self.internal.lock().reduce( func, |x| x.clone().into(), |vm| vm.ctx.empty_tuple.clone().into(), vm, - ) + )) } #[pymethod] diff --git a/crates/vm/src/builtins/bytes.rs b/crates/vm/src/builtins/bytes.rs index 14c2dff87..e7d41c050 100644 --- a/crates/vm/src/builtins/bytes.rs +++ b/crates/vm/src/builtins/bytes.rs @@ -773,14 +773,14 @@ impl PyBytesIterator { } #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let func = builtins_iter(vm); - self.internal.lock().reduce( + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let func = builtins_iter(vm)?; + Ok(self.internal.lock().reduce( func, |x| x.clone().into(), |vm| vm.ctx.empty_tuple.clone().into(), vm, - ) + )) } #[pymethod] diff --git a/crates/vm/src/builtins/code.rs b/crates/vm/src/builtins/code.rs index 2eb0e5a5f..d083f778a 100644 --- a/crates/vm/src/builtins/code.rs +++ b/crates/vm/src/builtins/code.rs @@ -709,6 +709,27 @@ impl fmt::Debug for PyCode { } } +impl PyCode { + /// Line number for a byte offset, or -1 when the linetable has no line. + pub fn addr2line(&self, lasti_bytes: i32) -> i32 { + if lasti_bytes < 0 { + return self.code.first_line_number.map_or(-1, |n| n.get() as i32); + } + let linetable = self.code.linetable.as_ref(); + if linetable.is_empty() { + return self.code.first_line_number.map_or(-1, |n| n.get() as i32); + } + let first_line = self.code.first_line_number.map_or(0, |n| n.get() as i32); + let mut range = PyCodeAddressRange::new(linetable, first_line); + while range.ar_end <= lasti_bytes { + if !range.advance() { + return -1; + } + } + range.ar_line + } +} + impl PyPayload for PyCode { #[inline] fn class(ctx: &Context) -> &'static Py { diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index 0304d7cbe..b223c0912 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -193,7 +193,7 @@ impl PyDict { match vm.call_method(exc.as_object(), "add_note", (vm.ctx.new_str(note),)) { Ok(_) => exc, Err(note_err) => { - note_err.set___context__(Some(exc)); + note_err.set_context(Some(exc)); note_err } } @@ -1252,8 +1252,8 @@ macro_rules! dict_view { } #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let iter = builtins_iter(vm); + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let iter = builtins_iter(vm)?; let internal = self.internal.lock(); let entries = match &internal.status { IterStatus::Active(dict) => { @@ -1269,7 +1269,7 @@ macro_rules! dict_view { } IterStatus::Exhausted => vec![], }; - vm.new_tuple((iter, (vm.ctx.new_list(entries),))) + Ok(vm.new_tuple((iter, (vm.ctx.new_list(entries),)))) } } @@ -1336,8 +1336,8 @@ macro_rules! dict_view { } #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let iter = builtins_iter(vm); + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let iter = builtins_iter(vm)?; let internal = self.internal.lock(); let entries = match &internal.status { IterStatus::Active(dict) => { @@ -1356,7 +1356,7 @@ macro_rules! dict_view { } IterStatus::Exhausted => vec![], }; - vm.new_tuple((iter, (vm.ctx.new_list(entries),))) + Ok(vm.new_tuple((iter, (vm.ctx.new_list(entries),)))) } #[pymethod] diff --git a/crates/vm/src/builtins/enumerate.rs b/crates/vm/src/builtins/enumerate.rs index a2c5b9641..cf9fbcde5 100644 --- a/crates/vm/src/builtins/enumerate.rs +++ b/crates/vm/src/builtins/enumerate.rs @@ -178,14 +178,14 @@ impl PyReverseSequenceIterator { } #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let func = builtins_reversed(vm); - self.internal.lock().reduce( + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let func = builtins_reversed(vm)?; + Ok(self.internal.lock().reduce( func, |x| x.clone(), |vm| vm.ctx.empty_tuple.clone().into(), vm, - ) + )) } } diff --git a/crates/vm/src/builtins/frame.rs b/crates/vm/src/builtins/frame.rs index 7b8ab8f2d..4dd4f92c9 100644 --- a/crates/vm/src/builtins/frame.rs +++ b/crates/vm/src/builtins/frame.rs @@ -436,7 +436,7 @@ impl Representable for FrameObject { fn repr_str(zelf: &Py, vm: &VirtualMachine) -> PyResult { let code = zelf.iframe().code(); let file_repr = code.source_path().to_owned().as_object().repr(vm)?; - let lineno = zelf.f_lineno(); + let lineno = zelf.lineno(); let name = code.code.obj_name.as_wtf8(); let ptr = zelf as *const Py as usize; Ok(format!( @@ -496,36 +496,28 @@ impl FrameObject { val * 2 } - #[pygetset] - pub fn f_lineno(&self) -> usize { - // For executing frames (on the TLS chain), read the live iframe's - // lasti directly rather than `self.lasti()`, which may reflect a - // stale snapshot taken before a nested call. lasti always points - // just past the last-fetched instruction (see the bytecode loop in - // `ExecutingFrame::run`), so `locations[lasti - 1]` is the source - // line of the instruction currently in flight — correct even when - // observed mid-CALL (e.g. sys._getframe, warnings.warn). - let live = self.find_live_source_iframe(); - let lasti = if !live.is_null() { - unsafe { (*live).lasti.load(Relaxed) } - } else { - self.lasti() - }; - // If lasti is 0, execution hasn't started yet - use first line number - if lasti == 0 { + /// Current line, or -1 when the linetable has no line for lasti. + pub fn lineno(&self) -> i32 { + let lasti_bytes = self.f_lasti() as i32; + // If lasti is 0, execution hasn't started yet - use first line number. + // Read the live iframe so a materialized copy that still has lasti==0 + // does not hide the current line while the frame is running. + if lasti_bytes == 0 { return self .iframe() .code() .first_line_number - .map_or(1, |n| n.get()); + .map_or(1, |n| n.get() as i32); } - // This lookup also covers returned frames (e.g. exception - // tracebacks), where `live` is null and `self.lasti()` reflects the - // frame's final position. - self.iframe().code().locations[lasti as usize - 1] - .0 - .line - .get() + // lasti is stored as the next instruction index (see FrameObject::run), + // so the executing opcode is at lasti-1 / lasti_bytes-2. + self.f_code().addr2line(lasti_bytes - 2) + } + + #[pygetset] + fn f_lineno(&self) -> Option { + let lineno = self.lineno(); + (lineno >= 0).then_some(lineno as usize) } #[pygetset(setter)] @@ -645,6 +637,37 @@ impl FrameObject { .cold() .pending_unwind_from_stack .store(start_stack, Relaxed); + // Bind None into any NULL localsplus slots the jump target may + // assume exist, rather than leaving LOAD_FAST to raise later. + let unbound = { + let fastlocals = unsafe { + let ptr = target as *const crate::frame::InterpreterFrame + as *mut crate::frame::InterpreterFrame; + (*ptr).localsplus.fastlocals() + }; + fastlocals.iter().filter(|slot| slot.is_none()).count() + }; + if unbound > 0 { + let s = if unbound == 1 { "" } else { "s" }; + crate::stdlib::_warnings::warn( + vm.ctx.exceptions.runtime_warning, + format!("assigning None to {unbound} unbound local{s}"), + 1, + vm, + )?; + let none = vm.ctx.none(); + let fastlocals = unsafe { + let ptr = target as *const crate::frame::InterpreterFrame + as *mut crate::frame::InterpreterFrame; + (*ptr).localsplus.fastlocals_mut() + }; + for slot in fastlocals.iter_mut() { + if slot.is_none() { + *slot = Some(none.clone()); + } + } + } + target.lasti.store(best_addr as u32, Relaxed); Ok(()) } diff --git a/crates/vm/src/builtins/iter.rs b/crates/vm/src/builtins/iter.rs index 2d231e9e6..b5d710bef 100644 --- a/crates/vm/src/builtins/iter.rs +++ b/crates/vm/src/builtins/iter.rs @@ -216,12 +216,12 @@ where locked_step(internal, |internal| internal.rev_next(f)) } -pub fn builtins_iter(vm: &VirtualMachine) -> PyObjectRef { - vm.builtins.get_attr("iter", vm).unwrap() +pub fn builtins_iter(vm: &VirtualMachine) -> PyResult { + vm.eval_get_builtin(vm.ctx.intern_str("iter")) } -pub fn builtins_reversed(vm: &VirtualMachine) -> PyObjectRef { - vm.builtins.get_attr("reversed", vm).unwrap() +pub fn builtins_reversed(vm: &VirtualMachine) -> PyResult { + vm.eval_get_builtin(vm.ctx.intern_str("reversed")) } #[pyclass(module = false, name = "iterator", traverse)] @@ -271,14 +271,14 @@ impl PySequenceIterator { } #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let func = builtins_iter(vm); - self.internal.lock().reduce( + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let func = builtins_iter(vm)?; + Ok(self.internal.lock().reduce( func, |x| x.clone(), |vm| vm.ctx.empty_tuple.clone().into(), vm, - ) + )) } #[pymethod] @@ -322,14 +322,14 @@ impl PyCallableIterator { } #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let func = builtins_iter(vm); + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let func = builtins_iter(vm)?; let status = self.status.read(); if let IterStatus::Active(callable) = &*status { let callable_obj: PyObjectRef = callable.clone().into(); - vm.new_tuple((func, (callable_obj, self.sentinel.clone()))) + Ok(vm.new_tuple((func, (callable_obj, self.sentinel.clone())))) } else { - vm.new_tuple((func, (vm.ctx.empty_tuple.clone(),))) + Ok(vm.new_tuple((func, (vm.ctx.empty_tuple.clone(),)))) } } } diff --git a/crates/vm/src/builtins/list.rs b/crates/vm/src/builtins/list.rs index 517477fd6..7939c828e 100644 --- a/crates/vm/src/builtins/list.rs +++ b/crates/vm/src/builtins/list.rs @@ -901,14 +901,14 @@ impl PyListIterator { } #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let func = builtins_iter(vm); - self.internal.lock().reduce( + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let func = builtins_iter(vm)?; + Ok(self.internal.lock().reduce( func, |x| x.clone().into(), |vm| vm.ctx.new_list(Vec::new()).into(), vm, - ) + )) } } @@ -965,14 +965,14 @@ impl PyListReverseIterator { } #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let func = builtins_reversed(vm); - self.internal.lock().reduce( + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let func = builtins_reversed(vm)?; + Ok(self.internal.lock().reduce( func, |x| x.clone().into(), |vm| vm.ctx.new_list(Vec::new()).into(), vm, - ) + )) } } diff --git a/crates/vm/src/builtins/memory.rs b/crates/vm/src/builtins/memory.rs index b67bfbe9f..b472e87c4 100644 --- a/crates/vm/src/builtins/memory.rs +++ b/crates/vm/src/builtins/memory.rs @@ -1699,14 +1699,14 @@ impl PyPayload for PyMemoryViewIterator { #[pyclass(flags(DISALLOW_INSTANTIATION), with(IterNext, Iterable))] impl PyMemoryViewIterator { #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let func = builtins_iter(vm); - self.internal.lock().reduce( + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let func = builtins_iter(vm)?; + Ok(self.internal.lock().reduce( func, |x| x.clone().into(), |vm| vm.ctx.empty_tuple.clone().into(), vm, - ) + )) } } diff --git a/crates/vm/src/builtins/object.rs b/crates/vm/src/builtins/object.rs index 1e92fae76..d52a77659 100644 --- a/crates/vm/src/builtins/object.rs +++ b/crates/vm/src/builtins/object.rs @@ -398,7 +398,36 @@ impl PyBaseObject { #[pymethod] pub fn __dir__(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { - zelf.dir(vm) + let mut names: Vec = Vec::new(); + + match zelf.get_attr(identifier!(vm, __dict__), vm) { + Ok(obj) => { + if let Ok(dict) = obj.downcast::() { + names.extend( + dict.into_iter() + .filter_map(|(k, _)| k.downcast_ref::().is_some().then_some(k)), + ); + } + } + Err(e) if e.fast_isinstance(vm.ctx.exceptions.attribute_error) => {} + Err(e) => return Err(e), + } + + match zelf.get_attr(identifier!(vm, __class__), vm) { + Ok(cls_obj) => { + if let Some(cls) = cls_obj.downcast_ref::() { + for (name, _) in cls.get_attributes(&vm.ctx) { + names.push(name.to_object()); + } + } + } + Err(e) if e.fast_isinstance(vm.ctx.exceptions.attribute_error) => {} + Err(e) => return Err(e), + } + + let lst = PyList::from(names); + lst.sort(Default::default(), vm)?; + Ok(lst) } #[pymethod] diff --git a/crates/vm/src/builtins/range.rs b/crates/vm/src/builtins/range.rs index db442d799..1252bcb5c 100644 --- a/crates/vm/src/builtins/range.rs +++ b/crates/vm/src/builtins/range.rs @@ -617,7 +617,7 @@ impl PyLongRangeIterator { } #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { range_iter_reduce( self.start.clone(), self.length.clone(), @@ -679,7 +679,7 @@ impl PyRangeIterator { } #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { range_iter_reduce( BigInt::from(self.start), BigInt::from(self.length), @@ -720,8 +720,8 @@ fn range_iter_reduce( step: BigInt, index: usize, vm: &VirtualMachine, -) -> PyTupleRef { - let iter = builtins_iter(vm); +) -> PyResult { + let iter = builtins_iter(vm)?; // CPython pickles the remaining range with a None state. next() increments // the index unconditionally, so clamp it to length before rebasing start. let index = BigInt::from(index).min(length.clone()); @@ -732,7 +732,7 @@ fn range_iter_reduce( stop: PyInt::from(stop).into_ref(&vm.ctx), step: PyInt::from(step).into_ref(&vm.ctx), }; - vm.new_tuple((iter, (range,), vm.ctx.none())) + Ok(vm.new_tuple((iter, (range,), vm.ctx.none()))) } // Silently clips state (i.e index) in range [0, usize::MAX]. diff --git a/crates/vm/src/builtins/set.rs b/crates/vm/src/builtins/set.rs index 44487e3e9..e59190c6f 100644 --- a/crates/vm/src/builtins/set.rs +++ b/crates/vm/src/builtins/set.rs @@ -583,7 +583,7 @@ impl PySetInner { "cannot use '{}' as a set element ({message})", item.class().name() )); - err.set___cause__(Some(cause)); + err.set_cause(Some(cause)); Err(err) } result => result, @@ -1534,10 +1534,13 @@ impl PySetIterator { } #[pymethod] - fn __reduce__(zelf: PyRef, vm: &VirtualMachine) -> (PyObjectRef, (PyObjectRef,)) { + fn __reduce__( + zelf: PyRef, + vm: &VirtualMachine, + ) -> PyResult<(PyObjectRef, (PyObjectRef,))> { let internal = zelf.internal.lock(); - ( - builtins_iter(vm), + Ok(( + builtins_iter(vm)?, (vm.ctx .new_list(match &internal.status { IterStatus::Exhausted => vec![], @@ -1550,7 +1553,7 @@ impl PySetIterator { .collect(), }) .into(),), - ) + )) } } diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index 8e48fa696..7cc6ac0ba 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -343,14 +343,14 @@ impl PyStrIterator { } #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let func = builtins_iter(vm); - self.internal.lock().0.reduce( + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let func = builtins_iter(vm)?; + Ok(self.internal.lock().0.reduce( func, |x| x.clone().into(), |vm| vm.ctx.empty_str.to_owned().into(), vm, - ) + )) } } @@ -548,7 +548,7 @@ impl PyStr { .code_points() .position(|c| c.to_char().is_none()) .unwrap(); - Err(vm.new_unicode_encode_error_real( + Err(vm.new_unicode_encode_error( identifier!(vm, utf_8).to_owned(), vm.ctx.new_str(self.data.clone()), start, @@ -927,6 +927,15 @@ impl PyStr { } } + /// Return `zelf` when it is an exact str; otherwise a new str copy. + fn result_unchanged(zelf: PyRef, vm: &VirtualMachine) -> PyRef { + if zelf.class().is(vm.ctx.types.str_type) { + zelf + } else { + vm.ctx.new_str(zelf.as_wtf8()) + } + } + #[pymethod] fn lstrip( zelf: PyRef, @@ -940,7 +949,7 @@ impl PyStr { |s| s.trim_start_matches(|c: CodePoint| c.is_char_and(unicode::classify::is_space)), ); if s == stripped { - zelf + Self::result_unchanged(zelf, vm) } else { vm.ctx.new_str(stripped) } @@ -959,7 +968,7 @@ impl PyStr { |s| s.trim_end_matches(|c: CodePoint| c.is_char_and(unicode::classify::is_space)), ); if s == stripped { - zelf + Self::result_unchanged(zelf, vm) } else { vm.ctx.new_str(stripped) } diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index e3fcc2589..ea9564bdf 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -705,14 +705,14 @@ impl PyTupleIterator { } #[pymethod] - fn __reduce__(&self, vm: &VirtualMachine) -> PyTupleRef { - let func = builtins_iter(vm); - self.internal.lock().reduce( + fn __reduce__(&self, vm: &VirtualMachine) -> PyResult { + let func = builtins_iter(vm)?; + Ok(self.internal.lock().reduce( func, |x| x.clone().into(), |vm| vm.ctx.empty_tuple.clone().into(), vm, - ) + )) } } diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 09669c2c0..3079aaebe 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -2300,43 +2300,31 @@ impl PyType { } #[pygetset] - fn __type_params__(&self, vm: &VirtualMachine) -> PyTupleRef { + fn __type_params__(&self, vm: &VirtualMachine) -> PyObjectRef { let key = identifier!(vm, __type_params__); if let Some(params) = self.attributes.get(key) - && let Ok(tuple) = params.downcast::() + // Builtin type namespaces store the getset wrapper under this + // name; heap types store the actual value. + && !params.class().is(vm.ctx.types.getset_type) { - return tuple; + return params; } - // Return empty tuple if not found or not a tuple - vm.ctx.empty_tuple.clone() + vm.ctx.empty_tuple.clone().into() } #[pygetset(setter)] - fn set___type_params__( - &self, - value: PySetterValue, - vm: &VirtualMachine, - ) -> PyResult<()> { + fn set___type_params__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { let key = identifier!(vm, __type_params__); match value { PySetterValue::Assign(val) => { self.check_set_special_type_attr(key, vm)?; let _prev_value = Self::with_type_lock(vm, || { self.modified_inner(); - self.attributes.insert(key, val.into()) + self.attributes.insert(key, val) }); } PySetterValue::Delete => { - if self.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { - return Err(vm.new_type_error(format!( - "cannot delete '__type_params__' attribute of immutable type '{}'", - self.slot_name() - ))); - } - let _prev_value = Self::with_type_lock(vm, || { - self.modified_inner(); - self.attributes.remove(key) - }); + return Err(vm.new_type_error("cannot delete '__type_params__'")); } } Ok(()) diff --git a/crates/vm/src/cformat.rs b/crates/vm/src/cformat.rs index 954386222..93ca42e27 100644 --- a/crates/vm/src/cformat.rs +++ b/crates/vm/src/cformat.rs @@ -32,6 +32,11 @@ fn spec_format_bytes( obj: PyObjectRef, ) -> PyResult> { match &spec.format_type { + CFormatType::Unsupported { ch, index } => Err(vm.new_value_error(format!( + "unsupported format character '{}' ({:#x}) at index {index}", + ch.to_char_lossy(), + ch.to_u32(), + ))), // Unlike strings, %r and %a are identical for bytes: the behaviour corresponds to // %a for strings (not %r) CFormatType::String(CFormatConversion::Repr | CFormatConversion::Ascii) => { @@ -174,6 +179,11 @@ fn spec_format_string( obj: PyObjectRef, ) -> PyResult { match &spec.format_type { + CFormatType::Unsupported { ch, index } => Err(vm.new_value_error(format!( + "unsupported format character '{}' ({:#x}) at index {index}", + ch.to_char_lossy(), + ch.to_u32(), + ))), CFormatType::String(conversion) => { let result = match conversion { CFormatConversion::Ascii => builtins::ascii(obj, vm)?.as_wtf8().to_owned(), diff --git a/crates/vm/src/codecs.rs b/crates/vm/src/codecs.rs index c5416c2ad..5ca706735 100644 --- a/crates/vm/src/codecs.rs +++ b/crates/vm/src/codecs.rs @@ -868,7 +868,7 @@ impl EncodeContext for PyEncodeContext<'_> { let reason = reason.expect( "should only ever pass reason: None if an exception is already set", ); - vm.new_unicode_encode_error_real( + vm.new_unicode_encode_error( vm.ctx.new_str(self.encoding), self.data.to_owned(), range.start.chars, diff --git a/crates/vm/src/coroutine.rs b/crates/vm/src/coroutine.rs index b4c8b01a2..ade2890cf 100644 --- a/crates/vm/src/coroutine.rs +++ b/crates/vm/src/coroutine.rs @@ -329,15 +329,15 @@ impl Coro { vm.new_runtime_error(format!("{} raised StopIteration", gen_name(jen, vm))); // PEP 479: chain __context__ as well as __cause__ to the // original StopIteration. - err.set___context__(Some(e.clone())); - err.set___cause__(Some(e)); + err.set_context(Some(e.clone())); + err.set_cause(Some(e)); Err(err) } else if jen.class().is(vm.ctx.types.async_generator) && e.fast_isinstance(vm.ctx.exceptions.stop_async_iteration) { let err = vm.new_runtime_error("async generator raised StopAsyncIteration"); - err.set___context__(Some(e.clone())); - err.set___cause__(Some(e)); + err.set_context(Some(e.clone())); + err.set_cause(Some(e)); Err(err) } else { Err(e) @@ -659,10 +659,10 @@ pub(crate) fn unraisable_while_closing( && let Some(frame) = coro.frame_opt() { let lasti = frame.lasti().saturating_mul(2); - let lineno = rustpython_compiler_core::OneIndexed::new(frame.f_lineno().max(1)) + let lineno = rustpython_compiler_core::OneIndexed::new(frame.lineno().max(1) as usize) .unwrap_or(rustpython_compiler_core::OneIndexed::MIN); let tb = PyTraceback::new(None, frame, lasti, lineno); - e.set_traceback_typed(Some(tb.into_ref(&vm.ctx))); + e.set_traceback(Some(tb.into_ref(&vm.ctx))); } let msg = jen .repr(vm) diff --git a/crates/vm/src/exception_group.rs b/crates/vm/src/exception_group.rs index dc697093f..58261712a 100644 --- a/crates/vm/src/exception_group.rs +++ b/crates/vm/src/exception_group.rs @@ -3,14 +3,14 @@ //! This module implements BaseExceptionGroup and ExceptionGroup with multiple inheritance support. use crate::builtins::{PyList, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef}; -use crate::function::{ArgIterable, FuncArgs}; +use crate::function::FuncArgs; +use crate::object::{Traverse, TraverseFn}; use crate::types::{PyTypeFlags, PyTypeSlots}; use crate::{ - AsObject, Context, Py, PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, - class::PyClassDef, + AsObject, Context, Py, PyAtomicRef, PyObject, PyObjectRef, PyRef, PyResult, VirtualMachine, }; use core::fmt::Write; -use rustpython_common::wtf8::{Wtf8, Wtf8Buf}; +use rustpython_common::wtf8::Wtf8Buf; use crate::exceptions::types::PyBaseException; @@ -49,13 +49,52 @@ pub(super) mod types { use crate::builtins::PyGenericAlias; use crate::types::{Constructor, Initializer}; - #[pyexception(name, base = PyBaseException, ctx = "base_exception_group")] - #[derive(Debug)] - #[repr(transparent)] - pub struct PyBaseExceptionGroup(PyBaseException); + #[pyexception(name, base = PyBaseException, ctx = "base_exception_group", traverse = "manual")] + #[repr(C)] + pub struct PyBaseExceptionGroup { + base: PyBaseException, + msg: PyAtomicRef, + excs: PyAtomicRef, + excs_str: PyAtomicRef>, + } + + impl crate::class::PySubclass for PyBaseExceptionGroup { + type Base = PyBaseException; + fn as_base(&self) -> &Self::Base { + &self.base + } + } + + impl core::fmt::Debug for PyBaseExceptionGroup { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PyBaseExceptionGroup") + .finish_non_exhaustive() + } + } + + unsafe impl Traverse for PyBaseExceptionGroup { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.base.traverse(tracer_fn); + tracer_fn(&self.msg); + tracer_fn(&self.excs); + if let Some(obj) = self.excs_str.deref() { + tracer_fn(obj); + } + } + } #[pyexception(with(Constructor, Initializer))] impl PyBaseExceptionGroup { + #[pygetset] + fn message(&self) -> PyObjectRef { + self.msg.to_owned() + } + + #[pygetset] + fn exceptions(&self) -> PyObjectRef { + self.excs.to_owned() + } + #[pyclassmethod] fn __class_getitem__( cls: PyTypeRef, @@ -66,19 +105,15 @@ pub(super) mod types { } #[pymethod] - fn derive( - zelf: PyRef, - excs: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult { - let message = zelf.get_arg(0).unwrap_or_else(|| vm.ctx.new_str("").into()); + fn derive(zelf: PyRef, excs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let message = zelf.msg.to_owned(); vm.invoke_exception(vm.ctx.exceptions.base_exception_group, vec![message, excs]) .map(|e| e.into()) } #[pymethod] fn subgroup( - zelf: PyRef, + zelf: PyRef, matcher_value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { @@ -130,7 +165,7 @@ pub(super) mod types { #[pymethod] fn split( - zelf: PyRef, + zelf: PyRef, matcher_value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { @@ -192,19 +227,12 @@ pub(super) mod types { } #[pymethod] - fn __str__(zelf: &Py, vm: &VirtualMachine) -> PyResult { - let message = zelf.get_arg(0).map(|m| m.str(vm)).transpose()?; - - let num_excs = zelf - .get_arg(1) - .and_then(|obj| obj.downcast_ref::().map(|t| t.len())) - .unwrap_or(0); + fn __str__(zelf: &Py, vm: &VirtualMachine) -> PyResult { + let message = zelf.msg.str(vm)?; + let num_excs = zelf.excs.downcast_ref::().map_or(0, |t| t.len()); let suffix = if num_excs == 1 { "" } else { "s" }; - let mut result = match message { - Some(s) => s.as_wtf8().to_owned(), - None => Wtf8Buf::new(), - }; + let mut result = message.as_wtf8().to_owned(); write!(result, " ({num_excs} sub-exception{suffix})").unwrap(); Ok(vm.ctx.new_str(result)) } @@ -212,52 +240,53 @@ pub(super) mod types { #[pyslot] fn slot_repr(zelf: &PyObject, vm: &VirtualMachine) -> PyResult { let zelf = zelf - .downcast_ref::() - .expect("exception group must be BaseException"); + .downcast_ref::() + .expect("exception group must be BaseExceptionGroup"); let class_name = zelf.class().name().to_owned(); - let message = zelf.get_arg(0).map(|m| m.repr(vm)).transpose()?; + let message = zelf.msg.repr(vm)?; + + let exceptions_str = if let Some(saved) = zelf.excs_str.to_owned() { + saved + .downcast::() + .map_err(|_| vm.new_type_error("__repr__ returned non-string"))? + } else { + let args = zelf.base.args(); + let exceptions_obj = + if args.len() == 2 && args[1].downcast_ref::().is_some() { + let list = match zelf.excs.downcast_ref::() { + Some(tuple) => vm.ctx.new_list(tuple.to_vec()), + None => vm.ctx.new_list(vec![]), + }; + list.into() + } else { + zelf.excs.to_owned() + }; + exceptions_obj.repr(vm)? + }; let mut result = Wtf8Buf::new(); write!(result, "{class_name}(").unwrap(); - let message_wtf8: &Wtf8 = message - .as_ref() - .map_or_else(|| "''".as_ref(), |s| s.as_wtf8()); - result.push_wtf8(message_wtf8); - result.push_str(", ["); - if let Some(exceptions_obj) = zelf.get_arg(1) { - let iter: ArgIterable = - ArgIterable::try_from_object(vm, exceptions_obj)?; - let mut first = true; - for exc in iter.iter(vm)? { - if !first { - result.push_str(", "); - } - first = false; - result.push_wtf8(exc?.repr(vm)?.as_wtf8()); - } - } - result.push_str("])"); + result.push_wtf8(message.as_wtf8()); + result.push_str(", "); + result.push_wtf8(exceptions_str.as_wtf8()); + result.push_str(")"); Ok(vm.ctx.new_str(result)) } } impl Constructor for PyBaseExceptionGroup { - type Args = crate::function::PosArgs; + type Args = FuncArgs; fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let args: Self::Args = args.bind_for(vm, Self::NAME)?; - let args = args.into_vec(); - // Validate exactly 2 positional arguments - if args.len() != 2 { + if args.args.len() != 2 { return Err(vm.new_type_error(format!( - "BaseExceptionGroup.__new__() takes exactly 2 positional arguments ({} given)", - args.len() + "BaseExceptionGroup.__new__() takes exactly 2 arguments ({} given)", + args.args.len() ))); } - // Validate message is str - let message = args[0].clone(); + let message = args.args[0].clone(); if !message.fast_isinstance(vm.ctx.types.str_type) { return Err(vm.new_type_error(format!( "argument 1 must be str, not {}", @@ -265,24 +294,29 @@ pub(super) mod types { ))); } - // Validate exceptions is a sequence - let exceptions_arg = &args[1]; + let exceptions_arg = &args.args[1]; exceptions_arg.try_sequence(vm).map_err(|_| { vm.new_type_error("second argument (exceptions) must be a sequence") })?; + let is_list = exceptions_arg.downcast_ref::().is_some(); + let is_tuple = exceptions_arg.downcast_ref::().is_some(); + let excs_str = if !is_list && !is_tuple { + Some(exceptions_arg.repr(vm)?.into()) + } else { + None + }; + let exceptions: Vec = exceptions_arg.try_to_value(vm).map_err(|_| { vm.new_type_error("second argument (exceptions) must be a sequence") })?; - // Validate non-empty if exceptions.is_empty() { return Err( vm.new_value_error("second argument (exceptions) must be a non-empty sequence") ); } - // Validate all items are BaseException instances let mut has_non_exception = false; for (i, exc) in exceptions.iter().enumerate() { if !exc.fast_isinstance(vm.ctx.exceptions.base_exception_type) { @@ -290,20 +324,14 @@ pub(super) mod types { "Item {i} of second argument (exceptions) is not an exception" ))); } - // Check if any exception is not an Exception subclass - // With dynamic ExceptionGroup (inherits from both BaseExceptionGroup and Exception), - // ExceptionGroup instances are automatically instances of Exception if !exc.fast_isinstance(vm.ctx.exceptions.exception_type) { has_non_exception = true; } } - // Get the dynamic ExceptionGroup type let exception_group_type = crate::exception_group::exception_group(); - // Determine the actual class to use let actual_cls = if cls.is(exception_group_type) { - // ExceptionGroup cannot contain BaseExceptions that are not Exception if has_non_exception { return Err( vm.new_type_error("Cannot nest BaseExceptions in an ExceptionGroup") @@ -311,14 +339,12 @@ pub(super) mod types { } cls } else if cls.is(vm.ctx.exceptions.base_exception_group) { - // Auto-convert to ExceptionGroup if all are Exception subclasses if !has_non_exception { exception_group_type.to_owned() } else { cls } } else { - // User-defined subclass if has_non_exception && cls.fast_issubclass(vm.ctx.exceptions.exception_type) { return Err(vm.new_type_error(format!( "Cannot nest BaseExceptions in '{}'", @@ -328,10 +354,23 @@ pub(super) mod types { cls }; - // Create the exception with (message, exceptions_tuple) as args - let exceptions_tuple = vm.ctx.new_tuple(exceptions); - let init_args = vec![message, exceptions_tuple.into()]; - PyBaseException::new(init_args, vm) + // Keep an exact tuple as-is so `.exceptions is original` for tuples. + let exceptions_tuple = if exceptions_arg.class().is(vm.ctx.types.tuple_type) { + exceptions_arg + .clone() + .downcast::() + .expect("exact tuple") + } else { + vm.ctx.new_tuple(exceptions) + }; + + let payload = Self { + base: PyBaseException::new(args.args.clone(), vm), + msg: message.into(), + excs: PyObjectRef::from(exceptions_tuple).into(), + excs_str: excs_str.into(), + }; + payload .into_ref_with_type_lazy_dict(vm, actual_cls) .map(Into::into) } @@ -345,18 +384,13 @@ pub(super) mod types { type Args = FuncArgs; fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - // BaseExceptionGroup_init: no kwargs allowed if !args.kwargs.is_empty() { return Err(vm.new_type_error(format!( "{} does not take keyword arguments", zelf.class().name() ))); } - // Do NOT call PyBaseException::slot_init here. - // slot_new already set args to (message, exceptions_tuple). - // Calling base init would overwrite with original args (message, exceptions_list). - let _ = (zelf, args, vm); - Ok(()) + PyBaseException::slot_init(zelf, args, vm) } fn init(_zelf: PyRef, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { @@ -370,13 +404,11 @@ pub(super) mod types { } fn get_exceptions_tuple( - exc: &Py, + exc: &Py, vm: &VirtualMachine, ) -> PyResult> { - let obj = exc - .get_arg(1) - .ok_or_else(|| vm.new_type_error("exceptions must be a tuple"))?; - let tuple = obj + let tuple = exc + .excs .downcast_ref::() .ok_or_else(|| vm.new_type_error("exceptions must be a tuple"))?; Ok(tuple.to_vec()) @@ -441,8 +473,8 @@ pub(super) mod types { } } - fn derive_and_copy_attributes( - orig: &Py, + pub(crate) fn derive_and_copy_attributes( + orig: &Py, excs: Vec, vm: &VirtualMachine, ) -> PyResult { @@ -456,17 +488,17 @@ pub(super) mod types { } // Copy traceback - if let Some(tb) = orig.__traceback__() { + if let Some(tb) = orig.base.__traceback__() { new_group.set_attr("__traceback__", tb, vm)?; } // Copy context - if let Some(ctx) = orig.__context__() { + if let Some(ctx) = orig.base.__context__() { new_group.set_attr("__context__", ctx, vm)?; } // Copy cause - if let Some(cause) = orig.__cause__() { + if let Some(cause) = orig.base.__cause__() { new_group.set_attr("__cause__", cause, vm)?; } diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index dc8f399e6..93c0ec8cd 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -398,7 +398,7 @@ impl VirtualMachine { let ctor = ExceptionCtor::try_from_object(self, exc_type)?; let exc = ctor.instantiate_value(exc_val, self)?; if let Some(tb) = Option::::try_from_object(self, exc_tb)? { - exc.set_traceback_typed(Some(tb)); + exc.set_traceback(Some(tb)); } Ok(exc) } @@ -654,7 +654,11 @@ impl PyBaseException { } #[pygetset(setter)] - fn set_args(&self, args: ArgIterable, vm: &VirtualMachine) -> PyResult<()> { + fn set_args(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let PySetterValue::Assign(value) = value else { + return Err(vm.new_type_error("args may not be deleted")); + }; + let args: ArgIterable = value.try_into_value(vm)?; let args = args.iter(vm)?.collect::>>()?; *self.args.write() = PyTuple::new_ref(args, &vm.ctx); Ok(()) @@ -666,7 +670,10 @@ impl PyBaseException { } #[pygetset(setter)] - pub fn set___traceback__(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + fn set___traceback__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let PySetterValue::Assign(value) = value else { + return Err(vm.new_type_error("__traceback__ may not be deleted")); + }; let traceback = if vm.is_none(&value) { None } else { @@ -677,12 +684,11 @@ impl PyBaseException { } } }; - self.set_traceback_typed(traceback); + self.set_traceback(traceback); Ok(()) } - // Helper method for internal use that doesn't require PyObjectRef - pub(crate) fn set_traceback_typed(&self, traceback: Option) { + pub fn set_traceback(&self, traceback: Option) { *self.traceback.write() = traceback; } @@ -692,7 +698,27 @@ impl PyBaseException { } #[pygetset(setter)] - pub fn set___cause__(&self, cause: Option>) { + fn set___cause__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let PySetterValue::Assign(value) = value else { + return Err(vm.new_type_error("__cause__ may not be deleted")); + }; + let cause = if vm.is_none(&value) { + None + } else { + match value.downcast::() { + Ok(exc) => Some(exc), + Err(_) => { + return Err(vm.new_type_error( + "exception cause must be None or derive from BaseException", + )); + } + } + }; + self.set_cause(cause); + Ok(()) + } + + pub fn set_cause(&self, cause: Option>) { let mut c = self.cause.write(); self.set_suppress_context(true); *c = cause; @@ -704,7 +730,27 @@ impl PyBaseException { } #[pygetset(setter)] - pub fn set___context__(&self, context: Option>) { + fn set___context__(&self, value: PySetterValue, vm: &VirtualMachine) -> PyResult<()> { + let PySetterValue::Assign(value) = value else { + return Err(vm.new_type_error("__context__ may not be deleted")); + }; + let context = if vm.is_none(&value) { + None + } else { + match value.downcast::() { + Ok(exc) => Some(exc), + Err(_) => { + return Err(vm.new_type_error( + "exception context must be None or derive from BaseException", + )); + } + } + }; + self.set_context(context); + Ok(()) + } + + pub fn set_context(&self, context: Option>) { *self.context.write() = context; } @@ -1006,10 +1052,7 @@ impl ExceptionZoo { PyBaseException::extend_class(ctx, excs.base_exception_type); // Sorted By Hierarchy then alphabetized. - extend_exception!(PyBaseExceptionGroup, ctx, excs.base_exception_group, { - "message" => ctx.new_readonly_getset("message", excs.base_exception_group, make_arg_getter(0)), - "exceptions" => ctx.new_readonly_getset("exceptions", excs.base_exception_group, make_arg_getter(1)), - }); + extend_exception!(PyBaseExceptionGroup, ctx, excs.base_exception_group); extend_exception!(PySystemExit, ctx, excs.system_exit); extend_exception!(PyKeyboardInterrupt, ctx, excs.keyboard_interrupt); @@ -1914,7 +1957,33 @@ pub(super) mod types { pub struct PyAttributeError(PyException); #[pyexception(with(Initializer))] - impl PyAttributeError {} + impl PyAttributeError { + #[pymethod] + fn __getstate__(zelf: PyBaseExceptionRef, vm: &VirtualMachine) -> PyResult { + let obj = zelf.as_object(); + let state = match obj.dict() { + Some(dict) => dict.copy().into_ref(&vm.ctx), + None => vm.ctx.new_dict(), + }; + let _ = state.del_item("obj", vm); + if let Ok(name) = obj.get_attr("name", vm) + && !vm.is_none(&name) + { + state.set_item("name", name, vm)?; + } + Ok(state.into()) + } + + #[pymethod] + fn __reduce__(zelf: PyBaseExceptionRef, vm: &VirtualMachine) -> PyResult { + let state = Self::__getstate__(zelf.clone(), vm)?; + Ok(vm.ctx.new_tuple(vec![ + zelf.class().to_owned().into(), + zelf.args().into(), + state, + ])) + } + } impl Initializer for PyAttributeError { type Args = FuncArgs; @@ -1996,9 +2065,10 @@ pub(super) mod types { // Check for any remaining invalid keyword arguments if let Some(invalid_key) = kwargs.keys().next() { - return Err(vm.new_type_error(format!( - "'{invalid_key}' is an invalid keyword argument for ImportError" - ))); + return Err(vm.new_unexpected_keyword_type_error( + Some("ImportError"), + &invalid_key.to_string(), + )); } if let Some(name) = name { @@ -2232,6 +2302,28 @@ pub(super) mod types { } fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if oserror_use_init(&cls) { + let payload = Self { + base: PyException(PyBaseException::new(vec![], vm)), + errno: None.into(), + strerror: None.into(), + filename: None.into(), + filename2: None.into(), + #[cfg(windows)] + winerror: None.into(), + written: AtomicCell::new(-1), + }; + return payload + .into_ref_with_type_lazy_dict(vm, cls) + .map(Into::into); + } + + if !args.kwargs.is_empty() { + return Err( + vm.new_type_error(format!("{}() takes no keyword arguments", cls.slot_name())) + ); + } + // We need this method, because of how `CPython` copies `init` // from `BaseException` in `SimpleExtendsException` macro. // See: `BaseException_new` @@ -2253,97 +2345,120 @@ pub(super) mod types { } } } - let payload = Self::py_new(&cls, args, vm)?; - payload - .into_ref_with_type_lazy_dict(vm, cls) - .map(Into::into) + let payload = Self::py_new(&cls, args.clone(), vm)?; + let obj = payload.into_ref_with_type_lazy_dict(vm, cls)?; + oserror_init(obj.as_object().to_owned(), args, vm)?; + Ok(obj.into()) } } + fn oserror_use_init(cls: &Py) -> bool { + let init = cls.slots.init.load(); + let new = cls.slots.new.load(); + let slot_init: fn(PyObjectRef, FuncArgs, &VirtualMachine) -> PyResult<()> = + PyOSError::slot_init; + let slot_new: fn(PyTypeRef, FuncArgs, &VirtualMachine) -> PyResult = PyOSError::slot_new; + !matches!(init, Some(f) if core::ptr::fn_addr_eq(f, slot_init)) + && matches!(new, Some(f) if core::ptr::fn_addr_eq(f, slot_new)) + } + + fn oserror_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + let len = args.args.len(); + let mut new_args = args; + + // All OSError subclasses use #[repr(transparent)] wrapping PyOSError, + // so we can safely access the PyOSError fields through pointer cast + // SAFETY: All OSError subclasses (FileNotFoundError, etc.) are + // #[repr(transparent)] wrappers around PyOSError with identical memory layout + #[allow(deprecated)] + let exc: &Py = zelf.downcast_ref::().unwrap(); + + // Check if this is BlockingIOError - need to handle characters_written + let is_blocking_io_error = zelf + .class() + .is(vm.ctx.exceptions.blocking_io_error.as_ref()); + + // SAFETY: slot_init is called during object initialization, + // so fields are None and swap result can be safely ignored + let mut set_filename = true; + if len <= 5 { + // Only set errno/strerror when args len is 2-5 + if 2 <= len { + let _ = unsafe { exc.errno.swap(Some(new_args.args[0].clone())) }; + let _ = unsafe { exc.strerror.swap(Some(new_args.args[1].clone())) }; + } + if 3 <= len { + let third_arg = &new_args.args[2]; + // BlockingIOError's 3rd argument can be the number of characters written + if is_blocking_io_error + && !vm.is_none(third_arg) + && crate::protocol::PyNumber::check(third_arg) + && let Ok(written) = third_arg.try_index(vm) + && let Ok(n) = written.try_to_primitive::(vm) + { + exc.written.store(n); + set_filename = false; + // The count leaves neither filename taken, so both are + // put back the way `py_new` found them. + let _ = unsafe { exc.filename.swap(None) }; + let _ = unsafe { exc.filename2.swap(None) }; + } + if set_filename { + let _ = unsafe { exc.filename.swap(Some(third_arg.clone())) }; + } + } + #[cfg(windows)] + if 4 <= len { + let winerror = new_args.args.get(3).cloned(); + // Store original winerror + let _ = unsafe { exc.winerror.swap(winerror.clone()) }; + + // Convert winerror to errno and update errno + args[0] + if let Some(errno) = winerror + .as_ref() + .and_then(|w| w.downcast_ref::()) + .and_then(|w| w.try_to_primitive::(vm).ok()) + .map(crate::host_env::os::winerror_to_errno) + { + let errno_obj = vm.new_pyobj(errno); + let _ = unsafe { exc.errno.swap(Some(errno_obj.clone())) }; + new_args.args[0] = errno_obj; + } + } + } + + // A second filename, and the two arguments the rest are cut back + // to, both follow the filename itself having been taken. + let has_filename = exc + .filename + .to_owned() + .as_ref() + .is_some_and(|f| !vm.is_none(f)); + if (3..=5).contains(&len) && has_filename { + if let Some(filename2) = new_args.args.get(4) + && !vm.is_none(filename2) + { + let _ = unsafe { exc.filename2.swap(Some(filename2.clone())) }; + } + new_args.args.truncate(2); + } + PyBaseException::slot_init(zelf, new_args, vm) + } + impl Initializer for PyOSError { type Args = FuncArgs; fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { - let len = args.args.len(); - let mut new_args = args; - - // All OSError subclasses use #[repr(transparent)] wrapping PyOSError, - // so we can safely access the PyOSError fields through pointer cast - // SAFETY: All OSError subclasses (FileNotFoundError, etc.) are - // #[repr(transparent)] wrappers around PyOSError with identical memory layout - #[allow(deprecated)] - let exc: &Py = zelf.downcast_ref::().unwrap(); - - // Check if this is BlockingIOError - need to handle characters_written - let is_blocking_io_error = - zelf.class() - .is(vm.ctx.exceptions.blocking_io_error.as_ref()); - - // SAFETY: slot_init is called during object initialization, - // so fields are None and swap result can be safely ignored - let mut set_filename = true; - if len <= 5 { - // Only set errno/strerror when args len is 2-5 - if 2 <= len { - let _ = unsafe { exc.errno.swap(Some(new_args.args[0].clone())) }; - let _ = unsafe { exc.strerror.swap(Some(new_args.args[1].clone())) }; - } - if 3 <= len { - let third_arg = &new_args.args[2]; - // BlockingIOError's 3rd argument can be the number of characters written - if is_blocking_io_error - && !vm.is_none(third_arg) - && crate::protocol::PyNumber::check(third_arg) - && let Ok(written) = third_arg.try_index(vm) - && let Ok(n) = written.try_to_primitive::(vm) - { - exc.written.store(n); - set_filename = false; - // The count leaves neither filename taken, so both are - // put back the way `py_new` found them. - let _ = unsafe { exc.filename.swap(None) }; - let _ = unsafe { exc.filename2.swap(None) }; - } - if set_filename { - let _ = unsafe { exc.filename.swap(Some(third_arg.clone())) }; - } - } - #[cfg(windows)] - if 4 <= len { - let winerror = new_args.args.get(3).cloned(); - // Store original winerror - let _ = unsafe { exc.winerror.swap(winerror.clone()) }; - - // Convert winerror to errno and update errno + args[0] - if let Some(errno) = winerror - .as_ref() - .and_then(|w| w.downcast_ref::()) - .and_then(|w| w.try_to_primitive::(vm).ok()) - .map(crate::host_env::os::winerror_to_errno) - { - let errno_obj = vm.new_pyobj(errno); - let _ = unsafe { exc.errno.swap(Some(errno_obj.clone())) }; - new_args.args[0] = errno_obj; - } - } + if !oserror_use_init(zelf.class()) { + return Ok(()); } - - // A second filename, and the two arguments the rest are cut back - // to, both follow the filename itself having been taken. - let has_filename = exc - .filename - .to_owned() - .as_ref() - .is_some_and(|f| !vm.is_none(f)); - if (3..=5).contains(&len) && has_filename { - if let Some(filename2) = new_args.args.get(4) - && !vm.is_none(filename2) - { - let _ = unsafe { exc.filename2.swap(Some(filename2.clone())) }; - } - new_args.args.truncate(2); + if !args.kwargs.is_empty() { + return Err(vm.new_type_error(format!( + "{}() takes no keyword arguments", + zelf.class().slot_name() + ))); } - PyBaseException::slot_init(zelf, new_args, vm) + oserror_init(zelf, args, vm) } fn init(_zelf: PyRef, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> { @@ -2564,12 +2679,11 @@ pub(super) mod types { #[pygetset(setter)] fn set_characters_written( &self, - value: Option, + value: PySetterValue, vm: &VirtualMachine, ) -> PyResult<()> { match value { - None => { - // Deleting the attribute + PySetterValue::Delete => { if self.written.load() == -1 { Err(vm.new_attribute_error("characters_written")) } else { @@ -2577,7 +2691,7 @@ pub(super) mod types { Ok(()) } } - Some(v) => { + PySetterValue::Assign(v) => { let n = v .try_index(vm)? .try_to_primitive::(vm) @@ -2818,11 +2932,12 @@ pub(super) mod types { .iter() .enumerate() { - if location_tup_len > i { - zelf.set_attr(attr, location_tuple[i].to_owned(), vm)?; + let value = if location_tup_len > i { + location_tuple[i].to_owned() } else { - break; - } + vm.ctx.none() + }; + zelf.set_attr(attr, value, vm)?; } } @@ -2882,10 +2997,92 @@ pub(super) mod types { ))) } - #[pyexception(name, base = PyValueError, ctx = "unicode_error", impl)] - #[derive(Debug)] - #[repr(transparent)] - pub struct PyUnicodeError(PyValueError); + fn unicode_error_attr_str( + exc: &Py, + name: &'static str, + vm: &VirtualMachine, + ) -> PyResult { + exc.as_object().get_attr(name, vm)?.str(vm) + } + + fn unicode_error_attr_isize( + exc: &Py, + name: &'static str, + vm: &VirtualMachine, + ) -> PyResult { + let value = exc.as_object().get_attr(name, vm)?; + match value.try_index(vm) { + Ok(n) => Ok(n.try_to_primitive::(vm).unwrap_or(0)), + Err(_) => Ok(0), + } + } + + fn unicode_error_require_object( + exc: &Py, + vm: &VirtualMachine, + ) -> PyResult { + match exc.as_object().get_attr("object", vm) { + Ok(obj) if !vm.is_none(&obj) => Ok(obj), + _ => Err(vm.new_type_error("UnicodeError 'object' attribute is not set")), + } + } + + #[pyexception(name, base = PyValueError, ctx = "unicode_error", traverse = "manual")] + #[repr(C)] + pub struct PyUnicodeError { + base: PyValueError, + object: PyAtomicRef>, + } + + impl crate::class::PySubclass for PyUnicodeError { + type Base = PyValueError; + fn as_base(&self) -> &Self::Base { + &self.base + } + } + + impl core::fmt::Debug for PyUnicodeError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PyUnicodeError").finish_non_exhaustive() + } + } + + unsafe impl Traverse for PyUnicodeError { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.base.0.0.traverse(tracer_fn); + if let Some(obj) = self.object.deref() { + tracer_fn(obj); + } + } + } + + #[pyexception(with(Constructor))] + impl PyUnicodeError { + #[pygetset] + fn object(&self) -> Option { + self.object.to_owned() + } + + #[pygetset(setter)] + fn set_object(&self, value: PySetterValue, vm: &VirtualMachine) { + let object = match value { + PySetterValue::Assign(v) => Some(v), + PySetterValue::Delete => None, + }; + self.object.swap_to_temporary_refs(object, vm); + } + } + + impl Constructor for PyUnicodeError { + type Args = FuncArgs; + + fn py_new(_cls: &Py, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Ok(Self { + base: PyValueError(PyException(PyBaseException::new(args.args, vm))), + object: None.into(), + }) + } + } #[pyexception(name, base = PyUnicodeError, ctx = "unicode_decode_error")] #[derive(Debug)] @@ -2896,31 +3093,35 @@ pub(super) mod types { impl PyUnicodeDecodeError { #[pymethod] fn __str__(zelf: &Py, vm: &VirtualMachine) -> PyResult { - let Ok(object) = zelf.as_object().get_attr("object", vm) else { - return Ok(vm.ctx.empty_str.to_owned()); - }; - let object: ArgBytesLike = object.try_into_value(vm)?; - let encoding: PyStrRef = zelf - .as_object() - .get_attr("encoding", vm)? - .try_into_value(vm)?; - let start: usize = zelf.as_object().get_attr("start", vm)?.try_into_value(vm)?; - let end: usize = zelf.as_object().get_attr("end", vm)?.try_into_value(vm)?; - let reason: PyStrRef = zelf - .as_object() - .get_attr("reason", vm)? - .try_into_value(vm)?; - Ok(vm.ctx.new_str(if start < object.len() && end <= object.len() && end == start + 1 { - let b = object.borrow_buf()[start]; - format!( - "'{encoding}' codec can't decode byte {b:#04x} in position {start}: {reason}" - ) - } else { - format!( - "'{encoding}' codec can't decode bytes in position {start}-{}: {reason}", - end - 1, - ) - })) + match zelf.as_object().get_attr("object", vm) { + Ok(obj) if !vm.is_none(&obj) => {} + _ => return Ok(vm.ctx.empty_str.to_owned()), + } + let encoding = unicode_error_attr_str(zelf, "encoding", vm)?; + let reason = unicode_error_attr_str(zelf, "reason", vm)?; + let object: ArgBytesLike = + unicode_error_require_object(zelf, vm)?.try_into_value(vm)?; + let start = unicode_error_attr_isize(zelf, "start", vm)?; + let end = unicode_error_attr_isize(zelf, "end", vm)?; + let start_u = usize::try_from(start).ok(); + let end_u = usize::try_from(end).ok(); + Ok(vm.ctx.new_str( + if let (Some(start_u), Some(end_u)) = (start_u, end_u) + && start_u < object.len() + && end_u <= object.len() + && end_u == start_u + 1 + { + let b = object.borrow_buf()[start_u]; + format!( + "'{encoding}' codec can't decode byte {b:#04x} in position {start}: {reason}" + ) + } else { + format!( + "'{encoding}' codec can't decode bytes in position {start}-{}: {reason}", + end - 1, + ) + }, + )) } } @@ -2955,32 +3156,35 @@ pub(super) mod types { impl PyUnicodeEncodeError { #[pymethod] fn __str__(zelf: &Py, vm: &VirtualMachine) -> PyResult { - let Ok(object) = zelf.as_object().get_attr("object", vm) else { - return Ok(vm.ctx.empty_str.to_owned()); - }; - let object: PyStrRef = object.try_into_value(vm)?; - let encoding: PyStrRef = zelf - .as_object() - .get_attr("encoding", vm)? - .try_into_value(vm)?; - let start: usize = zelf.as_object().get_attr("start", vm)?.try_into_value(vm)?; - let end: usize = zelf.as_object().get_attr("end", vm)?.try_into_value(vm)?; - let reason: PyStrRef = zelf - .as_object() - .get_attr("reason", vm)? - .try_into_value(vm)?; - Ok(vm.ctx.new_str(if start < object.char_len() && end <= object.char_len() && end == start + 1 { - let ch = object.as_wtf8().code_points().nth(start).unwrap(); - format!( - "'{encoding}' codec can't encode character '{}' in position {start}: {reason}", - UnicodeEscapeCodepoint(ch) - ) - } else { - format!( - "'{encoding}' codec can't encode characters in position {start}-{}: {reason}", - end - 1, - ) - })) + match zelf.as_object().get_attr("object", vm) { + Ok(obj) if !vm.is_none(&obj) => {} + _ => return Ok(vm.ctx.empty_str.to_owned()), + } + let encoding = unicode_error_attr_str(zelf, "encoding", vm)?; + let reason = unicode_error_attr_str(zelf, "reason", vm)?; + let object: PyStrRef = unicode_error_require_object(zelf, vm)?.try_into_value(vm)?; + let start = unicode_error_attr_isize(zelf, "start", vm)?; + let end = unicode_error_attr_isize(zelf, "end", vm)?; + let start_u = usize::try_from(start).ok(); + let end_u = usize::try_from(end).ok(); + Ok(vm.ctx.new_str( + if let (Some(start_u), Some(end_u)) = (start_u, end_u) + && start_u < object.char_len() + && end_u <= object.char_len() + && end_u == start_u + 1 + { + let ch = object.as_wtf8().code_points().nth(start_u).unwrap(); + format!( + "'{encoding}' codec can't encode character '{}' in position {start}: {reason}", + UnicodeEscapeCodepoint(ch) + ) + } else { + format!( + "'{encoding}' codec can't encode characters in position {start}-{}: {reason}", + end - 1, + ) + }, + )) } } @@ -3015,19 +3219,23 @@ pub(super) mod types { impl PyUnicodeTranslateError { #[pymethod] fn __str__(zelf: &Py, vm: &VirtualMachine) -> PyResult { - let Ok(object) = zelf.as_object().get_attr("object", vm) else { - return Ok(vm.ctx.empty_str.to_owned()); - }; - let object: PyStrRef = object.try_into_value(vm)?; - let start: usize = zelf.as_object().get_attr("start", vm)?.try_into_value(vm)?; - let end: usize = zelf.as_object().get_attr("end", vm)?.try_into_value(vm)?; - let reason: PyStrRef = zelf - .as_object() - .get_attr("reason", vm)? - .try_into_value(vm)?; + match zelf.as_object().get_attr("object", vm) { + Ok(obj) if !vm.is_none(&obj) => {} + _ => return Ok(vm.ctx.empty_str.to_owned()), + } + let reason = unicode_error_attr_str(zelf, "reason", vm)?; + let object: PyStrRef = unicode_error_require_object(zelf, vm)?.try_into_value(vm)?; + let start = unicode_error_attr_isize(zelf, "start", vm)?; + let end = unicode_error_attr_isize(zelf, "end", vm)?; + let start_u = usize::try_from(start).ok(); + let end_u = usize::try_from(end).ok(); Ok(vm.ctx.new_str( - if start < object.char_len() && end <= object.char_len() && end == start + 1 { - let ch = object.as_wtf8().code_points().nth(start).unwrap(); + if let (Some(start_u), Some(end_u)) = (start_u, end_u) + && start_u < object.char_len() + && end_u <= object.char_len() + && end_u == start_u + 1 + { + let ch = object.as_wtf8().code_points().nth(start_u).unwrap(); format!( "can't translate character '{}' in position {start}: {reason}", UnicodeEscapeCodepoint(ch) @@ -3198,7 +3406,7 @@ pub fn exception_group_match( && let Some(tb) = exc.__traceback__() && let Ok(wrapped_exc) = wrapped.clone().downcast::() { - let _ = wrapped_exc.set___traceback__(tb.into(), vm); + wrapped_exc.set_traceback(Some(tb)); } wrapped }; @@ -3404,7 +3612,8 @@ fn split_by_leaf_ids( return Ok(vm.ctx.none()); } - // Reconstruct using derive() to preserve the structure (not necessarily the subclass type) - let matched_tuple = vm.ctx.new_tuple(matched); - vm.call_method(exc, "derive", (matched_tuple,)) + let group = exc + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("expected a BaseExceptionGroup"))?; + crate::exception_group::types::derive_and_copy_attributes(group, matched, vm) } diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 00830737b..cfe747208 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -41,7 +41,7 @@ use core::sync::atomic; use core::sync::atomic::Ordering::{Acquire, Relaxed}; use itertools::Itertools; use malachite_bigint::BigInt; -use num_traits::Zero; +use num_traits::{ToPrimitive, Zero}; use rustpython_common::atomic::{PyAtomic, Radium}; use rustpython_common::{ lock::{OnceCell, PyMutex}, @@ -206,7 +206,11 @@ enum UnwindReason { /// We hit an exception, so unwind any try-except and finally blocks. The exception should be /// on top of the vm exception stack. - Raising { exception: PyBaseExceptionRef }, + Raising { + exception: PyBaseExceptionRef, + /// Instruction that raised; used for exception-table lookup. + offset: u32, + }, } /// Tracks who owns a frame. @@ -1435,7 +1439,7 @@ impl InterpreterFrame { } /// Synchronize `prev_line` to the line of the instruction currently - /// in flight (derived from `lasti`, same as `PyFrame::f_lineno`). + /// in flight (derived from `lasti`, same as `FrameObject::lineno`). /// /// `prev_line` is normally only updated on the cold 'line'-trace-event /// path (see the dispatch loop in `ExecutingFrame::run`), so it can go @@ -2663,13 +2667,14 @@ pub(crate) fn trampoline_handle_exception( if let Some((loc, _end_loc)) = exec.code.locations.get(idx) { let next = exception.__traceback__(); let new_traceback = PyTraceback::new(next, exec.frame_object(vm), idx as u32 * 2, loc.line); - exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + exception.set_traceback(Some(new_traceback.into_ref(&vm.ctx))); } exec.unwind_blocks( vm, UnwindReason::Raising { exception: exception.clone(), + offset: idx as u32, }, ) } @@ -3509,7 +3514,7 @@ impl ExecutingFrame<'_> { let mut do_extend_arg = false; // f_lineno for a live (currently executing) frame is derived - // lazily from lasti/locations (see `PyFrame::f_lineno`) rather + // lazily from lasti/locations (see `FrameObject::lineno`) rather // than maintained here on every instruction. lasti already // points past the instruction currently executing (see the // `self.lasti.store` above), so `locations[lasti - 1]` gives @@ -3566,10 +3571,16 @@ impl ExecutingFrame<'_> { idx as u32 * 2, loc.line, ); - exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + exception.set_traceback(Some(new_traceback.into_ref(&vm.ctx))); } vm.contextualize_exception(&exception); - frame.unwind_blocks(vm, UnwindReason::Raising { exception }) + frame.unwind_blocks( + vm, + UnwindReason::Raising { + exception, + offset: idx as u32, + }, + ) } match handle_signal_exception(self, exception, idx, vm) { Ok(None) => {} @@ -3656,8 +3667,7 @@ impl ExecutingFrame<'_> { new_traceback, loc.line ); - exception - .set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + exception.set_traceback(Some(new_traceback.into_ref(&vm.ctx))); } // _PyErr_SetObject sets __context__ only when the exception @@ -3671,7 +3681,13 @@ impl ExecutingFrame<'_> { } // Use exception table for zero-cost exception handling - frame.unwind_blocks(vm, UnwindReason::Raising { exception }) + frame.unwind_blocks( + vm, + UnwindReason::Raising { + exception, + offset: idx as u32, + }, + ) } // Check if this is a RERAISE instruction @@ -3756,18 +3772,6 @@ impl ExecutingFrame<'_> { exception }; - // Restore lasti from traceback so frame.f_lineno matches tb_lineno - // The traceback was created with the correct lasti when exception - // was first raised, but frame.lasti may have changed during cleanup - if let Some(tb) = exception.__traceback__() - && self.iframe().frame_obj().is_some_and(|fo| { - core::ptr::eq::>(&*tb.frame, fo) - }) - { - // This traceback entry is for this frame - restore its lasti - // tb.lasti is in bytes (idx * 2), convert back to instruction index - self.update_lasti(|i| *i = tb.lasti / 2); - } break Err(exception); } } @@ -3865,12 +3869,18 @@ impl ExecutingFrame<'_> { let next = err.__traceback__(); let new_traceback = PyTraceback::new(next, self.frame_object(vm), idx as u32 * 2, loc.line); - err.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + err.set_traceback(Some(new_traceback.into_ref(&vm.ctx))); } self.push_value(vm.ctx.none()); vm.chain_stack_item(&err); - return match self.unwind_blocks(vm, UnwindReason::Raising { exception: err }) { + return match self.unwind_blocks( + vm, + UnwindReason::Raising { + exception: err, + offset: idx as u32, + }, + ) { Ok(None) => { self.prev_line.set(0); self.run(vm) @@ -3910,12 +3920,18 @@ impl ExecutingFrame<'_> { idx as u32 * 2, loc.line, ); - err.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + err.set_traceback(Some(new_traceback.into_ref(&vm.ctx))); } self.push_value(vm.ctx.none()); vm.chain_stack_item(&err); - match self.unwind_blocks(vm, UnwindReason::Raising { exception: err }) { + match self.unwind_blocks( + vm, + UnwindReason::Raising { + exception: err, + offset: idx as u32, + }, + ) { Ok(None) => { self.prev_line.set(0); self.run(vm) @@ -3935,7 +3951,7 @@ impl ExecutingFrame<'_> { let exception = match ctor.instantiate_value(exc_val, vm) { Ok(exc) => { if let Some(tb) = Option::>::try_from_object(vm, exc_tb)? { - exc.set_traceback_typed(Some(tb)); + exc.set_traceback(Some(tb)); } exc } @@ -3949,7 +3965,7 @@ impl ExecutingFrame<'_> { let next = exception.__traceback__(); let new_traceback = PyTraceback::new(next, self.frame_object(vm), idx as u32 * 2, loc.line); - exception.set_traceback_typed(Some(new_traceback.into_ref(&vm.ctx))); + exception.set_traceback(Some(new_traceback.into_ref(&vm.ctx))); } // Fire PY_THROW and RAISE events before raising the exception. @@ -3986,7 +4002,13 @@ impl ExecutingFrame<'_> { // This is needed for exception handler to have correct stack state self.push_value(vm.ctx.none()); - match self.unwind_blocks(vm, UnwindReason::Raising { exception }) { + match self.unwind_blocks( + vm, + UnwindReason::Raising { + exception, + offset: idx as u32, + }, + ) { Ok(None) => { // Reset prev_line so that the first instruction in the handler // fires a LINE event. In CPython, gen_send_ex re-enters the @@ -4573,7 +4595,7 @@ impl ExecutingFrame<'_> { "'async for' received an invalid object from __anext__: {:.200}", next_iter.class().name() )); - err.set___cause__(Some(e)); + err.set_cause(Some(e)); err })? }; @@ -5445,19 +5467,16 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(result).into()); Ok(None) } - Instruction::Reraise { depth: _ } => { + Instruction::Reraise { depth } => { // inst(RERAISE, (values[oparg], exc -- values[oparg])) // - // RERAISE pops only `exc` from TOS. The `values` below it - // (lasti and optional prev_exc) stay on the stack — the - // outer exception handler's exception-table unwind will - // pop them down to its configured stack depth. - // - // `oparg` encodes how many values are preserved below exc - // (1 for simple reraise, 2 for with-block reraise where - // values[0]=lasti). Runtime-wise we don't need oparg since - // the exception table handles stack layout. + // Pops only `exc`. When oparg != 0, values[0] is the lasti + // pushed by the exception table; restore it before unwind so + // a later handler that also pushes lasti records the original + // raise. + let oparg = depth.get(arg); let exc = self.pop_value(); + self.restore_reraise_lasti(oparg); if let Some(exc_ref) = exc.downcast_ref::() { Err(exc_ref.to_owned()) @@ -8232,14 +8251,10 @@ impl ExecutingFrame<'_> { #[cfg_attr(feature = "flame-it", flame("FrameObject"))] fn import(&mut self, vm: &VirtualMachine, module_name: Option<&Py>) -> PyResult<()> { let module_name = module_name.unwrap_or(vm.ctx.empty_str); - let top = self.pop_value(); - let from_list = match >::try_from_object(vm, top)? { - Some(from_list) => from_list.try_into_typed::(vm)?, - None => vm.ctx.empty_tuple_typed().to_owned(), - }; + let from_list = self.pop_value(); let level = usize::try_from_object(vm, self.pop_value())?; - let module = vm.import_from(module_name, &from_list, level)?; + let module = vm.import_from(module_name, from_list, level)?; self.push_value(module); Ok(()) @@ -8410,17 +8425,9 @@ impl ExecutingFrame<'_> { fn unwind_blocks(&mut self, vm: &VirtualMachine, reason: UnwindReason) -> FrameResult { // use exception table for exception handling match reason { - UnwindReason::Raising { exception } => { - // Look up handler in exception table - // lasti points to NEXT instruction (already incremented in run loop) - // The exception occurred at the previous instruction - // Python uses signed int where INSTR_OFFSET() - 1 = -1 before first instruction. - // We use u32, so check for 0 explicitly. - if self.lasti() == 0 { - // No instruction executed yet, no handler can match - return Err(exception); - } - let offset = self.lasti() - 1; + UnwindReason::Raising { exception, offset } => { + // Look up handler from the raising instruction. lasti may + // already have been restored by RERAISE to the original raise. if let Some(entry) = bytecode::find_exception_handler(&self.code.exceptiontable, offset) { @@ -8439,10 +8446,11 @@ impl ExecutingFrame<'_> { let _ = self.localsplus.stack_pop(); } - // 2. If push_lasti=true (SETUP_CLEANUP), push lasti before exception - // pushes lasti as PyLong + // 2. If push_lasti=true, push the current lasti (which + // RERAISE may have restored to the original raise). if entry.push_lasti { - self.push_value(vm.ctx.new_int(offset as i32).into()); + let lasti = self.lasti().saturating_sub(1); + self.push_value(vm.ctx.new_int(lasti as i32).into()); } // 3. Push exception onto stack @@ -8464,6 +8472,31 @@ impl ExecutingFrame<'_> { } } + /// Restore lasti from the exception-table value still on the stack. + /// + /// `oparg` is RERAISE's operand: values[0] is lasti when oparg != 0. + /// lasti is stored as the next instruction index, so the saved index is + /// incremented by one. + fn restore_reraise_lasti(&mut self, oparg: u32) { + if oparg == 0 { + return; + } + let stack_len = self.localsplus.stack_len(); + if stack_len < oparg as usize { + return; + } + let Some(lasti_ref) = self.localsplus.stack_index(stack_len - oparg as usize) else { + return; + }; + let Some(int) = lasti_ref.as_object().downcast_ref::() else { + return; + }; + let Some(idx) = int.as_bigint().to_u32() else { + return; + }; + self.lasti.store(idx.saturating_add(1), Relaxed); + } + fn execute_store_subscript(&mut self, vm: &VirtualMachine) -> FrameResult { let idx = self.pop_stackref(); let obj = self.pop_stackref(); @@ -8888,7 +8921,7 @@ impl ExecutingFrame<'_> { #[cfg(debug_assertions)] debug!("Exception raised: {exception:?} with cause: {cause:?}"); if let Some(cause) = cause { - exception.set___cause__(cause); + exception.set_cause(cause); } Err(exception) } @@ -12086,8 +12119,8 @@ impl ExecutingFrame<'_> { // see in tracebacks (suppress_context becomes true), but // assertions that inspect __context__ also expect it set. let cause: Option = arg.downcast().ok(); - err.set___context__(cause.clone()); - err.set___cause__(cause); + err.set_context(cause.clone()); + err.set_cause(cause); Ok(err.into()) } else { // Not StopIteration, pass through for RERAISE diff --git a/crates/vm/src/import.rs b/crates/vm/src/import.rs index 6a89dc998..89ff2c98d 100644 --- a/crates/vm/src/import.rs +++ b/crates/vm/src/import.rs @@ -269,7 +269,7 @@ pub fn remove_importlib_frames(vm: &VirtualMachine, exc: &Py) { if let Some(tb) = exc.__traceback__() { let trimmed_tb = remove_importlib_frames_inner(vm, Some(tb), always_trim).0; - exc.set_traceback_typed(trimmed_tb); + exc.set_traceback(trimmed_tb); } } diff --git a/crates/vm/src/stdlib/_codecs.rs b/crates/vm/src/stdlib/_codecs.rs index ac45f20d7..a40f55590 100644 --- a/crates/vm/src/stdlib/_codecs.rs +++ b/crates/vm/src/stdlib/_codecs.rs @@ -1019,7 +1019,7 @@ mod _codecs_windows { _ => break, } } - return Err(vm.new_unicode_encode_error_real( + return Err(vm.new_unicode_encode_error( encoding_str, s.clone(), fail_pos, @@ -1070,7 +1070,7 @@ mod _codecs_windows { } // Character can't be encoded - call error handler - let exc = vm.new_unicode_encode_error_real( + let exc = vm.new_unicode_encode_error( encoding_str.clone(), s.clone(), pos, @@ -1097,7 +1097,7 @@ mod _codecs_windows { for rcp in rep_str.as_wtf8().code_points() { let rch = rcp.to_u32(); if rch > 127 { - return Err(vm.new_unicode_encode_error_real( + return Err(vm.new_unicode_encode_error( encoding_str, s.clone(), pos, diff --git a/crates/vm/src/stdlib/_interpreters.rs b/crates/vm/src/stdlib/_interpreters.rs index 02276c363..db8c4e29e 100644 --- a/crates/vm/src/stdlib/_interpreters.rs +++ b/crates/vm/src/stdlib/_interpreters.rs @@ -446,7 +446,7 @@ pub(crate) mod _interpreters { crate::Interpreter::create_subinterpreter_from_vm(vm, config).map_err(|msg| { let cause = vm.new_runtime_error(msg.to_owned()); let exc = interpreter_error(vm, "interpreter creation failed"); - exc.set___context__(Some(cause)); + exc.set_context(Some(cause)); exc })?; if reqrefs { diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs index 4854f268d..ab774939f 100644 --- a/crates/vm/src/stdlib/_io.rs +++ b/crates/vm/src/stdlib/_io.rs @@ -1401,7 +1401,7 @@ mod _io { "raw readinto() returned invalid length {n} (should have been between 0 and {len})" )); if let Some(cause) = type_error { - os_error.set___cause__(Some(cause)); + os_error.set_cause(Some(cause)); } return Err(os_error); } @@ -1976,7 +1976,7 @@ mod _io { fn exception_chain(e1: PyResult<()>, e2: PyResult) -> PyResult { match (e1, e2) { (Err(e1), Err(e)) => { - e.set___context__(Some(e1)); + e.set_context(Some(e1)); Err(e) } (Err(e), Ok(_)) | (Ok(()), Err(e)) => Err(e), @@ -5987,7 +5987,7 @@ mod fileio { }; match (flush_exc, close_err) { (Some(fe), Some(ce)) => { - ce.set___context__(Some(fe)); + ce.set_context(Some(fe)); Err(ce) } (Some(e), None) | (None, Some(e)) => Err(e), @@ -6441,7 +6441,7 @@ mod winconsoleio { }; match (flush_exc, close_err) { (Some(fe), Some(ce)) => { - ce.set___context__(Some(fe)); + ce.set_context(Some(fe)); Err(ce) } (Some(e), None) | (None, Some(e)) => Err(e), diff --git a/crates/vm/src/stdlib/_testinternalcapi.rs b/crates/vm/src/stdlib/_testinternalcapi.rs index 31b65521c..8fe01ff85 100644 --- a/crates/vm/src/stdlib/_testinternalcapi.rs +++ b/crates/vm/src/stdlib/_testinternalcapi.rs @@ -368,8 +368,8 @@ mod _testinternalcapi { } #[pyfunction] - fn iframe_getline(frame: PyRef) -> usize { - frame.f_lineno() + fn iframe_getline(frame: PyRef) -> i32 { + frame.lineno() } #[pyfunction] @@ -1616,7 +1616,7 @@ fn run_string_in_new_subinterp( let cause = vm.new_runtime_error(msg.to_owned()); let exc = crate::stdlib::_interpreters::interpreter_error(vm, "sub-interpreter creation failed"); - exc.set___context__(Some(cause)); + exc.set_context(Some(cause)); exc })?; let id = crate::vm::runtime::store_owned_interpreter(interp); diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index c0d6b7882..69cfc3e39 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -204,6 +204,104 @@ impl VirtualMachine { name == "utf-8" } + /// Load the source line for a compiler SyntaxError. Tokenizer errors keep + /// the in-memory line; compiler errors read the file named by the + /// exception, and get None when that file cannot be opened. + #[cfg(any(feature = "parser", feature = "compiler"))] + pub(crate) fn program_text(&self, filename: &str, lineno: usize) -> Option { + if lineno == 0 { + return None; + } + #[cfg(feature = "host_env")] + { + let buf = crate::host_env::fs::read(filename).ok()?; + let decoded = { + #[cfg(feature = "parser")] + { + let encoding = Self::detect_source_encoding(&buf); + if encoding.as_deref().is_none_or(Self::is_utf8_encoding) { + String::from_utf8_lossy(&buf).into_owned() + } else if encoding.as_deref() == Some("iso-8859-1") { + buf.iter().copied().map(char::from).collect() + } else { + let name = encoding.as_deref()?; + let bytes = self.ctx.new_bytes(buf); + self.state + .codec_registry + .decode_text(bytes.into(), name, None, self) + .ok()? + .to_string_lossy() + .into_owned() + } + } + #[cfg(not(feature = "parser"))] + { + String::from_utf8_lossy(&buf).into_owned() + } + }; + let mut remaining = decoded.as_str(); + if remaining.starts_with('\u{feff}') { + remaining = &remaining['\u{feff}'.len_utf8()..]; + } + remaining + .split_inclusive('\n') + .nth(lineno.checked_sub(1)?) + .map(str::to_owned) + } + #[cfg(not(feature = "host_env"))] + { + let _ = filename; + None + } + } + + #[cfg(feature = "parser")] + fn new_non_utf8_syntax_error( + &self, + filename: &str, + src: &[u8], + error_at: usize, + ) -> PyBaseExceptionRef { + let bad_byte = src[error_at]; + let line_start = src[..error_at] + .iter() + .rposition(|&b| b == b'\n') + .map_or(0, |i| i + 1); + let lineno = src[..error_at].iter().filter(|&&b| b == b'\n').count() + 1; + let offset = + core::str::from_utf8(&src[line_start..error_at]).map_or(1, |s| s.chars().count() + 1); + let line_end = src[line_start..] + .iter() + .position(|&b| b == b'\n') + .map_or(src.len(), |i| line_start + i); + let text = String::from_utf8_lossy(&src[line_start..line_end]).into_owned(); + let in_file = if filename.starts_with('<') { + String::new() + } else { + format!(" in file {filename}") + }; + let msg = format!( + "Non-UTF-8 code starting with '\\x{bad_byte:02x}'{in_file} \ + on line {lineno}, but no encoding declared; \ + see https://peps.python.org/pep-0263/ for details" + ); + let location = self.ctx.new_tuple(vec![ + self.ctx.new_str(filename).into(), + self.ctx.new_int(lineno).into(), + self.ctx.new_int(offset).into(), + self.ctx.new_str(text).into(), + self.ctx.new_int(lineno).into(), + self.ctx.new_int(offset).into(), + ]); + self.invoke_exception( + self.ctx.exceptions.syntax_error, + vec![self.ctx.new_str(msg.as_str()).into(), location.into()], + ) + .unwrap_or_else(|_| { + self.new_exception_msg(self.ctx.exceptions.syntax_error.to_owned(), msg.into()) + }) + } + #[cfg(feature = "parser")] pub(crate) fn decode_source_bytes( &self, @@ -223,34 +321,37 @@ impl VirtualMachine { let is_utf8 = encoding.as_deref().is_none_or(Self::is_utf8_encoding); if has_bom && !is_utf8 { let enc = encoding.as_deref().unwrap_or("utf-8"); - return Err(self.new_exception_msg( - self.ctx.exceptions.syntax_error.to_owned(), - format!("encoding problem: {enc} with BOM").into(), - )); + let after_bom = &source[3..]; + let line_end = after_bom + .iter() + .position(|&b| b == b'\n') + .unwrap_or(after_bom.len()); + let text = String::from_utf8_lossy(&after_bom[..line_end]).into_owned(); + let end_offset = text.chars().count() as i32; + let msg = format!("encoding problem: {enc} with BOM"); + let location = self.ctx.new_tuple(vec![ + self.ctx.new_str(filename).into(), + self.ctx.new_int(1).into(), + self.ctx.new_int(0).into(), + self.ctx.new_str(text).into(), + self.ctx.new_int(1).into(), + self.ctx.new_int(end_offset).into(), + ]); + return Err(self + .invoke_exception( + self.ctx.exceptions.syntax_error, + vec![self.ctx.new_str(msg.as_str()).into(), location.into()], + ) + .unwrap_or_else(|_| { + self.new_exception_msg(self.ctx.exceptions.syntax_error.to_owned(), msg.into()) + })); } if is_utf8 { let src = if has_bom { &source[3..] } else { source }; match core::str::from_utf8(src) { Ok(s) => Ok(s.to_owned()), - Err(e) => { - let bad_byte = src[e.valid_up_to()]; - let line = src[..e.valid_up_to()] - .iter() - .filter(|&&b| b == b'\n') - .count() - + 1; - Err(self.new_exception_msg( - self.ctx.exceptions.syntax_error.to_owned(), - format!( - "Non-UTF-8 code starting with '\\x{bad_byte:02x}' \ - on line {line}, but no encoding declared; \ - see https://peps.python.org/pep-0263/ for details \ - ({filename}, line {line})" - ) - .into(), - )) - } + Err(e) => Err(self.new_non_utf8_syntax_error(filename, src, e.valid_up_to())), } } else { let encoding = encoding.as_deref().unwrap(); diff --git a/crates/vm/src/vm/crossinterp.rs b/crates/vm/src/vm/crossinterp.rs index b7292d882..8aef4b3e2 100644 --- a/crates/vm/src/vm/crossinterp.rs +++ b/crates/vm/src/vm/crossinterp.rs @@ -354,7 +354,7 @@ fn not_shareable_error_from( cause: PyBaseExceptionRef, ) -> PyBaseExceptionRef { let exc = not_shareable_error(vm, msg); - exc.set___cause__(Some(cause)); + exc.set_cause(Some(cause)); exc } @@ -405,7 +405,7 @@ pub fn utf8_key<'a>(key: &'a PyObject, vm: &'a VirtualMachine) -> PyResult<&'a s .downcast_ref::() .ok_or_else(|| vm.new_type_error("bad argument type for built-in operation"))?; s.as_wtf8().as_str().map_err(|_| { - vm.new_unicode_encode_error_real( + vm.new_unicode_encode_error( vm.ctx.new_str("utf-8"), s.to_owned(), 0, diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 23b09e8c3..8a228c547 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -1281,7 +1281,7 @@ impl VirtualMachine { } let err = self.new_runtime_error(msg); - err.set___cause__(Some(import_err)); + err.set_cause(Some(import_err)); err })?; Ok(()) @@ -3044,8 +3044,7 @@ impl VirtualMachine { #[inline] pub fn import<'a>(&self, module_name: impl AsPyStr<'a>, level: usize) -> PyResult { let module_name = module_name.as_pystr(&self.ctx); - let from_list = self.ctx.empty_tuple_typed(); - self.import_inner(module_name, from_list, level) + self.import_inner(module_name, self.ctx.none(), level) } /// Call Python __import__ function caller with from_list. @@ -3054,30 +3053,69 @@ impl VirtualMachine { pub fn import_from<'a>( &self, module_name: impl AsPyStr<'a>, - from_list: &Py>, + from_list: impl Into, level: usize, ) -> PyResult { let module_name = module_name.as_pystr(&self.ctx); - self.import_inner(module_name, from_list, level) + self.import_inner(module_name, from_list.into(), level) } - fn import_inner( - &self, - module: &Py, - from_list: &Py>, - level: usize, - ) -> PyResult { + /// Look up `name` in the current frame's builtins (`_PyEval_GetBuiltin`). + /// A missing key becomes AttributeError with that name. + pub fn eval_get_builtin(&self, name: &'static PyStrInterned) -> PyResult { + let builtins = + crate::frame::current_builtins().unwrap_or_else(|| self.builtins.dict().into()); + if let Some(dict) = builtins.downcast_ref::() { + match dict.get_item_opt(name, self)? { + Some(value) => Ok(value), + None => Err(self.new_attribute_error(name.to_string())), + } + } else { + match builtins.get_item(name, self) { + Ok(value) => Ok(value), + Err(e) if e.fast_isinstance(self.ctx.exceptions.key_error) => { + Err(self.new_attribute_error(name.to_string())) + } + Err(e) => Err(e), + } + } + } + + fn import_inner(&self, module: &Py, from_list: PyObjectRef, level: usize) -> PyResult { + let builtins = + crate::frame::current_builtins().unwrap_or_else(|| self.builtins.dict().into()); + // The module-cache fast path assumes interpreter builtins. A frame + // whose f_builtins is a custom mapping (eval/exec) must go through + // that mapping's __import__. None and an empty tuple are both an + // empty from-list (`import name`). + let fromlist_empty = self.is_none(&from_list) + || from_list + .downcast_ref::() + .is_some_and(|tuple| tuple.is_empty()); if level == 0 - && from_list.as_slice().is_empty() + && fromlist_empty + && builtins.is(self.builtins.dict().as_object()) && let Some(cached) = self.try_import_cached(module)? { return Ok(cached); } - let import_func = self - .builtins - .get_attr(identifier!(self, __import__), self) - .map_err(|_| self.new_import_error("__import__ not found", module.to_owned()))?; + let import_func = if let Some(dict) = builtins.downcast_ref::() { + match dict.get_item_opt(identifier!(self, __import__), self)? { + Some(func) => func, + None => { + return Err(self.new_import_error("__import__ not found", module.to_owned())); + } + } + } else { + match builtins.get_item(identifier!(self, __import__), self) { + Ok(func) => func, + Err(e) if e.fast_isinstance(self.ctx.exceptions.key_error) => { + return Err(self.new_import_error("__import__ not found", module.to_owned())); + } + Err(e) => return Err(e), + } + }; let (locals, globals) = if let Some(globals) = crate::frame::current_globals() { // Locals fallback: use the heavy frame if available, otherwise @@ -3090,7 +3128,6 @@ impl VirtualMachine { } else { (None, None) }; - let from_list: PyObjectRef = from_list.to_owned().into(); import_func .call((module.to_owned(), globals, locals, from_list, level), self) .inspect_err(|exc| import::remove_importlib_frames(self, exc)) @@ -3602,7 +3639,7 @@ impl VirtualMachine { let mut slow_update_toggle = false; while let Some(context) = o.__context__() { if context.is(exception) { - o.set___context__(None); + o.set_context(None); break; } o = context; @@ -3615,7 +3652,7 @@ impl VirtualMachine { } slow_update_toggle = !slow_update_toggle; } - exception.set___context__(Some(context_exc)) + exception.set_context(Some(context_exc)) } } diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index 423fbc189..2dd6153d3 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -42,7 +42,7 @@ macro_rules! define_exception_fn { )] pub fn $fn_name(&self, msg: impl Into) -> $crate::builtins::PyBaseExceptionRef { let err = self.ctx.exceptions.$attr.to_owned(); - self.new_exception_msg(err, msg.into()) + self.new_simple_exception(err, vec![self.ctx.new_str(msg.into()).into()]) } }; } @@ -414,22 +414,42 @@ impl VirtualMachine { def.build_method(class, self) } - /// Instantiate an exception with arguments. - /// This function should only be used with builtin exception types; if a user-defined exception - /// type is passed in, it may not be fully initialized; try using - /// [`vm.invoke_exception()`][Self::invoke_exception] or - /// [`exceptions::ExceptionCtor`][crate::exceptions::ExceptionCtor] instead. - pub fn new_exception(&self, exc_type: PyTypeRef, args: Vec) -> PyBaseExceptionRef { + /// Allocate a `BaseException`-layout instance without running `__new__`/`__init__`. + /// + /// Only valid when `exc_type` uses the `BaseException` payload. Extra-payload + /// types (OSError, UnicodeError, ExceptionGroup, SystemExit, ...) must use + /// [`new_exception`][Self::new_exception] or + /// [`invoke_exception`][Self::invoke_exception]. + pub fn new_simple_exception( + &self, + exc_type: PyTypeRef, + args: Vec, + ) -> PyBaseExceptionRef { debug_assert_eq!( exc_type.slots.basicsize, core::mem::size_of::(), - "vm.new_exception() is only for exception types without additional payload. The given type '{}' is not allowed. Use vm.new_os_subtype_error() for OSError subtypes.", + "vm.new_simple_exception() requires a BaseException-sized type, got {}", exc_type.name() ); - PyBaseException::new(args, self) .into_ref_with_type_lazy_dict(self, exc_type) - .expect("vm.new_exception() called with an invalid exception type") + .expect("vm.new_simple_exception() called with an invalid exception type") + } + + /// Instantiate an exception with arguments. + /// + /// `BaseException`-sized types use + /// [`new_simple_exception`][Self::new_simple_exception]. Extra-payload types + /// run the type constructor so their fields are initialized. User-defined + /// types that override `__new__`/`__init__` should still use + /// [`invoke_exception`][Self::invoke_exception] or + /// [`exceptions::ExceptionCtor`][crate::exceptions::ExceptionCtor]. + pub fn new_exception(&self, exc_type: PyTypeRef, args: Vec) -> PyBaseExceptionRef { + if exc_type.slots.basicsize == core::mem::size_of::() { + self.new_simple_exception(exc_type, args) + } else { + self.invoke_exception(&exc_type, args).unwrap_or_else(|e| e) + } } /// Construct a built-in exception type that carries a payload, directly @@ -446,10 +466,14 @@ impl VirtualMachine { core::any::type_name::(), cls.name() ); - let payload = T::py_new(&cls, args.clone(), self)?; - let exc = payload.into_ref_with_type_lazy_dict(self, cls)?; - T::slot_init(exc.as_object().to_owned(), args, self)?; - Ok(exc) + let obj = T::slot_new(cls, args.clone(), self)?; + T::slot_init(obj.clone(), args, self)?; + obj.downcast().map_err(|obj| { + self.new_type_error(format!( + "payload constructor returned '{}'", + obj.class().name() + )) + }) } pub fn new_os_error(&self, msg: impl ToPyObject) -> PyRef { @@ -469,19 +493,11 @@ impl VirtualMachine { } /// Instantiate an exception with no arguments. - /// This function should only be used with builtin exception types; if a user-defined exception - /// type is passed in, it may not be fully initialized; try using - /// [`vm.invoke_exception()`][Self::invoke_exception] or - /// [`exceptions::ExceptionCtor`][crate::exceptions::ExceptionCtor] instead. pub fn new_exception_empty(&self, exc_type: PyTypeRef) -> PyBaseExceptionRef { self.new_exception(exc_type, vec![]) } /// Instantiate an exception with `msg` as the only argument. - /// This function should only be used with builtin exception types; if a user-defined exception - /// type is passed in, it may not be fully initialized; try using - /// [`vm.invoke_exception()`][Self::invoke_exception] or - /// [`exceptions::ExceptionCtor`][crate::exceptions::ExceptionCtor] instead. pub fn new_exception_msg(&self, exc_type: PyTypeRef, msg: Wtf8Buf) -> PyBaseExceptionRef { self.new_exception(exc_type, vec![self.ctx.new_str(msg).into()]) } @@ -629,30 +645,20 @@ impl VirtualMachine { ) -> PyBaseExceptionRef { let start = self.ctx.new_int(start); let end = self.ctx.new_int(end); - let exc = self.new_exception( - self.ctx.exceptions.unicode_decode_error.to_owned(), + self.invoke_exception( + self.ctx.exceptions.unicode_decode_error, vec![ - encoding.clone().into(), - object.clone().into(), - start.clone().into(), - end.clone().into(), - reason.clone().into(), + encoding.into(), + object.into(), + start.into(), + end.into(), + reason.into(), ], - ); - - set_attrs!( - exc.as_object(), self, unwrap, - "encoding" => encoding, - "object" => object, - "start" => start, - "end" => end, - "reason" => reason, - ); - - exc + ) + .expect("UnicodeDecodeError constructor") } - pub fn new_unicode_encode_error_real( + pub fn new_unicode_encode_error( &self, encoding: PyStrRef, object: PyStrRef, @@ -662,33 +668,55 @@ impl VirtualMachine { ) -> PyBaseExceptionRef { let start = self.ctx.new_int(start); let end = self.ctx.new_int(end); - let exc = self.new_exception( - self.ctx.exceptions.unicode_encode_error.to_owned(), + self.invoke_exception( + self.ctx.exceptions.unicode_encode_error, vec![ - encoding.clone().into(), - object.clone().into(), - start.clone().into(), - end.clone().into(), - reason.clone().into(), + encoding.into(), + object.into(), + start.into(), + end.into(), + reason.into(), ], - ); + ) + .expect("UnicodeEncodeError constructor") + } - set_attrs!( - exc.as_object(), self, unwrap, - "encoding" => encoding, - "object" => object, - "start" => start, - "end" => end, - "reason" => reason, - ); - - exc + #[cfg(feature = "parser")] + fn source_has_mixed_tabs_and_spaces(source: Option<&str>, error_line: usize) -> bool { + source.is_some_and(|source| { + let mut has_space_indent = false; + let mut has_tab_indent = false; + for (i, line) in source.lines().enumerate() { + if i + 1 > error_line { + break; + } + let rest = line.trim_start_matches([' ', '\t']); + // Blank and comment-only lines do not participate in indent. + if rest.is_empty() || rest.starts_with('#') { + continue; + } + let indent = &line.as_bytes()[..line.len() - rest.len()]; + if indent.is_empty() { + continue; + } + if indent.contains(&b' ') && indent.contains(&b'\t') { + return true; + } + if indent.contains(&b' ') { + has_space_indent = true; + } + if indent.contains(&b'\t') { + has_tab_indent = true; + } + } + has_space_indent && has_tab_indent + }) } // TODO: don't take ownership should make the success path faster pub fn new_key_error(&self, obj: PyObjectRef) -> PyBaseExceptionRef { let key_error = self.ctx.exceptions.key_error.to_owned(); - self.new_exception(key_error, vec![obj]) + self.new_simple_exception(key_error, vec![obj]) } #[cfg(any(feature = "parser", feature = "compiler"))] @@ -722,46 +750,18 @@ impl VirtualMachine { error: ruff_python_parser::ParseErrorType::Lexical( ruff_python_parser::LexicalErrorType::IndentationError, - ), + ) + | ruff_python_parser::ParseErrorType::UnexpectedIndentation, + location, .. }) => { - // Detect tab/space mixing to raise TabError instead of IndentationError. - // This checks both within a single line and across different lines. - let is_tab_error = source.is_some_and(|source| { - let mut has_space_indent = false; - let mut has_tab_indent = false; - for line in source.lines() { - let indent: Vec = line - .bytes() - .take_while(|&b| b == b' ' || b == b'\t') - .collect(); - if indent.is_empty() { - continue; - } - if indent.contains(&b' ') && indent.contains(&b'\t') { - return true; - } - if indent.contains(&b' ') { - has_space_indent = true; - } - if indent.contains(&b'\t') { - has_tab_indent = true; - } - } - has_space_indent && has_tab_indent - }); - if is_tab_error { + if Self::source_has_mixed_tabs_and_spaces(source, location.line.get()) { self.ctx.exceptions.tab_error } else { self.ctx.exceptions.indentation_error } } #[cfg(feature = "parser")] - crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { - error: ruff_python_parser::ParseErrorType::UnexpectedIndentation, - .. - }) => self.ctx.exceptions.indentation_error, - #[cfg(feature = "parser")] crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { error: ruff_python_parser::ParseErrorType::Lexical( @@ -877,7 +877,14 @@ impl VirtualMachine { Some(line + "\n") } - let mut statement = source.and_then(|src| get_statement(src, error.location())); + // Tokenizer/parser errors attach the current source line. Compiler + // errors load that line with ProgramText, which is None when the + // filename cannot be opened (compile of a string). + let mut statement = if matches!(error, crate::compiler::CompileError::Codegen(_)) { + self.program_text(error.source_path(), error.python_location().0) + } else { + source.and_then(|src| get_statement(src, error.location())) + }; let mut msg = error.to_string(); if !msg.starts_with("Exceeds the limit ") @@ -901,6 +908,7 @@ impl VirtualMachine { && !msg.starts_with("Assignment expressions are") && !msg.starts_with("Await expressions are") && !msg.starts_with("Underscores in numeric literals are") + && !msg.starts_with("Missing parentheses") && let Some(msg) = msg.get_mut(..1) { msg.make_ascii_lowercase(); diff --git a/crates/vm/src/warn.rs b/crates/vm/src/warn.rs index e64abf14e..aa801759f 100644 --- a/crates/vm/src/warn.rs +++ b/crates/vm/src/warn.rs @@ -617,7 +617,7 @@ fn setup_context( ( f.iframe().globals().to_owned(), f.iframe().code().source_path(), - f.f_lineno(), + f.lineno().max(0) as usize, ) } else if let Some(frame) = vm.current_frame() { // We have a frame but it wasn't found during stack walking