diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index d7dc88792..38828043d 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1739,6 +1739,26 @@ def skip_if_pgo_task(test): return test if ok else unittest.skip(msg)(test) +# XXX: RUSTPYTHON; Taken from 3.14.3 +def skip_if_unlimited_stack_size(test): + """Skip decorator for tests not run when an unlimited stack size is configured. + + Tests using support.infinite_recursion([...]) may otherwise run into + an infinite loop, running until the memory on the system is filled and + crashing due to OOM. + + See https://github.com/python/cpython/issues/143460. + """ + if is_emscripten or is_wasi or os.name == "nt": + return test + + import resource + curlim, maxlim = resource.getrlimit(resource.RLIMIT_STACK) + unlimited_stack_size_cond = curlim == maxlim and curlim in (-1, 0xFFFF_FFFF_FFFF_FFFF) + reason = "Not run due to unlimited stack size" + return unittest.skipIf(unlimited_stack_size_cond, reason)(test) + + def detect_api_mismatch(ref_api, other_api, *, ignore=()): """Returns the set of items in ref_api not in other_api, except for a defined list of items to be ignored in this check. diff --git a/Lib/test/test_tomllib/test_error.py b/Lib/test/test_tomllib/test_error.py index 72446267f..3a8587492 100644 --- a/Lib/test/test_tomllib/test_error.py +++ b/Lib/test/test_tomllib/test_error.py @@ -39,8 +39,19 @@ class TestError(unittest.TestCase): tomllib.loads("v = '\n'") self.assertTrue(" '\\n' " in str(exc_info.exception)) + def test_type_error(self): + with self.assertRaises(TypeError) as exc_info: + tomllib.loads(b"v = 1") # type: ignore[arg-type] + self.assertEqual(str(exc_info.exception), "Expected str object, not 'bytes'") + + with self.assertRaises(TypeError) as exc_info: + tomllib.loads(False) # type: ignore[arg-type] + self.assertEqual(str(exc_info.exception), "Expected str object, not 'bool'") + def test_module_name(self): - self.assertEqual(tomllib.TOMLDecodeError().__module__, tomllib.__name__) + self.assertEqual( + tomllib.TOMLDecodeError("", "", 0).__module__, tomllib.__name__ + ) def test_invalid_parse_float(self): def dict_returner(s: str) -> dict: @@ -55,3 +66,33 @@ class TestError(unittest.TestCase): self.assertEqual( str(exc_info.exception), "parse_float must not return dicts or lists" ) + + def test_deprecated_tomldecodeerror(self): + for args in [ + (), + ("err msg",), + (None,), + (None, "doc"), + ("err msg", None), + (None, "doc", None), + ("err msg", "doc", None), + ("one", "two", "three", "four"), + ("one", "two", 3, "four", "five"), + ]: + with self.assertWarns(DeprecationWarning): + e = tomllib.TOMLDecodeError(*args) # type: ignore[arg-type] + self.assertEqual(e.args, args) + + def test_tomldecodeerror(self): + msg = "error parsing" + doc = "v=1\n[table]\nv='val'" + pos = 13 + formatted_msg = "error parsing (at line 3, column 2)" + e = tomllib.TOMLDecodeError(msg, doc, pos) + self.assertEqual(e.args, (formatted_msg,)) + self.assertEqual(str(e), formatted_msg) + self.assertEqual(e.msg, msg) + self.assertEqual(e.doc, doc) + self.assertEqual(e.pos, pos) + self.assertEqual(e.lineno, 3) + self.assertEqual(e.colno, 2) diff --git a/Lib/test/test_tomllib/test_misc.py b/Lib/test/test_tomllib/test_misc.py index 9e677a337..118fde24d 100644 --- a/Lib/test/test_tomllib/test_misc.py +++ b/Lib/test/test_tomllib/test_misc.py @@ -5,6 +5,7 @@ import copy import datetime from decimal import Decimal as D +import importlib from pathlib import Path import sys import tempfile @@ -92,6 +93,7 @@ class TestMiscellaneous(unittest.TestCase): } self.assertEqual(obj_copy, expected_obj) + @support.skip_if_unlimited_stack_size def test_inline_array_recursion_limit(self): with support.infinite_recursion(max_depth=100): available = support.get_recursion_available() @@ -103,6 +105,7 @@ class TestMiscellaneous(unittest.TestCase): recursive_array_toml = "arr = " + nest_count * "[" + nest_count * "]" tomllib.loads(recursive_array_toml) + @support.skip_if_unlimited_stack_size def test_inline_table_recursion_limit(self): with support.infinite_recursion(max_depth=100): available = support.get_recursion_available() @@ -113,3 +116,11 @@ class TestMiscellaneous(unittest.TestCase): nest_count=nest_count): recursive_table_toml = nest_count * "key = {" + nest_count * "}" tomllib.loads(recursive_table_toml) + + def test_types_import(self): + """Test that `_types` module runs. + + The module is for type annotations only, so it is otherwise + never imported by tests. + """ + importlib.import_module(f"{tomllib.__name__}._types") diff --git a/Lib/tomllib/_parser.py b/Lib/tomllib/_parser.py index 9c80a6a54..3ee47aa9e 100644 --- a/Lib/tomllib/_parser.py +++ b/Lib/tomllib/_parser.py @@ -4,10 +4,7 @@ from __future__ import annotations -from collections.abc import Iterable -import string from types import MappingProxyType -from typing import Any, BinaryIO, NamedTuple from ._re import ( RE_DATETIME, @@ -17,7 +14,13 @@ from ._re import ( match_to_localtime, match_to_number, ) -from ._types import Key, ParseFloat, Pos + +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Iterable + from typing import IO, Any + + from ._types import Key, ParseFloat, Pos ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) @@ -33,9 +36,11 @@ ILLEGAL_COMMENT_CHARS = ILLEGAL_BASIC_STR_CHARS TOML_WS = frozenset(" \t") TOML_WS_AND_NEWLINE = TOML_WS | frozenset("\n") -BARE_KEY_CHARS = frozenset(string.ascii_letters + string.digits + "-_") +BARE_KEY_CHARS = frozenset( + "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789" "-_" +) KEY_INITIAL_CHARS = BARE_KEY_CHARS | frozenset("\"'") -HEXDIGIT_CHARS = frozenset(string.hexdigits) +HEXDIGIT_CHARS = frozenset("abcdef" "ABCDEF" "0123456789") BASIC_STR_ESCAPE_REPLACEMENTS = MappingProxyType( { @@ -50,11 +55,73 @@ BASIC_STR_ESCAPE_REPLACEMENTS = MappingProxyType( ) +class DEPRECATED_DEFAULT: + """Sentinel to be used as default arg during deprecation + period of TOMLDecodeError's free-form arguments.""" + + class TOMLDecodeError(ValueError): - """An error raised if a document is not valid TOML.""" + """An error raised if a document is not valid TOML. + + Adds the following attributes to ValueError: + msg: The unformatted error message + doc: The TOML document being parsed + pos: The index of doc where parsing failed + lineno: The line corresponding to pos + colno: The column corresponding to pos + """ + + def __init__( + self, + msg: str = DEPRECATED_DEFAULT, # type: ignore[assignment] + doc: str = DEPRECATED_DEFAULT, # type: ignore[assignment] + pos: Pos = DEPRECATED_DEFAULT, # type: ignore[assignment] + *args: Any, + ): + if ( + args + or not isinstance(msg, str) + or not isinstance(doc, str) + or not isinstance(pos, int) + ): + import warnings + + warnings.warn( + "Free-form arguments for TOMLDecodeError are deprecated. " + "Please set 'msg' (str), 'doc' (str) and 'pos' (int) arguments only.", + DeprecationWarning, + stacklevel=2, + ) + if pos is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = pos, *args + if doc is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = doc, *args + if msg is not DEPRECATED_DEFAULT: # type: ignore[comparison-overlap] + args = msg, *args + ValueError.__init__(self, *args) + return + + lineno = doc.count("\n", 0, pos) + 1 + if lineno == 1: + colno = pos + 1 + else: + colno = pos - doc.rindex("\n", 0, pos) + + if pos >= len(doc): + coord_repr = "end of document" + else: + coord_repr = f"line {lineno}, column {colno}" + errmsg = f"{msg} (at {coord_repr})" + ValueError.__init__(self, errmsg) + + self.msg = msg + self.doc = doc + self.pos = pos + self.lineno = lineno + self.colno = colno -def load(fp: BinaryIO, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: +def load(fp: IO[bytes], /, *, parse_float: ParseFloat = float) -> dict[str, Any]: """Parse TOML from a binary file object.""" b = fp.read() try: @@ -71,9 +138,14 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n # The spec allows converting "\r\n" to "\n", even in string # literals. Let's do so to simplify parsing. - src = s.replace("\r\n", "\n") + try: + src = s.replace("\r\n", "\n") + except (AttributeError, TypeError): + raise TypeError( + f"Expected str object, not '{type(s).__qualname__}'" + ) from None pos = 0 - out = Output(NestedDict(), Flags()) + out = Output() header: Key = () parse_float = make_safe_parse_float(parse_float) @@ -113,7 +185,7 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n pos, header = create_dict_rule(src, pos, out) pos = skip_chars(src, pos, TOML_WS) elif char != "#": - raise suffixed_err(src, pos, "Invalid statement") + raise TOMLDecodeError("Invalid statement", src, pos) # 3. Skip comment pos = skip_comment(src, pos) @@ -124,8 +196,8 @@ def loads(s: str, /, *, parse_float: ParseFloat = float) -> dict[str, Any]: # n except IndexError: break if char != "\n": - raise suffixed_err( - src, pos, "Expected newline or end of document after a statement" + raise TOMLDecodeError( + "Expected newline or end of document after a statement", src, pos ) pos += 1 @@ -224,9 +296,10 @@ class NestedDict: cont[last_key] = [{}] -class Output(NamedTuple): - data: NestedDict - flags: Flags +class Output: + def __init__(self) -> None: + self.data = NestedDict() + self.flags = Flags() def skip_chars(src: str, pos: Pos, chars: Iterable[str]) -> Pos: @@ -251,12 +324,12 @@ def skip_until( except ValueError: new_pos = len(src) if error_on_eof: - raise suffixed_err(src, new_pos, f"Expected {expect!r}") from None + raise TOMLDecodeError(f"Expected {expect!r}", src, new_pos) from None if not error_on.isdisjoint(src[pos:new_pos]): while src[pos] not in error_on: pos += 1 - raise suffixed_err(src, pos, f"Found invalid character {src[pos]!r}") + raise TOMLDecodeError(f"Found invalid character {src[pos]!r}", src, pos) return new_pos @@ -287,15 +360,17 @@ def create_dict_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: pos, key = parse_key(src, pos) if out.flags.is_(key, Flags.EXPLICIT_NEST) or out.flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot declare {key} twice") + raise TOMLDecodeError(f"Cannot declare {key} twice", src, pos) out.flags.set(key, Flags.EXPLICIT_NEST, recursive=False) try: out.data.get_or_create_nest(key) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if not src.startswith("]", pos): - raise suffixed_err(src, pos, "Expected ']' at the end of a table declaration") + raise TOMLDecodeError( + "Expected ']' at the end of a table declaration", src, pos + ) return pos + 1, key @@ -305,7 +380,7 @@ def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: pos, key = parse_key(src, pos) if out.flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") + raise TOMLDecodeError(f"Cannot mutate immutable namespace {key}", src, pos) # Free the namespace now that it points to another empty list item... out.flags.unset_all(key) # ...but this key precisely is still prohibited from table declaration @@ -313,10 +388,12 @@ def create_list_rule(src: str, pos: Pos, out: Output) -> tuple[Pos, Key]: try: out.data.append_nest_to_list(key) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if not src.startswith("]]", pos): - raise suffixed_err(src, pos, "Expected ']]' at the end of an array declaration") + raise TOMLDecodeError( + "Expected ']]' at the end of an array declaration", src, pos + ) return pos + 2, key @@ -331,22 +408,22 @@ def key_value_rule( for cont_key in relative_path_cont_keys: # Check that dotted key syntax does not redefine an existing table if out.flags.is_(cont_key, Flags.EXPLICIT_NEST): - raise suffixed_err(src, pos, f"Cannot redefine namespace {cont_key}") + raise TOMLDecodeError(f"Cannot redefine namespace {cont_key}", src, pos) # Containers in the relative path can't be opened with the table syntax or # dotted key/value syntax in following table sections. out.flags.add_pending(cont_key, Flags.EXPLICIT_NEST) if out.flags.is_(abs_key_parent, Flags.FROZEN): - raise suffixed_err( - src, pos, f"Cannot mutate immutable namespace {abs_key_parent}" + raise TOMLDecodeError( + f"Cannot mutate immutable namespace {abs_key_parent}", src, pos ) try: nest = out.data.get_or_create_nest(abs_key_parent) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if key_stem in nest: - raise suffixed_err(src, pos, "Cannot overwrite a value") + raise TOMLDecodeError("Cannot overwrite a value", src, pos) # Mark inline table and array namespaces recursively immutable if isinstance(value, (dict, list)): out.flags.set(header + key, Flags.FROZEN, recursive=True) @@ -363,7 +440,7 @@ def parse_key_value_pair( except IndexError: char = None if char != "=": - raise suffixed_err(src, pos, "Expected '=' after a key in a key/value pair") + raise TOMLDecodeError("Expected '=' after a key in a key/value pair", src, pos) pos += 1 pos = skip_chars(src, pos, TOML_WS) pos, value = parse_value(src, pos, parse_float) @@ -401,7 +478,7 @@ def parse_key_part(src: str, pos: Pos) -> tuple[Pos, str]: return parse_literal_str(src, pos) if char == '"': return parse_one_line_basic_str(src, pos) - raise suffixed_err(src, pos, "Invalid initial character for a key part") + raise TOMLDecodeError("Invalid initial character for a key part", src, pos) def parse_one_line_basic_str(src: str, pos: Pos) -> tuple[Pos, str]: @@ -425,7 +502,7 @@ def parse_array(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos, list[ if c == "]": return pos + 1, array if c != ",": - raise suffixed_err(src, pos, "Unclosed array") + raise TOMLDecodeError("Unclosed array", src, pos) pos += 1 pos = skip_comments_and_array_ws(src, pos) @@ -445,20 +522,20 @@ def parse_inline_table(src: str, pos: Pos, parse_float: ParseFloat) -> tuple[Pos pos, key, value = parse_key_value_pair(src, pos, parse_float) key_parent, key_stem = key[:-1], key[-1] if flags.is_(key, Flags.FROZEN): - raise suffixed_err(src, pos, f"Cannot mutate immutable namespace {key}") + raise TOMLDecodeError(f"Cannot mutate immutable namespace {key}", src, pos) try: nest = nested_dict.get_or_create_nest(key_parent, access_lists=False) except KeyError: - raise suffixed_err(src, pos, "Cannot overwrite a value") from None + raise TOMLDecodeError("Cannot overwrite a value", src, pos) from None if key_stem in nest: - raise suffixed_err(src, pos, f"Duplicate inline table key {key_stem!r}") + raise TOMLDecodeError(f"Duplicate inline table key {key_stem!r}", src, pos) nest[key_stem] = value pos = skip_chars(src, pos, TOML_WS) c = src[pos : pos + 1] if c == "}": return pos + 1, nested_dict.dict if c != ",": - raise suffixed_err(src, pos, "Unclosed inline table") + raise TOMLDecodeError("Unclosed inline table", src, pos) if isinstance(value, (dict, list)): flags.set(key, Flags.FROZEN, recursive=True) pos += 1 @@ -480,7 +557,7 @@ def parse_basic_str_escape( except IndexError: return pos, "" if char != "\n": - raise suffixed_err(src, pos, "Unescaped '\\' in a string") + raise TOMLDecodeError("Unescaped '\\' in a string", src, pos) pos += 1 pos = skip_chars(src, pos, TOML_WS_AND_NEWLINE) return pos, "" @@ -491,7 +568,7 @@ def parse_basic_str_escape( try: return pos, BASIC_STR_ESCAPE_REPLACEMENTS[escape_id] except KeyError: - raise suffixed_err(src, pos, "Unescaped '\\' in a string") from None + raise TOMLDecodeError("Unescaped '\\' in a string", src, pos) from None def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]: @@ -501,11 +578,13 @@ def parse_basic_str_escape_multiline(src: str, pos: Pos) -> tuple[Pos, str]: def parse_hex_char(src: str, pos: Pos, hex_len: int) -> tuple[Pos, str]: hex_str = src[pos : pos + hex_len] if len(hex_str) != hex_len or not HEXDIGIT_CHARS.issuperset(hex_str): - raise suffixed_err(src, pos, "Invalid hex value") + raise TOMLDecodeError("Invalid hex value", src, pos) pos += hex_len hex_int = int(hex_str, 16) if not is_unicode_scalar_value(hex_int): - raise suffixed_err(src, pos, "Escaped character is not a Unicode scalar value") + raise TOMLDecodeError( + "Escaped character is not a Unicode scalar value", src, pos + ) return pos, chr(hex_int) @@ -562,7 +641,7 @@ def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: try: char = src[pos] except IndexError: - raise suffixed_err(src, pos, "Unterminated string") from None + raise TOMLDecodeError("Unterminated string", src, pos) from None if char == '"': if not multiline: return pos + 1, result + src[start_pos:pos] @@ -577,7 +656,7 @@ def parse_basic_str(src: str, pos: Pos, *, multiline: bool) -> tuple[Pos, str]: start_pos = pos continue if char in error_on: - raise suffixed_err(src, pos, f"Illegal character {char!r}") + raise TOMLDecodeError(f"Illegal character {char!r}", src, pos) pos += 1 @@ -625,7 +704,7 @@ def parse_value( # noqa: C901 try: datetime_obj = match_to_datetime(datetime_match) except ValueError as e: - raise suffixed_err(src, pos, "Invalid date or datetime") from e + raise TOMLDecodeError("Invalid date or datetime", src, pos) from e return datetime_match.end(), datetime_obj localtime_match = RE_LOCALTIME.match(src, pos) if localtime_match: @@ -646,24 +725,7 @@ def parse_value( # noqa: C901 if first_four in {"-inf", "+inf", "-nan", "+nan"}: return pos + 4, parse_float(first_four) - raise suffixed_err(src, pos, "Invalid value") - - -def suffixed_err(src: str, pos: Pos, msg: str) -> TOMLDecodeError: - """Return a `TOMLDecodeError` where error message is suffixed with - coordinates in source.""" - - def coord_repr(src: str, pos: Pos) -> str: - if pos >= len(src): - return "end of document" - line = src.count("\n", 0, pos) + 1 - if line == 1: - column = pos + 1 - else: - column = pos - src.rindex("\n", 0, pos) - return f"line {line}, column {column}" - - return TOMLDecodeError(f"{msg} (at {coord_repr(src, pos)})") + raise TOMLDecodeError("Invalid value", src, pos) def is_unicode_scalar_value(codepoint: int) -> bool: diff --git a/Lib/tomllib/_re.py b/Lib/tomllib/_re.py index a97cab2f9..eb8beb197 100644 --- a/Lib/tomllib/_re.py +++ b/Lib/tomllib/_re.py @@ -7,9 +7,12 @@ from __future__ import annotations from datetime import date, datetime, time, timedelta, timezone, tzinfo from functools import lru_cache import re -from typing import Any -from ._types import ParseFloat +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Any + + from ._types import ParseFloat # E.g. # - 00:32:00.999999 @@ -84,6 +87,9 @@ def match_to_datetime(match: re.Match[str]) -> datetime | date: return datetime(year, month, day, hour, minute, sec, micros, tzinfo=tz) +# No need to limit cache size. This is only ever called on input +# that matched RE_DATETIME, so there is an implicit bound of +# 24 (hours) * 60 (minutes) * 2 (offset direction) = 2880. @lru_cache(maxsize=None) def cached_tz(hour_str: str, minute_str: str, sign_str: str) -> timezone: sign = 1 if sign_str == "+" else -1