diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index 70251dbb6..0f95a5750 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -55,7 +55,7 @@ def _days_before_year(year): def _days_in_month(year, month): "year, month -> number of days in that month in that year." - assert 1 <= month <= 12, month + assert 1 <= month <= 12, f"month must be in 1..12, not {month}" if month == 2 and _is_leap(year): return 29 return _DAYS_IN_MONTH[month] @@ -272,12 +272,12 @@ def _wrap_strftime(object, format, timetuple): newformat.append(Zreplace) # Note that datetime(1000, 1, 1).strftime('%G') == '1000' so # year 1000 for %G can go on the fast path. - elif ((ch in 'YG' or ch in 'FC') and - object.year < 1000 and _need_normalize_century()): + elif (ch in 'YGFC' and timetuple[0] < 1000 and + _need_normalize_century()): if ch == 'G': year = int(_time.strftime("%G", timetuple)) else: - year = object.year + year = timetuple[0] if ch == 'C': push('{:02}'.format(year // 100)) else: @@ -358,7 +358,8 @@ def _find_isoformat_datetime_separator(dtstr): def _parse_isoformat_date(dtstr): # It is assumed that this is an ASCII-only string of lengths 7, 8 or 10, # see the comment on Modules/_datetimemodule.c:_find_isoformat_datetime_separator - assert len(dtstr) in (7, 8, 10) + if len(dtstr) not in (7, 8, 10): + raise ValueError("Invalid isoformat string") year = int(dtstr[0:4]) has_sep = dtstr[4] == '-' @@ -1947,7 +1948,7 @@ class datetime(date): if became_next_day: year, month, day = date_components # Only wrap day/month when it was previously valid - if month <= 12 and day <= (days_in_month := _days_in_month(year, month)): + if 1 <= month <= 12 and day <= (days_in_month := _days_in_month(year, month)): # Calculate midnight of the next day day += 1 if day > days_in_month: diff --git a/Lib/_pyio.py b/Lib/_pyio.py index d0adef08e..cc336d293 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -1904,7 +1904,7 @@ class TextIOBase(IOBase): """Truncate size to pos, where pos is an int.""" self._unsupported("truncate") - def readline(self): + def readline(self, size=-1, /): """Read until newline or EOF. Returns an empty string if EOF is hit immediately. diff --git a/Lib/argparse.py b/Lib/argparse.py index 24f474056..a106a718a 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1829,6 +1829,9 @@ def _prog_name(prog=None): if modspec is None: # simple script return _os.path.basename(arg0) + if modspec.name != '__main__' and arg0 != modspec.origin: + # named module executed as main without altering sys.argv[0] + return _os.path.basename(arg0) py = _os.path.basename(_sys.executable) if modspec.name != '__main__': # imported module or package diff --git a/Lib/base64.py b/Lib/base64.py index 775ceab0e..0f84420d7 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -1,4 +1,4 @@ -"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" +"""Base16, Base32, Base64 (RFC 4648), Base85 and Ascii85 data encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support @@ -147,7 +147,7 @@ the input is incorrectly padded or if there are non-alphabet characters present in the input. ''' _B32_DECODE_MAP01_DOCSTRING = ''' -RFC 3548 allows for optional mapping of the digit 0 (zero) to the +RFC 4648 allows for optional mapping of the digit 0 (zero) to the letter O (oh), and for optional mapping of the digit 1 (one) to either the letter I (eye) or letter L (el). The optional argument map01 when not None, specifies which letter the digit 1 should be @@ -261,7 +261,7 @@ b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex', extra_args='') -# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns +# RFC 4648, Base 16 Alphabet specifies uppercase, but hexlify() returns # lowercase. The RFC also recommends against accepting input case # insensitively. def b16encode(s): diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 803de0c67..0aeca1ed7 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -553,6 +553,9 @@ class Counter(dict): or multiset. Elements are stored as dictionary keys and their counts are stored as dictionary values. + When constructed from a Mapping or Counter, the original object's + values will be used as the initial counts. + >>> c = Counter('abcdeabcdabcaba') # count elements from a string >>> c.most_common(3) # three most common elements diff --git a/Lib/configparser.py b/Lib/configparser.py index a53ac8727..3c452afe8 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -992,7 +992,9 @@ class RawConfigParser(MutableMapping): value = self._interpolation.before_write(self, section_name, key, value) if value is not None or not self._allow_no_value: - value = delimiter + str(value).replace('\n', '\n\t') + # Convert all possible line-endings into '\n\t' + value = (delimiter + str(value).replace('\r\n', '\n') + .replace('\r', '\n').replace('\n', '\n\t')) else: value = "" fp.write("{}{}\n".format(key, value)) diff --git a/Lib/csv.py b/Lib/csv.py index 0a627ba7a..5d15344ea 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -243,8 +243,14 @@ class Sniffer: def sniff(self, sample, delimiters=None): """ Returns a dialect (or None) corresponding to the sample + + If several delimiters fit the sample equally well, the + delimiters listed in the preferred attribute are preferred, in + that order, no matter how many times each of them occurs. """ + sample = sample.replace('\r\n', '\n').replace('\r', '\n') + quotechar, doublequote, delimiter, skipinitialspace = \ self._guess_quote_and_delimiter(sample, delimiters) if not delimiter: @@ -282,12 +288,16 @@ class Sniffer: """ import re + # The body of a quoted field ends at the first quote which is + # not doubled, as it does for a reader. A lazy ".*?" scans to + # the end of the sample instead, from every start: quadratically. + body = r'(?:(?P=quote){2}|(?!(?P=quote)).)*+' matches = [] - for restr in (r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?P=delim)', # ,".*?", - r'(?:^|\n)(?P["\']).*?(?P=quote)(?P[^\w\n"\'])(?P ?)', # ".*?", - r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?:$|\n)', # ,".*?" - r'(?:^|\n)(?P["\']).*?(?P=quote)(?:$|\n)'): # ".*?" (no delim, no space) - regexp = re.compile(restr, re.DOTALL | re.MULTILINE) + for restr in (r'(?P[^\w\n"\'])(?P ?)(?P["\'])%s(?P=quote)(?P=delim)', # ,"...", + r'(?:^|\n)(?P["\'])%s(?P=quote)(?P[^\w\n"\'])(?P ?)', # "...", + r'(?P[^\w\n"\'])(?P ?)(?P["\'])%s(?P=quote)(?:$|\n)', # ,"..." + r'(?:^|\n)(?P["\'])%s(?P=quote)(?:$|\n)'): # "..." (no delim, no space) + regexp = re.compile(restr % body, re.DOTALL | re.MULTILINE) matches = regexp.findall(data) if matches: break @@ -330,18 +340,22 @@ class Sniffer: delim = '' skipinitialspace = 0 - # if we see an extra quote between delimiters, we've got a - # double quoted format - dq_regexp = re.compile( - r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \ - {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE) - - - - if dq_regexp.search(data): - doublequote = True - else: - doublequote = False + # A doubled quote character inside a quoted field means + # a double quoted format. Match whole fields, so that a match + # cannot slide across field boundaries. + doublequote = False + if delim: + dq_regexp = re.compile( + r"(?:(?<=%(delim)s)|^)%(space)s%(quote)s" # ," + r"((?:%(quote)s%(quote)s|[^%(quote)s]++)*+)" # the body + r"%(quote)s(?:%(delim)s|$)" # ", + % {'delim': re.escape(delim), 'quote': quotechar, + # Skipping spaces after a space rescans them. + 'space': ' *+' if delim != ' ' else ''}, + re.MULTILINE) + dquotechar = quotechar * 2 + doublequote = any(dquotechar in m[1] + for m in dq_regexp.finditer(data)) return (quotechar, doublequote, delim, skipinitialspace) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index f9c1113b1..242a6d2ff 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1247,6 +1247,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, # classes with slots. These could be slightly more performant if we generated # the code instead of iterating over fields. But that can be a project for # another day, if performance becomes an issue. + def _dataclass_getstate(self): return [getattr(self, f.name) for f in fields(self)] diff --git a/Lib/difflib.py b/Lib/difflib.py index ac1ba4a6e..aef2b6f0f 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1977,6 +1977,8 @@ class HtmlDiff(object): # change tabs to spaces before it gets more difficult after we insert # markup + # it also removes trailing newlines, causing some diffs to be missed + # see: gh-71896 fromlines,tolines = self._tab_newline_replace(fromlines,tolines) # create diffs iterator which generates side by side from/to data diff --git a/Lib/glob.py b/Lib/glob.py index f8be46771..173272ba3 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -34,8 +34,8 @@ def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, If `dir_fd` is not None, it should be a file descriptor referring to a directory, and paths will then be relative to that directory. - If `include_hidden` is true, the patterns '*', '?', '**' will match - hidden directories. + If `include_hidden` is true, wildcards can match path segments beginning + with a dot ('.'). If `recursive` is true, the pattern '**' will match any files and zero or more directories and subdirectories. @@ -64,8 +64,8 @@ def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False, If `dir_fd` is not None, it should be a file descriptor referring to a directory, and paths will then be relative to that directory. - If `include_hidden` is true, the patterns '*', '?', '**' will match - hidden directories. + If `include_hidden` is true, wildcards can match path segments beginning + with a dot ('.'). If `recursive` is true, the pattern '**' will match any files and zero or more directories and subdirectories. diff --git a/Lib/inspect.py b/Lib/inspect.py index d3e406cae..cf61ae943 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -196,18 +196,18 @@ def ispackage(object): def ismethoddescriptor(object): """Return true if the object is a method descriptor. - But not if ismethod() or isclass() or isfunction() are true. + But not if ismethod(), isclass() or isfunction() is true. - This is new in Python 2.2, and, for example, is true of int.__add__. - An object passing this test has a __get__ attribute, but not a - __set__ attribute or a __delete__ attribute. Beyond that, the set - of attributes varies; __name__ is usually sensible, and __doc__ - often is. + An object passing this test (for example, int.__add__) has a __get__ + attribute, but not a __set__ attribute or a __delete__ attribute. + Beyond that, the set of attributes varies; __name__ is usually + sensible, and __doc__ often is. Methods implemented via descriptors that also pass one of the other - tests return false from the ismethoddescriptor() test, simply because - the other tests promise more -- you can, e.g., count on having the - __func__ attribute (etc) when an object passes ismethod().""" + tests (ismethod(), isclass(), isfunction()) make this function return + false, simply because those other tests promise more -- you can, for + example, count on having the __func__ attribute when an object passes + ismethod().""" if isclass(object) or ismethod(object) or isfunction(object): # mutual exclusion return False @@ -219,8 +219,13 @@ def ismethoddescriptor(object): def isdatadescriptor(object): """Return true if the object is a data descriptor. + But not if ismethod(), isclass() or isfunction() is true. + Data descriptors have a __set__ or a __delete__ attribute. Examples are - properties (defined in Python) and getsets and members (defined in C). + properties, getsets, and members. For the latter two (defined only in C + extension modules) more specific tests are available as well: + isgetsetdescriptor() and ismemberdescriptor(), respectively. + Typically, data descriptors will also have __name__ and __doc__ attributes (properties, getsets, and members have both of these attributes), but this is not guaranteed.""" @@ -2712,6 +2717,10 @@ class Parameter: raise ValueError(msg) self._kind = _POSITIONAL_ONLY name = 'implicit{}'.format(name[1:]) + elif name == '.format': + # gh-151665: Hidden parameter of compiler-generated annotation and type + # alias/typevar evaluators. Show it as "format". + name = 'format' # It's possible for C functions to have a positional-only parameter # where the name is a keyword, so for compatibility we'll allow it. diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index db87724a8..04e000e07 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -111,7 +111,7 @@ def py_scanstring(s, end, strict=True, if strict: #msg = "Invalid control character %r at" % (terminator,) msg = "Invalid control character {0!r} at".format(terminator) - raise JSONDecodeError(msg, s, end) + raise JSONDecodeError(msg, s, end - 1) else: _append(terminator) continue diff --git a/Lib/locale.py b/Lib/locale.py index 498af0873..6984a6a5e 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -545,12 +545,6 @@ def getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')): """ - import warnings - warnings._deprecated( - "locale.getdefaultlocale", - "{name!r} is deprecated and slated for removal in Python {remove}. " - "Use setlocale(), getencoding() and getlocale() instead.", - remove=(3, 15)) return _getdefaultlocale(envvars) diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 7d0f4c1fd..e7581d2ae 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -244,7 +244,7 @@ class MimeTypes: list of standard types, else to the list of non-standard types. """ - with open(filename, encoding='utf-8') as fp: + with open(filename, encoding='utf-8', errors='surrogateescape') as fp: self.readfp(fp, strict) def readfp(self, fp, strict=True): @@ -435,7 +435,7 @@ def init(files=None): def read_mime_types(file): try: - f = open(file, encoding='utf-8') + f = open(file, encoding='utf-8', errors='surrogateescape') except OSError: return None with f: diff --git a/Lib/ntpath.py b/Lib/ntpath.py index eb127ec26..36da5b7cb 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -345,7 +345,7 @@ def _isreservedname(name): def expanduser(path): """Expand ~ and ~user constructs. - If user or $HOME is unknown, do nothing.""" + If user or home directory is unknown, do nothing.""" path = os.fspath(path) if isinstance(path, bytes): seps = b'\\/' diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 1f8a6ef3d..091f8af01 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1876,6 +1876,7 @@ class Helper: 'async': ('async', ''), 'await': ('await', ''), 'break': ('break', 'while for'), + 'case': 'match', 'class': ('class', 'CLASSES SPECIALMETHODS'), 'continue': ('continue', 'while for'), 'def': ('function', ''), @@ -1887,11 +1888,12 @@ class Helper: 'for': ('for', 'break continue while'), 'from': 'import', 'global': ('global', 'nonlocal NAMESPACES'), - 'if': ('if', 'TRUTHVALUE'), + 'if': ('if', 'TRUTHVALUE match'), 'import': ('import', 'MODULES'), 'in': ('in', 'SEQUENCEMETHODS'), 'is': 'COMPARISON', 'lambda': ('lambda', 'FUNCTIONS'), + 'match': ('match', 'if'), 'nonlocal': ('nonlocal', 'global NAMESPACES'), 'not': 'BOOLEAN', 'or': 'BOOLEAN', diff --git a/Lib/pydoc_data/module_docs.py b/Lib/pydoc_data/module_docs.py index 69b9046f4..2d3543f29 100644 --- a/Lib/pydoc_data/module_docs.py +++ b/Lib/pydoc_data/module_docs.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Wed Jun 10 13:03:46 2026 +# Autogenerated by Sphinx on Wed Aug 5 13:29:41 2026 # as part of the release process. module_docs = { @@ -251,6 +251,7 @@ module_docs = { 'tkinter': 'tkinter#module-tkinter', 'tkinter.colorchooser': 'tkinter.colorchooser#module-tkinter.colorchooser', 'tkinter.commondialog': 'dialog#module-tkinter.commondialog', + 'tkinter.dialog': 'dialog#module-tkinter.dialog', 'tkinter.dnd': 'tkinter.dnd#module-tkinter.dnd', 'tkinter.filedialog': 'dialog#module-tkinter.filedialog', 'tkinter.font': 'tkinter.font#module-tkinter.font', diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 05d283c95..753d9d430 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Wed Jun 10 13:03:46 2026 +# Autogenerated by Sphinx on Wed Aug 5 13:29:41 2026 # as part of the release process. topics = { @@ -125,7 +125,7 @@ follows. inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3 This description does not necessarily apply to descriptor - attributes, such as properties created with "property()". + attributes, such as properties created with "@property". * If the target is a subscription: The primary expression in the reference is evaluated. Next, the subscript expression is evaluated. @@ -854,7 +854,7 @@ instances can redefine and override methods. This allows individual instances to acquire behaviors that differ from other instances of the same class. -The "property()" function is implemented as a data descriptor. +The "@property" decorator is implemented as a data descriptor. Accordingly, instances cannot override the behavior of a property. @@ -1683,7 +1683,7 @@ follow some consistency rules, if possible: The last two expressions apply to totally ordered collections (e.g. to sequences, but not to sets or mappings). See also the - "total_ordering()" decorator. + "@~functools.total_ordering" decorator. * The "hash()" result should be consistent with equality. Objects that are equal should either have the same hash value, or be marked as @@ -1946,12 +1946,12 @@ exception, and the exception occurs in the "try" clause of the inner handler, the outer handler will not handle the exception.) When an exception has been assigned using "as target", it is cleared -at the end of the "except" clause. This is as if +at the end of the "except" clause. This is as if: except E as N: foo -was translated to +was translated to: except E as N: try: @@ -1999,8 +1999,8 @@ have either "except" or "except*" clauses, but not both. The exception type for matching is mandatory in the case of "except*", so "except*:" is a syntax error. The type is interpreted as in the case of "except", but matching is performed on the exceptions contained in the group -that is being handled. An "TypeError" is raised if a matching type is -a subclass of "BaseExceptionGroup", because that would have ambiguous +that is being handled. A "TypeError" is raised if a matching type is a +subclass of "BaseExceptionGroup", because that would have ambiguous semantics. When an exception group is raised in the try block, each "except*" @@ -2015,7 +2015,7 @@ the group have been matched, or the last "except*" clause has run. After all "except*" clauses execute, the group of unhandled exceptions is merged with any exceptions that were raised or re-raised from -within "except*" clauses. This merged exception group propagates on.: +within "except*" clauses. This merged exception group propagates on: >>> try: ... raise ExceptionGroup("eg", @@ -2899,7 +2899,7 @@ as a list or a dictionary: if the function modifies the object (e.g. by appending an item to a list), the default parameter value is in effect modified. This is generally not what was intended. A way around this is to use "None" as the default, and explicitly test for -it in the body of the function, e.g.: +it in the body of the function, for example: def whats_on_the_telly(penguin=None): if penguin is None: @@ -3434,7 +3434,7 @@ enforced by the language, and in general annotations may contain arbitrary expressions. The presence of annotations does not change the runtime semantics of the code, except if some mechanism is used that introspects and uses the annotations (such as "dataclasses" or -"functools.singledispatch()"). +"@functools.singledispatch"). By default, annotations are lazily evaluated in an annotation scope. This means that they are not evaluated when the code containing the @@ -3786,7 +3786,7 @@ object.__ge__(self, other) operators or default implementations; for example, the truth of "(x>> flag = False + >>> match (100, 200): + ... case (100, 300): # Mismatch: 200 != 300 + ... print('Case 1') + ... case (100, 200) if flag: # Successful match, but guard fails + ... print('Case 2') + ... case (100, y): # Matches and binds y to 200 + ... print(f'Case 3, y: {y}') + ... case _: # Pattern not attempted + ... print('Case 4, I match anything!') + ... + Case 3, y: 200 + +In this case, "if flag" is a guard. Read more about that in the next +section. + + +Guards +====== + + guard: "if" assignment_expression + +A "guard" (which is part of the "case") must succeed for code inside +the "case" block to execute. It takes the form: "if" followed by an +expression. + +The logical flow of a "case" block with a "guard" follows: + +1. Check that the pattern in the "case" block succeeded. If the + pattern failed, the "guard" is not evaluated and the next "case" + block is checked. + +2. If the pattern succeeded, evaluate the "guard". + + * If the "guard" condition evaluates as true, the case block is + selected. + + * If the "guard" condition evaluates as false, the case block is + not selected. + + * If the "guard" raises an exception during evaluation, the + exception bubbles up. + +Guards are allowed to have side effects as they are expressions. +Guard evaluation must proceed from the first to the last case block, +one at a time, skipping case blocks whose pattern(s) don’t all +succeed. (I.e., guard evaluation must happen in order.) Guard +evaluation must stop once a case block is selected. + + +Irrefutable Case Blocks +======================= + +An irrefutable case block is a match-all case block. A match +statement may have at most one irrefutable case block, and it must be +last. + +A case block is considered irrefutable if it has no guard and its +pattern is irrefutable. A pattern is considered irrefutable if we can +prove from its syntax alone that it will always succeed. Only the +following patterns are irrefutable: + +* AS Patterns whose left-hand side is irrefutable + +* OR Patterns containing at least one irrefutable pattern + +* Capture Patterns + +* Wildcard Patterns + +* parenthesized irrefutable patterns + + +Patterns +======== + +Note: + + This section uses grammar notations beyond standard EBNF: + + * the notation "SEP.RULE+" is shorthand for "RULE (SEP RULE)*" + + * the notation "!RULE" is shorthand for a negative lookahead + assertion + +The top-level syntax for "patterns" is: + + patterns: open_sequence_pattern | pattern + pattern: as_pattern | or_pattern + closed_pattern: | literal_pattern + | capture_pattern + | wildcard_pattern + | value_pattern + | group_pattern + | sequence_pattern + | mapping_pattern + | class_pattern + +The descriptions below will include a description “in simple terms” of +what a pattern does for illustration purposes (credits to Raymond +Hettinger for a document that inspired most of the descriptions). Note +that these descriptions are purely for illustration purposes and **may +not** reflect the underlying implementation. Furthermore, they do not +cover all valid forms. + + +OR Patterns +----------- + +An OR pattern is two or more patterns separated by vertical bars "|". +Syntax: + + or_pattern: "|".closed_pattern+ + +Only the final subpattern may be irrefutable, and each subpattern must +bind the same set of names to avoid ambiguity. + +An OR pattern matches each of its subpatterns in turn to the subject +value, until one succeeds. The OR pattern is then considered +successful. Otherwise, if none of the subpatterns succeed, the OR +pattern fails. + +In simple terms, "P1 | P2 | ..." will try to match "P1", if it fails +it will try to match "P2", succeeding immediately if any succeeds, +failing otherwise. + + +AS Patterns +----------- + +An AS pattern matches an OR pattern on the left of the "as" keyword +against a subject. Syntax: + + as_pattern: or_pattern "as" capture_pattern + +If the OR pattern fails, the AS pattern fails. Otherwise, the AS +pattern binds the subject to the name on the right of the as keyword +and succeeds. "capture_pattern" cannot be a "_". + +In simple terms "P as NAME" will match with "P", and on success it +will set "NAME = ". + + +Literal Patterns +---------------- + +A literal pattern corresponds to most literals in Python. Syntax: + + literal_pattern: signed_number + | signed_number "+" NUMBER + | signed_number "-" NUMBER + | strings + | "None" + | "True" + | "False" + signed_number: ["-"] NUMBER + +The rule "strings" and the token "NUMBER" are defined in the standard +Python grammar. Triple-quoted strings are supported. Raw strings and +byte strings are supported. f-strings and t-strings are not +supported. + +The forms "signed_number '+' NUMBER" and "signed_number '-' NUMBER" +are for expressing complex numbers; they require a real number on the +left and an imaginary number on the right. E.g. "3 + 4j". + +In simple terms, "LITERAL" will succeed only if " == +LITERAL". For the singletons "None", "True" and "False", the "is" +operator is used. + + +Capture Patterns +---------------- + +A capture pattern binds the subject value to a name. Syntax: + + capture_pattern: !'_' NAME + +A single underscore "_" is not a capture pattern (this is what "!'_'" +expresses). It is instead treated as a "wildcard_pattern". + +In a given pattern, a given name can only be bound once. E.g. "case +x, x: ..." is invalid while "case [x] | x: ..." is allowed. + +Capture patterns always succeed. The binding follows scoping rules +established by the assignment expression operator in **PEP 572**; the +name becomes a local variable in the closest containing function scope +unless there’s an applicable "global" or "nonlocal" statement. + +In simple terms "NAME" will always succeed and it will set "NAME = +". + + +Wildcard Patterns +----------------- + +A wildcard pattern always succeeds (matches anything) and binds no +name. Syntax: + + wildcard_pattern: '_' + +"_" is a soft keyword within any pattern, but only within patterns. +It is an identifier, as usual, even within "match" subject +expressions, "guard"s, and "case" blocks. + +In simple terms, "_" will always succeed. + + +Value Patterns +-------------- + +A value pattern represents a named value in Python. Syntax: + + value_pattern: attr + attr: name_or_attr "." NAME + name_or_attr: attr | NAME + +The dotted name in the pattern is looked up using standard Python name +resolution rules. The pattern succeeds if the value found compares +equal to the subject value (using the "==" equality operator). + +In simple terms "NAME1.NAME2" will succeed only if " == +NAME1.NAME2" + +Note: + + If the same value occurs multiple times in the same match statement, + the interpreter may cache the first value found and reuse it rather + than repeat the same lookup. This cache is strictly tied to a given + execution of a given match statement. + + +Group Patterns +-------------- + +A group pattern allows users to add parentheses around patterns to +emphasize the intended grouping. Otherwise, it has no additional +syntax. Syntax: + + group_pattern: "(" pattern ")" + +In simple terms "(P)" has the same effect as "P". + + +Sequence Patterns +----------------- + +A sequence pattern contains several subpatterns to be matched against +sequence elements. The syntax is similar to the unpacking of a list or +tuple. + + sequence_pattern: "[" [maybe_sequence_pattern] "]" + | "(" [open_sequence_pattern] ")" + open_sequence_pattern: maybe_star_pattern "," [maybe_sequence_pattern] + maybe_sequence_pattern: ",".maybe_star_pattern+ ","? + maybe_star_pattern: star_pattern | pattern + star_pattern: "*" (capture_pattern | wildcard_pattern) + +There is no difference if parentheses or square brackets are used for +sequence patterns (i.e. "(...)" vs "[...]" ). + +Note: + + A single pattern enclosed in parentheses without a trailing comma + (e.g. "(3 | 4)") is a group pattern. While a single pattern enclosed + in square brackets (e.g. "[3 | 4]") is still a sequence pattern. + +At most one star subpattern may be in a sequence pattern. The star +subpattern may occur in any position. If no star subpattern is +present, the sequence pattern is a fixed-length sequence pattern; +otherwise it is a variable-length sequence pattern. + +The following is the logical flow for matching a sequence pattern +against a subject value: + +1. If the subject value is not a sequence [2], the sequence pattern + fails. + +2. If the subject value is an instance of "str", "bytes" or + "bytearray" the sequence pattern fails. + +3. The subsequent steps depend on whether the sequence pattern is + fixed or variable-length. + + If the sequence pattern is fixed-length: + + 1. If the length of the subject sequence is not equal to the number + of subpatterns, the sequence pattern fails + + 2. Subpatterns in the sequence pattern are matched to their + corresponding items in the subject sequence from left to right. + Matching stops as soon as a subpattern fails. If all + subpatterns succeed in matching their corresponding item, the + sequence pattern succeeds. + + Otherwise, if the sequence pattern is variable-length: + + 1. If the length of the subject sequence is less than the number of + non-star subpatterns, the sequence pattern fails. + + 2. The leading non-star subpatterns are matched to their + corresponding items as for fixed-length sequences. + + 3. If the previous step succeeds, the star subpattern matches a + list formed of the remaining subject items, excluding the + remaining items corresponding to non-star subpatterns following + the star subpattern. + + 4. Remaining non-star subpatterns are matched to their + corresponding subject items, as for a fixed-length sequence. + + Note: + + The length of the subject sequence is obtained via "len()" (i.e. + via the "__len__()" protocol). This length may be cached by the + interpreter in a similar manner as value patterns. + +In simple terms "[P1, P2, P3," … ", P]" matches only if all the +following happens: + +* check "" is a sequence + +* "len(subject) == " + +* "P1" matches "[0]" (note that this match can also bind + names) + +* "P2" matches "[1]" (note that this match can also bind + names) + +* … and so on for the corresponding pattern/element. + + +Mapping Patterns +---------------- + +A mapping pattern contains one or more key-value patterns. The syntax +is similar to the construction of a dictionary. Syntax: + + mapping_pattern: "{" [items_pattern] "}" + items_pattern: ",".key_value_pattern+ ","? + key_value_pattern: (literal_pattern | value_pattern) ":" pattern + | double_star_pattern + double_star_pattern: "**" capture_pattern + +At most one double star pattern may be in a mapping pattern. The +double star pattern must be the last subpattern in the mapping +pattern. + +Duplicate keys in mapping patterns are disallowed. Duplicate literal +keys will raise a "SyntaxError". Two keys that otherwise have the same +value will raise a "ValueError" at runtime. + +The following is the logical flow for matching a mapping pattern +against a subject value: + +1. If the subject value is not a mapping [3],the mapping pattern + fails. + +2. If every key given in the mapping pattern is present in the subject + mapping, and the pattern for each key matches the corresponding + item of the subject mapping, the mapping pattern succeeds. + +3. If duplicate keys are detected in the mapping pattern, the pattern + is considered invalid. A "SyntaxError" is raised for duplicate + literal values; or a "ValueError" for named keys of the same value. + +Note: + + Key-value pairs are matched using the two-argument form of the + mapping subject’s "get()" method. Matched key-value pairs must + already be present in the mapping, and not created on-the-fly via + "__missing__()" or "__getitem__()". + +In simple terms "{KEY1: P1, KEY2: P2, ... }" matches only if all the +following happens: + +* check "" is a mapping + +* "KEY1 in " + +* "P1" matches "[KEY1]" + +* … and so on for the corresponding KEY/pattern pair. + + +Class Patterns +-------------- + +A class pattern represents a class and its positional and keyword +arguments (if any). Syntax: + + class_pattern: name_or_attr "(" [pattern_arguments ","?] ")" + pattern_arguments: positional_patterns ["," keyword_patterns] + | keyword_patterns + positional_patterns: ",".pattern+ + keyword_patterns: ",".keyword_pattern+ + keyword_pattern: NAME "=" pattern + +The same keyword should not be repeated in class patterns. + +The following is the logical flow for matching a class pattern against +a subject value: + +1. If "name_or_attr" is not an instance of the builtin "type" , raise + "TypeError". + +2. If the subject value is not an instance of "name_or_attr" (tested + via "isinstance()"), the class pattern fails. + +3. If no pattern arguments are present, the pattern succeeds. + Otherwise, the subsequent steps depend on whether keyword or + positional argument patterns are present. + + For a number of built-in types (specified below), a single + positional subpattern is accepted which will match the entire + subject; for these types keyword patterns also work as for other + types. + + If only keyword patterns are present, they are processed as + follows, one by one: + + 1. The keyword is looked up as an attribute on the subject. + + * If this raises an exception other than "AttributeError", the + exception bubbles up. + + * If this raises "AttributeError", the class pattern has failed. + + * Else, the subpattern associated with the keyword pattern is + matched against the subject’s attribute value. If this fails, + the class pattern fails; if this succeeds, the match proceeds + to the next keyword. + + 2. If all keyword patterns succeed, the class pattern succeeds. + + If any positional patterns are present, they are converted to + keyword patterns using the "__match_args__" attribute on the class + "name_or_attr" before matching: + + 1. The equivalent of "getattr(cls, "__match_args__", ())" is + called. + + * If this raises an exception, the exception bubbles up. + + * If the returned value is not a tuple, the conversion fails and + "TypeError" is raised. + + * If there are more positional patterns than + "len(cls.__match_args__)", "TypeError" is raised. + + * Otherwise, positional pattern "i" is converted to a keyword + pattern using "__match_args__[i]" as the keyword. + "__match_args__[i]" must be a string; if not "TypeError" is + raised. + + * If there are duplicate keywords, "TypeError" is raised. + + See also: + + Customizing positional arguments in class pattern matching + + 2. Once all positional patterns have been converted to keyword + patterns, the match proceeds as if there were only keyword + patterns. + + For the following built-in types the handling of positional + subpatterns is different: + + * "bool" + + * "bytearray" + + * "bytes" + + * "dict" + + * "float" + + * "frozenset" + + * "int" + + * "list" + + * "set" + + * "str" + + * "tuple" + + These classes accept a single positional argument, and the pattern + there is matched against the whole object rather than an attribute. + For example "int(0|1)" matches the value "0", but not the value + "0.0". + +In simple terms "CLS(P1, attr=P2)" matches only if the following +happens: + +* "isinstance(, CLS)" + +* convert "P1" to a keyword pattern using "CLS.__match_args__" + +* For each keyword argument "attr=P2": + + * "hasattr(, "attr")" + + * "P2" matches ".attr" + +* … and so on for the corresponding keyword argument/pattern pair. + +See also: + + * **PEP 634** – Structural Pattern Matching: Specification + + * **PEP 636** – Structural Pattern Matching: Tutorial ''', 'naming': r'''Naming and binding ****************** @@ -8426,7 +9024,7 @@ object.__ge__(self, other) operators or default implementations; for example, the truth of "(x>> try: ... raise ExceptionGroup("eg", diff --git a/Lib/random.py b/Lib/random.py index 726a71e78..0e6a183d1 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -861,7 +861,11 @@ class Random(_random.Random): u = random() u -= 0.5 us = 0.5 - _fabs(u) - k = _floor((2.0 * a / us + b) * u + c) + try: + k = _floor((2.0 * a / us + b) * u + c) + except ZeroDivisionError: + # Reject case where random() returned 0.0 + continue if k < 0 or k > n: continue v = random() diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index c1bb6138a..1b71b71c5 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -3581,6 +3581,11 @@ class TestDateTime(TestDate): '2009-04-19T12:30:45.400 +02:30', # Space between ms and timezone (gh-130959) '2009-04-19T12:30:45.400 ', # Trailing space (gh-130959) '2009-04-19T12:30:45. 400', # Space before fraction (gh-130959) + '2020-2020', # Ambiguous 9-char date portion + '2009-04-19T12:30:45.+05:00', # Empty fraction before offset + '2009-04-19T12:30:45.-05:00', # Empty fraction before offset + '2009-04-19T12:30:45.Z', # Empty fraction before Z + '2009-04-19T12:30:45,+05:00', # Empty fraction (comma) before offset ] for bad_str in bad_strs: @@ -3597,6 +3602,7 @@ class TestDateTime(TestDate): "2009-04-01T12:30:90", # Second out of range "2009-04-01T12:90:45", # Minute out of range "2009-04-01T25:30:45", # Hour out of range + "2009-00-01T24:00:00", # Month below range "2009-13-01T24:00:00", # Month out of range "9999-12-31T24:00:00", # Year out of range ] @@ -3625,6 +3631,32 @@ class TestDateTime(TestDate): self.assertIs(dt.tzinfo, timezone.utc) + def test_fromisoformat_utc_subsecond_offset(self): + # A UTC offset whose whole-second part is zero but with a non-zero + # microsecond part must be preserved, not collapsed to UTC. + for us in (1, -1, 999999, -999999): + with self.subTest(microseconds=us): + tz = timezone(timedelta(microseconds=us)) + dt = self.theclass(2020, 6, 15, 12, 34, 56, tzinfo=tz) + rt = self.theclass.fromisoformat(dt.isoformat()) + self.assertEqual(rt.utcoffset(), timedelta(microseconds=us)) + self.assertEqual(rt, dt) + self.assertIsNot(rt.tzinfo, timezone.utc) + + tz = timezone(timedelta(hours=5, minutes=30, seconds=15, + microseconds=123456)) + dt = self.theclass(2020, 6, 15, 12, 34, 56, tzinfo=tz) + rt = self.theclass.fromisoformat(dt.isoformat()) + self.assertEqual(rt.utcoffset(), tz.utcoffset(None)) + self.assertEqual(rt, dt) + + for tstr in ('2020-06-15T12:34:56+00:00', + '2020-06-15T12:34:56+00:00:00.000000', + '2020-06-15T12:34:56Z'): + with self.subTest(tstr=tstr): + self.assertIs(self.theclass.fromisoformat(tstr).tzinfo, + timezone.utc) + def test_fromisoformat_subclass(self): class DateTimeSubclass(self.theclass): pass @@ -3908,6 +3940,11 @@ class TestTime(HarmlessMixedComparison, unittest.TestCase): self.assertEqual(t.strftime('\0'*1000), '\0'*1000) self.assertEqual(t.strftime('\0%I%p%Z\0%X'), f'\0{s1}\0{s2}') self.assertEqual(t.strftime('%I%p%Z\0%X\0'), f'{s1}\0{s2}\0') + # gh-152305: the year directives must not raise on a time. + for directive, expected in (('%Y', '1900'), ('%G', '1900'), + ('%C', '19'), ('%F', '1900-01-01')): + with self.subTest(directive=directive): + self.assertEqual(t.strftime(directive), expected) def test_format(self): t = self.theclass(1, 2, 3, 4) @@ -4805,6 +4842,10 @@ class TestTimeTZ(TestTime, TZInfoBase, unittest.TestCase): '12:30:45.400 +02:30', # Space between ms and timezone (gh-130959) '12:30:45.400 ', # Trailing space (gh-130959) '12:30:45. 400', # Space before fraction (gh-130959) + '12:30:45.+05:00', # Empty fraction before offset + '12:30:45.-05:00', # Empty fraction before offset + '12:30:45.Z', # Empty fraction before Z + '12:30:45,+05:00', # Empty fraction (comma) before offset ] for bad_str in bad_strs: diff --git a/Lib/test/mime.types b/Lib/test/mime.types index 68724430c..eb39a17b6 100644 --- a/Lib/test/mime.types +++ b/Lib/test/mime.types @@ -1,11 +1,11 @@ # This is a comment. I love comments. -*- indent-tabs-mode: t -*- -# This file controls what Internet media types are sent to the client for +# This file controls what internet media types are sent to the client for # given file extension(s). Sending the correct media type to the client # is important so they know how to handle the content of the file. # Extra types can either be added here or by using an AddType directive -# in your config files. For more information about Internet media types, -# please read RFC 2045, 2046, 2047, 2048, and 2077. The Internet media type +# in your config files. For more information about internet media types, +# please read RFC 2045, 2046, 2047, 2048, and 2077. The internet media type # registry is at . # IANA types diff --git a/Lib/test/mime.types2 b/Lib/test/mime.types2 new file mode 100644 index 000000000..b05c318d1 --- /dev/null +++ b/Lib/test/mime.types2 @@ -0,0 +1,3 @@ +# MIME type Extensions +testing/test2 test test2 +testing/test3 test test3 diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 627e395c4..e376c44fa 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -7128,6 +7128,19 @@ class TestProgName(TestCase): def test_module_compiled(self): self.test_module(compiled=True) + def test_module_as_main_without_altering_argv(self): + basename = 'module' + os_helper.FS_NONASCII + modulename = f'{self.dirname}.{basename}' + self.make_script(self.dirname, basename) + runner_source = textwrap.dedent(f'''\ + import runpy + runpy._run_module_as_main({modulename!r}, alter_argv=False) + ''') + runner = script_helper.make_script( + self.dirname, 'runner', runner_source) + self.check_usage(os.path.basename(runner), runner, + PYTHONPATH=os.curdir) + def test_package(self, compiled=False): basename = 'subpackage' + os_helper.FS_NONASCII packagename = f'{self.dirname}.{basename}' diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index ae6fec121..64e1d1e5a 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -5,7 +5,7 @@ import collections.abc import unittest from test import support -from test.support import import_helper +from test.support import import_helper, script_helper from test.support import os_helper from test.support import _2G from test.support import subTests @@ -35,6 +35,11 @@ typecodes = 'uwbBhHiIlLfdqQ' class MiscTest(unittest.TestCase): + def test_array_type_importable(self): + from array import ArrayType + + self.assertIs(array.array, ArrayType) + def test_array_is_sequence(self): self.assertIsInstance(array.array("B"), collections.abc.MutableSequence) self.assertIsInstance(array.array("B"), collections.abc.Reversible) @@ -45,6 +50,23 @@ class MiscTest(unittest.TestCase): self.assertRaises(TypeError, array.array, 'xx') self.assertRaises(ValueError, array.array, 'x') + @support.cpython_only + def test_does_not_crash_on_broken_imports(self): + # gh-153210 + code = """if 1: + import collections.abc + + del collections.abc.MutableSequence + + try: + import array # it used to crash before + except AttributeError: + pass + else: + raise AssertionError('AttributeError was not raised') + """ + script_helper.assert_python_ok('-c', code) + @support.cpython_only def test_disallow_instantiation(self): my_array = array.array("I") diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 8d8dd2a2b..4783943f7 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -526,6 +526,17 @@ boolean {0[0]} NO cf.get(self.default_section, "Foo"), "Bar", "could not locate option, expecting case-insensitive defaults") + def test_crlf_normalization(self): + cf = self.newconfig({"key1": "a\nb","key2": "a\rb", "key3": "a\r\nb", "key4": "a\r\nb"}) + buf = io.StringIO() + cf.write(buf) + cf_str = buf.getvalue() + self.assertNotIn("\r", cf_str) + self.assertNotIn("\r\n", cf_str) + self.assertEqual(cf_str.count("\n"), 10) + self.assertEqual(cf_str.count("\n\t"), 4) + self.assertTrue(cf_str.endswith("\n\n")) + def test_parse_errors(self): cf = self.newconfig() self.parse_error(cf, configparser.ParsingError, diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 0fbf026ae..2dd012c93 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1405,6 +1405,9 @@ ghi\0jkl sample18.append("v,twenty_one") # 'u' was not skipped sample18 = '\n'.join(sample18) + sample19 = ('time,title\r\n' + '2020-10-01,"Pocket - Save news, videos, stories and more"\r\n') + def test_issue43625(self): sniffer = csv.Sniffer() self.assertTrue(sniffer.has_header(self.sample12)) @@ -1484,6 +1487,9 @@ ghi\0jkl sniffer.sniff, self.sample15) self.assertRaisesRegex(csv.Error, "Could not determine delimiter", sniffer.sniff, self.sample16) + dialect = sniffer.sniff(self.sample19) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.quotechar, '"') def test_doublequote(self): sniffer = csv.Sniffer() @@ -1498,6 +1504,57 @@ ghi\0jkl dialect = sniffer.sniff(self.sample9) self.assertTrue(dialect.doublequote) + def test_sniff_regex_backtracking(self): + # gh-109638: this artificial sample used to take minutes. + sniffer = csv.Sniffer() + sample = '"",' * 100 + '"' * 100 + '0' + '"' * 100 + '0' + self.assertEqual(sniffer.sniff(sample).delimiter, ',') + + def test_sniff_doublequote_across_fields(self): + # A quoted field which contains the delimiter, followed by + # an empty quoted field, is not a doubled quote. + sniffer = csv.Sniffer() + sample = '",","",","\n' * 4 + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.quotechar, '"') + self.assertIs(dialect.doublequote, False) + self.assertEqual(next(csv.reader(StringIO(sample), dialect)), + [',', '', ',']) + + def test_sniff_doublequote_record_separators(self): + # The record separator ends a field as a delimiter does. + sniffer = csv.Sniffer() + for sep in '\n', '\r\n', '\r': + with self.subTest(sep=sep): + sample = ('x,"a""b"' + sep + 'y,"c"' + sep) * 2 + self.assertIs(sniffer.sniff(sample).doublequote, True) + sample = ('"",","' + sep) * 4 + self.assertIs(sniffer.sniff(sample).doublequote, False) + + def test_sniff_single_column(self): + # This sample used to be quadratic. + sniffer = csv.Sniffer() + sample = '"a"\n' + ' ' * 100000 + with self.assertRaisesRegex(csv.Error, "Could not determine delimiter"): + sniffer.sniff(sample, delimiters=',;') + + def test_sniff_space_delimiter(self): + # This sample used to be quadratic. + sniffer = csv.Sniffer() + sample = '"a" "b"\n' + ' ' * 100000 + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ' ') + self.assertIs(dialect.doublequote, False) + + def test_sniff_quoted_single_column(self): + # gh-98820: this sample used to take minutes. + sniffer = csv.Sniffer() + sample = '"abcdefghijklmnopqrstuvwxyz"\n' * 10000 + with self.assertRaisesRegex(csv.Error, "Could not determine delimiter"): + sniffer.sniff(sample, delimiters=',:|\t') + + class NUL: def write(s, *args): pass diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 36be5c99c..518b8c527 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -282,6 +282,29 @@ class TestSFpatches(unittest.TestCase): self.assertIn('content="text/html; charset=us-ascii"', output) self.assertIn('ımplıcıt', output) + def test_strip_trailing_newlines_before_diff(self): + # characterization test for the current buggy behavior + # see: gh-71896 + html_diff = difflib.HtmlDiff() + from_lines = [ + "Line 1: no newline after", + "Line 2: one newline after\n", + "Line 3: several newlines after\n\n\n\n\n", + ] + to_lines = [ + "Line 1: no newline after", + "Line 2: one newline after", # actually no \n + "Line 3: several newlines after", # actually no \n + ] + output = html_diff.make_table(from_lines, to_lines) + # we (currently) expect no line change, so all equal + self.assertNotIn('class="diff_add"', output) + self.assertNotIn('class="diff_chg"', output) + self.assertNotIn('class="diff_sub"', output) + self.assertEqual(output.count('>Line 1: no newline after<'), 2) + self.assertEqual(output.count('>Line 2: one newline after<'), 2) + self.assertEqual(output.count('>Line 3: several newlines after<'), 2) + class TestDiffer(unittest.TestCase): def test_close_matches_aligned(self): # Of the 4 closely matching pairs, we want 1 to match with 3, diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 8e49aa895..f91843562 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -1,6 +1,5 @@ from decimal import Decimal from test.support import cpython_only, verbose, is_android, linked_to_musl, os_helper -from test.support.warnings_helper import check_warnings from test.support.import_helper import ensure_lazy_imports, import_fresh_module from unittest import mock import unittest @@ -554,8 +553,7 @@ class TestMiscellaneous(unittest.TestCase): env.unset('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE') env.set('LC_CTYPE', 'UTF-8') - with check_warnings(('', DeprecationWarning)): - self.assertEqual(locale.getdefaultlocale(), (None, 'UTF-8')) + self.assertEqual(locale.getdefaultlocale(), (None, 'UTF-8')) finally: if orig_getlocale is not None: _locale._getdefaultlocale = orig_getlocale diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 5678536ff..943448d69 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -238,6 +238,19 @@ class MyIndexable(object): def __index__(self): return self.value +# int subclass with broken arithmetic operators; implementations must +# convert their arguments to exact ints instead of using these. +class BadIntSubclass(int): + def _binop(self, other='ignored', mod=None): + return 42 + __add__ = __radd__ = __sub__ = __rsub__ = _binop + __mul__ = __rmul__ = __mod__ = __rmod__ = _binop + __divmod__ = __rdivmod__ = __pow__ = __rpow__ = _binop + __floordiv__ = __rfloordiv__ = _binop + __lshift__ = __rlshift__ = __rshift__ = __rrshift__ = _binop + __and__ = __rand__ = __or__ = __ror__ = __xor__ = __rxor__ = _binop + __lt__ = __le__ = __gt__ = __ge__ = _binop + class BadDescr: def __get__(self, obj, objtype=None): raise ValueError @@ -1117,6 +1130,16 @@ class MathTests(unittest.TestCase): self.assertIs(type(s), int) self.assertEqual(s, 41) + # Overridden operators of an int subclass must not affect the + # result. + s = math.isqrt(BadIntSubclass(10**20)) + self.assertIs(type(s), int) + self.assertEqual(s, 10**10) + + s = math.isqrt(BadIntSubclass(10**20 - 1)) + self.assertIs(type(s), int) + self.assertEqual(s, 10**10 - 1) + with self.assertRaises(ValueError): math.isqrt(IntegerLike(-3)) diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index e263487e0..a7d1d724f 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -31,50 +31,11 @@ def tearDownModule(): mimetypes.knownfiles = knownfiles -class MimeTypesTestCase(unittest.TestCase): +class MimeTypesModuleTestCase(unittest.TestCase): def setUp(self): - self.db = mimetypes.MimeTypes() - - def test_case_sensitivity(self): - eq = self.assertEqual - eq(self.db.guess_file_type("foobar.html"), ("text/html", None)) - eq(self.db.guess_type("scheme:foobar.html"), ("text/html", None)) - eq(self.db.guess_file_type("foobar.HTML"), ("text/html", None)) - eq(self.db.guess_type("scheme:foobar.HTML"), ("text/html", None)) - eq(self.db.guess_file_type("foobar.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("scheme:foobar.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_file_type("foobar.TGZ"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("scheme:foobar.TGZ"), ("application/x-tar", "gzip")) - eq(self.db.guess_file_type("foobar.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("scheme:foobar.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_file_type("foobar.tar.z"), (None, None)) - eq(self.db.guess_type("scheme:foobar.tar.z"), (None, None)) - - def test_default_data(self): - eq = self.assertEqual - eq(self.db.guess_file_type("foo.html"), ("text/html", None)) - eq(self.db.guess_file_type("foo.HTML"), ("text/html", None)) - eq(self.db.guess_file_type("foo.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_file_type("foo.tar.gz"), ("application/x-tar", "gzip")) - eq(self.db.guess_file_type("foo.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_file_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) - eq(self.db.guess_file_type("foo.tar.xz"), ("application/x-tar", "xz")) - - def test_data_urls(self): - eq = self.assertEqual - guess_type = self.db.guess_type - eq(guess_type("data:invalidDataWithoutComma"), (None, None)) - eq(guess_type("data:,thisIsTextPlain"), ("text/plain", None)) - eq(guess_type("data:;base64,thisIsTextPlain"), ("text/plain", None)) - eq(guess_type("data:text/x-foo,thisIsTextXFoo"), ("text/x-foo", None)) - - def test_file_parsing(self): - eq = self.assertEqual - sio = io.StringIO("x-application/x-unittest pyunit\n") - self.db.readfp(sio) - eq(self.db.guess_file_type("foo.pyunit"), - ("x-application/x-unittest", None)) - eq(self.db.guess_extension("x-application/x-unittest"), ".pyunit") + mimetypes.inited = False + mimetypes._default_mime_types() + mimetypes._db = None def test_read_mime_types(self): eq = self.assertEqual @@ -106,103 +67,49 @@ class MimeTypesTestCase(unittest.TestCase): with unittest.mock.patch.object(mimetypes, 'open', return_value=fp) as mock_open: mime_dict = mimetypes.read_mime_types(filename) - mock_open.assert_called_with(filename, encoding='utf-8') + mock_open.assert_called_with(filename, encoding='utf-8', + errors='surrogateescape') eq(mime_dict[".Français"], "application/no-mans-land") - def test_non_standard_types(self): - eq = self.assertEqual - # First try strict - eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) - eq(self.db.guess_extension('image/jpg', strict=True), None) - # And then non-strict - eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) - eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) - eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) - eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') - eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') + def test_read_mime_types_invalid_utf8_comment(self): + with os_helper.temp_dir() as directory: + data = (b"# non-UTF-8 comment: \x83\n" + b"x-application/x-unittest pyunit\n") + file = os.path.join(directory, "sample.mimetype") + with open(file, "wb") as f: + f.write(data) - def test_filename_with_url_delimiters(self): - # bpo-38449: URL delimiters cases should be handled also. - # They would have different mime types if interpreted as URL as - # compared to when interpreted as filename because of the semicolon. - eq = self.assertEqual - gzip_expected = ('application/x-tar', 'gzip') - for name in ( - ';1.tar.gz', - '?1.tar.gz', - '#1.tar.gz', - '#1#.tar.gz', - ';1#.tar.gz', - ';&1=123;?.tar.gz', - '?k1=v1&k2=v2.tar.gz', - ): - for prefix in ('', '/', '\\', - 'c:', 'c:/', 'c:\\', 'c:/d/', 'c:\\d\\', - '//share/server/', '\\\\share\\server\\'): - path = prefix + name - with self.subTest(path=path): - eq(self.db.guess_file_type(path), gzip_expected) - eq(self.db.guess_type(path), gzip_expected) - expected = (None, None) if os.name == 'nt' else gzip_expected - for prefix in ('//', '\\\\', '//share/', '\\\\share\\'): - path = prefix + name - with self.subTest(path=path): - eq(self.db.guess_file_type(path), expected) - eq(self.db.guess_type(path), expected) - eq(self.db.guess_file_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) - eq(self.db.guess_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + mime_dict = mimetypes.read_mime_types(file) + self.assertEqual( + mime_dict[".pyunit"], "x-application/x-unittest") - eq(self.db.guess_file_type(r'foo/.tar.gz'), (None, 'gzip')) - eq(self.db.guess_type(r'foo/.tar.gz'), (None, 'gzip')) - expected = (None, 'gzip') if os.name == 'nt' else gzip_expected - eq(self.db.guess_file_type(r'foo\.tar.gz'), expected) - eq(self.db.guess_type(r'foo\.tar.gz'), expected) - eq(self.db.guess_type(r'scheme:foo\.tar.gz'), gzip_expected) + db = mimetypes.MimeTypes() + db.read(file) + self.assertEqual( + db.guess_file_type("sample.pyunit")[0], + "x-application/x-unittest") - def test_url(self): - result = self.db.guess_type('http://example.com/host.html') - result = self.db.guess_type('http://host.html') - msg = 'URL only has a host name, not a file' - self.assertSequenceEqual(result, (None, None), msg) - result = self.db.guess_type('http://example.com/host.html') - msg = 'Should be text/html' - self.assertSequenceEqual(result, ('text/html', None), msg) - result = self.db.guess_type('http://example.com/host.html#x.tar') - self.assertSequenceEqual(result, ('text/html', None)) - result = self.db.guess_type('http://example.com/host.html?q=x.tar') - self.assertSequenceEqual(result, ('text/html', None)) + mimetypes.init(files=[file]) + self.assertEqual( + mimetypes.guess_file_type("sample.pyunit")[0], + "x-application/x-unittest") - def test_guess_all_types(self): - # First try strict. Use a set here for testing the results because if - # test_urllib2 is run before test_mimetypes, global state is modified - # such that the 'all' set will have more items in it. - all = self.db.guess_all_extensions('text/plain', strict=True) - self.assertTrue(set(all) >= {'.bat', '.c', '.h', '.ksh', '.pl', '.txt'}) - self.assertEqual(len(set(all)), len(all)) # no duplicates - # And now non-strict - all = self.db.guess_all_extensions('image/jpg', strict=False) - self.assertEqual(all, ['.jpg']) - # And now for no hits - all = self.db.guess_all_extensions('image/jpg', strict=True) - self.assertEqual(all, []) - # And now for type existing in both strict and non-strict mappings. - self.db.add_type('test-type', '.strict-ext') - self.db.add_type('test-type', '.non-strict-ext', strict=False) - all = self.db.guess_all_extensions('test-type', strict=False) - self.assertEqual(all, ['.strict-ext', '.non-strict-ext']) - all = self.db.guess_all_extensions('test-type') - self.assertEqual(all, ['.strict-ext']) - # Test that changing the result list does not affect the global state - all.append('.no-such-ext') - all = self.db.guess_all_extensions('test-type') - self.assertNotIn('.no-such-ext', all) + def test_read_mime_types_invalid_utf8_type(self): + # A non-UTF-8 byte in a type or extension (not only in a comment) is + # preserved via surrogateescape, so the mapping is not corrupted. + with os_helper.temp_dir() as directory: + data = (b"x-application/x-unittest pyunit\n" + b"application/bad\x83 badext\x83\n") + file = os.path.join(directory, "sample.mimetype") + with open(file, "wb") as f: + f.write(data) - def test_encoding(self): - filename = support.findfile("mime.types") - mimes = mimetypes.MimeTypes([filename]) - exts = mimes.guess_all_extensions('application/vnd.geocube+xml', - strict=True) - self.assertEqual(exts, ['.g3', '.g\xb3']) + bad_type = b"application/bad\x83".decode("utf-8", "surrogateescape") + bad_ext = b".badext\x83".decode("utf-8", "surrogateescape") + + mime_dict = mimetypes.read_mime_types(file) + self.assertEqual(mime_dict[".pyunit"], "x-application/x-unittest") + self.assertEqual(mime_dict[bad_ext], bad_type) def test_init_reinitializes(self): # Issue 4936: make sure an init starts clean @@ -336,6 +243,203 @@ class MimeTypesTestCase(unittest.TestCase): self.assertEqual(types_map, mimetypes.types_map) self.assertEqual(common_types, mimetypes.common_types) + def test_init_files(self): + guess_file_type = mimetypes.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], None) + + filename = support.findfile("mime.types2") + mimetypes.init([filename]) + self.assertEqual(guess_file_type("file.test2")[0], "testing/test2") + + mimetypes.init() + self.assertEqual(guess_file_type("file.test2")[0], None) + + def test_init_knownfiles(self): + guess_file_type = mimetypes.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], None) + + filename = support.findfile("mime.types2") + mimetypes.knownfiles = [filename, "non-existent"] + self.addCleanup(mimetypes.knownfiles.clear) + + mimetypes.init() + self.assertEqual(guess_file_type("file.test2")[0], "testing/test2") + + def test_added_types_are_used(self): + mimetypes.add_type('testing/default-type', '') + mime_type, _ = mimetypes.guess_type('') + self.assertEqual(mime_type, 'testing/default-type') + + mime_type, _ = mimetypes.guess_type('test.myext') + self.assertEqual(mime_type, None) + + mimetypes.add_type('testing/type', '.myext') + mime_type, _ = mimetypes.guess_type('test.myext') + self.assertEqual(mime_type, 'testing/type') + + def test_add_type_with_undotted_extension_deprecated(self): + with self.assertWarns(DeprecationWarning): + mimetypes.add_type("testing/type", "undotted") + + +class MimeTypesClassTestCase(unittest.TestCase): + def setUp(self): + self.db = mimetypes.MimeTypes() + + def test_init_files(self): + guess_file_type = self.db.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], None) + + filename = support.findfile("mime.types2") + db = mimetypes.MimeTypes([filename]) + guess_file_type = db.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], "testing/test2") + + def test_init_knownfiles(self): + filename = support.findfile("mime.types2") + mimetypes.knownfiles = [filename, "non-existent"] + self.addCleanup(mimetypes.knownfiles.clear) + + db = mimetypes.MimeTypes() + guess_file_type = db.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], None) + + def test_case_sensitivity(self): + eq = self.assertEqual + eq(self.db.guess_file_type("foobar.html"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.html"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.HTML"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_type("scheme:foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_type("scheme:foobar.tar.z"), (None, None)) + + def test_default_data(self): + eq = self.assertEqual + eq(self.db.guess_file_type("foo.html"), ("text/html", None)) + eq(self.db.guess_file_type("foo.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foo.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.gz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) + eq(self.db.guess_file_type("foo.tar.xz"), ("application/x-tar", "xz")) + + def test_data_urls(self): + eq = self.assertEqual + guess_type = self.db.guess_type + eq(guess_type("data:invalidDataWithoutComma"), (None, None)) + eq(guess_type("data:,thisIsTextPlain"), ("text/plain", None)) + eq(guess_type("data:;base64,thisIsTextPlain"), ("text/plain", None)) + eq(guess_type("data:text/x-foo,thisIsTextXFoo"), ("text/x-foo", None)) + + def test_file_parsing(self): + eq = self.assertEqual + sio = io.StringIO("x-application/x-unittest pyunit\n") + self.db.readfp(sio) + eq(self.db.guess_file_type("foo.pyunit"), + ("x-application/x-unittest", None)) + eq(self.db.guess_extension("x-application/x-unittest"), ".pyunit") + + def test_non_standard_types(self): + eq = self.assertEqual + # First try strict + eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) + eq(self.db.guess_extension('image/jpg', strict=True), None) + # And then non-strict + eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) + eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') + eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') + + def test_filename_with_url_delimiters(self): + # bpo-38449: URL delimiters cases should be handled also. + # They would have different mime types if interpreted as URL as + # compared to when interpreted as filename because of the semicolon. + eq = self.assertEqual + gzip_expected = ('application/x-tar', 'gzip') + for name in ( + ';1.tar.gz', + '?1.tar.gz', + '#1.tar.gz', + '#1#.tar.gz', + ';1#.tar.gz', + ';&1=123;?.tar.gz', + '?k1=v1&k2=v2.tar.gz', + ): + for prefix in ('', '/', '\\', + 'c:', 'c:/', 'c:\\', 'c:/d/', 'c:\\d\\', + '//share/server/', '\\\\share\\server\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), gzip_expected) + eq(self.db.guess_type(path), gzip_expected) + expected = (None, None) if os.name == 'nt' else gzip_expected + for prefix in ('//', '\\\\', '//share/', '\\\\share\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), expected) + eq(self.db.guess_type(path), expected) + eq(self.db.guess_file_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + eq(self.db.guess_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + + eq(self.db.guess_file_type(r'foo/.tar.gz'), (None, 'gzip')) + eq(self.db.guess_type(r'foo/.tar.gz'), (None, 'gzip')) + expected = (None, 'gzip') if os.name == 'nt' else gzip_expected + eq(self.db.guess_file_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'scheme:foo\.tar.gz'), gzip_expected) + + def test_url(self): + result = self.db.guess_type('http://example.com/host.html') + result = self.db.guess_type('http://host.html') + msg = 'URL only has a host name, not a file' + self.assertSequenceEqual(result, (None, None), msg) + result = self.db.guess_type('http://example.com/host.html') + msg = 'Should be text/html' + self.assertSequenceEqual(result, ('text/html', None), msg) + result = self.db.guess_type('http://example.com/host.html#x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + result = self.db.guess_type('http://example.com/host.html?q=x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + + def test_guess_all_types(self): + # First try strict. Use a set here for testing the results because if + # test_urllib2 is run before test_mimetypes, global state is modified + # such that the 'all' set will have more items in it. + all = self.db.guess_all_extensions('text/plain', strict=True) + self.assertTrue(set(all) >= {'.bat', '.c', '.h', '.ksh', '.pl', '.txt'}) + self.assertEqual(len(set(all)), len(all)) # no duplicates + # And now non-strict + all = self.db.guess_all_extensions('image/jpg', strict=False) + self.assertEqual(all, ['.jpg']) + # And now for no hits + all = self.db.guess_all_extensions('image/jpg', strict=True) + self.assertEqual(all, []) + # And now for type existing in both strict and non-strict mappings. + self.db.add_type('test-type', '.strict-ext') + self.db.add_type('test-type', '.non-strict-ext', strict=False) + all = self.db.guess_all_extensions('test-type', strict=False) + self.assertEqual(all, ['.strict-ext', '.non-strict-ext']) + all = self.db.guess_all_extensions('test-type') + self.assertEqual(all, ['.strict-ext']) + # Test that changing the result list does not affect the global state + all.append('.no-such-ext') + all = self.db.guess_all_extensions('test-type') + self.assertNotIn('.no-such-ext', all) + + def test_encoding(self): + filename = support.findfile("mime.types") + mimes = mimetypes.MimeTypes([filename]) + exts = mimes.guess_all_extensions('application/vnd.geocube+xml', + strict=True) + self.assertEqual(exts, ['.g3', '.g\xb3']) + def test_path_like_ob(self): filename = "LICENSE.txt" filepath = os_helper.FakePath(filename) @@ -375,22 +479,6 @@ class MimeTypesTestCase(unittest.TestCase): self.assertEqual(self.db.guess_extension( type='image/jpg', strict=False), '.jpg') - def test_added_types_are_used(self): - mimetypes.add_type('testing/default-type', '') - mime_type, _ = mimetypes.guess_type('') - self.assertEqual(mime_type, 'testing/default-type') - - mime_type, _ = mimetypes.guess_type('test.myext') - self.assertEqual(mime_type, None) - - mimetypes.add_type('testing/type', '.myext') - mime_type, _ = mimetypes.guess_type('test.myext') - self.assertEqual(mime_type, 'testing/type') - - def test_add_type_with_undotted_extension_deprecated(self): - with self.assertWarns(DeprecationWarning): - mimetypes.add_type("testing/type", "undotted") - @unittest.skipUnless(sys.platform.startswith("win"), "Windows only") class Win32MimeTypesTestCase(unittest.TestCase): diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index f44c56da6..fc6c830be 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -2243,7 +2243,7 @@ class TestHelper(unittest.TestCase): def test_keywords(self): self.assertEqual(sorted(pydoc.Helper.keywords), - sorted(keyword.kwlist)) + sorted(keyword.kwlist + ['case', 'match'])) def test_interact_empty_line_continues(self): # gh-138568: test pressing Enter without input should continue in help session