diff --git a/Lib/importlib/_abc.py b/Lib/importlib/_abc.py index 693b46611..2598036da 100644 --- a/Lib/importlib/_abc.py +++ b/Lib/importlib/_abc.py @@ -19,21 +19,3 @@ class Loader(metaclass=abc.ABCMeta): # We don't define exec_module() here since that would break # hasattr checks we do to support backward compatibility. - - def load_module(self, fullname): - """Return the loaded module. - - The module must be added to sys.modules and have import-related - attributes set properly. The fullname is a str. - - ImportError is raised on failure. - - This method is deprecated in favor of loader.exec_module(). If - exec_module() exists then it is used to provide a backwards-compatible - functionality for this method. - - """ - if not hasattr(self, 'exec_module'): - raise ImportError - # Warning implemented in _load_module_shim(). - return _bootstrap._load_module_shim(self, fullname) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 9d911e1dc..75bc901e2 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -424,6 +424,64 @@ class _ModuleLockManager: self._lock.release() +def _get_module_chain(name): + """Return the chain of dotted-name prefixes from root to leaf. + + For example: 'a.b.c' -> ['a', 'a.b', 'a.b.c'] + """ + parts = name.split('.') + return ['.'.join(parts[:i+1]) for i in range(len(parts))] + + +class _HierarchicalLockManager: + """Manages acquisition of multiple module locks in hierarchical order. + + This prevents deadlocks by ensuring all threads acquire locks in the + same order (parent modules before child modules). + """ + + def __init__(self, name): + self._name = name + self._module_chain = _get_module_chain(name) + self._locks = [] + + def __enter__(self): + try: + for module_name in self._module_chain: + # Only acquire lock if module is not already fully loaded + module = sys.modules.get(module_name) + if (module is None or + getattr(getattr(module, "__spec__", None), + "_initializing", False)): + lock = _get_module_lock(module_name) + try: + lock.acquire() + except _DeadlockError: + if module_name == self._name: + raise + # The parent is being initialised by a thread that + # is (transitively) waiting on a lock we hold. + # Apply the same policy as _lock_unlock_module(): + # accept a partially-initialised parent for circular + # imports rather than failing the whole chain. + continue + self._locks.append((module_name, lock)) + except: + # __exit__ is not called when __enter__ raises (e.g. _DeadlockError + # on the leaf lock, or KeyboardInterrupt), so release whatever we + # already hold to avoid permanently leaking held module locks. + for module_name, lock in reversed(self._locks): + lock.release() + self._locks.clear() + raise + return self + + def __exit__(self, *args, **kwargs): + for module_name, lock in reversed(self._locks): + lock.release() + self._locks.clear() + + # The following two functions are for consumption by Python/import.c. def _get_module_lock(name): @@ -521,24 +579,6 @@ def _requires_frozen(fxn): return _requires_frozen_wrapper -# Typically used by loader classes as a method replacement. -def _load_module_shim(self, fullname): - """Load the specified module into sys.modules and return it. - - This method is deprecated. Use loader.exec_module() instead. - - """ - msg = ("the load_module() method is deprecated and slated for removal in " - "Python 3.15; use exec_module() instead") - _warnings.warn(msg, DeprecationWarning) - spec = spec_from_loader(fullname, self) - if fullname in sys.modules: - module = sys.modules[fullname] - _exec(spec, module) - return sys.modules[fullname] - else: - return _load(spec) - # Module specifications ####################################################### def _module_repr(module): @@ -583,8 +623,7 @@ class ModuleSpec: `has_location` indicates that a spec's "origin" reflects a location. When this is True, `__file__` attribute of the module is set. - `cached` is the location of the cached bytecode file, if any. It - corresponds to the `__cached__` attribute. + `cached` is the location of the cached bytecode file, if any. `submodule_search_locations` is the sequence of path entries to search when importing submodules. If set, is_package should be @@ -717,10 +756,6 @@ def _spec_from_module(module, loader=None, origin=None): origin = getattr(loader, '_ORIGIN', None) if not origin and location is not None: origin = location - try: - cached = module.__cached__ - except AttributeError: - cached = None try: submodule_search_locations = list(module.__path__) except AttributeError: @@ -728,7 +763,7 @@ def _spec_from_module(module, loader=None, origin=None): spec = ModuleSpec(name, loader, origin=origin) spec._set_fileattr = False if location is None else (origin == location) - spec.cached = cached + spec.cached = None spec.submodule_search_locations = submodule_search_locations return spec @@ -763,8 +798,7 @@ def _init_module_attrs(spec, module, *, override=False): # should also be None for consistency. While a bit of a hack, # this is the best place to ensure this consistency. # - # See # https://docs.python.org/3/library/importlib.html#importlib.abc.Loader.load_module - # and bpo-32305 + # See bpo-32305 module.__file__ = None try: module.__loader__ = loader @@ -789,7 +823,7 @@ def _init_module_attrs(spec, module, *, override=False): module.__path__ = spec.submodule_search_locations except AttributeError: pass - # __file__/__cached__ + # __file__ if spec.has_location: if override or getattr(module, '__file__', None) is None: try: @@ -797,12 +831,6 @@ def _init_module_attrs(spec, module, *, override=False): except AttributeError: pass - if override or getattr(module, '__cached__', None) is None: - if spec.cached is not None: - try: - module.__cached__ = spec.cached - except AttributeError: - pass return module @@ -844,7 +872,7 @@ def _module_repr_from_spec(spec): return f'' -# Used by importlib.reload() and _load_module_shim(). +# Used by importlib.reload(). def _exec(spec, module): """Execute the spec's specified module in an existing module's namespace.""" name = spec.name @@ -860,13 +888,7 @@ def _exec(spec, module): _init_module_attrs(spec, module, override=True) else: _init_module_attrs(spec, module, override=True) - if not hasattr(spec.loader, 'exec_module'): - msg = (f"{_object_name(spec.loader)}.exec_module() not found; " - "falling back to load_module()") - _warnings.warn(msg, ImportWarning) - spec.loader.load_module(name) - else: - spec.loader.exec_module(module) + spec.loader.exec_module(module) finally: # Update the order of insertion into sys.modules for module # clean-up at shutdown. @@ -874,53 +896,8 @@ def _exec(spec, module): sys.modules[spec.name] = module return module - -def _load_backward_compatible(spec): - # It is assumed that all callers have been warned about using load_module() - # appropriately before calling this function. - try: - spec.loader.load_module(spec.name) - except: - if spec.name in sys.modules: - module = sys.modules.pop(spec.name) - sys.modules[spec.name] = module - raise - # The module must be in sys.modules at this point! - # Move it to the end of sys.modules. - module = sys.modules.pop(spec.name) - sys.modules[spec.name] = module - if getattr(module, '__loader__', None) is None: - try: - module.__loader__ = spec.loader - except AttributeError: - pass - if getattr(module, '__package__', None) is None: - try: - # Since module.__path__ may not line up with - # spec.submodule_search_paths, we can't necessarily rely - # on spec.parent here. - module.__package__ = module.__name__ - if not hasattr(module, '__path__'): - module.__package__ = spec.name.rpartition('.')[0] - except AttributeError: - pass - if getattr(module, '__spec__', None) is None: - try: - module.__spec__ = spec - except AttributeError: - pass - return module - def _load_unlocked(spec): # A helper for direct use by the import system. - if spec.loader is not None: - # Not a namespace package. - if not hasattr(spec.loader, 'exec_module'): - msg = (f"{_object_name(spec.loader)}.exec_module() not found; " - "falling back to load_module()") - _warnings.warn(msg, ImportWarning) - return _load_backward_compatible(spec) - module = module_from_spec(spec) # This must be done before putting the module in sys.modules @@ -954,8 +931,7 @@ def _load_unlocked(spec): return module -# A method used during testing of _load_unlocked() and by -# _load_module_shim(). +# A method used during testing of _load_unlocked(). def _load(spec): """Return a new module object, loaded by the spec's loader. @@ -1020,8 +996,6 @@ class BuiltinImporter: """Return False as built-in modules are never packages.""" return False - load_module = classmethod(_load_module_shim) - class FrozenImporter: @@ -1178,25 +1152,6 @@ class FrozenImporter: code = _call_with_frames_removed(_imp.get_frozen_object, name) exec(code, module.__dict__) - @classmethod - def load_module(cls, fullname): - """Load a frozen module. - - This method is deprecated. Use exec_module() instead. - - """ - # Warning about deprecation implemented in _load_module_shim(). - module = _load_module_shim(cls, fullname) - info = _imp.find_frozen(fullname) - assert info is not None - _, ispkg, origname = info - module.__origname__ = origname - vars(module).pop('__file__', None) - if ispkg: - module.__path__ = [] - cls._fix_up_module(module) - return module - @classmethod @_requires_frozen def get_code(cls, fullname): @@ -1305,8 +1260,16 @@ def _sanity_check(name, package, level): _ERR_MSG_PREFIX = 'No module named ' -def _find_and_load_unlocked(name, import_): +def _find_and_load_unlocked(name, import_, *, lazy_submodule=False): path = None + sys.audit( + "import", + name, + path, + getattr(sys, "path", None), + getattr(sys, "meta_path", None), + getattr(sys, "path_hooks", None) + ) parent = name.rpartition('.')[0] parent_spec = None if parent: @@ -1320,6 +1283,8 @@ def _find_and_load_unlocked(name, import_): try: path = parent_module.__path__ except AttributeError: + if lazy_submodule: + return None msg = f'{_ERR_MSG_PREFIX}{name!r}; {parent!r} is not a package' raise ModuleNotFoundError(msg, name=name) from None parent_spec = parent_module.__spec__ @@ -1332,6 +1297,8 @@ def _find_and_load_unlocked(name, import_): child = name.rpartition('.')[2] spec = _find_spec(name, path) if spec is None: + if lazy_submodule: + return None raise ModuleNotFoundError(f'{_ERR_MSG_PREFIX}{name!r}', name=name) else: if parent_spec: @@ -1351,13 +1318,19 @@ def _find_and_load_unlocked(name, import_): except AttributeError: msg = f"Cannot set an attribute on {parent!r} for child module {child!r}" _warnings.warn(msg, ImportWarning) + # Set attributes to lazy submodules on the module. + try: + _imp._set_lazy_attributes(module, name) + except Exception as e: + msg = f"Cannot set lazy attributes on {name!r}: {e!r}" + _warnings.warn(msg, ImportWarning) return module _NEEDS_LOADING = object() -def _find_and_load(name, import_): +def _find_and_load(name, import_, *, lazy_submodule=False): """Find and load the module.""" # Optimization: we avoid unneeded module locking if the module @@ -1365,10 +1338,17 @@ def _find_and_load(name, import_): module = sys.modules.get(name, _NEEDS_LOADING) if (module is _NEEDS_LOADING or getattr(getattr(module, "__spec__", None), "_initializing", False)): - with _ModuleLockManager(name): + + if '.' in name: + lock_manager = _HierarchicalLockManager(name) + else: + lock_manager = _ModuleLockManager(name) + + with lock_manager: module = sys.modules.get(name, _NEEDS_LOADING) if module is _NEEDS_LOADING: - return _find_and_load_unlocked(name, import_) + return _find_and_load_unlocked( + name, import_, lazy_submodule=lazy_submodule) # Optimization: only call _bootstrap._lock_unlock_module() if # module.__spec__._initializing is True. @@ -1382,7 +1362,7 @@ def _find_and_load(name, import_): # to preserve normal semantics: the caller gets the exception from # the actual import failure rather than a synthetic error. if sys.modules.get(name) is not module: - return _find_and_load(name, import_) + return _find_and_load(name, import_, lazy_submodule=lazy_submodule) if module is None: message = f'import of {name} halted; None in sys.modules' @@ -1391,6 +1371,10 @@ def _find_and_load(name, import_): return module +def _find_and_load_lazy_submodule(name, import_): + return _find_and_load(name, import_, lazy_submodule=True) + + def _gcd_import(name, package=None, level=0): """Import and return the module based on its name, the package the call is being made from, and the level adjustment. diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 6a828ae75..a1cb729ef 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -236,7 +236,7 @@ BYTECODE_SUFFIXES = ['.pyc'] # Deprecated. DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES -def cache_from_source(path, debug_override=None, *, optimization=None): +def cache_from_source(path, *, optimization=None): """Given the path to a .py file, return the path to its .pyc file. The .py file does not need to exist; this simply returns the path to the @@ -247,20 +247,9 @@ def cache_from_source(path, debug_override=None, *, optimization=None): of the argument is taken and verified to be alphanumeric (else ValueError is raised). - The debug_override parameter is deprecated. If debug_override is not None, - a True value is the same as setting 'optimization' to the empty string - while a False value is equivalent to setting 'optimization' to '1'. - If sys.implementation.cache_tag is None then NotImplementedError is raised. """ - if debug_override is not None: - _warnings.warn('the debug_override parameter is deprecated; use ' - "'optimization' instead", DeprecationWarning) - if optimization is not None: - message = 'debug_override or optimization must be set to None' - raise TypeError(message) - optimization = '' if debug_override else 1 path = _os.fspath(path) head, tail = _path_split(path) base, sep, rest = tail.rpartition('.') @@ -548,8 +537,7 @@ def decode_source(source_bytes): import tokenize # To avoid bootstrap issues. source_bytes_readline = _io.BytesIO(source_bytes).readline encoding = tokenize.detect_encoding(source_bytes_readline) - newline_decoder = _io.IncrementalNewlineDecoder(None, True) - return newline_decoder.decode(source_bytes.decode(encoding[0])) + return _io.TextIOWrapper(_io.BytesIO(source_bytes), encoding=encoding[0], newline=None).read() # Module specifications ####################################################### @@ -758,11 +746,6 @@ class _LoaderBasics: 'get_code() returns None') _bootstrap._call_with_frames_removed(exec, code, module.__dict__) - def load_module(self, fullname): - """This method is deprecated.""" - # Warning implemented in _load_module_shim(). - return _bootstrap._load_module_shim(self, fullname) - class SourceLoader(_LoaderBasics): @@ -815,13 +798,14 @@ class SourceLoader(_LoaderBasics): name=fullname) from exc return decode_source(source_bytes) - def source_to_code(self, data, path, *, _optimize=-1): + def source_to_code(self, data, path, fullname=None, *, _optimize=-1): """Return the code object compiled from source. The 'data' argument can be any object type that compile() supports. """ return _bootstrap._call_with_frames_removed(compile, data, path, 'exec', - dont_inherit=True, optimize=_optimize) + dont_inherit=True, optimize=_optimize, + module=fullname) def get_code(self, fullname): """Concrete implementation of InspectLoader.get_code. @@ -890,7 +874,7 @@ class SourceLoader(_LoaderBasics): source_path=source_path) if source_bytes is None: source_bytes = self.get_data(source_path) - code_object = self.source_to_code(source_bytes, source_path) + code_object = self.source_to_code(source_bytes, source_path, fullname) _bootstrap._verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and source_mtime is not None): @@ -927,18 +911,6 @@ class FileLoader: def __hash__(self): return hash(self.name) ^ hash(self.path) - @_check_name - def load_module(self, fullname): - """Load a module from a file. - - This method is deprecated. Use exec_module() instead. - - """ - # The only reason for this method is for the name check. - # Issue #14857: Avoid the zero-argument form of super so the implementation - # of that form can be updated without breaking the frozen module. - return super(FileLoader, self).load_module(fullname) - @_check_name def get_filename(self, fullname): """Return the path to the source file as found by the finder.""" @@ -1082,12 +1054,18 @@ class ExtensionFileLoader(FileLoader, _LoaderBasics): return self.path -class _NamespacePath: - """Represents a namespace package's path. It uses the module name - to find its parent module, and from there it looks up the parent's - __path__. When this changes, the module's own path is recomputed, - using path_finder. For top-level modules, the parent module's path - is sys.path.""" +class NamespacePath: + """Represents a namespace package's path. + + It uses the module *name* to find its parent module, and from there it looks + up the parent's __path__. When this changes, the module's own path is + recomputed, using *path_finder*. The initial value is set to *path*. + + For top-level modules, the parent module's path is sys.path. + + *path_finder* should be a callable with the same signature as + MetaPathFinder.find_spec((fullname, path, target=None) -> spec). + """ # When invalidate_caches() is called, this epoch is incremented # https://bugs.python.org/issue45703 @@ -1112,7 +1090,15 @@ class _NamespacePath: def _get_parent_path(self): parent_module_name, path_attr_name = self._find_parent_path_names() - return getattr(sys.modules[parent_module_name], path_attr_name) + try: + module = sys.modules[parent_module_name] + except KeyError as e: + raise ModuleNotFoundError( + f"{parent_module_name!r} must be imported before finding {self._name!r}.", + name=parent_module_name, + ) from e + else: + return getattr(module, path_attr_name) def _recalculate(self): # If the parent's path has changed, recalculate _path @@ -1141,7 +1127,7 @@ class _NamespacePath: return len(self._recalculate()) def __repr__(self): - return f'_NamespacePath({self._path!r})' + return f'NamespacePath({self._path!r})' def __contains__(self, item): return item in self._recalculate() @@ -1150,12 +1136,16 @@ class _NamespacePath: self._path.append(item) +# For backwards-compatibility for anyone desperate enough to get at the class back in the day. +_NamespacePath = NamespacePath + + # This class is actually exposed publicly in a namespace package's __loader__ # attribute, so it should be available through a non-private name. # https://github.com/python/cpython/issues/92054 class NamespaceLoader: def __init__(self, name, path, path_finder): - self._path = _NamespacePath(name, path, path_finder) + self._path = NamespacePath(name, path, path_finder) def is_package(self, fullname): return True @@ -1164,7 +1154,7 @@ class NamespaceLoader: return '' def get_code(self, fullname): - return compile('', '', 'exec', dont_inherit=True) + return compile('', '', 'exec', dont_inherit=True, module=fullname) def create_module(self, spec): """Use default semantics for module creation.""" @@ -1172,18 +1162,6 @@ class NamespaceLoader: def exec_module(self, module): pass - def load_module(self, fullname): - """Load a namespace module. - - This method is deprecated. Use exec_module() instead. - - """ - # The import system never calls this method. - _bootstrap._verbose_message('namespace module loaded with path {!r}', - self._path) - # Warning implemented in _load_module_shim(). - return _bootstrap._load_module_shim(self, fullname) - def get_resource_reader(self, module): from importlib.readers import NamespaceReader return NamespaceReader(self._path) @@ -1210,9 +1188,9 @@ class PathFinder: del sys.path_importer_cache[name] elif hasattr(finder, 'invalidate_caches'): finder.invalidate_caches() - # Also invalidate the caches of _NamespacePaths + # Also invalidate the caches of NamespacePaths # https://bugs.python.org/issue45703 - _NamespacePath._epoch += 1 + NamespacePath._epoch += 1 from importlib.metadata import MetadataPathFinder MetadataPathFinder.invalidate_caches() @@ -1298,13 +1276,30 @@ class PathFinder: # We found at least one namespace path. Return a spec which # can create the namespace package. spec.origin = None - spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec) + spec.submodule_search_locations = NamespacePath(fullname, namespace_path, cls._get_spec) return spec else: return None else: return spec + @classmethod + def discover(cls, parent=None): + if parent is None: + path = sys.path + elif parent.submodule_search_locations is None: + raise ValueError(f'{parent} is not a package module') + else: + path = parent.submodule_search_locations + + for entry in set(path): + if not isinstance(entry, str): + continue + if (finder := cls._path_importer_cache(entry)) is None: + continue + if discover := getattr(finder, 'discover', None): + yield from discover(parent) + @staticmethod def find_distributions(*args, **kwargs): """ @@ -1454,6 +1449,37 @@ class FileFinder: return path_hook_for_FileFinder + def _find_children(self): + with _os.scandir(self.path) as scan_iterator: + while True: + try: + entry = next(scan_iterator) + if entry.name == _PYCACHE: + continue + # packages + if entry.is_dir() and '.' not in entry.name: + yield entry.name + # files + if entry.is_file(): + yield from { + entry.name.removesuffix(suffix) + for suffix, _ in self._loaders + if entry.name.endswith(suffix) + } + except OSError: + pass # ignore exceptions from next(scan_iterator) and os.DirEntry + except StopIteration: + break + + def discover(self, parent=None): + if parent and parent.submodule_search_locations is None: + raise ValueError(f'{parent} is not a package module') + + module_prefix = f'{parent.name}.' if parent else '' + for child_name in self._find_children(): + if spec := self.find_spec(module_prefix + child_name): + yield spec + def __repr__(self): return f'FileFinder({self.path!r})' @@ -1525,7 +1551,6 @@ def _fix_up_module(ns, name, pathname, cpathname=None): ns['__spec__'] = spec ns['__loader__'] = loader ns['__file__'] = pathname - ns['__cached__'] = cpathname except Exception: # Not important enough to report. pass diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 1e47495f6..9ca127ad9 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -45,6 +45,16 @@ class MetaPathFinder(metaclass=abc.ABCMeta): This method is used by importlib.invalidate_caches(). """ + def discover(self, parent=None): + """An optional method which searches for possible specs with given *parent* + module spec. If *parent* is *None*, MetaPathFinder.discover will search + for top-level modules. + + Returns an iterable of possible specs. + """ + return () + + _register(MetaPathFinder, machinery.BuiltinImporter, machinery.FrozenImporter, machinery.PathFinder, machinery.WindowsRegistryFinder) @@ -58,6 +68,15 @@ class PathEntryFinder(metaclass=abc.ABCMeta): This method is used by PathFinder.invalidate_caches(). """ + def discover(self, parent=None): + """An optional method which searches for possible specs with given + *parent* module spec. If *parent* is *None*, PathEntryFinder.discover + will search for top-level modules. + + Returns an iterable of possible specs. + """ + return () + _register(PathEntryFinder, machinery.FileFinder) @@ -108,7 +127,7 @@ class InspectLoader(Loader): source = self.get_source(fullname) if source is None: return None - return self.source_to_code(source) + return self.source_to_code(source, '', fullname) @abc.abstractmethod def get_source(self, fullname): @@ -120,15 +139,14 @@ class InspectLoader(Loader): raise ImportError @staticmethod - def source_to_code(data, path=''): + def source_to_code(data, path='', fullname=None): """Compile 'data' into a code object. The 'data' argument can be anything that compile() can handle. The'path' argument should be where the data was retrieved (when applicable).""" - return compile(data, path, 'exec', dont_inherit=True) + return compile(data, path, 'exec', dont_inherit=True, module=fullname) exec_module = _bootstrap_external._LoaderBasics.exec_module - load_module = _bootstrap_external._LoaderBasics.load_module _register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter, machinery.NamespaceLoader) @@ -163,9 +181,8 @@ class ExecutionLoader(InspectLoader): try: path = self.get_filename(fullname) except ImportError: - return self.source_to_code(source) - else: - return self.source_to_code(source, path) + path = '' + return self.source_to_code(source, path, fullname) _register( ExecutionLoader, diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index 63d726445..023f77d75 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -16,6 +16,7 @@ from ._bootstrap_external import SourcelessFileLoader from ._bootstrap_external import ExtensionFileLoader from ._bootstrap_external import AppleFrameworkLoader from ._bootstrap_external import NamespaceLoader +from ._bootstrap_external import NamespacePath def all_suffixes(): diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index 8ce62dd86..32f4b7d2d 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -1,40 +1,50 @@ +""" +APIs exposing metadata from third-party Python packages. + +This codebase is shared between importlib.metadata in the stdlib +and importlib_metadata in PyPI. See +https://github.com/python/importlib_metadata/wiki/Development-Methodology +for more detail. +""" + from __future__ import annotations -import os -import re import abc -import sys -import json +import collections import email -import types -import inspect -import pathlib -import zipfile -import operator -import textwrap -import warnings import functools import itertools +import operator +import os +import pathlib import posixpath -import collections - -from . import _meta -from ._collections import FreezableDefaultDict, Pair -from ._functools import method_cache, pass_none -from ._itertools import always_iterable, unique_everseen -from ._meta import PackageMetadata, SimplePath - +import re +import sys +import textwrap +import types +from collections.abc import Iterable, Mapping from contextlib import suppress from importlib import import_module from importlib.abc import MetaPathFinder from itertools import starmap -from typing import Any, Iterable, List, Mapping, Match, Optional, Set, cast +from typing import Any + +from . import _meta +from ._collections import FreezableDefaultDict, Pair +from ._context import ExceptionTrap +from ._functools import method_cache, noop, pass_none, passthrough +from ._itertools import always_iterable, bucket, unique_everseen +from ._meta import PackageMetadata, SimplePath +from ._typing import md_none __all__ = [ 'Distribution', 'DistributionFinder', 'PackageMetadata', 'PackageNotFoundError', + 'PackagePath', + 'MetadataNotFound', + 'SimplePath', 'distribution', 'distributions', 'entry_points', @@ -53,11 +63,15 @@ class PackageNotFoundError(ModuleNotFoundError): return f"No package metadata was found for {self.name}" @property - def name(self) -> str: # type: ignore[override] + def name(self) -> str: # type: ignore[override] # make readonly (name,) = self.args return name +class MetadataNotFound(FileNotFoundError): + """No metadata file is present in the distribution.""" + + class Sectioned: """ A simple entry point config parser for performance @@ -123,6 +137,12 @@ class Sectioned: return line and not line.startswith('#') +class _EntryPointMatch(types.SimpleNamespace): + module: str + attr: str + extras: str + + class EntryPoint: """An entry point as defined by Python packaging conventions. @@ -138,6 +158,30 @@ class EntryPoint: 'attr' >>> ep.extras ['extra1', 'extra2'] + + If the value package or module are not valid identifiers, a + ValueError is raised on access. + + >>> EntryPoint(name=None, group=None, value='invalid-name').module + Traceback (most recent call last): + ... + ValueError: ('Invalid object reference...invalid-name... + >>> EntryPoint(name=None, group=None, value='invalid-name').attr + Traceback (most recent call last): + ... + ValueError: ('Invalid object reference...invalid-name... + >>> EntryPoint(name=None, group=None, value='invalid-name').extras + Traceback (most recent call last): + ... + ValueError: ('Invalid object reference...invalid-name... + + The same thing happens on construction. + + >>> EntryPoint(name=None, group=None, value='invalid-name') + Traceback (most recent call last): + ... + ValueError: ('Invalid object reference...invalid-name... + """ pattern = re.compile( @@ -165,38 +209,44 @@ class EntryPoint: value: str group: str - dist: Optional[Distribution] = None + dist: Distribution | None = None def __init__(self, name: str, value: str, group: str) -> None: vars(self).update(name=name, value=value, group=group) + self.module def load(self) -> Any: """Load the entry point from its definition. If only a module is indicated by the value, return that module. Otherwise, return the named object. """ - match = cast(Match, self.pattern.match(self.value)) - module = import_module(match.group('module')) - attrs = filter(None, (match.group('attr') or '').split('.')) + module = import_module(self.module) + attrs = filter(None, (self.attr or '').split('.')) return functools.reduce(getattr, attrs, module) @property def module(self) -> str: - match = self.pattern.match(self.value) - assert match is not None - return match.group('module') + return self._match.module @property def attr(self) -> str: - match = self.pattern.match(self.value) - assert match is not None - return match.group('attr') + return self._match.attr @property - def extras(self) -> List[str]: + def extras(self) -> list[str]: + return re.findall(r'\w+', self._match.extras or '') + + @functools.cached_property + def _match(self) -> _EntryPointMatch: match = self.pattern.match(self.value) - assert match is not None - return re.findall(r'\w+', match.group('extras') or '') + if not match: + raise ValueError( + 'Invalid object reference. ' + 'See https://packaging.python.org' + '/en/latest/specifications/entry-points/#data-model', + self.value, + ) + return _EntryPointMatch(**match.groupdict()) def _for(self, dist): vars(self).update(dist=dist) @@ -222,9 +272,26 @@ class EntryPoint: >>> ep.matches(attr='bong') True """ + self._disallow_dist(params) attrs = (getattr(self, param) for param in params) return all(map(operator.eq, params.values(), attrs)) + @staticmethod + def _disallow_dist(params): + """ + Querying by dist is not allowed (dist objects are not comparable). + >>> EntryPoint(name='fan', value='fav', group='fag').matches(dist='foo') + Traceback (most recent call last): + ... + ValueError: "dist" is not suitable for matching... + """ + if "dist" in params: + raise ValueError( + '"dist" is not suitable for matching. ' + "Instead, use Distribution.entry_points.select() on a " + "located distribution." + ) + def _key(self): return self.name, self.value, self.group @@ -254,7 +321,7 @@ class EntryPoints(tuple): __slots__ = () - def __getitem__(self, name: str) -> EntryPoint: # type: ignore[override] + def __getitem__(self, name: str) -> EntryPoint: # type: ignore[override] # Work with str instead of int """ Get the EntryPoint in self matching name. """ @@ -278,14 +345,14 @@ class EntryPoints(tuple): return EntryPoints(ep for ep in self if ep.matches(**params)) @property - def names(self) -> Set[str]: + def names(self) -> set[str]: """ Return the set of all names of all entry points. """ return {ep.name for ep in self} @property - def groups(self) -> Set[str]: + def groups(self) -> set[str]: """ Return the set of all groups of all entry points. """ @@ -306,11 +373,11 @@ class EntryPoints(tuple): class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" - hash: Optional[FileHash] + hash: FileHash | None size: int dist: Distribution - def read_text(self, encoding: str = 'utf-8') -> str: # type: ignore[override] + def read_text(self, encoding: str = 'utf-8') -> str: return self.locate().read_text(encoding=encoding) def read_binary(self) -> bytes: @@ -329,27 +396,7 @@ class FileHash: return f'' -class DeprecatedNonAbstract: - # Required until Python 3.14 - def __new__(cls, *args, **kwargs): - all_names = { - name for subclass in inspect.getmro(cls) for name in vars(subclass) - } - abstract = { - name - for name in all_names - if getattr(getattr(cls, name), '__isabstractmethod__', False) - } - if abstract: - warnings.warn( - f"Unimplemented abstract methods {abstract}", - DeprecationWarning, - stacklevel=2, - ) - return super().__new__(cls) - - -class Distribution(DeprecatedNonAbstract): +class Distribution(metaclass=abc.ABCMeta): """ An abstract Python distribution package. @@ -361,7 +408,7 @@ class Distribution(DeprecatedNonAbstract): """ @abc.abstractmethod - def read_text(self, filename) -> Optional[str]: + def read_text(self, filename) -> str | None: """Attempt to load metadata file given by the name. Python distribution metadata is organized by blobs of text @@ -388,6 +435,17 @@ class Distribution(DeprecatedNonAbstract): """ Given a path to a file in this distribution, return a SimplePath to it. + + This method is used by callers of ``Distribution.files()`` to + locate files within the distribution. If it's possible for a + Distribution to represent files in the distribution as + ``SimplePath`` objects, it should implement this method + to resolve such objects. + + Some Distribution providers may elect not to resolve SimplePath + objects within the distribution by raising a + NotImplementedError, but consumers of such a Distribution would + be unable to invoke ``Distribution.files()``. """ @classmethod @@ -404,13 +462,13 @@ class Distribution(DeprecatedNonAbstract): if not name: raise ValueError("A distribution name is required.") try: - return next(iter(cls.discover(name=name))) + return next(iter(cls._prefer_valid(cls.discover(name=name)))) except StopIteration: - raise PackageNotFoundError(name) + raise PackageNotFoundError(name) from None @classmethod def discover( - cls, *, context: Optional[DistributionFinder.Context] = None, **kwargs + cls, *, context: DistributionFinder.Context | None = None, **kwargs ) -> Iterable[Distribution]: """Return an iterable of Distribution objects for all packages. @@ -428,6 +486,21 @@ class Distribution(DeprecatedNonAbstract): resolver(context) for resolver in cls._discover_resolvers() ) + @staticmethod + def _prefer_valid(dists: Iterable[Distribution]) -> Iterable[Distribution]: + """ + Prefer (move to the front) distributions that have metadata. + + Ref python/importlib_resources#489. + """ + + has_metadata = ExceptionTrap(MetadataNotFound).passes( + operator.attrgetter('metadata') + ) + + buckets = bucket(dists, has_metadata) + return itertools.chain(buckets[True], buckets[False]) + @staticmethod def at(path: str | os.PathLike[str]) -> Distribution: """Return a Distribution for the indicated metadata path. @@ -455,11 +528,11 @@ class Distribution(DeprecatedNonAbstract): Custom providers may provide the METADATA file or override this property. - """ - # deferred for performance (python/cpython#109829) - from . import _adapters - opt_text = ( + :raises MetadataNotFound: If no metadata file is present. + """ + + text = ( self.read_text('METADATA') or self.read_text('PKG-INFO') # This last clause is here to support old egg-info files. Its @@ -467,9 +540,21 @@ class Distribution(DeprecatedNonAbstract): # (which points to the egg-info file) attribute unchanged. or self.read_text('') ) - text = cast(str, opt_text) + return self._assemble_message(self._ensure_metadata_present(text)) + + @staticmethod + def _assemble_message(text: str) -> _meta.PackageMetadata: + # deferred for performance (python/cpython#109829) + from . import _adapters + return _adapters.Message(email.message_from_string(text)) + def _ensure_metadata_present(self, text: str | None) -> str: + if text is not None: + return text + + raise MetadataNotFound('No package metadata was found.') + @property def name(self) -> str: """Return the 'Name' metadata for the distribution package.""" @@ -496,7 +581,7 @@ class Distribution(DeprecatedNonAbstract): return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) @property - def files(self) -> Optional[List[PackagePath]]: + def files(self) -> list[PackagePath] | None: """Files in this distribution. :return: List of PackagePath for this distribution or None @@ -589,7 +674,7 @@ class Distribution(DeprecatedNonAbstract): return text and map('"{}"'.format, text.splitlines()) @property - def requires(self) -> Optional[List[str]]: + def requires(self) -> list[str] | None: """Generated requirements specified for this Distribution""" reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() return reqs and list(reqs) @@ -645,6 +730,9 @@ class Distribution(DeprecatedNonAbstract): return self._load_json('direct_url.json') def _load_json(self, filename): + # Deferred for performance (python/importlib_metadata#503) + import json + return pass_none(json.loads)( self.read_text(filename), object_hook=lambda data: types.SimpleNamespace(**data), @@ -692,7 +780,7 @@ class DistributionFinder(MetaPathFinder): vars(self).update(kwargs) @property - def path(self) -> List[str]: + def path(self) -> list[str]: """ The sequence of directory path that a distribution finder should search. @@ -713,6 +801,20 @@ class DistributionFinder(MetaPathFinder): """ +@passthrough +def _clear_after_fork(cached): + """Ensure ``func`` clears cached state after ``fork`` when supported. + + ``FastPath`` caches zip-backed ``pathlib.Path`` objects that retain a + reference to the parent's open ``ZipFile`` handle. Re-using a cached + instance in a forked child can therefore resurrect invalid file pointers + and trigger ``BadZipFile``/``OSError`` failures (python/importlib_metadata#520). + Registering ``cache_clear`` with ``os.register_at_fork`` keeps each process + on its own cache. + """ + getattr(os, 'register_at_fork', noop)(after_in_child=cached.cache_clear) + + class FastPath: """ Micro-optimized class for searching a root for children. @@ -729,7 +831,8 @@ class FastPath: True """ - @functools.lru_cache() # type: ignore + @_clear_after_fork # type: ignore[misc] + @functools.lru_cache() def __new__(cls, root): return super().__new__(cls) @@ -747,6 +850,9 @@ class FastPath: return [] def zip_children(self): + # deferred for performance (python/importlib_metadata#502) + import zipfile + zip_path = zipfile.Path(self.root) names = zip_path.root.namelist() self.joinpath = zip_path.joinpath @@ -841,7 +947,7 @@ class Prepared: normalized = None legacy_normalized = None - def __init__(self, name: Optional[str]): + def __init__(self, name: str | None): self.name = name if name is None: return @@ -852,8 +958,15 @@ class Prepared: def normalize(name): """ PEP 503 normalization plus dashes as underscores. + + Specifically avoids ``re.sub`` as prescribed for performance + benefits (see python/cpython#143658). """ - return re.sub(r"[-_.]+", "-", name).lower().replace('-', '_') + value = name.lower().replace("-", "_").replace(".", "_") + # Condense repeats + while "__" in value: + value = value.replace("__", "_") + return value @staticmethod def legacy_normalize(name): @@ -904,7 +1017,7 @@ class PathDistribution(Distribution): """ self._path = path - def read_text(self, filename: str | os.PathLike[str]) -> Optional[str]: + def read_text(self, filename: str | os.PathLike[str]) -> str | None: with suppress( FileNotFoundError, IsADirectoryError, @@ -973,6 +1086,7 @@ def metadata(distribution_name: str) -> _meta.PackageMetadata: :param distribution_name: The name of the distribution package to query. :return: A PackageMetadata containing the parsed metadata. + :raises MetadataNotFound: If no metadata file is present in the distribution. """ return Distribution.from_name(distribution_name).metadata @@ -1011,7 +1125,7 @@ def entry_points(**params) -> EntryPoints: return EntryPoints(eps).select(**params) -def files(distribution_name: str) -> Optional[List[PackagePath]]: +def files(distribution_name: str) -> list[PackagePath] | None: """Return a list of files for the named package. :param distribution_name: The name of the distribution package to query. @@ -1020,7 +1134,7 @@ def files(distribution_name: str) -> Optional[List[PackagePath]]: return distribution(distribution_name).files -def requires(distribution_name: str) -> Optional[List[str]]: +def requires(distribution_name: str) -> list[str] | None: """ Return a list of requirements for the named package. @@ -1030,7 +1144,7 @@ def requires(distribution_name: str) -> Optional[List[str]]: return distribution(distribution_name).requires -def packages_distributions() -> Mapping[str, List[str]]: +def packages_distributions() -> Mapping[str, list[str]]: """ Return a mapping of top-level packages to their distributions. @@ -1051,7 +1165,7 @@ def _top_level_declared(dist): return (dist.read_text('top_level.txt') or '').split() -def _topmost(name: PackagePath) -> Optional[str]: +def _topmost(name: PackagePath) -> str | None: """ Return the top-most parent as long as there is a parent. """ @@ -1077,11 +1191,10 @@ def _get_toplevel_name(name: PackagePath) -> str: >>> _get_toplevel_name(PackagePath('foo.dist-info')) 'foo.dist-info' """ - return _topmost(name) or ( - # python/typeshed#10328 - inspect.getmodulename(name) # type: ignore - or str(name) - ) + # Defer import of inspect for performance (python/cpython#118761) + import inspect + + return _topmost(name) or inspect.getmodulename(name) or str(name) def _top_level_inferred(dist): diff --git a/Lib/importlib/metadata/_adapters.py b/Lib/importlib/metadata/_adapters.py index 591168808..dede395d7 100644 --- a/Lib/importlib/metadata/_adapters.py +++ b/Lib/importlib/metadata/_adapters.py @@ -1,22 +1,59 @@ -import functools -import warnings +import email.message +import email.policy import re import textwrap -import email.message from ._text import FoldedCase -# Do not remove prior to 2024-01-01 or Python 3.14 -_warn = functools.partial( - warnings.warn, - "Implicit None on return values is deprecated and will raise KeyErrors.", - DeprecationWarning, - stacklevel=2, -) +class RawPolicy(email.policy.EmailPolicy): + def fold(self, name, value): + folded = self.linesep.join( + textwrap + .indent(value, prefix=' ' * 8, predicate=lambda line: True) + .lstrip() + .splitlines() + ) + return f'{name}: {folded}{self.linesep}' class Message(email.message.Message): + r""" + Specialized Message subclass to handle metadata naturally. + + Reads values that may have newlines in them and converts the + payload to the Description. + + >>> msg_text = textwrap.dedent(''' + ... Name: Foo + ... Version: 3.0 + ... License: blah + ... de-blah + ... + ... First line of description. + ... Second line of description. + ... + ... Fourth line! + ... ''').lstrip().replace('', '') + >>> msg = Message(email.message_from_string(msg_text)) + >>> msg['Description'] + 'First line of description.\nSecond line of description.\n\nFourth line!\n' + + Message should render even if values contain newlines. + + >>> print(msg) + Name: Foo + Version: 3.0 + License: blah + de-blah + Description: First line of description. + Second line of description. + + Fourth line! + + + """ + multiple_use_keys = set( map( FoldedCase, @@ -52,26 +89,36 @@ class Message(email.message.Message): def __getitem__(self, item): """ - Warn users that a ``KeyError`` can be expected when a - missing key is supplied. Ref python/importlib_metadata#371. + Override parent behavior to typical dict behavior. + + ``email.message.Message`` will emit None values for missing + keys. Typical mappings, including this ``Message``, will raise + a key error for missing keys. + + Ref python/importlib_metadata#371. """ res = super().__getitem__(item) if res is None: - _warn() + raise KeyError(item) return res def _repair_headers(self): def redent(value): "Correct for RFC822 indentation" - if not value or '\n' not in value: + indent = ' ' * 8 + if not value or '\n' + indent not in value: return value - return textwrap.dedent(' ' * 8 + value) + return textwrap.dedent(indent + value) headers = [(key, redent(value)) for key, value in vars(self)['_headers']] if self._payload: headers.append(('Description', self.get_payload())) + self.set_payload('') return headers + def as_string(self): + return super().as_string(policy=RawPolicy()) + @property def json(self): """ diff --git a/Lib/importlib/metadata/_collections.py b/Lib/importlib/metadata/_collections.py index cf0954e1a..fc5045d36 100644 --- a/Lib/importlib/metadata/_collections.py +++ b/Lib/importlib/metadata/_collections.py @@ -1,4 +1,5 @@ import collections +import typing # from jaraco.collections 3.3 @@ -24,7 +25,10 @@ class FreezableDefaultDict(collections.defaultdict): self._frozen = lambda key: self.default_factory() -class Pair(collections.namedtuple('Pair', 'name value')): +class Pair(typing.NamedTuple): + name: str + value: str + @classmethod def parse(cls, text): return cls(*map(str.strip, text.split("=", 1))) diff --git a/Lib/importlib/metadata/_context.py b/Lib/importlib/metadata/_context.py new file mode 100644 index 000000000..2635b164c --- /dev/null +++ b/Lib/importlib/metadata/_context.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import functools +import operator + + +# from jaraco.context 6.1 +class ExceptionTrap: + """ + A context manager that will catch certain exceptions and provide an + indication they occurred. + + >>> with ExceptionTrap() as trap: + ... raise Exception() + >>> bool(trap) + True + + >>> with ExceptionTrap() as trap: + ... pass + >>> bool(trap) + False + + >>> with ExceptionTrap(ValueError) as trap: + ... raise ValueError("1 + 1 is not 3") + >>> bool(trap) + True + >>> trap.value + ValueError('1 + 1 is not 3') + >>> trap.tb + + + >>> with ExceptionTrap(ValueError) as trap: + ... raise Exception() + Traceback (most recent call last): + ... + Exception + + >>> bool(trap) + False + """ + + exc_info = None, None, None + + def __init__(self, exceptions=(Exception,)): + self.exceptions = exceptions + + def __enter__(self): + return self + + @property + def type(self): + return self.exc_info[0] + + @property + def value(self): + return self.exc_info[1] + + @property + def tb(self): + return self.exc_info[2] + + def __exit__(self, *exc_info): + type = exc_info[0] + matches = type and issubclass(type, self.exceptions) + if matches: + self.exc_info = exc_info + return matches + + def __bool__(self): + return bool(self.type) + + def raises(self, func, *, _test=bool): + """ + Wrap func and replace the result with the truth + value of the trap (True if an exception occurred). + + First, give the decorator an alias to support Python 3.8 + Syntax. + + >>> raises = ExceptionTrap(ValueError).raises + + Now decorate a function that always fails. + + >>> @raises + ... def fail(): + ... raise ValueError('failed') + >>> fail() + True + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + with ExceptionTrap(self.exceptions) as trap: + func(*args, **kwargs) + return _test(trap) + + return wrapper + + def passes(self, func): + """ + Wrap func and replace the result with the truth + value of the trap (True if no exception). + + First, give the decorator an alias to support Python 3.8 + Syntax. + + >>> passes = ExceptionTrap(ValueError).passes + + Now decorate a function that always fails. + + >>> @passes + ... def fail(): + ... raise ValueError('failed') + + >>> fail() + False + """ + return self.raises(func, _test=operator.not_) diff --git a/Lib/importlib/metadata/_functools.py b/Lib/importlib/metadata/_functools.py index 71f66bd03..c159b46e4 100644 --- a/Lib/importlib/metadata/_functools.py +++ b/Lib/importlib/metadata/_functools.py @@ -1,5 +1,7 @@ -import types import functools +import types +from collections.abc import Callable +from typing import TypeVar # from jaraco.functools 3.3 @@ -102,3 +104,33 @@ def pass_none(func): return func(param, *args, **kwargs) return wrapper + + +# From jaraco.functools 4.4 +def noop(*args, **kwargs): + """ + A no-operation function that does nothing. + + >>> noop(1, 2, three=3) + """ + + +_T = TypeVar('_T') + + +# From jaraco.functools 4.4 +def passthrough(func: Callable[..., object]) -> Callable[[_T], _T]: + """ + Wrap the function to always return the first parameter. + + >>> passthrough(print)('3') + 3 + '3' + """ + + @functools.wraps(func) + def wrapper(first: _T, *args, **kwargs) -> _T: + func(first, *args, **kwargs) + return first + + return wrapper # type: ignore[return-value] diff --git a/Lib/importlib/metadata/_itertools.py b/Lib/importlib/metadata/_itertools.py index d4ca9b914..79d37198c 100644 --- a/Lib/importlib/metadata/_itertools.py +++ b/Lib/importlib/metadata/_itertools.py @@ -1,3 +1,4 @@ +from collections import defaultdict, deque from itertools import filterfalse @@ -71,3 +72,100 @@ def always_iterable(obj, base_type=(str, bytes)): return iter(obj) except TypeError: return iter((obj,)) + + +# Copied from more_itertools 10.3 +class bucket: + """Wrap *iterable* and return an object that buckets the iterable into + child iterables based on a *key* function. + + >>> iterable = ['a1', 'b1', 'c1', 'a2', 'b2', 'c2', 'b3'] + >>> s = bucket(iterable, key=lambda x: x[0]) # Bucket by 1st character + >>> sorted(list(s)) # Get the keys + ['a', 'b', 'c'] + >>> a_iterable = s['a'] + >>> next(a_iterable) + 'a1' + >>> next(a_iterable) + 'a2' + >>> list(s['b']) + ['b1', 'b2', 'b3'] + + The original iterable will be advanced and its items will be cached until + they are used by the child iterables. This may require significant storage. + + By default, attempting to select a bucket to which no items belong will + exhaust the iterable and cache all values. + If you specify a *validator* function, selected buckets will instead be + checked against it. + + >>> from itertools import count + >>> it = count(1, 2) # Infinite sequence of odd numbers + >>> key = lambda x: x % 10 # Bucket by last digit + >>> validator = lambda x: x in {1, 3, 5, 7, 9} # Odd digits only + >>> s = bucket(it, key=key, validator=validator) + >>> 2 in s + False + >>> list(s[2]) + [] + + """ + + def __init__(self, iterable, key, validator=None): + self._it = iter(iterable) + self._key = key + self._cache = defaultdict(deque) + self._validator = validator or (lambda x: True) + + def __contains__(self, value): + if not self._validator(value): + return False + + try: + item = next(self[value]) + except StopIteration: + return False + else: + self._cache[value].appendleft(item) + + return True + + def _get_values(self, value): + """ + Helper to yield items from the parent iterator that match *value*. + Items that don't match are stored in the local cache as they + are encountered. + """ + while True: + # If we've cached some items that match the target value, emit + # the first one and evict it from the cache. + if self._cache[value]: + yield self._cache[value].popleft() + # Otherwise we need to advance the parent iterator to search for + # a matching item, caching the rest. + else: + while True: + try: + item = next(self._it) + except StopIteration: + return + item_value = self._key(item) + if item_value == value: + yield item + break + elif self._validator(item_value): + self._cache[item_value].append(item) + + def __iter__(self): + for item in self._it: + item_value = self._key(item) + if self._validator(item_value): + self._cache[item_value].append(item) + + yield from self._cache.keys() + + def __getitem__(self, value): + if not self._validator(value): + return iter(()) + + return self._get_values(value) diff --git a/Lib/importlib/metadata/_meta.py b/Lib/importlib/metadata/_meta.py index 1927d0f62..0c20eff3d 100644 --- a/Lib/importlib/metadata/_meta.py +++ b/Lib/importlib/metadata/_meta.py @@ -1,9 +1,13 @@ from __future__ import annotations import os -from typing import Protocol -from typing import Any, Dict, Iterator, List, Optional, TypeVar, Union, overload - +from collections.abc import Iterator +from typing import ( + Any, + Protocol, + TypeVar, + overload, +) _T = TypeVar("_T") @@ -20,25 +24,25 @@ class PackageMetadata(Protocol): @overload def get( self, name: str, failobj: None = None - ) -> Optional[str]: ... # pragma: no cover + ) -> str | None: ... # pragma: no cover @overload - def get(self, name: str, failobj: _T) -> Union[str, _T]: ... # pragma: no cover + def get(self, name: str, failobj: _T) -> str | _T: ... # pragma: no cover # overload per python/importlib_metadata#435 @overload def get_all( self, name: str, failobj: None = None - ) -> Optional[List[Any]]: ... # pragma: no cover + ) -> list[Any] | None: ... # pragma: no cover @overload - def get_all(self, name: str, failobj: _T) -> Union[List[Any], _T]: + def get_all(self, name: str, failobj: _T) -> list[Any] | _T: """ Return all values associated with a possibly multi-valued key. """ @property - def json(self) -> Dict[str, Union[str, List[str]]]: + def json(self) -> dict[str, str | list[str]]: """ A JSON-compatible form of the metadata. """ @@ -50,11 +54,11 @@ class SimplePath(Protocol): """ def joinpath( - self, other: Union[str, os.PathLike[str]] + self, other: str | os.PathLike[str] ) -> SimplePath: ... # pragma: no cover def __truediv__( - self, other: Union[str, os.PathLike[str]] + self, other: str | os.PathLike[str] ) -> SimplePath: ... # pragma: no cover @property diff --git a/Lib/importlib/metadata/_typing.py b/Lib/importlib/metadata/_typing.py new file mode 100644 index 000000000..32b1d2b98 --- /dev/null +++ b/Lib/importlib/metadata/_typing.py @@ -0,0 +1,15 @@ +import functools +import typing + +from ._meta import PackageMetadata + +md_none = functools.partial(typing.cast, PackageMetadata) +""" +Suppress type errors for optional metadata. + +Although Distribution.metadata can return None when metadata is corrupt +and thus None, allow callers to assume it's not None and crash if +that's the case. + +# python/importlib_metadata#493 +""" diff --git a/Lib/importlib/resources/__init__.py b/Lib/importlib/resources/__init__.py index 723c9f9eb..27d6c7f89 100644 --- a/Lib/importlib/resources/__init__.py +++ b/Lib/importlib/resources/__init__.py @@ -8,12 +8,11 @@ for more detail. """ from ._common import ( + Anchor, + Package, as_file, files, - Package, - Anchor, ) - from ._functional import ( contents, is_resource, @@ -23,10 +22,8 @@ from ._functional import ( read_binary, read_text, ) - from .abc import ResourceReader - __all__ = [ 'Package', 'Anchor', diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index 4e9014c45..6f87d7749 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -1,54 +1,20 @@ +import contextlib +import functools +import importlib +import inspect +import itertools import os import pathlib import tempfile -import functools -import contextlib import types -import importlib -import inspect -import warnings -import itertools +from typing import Optional, cast -from typing import Union, Optional, cast from .abc import ResourceReader, Traversable -Package = Union[types.ModuleType, str] +Package = types.ModuleType | str Anchor = Package -def package_to_anchor(func): - """ - Replace 'package' parameter as 'anchor' and warn about the change. - - Other errors should fall through. - - >>> files('a', 'b') - Traceback (most recent call last): - TypeError: files() takes from 0 to 1 positional arguments but 2 were given - - Remove this compatibility in Python 3.14. - """ - undefined = object() - - @functools.wraps(func) - def wrapper(anchor=undefined, package=undefined): - if package is not undefined: - if anchor is not undefined: - return func(anchor, package) - warnings.warn( - "First parameter to files is renamed to 'anchor'", - DeprecationWarning, - stacklevel=2, - ) - return func(package) - elif anchor is undefined: - return func() - return func(anchor) - - return wrapper - - -@package_to_anchor def files(anchor: Optional[Anchor] = None) -> Traversable: """ Get a Traversable resource for an anchor. @@ -105,6 +71,19 @@ def _infer_caller(): return next(callers).frame +def _assert_spec(package: types.ModuleType) -> None: + """ + Provide a nicer error message when package is ``__main__`` + and its ``__spec__`` is ``None`` + (https://docs.python.org/3/reference/import.html#main-spec). + """ + if package.__spec__ is None: + raise TypeError( + f"Cannot access resources for '{package.__name__}' " + "as it does not appear to correspond to an importable module (its __spec__ is None)." + ) + + def from_package(package: types.ModuleType): """ Return a Traversable object for the given package. @@ -113,6 +92,7 @@ def from_package(package: types.ModuleType): # deferred for performance (python/cpython#109829) from ._adapters import wrap_spec + _assert_spec(package) spec = wrap_spec(package) reader = spec.loader.get_resource_reader(spec.name) return reader.files() diff --git a/Lib/importlib/resources/_functional.py b/Lib/importlib/resources/_functional.py index f59416f2d..b08a5c6ef 100644 --- a/Lib/importlib/resources/_functional.py +++ b/Lib/importlib/resources/_functional.py @@ -2,8 +2,8 @@ import warnings -from ._common import files, as_file - +from ._common import as_file, files +from .abc import TraversalError _MISSING = object() @@ -42,7 +42,10 @@ def is_resource(anchor, *path_names): Otherwise returns ``False``. """ - return _get_resource(anchor, path_names).is_file() + try: + return _get_resource(anchor, path_names).is_file() + except TraversalError: + return False def contents(anchor, *path_names): diff --git a/Lib/importlib/resources/abc.py b/Lib/importlib/resources/abc.py index 6750a7aaf..0b5fdee80 100644 --- a/Lib/importlib/resources/abc.py +++ b/Lib/importlib/resources/abc.py @@ -1,14 +1,22 @@ import abc -import io import itertools import os import pathlib -from typing import Any, BinaryIO, Iterable, Iterator, NoReturn, Text, Optional -from typing import runtime_checkable, Protocol -from typing import Union +from collections.abc import Iterable, Iterator +from typing import ( + Any, + BinaryIO, + Literal, + NoReturn, + Optional, + Protocol, + Text, + TextIO, + overload, + runtime_checkable, +) - -StrPath = Union[str, os.PathLike[str]] +StrPath = str | os.PathLike[str] __all__ = ["ResourceReader", "Traversable", "TraversableResources"] @@ -82,11 +90,13 @@ class Traversable(Protocol): with self.open('rb') as strm: return strm.read() - def read_text(self, encoding: Optional[str] = None) -> str: + def read_text( + self, encoding: Optional[str] = None, errors: Optional[str] = None + ) -> str: """ Read contents of self as text """ - with self.open(encoding=encoding) as strm: + with self.open(encoding=encoding, errors=errors) as strm: return strm.read() @abc.abstractmethod @@ -132,8 +142,14 @@ class Traversable(Protocol): """ return self.joinpath(child) + @overload + def open(self, mode: Literal['r'] = 'r', *args: Any, **kwargs: Any) -> TextIO: ... + + @overload + def open(self, mode: Literal['rb'], *args: Any, **kwargs: Any) -> BinaryIO: ... + @abc.abstractmethod - def open(self, mode='r', *args, **kwargs): + def open(self, mode: str = 'r', *args: Any, **kwargs: Any) -> TextIO | BinaryIO: """ mode may be 'r' or 'rb' to open as text or binary. Return a handle suitable for reading (same as pathlib.Path.open). @@ -160,7 +176,7 @@ class TraversableResources(ResourceReader): def files(self) -> "Traversable": """Return a Traversable object for the loaded package.""" - def open_resource(self, resource: StrPath) -> io.BufferedReader: + def open_resource(self, resource: StrPath) -> BinaryIO: return self.files().joinpath(resource).open('rb') def resource_path(self, resource: Any) -> NoReturn: diff --git a/Lib/importlib/resources/readers.py b/Lib/importlib/resources/readers.py index 70fc7e2b9..5d0ae46d6 100644 --- a/Lib/importlib/resources/readers.py +++ b/Lib/importlib/resources/readers.py @@ -3,15 +3,14 @@ from __future__ import annotations import collections import contextlib import itertools -import pathlib import operator +import pathlib import re import warnings import zipfile from collections.abc import Iterator from . import abc - from ._itertools import only diff --git a/Lib/importlib/resources/simple.py b/Lib/importlib/resources/simple.py index 2e75299b1..5e182d126 100644 --- a/Lib/importlib/resources/simple.py +++ b/Lib/importlib/resources/simple.py @@ -5,7 +5,7 @@ Interface adapters for low-level readers. import abc import io import itertools -from typing import BinaryIO, List +from typing import BinaryIO from .abc import Traversable, TraversableResources @@ -24,14 +24,14 @@ class SimpleReader(abc.ABC): """ @abc.abstractmethod - def children(self) -> List['SimpleReader']: + def children(self) -> list['SimpleReader']: """ Obtain an iterable of SimpleReader for available child containers (e.g. directories). """ @abc.abstractmethod - def resources(self) -> List[str]: + def resources(self) -> list[str]: """ Obtain available named resources for this virtual package. """ diff --git a/Lib/inspect.py b/Lib/inspect.py index cf61ae943..92664f46d 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -153,9 +153,9 @@ import importlib.machinery import itertools import linecache import os -import re +lazy import re import sys -import tokenize +lazy import tokenize import token import types import functools @@ -163,9 +163,9 @@ import builtins from keyword import iskeyword from operator import attrgetter from collections import namedtuple, OrderedDict -from weakref import ref as make_weakref +from _weakref import ref as make_weakref -# Create constants for the compiler flags in Include/code.h +# Create constants for the compiler flags in Include/cpython/code.h # We try to get them from dis to avoid duplication mod_dict = globals() for k, v in dis.COMPILER_FLAG_NAMES.items(): @@ -421,7 +421,6 @@ def iscode(object): co_freevars tuple of names of free variables co_posonlyargcount number of positional only arguments co_kwonlyargcount number of keyword only arguments (not including ** arg) - co_lnotab encoded mapping of line numbers to bytecode indices co_name name with which this code object was defined co_names tuple of names other than arguments and function locals co_nlocals number of local variables @@ -712,8 +711,8 @@ def _findclass(func): return None return cls -def _finddoc(obj): - if isclass(obj): +def _finddoc(obj, *, search_in_class=True): + if search_in_class and isclass(obj): for base in obj.__mro__: if base is not object: try: @@ -753,6 +752,12 @@ def _finddoc(obj): cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None + # Should be tested before ismethoddescriptor() + elif isinstance(obj, functools.cached_property): + name = obj.attrname + cls = _findclass(obj.func) + if cls is None or getattr(cls, name) is not obj: + return None elif ismethoddescriptor(obj) or isdatadescriptor(obj): name = obj.__name__ cls = obj.__objclass__ @@ -773,19 +778,37 @@ def _finddoc(obj): return doc return None -def getdoc(object): +def _getowndoc(obj): + """Get the documentation string for an object if it is not + inherited from its class.""" + try: + doc = object.__getattribute__(obj, '__doc__') + if doc is None: + return None + if obj is not type: + typedoc = type(obj).__doc__ + if isinstance(typedoc, str) and typedoc == doc: + return None + return doc + except AttributeError: + return None + +def getdoc(object, *, fallback_to_class_doc=True, inherit_class_doc=True): """Get the documentation string for an object. All tabs are expanded to spaces. To clean up docstrings that are indented to line up with blocks of code, any whitespace than can be uniformly removed from the second line onwards is removed.""" - try: - doc = object.__doc__ - except AttributeError: - return None + if fallback_to_class_doc: + try: + doc = object.__doc__ + except AttributeError: + return None + else: + doc = _getowndoc(object) if doc is None: try: - doc = _finddoc(object) + doc = _finddoc(object, search_in_class=inherit_class_doc) except (AttributeError, TypeError): return None if not isinstance(doc, str): @@ -1236,11 +1259,10 @@ def getargs(co): FullArgSpec = namedtuple('FullArgSpec', 'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations') -def getfullargspec(func): +def getfullargspec(func, *, annotation_format=Format.VALUE): """Get the names and default values of a callable object's parameters. - A tuple of seven things is returned: - (args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations). + A FullArgSpec namedtuple is returned, which has the following attributes: 'args' is a list of the parameter names. 'varargs' and 'varkw' are the names of the * and ** parameters or None. 'defaults' is an n-tuple of the default values of the last n parameters. @@ -1248,6 +1270,9 @@ def getfullargspec(func): 'kwonlydefaults' is a dictionary mapping names from kwonlyargs to defaults. 'annotations' is a dictionary mapping parameter names to annotations. + The *annotation_format* parameter controls the format of the annotations. + See the annotationlib documentation for details. + Notable differences from inspect.signature(): - the "self" parameter is always reported, even for bound methods - wrapper chains defined by __wrapped__ *not* unwrapped automatically @@ -1273,7 +1298,8 @@ def getfullargspec(func): follow_wrapper_chains=False, skip_bound_arg=False, sigcls=Signature, - eval_str=False) + eval_str=False, + annotation_format=annotation_format) except Exception as ex: # Most of the times 'signature' will raise ValueError. # But, it can also raise AttributeError, and, maybe something @@ -1615,7 +1641,6 @@ def getframeinfo(frame, context=1): def getlineno(frame): """Get the line number from a frame object, allowing for optimization.""" - # FrameType.f_lineno is now a descriptor that grovels co_lnotab return frame.f_lineno _FrameInfo = namedtuple('_FrameInfo', ('frame',) + Traceback._fields) @@ -1709,7 +1734,8 @@ def _shadowed_dict_from_weakref_mro_tuple(*weakref_mro): class_dict = dunder_dict['__dict__'] if not (type(class_dict) is types.GetSetDescriptorType and class_dict.__name__ == "__dict__" and - class_dict.__objclass__ is entry): + (class_dict.__objclass__ is object or + class_dict.__objclass__ is entry)): return class_dict return _sentinel @@ -1793,13 +1819,7 @@ def getgeneratorstate(generator): GEN_SUSPENDED: Currently suspended at a yield expression. GEN_CLOSED: Execution has completed. """ - if generator.gi_running: - return GEN_RUNNING - if generator.gi_suspended: - return GEN_SUSPENDED - if generator.gi_frame is None: - return GEN_CLOSED - return GEN_CREATED + return generator.gi_state def getgeneratorlocals(generator): @@ -1835,13 +1855,7 @@ def getcoroutinestate(coroutine): CORO_SUSPENDED: Currently suspended at an await expression. CORO_CLOSED: Execution has completed. """ - if coroutine.cr_running: - return CORO_RUNNING - if coroutine.cr_suspended: - return CORO_SUSPENDED - if coroutine.cr_frame is None: - return CORO_CLOSED - return CORO_CREATED + return coroutine.cr_state def getcoroutinelocals(coroutine): @@ -1874,13 +1888,7 @@ def getasyncgenstate(agen): AGEN_SUSPENDED: Currently suspended at a yield expression. AGEN_CLOSED: Execution has completed. """ - if agen.ag_running: - return AGEN_RUNNING - if agen.ag_suspended: - return AGEN_SUSPENDED - if agen.ag_frame is None: - return AGEN_CLOSED - return AGEN_CREATED + return agen.ag_state def getasyncgenlocals(agen): @@ -2121,7 +2129,6 @@ def _signature_strip_non_python_syntax(signature): current_parameter = 0 OP = token.OP - ERRORTOKEN = token.ERRORTOKEN # token stream always starts with ENCODING token, skip it t = next(token_stream) @@ -2198,7 +2205,8 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): except NameError: raise ValueError - if isinstance(value, (str, int, float, bytes, bool, type(None))): + if isinstance(value, (str, int, float, bytes, bool, type(None), + sentinel)): return ast.Constant(value) raise ValueError @@ -3356,22 +3364,113 @@ class BufferFlags(enum.IntFlag): WRITE = 0x200 +def _get_details_for_cli(module, nominal_target, resolved_target): + # Determine if the given module name is an alias for another module, + # or if it is reexporting a name that is actually defined elsewhere + resolved_module = getmodule(resolved_target) + if resolved_module is not None and resolved_module is not module: + # Referenced target indicates it was defined somewhere else, + # so report the details of that module rather than the lookup module + module = resolved_module + reported_module_name = module.__name__ + # Ensure the reported source file reflects the actual defining location + try: + source_file = getsourcefile(resolved_target) + except Exception: + try: + source_file = getsourcefile(module) + except Exception: + source_file = None + # Determine if the nominal target location is its defining location + if resolved_target is module: + reported_target = reported_module_name + else: + reported_qualname = getattr(resolved_target, "__qualname__", None) + if not reported_qualname: + reported_qualname = nominal_target.partition(":")[2] + reported_target = f"{reported_module_name}:{reported_qualname}" + # Special case for looking up functions in frozen modules + if source_file == f"": + source_file = module.__file__ + # Populate the actual details to be reported + details = { + "target": reported_target, + "origin": module.__spec__.origin, + "cached": module.__spec__.cached, + "source": source_file, + } + if reported_target != nominal_target: + details["alias"] = nominal_target + error = None + if not source_file: + if module.__name__ in sys.builtin_module_names: + error = "No source code available for builtin module" + else: + error = "No source code available for defining module" + if resolved_target is module: + details["loader"] = repr(module.__spec__.loader) + if hasattr(module, '__path__'): + details["submodule_paths"] = str(module.__path__) + elif source_file: + try: + __, lineno = findsource(resolved_target) + except Exception: + error = "Failed to retrieve source code for given target" + else: + details["lineno"] = lineno + if error: + details["error"] = error + return details + +def _render_details_for_cli(details): + resolved_target = details["target"] + alias = details.get("alias") + if alias: + rendered_target = f'{resolved_target} (looked up as "{alias}")' + else: + rendered_target = resolved_target + lines = [ + f'Target: {rendered_target}', + f'Origin: {details["origin"]}', + f'Source: {details["source"]}', + f'Cached: {details["cached"]}', + ] + loader = details.get("loader") + if loader: + lines.append(f'Loader: {loader}') + submodule_paths = details.get("submodule_paths") + if submodule_paths: + lines.append(f'Submodule search paths: {submodule_paths}') + else: + error = details.get("error") + if error: + # The error is only informational when retrieving object details + lines.append(error) + else: + lines.append(f'Line: {details["lineno"]}') + + lines.append("") + return "\n".join(lines) + + def _main(): """ Logic for inspecting an object given at command line """ import argparse import importlib - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument( 'object', help="The object to be analysed. " - "It supports the 'module:qualname' syntax") + "It supports the `module:qualname` syntax") parser.add_argument( '-d', '--details', action='store_true', help='Display info about the module rather than its source code') args = parser.parse_args() + # We don't use `pkgutil.resolve_name` here because we want to obtain + # references to both the module *and* the fully resolved target object target = args.object mod_name, has_attrs, attrs = target.partition(":") try: @@ -3389,29 +3488,16 @@ def _main(): for part in parts: obj = getattr(obj, part) - if module.__name__ in sys.builtin_module_names: - print("Can't get info for builtin modules.", file=sys.stderr) - sys.exit(1) - + details = _get_details_for_cli(module, target, obj) if args.details: - print('Target: {}'.format(target)) - print('Origin: {}'.format(getsourcefile(module))) - print('Cached: {}'.format(module.__cached__)) - if obj is module: - print('Loader: {}'.format(repr(module.__loader__))) - if hasattr(module, '__path__'): - print('Submodule search path: {}'.format(module.__path__)) - else: - try: - __, lineno = findsource(obj) - except Exception: - pass - else: - print('Line: {}'.format(lineno)) - - print('\n') + print(_render_details_for_cli(details)) else: - print(getsource(obj)) + # Attempt to render target source details + error = details.get("error") + if error: + sys.exit(error) + else: + print(getsource(obj)) if __name__ == "__main__": diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 4c7eac0b7..e8a3d176a 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -4,6 +4,7 @@ import importlib import importlib.machinery import importlib.util import os +import py_compile import shutil import sys import textwrap @@ -49,20 +50,31 @@ def forget(modname): # combinations of PEP 3147/488 and legacy pyc files. unlink(source + 'c') for opt in ('', 1, 2): - unlink(importlib.util.cache_from_source(source, optimization=opt)) + try: + unlink(importlib.util.cache_from_source(source, optimization=opt)) + except NotImplementedError: + pass -def make_legacy_pyc(source): +def make_legacy_pyc(source, allow_compile=False): """Move a PEP 3147/488 pyc file to its legacy pyc location. :param source: The file system path to the source file. The source file - does not need to exist, however the PEP 3147/488 pyc file must exist. + does not need to exist, however the PEP 3147/488 pyc file must exist or + allow_compile must be set. + :param allow_compile: If True, uses py_compile to create a .pyc if it does + not exist. This should be passed as True if cache_tag may be None. :return: The file system path to the legacy pyc file. """ - pyc_file = importlib.util.cache_from_source(source) assert source.endswith('.py') legacy_pyc = source + 'c' - shutil.move(pyc_file, legacy_pyc) + try: + pyc_file = importlib.util.cache_from_source(source) + shutil.move(pyc_file, legacy_pyc) + except FileNotFoundError, NotImplementedError: + if not allow_compile: + raise + py_compile.compile(source, legacy_pyc, doraise=True) return legacy_pyc @@ -313,7 +325,7 @@ def ready_to_import(name=None, source=""): sys.modules.pop(name, None) -def ensure_lazy_imports(imported_module, modules_to_block): +def ensure_lazy_imports(imported_module, modules_to_block, *, additional_code=None): """Test that when imported_module is imported, none of the modules in modules_to_block are imported as a side effect.""" modules_to_block = frozenset(modules_to_block) @@ -331,6 +343,16 @@ def ensure_lazy_imports(imported_module, modules_to_block): raise AssertionError(f'unexpectedly imported after importing {imported_module}: {{after}}') """ ) + if additional_code: + script += additional_code + script += textwrap.dedent( + f""" + if unexpected := modules_to_block & sys.modules.keys(): + after = ", ".join(unexpected) + raise AssertionError(f'unexpectedly imported after additional code: {{after}}') + """ + ) + from .script_helper import assert_python_ok assert_python_ok("-S", "-c", script) diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index daf606094..12d34dedf 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -1,6 +1,7 @@ import collections.abc import contextlib import errno +import functools import logging import os import re @@ -433,7 +434,10 @@ if sys.platform.startswith("win"): file=sys.__stderr__) mode = 0 if stat.S_ISDIR(mode): - _waitfor(_rmtree_inner, fullname, waitall=True) + # Do not follow junctions, which os.lstat() reports + # as directories. + if not os.path.isjunction(fullname): + _waitfor(_rmtree_inner, fullname, waitall=True) _force_run(fullname, os.rmdir, fullname) else: _force_run(fullname, os.unlink, fullname) @@ -806,6 +810,48 @@ class EnvironmentVarGuard(collections.abc.MutableMapping): os.environ = self._environ +def without_source_date_epoch(fxn): + """Runs function with SOURCE_DATE_EPOCH unset.""" + @functools.wraps(fxn) + def wrapper(*args, **kwargs): + with EnvironmentVarGuard() as env: + env.unset('SOURCE_DATE_EPOCH') + return fxn(*args, **kwargs) + return wrapper + + +_MISSING = sentinel("MISSING") + +def with_source_date_epoch(fxn=_MISSING, *, epoch=123456789): + """Runs function with SOURCE_DATE_EPOCH set to *epoch*.""" + if fxn is _MISSING: + return functools.partial(with_source_date_epoch, epoch=epoch) + + @functools.wraps(fxn) + def wrapper(*args, **kwargs): + with EnvironmentVarGuard() as env: + env['SOURCE_DATE_EPOCH'] = str(epoch) + return fxn(*args, **kwargs) + return wrapper + + +# Run tests with SOURCE_DATE_EPOCH set or unset explicitly. +class SourceDateEpochTestMeta(type(unittest.TestCase)): + def __new__(mcls, name, bases, dct, *, source_date_epoch): + cls = super().__new__(mcls, name, bases, dct) + + for attr in dir(cls): + if attr.startswith('test_'): + meth = getattr(cls, attr) + if source_date_epoch: + wrapper = with_source_date_epoch(meth) + else: + wrapper = without_source_date_epoch(meth) + setattr(cls, attr, wrapper) + + return cls + + try: if support.MS_WINDOWS: import ctypes diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 3233a7275..5d8e764b5 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -15,6 +15,7 @@ import marshal import os import py_compile import random +import re import shutil import stat import subprocess @@ -23,6 +24,7 @@ import textwrap import threading import time import types +import warnings import unittest from unittest import mock import _imp @@ -52,7 +54,7 @@ from test.support.os_helper import ( TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE) from test.support import script_helper from test.support import threading_helper -from test.test_importlib.util import uncache +from test.test_importlib.util import uncache, temporary_pycache_prefix from types import ModuleType try: import _testsinglephase @@ -64,8 +66,10 @@ except ImportError: _testmultiphase = None try: import _interpreters + import concurrent.interpreters except ModuleNotFoundError: _interpreters = None + concurrent = None try: import _testinternalcapi except ImportError: @@ -73,7 +77,7 @@ except ImportError: skip_if_dont_write_bytecode = unittest.skipIf( - sys.dont_write_bytecode, + sys.dont_write_bytecode or sys.implementation.cache_tag is None, "test meaningful only when writing bytecode") @@ -363,11 +367,11 @@ class ImportTests(unittest.TestCase): def test_import_null_byte_in_name_raises_ModuleNotFoundError(self): # gh-150633: module names containing null bytes should not # lead to duplicates in sys.modules - before = set(sys.modules.keys()) + before = set(sys.modules) with self.assertRaises(ModuleNotFoundError): __import__('zipimport\x00junk') - self.assertEqual(set(sys.modules.keys()), before) + self.assertEqual(set(sys.modules), before) def test_from_import_missing_module_raises_ModuleNotFoundError(self): with self.assertRaises(ModuleNotFoundError): @@ -422,7 +426,6 @@ class ImportTests(unittest.TestCase): self.assertIsNotNone(cm.exception) def test_from_import_star_invalid_type(self): - import re with ready_to_import() as (name, path): with open(path, 'w', encoding='utf-8') as f: f.write("__all__ = [b'invalid_type']") @@ -511,7 +514,7 @@ class ImportTests(unittest.TestCase): try: # Compile & remove .py file; we only need .pyc. # Bytecode must be relocated from the PEP 3147 bytecode-only location. - py_compile.compile(filename) + make_legacy_pyc(filename, allow_compile=True) finally: unlink(filename) @@ -521,7 +524,6 @@ class ImportTests(unittest.TestCase): namespace = {} try: - make_legacy_pyc(filename) # This used to crash. exec('import ' + module, None, namespace) finally: @@ -786,7 +788,7 @@ class ImportTests(unittest.TestCase): pydname = importlib.util.find_spec("_sqlite3").origin depname = os.path.join( os.path.dirname(pydname), - "sqlite3{}.dll".format("_d" if "_d" in pydname else "")) + "sqlite3{}.dll".format("_d" if "_d" in os.path.basename(pydname) else "")) with os_helper.temp_dir() as tmp: tmp2 = os.path.join(tmp, "DLLs") @@ -1261,6 +1263,44 @@ os.does_not_exist origin = "a\x00b" _imp.create_dynamic(Spec2()) + def test_create_builtin(self): + class Spec: + pass + spec = Spec() + + spec.name = "sys" + self.assertIs(_imp.create_builtin(spec), sys) + + spec.name = None + with self.assertRaisesRegex(TypeError, 'name must be string, not NoneType'): + _imp.create_builtin(spec) + + # gh-142029 + spec.name = "nonexistent_lib" + with self.assertRaises(ModuleNotFoundError): + _imp.create_builtin(spec) + + # gh-142029 + spec.name = "" + with self.assertRaisesRegex(ValueError, 'name must not be empty'): + _imp.create_builtin(spec) + + def test_filter_syntax_warnings_by_module(self): + module_re = r'test\.test_import\.data\.syntax_warnings\z' + unload('test.test_import.data.syntax_warnings') + with (os_helper.temp_dir() as tmpdir, + temporary_pycache_prefix(tmpdir), + warnings.catch_warnings(record=True) as wlog): + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + warnings.filterwarnings('error', module='syntax_warnings') + import test.test_import.data.syntax_warnings + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + filename = test.test_import.data.syntax_warnings.__file__ + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + @skip_if_dont_write_bytecode class FilePermissionTests(unittest.TestCase): @@ -1369,7 +1409,10 @@ func_filename = func.__code__.co_filename """ dir_name = os.path.abspath(TESTFN) file_name = os.path.join(dir_name, module_name) + os.extsep + "py" - compiled_name = importlib.util.cache_from_source(file_name) + try: + compiled_name = importlib.util.cache_from_source(file_name) + except NotImplementedError: + compiled_name = None def setUp(self): self.sys_path = sys.path[:] @@ -1387,7 +1430,8 @@ func_filename = func.__code__.co_filename else: unload(self.module_name) unlink(self.file_name) - unlink(self.compiled_name) + if self.compiled_name: + unlink(self.compiled_name) rmtree(self.dir_name) def import_module(self): @@ -1406,6 +1450,8 @@ func_filename = func.__code__.co_filename self.assertEqual(mod.code_filename, self.file_name) self.assertEqual(mod.func_filename, self.file_name) + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag is not None') def test_incorrect_code_name(self): py_compile.compile(self.file_name, dfile="another_module.py") mod = self.import_module() @@ -1415,9 +1461,9 @@ func_filename = func.__code__.co_filename def test_module_without_source(self): target = "another_module.py" - py_compile.compile(self.file_name, dfile=target) + pyc_file = self.file_name + 'c' + py_compile.compile(self.file_name, pyc_file, dfile=target) os.remove(self.file_name) - pyc_file = make_legacy_pyc(self.file_name) importlib.invalidate_caches() mod = self.import_module() self.assertEqual(mod.module_filename, pyc_file) @@ -1425,8 +1471,9 @@ func_filename = func.__code__.co_filename self.assertEqual(mod.func_filename, target) def test_foreign_code(self): - py_compile.compile(self.file_name) - with open(self.compiled_name, "rb") as f: + compiled_name = self.compiled_name or (self.file_name + 'c') + py_compile.compile(self.file_name, compiled_name) + with open(compiled_name, "rb") as f: header = f.read(16) code = marshal.load(f) constants = list(code.co_consts) @@ -1434,9 +1481,11 @@ func_filename = func.__code__.co_filename pos = constants.index(1000) constants[pos] = foreign_code code = code.replace(co_consts=tuple(constants)) - with open(self.compiled_name, "wb") as f: + with open(compiled_name, "wb") as f: f.write(header) marshal.dump(code, f) + if not self.compiled_name: + os.remove(self.file_name) mod = self.import_module() self.assertEqual(mod.constant.co_filename, foreign_code.co_filename) @@ -1621,6 +1670,11 @@ class PycacheTests(unittest.TestCase): unlink(self.source) def setUp(self): + # These tests assume bytecode is written next to the source in a + # local __pycache__ directory, so neutralize any pycache prefix (e.g. + # when the test suite is run with PYTHONPYCACHEPREFIX set). + self._orig_pycache_prefix = sys.pycache_prefix + sys.pycache_prefix = None self.source = TESTFN + '.py' self._clean() with open(self.source, 'w', encoding='utf-8') as fp: @@ -1632,6 +1686,7 @@ class PycacheTests(unittest.TestCase): assert sys.path[0] == os.curdir, 'Unexpected sys.path[0]' del sys.path[0] self._clean() + sys.pycache_prefix = self._orig_pycache_prefix @skip_if_dont_write_bytecode def test_import_pyc_path(self): @@ -1691,78 +1746,6 @@ class PycacheTests(unittest.TestCase): finally: os.remove(pyc_file) - def test___cached__(self): - # Modules now also have an __cached__ that points to the pyc file. - m = __import__(TESTFN) - pyc_file = importlib.util.cache_from_source(TESTFN + '.py') - self.assertEqual(m.__cached__, os.path.join(os.getcwd(), pyc_file)) - - @skip_if_dont_write_bytecode - def test___cached___legacy_pyc(self): - # Like test___cached__() except that for backward compatibility, - # when the pyc file lives where the py file would have been (and named - # without the tag), it is importable. The __cached__ of the imported - # module is the pyc location. - __import__(TESTFN) - # pyc_file gets removed in _clean() via tearDown(). - pyc_file = make_legacy_pyc(self.source) - os.remove(self.source) - unload(TESTFN) - importlib.invalidate_caches() - m = __import__(TESTFN) - self.assertEqual(m.__cached__, - os.path.join(os.getcwd(), os.path.relpath(pyc_file))) - - @skip_if_dont_write_bytecode - def test_package___cached__(self): - # Like test___cached__ but for packages. - def cleanup(): - rmtree('pep3147') - unload('pep3147.foo') - unload('pep3147') - os.mkdir('pep3147') - self.addCleanup(cleanup) - # Touch the __init__.py - with open(os.path.join('pep3147', '__init__.py'), 'wb'): - pass - with open(os.path.join('pep3147', 'foo.py'), 'wb'): - pass - importlib.invalidate_caches() - m = __import__('pep3147.foo') - init_pyc = importlib.util.cache_from_source( - os.path.join('pep3147', '__init__.py')) - self.assertEqual(m.__cached__, os.path.join(os.getcwd(), init_pyc)) - foo_pyc = importlib.util.cache_from_source(os.path.join('pep3147', 'foo.py')) - self.assertEqual(sys.modules['pep3147.foo'].__cached__, - os.path.join(os.getcwd(), foo_pyc)) - - def test_package___cached___from_pyc(self): - # Like test___cached__ but ensuring __cached__ when imported from a - # PEP 3147 pyc file. - def cleanup(): - rmtree('pep3147') - unload('pep3147.foo') - unload('pep3147') - os.mkdir('pep3147') - self.addCleanup(cleanup) - # Touch the __init__.py - with open(os.path.join('pep3147', '__init__.py'), 'wb'): - pass - with open(os.path.join('pep3147', 'foo.py'), 'wb'): - pass - importlib.invalidate_caches() - m = __import__('pep3147.foo') - unload('pep3147.foo') - unload('pep3147') - importlib.invalidate_caches() - m = __import__('pep3147.foo') - init_pyc = importlib.util.cache_from_source( - os.path.join('pep3147', '__init__.py')) - self.assertEqual(m.__cached__, os.path.join(os.getcwd(), init_pyc)) - foo_pyc = importlib.util.cache_from_source(os.path.join('pep3147', 'foo.py')) - self.assertEqual(sys.modules['pep3147.foo'].__cached__, - os.path.join(os.getcwd(), foo_pyc)) - def test_recompute_pyc_same_second(self): # Even when the source file doesn't change timestamp, a change in # source size is enough to trigger recomputation of the pyc file. @@ -2031,10 +2014,6 @@ class ImportTracebackTests(unittest.TestCase): # away from the traceback. self.create_module("foo", "") importlib = sys.modules['_frozen_importlib_external'] - if 'load_module' in vars(importlib.SourceLoader): - old_exec_module = importlib.SourceLoader.exec_module - else: - old_exec_module = None try: def exec_module(*args): 1/0 @@ -2047,10 +2026,7 @@ class ImportTracebackTests(unittest.TestCase): self.fail("ZeroDivisionError should have been raised") self.assert_traceback(tb, [__file__, '>> @suppress(KeyError) - ... def key_error(): - ... {}[''] - >>> key_error() - """ diff --git a/Lib/test/test_importlib/_path.py b/Lib/test/test_importlib/_path.py deleted file mode 100644 index 71a704389..000000000 --- a/Lib/test/test_importlib/_path.py +++ /dev/null @@ -1,109 +0,0 @@ -# from jaraco.path 3.5 - -import functools -import pathlib -from typing import Dict, Union - -try: - from typing import Protocol, runtime_checkable -except ImportError: # pragma: no cover - # Python 3.7 - from typing_extensions import Protocol, runtime_checkable # type: ignore - - -FilesSpec = Dict[str, Union[str, bytes, 'FilesSpec']] # type: ignore - - -@runtime_checkable -class TreeMaker(Protocol): - def __truediv__(self, *args, **kwargs): - ... # pragma: no cover - - def mkdir(self, **kwargs): - ... # pragma: no cover - - def write_text(self, content, **kwargs): - ... # pragma: no cover - - def write_bytes(self, content): - ... # pragma: no cover - - -def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: - return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore - - -def build( - spec: FilesSpec, - prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore -): - """ - Build a set of files/directories, as described by the spec. - - Each key represents a pathname, and the value represents - the content. Content may be a nested directory. - - >>> spec = { - ... 'README.txt': "A README file", - ... "foo": { - ... "__init__.py": "", - ... "bar": { - ... "__init__.py": "", - ... }, - ... "baz.py": "# Some code", - ... } - ... } - >>> target = getfixture('tmp_path') - >>> build(spec, target) - >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') - '# Some code' - """ - for name, contents in spec.items(): - create(contents, _ensure_tree_maker(prefix) / name) - - -@functools.singledispatch -def create(content: Union[str, bytes, FilesSpec], path): - path.mkdir(exist_ok=True) - build(content, prefix=path) # type: ignore - - -@create.register -def _(content: bytes, path): - path.write_bytes(content) - - -@create.register -def _(content: str, path): - path.write_text(content, encoding='utf-8') - - -@create.register -def _(content: str, path): - path.write_text(content, encoding='utf-8') - - -class Recording: - """ - A TreeMaker object that records everything that would be written. - - >>> r = Recording() - >>> build({'foo': {'foo1.txt': 'yes'}, 'bar.txt': 'abc'}, r) - >>> r.record - ['foo/foo1.txt', 'bar.txt'] - """ - - def __init__(self, loc=pathlib.PurePosixPath(), record=None): - self.loc = loc - self.record = record if record is not None else [] - - def __truediv__(self, other): - return Recording(self.loc / other, self.record) - - def write_text(self, content, **kwargs): - self.record.append(str(self.loc)) - - write_bytes = write_text - - def mkdir(self, **kwargs): - return diff --git a/Lib/test/test_importlib/abc.py b/Lib/test/test_importlib/abc.py index 5d4b95876..0da98e0c9 100644 --- a/Lib/test/test_importlib/abc.py +++ b/Lib/test/test_importlib/abc.py @@ -40,22 +40,6 @@ class FinderTests(metaclass=abc.ABCMeta): class LoaderTests(metaclass=abc.ABCMeta): - @abc.abstractmethod - def test_module(self): - """A module should load without issue. - - After the loader returns the module should be in sys.modules. - - Attributes to verify: - - * __file__ - * __loader__ - * __name__ - * No __path__ - - """ - pass - @abc.abstractmethod def test_package(self): """Loading a package should work. diff --git a/Lib/test/test_importlib/builtin/test_loader.py b/Lib/test/test_importlib/builtin/test_loader.py index 7e9d1b196..2a5c2a8f3 100644 --- a/Lib/test/test_importlib/builtin/test_loader.py +++ b/Lib/test/test_importlib/builtin/test_loader.py @@ -7,70 +7,6 @@ import types import unittest import warnings -@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module') -class LoaderTests(abc.LoaderTests): - - """Test load_module() for built-in modules.""" - - def setUp(self): - self.verification = {'__name__': 'errno', '__package__': '', - '__loader__': self.machinery.BuiltinImporter} - - def verify(self, module): - """Verify that the module matches against what it should have.""" - self.assertIsInstance(module, types.ModuleType) - for attr, value in self.verification.items(): - self.assertEqual(getattr(module, attr), value) - self.assertIn(module.__name__, sys.modules) - - def load_module(self, name): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return self.machinery.BuiltinImporter.load_module(name) - - def test_module(self): - # Common case. - with util.uncache(util.BUILTINS.good_name): - module = self.load_module(util.BUILTINS.good_name) - self.verify(module) - - # Built-in modules cannot be a package. - test_package = test_lacking_parent = None - - # No way to force an import failure. - test_state_after_failure = None - - def test_module_reuse(self): - # Test that the same module is used in a reload. - with util.uncache(util.BUILTINS.good_name): - module1 = self.load_module(util.BUILTINS.good_name) - module2 = self.load_module(util.BUILTINS.good_name) - self.assertIs(module1, module2) - - def test_unloadable(self): - name = 'dssdsdfff' - assert name not in sys.builtin_module_names - with self.assertRaises(ImportError) as cm: - self.load_module(name) - self.assertEqual(cm.exception.name, name) - - def test_already_imported(self): - # Using the name of a module already imported but not a built-in should - # still fail. - module_name = 'builtin_reload_test' - assert module_name not in sys.builtin_module_names - with util.uncache(module_name): - module = types.ModuleType(module_name) - sys.modules[module_name] = module - with self.assertRaises(ImportError) as cm: - self.load_module(module_name) - self.assertEqual(cm.exception.name, module_name) - - -(Frozen_LoaderTests, - Source_LoaderTests - ) = util.test_both(LoaderTests, machinery=machinery) - @unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module') class InspectLoaderTests: diff --git a/Lib/test/test_importlib/data/__init__.py b/Lib/test/test_importlib/data/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl b/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl deleted file mode 100644 index 641ab07f7..000000000 Binary files a/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl and /dev/null differ diff --git a/Lib/test/test_importlib/data/example-21.12-py3.6.egg b/Lib/test/test_importlib/data/example-21.12-py3.6.egg deleted file mode 100644 index cdb298a19..000000000 Binary files a/Lib/test/test_importlib/data/example-21.12-py3.6.egg and /dev/null differ diff --git a/Lib/test/test_importlib/data/example2-1.0.0-py3-none-any.whl b/Lib/test/test_importlib/data/example2-1.0.0-py3-none-any.whl deleted file mode 100644 index 5ca93657f..000000000 Binary files a/Lib/test/test_importlib/data/example2-1.0.0-py3-none-any.whl and /dev/null differ diff --git a/Lib/test/test_importlib/data01/binary.file b/Lib/test/test_importlib/data01/binary.file deleted file mode 100644 index eaf36c1da..000000000 Binary files a/Lib/test/test_importlib/data01/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/data01/subdirectory/binary.file b/Lib/test/test_importlib/data01/subdirectory/binary.file deleted file mode 100644 index eaf36c1da..000000000 Binary files a/Lib/test/test_importlib/data01/subdirectory/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/data01/utf-16.file b/Lib/test/test_importlib/data01/utf-16.file deleted file mode 100644 index 2cb772295..000000000 Binary files a/Lib/test/test_importlib/data01/utf-16.file and /dev/null differ diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index cdc8884d6..e33c04656 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -1,10 +1,11 @@ -from test.support import is_apple_mobile +from test.support import is_apple_mobile, Py_GIL_DISABLED from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') import unittest import sys +import sysconfig class FinderTests(abc.FinderTests): @@ -59,6 +60,28 @@ class FinderTests(abc.FinderTests): def test_failure(self): self.assertIsNone(self.find_spec('asdfjkl;')) + def test_abi3_extension_suffixes(self): + suffixes = self.machinery.EXTENSION_SUFFIXES + platform = sysconfig.get_config_var("SOABI_PLATFORM") + if 'win32' in sys.platform: + # Either "_d.pyd" or ".pyd" must be in suffixes + self.assertTrue({"_d.pyd", ".pyd"}.intersection(suffixes)) + elif 'cygwin' in sys.platform: + pass + else: + if Py_GIL_DISABLED: + self.assertNotIn(".abi3.so", suffixes) + else: + self.assertIn(".abi3.so", suffixes) + self.assertIn(".abi3t.so", suffixes) + + if platform: + if Py_GIL_DISABLED: + self.assertNotIn(f".abi3-{platform}.so", suffixes) + else: + self.assertIn(f".abi3-{platform}.so", suffixes) + self.assertIn(f".abi3t-{platform}.so", suffixes) + (Frozen_FinderTests, Source_FinderTests diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 0dd21e079..15d5a7d47 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -35,11 +35,6 @@ class LoaderTests: self.loader = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) - def load_module(self, fullname): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return self.loader.load_module(fullname) - def test_equality(self): other = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertEqual(self.loader, other) @@ -48,25 +43,6 @@ class LoaderTests: other = self.LoaderClass('_' + util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertNotEqual(self.loader, other) - def test_load_module_API(self): - # Test the default argument for load_module(). - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - self.loader.load_module() - self.loader.load_module(None) - with self.assertRaises(ImportError): - self.load_module('XXX') - - def test_module(self): - with util.uncache(util.EXTENSIONS.name): - module = self.load_module(util.EXTENSIONS.name) - for attr, value in [('__name__', util.EXTENSIONS.name), - ('__file__', util.EXTENSIONS.file_path), - ('__package__', '')]: - self.assertEqual(getattr(module, attr), value) - self.assertIn(util.EXTENSIONS.name, sys.modules) - self.assertIsInstance(module.__loader__, self.LoaderClass) - # No extension module as __init__ available for testing. test_package = None @@ -76,18 +52,6 @@ class LoaderTests: # No easy way to trigger a failure after a successful import. test_state_after_failure = None - def test_unloadable(self): - name = 'asdfjkl;' - with self.assertRaises(ImportError) as cm: - self.load_module(name) - self.assertEqual(cm.exception.name, name) - - def test_module_reuse(self): - with util.uncache(util.EXTENSIONS.name): - module1 = self.load_module(util.EXTENSIONS.name) - module2 = self.load_module(util.EXTENSIONS.name) - self.assertIs(module1, module2) - def test_is_package(self): self.assertFalse(self.loader.is_package(util.EXTENSIONS.name)) for suffix in self.machinery.EXTENSION_SUFFIXES: @@ -126,11 +90,6 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests): self.loader = self.LoaderClass(self.name, self.spec.origin) - def load_module(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return self.loader.load_module(self.name) - def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin @@ -140,19 +99,6 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests): loader.exec_module(module) return module - def test_module(self): - # Test loading an extension module. - with util.uncache(self.name): - module = self.load_module() - for attr, value in [('__name__', self.name), - ('__file__', self.spec.origin), - ('__package__', '')]: - self.assertEqual(getattr(module, attr), value) - with self.assertRaises(AttributeError): - module.__path__ - self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, self.LoaderClass) - # No extension module as __init__ available for testing. test_package = None @@ -213,12 +159,6 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests): assert self.spec self.loader = self.LoaderClass(self.name, self.spec.origin) - def load_module(self): - # Load the module from the test extension. - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return self.loader.load_module(self.name) - def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin @@ -237,60 +177,6 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests): # Handling failure on reload is the up to the module. test_state_after_failure = None - def test_module(self): - # Test loading an extension module. - with util.uncache(self.name): - module = self.load_module() - for attr, value in [('__name__', self.name), - ('__file__', self.spec.origin), - ('__package__', '')]: - self.assertEqual(getattr(module, attr), value) - with self.assertRaises(AttributeError): - module.__path__ - self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, self.LoaderClass) - - def test_functionality(self): - # Test basic functionality of stuff defined in an extension module. - with util.uncache(self.name): - module = self.load_module() - self.assertIsInstance(module, types.ModuleType) - ex = module.Example() - self.assertEqual(ex.demo('abcd'), 'abcd') - self.assertEqual(ex.demo(), None) - with self.assertRaises(AttributeError): - ex.abc - ex.abc = 0 - self.assertEqual(ex.abc, 0) - self.assertEqual(module.foo(9, 9), 18) - self.assertIsInstance(module.Str(), str) - self.assertEqual(module.Str(1) + '23', '123') - with self.assertRaises(module.error): - raise module.error() - self.assertEqual(module.int_const, 1969) - self.assertEqual(module.str_const, 'something different') - - def test_reload(self): - # Test that reload didn't re-set the module's attributes. - with util.uncache(self.name): - module = self.load_module() - ex_class = module.Example - importlib.reload(module) - self.assertIs(ex_class, module.Example) - - def test_try_registration(self): - # Assert that the PyState_{Find,Add,Remove}Module C API doesn't work. - with util.uncache(self.name): - module = self.load_module() - with self.subTest('PyState_FindModule'): - self.assertEqual(module.call_state_registration_func(0), None) - with self.subTest('PyState_AddModule'): - with self.assertRaises(SystemError): - module.call_state_registration_func(1) - with self.subTest('PyState_RemoveModule'): - with self.assertRaises(SystemError): - module.call_state_registration_func(2) - def test_load_submodule(self): # Test loading a simulated submodule. module = self.load_module_by_name('pkg.' + self.name) diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py deleted file mode 100644 index 73e5da2ba..000000000 --- a/Lib/test/test_importlib/fixtures.py +++ /dev/null @@ -1,382 +0,0 @@ -import os -import sys -import copy -import shutil -import pathlib -import tempfile -import textwrap -import functools -import contextlib - -from test.support.os_helper import FS_NONASCII -from test.support import requires_zlib - -from . import _path -from ._path import FilesSpec - - -try: - from importlib import resources # type: ignore - - getattr(resources, 'files') - getattr(resources, 'as_file') -except (ImportError, AttributeError): - import importlib_resources as resources # type: ignore - - -@contextlib.contextmanager -def tempdir(): - tmpdir = tempfile.mkdtemp() - try: - yield pathlib.Path(tmpdir) - finally: - shutil.rmtree(tmpdir) - - -@contextlib.contextmanager -def save_cwd(): - orig = os.getcwd() - try: - yield - finally: - os.chdir(orig) - - -@contextlib.contextmanager -def tempdir_as_cwd(): - with tempdir() as tmp: - with save_cwd(): - os.chdir(str(tmp)) - yield tmp - - -@contextlib.contextmanager -def install_finder(finder): - sys.meta_path.append(finder) - try: - yield - finally: - sys.meta_path.remove(finder) - - -class Fixtures: - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - - -class SiteDir(Fixtures): - def setUp(self): - super().setUp() - self.site_dir = self.fixtures.enter_context(tempdir()) - - -class OnSysPath(Fixtures): - @staticmethod - @contextlib.contextmanager - def add_sys_path(dir): - sys.path[:0] = [str(dir)] - try: - yield - finally: - sys.path.remove(str(dir)) - - def setUp(self): - super().setUp() - self.fixtures.enter_context(self.add_sys_path(self.site_dir)) - - -class DistInfoPkg(OnSysPath, SiteDir): - files: FilesSpec = { - "distinfo_pkg-1.0.0.dist-info": { - "METADATA": """ - Name: distinfo-pkg - Author: Steven Ma - Version: 1.0.0 - Requires-Dist: wheel >= 1.0 - Requires-Dist: pytest; extra == 'test' - Keywords: sample package - - Once upon a time - There was a distinfo pkg - """, - "RECORD": "mod.py,sha256=abc,20\n", - "entry_points.txt": """ - [entries] - main = mod:main - ns:sub = mod:main - """, - }, - "mod.py": """ - def main(): - print("hello world") - """, - } - - def setUp(self): - super().setUp() - build_files(DistInfoPkg.files, self.site_dir) - - def make_uppercase(self): - """ - Rewrite metadata with everything uppercase. - """ - shutil.rmtree(self.site_dir / "distinfo_pkg-1.0.0.dist-info") - files = copy.deepcopy(DistInfoPkg.files) - info = files["distinfo_pkg-1.0.0.dist-info"] - info["METADATA"] = info["METADATA"].upper() - build_files(files, self.site_dir) - - -class DistInfoPkgWithDot(OnSysPath, SiteDir): - files: FilesSpec = { - "pkg_dot-1.0.0.dist-info": { - "METADATA": """ - Name: pkg.dot - Version: 1.0.0 - """, - }, - } - - def setUp(self): - super().setUp() - build_files(DistInfoPkgWithDot.files, self.site_dir) - - -class DistInfoPkgWithDotLegacy(OnSysPath, SiteDir): - files: FilesSpec = { - "pkg.dot-1.0.0.dist-info": { - "METADATA": """ - Name: pkg.dot - Version: 1.0.0 - """, - }, - "pkg.lot.egg-info": { - "METADATA": """ - Name: pkg.lot - Version: 1.0.0 - """, - }, - } - - def setUp(self): - super().setUp() - build_files(DistInfoPkgWithDotLegacy.files, self.site_dir) - - -class DistInfoPkgOffPath(SiteDir): - def setUp(self): - super().setUp() - build_files(DistInfoPkg.files, self.site_dir) - - -class EggInfoPkg(OnSysPath, SiteDir): - files: FilesSpec = { - "egginfo_pkg.egg-info": { - "PKG-INFO": """ - Name: egginfo-pkg - Author: Steven Ma - License: Unknown - Version: 1.0.0 - Classifier: Intended Audience :: Developers - Classifier: Topic :: Software Development :: Libraries - Keywords: sample package - Description: Once upon a time - There was an egginfo package - """, - "SOURCES.txt": """ - mod.py - egginfo_pkg.egg-info/top_level.txt - """, - "entry_points.txt": """ - [entries] - main = mod:main - """, - "requires.txt": """ - wheel >= 1.0; python_version >= "2.7" - [test] - pytest - """, - "top_level.txt": "mod\n", - }, - "mod.py": """ - def main(): - print("hello world") - """, - } - - def setUp(self): - super().setUp() - build_files(EggInfoPkg.files, prefix=self.site_dir) - - -class EggInfoPkgPipInstalledNoToplevel(OnSysPath, SiteDir): - files: FilesSpec = { - "egg_with_module_pkg.egg-info": { - "PKG-INFO": "Name: egg_with_module-pkg", - # SOURCES.txt is made from the source archive, and contains files - # (setup.py) that are not present after installation. - "SOURCES.txt": """ - egg_with_module.py - setup.py - egg_with_module_pkg.egg-info/PKG-INFO - egg_with_module_pkg.egg-info/SOURCES.txt - egg_with_module_pkg.egg-info/top_level.txt - """, - # installed-files.txt is written by pip, and is a strictly more - # accurate source than SOURCES.txt as to the installed contents of - # the package. - "installed-files.txt": """ - ../egg_with_module.py - PKG-INFO - SOURCES.txt - top_level.txt - """, - # missing top_level.txt (to trigger fallback to installed-files.txt) - }, - "egg_with_module.py": """ - def main(): - print("hello world") - """, - } - - def setUp(self): - super().setUp() - build_files(EggInfoPkgPipInstalledNoToplevel.files, prefix=self.site_dir) - - -class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteDir): - files: FilesSpec = { - "egg_with_no_modules_pkg.egg-info": { - "PKG-INFO": "Name: egg_with_no_modules-pkg", - # SOURCES.txt is made from the source archive, and contains files - # (setup.py) that are not present after installation. - "SOURCES.txt": """ - setup.py - egg_with_no_modules_pkg.egg-info/PKG-INFO - egg_with_no_modules_pkg.egg-info/SOURCES.txt - egg_with_no_modules_pkg.egg-info/top_level.txt - """, - # installed-files.txt is written by pip, and is a strictly more - # accurate source than SOURCES.txt as to the installed contents of - # the package. - "installed-files.txt": """ - PKG-INFO - SOURCES.txt - top_level.txt - """, - # top_level.txt correctly reflects that no modules are installed - "top_level.txt": b"\n", - }, - } - - def setUp(self): - super().setUp() - build_files(EggInfoPkgPipInstalledNoModules.files, prefix=self.site_dir) - - -class EggInfoPkgSourcesFallback(OnSysPath, SiteDir): - files: FilesSpec = { - "sources_fallback_pkg.egg-info": { - "PKG-INFO": "Name: sources_fallback-pkg", - # SOURCES.txt is made from the source archive, and contains files - # (setup.py) that are not present after installation. - "SOURCES.txt": """ - sources_fallback.py - setup.py - sources_fallback_pkg.egg-info/PKG-INFO - sources_fallback_pkg.egg-info/SOURCES.txt - """, - # missing installed-files.txt (i.e. not installed by pip) and - # missing top_level.txt (to trigger fallback to SOURCES.txt) - }, - "sources_fallback.py": """ - def main(): - print("hello world") - """, - } - - def setUp(self): - super().setUp() - build_files(EggInfoPkgSourcesFallback.files, prefix=self.site_dir) - - -class EggInfoFile(OnSysPath, SiteDir): - files: FilesSpec = { - "egginfo_file.egg-info": """ - Metadata-Version: 1.0 - Name: egginfo_file - Version: 0.1 - Summary: An example package - Home-page: www.example.com - Author: Eric Haffa-Vee - Author-email: eric@example.coms - License: UNKNOWN - Description: UNKNOWN - Platform: UNKNOWN - """, - } - - def setUp(self): - super().setUp() - build_files(EggInfoFile.files, prefix=self.site_dir) - - -# dedent all text strings before writing -orig = _path.create.registry[str] -_path.create.register(str, lambda content, path: orig(DALS(content), path)) - - -build_files = _path.build - - -def build_record(file_defs): - return ''.join(f'{name},,\n' for name in record_names(file_defs)) - - -def record_names(file_defs): - recording = _path.Recording() - _path.build(file_defs, recording) - return recording.record - - -class FileBuilder: - def unicode_filename(self): - return FS_NONASCII or self.skip("File system does not support non-ascii.") - - -def DALS(str): - "Dedent and left-strip" - return textwrap.dedent(str).lstrip() - - -@requires_zlib() -class ZipFixtures: - root = 'test.test_importlib.data' - - def _fixture_on_path(self, filename): - pkg_file = resources.files(self.root).joinpath(filename) - file = self.resources.enter_context(resources.as_file(pkg_file)) - assert file.name.startswith('example'), file.name - sys.path.insert(0, str(file)) - self.resources.callback(sys.path.pop, 0) - - def setUp(self): - # Add self.zip_name to the front of sys.path. - self.resources = contextlib.ExitStack() - self.addCleanup(self.resources.close) - - -def parameterize(*args_set): - """Run test method with a series of parameters.""" - - def wrapper(func): - @functools.wraps(func) - def _inner(self): - for args in args_set: - with self.subTest(**args): - func(self, **args) - - return _inner - - return wrapper diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py index d6ad590b3..ea1e29a4e 100644 --- a/Lib/test/test_importlib/import_/test_api.py +++ b/Lib/test/test_importlib/import_/test_api.py @@ -27,13 +27,6 @@ class BadSpecFinderLoader: raise ImportError('I cannot be loaded!') -class BadLoaderFinder: - @classmethod - def load_module(cls, fullname): - if fullname == SUBMOD_NAME: - raise ImportError('I cannot be loaded!') - - class APITest: """Test API-specific details for __import__ (e.g. raising the right @@ -93,45 +86,6 @@ class APITest: self.assertEqual(cm.exception.name, SUBMOD_NAME) -class OldAPITests(APITest): - bad_finder_loader = BadLoaderFinder - - def test_raises_ModuleNotFoundError(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_raises_ModuleNotFoundError() - - def test_name_requires_rparition(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_name_requires_rparition() - - def test_negative_level(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_negative_level() - - def test_nonexistent_fromlist_entry(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_nonexistent_fromlist_entry() - - def test_fromlist_load_error_propagates(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_fromlist_load_error_propagates - - def test_blocked_fromlist(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_blocked_fromlist() - - -(Frozen_OldAPITests, - Source_OldAPITests - ) = util.test_both(OldAPITests, __import__=util.__import__) - - class SpecAPITests(APITest): bad_finder_loader = BadSpecFinderLoader diff --git a/Lib/test/test_importlib/import_/test_helpers.py b/Lib/test/test_importlib/import_/test_helpers.py index 550f88d1d..7587276a4 100644 --- a/Lib/test/test_importlib/import_/test_helpers.py +++ b/Lib/test/test_importlib/import_/test_helpers.py @@ -19,8 +19,7 @@ class FixUpModuleTests: ns = {"__spec__": spec} _bootstrap_external._fix_up_module(ns, name, path) - expected = {"__spec__": spec, "__loader__": loader, "__file__": path, - "__cached__": None} + expected = {"__spec__": spec, "__loader__": loader, "__file__": path} self.assertEqual(ns, expected) def test_no_loader_no_spec_but_sourceless(self): @@ -29,7 +28,7 @@ class FixUpModuleTests: ns = {} _bootstrap_external._fix_up_module(ns, name, path, path) - expected = {"__file__": path, "__cached__": path} + expected = {"__file__": path} for key, val in expected.items(): with self.subTest(f"{key}: {val}"): @@ -51,7 +50,7 @@ class FixUpModuleTests: ns = {} _bootstrap_external._fix_up_module(ns, name, path) - expected = {"__file__": path, "__cached__": None} + expected = {"__file__": path} for key, val in expected.items(): with self.subTest(f"{key}: {val}"): diff --git a/Lib/test/test_importlib/metadata/__init__.py b/Lib/test/test_importlib/metadata/__init__.py index 8b1378917..e69de29bb 100644 --- a/Lib/test/test_importlib/metadata/__init__.py +++ b/Lib/test/test_importlib/metadata/__init__.py @@ -1 +0,0 @@ - diff --git a/Lib/test/test_importlib/metadata/_path.py b/Lib/test/test_importlib/metadata/_path.py index b3cfb9cd5..e63d889f9 100644 --- a/Lib/test/test_importlib/metadata/_path.py +++ b/Lib/test/test_importlib/metadata/_path.py @@ -1,9 +1,14 @@ -# from jaraco.path 3.7 +# from jaraco.path 3.7.2 + +from __future__ import annotations import functools import pathlib -from typing import Dict, Protocol, Union -from typing import runtime_checkable +from collections.abc import Mapping +from typing import TYPE_CHECKING, Protocol, Union, runtime_checkable + +if TYPE_CHECKING: + from typing_extensions import Self class Symlink(str): @@ -12,29 +17,25 @@ class Symlink(str): """ -FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] # type: ignore +FilesSpec = Mapping[str, Union[str, bytes, Symlink, 'FilesSpec']] @runtime_checkable class TreeMaker(Protocol): - def __truediv__(self, *args, **kwargs): ... # pragma: no cover - - def mkdir(self, **kwargs): ... # pragma: no cover - - def write_text(self, content, **kwargs): ... # pragma: no cover - - def write_bytes(self, content): ... # pragma: no cover - - def symlink_to(self, target): ... # pragma: no cover + def __truediv__(self, other, /) -> Self: ... + def mkdir(self, *, exist_ok) -> object: ... + def write_text(self, content, /, *, encoding) -> object: ... + def write_bytes(self, content, /) -> object: ... + def symlink_to(self, target, /) -> object: ... -def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: - return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore +def _ensure_tree_maker(obj: str | TreeMaker) -> TreeMaker: + return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) def build( spec: FilesSpec, - prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore + prefix: str | TreeMaker = pathlib.Path(), ): """ Build a set of files/directories, as described by the spec. @@ -66,23 +67,24 @@ def build( @functools.singledispatch -def create(content: Union[str, bytes, FilesSpec], path): +def create(content: str | bytes | FilesSpec, path: TreeMaker) -> None: path.mkdir(exist_ok=True) - build(content, prefix=path) # type: ignore + # Mypy only looks at the signature of the main singledispatch method. So it must contain the complete Union + build(content, prefix=path) # type: ignore[arg-type] # python/mypy#11727 @create.register -def _(content: bytes, path): +def _(content: bytes, path: TreeMaker) -> None: path.write_bytes(content) @create.register -def _(content: str, path): +def _(content: str, path: TreeMaker) -> None: path.write_text(content, encoding='utf-8') @create.register -def _(content: Symlink, path): +def _(content: Symlink, path: TreeMaker) -> None: path.symlink_to(content) diff --git a/Lib/test/test_importlib/metadata/data/__init__.py b/Lib/test/test_importlib/metadata/data/__init__.py index 8b1378917..e69de29bb 100644 --- a/Lib/test/test_importlib/metadata/data/__init__.py +++ b/Lib/test/test_importlib/metadata/data/__init__.py @@ -1 +0,0 @@ - diff --git a/Lib/test/test_importlib/metadata/fixtures.py b/Lib/test/test_importlib/metadata/fixtures.py index 826b1b325..3283697d4 100644 --- a/Lib/test/test_importlib/metadata/fixtures.py +++ b/Lib/test/test_importlib/metadata/fixtures.py @@ -1,11 +1,12 @@ -import sys -import copy -import json -import shutil -import pathlib -import textwrap -import functools import contextlib +import copy +import functools +import json +import pathlib +import shutil +import sys +import textwrap +from importlib import resources from test.support import import_helper from test.support import os_helper @@ -15,15 +16,6 @@ from . import _path from ._path import FilesSpec -try: - from importlib import resources # type: ignore - - getattr(resources, 'files') - getattr(resources, 'as_file') -except (ImportError, AttributeError): - import importlib_resources as resources # type: ignore - - @contextlib.contextmanager def tmp_path(): """ diff --git a/Lib/test/test_importlib/metadata/test_api.py b/Lib/test/test_importlib/metadata/test_api.py index 2256e0c50..5449f0484 100644 --- a/Lib/test/test_importlib/metadata/test_api.py +++ b/Lib/test/test_importlib/metadata/test_api.py @@ -1,14 +1,12 @@ +import importlib import re import textwrap import unittest -import warnings -import importlib -import contextlib -from . import fixtures from importlib.metadata import ( Distribution, PackageNotFoundError, + Prepared, distribution, entry_points, files, @@ -17,12 +15,7 @@ from importlib.metadata import ( version, ) - -@contextlib.contextmanager -def suppress_known_deprecation(): - with warnings.catch_warnings(record=True) as ctx: - warnings.simplefilter('default', category=DeprecationWarning) - yield ctx +from . import fixtures class APITests( @@ -153,13 +146,13 @@ class APITests( classifiers = md.get_all('Classifier') assert 'Topic :: Software Development :: Libraries' in classifiers - def test_missing_key_legacy(self): + def test_missing_key(self): """ - Requesting a missing key will still return None, but warn. + Requesting a missing key raises KeyError. """ md = metadata('distinfo-pkg') - with suppress_known_deprecation(): - assert md['does-not-exist'] is None + with self.assertRaises(KeyError): + md['does-not-exist'] def test_get_key(self): """ @@ -321,3 +314,34 @@ class InvalidateCache(unittest.TestCase): def test_invalidate_cache(self): # No externally observable behavior, but ensures test coverage... importlib.invalidate_caches() + + +class PreparedTests(unittest.TestCase): + @fixtures.parameterize( + # Simple + dict(input='sample', expected='sample'), + # Mixed case + dict(input='Sample', expected='sample'), + dict(input='SAMPLE', expected='sample'), + dict(input='SaMpLe', expected='sample'), + # Separator conversions + dict(input='sample-pkg', expected='sample_pkg'), + dict(input='sample.pkg', expected='sample_pkg'), + dict(input='sample_pkg', expected='sample_pkg'), + # Multiple separators + dict(input='sample---pkg', expected='sample_pkg'), + dict(input='sample___pkg', expected='sample_pkg'), + dict(input='sample...pkg', expected='sample_pkg'), + # Mixed separators + dict(input='sample-._pkg', expected='sample_pkg'), + dict(input='sample_.-pkg', expected='sample_pkg'), + # Complex + dict(input='Sample__Pkg-name.foo', expected='sample_pkg_name_foo'), + dict(input='Sample__Pkg.name__foo', expected='sample_pkg_name_foo'), + # Uppercase with separators + dict(input='SAMPLE-PKG', expected='sample_pkg'), + dict(input='Sample.Pkg', expected='sample_pkg'), + dict(input='SAMPLE_PKG', expected='sample_pkg'), + ) + def test_normalize(self, input, expected): + self.assertEqual(Prepared.normalize(input), expected) diff --git a/Lib/test/test_importlib/metadata/test_main.py b/Lib/test/test_importlib/metadata/test_main.py index e4218076f..aae052160 100644 --- a/Lib/test/test_importlib/metadata/test_main.py +++ b/Lib/test/test_importlib/metadata/test_main.py @@ -1,23 +1,18 @@ -import re -import pickle -import unittest -import warnings import importlib -import importlib.metadata -import contextlib -from test.support import os_helper +import pickle +import re +import unittest try: import pyfakefs.fake_filesystem_unittest as ffs except ImportError: from .stubs import fake_filesystem_unittest as ffs +from test.support import os_helper -from . import fixtures -from ._context import suppress -from ._path import Symlink from importlib.metadata import ( Distribution, EntryPoint, + MetadataNotFound, PackageNotFoundError, _unique, distributions, @@ -27,12 +22,8 @@ from importlib.metadata import ( version, ) - -@contextlib.contextmanager -def suppress_known_deprecation(): - with warnings.catch_warnings(record=True) as ctx: - warnings.simplefilter('default', category=DeprecationWarning) - yield ctx +from . import fixtures +from ._path import Symlink class BasicTests(fixtures.DistInfoPkg, unittest.TestCase): @@ -59,9 +50,6 @@ class BasicTests(fixtures.DistInfoPkg, unittest.TestCase): assert "metadata" in str(ctx.exception) - # expected to fail until ABC is enforced - @suppress(AssertionError) - @suppress_known_deprecation() def test_abc_enforced(self): with self.assertRaises(TypeError): type('DistributionSubclass', (Distribution,), {})() @@ -146,6 +134,43 @@ class NameNormalizationTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.Test assert len(after) == len(before) +class InvalidMetadataTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + @staticmethod + def make_pkg(name, files=dict(METADATA="VERSION: 1.0")): + """ + Create metadata for a dist-info package with name and files. + """ + return { + f'{name}.dist-info': files, + } + + def test_valid_dists_preferred(self): + """ + Dists with metadata should be preferred when discovered by name. + + Ref python/importlib_metadata#489. + """ + # create three dists with the valid one in the middle (lexicographically) + # such that on most file systems, the valid one is never naturally first. + fixtures.build_files(self.make_pkg('foo-4.0', files={}), self.site_dir) + fixtures.build_files(self.make_pkg('foo-4.1'), self.site_dir) + fixtures.build_files(self.make_pkg('foo-4.2', files={}), self.site_dir) + dist = Distribution.from_name('foo') + assert dist.version == "1.0" + + def test_missing_metadata(self): + """ + Dists with a missing metadata file should raise ``MetadataNotFound``. + + Ref python/importlib_metadata#493 and python/cpython#143387. + """ + fixtures.build_files(self.make_pkg('foo-4.3', files={}), self.site_dir) + with self.assertRaises(MetadataNotFound): + Distribution.from_name('foo').metadata + with self.assertRaises(MetadataNotFound): + metadata('foo') + + class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): @staticmethod def pkg_with_non_ascii_description(site_dir): diff --git a/Lib/test/test_importlib/metadata/test_zip.py b/Lib/test/test_importlib/metadata/test_zip.py index 276f6288c..971685496 100644 --- a/Lib/test/test_importlib/metadata/test_zip.py +++ b/Lib/test/test_importlib/metadata/test_zip.py @@ -1,8 +1,12 @@ +import multiprocessing +import os import sys import unittest -from . import fixtures +from test.support import requires_fork, warnings_helper + from importlib.metadata import ( + FastPath, PackageNotFoundError, distribution, distributions, @@ -11,6 +15,8 @@ from importlib.metadata import ( version, ) +from . import fixtures + class TestZip(fixtures.ZipFixtures, unittest.TestCase): def setUp(self): @@ -46,6 +52,39 @@ class TestZip(fixtures.ZipFixtures, unittest.TestCase): dists = list(distributions(path=sys.path[:1])) assert len(dists) == 1 + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + @requires_fork() + @unittest.skipUnless( + hasattr(os, 'register_at_fork') + and 'fork' in multiprocessing.get_all_start_methods(), + 'requires fork-based multiprocessing support', + ) + def test_fastpath_cache_cleared_in_forked_child(self): + zip_path = sys.path[0] + + FastPath(zip_path) + assert FastPath.__new__.cache_info().currsize >= 1 + + ctx = multiprocessing.get_context('fork') + parent_conn, child_conn = ctx.Pipe() + + def child(conn, root): + try: + before = FastPath.__new__.cache_info().currsize + FastPath(root) + after = FastPath.__new__.cache_info().currsize + conn.send((before, after)) + finally: + conn.close() + + proc = ctx.Process(target=child, args=(child_conn, zip_path)) + proc.start() + child_conn.close() + cache_sizes = parent_conn.recv() + proc.join() + + self.assertEqual(cache_sizes, (0, 1)) + class TestEgg(TestZip): def setUp(self): diff --git a/Lib/test/test_importlib/namespace_pkgs/foo/README.md b/Lib/test/test_importlib/namespace_pkgs/foo/README.md new file mode 100644 index 000000000..9f6bc7494 --- /dev/null +++ b/Lib/test/test_importlib/namespace_pkgs/foo/README.md @@ -0,0 +1,2 @@ +This directory should not be a package, but should share a name with an +unrelated subpackage. diff --git a/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py index 8b1378917..e69de29bb 100644 --- a/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py +++ b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py @@ -1 +0,0 @@ - diff --git a/Lib/test/test_importlib/namespacedata01/binary.file b/Lib/test/test_importlib/namespacedata01/binary.file deleted file mode 100644 index eaf36c1da..000000000 Binary files a/Lib/test/test_importlib/namespacedata01/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/namespacedata01/utf-16.file b/Lib/test/test_importlib/namespacedata01/utf-16.file deleted file mode 100644 index 2cb772295..000000000 Binary files a/Lib/test/test_importlib/namespacedata01/utf-16.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/__init__.py b/Lib/test/test_importlib/resources/__init__.py index 8b1378917..e69de29bb 100644 --- a/Lib/test/test_importlib/resources/__init__.py +++ b/Lib/test/test_importlib/resources/__init__.py @@ -1 +0,0 @@ - diff --git a/Lib/test/test_importlib/resources/_path.py b/Lib/test/test_importlib/resources/_path.py index b144628cb..3720af7c5 100644 --- a/Lib/test/test_importlib/resources/_path.py +++ b/Lib/test/test_importlib/resources/_path.py @@ -1,10 +1,6 @@ -import pathlib import functools - -from typing import Dict, Union -from typing import runtime_checkable -from typing import Protocol - +import pathlib +from typing import Protocol, Union, runtime_checkable #### # from jaraco.path 3.7.1 @@ -16,7 +12,7 @@ class Symlink(str): """ -FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] +FilesSpec = dict[str, Union[str, bytes, Symlink, 'FilesSpec']] @runtime_checkable @@ -32,13 +28,13 @@ class TreeMaker(Protocol): def symlink_to(self, target): ... # pragma: no cover -def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: +def _ensure_tree_maker(obj: str | TreeMaker) -> TreeMaker: return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore[return-value] def build( spec: FilesSpec, - prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore[assignment] + prefix: str | TreeMaker = pathlib.Path(), # type: ignore[assignment] ): """ Build a set of files/directories, as described by the spec. @@ -70,7 +66,7 @@ def build( @functools.singledispatch -def create(content: Union[str, bytes, FilesSpec], path): +def create(content: str | bytes | FilesSpec, path): path.mkdir(exist_ok=True) build(content, prefix=path) # type: ignore[arg-type] diff --git a/Lib/test/test_importlib/resources/data01/__init__.py b/Lib/test/test_importlib/resources/data01/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/Lib/test/test_importlib/resources/data01/binary.file b/Lib/test/test_importlib/resources/data01/binary.file deleted file mode 100644 index eaf36c1da..000000000 Binary files a/Lib/test/test_importlib/resources/data01/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/data01/subdirectory/__init__.py b/Lib/test/test_importlib/resources/data01/subdirectory/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/Lib/test/test_importlib/resources/data01/subdirectory/binary.file b/Lib/test/test_importlib/resources/data01/subdirectory/binary.file deleted file mode 100644 index eaf36c1da..000000000 Binary files a/Lib/test/test_importlib/resources/data01/subdirectory/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/data01/utf-16.file b/Lib/test/test_importlib/resources/data01/utf-16.file deleted file mode 100644 index 2cb772295..000000000 Binary files a/Lib/test/test_importlib/resources/data01/utf-16.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/data01/utf-8.file b/Lib/test/test_importlib/resources/data01/utf-8.file deleted file mode 100644 index 1c0132ad9..000000000 --- a/Lib/test/test_importlib/resources/data01/utf-8.file +++ /dev/null @@ -1 +0,0 @@ -Hello, UTF-8 world! diff --git a/Lib/test/test_importlib/resources/data02/__init__.py b/Lib/test/test_importlib/resources/data02/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/Lib/test/test_importlib/resources/data02/one/__init__.py b/Lib/test/test_importlib/resources/data02/one/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/Lib/test/test_importlib/resources/data02/one/resource1.txt b/Lib/test/test_importlib/resources/data02/one/resource1.txt deleted file mode 100644 index 61a813e40..000000000 --- a/Lib/test/test_importlib/resources/data02/one/resource1.txt +++ /dev/null @@ -1 +0,0 @@ -one resource diff --git a/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt b/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt deleted file mode 100644 index 48f587a2d..000000000 --- a/Lib/test/test_importlib/resources/data02/subdirectory/subsubdir/resource.txt +++ /dev/null @@ -1 +0,0 @@ -a resource \ No newline at end of file diff --git a/Lib/test/test_importlib/resources/data02/two/__init__.py b/Lib/test/test_importlib/resources/data02/two/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/Lib/test/test_importlib/resources/data02/two/resource2.txt b/Lib/test/test_importlib/resources/data02/two/resource2.txt deleted file mode 100644 index a80ce46ea..000000000 --- a/Lib/test/test_importlib/resources/data02/two/resource2.txt +++ /dev/null @@ -1 +0,0 @@ -two resource diff --git a/Lib/test/test_importlib/resources/data03/__init__.py b/Lib/test/test_importlib/resources/data03/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/portion1/__init__.py b/Lib/test/test_importlib/resources/data03/namespace/portion1/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/portion2/__init__.py b/Lib/test/test_importlib/resources/data03/namespace/portion2/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/Lib/test/test_importlib/resources/data03/namespace/resource1.txt b/Lib/test/test_importlib/resources/data03/namespace/resource1.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/Lib/test/test_importlib/resources/namespacedata01/binary.file b/Lib/test/test_importlib/resources/namespacedata01/binary.file deleted file mode 100644 index eaf36c1da..000000000 Binary files a/Lib/test/test_importlib/resources/namespacedata01/binary.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/namespacedata01/utf-16.file b/Lib/test/test_importlib/resources/namespacedata01/utf-16.file deleted file mode 100644 index 2cb772295..000000000 Binary files a/Lib/test/test_importlib/resources/namespacedata01/utf-16.file and /dev/null differ diff --git a/Lib/test/test_importlib/resources/namespacedata01/utf-8.file b/Lib/test/test_importlib/resources/namespacedata01/utf-8.file deleted file mode 100644 index 1c0132ad9..000000000 --- a/Lib/test/test_importlib/resources/namespacedata01/utf-8.file +++ /dev/null @@ -1 +0,0 @@ -Hello, UTF-8 world! diff --git a/Lib/test/test_importlib/resources/test_compatibilty_files.py b/Lib/test/test_importlib/resources/test_compatibilty_files.py index bcf608d9e..2fd39bedf 100644 --- a/Lib/test/test_importlib/resources/test_compatibilty_files.py +++ b/Lib/test/test_importlib/resources/test_compatibilty_files.py @@ -1,8 +1,6 @@ +import importlib.resources as resources import io import unittest - -from importlib import resources - from importlib.resources._adapters import ( CompatibilityFiles, wrap_spec, @@ -26,51 +24,46 @@ class CompatibilityFilesTests(unittest.TestCase): return resources.files(self.package) def test_spec_path_iter(self): - self.assertEqual( - sorted(path.name for path in self.files.iterdir()), - ['a', 'b', 'c'], - ) + assert sorted(path.name for path in self.files.iterdir()) == ['a', 'b', 'c'] def test_child_path_iter(self): - self.assertEqual(list((self.files / 'a').iterdir()), []) + assert list((self.files / 'a').iterdir()) == [] def test_orphan_path_iter(self): - self.assertEqual(list((self.files / 'a' / 'a').iterdir()), []) - self.assertEqual(list((self.files / 'a' / 'a' / 'a').iterdir()), []) + assert list((self.files / 'a' / 'a').iterdir()) == [] + assert list((self.files / 'a' / 'a' / 'a').iterdir()) == [] def test_spec_path_is(self): - self.assertFalse(self.files.is_file()) - self.assertFalse(self.files.is_dir()) + assert not self.files.is_file() + assert not self.files.is_dir() def test_child_path_is(self): - self.assertTrue((self.files / 'a').is_file()) - self.assertFalse((self.files / 'a').is_dir()) + assert (self.files / 'a').is_file() + assert not (self.files / 'a').is_dir() def test_orphan_path_is(self): - self.assertFalse((self.files / 'a' / 'a').is_file()) - self.assertFalse((self.files / 'a' / 'a').is_dir()) - self.assertFalse((self.files / 'a' / 'a' / 'a').is_file()) - self.assertFalse((self.files / 'a' / 'a' / 'a').is_dir()) + assert not (self.files / 'a' / 'a').is_file() + assert not (self.files / 'a' / 'a').is_dir() + assert not (self.files / 'a' / 'a' / 'a').is_file() + assert not (self.files / 'a' / 'a' / 'a').is_dir() def test_spec_path_name(self): - self.assertEqual(self.files.name, 'testingpackage') + assert self.files.name == 'testingpackage' def test_child_path_name(self): - self.assertEqual((self.files / 'a').name, 'a') + assert (self.files / 'a').name == 'a' def test_orphan_path_name(self): - self.assertEqual((self.files / 'a' / 'b').name, 'b') - self.assertEqual((self.files / 'a' / 'b' / 'c').name, 'c') + assert (self.files / 'a' / 'b').name == 'b' + assert (self.files / 'a' / 'b' / 'c').name == 'c' def test_spec_path_open(self): - self.assertEqual(self.files.read_bytes(), b'Hello, world!') - self.assertEqual(self.files.read_text(encoding='utf-8'), 'Hello, world!') + assert self.files.read_bytes() == b'Hello, world!' + assert self.files.read_text(encoding='utf-8') == 'Hello, world!' def test_child_path_open(self): - self.assertEqual((self.files / 'a').read_bytes(), b'Hello, world!') - self.assertEqual( - (self.files / 'a').read_text(encoding='utf-8'), 'Hello, world!' - ) + assert (self.files / 'a').read_bytes() == b'Hello, world!' + assert (self.files / 'a').read_text(encoding='utf-8') == 'Hello, world!' def test_orphan_path_open(self): with self.assertRaises(FileNotFoundError): @@ -88,7 +81,7 @@ class CompatibilityFilesTests(unittest.TestCase): def test_wrap_spec(self): spec = wrap_spec(self.package) - self.assertIsInstance(spec.loader.get_resource_reader(None), CompatibilityFiles) + assert isinstance(spec.loader.get_resource_reader(None), CompatibilityFiles) class CompatibilityFilesNoReaderTests(unittest.TestCase): @@ -101,4 +94,4 @@ class CompatibilityFilesNoReaderTests(unittest.TestCase): return resources.files(self.package) def test_spec_path_joinpath(self): - self.assertIsInstance(self.files / 'a', CompatibilityFiles.OrphanPath) + assert isinstance(self.files / 'a', CompatibilityFiles.OrphanPath) diff --git a/Lib/test/test_importlib/resources/test_contents.py b/Lib/test/test_importlib/resources/test_contents.py index 4e4e0e9c3..bdc158d85 100644 --- a/Lib/test/test_importlib/resources/test_contents.py +++ b/Lib/test/test_importlib/resources/test_contents.py @@ -1,5 +1,5 @@ +import importlib.resources as resources import unittest -from importlib import resources from . import util diff --git a/Lib/test/test_importlib/resources/test_custom.py b/Lib/test/test_importlib/resources/test_custom.py index 640f90fc0..a7fc6bc35 100644 --- a/Lib/test/test_importlib/resources/test_custom.py +++ b/Lib/test/test_importlib/resources/test_custom.py @@ -1,12 +1,12 @@ -import unittest import contextlib +import importlib.resources as resources import pathlib +import unittest +from importlib.resources import abc +from importlib.resources.abc import ResourceReader, TraversableResources from test.support import os_helper -from importlib import resources -from importlib.resources import abc -from importlib.resources.abc import TraversableResources, ResourceReader from . import util diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 3ce44999f..c922d32ce 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -1,15 +1,16 @@ +import contextlib +import importlib +import importlib.resources as resources import pathlib import py_compile import textwrap import unittest import warnings -import importlib -import contextlib - -from importlib import resources from importlib.resources.abc import Traversable + +from test.support import import_helper, os_helper + from . import util -from test.support import os_helper, import_helper @contextlib.contextmanager @@ -36,15 +37,7 @@ class FilesTests: def test_joinpath_with_multiple_args(self): files = resources.files(self.data) binfile = files.joinpath('subdirectory', 'binary.file') - self.assertTrue(binfile.is_file()) - - def test_old_parameter(self): - """ - Files used to take a 'package' parameter. Make sure anyone - passing by name is still supported. - """ - with suppress_known_deprecation(): - resources.files(package=self.data) + assert binfile.is_file() class OpenDiskTests(FilesTests, util.DiskSetup, unittest.TestCase): @@ -70,7 +63,7 @@ class OpenNamespaceTests(FilesTests, util.DiskSetup, unittest.TestCase): to cause the ``PathEntryFinder`` to be called when searching for packages. In that case, resources should still be loadable. """ - import namespacedata01 + import namespacedata01 # type: ignore[import-not-found] namespacedata01.__path__.append( '__editable__.sample_namespace-1.0.finder.__path_hook__' @@ -161,7 +154,9 @@ class ImplicitContextFiles: sources = pathlib.Path(resources.__file__).parent for source_path in sources.glob('**/*.py'): - c_path = c_resources.joinpath(source_path.relative_to(sources)).with_suffix('.pyc') + c_path = c_resources.joinpath(source_path.relative_to(sources)).with_suffix( + '.pyc' + ) py_compile.compile(source_path, c_path) self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site)) diff --git a/Lib/test/test_importlib/resources/test_functional.py b/Lib/test/test_importlib/resources/test_functional.py index e8d25fa4d..9cec6af9a 100644 --- a/Lib/test/test_importlib/resources/test_functional.py +++ b/Lib/test/test_importlib/resources/test_functional.py @@ -1,17 +1,12 @@ -import unittest -import os import importlib +import importlib.resources as resources +import os +import unittest from test.support import warnings_helper -from importlib import resources - from . import util -# Since the functional API forwards to Traversable, we only test -# filesystem resources here -- not zip files, namespace packages etc. -# We do test for two kinds of Anchor, though. - class StringAnchorMixin: anchor01 = 'data01' @@ -28,7 +23,7 @@ class ModuleAnchorMixin: return importlib.import_module('data02') -class FunctionalAPIBase(util.DiskSetup): +class FunctionalAPIBase: def setUp(self): super().setUp() self.load_fixture('data02') @@ -43,83 +38,90 @@ class FunctionalAPIBase(util.DiskSetup): with self.subTest(path_parts=path_parts): yield path_parts + @staticmethod + def remove_utf16_bom(string): + """Remove an architecture-specific UTF-16 BOM prefix when present. + + Some platforms surface UTF-16 BOM bytes as escaped text when the + fixture is intentionally decoded as UTF-8 with ``errors='backslashreplace'``. + Strip that prefix so assertions validate content consistently.""" + for bom in ('\\xff\\xfe', '\\xfe\\xff', '\ufeff'): + if string.startswith(bom): + return string[len(bom) :] + return string + def test_read_text(self): - self.assertEqual( - resources.read_text(self.anchor01, 'utf-8.file'), - 'Hello, UTF-8 world!\n', + assert ( + resources.read_text(self.anchor01, 'utf-8.file') == 'Hello, UTF-8 world!\n' ) - self.assertEqual( + assert ( resources.read_text( self.anchor02, 'subdirectory', 'subsubdir', 'resource.txt', encoding='utf-8', - ), - 'a resource', + ) + == 'a resource' ) for path_parts in self._gen_resourcetxt_path_parts(): - self.assertEqual( + assert ( resources.read_text( self.anchor02, *path_parts, encoding='utf-8', - ), - 'a resource', + ) + == 'a resource' ) # Use generic OSError, since e.g. attempting to read a directory can # fail with PermissionError rather than IsADirectoryError with self.assertRaises(OSError): resources.read_text(self.anchor01) - with self.assertRaises(OSError): + with self.assertRaises((OSError, resources.abc.TraversalError)): resources.read_text(self.anchor01, 'no-such-file') with self.assertRaises(UnicodeDecodeError): resources.read_text(self.anchor01, 'utf-16.file') - self.assertEqual( + assert ( resources.read_text( self.anchor01, 'binary.file', encoding='latin1', - ), - '\x00\x01\x02\x03', + ) + == '\x00\x01\x02\x03' ) - self.assertEndsWith( # ignore the BOM + assert self.remove_utf16_bom( resources.read_text( self.anchor01, 'utf-16.file', errors='backslashreplace', ), - 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( - errors='backslashreplace', - ), + ) == 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', ) def test_read_binary(self): - self.assertEqual( - resources.read_binary(self.anchor01, 'utf-8.file'), - b'Hello, UTF-8 world!\n', + assert ( + resources.read_binary(self.anchor01, 'utf-8.file') + == b'Hello, UTF-8 world!\n' ) for path_parts in self._gen_resourcetxt_path_parts(): - self.assertEqual( - resources.read_binary(self.anchor02, *path_parts), - b'a resource', - ) + assert resources.read_binary(self.anchor02, *path_parts) == b'a resource' def test_open_text(self): with resources.open_text(self.anchor01, 'utf-8.file') as f: - self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + assert f.read() == 'Hello, UTF-8 world!\n' for path_parts in self._gen_resourcetxt_path_parts(): with resources.open_text( self.anchor02, *path_parts, encoding='utf-8', ) as f: - self.assertEqual(f.read(), 'a resource') + assert f.read() == 'a resource' # Use generic OSError, since e.g. attempting to read a directory can # fail with PermissionError rather than IsADirectoryError with self.assertRaises(OSError): resources.open_text(self.anchor01) - with self.assertRaises(OSError): + with self.assertRaises((OSError, resources.abc.TraversalError)): resources.open_text(self.anchor01, 'no-such-file') with resources.open_text(self.anchor01, 'utf-16.file') as f: with self.assertRaises(UnicodeDecodeError): @@ -129,71 +131,70 @@ class FunctionalAPIBase(util.DiskSetup): 'binary.file', encoding='latin1', ) as f: - self.assertEqual(f.read(), '\x00\x01\x02\x03') + assert f.read() == '\x00\x01\x02\x03' with resources.open_text( self.anchor01, 'utf-16.file', errors='backslashreplace', ) as f: - self.assertEndsWith( # ignore the BOM - f.read(), - 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( - errors='backslashreplace', - ), + assert self.remove_utf16_bom(f.read()) == 'Hello, UTF-16 world!\n'.encode( + 'utf-16-le' + ).decode( + errors='backslashreplace', ) def test_open_binary(self): with resources.open_binary(self.anchor01, 'utf-8.file') as f: - self.assertEqual(f.read(), b'Hello, UTF-8 world!\n') + assert f.read() == b'Hello, UTF-8 world!\n' for path_parts in self._gen_resourcetxt_path_parts(): with resources.open_binary( self.anchor02, *path_parts, ) as f: - self.assertEqual(f.read(), b'a resource') + assert f.read() == b'a resource' def test_path(self): with resources.path(self.anchor01, 'utf-8.file') as path: with open(str(path), encoding='utf-8') as f: - self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + assert f.read() == 'Hello, UTF-8 world!\n' with resources.path(self.anchor01) as path: with open(os.path.join(path, 'utf-8.file'), encoding='utf-8') as f: - self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + assert f.read() == 'Hello, UTF-8 world!\n' def test_is_resource(self): is_resource = resources.is_resource - self.assertTrue(is_resource(self.anchor01, 'utf-8.file')) - self.assertFalse(is_resource(self.anchor01, 'no_such_file')) - self.assertFalse(is_resource(self.anchor01)) - self.assertFalse(is_resource(self.anchor01, 'subdirectory')) + assert is_resource(self.anchor01, 'utf-8.file') + assert not is_resource(self.anchor01, 'no_such_file') + assert not is_resource(self.anchor01) + assert not is_resource(self.anchor01, 'subdirectory') for path_parts in self._gen_resourcetxt_path_parts(): - self.assertTrue(is_resource(self.anchor02, *path_parts)) + assert is_resource(self.anchor02, *path_parts) def test_contents(self): with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): c = resources.contents(self.anchor01) - self.assertGreaterEqual( - set(c), - {'utf-8.file', 'utf-16.file', 'binary.file', 'subdirectory'}, - ) - with self.assertRaises(OSError), warnings_helper.check_warnings(( - ".*contents.*", - DeprecationWarning, - )): + assert set(c) >= {'utf-8.file', 'utf-16.file', 'binary.file', 'subdirectory'} + with ( + self.assertRaises(OSError), + warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )), + ): list(resources.contents(self.anchor01, 'utf-8.file')) for path_parts in self._gen_resourcetxt_path_parts(): - with self.assertRaises(OSError), warnings_helper.check_warnings(( - ".*contents.*", - DeprecationWarning, - )): + with ( + self.assertRaises((OSError, resources.abc.TraversalError)), + warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )), + ): list(resources.contents(self.anchor01, *path_parts)) with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): c = resources.contents(self.anchor01, 'subdirectory') - self.assertGreaterEqual( - set(c), - {'binary.file'}, - ) + assert set(c) >= {'binary.file'} @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_common_errors(self): @@ -233,17 +234,28 @@ class FunctionalAPIBase(util.DiskSetup): ) -class FunctionalAPITest_StringAnchor( +class FunctionalAPITest_StringAnchor_Disk( StringAnchorMixin, FunctionalAPIBase, + util.DiskSetup, unittest.TestCase, ): pass -class FunctionalAPITest_ModuleAnchor( +class FunctionalAPITest_ModuleAnchor_Disk( ModuleAnchorMixin, FunctionalAPIBase, + util.DiskSetup, + unittest.TestCase, +): + pass + + +class FunctionalAPITest_StringAnchor_Memory( + StringAnchorMixin, + FunctionalAPIBase, + util.MemorySetup, unittest.TestCase, ): pass diff --git a/Lib/test/test_importlib/resources/test_open.py b/Lib/test/test_importlib/resources/test_open.py index 8c00378ad..950f71db0 100644 --- a/Lib/test/test_importlib/resources/test_open.py +++ b/Lib/test/test_importlib/resources/test_open.py @@ -1,6 +1,6 @@ +import importlib.resources as resources import unittest -from importlib import resources from . import util @@ -23,19 +23,19 @@ class OpenTests: target = resources.files(self.data) / 'binary.file' with target.open('rb') as fp: result = fp.read() - self.assertEqual(result, bytes(range(4))) + assert result == bytes(range(4)) def test_open_text_default_encoding(self): target = resources.files(self.data) / 'utf-8.file' with target.open(encoding='utf-8') as fp: result = fp.read() - self.assertEqual(result, 'Hello, UTF-8 world!\n') + assert result == 'Hello, UTF-8 world!\n' def test_open_text_given_encoding(self): target = resources.files(self.data) / 'utf-16.file' with target.open(encoding='utf-16', errors='strict') as fp: result = fp.read() - self.assertEqual(result, 'Hello, UTF-16 world!\n') + assert result == 'Hello, UTF-16 world!\n' def test_open_text_with_errors(self): """ @@ -46,11 +46,10 @@ class OpenTests: self.assertRaises(UnicodeError, fp.read) with target.open(encoding='utf-8', errors='ignore') as fp: result = fp.read() - self.assertEqual( - result, + assert result == ( 'H\x00e\x00l\x00l\x00o\x00,\x00 ' '\x00U\x00T\x00F\x00-\x001\x006\x00 ' - '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00', + '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00' ) def test_open_binary_FileNotFoundError(self): diff --git a/Lib/test/test_importlib/resources/test_path.py b/Lib/test/test_importlib/resources/test_path.py index 903911f57..162344e5d 100644 --- a/Lib/test/test_importlib/resources/test_path.py +++ b/Lib/test/test_importlib/resources/test_path.py @@ -1,8 +1,8 @@ +import importlib.resources as resources import io import pathlib import unittest -from importlib import resources from . import util @@ -19,9 +19,9 @@ class PathTests: """ target = resources.files(self.data) / 'utf-8.file' with resources.as_file(target) as path: - self.assertIsInstance(path, pathlib.Path) - self.assertEndsWith(path.name, "utf-8.file") - self.assertEqual('Hello, UTF-8 world!\n', path.read_text(encoding='utf-8')) + assert isinstance(path, pathlib.Path) + assert path.name.endswith("utf-8.file"), repr(path) + assert 'Hello, UTF-8 world!\n' == path.read_text(encoding='utf-8') class PathDiskTests(PathTests, util.DiskSetup, unittest.TestCase): diff --git a/Lib/test/test_importlib/resources/test_read.py b/Lib/test/test_importlib/resources/test_read.py index 59c237d96..4085a64b0 100644 --- a/Lib/test/test_importlib/resources/test_read.py +++ b/Lib/test/test_importlib/resources/test_read.py @@ -1,6 +1,6 @@ +import importlib.resources as resources import unittest - -from importlib import import_module, resources +from importlib import import_module from . import util @@ -18,23 +18,25 @@ class CommonTextTests(util.CommonTests, unittest.TestCase): class ReadTests: def test_read_bytes(self): result = resources.files(self.data).joinpath('binary.file').read_bytes() - self.assertEqual(result, bytes(range(4))) + assert result == bytes(range(4)) def test_read_text_default_encoding(self): result = ( - resources.files(self.data) + resources + .files(self.data) .joinpath('utf-8.file') .read_text(encoding='utf-8') ) - self.assertEqual(result, 'Hello, UTF-8 world!\n') + assert result == 'Hello, UTF-8 world!\n' def test_read_text_given_encoding(self): result = ( - resources.files(self.data) + resources + .files(self.data) .joinpath('utf-16.file') .read_text(encoding='utf-16') ) - self.assertEqual(result, 'Hello, UTF-16 world!\n') + assert result == 'Hello, UTF-16 world!\n' def test_read_text_with_errors(self): """ @@ -43,11 +45,10 @@ class ReadTests: target = resources.files(self.data) / 'utf-16.file' self.assertRaises(UnicodeError, target.read_text, encoding='utf-8') result = target.read_text(encoding='utf-8', errors='ignore') - self.assertEqual( - result, + assert result == ( 'H\x00e\x00l\x00l\x00o\x00,\x00 ' '\x00U\x00T\x00F\x00-\x001\x006\x00 ' - '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00', + '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00' ) @@ -59,13 +60,13 @@ class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): submodule = import_module('data01.subdirectory') result = resources.files(submodule).joinpath('binary.file').read_bytes() - self.assertEqual(result, bytes(range(4, 8))) + assert result == bytes(range(4, 8)) def test_read_submodule_resource_by_name(self): result = ( resources.files('data01.subdirectory').joinpath('binary.file').read_bytes() ) - self.assertEqual(result, bytes(range(4, 8))) + assert result == bytes(range(4, 8)) class ReadNamespaceTests(ReadTests, util.DiskSetup, unittest.TestCase): @@ -78,15 +79,16 @@ class ReadNamespaceZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): submodule = import_module('namespacedata01.subdirectory') result = resources.files(submodule).joinpath('binary.file').read_bytes() - self.assertEqual(result, bytes(range(12, 16))) + assert result == bytes(range(12, 16)) def test_read_submodule_resource_by_name(self): result = ( - resources.files('namespacedata01.subdirectory') + resources + .files('namespacedata01.subdirectory') .joinpath('binary.file') .read_bytes() ) - self.assertEqual(result, bytes(range(12, 16))) + assert result == bytes(range(12, 16)) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/test_reader.py b/Lib/test/test_importlib/resources/test_reader.py index ed5693ab4..691a78bb0 100644 --- a/Lib/test/test_importlib/resources/test_reader.py +++ b/Lib/test/test_importlib/resources/test_reader.py @@ -1,9 +1,8 @@ import os.path import pathlib import unittest - from importlib import import_module -from importlib.readers import MultiplexedPath, NamespaceReader +from importlib.resources.readers import MultiplexedPath, NamespaceReader from . import util @@ -31,9 +30,7 @@ class MultiplexedPathTest(util.DiskSetup, unittest.TestCase): contents.remove('__pycache__') except (KeyError, ValueError): pass - self.assertEqual( - contents, {'subdirectory', 'binary.file', 'utf-16.file', 'utf-8.file'} - ) + assert contents == {'subdirectory', 'binary.file', 'utf-16.file', 'utf-8.file'} def test_iterdir_duplicate(self): contents = { @@ -44,16 +41,19 @@ class MultiplexedPathTest(util.DiskSetup, unittest.TestCase): contents.remove(remove) except (KeyError, ValueError): pass - self.assertEqual( - contents, - {'__init__.py', 'binary.file', 'subdirectory', 'utf-16.file', 'utf-8.file'}, - ) + assert contents == { + '__init__.py', + 'binary.file', + 'subdirectory', + 'utf-16.file', + 'utf-8.file', + } def test_is_dir(self): - self.assertEqual(MultiplexedPath(self.folder).is_dir(), True) + assert MultiplexedPath(self.folder).is_dir() def test_is_file(self): - self.assertEqual(MultiplexedPath(self.folder).is_file(), False) + assert not MultiplexedPath(self.folder).is_file() def test_open_file(self): path = MultiplexedPath(self.folder) @@ -67,19 +67,17 @@ class MultiplexedPathTest(util.DiskSetup, unittest.TestCase): def test_join_path(self): prefix = str(self.folder.parent) path = MultiplexedPath(self.folder, self.data01) - self.assertEqual( - str(path.joinpath('binary.file'))[len(prefix) + 1 :], - os.path.join('namespacedata01', 'binary.file'), + assert str(path.joinpath('binary.file'))[len(prefix) + 1 :] == os.path.join( + 'namespacedata01', 'binary.file' ) sub = path.joinpath('subdirectory') assert isinstance(sub, MultiplexedPath) assert 'namespacedata01' in str(sub) assert 'data01' in str(sub) - self.assertEqual( - str(path.joinpath('imaginary'))[len(prefix) + 1 :], - os.path.join('namespacedata01', 'imaginary'), + assert str(path.joinpath('imaginary'))[len(prefix) + 1 :] == os.path.join( + 'namespacedata01', 'imaginary' ) - self.assertEqual(path.joinpath(), path) + assert path.joinpath() == path def test_join_path_compound(self): path = MultiplexedPath(self.folder) @@ -88,23 +86,16 @@ class MultiplexedPathTest(util.DiskSetup, unittest.TestCase): def test_join_path_common_subdir(self): prefix = str(self.data02.parent) path = MultiplexedPath(self.data01, self.data02) - self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) - self.assertEqual( - str(path.joinpath('subdirectory', 'subsubdir'))[len(prefix) + 1 :], - os.path.join('data02', 'subdirectory', 'subsubdir'), + assert isinstance(path.joinpath('subdirectory'), MultiplexedPath) + assert str(path.joinpath('subdirectory', 'subsubdir'))[len(prefix) + 1 :] == ( + os.path.join('data02', 'subdirectory', 'subsubdir') ) def test_repr(self): - self.assertEqual( - repr(MultiplexedPath(self.folder)), - f"MultiplexedPath('{self.folder}')", - ) + assert repr(MultiplexedPath(self.folder)) == f"MultiplexedPath('{self.folder}')" def test_name(self): - self.assertEqual( - MultiplexedPath(self.folder).name, - os.path.basename(self.folder), - ) + assert MultiplexedPath(self.folder).name == os.path.basename(self.folder) class NamespaceReaderTest(util.DiskSetup, unittest.TestCase): @@ -119,18 +110,14 @@ class NamespaceReaderTest(util.DiskSetup, unittest.TestCase): reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) root = self.data.__path__[0] - self.assertEqual( - reader.resource_path('binary.file'), os.path.join(root, 'binary.file') - ) - self.assertEqual( - reader.resource_path('imaginary'), os.path.join(root, 'imaginary') - ) + assert reader.resource_path('binary.file') == os.path.join(root, 'binary.file') + assert reader.resource_path('imaginary') == os.path.join(root, 'imaginary') def test_files(self): reader = NamespaceReader(self.data.__spec__.submodule_search_locations) root = self.data.__path__[0] - self.assertIsInstance(reader.files(), MultiplexedPath) - self.assertEqual(repr(reader.files()), f"MultiplexedPath('{root}')") + assert isinstance(reader.files(), MultiplexedPath) + assert repr(reader.files()) == f"MultiplexedPath('{root}')" if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/test_resource.py b/Lib/test/test_importlib/resources/test_resource.py index fcede14b8..b114b1f0d 100644 --- a/Lib/test/test_importlib/resources/test_resource.py +++ b/Lib/test/test_importlib/resources/test_resource.py @@ -1,7 +1,9 @@ +import importlib.resources as resources +import types import unittest +from importlib import import_module from . import util -from importlib import resources, import_module class ResourceTests: @@ -9,16 +11,16 @@ class ResourceTests: def test_is_file_exists(self): target = resources.files(self.data) / 'binary.file' - self.assertTrue(target.is_file()) + assert target.is_file() def test_is_file_missing(self): target = resources.files(self.data) / 'not-a-file' - self.assertFalse(target.is_file()) + assert not target.is_file() def test_is_dir(self): target = resources.files(self.data) / 'subdirectory' - self.assertFalse(target.is_file()) - self.assertTrue(target.is_dir()) + assert not target.is_file() + assert target.is_dir() class ResourceDiskTests(ResourceTests, util.DiskSetup, unittest.TestCase): @@ -38,7 +40,7 @@ class ResourceLoaderTests(util.DiskSetup, unittest.TestCase): package = util.create_package( file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) - self.assertEqual(names(resources.files(package)), {'A', 'B', 'C'}) + assert names(resources.files(package)) == {'A', 'B', 'C'} def test_is_file(self): package = util.create_package( @@ -46,7 +48,7 @@ class ResourceLoaderTests(util.DiskSetup, unittest.TestCase): path=self.data.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'], ) - self.assertTrue(resources.files(package).joinpath('B').is_file()) + assert resources.files(package).joinpath('B').is_file() def test_is_dir(self): package = util.create_package( @@ -54,7 +56,7 @@ class ResourceLoaderTests(util.DiskSetup, unittest.TestCase): path=self.data.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'], ) - self.assertTrue(resources.files(package).joinpath('D').is_dir()) + assert resources.files(package).joinpath('D').is_dir() def test_resource_missing(self): package = util.create_package( @@ -62,7 +64,7 @@ class ResourceLoaderTests(util.DiskSetup, unittest.TestCase): path=self.data.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'], ) - self.assertFalse(resources.files(package).joinpath('Z').is_file()) + assert not resources.files(package).joinpath('Z').is_file() class ResourceCornerCaseTests(util.DiskSetup, unittest.TestCase): @@ -82,30 +84,26 @@ class ResourceCornerCaseTests(util.DiskSetup, unittest.TestCase): module.__file__ = '/path/which/shall/not/be/named' module.__spec__.loader = module.__loader__ module.__spec__.origin = module.__file__ - self.assertFalse(resources.files(module).joinpath('A').is_file()) + assert not resources.files(module).joinpath('A').is_file() class ResourceFromZipsTest01(util.ZipSetup, unittest.TestCase): def test_is_submodule_resource(self): submodule = import_module('data01.subdirectory') - self.assertTrue(resources.files(submodule).joinpath('binary.file').is_file()) + assert resources.files(submodule).joinpath('binary.file').is_file() def test_read_submodule_resource_by_name(self): - self.assertTrue( - resources.files('data01.subdirectory').joinpath('binary.file').is_file() - ) + assert resources.files('data01.subdirectory').joinpath('binary.file').is_file() def test_submodule_contents(self): submodule = import_module('data01.subdirectory') - self.assertEqual( - names(resources.files(submodule)), {'__init__.py', 'binary.file'} - ) + assert names(resources.files(submodule)) == {'__init__.py', 'binary.file'} def test_submodule_contents_by_name(self): - self.assertEqual( - names(resources.files('data01.subdirectory')), - {'__init__.py', 'binary.file'}, - ) + assert names(resources.files('data01.subdirectory')) == { + '__init__.py', + 'binary.file', + } def test_as_file_directory(self): with resources.as_file(resources.files('data01')) as data: @@ -124,14 +122,8 @@ class ResourceFromZipsTest02(util.ZipSetup, unittest.TestCase): Test thata zip with two unrelated subpackages return distinct resources. Ref python/importlib_resources#44. """ - self.assertEqual( - names(resources.files('data02.one')), - {'__init__.py', 'resource1.txt'}, - ) - self.assertEqual( - names(resources.files('data02.two')), - {'__init__.py', 'resource2.txt'}, - ) + assert names(resources.files('data02.one')) == {'__init__.py', 'resource1.txt'} + assert names(resources.files('data02.two')) == {'__init__.py', 'resource2.txt'} class DeletingZipsTest(util.ZipSetup, unittest.TestCase): @@ -168,16 +160,15 @@ class DeletingZipsTest(util.ZipSetup, unittest.TestCase): class ResourceFromNamespaceTests: def test_is_submodule_resource(self): - self.assertTrue( - resources.files(import_module('namespacedata01')) + assert ( + resources + .files(import_module('namespacedata01')) .joinpath('binary.file') .is_file() ) def test_read_submodule_resource_by_name(self): - self.assertTrue( - resources.files('namespacedata01').joinpath('binary.file').is_file() - ) + assert resources.files('namespacedata01').joinpath('binary.file').is_file() def test_submodule_contents(self): contents = names(resources.files(import_module('namespacedata01'))) @@ -185,9 +176,7 @@ class ResourceFromNamespaceTests: contents.remove('__pycache__') except KeyError: pass - self.assertEqual( - contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} - ) + assert contents == {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} def test_submodule_contents_by_name(self): contents = names(resources.files('namespacedata01')) @@ -195,9 +184,7 @@ class ResourceFromNamespaceTests: contents.remove('__pycache__') except KeyError: pass - self.assertEqual( - contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} - ) + assert contents == {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} def test_submodule_sub_contents(self): contents = names(resources.files(import_module('namespacedata01.subdirectory'))) @@ -205,7 +192,7 @@ class ResourceFromNamespaceTests: contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file'}) + assert contents == {'binary.file'} def test_submodule_sub_contents_by_name(self): contents = names(resources.files('namespacedata01.subdirectory')) @@ -213,7 +200,7 @@ class ResourceFromNamespaceTests: contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file'}) + assert contents == {'binary.file'} class ResourceFromNamespaceDiskTests( @@ -232,5 +219,24 @@ class ResourceFromNamespaceZipTests( MODULE = 'namespacedata01' +class MainModuleTests(unittest.TestCase): + def test_main_module_with_none_spec(self): + """ + __main__ module with no spec should raise TypeError (for clarity). + + See python/cpython#138531 for details. + """ + # construct a __main__ module with no __spec__. + mainmodule = types.ModuleType("__main__") + + assert mainmodule.__spec__ is None + + with self.assertRaises( + TypeError, + msg="Cannot access resources for '__main__' as it does not appear to correspond to an importable module (its __spec__ is None).", + ): + resources.files(mainmodule) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_util.py b/Lib/test/test_importlib/resources/test_util.py new file mode 100644 index 000000000..de304b6f3 --- /dev/null +++ b/Lib/test/test_importlib/resources/test_util.py @@ -0,0 +1,29 @@ +import unittest + +from .util import MemorySetup, Traversable + + +class TestMemoryTraversableImplementation(unittest.TestCase): + def test_concrete_methods_are_not_overridden(self): + """`MemoryTraversable` must not override `Traversable` concrete methods. + + This test is not an attempt to enforce a particular `Traversable` protocol; + it merely catches changes in the `Traversable` abstract/concrete methods + that have not been mirrored in the `MemoryTraversable` subclass. + """ + + traversable_concrete_methods = { + method + for method, value in Traversable.__dict__.items() + if callable(value) and method not in Traversable.__abstractmethods__ + } + memory_traversable_concrete_methods = { + method + for method, value in MemorySetup.MemoryTraversable.__dict__.items() + if callable(value) and not method.startswith("__") + } + overridden_methods = ( + memory_traversable_concrete_methods & traversable_concrete_methods + ) + + assert not overridden_methods diff --git a/Lib/test/test_importlib/resources/update-zips.py b/Lib/test/test_importlib/resources/update-zips.py deleted file mode 100755 index 231334aa7..000000000 --- a/Lib/test/test_importlib/resources/update-zips.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Generate the zip test data files. - -Run to build the tests/zipdataNN/ziptestdata.zip files from -files in tests/dataNN. - -Replaces the file with the working copy, but does commit anything -to the source repo. -""" - -import contextlib -import os -import pathlib -import zipfile - - -def main(): - """ - >>> from unittest import mock - >>> monkeypatch = getfixture('monkeypatch') - >>> monkeypatch.setattr(zipfile, 'ZipFile', mock.MagicMock()) - >>> print(); main() # print workaround for bpo-32509 - - ...data01... -> ziptestdata/... - ... - ...data02... -> ziptestdata/... - ... - """ - suffixes = '01', '02' - tuple(map(generate, suffixes)) - - -def generate(suffix): - root = pathlib.Path(__file__).parent.relative_to(os.getcwd()) - zfpath = root / f'zipdata{suffix}/ziptestdata.zip' - with zipfile.ZipFile(zfpath, 'w') as zf: - for src, rel in walk(root / f'data{suffix}'): - dst = 'ziptestdata' / pathlib.PurePosixPath(rel.as_posix()) - print(src, '->', dst) - zf.write(src, dst) - - -def walk(datapath): - for dirpath, dirnames, filenames in os.walk(datapath): - with contextlib.suppress(ValueError): - dirnames.remove('__pycache__') - for filename in filenames: - res = pathlib.Path(dirpath) / filename - rel = res.relative_to(datapath) - yield res, rel - - -__name__ == '__main__' and main() diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index e2d995f59..85b5c6151 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -1,18 +1,18 @@ import abc +import contextlib +import functools import importlib import io +import pathlib import sys import types -import pathlib -import contextlib - -from importlib.resources.abc import ResourceReader -from test.support import import_helper, os_helper -from . import zip as zip_ -from . import _path - - from importlib.machinery import ModuleSpec +from importlib.resources.abc import ResourceReader, Traversable, TraversableResources + +from test.support import import_helper, os_helper + +from . import _path +from . import zip as zip_ class Reader(ResourceReader): @@ -122,7 +122,7 @@ class CommonTestsBase(metaclass=abc.ABCMeta): bytes_data = io.BytesIO(b'Hello, world!') package = create_package(file=bytes_data, path=FileNotFoundError()) self.execute(package, 'utf-8.file') - self.assertEqual(package.__loader__._path, 'utf-8.file') + assert package.__loader__._path == 'utf-8.file' def test_extant_path(self): # Attempting to open or read or request the path when the @@ -133,7 +133,7 @@ class CommonTestsBase(metaclass=abc.ABCMeta): path = __file__ package = create_package(file=bytes_data, path=path) self.execute(package, 'utf-8.file') - self.assertEqual(package.__loader__._path, 'utf-8.file') + assert package.__loader__._path == 'utf-8.file' def test_useless_loader(self): package = create_package(file=FileNotFoundError(), path=FileNotFoundError()) @@ -202,5 +202,108 @@ class DiskSetup(ModuleSetup): self.fixtures.enter_context(import_helper.DirsOnSysPath(temp_dir)) +class MemorySetup(ModuleSetup): + """Support loading a module in memory.""" + + MODULE = 'data01' + + def load_fixture(self, module): + self.fixtures.enter_context(self.augment_sys_metapath(module)) + return importlib.import_module(module) + + @contextlib.contextmanager + def augment_sys_metapath(self, module): + finder_instance = self.MemoryFinder(module) + sys.meta_path.append(finder_instance) + yield + sys.meta_path.remove(finder_instance) + + class MemoryFinder(importlib.abc.MetaPathFinder): + def __init__(self, module): + self._module = module + + def find_spec(self, fullname, path, target=None): + if fullname != self._module: + return None + + return importlib.machinery.ModuleSpec( + name=fullname, + loader=MemorySetup.MemoryLoader(self._module), + is_package=True, + ) + + class MemoryLoader(importlib.abc.Loader): + def __init__(self, module): + self._module = module + + def exec_module(self, module): + pass + + def get_resource_reader(self, fullname): + return MemorySetup.MemoryTraversableResources(self._module, fullname) + + class MemoryTraversableResources(TraversableResources): + def __init__(self, module, fullname): + self._module = module + self._fullname = fullname + + def files(self): + return MemorySetup.MemoryTraversable(self._module, self._fullname) + + class MemoryTraversable(Traversable): + """Implement only the abstract methods of `Traversable`. + + Besides `.__init__()`, no other methods may be implemented or overridden. + This is critical for validating the concrete `Traversable` implementations. + """ + + def __init__(self, module, fullname): + self._module = module + self._fullname = fullname + + def _resolve(self): + """ + Fully traverse the `fixtures` dictionary. + + This should be wrapped in a `try/except KeyError` + but it is not currently needed and lowers the code coverage numbers. + """ + path = pathlib.PurePosixPath(self._fullname) + return functools.reduce(lambda d, p: d[p], path.parts, fixtures) + + def iterdir(self): + directory = self._resolve() + if not isinstance(directory, dict): + # Filesystem openers raise OSError, and that exception is mirrored here. + raise OSError(f"{self._fullname} is not a directory") + for path in directory: + yield MemorySetup.MemoryTraversable( + self._module, f"{self._fullname}/{path}" + ) + + def is_dir(self) -> bool: + return isinstance(self._resolve(), dict) + + def is_file(self) -> bool: + return not self.is_dir() + + def open(self, mode='r', encoding=None, errors=None, *_, **__): + contents = self._resolve() + if isinstance(contents, dict): + # Filesystem openers raise OSError when attempting to open a directory, + # and that exception is mirrored here. + raise OSError(f"{self._fullname} is a directory") + if isinstance(contents, str): + contents = contents.encode("utf-8") + result = io.BytesIO(contents) + if "b" in mode: + return result + return io.TextIOWrapper(result, encoding=encoding, errors=errors) + + @property + def name(self): + return pathlib.PurePosixPath(self._fullname).name + + class CommonTests(DiskSetup, CommonTestsBase): pass diff --git a/Lib/test/test_importlib/resources/zipdata01/__init__.py b/Lib/test/test_importlib/resources/zipdata01/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip b/Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip deleted file mode 100644 index 9a3bb0739..000000000 Binary files a/Lib/test/test_importlib/resources/zipdata01/ziptestdata.zip and /dev/null differ diff --git a/Lib/test/test_importlib/resources/zipdata02/__init__.py b/Lib/test/test_importlib/resources/zipdata02/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip b/Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip deleted file mode 100644 index d63ff512d..000000000 Binary files a/Lib/test/test_importlib/resources/zipdata02/ziptestdata.zip and /dev/null differ diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py index f35adec1a..ca2cc045f 100644 --- a/Lib/test/test_importlib/source/test_file_loader.py +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -5,24 +5,21 @@ importlib_abc = util.import_importlib('importlib.abc') machinery = util.import_importlib('importlib.machinery') importlib_util = util.import_importlib('importlib.util') -import errno import marshal import os import py_compile -import shutil import stat import sys import types import unittest -import warnings -from test.support.import_helper import make_legacy_pyc, unload +from test.support.import_helper import make_legacy_pyc -from test.test_py_compile import without_source_date_epoch -from test.test_py_compile import SourceDateEpochTestMeta +from test.support.os_helper import without_source_date_epoch +from test.support.os_helper import SourceDateEpochTestMeta -class SimpleTest(abc.LoaderTests): +class SimpleTest: """Should have no issue importing a source module [basic]. And if there is a syntax error, it should raise a SyntaxError [syntax error]. @@ -34,17 +31,6 @@ class SimpleTest(abc.LoaderTests): self.filepath = os.path.join('ham', self.name + '.py') self.loader = self.machinery.SourceFileLoader(self.name, self.filepath) - def test_load_module_API(self): - class Tester(self.abc.FileLoader): - def get_source(self, _): return 'attr = 42' - def is_package(self, _): return False - - loader = Tester('blah', 'blah.py') - self.addCleanup(unload, 'blah') - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = loader.load_module() # Should not raise an exception. - def test_get_filename_API(self): # If fullname is not set then assume self.path is desired. class Tester(self.abc.FileLoader): @@ -69,173 +55,11 @@ class SimpleTest(abc.LoaderTests): other = self.machinery.SourceFileLoader('_' + self.name, self.filepath) self.assertNotEqual(self.loader, other) - # [basic] - def test_module(self): - with util.create_modules('_temp') as mapping: - loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = loader.load_module('_temp') - self.assertIn('_temp', sys.modules) - check = {'__name__': '_temp', '__file__': mapping['_temp'], - '__package__': ''} - for attr, value in check.items(): - self.assertEqual(getattr(module, attr), value) - - def test_package(self): - with util.create_modules('_pkg.__init__') as mapping: - loader = self.machinery.SourceFileLoader('_pkg', - mapping['_pkg.__init__']) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = loader.load_module('_pkg') - self.assertIn('_pkg', sys.modules) - check = {'__name__': '_pkg', '__file__': mapping['_pkg.__init__'], - '__path__': [os.path.dirname(mapping['_pkg.__init__'])], - '__package__': '_pkg'} - for attr, value in check.items(): - self.assertEqual(getattr(module, attr), value) - - - def test_lacking_parent(self): - with util.create_modules('_pkg.__init__', '_pkg.mod')as mapping: - loader = self.machinery.SourceFileLoader('_pkg.mod', - mapping['_pkg.mod']) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = loader.load_module('_pkg.mod') - self.assertIn('_pkg.mod', sys.modules) - check = {'__name__': '_pkg.mod', '__file__': mapping['_pkg.mod'], - '__package__': '_pkg'} - for attr, value in check.items(): - self.assertEqual(getattr(module, attr), value) def fake_mtime(self, fxn): """Fake mtime to always be higher than expected.""" return lambda name: fxn(name) + 1 - def test_module_reuse(self): - with util.create_modules('_temp') as mapping: - loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = loader.load_module('_temp') - module_id = id(module) - module_dict_id = id(module.__dict__) - with open(mapping['_temp'], 'w', encoding='utf-8') as file: - file.write("testing_var = 42\n") - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = loader.load_module('_temp') - self.assertIn('testing_var', module.__dict__, - "'testing_var' not in " - "{0}".format(list(module.__dict__.keys()))) - self.assertEqual(module, sys.modules['_temp']) - self.assertEqual(id(module), module_id) - self.assertEqual(id(module.__dict__), module_dict_id) - - def test_state_after_failure(self): - # A failed reload should leave the original module intact. - attributes = ('__file__', '__path__', '__package__') - value = '' - name = '_temp' - with util.create_modules(name) as mapping: - orig_module = types.ModuleType(name) - for attr in attributes: - setattr(orig_module, attr, value) - with open(mapping[name], 'w', encoding='utf-8') as file: - file.write('+++ bad syntax +++') - loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) - with self.assertRaises(SyntaxError): - loader.exec_module(orig_module) - for attr in attributes: - self.assertEqual(getattr(orig_module, attr), value) - with self.assertRaises(SyntaxError): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - loader.load_module(name) - for attr in attributes: - self.assertEqual(getattr(orig_module, attr), value) - - # [syntax error] - def test_bad_syntax(self): - with util.create_modules('_temp') as mapping: - with open(mapping['_temp'], 'w', encoding='utf-8') as file: - file.write('=') - loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) - with self.assertRaises(SyntaxError): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - loader.load_module('_temp') - self.assertNotIn('_temp', sys.modules) - - def test_file_from_empty_string_dir(self): - # Loading a module found from an empty string entry on sys.path should - # not only work, but keep all attributes relative. - file_path = '_temp.py' - with open(file_path, 'w', encoding='utf-8') as file: - file.write("# test file for importlib") - try: - with util.uncache('_temp'): - loader = self.machinery.SourceFileLoader('_temp', file_path) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - mod = loader.load_module('_temp') - self.assertEqual(file_path, mod.__file__) - self.assertEqual(self.util.cache_from_source(file_path), - mod.__cached__) - finally: - os.unlink(file_path) - pycache = os.path.dirname(self.util.cache_from_source(file_path)) - if os.path.exists(pycache): - shutil.rmtree(pycache) - - @util.writes_bytecode_files - def test_timestamp_overflow(self): - # When a modification timestamp is larger than 2**32, it should be - # truncated rather than raise an OverflowError. - with util.create_modules('_temp') as mapping: - source = mapping['_temp'] - compiled = self.util.cache_from_source(source) - with open(source, 'w', encoding='utf-8') as f: - f.write("x = 5") - try: - os.utime(source, (2 ** 33 - 5, 2 ** 33 - 5)) - except OverflowError: - self.skipTest("cannot set modification time to large integer") - except OSError as e: - if e.errno != getattr(errno, 'EOVERFLOW', None): - raise - self.skipTest("cannot set modification time to large integer ({})".format(e)) - loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) - # PEP 451 - module = types.ModuleType('_temp') - module.__spec__ = self.util.spec_from_loader('_temp', loader) - loader.exec_module(module) - self.assertEqual(module.x, 5) - self.assertTrue(os.path.exists(compiled)) - os.unlink(compiled) - # PEP 302 - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - mod = loader.load_module('_temp') - # Sanity checks. - self.assertEqual(mod.__cached__, compiled) - self.assertEqual(mod.x, 5) - # The pyc file was created. - self.assertTrue(os.path.exists(compiled)) - - def test_unloadable(self): - loader = self.machinery.SourceFileLoader('good name', {}) - module = types.ModuleType('bad name') - module.__spec__ = self.machinery.ModuleSpec('bad name', loader) - with self.assertRaises(ImportError): - loader.exec_module(module) - with self.assertRaises(ImportError): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - loader.load_module('bad name') - @util.writes_bytecode_files def test_checked_hash_based_pyc(self): with util.create_modules('_temp') as mapping: @@ -386,12 +210,21 @@ class BadBytecodeTest: del sys.modules['_temp'] except KeyError: pass - py_compile.compile(mapping[name], invalidation_mode=invalidation_mode) - if not del_source: - bytecode_path = self.util.cache_from_source(mapping[name]) + if sys.implementation.cache_tag is None: + if del_source: + bytecode_path = mapping[name] + 'c' + py_compile.compile(mapping[name], bytecode_path, + invalidation_mode=invalidation_mode) + os.unlink(mapping[name]) + else: + raise unittest.SkipTest('requires sys.implementation.cache_tag') else: - os.unlink(mapping[name]) - bytecode_path = make_legacy_pyc(mapping[name]) + py_compile.compile(mapping[name], invalidation_mode=invalidation_mode) + if not del_source: + bytecode_path = self.util.cache_from_source(mapping[name]) + else: + os.unlink(mapping[name]) + bytecode_path = make_legacy_pyc(mapping[name]) if manipulator: with open(bytecode_path, 'rb') as file: bc = file.read() @@ -511,16 +344,6 @@ class BadBytecodeTestPEP451(BadBytecodeTest): loader.exec_module(module) -class BadBytecodeTestPEP302(BadBytecodeTest): - - def import_(self, file, module_name): - loader = self.loader(module_name, file) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = loader.load_module(module_name) - self.assertIn(module_name, sys.modules) - - class SourceLoaderBadBytecodeTest: @classmethod @@ -681,18 +504,6 @@ class SourceLoaderBadBytecodeTestPEP451( util=importlib_util) -class SourceLoaderBadBytecodeTestPEP302( - SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): - pass - - -(Frozen_SourceBadBytecodePEP302, - Source_SourceBadBytecodePEP302 - ) = util.test_both(SourceLoaderBadBytecodeTestPEP302, importlib=importlib, - machinery=machinery, abc=importlib_abc, - util=importlib_util) - - class SourcelessLoaderBadBytecodeTest: @classmethod @@ -779,17 +590,5 @@ class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, util=importlib_util) -class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, - BadBytecodeTestPEP302): - pass - - -(Frozen_SourcelessBadBytecodePEP302, - Source_SourcelessBadBytecodePEP302 - ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib, - machinery=machinery, abc=importlib_abc, - util=importlib_util) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py index 4de736a6b..91865b997 100644 --- a/Lib/test/test_importlib/source/test_finder.py +++ b/Lib/test/test_importlib/source/test_finder.py @@ -57,6 +57,8 @@ class FinderTests(abc.FinderTests): """ if create is None: create = {test} + if (compile_ or unlink) and sys.implementation.cache_tag is None: + raise unittest.SkipTest('requires sys.implementation.cache_tag') with util.create_modules(*create) as mapping: if compile_: for name in compile_: @@ -73,7 +75,7 @@ class FinderTests(abc.FinderTests): if error.errno != errno.ENOENT: raise loader = self.import_(mapping['.root'], test) - self.assertHasAttr(loader, 'load_module') + self.assertHasAttr(loader, 'exec_module') return loader def test_module(self): @@ -100,7 +102,7 @@ class FinderTests(abc.FinderTests): with util.create_modules('pkg.__init__', 'pkg.sub') as mapping: pkg_dir = os.path.dirname(mapping['pkg.__init__']) loader = self.import_(pkg_dir, 'pkg.sub') - self.assertHasAttr(loader, 'load_module') + self.assertHasAttr(loader, 'exec_module') # [sub package] def test_package_in_package(self): @@ -108,7 +110,7 @@ class FinderTests(abc.FinderTests): with context as mapping: pkg_dir = os.path.dirname(mapping['pkg.__init__']) loader = self.import_(pkg_dir, 'pkg.sub') - self.assertHasAttr(loader, 'load_module') + self.assertHasAttr(loader, 'exec_module') # [package over modules] def test_package_over_module(self): @@ -129,7 +131,7 @@ class FinderTests(abc.FinderTests): file.write("# test file for importlib") try: loader = self._find(finder, 'mod', loader_only=True) - self.assertHasAttr(loader, 'load_module') + self.assertHasAttr(loader, 'exec_module') finally: os.unlink('mod.py') diff --git a/Lib/test/test_importlib/source/test_source_encoding.py b/Lib/test/test_importlib/source/test_source_encoding.py index c09c9aa12..fada07877 100644 --- a/Lib/test/test_importlib/source/test_source_encoding.py +++ b/Lib/test/test_importlib/source/test_source_encoding.py @@ -102,19 +102,6 @@ class EncodingTestPEP451(EncodingTest): ) = util.test_both(EncodingTestPEP451, machinery=machinery) -class EncodingTestPEP302(EncodingTest): - - def load(self, loader): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - return loader.load_module(self.module_name) - - -(Frozen_EncodingTestPEP302, - Source_EncodingTestPEP302 - ) = util.test_both(EncodingTestPEP302, machinery=machinery) - - class LineEndingTest: r"""Source written with the three types of line endings (\n, \r\n, \r) @@ -158,18 +145,5 @@ class LineEndingTestPEP451(LineEndingTest): ) = util.test_both(LineEndingTestPEP451, machinery=machinery) -class LineEndingTestPEP302(LineEndingTest): - - def load(self, loader, module_name): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - return loader.load_module(module_name) - - -(Frozen_LineEndingTestPEP302, - Source_LineEndingTestPEP302 - ) = util.test_both(LineEndingTestPEP302, machinery=machinery) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/stubs.py b/Lib/test/test_importlib/stubs.py deleted file mode 100644 index e5b011c39..000000000 --- a/Lib/test/test_importlib/stubs.py +++ /dev/null @@ -1,10 +0,0 @@ -import unittest - - -class fake_filesystem_unittest: - """ - Stubbed version of the pyfakefs module - """ - class TestCase(unittest.TestCase): - def setUpPyfakefs(self): - self.skipTest("pyfakefs not available") diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index dd943210f..b24c57664 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -194,10 +194,6 @@ class LoaderDefaultsTests(ABCTestHarness): spec = 'a spec' self.assertIsNone(self.ins.create_module(spec)) - def test_load_module(self): - with self.assertRaises(ImportError): - self.ins.load_module('something') - def test_module_repr(self): mod = types.ModuleType('blah') with warnings.catch_warnings(): @@ -358,38 +354,6 @@ class LoaderLoadModuleTests: return SpecLoader() - def test_fresh(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - loader = self.loader() - name = 'blah' - with test_util.uncache(name): - loader.load_module(name) - module = loader.found - self.assertIs(sys.modules[name], module) - self.assertEqual(loader, module.__loader__) - self.assertEqual(loader, module.__spec__.loader) - self.assertEqual(name, module.__name__) - self.assertEqual(name, module.__spec__.name) - self.assertIsNotNone(module.__path__) - self.assertIsNotNone(module.__path__, - module.__spec__.submodule_search_locations) - - def test_reload(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - name = 'blah' - loader = self.loader() - module = types.ModuleType(name) - module.__spec__ = self.util.spec_from_loader(name, loader) - module.__loader__ = loader - with test_util.uncache(name): - sys.modules[name] = module - loader.load_module(name) - found = loader.found - self.assertIs(found, sys.modules[name]) - self.assertIs(module, sys.modules[name]) - (Frozen_LoaderLoadModuleTests, Source_LoaderLoadModuleTests @@ -478,59 +442,6 @@ class InspectLoaderGetCodeTests: InspectLoaderSubclass=SPLIT_IL) -class InspectLoaderLoadModuleTests: - - """Test InspectLoader.load_module().""" - - module_name = 'blah' - - def setUp(self): - import_helper.unload(self.module_name) - self.addCleanup(import_helper.unload, self.module_name) - - def load(self, loader): - spec = self.util.spec_from_loader(self.module_name, loader) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - return self.init._bootstrap._load_unlocked(spec) - - def mock_get_code(self): - return mock.patch.object(self.InspectLoaderSubclass, 'get_code') - - def test_get_code_ImportError(self): - # If get_code() raises ImportError, it should propagate. - with self.mock_get_code() as mocked_get_code: - mocked_get_code.side_effect = ImportError - with self.assertRaises(ImportError): - loader = self.InspectLoaderSubclass() - self.load(loader) - - def test_get_code_None(self): - # If get_code() returns None, raise ImportError. - with self.mock_get_code() as mocked_get_code: - mocked_get_code.return_value = None - with self.assertRaises(ImportError): - loader = self.InspectLoaderSubclass() - self.load(loader) - - def test_module_returned(self): - # The loaded module should be returned. - code = compile('attr = 42', '', 'exec') - with self.mock_get_code() as mocked_get_code: - mocked_get_code.return_value = code - loader = self.InspectLoaderSubclass() - module = self.load(loader) - self.assertEqual(module, sys.modules[self.module_name]) - - -(Frozen_ILLoadModuleTests, - Source_ILLoadModuleTests - ) = test_util.test_both(InspectLoaderLoadModuleTests, - InspectLoaderSubclass=SPLIT_IL, - init=init, - util=util) - - ##### ExecutionLoader concrete methods ######################################### class ExecutionLoaderGetCodeTests: @@ -599,8 +510,7 @@ class ExecutionLoaderGetCodeTests: class SourceOnlyLoader: # Globals that should be defined for all modules. - source = (b"_ = '::'.join([__name__, __file__, __cached__, __package__, " - b"repr(__loader__)])") + source = (b"_ = '::'.join([__name__, __file__, __package__, repr(__loader__)])") def __init__(self, path): self.path = path @@ -623,7 +533,10 @@ class SourceLoader(SourceOnlyLoader): def __init__(self, path, magic=None): super().__init__(path) - self.bytecode_path = self.util.cache_from_source(self.path) + try: + self.bytecode_path = self.util.cache_from_source(self.path) + except NotImplementedError: + self.bytecode_path = None self.source_size = len(self.source) if magic is None: magic = self.util.MAGIC_NUMBER @@ -669,26 +582,26 @@ class SourceLoaderTestHarness: module_name = 'mod' self.path = os.path.join(self.package, '.'.join(['mod', 'py'])) self.name = '.'.join([self.package, module_name]) - self.cached = self.util.cache_from_source(self.path) + try: + self.cached = self.util.cache_from_source(self.path) + except NotImplementedError: + self.cached = None self.loader = self.loader_mock(self.path, **kwargs) def verify_module(self, module): self.assertEqual(module.__name__, self.name) self.assertEqual(module.__file__, self.path) - self.assertEqual(module.__cached__, self.cached) self.assertEqual(module.__package__, self.package) self.assertEqual(module.__loader__, self.loader) values = module._.split('::') self.assertEqual(values[0], self.name) self.assertEqual(values[1], self.path) - self.assertEqual(values[2], self.cached) - self.assertEqual(values[3], self.package) - self.assertEqual(values[4], repr(self.loader)) + self.assertEqual(values[2], self.package) + self.assertEqual(values[3], repr(self.loader)) def verify_code(self, code_object): module = types.ModuleType(self.name) module.__file__ = self.path - module.__cached__ = self.cached module.__package__ = self.package module.__loader__ = self.loader module.__path__ = [] @@ -729,34 +642,6 @@ class SourceOnlyLoaderTests(SourceLoaderTestHarness): code = self.loader.source_to_code(self.loader.source, self.path) self.verify_code(code) - def test_load_module(self): - # Loading a module should set __name__, __loader__, __package__, - # __path__ (for packages), __file__, and __cached__. - # The module should also be put into sys.modules. - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - with test_util.uncache(self.name): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = self.loader.load_module(self.name) - self.verify_module(module) - self.assertEqual(module.__path__, [os.path.dirname(self.path)]) - self.assertIn(self.name, sys.modules) - - def test_package_settings(self): - # __package__ needs to be set, while __path__ is set on if the module - # is a package. - # Testing the values for a package are covered by test_load_module. - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - self.setUp(is_package=False) - with test_util.uncache(self.name): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - module = self.loader.load_module(self.name) - self.verify_module(module) - self.assertNotHasAttr(module, '__path__') - def test_get_source_encoding(self): # Source is considered encoded in UTF-8 by default unless otherwise # specified by an encoding line. @@ -777,6 +662,8 @@ class SourceOnlyLoaderTests(SourceLoaderTestHarness): @unittest.skipIf(sys.dont_write_bytecode, "sys.dont_write_bytecode is true") +@unittest.skipIf(sys.implementation.cache_tag is None, + "sys.implementation.cache_tag is None") class SourceLoaderBytecodeTests(SourceLoaderTestHarness): """Test importlib.abc.SourceLoader's use of bytecode. @@ -904,7 +791,7 @@ class SourceLoaderGetSourceTests: mock = self.SourceOnlyLoaderMock('mod.file') source = "x = 42\r\ny = -13\r\n" mock.source = source.encode('utf-8') - expect = io.IncrementalNewlineDecoder(None, True).decode(source) + expect = io.StringIO(source, newline=None).getvalue() self.assertEqual(mock.get_source(name), expect) diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index 1bc531a2f..70d93d693 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -231,11 +231,9 @@ class ReloadTests: # Start as a plain module. self.init.invalidate_caches() path = os.path.join(cwd, name + '.py') - cached = self.util.cache_from_source(path) expected = {'__name__': name, '__package__': '', '__file__': path, - '__cached__': cached, '__doc__': None, } os_helper.create_empty_file(path) @@ -252,11 +250,9 @@ class ReloadTests: # Change to a package. self.init.invalidate_caches() init_path = os.path.join(cwd, name, '__init__.py') - cached = self.util.cache_from_source(init_path) expected = {'__name__': name, '__package__': name, '__file__': init_path, - '__cached__': cached, '__path__': [os.path.dirname(init_path)], '__doc__': None, } @@ -283,7 +279,6 @@ class ReloadTests: # Start as a namespace package. self.init.invalidate_caches() bad_path = os.path.join(cwd, name, '__init.py') - cached = self.util.cache_from_source(bad_path) expected = {'__name__': name, '__package__': name, '__doc__': None, @@ -312,11 +307,9 @@ class ReloadTests: # Change to a regular package. self.init.invalidate_caches() init_path = os.path.join(cwd, name, '__init__.py') - cached = self.util.cache_from_source(init_path) expected = {'__name__': name, '__package__': name, '__file__': init_path, - '__cached__': cached, '__path__': [os.path.dirname(init_path)], '__doc__': None, 'eggs': None, diff --git a/Lib/test/test_importlib/test_discover.py b/Lib/test/test_importlib/test_discover.py new file mode 100644 index 000000000..c4ab7b698 --- /dev/null +++ b/Lib/test/test_importlib/test_discover.py @@ -0,0 +1,121 @@ +from unittest.mock import Mock + +from test.test_importlib import util + +importlib = util.import_importlib('importlib') +machinery = util.import_importlib('importlib.machinery') + + +class DiscoverableFinder: + def __init__(self, discover=[]): + self._discovered_values = discover + + def find_spec(self, fullname, path=None, target=None): + raise NotImplementedError + + def discover(self, parent=None): + yield from self._discovered_values + + +class TestPathFinder: + """PathFinder implements MetaPathFinder, which uses the PathEntryFinder(s) + registered in sys.path_hooks (and sys.path_importer_cache) to search + sys.path or the parent's __path__. + + PathFinder.discover() should redirect to the .discover() method of the + PathEntryFinder for each path entry. + """ + + def test_search_path_hooks_top_level(self): + modules = [ + self.machinery.ModuleSpec(name='example1', loader=None), + self.machinery.ModuleSpec(name='example2', loader=None), + self.machinery.ModuleSpec(name='example3', loader=None), + ] + + with util.import_state( + path_importer_cache={ + 'discoverable': DiscoverableFinder(discover=modules), + }, + path=['discoverable'], + ): + discovered = list(self.machinery.PathFinder.discover()) + + self.assertEqual(discovered, modules) + + + def test_search_path_hooks_parent(self): + parent = self.machinery.ModuleSpec(name='example', loader=None, is_package=True) + parent.submodule_search_locations.append('discoverable') + + children = [ + self.machinery.ModuleSpec(name='example.child1', loader=None), + self.machinery.ModuleSpec(name='example.child2', loader=None), + self.machinery.ModuleSpec(name='example.child3', loader=None), + ] + + with util.import_state( + path_importer_cache={ + 'discoverable': DiscoverableFinder(discover=children) + }, + path=[], + ): + discovered = list(self.machinery.PathFinder.discover(parent)) + + self.assertEqual(discovered, children) + + def test_invalid_parent(self): + parent = self.machinery.ModuleSpec(name='example', loader=None) + with self.assertRaises(ValueError): + list(self.machinery.PathFinder.discover(parent)) + + +( + Frozen_TestPathFinder, + Source_TestPathFinder, +) = util.test_both(TestPathFinder, importlib=importlib, machinery=machinery) + + +class TestFileFinder: + """FileFinder implements PathEntryFinder and provides the base finder + implementation to search the file system. + """ + + def get_finder(self, path): + loader_details = [ + (self.machinery.SourceFileLoader, self.machinery.SOURCE_SUFFIXES), + (self.machinery.SourcelessFileLoader, self.machinery.BYTECODE_SUFFIXES), + ] + return self.machinery.FileFinder(path, *loader_details) + + def test_discover_top_level(self): + modules = {'example1', 'example2', 'example3'} + with util.create_modules(*modules) as mapping: + finder = self.get_finder(mapping['.root']) + discovered = list(finder.discover()) + self.assertEqual({spec.name for spec in discovered}, modules) + + def test_discover_parent(self): + modules = { + 'example.child1', + 'example.child2', + 'example.child3', + } + with util.create_modules(*modules) as mapping: + example = self.get_finder(mapping['.root']).find_spec('example') + finder = self.get_finder(example.submodule_search_locations[0]) + discovered = list(finder.discover(example)) + self.assertEqual({spec.name for spec in discovered}, modules) + + def test_invalid_parent(self): + with util.create_modules('example') as mapping: + finder = self.get_finder(mapping['.root']) + example = finder.find_spec('example') + with self.assertRaises(ValueError): + list(finder.discover(example)) + + +( + Frozen_TestFileFinder, + Source_TestFileFinder, +) = util.test_both(TestFileFinder, importlib=importlib, machinery=machinery) diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py deleted file mode 100644 index 81f683799..000000000 --- a/Lib/test/test_importlib/test_main.py +++ /dev/null @@ -1,427 +0,0 @@ -import re -import pickle -import unittest -import warnings -import importlib.metadata -import contextlib -import itertools - -try: - import pyfakefs.fake_filesystem_unittest as ffs -except ImportError: - from .stubs import fake_filesystem_unittest as ffs - -from . import fixtures -from ._context import suppress -from importlib.metadata import ( - Distribution, - EntryPoint, - PackageNotFoundError, - _unique, - distributions, - entry_points, - metadata, - packages_distributions, - version, -) - - -@contextlib.contextmanager -def suppress_known_deprecation(): - with warnings.catch_warnings(record=True) as ctx: - warnings.simplefilter('default', category=DeprecationWarning) - yield ctx - - -class BasicTests(fixtures.DistInfoPkg, unittest.TestCase): - version_pattern = r'\d+\.\d+(\.\d)?' - - def test_retrieves_version_of_self(self): - dist = Distribution.from_name('distinfo-pkg') - assert isinstance(dist.version, str) - assert re.match(self.version_pattern, dist.version) - - def test_for_name_does_not_exist(self): - with self.assertRaises(PackageNotFoundError): - Distribution.from_name('does-not-exist') - - def test_package_not_found_mentions_metadata(self): - """ - When a package is not found, that could indicate that the - package is not installed or that it is installed without - metadata. Ensure the exception mentions metadata to help - guide users toward the cause. See #124. - """ - with self.assertRaises(PackageNotFoundError) as ctx: - Distribution.from_name('does-not-exist') - - assert "metadata" in str(ctx.exception) - - # expected to fail until ABC is enforced - @suppress(AssertionError) - @suppress_known_deprecation() - def test_abc_enforced(self): - with self.assertRaises(TypeError): - type('DistributionSubclass', (Distribution,), {})() - - @fixtures.parameterize( - dict(name=None), - dict(name=''), - ) - def test_invalid_inputs_to_from_name(self, name): - with self.assertRaises(ValueError): - Distribution.from_name(name) - - -class ImportTests(fixtures.DistInfoPkg, unittest.TestCase): - def test_import_nonexistent_module(self): - # Ensure that the MetadataPathFinder does not crash an import of a - # non-existent module. - with self.assertRaises(ImportError): - importlib.import_module('does_not_exist') - - def test_resolve(self): - ep = entry_points(group='entries')['main'] - self.assertEqual(ep.load().__name__, "main") - - def test_entrypoint_with_colon_in_name(self): - ep = entry_points(group='entries')['ns:sub'] - self.assertEqual(ep.value, 'mod:main') - - def test_resolve_without_attr(self): - ep = EntryPoint( - name='ep', - value='importlib.metadata', - group='grp', - ) - assert ep.load() is importlib.metadata - - -class NameNormalizationTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): - @staticmethod - def make_pkg(name): - """ - Create minimal metadata for a dist-info package with - the indicated name on the file system. - """ - return { - f'{name}.dist-info': { - 'METADATA': 'VERSION: 1.0\n', - }, - } - - def test_dashes_in_dist_name_found_as_underscores(self): - """ - For a package with a dash in the name, the dist-info metadata - uses underscores in the name. Ensure the metadata loads. - """ - fixtures.build_files(self.make_pkg('my_pkg'), self.site_dir) - assert version('my-pkg') == '1.0' - - def test_dist_name_found_as_any_case(self): - """ - Ensure the metadata loads when queried with any case. - """ - pkg_name = 'CherryPy' - fixtures.build_files(self.make_pkg(pkg_name), self.site_dir) - assert version(pkg_name) == '1.0' - assert version(pkg_name.lower()) == '1.0' - assert version(pkg_name.upper()) == '1.0' - - def test_unique_distributions(self): - """ - Two distributions varying only by non-normalized name on - the file system should resolve as the same. - """ - fixtures.build_files(self.make_pkg('abc'), self.site_dir) - before = list(_unique(distributions())) - - alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) - self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) - fixtures.build_files(self.make_pkg('ABC'), alt_site_dir) - after = list(_unique(distributions())) - - assert len(after) == len(before) - - -class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): - @staticmethod - def pkg_with_non_ascii_description(site_dir): - """ - Create minimal metadata for a package with non-ASCII in - the description. - """ - contents = { - 'portend.dist-info': { - 'METADATA': 'Description: pôrˈtend', - }, - } - fixtures.build_files(contents, site_dir) - return 'portend' - - @staticmethod - def pkg_with_non_ascii_description_egg_info(site_dir): - """ - Create minimal metadata for an egg-info package with - non-ASCII in the description. - """ - contents = { - 'portend.dist-info': { - 'METADATA': """ - Name: portend - - pôrˈtend""", - }, - } - fixtures.build_files(contents, site_dir) - return 'portend' - - def test_metadata_loads(self): - pkg_name = self.pkg_with_non_ascii_description(self.site_dir) - meta = metadata(pkg_name) - assert meta['Description'] == 'pôrˈtend' - - def test_metadata_loads_egg_info(self): - pkg_name = self.pkg_with_non_ascii_description_egg_info(self.site_dir) - meta = metadata(pkg_name) - assert meta['Description'] == 'pôrˈtend' - - -class DiscoveryTests( - fixtures.EggInfoPkg, - fixtures.EggInfoPkgPipInstalledNoToplevel, - fixtures.EggInfoPkgPipInstalledNoModules, - fixtures.EggInfoPkgSourcesFallback, - fixtures.DistInfoPkg, - unittest.TestCase, -): - def test_package_discovery(self): - dists = list(distributions()) - assert all(isinstance(dist, Distribution) for dist in dists) - assert any(dist.metadata['Name'] == 'egginfo-pkg' for dist in dists) - assert any(dist.metadata['Name'] == 'egg_with_module-pkg' for dist in dists) - assert any(dist.metadata['Name'] == 'egg_with_no_modules-pkg' for dist in dists) - assert any(dist.metadata['Name'] == 'sources_fallback-pkg' for dist in dists) - assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists) - - def test_invalid_usage(self): - with self.assertRaises(ValueError): - list(distributions(context='something', name='else')) - - -class DirectoryTest(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): - def test_egg_info(self): - # make an `EGG-INFO` directory that's unrelated - self.site_dir.joinpath('EGG-INFO').mkdir() - # used to crash with `IsADirectoryError` - with self.assertRaises(PackageNotFoundError): - version('unknown-package') - - def test_egg(self): - egg = self.site_dir.joinpath('foo-3.6.egg') - egg.mkdir() - with self.add_sys_path(egg): - with self.assertRaises(PackageNotFoundError): - version('foo') - - -class MissingSysPath(fixtures.OnSysPath, unittest.TestCase): - site_dir = '/does-not-exist' - - def test_discovery(self): - """ - Discovering distributions should succeed even if - there is an invalid path on sys.path. - """ - importlib.metadata.distributions() - - -class InaccessibleSysPath(fixtures.OnSysPath, ffs.TestCase): - site_dir = '/access-denied' - - def setUp(self): - super().setUp() - self.setUpPyfakefs() - self.fs.create_dir(self.site_dir, perm_bits=000) - - def test_discovery(self): - """ - Discovering distributions should succeed even if - there is an invalid path on sys.path. - """ - list(importlib.metadata.distributions()) - - -class TestEntryPoints(unittest.TestCase): - def __init__(self, *args): - super().__init__(*args) - self.ep = importlib.metadata.EntryPoint( - name='name', value='value', group='group' - ) - - def test_entry_point_pickleable(self): - revived = pickle.loads(pickle.dumps(self.ep)) - assert revived == self.ep - - def test_positional_args(self): - """ - Capture legacy (namedtuple) construction, discouraged. - """ - EntryPoint('name', 'value', 'group') - - def test_immutable(self): - """EntryPoints should be immutable""" - with self.assertRaises(AttributeError): - self.ep.name = 'badactor' - - def test_repr(self): - assert 'EntryPoint' in repr(self.ep) - assert 'name=' in repr(self.ep) - assert "'name'" in repr(self.ep) - - def test_hashable(self): - """EntryPoints should be hashable""" - hash(self.ep) - - def test_module(self): - assert self.ep.module == 'value' - - def test_attr(self): - assert self.ep.attr is None - - def test_sortable(self): - """ - EntryPoint objects are sortable, but result is undefined. - """ - sorted( - [ - EntryPoint(name='b', value='val', group='group'), - EntryPoint(name='a', value='val', group='group'), - ] - ) - - -class FileSystem( - fixtures.OnSysPath, fixtures.SiteDir, fixtures.FileBuilder, unittest.TestCase -): - def test_unicode_dir_on_sys_path(self): - """ - Ensure a Unicode subdirectory of a directory on sys.path - does not crash. - """ - fixtures.build_files( - {self.unicode_filename(): {}}, - prefix=self.site_dir, - ) - list(distributions()) - - -class PackagesDistributionsPrebuiltTest(fixtures.ZipFixtures, unittest.TestCase): - def test_packages_distributions_example(self): - self._fixture_on_path('example-21.12-py3-none-any.whl') - assert packages_distributions()['example'] == ['example'] - - def test_packages_distributions_example2(self): - """ - Test packages_distributions on a wheel built - by trampolim. - """ - self._fixture_on_path('example2-1.0.0-py3-none-any.whl') - assert packages_distributions()['example2'] == ['example2'] - - -class PackagesDistributionsTest( - fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase -): - def test_packages_distributions_neither_toplevel_nor_files(self): - """ - Test a package built without 'top-level.txt' or a file list. - """ - fixtures.build_files( - { - 'trim_example-1.0.0.dist-info': { - 'METADATA': """ - Name: trim_example - Version: 1.0.0 - """, - } - }, - prefix=self.site_dir, - ) - packages_distributions() - - def test_packages_distributions_all_module_types(self): - """ - Test top-level modules detected on a package without 'top-level.txt'. - """ - suffixes = importlib.machinery.all_suffixes() - metadata = dict( - METADATA=""" - Name: all_distributions - Version: 1.0.0 - """, - ) - files = { - 'all_distributions-1.0.0.dist-info': metadata, - } - for i, suffix in enumerate(suffixes): - files.update( - { - f'importable-name {i}{suffix}': '', - f'in_namespace_{i}': { - f'mod{suffix}': '', - }, - f'in_package_{i}': { - '__init__.py': '', - f'mod{suffix}': '', - }, - } - ) - metadata.update(RECORD=fixtures.build_record(files)) - fixtures.build_files(files, prefix=self.site_dir) - - distributions = packages_distributions() - - for i in range(len(suffixes)): - assert distributions[f'importable-name {i}'] == ['all_distributions'] - assert distributions[f'in_namespace_{i}'] == ['all_distributions'] - assert distributions[f'in_package_{i}'] == ['all_distributions'] - - assert not any(name.endswith('.dist-info') for name in distributions) - - -class PackagesDistributionsEggTest( - fixtures.EggInfoPkg, - fixtures.EggInfoPkgPipInstalledNoToplevel, - fixtures.EggInfoPkgPipInstalledNoModules, - fixtures.EggInfoPkgSourcesFallback, - unittest.TestCase, -): - def test_packages_distributions_on_eggs(self): - """ - Test old-style egg packages with a variation of 'top_level.txt', - 'SOURCES.txt', and 'installed-files.txt', available. - """ - distributions = packages_distributions() - - def import_names_from_package(package_name): - return { - import_name - for import_name, package_names in distributions.items() - if package_name in package_names - } - - # egginfo-pkg declares one import ('mod') via top_level.txt - assert import_names_from_package('egginfo-pkg') == {'mod'} - - # egg_with_module-pkg has one import ('egg_with_module') inferred from - # installed-files.txt (top_level.txt is missing) - assert import_names_from_package('egg_with_module-pkg') == {'egg_with_module'} - - # egg_with_no_modules-pkg should not be associated with any import names - # (top_level.txt is empty, and installed-files.txt has no .py files) - assert import_names_from_package('egg_with_no_modules-pkg') == set() - - # sources_fallback-pkg has one import ('sources_fallback') inferred from - # SOURCES.txt (top_level.txt and installed-files.txt is missing) - assert import_names_from_package('sources_fallback-pkg') == {'sources_fallback'} diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/test_metadata_api.py deleted file mode 100644 index 33c6e85ee..000000000 --- a/Lib/test/test_importlib/test_metadata_api.py +++ /dev/null @@ -1,322 +0,0 @@ -import re -import textwrap -import unittest -import warnings -import importlib -import contextlib - -from . import fixtures -from importlib.metadata import ( - Distribution, - PackageNotFoundError, - distribution, - entry_points, - files, - metadata, - requires, - version, -) - - -@contextlib.contextmanager -def suppress_known_deprecation(): - with warnings.catch_warnings(record=True) as ctx: - warnings.simplefilter('default', category=DeprecationWarning) - yield ctx - - -class APITests( - fixtures.EggInfoPkg, - fixtures.EggInfoPkgPipInstalledNoToplevel, - fixtures.EggInfoPkgPipInstalledNoModules, - fixtures.EggInfoPkgSourcesFallback, - fixtures.DistInfoPkg, - fixtures.DistInfoPkgWithDot, - fixtures.EggInfoFile, - unittest.TestCase, -): - version_pattern = r'\d+\.\d+(\.\d)?' - - def test_retrieves_version_of_self(self): - pkg_version = version('egginfo-pkg') - assert isinstance(pkg_version, str) - assert re.match(self.version_pattern, pkg_version) - - def test_retrieves_version_of_distinfo_pkg(self): - pkg_version = version('distinfo-pkg') - assert isinstance(pkg_version, str) - assert re.match(self.version_pattern, pkg_version) - - def test_for_name_does_not_exist(self): - with self.assertRaises(PackageNotFoundError): - distribution('does-not-exist') - - def test_name_normalization(self): - names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot' - for name in names: - with self.subTest(name): - assert distribution(name).metadata['Name'] == 'pkg.dot' - - def test_prefix_not_matched(self): - prefixes = 'p', 'pkg', 'pkg.' - for prefix in prefixes: - with self.subTest(prefix): - with self.assertRaises(PackageNotFoundError): - distribution(prefix) - - def test_for_top_level(self): - tests = [ - ('egginfo-pkg', 'mod'), - ('egg_with_no_modules-pkg', ''), - ] - for pkg_name, expect_content in tests: - with self.subTest(pkg_name): - self.assertEqual( - distribution(pkg_name).read_text('top_level.txt').strip(), - expect_content, - ) - - def test_read_text(self): - tests = [ - ('egginfo-pkg', 'mod\n'), - ('egg_with_no_modules-pkg', '\n'), - ] - for pkg_name, expect_content in tests: - with self.subTest(pkg_name): - top_level = [ - path for path in files(pkg_name) if path.name == 'top_level.txt' - ][0] - self.assertEqual(top_level.read_text(), expect_content) - - def test_entry_points(self): - eps = entry_points() - assert 'entries' in eps.groups - entries = eps.select(group='entries') - assert 'main' in entries.names - ep = entries['main'] - self.assertEqual(ep.value, 'mod:main') - self.assertEqual(ep.extras, []) - - def test_entry_points_distribution(self): - entries = entry_points(group='entries') - for entry in ("main", "ns:sub"): - ep = entries[entry] - self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg')) - self.assertEqual(ep.dist.version, "1.0.0") - - def test_entry_points_unique_packages_normalized(self): - """ - Entry points should only be exposed for the first package - on sys.path with a given name (even when normalized). - """ - alt_site_dir = self.fixtures.enter_context(fixtures.tempdir()) - self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) - alt_pkg = { - "DistInfo_pkg-1.1.0.dist-info": { - "METADATA": """ - Name: distinfo-pkg - Version: 1.1.0 - """, - "entry_points.txt": """ - [entries] - main = mod:altmain - """, - }, - } - fixtures.build_files(alt_pkg, alt_site_dir) - entries = entry_points(group='entries') - assert not any( - ep.dist.name == 'distinfo-pkg' and ep.dist.version == '1.0.0' - for ep in entries - ) - # ns:sub doesn't exist in alt_pkg - assert 'ns:sub' not in entries.names - - def test_entry_points_missing_name(self): - with self.assertRaises(KeyError): - entry_points(group='entries')['missing'] - - def test_entry_points_missing_group(self): - assert entry_points(group='missing') == () - - def test_entry_points_allows_no_attributes(self): - ep = entry_points().select(group='entries', name='main') - with self.assertRaises(AttributeError): - ep.foo = 4 - - def test_metadata_for_this_package(self): - md = metadata('egginfo-pkg') - assert md['author'] == 'Steven Ma' - assert md['LICENSE'] == 'Unknown' - assert md['Name'] == 'egginfo-pkg' - classifiers = md.get_all('Classifier') - assert 'Topic :: Software Development :: Libraries' in classifiers - - def test_missing_key_legacy(self): - """ - Requesting a missing key will still return None, but warn. - """ - md = metadata('distinfo-pkg') - with suppress_known_deprecation(): - assert md['does-not-exist'] is None - - def test_get_key(self): - """ - Getting a key gets the key. - """ - md = metadata('egginfo-pkg') - assert md.get('Name') == 'egginfo-pkg' - - def test_get_missing_key(self): - """ - Requesting a missing key will return None. - """ - md = metadata('distinfo-pkg') - assert md.get('does-not-exist') is None - - @staticmethod - def _test_files(files): - root = files[0].root - for file in files: - assert file.root == root - assert not file.hash or file.hash.value - assert not file.hash or file.hash.mode == 'sha256' - assert not file.size or file.size >= 0 - assert file.locate().exists() - assert isinstance(file.read_binary(), bytes) - if file.name.endswith('.py'): - file.read_text() - - def test_file_hash_repr(self): - util = [p for p in files('distinfo-pkg') if p.name == 'mod.py'][0] - self.assertRegex(repr(util.hash), '') - - def test_files_dist_info(self): - self._test_files(files('distinfo-pkg')) - - def test_files_egg_info(self): - self._test_files(files('egginfo-pkg')) - self._test_files(files('egg_with_module-pkg')) - self._test_files(files('egg_with_no_modules-pkg')) - self._test_files(files('sources_fallback-pkg')) - - def test_version_egg_info_file(self): - self.assertEqual(version('egginfo-file'), '0.1') - - def test_requires_egg_info_file(self): - requirements = requires('egginfo-file') - self.assertIsNone(requirements) - - def test_requires_egg_info(self): - deps = requires('egginfo-pkg') - assert len(deps) == 2 - assert any(dep == 'wheel >= 1.0; python_version >= "2.7"' for dep in deps) - - def test_requires_egg_info_empty(self): - fixtures.build_files( - { - 'requires.txt': '', - }, - self.site_dir.joinpath('egginfo_pkg.egg-info'), - ) - deps = requires('egginfo-pkg') - assert deps == [] - - def test_requires_dist_info(self): - deps = requires('distinfo-pkg') - assert len(deps) == 2 - assert all(deps) - assert 'wheel >= 1.0' in deps - assert "pytest; extra == 'test'" in deps - - def test_more_complex_deps_requires_text(self): - requires = textwrap.dedent( - """ - dep1 - dep2 - - [:python_version < "3"] - dep3 - - [extra1] - dep4 - dep6@ git+https://example.com/python/dep.git@v1.0.0 - - [extra2:python_version < "3"] - dep5 - """ - ) - deps = sorted(Distribution._deps_from_requires_text(requires)) - expected = [ - 'dep1', - 'dep2', - 'dep3; python_version < "3"', - 'dep4; extra == "extra1"', - 'dep5; (python_version < "3") and extra == "extra2"', - 'dep6@ git+https://example.com/python/dep.git@v1.0.0 ; extra == "extra1"', - ] - # It's important that the environment marker expression be - # wrapped in parentheses to avoid the following 'and' binding more - # tightly than some other part of the environment expression. - - assert deps == expected - - def test_as_json(self): - md = metadata('distinfo-pkg').json - assert 'name' in md - assert md['keywords'] == ['sample', 'package'] - desc = md['description'] - assert desc.startswith('Once upon a time\nThere was') - assert len(md['requires_dist']) == 2 - - def test_as_json_egg_info(self): - md = metadata('egginfo-pkg').json - assert 'name' in md - assert md['keywords'] == ['sample', 'package'] - desc = md['description'] - assert desc.startswith('Once upon a time\nThere was') - assert len(md['classifier']) == 2 - - def test_as_json_odd_case(self): - self.make_uppercase() - md = metadata('distinfo-pkg').json - assert 'name' in md - assert len(md['requires_dist']) == 2 - assert md['keywords'] == ['SAMPLE', 'PACKAGE'] - - -class LegacyDots(fixtures.DistInfoPkgWithDotLegacy, unittest.TestCase): - def test_name_normalization(self): - names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot' - for name in names: - with self.subTest(name): - assert distribution(name).metadata['Name'] == 'pkg.dot' - - def test_name_normalization_versionless_egg_info(self): - names = 'pkg.lot', 'pkg_lot', 'pkg-lot', 'pkg..lot', 'Pkg.Lot' - for name in names: - with self.subTest(name): - assert distribution(name).metadata['Name'] == 'pkg.lot' - - -class OffSysPathTests(fixtures.DistInfoPkgOffPath, unittest.TestCase): - def test_find_distributions_specified_path(self): - dists = Distribution.discover(path=[str(self.site_dir)]) - assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists) - - def test_distribution_at_pathlib(self): - """Demonstrate how to load metadata direct from a directory.""" - dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' - dist = Distribution.at(dist_info_path) - assert dist.version == '1.0.0' - - def test_distribution_at_str(self): - dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' - dist = Distribution.at(str(dist_info_path)) - assert dist.version == '1.0.0' - - -class InvalidateCache(unittest.TestCase): - def test_invalidate_cache(self): - # No externally observable behavior, but ensures test coverage... - importlib.invalidate_caches() diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 6ca0978f9..522ed9fa4 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -318,6 +318,17 @@ class ModuleAndNamespacePackageInSameDir(NamespacePackageTest): self.assertEqual(a_test.attr, 'in module') +class NamespaceSubpackageSameName(NamespacePackageTest): + paths = [''] + + def test_namespace_subpackage_shares_name_with_directory(self): + submodule_path = 'project4.foo' + with self.assertRaises(ModuleNotFoundError) as cm: + importlib.machinery.PathFinder.find_spec(submodule_path) + + self.assertEqual(cm.exception.name, 'project4') + + class ReloadTests(NamespacePackageTest): paths = ['portion1'] diff --git a/Lib/test/test_importlib/test_pkg_import.py b/Lib/test/test_importlib/test_pkg_import.py index 5ffae6222..287684efc 100644 --- a/Lib/test/test_importlib/test_pkg_import.py +++ b/Lib/test/test_importlib/test_pkg_import.py @@ -39,9 +39,12 @@ class TestImport(unittest.TestCase): self.remove_modules() def rewrite_file(self, contents): - compiled_path = cache_from_source(self.module_path) - if os.path.exists(compiled_path): - os.remove(compiled_path) + try: + compiled_path = cache_from_source(self.module_path) + if os.path.exists(compiled_path): + os.remove(compiled_path) + except NotImplementedError: + pass with open(self.module_path, 'w', encoding='utf-8') as f: f.write(contents) diff --git a/Lib/test/test_importlib/test_spec.py b/Lib/test/test_importlib/test_spec.py index aebeabaf8..77b622894 100644 --- a/Lib/test/test_importlib/test_spec.py +++ b/Lib/test/test_importlib/test_spec.py @@ -52,7 +52,10 @@ class ModuleSpecTests: def setUp(self): self.name = 'spam' self.path = 'spam.py' - self.cached = self.util.cache_from_source(self.path) + try: + self.cached = self.util.cache_from_source(self.path) + except NotImplementedError: + self.cached = None self.loader = TestLoader() self.spec = self.machinery.ModuleSpec(self.name, self.loader) self.loc_spec = self.machinery.ModuleSpec(self.name, self.loader, @@ -184,6 +187,8 @@ class ModuleSpecTests: self.assertIs(spec.cached, None) + @unittest.skipIf(sys.implementation.cache_tag is None, + "sys.implementation.cache_tag is None") def test_cached_source(self): expected = self.util.cache_from_source(self.path) @@ -224,7 +229,10 @@ class ModuleSpecMethodsTests: def setUp(self): self.name = 'spam' self.path = 'spam.py' - self.cached = self.util.cache_from_source(self.path) + try: + self.cached = self.util.cache_from_source(self.path) + except NotImplementedError: + self.cached = None self.loader = TestLoader() self.spec = self.machinery.ModuleSpec(self.name, self.loader) self.loc_spec = self.machinery.ModuleSpec(self.name, self.loader, @@ -287,20 +295,6 @@ class ModuleSpecMethodsTests: loaded = self.bootstrap._load(self.spec) self.assertNotIn(self.spec.name, sys.modules) - def test_load_legacy_attributes_immutable(self): - module = object() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - class ImmutableLoader(TestLoader): - def load_module(self, name): - sys.modules[name] = module - return module - self.spec.loader = ImmutableLoader() - with CleanImport(self.spec.name): - loaded = self.bootstrap._load(self.spec) - - self.assertIs(sys.modules[self.spec.name], module) - # reload() def test_reload(self): @@ -350,7 +344,6 @@ class ModuleSpecMethodsTests: self.assertIs(loaded.__spec__, self.spec) self.assertNotHasAttr(loaded, '__path__') self.assertNotHasAttr(loaded, '__file__') - self.assertNotHasAttr(loaded, '__cached__') (Frozen_ModuleSpecMethodsTests, @@ -364,7 +357,10 @@ class FactoryTests: def setUp(self): self.name = 'spam' self.path = os.path.abspath('spam.py') - self.cached = self.util.cache_from_source(self.path) + try: + self.cached = self.util.cache_from_source(self.path) + except NotImplementedError: + self.cached = None self.loader = TestLoader() self.fileloader = TestLoader(self.path) self.pkgloader = TestLoader(self.path, True) diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 0e0d0721e..f68f1d83a 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -326,6 +326,143 @@ raise RuntimeError("Intentional import failure") # Neither thread should have errors about stale modules self.assertEqual(errors, [], f"Race condition detected: {errors}") + def test_hierarchical_import_deadlock(self): + # Regression test for bpo-38884 / gh-83065 + # Tests that concurrent imports at different hierarchy levels + # don't deadlock when parent imports child in __init__.py + + # Create package structure: + # package/__init__.py: from package import subpackage + # package/subpackage/__init__.py: from package.subpackage.module import * + # package/subpackage/module.py: class SomeClass: pass + + pkg_dir = os.path.join(TESTFN, 'hier_deadlock_pkg') + os.makedirs(pkg_dir) + self.addCleanup(shutil.rmtree, TESTFN) + + subpkg_dir = os.path.join(pkg_dir, 'subpackage') + os.makedirs(subpkg_dir) + + with open(os.path.join(pkg_dir, "__init__.py"), "w") as f: + f.write("from hier_deadlock_pkg import subpackage\n") + + with open(os.path.join(subpkg_dir, "__init__.py"), "w") as f: + f.write("from hier_deadlock_pkg.subpackage.module import *\n") + + with open(os.path.join(subpkg_dir, "module.py"), "w") as f: + f.write("class SomeClass:\n pass\n") + + sys.path.insert(0, TESTFN) + self.addCleanup(sys.path.remove, TESTFN) + self.addCleanup(forget, 'hier_deadlock_pkg') + self.addCleanup(forget, 'hier_deadlock_pkg.subpackage') + self.addCleanup(forget, 'hier_deadlock_pkg.subpackage.module') + + importlib.invalidate_caches() + + errors = [] + results = [] + barrier = threading.Barrier(2) + + def t1(): + barrier.wait() + try: + import hier_deadlock_pkg.subpackage + results.append('t1_success') + except Exception as e: + errors.append(('t1', type(e).__name__, str(e))) + + def t2(): + barrier.wait() + try: + import hier_deadlock_pkg.subpackage.module + results.append('t2_success') + except Exception as e: + errors.append(('t2', type(e).__name__, str(e))) + + # Run multiple times to increase chance of hitting race condition + for i in range(10): + for mod in ['hier_deadlock_pkg', 'hier_deadlock_pkg.subpackage', + 'hier_deadlock_pkg.subpackage.module']: + sys.modules.pop(mod, None) + + errors.clear() + results.clear() + barrier.reset() + + thread1 = threading.Thread(target=t1) + thread2 = threading.Thread(target=t2) + + thread1.start() + thread2.start() + + thread1.join(timeout=5) + thread2.join(timeout=5) + + if thread1.is_alive() or thread2.is_alive(): + self.fail(f"Threads deadlocked on iteration {i}") + + self.assertEqual( + errors, [], + f"Import(s) failed on iteration {i}: {errors}") + self.assertEqual( + sorted(results), ['t1_success', 't2_success'], + f"Not all imports succeeded on iteration {i}: {results}") + + def test_cross_package_circular_import(self): + # Two packages whose __init__.py each import a submodule of the + # other. Concurrent imports of submodules of each must not raise + # _DeadlockError; the import system accepts a partially-initialised + # parent in this case (see _lock_unlock_module). + os.makedirs(os.path.join(TESTFN, "circ_a")) + os.makedirs(os.path.join(TESTFN, "circ_b")) + self.addCleanup(shutil.rmtree, TESTFN) + with open(os.path.join(TESTFN, "circ_a", "__init__.py"), "w") as f: + f.write("import time; time.sleep(0.03)\nimport circ_b.other\n") + with open(os.path.join(TESTFN, "circ_b", "__init__.py"), "w") as f: + f.write("import time; time.sleep(0.03)\nimport circ_a.other\n") + for pkg in ("circ_a", "circ_b"): + for mod in ("sub.py", "other.py"): + with open(os.path.join(TESTFN, pkg, mod), "w") as f: + f.write("X = 1\n") + + sys.path.insert(0, TESTFN) + self.addCleanup(sys.path.remove, TESTFN) + for mod in ("circ_a", "circ_a.sub", "circ_a.other", + "circ_b", "circ_b.sub", "circ_b.other"): + self.addCleanup(forget, mod) + importlib.invalidate_caches() + + errors = [] + barrier = threading.Barrier(2) + + def do_import(name): + barrier.wait() + try: + importlib.import_module(name) + except Exception as e: + errors.append((name, type(e).__name__, str(e))) + + for i in range(10): + for mod in ("circ_a", "circ_a.sub", "circ_a.other", + "circ_b", "circ_b.sub", "circ_b.other"): + sys.modules.pop(mod, None) + errors.clear() + barrier.reset() + + thread1 = threading.Thread(target=do_import, args=("circ_a.sub",)) + thread2 = threading.Thread(target=do_import, args=("circ_b.sub",)) + thread1.start() + thread2.start() + thread1.join(timeout=5) + thread2.join(timeout=5) + + if thread1.is_alive() or thread2.is_alive(): + self.fail(f"Threads deadlocked on iteration {i}") + self.assertEqual( + errors, [], + f"Import(s) failed on iteration {i}: {errors}") + def setUpModule(): thread_info = threading_helper.threading_setup() diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 8c14b9627..8a911c1b7 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -124,12 +124,6 @@ class ModuleFromSpecTests: module = self.util.module_from_spec(spec) self.assertEqual(module.__file__, spec.origin) - def test___cached__(self): - spec = self.machinery.ModuleSpec('test', object()) - spec.cached = 'some/path' - spec.has_location = True - module = self.util.module_from_spec(spec) - self.assertEqual(module.__cached__, spec.cached) (Frozen_ModuleFromSpecTests, Source_ModuleFromSpecTests @@ -334,6 +328,17 @@ class PEP3147Tests: tag = sys.implementation.cache_tag + def setUp(self): + # Most of these tests assume the default (unset) pycache prefix, so + # clear it for the duration of the test (e.g. when the test suite is + # run with PYTHONPYCACHEPREFIX set). Tests that need a specific prefix + # set their own via util.temporary_pycache_prefix(). + self._orig_pycache_prefix = sys.pycache_prefix + sys.pycache_prefix = None + + def tearDown(self): + sys.pycache_prefix = self._orig_pycache_prefix + @unittest.skipIf(sys.implementation.cache_tag is None, 'requires sys.implementation.cache_tag not be None') def test_cache_from_source(self): @@ -351,6 +356,8 @@ class PEP3147Tests: with self.assertRaises(NotImplementedError): self.util.cache_from_source('whatever.py') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_cache_from_source_no_dot(self): # Directory with a dot, filename without dot. path = os.path.join('foo.bar', 'file') @@ -359,47 +366,16 @@ class PEP3147Tests: self.assertEqual(self.util.cache_from_source(path, optimization=''), expect) - def test_cache_from_source_debug_override(self): - # Given the path to a .py file, return the path to its PEP 3147/PEP 488 - # defined .pyc file (i.e. under __pycache__). - path = os.path.join('foo', 'bar', 'baz', 'qux.py') - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - self.assertEqual(self.util.cache_from_source(path, False), - self.util.cache_from_source(path, optimization=1)) - self.assertEqual(self.util.cache_from_source(path, True), - self.util.cache_from_source(path, optimization='')) - with warnings.catch_warnings(): - warnings.simplefilter('error') - with self.assertRaises(DeprecationWarning): - self.util.cache_from_source(path, False) - with self.assertRaises(DeprecationWarning): - self.util.cache_from_source(path, True) - + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_cache_from_source_cwd(self): path = 'foo.py' expect = os.path.join('__pycache__', 'foo.{}.pyc'.format(self.tag)) self.assertEqual(self.util.cache_from_source(path, optimization=''), expect) - def test_cache_from_source_override(self): - # When debug_override is not None, it can be any true-ish or false-ish - # value. - path = os.path.join('foo', 'bar', 'baz.py') - # However if the bool-ishness can't be determined, the exception - # propagates. - class Bearish: - def __bool__(self): raise RuntimeError - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - self.assertEqual(self.util.cache_from_source(path, []), - self.util.cache_from_source(path, optimization=1)) - self.assertEqual(self.util.cache_from_source(path, [17]), - self.util.cache_from_source(path, optimization='')) - with self.assertRaises(RuntimeError): - self.util.cache_from_source('/foo/bar/baz.py', Bearish()) - - + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_cache_from_source_optimization_empty_string(self): # Setting 'optimization' to '' leads to no optimization tag (PEP 488). path = 'foo.py' @@ -407,6 +383,8 @@ class PEP3147Tests: self.assertEqual(self.util.cache_from_source(path, optimization=''), expect) + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_cache_from_source_optimization_None(self): # Setting 'optimization' to None uses the interpreter's optimization. # (PEP 488) @@ -423,6 +401,8 @@ class PEP3147Tests: self.assertEqual(self.util.cache_from_source(path, optimization=None), expect) + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_cache_from_source_optimization_set(self): # The 'optimization' parameter accepts anything that has a string repr # that passes str.alnum(). @@ -440,6 +420,8 @@ class PEP3147Tests: with self.assertRaises(ValueError): self.util.cache_from_source(path, optimization='path/is/bad') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_cache_from_source_debug_override_optimization_both_set(self): # Can only set one of the optimization-related parameters. with warnings.catch_warnings(): @@ -449,6 +431,8 @@ class PEP3147Tests: @unittest.skipUnless(os.sep == '\\' and os.altsep == '/', 'test meaningful only where os.altsep is defined') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_sep_altsep_and_sep_cache_from_source(self): # Windows path and PEP 3147 where sep is right of altsep. self.assertEqual( @@ -481,44 +465,60 @@ class PEP3147Tests: with self.assertRaises(NotImplementedError): self.util.source_from_cache(path) + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_bad_path(self): # When the path to a pyc file is not in PEP 3147 format, a ValueError # is raised. self.assertRaises( ValueError, self.util.source_from_cache, '/foo/bar/bazqux.pyc') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_no_slash(self): # No slashes at all in path -> ValueError self.assertRaises( ValueError, self.util.source_from_cache, 'foo.cpython-32.pyc') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_too_few_dots(self): # Too few dots in final path component -> ValueError self.assertRaises( ValueError, self.util.source_from_cache, '__pycache__/foo.pyc') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_too_many_dots(self): with self.assertRaises(ValueError): self.util.source_from_cache( '__pycache__/foo.cpython-32.opt-1.foo.pyc') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_not_opt(self): # Non-`opt-` path component -> ValueError self.assertRaises( ValueError, self.util.source_from_cache, '__pycache__/foo.cpython-32.foo.pyc') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_no__pycache__(self): # Another problem with the path -> ValueError self.assertRaises( ValueError, self.util.source_from_cache, '/foo/bar/foo.cpython-32.foo.pyc') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_optimized_bytecode(self): # Optimized bytecode is not an issue. path = os.path.join('__pycache__', 'foo.{}.opt-1.pyc'.format(self.tag)) self.assertEqual(self.util.source_from_cache(path), 'foo.py') + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_missing_optimization(self): # An empty optimization level is a no-no. path = os.path.join('__pycache__', 'foo.{}.opt-.pyc'.format(self.tag)) @@ -633,7 +633,7 @@ class MagicNumberTests(unittest.TestCase): 'only applies to candidate or final python release levels' ) def test_magic_number(self): - # Each python minor release should generally have a MAGIC_NUMBER + # Each Python feature release should generally have a MAGIC_NUMBER # that does not change once the release reaches candidate status. # Once a release reaches candidate status, the value of the constant @@ -643,11 +643,11 @@ class MagicNumberTests(unittest.TestCase): # In exceptional cases, it may be required to change the MAGIC_NUMBER # for a maintenance release. In this case the change should be - # discussed in python-dev. If a change is required, community + # discussed in on Discourse. If a change is required, community # stakeholders such as OS package maintainers must be notified # in advance. Such exceptional releases will then require an # adjustment to this test case. - EXPECTED_MAGIC_NUMBER = 3627 + EXPECTED_MAGIC_NUMBER = 3666 actual = int.from_bytes(importlib.util.MAGIC_NUMBER[:2], 'little') msg = ( @@ -659,7 +659,7 @@ class MagicNumberTests(unittest.TestCase): "magic number in this test to the current MAGIC_NUMBER to " "continue with the release.\n\n" "Changing the MAGIC_NUMBER for a maintenance release " - "requires discussion in python-dev and notification of " + "requires discussion on Discourse and notification of " "community stakeholders." ) self.assertEqual(EXPECTED_MAGIC_NUMBER, actual, msg) diff --git a/Lib/test/test_importlib/test_zip.py b/Lib/test/test_importlib/test_zip.py deleted file mode 100644 index 276f6288c..000000000 --- a/Lib/test/test_importlib/test_zip.py +++ /dev/null @@ -1,62 +0,0 @@ -import sys -import unittest - -from . import fixtures -from importlib.metadata import ( - PackageNotFoundError, - distribution, - distributions, - entry_points, - files, - version, -) - - -class TestZip(fixtures.ZipFixtures, unittest.TestCase): - def setUp(self): - super().setUp() - self._fixture_on_path('example-21.12-py3-none-any.whl') - - def test_zip_version(self): - self.assertEqual(version('example'), '21.12') - - def test_zip_version_does_not_match(self): - with self.assertRaises(PackageNotFoundError): - version('definitely-not-installed') - - def test_zip_entry_points(self): - scripts = entry_points(group='console_scripts') - entry_point = scripts['example'] - self.assertEqual(entry_point.value, 'example:main') - entry_point = scripts['Example'] - self.assertEqual(entry_point.value, 'example:main') - - def test_missing_metadata(self): - self.assertIsNone(distribution('example').read_text('does not exist')) - - def test_case_insensitive(self): - self.assertEqual(version('Example'), '21.12') - - def test_files(self): - for file in files('example'): - path = str(file.dist.locate_file(file)) - assert '.whl/' in path, path - - def test_one_distribution(self): - dists = list(distributions(path=sys.path[:1])) - assert len(dists) == 1 - - -class TestEgg(TestZip): - def setUp(self): - super().setUp() - self._fixture_on_path('example-21.12-py3.6.egg') - - def test_files(self): - for file in files('example'): - path = str(file.dist.locate_file(file)) - assert '.egg/' in path, path - - def test_normalized_name(self): - dist = distribution('example') - assert dist._normalized_name == 'example' diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index 2eaefd687..33f82e773 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -294,6 +294,9 @@ def writes_bytecode_files(fxn): tests that require it to be set to False.""" if sys.dont_write_bytecode: return unittest.skip("relies on writing bytecode")(fxn) + if sys.implementation.cache_tag is None: + return unittest.skip("requires sys.implementation.cache_tag to not be None")(fxn) + @functools.wraps(fxn) def wrapper(*args, **kwargs): original = sys.dont_write_bytecode diff --git a/Lib/test/test_importlib/zipdata01/ziptestdata.zip b/Lib/test/test_importlib/zipdata01/ziptestdata.zip deleted file mode 100644 index 9a3bb0739..000000000 Binary files a/Lib/test/test_importlib/zipdata01/ziptestdata.zip and /dev/null differ diff --git a/Lib/test/test_importlib/zipdata02/ziptestdata.zip b/Lib/test/test_importlib/zipdata02/ziptestdata.zip deleted file mode 100644 index d63ff512d..000000000 Binary files a/Lib/test/test_importlib/zipdata02/ziptestdata.zip and /dev/null differ diff --git a/Lib/test/test_inspect/inspect_fodder3.py b/Lib/test/test_inspect/inspect_fodder3.py new file mode 100644 index 000000000..ea2481edf --- /dev/null +++ b/Lib/test/test_inspect/inspect_fodder3.py @@ -0,0 +1,39 @@ +from functools import cached_property + +# docstring in parent, inherited in child +class ParentInheritDoc: + @cached_property + def foo(self): + """docstring for foo defined in parent""" + +class ChildInheritDoc(ParentInheritDoc): + pass + +class ChildInheritDefineDoc(ParentInheritDoc): + @cached_property + def foo(self): + pass + +# Redefine foo as something other than cached_property +class ChildPropertyFoo(ParentInheritDoc): + @property + def foo(self): + """docstring for the property foo""" + +class ChildMethodFoo(ParentInheritDoc): + def foo(self): + """docstring for the method foo""" + +# docstring in child but not parent +class ParentNoDoc: + @cached_property + def foo(self): + pass + +class ChildNoDoc(ParentNoDoc): + pass + +class ChildDefineDoc(ParentNoDoc): + @cached_property + def foo(self): + """docstring for foo defined in child""" diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index c7c9a2930..41fc33b7c 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -47,6 +47,7 @@ from test import support from test.test_inspect import inspect_fodder as mod from test.test_inspect import inspect_fodder2 as mod2 +from test.test_inspect import inspect_fodder3 as mod3 from test.test_inspect import inspect_stringized_annotations from test.test_inspect import inspect_deferred_annotations @@ -173,6 +174,15 @@ class custom_descriptor: return self.func.__get__(instance, owner) +class TestImportTime(unittest.TestCase): + + @cpython_only + def test_lazy_import(self): + import_helper.ensure_lazy_imports( + "inspect", {"re", "tokenize"} + ) + + class TestPredicates(IsTestBase): def test_excluding_predicates(self): @@ -689,10 +699,56 @@ class TestRetrievingSourceCode(GetSourceBase): self.assertEqual(inspect.getdoc(mod.FesteringGob.contradiction), 'The automatic gainsaying.') + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_getdoc_inherited_class_doc(self): + class A: + """Common base class""" + class B(A): + pass + + a = A() + self.assertEqual(inspect.getdoc(A), 'Common base class') + self.assertEqual(inspect.getdoc(A, inherit_class_doc=False), + 'Common base class') + self.assertEqual(inspect.getdoc(a), 'Common base class') + self.assertIsNone(inspect.getdoc(a, fallback_to_class_doc=False)) + a.__doc__ = 'Instance' + self.assertEqual(inspect.getdoc(a, fallback_to_class_doc=False), + 'Instance') + + b = B() + self.assertEqual(inspect.getdoc(B), 'Common base class') + self.assertIsNone(inspect.getdoc(B, inherit_class_doc=False)) + self.assertIsNone(inspect.getdoc(b)) + self.assertIsNone(inspect.getdoc(b, fallback_to_class_doc=False)) + b.__doc__ = 'Instance' + self.assertEqual(inspect.getdoc(b, fallback_to_class_doc=False), 'Instance') + + def test_getdoc_inherited_cached_property(self): + doc = inspect.getdoc(mod3.ParentInheritDoc.foo) + self.assertEqual(doc, 'docstring for foo defined in parent') + self.assertEqual(inspect.getdoc(mod3.ChildInheritDoc.foo), doc) + self.assertEqual(inspect.getdoc(mod3.ChildInheritDefineDoc.foo), doc) + + def test_getdoc_redefine_cached_property_as_other(self): + self.assertEqual(inspect.getdoc(mod3.ChildPropertyFoo.foo), + 'docstring for the property foo') + self.assertEqual(inspect.getdoc(mod3.ChildMethodFoo.foo), + 'docstring for the method foo') + + def test_getdoc_define_cached_property(self): + self.assertEqual(inspect.getdoc(mod3.ChildDefineDoc.foo), + 'docstring for foo defined in child') + + def test_getdoc_nodoc_inherited(self): + self.assertIsNone(inspect.getdoc(mod3.ChildNoDoc.foo)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, "test requires docstrings") def test_finddoc(self): finddoc = inspect._finddoc self.assertEqual(finddoc(int), int.__doc__) + self.assertIsNone(finddoc(int, search_in_class=False)) self.assertEqual(finddoc(int.to_bytes), int.to_bytes.__doc__) self.assertEqual(finddoc(int().to_bytes), int.to_bytes.__doc__) self.assertEqual(finddoc(int.from_bytes), int.from_bytes.__doc__) @@ -1300,9 +1356,10 @@ class TestClassesAndFunctions(unittest.TestCase): varkw_e=None, defaults_e=None, posonlyargs_e=[], kwonlyargs_e=[], kwonlydefaults_e=None, - ann_e={}): + ann_e={}, + annotation_format=Format.VALUE): args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ - inspect.getfullargspec(routine) + inspect.getfullargspec(routine, annotation_format=annotation_format) self.assertEqual(args, args_e) self.assertEqual(varargs, varargs_e) self.assertEqual(varkw, varkw_e) @@ -1335,6 +1392,19 @@ class TestClassesAndFunctions(unittest.TestCase): kwonlyargs_e=['e', 'f'], kwonlydefaults_e={'e': 4, 'f': 5}) + def get_getfullargspec_with_undefined_names_in_annotations(self): + def my_func(a: undefined_name): + pass + + with self.assertRaises(NameError): + inspect.getfullargspec(my_func) + + self.assertFullArgSpecEquals(my_func, ['a'], ann_e={'a': 'undefined_name'}, + annotation_format=Format.STRING) + + arg_spec = inspect.getfullargspec(my_func, annotation_format=Format.FORWARDREF) + self.assertIsInstance(arg_spec.annotations['a'], ForwardRef) + def test_argspec_api_ignores_wrapped(self): # Issue 20684: low level introspection API must ignore __wrapped__ @functools.wraps(mod.spam) @@ -6081,7 +6151,8 @@ class TestSignatureDefinitions(unittest.TestCase): self.assertRaises(ValueError, inspect.signature, getattr(cls, name)) def test_builtins_have_signatures(self): - no_signature = {'type', 'super', 'bytearray', 'bytes', 'dict', 'int', 'str'} + no_signature = {'type', 'super', 'bytearray', 'bytes', + 'dict', 'frozendict', 'int', 'str'} # These need PEP 457 groups needs_groups = {"range", "slice", "dir", "getattr", "next", "iter", "vars"} @@ -6113,6 +6184,7 @@ class TestSignatureDefinitions(unittest.TestCase): 'AsyncGeneratorType': {'athrow'}, 'CoroutineType': {'throw'}, 'GeneratorType': {'throw'}, + 'FrameLocalsProxyType': {'setdefault', 'pop', 'get'}, } self._test_module_has_signatures(types, unsupported_signature=unsupported_signature, @@ -6157,6 +6229,21 @@ class TestSignatureDefinitions(unittest.TestCase): import collections.abc self._test_module_has_signatures(collections.abc) + def test_datetime_module_has_signatures(self): + # Only test if the C implementation is available. + import_helper.import_module('_datetime') + import datetime + no_signature = {'tzinfo'} + unsupported_signature = {'timezone'} + methods_unsupported_signature = { + 'date': {'replace'}, + 'time': {'replace'}, + 'datetime': {'replace', 'combine'}, + } + self._test_module_has_signatures(datetime, + no_signature, unsupported_signature, + methods_unsupported_signature=methods_unsupported_signature) + def test_errno_module_has_signatures(self): import errno self._test_module_has_signatures(errno) @@ -6169,8 +6256,7 @@ class TestSignatureDefinitions(unittest.TestCase): self._test_module_has_signatures(faulthandler, unsupported_signature=unsupported_signature) def test_functools_module_has_signatures(self): - unsupported_signature = {"reduce"} - self._test_module_has_signatures(functools, unsupported_signature=unsupported_signature) + self._test_module_has_signatures(functools) def test_gc_module_has_signatures(self): import gc @@ -6214,7 +6300,10 @@ class TestSignatureDefinitions(unittest.TestCase): def test_re_module_has_signatures(self): import re - methods_no_signature = {'Match': {'group'}} + methods_no_signature = { + 'Match': {'group'}, + 'Pattern': {'match'}, # It is now an alias for prefixmatch + } self._test_module_has_signatures(re, methods_no_signature=methods_no_signature, good_exceptions={'error', 'PatternError'}) @@ -6227,6 +6316,10 @@ class TestSignatureDefinitions(unittest.TestCase): import stat self._test_module_has_signatures(stat) + def test_struct_module_has_signatures(self): + import struct + self._test_module_has_signatures(struct) + def test_string_module_has_signatures(self): import string self._test_module_has_signatures(string) @@ -6284,7 +6377,7 @@ class TestSignatureDefinitions(unittest.TestCase): methods_unsupported_signature=methods_unsupported_signature) def test_warnings_module_has_signatures(self): - unsupported_signature = {'warn', 'warn_explicit'} + unsupported_signature = {'warn_explicit'} self._test_module_has_signatures(warnings, unsupported_signature=unsupported_signature) def test_weakref_module_has_signatures(self): @@ -6415,8 +6508,19 @@ class TestUnwrap(unittest.TestCase): self.assertIs(inspect.unwrap(staticmethod(classmethod)), classmethod) self.assertIs(inspect.unwrap(classmethod(staticmethod)), staticmethod) +def _clean_object_ids(text): + # Helper to handle "" details in CLI output checks + import re + detect = r"object at 0x([0-9A-Fa-f]+)>" + replace = "object at 0x...>" + return re.sub(detect, replace, text) + +class TestModuleCLI(unittest.TestCase): + + BUILTIN_ERROR = "No source code available for builtin module" + NO_SOURCE_ERROR = "No source code available for defining module" + NO_SOURCE_TARGET_ERROR = "Failed to retrieve source code for given target" -class TestMain(unittest.TestCase): @staticmethod def _expected_cached(module): # assert_python_ok() runs the subprocess in isolated mode (-I), which @@ -6457,27 +6561,228 @@ class TestMain(unittest.TestCase): inspect.getsource(ThreadPoolExecutor).splitlines()) self.assertEqual(err, b'') - def test_builtins(self): + def test_error_builtins(self): _, out, err = assert_python_failure('-m', 'inspect', 'sys') lines = err.decode().splitlines() - self.assertEqual(lines, ["Can't get info for builtin modules."]) + self.assertEqual(lines, [self.BUILTIN_ERROR]) - def test_details(self): - module = importlib.import_module('unittest') + @unittest.skipIf(importlib.util.find_spec("_testcapi") is None, "TODO: RUSTPYTHON; _testcapi extension fixture is unavailable") + def test_error_extension(self): + module_name = "_testcapi" + if module_name in sys.builtin_module_names: + # WASI test environment has even _testcapi as a builtin module + expected_error = self.BUILTIN_ERROR + else: + expected_error = self.NO_SOURCE_ERROR + _, out, err = assert_python_failure('-m', 'inspect', + module_name) + lines = err.decode().splitlines() + self.assertEqual(lines, [expected_error]) + + def test_error_data(self): + _, out, err = assert_python_failure('-m', 'inspect', + 'importlib.machinery:SOURCE_SUFFIXES') + lines = err.decode().splitlines() + self.assertEqual(lines, [self.NO_SOURCE_TARGET_ERROR]) + + def test_details_option_with_package(self): + module_name = 'unittest' + module = importlib.import_module(module_name) args = support.optim_args_from_interpreter_flags() rc, out, err = assert_python_ok(*args, '-m', 'inspect', - 'unittest', '--details') - output = out.decode() + module_name, '--details') cached = self._expected_cached(module) - # Just a quick sanity check on the output - self.assertIn(module.__spec__.name, output) - self.assertIn(module.__name__, output) - self.assertIn(module.__spec__.origin, output) - self.assertIn(module.__file__, output) - self.assertIn(cached, output) + # Full rendering check on the expected output + expected_lines = [ + f"Target: {module.__name__}", # No aliasing + f"Origin: {module.__spec__.origin}", + f"Source: {module.__file__}", + f"Cached: {cached}", # None is still displayed + f"Loader: {_clean_object_ids(repr(module.__spec__.loader))}", + f"Submodule search paths: {module.__path__}", + "", + ] + output_lines = _clean_object_ids(out.decode()).splitlines() + self.assertEqual(output_lines, expected_lines) self.assertEqual(err, b'') + def test_details_option_with_builtin_module(self): + # Also an end-to-end test of non-package lookups + module_name = 'sys' + module = importlib.import_module(module_name) + args = support.optim_args_from_interpreter_flags() + rc, out, err = assert_python_ok(*args, '-m', 'inspect', + module_name, '--details') + # Full rendering check on the expected output + # No error is reported when just fetching the module details + expected_lines = [ + f"Target: {module.__name__}", # No aliasing + f"Origin: {module.__spec__.origin}", + "Source: None", + "Cached: None", + f"Loader: {_clean_object_ids(repr(module.__spec__.loader))}", + "", + ] + output_lines = _clean_object_ids(out.decode()).splitlines() + self.assertEqual(output_lines, expected_lines) + self.assertEqual(err, b'') + + def test_details_option_with_data_target(self): + # Also an end-to-end test of non-module lookups without aliasing + module_name = 'importlib.machinery' + cli_target = f"{module_name}:SOURCE_SUFFIXES" + module = importlib.import_module(module_name) + args = support.optim_args_from_interpreter_flags() + rc, out, err = assert_python_ok(*args, '-m', 'inspect', + cli_target, '--details') + cached = self._expected_cached(module) + # Full rendering check on the expected output + # The error is only informational when reading source details + expected_lines = [ + f"Target: {cli_target}", # No aliasing + f"Origin: {module.__spec__.origin}", + f"Source: {module.__file__}", + f"Cached: {cached}", # None is still displayed + self.NO_SOURCE_TARGET_ERROR, + "", + ] + output_lines = out.decode().splitlines() + self.assertEqual(output_lines, expected_lines) + self.assertEqual(err, b'') + + @unittest.skipIf(not os.path.exists(os.path.__file__), "Needs frozen source file") + def test_details_option_with_aliased_target(self): + # Also an end-to-end test of successful non-module lookups + module = importlib.import_module("os.path") + target = module.join + cli_target = "os:path.join" # Defining module is os.path, not os + defining_target = f"{target.__module__}:{target.__qualname__}" + + args = support.optim_args_from_interpreter_flags() + rc, out, err = assert_python_ok(*args, '-m', 'inspect', + cli_target, '--details') + cached = self._expected_cached(module) + # Full rendering check on the expected output + expected_lines = [ + f'Target: {defining_target} (looked up as "{cli_target}")', + f"Origin: {module.__spec__.origin}", + f"Source: {module.__file__}", + f"Cached: {cached}", # None is still displayed + f"Line: {inspect.findsource(target)[1]}", + "", + ] + output_lines = out.decode().splitlines() + self.assertEqual(output_lines, expected_lines) + self.assertEqual(err, b'') + + def _check_details(self, module, details, other_expected_keys=(), *, alias=None, error=None): + expected_keys = {"target", "origin", "source", "cached"} + if other_expected_keys: + expected_keys |= other_expected_keys + if alias is not None: + expected_keys.add("alias") + if error is not None: + expected_keys.add("error") + self.assertEqual(set(details.keys()), expected_keys) + self.assertEqual(module.__spec__.origin, details["origin"]) + try: + expected_source = inspect.getsourcefile(module) + except Exception: + expected_source = None + if expected_source and expected_source.startswith("'. # The directory of argv[0] must match the directory of the Python # executable for the Popen() call to python to succeed as the directory - # path may be used by Py_GetPath() to build the default module search - # path. + # path may be used by PyConfig_Get("module_search_paths") to build the + # default module search path. stdin_fname = os.path.join(os.path.dirname(sys.executable), "") cmd_line = [stdin_fname, '-E', '-i'] cmd_line.extend(args) @@ -6568,7 +6875,5 @@ class TestRepl(unittest.TestCase): self.assertIn(expected, output) - - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_pkg.py b/Lib/test/test_pkg.py index d2b724db4..0a366e2a5 100644 --- a/Lib/test/test_pkg.py +++ b/Lib/test/test_pkg.py @@ -198,15 +198,15 @@ class TestPkg(unittest.TestCase): import t5 self.assertEqual(fixdir(dir(t5)), - ['__cached__', '__doc__', '__file__', '__loader__', - '__name__', '__package__', '__path__', '__spec__', - 'foo', 'string', 't5']) + ['__doc__', '__file__', '__loader__', '__name__', + '__package__', '__path__', '__spec__', 'foo', + 'string', 't5']) self.assertEqual(fixdir(dir(t5.foo)), - ['__cached__', '__doc__', '__file__', '__loader__', - '__name__', '__package__', '__spec__', 'string']) + ['__doc__', '__file__', '__loader__', '__name__', + '__package__', '__spec__', 'string']) self.assertEqual(fixdir(dir(t5.string)), - ['__cached__', '__doc__', '__file__', '__loader__', - '__name__', '__package__', '__spec__', 'spam']) + ['__doc__', '__file__', '__loader__', '__name__', + '__package__', '__spec__', 'spam']) def test_6(self): hier = [ @@ -221,14 +221,13 @@ class TestPkg(unittest.TestCase): import t6 self.assertEqual(fixdir(dir(t6)), - ['__all__', '__cached__', '__doc__', '__file__', - '__loader__', '__name__', '__package__', '__path__', - '__spec__']) + ['__all__', '__doc__', '__file__', '__loader__', + '__name__', '__package__', '__path__', '__spec__']) s = """ import t6 from t6 import * self.assertEqual(fixdir(dir(t6)), - ['__all__', '__cached__', '__doc__', '__file__', + ['__all__', '__doc__', '__file__', '__loader__', '__name__', '__package__', '__path__', '__spec__', 'eggs', 'ham', 'spam']) self.assertEqual(dir(), ['eggs', 'ham', 'self', 'spam', 't6']) @@ -256,20 +255,19 @@ class TestPkg(unittest.TestCase): t7, sub, subsub = None, None, None import t7 as tas self.assertEqual(fixdir(dir(tas)), - ['__cached__', '__doc__', '__file__', '__loader__', - '__name__', '__package__', '__path__', '__spec__']) + ['__doc__', '__file__', '__loader__', '__name__', + '__package__', '__path__', '__spec__']) self.assertFalse(t7) from t7 import sub as subpar self.assertEqual(fixdir(dir(subpar)), - ['__cached__', '__doc__', '__file__', '__loader__', - '__name__', '__package__', '__path__', '__spec__']) + ['__doc__', '__file__', '__loader__', '__name__', + '__package__', '__path__', '__spec__']) self.assertFalse(t7) self.assertFalse(sub) from t7.sub import subsub as subsubsub self.assertEqual(fixdir(dir(subsubsub)), - ['__cached__', '__doc__', '__file__', '__loader__', - '__name__', '__package__', '__path__', '__spec__', - 'spam']) + ['__doc__', '__file__', '__loader__', '__name__', + '__package__', '__path__', '__spec__', 'spam']) self.assertFalse(t7) self.assertFalse(sub) self.assertFalse(subsub) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index e07fa568a..57b12609f 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -8,7 +8,7 @@ from test.support.script_helper import assert_python_ok from test.support.import_helper import import_fresh_module import collections.abc -from collections import namedtuple, UserDict +from collections import namedtuple, UserDict, OrderedDict import copy # XXX: RUSTPYTHON try: @@ -27,6 +27,7 @@ import weakref import typing import unittest # XXX: RUSTPYTHON; importing to be able to skip tests +import re c_types = import_fresh_module('types', fresh=['_types']) py_types = import_fresh_module('types', blocked=['_types']) @@ -45,8 +46,9 @@ def clear_typing_caches(): class TypesTests(unittest.TestCase): + @unittest.expectedFailureIf(not hasattr(c_types, "LazyImportType"), "TODO: RUSTPYTHON; PEP 810 LazyImportType is not implemented") def test_names(self): - c_only_names = {'CapsuleType'} + c_only_names = {'CapsuleType', 'LazyImportType'} ignored = {'new_class', 'resolve_bases', 'prepare_class', 'get_original_bases', 'DynamicClassAttribute', 'coroutine'} @@ -58,11 +60,13 @@ class TypesTests(unittest.TestCase): 'AsyncGeneratorType', 'BuiltinFunctionType', 'BuiltinMethodType', 'CapsuleType', 'CellType', 'ClassMethodDescriptorType', 'CodeType', 'CoroutineType', 'EllipsisType', 'FrameType', 'FunctionType', + 'FrameLocalsProxyType', 'GeneratorType', 'GenericAlias', 'GetSetDescriptorType', - 'LambdaType', 'MappingProxyType', 'MemberDescriptorType', - 'MethodDescriptorType', 'MethodType', 'MethodWrapperType', - 'ModuleType', 'NoneType', 'NotImplementedType', 'SimpleNamespace', - 'TracebackType', 'UnionType', 'WrapperDescriptorType', + 'LambdaType', 'LazyImportType', 'MappingProxyType', + 'MemberDescriptorType', 'MethodDescriptorType', 'MethodType', + 'MethodWrapperType', 'ModuleType', 'NoneType', + 'NotImplementedType', 'SimpleNamespace', 'TracebackType', + 'UnionType', 'WrapperDescriptorType', } self.assertEqual(all_names, set(c_types.__all__)) self.assertEqual(all_names - c_only_names, set(py_types.__all__)) @@ -717,6 +721,19 @@ class TypesTests(unittest.TestCase): """ assert_python_ok("-c", code) + def test_frame_locals_proxy_type(self): + self.assertIsInstance(types.FrameLocalsProxyType, type) + if MISSING_C_DOCSTRINGS: + self.assertIsNone(types.FrameLocalsProxyType.__doc__) + else: + self.assertIsInstance(types.FrameLocalsProxyType.__doc__, str) + self.assertEqual(types.FrameLocalsProxyType.__module__, 'builtins') + self.assertEqual(types.FrameLocalsProxyType.__name__, 'FrameLocalsProxy') + + frame = inspect.currentframe() + self.assertIsNotNone(frame) + self.assertIsInstance(frame.f_locals, types.FrameLocalsProxyType) + class UnionTests(unittest.TestCase): @@ -1382,6 +1399,128 @@ class MappingProxyTests(unittest.TestCase): view = self.mappingproxy(mapping) self.assertEqual(hash(view), hash(mapping)) + def check_richcompare(self, mapping_type): + data1 = mapping_type({'a': 1}) + data2 = mapping_type({'a': 2}) + mp1 = self.mappingproxy(data1) + copy1 = self.mappingproxy(data1) + mp2 = self.mappingproxy(data2) + + self.assertTrue(mp1 == data1) + self.assertTrue(data1 == mp1) + self.assertTrue(copy1 == data1) + + self.assertTrue(mp1 == copy1) + self.assertFalse(mp1 != copy1) + + self.assertFalse(mp1 == mp2) + self.assertTrue(mp1 != mp2) + self.assertFalse(mp2 == data1) + self.assertTrue(mp2 != data1) + self.assertTrue(data1 != mp2) + + msg = "not supported between instances of 'mappingproxy' and 'mappingproxy'" + with self.assertRaisesRegex(TypeError, msg): + mp1 > mp2 + with self.assertRaisesRegex(TypeError, msg): + mp1 < copy1 + with self.assertRaisesRegex(TypeError, msg): + mp2 >= mp2 + with self.assertRaisesRegex(TypeError, msg): + copy1 <= mp1 + + if mapping_type.__module__ == 'collections': + mapping_name = f'{mapping_type.__module__}.{mapping_type.__name__}' + else: + mapping_name = mapping_type.__name__ + with self.assertRaisesRegex( + TypeError, + f"not supported between instances of 'mappingproxy' and '{mapping_name}'", + ): + copy1 <= data1 + + class Evil: + def __eq__(self, other): + return other + + result = (mp1 == Evil()) + self.assertIs(type(result), mapping_type) + if mapping_type == dict: + # Evil.__eq__() gets a copy of the mapping + self.assertEqual(result, data1) + else: + self.assertIs(result, data1) + + def test_richcompare_dict(self): + self.check_richcompare(dict) + + def test_richcompare_custom_mapping(self): + class CustomMapping(collections.abc.Mapping): + def __init__(self, data): + self._data = data + + def __getitem__(self, key): + return self._data[key] + + def __iter__(self): + return iter(self._data) + + def __len__(self): + return len(self._data) + + def __contains__(self, item): + return item in self._data + + self.check_richcompare(CustomMapping) + + class CustomMapping2(CustomMapping): + def __eq__(self, other): + return ( + isinstance(other, CustomMapping) + and self._data == other._data + ) + + mp1 = self.mappingproxy(CustomMapping({'a': 1})) + self.assertEqual(mp1, CustomMapping2({'a': 1})) + self.assertNotEqual(CustomMapping2({'a': 1}), mp1) + + def test_richcompare_odict(self): + self.check_richcompare(OrderedDict) + + def test_richcompare_frozendict(self): + self.check_richcompare(frozendict) + + def test_richcompare_evil(self): + # This test used to mutate the list dictionary, + # but MappingProxyType now creates a copy to call `__eq__`: + # https://github.com/python/cpython/issues/152405 + key = "__mappingproxy_crash_key__" + + class Evil: + def __eq__(self, other): + other[key] = 1 + return other + + # Checks that it does not mutate the internals of `MappingProxyType`: + dc = {} + leaked = self.mappingproxy(dc) == Evil() + self.assertIs(type(leaked), dict) + self.assertIn(key, leaked) + self.assertNotIn(key, dc) + + # Exposes the internals of `MappingProxyType` via richcompare: + leaked = vars(list) == Evil() + self.assertIs(type(leaked), dict) + self.assertIn(key, leaked) + name = "__mappingproxy_crash_probe__" + leaked[name] = lambda self: "probe" + self.assertIn(name, leaked) + self.assertNotHasAttr(list, key) + self.assertNotHasAttr(list, name) # it used to return `True` + del leaked[name] + self.assertNotIn(name, leaked) + self.assertNotHasAttr(list, name) # it used to crash + class ClassCreationTests(unittest.TestCase): @@ -2016,6 +2155,24 @@ class SimpleNamespaceTests(unittest.TestCase): self.assertEqual(ns1, ns2) self.assertNotEqual(ns2, types.SimpleNamespace()) + def test_richcompare_unsupported(self): + ns1 = types.SimpleNamespace(x=1) + ns2 = types.SimpleNamespace(y=2) + + msg = re.escape( + "not supported between instances of " + "'types.SimpleNamespace' and 'types.SimpleNamespace'" + ) + + with self.assertRaisesRegex(TypeError, msg): + ns1 > ns2 + with self.assertRaisesRegex(TypeError, msg): + ns1 >= ns2 + with self.assertRaisesRegex(TypeError, msg): + ns1 < ns2 + with self.assertRaisesRegex(TypeError, msg): + ns1 <= ns2 + def test_nested(self): ns1 = types.SimpleNamespace(a=1, b=2) ns2 = types.SimpleNamespace() diff --git a/Lib/types.py b/Lib/types.py index 60a4b22c2..6c069591a 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -58,7 +58,10 @@ except ImportError: raise TypeError except TypeError as exc: TracebackType = type(exc.__traceback__) - FrameType = type(exc.__traceback__.tb_frame) + + _f = (lambda: sys._getframe())() + FrameType = type(_f) + FrameLocalsProxyType = type(_f.f_locals) GetSetDescriptorType = type(FunctionType.__code__) MemberDescriptorType = type(FunctionType.__globals__) @@ -73,6 +76,9 @@ except ImportError: # CapsuleType cannot be accessed from pure Python, # so there is no fallback definition. + # LazyImportType in pure Python cannot be guaranteed + # without overriding the filter, so there is no fallback. + del sys, _f, _g, _C, _c, _ag, _cell_factory # Not for export @@ -251,7 +257,6 @@ class DynamicClassAttribute: class _GeneratorWrapper: - # TODO: Implement this in C. def __init__(self, gen): self.__wrapped = gen self.__isgen = gen.__class__ is GeneratorType @@ -278,6 +283,12 @@ class _GeneratorWrapper: @property def gi_suspended(self): return self.__wrapped.gi_suspended + @property + def gi_state(self): + return self.__wrapped.gi_state + @property + def cr_state(self): + return self.__wrapped.gi_state.replace('GEN_', 'CORO_') cr_code = gi_code cr_frame = gi_frame cr_running = gi_running @@ -310,7 +321,6 @@ def coroutine(func): # Check if 'func' is a generator function. # (0x20 == CO_GENERATOR) if co_flags & 0x20: - # TODO: Implement this in C. co = func.__code__ # 0x100 == CO_ITERABLE_COROUTINE func.__code__ = co.replace(co_flags=co.co_flags | 0x100) diff --git a/crates/vm/src/builtins/asyncgenerator.rs b/crates/vm/src/builtins/asyncgenerator.rs index 91fb744dc..810683377 100644 --- a/crates/vm/src/builtins/asyncgenerator.rs +++ b/crates/vm/src/builtins/asyncgenerator.rs @@ -1,3 +1,5 @@ +// spell-checker: ignore AGEN + use super::{PyCode, PyGenerator, PyGenericAlias, PyStrRef, PyType, PyTypeRef}; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, @@ -162,6 +164,15 @@ impl Py { self.inner.code() } #[pygetset] + fn ag_state(&self) -> &'static str { + self.inner.state_name([ + "AGEN_CREATED", + "AGEN_RUNNING", + "AGEN_SUSPENDED", + "AGEN_CLOSED", + ]) + } + #[pygetset] fn ag_suspended(&self, _vm: &VirtualMachine) -> bool { self.inner.suspended() } diff --git a/crates/vm/src/builtins/coroutine.rs b/crates/vm/src/builtins/coroutine.rs index a0d4a581a..93481c41a 100644 --- a/crates/vm/src/builtins/coroutine.rs +++ b/crates/vm/src/builtins/coroutine.rs @@ -148,6 +148,15 @@ impl Py { self.inner.code() } #[pygetset] + fn cr_state(&self) -> &'static str { + self.inner.state_name([ + "CORO_CREATED", + "CORO_RUNNING", + "CORO_SUSPENDED", + "CORO_CLOSED", + ]) + } + #[pygetset] fn cr_suspended(&self, _vm: &VirtualMachine) -> bool { self.inner.suspended() } diff --git a/crates/vm/src/builtins/frame_locals_proxy.rs b/crates/vm/src/builtins/frame_locals_proxy.rs index 28da2341f..8cb98d779 100644 --- a/crates/vm/src/builtins/frame_locals_proxy.rs +++ b/crates/vm/src/builtins/frame_locals_proxy.rs @@ -21,6 +21,13 @@ use crate::{ use rustpython_common::lock::LazyLock; use rustpython_common::wtf8::Wtf8Buf; +/// FrameLocalsProxy($frame) +/// -- +/// +/// Create a write-through view of the locals dictionary for a frame. +/// +/// frame +/// the frame object to wrap. #[pyclass( module = false, name = "FrameLocalsProxy", diff --git a/crates/vm/src/coroutine.rs b/crates/vm/src/coroutine.rs index feb95d49c..b1ea72b13 100644 --- a/crates/vm/src/coroutine.rs +++ b/crates/vm/src/coroutine.rs @@ -527,6 +527,21 @@ impl Coro { self.started.load() } + pub(crate) fn state_name(&self, names: [&'static str; 4]) -> &'static str { + // Closing publishes the terminal state before releasing frame locals, + // whose finalizers can observe us while the running claim is held. + let index = if self.closed() { + 3 + } else if self.running() { + 1 + } else if self.started() { + 2 + } else { + 0 + }; + names[index] + } + pub fn suspended(&self) -> bool { !self.closed.load() && !self.running.load() && self.started() } diff --git a/crates/vm/src/import.rs b/crates/vm/src/import.rs index 3fb6e5fdc..34911348b 100644 --- a/crates/vm/src/import.rs +++ b/crates/vm/src/import.rs @@ -29,7 +29,7 @@ pub(crate) fn init_importlib_base(vm: &mut VirtualMachine) -> PyResult PyObjectRef { + vm.ctx.types.frame_locals_proxy_type.to_owned().into() + } + #[pyattr] fn FrameType(vm: &VirtualMachine) -> PyObjectRef { vm.ctx.types.frame_type.to_owned().into() diff --git a/extra_tests/snippets/vm_frame_locals.py b/extra_tests/snippets/vm_frame_locals.py index 608167c2a..4313b07db 100644 --- a/extra_tests/snippets/vm_frame_locals.py +++ b/extra_tests/snippets/vm_frame_locals.py @@ -230,6 +230,33 @@ value = [ frame.clear() self.assertEqual(len(events), 1) + def test_closed_coroutine_state_is_visible_to_frame_local_finalizers(self): + events = [] + + class ReentrantFinalizer: + def __del__(self): + events.append(getattr(owner, state_attribute)) + + async def coroutine(value): + return value + + async def async_generator(value): + yield value + + for factory, state_attribute, closed in ( + (coroutine, "cr_state", "CORO_CLOSED"), + (async_generator, "ag_state", "AGEN_CLOSED"), + ): + with self.subTest(state_attribute=state_attribute): + events.clear() + owner = factory(ReentrantFinalizer()) + if state_attribute == "cr_state": + owner.close() + else: + with self.assertRaises(StopIteration): + owner.aclose().send(None) + self.assertEqual(events, [closed]) + def test_unoptimized_frame_locals_are_readable_from_another_thread(self): ready = threading.Event() release = threading.Event() diff --git a/scripts/update_lib/cmd_migrate.py b/scripts/update_lib/cmd_migrate.py index 6dea08a3c..f9b8a5ca2 100644 --- a/scripts/update_lib/cmd_migrate.py +++ b/scripts/update_lib/cmd_migrate.py @@ -5,6 +5,7 @@ Migrate test file(s) from CPython, preserving RustPython markers. Usage: python scripts/update_lib migrate cpython/Lib/test/test_foo.py python scripts/update_lib migrate cpython/Lib/test/test_foo.py --patches remaining.json + python scripts/update_lib migrate cpython/Lib/test/test_foo --canonical-removals removals.json --dry-run Remaining metadata names the exact repository-relative target and its existing decorator patches. An optional "doctests" object selects existing guarded inputs @@ -12,6 +13,13 @@ by exact source text, mapping each to a single-line reason. Guarded doctests nee this explicit selection; an empty object retires all their guards. Canonical expected output is never copied from the old file. +Canonical removals are a separate, explicit operation. JSON source metadata +pins path (Lib/test/test_*), commit and tree; removals maps explicit relative +filenames to sha256 and upstream_commit. The caller must first verify and +review historical provenance, including release-branch evidence. This command +checks pinned canonical absence and exact local bytes; it does not authenticate +historical evidence. --dry-run performs all validation without deleting files. + This will: 1. Extract patches from Lib/test/test_foo.py (if exists) 2. Apply them to cpython/Lib/test/test_foo.py @@ -25,8 +33,11 @@ import collections import copy import hashlib import json +import os import pathlib +import re import shutil +import subprocess import sys import textwrap @@ -51,6 +62,33 @@ else: EXCEPTION_LOOP = """for excname in PYTHON2_EXCEPTIONS: NAME_MAPPING[("exceptions", excname)] = ("builtins", excname) """ +MULTIPHASE_UPSTREAM = ( + "# gh-116303: Skip test module dependent tests if test modules are unavailable\n" + 'import_helper.import_module("_testmultiphase")\n' +) +MULTIPHASE_LOCAL = """try: # TODO: RUSTPYTHON + import_helper.import_module("_testmultiphase") +except unittest.SkipTest: + _testmultiphase = None +""" +LOCK_SPECIALIZATION = """(Frozen_LifetimeTests, + Source_LifetimeTests + ) = test_util.test_both(LifetimeTests, init=init) +""" +LOCK_SKIP = """# TODO: RUSTPYTHON; dead weakref module locks not cleaned up in frozen bootstrap +Frozen_LifetimeTests.test_all_locks = unittest.skip("TODO: RUSTPYTHON")( + Frozen_LifetimeTests.test_all_locks +) +""" +TYPES_DATETIME_GUARD = """# XXX: RUSTPYTHON +try: + import _datetime +except ImportError: + _datetime = None +""" +TYPES_UNITTEST_IMPORT = ( + "import unittest # XXX: RUSTPYTHON; importing to be able to skip tests\n" +) BYTES_CAPI_UPSTREAM = ( " _testlimitedcapi = import_helper.import_module('_testlimitedcapi')\n" ) @@ -258,6 +296,78 @@ def _jit_block(contents: str) -> bool: ) +def _preserve_top_level_block(old, new, upstream, local): + """Require exact, consecutive top-level AST statements and comment bytes.""" + for contents, fragment in ((old, local), (new, upstream)): + tree, expected = ast.parse(contents), ast.parse(fragment).body + nodes = [ + _unique_anchor( + contents, + tree, + lambda node: ast.dump(node) == ast.dump(item), + direct=True, + ) + for item in expected + ] + start = tree.body.index(nodes[0]) + lines = contents.splitlines(keepends=True) + first = nodes[0].lineno - expected[0].lineno + if ( + tree.body[start : start + len(nodes)] != nodes + or contents.count(fragment) != 1 + or "".join(lines[first : nodes[-1].end_lineno]) != fragment + ): + raise ValueError("Changed or ambiguous migration customization anchor") + return new.replace(upstream, local, 1) + + +def _preserve_multiphase_guard(old, new): + for contents, local in ((old, True), (new, False)): + tree = ast.parse(contents) + if sum( + isinstance(n, ast.Constant) and n.value == "_testmultiphase" + for n in ast.walk(tree) + ) != 1 or sum( + isinstance(n, ast.Name) and n.id == "_testmultiphase" + for n in ast.walk(tree) + ) != int(local): + raise ValueError("Ambiguous _testmultiphase anchor") + return _preserve_top_level_block(old, new, MULTIPHASE_UPSTREAM, MULTIPHASE_LOCAL) + + +def _preserve_frozen_lock_skip(old, new): + methods = [] + for contents, local in ((old, True), (new, False)): + tree = ast.parse(contents) + owner = _unique_anchor( + contents, + tree, + lambda n: isinstance(n, ast.ClassDef) and n.name == "LifetimeTests", + direct=True, + ) + method = _unique_anchor( + contents, + owner, + lambda n: isinstance(n, ast.FunctionDef) and n.name == "test_all_locks", + direct=True, + ) + methods.append(ast.dump(method)) + if sum( + isinstance(n, ast.Name) and n.id == "Frozen_LifetimeTests" + for n in ast.walk(tree) + ) != (3 if local else 1): + raise ValueError("Ambiguous frozen lock specialization") + if methods[0] != methods[1]: + raise ValueError("Frozen lock test changed; review its customization") + setup = "def setUpModule():\n thread_info = threading_helper.threading_setup()\n unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info)\n" + return _preserve_top_level_block( + old, + new, + LOCK_SPECIALIZATION + "\n\n" + setup, + LOCK_SPECIALIZATION + "\n" + LOCK_SKIP + "\n" + setup, + ) + + def preserve_migration_customizations(path: pathlib.Path, old: str, new: str) -> str: """Fail closed if either known local patch or its canonical anchor changes.""" parts = path.parts @@ -299,6 +409,31 @@ def preserve_migration_customizations(path: pathlib.Path, old: str, new: str) -> lines = new.splitlines(keepends=True) lines[anchor.lineno - 1 : anchor.end_lineno] = [BYTES_CAPI_LOCAL] return _preserve_bytes_allocations(old, "".join(lines)) + if parts == ("Lib", "test", "test_importlib", "util.py"): + return _preserve_multiphase_guard(old, new) + if parts == ("Lib", "test", "test_importlib", "test_locks.py"): + return _preserve_frozen_lock_skip(old, new) + if parts == ("Lib", "test", "test_types.py"): + for contents in (old, new): + if ( + sum( + isinstance(node, ast.alias) and node.name == "_datetime" + for node in ast.walk(ast.parse(contents)) + ) + != 1 + ): + raise ValueError("Ambiguous test_types _datetime import") + if any( + isinstance(node, ast.alias) and node.name == "unittest" + for node in ast.walk(ast.parse(new)) + ): + raise ValueError("Canonical test_types already imports unittest directly") + new = _preserve_top_level_block( + old, new, "import _datetime\n", TYPES_DATETIME_GUARD + ) + return _preserve_top_level_block( + old, new, "import typing\n", "import typing\n\n" + TYPES_UNITTEST_IMPORT + ) return new @@ -622,6 +757,8 @@ def patch_single_content( ) if selection is not None: src_content = apply_doctest_metadata(old_content, src_content, selection) + if not src_content and not patches: + return src_content return apply_patches(src_content, patches) @@ -849,6 +986,159 @@ def migrate_package( legacy.unlink() +def remove_canonical_files(src_dir, manifest_path, *, dry_run=False): + """Remove only explicitly reviewed files, validating the whole batch first.""" + + def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"Duplicate removal metadata key: {key}") + result[key] = value + return result + + def relative(name): + path = pathlib.PurePosixPath(name) if isinstance(name, str) else None + if ( + not path + or not path.parts + or path.is_absolute() + or ".." in path.parts + or "\\" in name + or path.as_posix() != name + or "\0" in name + ): + raise ValueError("Expected an exact relative POSIX filename") + return pathlib.Path(path) + + def no_symlinks(path): + if any(p.is_symlink() or p.is_junction() for p in (path, *path.parents)): + raise ValueError(f"Symlink or junction in removal path: {path}") + + def git(*args): + result = subprocess.run( + ["git", "-C", str(src_dir), *args], + env={ + **os.environ, + "GIT_NO_LAZY_FETCH": "1", + "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_TERMINAL_PROMPT": "0", + }, + capture_output=True, + check=False, + ) + if result.returncode: + raise ValueError("Cannot verify pinned canonical source") + return result.stdout.decode().strip() + + document = json.loads( + manifest_path.read_text(encoding="utf-8"), object_pairs_hook=unique_object + ) + if not isinstance(document, dict) or set(document) != {"source", "removals"}: + raise ValueError("Expected source and removals metadata") + source, removals = document["source"], document["removals"] + if not isinstance(source, dict) or set(source) != {"path", "commit", "tree"}: + raise ValueError("Expected source path, commit and tree") + package = relative(source["path"]) + if ( + len(package.parts) != 3 + or package.parts[:2] != ("Lib", "test") + or not package.name.startswith("test_") + or not package.name.isidentifier() + ): + raise ValueError("Removals require an exact Lib/test/test_* package") + if any( + not isinstance(source[k], str) or not re.fullmatch(r"[0-9a-f]{40}", source[k]) + for k in ("commit", "tree") + ): + raise ValueError("Source commit and tree must be full Git hashes") + if not isinstance(removals, dict) or not removals: + raise ValueError("Removals must be a nonempty filename map") + src_dir = src_dir.absolute() + destination = pathlib.Path.cwd() / package + for path in (src_dir, destination): + no_symlinks(path) + if not path.is_dir(): + raise ValueError("Source and destination packages must exist") + if ( + src_dir == destination + or src_dir in destination.parents + or destination in src_dir.parents + ): + raise ValueError("Source and destination overlap") + if ( + src_dir != pathlib.Path(git("rev-parse", "--show-toplevel")) / package + or git("rev-parse", "HEAD") != source["commit"] + or git("rev-parse", f"HEAD:{package.as_posix()}") != source["tree"] + ): + raise ValueError("Canonical source identity differs from the pinned input") + canonical = set( + git( + "ls-tree", + "--full-tree", + "-r", + "--name-only", + "-z", + "HEAD", + "--", + package.as_posix(), + ).split("\0") + ) + snapshots, receipt = {}, {"source": source, "dry_run": dry_run, "removals": {}} + for name, entry in sorted(removals.items()): + path = relative(name) + if ( + not isinstance(entry, dict) + or set(entry) != {"sha256", "upstream_commit"} + or any( + not isinstance(entry[k], str) + or not re.fullmatch(f"[0-9a-f]{{{length}}}", entry[k]) + for k, length in (("sha256", 64), ("upstream_commit", 40)) + ) + ): + raise ValueError(f"Missing or invalid hash/provenance pins: {name}") + target = destination / path + no_symlinks(target) + no_symlinks(src_dir / path) + if (package / path).as_posix() in canonical or (src_dir / path).exists(): + raise ValueError(f"Selected file is still canonical: {name}") + if target.exists() and not target.is_file(): + raise ValueError(f"Removal target is not a regular file: {name}") + data = target.read_bytes() if target.exists() else None + if data is not None and ( + hashlib.sha256(data).hexdigest() != entry["sha256"] + or b"rustpython" in data.lower() + ): + raise ValueError(f"Changed bytes or RustPython customization: {name}") + snapshots[target] = data + receipt["removals"][name] = { + **entry, + "status": "already_absent" + if data is None + else "would_remove" + if dry_run + else "removed", + } + if git("rev-parse", "HEAD") != source["commit"]: + raise ValueError("Canonical source changed during validation") + for target, data in snapshots.items(): + no_symlinks(target) + canonical_path = src_dir / target.relative_to(destination) + no_symlinks(canonical_path) + if canonical_path.exists(): + raise ValueError("Canonical source changed during validation") + if (target.exists() and not target.is_file()) or ( + target.read_bytes() if target.exists() else None + ) != data: + raise ValueError(f"Removal target changed during validation: {target}") + if not dry_run: + for target, data in snapshots.items(): + if data is not None: + target.unlink() + print(json.dumps(receipt, sort_keys=True)) + return receipt + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description=__doc__, @@ -865,6 +1155,16 @@ def main(argv: list[str] | None = None) -> int: type=pathlib.Path, help="Explicit legacy-file/class to package-file JSON mapping", ) + parser.add_argument( + "--canonical-removals", + type=pathlib.Path, + help="Remove only selected unchanged historical files absent from a pinned canonical tree", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Validate canonical removals without deleting", + ) parser.add_argument( "--patches", @@ -882,6 +1182,27 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) try: + if args.canonical_removals is not None: + if ( + args.legacy_map is not None + or args.patches is not None + or args.support_assignments + or args.support_exports + or args.source_sha256 + or args.destination_sha256 + ): + raise ValueError( + "Canonical removals cannot combine with other migration modes" + ) + try: + remove_canonical_files( + args.path, args.canonical_removals, dry_run=args.dry_run + ) + except (OSError, UnicodeError) as exc: + raise ValueError(str(exc)) from exc + return 0 + if args.dry_run: + raise ValueError("--dry-run requires --canonical-removals") if args.support_assignments: if args.legacy_map is not None or args.patches is not None: raise ValueError( diff --git a/scripts/update_lib/tests/test_migrate.py b/scripts/update_lib/tests/test_migrate.py index 3f85165d5..c6f0fa172 100644 --- a/scripts/update_lib/tests/test_migrate.py +++ b/scripts/update_lib/tests/test_migrate.py @@ -4,7 +4,9 @@ import contextlib import hashlib import io import json +import os import pathlib +import subprocess import tempfile import unittest from pathlib import Path @@ -18,8 +20,14 @@ from update_lib.cmd_migrate import ( EXCEPTION_LOOP, JIT_ERROR, LOCAL_CALL, + LOCK_SKIP, + LOCK_SPECIALIZATION, + MULTIPHASE_LOCAL, + MULTIPHASE_UPSTREAM, SYSCONFIG_LOCAL, SYSCONFIG_UPSTREAM, + TYPES_DATETIME_GUARD, + TYPES_UNITTEST_IMPORT, copy_support_assignments, main, migrate_package, @@ -27,6 +35,7 @@ from update_lib.cmd_migrate import ( patch_file, patch_single_content, preserve_migration_customizations, + remove_canonical_files, ) from update_lib.patch_spec import ( CANONICAL_LOAD_TESTS, @@ -711,6 +720,17 @@ class TestFoo(unittest.TestCase): class TestPatchDirectory(unittest.TestCase): """Tests for patch_directory function.""" + def test_empty_canonical_fixture_stays_byte_empty(self): + with tempfile.TemporaryDirectory() as temporary: + source, target = Path(temporary) / "canonical", Path(temporary) / "local" + source.mkdir() + target.mkdir() + (source / "empty.py").write_bytes(b"") + (target / "empty.py").write_bytes(b"\n") + for _ in range(2): + patch_directory(source, target, verbose=False) + self.assertEqual((target / "empty.py").read_bytes(), b"") + def test_excludes_cache_directories_but_preserves_canonical_assets(self): with tempfile.TemporaryDirectory() as temporary: root = pathlib.Path(temporary) @@ -1282,5 +1302,362 @@ class TestMigratePackage(unittest.TestCase): self.assert_rejected() +class TestCanonicalRemovals(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.source = self.root / "canonical/Lib/test/test_example" + self.target = self.root / "local/Lib/test/test_example" + self.source.mkdir(parents=True) + self.target.mkdir(parents=True) + (self.source / "__init__.py").write_bytes(b"") + self.git("init", "-q") + self.git("add", ".") + self.git("commit", "-qm", "canonical test fixture") + self.document = { + "source": { + "path": "Lib/test/test_example", + "commit": self.git("rev-parse", "HEAD"), + "tree": self.git("rev-parse", "HEAD:Lib/test/test_example"), + }, + "removals": {}, + } + for name, data in (("a.py", b"old\n"), ("data/z.bin", b"\x00\xff")): + path = self.target / name + path.parent.mkdir(exist_ok=True) + path.write_bytes(data) + self.document["removals"][name] = { + "sha256": hashlib.sha256(data).hexdigest(), + "upstream_commit": "1" * 40, + } + self.manifest = self.root / "removals.json" + + def git(self, *args): + return subprocess.check_output( + ["git", "-C", str(self.source.parents[2]), *args], + env={ + **os.environ, + "GIT_AUTHOR_NAME": "Fixture", + "GIT_AUTHOR_EMAIL": "fixture@example.invalid", + "GIT_COMMITTER_NAME": "Fixture", + "GIT_COMMITTER_EMAIL": "fixture@example.invalid", + }, + text=True, + ).strip() + + def run_removal(self, dry_run=False): + self.manifest.write_text(json.dumps(self.document)) + with ( + contextlib.chdir(self.target.parents[2]), + contextlib.redirect_stdout(io.StringIO()), + ): + return remove_canonical_files(self.source, self.manifest, dry_run=dry_run) + + def test_dry_run_explicit_scope_receipt_and_idempotence(self): + (self.target / "unselected").write_text("keep") + for dry_run, expected in ( + (True, "would_remove"), + (False, "removed"), + (False, "already_absent"), + ): + receipt = self.run_removal(dry_run) + self.assertEqual(receipt["source"], self.document["source"]) + self.assertEqual( + {v["status"] for v in receipt["removals"].values()}, {expected} + ) + self.assertEqual((self.target / "a.py").exists(), dry_run) + self.assertEqual((self.target / "unselected").read_text(), "keep") + + def test_junction_components_reject_the_whole_batch(self): + before = { + name: (self.target / name).read_bytes() + for name in self.document["removals"] + } + for junction in ( + self.source, + self.source.parent, + self.source / "data", + self.target, + self.target.parent, + self.target / "data", + ): + for dry_run in (True, False): + with self.subTest(junction=junction, dry_run=dry_run): + self.assertFalse(junction.is_symlink()) + with ( + patch.object( + Path, + "is_junction", + autospec=True, + side_effect=lambda path: path == junction, + ), + self.assertRaisesRegex(ValueError, "[Jj]unction"), + ): + self.run_removal(dry_run) + self.assertEqual( + {name: (self.target / name).read_bytes() for name in before}, + before, + ) + + def test_junction_revalidation_prevents_partial_removal(self): + for junction in (self.target / "data", self.source / "data"): + checks = 0 + + def becomes_junction(path): + nonlocal checks + if path == junction: + checks += 1 + return checks > 1 + return False + + with self.subTest(junction=junction): + with ( + patch.object( + Path, "is_junction", autospec=True, side_effect=becomes_junction + ), + self.assertRaisesRegex(ValueError, "[Jj]unction"), + ): + self.run_removal() + self.assertEqual(checks, 2) + self.assertEqual((self.target / "a.py").read_bytes(), b"old\n") + self.assertEqual((self.target / "data/z.bin").read_bytes(), b"\x00\xff") + + def test_utf8_manifest_ignores_non_utf8_default_encoding(self): + name = "caf\u00e9.py" + target = self.target / name + target.write_bytes(b"old\n") + self.document["removals"][name] = dict(self.document["removals"]["a.py"]) + self.manifest.write_bytes( + json.dumps(self.document, ensure_ascii=False).encode("utf-8") + ) + with ( + patch( + "pathlib.io.text_encoding", + side_effect=lambda encoding, *args: ( + "latin-1" if encoding is None else encoding + ), + ), + contextlib.chdir(self.target.parents[2]), + contextlib.redirect_stdout(io.StringIO()), + ): + self.assertNotEqual( + self.manifest.read_text(), self.manifest.read_text(encoding="utf-8") + ) + for dry_run, expected in ( + (True, "would_remove"), + (False, "removed"), + (False, "already_absent"), + ): + receipt = remove_canonical_files( + self.source, self.manifest, dry_run=dry_run + ) + self.assertEqual( + set(receipt["removals"]), set(self.document["removals"]) + ) + self.assertEqual(receipt["removals"][name]["status"], expected) + self.assertEqual(target.exists(), dry_run) + + def test_cli_dry_run_never_migrates_or_changes_destination_bytes(self): + def snapshot(): + return { + p.relative_to(self.target): p.read_bytes() + for p in self.target.rglob("*") + if p.is_file() + } + + before = snapshot() + self.manifest.write_text(json.dumps(self.document)) + with contextlib.ExitStack() as stack: + stack.enter_context(contextlib.chdir(self.target.parents[2])) + stack.enter_context(contextlib.redirect_stdout(io.StringIO())) + for name in ( + "patch_file", + "patch_directory", + "migrate_package", + "migrate_support_assignments", + ): + stack.enter_context( + patch( + f"update_lib.cmd_migrate.{name}", + side_effect=AssertionError("Dry-run must not migrate"), + ) + ) + self.assertEqual( + main( + [ + str(self.source), + "--canonical-removals", + str(self.manifest), + "--dry-run", + ] + ), + 0, + ) + self.assertEqual(snapshot(), before) + + def test_hash_customization_and_all_batch_preflight(self): + path = self.target / "data/z.bin" + for data, repin in ((b"changed", False), (b"# TODO: RUSTPYTHON", True)): + path.write_bytes(data) + if repin: + self.document["removals"]["data/z.bin"]["sha256"] = hashlib.sha256( + data + ).hexdigest() + with self.assertRaises(ValueError): + self.run_removal() + self.assertEqual((self.target / "a.py").read_bytes(), b"old\n") + + def test_canonical_source_identity_and_tracked_absence(self): + for key, value in ( + ("commit", "0" * 40), + ("tree", "0" * 40), + ("path", "Lib/test/test_other"), + ): + old = self.document["source"][key] + self.document["source"][key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + self.run_removal() + self.document["source"][key] = old + (self.source / "__init__.py").unlink() + self.document["removals"]["__init__.py"] = self.document["removals"]["a.py"] + with self.assertRaisesRegex(ValueError, "still canonical"): + self.run_removal() + self.assertTrue((self.target / "a.py").exists()) + + def test_invalid_paths_symlinks_directories_and_empty_selection(self): + original = self.document["removals"] + for name in ("../a.py", "/a.py", "./a.py", "data//z.bin", "data\\z.bin"): + self.document["removals"] = {name: original["a.py"]} + with self.subTest(name=name), self.assertRaises(ValueError): + self.run_removal() + self.document["removals"] = {} + with self.assertRaises(ValueError): + self.run_removal() + self.document["removals"] = original + target = self.target / "a.py" + target.unlink() + target.symlink_to(self.target / "data/z.bin") + with self.assertRaisesRegex(ValueError, "Symlink"): + self.run_removal() + target.unlink() + target.mkdir() + with self.assertRaisesRegex(ValueError, "regular file"): + self.run_removal() + + def test_cli_refuses_combined_modes_and_duplicate_metadata(self): + for option in ("--patches", "--legacy-map", "--support-assignments"): + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual( + main( + [ + str(self.source), + "--canonical-removals", + "unused", + option, + "unused", + ] + ), + 1, + ) + self.manifest.write_text('{"source": {}, "source": {}}') + with self.assertRaisesRegex(ValueError, "Duplicate"): + remove_canonical_files(self.source, self.manifest) + + +class TestImportlibGuardMigration(unittest.TestCase): + def test_exact_guards_preserved_and_changed_ambiguous_anchors_refused(self): + setup = "def setUpModule():\n thread_info = threading_helper.threading_setup()\n unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info)\n" + cases = ( + ("util.py", MULTIPHASE_LOCAL, MULTIPHASE_UPSTREAM), + ( + "test_locks.py", + "class LifetimeTests:\n def test_all_locks(self): pass\n\n" + + LOCK_SPECIALIZATION + + "\n" + + LOCK_SKIP + + "\n" + + setup, + "class LifetimeTests:\n def test_all_locks(self): pass\n\n" + + LOCK_SPECIALIZATION + + "\n\n" + + setup, + ), + ) + for filename, old, new in cases: + path = Path("Lib/test/test_importlib") / filename + result = preserve_migration_customizations(path, old, new + "VALUE = 2\n") + self.assertEqual(result, old + "VALUE = 2\n") + self.assertEqual( + preserve_migration_customizations(path, result, new + "VALUE = 2\n"), + result, + ) + for changed_old, changed_new in ( + (old + old, new), + (old, new + new), + (old.replace("TODO: RUSTPYTHON", "changed"), new), + ( + old, + "if True:\n" + + "".join(" " + line for line in new.splitlines(keepends=True)), + ), + ): + with self.subTest(filename=filename), self.assertRaises(ValueError): + preserve_migration_customizations(path, changed_old, changed_new) + + +class TestTypesGuardMigration(unittest.TestCase): + path = Path("Lib/test/test_types.py") + old = ( + "import copy\n" + + TYPES_DATETIME_GUARD + + "import gc\nimport unittest.mock\nimport typing\n\n" + + TYPES_UNITTEST_IMPORT + + "\nVALUE = 1\n" + ) + new = ( + "import copy\nimport _datetime\nimport gc\nimport unittest.mock\n" + "import typing\nimport re\n\nVALUE = 2\n" + ) + + def test_exact_guards_and_unrelated_canonical_changes_survive_idempotently(self): + expected = self.new.replace("import _datetime\n", TYPES_DATETIME_GUARD).replace( + "import typing\n", "import typing\n\n" + TYPES_UNITTEST_IMPORT + ) + result = preserve_migration_customizations(self.path, self.old, self.new) + self.assertEqual(result, expected) + self.assertEqual( + preserve_migration_customizations(self.path, result, self.new), expected + ) + + def test_changed_ambiguous_or_misplaced_local_guards_are_refused(self): + for old in ( + self.old.replace("except ImportError:", "except Exception:"), + self.old.replace("_datetime = None", "_datetime = False"), + self.old.replace("# XXX: RUSTPYTHON\n", "# changed\n"), + self.old.replace("importing to be able to skip tests", "changed reason"), + self.old + TYPES_DATETIME_GUARD, + self.old + "import _datetime\n", + self.old + TYPES_UNITTEST_IMPORT, + self.old.replace("import typing\n\n", "import typing\nVALUE = 3\n\n"), + ): + with self.subTest(old=old), self.assertRaises(ValueError): + preserve_migration_customizations(self.path, old, self.new) + + def test_changed_ambiguous_or_nested_canonical_anchors_are_refused(self): + for new in ( + self.new.replace("import _datetime", "import _datetime as other"), + self.new.replace("import _datetime\n", "if True:\n import _datetime\n"), + self.new.replace("import _datetime\n", 'TEXT = "import _datetime\\n"\n'), + self.new.replace("import typing", "import typing as other"), + self.new.replace("import typing\n", "if True:\n import typing\n"), + self.new + "import _datetime\n", + self.new + "import typing\n", + self.new + "import unittest\n", + ): + with self.subTest(new=new), self.assertRaises(ValueError): + preserve_migration_customizations(self.path, self.old, new) + + if __name__ == "__main__": unittest.main()