From afdf8cefe8760b04424bd68fe2722c258c0fb6ea Mon Sep 17 00:00:00 2001 From: CPython Developers <> Date: Wed, 4 Feb 2026 16:48:27 +0900 Subject: [PATCH] Update importlib from v3.14.2 --- Lib/importlib/_bootstrap.py | 22 +- Lib/importlib/_bootstrap_external.py | 284 +---------- Lib/importlib/abc.py | 25 +- Lib/importlib/machinery.py | 35 +- Lib/importlib/resources/_common.py | 6 +- Lib/importlib/resources/_legacy.py | 120 ----- Lib/importlib/util.py | 9 +- Lib/test/lock_tests.py | 54 +- .../test_importlib/builtin/test_finder.py | 1 - .../extension/_test_nonmodule_cases.py | 44 ++ .../extension/test_case_sensitivity.py | 4 +- .../test_importlib/extension/test_finder.py | 29 +- .../test_importlib/extension/test_loader.py | 106 ++-- .../extension/test_path_hook.py | 4 +- Lib/test/test_importlib/frozen/test_finder.py | 9 - Lib/test/test_importlib/frozen/test_loader.py | 10 +- .../test_importlib/import_/test___loader__.py | 3 - .../test_importlib/import_/test_caching.py | 4 +- .../test_importlib/import_/test_fromlist.py | 16 +- .../test_importlib/import_/test_meta_path.py | 2 +- .../test_importlib/import_/test_packages.py | 1 - Lib/test/test_importlib/import_/test_path.py | 29 +- .../import_/test_relative_imports.py | 21 +- Lib/test/test_importlib/metadata/__init__.py | 1 + Lib/test/test_importlib/metadata/_context.py | 13 + Lib/test/test_importlib/metadata/_path.py | 115 +++++ .../test_importlib/metadata/data/__init__.py | 1 + .../data/example-21.12-py3-none-any.whl | Bin 0 -> 1455 bytes .../metadata/data/example-21.12-py3.6.egg | Bin 0 -> 1497 bytes .../data/example2-1.0.0-py3-none-any.whl | Bin 0 -> 1167 bytes .../data/sources/example/example/__init__.py | 2 + .../metadata/data/sources/example/setup.py | 11 + .../sources/example2/example2/__init__.py | 2 + .../data/sources/example2/pyproject.toml | 10 + Lib/test/test_importlib/metadata/fixtures.py | 395 +++++++++++++++ Lib/test/test_importlib/metadata/stubs.py | 10 + Lib/test/test_importlib/metadata/test_api.py | 323 ++++++++++++ Lib/test/test_importlib/metadata/test_main.py | 468 ++++++++++++++++++ Lib/test/test_importlib/metadata/test_zip.py | 62 +++ .../not_a_namespace_pkg/foo/__init__.py | 1 + Lib/test/test_importlib/resources/__init__.py | 1 + Lib/test/test_importlib/resources/_path.py | 52 +- .../test_importlib/resources/test_contents.py | 17 +- .../test_importlib/resources/test_custom.py | 6 +- .../test_importlib/resources/test_files.py | 176 +++++-- .../resources/test_functional.py | 249 ++++++++++ .../test_importlib/resources/test_open.py | 19 +- .../test_importlib/resources/test_path.py | 19 +- .../test_importlib/resources/test_read.py | 42 +- .../test_importlib/resources/test_reader.py | 63 ++- .../test_importlib/resources/test_resource.py | 169 +++---- Lib/test/test_importlib/resources/util.py | 111 +++-- Lib/test/test_importlib/resources/zip.py | 24 + .../source/test_case_sensitivity.py | 1 - .../test_importlib/source/test_file_loader.py | 5 + Lib/test/test_importlib/source/test_finder.py | 9 +- .../test_importlib/source/test_path_hook.py | 6 +- .../source/test_source_encoding.py | 9 +- Lib/test/test_importlib/test_abc.py | 40 +- Lib/test/test_importlib/test_api.py | 74 ++- Lib/test/test_importlib/test_lazy.py | 92 +++- Lib/test/test_importlib/test_locks.py | 14 +- .../test_importlib/test_namespace_pkgs.py | 29 +- Lib/test/test_importlib/test_pkg_import.py | 2 +- Lib/test/test_importlib/test_spec.py | 24 +- .../test_importlib/test_threaded_import.py | 29 +- Lib/test/test_importlib/test_util.py | 147 +++++- Lib/test/test_importlib/test_windows.py | 45 +- Lib/test/test_importlib/util.py | 54 +- Lib/test/test_py_compile.py | 9 +- 70 files changed, 2850 insertions(+), 939 deletions(-) delete mode 100644 Lib/importlib/resources/_legacy.py create mode 100644 Lib/test/test_importlib/extension/_test_nonmodule_cases.py create mode 100644 Lib/test/test_importlib/metadata/__init__.py create mode 100644 Lib/test/test_importlib/metadata/_context.py create mode 100644 Lib/test/test_importlib/metadata/_path.py create mode 100644 Lib/test/test_importlib/metadata/data/__init__.py create mode 100644 Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl create mode 100644 Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg create mode 100644 Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl create mode 100644 Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py create mode 100644 Lib/test/test_importlib/metadata/data/sources/example/setup.py create mode 100644 Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py create mode 100644 Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml create mode 100644 Lib/test/test_importlib/metadata/fixtures.py create mode 100644 Lib/test/test_importlib/metadata/stubs.py create mode 100644 Lib/test/test_importlib/metadata/test_api.py create mode 100644 Lib/test/test_importlib/metadata/test_main.py create mode 100644 Lib/test/test_importlib/metadata/test_zip.py create mode 100644 Lib/test/test_importlib/resources/test_functional.py create mode 100644 Lib/test/test_importlib/resources/zip.py diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 68d993cac..499da1e04 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -382,6 +382,9 @@ class _ModuleLock: self.waiters.pop() self.wakeup.release() + def locked(self): + return bool(self.count) + def __repr__(self): return f'_ModuleLock({self.name!r}) at {id(self)}' @@ -490,8 +493,7 @@ def _call_with_frames_removed(f, *args, **kwds): def _verbose_message(message, *args, verbosity=1): """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" - # XXX RUSTPYTHON: hasattr check because we might be bootstrapping and we wouldn't have stderr yet - if sys.flags.verbose >= verbosity and hasattr(sys, "stderr"): + if sys.flags.verbose >= verbosity: if not message.startswith(('#', 'import ')): message = '# ' + message print(message.format(*args), file=sys.stderr) @@ -1242,10 +1244,12 @@ def _find_spec(name, path, target=None): """Find a module's spec.""" meta_path = sys.meta_path if meta_path is None: - # PyImport_Cleanup() is running or has been called. raise ImportError("sys.meta_path is None, Python is likely " "shutting down") + # gh-130094: Copy sys.meta_path so that we have a consistent view of the + # list while iterating over it. + meta_path = list(meta_path) if not meta_path: _warnings.warn('sys.meta_path is empty', ImportWarning) @@ -1300,7 +1304,6 @@ def _sanity_check(name, package, level): _ERR_MSG_PREFIX = 'No module named ' -_ERR_MSG = _ERR_MSG_PREFIX + '{!r}' def _find_and_load_unlocked(name, import_): path = None @@ -1310,8 +1313,9 @@ def _find_and_load_unlocked(name, import_): if parent not in sys.modules: _call_with_frames_removed(import_, parent) # Crazy side-effects! - if name in sys.modules: - return sys.modules[name] + module = sys.modules.get(name) + if module is not None: + return module parent_module = sys.modules[parent] try: path = parent_module.__path__ @@ -1319,6 +1323,12 @@ def _find_and_load_unlocked(name, import_): msg = f'{_ERR_MSG_PREFIX}{name!r}; {parent!r} is not a package' raise ModuleNotFoundError(msg, name=name) from None parent_spec = parent_module.__spec__ + if getattr(parent_spec, '_initializing', False): + _call_with_frames_removed(import_, parent) + # Crazy side-effects (again)! + module = sys.modules.get(name) + if module is not None: + return module child = name.rpartition('.')[2] spec = _find_spec(name, path) if spec is None: diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 89ce8c09c..95ce14b2c 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -221,277 +221,7 @@ def _write_atomic(path, data, mode=0o666): _code_type = type(_write_atomic.__code__) - -# Finder/loader utility code ############################################### - -# Magic word to reject .pyc files generated by other Python versions. -# It should change for each incompatible change to the bytecode. -# -# The value of CR and LF is incorporated so if you ever read or write -# a .pyc file in text mode the magic number will be wrong; also, the -# Apple MPW compiler swaps their values, botching string constants. -# -# There were a variety of old schemes for setting the magic number. -# The current working scheme is to increment the previous value by -# 10. -# -# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic -# number also includes a new "magic tag", i.e. a human readable string used -# to represent the magic number in __pycache__ directories. When you change -# the magic number, you must also set a new unique magic tag. Generally this -# can be named after the Python major version of the magic number bump, but -# it can really be anything, as long as it's different than anything else -# that's come before. The tags are included in the following table, starting -# with Python 3.2a0. -# -# Known values: -# Python 1.5: 20121 -# Python 1.5.1: 20121 -# Python 1.5.2: 20121 -# Python 1.6: 50428 -# Python 2.0: 50823 -# Python 2.0.1: 50823 -# Python 2.1: 60202 -# Python 2.1.1: 60202 -# Python 2.1.2: 60202 -# Python 2.2: 60717 -# Python 2.3a0: 62011 -# Python 2.3a0: 62021 -# Python 2.3a0: 62011 (!) -# Python 2.4a0: 62041 -# Python 2.4a3: 62051 -# Python 2.4b1: 62061 -# Python 2.5a0: 62071 -# Python 2.5a0: 62081 (ast-branch) -# Python 2.5a0: 62091 (with) -# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) -# Python 2.5b3: 62101 (fix wrong code: for x, in ...) -# Python 2.5b3: 62111 (fix wrong code: x += yield) -# Python 2.5c1: 62121 (fix wrong lnotab with for loops and -# storing constants that should have been removed) -# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) -# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) -# Python 2.6a1: 62161 (WITH_CLEANUP optimization) -# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) -# Python 2.7a0: 62181 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) -# Python 2.7a0 62191 (introduce SETUP_WITH) -# Python 2.7a0 62201 (introduce BUILD_SET) -# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) -# Python 3000: 3000 -# 3010 (removed UNARY_CONVERT) -# 3020 (added BUILD_SET) -# 3030 (added keyword-only parameters) -# 3040 (added signature annotations) -# 3050 (print becomes a function) -# 3060 (PEP 3115 metaclass syntax) -# 3061 (string literals become unicode) -# 3071 (PEP 3109 raise changes) -# 3081 (PEP 3137 make __file__ and __name__ unicode) -# 3091 (kill str8 interning) -# 3101 (merge from 2.6a0, see 62151) -# 3103 (__file__ points to source file) -# Python 3.0a4: 3111 (WITH_CLEANUP optimization). -# Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT - #3021) -# Python 3.1a1: 3141 (optimize list, set and dict comprehensions: -# change LIST_APPEND and SET_ADD, add MAP_ADD #2183) -# Python 3.1a1: 3151 (optimize conditional branches: -# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE - #4715) -# Python 3.2a1: 3160 (add SETUP_WITH #6101) -# tag: cpython-32 -# Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225) -# tag: cpython-32 -# Python 3.2a3 3180 (add DELETE_DEREF #4617) -# Python 3.3a1 3190 (__class__ super closure changed) -# Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448) -# Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645) -# Python 3.3a2 3220 (changed PEP 380 implementation #14230) -# Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857) -# Python 3.4a1 3250 (evaluate positional default arguments before -# keyword-only defaults #16967) -# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override -# free vars #17853) -# Python 3.4a1 3270 (various tweaks to the __class__ closure #12370) -# Python 3.4a1 3280 (remove implicit class argument) -# Python 3.4a4 3290 (changes to __qualname__ computation #19301) -# Python 3.4a4 3300 (more changes to __qualname__ computation #19301) -# Python 3.4rc2 3310 (alter __qualname__ computation #20625) -# Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176) -# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292) -# Python 3.5b2 3340 (fix dictionary display evaluation order #11205) -# Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400) -# Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286) -# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) -# Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107) -# Python 3.6a2 3370 (16 bit wordcode #26647) -# Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140) -# Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE -# #27095) -# Python 3.6b1 3373 (add BUILD_STRING opcode #27078) -# Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes -# #27985) -# Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL - #27213) -# Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) -# Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) -# Python 3.6rc1 3379 (more thorough __class__ validation #23722) -# Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110) -# Python 3.7a2 3391 (update GET_AITER #31709) -# Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) -# Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) -# Python 3.7b5 3394 (restored docstring as the first stmt in the body; -# this might affected the first line number #32911) -# Python 3.8a1 3400 (move frame block handling to compiler #17611) -# Python 3.8a1 3401 (add END_ASYNC_FOR #33041) -# Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) -# Python 3.8b2 3411 (Reverse evaluation order of key: value in dict -# comprehensions #35224) -# Python 3.8b2 3412 (Swap the position of positional args and positional -# only args in ast.arguments #37593) -# Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) -# Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880) -# Python 3.9a0 3421 (simplified bytecode for with blocks #32949) -# Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387) -# Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156) -# Python 3.9a2 3424 (simplify bytecodes for *value unpacking) -# Python 3.9a2 3425 (simplify bytecodes for **value unpacking) -# Python 3.10a1 3430 (Make 'annotations' future by default) -# Python 3.10a1 3431 (New line number table format -- PEP 626) -# Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202) -# Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0) -# Python 3.10a6 3434 (PEP 634: Structural Pattern Matching) -# Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets). -# Python 3.10b1 3436 (Add GEN_START bytecode #43683) -# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!) -# Python 3.10b1 3438 Safer line number table handling. -# Python 3.10b1 3439 (Add ROT_N) -# Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling) -# Python 3.11a1 3451 (Add CALL_METHOD_KW) -# Python 3.11a1 3452 (drop nlocals from marshaled code objects) -# Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds) -# Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693) -# Python 3.11a1 3455 (add MAKE_CELL bpo-43693) -# Python 3.11a1 3456 (interleave cell args bpo-43693) -# Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693) -# Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD) -# Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions) -# Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530) -# Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards) -# Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change -# MATCH_CLASS and MATCH_KEYS, and add COPY) -# Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the -# active exception) -# Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*/INPLACE_* into -# BINARY_OP) -# Python 3.11a3 3465 (Add COPY_FREE_VARS opcode) -# Python 3.11a4 3466 (bpo-45292: PEP-654 except*) -# Python 3.11a4 3467 (Change CALL_xxx opcodes) -# Python 3.11a4 3468 (Add SEND opcode) -# Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info) -# Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti) -# Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE) -# Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP) -# Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes) -# Python 3.11a4 3474 (Add RESUME opcode) -# Python 3.11a5 3475 (Add RETURN_GENERATOR opcode) -# Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode) -# Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and -# ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP) -# Python 3.11a5 3478 (New CALL opcodes) -# Python 3.11a5 3479 (Add PUSH_NULL opcode) -# Python 3.11a5 3480 (New CALL opcodes, second iteration) -# Python 3.11a5 3481 (Use inline cache for BINARY_OP) -# Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL) -# Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR) -# Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and -# STORE_ATTR) -# Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE) -# Python 3.11a6 3486 (Use inline caching for PRECALL and CALL) -# Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism) -# Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL) -# Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE) -# Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH) -# Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH, -# add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual) -# Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) -# Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) -# Python 3.11a7 3494 (New location info table) -# Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626) -# Python 3.12a1 3500 (Remove PRECALL opcode) -# Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) -# Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) -# Python 3.12a1 3503 (Shrink LOAD_METHOD cache) -# Python 3.12a1 3504 (Merge LOAD_METHOD back into LOAD_ATTR) -# Python 3.12a1 3505 (Specialization/Cache for FOR_ITER) -# Python 3.12a1 3506 (Add BINARY_SLICE and STORE_SLICE instructions) -# Python 3.12a1 3507 (Set lineno of module's RESUME to 0) -# Python 3.12a1 3508 (Add CLEANUP_THROW) -# Python 3.12a1 3509 (Conditional jumps only jump forward) -# Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack) -# Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction) -# Python 3.12a2 3512 (Remove all unused consts from code objects) -# Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR) -# Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE) -# Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) -# Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) -# Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) -# Python 3.12a6 3518 (Add RETURN_CONST instruction) -# Python 3.12a6 3519 (Modify SEND instruction) -# Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) -# Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) -# Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) -# Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) -# Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) -# Python 3.12b1 3525 (Shrink the CALL caches) -# Python 3.12b1 3526 (Add instrumentation support) -# Python 3.12b1 3527 (Add LOAD_SUPER_ATTR) -# Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) -# Python 3.12b1 3529 (Inline list/dict/set comprehensions) -# Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches) -# Python 3.12b1 3531 (Add PEP 695 changes) -# Python 3.13a1 3550 (Plugin optimizer support) -# Python 3.13a1 3551 (Compact superinstructions) -# Python 3.13a1 3552 (Remove LOAD_FAST__LOAD_CONST and LOAD_CONST__LOAD_FAST) -# Python 3.13a1 3553 (Add SET_FUNCTION_ATTRIBUTE) -# Python 3.13a1 3554 (more efficient bytecodes for f-strings) -# Python 3.13a1 3555 (generate specialized opcodes metadata from bytecodes.c) -# Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op) -# Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) -# Python 3.13a1 3558 (Reorder the stack items for CALL) -# Python 3.13a1 3559 (Generate opcode IDs from bytecodes.c) -# Python 3.13a1 3560 (Add RESUME_CHECK instruction) -# Python 3.13a1 3561 (Add cache entry to branch instructions) -# Python 3.13a1 3562 (Assign opcode IDs for internal ops in separate range) -# Python 3.13a1 3563 (Add CALL_KW and remove KW_NAMES) -# Python 3.13a1 3564 (Removed oparg from YIELD_VALUE, changed oparg values of RESUME) -# Python 3.13a1 3565 (Oparg of YIELD_VALUE indicates whether it is in a yield-from) -# Python 3.13a1 3566 (Emit JUMP_NO_INTERRUPT instead of JUMP for non-loop no-lineno cases) -# Python 3.13a1 3567 (Reimplement line number propagation by the compiler) -# Python 3.13a1 3568 (Change semantics of END_FOR) -# Python 3.13a5 3569 (Specialize CONTAINS_OP) -# Python 3.13a6 3570 (Add __firstlineno__ class attribute) -# Python 3.13b1 3571 (Fix miscompilation of private names in generic classes) - -# Python 3.14 will start with 3600 - -# Please don't copy-paste the same pre-release tag for new entries above!!! -# You should always use the *upcoming* tag. For example, if 3.12a6 came out -# a week ago, I should put "Python 3.12a7" next to my new magic number. - -# MAGIC must change whenever the bytecode emitted by the compiler may no -# longer be understood by older implementations of the eval loop (usually -# due to the addition of new opcodes). -# -# Starting with Python 3.11, Python 3.n starts with magic number 2900+50n. -# -# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array -# in PC/launcher.c must also be updated. - -MAGIC_NUMBER = (2996).to_bytes(2, 'little') + b'\r\n' - -_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c +MAGIC_NUMBER = _imp.pyc_magic_number_token.to_bytes(4, 'little') _PYCACHE = '__pycache__' _OPT = 'opt-' @@ -983,6 +713,12 @@ class WindowsRegistryFinder: @classmethod def find_spec(cls, fullname, path=None, target=None): + _warnings.warn('importlib.machinery.WindowsRegistryFinder is ' + 'deprecated; use site configuration instead. ' + 'Future versions of Python may not enable this ' + 'finder by default.', + DeprecationWarning, stacklevel=2) + filepath = cls._search_registry(fullname) if filepath is None: return None @@ -1131,7 +867,7 @@ class SourceLoader(_LoaderBasics): _imp.check_hash_based_pycs == 'always')): source_bytes = self.get_data(source_path) source_hash = _imp.source_hash( - _RAW_MAGIC_NUMBER, + _imp.pyc_magic_number_token, source_bytes, ) _validate_hash_pyc(data, source_hash, fullname, @@ -1160,7 +896,7 @@ class SourceLoader(_LoaderBasics): source_mtime is not None): if hash_based: if source_hash is None: - source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER, + source_hash = _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) data = _code_to_hash_pyc(code_object, source_hash, check_source) else: @@ -1505,7 +1241,7 @@ class PathFinder: if path == '': try: path = _os.getcwd() - except FileNotFoundError: + except (FileNotFoundError, PermissionError): # Don't cache the failure as the cwd can easily change to # a valid directory later on. return None diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 37fef357f..1e47495f6 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -13,9 +13,6 @@ except ImportError: _frozen_importlib_external = _bootstrap_external from ._abc import Loader import abc -import warnings - -from .resources import abc as _resources_abc __all__ = [ @@ -25,19 +22,6 @@ __all__ = [ ] -def __getattr__(name): - """ - For backwards compatibility, continue to make names - from _resources_abc available through this module. #93963 - """ - if name in _resources_abc.__all__: - obj = getattr(_resources_abc, name) - warnings._deprecated(f"{__name__}.{name}", remove=(3, 14)) - globals()[name] = obj - return obj - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') - - def _register(abstract_cls, *classes): for cls in classes: abstract_cls.register(cls) @@ -80,10 +64,13 @@ _register(PathEntryFinder, machinery.FileFinder) class ResourceLoader(Loader): """Abstract base class for loaders which can return data from their - back-end storage. + back-end storage to facilitate reading data to perform an import. This ABC represents one of the optional protocols specified by PEP 302. + For directly loading resources, use TraversableResources instead. This class + primarily exists for backwards compatibility with other ABCs in this module. + """ @abc.abstractmethod @@ -215,6 +202,10 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo def path_mtime(self, path): """Return the (int) modification time for the path (str).""" + import warnings + warnings.warn('SourceLoader.path_mtime is deprecated in favour of ' + 'SourceLoader.path_stats().', + DeprecationWarning, stacklevel=2) if self.path_stats.__func__ is SourceLoader.path_stats: raise OSError return int(self.path_stats(path)['mtime']) diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index fbd30b159..63d726445 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -3,9 +3,11 @@ from ._bootstrap import ModuleSpec from ._bootstrap import BuiltinImporter from ._bootstrap import FrozenImporter -from ._bootstrap_external import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES, - OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES, - EXTENSION_SUFFIXES) +from ._bootstrap_external import ( + SOURCE_SUFFIXES, BYTECODE_SUFFIXES, EXTENSION_SUFFIXES, + DEBUG_BYTECODE_SUFFIXES as _DEBUG_BYTECODE_SUFFIXES, + OPTIMIZED_BYTECODE_SUFFIXES as _OPTIMIZED_BYTECODE_SUFFIXES +) from ._bootstrap_external import WindowsRegistryFinder from ._bootstrap_external import PathFinder from ._bootstrap_external import FileFinder @@ -19,3 +21,30 @@ from ._bootstrap_external import NamespaceLoader def all_suffixes(): """Returns a list of all recognized module suffixes for this process""" return SOURCE_SUFFIXES + BYTECODE_SUFFIXES + EXTENSION_SUFFIXES + + +__all__ = ['AppleFrameworkLoader', 'BYTECODE_SUFFIXES', 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', 'FileFinder', 'FrozenImporter', 'ModuleSpec', + 'NamespaceLoader', 'OPTIMIZED_BYTECODE_SUFFIXES', 'PathFinder', + 'SOURCE_SUFFIXES', 'SourceFileLoader', 'SourcelessFileLoader', + 'WindowsRegistryFinder', 'all_suffixes'] + + +def __getattr__(name): + import warnings + + if name == 'DEBUG_BYTECODE_SUFFIXES': + warnings.warn('importlib.machinery.DEBUG_BYTECODE_SUFFIXES is ' + 'deprecated; use importlib.machinery.BYTECODE_SUFFIXES ' + 'instead.', + DeprecationWarning, stacklevel=2) + return _DEBUG_BYTECODE_SUFFIXES + elif name == 'OPTIMIZED_BYTECODE_SUFFIXES': + warnings.warn('importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES is ' + 'deprecated; use importlib.machinery.BYTECODE_SUFFIXES ' + 'instead.', + DeprecationWarning, stacklevel=2) + return _OPTIMIZED_BYTECODE_SUFFIXES + + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index d9128266a..4e9014c45 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -77,12 +77,12 @@ def resolve(cand: Optional[Anchor]) -> types.ModuleType: return cast(types.ModuleType, cand) -@resolve.register(str) # TODO: RUSTPYTHON; manual type annotation +@resolve.register def _(cand: str) -> types.ModuleType: return importlib.import_module(cand) -@resolve.register(type(None)) # TODO: RUSTPYTHON; manual type annotation +@resolve.register def _(cand: None) -> types.ModuleType: return resolve(_infer_caller().f_globals['__name__']) @@ -183,7 +183,7 @@ def _(path): @contextlib.contextmanager def _temp_path(dir: tempfile.TemporaryDirectory): """ - Wrap tempfile.TemporyDirectory to return a pathlib object. + Wrap tempfile.TemporaryDirectory to return a pathlib object. """ with dir as result: yield pathlib.Path(result) diff --git a/Lib/importlib/resources/_legacy.py b/Lib/importlib/resources/_legacy.py deleted file mode 100644 index b1ea8105d..000000000 --- a/Lib/importlib/resources/_legacy.py +++ /dev/null @@ -1,120 +0,0 @@ -import functools -import os -import pathlib -import types -import warnings - -from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any - -from . import _common - -Package = Union[types.ModuleType, str] -Resource = str - - -def deprecated(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - warnings.warn( - f"{func.__name__} is deprecated. Use files() instead. " - "Refer to https://importlib-resources.readthedocs.io" - "/en/latest/using.html#migrating-from-legacy for migration advice.", - DeprecationWarning, - stacklevel=2, - ) - return func(*args, **kwargs) - - return wrapper - - -def normalize_path(path: Any) -> str: - """Normalize a path by ensuring it is a string. - - If the resulting string contains path separators, an exception is raised. - """ - str_path = str(path) - parent, file_name = os.path.split(str_path) - if parent: - raise ValueError(f'{path!r} must be only a file name') - return file_name - - -@deprecated -def open_binary(package: Package, resource: Resource) -> BinaryIO: - """Return a file-like object opened for binary reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open('rb') - - -@deprecated -def read_binary(package: Package, resource: Resource) -> bytes: - """Return the binary contents of the resource.""" - return (_common.files(package) / normalize_path(resource)).read_bytes() - - -@deprecated -def open_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> TextIO: - """Return a file-like object opened for text reading of the resource.""" - return (_common.files(package) / normalize_path(resource)).open( - 'r', encoding=encoding, errors=errors - ) - - -@deprecated -def read_text( - package: Package, - resource: Resource, - encoding: str = 'utf-8', - errors: str = 'strict', -) -> str: - """Return the decoded string of the resource. - - The decoding-related arguments have the same semantics as those of - bytes.decode(). - """ - with open_text(package, resource, encoding, errors) as fp: - return fp.read() - - -@deprecated -def contents(package: Package) -> Iterable[str]: - """Return an iterable of entries in `package`. - - Note that not all entries are resources. Specifically, directories are - not considered resources. Use `is_resource()` on each entry returned here - to check if it is a resource or not. - """ - return [path.name for path in _common.files(package).iterdir()] - - -@deprecated -def is_resource(package: Package, name: str) -> bool: - """True if `name` is a resource inside `package`. - - Directories are *not* resources. - """ - resource = normalize_path(name) - return any( - traversable.name == resource and traversable.is_file() - for traversable in _common.files(package).iterdir() - ) - - -@deprecated -def path( - package: Package, - resource: Resource, -) -> ContextManager[pathlib.Path]: - """A context manager providing a file path object to the resource. - - If the resource does not already exist on its own on the file system, - a temporary file will be created. If the file was created, the file - will be deleted upon exiting the context manager (no exception is - raised if the file was deleted prior to the context manager - exiting). - """ - return _common.as_file(_common.files(package) / normalize_path(resource)) diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 284206b62..2b564e9b5 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -5,7 +5,6 @@ from ._bootstrap import _resolve_name from ._bootstrap import spec_from_loader from ._bootstrap import _find_spec from ._bootstrap_external import MAGIC_NUMBER -from ._bootstrap_external import _RAW_MAGIC_NUMBER from ._bootstrap_external import cache_from_source from ._bootstrap_external import decode_source from ._bootstrap_external import source_from_cache @@ -18,7 +17,7 @@ import types def source_hash(source_bytes): "Return the hash of *source_bytes* as used in hash-based pyc files." - return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes) + return _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) def resolve_name(name, package): @@ -272,3 +271,9 @@ class LazyLoader(Loader): loader_state['is_loading'] = False module.__spec__.loader_state = loader_state module.__class__ = _LazyModule + + +__all__ = ['LazyLoader', 'Loader', 'MAGIC_NUMBER', + 'cache_from_source', 'decode_source', 'find_spec', + 'module_from_spec', 'resolve_name', 'source_from_cache', + 'source_hash', 'spec_from_file_location', 'spec_from_loader'] diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 4031bfaeb..8c7a4f765 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -162,7 +162,7 @@ class BaseLockTests(BaseTestCase): self.assertFalse(result[0]) lock.release() - @unittest.skip('TODO: RUSTPYTHON; sometimes hangs') + @unittest.skip("TODO: RUSTPYTHON; sometimes hangs") def test_acquire_contended(self): lock = self.locktype() lock.acquire() @@ -210,7 +210,7 @@ class BaseLockTests(BaseTestCase): with Bunch(f, 1): pass - @unittest.skip('TODO: RUSTPYTHON; sometimes hangs') + @unittest.skip("TODO: RUSTPYTHON; sometimes hangs") def test_thread_leak(self): # The lock shouldn't leak a Thread instance when used from a foreign # (non-threading) thread. @@ -334,6 +334,26 @@ class RLockTests(BaseLockTests): """ Tests for recursive locks. """ + def test_repr_count(self): + # see gh-134322: check that count values are correct: + # when a rlock is just created, + # in a second thread when rlock is acquired in the main thread. + lock = self.locktype() + self.assertIn("count=0", repr(lock)) + self.assertIn("') self.assertIsNone(spec) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_not_using_frozen(self): finder = self.machinery.FrozenImporter with import_helper.frozen_modules(enabled=False): diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index b1eb399d9..613276348 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -3,11 +3,8 @@ from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') from test.support import captured_stdout, import_helper, STDLIB_DIR -import _imp import contextlib -import marshal import os.path -import sys import types import unittest import warnings @@ -64,7 +61,7 @@ class ExecModuleTests(abc.LoaderTests): module.main() self.assertTrue(module.initialized) - self.assertTrue(hasattr(module, '__spec__')) + self.assertHasAttr(module, '__spec__') self.assertEqual(module.__spec__.origin, 'frozen') return module, stdout.getvalue() @@ -75,7 +72,7 @@ class ExecModuleTests(abc.LoaderTests): for attr, value in check.items(): self.assertEqual(getattr(module, attr), value) self.assertEqual(output, 'Hello world!\n') - self.assertTrue(hasattr(module, '__spec__')) + self.assertHasAttr(module, '__spec__') self.assertEqual(module.__spec__.loader_state.origname, name) @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON") @@ -92,7 +89,6 @@ class ExecModuleTests(abc.LoaderTests): self.assertEqual(output, 'Hello world!\n') self.assertEqual(module.__spec__.loader_state.origname, name) - @unittest.skipIf(sys.platform == 'win32', "TODO:RUSTPYTHON Flaky on Windows") def test_lacking_parent(self): name = '__phello__.spam' with util.uncache('__phello__'): @@ -141,7 +137,7 @@ class InspectLoaderTests: exec(code, mod.__dict__) with captured_stdout() as stdout: mod.main() - self.assertTrue(hasattr(mod, 'initialized')) + self.assertHasAttr(mod, 'initialized') self.assertEqual(stdout.getvalue(), 'Hello world!\n') def test_get_source(self): diff --git a/Lib/test/test_importlib/import_/test___loader__.py b/Lib/test/test_importlib/import_/test___loader__.py index a14163919..858b37eff 100644 --- a/Lib/test/test_importlib/import_/test___loader__.py +++ b/Lib/test/test_importlib/import_/test___loader__.py @@ -1,8 +1,5 @@ from importlib import machinery -import sys -import types import unittest -import warnings from test.test_importlib import util diff --git a/Lib/test/test_importlib/import_/test_caching.py b/Lib/test/test_importlib/import_/test_caching.py index aedf0fd4f..718e7d041 100644 --- a/Lib/test/test_importlib/import_/test_caching.py +++ b/Lib/test/test_importlib/import_/test_caching.py @@ -78,7 +78,7 @@ class ImportlibUseCache(UseCache, unittest.TestCase): with self.create_mock('pkg.__init__', 'pkg.module') as importer: with util.import_state(meta_path=[importer]): module = self.__import__('pkg.module') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(id(module.module), id(sys.modules['pkg.module'])) @@ -88,7 +88,7 @@ class ImportlibUseCache(UseCache, unittest.TestCase): with self.create_mock('pkg.__init__', 'pkg.module') as importer: with util.import_state(meta_path=[importer]): module = self.__import__('pkg', fromlist=['module']) - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(id(module.module), id(sys.modules['pkg.module'])) diff --git a/Lib/test/test_importlib/import_/test_fromlist.py b/Lib/test/test_importlib/import_/test_fromlist.py index 4b4b9bc3f..feccc7be0 100644 --- a/Lib/test/test_importlib/import_/test_fromlist.py +++ b/Lib/test/test_importlib/import_/test_fromlist.py @@ -63,7 +63,7 @@ class HandlingFromlist: with util.import_state(meta_path=[importer]): module = self.__import__('module', fromlist=['non_existent']) self.assertEqual(module.__name__, 'module') - self.assertFalse(hasattr(module, 'non_existent')) + self.assertNotHasAttr(module, 'non_existent') def test_module_from_package(self): # [module] @@ -71,7 +71,7 @@ class HandlingFromlist: with util.import_state(meta_path=[importer]): module = self.__import__('pkg', fromlist=['module']) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(module.module.__name__, 'pkg.module') def test_nonexistent_from_package(self): @@ -79,7 +79,7 @@ class HandlingFromlist: with util.import_state(meta_path=[importer]): module = self.__import__('pkg', fromlist=['non_existent']) self.assertEqual(module.__name__, 'pkg') - self.assertFalse(hasattr(module, 'non_existent')) + self.assertNotHasAttr(module, 'non_existent') def test_module_from_package_triggers_ModuleNotFoundError(self): # If a submodule causes an ModuleNotFoundError because it tries @@ -107,7 +107,7 @@ class HandlingFromlist: mock['pkg'].__all__ = ['module'] module = self.__import__('pkg', fromlist=fromlist) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(module.module.__name__, 'pkg.module') def test_using_star(self): @@ -125,8 +125,8 @@ class HandlingFromlist: mock['pkg'].__all__ = ['module1'] module = self.__import__('pkg', fromlist=['module2', '*']) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module1')) - self.assertTrue(hasattr(module, 'module2')) + self.assertHasAttr(module, 'module1') + self.assertHasAttr(module, 'module2') self.assertEqual(module.module1.__name__, 'pkg.module1') self.assertEqual(module.module2.__name__, 'pkg.module2') @@ -136,7 +136,7 @@ class HandlingFromlist: importer['pkg'].__all__ = ['non_existent'] module = self.__import__('pkg', fromlist=['*']) self.assertEqual(module.__name__, 'pkg') - self.assertFalse(hasattr(module, 'non_existent')) + self.assertNotHasAttr(module, 'non_existent') def test_star_in_all(self): with util.mock_spec('pkg.__init__') as importer: @@ -144,7 +144,7 @@ class HandlingFromlist: importer['pkg'].__all__ = ['*'] module = self.__import__('pkg', fromlist=['*']) self.assertEqual(module.__name__, 'pkg') - self.assertFalse(hasattr(module, '*')) + self.assertNotHasAttr(module, '*') def test_invalid_type(self): with util.mock_spec('pkg.__init__') as importer: diff --git a/Lib/test/test_importlib/import_/test_meta_path.py b/Lib/test/test_importlib/import_/test_meta_path.py index 8689017ba..4c00f6068 100644 --- a/Lib/test/test_importlib/import_/test_meta_path.py +++ b/Lib/test/test_importlib/import_/test_meta_path.py @@ -43,7 +43,7 @@ class CallingOrder: self.assertIsNone(importlib._bootstrap._find_spec('nothing', None)) self.assertEqual(len(w), 1) - self.assertTrue(issubclass(w[-1].category, ImportWarning)) + self.assertIsSubclass(w[-1].category, ImportWarning) (Frozen_CallingOrder, diff --git a/Lib/test/test_importlib/import_/test_packages.py b/Lib/test/test_importlib/import_/test_packages.py index eb0831f7d..0c29d6083 100644 --- a/Lib/test/test_importlib/import_/test_packages.py +++ b/Lib/test/test_importlib/import_/test_packages.py @@ -1,7 +1,6 @@ from test.test_importlib import util import sys import unittest -from test import support from test.support import import_helper diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py index 89b52fbd1..79e0bdca9 100644 --- a/Lib/test/test_importlib/import_/test_path.py +++ b/Lib/test/test_importlib/import_/test_path.py @@ -1,3 +1,4 @@ +from test.support import os_helper from test.test_importlib import util importlib = util.import_importlib('importlib') @@ -80,7 +81,7 @@ class FinderTests: self.assertIsNone(self.find('os')) self.assertIsNone(sys.path_importer_cache[path_entry]) self.assertEqual(len(w), 1) - self.assertTrue(issubclass(w[-1].category, ImportWarning)) + self.assertIsSubclass(w[-1].category, ImportWarning) def test_path_importer_cache_empty_string(self): # The empty string should create a finder using the cwd. @@ -153,6 +154,32 @@ class FinderTests: # Do not want FileNotFoundError raised. self.assertIsNone(self.machinery.PathFinder.find_spec('whatever')) + @os_helper.skip_unless_working_chmod + def test_permission_error_cwd(self): + # gh-115911: Test that an unreadable CWD does not break imports, in + # particular during early stages of interpreter startup. + + def noop_hook(*args): + raise ImportError + + with ( + os_helper.temp_dir() as new_dir, + os_helper.save_mode(new_dir), + os_helper.change_cwd(new_dir), + util.import_state(path=[''], path_hooks=[noop_hook]), + ): + # chmod() is done here (inside the 'with' block) because the order + # of teardown operations cannot be the reverse of setup order. See + # https://github.com/python/cpython/pull/116131#discussion_r1739649390 + try: + os.chmod(new_dir, 0o000) + except OSError: + self.skipTest("platform does not allow " + "changing mode of the cwd") + + # Do not want PermissionError raised. + self.assertIsNone(self.machinery.PathFinder.find_spec('whatever')) + def test_invalidate_caches_finders(self): # Finders with an invalidate_caches() method have it called. class FakeFinder: diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index 99c24f1fd..1549cbe96 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -81,7 +81,7 @@ class RelativeImports: self.__import__('pkg') # For __import__(). module = self.__import__('', global_, fromlist=['mod2'], level=1) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'mod2')) + self.assertHasAttr(module, 'mod2') self.assertEqual(module.mod2.attr, 'pkg.mod2') self.relative_import_test(create, globals_, callback) @@ -107,7 +107,7 @@ class RelativeImports: module = self.__import__('', global_, fromlist=['module'], level=1) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'module')) + self.assertHasAttr(module, 'module') self.assertEqual(module.module.attr, 'pkg.module') self.relative_import_test(create, globals_, callback) @@ -131,7 +131,7 @@ class RelativeImports: module = self.__import__('', global_, fromlist=['subpkg2'], level=2) self.assertEqual(module.__name__, 'pkg') - self.assertTrue(hasattr(module, 'subpkg2')) + self.assertHasAttr(module, 'subpkg2') self.assertEqual(module.subpkg2.attr, 'pkg.subpkg2.__init__') self.relative_import_test(create, globals_, callback) @@ -223,6 +223,21 @@ class RelativeImports: self.__import__('sys', {'__package__': '', '__spec__': None}, level=1) + def test_malicious_relative_import(self): + # https://github.com/python/cpython/issues/134100 + # Test to make sure UAF bug with error msg doesn't come back to life + import sys + loooong = "".ljust(0x23000, "b") + name = f"a.{loooong}.c" + + with util.uncache(name): + sys.modules[name] = {} + with self.assertRaisesRegex( + KeyError, + r"'a\.b+' not in sys\.modules as expected" + ): + __import__(f"{loooong}.c", {"__package__": "a"}, level=1) + (Frozen_RelativeImports, Source_RelativeImports diff --git a/Lib/test/test_importlib/metadata/__init__.py b/Lib/test/test_importlib/metadata/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/Lib/test/test_importlib/metadata/__init__.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_importlib/metadata/_context.py b/Lib/test/test_importlib/metadata/_context.py new file mode 100644 index 000000000..8a53eb55d --- /dev/null +++ b/Lib/test/test_importlib/metadata/_context.py @@ -0,0 +1,13 @@ +import contextlib + + +# from jaraco.context 4.3 +class suppress(contextlib.suppress, contextlib.ContextDecorator): + """ + A version of contextlib.suppress with decorator support. + + >>> @suppress(KeyError) + ... def key_error(): + ... {}[''] + >>> key_error() + """ diff --git a/Lib/test/test_importlib/metadata/_path.py b/Lib/test/test_importlib/metadata/_path.py new file mode 100644 index 000000000..b3cfb9cd5 --- /dev/null +++ b/Lib/test/test_importlib/metadata/_path.py @@ -0,0 +1,115 @@ +# from jaraco.path 3.7 + +import functools +import pathlib +from typing import Dict, Protocol, Union +from typing import runtime_checkable + + +class Symlink(str): + """ + A string indicating the target of a symlink. + """ + + +FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] # type: ignore + + +@runtime_checkable +class TreeMaker(Protocol): + def __truediv__(self, *args, **kwargs): ... # pragma: no cover + + def mkdir(self, **kwargs): ... # pragma: no cover + + def write_text(self, content, **kwargs): ... # pragma: no cover + + def write_bytes(self, content): ... # pragma: no cover + + def symlink_to(self, target): ... # pragma: no cover + + +def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: + return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore + + +def build( + spec: FilesSpec, + prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore +): + """ + Build a set of files/directories, as described by the spec. + + Each key represents a pathname, and the value represents + the content. Content may be a nested directory. + + >>> spec = { + ... 'README.txt': "A README file", + ... "foo": { + ... "__init__.py": "", + ... "bar": { + ... "__init__.py": "", + ... }, + ... "baz.py": "# Some code", + ... "bar.py": Symlink("baz.py"), + ... }, + ... "bing": Symlink("foo"), + ... } + >>> target = getfixture('tmp_path') + >>> build(spec, target) + >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') + '# Some code' + >>> target.joinpath('bing/bar.py').read_text(encoding='utf-8') + '# Some code' + """ + for name, contents in spec.items(): + create(contents, _ensure_tree_maker(prefix) / name) + + +@functools.singledispatch +def create(content: Union[str, bytes, FilesSpec], path): + path.mkdir(exist_ok=True) + build(content, prefix=path) # type: ignore + + +@create.register +def _(content: bytes, path): + path.write_bytes(content) + + +@create.register +def _(content: str, path): + path.write_text(content, encoding='utf-8') + + +@create.register +def _(content: Symlink, path): + path.symlink_to(content) + + +class Recording: + """ + A TreeMaker object that records everything that would be written. + + >>> r = Recording() + >>> build({'foo': {'foo1.txt': 'yes'}, 'bar.txt': 'abc'}, r) + >>> r.record + ['foo/foo1.txt', 'bar.txt'] + """ + + def __init__(self, loc=pathlib.PurePosixPath(), record=None): + self.loc = loc + self.record = record if record is not None else [] + + def __truediv__(self, other): + return Recording(self.loc / other, self.record) + + def write_text(self, content, **kwargs): + self.record.append(str(self.loc)) + + write_bytes = write_text + + def mkdir(self, **kwargs): + return + + def symlink_to(self, target): + pass diff --git a/Lib/test/test_importlib/metadata/data/__init__.py b/Lib/test/test_importlib/metadata/data/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/__init__.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl b/Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..641ab07f7aadd5c3ffe199b1e397b84504444994 GIT binary patch literal 1455 zcmWIWW@Zs#U|`^2sH#5hC#89?LJ`OVVPPOntw_u*$Vt_YkI&4@EQycTE2#AL^bJ1Y zd*;mL3tJuqF*Gf@GU?JH8`iH^x{lmwniEp0#}EKFaSqT#mVX%$)j*SW1F-_aL|r38 zJwqeCl+5B1-ORkSe0^Wn5Jwls5Xac=ja-KeL|ngr7ya2Na>+2-+_Beb<+qte)Tv+?!=mXP3IU>yCSok_B-aII-Kw3 zQMDMLgF#ppkHf<~TwQ&*&wKcuy~yjWt99!gle zr#}12zzYT!jLohYUwp!K>b(BNuR7OwHFUksp7hxiq@k(h`;^PmXM2#w3X^MRPoMX` z>U(DOlSfQw;W@=u^`JJ;IgUWAfyY^?c_l@a@df#rc_qbqB^4#ze&^0>pF8i_tM8|G zN=HMp?`hFmk2AU_JVQ5xdbDm>QzUHsc}LJj?qWtX3-6|I25|u`;s9b*JQkMZ7sThJ zmZj!^Z1aYg{S@8xHoufE=rJ-dTm)h{Jcb3iI{OE?l%CmH)a1axaN%A)u3Q2l zE(Z%%TzIplh-=e{bsROdT|LH`74jc0G1}d-ie(BgQ)O((bmtPEAigxe^3+zD^2ZPA z=Kj_@llnHb%pqs>={~b{JFAWU-FsQLpa0z(9XLnn{J!+d2H)mi39tFM zWw~J?v;DQj2W5<}*`3O4x;giZz4dIBjE_o=poHl8P!xz%}R z^2PdUhwx4OykW7N0@LpsPXda2KAh-#t~vYAon3*GEN%;5X|j5I`Pcv4f`xLoSdG>m z=t|fwwtn6&&3ktpt-?;-J{BOG-L!7y`!yP=+rIT|ef?5o>krldZ$>5&X54uh7~Ej6 zq!C16$=S#TV9UA??FDpj9zYni2#-I~9cJVf#soY@LK1&~H!B-RJsS|72YTxiGl&NO07LVy literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg b/Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg new file mode 100644 index 0000000000000000000000000000000000000000..cdb298a19b09d360d18f72a91e9dd70d7d7f54ec GIT binary patch literal 1497 zcmWIWW@Zs#U|`^2c;|fF@3nF0krp8DC=d$*v8%hguBV@yzkYx>gb~~8$aTm-!1cSO ztKNj9U1!!ka0&=sxZ<^c4f@#kCS*N+n7GL42YfZ! zEk5xYo$43J*3tWYGog!X+Wp$0%DLy;7$(q4-Im54c05E zC@DSV$kpt?s$Pc9T!_{7cXah`>i&=P4|_F+Kpf9t5RHa za$M}sCdvd)D?c%>-)L{RYwfgc?`B2cv%3AJhgrnPA#S40>GXa36wF*N9Z0I@t{X7}-NOjPvdHdANi9gtOG(X3u8hyg%*!qYIq)$P1FG&RzN!bcfw~-l zSQ1%xYF7CNi(Cm9!wASN{?g`J(4WS;b8`cyF z8-Lyrw2`}*5zU6XDV#xEK$AIuSPa>QlKg`BoYb<^9FW7jA%;CgcOSCvP!dX2W`S;T zVp=MSkE(!4OG@)#g(A=%5EcgF)QZI1f}B+S`1s7c%#!$cy@E<_Pv77ZzGu!{zOdy{ z5JS_VE0ZoQvSIytrR&I@ra2*{dT2gSDqd(m7iiKmAO?jM!o-5gB$9^ zB^mj7y2fTei;|6>J$69*1_n8CRa9r%0Q~F|js{Hj!lsY`)J@ zQc8Oq-tYt+X9_ybF(v)e?xXXfCeDbSFnQOcg&Zk-O+Qj6gv^>bdC~kSlOkglO`kk7 zJZfQdVxm}n+M@Zh=B?wux=dbv{>dG%t&2B zJp&+2$t*6>&CE;7*Y|Y|addGEas2G7>*?dE<9UVGTUYDcne&^23@#Zz`s97acWaQx zT8}fjCp=9bHJz4xCHX?KfDz5MzpI@lg#c{|1!6fM#%Wu)hpVfP=qX381_c3z1H0GX z;9nzg_m4wKcZZds58V*ju&HNFk+AXS9XpUc4~)G%DhBEQfRCIbeChj-ICy*M3ZtsFSrmBkh=j1cEb;Q7d&Ar$@L#zb9$Vue#oPCb0aL5c71K zRb5sBzwheY+{SQ7@b}`L8Pcy__>+KgTMyzlO=zO+(sU7k_Wpmh%gXI+`S`plJ6=}#w3+U_l2vF`iB zqo-v9ADM30!*%9D@)Xl3zF2Lkwa0c{FP|5G)j?_Q$FqTtW^_tTQu-Ku&G~Ua*rn%X zk01E{s@r+&z`y6NP4SD4uqS+B4)A7V5@E)j^nu|A21^=26f7MfX9Q#uu%&c}eg=jH z#$Y59aHe~7K4X*_)LYQ Vz5s7lHjoBZAZ!3C|H%a60RRCxggXEL literal 0 HcmV?d00001 diff --git a/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py b/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py new file mode 100644 index 000000000..ba73b7433 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py @@ -0,0 +1,2 @@ +def main(): + return 'example' diff --git a/Lib/test/test_importlib/metadata/data/sources/example/setup.py b/Lib/test/test_importlib/metadata/data/sources/example/setup.py new file mode 100644 index 000000000..479488a03 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup + +setup( + name='example', + version='21.12', + license='Apache Software License', + packages=['example'], + entry_points={ + 'console_scripts': ['example = example:main', 'Example=example:main'], + }, +) diff --git a/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py b/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py new file mode 100644 index 000000000..de645c2e8 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py @@ -0,0 +1,2 @@ +def main(): + return "example" diff --git a/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml b/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml new file mode 100644 index 000000000..011f4751f --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +build-backend = 'trampolim' +requires = ['trampolim'] + +[project] +name = 'example2' +version = '1.0.0' + +[project.scripts] +example = 'example2:main' diff --git a/Lib/test/test_importlib/metadata/fixtures.py b/Lib/test/test_importlib/metadata/fixtures.py new file mode 100644 index 000000000..826b1b325 --- /dev/null +++ b/Lib/test/test_importlib/metadata/fixtures.py @@ -0,0 +1,395 @@ +import sys +import copy +import json +import shutil +import pathlib +import textwrap +import functools +import contextlib + +from test.support import import_helper +from test.support import os_helper +from test.support import requires_zlib + +from . import _path +from ._path import FilesSpec + + +try: + from importlib import resources # type: ignore + + getattr(resources, 'files') + getattr(resources, 'as_file') +except (ImportError, AttributeError): + import importlib_resources as resources # type: ignore + + +@contextlib.contextmanager +def tmp_path(): + """ + Like os_helper.temp_dir, but yields a pathlib.Path. + """ + with os_helper.temp_dir() as path: + yield pathlib.Path(path) + + +@contextlib.contextmanager +def install_finder(finder): + sys.meta_path.append(finder) + try: + yield + finally: + sys.meta_path.remove(finder) + + +class Fixtures: + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + +class SiteDir(Fixtures): + def setUp(self): + super().setUp() + self.site_dir = self.fixtures.enter_context(tmp_path()) + + +class OnSysPath(Fixtures): + @staticmethod + @contextlib.contextmanager + def add_sys_path(dir): + sys.path[:0] = [str(dir)] + try: + yield + finally: + sys.path.remove(str(dir)) + + def setUp(self): + super().setUp() + self.fixtures.enter_context(self.add_sys_path(self.site_dir)) + self.fixtures.enter_context(import_helper.isolated_modules()) + + +class SiteBuilder(SiteDir): + def setUp(self): + super().setUp() + for cls in self.__class__.mro(): + with contextlib.suppress(AttributeError): + build_files(cls.files, prefix=self.site_dir) + + +class DistInfoPkg(OnSysPath, SiteBuilder): + files: FilesSpec = { + "distinfo_pkg-1.0.0.dist-info": { + "METADATA": """ + Name: distinfo-pkg + Author: Steven Ma + Version: 1.0.0 + Requires-Dist: wheel >= 1.0 + Requires-Dist: pytest; extra == 'test' + Keywords: sample package + + Once upon a time + There was a distinfo pkg + """, + "RECORD": "mod.py,sha256=abc,20\n", + "entry_points.txt": """ + [entries] + main = mod:main + ns:sub = mod:main + """, + }, + "mod.py": """ + def main(): + print("hello world") + """, + } + + def make_uppercase(self): + """ + Rewrite metadata with everything uppercase. + """ + shutil.rmtree(self.site_dir / "distinfo_pkg-1.0.0.dist-info") + files = copy.deepcopy(DistInfoPkg.files) + info = files["distinfo_pkg-1.0.0.dist-info"] + info["METADATA"] = info["METADATA"].upper() + build_files(files, self.site_dir) + + +class DistInfoPkgEditable(DistInfoPkg): + """ + Package with a PEP 660 direct_url.json. + """ + + some_hash = '524127ce937f7cb65665130c695abd18ca386f60bb29687efb976faa1596fdcc' + files: FilesSpec = { + 'distinfo_pkg-1.0.0.dist-info': { + 'direct_url.json': json.dumps({ + "archive_info": { + "hash": f"sha256={some_hash}", + "hashes": {"sha256": f"{some_hash}"}, + }, + "url": "file:///path/to/distinfo_pkg-1.0.0.editable-py3-none-any.whl", + }) + }, + } + + +class DistInfoPkgWithDot(OnSysPath, SiteBuilder): + files: FilesSpec = { + "pkg_dot-1.0.0.dist-info": { + "METADATA": """ + Name: pkg.dot + Version: 1.0.0 + """, + }, + } + + +class DistInfoPkgWithDotLegacy(OnSysPath, SiteBuilder): + files: FilesSpec = { + "pkg.dot-1.0.0.dist-info": { + "METADATA": """ + Name: pkg.dot + Version: 1.0.0 + """, + }, + "pkg.lot.egg-info": { + "METADATA": """ + Name: pkg.lot + Version: 1.0.0 + """, + }, + } + + +class DistInfoPkgOffPath(SiteBuilder): + files = DistInfoPkg.files + + +class EggInfoPkg(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egginfo_pkg.egg-info": { + "PKG-INFO": """ + Name: egginfo-pkg + Author: Steven Ma + License: Unknown + Version: 1.0.0 + Classifier: Intended Audience :: Developers + Classifier: Topic :: Software Development :: Libraries + Keywords: sample package + Description: Once upon a time + There was an egginfo package + """, + "SOURCES.txt": """ + mod.py + egginfo_pkg.egg-info/top_level.txt + """, + "entry_points.txt": """ + [entries] + main = mod:main + """, + "requires.txt": """ + wheel >= 1.0; python_version >= "2.7" + [test] + pytest + """, + "top_level.txt": "mod\n", + }, + "mod.py": """ + def main(): + print("hello world") + """, + } + + +class EggInfoPkgPipInstalledNoToplevel(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egg_with_module_pkg.egg-info": { + "PKG-INFO": "Name: egg_with_module-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + egg_with_module.py + setup.py + egg_with_module_pkg.egg-info/PKG-INFO + egg_with_module_pkg.egg-info/SOURCES.txt + egg_with_module_pkg.egg-info/top_level.txt + """, + # installed-files.txt is written by pip, and is a strictly more + # accurate source than SOURCES.txt as to the installed contents of + # the package. + "installed-files.txt": """ + ../egg_with_module.py + PKG-INFO + SOURCES.txt + top_level.txt + """, + # missing top_level.txt (to trigger fallback to installed-files.txt) + }, + "egg_with_module.py": """ + def main(): + print("hello world") + """, + } + + +class EggInfoPkgPipInstalledExternalDataFiles(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egg_with_module_pkg.egg-info": { + "PKG-INFO": "Name: egg_with_module-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + egg_with_module.py + setup.py + egg_with_module.json + egg_with_module_pkg.egg-info/PKG-INFO + egg_with_module_pkg.egg-info/SOURCES.txt + egg_with_module_pkg.egg-info/top_level.txt + """, + # installed-files.txt is written by pip, and is a strictly more + # accurate source than SOURCES.txt as to the installed contents of + # the package. + "installed-files.txt": """ + ../../../etc/jupyter/jupyter_notebook_config.d/relative.json + /etc/jupyter/jupyter_notebook_config.d/absolute.json + ../egg_with_module.py + PKG-INFO + SOURCES.txt + top_level.txt + """, + # missing top_level.txt (to trigger fallback to installed-files.txt) + }, + "egg_with_module.py": """ + def main(): + print("hello world") + """, + } + + +class EggInfoPkgPipInstalledNoModules(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egg_with_no_modules_pkg.egg-info": { + "PKG-INFO": "Name: egg_with_no_modules-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + setup.py + egg_with_no_modules_pkg.egg-info/PKG-INFO + egg_with_no_modules_pkg.egg-info/SOURCES.txt + egg_with_no_modules_pkg.egg-info/top_level.txt + """, + # installed-files.txt is written by pip, and is a strictly more + # accurate source than SOURCES.txt as to the installed contents of + # the package. + "installed-files.txt": """ + PKG-INFO + SOURCES.txt + top_level.txt + """, + # top_level.txt correctly reflects that no modules are installed + "top_level.txt": b"\n", + }, + } + + +class EggInfoPkgSourcesFallback(OnSysPath, SiteBuilder): + files: FilesSpec = { + "sources_fallback_pkg.egg-info": { + "PKG-INFO": "Name: sources_fallback-pkg", + # SOURCES.txt is made from the source archive, and contains files + # (setup.py) that are not present after installation. + "SOURCES.txt": """ + sources_fallback.py + setup.py + sources_fallback_pkg.egg-info/PKG-INFO + sources_fallback_pkg.egg-info/SOURCES.txt + """, + # missing installed-files.txt (i.e. not installed by pip) and + # missing top_level.txt (to trigger fallback to SOURCES.txt) + }, + "sources_fallback.py": """ + def main(): + print("hello world") + """, + } + + +class EggInfoFile(OnSysPath, SiteBuilder): + files: FilesSpec = { + "egginfo_file.egg-info": """ + Metadata-Version: 1.0 + Name: egginfo_file + Version: 0.1 + Summary: An example package + Home-page: www.example.com + Author: Eric Haffa-Vee + Author-email: eric@example.coms + License: UNKNOWN + Description: UNKNOWN + Platform: UNKNOWN + """, + } + + +# dedent all text strings before writing +orig = _path.create.registry[str] +_path.create.register(str, lambda content, path: orig(DALS(content), path)) + + +build_files = _path.build + + +def build_record(file_defs): + return ''.join(f'{name},,\n' for name in record_names(file_defs)) + + +def record_names(file_defs): + recording = _path.Recording() + _path.build(file_defs, recording) + return recording.record + + +class FileBuilder: + def unicode_filename(self): + return os_helper.FS_NONASCII or self.skip( + "File system does not support non-ascii." + ) + + +def DALS(str): + "Dedent and left-strip" + return textwrap.dedent(str).lstrip() + + +@requires_zlib() +class ZipFixtures: + root = 'test.test_importlib.metadata.data' + + def _fixture_on_path(self, filename): + pkg_file = resources.files(self.root).joinpath(filename) + file = self.resources.enter_context(resources.as_file(pkg_file)) + assert file.name.startswith('example'), file.name + sys.path.insert(0, str(file)) + self.resources.callback(sys.path.pop, 0) + + def setUp(self): + # Add self.zip_name to the front of sys.path. + self.resources = contextlib.ExitStack() + self.addCleanup(self.resources.close) + + +def parameterize(*args_set): + """Run test method with a series of parameters.""" + + def wrapper(func): + @functools.wraps(func) + def _inner(self): + for args in args_set: + with self.subTest(**args): + func(self, **args) + + return _inner + + return wrapper diff --git a/Lib/test/test_importlib/metadata/stubs.py b/Lib/test/test_importlib/metadata/stubs.py new file mode 100644 index 000000000..e5b011c39 --- /dev/null +++ b/Lib/test/test_importlib/metadata/stubs.py @@ -0,0 +1,10 @@ +import unittest + + +class fake_filesystem_unittest: + """ + Stubbed version of the pyfakefs module + """ + class TestCase(unittest.TestCase): + def setUpPyfakefs(self): + self.skipTest("pyfakefs not available") diff --git a/Lib/test/test_importlib/metadata/test_api.py b/Lib/test/test_importlib/metadata/test_api.py new file mode 100644 index 000000000..2256e0c50 --- /dev/null +++ b/Lib/test/test_importlib/metadata/test_api.py @@ -0,0 +1,323 @@ +import re +import textwrap +import unittest +import warnings +import importlib +import contextlib + +from . import fixtures +from importlib.metadata import ( + Distribution, + PackageNotFoundError, + distribution, + entry_points, + files, + metadata, + requires, + version, +) + + +@contextlib.contextmanager +def suppress_known_deprecation(): + with warnings.catch_warnings(record=True) as ctx: + warnings.simplefilter('default', category=DeprecationWarning) + yield ctx + + +class APITests( + fixtures.EggInfoPkg, + fixtures.EggInfoPkgPipInstalledNoToplevel, + fixtures.EggInfoPkgPipInstalledNoModules, + fixtures.EggInfoPkgPipInstalledExternalDataFiles, + fixtures.EggInfoPkgSourcesFallback, + fixtures.DistInfoPkg, + fixtures.DistInfoPkgWithDot, + fixtures.EggInfoFile, + unittest.TestCase, +): + version_pattern = r'\d+\.\d+(\.\d)?' + + def test_retrieves_version_of_self(self): + pkg_version = version('egginfo-pkg') + assert isinstance(pkg_version, str) + assert re.match(self.version_pattern, pkg_version) + + def test_retrieves_version_of_distinfo_pkg(self): + pkg_version = version('distinfo-pkg') + assert isinstance(pkg_version, str) + assert re.match(self.version_pattern, pkg_version) + + def test_for_name_does_not_exist(self): + with self.assertRaises(PackageNotFoundError): + distribution('does-not-exist') + + def test_name_normalization(self): + names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot' + for name in names: + with self.subTest(name): + assert distribution(name).metadata['Name'] == 'pkg.dot' + + def test_prefix_not_matched(self): + prefixes = 'p', 'pkg', 'pkg.' + for prefix in prefixes: + with self.subTest(prefix): + with self.assertRaises(PackageNotFoundError): + distribution(prefix) + + def test_for_top_level(self): + tests = [ + ('egginfo-pkg', 'mod'), + ('egg_with_no_modules-pkg', ''), + ] + for pkg_name, expect_content in tests: + with self.subTest(pkg_name): + self.assertEqual( + distribution(pkg_name).read_text('top_level.txt').strip(), + expect_content, + ) + + def test_read_text(self): + tests = [ + ('egginfo-pkg', 'mod\n'), + ('egg_with_no_modules-pkg', '\n'), + ] + for pkg_name, expect_content in tests: + with self.subTest(pkg_name): + top_level = [ + path for path in files(pkg_name) if path.name == 'top_level.txt' + ][0] + self.assertEqual(top_level.read_text(), expect_content) + + def test_entry_points(self): + eps = entry_points() + assert 'entries' in eps.groups + entries = eps.select(group='entries') + assert 'main' in entries.names + ep = entries['main'] + self.assertEqual(ep.value, 'mod:main') + self.assertEqual(ep.extras, []) + + def test_entry_points_distribution(self): + entries = entry_points(group='entries') + for entry in ("main", "ns:sub"): + ep = entries[entry] + self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg')) + self.assertEqual(ep.dist.version, "1.0.0") + + def test_entry_points_unique_packages_normalized(self): + """ + Entry points should only be exposed for the first package + on sys.path with a given name (even when normalized). + """ + alt_site_dir = self.fixtures.enter_context(fixtures.tmp_path()) + self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) + alt_pkg = { + "DistInfo_pkg-1.1.0.dist-info": { + "METADATA": """ + Name: distinfo-pkg + Version: 1.1.0 + """, + "entry_points.txt": """ + [entries] + main = mod:altmain + """, + }, + } + fixtures.build_files(alt_pkg, alt_site_dir) + entries = entry_points(group='entries') + assert not any( + ep.dist.name == 'distinfo-pkg' and ep.dist.version == '1.0.0' + for ep in entries + ) + # ns:sub doesn't exist in alt_pkg + assert 'ns:sub' not in entries.names + + def test_entry_points_missing_name(self): + with self.assertRaises(KeyError): + entry_points(group='entries')['missing'] + + def test_entry_points_missing_group(self): + assert entry_points(group='missing') == () + + def test_entry_points_allows_no_attributes(self): + ep = entry_points().select(group='entries', name='main') + with self.assertRaises(AttributeError): + ep.foo = 4 + + def test_metadata_for_this_package(self): + md = metadata('egginfo-pkg') + assert md['author'] == 'Steven Ma' + assert md['LICENSE'] == 'Unknown' + assert md['Name'] == 'egginfo-pkg' + classifiers = md.get_all('Classifier') + assert 'Topic :: Software Development :: Libraries' in classifiers + + def test_missing_key_legacy(self): + """ + Requesting a missing key will still return None, but warn. + """ + md = metadata('distinfo-pkg') + with suppress_known_deprecation(): + assert md['does-not-exist'] is None + + def test_get_key(self): + """ + Getting a key gets the key. + """ + md = metadata('egginfo-pkg') + assert md.get('Name') == 'egginfo-pkg' + + def test_get_missing_key(self): + """ + Requesting a missing key will return None. + """ + md = metadata('distinfo-pkg') + assert md.get('does-not-exist') is None + + @staticmethod + def _test_files(files): + root = files[0].root + for file in files: + assert file.root == root + assert not file.hash or file.hash.value + assert not file.hash or file.hash.mode == 'sha256' + assert not file.size or file.size >= 0 + assert file.locate().exists() + assert isinstance(file.read_binary(), bytes) + if file.name.endswith('.py'): + file.read_text() + + def test_file_hash_repr(self): + util = [p for p in files('distinfo-pkg') if p.name == 'mod.py'][0] + self.assertRegex(repr(util.hash), '') + + def test_files_dist_info(self): + self._test_files(files('distinfo-pkg')) + + def test_files_egg_info(self): + self._test_files(files('egginfo-pkg')) + self._test_files(files('egg_with_module-pkg')) + self._test_files(files('egg_with_no_modules-pkg')) + self._test_files(files('sources_fallback-pkg')) + + def test_version_egg_info_file(self): + self.assertEqual(version('egginfo-file'), '0.1') + + def test_requires_egg_info_file(self): + requirements = requires('egginfo-file') + self.assertIsNone(requirements) + + def test_requires_egg_info(self): + deps = requires('egginfo-pkg') + assert len(deps) == 2 + assert any(dep == 'wheel >= 1.0; python_version >= "2.7"' for dep in deps) + + def test_requires_egg_info_empty(self): + fixtures.build_files( + { + 'requires.txt': '', + }, + self.site_dir.joinpath('egginfo_pkg.egg-info'), + ) + deps = requires('egginfo-pkg') + assert deps == [] + + def test_requires_dist_info(self): + deps = requires('distinfo-pkg') + assert len(deps) == 2 + assert all(deps) + assert 'wheel >= 1.0' in deps + assert "pytest; extra == 'test'" in deps + + def test_more_complex_deps_requires_text(self): + requires = textwrap.dedent( + """ + dep1 + dep2 + + [:python_version < "3"] + dep3 + + [extra1] + dep4 + dep6@ git+https://example.com/python/dep.git@v1.0.0 + + [extra2:python_version < "3"] + dep5 + """ + ) + deps = sorted(Distribution._deps_from_requires_text(requires)) + expected = [ + 'dep1', + 'dep2', + 'dep3; python_version < "3"', + 'dep4; extra == "extra1"', + 'dep5; (python_version < "3") and extra == "extra2"', + 'dep6@ git+https://example.com/python/dep.git@v1.0.0 ; extra == "extra1"', + ] + # It's important that the environment marker expression be + # wrapped in parentheses to avoid the following 'and' binding more + # tightly than some other part of the environment expression. + + assert deps == expected + + def test_as_json(self): + md = metadata('distinfo-pkg').json + assert 'name' in md + assert md['keywords'] == ['sample', 'package'] + desc = md['description'] + assert desc.startswith('Once upon a time\nThere was') + assert len(md['requires_dist']) == 2 + + def test_as_json_egg_info(self): + md = metadata('egginfo-pkg').json + assert 'name' in md + assert md['keywords'] == ['sample', 'package'] + desc = md['description'] + assert desc.startswith('Once upon a time\nThere was') + assert len(md['classifier']) == 2 + + def test_as_json_odd_case(self): + self.make_uppercase() + md = metadata('distinfo-pkg').json + assert 'name' in md + assert len(md['requires_dist']) == 2 + assert md['keywords'] == ['SAMPLE', 'PACKAGE'] + + +class LegacyDots(fixtures.DistInfoPkgWithDotLegacy, unittest.TestCase): + def test_name_normalization(self): + names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot' + for name in names: + with self.subTest(name): + assert distribution(name).metadata['Name'] == 'pkg.dot' + + def test_name_normalization_versionless_egg_info(self): + names = 'pkg.lot', 'pkg_lot', 'pkg-lot', 'pkg..lot', 'Pkg.Lot' + for name in names: + with self.subTest(name): + assert distribution(name).metadata['Name'] == 'pkg.lot' + + +class OffSysPathTests(fixtures.DistInfoPkgOffPath, unittest.TestCase): + def test_find_distributions_specified_path(self): + dists = Distribution.discover(path=[str(self.site_dir)]) + assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists) + + def test_distribution_at_pathlib(self): + """Demonstrate how to load metadata direct from a directory.""" + dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' + dist = Distribution.at(dist_info_path) + assert dist.version == '1.0.0' + + def test_distribution_at_str(self): + dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' + dist = Distribution.at(str(dist_info_path)) + assert dist.version == '1.0.0' + + +class InvalidateCache(unittest.TestCase): + def test_invalidate_cache(self): + # No externally observable behavior, but ensures test coverage... + importlib.invalidate_caches() diff --git a/Lib/test/test_importlib/metadata/test_main.py b/Lib/test/test_importlib/metadata/test_main.py new file mode 100644 index 000000000..e4218076f --- /dev/null +++ b/Lib/test/test_importlib/metadata/test_main.py @@ -0,0 +1,468 @@ +import re +import pickle +import unittest +import warnings +import importlib +import importlib.metadata +import contextlib +from test.support import os_helper + +try: + import pyfakefs.fake_filesystem_unittest as ffs +except ImportError: + from .stubs import fake_filesystem_unittest as ffs + +from . import fixtures +from ._context import suppress +from ._path import Symlink +from importlib.metadata import ( + Distribution, + EntryPoint, + PackageNotFoundError, + _unique, + distributions, + entry_points, + metadata, + packages_distributions, + version, +) + + +@contextlib.contextmanager +def suppress_known_deprecation(): + with warnings.catch_warnings(record=True) as ctx: + warnings.simplefilter('default', category=DeprecationWarning) + yield ctx + + +class BasicTests(fixtures.DistInfoPkg, unittest.TestCase): + version_pattern = r'\d+\.\d+(\.\d)?' + + def test_retrieves_version_of_self(self): + dist = Distribution.from_name('distinfo-pkg') + assert isinstance(dist.version, str) + assert re.match(self.version_pattern, dist.version) + + def test_for_name_does_not_exist(self): + with self.assertRaises(PackageNotFoundError): + Distribution.from_name('does-not-exist') + + def test_package_not_found_mentions_metadata(self): + """ + When a package is not found, that could indicate that the + package is not installed or that it is installed without + metadata. Ensure the exception mentions metadata to help + guide users toward the cause. See #124. + """ + with self.assertRaises(PackageNotFoundError) as ctx: + Distribution.from_name('does-not-exist') + + assert "metadata" in str(ctx.exception) + + # expected to fail until ABC is enforced + @suppress(AssertionError) + @suppress_known_deprecation() + def test_abc_enforced(self): + with self.assertRaises(TypeError): + type('DistributionSubclass', (Distribution,), {})() + + @fixtures.parameterize( + dict(name=None), + dict(name=''), + ) + def test_invalid_inputs_to_from_name(self, name): + with self.assertRaises(Exception): + Distribution.from_name(name) + + +class ImportTests(fixtures.DistInfoPkg, unittest.TestCase): + def test_import_nonexistent_module(self): + # Ensure that the MetadataPathFinder does not crash an import of a + # non-existent module. + with self.assertRaises(ImportError): + importlib.import_module('does_not_exist') + + def test_resolve(self): + ep = entry_points(group='entries')['main'] + self.assertEqual(ep.load().__name__, "main") + + def test_entrypoint_with_colon_in_name(self): + ep = entry_points(group='entries')['ns:sub'] + self.assertEqual(ep.value, 'mod:main') + + def test_resolve_without_attr(self): + ep = EntryPoint( + name='ep', + value='importlib.metadata', + group='grp', + ) + assert ep.load() is importlib.metadata + + +class NameNormalizationTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + @staticmethod + def make_pkg(name): + """ + Create minimal metadata for a dist-info package with + the indicated name on the file system. + """ + return { + f'{name}.dist-info': { + 'METADATA': 'VERSION: 1.0\n', + }, + } + + def test_dashes_in_dist_name_found_as_underscores(self): + """ + For a package with a dash in the name, the dist-info metadata + uses underscores in the name. Ensure the metadata loads. + """ + fixtures.build_files(self.make_pkg('my_pkg'), self.site_dir) + assert version('my-pkg') == '1.0' + + def test_dist_name_found_as_any_case(self): + """ + Ensure the metadata loads when queried with any case. + """ + pkg_name = 'CherryPy' + fixtures.build_files(self.make_pkg(pkg_name), self.site_dir) + assert version(pkg_name) == '1.0' + assert version(pkg_name.lower()) == '1.0' + assert version(pkg_name.upper()) == '1.0' + + def test_unique_distributions(self): + """ + Two distributions varying only by non-normalized name on + the file system should resolve as the same. + """ + fixtures.build_files(self.make_pkg('abc'), self.site_dir) + before = list(_unique(distributions())) + + alt_site_dir = self.fixtures.enter_context(fixtures.tmp_path()) + self.fixtures.enter_context(self.add_sys_path(alt_site_dir)) + fixtures.build_files(self.make_pkg('ABC'), alt_site_dir) + after = list(_unique(distributions())) + + assert len(after) == len(before) + + +class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + @staticmethod + def pkg_with_non_ascii_description(site_dir): + """ + Create minimal metadata for a package with non-ASCII in + the description. + """ + contents = { + 'portend.dist-info': { + 'METADATA': 'Description: pôrˈtend', + }, + } + fixtures.build_files(contents, site_dir) + return 'portend' + + @staticmethod + def pkg_with_non_ascii_description_egg_info(site_dir): + """ + Create minimal metadata for an egg-info package with + non-ASCII in the description. + """ + contents = { + 'portend.dist-info': { + 'METADATA': """ + Name: portend + + pôrˈtend""", + }, + } + fixtures.build_files(contents, site_dir) + return 'portend' + + def test_metadata_loads(self): + pkg_name = self.pkg_with_non_ascii_description(self.site_dir) + meta = metadata(pkg_name) + assert meta['Description'] == 'pôrˈtend' + + def test_metadata_loads_egg_info(self): + pkg_name = self.pkg_with_non_ascii_description_egg_info(self.site_dir) + meta = metadata(pkg_name) + assert meta['Description'] == 'pôrˈtend' + + +class DiscoveryTests( + fixtures.EggInfoPkg, + fixtures.EggInfoPkgPipInstalledNoToplevel, + fixtures.EggInfoPkgPipInstalledNoModules, + fixtures.EggInfoPkgSourcesFallback, + fixtures.DistInfoPkg, + unittest.TestCase, +): + def test_package_discovery(self): + dists = list(distributions()) + assert all(isinstance(dist, Distribution) for dist in dists) + assert any(dist.metadata['Name'] == 'egginfo-pkg' for dist in dists) + assert any(dist.metadata['Name'] == 'egg_with_module-pkg' for dist in dists) + assert any(dist.metadata['Name'] == 'egg_with_no_modules-pkg' for dist in dists) + assert any(dist.metadata['Name'] == 'sources_fallback-pkg' for dist in dists) + assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists) + + def test_invalid_usage(self): + with self.assertRaises(ValueError): + list(distributions(context='something', name='else')) + + def test_interleaved_discovery(self): + """ + Ensure interleaved searches are safe. + + When the search is cached, it is possible for searches to be + interleaved, so make sure those use-cases are safe. + + Ref #293 + """ + dists = distributions() + next(dists) + version('egginfo-pkg') + next(dists) + + +class DirectoryTest(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + def test_egg_info(self): + # make an `EGG-INFO` directory that's unrelated + self.site_dir.joinpath('EGG-INFO').mkdir() + # used to crash with `IsADirectoryError` + with self.assertRaises(PackageNotFoundError): + version('unknown-package') + + def test_egg(self): + egg = self.site_dir.joinpath('foo-3.6.egg') + egg.mkdir() + with self.add_sys_path(egg): + with self.assertRaises(PackageNotFoundError): + version('foo') + + +class MissingSysPath(fixtures.OnSysPath, unittest.TestCase): + site_dir = '/does-not-exist' + + def test_discovery(self): + """ + Discovering distributions should succeed even if + there is an invalid path on sys.path. + """ + importlib.metadata.distributions() + + +class InaccessibleSysPath(fixtures.OnSysPath, ffs.TestCase): + site_dir = '/access-denied' + + def setUp(self): + super().setUp() + self.setUpPyfakefs() + self.fs.create_dir(self.site_dir, perm_bits=000) + + def test_discovery(self): + """ + Discovering distributions should succeed even if + there is an invalid path on sys.path. + """ + list(importlib.metadata.distributions()) + + +class TestEntryPoints(unittest.TestCase): + def __init__(self, *args): + super().__init__(*args) + self.ep = importlib.metadata.EntryPoint( + name='name', value='value', group='group' + ) + + def test_entry_point_pickleable(self): + revived = pickle.loads(pickle.dumps(self.ep)) + assert revived == self.ep + + def test_positional_args(self): + """ + Capture legacy (namedtuple) construction, discouraged. + """ + EntryPoint('name', 'value', 'group') + + def test_immutable(self): + """EntryPoints should be immutable""" + with self.assertRaises(AttributeError): + self.ep.name = 'badactor' + + def test_repr(self): + assert 'EntryPoint' in repr(self.ep) + assert 'name=' in repr(self.ep) + assert "'name'" in repr(self.ep) + + def test_hashable(self): + """EntryPoints should be hashable""" + hash(self.ep) + + def test_module(self): + assert self.ep.module == 'value' + + def test_attr(self): + assert self.ep.attr is None + + def test_sortable(self): + """ + EntryPoint objects are sortable, but result is undefined. + """ + sorted([ + EntryPoint(name='b', value='val', group='group'), + EntryPoint(name='a', value='val', group='group'), + ]) + + +class FileSystem( + fixtures.OnSysPath, fixtures.SiteDir, fixtures.FileBuilder, unittest.TestCase +): + def test_unicode_dir_on_sys_path(self): + """ + Ensure a Unicode subdirectory of a directory on sys.path + does not crash. + """ + fixtures.build_files( + {self.unicode_filename(): {}}, + prefix=self.site_dir, + ) + list(distributions()) + + +class PackagesDistributionsPrebuiltTest(fixtures.ZipFixtures, unittest.TestCase): + def test_packages_distributions_example(self): + self._fixture_on_path('example-21.12-py3-none-any.whl') + assert packages_distributions()['example'] == ['example'] + + def test_packages_distributions_example2(self): + """ + Test packages_distributions on a wheel built + by trampolim. + """ + self._fixture_on_path('example2-1.0.0-py3-none-any.whl') + assert packages_distributions()['example2'] == ['example2'] + + +class PackagesDistributionsTest( + fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase +): + def test_packages_distributions_neither_toplevel_nor_files(self): + """ + Test a package built without 'top-level.txt' or a file list. + """ + fixtures.build_files( + { + 'trim_example-1.0.0.dist-info': { + 'METADATA': """ + Name: trim_example + Version: 1.0.0 + """, + } + }, + prefix=self.site_dir, + ) + packages_distributions() + + def test_packages_distributions_all_module_types(self): + """ + Test top-level modules detected on a package without 'top-level.txt'. + """ + suffixes = importlib.machinery.all_suffixes() + metadata = dict( + METADATA=""" + Name: all_distributions + Version: 1.0.0 + """, + ) + files = { + 'all_distributions-1.0.0.dist-info': metadata, + } + for i, suffix in enumerate(suffixes): + files.update({ + f'importable-name {i}{suffix}': '', + f'in_namespace_{i}': { + f'mod{suffix}': '', + }, + f'in_package_{i}': { + '__init__.py': '', + f'mod{suffix}': '', + }, + }) + metadata.update(RECORD=fixtures.build_record(files)) + fixtures.build_files(files, prefix=self.site_dir) + + distributions = packages_distributions() + + for i in range(len(suffixes)): + assert distributions[f'importable-name {i}'] == ['all_distributions'] + assert distributions[f'in_namespace_{i}'] == ['all_distributions'] + assert distributions[f'in_package_{i}'] == ['all_distributions'] + + assert not any(name.endswith('.dist-info') for name in distributions) + + @os_helper.skip_unless_symlink + def test_packages_distributions_symlinked_top_level(self) -> None: + """ + Distribution is resolvable from a simple top-level symlink in RECORD. + See #452. + """ + + files: fixtures.FilesSpec = { + "symlinked_pkg-1.0.0.dist-info": { + "METADATA": """ + Name: symlinked-pkg + Version: 1.0.0 + """, + "RECORD": "symlinked,,\n", + }, + ".symlink.target": {}, + "symlinked": Symlink(".symlink.target"), + } + + fixtures.build_files(files, self.site_dir) + assert packages_distributions()['symlinked'] == ['symlinked-pkg'] + + +class PackagesDistributionsEggTest( + fixtures.EggInfoPkg, + fixtures.EggInfoPkgPipInstalledNoToplevel, + fixtures.EggInfoPkgPipInstalledNoModules, + fixtures.EggInfoPkgSourcesFallback, + unittest.TestCase, +): + def test_packages_distributions_on_eggs(self): + """ + Test old-style egg packages with a variation of 'top_level.txt', + 'SOURCES.txt', and 'installed-files.txt', available. + """ + distributions = packages_distributions() + + def import_names_from_package(package_name): + return { + import_name + for import_name, package_names in distributions.items() + if package_name in package_names + } + + # egginfo-pkg declares one import ('mod') via top_level.txt + assert import_names_from_package('egginfo-pkg') == {'mod'} + + # egg_with_module-pkg has one import ('egg_with_module') inferred from + # installed-files.txt (top_level.txt is missing) + assert import_names_from_package('egg_with_module-pkg') == {'egg_with_module'} + + # egg_with_no_modules-pkg should not be associated with any import names + # (top_level.txt is empty, and installed-files.txt has no .py files) + assert import_names_from_package('egg_with_no_modules-pkg') == set() + + # sources_fallback-pkg has one import ('sources_fallback') inferred from + # SOURCES.txt (top_level.txt and installed-files.txt is missing) + assert import_names_from_package('sources_fallback-pkg') == {'sources_fallback'} + + +class EditableDistributionTest(fixtures.DistInfoPkgEditable, unittest.TestCase): + def test_origin(self): + dist = Distribution.from_name('distinfo-pkg') + assert dist.origin.url.endswith('.whl') + assert dist.origin.archive_info.hashes.sha256 diff --git a/Lib/test/test_importlib/metadata/test_zip.py b/Lib/test/test_importlib/metadata/test_zip.py new file mode 100644 index 000000000..276f6288c --- /dev/null +++ b/Lib/test/test_importlib/metadata/test_zip.py @@ -0,0 +1,62 @@ +import sys +import unittest + +from . import fixtures +from importlib.metadata import ( + PackageNotFoundError, + distribution, + distributions, + entry_points, + files, + version, +) + + +class TestZip(fixtures.ZipFixtures, unittest.TestCase): + def setUp(self): + super().setUp() + self._fixture_on_path('example-21.12-py3-none-any.whl') + + def test_zip_version(self): + self.assertEqual(version('example'), '21.12') + + def test_zip_version_does_not_match(self): + with self.assertRaises(PackageNotFoundError): + version('definitely-not-installed') + + def test_zip_entry_points(self): + scripts = entry_points(group='console_scripts') + entry_point = scripts['example'] + self.assertEqual(entry_point.value, 'example:main') + entry_point = scripts['Example'] + self.assertEqual(entry_point.value, 'example:main') + + def test_missing_metadata(self): + self.assertIsNone(distribution('example').read_text('does not exist')) + + def test_case_insensitive(self): + self.assertEqual(version('Example'), '21.12') + + def test_files(self): + for file in files('example'): + path = str(file.dist.locate_file(file)) + assert '.whl/' in path, path + + def test_one_distribution(self): + dists = list(distributions(path=sys.path[:1])) + assert len(dists) == 1 + + +class TestEgg(TestZip): + def setUp(self): + super().setUp() + self._fixture_on_path('example-21.12-py3.6.egg') + + def test_files(self): + for file in files('example'): + path = str(file.dist.locate_file(file)) + assert '.egg/' in path, path + + def test_normalized_name(self): + dist = distribution('example') + assert dist._normalized_name == 'example' diff --git a/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py index e69de29bb..8b1378917 100644 --- a/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py +++ b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_importlib/resources/__init__.py b/Lib/test/test_importlib/resources/__init__.py index e69de29bb..8b1378917 100644 --- a/Lib/test/test_importlib/resources/__init__.py +++ b/Lib/test/test_importlib/resources/__init__.py @@ -0,0 +1 @@ + diff --git a/Lib/test/test_importlib/resources/_path.py b/Lib/test/test_importlib/resources/_path.py index 1f97c9614..b144628cb 100644 --- a/Lib/test/test_importlib/resources/_path.py +++ b/Lib/test/test_importlib/resources/_path.py @@ -2,15 +2,44 @@ import pathlib import functools from typing import Dict, Union +from typing import runtime_checkable +from typing import Protocol #### -# from jaraco.path 3.4.1 - -FilesSpec = Dict[str, Union[str, bytes, 'FilesSpec']] # type: ignore +# from jaraco.path 3.7.1 -def build(spec: FilesSpec, prefix=pathlib.Path()): +class Symlink(str): + """ + A string indicating the target of a symlink. + """ + + +FilesSpec = Dict[str, Union[str, bytes, Symlink, 'FilesSpec']] + + +@runtime_checkable +class TreeMaker(Protocol): + def __truediv__(self, *args, **kwargs): ... # pragma: no cover + + def mkdir(self, **kwargs): ... # pragma: no cover + + def write_text(self, content, **kwargs): ... # pragma: no cover + + def write_bytes(self, content): ... # pragma: no cover + + def symlink_to(self, target): ... # pragma: no cover + + +def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: + return obj if isinstance(obj, TreeMaker) else pathlib.Path(obj) # type: ignore[return-value] + + +def build( + spec: FilesSpec, + prefix: Union[str, TreeMaker] = pathlib.Path(), # type: ignore[assignment] +): """ Build a set of files/directories, as described by the spec. @@ -25,21 +54,25 @@ def build(spec: FilesSpec, prefix=pathlib.Path()): ... "__init__.py": "", ... }, ... "baz.py": "# Some code", - ... } + ... "bar.py": Symlink("baz.py"), + ... }, + ... "bing": Symlink("foo"), ... } >>> target = getfixture('tmp_path') >>> build(spec, target) >>> target.joinpath('foo/baz.py').read_text(encoding='utf-8') '# Some code' + >>> target.joinpath('bing/bar.py').read_text(encoding='utf-8') + '# Some code' """ for name, contents in spec.items(): - create(contents, pathlib.Path(prefix) / name) + create(contents, _ensure_tree_maker(prefix) / name) @functools.singledispatch def create(content: Union[str, bytes, FilesSpec], path): path.mkdir(exist_ok=True) - build(content, prefix=path) # type: ignore + build(content, prefix=path) # type: ignore[arg-type] @create.register @@ -52,5 +85,10 @@ def _(content: str, path): path.write_text(content, encoding='utf-8') +@create.register +def _(content: Symlink, path): + path.symlink_to(content) + + # end from jaraco.path #### diff --git a/Lib/test/test_importlib/resources/test_contents.py b/Lib/test/test_importlib/resources/test_contents.py index 1a13f043a..4e4e0e9c3 100644 --- a/Lib/test/test_importlib/resources/test_contents.py +++ b/Lib/test/test_importlib/resources/test_contents.py @@ -1,7 +1,6 @@ import unittest from importlib import resources -from . import data01 from . import util @@ -19,25 +18,21 @@ class ContentsTests: assert self.expected <= contents -class ContentsDiskTests(ContentsTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class ContentsDiskTests(ContentsTests, util.DiskSetup, unittest.TestCase): + pass class ContentsZipTests(ContentsTests, util.ZipSetup, unittest.TestCase): pass -class ContentsNamespaceTests(ContentsTests, unittest.TestCase): +class ContentsNamespaceTests(ContentsTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + expected = { # no __init__ because of namespace design - # no subdirectory as incidental difference in fixture 'binary.file', + 'subdirectory', 'utf-16.file', 'utf-8.file', } - - def setUp(self): - from . import namespacedata01 - - self.data = namespacedata01 diff --git a/Lib/test/test_importlib/resources/test_custom.py b/Lib/test/test_importlib/resources/test_custom.py index 73127209a..640f90fc0 100644 --- a/Lib/test/test_importlib/resources/test_custom.py +++ b/Lib/test/test_importlib/resources/test_custom.py @@ -5,6 +5,7 @@ import pathlib from test.support import os_helper from importlib import resources +from importlib.resources import abc from importlib.resources.abc import TraversableResources, ResourceReader from . import util @@ -39,8 +40,9 @@ class CustomTraversableResourcesTests(unittest.TestCase): self.addCleanup(self.fixtures.close) def test_custom_loader(self): - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + temp_dir = pathlib.Path(self.fixtures.enter_context(os_helper.temp_dir())) loader = SimpleLoader(MagicResources(temp_dir)) pkg = util.create_package_from_loader(loader) files = resources.files(pkg) - assert files is temp_dir + assert isinstance(files, abc.Traversable) + assert list(files.iterdir()) == [] diff --git a/Lib/test/test_importlib/resources/test_files.py b/Lib/test/test_importlib/resources/test_files.py index 0e9c5c79a..f2165730b 100644 --- a/Lib/test/test_importlib/resources/test_files.py +++ b/Lib/test/test_importlib/resources/test_files.py @@ -1,4 +1,5 @@ -import typing +import pathlib +import py_compile import textwrap import unittest import warnings @@ -7,11 +8,8 @@ import contextlib from importlib import resources from importlib.resources.abc import Traversable -from . import data01 from . import util -from . import _path -from test.support import os_helper -from test.support import import_helper +from test.support import os_helper, import_helper @contextlib.contextmanager @@ -32,13 +30,14 @@ class FilesTests: actual = files.joinpath('utf-8.file').read_text(encoding='utf-8') assert actual == 'Hello, UTF-8 world!\n' - @unittest.skipUnless( - hasattr(typing, 'runtime_checkable'), - "Only suitable when typing supports runtime_checkable", - ) def test_traversable(self): assert isinstance(resources.files(self.data), Traversable) + def test_joinpath_with_multiple_args(self): + files = resources.files(self.data) + binfile = files.joinpath('subdirectory', 'binary.file') + self.assertTrue(binfile.is_file()) + def test_old_parameter(self): """ Files used to take a 'package' parameter. Make sure anyone @@ -48,74 +47,153 @@ class FilesTests: resources.files(package=self.data) -class OpenDiskTests(FilesTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class OpenDiskTests(FilesTests, util.DiskSetup, unittest.TestCase): + pass - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON, line ending issue") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; line ending issue") def test_read_bytes(self): - super().test_read_bytes() + return super().test_read_bytes() class OpenZipTests(FilesTests, util.ZipSetup, unittest.TestCase): pass -class OpenNamespaceTests(FilesTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 +class OpenNamespaceTests(FilesTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' - self.data = namespacedata01 + def test_non_paths_in_dunder_path(self): + """ + Non-path items in a namespace package's ``__path__`` are ignored. - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON, line ending issue") + As reported in python/importlib_resources#311, some tools + like Setuptools, when creating editable packages, will inject + non-paths into a namespace package's ``__path__``, a + sentinel like + ``__editable__.sample_namespace-1.0.finder.__path_hook__`` + to cause the ``PathEntryFinder`` to be called when searching + for packages. In that case, resources should still be loadable. + """ + import namespacedata01 + + namespacedata01.__path__.append( + '__editable__.sample_namespace-1.0.finder.__path_hook__' + ) + + resources.files(namespacedata01) + + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; line ending issue") def test_read_bytes(self): - super().test_read_bytes() + return super().test_read_bytes() -class SiteDir: - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - self.site_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(import_helper.DirsOnSysPath(self.site_dir)) - self.fixtures.enter_context(import_helper.CleanImport()) +class OpenNamespaceZipTests(FilesTests, util.ZipSetup, unittest.TestCase): + ZIP_MODULE = 'namespacedata01' -class ModulesFilesTests(SiteDir, unittest.TestCase): +class DirectSpec: + """ + Override behavior of ModuleSetup to write a full spec directly. + """ + + MODULE = 'unused' + + def load_fixture(self, name): + self.tree_on_path(self.spec) + + +class ModulesFiles: + spec = { + 'mod.py': '', + 'res.txt': 'resources are the best', + } + def test_module_resources(self): """ A module can have resources found adjacent to the module. """ - spec = { - 'mod.py': '', - 'res.txt': 'resources are the best', - } - _path.build(spec, self.site_dir) - import mod + import mod # type: ignore[import-not-found] actual = resources.files(mod).joinpath('res.txt').read_text(encoding='utf-8') - assert actual == spec['res.txt'] + assert actual == self.spec['res.txt'] -class ImplicitContextFilesTests(SiteDir, unittest.TestCase): - def test_implicit_files(self): +class ModuleFilesDiskTests(DirectSpec, util.DiskSetup, ModulesFiles, unittest.TestCase): + pass + + +class ModuleFilesZipTests(DirectSpec, util.ZipSetup, ModulesFiles, unittest.TestCase): + pass + + +class ImplicitContextFiles: + set_val = textwrap.dedent( + f""" + import {resources.__name__} as res + val = res.files().joinpath('res.txt').read_text(encoding='utf-8') + """ + ) + spec = { + 'somepkg': { + '__init__.py': set_val, + 'submod.py': set_val, + 'res.txt': 'resources are the best', + }, + 'frozenpkg': { + '__init__.py': set_val.replace(resources.__name__, 'c_resources'), + 'res.txt': 'resources are the best', + }, + } + + def test_implicit_files_package(self): """ Without any parameter, files() will infer the location as the caller. """ - spec = { - 'somepkg': { - '__init__.py': textwrap.dedent( - """ - import importlib.resources as res - val = res.files().joinpath('res.txt').read_text(encoding='utf-8') - """ - ), - 'res.txt': 'resources are the best', - }, - } - _path.build(spec, self.site_dir) assert importlib.import_module('somepkg').val == 'resources are the best' + def test_implicit_files_submodule(self): + """ + Without any parameter, files() will infer the location as the caller. + """ + assert importlib.import_module('somepkg.submod').val == 'resources are the best' + + def _compile_importlib(self): + """ + Make a compiled-only copy of the importlib resources package. + + Currently only code is copied, as importlib resources doesn't itself + have any resources. + """ + bin_site = self.fixtures.enter_context(os_helper.temp_dir()) + c_resources = pathlib.Path(bin_site, 'c_resources') + sources = pathlib.Path(resources.__file__).parent + + for source_path in sources.glob('**/*.py'): + c_path = c_resources.joinpath(source_path.relative_to(sources)).with_suffix('.pyc') + py_compile.compile(source_path, c_path) + self.fixtures.enter_context(import_helper.DirsOnSysPath(bin_site)) + + def test_implicit_files_with_compiled_importlib(self): + """ + Caller detection works for compiled-only resources module. + + python/cpython#123085 + """ + self._compile_importlib() + assert importlib.import_module('frozenpkg').val == 'resources are the best' + + +class ImplicitContextFilesDiskTests( + DirectSpec, util.DiskSetup, ImplicitContextFiles, unittest.TestCase +): + pass + + +class ImplicitContextFilesZipTests( + DirectSpec, util.ZipSetup, ImplicitContextFiles, unittest.TestCase +): + pass + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_functional.py b/Lib/test/test_importlib/resources/test_functional.py new file mode 100644 index 000000000..e8d25fa4d --- /dev/null +++ b/Lib/test/test_importlib/resources/test_functional.py @@ -0,0 +1,249 @@ +import unittest +import os +import importlib + +from test.support import warnings_helper + +from importlib import resources + +from . import util + +# Since the functional API forwards to Traversable, we only test +# filesystem resources here -- not zip files, namespace packages etc. +# We do test for two kinds of Anchor, though. + + +class StringAnchorMixin: + anchor01 = 'data01' + anchor02 = 'data02' + + +class ModuleAnchorMixin: + @property + def anchor01(self): + return importlib.import_module('data01') + + @property + def anchor02(self): + return importlib.import_module('data02') + + +class FunctionalAPIBase(util.DiskSetup): + def setUp(self): + super().setUp() + self.load_fixture('data02') + + def _gen_resourcetxt_path_parts(self): + """Yield various names of a text file in anchor02, each in a subTest""" + for path_parts in ( + ('subdirectory', 'subsubdir', 'resource.txt'), + ('subdirectory/subsubdir/resource.txt',), + ('subdirectory/subsubdir', 'resource.txt'), + ): + with self.subTest(path_parts=path_parts): + yield path_parts + + def test_read_text(self): + self.assertEqual( + resources.read_text(self.anchor01, 'utf-8.file'), + 'Hello, UTF-8 world!\n', + ) + self.assertEqual( + resources.read_text( + self.anchor02, + 'subdirectory', + 'subsubdir', + 'resource.txt', + encoding='utf-8', + ), + 'a resource', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_text( + self.anchor02, + *path_parts, + encoding='utf-8', + ), + 'a resource', + ) + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.read_text(self.anchor01) + with self.assertRaises(OSError): + resources.read_text(self.anchor01, 'no-such-file') + with self.assertRaises(UnicodeDecodeError): + resources.read_text(self.anchor01, 'utf-16.file') + self.assertEqual( + resources.read_text( + self.anchor01, + 'binary.file', + encoding='latin1', + ), + '\x00\x01\x02\x03', + ) + self.assertEndsWith( # ignore the BOM + resources.read_text( + self.anchor01, + 'utf-16.file', + errors='backslashreplace', + ), + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', + ), + ) + + def test_read_binary(self): + self.assertEqual( + resources.read_binary(self.anchor01, 'utf-8.file'), + b'Hello, UTF-8 world!\n', + ) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertEqual( + resources.read_binary(self.anchor02, *path_parts), + b'a resource', + ) + + def test_open_text(self): + with resources.open_text(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_text( + self.anchor02, + *path_parts, + encoding='utf-8', + ) as f: + self.assertEqual(f.read(), 'a resource') + # Use generic OSError, since e.g. attempting to read a directory can + # fail with PermissionError rather than IsADirectoryError + with self.assertRaises(OSError): + resources.open_text(self.anchor01) + with self.assertRaises(OSError): + resources.open_text(self.anchor01, 'no-such-file') + with resources.open_text(self.anchor01, 'utf-16.file') as f: + with self.assertRaises(UnicodeDecodeError): + f.read() + with resources.open_text( + self.anchor01, + 'binary.file', + encoding='latin1', + ) as f: + self.assertEqual(f.read(), '\x00\x01\x02\x03') + with resources.open_text( + self.anchor01, + 'utf-16.file', + errors='backslashreplace', + ) as f: + self.assertEndsWith( # ignore the BOM + f.read(), + 'Hello, UTF-16 world!\n'.encode('utf-16-le').decode( + errors='backslashreplace', + ), + ) + + def test_open_binary(self): + with resources.open_binary(self.anchor01, 'utf-8.file') as f: + self.assertEqual(f.read(), b'Hello, UTF-8 world!\n') + for path_parts in self._gen_resourcetxt_path_parts(): + with resources.open_binary( + self.anchor02, + *path_parts, + ) as f: + self.assertEqual(f.read(), b'a resource') + + def test_path(self): + with resources.path(self.anchor01, 'utf-8.file') as path: + with open(str(path), encoding='utf-8') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + with resources.path(self.anchor01) as path: + with open(os.path.join(path, 'utf-8.file'), encoding='utf-8') as f: + self.assertEqual(f.read(), 'Hello, UTF-8 world!\n') + + def test_is_resource(self): + is_resource = resources.is_resource + self.assertTrue(is_resource(self.anchor01, 'utf-8.file')) + self.assertFalse(is_resource(self.anchor01, 'no_such_file')) + self.assertFalse(is_resource(self.anchor01)) + self.assertFalse(is_resource(self.anchor01, 'subdirectory')) + for path_parts in self._gen_resourcetxt_path_parts(): + self.assertTrue(is_resource(self.anchor02, *path_parts)) + + def test_contents(self): + with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01) + self.assertGreaterEqual( + set(c), + {'utf-8.file', 'utf-16.file', 'binary.file', 'subdirectory'}, + ) + with self.assertRaises(OSError), warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )): + list(resources.contents(self.anchor01, 'utf-8.file')) + + for path_parts in self._gen_resourcetxt_path_parts(): + with self.assertRaises(OSError), warnings_helper.check_warnings(( + ".*contents.*", + DeprecationWarning, + )): + list(resources.contents(self.anchor01, *path_parts)) + with warnings_helper.check_warnings((".*contents.*", DeprecationWarning)): + c = resources.contents(self.anchor01, 'subdirectory') + self.assertGreaterEqual( + set(c), + {'binary.file'}, + ) + + @warnings_helper.ignore_warnings(category=DeprecationWarning) + def test_common_errors(self): + for func in ( + resources.read_text, + resources.read_binary, + resources.open_text, + resources.open_binary, + resources.path, + resources.is_resource, + resources.contents, + ): + with self.subTest(func=func): + # Rejecting None anchor + with self.assertRaises(TypeError): + func(None) + # Rejecting invalid anchor type + with self.assertRaises((TypeError, AttributeError)): + func(1234) + # Unknown module + with self.assertRaises(ModuleNotFoundError): + func('$missing module$') + + def test_text_errors(self): + for func in ( + resources.read_text, + resources.open_text, + ): + with self.subTest(func=func): + # Multiple path arguments need explicit encoding argument. + with self.assertRaises(TypeError): + func( + self.anchor02, + 'subdirectory', + 'subsubdir', + 'resource.txt', + ) + + +class FunctionalAPITest_StringAnchor( + StringAnchorMixin, + FunctionalAPIBase, + unittest.TestCase, +): + pass + + +class FunctionalAPITest_ModuleAnchor( + ModuleAnchorMixin, + FunctionalAPIBase, + unittest.TestCase, +): + pass diff --git a/Lib/test/test_importlib/resources/test_open.py b/Lib/test/test_importlib/resources/test_open.py index 86becb4bf..8c00378ad 100644 --- a/Lib/test/test_importlib/resources/test_open.py +++ b/Lib/test/test_importlib/resources/test_open.py @@ -1,7 +1,6 @@ import unittest from importlib import resources -from . import data01 from . import util @@ -24,7 +23,7 @@ class OpenTests: target = resources.files(self.data) / 'binary.file' with target.open('rb') as fp: result = fp.read() - self.assertEqual(result, b'\x00\x01\x02\x03') + self.assertEqual(result, bytes(range(4))) def test_open_text_default_encoding(self): target = resources.files(self.data) / 'utf-8.file' @@ -65,21 +64,21 @@ class OpenTests: target.open(encoding='utf-8') -class OpenDiskTests(OpenTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class OpenDiskTests(OpenTests, util.DiskSetup, unittest.TestCase): + pass -class OpenDiskNamespaceTests(OpenTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - - self.data = namespacedata01 +class OpenDiskNamespaceTests(OpenTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): pass +class OpenNamespaceZipTests(OpenTests, util.ZipSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/resources/test_path.py b/Lib/test/test_importlib/resources/test_path.py index 34a6bdd2d..903911f57 100644 --- a/Lib/test/test_importlib/resources/test_path.py +++ b/Lib/test/test_importlib/resources/test_path.py @@ -1,8 +1,8 @@ import io +import pathlib import unittest from importlib import resources -from . import data01 from . import util @@ -15,23 +15,16 @@ class CommonTests(util.CommonTests, unittest.TestCase): class PathTests: def test_reading(self): """ - Path should be readable. - - Test also implicitly verifies the returned object is a pathlib.Path - instance. + Path should be readable and a pathlib.Path instance. """ target = resources.files(self.data) / 'utf-8.file' with resources.as_file(target) as path: - self.assertTrue(path.name.endswith("utf-8.file"), repr(path)) - # pathlib.Path.read_text() was introduced in Python 3.5. - with path.open('r', encoding='utf-8') as file: - text = file.read() - self.assertEqual('Hello, UTF-8 world!\n', text) + self.assertIsInstance(path, pathlib.Path) + self.assertEndsWith(path.name, "utf-8.file") + self.assertEqual('Hello, UTF-8 world!\n', path.read_text(encoding='utf-8')) -class PathDiskTests(PathTests, unittest.TestCase): - data = data01 - +class PathDiskTests(PathTests, util.DiskSetup, unittest.TestCase): def test_natural_path(self): # Guarantee the internal implementation detail that # file-system-backed resources do not get the tempdir diff --git a/Lib/test/test_importlib/resources/test_read.py b/Lib/test/test_importlib/resources/test_read.py index 088982681..59c237d96 100644 --- a/Lib/test/test_importlib/resources/test_read.py +++ b/Lib/test/test_importlib/resources/test_read.py @@ -1,7 +1,7 @@ import unittest from importlib import import_module, resources -from . import data01 + from . import util @@ -18,7 +18,7 @@ class CommonTextTests(util.CommonTests, unittest.TestCase): class ReadTests: def test_read_bytes(self): result = resources.files(self.data).joinpath('binary.file').read_bytes() - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4))) def test_read_text_default_encoding(self): result = ( @@ -51,30 +51,42 @@ class ReadTests: ) -class ReadDiskTests(ReadTests, unittest.TestCase): - data = data01 +class ReadDiskTests(ReadTests, util.DiskSetup, unittest.TestCase): + pass class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): def test_read_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') result = resources.files(submodule).joinpath('binary.file').read_bytes() - self.assertEqual(result, b'\0\1\2\3') + self.assertEqual(result, bytes(range(4, 8))) def test_read_submodule_resource_by_name(self): result = ( - resources.files('ziptestdata.subdirectory') + resources.files('data01.subdirectory').joinpath('binary.file').read_bytes() + ) + self.assertEqual(result, bytes(range(4, 8))) + + +class ReadNamespaceTests(ReadTests, util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + +class ReadNamespaceZipTests(ReadTests, util.ZipSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def test_read_submodule_resource(self): + submodule = import_module('namespacedata01.subdirectory') + result = resources.files(submodule).joinpath('binary.file').read_bytes() + self.assertEqual(result, bytes(range(12, 16))) + + def test_read_submodule_resource_by_name(self): + result = ( + resources.files('namespacedata01.subdirectory') .joinpath('binary.file') .read_bytes() ) - self.assertEqual(result, b'\0\1\2\3') - - -class ReadNamespaceTests(ReadTests, unittest.TestCase): - def setUp(self): - from . import namespacedata01 - - self.data = namespacedata01 + self.assertEqual(result, bytes(range(12, 16))) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/test_reader.py b/Lib/test/test_importlib/resources/test_reader.py index 8670f72a3..ed5693ab4 100644 --- a/Lib/test/test_importlib/resources/test_reader.py +++ b/Lib/test/test_importlib/resources/test_reader.py @@ -1,17 +1,21 @@ import os.path -import sys import pathlib import unittest from importlib import import_module from importlib.readers import MultiplexedPath, NamespaceReader +from . import util -class MultiplexedPathTest(unittest.TestCase): - @classmethod - def setUpClass(cls): - path = pathlib.Path(__file__).parent / 'namespacedata01' - cls.folder = str(path) + +class MultiplexedPathTest(util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' + + def setUp(self): + super().setUp() + self.folder = pathlib.Path(self.data.__path__[0]) + self.data01 = pathlib.Path(self.load_fixture('data01').__file__).parent + self.data02 = pathlib.Path(self.load_fixture('data02').__file__).parent def test_init_no_paths(self): with self.assertRaises(FileNotFoundError): @@ -19,7 +23,7 @@ class MultiplexedPathTest(unittest.TestCase): def test_init_file(self): with self.assertRaises(NotADirectoryError): - MultiplexedPath(os.path.join(self.folder, 'binary.file')) + MultiplexedPath(self.folder / 'binary.file') def test_iterdir(self): contents = {path.name for path in MultiplexedPath(self.folder).iterdir()} @@ -27,12 +31,13 @@ class MultiplexedPathTest(unittest.TestCase): contents.remove('__pycache__') except (KeyError, ValueError): pass - self.assertEqual(contents, {'binary.file', 'utf-16.file', 'utf-8.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-16.file', 'utf-8.file'} + ) def test_iterdir_duplicate(self): - data01 = os.path.abspath(os.path.join(__file__, '..', 'data01')) contents = { - path.name for path in MultiplexedPath(self.folder, data01).iterdir() + path.name for path in MultiplexedPath(self.folder, self.data01).iterdir() } for remove in ('__pycache__', '__init__.pyc'): try: @@ -60,17 +65,16 @@ class MultiplexedPathTest(unittest.TestCase): path.open() def test_join_path(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') - path = MultiplexedPath(self.folder, data01) + prefix = str(self.folder.parent) + path = MultiplexedPath(self.folder, self.data01) self.assertEqual( str(path.joinpath('binary.file'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'binary.file'), ) - self.assertEqual( - str(path.joinpath('subdirectory'))[len(prefix) + 1 :], - os.path.join('data01', 'subdirectory'), - ) + sub = path.joinpath('subdirectory') + assert isinstance(sub, MultiplexedPath) + assert 'namespacedata01' in str(sub) + assert 'data01' in str(sub) self.assertEqual( str(path.joinpath('imaginary'))[len(prefix) + 1 :], os.path.join('namespacedata01', 'imaginary'), @@ -82,10 +86,8 @@ class MultiplexedPathTest(unittest.TestCase): assert not path.joinpath('imaginary/foo.py').exists() def test_join_path_common_subdir(self): - prefix = os.path.abspath(os.path.join(__file__, '..')) - data01 = os.path.join(prefix, 'data01') - data02 = os.path.join(prefix, 'data02') - path = MultiplexedPath(data01, data02) + prefix = str(self.data02.parent) + path = MultiplexedPath(self.data01, self.data02) self.assertIsInstance(path.joinpath('subdirectory'), MultiplexedPath) self.assertEqual( str(path.joinpath('subdirectory', 'subsubdir'))[len(prefix) + 1 :], @@ -105,16 +107,8 @@ class MultiplexedPathTest(unittest.TestCase): ) -class NamespaceReaderTest(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) +class NamespaceReaderTest(util.DiskSetup, unittest.TestCase): + MODULE = 'namespacedata01' def test_init_error(self): with self.assertRaises(ValueError): @@ -124,7 +118,7 @@ class NamespaceReaderTest(unittest.TestCase): namespacedata01 = import_module('namespacedata01') reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) - root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + root = self.data.__path__[0] self.assertEqual( reader.resource_path('binary.file'), os.path.join(root, 'binary.file') ) @@ -133,9 +127,8 @@ class NamespaceReaderTest(unittest.TestCase): ) def test_files(self): - namespacedata01 = import_module('namespacedata01') - reader = NamespaceReader(namespacedata01.__spec__.submodule_search_locations) - root = os.path.abspath(os.path.join(__file__, '..', 'namespacedata01')) + reader = NamespaceReader(self.data.__spec__.submodule_search_locations) + root = self.data.__path__[0] self.assertIsInstance(reader.files(), MultiplexedPath) self.assertEqual(repr(reader.files()), f"MultiplexedPath('{root}')") diff --git a/Lib/test/test_importlib/resources/test_resource.py b/Lib/test/test_importlib/resources/test_resource.py index 54d32d9b2..7e5e5903f 100644 --- a/Lib/test/test_importlib/resources/test_resource.py +++ b/Lib/test/test_importlib/resources/test_resource.py @@ -1,15 +1,8 @@ -import contextlib -import sys +import os import unittest -import uuid -import pathlib -from . import data01 -from . import zipdata01, zipdata02 from . import util from importlib import resources, import_module -from test.support import import_helper, os_helper -from test.support.os_helper import unlink class ResourceTests: @@ -29,9 +22,8 @@ class ResourceTests: self.assertTrue(target.is_dir()) -class ResourceDiskTests(ResourceTests, unittest.TestCase): - def setUp(self): - self.data = data01 +class ResourceDiskTests(ResourceTests, util.DiskSetup, unittest.TestCase): + pass class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase): @@ -42,33 +34,39 @@ def names(traversable): return {item.name for item in traversable.iterdir()} -class ResourceLoaderTests(unittest.TestCase): +class ResourceLoaderTests(util.DiskSetup, unittest.TestCase): def test_resource_contents(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) self.assertEqual(names(resources.files(package)), {'A', 'B', 'C'}) def test_is_file(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertTrue(resources.files(package).joinpath('B').is_file()) def test_is_dir(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertTrue(resources.files(package).joinpath('D').is_dir()) def test_resource_missing(self): package = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C', 'D/E', 'D/F'] + file=self.data, + path=self.data.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F'], ) self.assertFalse(resources.files(package).joinpath('Z').is_file()) -class ResourceCornerCaseTests(unittest.TestCase): +class ResourceCornerCaseTests(util.DiskSetup, unittest.TestCase): def test_package_has_no_reader_fallback(self): """ Test odd ball packages which: @@ -77,7 +75,7 @@ class ResourceCornerCaseTests(unittest.TestCase): # 3. Are not in a zip file """ module = util.create_package( - file=data01, path=data01.__file__, contents=['A', 'B', 'C'] + file=self.data, path=self.data.__file__, contents=['A', 'B', 'C'] ) # Give the module a dummy loader. module.__loader__ = object() @@ -88,43 +86,39 @@ class ResourceCornerCaseTests(unittest.TestCase): self.assertFalse(resources.files(module).joinpath('A').is_file()) -class ResourceFromZipsTest01(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata01 # type: ignore - +class ResourceFromZipsTest01(util.ZipSetup, unittest.TestCase): def test_is_submodule_resource(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertTrue(resources.files(submodule).joinpath('binary.file').is_file()) def test_read_submodule_resource_by_name(self): self.assertTrue( - resources.files('ziptestdata.subdirectory') - .joinpath('binary.file') - .is_file() + resources.files('data01.subdirectory').joinpath('binary.file').is_file() ) def test_submodule_contents(self): - submodule = import_module('ziptestdata.subdirectory') + submodule = import_module('data01.subdirectory') self.assertEqual( names(resources.files(submodule)), {'__init__.py', 'binary.file'} ) def test_submodule_contents_by_name(self): self.assertEqual( - names(resources.files('ziptestdata.subdirectory')), + names(resources.files('data01.subdirectory')), {'__init__.py', 'binary.file'}, ) def test_as_file_directory(self): - with resources.as_file(resources.files('ziptestdata')) as data: - assert data.name == 'ziptestdata' + with resources.as_file(resources.files('data01')) as data: + assert data.name == 'data01' assert data.is_dir() assert data.joinpath('subdirectory').is_dir() assert len(list(data.iterdir())) assert not data.parent.exists() -class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): - ZIP_MODULE = zipdata02 # type: ignore +class ResourceFromZipsTest02(util.ZipSetup, unittest.TestCase): + MODULE = 'data02' def test_unrelated_contents(self): """ @@ -132,98 +126,49 @@ class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): distinct resources. Ref python/importlib_resources#44. """ self.assertEqual( - names(resources.files('ziptestdata.one')), + names(resources.files('data02.one')), {'__init__.py', 'resource1.txt'}, ) self.assertEqual( - names(resources.files('ziptestdata.two')), + names(resources.files('data02.two')), {'__init__.py', 'resource2.txt'}, ) -@contextlib.contextmanager -def zip_on_path(dir): - data_path = pathlib.Path(zipdata01.__file__) - source_zip_path = data_path.parent.joinpath('ziptestdata.zip') - zip_path = pathlib.Path(dir) / f'{uuid.uuid4()}.zip' - zip_path.write_bytes(source_zip_path.read_bytes()) - sys.path.append(str(zip_path)) - import_module('ziptestdata') - - try: - yield - finally: - with contextlib.suppress(ValueError): - sys.path.remove(str(zip_path)) - - with contextlib.suppress(KeyError): - del sys.path_importer_cache[str(zip_path)] - del sys.modules['ziptestdata'] - - with contextlib.suppress(OSError): - unlink(zip_path) - - -class DeletingZipsTest(unittest.TestCase): +class DeletingZipsTest(util.ZipSetup, unittest.TestCase): """Having accessed resources in a zip file should not keep an open reference to the zip. """ - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) - - temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) - self.fixtures.enter_context(zip_on_path(temp_dir)) - def test_iterdir_does_not_keep_open(self): - [item.name for item in resources.files('ziptestdata').iterdir()] + [item.name for item in resources.files('data01').iterdir()] def test_is_file_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').is_file() + resources.files('data01').joinpath('binary.file').is_file() def test_is_file_failure_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('not-present').is_file() + resources.files('data01').joinpath('not-present').is_file() @unittest.skip("Desired but not supported.") def test_as_file_does_not_keep_open(self): # pragma: no cover - resources.as_file(resources.files('ziptestdata') / 'binary.file') + resources.as_file(resources.files('data01') / 'binary.file') - import os # TODO: RUSTPYTHON see below - @unittest.skipIf( - 'RUSTPYTHON_SKIP_ENV_POLLUTERS' in os.environ, - "TODO: RUSTPYTHON environment pollution when running rustpython -m test --fail-env-changed due to tmpfile leak" - ) + @unittest.skipIf("RUSTPYTHON_SKIP_ENV_POLLUTERS" in os.environ, "TODO: RUSTPYTHON; environment pollution when running rustpython -m test --fail-env-changed due to tmpfile leak") def test_entered_path_does_not_keep_open(self): """ Mimic what certifi does on import to make its bundle available for the process duration. """ - resources.as_file(resources.files('ziptestdata') / 'binary.file').__enter__() + resources.as_file(resources.files('data01') / 'binary.file').__enter__() def test_read_binary_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('binary.file').read_bytes() + resources.files('data01').joinpath('binary.file').read_bytes() def test_read_text_does_not_keep_open(self): - resources.files('ziptestdata').joinpath('utf-8.file').read_text( - encoding='utf-8' - ) + resources.files('data01').joinpath('utf-8.file').read_text(encoding='utf-8') -class ResourceFromNamespaceTest01(unittest.TestCase): - site_dir = str(pathlib.Path(__file__).parent) - - @classmethod - def setUpClass(cls): - sys.path.append(cls.site_dir) - - @classmethod - def tearDownClass(cls): - sys.path.remove(cls.site_dir) - +class ResourceFromNamespaceTests: def test_is_submodule_resource(self): self.assertTrue( resources.files(import_module('namespacedata01')) @@ -242,7 +187,9 @@ class ResourceFromNamespaceTest01(unittest.TestCase): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) def test_submodule_contents_by_name(self): contents = names(resources.files('namespacedata01')) @@ -250,7 +197,41 @@ class ResourceFromNamespaceTest01(unittest.TestCase): contents.remove('__pycache__') except KeyError: pass - self.assertEqual(contents, {'binary.file', 'utf-8.file', 'utf-16.file'}) + self.assertEqual( + contents, {'subdirectory', 'binary.file', 'utf-8.file', 'utf-16.file'} + ) + + def test_submodule_sub_contents(self): + contents = names(resources.files(import_module('namespacedata01.subdirectory'))) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + def test_submodule_sub_contents_by_name(self): + contents = names(resources.files('namespacedata01.subdirectory')) + try: + contents.remove('__pycache__') + except KeyError: + pass + self.assertEqual(contents, {'binary.file'}) + + +class ResourceFromNamespaceDiskTests( + util.DiskSetup, + ResourceFromNamespaceTests, + unittest.TestCase, +): + MODULE = 'namespacedata01' + + +class ResourceFromNamespaceZipTests( + util.ZipSetup, + ResourceFromNamespaceTests, + unittest.TestCase, +): + MODULE = 'namespacedata01' if __name__ == '__main__': diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index dbe6ee814..e2d995f59 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -4,11 +4,12 @@ import io import sys import types import pathlib +import contextlib -from . import data01 -from . import zipdata01 from importlib.resources.abc import ResourceReader -from test.support import import_helper +from test.support import import_helper, os_helper +from . import zip as zip_ +from . import _path from importlib.machinery import ModuleSpec @@ -67,7 +68,7 @@ def create_package(file=None, path=None, is_package=True, contents=()): ) -class CommonTests(metaclass=abc.ABCMeta): +class CommonTestsBase(metaclass=abc.ABCMeta): """ Tests shared by test_open, test_path, and test_read. """ @@ -83,34 +84,34 @@ class CommonTests(metaclass=abc.ABCMeta): """ Passing in the package name should succeed. """ - self.execute(data01.__name__, 'utf-8.file') + self.execute(self.data.__name__, 'utf-8.file') def test_package_object(self): """ Passing in the package itself should succeed. """ - self.execute(data01, 'utf-8.file') + self.execute(self.data, 'utf-8.file') def test_string_path(self): """ Passing in a string for the path should succeed. """ path = 'utf-8.file' - self.execute(data01, path) + self.execute(self.data, path) def test_pathlib_path(self): """ Passing in a pathlib.PurePath object for the path should succeed. """ path = pathlib.PurePath('utf-8.file') - self.execute(data01, path) + self.execute(self.data, path) def test_importing_module_as_side_effect(self): """ The anchor package can already be imported. """ - del sys.modules[data01.__name__] - self.execute(data01.__name__, 'utf-8.file') + del sys.modules[self.data.__name__] + self.execute(self.data.__name__, 'utf-8.file') def test_missing_path(self): """ @@ -140,40 +141,66 @@ class CommonTests(metaclass=abc.ABCMeta): self.execute(package, 'utf-8.file') -class ZipSetupBase: - ZIP_MODULE = None +fixtures = dict( + data01={ + '__init__.py': '', + 'binary.file': bytes(range(4)), + 'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'), + 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), + 'subdirectory': { + '__init__.py': '', + 'binary.file': bytes(range(4, 8)), + }, + }, + data02={ + '__init__.py': '', + 'one': {'__init__.py': '', 'resource1.txt': 'one resource'}, + 'two': {'__init__.py': '', 'resource2.txt': 'two resource'}, + 'subdirectory': {'subsubdir': {'resource.txt': 'a resource'}}, + }, + namespacedata01={ + 'binary.file': bytes(range(4)), + 'utf-16.file': '\ufeffHello, UTF-16 world!\n'.encode('utf-16-le'), + 'utf-8.file': 'Hello, UTF-8 world!\n'.encode('utf-8'), + 'subdirectory': { + 'binary.file': bytes(range(12, 16)), + }, + }, +) - @classmethod - def setUpClass(cls): - data_path = pathlib.Path(cls.ZIP_MODULE.__file__) - data_dir = data_path.parent - cls._zip_path = str(data_dir / 'ziptestdata.zip') - sys.path.append(cls._zip_path) - cls.data = importlib.import_module('ziptestdata') - - @classmethod - def tearDownClass(cls): - try: - sys.path.remove(cls._zip_path) - except ValueError: - pass - - try: - del sys.path_importer_cache[cls._zip_path] - del sys.modules[cls.data.__name__] - except KeyError: - pass - - try: - del cls.data - del cls._zip_path - except AttributeError: - pass +class ModuleSetup: def setUp(self): - modules = import_helper.modules_setup() - self.addCleanup(import_helper.modules_cleanup, *modules) + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + self.fixtures.enter_context(import_helper.isolated_modules()) + self.data = self.load_fixture(self.MODULE) + + def load_fixture(self, module): + self.tree_on_path({module: fixtures[module]}) + return importlib.import_module(module) -class ZipSetup(ZipSetupBase): - ZIP_MODULE = zipdata01 # type: ignore +class ZipSetup(ModuleSetup): + MODULE = 'data01' + + def tree_on_path(self, spec): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + modules = pathlib.Path(temp_dir) / 'zipped modules.zip' + self.fixtures.enter_context( + import_helper.DirsOnSysPath(str(zip_.make_zip_file(spec, modules))) + ) + + +class DiskSetup(ModuleSetup): + MODULE = 'data01' + + def tree_on_path(self, spec): + temp_dir = self.fixtures.enter_context(os_helper.temp_dir()) + _path.build(spec, pathlib.Path(temp_dir)) + self.fixtures.enter_context(import_helper.DirsOnSysPath(temp_dir)) + + +class CommonTests(DiskSetup, CommonTestsBase): + pass diff --git a/Lib/test/test_importlib/resources/zip.py b/Lib/test/test_importlib/resources/zip.py new file mode 100644 index 000000000..fc453f020 --- /dev/null +++ b/Lib/test/test_importlib/resources/zip.py @@ -0,0 +1,24 @@ +""" +Generate zip test data files. +""" + +import zipfile + + +def make_zip_file(tree, dst): + """ + Zip the files in tree into a new zipfile at dst. + """ + with zipfile.ZipFile(dst, 'w') as zf: + for name, contents in walk(tree): + zf.writestr(name, contents) + zipfile._path.CompleteDirs.inject(zf) + return dst + + +def walk(tree, prefix=''): + for name, contents in tree.items(): + if isinstance(contents, dict): + yield from walk(contents, prefix=f'{prefix}{name}/') + else: + yield f'{prefix}{name}', contents diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py index 6a0631331..e52829e62 100644 --- a/Lib/test/test_importlib/source/test_case_sensitivity.py +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -9,7 +9,6 @@ machinery = util.import_importlib('importlib.machinery') import os from test.support import os_helper import unittest -import warnings @util.case_insensitive_tests diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py index fa0efee8d..f35adec1a 100644 --- a/Lib/test/test_importlib/source/test_file_loader.py +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -359,6 +359,7 @@ class SimpleTest(abc.LoaderTests): ) = util.test_both(SimpleTest, importlib=importlib, machinery=machinery, abc=importlib_abc, util=importlib_util) + class SourceDateEpochTestMeta(SourceDateEpochTestMeta, type(Source_SimpleTest)): pass @@ -679,6 +680,7 @@ class SourceLoaderBadBytecodeTestPEP451( machinery=machinery, abc=importlib_abc, util=importlib_util) + class SourceLoaderBadBytecodeTestPEP302( SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): pass @@ -690,6 +692,7 @@ class SourceLoaderBadBytecodeTestPEP302( machinery=machinery, abc=importlib_abc, util=importlib_util) + class SourcelessLoaderBadBytecodeTest: @classmethod @@ -775,6 +778,7 @@ class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, machinery=machinery, abc=importlib_abc, util=importlib_util) + class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, BadBytecodeTestPEP302): pass @@ -786,5 +790,6 @@ class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, machinery=machinery, abc=importlib_abc, util=importlib_util) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py index 12db7c7d3..4de736a6b 100644 --- a/Lib/test/test_importlib/source/test_finder.py +++ b/Lib/test/test_importlib/source/test_finder.py @@ -10,7 +10,6 @@ import sys import tempfile from test.support.import_helper import make_legacy_pyc import unittest -import warnings class FinderTests(abc.FinderTests): @@ -74,7 +73,7 @@ class FinderTests(abc.FinderTests): if error.errno != errno.ENOENT: raise loader = self.import_(mapping['.root'], test) - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') return loader def test_module(self): @@ -101,7 +100,7 @@ class FinderTests(abc.FinderTests): with util.create_modules('pkg.__init__', 'pkg.sub') as mapping: pkg_dir = os.path.dirname(mapping['pkg.__init__']) loader = self.import_(pkg_dir, 'pkg.sub') - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') # [sub package] def test_package_in_package(self): @@ -109,7 +108,7 @@ class FinderTests(abc.FinderTests): with context as mapping: pkg_dir = os.path.dirname(mapping['pkg.__init__']) loader = self.import_(pkg_dir, 'pkg.sub') - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') # [package over modules] def test_package_over_module(self): @@ -130,7 +129,7 @@ class FinderTests(abc.FinderTests): file.write("# test file for importlib") try: loader = self._find(finder, 'mod', loader_only=True) - self.assertTrue(hasattr(loader, 'load_module')) + self.assertHasAttr(loader, 'load_module') finally: os.unlink('mod.py') diff --git a/Lib/test/test_importlib/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py index f274330e0..6e1c23e6a 100644 --- a/Lib/test/test_importlib/source/test_path_hook.py +++ b/Lib/test/test_importlib/source/test_path_hook.py @@ -15,12 +15,12 @@ class PathHookTest: def test_success(self): with util.create_modules('dummy') as mapping: - self.assertTrue(hasattr(self.path_hook()(mapping['.root']), - 'find_spec')) + self.assertHasAttr(self.path_hook()(mapping['.root']), + 'find_spec') def test_empty_string(self): # The empty string represents the cwd. - self.assertTrue(hasattr(self.path_hook()(''), 'find_spec')) + self.assertHasAttr(self.path_hook()(''), 'find_spec') (Frozen_PathHookTest, diff --git a/Lib/test/test_importlib/source/test_source_encoding.py b/Lib/test/test_importlib/source/test_source_encoding.py index 4f206accf..d65d51d0c 100644 --- a/Lib/test/test_importlib/source/test_source_encoding.py +++ b/Lib/test/test_importlib/source/test_source_encoding.py @@ -61,17 +61,15 @@ class EncodingTest: def test_default_encoding(self): self.run_test(self.source_line.encode('utf-8')) - # TODO: RUSTPYTHON, UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 17 - @unittest.expectedFailure # [encoding first line] + @unittest.expectedFailure # TODO: RUSTPYTHON; UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 17 def test_encoding_on_first_line(self): encoding = 'Latin-1' source = self.create_source(encoding) self.run_test(source) - # TODO: RUSTPYTHON, UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 34 - @unittest.expectedFailure # [encoding second line] + @unittest.expectedFailure # TODO: RUSTPYTHON; UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 34 def test_encoding_on_second_line(self): source = b"#/usr/bin/python\n" + self.create_source('Latin-1') self.run_test(source) @@ -85,9 +83,8 @@ class EncodingTest: source = codecs.BOM_UTF8 + self.create_source('utf-8') self.run_test(source) - # TODO: RUSTPYTHON, UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 20 - @unittest.expectedFailure # [BOM conflict] + @unittest.expectedFailure # TODO: RUSTPYTHON; UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 20 def test_bom_conflict(self): source = codecs.BOM_UTF8 + self.create_source('latin-1') with self.assertRaises(SyntaxError): diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index 603125f6d..dd943210f 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -43,14 +43,12 @@ class InheritanceTests: def test_subclasses(self): # Test that the expected subclasses inherit. for subclass in self.subclasses: - self.assertTrue(issubclass(subclass, self.__test), - "{0} is not a subclass of {1}".format(subclass, self.__test)) + self.assertIsSubclass(subclass, self.__test) def test_superclasses(self): # Test that the class inherits from the expected superclasses. for superclass in self.superclasses: - self.assertTrue(issubclass(self.__test, superclass), - "{0} is not a superclass of {1}".format(superclass, self.__test)) + self.assertIsSubclass(self.__test, superclass) class MetaPathFinder(InheritanceTests): @@ -416,14 +414,14 @@ class InspectLoaderSourceToCodeTests: # Since compile() can handle strings, so should source_to_code(). source = 'attr = 42' module = self.source_to_module(source) - self.assertTrue(hasattr(module, 'attr')) + self.assertHasAttr(module, 'attr') self.assertEqual(module.attr, 42) def test_source_to_code_bytes(self): # Since compile() can handle bytes, so should source_to_code(). source = b'attr = 42' module = self.source_to_module(source) - self.assertTrue(hasattr(module, 'attr')) + self.assertHasAttr(module, 'attr') self.assertEqual(module.attr, 42) def test_source_to_code_path(self): @@ -757,7 +755,7 @@ class SourceOnlyLoaderTests(SourceLoaderTestHarness): warnings.simplefilter('ignore', DeprecationWarning) module = self.loader.load_module(self.name) self.verify_module(module) - self.assertFalse(hasattr(module, '__path__')) + self.assertNotHasAttr(module, '__path__') def test_get_source_encoding(self): # Source is considered encoded in UTF-8 by default unless otherwise @@ -795,6 +793,9 @@ class SourceLoaderBytecodeTests(SourceLoaderTestHarness): data.extend(self.init._pack_uint32(0)) data.extend(self.init._pack_uint32(self.loader.source_mtime)) data.extend(self.init._pack_uint32(self.loader.source_size)) + # Make sure there's > 1 reference to code_object so that the + # marshaled representation below matches the cached representation + l = [code_object] data.extend(marshal.dumps(code_object)) self.assertEqual(self.loader.written[self.cached], bytes(data)) @@ -913,5 +914,30 @@ class SourceLoaderGetSourceTests: SourceOnlyLoaderMock=SPLIT_SOL) +class SourceLoaderDeprecationWarningsTests(unittest.TestCase): + """Tests SourceLoader deprecation warnings.""" + + def test_deprecated_path_mtime(self): + from importlib.abc import SourceLoader + class DummySourceLoader(SourceLoader): + def get_data(self, path): + return b'' + + def get_filename(self, fullname): + return 'foo.py' + + def path_stats(self, path): + return {'mtime': 1} + + loader = DummySourceLoader() + + with self.assertWarnsRegex( + DeprecationWarning, + r"SourceLoader\.path_mtime is deprecated in favour of " + r"SourceLoader\.path_stats\(\)\." + ): + loader.path_mtime('foo.py') + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index ecf2c47c4..1bc531a2f 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -6,11 +6,12 @@ machinery = test_util.import_importlib('importlib.machinery') import os.path import sys +from test import support from test.support import import_helper from test.support import os_helper +import traceback import types import unittest -import warnings class ImportModuleTests: @@ -354,6 +355,20 @@ class ReloadTests: with self.assertRaises(ModuleNotFoundError): self.init.reload(module) + def test_reload_traceback_with_non_str(self): + # gh-125519 + with support.captured_stdout() as stdout: + try: + self.init.reload("typing") + except TypeError as exc: + traceback.print_exception(exc, file=stdout) + else: + self.fail("Expected TypeError to be raised") + printed_traceback = stdout.getvalue() + self.assertIn("TypeError", printed_traceback) + self.assertNotIn("AttributeError", printed_traceback) + self.assertNotIn("module.__spec__.name", printed_traceback) + (Frozen_ReloadTests, Source_ReloadTests @@ -415,8 +430,7 @@ class StartupTests: for name, module in sys.modules.items(): if isinstance(module, types.ModuleType): with self.subTest(name=name): - self.assertTrue(hasattr(module, '__loader__'), - '{!r} lacks a __loader__ attribute'.format(name)) + self.assertHasAttr(module, '__loader__') if self.machinery.BuiltinImporter.find_spec(name): self.assertIsNot(module.__loader__, None) elif self.machinery.FrozenImporter.find_spec(name): @@ -426,7 +440,7 @@ class StartupTests: for name, module in sys.modules.items(): if isinstance(module, types.ModuleType): with self.subTest(name=name): - self.assertTrue(hasattr(module, '__spec__')) + self.assertHasAttr(module, '__spec__') if self.machinery.BuiltinImporter.find_spec(name): self.assertIsNot(module.__spec__, None) elif self.machinery.FrozenImporter.find_spec(name): @@ -438,5 +452,57 @@ class StartupTests: ) = test_util.test_both(StartupTests, machinery=machinery) +class TestModuleAll(unittest.TestCase): + def test_machinery(self): + extra = ( + # from importlib._bootstrap and importlib._bootstrap_external + 'AppleFrameworkLoader', + 'BYTECODE_SUFFIXES', + 'BuiltinImporter', + 'DEBUG_BYTECODE_SUFFIXES', + 'EXTENSION_SUFFIXES', + 'ExtensionFileLoader', + 'FileFinder', + 'FrozenImporter', + 'ModuleSpec', + 'NamespaceLoader', + 'OPTIMIZED_BYTECODE_SUFFIXES', + 'PathFinder', + 'SOURCE_SUFFIXES', + 'SourceFileLoader', + 'SourcelessFileLoader', + 'WindowsRegistryFinder', + ) + support.check__all__(self, machinery['Source'], extra=extra) + + def test_util(self): + extra = ( + # from importlib.abc, importlib._bootstrap + # and importlib._bootstrap_external + 'Loader', + 'MAGIC_NUMBER', + 'cache_from_source', + 'decode_source', + 'module_from_spec', + 'source_from_cache', + 'spec_from_file_location', + 'spec_from_loader', + ) + support.check__all__(self, util['Source'], extra=extra) + + +class TestDeprecations(unittest.TestCase): + def test_machinery_deprecated_attributes(self): + from importlib import machinery + attributes = ( + 'DEBUG_BYTECODE_SUFFIXES', + 'OPTIMIZED_BYTECODE_SUFFIXES', + ) + for attr in attributes: + with self.subTest(attr=attr): + with self.assertWarns(DeprecationWarning): + getattr(machinery, attr) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py index cc993f333..e48fad889 100644 --- a/Lib/test/test_importlib/test_lazy.py +++ b/Lib/test/test_importlib/test_lazy.py @@ -2,9 +2,12 @@ import importlib from importlib import abc from importlib import util import sys +import time +import threading import types import unittest +from test.support import threading_helper from test.test_importlib import util as test_util @@ -40,6 +43,7 @@ class TestingImporter(abc.MetaPathFinder, abc.Loader): module_name = 'lazy_loader_test' mutated_name = 'changed' loaded = None + load_count = 0 source_code = 'attr = 42; __name__ = {!r}'.format(mutated_name) def find_spec(self, name, path, target=None): @@ -48,8 +52,10 @@ class TestingImporter(abc.MetaPathFinder, abc.Loader): return util.spec_from_loader(name, util.LazyLoader(self)) def exec_module(self, module): + time.sleep(0.01) # Simulate a slow load. exec(self.source_code, module.__dict__) self.loaded = module + self.load_count += 1 class LazyLoaderTests(unittest.TestCase): @@ -59,8 +65,9 @@ class LazyLoaderTests(unittest.TestCase): # Classes that don't define exec_module() trigger TypeError. util.LazyLoader(object) - def new_module(self, source_code=None): - loader = TestingImporter() + def new_module(self, source_code=None, loader=None): + if loader is None: + loader = TestingImporter() if source_code is not None: loader.source_code = source_code spec = util.spec_from_loader(TestingImporter.module_name, @@ -118,12 +125,12 @@ class LazyLoaderTests(unittest.TestCase): # Deleting an attribute should stay deleted. module = self.new_module() del module.attr - self.assertFalse(hasattr(module, 'attr')) + self.assertNotHasAttr(module, 'attr') def test_delete_preexisting_attr(self): module = self.new_module() del module.__name__ - self.assertFalse(hasattr(module, '__name__')) + self.assertNotHasAttr(module, '__name__') def test_module_substitution_error(self): with test_util.uncache(TestingImporter.module_name): @@ -140,6 +147,83 @@ class LazyLoaderTests(unittest.TestCase): # Force the load; just care that no exception is raised. module.__name__ + @threading_helper.requires_working_threading() + def test_module_load_race(self): + with test_util.uncache(TestingImporter.module_name): + loader = TestingImporter() + module = self.new_module(loader=loader) + self.assertEqual(loader.load_count, 0) + + class RaisingThread(threading.Thread): + exc = None + def run(self): + try: + super().run() + except Exception as exc: + self.exc = exc + + def access_module(): + return module.attr + + threads = [] + for _ in range(2): + threads.append(thread := RaisingThread(target=access_module)) + thread.start() + + # Races could cause errors + for thread in threads: + thread.join() + self.assertIsNone(thread.exc) + + # Or multiple load attempts + self.assertEqual(loader.load_count, 1) + + def test_lazy_self_referential_modules(self): + # Directory modules with submodules that reference the parent can attempt to access + # the parent module during a load. Verify that this common pattern works with lazy loading. + # json is a good example in the stdlib. + json_modules = [name for name in sys.modules if name.startswith('json')] + with test_util.uncache(*json_modules): + # Standard lazy loading, unwrapped + spec = util.find_spec('json') + loader = util.LazyLoader(spec.loader) + spec.loader = loader + module = util.module_from_spec(spec) + sys.modules['json'] = module + loader.exec_module(module) + + # Trigger load with attribute lookup, ensure expected behavior + test_load = module.loads('{}') + self.assertEqual(test_load, {}) + + def test_lazy_module_type_override(self): + # Verify that lazy loading works with a module that modifies + # its __class__ to be a custom type. + + # Example module from PEP 726 + module = self.new_module(source_code="""\ +import sys +from types import ModuleType + +CONSTANT = 3.14 + +class ImmutableModule(ModuleType): + def __setattr__(self, name, value): + raise AttributeError('Read-only attribute!') + + def __delattr__(self, name): + raise AttributeError('Read-only attribute!') + +sys.modules[__name__].__class__ = ImmutableModule +""") + sys.modules[TestingImporter.module_name] = module + self.assertIsInstance(module, util._LazyModule) + self.assertEqual(module.CONSTANT, 3.14) + with self.assertRaises(AttributeError): + module.CONSTANT = 2.71 + with self.assertRaises(AttributeError): + del module.CONSTANT + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py index edf0329c7..9406f06ad 100644 --- a/Lib/test/test_importlib/test_locks.py +++ b/Lib/test/test_importlib/test_locks.py @@ -34,6 +34,7 @@ class ModuleLockAsRLockTests: # lock status in repr unsupported test_repr = None test_locked_repr = None + test_repr_count = None def tearDown(self): for splitinit in init.values(): @@ -49,7 +50,6 @@ LOCK_TYPES = {kind: splitinit._bootstrap._ModuleLock LockType=LOCK_TYPES) -@unittest.skipIf(sys.platform == "darwin", "TODO: RUSTPYTHON") class DeadlockAvoidanceTests: def setUp(self): @@ -99,7 +99,7 @@ class DeadlockAvoidanceTests: self.assertEqual(len(results), NTHREADS) return results - @unittest.skip("TODO: RUSTPYTHON, sometimes hangs") + @unittest.skip("TODO: RUSTPYTHON; sometimes hangs") def test_deadlock(self): results = self.run_deadlock_avoidance_test(True) # At least one of the threads detected a potential deadlock on its @@ -109,7 +109,7 @@ class DeadlockAvoidanceTests: self.assertGreaterEqual(nb_deadlocks, 1) self.assertEqual(results.count((True, True)), len(results) - nb_deadlocks) - @unittest.skip("TODO: RUSTPYTHON, flaky test") + @unittest.skip("TODO: RUSTPYTHON; flaky test") def test_no_deadlock(self): results = self.run_deadlock_avoidance_test(False) self.assertEqual(results.count((True, False)), 0) @@ -148,10 +148,10 @@ class LifetimeTests: self.assertEqual(0, len(self.bootstrap._module_locks), self.bootstrap._module_locks) -# TODO: RUSTPYTHON -# (Frozen_LifetimeTests, -# Source_LifetimeTests -# ) = test_util.test_both(LifetimeTests, init=init) + +(Frozen_LifetimeTests, + Source_LifetimeTests + ) = test_util.test_both(LifetimeTests, init=init) def setUpModule(): diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py index 97ce4df08..6ca0978f9 100644 --- a/Lib/test/test_importlib/test_namespace_pkgs.py +++ b/Lib/test/test_importlib/test_namespace_pkgs.py @@ -6,7 +6,6 @@ import os import sys import tempfile import unittest -import warnings from test.test_importlib import util @@ -81,7 +80,7 @@ class SingleNamespacePackage(NamespacePackageTest): def test_simple_repr(self): import foo.one - assert repr(foo).startswith(": module (.*) does not support loading in subinterpreters") - def run_with_own_gil(self, script): - interpid = _interpreters.create(isolated=True) - try: - _interpreters.run_string(interpid, script) - except _interpreters.RunFailedError as exc: - if m := self.ERROR.match(str(exc)): - modname, = m.groups() - raise ImportError(modname) + interpid = _interpreters.create('isolated') + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + excsnap = _interpreters.exec(interpid, script) + if excsnap is not None: + if excsnap.type.__name__ == 'ImportError': + raise ImportError(excsnap.msg) def run_with_shared_gil(self, script): - interpid = _interpreters.create(isolated=False) - try: - _interpreters.run_string(interpid, script) - except _interpreters.RunFailedError as exc: - if m := self.ERROR.match(str(exc)): - modname, = m.groups() - raise ImportError(modname) + interpid = _interpreters.create('legacy') + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + excsnap = _interpreters.exec(interpid, script) + if excsnap is not None: + if excsnap.type.__name__ == 'ImportError': + raise ImportError(excsnap.msg) @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") + # gh-117649: single-phase init modules are not currently supported in + # subinterpreters in the free-threaded build + @support.expected_failure_if_gil_disabled() def test_single_phase_init_module(self): script = textwrap.dedent(''' from importlib.util import _incompatible_extension_module_restrictions @@ -702,14 +722,22 @@ class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): self.run_with_own_gil(script) @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") + @support.requires_gil_enabled("gh-117649: not supported in free-threaded build") def test_incomplete_multi_phase_init_module(self): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + prescript = textwrap.dedent(f''' from importlib.util import spec_from_loader, module_from_spec - from importlib.machinery import ExtensionFileLoader + from importlib.machinery import {loader} name = '_test_shared_gil_only' filename = {_testmultiphase.__file__!r} - loader = ExtensionFileLoader(name, filename) + loader = {loader}(name, filename) spec = spec_from_loader(name, loader) ''') @@ -760,5 +788,74 @@ class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): self.run_with_own_gil(script) +class PatchAtomicWrites: + def __init__(self, truncate_at_length, never_complete=False): + self.truncate_at_length = truncate_at_length + self.never_complete = never_complete + self.seen_write = False + self._children = [] + + def __enter__(self): + import _pyio + + oldwrite = os.write + + # Emulate an os.write that only writes partial data. + def write(fd, data): + if self.seen_write and self.never_complete: + return None + self.seen_write = True + return oldwrite(fd, data[:self.truncate_at_length]) + + # Need to patch _io to be _pyio, so that io.FileIO is affected by the + # os.write patch. + self.children = [ + support.swap_attr(_bootstrap_external, '_io', _pyio), + support.swap_attr(os, 'write', write) + ] + for child in self.children: + child.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + for child in self.children: + child.__exit__(exc_type, exc_val, exc_tb) + + +class MiscTests(unittest.TestCase): + + def test_atomic_write_retries_incomplete_writes(self): + truncate_at_length = 100 + length = truncate_at_length * 2 + + with PatchAtomicWrites(truncate_at_length=truncate_at_length) as cm: + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * length + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + self.assertTrue(cm.seen_write) + + self.assertEqual(os.stat(support.os_helper.TESTFN).st_size, length) + os.unlink(support.os_helper.TESTFN) + + def test_atomic_write_errors_if_unable_to_complete(self): + truncate_at_length = 100 + + with ( + PatchAtomicWrites( + truncate_at_length=truncate_at_length, never_complete=True, + ) as cm, + self.assertRaises(OSError) + ): + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * (truncate_at_length * 2) + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + self.assertTrue(cm.seen_write) + + with self.assertRaises(OSError): + os.stat(support.os_helper.TESTFN) # Check that the file did not get written. + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index f8a9ead9a..bef4fb46f 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -5,7 +5,7 @@ import os import re import sys import unittest -import warnings +from test import support from test.support import import_helper from contextlib import contextmanager from test.test_importlib.util import temp_module @@ -91,31 +91,60 @@ class WindowsRegistryFinderTests: test_module = "spamham{}".format(os.getpid()) def test_find_spec_missing(self): - spec = self.machinery.WindowsRegistryFinder.find_spec('spam') + with self.assertWarnsRegex( + DeprecationWarning, + r"importlib\.machinery\.WindowsRegistryFinder is deprecated; " + r"use site configuration instead\. Future versions of Python may " + r"not enable this finder by default\." + ): + spec = self.machinery.WindowsRegistryFinder.find_spec('spam') self.assertIsNone(spec) def test_module_found(self): with setup_module(self.machinery, self.test_module): - spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) + with self.assertWarnsRegex( + DeprecationWarning, + r"importlib\.machinery\.WindowsRegistryFinder is deprecated; " + r"use site configuration instead\. Future versions of Python may " + r"not enable this finder by default\." + ): + spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) self.assertIsNotNone(spec) def test_module_not_found(self): with setup_module(self.machinery, self.test_module, path="."): - spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) + with self.assertWarnsRegex( + DeprecationWarning, + r"importlib\.machinery\.WindowsRegistryFinder is deprecated; " + r"use site configuration instead\. Future versions of Python may " + r"not enable this finder by default\." + ): + spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) self.assertIsNone(spec) + def test_raises_deprecation_warning(self): + # WindowsRegistryFinder is not meant to be instantiated, so the + # deprecation warning is raised in the 'find_spec' method instead. + with self.assertWarnsRegex( + DeprecationWarning, + r"importlib\.machinery\.WindowsRegistryFinder is deprecated; " + r"use site configuration instead\. Future versions of Python may " + r"not enable this finder by default\." + ): + self.machinery.WindowsRegistryFinder.find_spec('spam') + (Frozen_WindowsRegistryFinderTests, Source_WindowsRegistryFinderTests ) = test_util.test_both(WindowsRegistryFinderTests, machinery=machinery) @unittest.skipUnless(sys.platform.startswith('win'), 'requires Windows') class WindowsExtensionSuffixTests: - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tagged_suffix(self): suffixes = self.machinery.EXTENSION_SUFFIXES - expected_tag = ".cp{0.major}{0.minor}-{1}.pyd".format(sys.version_info, - re.sub('[^a-zA-Z0-9]', '_', get_platform())) + abi_flags = "t" if support.Py_GIL_DISABLED else "" + ver = sys.version_info + platform = re.sub('[^a-zA-Z0-9]', '_', get_platform()) + expected_tag = f".cp{ver.major}{ver.minor}{abi_flags}-{platform}.pyd" try: untagged_i = suffixes.index(".pyd") except ValueError: diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index c25be096e..edbe78545 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -6,13 +6,17 @@ from importlib import machinery, util, invalidate_caches import marshal import os import os.path +from test import support from test.support import import_helper +from test.support import is_apple_mobile from test.support import os_helper import unittest import sys import tempfile import types +_testsinglephase = import_helper.import_module("_testsinglephase") + BUILTINS = types.SimpleNamespace() BUILTINS.good_name = None @@ -22,25 +26,39 @@ if 'errno' in sys.builtin_module_names: if 'importlib' not in sys.builtin_module_names: BUILTINS.bad_name = 'importlib' -EXTENSIONS = types.SimpleNamespace() -EXTENSIONS.path = None -EXTENSIONS.ext = None -EXTENSIONS.filename = None -EXTENSIONS.file_path = None -EXTENSIONS.name = '_testsinglephase' +if support.is_wasi: + # dlopen() is a shim for WASI as of WASI SDK which fails by default. + # We don't provide an implementation, so tests will fail. + # But we also don't want to turn off dynamic loading for those that provide + # a working implementation. + def _extension_details(): + global EXTENSIONS + EXTENSIONS = None +else: + EXTENSIONS = types.SimpleNamespace() + EXTENSIONS.path = None + EXTENSIONS.ext = None + EXTENSIONS.filename = None + EXTENSIONS.file_path = None + EXTENSIONS.name = '_testsinglephase' -def _extension_details(): - global EXTENSIONS - for path in sys.path: - for ext in machinery.EXTENSION_SUFFIXES: - filename = EXTENSIONS.name + ext - file_path = os.path.join(path, filename) - if os.path.exists(file_path): - EXTENSIONS.path = path - EXTENSIONS.ext = ext - EXTENSIONS.filename = filename - EXTENSIONS.file_path = file_path - return + def _extension_details(): + global EXTENSIONS + for path in sys.path: + for ext in machinery.EXTENSION_SUFFIXES: + # Apple mobile platforms mechanically load .so files, + # but the findable files are labelled .fwork + if is_apple_mobile: + ext = ext.replace(".so", ".fwork") + + filename = EXTENSIONS.name + ext + file_path = os.path.join(path, filename) + if os.path.exists(file_path): + EXTENSIONS.path = path + EXTENSIONS.ext = ext + EXTENSIONS.filename = filename + EXTENSIONS.file_path = file_path + return _extension_details() diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index 3ea5d6d38..54786505d 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -109,12 +109,7 @@ class PyCompileTestsBase: self.assertTrue(os.path.exists(self.pyc_path)) self.assertFalse(os.path.exists(self.cache_path)) - @unittest.expectedFailureIf( - sys.platform == "darwin" and int( - __import__("platform").release().split(".")[0] - ) < 20, - "TODO: RUSTPYTHON" - ) + @unittest.expectedFailureIf(sys.platform == "darwin" and int(__import__("platform").release().split(".")[0]) < 20, "TODO: RUSTPYTHON") def test_relative_path(self): py_compile.compile(os.path.relpath(self.source_path), os.path.relpath(self.pyc_path)) @@ -203,7 +198,7 @@ class PyCompileTestsBase: fp.read(), 'test', {}) self.assertEqual(flags, 0b1) - @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.expectedFailure # TODO: RUSTPYTHON def test_quiet(self): bad_coding = os.path.join(os.path.dirname(__file__), 'tokenizedata',