Merge remote-tracking branch 'origin/master' into try-to-use-local

.github/workflows/ci.yaml

@@ -25,15 +25,21 @@ jobs:
       - name: Cache cargo dependencies
         uses: actions/cache@v2
         with:
-          key: ${{ runner.os }}-rust_tests-${{ hashFiles('Cargo.lock') }}
-          path: target
-          restore-keys: |
-            ${{ runner.os }}-rust_tests-
+          path: |
+              ~/.cargo/registry
+              ~/.cargo/git
+              target
+          key: ${{ runner.os }}-debug_opt3-${{ hashFiles('**/Cargo.lock') }}
       - name: run rust tests
         uses: actions-rs/cargo@v1
         with:
           command: test
           args: --verbose ${{ env.CARGO_ARGS }}
+      - name: check compilation without threading
+        uses: actions-rs/cargo@v1
+        with:
+          command: check
+          args: ${{ env.CARGO_ARGS }} --no-default-features
 
   snippets_cpython:
     name: Run snippets and cpython tests
@@ -51,10 +57,11 @@ jobs:
       - name: Cache cargo dependencies
         uses: actions/cache@v2
         with:
-          key: ${{ runner.os }}-snippets-${{ hashFiles('Cargo.lock') }}
-          path: target
-          restore-keys: |
-            ${{ runner.os }}-snippets-
+          path: |
+              ~/.cargo/registry
+              ~/.cargo/git
+              target
+          key: ${{ runner.os }}-release-${{ hashFiles('**/Cargo.lock') }}
       - name: build rustpython
         uses: actions-rs/cargo@v1
         with:
@@ -77,9 +84,14 @@ jobs:
         run: target/release/rustpython -m test -v
         env:
           RUSTPYTHONPATH: ${{ github.workspace }}/Lib
-        if: runner.os != 'Windows'
+        if: runner.os == 'Linux'
+      - name: run cpython tests (macOS lightweight)
+        run: target/release/rustpython -m test -x test_argparse -x test_json -x test_bytes -x test_long -v
+        env:
+          RUSTPYTHONPATH: ${{ github.workspace }}/Lib
+        if: runner.os == 'macOS'
 
-  format:
+  lint:
     name: Check Rust code with rustfmt and clippy
     runs-on: ubuntu-latest
     steps:
@@ -100,12 +112,6 @@ jobs:
         with:
           command: clippy
           args: ${{ env.CARGO_ARGS }} -- -Dwarnings
-
-  lint:
-    name: Lint Python code with flake8
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
       - uses: actions/setup-python@v1
         with:
           python-version: 3.8
@@ -116,16 +122,20 @@ jobs:
 
   wasm:
     name: Check the WASM package and demo
+    needs: rust_tests
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@master
       - name: Cache cargo dependencies
         uses: actions/cache@v2
         with:
-          key: ${{ runner.os }}-wasm-${{ hashFiles('**/Cargo.lock') }}
-          path: target
+          path: |
+              ~/.cargo/registry
+              ~/.cargo/git
+              target
+          key: ${{ runner.os }}-wasm_opt3-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: |
-            ${{ runner.os }}-wasm-
+               ${{ runner.os }}-debug_opt3-${{ hashFiles('**/Cargo.lock') }}
       - name: install wasm-pack
         run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
       - name: install geckodriver

.gitpod.Dockerfile

@@ -11,4 +11,11 @@ RUN rm -rf ~/.rustup && \
     curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && \
     rustup target add wasm32-unknown-unknown
 
+RUN sudo apt-get -q update \
+    && sudo apt-get install -yq \
+        libpython3.6 \
+        rust-lldb \
+    && sudo rm -rf /var/lib/apt/lists/*
+ENV RUST_LLDB=/usr/bin/lldb-8
+
 USER root

.gitpod.yml

@@ -1,2 +1,6 @@
 image:
   file: .gitpod.Dockerfile
+
+vscode:
+  extensions:
+    - vadimcn.vscode-lldb@1.5.3:vTh/rWhvJ5nQpeAVsD20QA==

.theia/launch.json

@@ -0,0 +1,16 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  "version": "0.2.0",
+  "configurations": [
+      {
+          "type": "lldb",
+          "request": "launch",
+          "name": "Debug Rust Code",
+          //"preLaunchTask": "cargo",
+          "program": "${workspaceFolder}/target/debug/rustpython",
+          "cwd": "${workspaceFolder}",
+          //"valuesFormatting": "parseText"
+      }
+  ]
+}

.theia/settings.json

@@ -0,0 +1,8 @@
+{
+    "cpp.buildConfigurations": [
+        {
+            "name": "",
+            "directory": ""
+        },
+    ]
+}

Cargo.toml

@@ -6,17 +6,20 @@ edition = "2018"
 description = "A python interpreter written in rust."
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
+include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
 
 [workspace]
-members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "examples/freeze", "vm/pylib-crate"]
+members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode", "vm/pylib-crate", "common"]
 
 [[bench]]
 name = "bench"
 path = "./benchmarks/bench.rs"
 
 [features]
+default = ["threading", "pylib"]
 flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
 freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
+threading = ["rustpython-vm/threading"]
 
 ssl = ["rustpython-vm/ssl"]
 
@@ -24,9 +27,10 @@ ssl = ["rustpython-vm/ssl"]
 log = "0.4"
 env_logger = "0.7"
 clap = "2.33"
-rustpython-compiler = {path = "compiler", version = "0.1.1"}
-rustpython-parser = {path = "parser", version = "0.1.1"}
-rustpython-vm = {path = "vm", version = "0.1.1"}
+rustpython-compiler = { path = "compiler", version = "0.1.1" }
+rustpython-parser = { path = "parser", version = "0.1.1" }
+rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] }
+pylib = { package = "rustpython-pylib", path = "vm/pylib-crate", version = "0.1.0", default-features = false, optional = true }
 dirs = { package = "dirs-next", version = "1.0" }
 num-traits = "0.2.8"
 cfg-if = "0.1"
@@ -45,6 +49,9 @@ version = "0.2"
 name = "rustpython"
 path = "src/main.rs"
 
+[profile.dev.package."*"]
+opt-level = 3
+
 [patch.crates-io]
 # REDOX START, Uncommment when you want to compile/check with redoxer
 # # following patches are just waiting on a new version to be released to crates.io

Lib/_osx_support.py

@@ -0,0 +1,502 @@
+"""Shared OS X support functions."""
+
+import os
+import re
+import sys
+
+__all__ = [
+    'compiler_fixup',
+    'customize_config_vars',
+    'customize_compiler',
+    'get_platform_osx',
+]
+
+# configuration variables that may contain universal build flags,
+# like "-arch" or "-isdkroot", that may need customization for
+# the user environment
+_UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS',
+                            'BLDSHARED', 'LDSHARED', 'CC', 'CXX',
+                            'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS',
+                            'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS')
+
+# configuration variables that may contain compiler calls
+_COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX')
+
+# prefix added to original configuration variable names
+_INITPRE = '_OSX_SUPPORT_INITIAL_'
+
+
+def _find_executable(executable, path=None):
+    """Tries to find 'executable' in the directories listed in 'path'.
+
+    A string listing directories separated by 'os.pathsep'; defaults to
+    os.environ['PATH'].  Returns the complete filename or None if not found.
+    """
+    if path is None:
+        path = os.environ['PATH']
+
+    paths = path.split(os.pathsep)
+    base, ext = os.path.splitext(executable)
+
+    if (sys.platform == 'win32') and (ext != '.exe'):
+        executable = executable + '.exe'
+
+    if not os.path.isfile(executable):
+        for p in paths:
+            f = os.path.join(p, executable)
+            if os.path.isfile(f):
+                # the file exists, we have a shot at spawn working
+                return f
+        return None
+    else:
+        return executable
+
+
+def _read_output(commandstring):
+    """Output from successful command execution or None"""
+    # Similar to os.popen(commandstring, "r").read(),
+    # but without actually using os.popen because that
+    # function is not usable during python bootstrap.
+    # tempfile is also not available then.
+    import contextlib
+    try:
+        import tempfile
+        fp = tempfile.NamedTemporaryFile()
+    except ImportError:
+        fp = open("/tmp/_osx_support.%s"%(
+            os.getpid(),), "w+b")
+
+    with contextlib.closing(fp) as fp:
+        cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name)
+        return fp.read().decode('utf-8').strip() if not os.system(cmd) else None
+
+
+def _find_build_tool(toolname):
+    """Find a build tool on current path or using xcrun"""
+    return (_find_executable(toolname)
+                or _read_output("/usr/bin/xcrun -find %s" % (toolname,))
+                or ''
+            )
+
+_SYSTEM_VERSION = None
+
+def _get_system_version():
+    """Return the OS X system version as a string"""
+    # Reading this plist is a documented way to get the system
+    # version (see the documentation for the Gestalt Manager)
+    # We avoid using platform.mac_ver to avoid possible bootstrap issues during
+    # the build of Python itself (distutils is used to build standard library
+    # extensions).
+
+    global _SYSTEM_VERSION
+
+    if _SYSTEM_VERSION is None:
+        _SYSTEM_VERSION = ''
+        try:
+            f = open('/System/Library/CoreServices/SystemVersion.plist')
+        except OSError:
+            # We're on a plain darwin box, fall back to the default
+            # behaviour.
+            pass
+        else:
+            try:
+                m = re.search(r'<key>ProductUserVisibleVersion</key>\s*'
+                              r'<string>(.*?)</string>', f.read())
+            finally:
+                f.close()
+            if m is not None:
+                _SYSTEM_VERSION = '.'.join(m.group(1).split('.')[:2])
+            # else: fall back to the default behaviour
+
+    return _SYSTEM_VERSION
+
+def _remove_original_values(_config_vars):
+    """Remove original unmodified values for testing"""
+    # This is needed for higher-level cross-platform tests of get_platform.
+    for k in list(_config_vars):
+        if k.startswith(_INITPRE):
+            del _config_vars[k]
+
+def _save_modified_value(_config_vars, cv, newvalue):
+    """Save modified and original unmodified value of configuration var"""
+
+    oldvalue = _config_vars.get(cv, '')
+    if (oldvalue != newvalue) and (_INITPRE + cv not in _config_vars):
+        _config_vars[_INITPRE + cv] = oldvalue
+    _config_vars[cv] = newvalue
+
+def _supports_universal_builds():
+    """Returns True if universal builds are supported on this system"""
+    # As an approximation, we assume that if we are running on 10.4 or above,
+    # then we are running with an Xcode environment that supports universal
+    # builds, in particular -isysroot and -arch arguments to the compiler. This
+    # is in support of allowing 10.4 universal builds to run on 10.3.x systems.
+
+    osx_version = _get_system_version()
+    if osx_version:
+        try:
+            osx_version = tuple(int(i) for i in osx_version.split('.'))
+        except ValueError:
+            osx_version = ''
+    return bool(osx_version >= (10, 4)) if osx_version else False
+
+
+def _find_appropriate_compiler(_config_vars):
+    """Find appropriate C compiler for extension module builds"""
+
+    # Issue #13590:
+    #    The OSX location for the compiler varies between OSX
+    #    (or rather Xcode) releases.  With older releases (up-to 10.5)
+    #    the compiler is in /usr/bin, with newer releases the compiler
+    #    can only be found inside Xcode.app if the "Command Line Tools"
+    #    are not installed.
+    #
+    #    Furthermore, the compiler that can be used varies between
+    #    Xcode releases. Up to Xcode 4 it was possible to use 'gcc-4.2'
+    #    as the compiler, after that 'clang' should be used because
+    #    gcc-4.2 is either not present, or a copy of 'llvm-gcc' that
+    #    miscompiles Python.
+
+    # skip checks if the compiler was overridden with a CC env variable
+    if 'CC' in os.environ:
+        return _config_vars
+
+    # The CC config var might contain additional arguments.
+    # Ignore them while searching.
+    cc = oldcc = _config_vars['CC'].split()[0]
+    if not _find_executable(cc):
+        # Compiler is not found on the shell search PATH.
+        # Now search for clang, first on PATH (if the Command LIne
+        # Tools have been installed in / or if the user has provided
+        # another location via CC).  If not found, try using xcrun
+        # to find an uninstalled clang (within a selected Xcode).
+
+        # NOTE: Cannot use subprocess here because of bootstrap
+        # issues when building Python itself (and os.popen is
+        # implemented on top of subprocess and is therefore not
+        # usable as well)
+
+        cc = _find_build_tool('clang')
+
+    elif os.path.basename(cc).startswith('gcc'):
+        # Compiler is GCC, check if it is LLVM-GCC
+        data = _read_output("'%s' --version"
+                             % (cc.replace("'", "'\"'\"'"),))
+        if data and 'llvm-gcc' in data:
+            # Found LLVM-GCC, fall back to clang
+            cc = _find_build_tool('clang')
+
+    if not cc:
+        raise SystemError(
+               "Cannot locate working compiler")
+
+    if cc != oldcc:
+        # Found a replacement compiler.
+        # Modify config vars using new compiler, if not already explicitly
+        # overridden by an env variable, preserving additional arguments.
+        for cv in _COMPILER_CONFIG_VARS:
+            if cv in _config_vars and cv not in os.environ:
+                cv_split = _config_vars[cv].split()
+                cv_split[0] = cc if cv != 'CXX' else cc + '++'
+                _save_modified_value(_config_vars, cv, ' '.join(cv_split))
+
+    return _config_vars
+
+
+def _remove_universal_flags(_config_vars):
+    """Remove all universal build arguments from config vars"""
+
+    for cv in _UNIVERSAL_CONFIG_VARS:
+        # Do not alter a config var explicitly overridden by env var
+        if cv in _config_vars and cv not in os.environ:
+            flags = _config_vars[cv]
+            flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII)
+            flags = re.sub('-isysroot [^ \t]*', ' ', flags)
+            _save_modified_value(_config_vars, cv, flags)
+
+    return _config_vars
+
+
+def _remove_unsupported_archs(_config_vars):
+    """Remove any unsupported archs from config vars"""
+    # Different Xcode releases support different sets for '-arch'
+    # flags. In particular, Xcode 4.x no longer supports the
+    # PPC architectures.
+    #
+    # This code automatically removes '-arch ppc' and '-arch ppc64'
+    # when these are not supported. That makes it possible to
+    # build extensions on OSX 10.7 and later with the prebuilt
+    # 32-bit installer on the python.org website.
+
+    # skip checks if the compiler was overridden with a CC env variable
+    if 'CC' in os.environ:
+        return _config_vars
+
+    if re.search(r'-arch\s+ppc', _config_vars['CFLAGS']) is not None:
+        # NOTE: Cannot use subprocess here because of bootstrap
+        # issues when building Python itself
+        status = os.system(
+            """echo 'int main{};' | """
+            """'%s' -c -arch ppc -x c -o /dev/null /dev/null 2>/dev/null"""
+            %(_config_vars['CC'].replace("'", "'\"'\"'"),))
+        if status:
+            # The compile failed for some reason.  Because of differences
+            # across Xcode and compiler versions, there is no reliable way
+            # to be sure why it failed.  Assume here it was due to lack of
+            # PPC support and remove the related '-arch' flags from each
+            # config variables not explicitly overridden by an environment
+            # variable.  If the error was for some other reason, we hope the
+            # failure will show up again when trying to compile an extension
+            # module.
+            for cv in _UNIVERSAL_CONFIG_VARS:
+                if cv in _config_vars and cv not in os.environ:
+                    flags = _config_vars[cv]
+                    flags = re.sub(r'-arch\s+ppc\w*\s', ' ', flags)
+                    _save_modified_value(_config_vars, cv, flags)
+
+    return _config_vars
+
+
+def _override_all_archs(_config_vars):
+    """Allow override of all archs with ARCHFLAGS env var"""
+    # NOTE: This name was introduced by Apple in OSX 10.5 and
+    # is used by several scripting languages distributed with
+    # that OS release.
+    if 'ARCHFLAGS' in os.environ:
+        arch = os.environ['ARCHFLAGS']
+        for cv in _UNIVERSAL_CONFIG_VARS:
+            if cv in _config_vars and '-arch' in _config_vars[cv]:
+                flags = _config_vars[cv]
+                flags = re.sub(r'-arch\s+\w+\s', ' ', flags)
+                flags = flags + ' ' + arch
+                _save_modified_value(_config_vars, cv, flags)
+
+    return _config_vars
+
+
+def _check_for_unavailable_sdk(_config_vars):
+    """Remove references to any SDKs not available"""
+    # If we're on OSX 10.5 or later and the user tries to
+    # compile an extension using an SDK that is not present
+    # on the current machine it is better to not use an SDK
+    # than to fail.  This is particularly important with
+    # the standalone Command Line Tools alternative to a
+    # full-blown Xcode install since the CLT packages do not
+    # provide SDKs.  If the SDK is not present, it is assumed
+    # that the header files and dev libs have been installed
+    # to /usr and /System/Library by either a standalone CLT
+    # package or the CLT component within Xcode.
+    cflags = _config_vars.get('CFLAGS', '')
+    m = re.search(r'-isysroot\s+(\S+)', cflags)
+    if m is not None:
+        sdk = m.group(1)
+        if not os.path.exists(sdk):
+            for cv in _UNIVERSAL_CONFIG_VARS:
+                # Do not alter a config var explicitly overridden by env var
+                if cv in _config_vars and cv not in os.environ:
+                    flags = _config_vars[cv]
+                    flags = re.sub(r'-isysroot\s+\S+(?:\s|$)', ' ', flags)
+                    _save_modified_value(_config_vars, cv, flags)
+
+    return _config_vars
+
+
+def compiler_fixup(compiler_so, cc_args):
+    """
+    This function will strip '-isysroot PATH' and '-arch ARCH' from the
+    compile flags if the user has specified one them in extra_compile_flags.
+
+    This is needed because '-arch ARCH' adds another architecture to the
+    build, without a way to remove an architecture. Furthermore GCC will
+    barf if multiple '-isysroot' arguments are present.
+    """
+    stripArch = stripSysroot = False
+
+    compiler_so = list(compiler_so)
+
+    if not _supports_universal_builds():
+        # OSX before 10.4.0, these don't support -arch and -isysroot at
+        # all.
+        stripArch = stripSysroot = True
+    else:
+        stripArch = '-arch' in cc_args
+        stripSysroot = '-isysroot' in cc_args
+
+    if stripArch or 'ARCHFLAGS' in os.environ:
+        while True:
+            try:
+                index = compiler_so.index('-arch')
+                # Strip this argument and the next one:
+                del compiler_so[index:index+2]
+            except ValueError:
+                break
+
+    if 'ARCHFLAGS' in os.environ and not stripArch:
+        # User specified different -arch flags in the environ,
+        # see also distutils.sysconfig
+        compiler_so = compiler_so + os.environ['ARCHFLAGS'].split()
+
+    if stripSysroot:
+        while True:
+            try:
+                index = compiler_so.index('-isysroot')
+                # Strip this argument and the next one:
+                del compiler_so[index:index+2]
+            except ValueError:
+                break
+
+    # Check if the SDK that is used during compilation actually exists,
+    # the universal build requires the usage of a universal SDK and not all
+    # users have that installed by default.
+    sysroot = None
+    if '-isysroot' in cc_args:
+        idx = cc_args.index('-isysroot')
+        sysroot = cc_args[idx+1]
+    elif '-isysroot' in compiler_so:
+        idx = compiler_so.index('-isysroot')
+        sysroot = compiler_so[idx+1]
+
+    if sysroot and not os.path.isdir(sysroot):
+        from distutils import log
+        log.warn("Compiling with an SDK that doesn't seem to exist: %s",
+                sysroot)
+        log.warn("Please check your Xcode installation")
+
+    return compiler_so
+
+
+def customize_config_vars(_config_vars):
+    """Customize Python build configuration variables.
+
+    Called internally from sysconfig with a mutable mapping
+    containing name/value pairs parsed from the configured
+    makefile used to build this interpreter.  Returns
+    the mapping updated as needed to reflect the environment
+    in which the interpreter is running; in the case of
+    a Python from a binary installer, the installed
+    environment may be very different from the build
+    environment, i.e. different OS levels, different
+    built tools, different available CPU architectures.
+
+    This customization is performed whenever
+    distutils.sysconfig.get_config_vars() is first
+    called.  It may be used in environments where no
+    compilers are present, i.e. when installing pure
+    Python dists.  Customization of compiler paths
+    and detection of unavailable archs is deferred
+    until the first extension module build is
+    requested (in distutils.sysconfig.customize_compiler).
+
+    Currently called from distutils.sysconfig
+    """
+
+    if not _supports_universal_builds():
+        # On Mac OS X before 10.4, check if -arch and -isysroot
+        # are in CFLAGS or LDFLAGS and remove them if they are.
+        # This is needed when building extensions on a 10.3 system
+        # using a universal build of python.
+        _remove_universal_flags(_config_vars)
+
+    # Allow user to override all archs with ARCHFLAGS env var
+    _override_all_archs(_config_vars)
+
+    # Remove references to sdks that are not found
+    _check_for_unavailable_sdk(_config_vars)
+
+    return _config_vars
+
+
+def customize_compiler(_config_vars):
+    """Customize compiler path and configuration variables.
+
+    This customization is performed when the first
+    extension module build is requested
+    in distutils.sysconfig.customize_compiler).
+    """
+
+    # Find a compiler to use for extension module builds
+    _find_appropriate_compiler(_config_vars)
+
+    # Remove ppc arch flags if not supported here
+    _remove_unsupported_archs(_config_vars)
+
+    # Allow user to override all archs with ARCHFLAGS env var
+    _override_all_archs(_config_vars)
+
+    return _config_vars
+
+
+def get_platform_osx(_config_vars, osname, release, machine):
+    """Filter values for get_platform()"""
+    # called from get_platform() in sysconfig and distutils.util
+    #
+    # For our purposes, we'll assume that the system version from
+    # distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set
+    # to. This makes the compatibility story a bit more sane because the
+    # machine is going to compile and link as if it were
+    # MACOSX_DEPLOYMENT_TARGET.
+
+    macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '')
+    macrelease = _get_system_version() or macver
+    macver = macver or macrelease
+
+    if macver:
+        release = macver
+        osname = "macosx"
+
+        # Use the original CFLAGS value, if available, so that we
+        # return the same machine type for the platform string.
+        # Otherwise, distutils may consider this a cross-compiling
+        # case and disallow installs.
+        cflags = _config_vars.get(_INITPRE+'CFLAGS',
+                                    _config_vars.get('CFLAGS', ''))
+        if macrelease:
+            try:
+                macrelease = tuple(int(i) for i in macrelease.split('.')[0:2])
+            except ValueError:
+                macrelease = (10, 0)
+        else:
+            # assume no universal support
+            macrelease = (10, 0)
+
+        if (macrelease >= (10, 4)) and '-arch' in cflags.strip():
+            # The universal build will build fat binaries, but not on
+            # systems before 10.4
+
+            machine = 'fat'
+
+            archs = re.findall(r'-arch\s+(\S+)', cflags)
+            archs = tuple(sorted(set(archs)))
+
+            if len(archs) == 1:
+                machine = archs[0]
+            elif archs == ('i386', 'ppc'):
+                machine = 'fat'
+            elif archs == ('i386', 'x86_64'):
+                machine = 'intel'
+            elif archs == ('i386', 'ppc', 'x86_64'):
+                machine = 'fat3'
+            elif archs == ('ppc64', 'x86_64'):
+                machine = 'fat64'
+            elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'):
+                machine = 'universal'
+            else:
+                raise ValueError(
+                   "Don't know machine value for archs=%r" % (archs,))
+
+        elif machine == 'i386':
+            # On OSX the machine type returned by uname is always the
+            # 32-bit variant, even if the executable architecture is
+            # the 64-bit variant
+            if sys.maxsize >= 2**32:
+                machine = 'x86_64'
+
+        elif machine in ('PowerPC', 'Power_Macintosh'):
+            # Pick a sane name for the PPC architecture.
+            # See 'i386' case
+            if sys.maxsize >= 2**32:
+                machine = 'ppc64'
+            else:
+                machine = 'ppc'
+
+    return (osname, release, machine)

Lib/cgi.py

@@ -0,0 +1,992 @@
+#! /usr/local/bin/python
+
+# NOTE: the above "/usr/local/bin/python" is NOT a mistake.  It is
+# intentionally NOT "/usr/bin/env python".  On many systems
+# (e.g. Solaris), /usr/local/bin is not in $PATH as passed to CGI
+# scripts, and /usr/local/bin is the default directory where Python is
+# installed, so /usr/bin/env would be unable to find python.  Granted,
+# binary installations by Linux vendors often install Python in
+# /usr/bin.  So let those vendors patch cgi.py to match their choice
+# of installation.
+
+"""Support module for CGI (Common Gateway Interface) scripts.
+
+This module defines a number of utilities for use by CGI scripts
+written in Python.
+"""
+
+# History
+# -------
+#
+# Michael McLay started this module.  Steve Majewski changed the
+# interface to SvFormContentDict and FormContentDict.  The multipart
+# parsing was inspired by code submitted by Andreas Paepcke.  Guido van
+# Rossum rewrote, reformatted and documented the module and is currently
+# responsible for its maintenance.
+#
+
+__version__ = "2.6"
+
+
+# Imports
+# =======
+
+from io import StringIO, BytesIO, TextIOWrapper
+from collections.abc import Mapping
+import sys
+import os
+import urllib.parse
+from email.parser import FeedParser
+from email.message import Message
+import html
+import locale
+import tempfile
+
+__all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_multipart",
+           "parse_header", "test", "print_exception", "print_environ",
+           "print_form", "print_directory", "print_arguments",
+           "print_environ_usage"]
+
+# Logging support
+# ===============
+
+logfile = ""            # Filename to log to, if not empty
+logfp = None            # File object to log to, if not None
+
+def initlog(*allargs):
+    """Write a log message, if there is a log file.
+
+    Even though this function is called initlog(), you should always
+    use log(); log is a variable that is set either to initlog
+    (initially), to dolog (once the log file has been opened), or to
+    nolog (when logging is disabled).
+
+    The first argument is a format string; the remaining arguments (if
+    any) are arguments to the % operator, so e.g.
+        log("%s: %s", "a", "b")
+    will write "a: b" to the log file, followed by a newline.
+
+    If the global logfp is not None, it should be a file object to
+    which log data is written.
+
+    If the global logfp is None, the global logfile may be a string
+    giving a filename to open, in append mode.  This file should be
+    world writable!!!  If the file can't be opened, logging is
+    silently disabled (since there is no safe place where we could
+    send an error message).
+
+    """
+    global log, logfile, logfp
+    if logfile and not logfp:
+        try:
+            logfp = open(logfile, "a")
+        except OSError:
+            pass
+    if not logfp:
+        log = nolog
+    else:
+        log = dolog
+    log(*allargs)
+
+def dolog(fmt, *args):
+    """Write a log message to the log file.  See initlog() for docs."""
+    logfp.write(fmt%args + "\n")
+
+def nolog(*allargs):
+    """Dummy function, assigned to log when logging is disabled."""
+    pass
+
+def closelog():
+    """Close the log file."""
+    global log, logfile, logfp
+    logfile = ''
+    if logfp:
+        logfp.close()
+        logfp = None
+    log = initlog
+
+log = initlog           # The current logging function
+
+
+# Parsing functions
+# =================
+
+# Maximum input we will accept when REQUEST_METHOD is POST
+# 0 ==> unlimited input
+maxlen = 0
+
+def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0):
+    """Parse a query in the environment or from a file (default stdin)
+
+        Arguments, all optional:
+
+        fp              : file pointer; default: sys.stdin.buffer
+
+        environ         : environment dictionary; default: os.environ
+
+        keep_blank_values: flag indicating whether blank values in
+            percent-encoded forms should be treated as blank strings.
+            A true value indicates that blanks should be retained as
+            blank strings.  The default false value indicates that
+            blank values are to be ignored and treated as if they were
+            not included.
+
+        strict_parsing: flag indicating what to do with parsing errors.
+            If false (the default), errors are silently ignored.
+            If true, errors raise a ValueError exception.
+    """
+    if fp is None:
+        fp = sys.stdin
+
+    # field keys and values (except for files) are returned as strings
+    # an encoding is required to decode the bytes read from self.fp
+    if hasattr(fp,'encoding'):
+        encoding = fp.encoding
+    else:
+        encoding = 'latin-1'
+
+    # fp.read() must return bytes
+    if isinstance(fp, TextIOWrapper):
+        fp = fp.buffer
+
+    if not 'REQUEST_METHOD' in environ:
+        environ['REQUEST_METHOD'] = 'GET'       # For testing stand-alone
+    if environ['REQUEST_METHOD'] == 'POST':
+        ctype, pdict = parse_header(environ['CONTENT_TYPE'])
+        if ctype == 'multipart/form-data':
+            return parse_multipart(fp, pdict)
+        elif ctype == 'application/x-www-form-urlencoded':
+            clength = int(environ['CONTENT_LENGTH'])
+            if maxlen and clength > maxlen:
+                raise ValueError('Maximum content length exceeded')
+            qs = fp.read(clength).decode(encoding)
+        else:
+            qs = ''                     # Unknown content-type
+        if 'QUERY_STRING' in environ:
+            if qs: qs = qs + '&'
+            qs = qs + environ['QUERY_STRING']
+        elif sys.argv[1:]:
+            if qs: qs = qs + '&'
+            qs = qs + sys.argv[1]
+        environ['QUERY_STRING'] = qs    # XXX Shouldn't, really
+    elif 'QUERY_STRING' in environ:
+        qs = environ['QUERY_STRING']
+    else:
+        if sys.argv[1:]:
+            qs = sys.argv[1]
+        else:
+            qs = ""
+        environ['QUERY_STRING'] = qs    # XXX Shouldn't, really
+    return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing,
+                                 encoding=encoding)
+
+
+def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"):
+    """Parse multipart input.
+
+    Arguments:
+    fp   : input file
+    pdict: dictionary containing other parameters of content-type header
+    encoding, errors: request encoding and error handler, passed to
+        FieldStorage
+
+    Returns a dictionary just like parse_qs(): keys are the field names, each
+    value is a list of values for that field. For non-file fields, the value
+    is a list of strings.
+    """
+    # RFC 2026, Section 5.1 : The "multipart" boundary delimiters are always
+    # represented as 7bit US-ASCII.
+    boundary = pdict['boundary'].decode('ascii')
+    ctype = "multipart/form-data; boundary={}".format(boundary)
+    headers = Message()
+    headers.set_type(ctype)
+    headers['Content-Length'] = pdict['CONTENT-LENGTH']
+    fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors,
+        environ={'REQUEST_METHOD': 'POST'})
+    return {k: fs.getlist(k) for k in fs}
+
+def _parseparam(s):
+    while s[:1] == ';':
+        s = s[1:]
+        end = s.find(';')
+        while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2:
+            end = s.find(';', end + 1)
+        if end < 0:
+            end = len(s)
+        f = s[:end]
+        yield f.strip()
+        s = s[end:]
+
+def parse_header(line):
+    """Parse a Content-type like header.
+
+    Return the main content-type and a dictionary of options.
+
+    """
+    parts = _parseparam(';' + line)
+    key = parts.__next__()
+    pdict = {}
+    for p in parts:
+        i = p.find('=')
+        if i >= 0:
+            name = p[:i].strip().lower()
+            value = p[i+1:].strip()
+            if len(value) >= 2 and value[0] == value[-1] == '"':
+                value = value[1:-1]
+                value = value.replace('\\\\', '\\').replace('\\"', '"')
+            pdict[name] = value
+    return key, pdict
+
+
+# Classes for field storage
+# =========================
+
+class MiniFieldStorage:
+
+    """Like FieldStorage, for use when no file uploads are possible."""
+
+    # Dummy attributes
+    filename = None
+    list = None
+    type = None
+    file = None
+    type_options = {}
+    disposition = None
+    disposition_options = {}
+    headers = {}
+
+    def __init__(self, name, value):
+        """Constructor from field name and value."""
+        self.name = name
+        self.value = value
+        # self.file = StringIO(value)
+
+    def __repr__(self):
+        """Return printable representation."""
+        return "MiniFieldStorage(%r, %r)" % (self.name, self.value)
+
+
+class FieldStorage:
+
+    """Store a sequence of fields, reading multipart/form-data.
+
+    This class provides naming, typing, files stored on disk, and
+    more.  At the top level, it is accessible like a dictionary, whose
+    keys are the field names.  (Note: None can occur as a field name.)
+    The items are either a Python list (if there's multiple values) or
+    another FieldStorage or MiniFieldStorage object.  If it's a single
+    object, it has the following attributes:
+
+    name: the field name, if specified; otherwise None
+
+    filename: the filename, if specified; otherwise None; this is the
+        client side filename, *not* the file name on which it is
+        stored (that's a temporary file you don't deal with)
+
+    value: the value as a *string*; for file uploads, this
+        transparently reads the file every time you request the value
+        and returns *bytes*
+
+    file: the file(-like) object from which you can read the data *as
+        bytes* ; None if the data is stored a simple string
+
+    type: the content-type, or None if not specified
+
+    type_options: dictionary of options specified on the content-type
+        line
+
+    disposition: content-disposition, or None if not specified
+
+    disposition_options: dictionary of corresponding options
+
+    headers: a dictionary(-like) object (sometimes email.message.Message or a
+        subclass thereof) containing *all* headers
+
+    The class is subclassable, mostly for the purpose of overriding
+    the make_file() method, which is called internally to come up with
+    a file open for reading and writing.  This makes it possible to
+    override the default choice of storing all files in a temporary
+    directory and unlinking them as soon as they have been opened.
+
+    """
+    def __init__(self, fp=None, headers=None, outerboundary=b'',
+                 environ=os.environ, keep_blank_values=0, strict_parsing=0,
+                 limit=None, encoding='utf-8', errors='replace',
+                 max_num_fields=None):
+        """Constructor.  Read multipart/* until last part.
+
+        Arguments, all optional:
+
+        fp              : file pointer; default: sys.stdin.buffer
+            (not used when the request method is GET)
+            Can be :
+            1. a TextIOWrapper object
+            2. an object whose read() and readline() methods return bytes
+
+        headers         : header dictionary-like object; default:
+            taken from environ as per CGI spec
+
+        outerboundary   : terminating multipart boundary
+            (for internal use only)
+
+        environ         : environment dictionary; default: os.environ
+
+        keep_blank_values: flag indicating whether blank values in
+            percent-encoded forms should be treated as blank strings.
+            A true value indicates that blanks should be retained as
+            blank strings.  The default false value indicates that
+            blank values are to be ignored and treated as if they were
+            not included.
+
+        strict_parsing: flag indicating what to do with parsing errors.
+            If false (the default), errors are silently ignored.
+            If true, errors raise a ValueError exception.
+
+        limit : used internally to read parts of multipart/form-data forms,
+            to exit from the reading loop when reached. It is the difference
+            between the form content-length and the number of bytes already
+            read
+
+        encoding, errors : the encoding and error handler used to decode the
+            binary stream to strings. Must be the same as the charset defined
+            for the page sending the form (content-type : meta http-equiv or
+            header)
+
+        max_num_fields: int. If set, then __init__ throws a ValueError
+            if there are more than n fields read by parse_qsl().
+
+        """
+        method = 'GET'
+        self.keep_blank_values = keep_blank_values
+        self.strict_parsing = strict_parsing
+        self.max_num_fields = max_num_fields
+        if 'REQUEST_METHOD' in environ:
+            method = environ['REQUEST_METHOD'].upper()
+        self.qs_on_post = None
+        if method == 'GET' or method == 'HEAD':
+            if 'QUERY_STRING' in environ:
+                qs = environ['QUERY_STRING']
+            elif sys.argv[1:]:
+                qs = sys.argv[1]
+            else:
+                qs = ""
+            qs = qs.encode(locale.getpreferredencoding(), 'surrogateescape')
+            fp = BytesIO(qs)
+            if headers is None:
+                headers = {'content-type':
+                           "application/x-www-form-urlencoded"}
+        if headers is None:
+            headers = {}
+            if method == 'POST':
+                # Set default content-type for POST to what's traditional
+                headers['content-type'] = "application/x-www-form-urlencoded"
+            if 'CONTENT_TYPE' in environ:
+                headers['content-type'] = environ['CONTENT_TYPE']
+            if 'QUERY_STRING' in environ:
+                self.qs_on_post = environ['QUERY_STRING']
+            if 'CONTENT_LENGTH' in environ:
+                headers['content-length'] = environ['CONTENT_LENGTH']
+        else:
+            if not (isinstance(headers, (Mapping, Message))):
+                raise TypeError("headers must be mapping or an instance of "
+                                "email.message.Message")
+        self.headers = headers
+        if fp is None:
+            self.fp = sys.stdin.buffer
+        # self.fp.read() must return bytes
+        elif isinstance(fp, TextIOWrapper):
+            self.fp = fp.buffer
+        else:
+            if not (hasattr(fp, 'read') and hasattr(fp, 'readline')):
+                raise TypeError("fp must be file pointer")
+            self.fp = fp
+
+        self.encoding = encoding
+        self.errors = errors
+
+        if not isinstance(outerboundary, bytes):
+            raise TypeError('outerboundary must be bytes, not %s'
+                            % type(outerboundary).__name__)
+        self.outerboundary = outerboundary
+
+        self.bytes_read = 0
+        self.limit = limit
+
+        # Process content-disposition header
+        cdisp, pdict = "", {}
+        if 'content-disposition' in self.headers:
+            cdisp, pdict = parse_header(self.headers['content-disposition'])
+        self.disposition = cdisp
+        self.disposition_options = pdict
+        self.name = None
+        if 'name' in pdict:
+            self.name = pdict['name']
+        self.filename = None
+        if 'filename' in pdict:
+            self.filename = pdict['filename']
+        self._binary_file = self.filename is not None
+
+        # Process content-type header
+        #
+        # Honor any existing content-type header.  But if there is no
+        # content-type header, use some sensible defaults.  Assume
+        # outerboundary is "" at the outer level, but something non-false
+        # inside a multi-part.  The default for an inner part is text/plain,
+        # but for an outer part it should be urlencoded.  This should catch
+        # bogus clients which erroneously forget to include a content-type
+        # header.
+        #
+        # See below for what we do if there does exist a content-type header,
+        # but it happens to be something we don't understand.
+        if 'content-type' in self.headers:
+            ctype, pdict = parse_header(self.headers['content-type'])
+        elif self.outerboundary or method != 'POST':
+            ctype, pdict = "text/plain", {}
+        else:
+            ctype, pdict = 'application/x-www-form-urlencoded', {}
+        self.type = ctype
+        self.type_options = pdict
+        if 'boundary' in pdict:
+            self.innerboundary = pdict['boundary'].encode(self.encoding,
+                                                          self.errors)
+        else:
+            self.innerboundary = b""
+
+        clen = -1
+        if 'content-length' in self.headers:
+            try:
+                clen = int(self.headers['content-length'])
+            except ValueError:
+                pass
+            if maxlen and clen > maxlen:
+                raise ValueError('Maximum content length exceeded')
+        self.length = clen
+        if self.limit is None and clen >= 0:
+            self.limit = clen
+
+        self.list = self.file = None
+        self.done = 0
+        if ctype == 'application/x-www-form-urlencoded':
+            self.read_urlencoded()
+        elif ctype[:10] == 'multipart/':
+            self.read_multi(environ, keep_blank_values, strict_parsing)
+        else:
+            self.read_single()
+
+    def __del__(self):
+        try:
+            self.file.close()
+        except AttributeError:
+            pass
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.file.close()
+
+    def __repr__(self):
+        """Return a printable representation."""
+        return "FieldStorage(%r, %r, %r)" % (
+                self.name, self.filename, self.value)
+
+    def __iter__(self):
+        return iter(self.keys())
+
+    def __getattr__(self, name):
+        if name != 'value':
+            raise AttributeError(name)
+        if self.file:
+            self.file.seek(0)
+            value = self.file.read()
+            self.file.seek(0)
+        elif self.list is not None:
+            value = self.list
+        else:
+            value = None
+        return value
+
+    def __getitem__(self, key):
+        """Dictionary style indexing."""
+        if self.list is None:
+            raise TypeError("not indexable")
+        found = []
+        for item in self.list:
+            if item.name == key: found.append(item)
+        if not found:
+            raise KeyError(key)
+        if len(found) == 1:
+            return found[0]
+        else:
+            return found
+
+    def getvalue(self, key, default=None):
+        """Dictionary style get() method, including 'value' lookup."""
+        if key in self:
+            value = self[key]
+            if isinstance(value, list):
+                return [x.value for x in value]
+            else:
+                return value.value
+        else:
+            return default
+
+    def getfirst(self, key, default=None):
+        """ Return the first value received."""
+        if key in self:
+            value = self[key]
+            if isinstance(value, list):
+                return value[0].value
+            else:
+                return value.value
+        else:
+            return default
+
+    def getlist(self, key):
+        """ Return list of received values."""
+        if key in self:
+            value = self[key]
+            if isinstance(value, list):
+                return [x.value for x in value]
+            else:
+                return [value.value]
+        else:
+            return []
+
+    def keys(self):
+        """Dictionary style keys() method."""
+        if self.list is None:
+            raise TypeError("not indexable")
+        return list(set(item.name for item in self.list))
+
+    def __contains__(self, key):
+        """Dictionary style __contains__ method."""
+        if self.list is None:
+            raise TypeError("not indexable")
+        return any(item.name == key for item in self.list)
+
+    def __len__(self):
+        """Dictionary style len(x) support."""
+        return len(self.keys())
+
+    def __bool__(self):
+        if self.list is None:
+            raise TypeError("Cannot be converted to bool.")
+        return bool(self.list)
+
+    def read_urlencoded(self):
+        """Internal: read data in query string format."""
+        qs = self.fp.read(self.length)
+        if not isinstance(qs, bytes):
+            raise ValueError("%s should return bytes, got %s" \
+                             % (self.fp, type(qs).__name__))
+        qs = qs.decode(self.encoding, self.errors)
+        if self.qs_on_post:
+            qs += '&' + self.qs_on_post
+        query = urllib.parse.parse_qsl(
+            qs, self.keep_blank_values, self.strict_parsing,
+            encoding=self.encoding, errors=self.errors,
+            max_num_fields=self.max_num_fields)
+        self.list = [MiniFieldStorage(key, value) for key, value in query]
+        self.skip_lines()
+
+    FieldStorageClass = None
+
+    def read_multi(self, environ, keep_blank_values, strict_parsing):
+        """Internal: read a part that is itself multipart."""
+        ib = self.innerboundary
+        if not valid_boundary(ib):
+            raise ValueError('Invalid boundary in multipart form: %r' % (ib,))
+        self.list = []
+        if self.qs_on_post:
+            query = urllib.parse.parse_qsl(
+                self.qs_on_post, self.keep_blank_values, self.strict_parsing,
+                encoding=self.encoding, errors=self.errors,
+                max_num_fields=self.max_num_fields)
+            self.list.extend(MiniFieldStorage(key, value) for key, value in query)
+
+        klass = self.FieldStorageClass or self.__class__
+        first_line = self.fp.readline() # bytes
+        if not isinstance(first_line, bytes):
+            raise ValueError("%s should return bytes, got %s" \
+                             % (self.fp, type(first_line).__name__))
+        self.bytes_read += len(first_line)
+
+        # Ensure that we consume the file until we've hit our inner boundary
+        while (first_line.strip() != (b"--" + self.innerboundary) and
+                first_line):
+            first_line = self.fp.readline()
+            self.bytes_read += len(first_line)
+
+        # Propagate max_num_fields into the sub class appropriately
+        max_num_fields = self.max_num_fields
+        if max_num_fields is not None:
+            max_num_fields -= len(self.list)
+
+        while True:
+            parser = FeedParser()
+            hdr_text = b""
+            while True:
+                data = self.fp.readline()
+                hdr_text += data
+                if not data.strip():
+                    break
+            if not hdr_text:
+                break
+            # parser takes strings, not bytes
+            self.bytes_read += len(hdr_text)
+            parser.feed(hdr_text.decode(self.encoding, self.errors))
+            headers = parser.close()
+
+            # Some clients add Content-Length for part headers, ignore them
+            if 'content-length' in headers:
+                del headers['content-length']
+
+            limit = None if self.limit is None \
+                else self.limit - self.bytes_read
+            part = klass(self.fp, headers, ib, environ, keep_blank_values,
+                         strict_parsing, limit,
+                         self.encoding, self.errors, max_num_fields)
+
+            if max_num_fields is not None:
+                max_num_fields -= 1
+                if part.list:
+                    max_num_fields -= len(part.list)
+                if max_num_fields < 0:
+                    raise ValueError('Max number of fields exceeded')
+
+            self.bytes_read += part.bytes_read
+            self.list.append(part)
+            if part.done or self.bytes_read >= self.length > 0:
+                break
+        self.skip_lines()
+
+    def read_single(self):
+        """Internal: read an atomic part."""
+        if self.length >= 0:
+            self.read_binary()
+            self.skip_lines()
+        else:
+            self.read_lines()
+        self.file.seek(0)
+
+    bufsize = 8*1024            # I/O buffering size for copy to file
+
+    def read_binary(self):
+        """Internal: read binary data."""
+        self.file = self.make_file()
+        todo = self.length
+        if todo >= 0:
+            while todo > 0:
+                data = self.fp.read(min(todo, self.bufsize)) # bytes
+                if not isinstance(data, bytes):
+                    raise ValueError("%s should return bytes, got %s"
+                                     % (self.fp, type(data).__name__))
+                self.bytes_read += len(data)
+                if not data:
+                    self.done = -1
+                    break
+                self.file.write(data)
+                todo = todo - len(data)
+
+    def read_lines(self):
+        """Internal: read lines until EOF or outerboundary."""
+        if self._binary_file:
+            self.file = self.__file = BytesIO() # store data as bytes for files
+        else:
+            self.file = self.__file = StringIO() # as strings for other fields
+        if self.outerboundary:
+            self.read_lines_to_outerboundary()
+        else:
+            self.read_lines_to_eof()
+
+    def __write(self, line):
+        """line is always bytes, not string"""
+        if self.__file is not None:
+            if self.__file.tell() + len(line) > 1000:
+                self.file = self.make_file()
+                data = self.__file.getvalue()
+                self.file.write(data)
+                self.__file = None
+        if self._binary_file:
+            # keep bytes
+            self.file.write(line)
+        else:
+            # decode to string
+            self.file.write(line.decode(self.encoding, self.errors))
+
+    def read_lines_to_eof(self):
+        """Internal: read lines until EOF."""
+        while 1:
+            line = self.fp.readline(1<<16) # bytes
+            self.bytes_read += len(line)
+            if not line:
+                self.done = -1
+                break
+            self.__write(line)
+
+    def read_lines_to_outerboundary(self):
+        """Internal: read lines until outerboundary.
+        Data is read as bytes: boundaries and line ends must be converted
+        to bytes for comparisons.
+        """
+        next_boundary = b"--" + self.outerboundary
+        last_boundary = next_boundary + b"--"
+        delim = b""
+        last_line_lfend = True
+        _read = 0
+        while 1:
+            if self.limit is not None and _read >= self.limit:
+                break
+            line = self.fp.readline(1<<16) # bytes
+            self.bytes_read += len(line)
+            _read += len(line)
+            if not line:
+                self.done = -1
+                break
+            if delim == b"\r":
+                line = delim + line
+                delim = b""
+            if line.startswith(b"--") and last_line_lfend:
+                strippedline = line.rstrip()
+                if strippedline == next_boundary:
+                    break
+                if strippedline == last_boundary:
+                    self.done = 1
+                    break
+            odelim = delim
+            if line.endswith(b"\r\n"):
+                delim = b"\r\n"
+                line = line[:-2]
+                last_line_lfend = True
+            elif line.endswith(b"\n"):
+                delim = b"\n"
+                line = line[:-1]
+                last_line_lfend = True
+            elif line.endswith(b"\r"):
+                # We may interrupt \r\n sequences if they span the 2**16
+                # byte boundary
+                delim = b"\r"
+                line = line[:-1]
+                last_line_lfend = False
+            else:
+                delim = b""
+                last_line_lfend = False
+            self.__write(odelim + line)
+
+    def skip_lines(self):
+        """Internal: skip lines until outer boundary if defined."""
+        if not self.outerboundary or self.done:
+            return
+        next_boundary = b"--" + self.outerboundary
+        last_boundary = next_boundary + b"--"
+        last_line_lfend = True
+        while True:
+            line = self.fp.readline(1<<16)
+            self.bytes_read += len(line)
+            if not line:
+                self.done = -1
+                break
+            if line.endswith(b"--") and last_line_lfend:
+                strippedline = line.strip()
+                if strippedline == next_boundary:
+                    break
+                if strippedline == last_boundary:
+                    self.done = 1
+                    break
+            last_line_lfend = line.endswith(b'\n')
+
+    def make_file(self):
+        """Overridable: return a readable & writable file.
+
+        The file will be used as follows:
+        - data is written to it
+        - seek(0)
+        - data is read from it
+
+        The file is opened in binary mode for files, in text mode
+        for other fields
+
+        This version opens a temporary file for reading and writing,
+        and immediately deletes (unlinks) it.  The trick (on Unix!) is
+        that the file can still be used, but it can't be opened by
+        another process, and it will automatically be deleted when it
+        is closed or when the current process terminates.
+
+        If you want a more permanent file, you derive a class which
+        overrides this method.  If you want a visible temporary file
+        that is nevertheless automatically deleted when the script
+        terminates, try defining a __del__ method in a derived class
+        which unlinks the temporary files you have created.
+
+        """
+        if self._binary_file:
+            return tempfile.TemporaryFile("wb+")
+        else:
+            return tempfile.TemporaryFile("w+",
+                encoding=self.encoding, newline = '\n')
+
+
+# Test/debug code
+# ===============
+
+def test(environ=os.environ):
+    """Robust test CGI script, usable as main program.
+
+    Write minimal HTTP headers and dump all information provided to
+    the script in HTML form.
+
+    """
+    print("Content-type: text/html")
+    print()
+    sys.stderr = sys.stdout
+    try:
+        form = FieldStorage()   # Replace with other classes to test those
+        print_directory()
+        print_arguments()
+        print_form(form)
+        print_environ(environ)
+        print_environ_usage()
+        def f():
+            exec("testing print_exception() -- <I>italics?</I>")
+        def g(f=f):
+            f()
+        print("<H3>What follows is a test, not an actual exception:</H3>")
+        g()
+    except:
+        print_exception()
+
+    print("<H1>Second try with a small maxlen...</H1>")
+
+    global maxlen
+    maxlen = 50
+    try:
+        form = FieldStorage()   # Replace with other classes to test those
+        print_directory()
+        print_arguments()
+        print_form(form)
+        print_environ(environ)
+    except:
+        print_exception()
+
+def print_exception(type=None, value=None, tb=None, limit=None):
+    if type is None:
+        type, value, tb = sys.exc_info()
+    import traceback
+    print()
+    print("<H3>Traceback (most recent call last):</H3>")
+    list = traceback.format_tb(tb, limit) + \
+           traceback.format_exception_only(type, value)
+    print("<PRE>%s<B>%s</B></PRE>" % (
+        html.escape("".join(list[:-1])),
+        html.escape(list[-1]),
+        ))
+    del tb
+
+def print_environ(environ=os.environ):
+    """Dump the shell environment as HTML."""
+    keys = sorted(environ.keys())
+    print()
+    print("<H3>Shell Environment:</H3>")
+    print("<DL>")
+    for key in keys:
+        print("<DT>", html.escape(key), "<DD>", html.escape(environ[key]))
+    print("</DL>")
+    print()
+
+def print_form(form):
+    """Dump the contents of a form as HTML."""
+    keys = sorted(form.keys())
+    print()
+    print("<H3>Form Contents:</H3>")
+    if not keys:
+        print("<P>No form fields.")
+    print("<DL>")
+    for key in keys:
+        print("<DT>" + html.escape(key) + ":", end=' ')
+        value = form[key]
+        print("<i>" + html.escape(repr(type(value))) + "</i>")
+        print("<DD>" + html.escape(repr(value)))
+    print("</DL>")
+    print()
+
+def print_directory():
+    """Dump the current directory as HTML."""
+    print()
+    print("<H3>Current Working Directory:</H3>")
+    try:
+        pwd = os.getcwd()
+    except OSError as msg:
+        print("OSError:", html.escape(str(msg)))
+    else:
+        print(html.escape(pwd))
+    print()
+
+def print_arguments():
+    print()
+    print("<H3>Command Line Arguments:</H3>")
+    print()
+    print(sys.argv)
+    print()
+
+def print_environ_usage():
+    """Dump a list of environment variables used by CGI as HTML."""
+    print("""
+<H3>These environment variables could have been set:</H3>
+<UL>
+<LI>AUTH_TYPE
+<LI>CONTENT_LENGTH
+<LI>CONTENT_TYPE
+<LI>DATE_GMT
+<LI>DATE_LOCAL
+<LI>DOCUMENT_NAME
+<LI>DOCUMENT_ROOT
+<LI>DOCUMENT_URI
+<LI>GATEWAY_INTERFACE
+<LI>LAST_MODIFIED
+<LI>PATH
+<LI>PATH_INFO
+<LI>PATH_TRANSLATED
+<LI>QUERY_STRING
+<LI>REMOTE_ADDR
+<LI>REMOTE_HOST
+<LI>REMOTE_IDENT
+<LI>REMOTE_USER
+<LI>REQUEST_METHOD
+<LI>SCRIPT_NAME
+<LI>SERVER_NAME
+<LI>SERVER_PORT
+<LI>SERVER_PROTOCOL
+<LI>SERVER_ROOT
+<LI>SERVER_SOFTWARE
+</UL>
+In addition, HTTP headers sent by the server may be passed in the
+environment as well.  Here are some common variable names:
+<UL>
+<LI>HTTP_ACCEPT
+<LI>HTTP_CONNECTION
+<LI>HTTP_HOST
+<LI>HTTP_PRAGMA
+<LI>HTTP_REFERER
+<LI>HTTP_USER_AGENT
+</UL>
+""")
+
+
+# Utilities
+# =========
+
+def valid_boundary(s):
+    import re
+    if isinstance(s, bytes):
+        _vb_pattern = b"^[ -~]{0,200}[!-~]$"
+    else:
+        _vb_pattern = "^[ -~]{0,200}[!-~]$"
+    return re.match(_vb_pattern, s)
+
+# Invoke mainline
+# ===============
+
+# Call test() when this file is run as a script (not imported as a module)
+if __name__ == '__main__':
+    test()

Lib/imp.py

@@ -0,0 +1,345 @@
+"""This module provides the components needed to build your own __import__
+function.  Undocumented functions are obsolete.
+
+In most cases it is preferred you consider using the importlib module's
+functionality over this module.
+
+"""
+# (Probably) need to stay in _imp
+from _imp import (lock_held, acquire_lock, release_lock,
+                  get_frozen_object, is_frozen_package,
+                  init_frozen, is_builtin, is_frozen,
+                  _fix_co_filename)
+try:
+    from _imp import create_dynamic
+except ImportError:
+    # Platform doesn't support dynamic loading.
+    create_dynamic = None
+
+from importlib._bootstrap import _ERR_MSG, _exec, _load, _builtin_from_name
+from importlib._bootstrap_external import SourcelessFileLoader
+
+from importlib import machinery
+from importlib import util
+import importlib
+import os
+import sys
+import tokenize
+import types
+import warnings
+
+warnings.warn("the imp module is deprecated in favour of importlib; "
+              "see the module's documentation for alternative uses",
+              DeprecationWarning, stacklevel=2)
+
+# DEPRECATED
+SEARCH_ERROR = 0
+PY_SOURCE = 1
+PY_COMPILED = 2
+C_EXTENSION = 3
+PY_RESOURCE = 4
+PKG_DIRECTORY = 5
+C_BUILTIN = 6
+PY_FROZEN = 7
+PY_CODERESOURCE = 8
+IMP_HOOK = 9
+
+
+def new_module(name):
+    """**DEPRECATED**
+
+    Create a new module.
+
+    The module is not entered into sys.modules.
+
+    """
+    return types.ModuleType(name)
+
+
+def get_magic():
+    """**DEPRECATED**
+
+    Return the magic number for .pyc files.
+    """
+    return util.MAGIC_NUMBER
+
+
+def get_tag():
+    """Return the magic tag for .pyc files."""
+    return sys.implementation.cache_tag
+
+
+def cache_from_source(path, debug_override=None):
+    """**DEPRECATED**
+
+    Given the path to a .py file, return the path to its .pyc file.
+
+    The .py file does not need to exist; this simply returns the path to the
+    .pyc file calculated as if the .py file were imported.
+
+    If debug_override is not None, then it must be a boolean and is used in
+    place of sys.flags.optimize.
+
+    If sys.implementation.cache_tag is None then NotImplementedError is raised.
+
+    """
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore')
+        return util.cache_from_source(path, debug_override)
+
+
+def source_from_cache(path):
+    """**DEPRECATED**
+
+    Given the path to a .pyc. file, return the path to its .py file.
+
+    The .pyc file does not need to exist; this simply returns the path to
+    the .py file calculated to correspond to the .pyc file.  If path does
+    not conform to PEP 3147 format, ValueError will be raised. If
+    sys.implementation.cache_tag is None then NotImplementedError is raised.
+
+    """
+    return util.source_from_cache(path)
+
+
+def get_suffixes():
+    """**DEPRECATED**"""
+    extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES]
+    source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES]
+    bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES]
+
+    return extensions + source + bytecode
+
+
+class NullImporter:
+
+    """**DEPRECATED**
+
+    Null import object.
+
+    """
+
+    def __init__(self, path):
+        if path == '':
+            raise ImportError('empty pathname', path='')
+        elif os.path.isdir(path):
+            raise ImportError('existing directory', path=path)
+
+    def find_module(self, fullname):
+        """Always returns None."""
+        return None
+
+
+class _HackedGetData:
+
+    """Compatibility support for 'file' arguments of various load_*()
+    functions."""
+
+    def __init__(self, fullname, path, file=None):
+        super().__init__(fullname, path)
+        self.file = file
+
+    def get_data(self, path):
+        """Gross hack to contort loader to deal w/ load_*()'s bad API."""
+        if self.file and path == self.path:
+            # The contract of get_data() requires us to return bytes. Reopen the
+            # file in binary mode if needed.
+            if not self.file.closed:
+                file = self.file
+                if 'b' not in file.mode:
+                    file.close()
+            if self.file.closed:
+                self.file = file = open(self.path, 'rb')
+
+            with file:
+                return file.read()
+        else:
+            return super().get_data(path)
+
+
+class _LoadSourceCompatibility(_HackedGetData, machinery.SourceFileLoader):
+
+    """Compatibility support for implementing load_source()."""
+
+
+def load_source(name, pathname, file=None):
+    loader = _LoadSourceCompatibility(name, pathname, file)
+    spec = util.spec_from_file_location(name, pathname, loader=loader)
+    if name in sys.modules:
+        module = _exec(spec, sys.modules[name])
+    else:
+        module = _load(spec)
+    # To allow reloading to potentially work, use a non-hacked loader which
+    # won't rely on a now-closed file object.
+    module.__loader__ = machinery.SourceFileLoader(name, pathname)
+    module.__spec__.loader = module.__loader__
+    return module
+
+
+class _LoadCompiledCompatibility(_HackedGetData, SourcelessFileLoader):
+
+    """Compatibility support for implementing load_compiled()."""
+
+
+def load_compiled(name, pathname, file=None):
+    """**DEPRECATED**"""
+    loader = _LoadCompiledCompatibility(name, pathname, file)
+    spec = util.spec_from_file_location(name, pathname, loader=loader)
+    if name in sys.modules:
+        module = _exec(spec, sys.modules[name])
+    else:
+        module = _load(spec)
+    # To allow reloading to potentially work, use a non-hacked loader which
+    # won't rely on a now-closed file object.
+    module.__loader__ = SourcelessFileLoader(name, pathname)
+    module.__spec__.loader = module.__loader__
+    return module
+
+
+def load_package(name, path):
+    """**DEPRECATED**"""
+    if os.path.isdir(path):
+        extensions = (machinery.SOURCE_SUFFIXES[:] +
+                      machinery.BYTECODE_SUFFIXES[:])
+        for extension in extensions:
+            init_path = os.path.join(path, '__init__' + extension)
+            if os.path.exists(init_path):
+                path = init_path
+                break
+        else:
+            raise ValueError('{!r} is not a package'.format(path))
+    spec = util.spec_from_file_location(name, path,
+                                        submodule_search_locations=[])
+    if name in sys.modules:
+        return _exec(spec, sys.modules[name])
+    else:
+        return _load(spec)
+
+
+def load_module(name, file, filename, details):
+    """**DEPRECATED**
+
+    Load a module, given information returned by find_module().
+
+    The module name must include the full package name, if any.
+
+    """
+    suffix, mode, type_ = details
+    if mode and (not mode.startswith(('r', 'U')) or '+' in mode):
+        raise ValueError('invalid file open mode {!r}'.format(mode))
+    elif file is None and type_ in {PY_SOURCE, PY_COMPILED}:
+        msg = 'file object required for import (type code {})'.format(type_)
+        raise ValueError(msg)
+    elif type_ == PY_SOURCE:
+        return load_source(name, filename, file)
+    elif type_ == PY_COMPILED:
+        return load_compiled(name, filename, file)
+    elif type_ == C_EXTENSION and load_dynamic is not None:
+        if file is None:
+            with open(filename, 'rb') as opened_file:
+                return load_dynamic(name, filename, opened_file)
+        else:
+            return load_dynamic(name, filename, file)
+    elif type_ == PKG_DIRECTORY:
+        return load_package(name, filename)
+    elif type_ == C_BUILTIN:
+        return init_builtin(name)
+    elif type_ == PY_FROZEN:
+        return init_frozen(name)
+    else:
+        msg =  "Don't know how to import {} (type code {})".format(name, type_)
+        raise ImportError(msg, name=name)
+
+
+def find_module(name, path=None):
+    """**DEPRECATED**
+
+    Search for a module.
+
+    If path is omitted or None, search for a built-in, frozen or special
+    module and continue search in sys.path. The module name cannot
+    contain '.'; to search for a submodule of a package, pass the
+    submodule name and the package's __path__.
+
+    """
+    if not isinstance(name, str):
+        raise TypeError("'name' must be a str, not {}".format(type(name)))
+    elif not isinstance(path, (type(None), list)):
+        # Backwards-compatibility
+        raise RuntimeError("'path' must be None or a list, "
+                           "not {}".format(type(path)))
+
+    if path is None:
+        if is_builtin(name):
+            return None, None, ('', '', C_BUILTIN)
+        elif is_frozen(name):
+            return None, None, ('', '', PY_FROZEN)
+        else:
+            path = sys.path
+
+    for entry in path:
+        package_directory = os.path.join(entry, name)
+        for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]:
+            package_file_name = '__init__' + suffix
+            file_path = os.path.join(package_directory, package_file_name)
+            if os.path.isfile(file_path):
+                return None, package_directory, ('', '', PKG_DIRECTORY)
+        for suffix, mode, type_ in get_suffixes():
+            file_name = name + suffix
+            file_path = os.path.join(entry, file_name)
+            if os.path.isfile(file_path):
+                break
+        else:
+            continue
+        break  # Break out of outer loop when breaking out of inner loop.
+    else:
+        raise ImportError(_ERR_MSG.format(name), name=name)
+
+    encoding = None
+    if 'b' not in mode:
+        with open(file_path, 'rb') as file:
+            encoding = tokenize.detect_encoding(file.readline)[0]
+    file = open(file_path, mode, encoding=encoding)
+    return file, file_path, (suffix, mode, type_)
+
+
+def reload(module):
+    """**DEPRECATED**
+
+    Reload the module and return it.
+
+    The module must have been successfully imported before.
+
+    """
+    return importlib.reload(module)
+
+
+def init_builtin(name):
+    """**DEPRECATED**
+
+    Load and return a built-in module by name, or None is such module doesn't
+    exist
+    """
+    try:
+        return _builtin_from_name(name)
+    except ImportError:
+        return None
+
+
+if create_dynamic:
+    def load_dynamic(name, path, file=None):
+        """**DEPRECATED**
+
+        Load an extension module.
+        """
+        import importlib.machinery
+        loader = importlib.machinery.ExtensionFileLoader(name, path)
+
+        # Issue #24748: Skip the sys.modules check in _load_module_shim;
+        # always load new extension
+        spec = importlib.machinery.ModuleSpec(
+            name=name, loader=loader, origin=path)
+        return _load(spec)
+
+else:
+    load_dynamic = None

Lib/importlib/__init__.py

@@ -48,8 +48,8 @@ else:
     sys.modules['importlib._bootstrap_external'] = _bootstrap_external
 
 # To simplify imports in test code
-_w_long = _bootstrap_external._w_long
-_r_long = _bootstrap_external._r_long
+_pack_uint32 = _bootstrap_external._pack_uint32
+_unpack_uint32 = _bootstrap_external._unpack_uint32
 
 # Fully bootstrapped at this point, import whatever you like, circular
 # dependencies and startup overhead minimisation permitting :)
@@ -79,7 +79,8 @@ def find_loader(name, path=None):
     This function is deprecated in favor of importlib.util.find_spec().
 
     """
-    warnings.warn('Use importlib.util.find_spec() instead.',
+    warnings.warn('Deprecated since Python 3.4. '
+                  'Use importlib.util.find_spec() instead.',
                   DeprecationWarning, stacklevel=2)
     try:
         loader = sys.modules[name].__loader__
@@ -136,7 +137,7 @@ def reload(module):
 
     """
     if not module or not isinstance(module, types.ModuleType):
-        raise TypeError("reload() argument must be module")
+        raise TypeError("reload() argument must be a module")
     try:
         name = module.__spec__.name
     except AttributeError:
@@ -163,6 +164,8 @@ def reload(module):
             pkgpath = None
         target = module
         spec = module.__spec__ = _bootstrap._find_spec(name, pkgpath, target)
+        if spec is None:
+            raise ModuleNotFoundError(f"spec not found for the module {name!r}", name=name)
         _bootstrap._exec(spec, module)
         # The module may have replaced itself in sys.modules!
         return sys.modules[name]

Lib/importlib/_bootstrap_external.py

@@ -26,16 +26,6 @@ _CASE_INSENSITIVE_PLATFORMS =  (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY
                                 + _CASE_INSENSITIVE_PLATFORMS_STR_KEY)
 
 
-def _w_long(x):
-    """Convert a 32-bit integer to little-endian."""
-    return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little')
-
-
-def _r_long(int_bytes):
-    """Convert 4 bytes in little-endian to an integer."""
-    return int.from_bytes(int_bytes, 'little')
-
-
 def _make_relax_case():
     if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
         if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS_STR_KEY):
@@ -1597,7 +1587,10 @@ def _setup(_bootstrap_module):
     setattr(self_module, '_pathseps_with_colon', {f':{s}' for s in path_separators})
 
     # Directly load the _thread module (needed during bootstrap).
-    thread_module = _bootstrap._builtin_from_name('_thread')
+    try:
+        thread_module = _bootstrap._builtin_from_name('_thread')
+    except ImportError:
+        thread_module = None
     setattr(self_module, '_thread', thread_module)
 
     # Directly load the _weakref module (needed during bootstrap).

Lib/importlib/abc.py

@@ -13,6 +13,7 @@ try:
 except ImportError as exc:
     _frozen_importlib_external = _bootstrap_external
 import abc
+import warnings
 
 
 def _register(abstract_cls, *classes):
@@ -34,6 +35,8 @@ class Finder(metaclass=abc.ABCMeta):
     reimplementations of the import system.  Otherwise, finder
     implementations should derive from the more specific MetaPathFinder
     or PathEntryFinder ABCs.
+
+    Deprecated since Python 3.3
     """
 
     @abc.abstractmethod
@@ -57,11 +60,16 @@ class MetaPathFinder(Finder):
         If no module is found, return None.  The fullname is a str and
         the path is a list of strings or None.
 
-        This method is deprecated in favor of finder.find_spec(). If find_spec()
-        exists then backwards-compatible functionality is provided for this
-        method.
+        This method is deprecated since Python 3.4 in favor of
+        finder.find_spec(). If find_spec() exists then backwards-compatible
+        functionality is provided for this method.
 
         """
+        warnings.warn("MetaPathFinder.find_module() is deprecated since Python "
+                      "3.4 in favor of MetaPathFinder.find_spec() "
+                      "(available since 3.4)",
+                      DeprecationWarning,
+                      stacklevel=2)
         if not hasattr(self, 'find_spec'):
             return None
         found = self.find_spec(fullname, path)
@@ -94,10 +102,15 @@ class PathEntryFinder(Finder):
         The portion will be discarded if another path entry finder
         locates the module as a normal module or package.
 
-        This method is deprecated in favor of finder.find_spec(). If find_spec()
-        is provided than backwards-compatible functionality is provided.
-
+        This method is deprecated since Python 3.4 in favor of
+        finder.find_spec(). If find_spec() is provided than backwards-compatible
+        functionality is provided.
         """
+        warnings.warn("PathEntryFinder.find_loader() is deprecated since Python "
+                      "3.4 in favor of PathEntryFinder.find_spec() "
+                      "(available since 3.4)",
+                      DeprecationWarning,
+                      stacklevel=2)
         if not hasattr(self, 'find_spec'):
             return None, []
         found = self.find_spec(fullname)
@@ -180,7 +193,7 @@ class ResourceLoader(Loader):
     def get_data(self, path):
         """Abstract method which when implemented should return the bytes for
         the specified path.  The path must be a str."""
-        raise IOError
+        raise OSError
 
 
 class InspectLoader(Loader):
@@ -302,7 +315,7 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo
     def path_mtime(self, path):
         """Return the (int) modification time for the path (str)."""
         if self.path_stats.__func__ is SourceLoader.path_stats:
-            raise IOError
+            raise OSError
         return int(self.path_stats(path)['mtime'])
 
     def path_stats(self, path):
@@ -313,7 +326,7 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo
         - 'size' (optional) is the size in bytes of the source code.
         """
         if self.path_mtime.__func__ is SourceLoader.path_mtime:
-            raise IOError
+            raise OSError
         return {'mtime': self.path_mtime(path)}
 
     def set_data(self, path, data):
@@ -327,3 +340,49 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo
         """
 
 _register(SourceLoader, machinery.SourceFileLoader)
+
+
+class ResourceReader(metaclass=abc.ABCMeta):
+
+    """Abstract base class to provide resource-reading support.
+
+    Loaders that support resource reading are expected to implement
+    the ``get_resource_reader(fullname)`` method and have it either return None
+    or an object compatible with this ABC.
+    """
+
+    @abc.abstractmethod
+    def open_resource(self, resource):
+        """Return an opened, file-like object for binary reading.
+
+        The 'resource' argument is expected to represent only a file name
+        and thus not contain any subdirectory components.
+
+        If the resource cannot be found, FileNotFoundError is raised.
+        """
+        raise FileNotFoundError
+
+    @abc.abstractmethod
+    def resource_path(self, resource):
+        """Return the file system path to the specified resource.
+
+        The 'resource' argument is expected to represent only a file name
+        and thus not contain any subdirectory components.
+
+        If the resource does not exist on the file system, raise
+        FileNotFoundError.
+        """
+        raise FileNotFoundError
+
+    @abc.abstractmethod
+    def is_resource(self, name):
+        """Return True if the named 'name' is consider a resource."""
+        raise FileNotFoundError
+
+    @abc.abstractmethod
+    def contents(self):
+        """Return an iterable of strings over the contents of the package."""
+        return []
+
+
+_register(ResourceReader, machinery.SourceFileLoader)

Lib/importlib/resources.py

@@ -0,0 +1,259 @@
+import os
+import tempfile
+
+from . import abc as resources_abc
+from contextlib import contextmanager, suppress
+from importlib import import_module
+from importlib.abc import ResourceLoader
+from io import BytesIO, TextIOWrapper
+from pathlib import Path
+from types import ModuleType
+from typing import Iterable, Iterator, Optional, Set, Union   # noqa: F401
+from typing import cast
+from typing.io import BinaryIO, TextIO
+from zipimport import ZipImportError
+
+
+__all__ = [
+    'Package',
+    'Resource',
+    'contents',
+    'is_resource',
+    'open_binary',
+    'open_text',
+    'path',
+    'read_binary',
+    'read_text',
+    ]
+
+
+Package = Union[str, ModuleType]
+Resource = Union[str, os.PathLike]
+
+
+def _get_package(package) -> ModuleType:
+    """Take a package name or module object and return the module.
+
+    If a name, the module is imported.  If the passed or imported module
+    object is not a package, raise an exception.
+    """
+    if hasattr(package, '__spec__'):
+        if package.__spec__.submodule_search_locations is None:
+            raise TypeError('{!r} is not a package'.format(
+                package.__spec__.name))
+        else:
+            return package
+    else:
+        module = import_module(package)
+        if module.__spec__.submodule_search_locations is None:
+            raise TypeError('{!r} is not a package'.format(package))
+        else:
+            return module
+
+
+def _normalize_path(path) -> str:
+    """Normalize a path by ensuring it is a string.
+
+    If the resulting string contains path separators, an exception is raised.
+    """
+    parent, file_name = os.path.split(path)
+    if parent:
+        raise ValueError('{!r} must be only a file name'.format(path))
+    else:
+        return file_name
+
+
+def _get_resource_reader(
+        package: ModuleType) -> Optional[resources_abc.ResourceReader]:
+    # Return the package's loader if it's a ResourceReader.  We can't use
+    # a issubclass() check here because apparently abc.'s __subclasscheck__()
+    # hook wants to create a weak reference to the object, but
+    # zipimport.zipimporter does not support weak references, resulting in a
+    # TypeError.  That seems terrible.
+    spec = package.__spec__
+    if hasattr(spec.loader, 'get_resource_reader'):
+        return cast(resources_abc.ResourceReader,
+                    spec.loader.get_resource_reader(spec.name))
+    return None
+
+
+def _check_location(package):
+    if package.__spec__.origin is None or not package.__spec__.has_location:
+        raise FileNotFoundError(f'Package has no location {package!r}')
+
+
+def open_binary(package: Package, resource: Resource) -> BinaryIO:
+    """Return a file-like object opened for binary reading of the resource."""
+    resource = _normalize_path(resource)
+    package = _get_package(package)
+    reader = _get_resource_reader(package)
+    if reader is not None:
+        return reader.open_resource(resource)
+    _check_location(package)
+    absolute_package_path = os.path.abspath(package.__spec__.origin)
+    package_path = os.path.dirname(absolute_package_path)
+    full_path = os.path.join(package_path, resource)
+    try:
+        return open(full_path, mode='rb')
+    except OSError:
+        # Just assume the loader is a resource loader; all the relevant
+        # importlib.machinery loaders are and an AttributeError for
+        # get_data() will make it clear what is needed from the loader.
+        loader = cast(ResourceLoader, package.__spec__.loader)
+        data = None
+        if hasattr(package.__spec__.loader, 'get_data'):
+            with suppress(OSError):
+                data = loader.get_data(full_path)
+        if data is None:
+            package_name = package.__spec__.name
+            message = '{!r} resource not found in {!r}'.format(
+                resource, package_name)
+            raise FileNotFoundError(message)
+        else:
+            return BytesIO(data)
+
+
+def open_text(package: Package,
+              resource: Resource,
+              encoding: str = 'utf-8',
+              errors: str = 'strict') -> TextIO:
+    """Return a file-like object opened for text reading of the resource."""
+    resource = _normalize_path(resource)
+    package = _get_package(package)
+    reader = _get_resource_reader(package)
+    if reader is not None:
+        return TextIOWrapper(reader.open_resource(resource), encoding, errors)
+    _check_location(package)
+    absolute_package_path = os.path.abspath(package.__spec__.origin)
+    package_path = os.path.dirname(absolute_package_path)
+    full_path = os.path.join(package_path, resource)
+    try:
+        return open(full_path, mode='r', encoding=encoding, errors=errors)
+    except OSError:
+        # Just assume the loader is a resource loader; all the relevant
+        # importlib.machinery loaders are and an AttributeError for
+        # get_data() will make it clear what is needed from the loader.
+        loader = cast(ResourceLoader, package.__spec__.loader)
+        data = None
+        if hasattr(package.__spec__.loader, 'get_data'):
+            with suppress(OSError):
+                data = loader.get_data(full_path)
+        if data is None:
+            package_name = package.__spec__.name
+            message = '{!r} resource not found in {!r}'.format(
+                resource, package_name)
+            raise FileNotFoundError(message)
+        else:
+            return TextIOWrapper(BytesIO(data), encoding, errors)
+
+
+def read_binary(package: Package, resource: Resource) -> bytes:
+    """Return the binary contents of the resource."""
+    resource = _normalize_path(resource)
+    package = _get_package(package)
+    with open_binary(package, resource) as fp:
+        return fp.read()
+
+
+def read_text(package: Package,
+              resource: Resource,
+              encoding: str = 'utf-8',
+              errors: str = 'strict') -> str:
+    """Return the decoded string of the resource.
+
+    The decoding-related arguments have the same semantics as those of
+    bytes.decode().
+    """
+    resource = _normalize_path(resource)
+    package = _get_package(package)
+    with open_text(package, resource, encoding, errors) as fp:
+        return fp.read()
+
+
+@contextmanager
+def path(package: Package, resource: Resource) -> Iterator[Path]:
+    """A context manager providing a file path object to the resource.
+
+    If the resource does not already exist on its own on the file system,
+    a temporary file will be created. If the file was created, the file
+    will be deleted upon exiting the context manager (no exception is
+    raised if the file was deleted prior to the context manager
+    exiting).
+    """
+    resource = _normalize_path(resource)
+    package = _get_package(package)
+    reader = _get_resource_reader(package)
+    if reader is not None:
+        try:
+            yield Path(reader.resource_path(resource))
+            return
+        except FileNotFoundError:
+            pass
+    else:
+        _check_location(package)
+    # Fall-through for both the lack of resource_path() *and* if
+    # resource_path() raises FileNotFoundError.
+    package_directory = Path(package.__spec__.origin).parent
+    file_path = package_directory / resource
+    if file_path.exists():
+        yield file_path
+    else:
+        with open_binary(package, resource) as fp:
+            data = fp.read()
+        # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try'
+        # blocks due to the need to close the temporary file to work on
+        # Windows properly.
+        fd, raw_path = tempfile.mkstemp()
+        try:
+            os.write(fd, data)
+            os.close(fd)
+            yield Path(raw_path)
+        finally:
+            try:
+                os.remove(raw_path)
+            except FileNotFoundError:
+                pass
+
+
+def is_resource(package: Package, name: str) -> bool:
+    """True if 'name' is a resource inside 'package'.
+
+    Directories are *not* resources.
+    """
+    package = _get_package(package)
+    _normalize_path(name)
+    reader = _get_resource_reader(package)
+    if reader is not None:
+        return reader.is_resource(name)
+    try:
+        package_contents = set(contents(package))
+    except (NotADirectoryError, FileNotFoundError):
+        return False
+    if name not in package_contents:
+        return False
+    # Just because the given file_name lives as an entry in the package's
+    # contents doesn't necessarily mean it's a resource.  Directories are not
+    # resources, so let's try to find out if it's a directory or not.
+    path = Path(package.__spec__.origin).parent / name
+    return path.is_file()
+
+
+def contents(package: Package) -> Iterable[str]:
+    """Return an iterable of entries in 'package'.
+
+    Note that not all entries are resources.  Specifically, directories are
+    not considered resources.  Use `is_resource()` on each entry returned here
+    to check if it is a resource or not.
+    """
+    package = _get_package(package)
+    reader = _get_resource_reader(package)
+    if reader is not None:
+        return reader.contents()
+    # Is the package a namespace package?  By definition, namespace packages
+    # cannot have resources.  We could use _check_location() and catch the
+    # exception, but that's extra work, so just inline the check.
+    elif package.__spec__.origin is None or not package.__spec__.has_location:
+        return ()
+    else:
+        package_directory = Path(package.__spec__.origin).parent
+        return os.listdir(package_directory)

Lib/importlib/util.py

@@ -5,18 +5,25 @@ from ._bootstrap import _resolve_name
 from ._bootstrap import spec_from_loader
 from ._bootstrap import _find_spec
 from ._bootstrap_external import MAGIC_NUMBER
+from ._bootstrap_external import _RAW_MAGIC_NUMBER
 from ._bootstrap_external import cache_from_source
 from ._bootstrap_external import decode_source
 from ._bootstrap_external import source_from_cache
 from ._bootstrap_external import spec_from_file_location
 
 from contextlib import contextmanager
+import _imp
 import functools
 import sys
 import types
 import warnings
 
 
+def source_hash(source_bytes):
+    "Return the hash of *source_bytes* as used in hash-based pyc files."
+    return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes)
+
+
 def resolve_name(name, package):
     """Resolve a relative module name to an absolute one."""
     if not name.startswith('.'):
@@ -84,11 +91,16 @@ def find_spec(name, package=None):
     if fullname not in sys.modules:
         parent_name = fullname.rpartition('.')[0]
         if parent_name:
-            # Use builtins.__import__() in case someone replaced it.
             parent = __import__(parent_name, fromlist=['__path__'])
-            return _find_spec(fullname, parent.__path__)
+            try:
+                parent_path = parent.__path__
+            except AttributeError as e:
+                raise ModuleNotFoundError(
+                    f"__path__ attribute not found on {parent_name!r} "
+                    f"while trying to find {fullname!r}", name=fullname) from e
         else:
-            return _find_spec(fullname, None)
+            parent_path = None
+        return _find_spec(fullname, parent_path)
     else:
         module = sys.modules[fullname]
         if module is None:

Lib/py_compile.py

@@ -3,6 +3,7 @@
 This module has intimate knowledge of the format of .pyc files.
 """
 
+import enum
 import importlib._bootstrap_external
 import importlib.machinery
 import importlib.util
@@ -11,7 +12,7 @@ import os.path
 import sys
 import traceback
 
-__all__ = ["compile", "main", "PyCompileError"]
+__all__ = ["compile", "main", "PyCompileError", "PycInvalidationMode"]
 
 
 class PyCompileError(Exception):
@@ -62,7 +63,21 @@ class PyCompileError(Exception):
         return self.msg
 
 
-def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
+class PycInvalidationMode(enum.Enum):
+    TIMESTAMP = 1
+    CHECKED_HASH = 2
+    UNCHECKED_HASH = 3
+
+
+def _get_default_invalidation_mode():
+    if os.environ.get('SOURCE_DATE_EPOCH'):
+        return PycInvalidationMode.CHECKED_HASH
+    else:
+        return PycInvalidationMode.TIMESTAMP
+
+
+def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1,
+            invalidation_mode=None):
     """Byte-compile one Python source file to Python bytecode.
 
     :param file: The source file name.
@@ -79,6 +94,7 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
     :param optimize: The optimization level for the compiler.  Valid values
         are -1, 0, 1 and 2.  A value of -1 means to use the optimization
         level of the current interpreter, as given by -O command line options.
+    :param invalidation_mode:
 
     :return: Path to the resulting byte compiled file.
 
@@ -103,6 +119,8 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
     the resulting file would be regular and thus not the same type of file as
     it was previously.
     """
+    if invalidation_mode is None:
+        invalidation_mode = _get_default_invalidation_mode()
     if cfile is None:
         if optimize >= 0:
             optimization = optimize if optimize >= 1 else ''
@@ -136,9 +154,17 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
             os.makedirs(dirname)
     except FileExistsError:
         pass
-    source_stats = loader.path_stats(file)
-    bytecode = importlib._bootstrap_external._code_to_bytecode(
+    if invalidation_mode == PycInvalidationMode.TIMESTAMP:
+        source_stats = loader.path_stats(file)
+        bytecode = importlib._bootstrap_external._code_to_timestamp_pyc(
             code, source_stats['mtime'], source_stats['size'])
+    else:
+        source_hash = importlib.util.source_hash(source_bytes)
+        bytecode = importlib._bootstrap_external._code_to_hash_pyc(
+            code,
+            source_hash,
+            (invalidation_mode == PycInvalidationMode.CHECKED_HASH),
+        )
     mode = importlib._bootstrap_external._calc_mode(file)
     importlib._bootstrap_external._write_atomic(cfile, bytecode, mode)
     return cfile

Lib/test/support/__init__.py

@@ -2568,6 +2568,31 @@ def skip_if_buggy_ucrt_strfptime(test):
             _buggy_ucrt = False
     return unittest.skip("buggy MSVC UCRT strptime/strftime")(test) if _buggy_ucrt else test
 
+
+def skip_if_broken_multiprocessing_synchronize():
+    """
+    Skip tests if the multiprocessing.synchronize module is missing, if there
+    is no available semaphore implementation, or if creating a lock raises an
+    OSError (on Linux only).
+    """
+
+    # Skip tests if the _multiprocessing extension is missing.
+    import_module('_multiprocessing')
+
+    # Skip tests if there is no available semaphore implementation:
+    # multiprocessing.synchronize requires _multiprocessing.SemLock.
+    synchronize = import_module('multiprocessing.synchronize')
+
+    if sys.platform == "linux":
+        try:
+            # bpo-38377: On Linux, creating a semaphore fails with OSError
+            # if the current user does not have the permission to create
+            # a file in /dev/shm/ directory.
+            synchronize.Lock(ctx=None)
+        except OSError as exc:
+            raise unittest.SkipTest(f"broken multiprocessing SemLock: {exc!r}")
+
+
 class PythonSymlink:
     """Creates a symlink for the current Python executable"""
     def __init__(self, link=None):

Lib/test/test___future__.py

@@ -0,0 +1,61 @@
+import unittest
+import __future__
+
+GOOD_SERIALS = ("alpha", "beta", "candidate", "final")
+
+features = __future__.all_feature_names
+
+class FutureTest(unittest.TestCase):
+
+    def test_names(self):
+        # Verify that all_feature_names appears correct.
+        given_feature_names = features[:]
+        for name in dir(__future__):
+            obj = getattr(__future__, name, None)
+            if obj is not None and isinstance(obj, __future__._Feature):
+                self.assertTrue(
+                    name in given_feature_names,
+                    "%r should have been in all_feature_names" % name
+                )
+                given_feature_names.remove(name)
+        self.assertEqual(len(given_feature_names), 0,
+               "all_feature_names has too much: %r" % given_feature_names)
+
+    def test_attributes(self):
+        for feature in features:
+            value = getattr(__future__, feature)
+
+            optional = value.getOptionalRelease()
+            mandatory = value.getMandatoryRelease()
+
+            a = self.assertTrue
+            e = self.assertEqual
+            def check(t, name):
+                a(isinstance(t, tuple), "%s isn't tuple" % name)
+                e(len(t), 5, "%s isn't 5-tuple" % name)
+                (major, minor, micro, level, serial) = t
+                a(isinstance(major, int), "%s major isn't int"  % name)
+                a(isinstance(minor, int), "%s minor isn't int" % name)
+                a(isinstance(micro, int), "%s micro isn't int" % name)
+                a(isinstance(level, str),
+                    "%s level isn't string" % name)
+                a(level in GOOD_SERIALS,
+                       "%s level string has unknown value" % name)
+                a(isinstance(serial, int), "%s serial isn't int" % name)
+
+            check(optional, "optional")
+            if mandatory is not None:
+                check(mandatory, "mandatory")
+                a(optional < mandatory,
+                       "optional not less than mandatory, and mandatory not None")
+
+            a(hasattr(value, "compiler_flag"),
+                   "feature is missing a .compiler_flag attr")
+            # Make sure the compile accepts the flag.
+            compile("", "<test>", "exec", value.compiler_flag)
+            a(isinstance(getattr(value, "compiler_flag"), int),
+                   ".compiler_flag isn't int")
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_abstract_numbers.py

@@ -0,0 +1,44 @@
+"""Unit tests for numbers.py."""
+
+import math
+import operator
+import unittest
+from numbers import Complex, Real, Rational, Integral
+
+class TestNumbers(unittest.TestCase):
+    def test_int(self):
+        self.assertTrue(issubclass(int, Integral))
+        self.assertTrue(issubclass(int, Complex))
+
+        self.assertEqual(7, int(7).real)
+        self.assertEqual(0, int(7).imag)
+        self.assertEqual(7, int(7).conjugate())
+        self.assertEqual(-7, int(-7).conjugate())
+        self.assertEqual(7, int(7).numerator)
+        self.assertEqual(1, int(7).denominator)
+
+    def test_float(self):
+        self.assertFalse(issubclass(float, Rational))
+        self.assertTrue(issubclass(float, Real))
+
+        self.assertEqual(7.3, float(7.3).real)
+        self.assertEqual(0, float(7.3).imag)
+        self.assertEqual(7.3, float(7.3).conjugate())
+        self.assertEqual(-7.3, float(-7.3).conjugate())
+
+    def test_complex(self):
+        self.assertFalse(issubclass(complex, Real))
+        self.assertTrue(issubclass(complex, Complex))
+
+        c1, c2 = complex(3, 2), complex(4,1)
+        # XXX: This is not ideal, but see the comment in math_trunc().
+        self.assertRaises(TypeError, math.trunc, c1)
+        self.assertRaises(TypeError, operator.mod, c1, c2)
+        self.assertRaises(TypeError, divmod, c1, c2)
+        self.assertRaises(TypeError, operator.floordiv, c1, c2)
+        self.assertRaises(TypeError, float, c1)
+        self.assertRaises(TypeError, int, c1)
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_array.py

@@ -918,8 +918,6 @@ class BaseTest:
                     del a[start:stop:step]
                     self.assertEqual(a, array.array(self.typecode, L))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_index(self):
         example = 2*self.example
         a = array.array(self.typecode, example)
@@ -929,8 +927,6 @@ class BaseTest:
         self.assertRaises(ValueError, a.index, None)
         self.assertRaises(ValueError, a.index, self.outside)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_count(self):
         example = 2*self.example
         a = array.array(self.typecode, example)
@@ -940,8 +936,6 @@ class BaseTest:
         self.assertEqual(a.count(self.outside), 0)
         self.assertEqual(a.count(None), 0)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_remove(self):
         for x in self.example:
             example = 2*self.example

Lib/test/test_builtin.py

@@ -446,8 +446,6 @@ class BuiltinTest(unittest.TestCase):
         # test that object has a __dir__()
         self.assertEqual(sorted([].__dir__()), dir([]))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_divmod(self):
         self.assertEqual(divmod(12, 7), (1, 5))
         self.assertEqual(divmod(-12, 7), (-2, 2))

Lib/test/test_cgi.py

@@ -0,0 +1,609 @@
+import cgi
+import os
+import sys
+import tempfile
+import unittest
+from collections import namedtuple
+from io import StringIO, BytesIO
+from test import support
+
+class HackedSysModule:
+    # The regression test will have real values in sys.argv, which
+    # will completely confuse the test of the cgi module
+    argv = []
+    stdin = sys.stdin
+
+cgi.sys = HackedSysModule()
+
+class ComparableException:
+    def __init__(self, err):
+        self.err = err
+
+    def __str__(self):
+        return str(self.err)
+
+    def __eq__(self, anExc):
+        if not isinstance(anExc, Exception):
+            return NotImplemented
+        return (self.err.__class__ == anExc.__class__ and
+                self.err.args == anExc.args)
+
+    def __getattr__(self, attr):
+        return getattr(self.err, attr)
+
+def do_test(buf, method):
+    env = {}
+    if method == "GET":
+        fp = None
+        env['REQUEST_METHOD'] = 'GET'
+        env['QUERY_STRING'] = buf
+    elif method == "POST":
+        fp = BytesIO(buf.encode('latin-1')) # FieldStorage expects bytes
+        env['REQUEST_METHOD'] = 'POST'
+        env['CONTENT_TYPE'] = 'application/x-www-form-urlencoded'
+        env['CONTENT_LENGTH'] = str(len(buf))
+    else:
+        raise ValueError("unknown method: %s" % method)
+    try:
+        return cgi.parse(fp, env, strict_parsing=1)
+    except Exception as err:
+        return ComparableException(err)
+
+parse_strict_test_cases = [
+    ("", ValueError("bad query field: ''")),
+    ("&", ValueError("bad query field: ''")),
+    ("&&", ValueError("bad query field: ''")),
+    (";", ValueError("bad query field: ''")),
+    (";&;", ValueError("bad query field: ''")),
+    # Should the next few really be valid?
+    ("=", {}),
+    ("=&=", {}),
+    ("=;=", {}),
+    # This rest seem to make sense
+    ("=a", {'': ['a']}),
+    ("&=a", ValueError("bad query field: ''")),
+    ("=a&", ValueError("bad query field: ''")),
+    ("=&a", ValueError("bad query field: 'a'")),
+    ("b=a", {'b': ['a']}),
+    ("b+=a", {'b ': ['a']}),
+    ("a=b=a", {'a': ['b=a']}),
+    ("a=+b=a", {'a': [' b=a']}),
+    ("&b=a", ValueError("bad query field: ''")),
+    ("b&=a", ValueError("bad query field: 'b'")),
+    ("a=a+b&b=b+c", {'a': ['a b'], 'b': ['b c']}),
+    ("a=a+b&a=b+a", {'a': ['a b', 'b a']}),
+    ("x=1&y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
+    ("x=1;y=2.0&z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
+    ("x=1;y=2.0;z=2-3.%2b0", {'x': ['1'], 'y': ['2.0'], 'z': ['2-3.+0']}),
+    ("Hbc5161168c542333633315dee1182227:key_store_seqid=400006&cuyer=r&view=bustomer&order_id=0bb2e248638833d48cb7fed300000f1b&expire=964546263&lobale=en-US&kid=130003.300038&ss=env",
+     {'Hbc5161168c542333633315dee1182227:key_store_seqid': ['400006'],
+      'cuyer': ['r'],
+      'expire': ['964546263'],
+      'kid': ['130003.300038'],
+      'lobale': ['en-US'],
+      'order_id': ['0bb2e248638833d48cb7fed300000f1b'],
+      'ss': ['env'],
+      'view': ['bustomer'],
+      }),
+
+    ("group_id=5470&set=custom&_assigned_to=31392&_status=1&_category=100&SUBMIT=Browse",
+     {'SUBMIT': ['Browse'],
+      '_assigned_to': ['31392'],
+      '_category': ['100'],
+      '_status': ['1'],
+      'group_id': ['5470'],
+      'set': ['custom'],
+      })
+    ]
+
+def norm(seq):
+    return sorted(seq, key=repr)
+
+def first_elts(list):
+    return [p[0] for p in list]
+
+def first_second_elts(list):
+    return [(p[0], p[1][0]) for p in list]
+
+def gen_result(data, environ):
+    encoding = 'latin-1'
+    fake_stdin = BytesIO(data.encode(encoding))
+    fake_stdin.seek(0)
+    form = cgi.FieldStorage(fp=fake_stdin, environ=environ, encoding=encoding)
+
+    result = {}
+    for k, v in dict(form).items():
+        result[k] = isinstance(v, list) and form.getlist(k) or v.value
+
+    return result
+
+class CgiTests(unittest.TestCase):
+
+    def test_parse_multipart(self):
+        fp = BytesIO(POSTDATA.encode('latin1'))
+        env = {'boundary': BOUNDARY.encode('latin1'),
+               'CONTENT-LENGTH': '558'}
+        result = cgi.parse_multipart(fp, env)
+        expected = {'submit': [' Add '], 'id': ['1234'],
+                    'file': [b'Testing 123.\n'], 'title': ['']}
+        self.assertEqual(result, expected)
+
+    # TODO RUSTPYTHON - see https://github.com/RustPython/RustPython/issues/935
+    @unittest.expectedFailure
+    def test_parse_multipart_invalid_encoding(self):
+        BOUNDARY = "JfISa01"
+        POSTDATA = """--JfISa01
+Content-Disposition: form-data; name="submit-name"
+Content-Length: 3
+
+\u2603
+--JfISa01"""
+        fp = BytesIO(POSTDATA.encode('utf8'))
+        env = {'boundary': BOUNDARY.encode('latin1'),
+               'CONTENT-LENGTH': str(len(POSTDATA.encode('utf8')))}
+        result = cgi.parse_multipart(fp, env, encoding="ascii",
+                                     errors="surrogateescape")
+        expected = {'submit-name': ["\udce2\udc98\udc83"]}
+        self.assertEqual(result, expected)
+        self.assertEqual("\u2603".encode('utf8'),
+                         result["submit-name"][0].encode('utf8', 'surrogateescape'))
+
+    def test_fieldstorage_properties(self):
+        fs = cgi.FieldStorage()
+        self.assertFalse(fs)
+        self.assertIn("FieldStorage", repr(fs))
+        self.assertEqual(list(fs), list(fs.keys()))
+        fs.list.append(namedtuple('MockFieldStorage', 'name')('fieldvalue'))
+        self.assertTrue(fs)
+
+    def test_fieldstorage_invalid(self):
+        self.assertRaises(TypeError, cgi.FieldStorage, "not-a-file-obj",
+                                                            environ={"REQUEST_METHOD":"PUT"})
+        self.assertRaises(TypeError, cgi.FieldStorage, "foo", "bar")
+        fs = cgi.FieldStorage(headers={'content-type':'text/plain'})
+        self.assertRaises(TypeError, bool, fs)
+
+    def test_strict(self):
+        for orig, expect in parse_strict_test_cases:
+            # Test basic parsing
+            d = do_test(orig, "GET")
+            self.assertEqual(d, expect, "Error parsing %s method GET" % repr(orig))
+            d = do_test(orig, "POST")
+            self.assertEqual(d, expect, "Error parsing %s method POST" % repr(orig))
+
+            env = {'QUERY_STRING': orig}
+            fs = cgi.FieldStorage(environ=env)
+            if isinstance(expect, dict):
+                # test dict interface
+                self.assertEqual(len(expect), len(fs))
+                self.assertCountEqual(expect.keys(), fs.keys())
+                ##self.assertEqual(norm(expect.values()), norm(fs.values()))
+                ##self.assertEqual(norm(expect.items()), norm(fs.items()))
+                self.assertEqual(fs.getvalue("nonexistent field", "default"), "default")
+                # test individual fields
+                for key in expect.keys():
+                    expect_val = expect[key]
+                    self.assertIn(key, fs)
+                    if len(expect_val) > 1:
+                        self.assertEqual(fs.getvalue(key), expect_val)
+                    else:
+                        self.assertEqual(fs.getvalue(key), expect_val[0])
+
+    def test_log(self):
+        cgi.log("Testing")
+
+        cgi.logfp = StringIO()
+        cgi.initlog("%s", "Testing initlog 1")
+        cgi.log("%s", "Testing log 2")
+        self.assertEqual(cgi.logfp.getvalue(), "Testing initlog 1\nTesting log 2\n")
+        if os.path.exists(os.devnull):
+            cgi.logfp = None
+            cgi.logfile = os.devnull
+            cgi.initlog("%s", "Testing log 3")
+            self.addCleanup(cgi.closelog)
+            cgi.log("Testing log 4")
+
+    # TODO RUSTPYTHON
+    @unittest.expectedFailure
+    def test_fieldstorage_readline(self):
+        # FieldStorage uses readline, which has the capacity to read all
+        # contents of the input file into memory; we use readline's size argument
+        # to prevent that for files that do not contain any newlines in
+        # non-GET/HEAD requests
+        class TestReadlineFile:
+            def __init__(self, file):
+                self.file = file
+                self.numcalls = 0
+
+            def readline(self, size=None):
+                self.numcalls += 1
+                if size:
+                    return self.file.readline(size)
+                else:
+                    return self.file.readline()
+
+            def __getattr__(self, name):
+                file = self.__dict__['file']
+                a = getattr(file, name)
+                if not isinstance(a, int):
+                    setattr(self, name, a)
+                return a
+
+        f = TestReadlineFile(tempfile.TemporaryFile("wb+"))
+        self.addCleanup(f.close)
+        f.write(b'x' * 256 * 1024)
+        f.seek(0)
+        env = {'REQUEST_METHOD':'PUT'}
+        fs = cgi.FieldStorage(fp=f, environ=env)
+        self.addCleanup(fs.file.close)
+        # if we're not chunking properly, readline is only called twice
+        # (by read_binary); if we are chunking properly, it will be called 5 times
+        # as long as the chunksize is 1 << 16.
+        self.assertGreater(f.numcalls, 2)
+        f.close()
+
+    def test_fieldstorage_multipart(self):
+        #Test basic FieldStorage multipart parsing
+        env = {
+            'REQUEST_METHOD': 'POST',
+            'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY),
+            'CONTENT_LENGTH': '558'}
+        fp = BytesIO(POSTDATA.encode('latin-1'))
+        fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1")
+        self.assertEqual(len(fs.list), 4)
+        expect = [{'name':'id', 'filename':None, 'value':'1234'},
+                  {'name':'title', 'filename':None, 'value':''},
+                  {'name':'file', 'filename':'test.txt', 'value':b'Testing 123.\n'},
+                  {'name':'submit', 'filename':None, 'value':' Add '}]
+        for x in range(len(fs.list)):
+            for k, exp in expect[x].items():
+                got = getattr(fs.list[x], k)
+                self.assertEqual(got, exp)
+
+    def test_fieldstorage_multipart_leading_whitespace(self):
+        env = {
+            'REQUEST_METHOD': 'POST',
+            'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY),
+            'CONTENT_LENGTH': '560'}
+        # Add some leading whitespace to our post data that will cause the
+        # first line to not be the innerboundary.
+        fp = BytesIO(b"\r\n" + POSTDATA.encode('latin-1'))
+        fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1")
+        self.assertEqual(len(fs.list), 4)
+        expect = [{'name':'id', 'filename':None, 'value':'1234'},
+                  {'name':'title', 'filename':None, 'value':''},
+                  {'name':'file', 'filename':'test.txt', 'value':b'Testing 123.\n'},
+                  {'name':'submit', 'filename':None, 'value':' Add '}]
+        for x in range(len(fs.list)):
+            for k, exp in expect[x].items():
+                got = getattr(fs.list[x], k)
+                self.assertEqual(got, exp)
+
+    def test_fieldstorage_multipart_non_ascii(self):
+        #Test basic FieldStorage multipart parsing
+        env = {'REQUEST_METHOD':'POST',
+            'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY),
+            'CONTENT_LENGTH':'558'}
+        for encoding in ['iso-8859-1','utf-8']:
+            fp = BytesIO(POSTDATA_NON_ASCII.encode(encoding))
+            fs = cgi.FieldStorage(fp, environ=env,encoding=encoding)
+            self.assertEqual(len(fs.list), 1)
+            expect = [{'name':'id', 'filename':None, 'value':'\xe7\xf1\x80'}]
+            for x in range(len(fs.list)):
+                for k, exp in expect[x].items():
+                    got = getattr(fs.list[x], k)
+                    self.assertEqual(got, exp)
+
+    # TODO RUSTPYTHON
+    @unittest.expectedFailure
+    def test_fieldstorage_multipart_maxline(self):
+        # Issue #18167
+        maxline = 1 << 16
+        self.maxDiff = None
+        def check(content):
+            data = """---123
+Content-Disposition: form-data; name="upload"; filename="fake.txt"
+Content-Type: text/plain
+
+%s
+---123--
+""".replace('\n', '\r\n') % content
+            environ = {
+                'CONTENT_LENGTH':   str(len(data)),
+                'CONTENT_TYPE':     'multipart/form-data; boundary=-123',
+                'REQUEST_METHOD':   'POST',
+            }
+            self.assertEqual(gen_result(data, environ),
+                             {'upload': content.encode('latin1')})
+        check('x' * (maxline - 1))
+        check('x' * (maxline - 1) + '\r')
+        check('x' * (maxline - 1) + '\r' + 'y' * (maxline - 1))
+
+    def test_fieldstorage_multipart_w3c(self):
+        # Test basic FieldStorage multipart parsing (W3C sample)
+        env = {
+            'REQUEST_METHOD': 'POST',
+            'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY_W3),
+            'CONTENT_LENGTH': str(len(POSTDATA_W3))}
+        fp = BytesIO(POSTDATA_W3.encode('latin-1'))
+        fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1")
+        self.assertEqual(len(fs.list), 2)
+        self.assertEqual(fs.list[0].name, 'submit-name')
+        self.assertEqual(fs.list[0].value, 'Larry')
+        self.assertEqual(fs.list[1].name, 'files')
+        files = fs.list[1].value
+        self.assertEqual(len(files), 2)
+        expect = [{'name': None, 'filename': 'file1.txt', 'value': b'... contents of file1.txt ...'},
+                  {'name': None, 'filename': 'file2.gif', 'value': b'...contents of file2.gif...'}]
+        for x in range(len(files)):
+            for k, exp in expect[x].items():
+                got = getattr(files[x], k)
+                self.assertEqual(got, exp)
+
+    def test_fieldstorage_part_content_length(self):
+        BOUNDARY = "JfISa01"
+        POSTDATA = """--JfISa01
+Content-Disposition: form-data; name="submit-name"
+Content-Length: 5
+
+Larry
+--JfISa01"""
+        env = {
+            'REQUEST_METHOD': 'POST',
+            'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY),
+            'CONTENT_LENGTH': str(len(POSTDATA))}
+        fp = BytesIO(POSTDATA.encode('latin-1'))
+        fs = cgi.FieldStorage(fp, environ=env, encoding="latin-1")
+        self.assertEqual(len(fs.list), 1)
+        self.assertEqual(fs.list[0].name, 'submit-name')
+        self.assertEqual(fs.list[0].value, 'Larry')
+
+    def test_field_storage_multipart_no_content_length(self):
+        fp = BytesIO(b"""--MyBoundary
+Content-Disposition: form-data; name="my-arg"; filename="foo"
+
+Test
+
+--MyBoundary--
+""")
+        env = {
+            "REQUEST_METHOD": "POST",
+            "CONTENT_TYPE": "multipart/form-data; boundary=MyBoundary",
+            "wsgi.input": fp,
+        }
+        fields = cgi.FieldStorage(fp, environ=env)
+
+        self.assertEqual(len(fields["my-arg"].file.read()), 5)
+
+    def test_fieldstorage_as_context_manager(self):
+        fp = BytesIO(b'x' * 10)
+        env = {'REQUEST_METHOD': 'PUT'}
+        with cgi.FieldStorage(fp=fp, environ=env) as fs:
+            content = fs.file.read()
+            self.assertFalse(fs.file.closed)
+        self.assertTrue(fs.file.closed)
+        self.assertEqual(content, 'x' * 10)
+        with self.assertRaisesRegex(ValueError, 'I/O operation on closed file'):
+            fs.file.read()
+
+    _qs_result = {
+        'key1': 'value1',
+        'key2': ['value2x', 'value2y'],
+        'key3': 'value3',
+        'key4': 'value4'
+    }
+    def testQSAndUrlEncode(self):
+        data = "key2=value2x&key3=value3&key4=value4"
+        environ = {
+            'CONTENT_LENGTH':   str(len(data)),
+            'CONTENT_TYPE':     'application/x-www-form-urlencoded',
+            'QUERY_STRING':     'key1=value1&key2=value2y',
+            'REQUEST_METHOD':   'POST',
+        }
+        v = gen_result(data, environ)
+        self.assertEqual(self._qs_result, v)
+
+    def test_max_num_fields(self):
+        # For application/x-www-form-urlencoded
+        data = '&'.join(['a=a']*11)
+        environ = {
+            'CONTENT_LENGTH': str(len(data)),
+            'CONTENT_TYPE': 'application/x-www-form-urlencoded',
+            'REQUEST_METHOD': 'POST',
+        }
+
+        with self.assertRaises(ValueError):
+            cgi.FieldStorage(
+                fp=BytesIO(data.encode()),
+                environ=environ,
+                max_num_fields=10,
+            )
+
+        # For multipart/form-data
+        data = """---123
+Content-Disposition: form-data; name="a"
+
+3
+---123
+Content-Type: application/x-www-form-urlencoded
+
+a=4
+---123
+Content-Type: application/x-www-form-urlencoded
+
+a=5
+---123--
+"""
+        environ = {
+            'CONTENT_LENGTH':   str(len(data)),
+            'CONTENT_TYPE':     'multipart/form-data; boundary=-123',
+            'QUERY_STRING':     'a=1&a=2',
+            'REQUEST_METHOD':   'POST',
+        }
+
+        # 2 GET entities
+        # 1 top level POST entities
+        # 1 entity within the second POST entity
+        # 1 entity within the third POST entity
+        with self.assertRaises(ValueError):
+            cgi.FieldStorage(
+                fp=BytesIO(data.encode()),
+                environ=environ,
+                max_num_fields=4,
+            )
+        cgi.FieldStorage(
+            fp=BytesIO(data.encode()),
+            environ=environ,
+            max_num_fields=5,
+        )
+
+    def testQSAndFormData(self):
+        data = """---123
+Content-Disposition: form-data; name="key2"
+
+value2y
+---123
+Content-Disposition: form-data; name="key3"
+
+value3
+---123
+Content-Disposition: form-data; name="key4"
+
+value4
+---123--
+"""
+        environ = {
+            'CONTENT_LENGTH':   str(len(data)),
+            'CONTENT_TYPE':     'multipart/form-data; boundary=-123',
+            'QUERY_STRING':     'key1=value1&key2=value2x',
+            'REQUEST_METHOD':   'POST',
+        }
+        v = gen_result(data, environ)
+        self.assertEqual(self._qs_result, v)
+
+    def testQSAndFormDataFile(self):
+        data = """---123
+Content-Disposition: form-data; name="key2"
+
+value2y
+---123
+Content-Disposition: form-data; name="key3"
+
+value3
+---123
+Content-Disposition: form-data; name="key4"
+
+value4
+---123
+Content-Disposition: form-data; name="upload"; filename="fake.txt"
+Content-Type: text/plain
+
+this is the content of the fake file
+
+---123--
+"""
+        environ = {
+            'CONTENT_LENGTH':   str(len(data)),
+            'CONTENT_TYPE':     'multipart/form-data; boundary=-123',
+            'QUERY_STRING':     'key1=value1&key2=value2x',
+            'REQUEST_METHOD':   'POST',
+        }
+        result = self._qs_result.copy()
+        result.update({
+            'upload': b'this is the content of the fake file\n'
+        })
+        v = gen_result(data, environ)
+        self.assertEqual(result, v)
+
+    def test_parse_header(self):
+        self.assertEqual(
+            cgi.parse_header("text/plain"),
+            ("text/plain", {}))
+        self.assertEqual(
+            cgi.parse_header("text/vnd.just.made.this.up ; "),
+            ("text/vnd.just.made.this.up", {}))
+        self.assertEqual(
+            cgi.parse_header("text/plain;charset=us-ascii"),
+            ("text/plain", {"charset": "us-ascii"}))
+        self.assertEqual(
+            cgi.parse_header('text/plain ; charset="us-ascii"'),
+            ("text/plain", {"charset": "us-ascii"}))
+        self.assertEqual(
+            cgi.parse_header('text/plain ; charset="us-ascii"; another=opt'),
+            ("text/plain", {"charset": "us-ascii", "another": "opt"}))
+        self.assertEqual(
+            cgi.parse_header('attachment; filename="silly.txt"'),
+            ("attachment", {"filename": "silly.txt"}))
+        self.assertEqual(
+            cgi.parse_header('attachment; filename="strange;name"'),
+            ("attachment", {"filename": "strange;name"}))
+        self.assertEqual(
+            cgi.parse_header('attachment; filename="strange;name";size=123;'),
+            ("attachment", {"filename": "strange;name", "size": "123"}))
+        self.assertEqual(
+            cgi.parse_header('form-data; name="files"; filename="fo\\"o;bar"'),
+            ("form-data", {"name": "files", "filename": 'fo"o;bar'}))
+
+    def test_all(self):
+        blacklist = {"logfile", "logfp", "initlog", "dolog", "nolog",
+                     "closelog", "log", "maxlen", "valid_boundary"}
+        support.check__all__(self, cgi, blacklist=blacklist)
+
+
+BOUNDARY = "---------------------------721837373350705526688164684"
+
+POSTDATA = """-----------------------------721837373350705526688164684
+Content-Disposition: form-data; name="id"
+
+1234
+-----------------------------721837373350705526688164684
+Content-Disposition: form-data; name="title"
+
+
+-----------------------------721837373350705526688164684
+Content-Disposition: form-data; name="file"; filename="test.txt"
+Content-Type: text/plain
+
+Testing 123.
+
+-----------------------------721837373350705526688164684
+Content-Disposition: form-data; name="submit"
+
+ Add\x20
+-----------------------------721837373350705526688164684--
+"""
+
+POSTDATA_NON_ASCII = """-----------------------------721837373350705526688164684
+Content-Disposition: form-data; name="id"
+
+\xe7\xf1\x80
+-----------------------------721837373350705526688164684
+"""
+
+# http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4
+BOUNDARY_W3 = "AaB03x"
+POSTDATA_W3 = """--AaB03x
+Content-Disposition: form-data; name="submit-name"
+
+Larry
+--AaB03x
+Content-Disposition: form-data; name="files"
+Content-Type: multipart/mixed; boundary=BbC04y
+
+--BbC04y
+Content-Disposition: file; filename="file1.txt"
+Content-Type: text/plain
+
+... contents of file1.txt ...
+--BbC04y
+Content-Disposition: file; filename="file2.gif"
+Content-Type: image/gif
+Content-Transfer-Encoding: binary
+
+...contents of file2.gif...
+--BbC04y--
+--AaB03x--
+"""
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_cmd.py

@@ -0,0 +1,242 @@
+"""
+Test script for the 'cmd' module
+Original by Michael Schneider
+"""
+
+
+import cmd
+import sys
+import unittest
+import io
+from test import support
+
+class samplecmdclass(cmd.Cmd):
+    """
+    Instance the sampleclass:
+    >>> mycmd = samplecmdclass()
+
+    Test for the function parseline():
+    >>> mycmd.parseline("")
+    (None, None, '')
+    >>> mycmd.parseline("?")
+    ('help', '', 'help ')
+    >>> mycmd.parseline("?help")
+    ('help', 'help', 'help help')
+    >>> mycmd.parseline("!")
+    ('shell', '', 'shell ')
+    >>> mycmd.parseline("!command")
+    ('shell', 'command', 'shell command')
+    >>> mycmd.parseline("func")
+    ('func', '', 'func')
+    >>> mycmd.parseline("func arg1")
+    ('func', 'arg1', 'func arg1')
+
+
+    Test for the function onecmd():
+    >>> mycmd.onecmd("")
+    >>> mycmd.onecmd("add 4 5")
+    9
+    >>> mycmd.onecmd("")
+    9
+    >>> mycmd.onecmd("test")
+    *** Unknown syntax: test
+
+    Test for the function emptyline():
+    >>> mycmd.emptyline()
+    *** Unknown syntax: test
+
+    Test for the function default():
+    >>> mycmd.default("default")
+    *** Unknown syntax: default
+
+    Test for the function completedefault():
+    >>> mycmd.completedefault()
+    This is the completedefault method
+    >>> mycmd.completenames("a")
+    ['add']
+
+    Test for the function completenames():
+    >>> mycmd.completenames("12")
+    []
+    >>> mycmd.completenames("help")
+    ['help']
+
+    Test for the function complete_help():
+    >>> mycmd.complete_help("a")
+    ['add']
+    >>> mycmd.complete_help("he")
+    ['help']
+    >>> mycmd.complete_help("12")
+    []
+    >>> sorted(mycmd.complete_help(""))
+    ['add', 'exit', 'help', 'shell']
+
+    Test for the function do_help():
+    >>> mycmd.do_help("testet")
+    *** No help on testet
+    >>> mycmd.do_help("add")
+    help text for add
+    >>> mycmd.onecmd("help add")
+    help text for add
+    >>> mycmd.do_help("")
+    <BLANKLINE>
+    Documented commands (type help <topic>):
+    ========================================
+    add  help
+    <BLANKLINE>
+    Undocumented commands:
+    ======================
+    exit  shell
+    <BLANKLINE>
+
+    Test for the function print_topics():
+    >>> mycmd.print_topics("header", ["command1", "command2"], 2 ,10)
+    header
+    ======
+    command1
+    command2
+    <BLANKLINE>
+
+    Test for the function columnize():
+    >>> mycmd.columnize([str(i) for i in range(20)])
+    0  1  2  3  4  5  6  7  8  9  10  11  12  13  14  15  16  17  18  19
+    >>> mycmd.columnize([str(i) for i in range(20)], 10)
+    0  7   14
+    1  8   15
+    2  9   16
+    3  10  17
+    4  11  18
+    5  12  19
+    6  13
+
+    This is an interactive test, put some commands in the cmdqueue attribute
+    and let it execute
+    This test includes the preloop(), postloop(), default(), emptyline(),
+    parseline(), do_help() functions
+    >>> mycmd.use_rawinput=0
+    >>> mycmd.cmdqueue=["", "add", "add 4 5", "help", "help add","exit"]
+    >>> mycmd.cmdloop()
+    Hello from preloop
+    help text for add
+    *** invalid number of arguments
+    9
+    <BLANKLINE>
+    Documented commands (type help <topic>):
+    ========================================
+    add  help
+    <BLANKLINE>
+    Undocumented commands:
+    ======================
+    exit  shell
+    <BLANKLINE>
+    help text for add
+    Hello from postloop
+    """
+
+    def preloop(self):
+        print("Hello from preloop")
+
+    def postloop(self):
+        print("Hello from postloop")
+
+    def completedefault(self, *ignored):
+        print("This is the completedefault method")
+
+    def complete_command(self):
+        print("complete command")
+
+    def do_shell(self, s):
+        pass
+
+    def do_add(self, s):
+        l = s.split()
+        if len(l) != 2:
+            print("*** invalid number of arguments")
+            return
+        try:
+            l = [int(i) for i in l]
+        except ValueError:
+            print("*** arguments should be numbers")
+            return
+        print(l[0]+l[1])
+
+    def help_add(self):
+        print("help text for add")
+        return
+
+    def do_exit(self, arg):
+        return True
+
+
+class TestAlternateInput(unittest.TestCase):
+
+    class simplecmd(cmd.Cmd):
+
+        def do_print(self, args):
+            print(args, file=self.stdout)
+
+        def do_EOF(self, args):
+            return True
+
+
+    class simplecmd2(simplecmd):
+
+        def do_EOF(self, args):
+            print('*** Unknown syntax: EOF', file=self.stdout)
+            return True
+
+
+    def test_file_with_missing_final_nl(self):
+        input = io.StringIO("print test\nprint test2")
+        output = io.StringIO()
+        cmd = self.simplecmd(stdin=input, stdout=output)
+        cmd.use_rawinput = False
+        cmd.cmdloop()
+        self.assertMultiLineEqual(output.getvalue(),
+            ("(Cmd) test\n"
+             "(Cmd) test2\n"
+             "(Cmd) "))
+
+
+    def test_input_reset_at_EOF(self):
+        input = io.StringIO("print test\nprint test2")
+        output = io.StringIO()
+        cmd = self.simplecmd2(stdin=input, stdout=output)
+        cmd.use_rawinput = False
+        cmd.cmdloop()
+        self.assertMultiLineEqual(output.getvalue(),
+            ("(Cmd) test\n"
+             "(Cmd) test2\n"
+             "(Cmd) *** Unknown syntax: EOF\n"))
+        input = io.StringIO("print \n\n")
+        output = io.StringIO()
+        cmd.stdin = input
+        cmd.stdout = output
+        cmd.cmdloop()
+        self.assertMultiLineEqual(output.getvalue(),
+            ("(Cmd) \n"
+             "(Cmd) \n"
+             "(Cmd) *** Unknown syntax: EOF\n"))
+
+
+def test_main(verbose=None):
+    from test import test_cmd
+    support.run_doctest(test_cmd, verbose)
+    support.run_unittest(TestAlternateInput)
+
+def test_coverage(coverdir):
+    trace = support.import_module('trace')
+    tracer=trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,],
+                        trace=0, count=1)
+    tracer.run('import importlib; importlib.reload(cmd); test_main()')
+    r=tracer.results()
+    print("Writing coverage results...")
+    r.write_results(show_missing=True, summary=True, coverdir=coverdir)
+
+if __name__ == "__main__":
+    if "-c" in sys.argv:
+        test_coverage('/tmp/cmd.cover')
+    elif "-i" in sys.argv:
+        samplecmdclass().cmdloop()
+    else:
+        test_main()

Lib/test/test_dis.py

@@ -0,0 +1,60 @@
+import subprocess
+import sys
+import unittest
+
+# This only tests that it prints something in order
+# to avoid changing this test if the bytecode changes
+
+# These tests start a new process instead of redirecting stdout because
+# stdout is being written to by rust code, which currently can't be
+# redirected by reassigning sys.stdout
+
+
+class TestDis(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.setup = """
+import dis
+def tested_func(): pass
+"""
+        cls.command = (sys.executable, "-c")
+
+    def test_dis(self):
+        test_code = f"""
+{self.setup}
+dis.dis(tested_func)
+dis.dis("x = 2; print(x)")
+"""
+
+        result = subprocess.run(
+            self.command + (test_code,), capture_output=True
+        )
+        self.assertNotEqual("", result.stdout.decode())
+        self.assertEqual("", result.stderr.decode())
+
+    def test_disassemble(self):
+        test_code = f"""
+{self.setup}
+dis.disassemble(tested_func)
+"""
+        result = subprocess.run(
+            self.command + (test_code,), capture_output=True
+        )
+        # In CPython this would raise an AttributeError, not a
+        # TypeError because dis is implemented in python in CPython and
+        # as such the type mismatch wouldn't be caught immeadiately
+        self.assertIn("TypeError", result.stderr.decode())
+
+        test_code = f"""
+{self.setup}
+dis.disassemble(tested_func.__code__)
+"""
+        result = subprocess.run(
+            self.command + (test_code,), capture_output=True
+        )
+        self.assertNotEqual("", result.stdout.decode())
+        self.assertEqual("", result.stderr.decode())
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_fractions.py

@@ -392,8 +392,6 @@ class FractionTest(unittest.TestCase):
         self.assertTypedEquals(F(-2, 10), round(F(-15, 100), 1))
         self.assertTypedEquals(F(-2, 10), round(F(-25, 100), 1))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def testArithmetic(self):
         self.assertEqual(F(1, 2), F(1, 10) + F(2, 5))
         self.assertEqual(F(-3, 10), F(1, 10) - F(2, 5))

Lib/test/test_imp.py

@@ -0,0 +1,472 @@
+import importlib
+import importlib.util
+import os
+import os.path
+import py_compile
+import sys
+from test import support
+from test.support import script_helper
+import unittest
+import warnings
+with warnings.catch_warnings():
+    warnings.simplefilter('ignore', DeprecationWarning)
+    import imp
+import _imp
+
+
+def requires_load_dynamic(meth):
+    """Decorator to skip a test if not running under CPython or lacking
+    imp.load_dynamic()."""
+    meth = support.cpython_only(meth)
+    return unittest.skipIf(not hasattr(imp, 'load_dynamic'),
+                           'imp.load_dynamic() required')(meth)
+
+
+class LockTests(unittest.TestCase):
+
+    """Very basic test of import lock functions."""
+
+    def verify_lock_state(self, expected):
+        self.assertEqual(imp.lock_held(), expected,
+                             "expected imp.lock_held() to be %r" % expected)
+    def testLock(self):
+        LOOPS = 50
+
+        # The import lock may already be held, e.g. if the test suite is run
+        # via "import test.autotest".
+        lock_held_at_start = imp.lock_held()
+        self.verify_lock_state(lock_held_at_start)
+
+        for i in range(LOOPS):
+            imp.acquire_lock()
+            self.verify_lock_state(True)
+
+        for i in range(LOOPS):
+            imp.release_lock()
+
+        # The original state should be restored now.
+        self.verify_lock_state(lock_held_at_start)
+
+        if not lock_held_at_start:
+            try:
+                imp.release_lock()
+            except RuntimeError:
+                pass
+            else:
+                self.fail("release_lock() without lock should raise "
+                            "RuntimeError")
+
+class ImportTests(unittest.TestCase):
+    # TODO: RustPython
+    # def setUp(self):
+    #     mod = importlib.import_module('test.encoded_modules')
+    #     self.test_strings = mod.test_strings
+    #     self.test_path = mod.__path__
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_import_encoded_module(self):
+        for modname, encoding, teststr in self.test_strings:
+            mod = importlib.import_module('test.encoded_modules.'
+                                          'module_' + modname)
+            self.assertEqual(teststr, mod.test)
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_find_module_encoding(self):
+        for mod, encoding, _ in self.test_strings:
+            with imp.find_module('module_' + mod, self.test_path)[0] as fd:
+                self.assertEqual(fd.encoding, encoding)
+
+        path = [os.path.dirname(__file__)]
+        with self.assertRaises(SyntaxError):
+            imp.find_module('badsyntax_pep3120', path)
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_issue1267(self):
+        for mod, encoding, _ in self.test_strings:
+            fp, filename, info  = imp.find_module('module_' + mod,
+                                                  self.test_path)
+            with fp:
+                self.assertNotEqual(fp, None)
+                self.assertEqual(fp.encoding, encoding)
+                self.assertEqual(fp.tell(), 0)
+                self.assertEqual(fp.readline(), '# test %s encoding\n'
+                                 % encoding)
+
+        fp, filename, info = imp.find_module("tokenize")
+        with fp:
+            self.assertNotEqual(fp, None)
+            self.assertEqual(fp.encoding, "utf-8")
+            self.assertEqual(fp.tell(), 0)
+            self.assertEqual(fp.readline(),
+                             '"""Tokenization help for Python programs.\n')
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_issue3594(self):
+        temp_mod_name = 'test_imp_helper'
+        sys.path.insert(0, '.')
+        try:
+            with open(temp_mod_name + '.py', 'w') as file:
+                file.write("# coding: cp1252\nu = 'test.test_imp'\n")
+            file, filename, info = imp.find_module(temp_mod_name)
+            file.close()
+            self.assertEqual(file.encoding, 'cp1252')
+        finally:
+            del sys.path[0]
+            support.unlink(temp_mod_name + '.py')
+            support.unlink(temp_mod_name + '.pyc')
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_issue5604(self):
+        # Test cannot cover imp.load_compiled function.
+        # Martin von Loewis note what shared library cannot have non-ascii
+        # character because init_xxx function cannot be compiled
+        # and issue never happens for dynamic modules.
+        # But sources modified to follow generic way for processing paths.
+
+        # the return encoding could be uppercase or None
+        fs_encoding = sys.getfilesystemencoding()
+
+        # covers utf-8 and Windows ANSI code pages
+        # one non-space symbol from every page
+        # (http://en.wikipedia.org/wiki/Code_page)
+        known_locales = {
+            'utf-8' : b'\xc3\xa4',
+            'cp1250' : b'\x8C',
+            'cp1251' : b'\xc0',
+            'cp1252' : b'\xc0',
+            'cp1253' : b'\xc1',
+            'cp1254' : b'\xc0',
+            'cp1255' : b'\xe0',
+            'cp1256' : b'\xe0',
+            'cp1257' : b'\xc0',
+            'cp1258' : b'\xc0',
+            }
+
+        if sys.platform == 'darwin':
+            self.assertEqual(fs_encoding, 'utf-8')
+            # Mac OS X uses the Normal Form D decomposition
+            # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html
+            special_char = b'a\xcc\x88'
+        else:
+            special_char = known_locales.get(fs_encoding)
+
+        if not special_char:
+            self.skipTest("can't run this test with %s as filesystem encoding"
+                          % fs_encoding)
+        decoded_char = special_char.decode(fs_encoding)
+        temp_mod_name = 'test_imp_helper_' + decoded_char
+        test_package_name = 'test_imp_helper_package_' + decoded_char
+        init_file_name = os.path.join(test_package_name, '__init__.py')
+        try:
+            # if the curdir is not in sys.path the test fails when run with
+            # ./python ./Lib/test/regrtest.py test_imp
+            sys.path.insert(0, os.curdir)
+            with open(temp_mod_name + '.py', 'w') as file:
+                file.write('a = 1\n')
+            file, filename, info = imp.find_module(temp_mod_name)
+            with file:
+                self.assertIsNotNone(file)
+                self.assertTrue(filename[:-3].endswith(temp_mod_name))
+                self.assertEqual(info[0], '.py')
+                self.assertEqual(info[1], 'r')
+                self.assertEqual(info[2], imp.PY_SOURCE)
+
+                mod = imp.load_module(temp_mod_name, file, filename, info)
+                self.assertEqual(mod.a, 1)
+
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore')
+                mod = imp.load_source(temp_mod_name, temp_mod_name + '.py')
+            self.assertEqual(mod.a, 1)
+
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore')
+                if not sys.dont_write_bytecode:
+                    mod = imp.load_compiled(
+                        temp_mod_name,
+                        imp.cache_from_source(temp_mod_name + '.py'))
+            self.assertEqual(mod.a, 1)
+
+            if not os.path.exists(test_package_name):
+                os.mkdir(test_package_name)
+            with open(init_file_name, 'w') as file:
+                file.write('b = 2\n')
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore')
+                package = imp.load_package(test_package_name, test_package_name)
+            self.assertEqual(package.b, 2)
+        finally:
+            del sys.path[0]
+            for ext in ('.py', '.pyc'):
+                support.unlink(temp_mod_name + ext)
+                support.unlink(init_file_name + ext)
+            support.rmtree(test_package_name)
+            support.rmtree('__pycache__')
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_issue9319(self):
+        path = os.path.dirname(__file__)
+        self.assertRaises(SyntaxError,
+                          imp.find_module, "badsyntax_pep3120", [path])
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_load_from_source(self):
+        # Verify that the imp module can correctly load and find .py files
+        # XXX (ncoghlan): It would be nice to use support.CleanImport
+        # here, but that breaks because the os module registers some
+        # handlers in copy_reg on import. Since CleanImport doesn't
+        # revert that registration, the module is left in a broken
+        # state after reversion. Reinitialising the module contents
+        # and just reverting os.environ to its previous state is an OK
+        # workaround
+        orig_path = os.path
+        orig_getenv = os.getenv
+        with support.EnvironmentVarGuard():
+            x = imp.find_module("os")
+            self.addCleanup(x[0].close)
+            new_os = imp.load_module("os", *x)
+            self.assertIs(os, new_os)
+            self.assertIs(orig_path, new_os.path)
+            self.assertIsNot(orig_getenv, new_os.getenv)
+
+    @requires_load_dynamic
+    def test_issue15828_load_extensions(self):
+        # Issue 15828 picked up that the adapter between the old imp API
+        # and importlib couldn't handle C extensions
+        example = "_heapq"
+        x = imp.find_module(example)
+        file_ = x[0]
+        if file_ is not None:
+            self.addCleanup(file_.close)
+        mod = imp.load_module(example, *x)
+        self.assertEqual(mod.__name__, example)
+
+    @requires_load_dynamic
+    def test_issue16421_multiple_modules_in_one_dll(self):
+        # Issue 16421: loading several modules from the same compiled file fails
+        m = '_testimportmultiple'
+        fileobj, pathname, description = imp.find_module(m)
+        fileobj.close()
+        mod0 = imp.load_dynamic(m, pathname)
+        mod1 = imp.load_dynamic('_testimportmultiple_foo', pathname)
+        mod2 = imp.load_dynamic('_testimportmultiple_bar', pathname)
+        self.assertEqual(mod0.__name__, m)
+        self.assertEqual(mod1.__name__, '_testimportmultiple_foo')
+        self.assertEqual(mod2.__name__, '_testimportmultiple_bar')
+        with self.assertRaises(ImportError):
+            imp.load_dynamic('nonexistent', pathname)
+
+    @requires_load_dynamic
+    def test_load_dynamic_ImportError_path(self):
+        # Issue #1559549 added `name` and `path` attributes to ImportError
+        # in order to provide better detail. Issue #10854 implemented those
+        # attributes on import failures of extensions on Windows.
+        path = 'bogus file path'
+        name = 'extension'
+        with self.assertRaises(ImportError) as err:
+            imp.load_dynamic(name, path)
+        self.assertIn(path, err.exception.path)
+        self.assertEqual(name, err.exception.name)
+
+    @requires_load_dynamic
+    def test_load_module_extension_file_is_None(self):
+        # When loading an extension module and the file is None, open one
+        # on the behalf of imp.load_dynamic().
+        # Issue #15902
+        name = '_testimportmultiple'
+        found = imp.find_module(name)
+        if found[0] is not None:
+            found[0].close()
+        if found[2][2] != imp.C_EXTENSION:
+            self.skipTest("found module doesn't appear to be a C extension")
+        imp.load_module(name, None, *found[1:])
+
+    @requires_load_dynamic
+    def test_issue24748_load_module_skips_sys_modules_check(self):
+        name = 'test.imp_dummy'
+        try:
+            del sys.modules[name]
+        except KeyError:
+            pass
+        try:
+            module = importlib.import_module(name)
+            spec = importlib.util.find_spec('_testmultiphase')
+            module = imp.load_dynamic(name, spec.origin)
+            self.assertEqual(module.__name__, name)
+            self.assertEqual(module.__spec__.name, name)
+            self.assertEqual(module.__spec__.origin, spec.origin)
+            self.assertRaises(AttributeError, getattr, module, 'dummy_name')
+            self.assertEqual(module.int_const, 1969)
+            self.assertIs(sys.modules[name], module)
+        finally:
+            try:
+                del sys.modules[name]
+            except KeyError:
+                pass
+
+    @unittest.skipIf(sys.dont_write_bytecode,
+        "test meaningful only when writing bytecode")
+    def test_bug7732(self):
+        with support.temp_cwd():
+            source = support.TESTFN + '.py'
+            os.mkdir(source)
+            self.assertRaisesRegex(ImportError, '^No module',
+                imp.find_module, support.TESTFN, ["."])
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_multiple_calls_to_get_data(self):
+        # Issue #18755: make sure multiple calls to get_data() can succeed.
+        loader = imp._LoadSourceCompatibility('imp', imp.__file__,
+                                              open(imp.__file__))
+        loader.get_data(imp.__file__)  # File should be closed
+        loader.get_data(imp.__file__)  # Will need to create a newly opened file
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_load_source(self):
+        # Create a temporary module since load_source(name) modifies
+        # sys.modules[name] attributes like __loader___
+        modname = f"tmp{__name__}"
+        mod = type(sys.modules[__name__])(modname)
+        with support.swap_item(sys.modules, modname, mod):
+            with self.assertRaisesRegex(ValueError, 'embedded null'):
+                imp.load_source(modname, __file__ + "\0")
+
+    @support.cpython_only
+    def test_issue31315(self):
+        # There shouldn't be an assertion failure in imp.create_dynamic(),
+        # when spec.name is not a string.
+        create_dynamic = support.get_attribute(imp, 'create_dynamic')
+        class BadSpec:
+            name = None
+            origin = 'foo'
+        with self.assertRaises(TypeError):
+            create_dynamic(BadSpec())
+
+    def test_issue_35321(self):
+        # Both _frozen_importlib and _frozen_importlib_external
+        # should have a spec origin of "frozen" and
+        # no need to clean up imports in this case.
+
+        import _frozen_importlib_external
+        self.assertEqual(_frozen_importlib_external.__spec__.origin, "frozen")
+
+        import _frozen_importlib
+        self.assertEqual(_frozen_importlib.__spec__.origin, "frozen")
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_source_hash(self):
+        self.assertEqual(_imp.source_hash(42, b'hi'), b'\xc6\xe7Z\r\x03:}\xab')
+        self.assertEqual(_imp.source_hash(43, b'hi'), b'\x85\x9765\xf8\x9a\x8b9')
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_pyc_invalidation_mode_from_cmdline(self):
+        cases = [
+            ([], "default"),
+            (["--check-hash-based-pycs", "default"], "default"),
+            (["--check-hash-based-pycs", "always"], "always"),
+            (["--check-hash-based-pycs", "never"], "never"),
+        ]
+        for interp_args, expected in cases:
+            args = interp_args + [
+                "-c",
+                "import _imp; print(_imp.check_hash_based_pycs)",
+            ]
+            res = script_helper.assert_python_ok(*args)
+            self.assertEqual(res.out.strip().decode('utf-8'), expected)
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_find_and_load_checked_pyc(self):
+        # issue 34056
+        with support.temp_cwd():
+            with open('mymod.py', 'wb') as fp:
+                fp.write(b'x = 42\n')
+            py_compile.compile(
+                'mymod.py',
+                doraise=True,
+                invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+            )
+            file, path, description = imp.find_module('mymod', path=['.'])
+            mod = imp.load_module('mymod', file, path, description)
+        self.assertEqual(mod.x, 42)
+
+
+class ReloadTests(unittest.TestCase):
+
+    """Very basic tests to make sure that imp.reload() operates just like
+    reload()."""
+
+    def test_source(self):
+        # XXX (ncoghlan): It would be nice to use test.support.CleanImport
+        # here, but that breaks because the os module registers some
+        # handlers in copy_reg on import. Since CleanImport doesn't
+        # revert that registration, the module is left in a broken
+        # state after reversion. Reinitialising the module contents
+        # and just reverting os.environ to its previous state is an OK
+        # workaround
+        with support.EnvironmentVarGuard():
+            import os
+            imp.reload(os)
+
+    def test_extension(self):
+        with support.CleanImport('time'):
+            import time
+            imp.reload(time)
+
+    def test_builtin(self):
+        with support.CleanImport('marshal'):
+            import marshal
+            imp.reload(marshal)
+
+    def test_with_deleted_parent(self):
+        # see #18681
+        from html import parser
+        html = sys.modules.pop('html')
+        def cleanup():
+            sys.modules['html'] = html
+        self.addCleanup(cleanup)
+        with self.assertRaisesRegex(ImportError, 'html'):
+            imp.reload(parser)
+
+
+class PEP3147Tests(unittest.TestCase):
+    """Tests of PEP 3147."""
+
+    tag = imp.get_tag()
+
+    @unittest.skipUnless(sys.implementation.cache_tag is not None,
+                         'requires sys.implementation.cache_tag not be None')
+    def test_cache_from_source(self):
+        # Given the path to a .py file, return the path to its PEP 3147
+        # defined .pyc file (i.e. under __pycache__).
+        path = os.path.join('foo', 'bar', 'baz', 'qux.py')
+        expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
+                              'qux.{}.pyc'.format(self.tag))
+        self.assertEqual(imp.cache_from_source(path, True), expect)
+
+    @unittest.skipUnless(sys.implementation.cache_tag is not None,
+                         'requires sys.implementation.cache_tag to not be '
+                         'None')
+    def test_source_from_cache(self):
+        # Given the path to a PEP 3147 defined .pyc file, return the path to
+        # its source.  This tests the good path.
+        path = os.path.join('foo', 'bar', 'baz', '__pycache__',
+                            'qux.{}.pyc'.format(self.tag))
+        expect = os.path.join('foo', 'bar', 'baz', 'qux.py')
+        self.assertEqual(imp.source_from_cache(path), expect)
+
+
+class NullImporterTests(unittest.TestCase):
+    @unittest.skipIf(support.TESTFN_UNENCODABLE is None,
+                     "Need an undecodeable filename")
+    def test_unencodeable(self):
+        name = support.TESTFN_UNENCODABLE
+        os.mkdir(name)
+        try:
+            self.assertRaises(ImportError, imp.NullImporter, name)
+        finally:
+            os.rmdir(name)
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_importlib/__init__.py

@@ -0,0 +1,5 @@
+import os
+from test.support import load_package_tests
+
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)

Lib/test/test_importlib/__main__.py

@@ -0,0 +1,4 @@
+from . import load_tests
+import unittest
+
+unittest.main()

Lib/test/test_importlib/abc.py

@@ -0,0 +1,93 @@
+import abc
+
+
+class FinderTests(metaclass=abc.ABCMeta):
+
+    """Basic tests for a finder to pass."""
+
+    @abc.abstractmethod
+    def test_module(self):
+        # Test importing a top-level module.
+        pass
+
+    @abc.abstractmethod
+    def test_package(self):
+        # Test importing a package.
+        pass
+
+    @abc.abstractmethod
+    def test_module_in_package(self):
+        # Test importing a module contained within a package.
+        # A value for 'path' should be used if for a meta_path finder.
+        pass
+
+    @abc.abstractmethod
+    def test_package_in_package(self):
+        # Test importing a subpackage.
+        # A value for 'path' should be used if for a meta_path finder.
+        pass
+
+    @abc.abstractmethod
+    def test_package_over_module(self):
+        # Test that packages are chosen over modules.
+        pass
+
+    @abc.abstractmethod
+    def test_failure(self):
+        # Test trying to find a module that cannot be handled.
+        pass
+
+
+class LoaderTests(metaclass=abc.ABCMeta):
+
+    @abc.abstractmethod
+    def test_module(self):
+        """A module should load without issue.
+
+        After the loader returns the module should be in sys.modules.
+
+        Attributes to verify:
+
+            * __file__
+            * __loader__
+            * __name__
+            * No __path__
+
+        """
+        pass
+
+    @abc.abstractmethod
+    def test_package(self):
+        """Loading a package should work.
+
+        After the loader returns the module should be in sys.modules.
+
+        Attributes to verify:
+
+            * __name__
+            * __file__
+            * __package__
+            * __path__
+            * __loader__
+
+        """
+        pass
+
+    @abc.abstractmethod
+    def test_lacking_parent(self):
+        """A loader should not be dependent on it's parent package being
+        imported."""
+        pass
+
+    @abc.abstractmethod
+    def test_state_after_failure(self):
+        """If a module is already in sys.modules and a reload fails
+        (e.g. a SyntaxError), the module should be in the state it was before
+        the reload began."""
+        pass
+
+    @abc.abstractmethod
+    def test_unloadable(self):
+        """Test ImportError is raised when the loader is asked to load a module
+        it can't."""
+        pass

Lib/test/test_importlib/builtin/__init__.py

@@ -0,0 +1,5 @@
+import os
+from test.support import load_package_tests
+
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)

Lib/test/test_importlib/builtin/__main__.py

@@ -0,0 +1,4 @@
+from . import load_tests
+import unittest
+
+unittest.main()

Lib/test/test_importlib/builtin/test_finder.py

@@ -0,0 +1,90 @@
+from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import sys
+import unittest
+
+
+@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module')
+class FindSpecTests(abc.FinderTests):
+
+    """Test find_spec() for built-in modules."""
+
+    def test_module(self):
+        # Common case.
+        with util.uncache(util.BUILTINS.good_name):
+            found = self.machinery.BuiltinImporter.find_spec(util.BUILTINS.good_name)
+            self.assertTrue(found)
+            self.assertEqual(found.origin, 'built-in')
+
+    # Built-in modules cannot be a package.
+    test_package = None
+
+    # Built-in modules cannot be in a package.
+    test_module_in_package = None
+
+    # Built-in modules cannot be a package.
+    test_package_in_package = None
+
+    # Built-in modules cannot be a package.
+    test_package_over_module = None
+
+    def test_failure(self):
+        name = 'importlib'
+        assert name not in sys.builtin_module_names
+        spec = self.machinery.BuiltinImporter.find_spec(name)
+        self.assertIsNone(spec)
+
+    def test_ignore_path(self):
+        # The value for 'path' should always trigger a failed import.
+        with util.uncache(util.BUILTINS.good_name):
+            spec = self.machinery.BuiltinImporter.find_spec(util.BUILTINS.good_name,
+                                                            ['pkg'])
+            self.assertIsNone(spec)
+
+
+(Frozen_FindSpecTests,
+ Source_FindSpecTests
+ ) = util.test_both(FindSpecTests, machinery=machinery)
+
+
+@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module')
+class FinderTests(abc.FinderTests):
+
+    """Test find_module() for built-in modules."""
+
+    def test_module(self):
+        # Common case.
+        with util.uncache(util.BUILTINS.good_name):
+            found = self.machinery.BuiltinImporter.find_module(util.BUILTINS.good_name)
+            self.assertTrue(found)
+            self.assertTrue(hasattr(found, 'load_module'))
+
+    # Built-in modules cannot be a package.
+    test_package = test_package_in_package = test_package_over_module = None
+
+    # Built-in modules cannot be in a package.
+    test_module_in_package = None
+
+    def test_failure(self):
+        assert 'importlib' not in sys.builtin_module_names
+        loader = self.machinery.BuiltinImporter.find_module('importlib')
+        self.assertIsNone(loader)
+
+    def test_ignore_path(self):
+        # The value for 'path' should always trigger a failed import.
+        with util.uncache(util.BUILTINS.good_name):
+            loader = self.machinery.BuiltinImporter.find_module(util.BUILTINS.good_name,
+                                                            ['pkg'])
+            self.assertIsNone(loader)
+
+
+(Frozen_FinderTests,
+ Source_FinderTests
+ ) = util.test_both(FinderTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/builtin/test_loader.py

@@ -0,0 +1,108 @@
+from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import sys
+import types
+import unittest
+
+@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module')
+class LoaderTests(abc.LoaderTests):
+
+    """Test load_module() for built-in modules."""
+
+    def setUp(self):
+        self.verification = {'__name__': 'errno', '__package__': '',
+                             '__loader__': self.machinery.BuiltinImporter}
+
+    def verify(self, module):
+        """Verify that the module matches against what it should have."""
+        self.assertIsInstance(module, types.ModuleType)
+        for attr, value in self.verification.items():
+            self.assertEqual(getattr(module, attr), value)
+        self.assertIn(module.__name__, sys.modules)
+
+    def load_module(self, name):
+        return self.machinery.BuiltinImporter.load_module(name)
+
+    def test_module(self):
+        # Common case.
+        with util.uncache(util.BUILTINS.good_name):
+            module = self.load_module(util.BUILTINS.good_name)
+            self.verify(module)
+
+    # Built-in modules cannot be a package.
+    test_package = test_lacking_parent = None
+
+    # No way to force an import failure.
+    test_state_after_failure = None
+
+    def test_module_reuse(self):
+        # Test that the same module is used in a reload.
+        with util.uncache(util.BUILTINS.good_name):
+            module1 = self.load_module(util.BUILTINS.good_name)
+            module2 = self.load_module(util.BUILTINS.good_name)
+            self.assertIs(module1, module2)
+
+    def test_unloadable(self):
+        name = 'dssdsdfff'
+        assert name not in sys.builtin_module_names
+        with self.assertRaises(ImportError) as cm:
+            self.load_module(name)
+        self.assertEqual(cm.exception.name, name)
+
+    def test_already_imported(self):
+        # Using the name of a module already imported but not a built-in should
+        # still fail.
+        module_name = 'builtin_reload_test'
+        assert module_name not in sys.builtin_module_names
+        with util.uncache(module_name):
+            module = types.ModuleType(module_name)
+            sys.modules[module_name] = module
+        with self.assertRaises(ImportError) as cm:
+            self.load_module(module_name)
+        self.assertEqual(cm.exception.name, module_name)
+
+
+(Frozen_LoaderTests,
+ Source_LoaderTests
+ ) = util.test_both(LoaderTests, machinery=machinery)
+
+
+@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module')
+class InspectLoaderTests:
+
+    """Tests for InspectLoader methods for BuiltinImporter."""
+
+    def test_get_code(self):
+        # There is no code object.
+        result = self.machinery.BuiltinImporter.get_code(util.BUILTINS.good_name)
+        self.assertIsNone(result)
+
+    def test_get_source(self):
+        # There is no source.
+        result = self.machinery.BuiltinImporter.get_source(util.BUILTINS.good_name)
+        self.assertIsNone(result)
+
+    def test_is_package(self):
+        # Cannot be a package.
+        result = self.machinery.BuiltinImporter.is_package(util.BUILTINS.good_name)
+        self.assertFalse(result)
+
+    @unittest.skipIf(util.BUILTINS.bad_name is None, 'all modules are built in')
+    def test_not_builtin(self):
+        # Modules not built-in should raise ImportError.
+        for meth_name in ('get_code', 'get_source', 'is_package'):
+            method = getattr(self.machinery.BuiltinImporter, meth_name)
+        with self.assertRaises(ImportError) as cm:
+            method(util.BUILTINS.bad_name)
+
+
+(Frozen_InspectLoaderTests,
+ Source_InspectLoaderTests
+ ) = util.test_both(InspectLoaderTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/data01/utf-8.file

@@ -0,0 +1 @@
+Hello, UTF-8 world!

Lib/test/test_importlib/data02/one/resource1.txt

@@ -0,0 +1 @@
+one resource

Lib/test/test_importlib/data02/two/resource2.txt

@@ -0,0 +1 @@
+two resource

Lib/test/test_importlib/extension/__init__.py

@@ -0,0 +1,5 @@
+import os
+from test.support import load_package_tests
+
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)

Lib/test/test_importlib/extension/__main__.py

@@ -0,0 +1,4 @@
+from . import load_tests
+import unittest
+
+unittest.main()

Lib/test/test_importlib/extension/test_case_sensitivity.py

@@ -0,0 +1,46 @@
+from importlib import _bootstrap_external
+from test import support
+import unittest
+
+from .. import util
+
+importlib = util.import_importlib('importlib')
+machinery = util.import_importlib('importlib.machinery')
+
+
+@unittest.skipIf(util.EXTENSIONS.filename is None, '_testcapi not available')
+@util.case_insensitive_tests
+class ExtensionModuleCaseSensitivityTest(util.CASEOKTestBase):
+
+    def find_module(self):
+        good_name = util.EXTENSIONS.name
+        bad_name = good_name.upper()
+        assert good_name != bad_name
+        finder = self.machinery.FileFinder(util.EXTENSIONS.path,
+                                          (self.machinery.ExtensionFileLoader,
+                                           self.machinery.EXTENSION_SUFFIXES))
+        return finder.find_module(bad_name)
+
+    def test_case_sensitive(self):
+        with support.EnvironmentVarGuard() as env:
+            env.unset('PYTHONCASEOK')
+            self.caseok_env_changed(should_exist=False)
+            loader = self.find_module()
+            self.assertIsNone(loader)
+
+    def test_case_insensitivity(self):
+        with support.EnvironmentVarGuard() as env:
+            env.set('PYTHONCASEOK', '1')
+            self.caseok_env_changed(should_exist=True)
+            loader = self.find_module()
+            self.assertTrue(hasattr(loader, 'load_module'))
+
+
+(Frozen_ExtensionCaseSensitivity,
+ Source_ExtensionCaseSensitivity
+ ) = util.test_both(ExtensionModuleCaseSensitivityTest, importlib=importlib,
+                    machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/extension/test_finder.py

@@ -0,0 +1,46 @@
+from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import unittest
+import warnings
+
+
+class FinderTests(abc.FinderTests):
+
+    """Test the finder for extension modules."""
+
+    def find_module(self, fullname):
+        importer = self.machinery.FileFinder(util.EXTENSIONS.path,
+                                            (self.machinery.ExtensionFileLoader,
+                                             self.machinery.EXTENSION_SUFFIXES))
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', DeprecationWarning)
+            return importer.find_module(fullname)
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_module(self):
+        self.assertTrue(self.find_module(util.EXTENSIONS.name))
+
+    # No extension module as an __init__ available for testing.
+    test_package = test_package_in_package = None
+
+    # No extension module in a package available for testing.
+    test_module_in_package = None
+
+    # Extension modules cannot be an __init__ for a package.
+    test_package_over_module = None
+
+    def test_failure(self):
+        self.assertIsNone(self.find_module('asdfjkl;'))
+
+
+(Frozen_FinderTests,
+ Source_FinderTests
+ ) = util.test_both(FinderTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/extension/test_loader.py

@@ -0,0 +1,306 @@
+from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import os.path
+import sys
+import types
+import unittest
+import importlib.util
+import importlib
+from test.support.script_helper import assert_python_failure
+
+class LoaderTests(abc.LoaderTests):
+
+    """Test load_module() for extension modules."""
+
+    def setUp(self):
+        self.loader = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name,
+                                                         util.EXTENSIONS.file_path)
+
+    def load_module(self, fullname):
+        return self.loader.load_module(fullname)
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_load_module_API(self):
+        # Test the default argument for load_module().
+        self.loader.load_module()
+        self.loader.load_module(None)
+        with self.assertRaises(ImportError):
+            self.load_module('XXX')
+
+    def test_equality(self):
+        other = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name,
+                                                   util.EXTENSIONS.file_path)
+        self.assertEqual(self.loader, other)
+
+    def test_inequality(self):
+        other = self.machinery.ExtensionFileLoader('_' + util.EXTENSIONS.name,
+                                                   util.EXTENSIONS.file_path)
+        self.assertNotEqual(self.loader, other)
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_module(self):
+        with util.uncache(util.EXTENSIONS.name):
+            module = self.load_module(util.EXTENSIONS.name)
+            for attr, value in [('__name__', util.EXTENSIONS.name),
+                                ('__file__', util.EXTENSIONS.file_path),
+                                ('__package__', '')]:
+                self.assertEqual(getattr(module, attr), value)
+            self.assertIn(util.EXTENSIONS.name, sys.modules)
+            self.assertIsInstance(module.__loader__,
+                                  self.machinery.ExtensionFileLoader)
+
+    # No extension module as __init__ available for testing.
+    test_package = None
+
+    # No extension module in a package available for testing.
+    test_lacking_parent = None
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_module_reuse(self):
+        with util.uncache(util.EXTENSIONS.name):
+            module1 = self.load_module(util.EXTENSIONS.name)
+            module2 = self.load_module(util.EXTENSIONS.name)
+            self.assertIs(module1, module2)
+
+    # No easy way to trigger a failure after a successful import.
+    test_state_after_failure = None
+
+    def test_unloadable(self):
+        name = 'asdfjkl;'
+        with self.assertRaises(ImportError) as cm:
+            self.load_module(name)
+        self.assertEqual(cm.exception.name, name)
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_is_package(self):
+        self.assertFalse(self.loader.is_package(util.EXTENSIONS.name))
+        for suffix in self.machinery.EXTENSION_SUFFIXES:
+            path = os.path.join('some', 'path', 'pkg', '__init__' + suffix)
+            loader = self.machinery.ExtensionFileLoader('pkg', path)
+            self.assertTrue(loader.is_package('pkg'))
+
+(Frozen_LoaderTests,
+ Source_LoaderTests
+ ) = util.test_both(LoaderTests, machinery=machinery)
+
+@unittest.skip("TODO: RUSTPYTHON")
+class MultiPhaseExtensionModuleTests(abc.LoaderTests):
+    """Test loading extension modules with multi-phase initialization (PEP 489)
+    """
+
+    def setUp(self):
+        self.name = '_testmultiphase'
+        finder = self.machinery.FileFinder(None)
+        self.spec = importlib.util.find_spec(self.name)
+        assert self.spec
+        self.loader = self.machinery.ExtensionFileLoader(
+            self.name, self.spec.origin)
+
+    # No extension module as __init__ available for testing.
+    test_package = None
+
+    # No extension module in a package available for testing.
+    test_lacking_parent = None
+
+    # Handling failure on reload is the up to the module.
+    test_state_after_failure = None
+
+    def test_module(self):
+        '''Test loading an extension module'''
+        with util.uncache(self.name):
+            module = self.load_module()
+            for attr, value in [('__name__', self.name),
+                                ('__file__', self.spec.origin),
+                                ('__package__', '')]:
+                self.assertEqual(getattr(module, attr), value)
+            with self.assertRaises(AttributeError):
+                module.__path__
+            self.assertIs(module, sys.modules[self.name])
+            self.assertIsInstance(module.__loader__,
+                                  self.machinery.ExtensionFileLoader)
+
+    def test_functionality(self):
+        '''Test basic functionality of stuff defined in an extension module'''
+        with util.uncache(self.name):
+            module = self.load_module()
+            self.assertIsInstance(module, types.ModuleType)
+            ex = module.Example()
+            self.assertEqual(ex.demo('abcd'), 'abcd')
+            self.assertEqual(ex.demo(), None)
+            with self.assertRaises(AttributeError):
+                ex.abc
+            ex.abc = 0
+            self.assertEqual(ex.abc, 0)
+            self.assertEqual(module.foo(9, 9), 18)
+            self.assertIsInstance(module.Str(), str)
+            self.assertEqual(module.Str(1) + '23', '123')
+            with self.assertRaises(module.error):
+                raise module.error()
+            self.assertEqual(module.int_const, 1969)
+            self.assertEqual(module.str_const, 'something different')
+
+    def test_reload(self):
+        '''Test that reload didn't re-set the module's attributes'''
+        with util.uncache(self.name):
+            module = self.load_module()
+            ex_class = module.Example
+            importlib.reload(module)
+            self.assertIs(ex_class, module.Example)
+
+    def test_try_registration(self):
+        '''Assert that the PyState_{Find,Add,Remove}Module C API doesn't work'''
+        module = self.load_module()
+        with self.subTest('PyState_FindModule'):
+            self.assertEqual(module.call_state_registration_func(0), None)
+        with self.subTest('PyState_AddModule'):
+            with self.assertRaises(SystemError):
+                module.call_state_registration_func(1)
+        with self.subTest('PyState_RemoveModule'):
+            with self.assertRaises(SystemError):
+                module.call_state_registration_func(2)
+
+    def load_module(self):
+        '''Load the module from the test extension'''
+        return self.loader.load_module(self.name)
+
+    def load_module_by_name(self, fullname):
+        '''Load a module from the test extension by name'''
+        origin = self.spec.origin
+        loader = self.machinery.ExtensionFileLoader(fullname, origin)
+        spec = importlib.util.spec_from_loader(fullname, loader)
+        module = importlib.util.module_from_spec(spec)
+        loader.exec_module(module)
+        return module
+
+    def test_load_submodule(self):
+        '''Test loading a simulated submodule'''
+        module = self.load_module_by_name('pkg.' + self.name)
+        self.assertIsInstance(module, types.ModuleType)
+        self.assertEqual(module.__name__, 'pkg.' + self.name)
+        self.assertEqual(module.str_const, 'something different')
+
+    def test_load_short_name(self):
+        '''Test loading module with a one-character name'''
+        module = self.load_module_by_name('x')
+        self.assertIsInstance(module, types.ModuleType)
+        self.assertEqual(module.__name__, 'x')
+        self.assertEqual(module.str_const, 'something different')
+        self.assertNotIn('x', sys.modules)
+
+    def test_load_twice(self):
+        '''Test that 2 loads result in 2 module objects'''
+        module1 = self.load_module_by_name(self.name)
+        module2 = self.load_module_by_name(self.name)
+        self.assertIsNot(module1, module2)
+
+    def test_unloadable(self):
+        '''Test nonexistent module'''
+        name = 'asdfjkl;'
+        with self.assertRaises(ImportError) as cm:
+            self.load_module_by_name(name)
+        self.assertEqual(cm.exception.name, name)
+
+    def test_unloadable_nonascii(self):
+        '''Test behavior with nonexistent module with non-ASCII name'''
+        name = 'fo\xf3'
+        with self.assertRaises(ImportError) as cm:
+            self.load_module_by_name(name)
+        self.assertEqual(cm.exception.name, name)
+
+    def test_nonmodule(self):
+        '''Test returning a non-module object from create works'''
+        name = self.name + '_nonmodule'
+        mod = self.load_module_by_name(name)
+        self.assertNotEqual(type(mod), type(unittest))
+        self.assertEqual(mod.three, 3)
+
+    # issue 27782
+    def test_nonmodule_with_methods(self):
+        '''Test creating a non-module object with methods defined'''
+        name = self.name + '_nonmodule_with_methods'
+        mod = self.load_module_by_name(name)
+        self.assertNotEqual(type(mod), type(unittest))
+        self.assertEqual(mod.three, 3)
+        self.assertEqual(mod.bar(10, 1), 9)
+
+    def test_null_slots(self):
+        '''Test that NULL slots aren't a problem'''
+        name = self.name + '_null_slots'
+        module = self.load_module_by_name(name)
+        self.assertIsInstance(module, types.ModuleType)
+        self.assertEqual(module.__name__, name)
+
+    def test_bad_modules(self):
+        '''Test SystemError is raised for misbehaving extensions'''
+        for name_base in [
+                'bad_slot_large',
+                'bad_slot_negative',
+                'create_int_with_state',
+                'negative_size',
+                'export_null',
+                'export_uninitialized',
+                'export_raise',
+                'export_unreported_exception',
+                'create_null',
+                'create_raise',
+                'create_unreported_exception',
+                'nonmodule_with_exec_slots',
+                'exec_err',
+                'exec_raise',
+                'exec_unreported_exception',
+                ]:
+            with self.subTest(name_base):
+                name = self.name + '_' + name_base
+                with self.assertRaises(SystemError):
+                    self.load_module_by_name(name)
+
+    def test_nonascii(self):
+        '''Test that modules with non-ASCII names can be loaded'''
+        # punycode behaves slightly differently in some-ASCII and no-ASCII
+        # cases, so test both
+        cases = [
+            (self.name + '_zkou\u0161ka_na\u010dten\xed', 'Czech'),
+            ('\uff3f\u30a4\u30f3\u30dd\u30fc\u30c8\u30c6\u30b9\u30c8',
+             'Japanese'),
+            ]
+        for name, lang in cases:
+            with self.subTest(name):
+                module = self.load_module_by_name(name)
+                self.assertEqual(module.__name__, name)
+                self.assertEqual(module.__doc__, "Module named in %s" % lang)
+
+    @unittest.skipIf(not hasattr(sys, 'gettotalrefcount'),
+            '--with-pydebug has to be enabled for this test')
+    def test_bad_traverse(self):
+        ''' Issue #32374: Test that traverse fails when accessing per-module
+            state before Py_mod_exec was executed.
+            (Multiphase initialization modules only)
+        '''
+        script = """if True:
+                try:
+                    from test import support
+                    import importlib.util as util
+                    spec = util.find_spec('_testmultiphase')
+                    spec.name = '_testmultiphase_with_bad_traverse'
+
+                    with support.SuppressCrashReport():
+                        m = spec.loader.create_module(spec)
+                except:
+                    # Prevent Python-level exceptions from
+                    # ending the process with non-zero status
+                    # (We are testing for a crash in C-code)
+                    pass"""
+        assert_python_failure("-c", script)
+
+
+(Frozen_MultiPhaseExtensionModuleTests,
+ Source_MultiPhaseExtensionModuleTests
+ ) = util.test_both(MultiPhaseExtensionModuleTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/extension/test_path_hook.py

@@ -0,0 +1,31 @@
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import unittest
+
+
+class PathHookTests:
+
+    """Test the path hook for extension modules."""
+    # XXX Should it only succeed for pre-existing directories?
+    # XXX Should it only work for directories containing an extension module?
+
+    def hook(self, entry):
+        return self.machinery.FileFinder.path_hook(
+                (self.machinery.ExtensionFileLoader,
+                 self.machinery.EXTENSION_SUFFIXES))(entry)
+
+    def test_success(self):
+        # Path hook should handle a directory where a known extension module
+        # exists.
+        self.assertTrue(hasattr(self.hook(util.EXTENSIONS.path), 'find_module'))
+
+
+(Frozen_PathHooksTests,
+ Source_PathHooksTests
+ ) = util.test_both(PathHookTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/frozen/__init__.py

@@ -0,0 +1,5 @@
+import os
+from test.support import load_package_tests
+
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)

Lib/test/test_importlib/frozen/__main__.py

@@ -0,0 +1,4 @@
+from . import load_tests
+import unittest
+
+unittest.main()

Lib/test/test_importlib/frozen/test_finder.py

@@ -0,0 +1,92 @@
+from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import unittest
+
+
+class FindSpecTests(abc.FinderTests):
+
+    """Test finding frozen modules."""
+
+    def find(self, name, path=None):
+        finder = self.machinery.FrozenImporter
+        return finder.find_spec(name, path)
+
+    def test_module(self):
+        name = '__hello__'
+        spec = self.find(name)
+        self.assertEqual(spec.origin, 'frozen')
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_package(self):
+        spec = self.find('__phello__')
+        self.assertIsNotNone(spec)
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_module_in_package(self):
+        spec = self.find('__phello__.spam', ['__phello__'])
+        self.assertIsNotNone(spec)
+
+    # No frozen package within another package to test with.
+    test_package_in_package = None
+
+    # No easy way to test.
+    test_package_over_module = None
+
+    def test_failure(self):
+        spec = self.find('<not real>')
+        self.assertIsNone(spec)
+
+
+(Frozen_FindSpecTests,
+ Source_FindSpecTests
+ ) = util.test_both(FindSpecTests, machinery=machinery)
+
+
+class FinderTests(abc.FinderTests):
+
+    """Test finding frozen modules."""
+
+    def find(self, name, path=None):
+        finder = self.machinery.FrozenImporter
+        return finder.find_module(name, path)
+
+    def test_module(self):
+        name = '__hello__'
+        loader = self.find(name)
+        self.assertTrue(hasattr(loader, 'load_module'))
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_package(self):
+        loader = self.find('__phello__')
+        self.assertTrue(hasattr(loader, 'load_module'))
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_module_in_package(self):
+        loader = self.find('__phello__.spam', ['__phello__'])
+        self.assertTrue(hasattr(loader, 'load_module'))
+
+    # No frozen package within another package to test with.
+    test_package_in_package = None
+
+    # No easy way to test.
+    test_package_over_module = None
+
+    def test_failure(self):
+        loader = self.find('<not real>')
+        self.assertIsNone(loader)
+
+
+(Frozen_FinderTests,
+ Source_FinderTests
+ ) = util.test_both(FinderTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/frozen/test_loader.py

@@ -0,0 +1,230 @@
+from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+from test.support import captured_stdout
+import types
+import unittest
+import warnings
+
+
+class ExecModuleTests(abc.LoaderTests):
+
+    def exec_module(self, name):
+        with util.uncache(name), captured_stdout() as stdout:
+            spec = self.machinery.ModuleSpec(
+                    name, self.machinery.FrozenImporter, origin='frozen',
+                    is_package=self.machinery.FrozenImporter.is_package(name))
+            module = types.ModuleType(name)
+            module.__spec__ = spec
+            assert not hasattr(module, 'initialized')
+            self.machinery.FrozenImporter.exec_module(module)
+            self.assertTrue(module.initialized)
+            self.assertTrue(hasattr(module, '__spec__'))
+            self.assertEqual(module.__spec__.origin, 'frozen')
+            return module, stdout.getvalue()
+
+    def test_module(self):
+        name = '__hello__'
+        module, output = self.exec_module(name)
+        check = {'__name__': name}
+        for attr, value in check.items():
+            self.assertEqual(getattr(module, attr), value)
+        self.assertEqual(output, 'Hello world!\n')
+        self.assertTrue(hasattr(module, '__spec__'))
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_package(self):
+        name = '__phello__'
+        module, output = self.exec_module(name)
+        check = {'__name__': name}
+        for attr, value in check.items():
+            attr_value = getattr(module, attr)
+            self.assertEqual(attr_value, value,
+                        'for {name}.{attr}, {given!r} != {expected!r}'.format(
+                                 name=name, attr=attr, given=attr_value,
+                                 expected=value))
+        self.assertEqual(output, 'Hello world!\n')
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_lacking_parent(self):
+        name = '__phello__.spam'
+        with util.uncache('__phello__'):
+            module, output = self.exec_module(name)
+            check = {'__name__': name}
+            for attr, value in check.items():
+                attr_value = getattr(module, attr)
+                self.assertEqual(attr_value, value,
+                        'for {name}.{attr}, {given} != {expected!r}'.format(
+                                 name=name, attr=attr, given=attr_value,
+                                 expected=value))
+            self.assertEqual(output, 'Hello world!\n')
+
+    def test_module_repr(self):
+        name = '__hello__'
+        module, output = self.exec_module(name)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', DeprecationWarning)
+            repr_str = self.machinery.FrozenImporter.module_repr(module)
+        self.assertEqual(repr_str,
+                         "<module '__hello__' (frozen)>")
+
+    def test_module_repr_indirect(self):
+        name = '__hello__'
+        module, output = self.exec_module(name)
+        self.assertEqual(repr(module),
+                         "<module '__hello__' (frozen)>")
+
+    # No way to trigger an error in a frozen module.
+    test_state_after_failure = None
+
+    def test_unloadable(self):
+        assert self.machinery.FrozenImporter.find_module('_not_real') is None
+        with self.assertRaises(ImportError) as cm:
+            self.exec_module('_not_real')
+        self.assertEqual(cm.exception.name, '_not_real')
+
+
+(Frozen_ExecModuleTests,
+ Source_ExecModuleTests
+ ) = util.test_both(ExecModuleTests, machinery=machinery)
+
+
+class LoaderTests(abc.LoaderTests):
+
+    def test_module(self):
+        with util.uncache('__hello__'), captured_stdout() as stdout:
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                module = self.machinery.FrozenImporter.load_module('__hello__')
+            check = {'__name__': '__hello__',
+                    '__package__': '',
+                    '__loader__': self.machinery.FrozenImporter,
+                    }
+            for attr, value in check.items():
+                self.assertEqual(getattr(module, attr), value)
+            self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+            self.assertFalse(hasattr(module, '__file__'))
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_package(self):
+        with util.uncache('__phello__'),  captured_stdout() as stdout:
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                module = self.machinery.FrozenImporter.load_module('__phello__')
+            check = {'__name__': '__phello__',
+                     '__package__': '__phello__',
+                     '__path__': [],
+                     '__loader__': self.machinery.FrozenImporter,
+                     }
+            for attr, value in check.items():
+                attr_value = getattr(module, attr)
+                self.assertEqual(attr_value, value,
+                                 "for __phello__.%s, %r != %r" %
+                                 (attr, attr_value, value))
+            self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+            self.assertFalse(hasattr(module, '__file__'))
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_lacking_parent(self):
+        with util.uncache('__phello__', '__phello__.spam'), \
+             captured_stdout() as stdout:
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                module = self.machinery.FrozenImporter.load_module('__phello__.spam')
+            check = {'__name__': '__phello__.spam',
+                    '__package__': '__phello__',
+                    '__loader__': self.machinery.FrozenImporter,
+                    }
+            for attr, value in check.items():
+                attr_value = getattr(module, attr)
+                self.assertEqual(attr_value, value,
+                                 "for __phello__.spam.%s, %r != %r" %
+                                 (attr, attr_value, value))
+            self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+            self.assertFalse(hasattr(module, '__file__'))
+
+    def test_module_reuse(self):
+        with util.uncache('__hello__'), captured_stdout() as stdout:
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                module1 = self.machinery.FrozenImporter.load_module('__hello__')
+                module2 = self.machinery.FrozenImporter.load_module('__hello__')
+            self.assertIs(module1, module2)
+            self.assertEqual(stdout.getvalue(),
+                             'Hello world!\nHello world!\n')
+
+    def test_module_repr(self):
+        with util.uncache('__hello__'), captured_stdout():
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                module = self.machinery.FrozenImporter.load_module('__hello__')
+                repr_str = self.machinery.FrozenImporter.module_repr(module)
+            self.assertEqual(repr_str,
+                             "<module '__hello__' (frozen)>")
+
+    def test_module_repr_indirect(self):
+        with util.uncache('__hello__'), captured_stdout():
+            module = self.machinery.FrozenImporter.load_module('__hello__')
+        self.assertEqual(repr(module),
+                         "<module '__hello__' (frozen)>")
+
+    # No way to trigger an error in a frozen module.
+    test_state_after_failure = None
+
+    def test_unloadable(self):
+        assert self.machinery.FrozenImporter.find_module('_not_real') is None
+        with self.assertRaises(ImportError) as cm:
+            self.machinery.FrozenImporter.load_module('_not_real')
+        self.assertEqual(cm.exception.name, '_not_real')
+
+
+(Frozen_LoaderTests,
+ Source_LoaderTests
+ ) = util.test_both(LoaderTests, machinery=machinery)
+
+
+class InspectLoaderTests:
+
+    """Tests for the InspectLoader methods for FrozenImporter."""
+
+    def test_get_code(self):
+        # Make sure that the code object is good.
+        name = '__hello__'
+        with captured_stdout() as stdout:
+            code = self.machinery.FrozenImporter.get_code(name)
+            mod = types.ModuleType(name)
+            exec(code, mod.__dict__)
+            self.assertTrue(hasattr(mod, 'initialized'))
+            self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+
+    def test_get_source(self):
+        # Should always return None.
+        result = self.machinery.FrozenImporter.get_source('__hello__')
+        self.assertIsNone(result)
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_is_package(self):
+        # Should be able to tell what is a package.
+        test_for = (('__hello__', False), ('__phello__', True),
+                    ('__phello__.spam', False))
+        for name, is_package in test_for:
+            result = self.machinery.FrozenImporter.is_package(name)
+            self.assertEqual(bool(result), is_package)
+
+    def test_failure(self):
+        # Raise ImportError for modules that are not frozen.
+        for meth_name in ('get_code', 'get_source', 'is_package'):
+            method = getattr(self.machinery.FrozenImporter, meth_name)
+            with self.assertRaises(ImportError) as cm:
+                method('importlib')
+            self.assertEqual(cm.exception.name, 'importlib')
+
+(Frozen_ILTests,
+ Source_ILTests
+ ) = util.test_both(InspectLoaderTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/import_/__init__.py

@@ -0,0 +1,5 @@
+import os
+from test.support import load_package_tests
+
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)

Lib/test/test_importlib/import_/__main__.py

@@ -0,0 +1,4 @@
+from . import load_tests
+import unittest
+
+unittest.main()

Lib/test/test_importlib/import_/test___loader__.py

@@ -0,0 +1,75 @@
+from importlib import machinery
+import sys
+import types
+import unittest
+
+from .. import util
+
+
+class SpecLoaderMock:
+
+    def find_spec(self, fullname, path=None, target=None):
+        return machinery.ModuleSpec(fullname, self)
+
+    def create_module(self, spec):
+        return None
+
+    def exec_module(self, module):
+        pass
+
+
+class SpecLoaderAttributeTests:
+
+    def test___loader__(self):
+        loader = SpecLoaderMock()
+        with util.uncache('blah'), util.import_state(meta_path=[loader]):
+            module = self.__import__('blah')
+        self.assertEqual(loader, module.__loader__)
+
+
+(Frozen_SpecTests,
+ Source_SpecTests
+ ) = util.test_both(SpecLoaderAttributeTests, __import__=util.__import__)
+
+
+class LoaderMock:
+
+    def find_module(self, fullname, path=None):
+        return self
+
+    def load_module(self, fullname):
+        sys.modules[fullname] = self.module
+        return self.module
+
+
+class LoaderAttributeTests:
+
+    def test___loader___missing(self):
+        module = types.ModuleType('blah')
+        try:
+            del module.__loader__
+        except AttributeError:
+            pass
+        loader = LoaderMock()
+        loader.module = module
+        with util.uncache('blah'), util.import_state(meta_path=[loader]):
+            module = self.__import__('blah')
+        self.assertEqual(loader, module.__loader__)
+
+    def test___loader___is_None(self):
+        module = types.ModuleType('blah')
+        module.__loader__ = None
+        loader = LoaderMock()
+        loader.module = module
+        with util.uncache('blah'), util.import_state(meta_path=[loader]):
+            returned_module = self.__import__('blah')
+        self.assertEqual(loader, module.__loader__)
+
+
+(Frozen_Tests,
+ Source_Tests
+ ) = util.test_both(LoaderAttributeTests, __import__=util.__import__)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/import_/test___package__.py

@@ -0,0 +1,167 @@
+"""PEP 366 ("Main module explicit relative imports") specifies the
+semantics for the __package__ attribute on modules. This attribute is
+used, when available, to detect which package a module belongs to (instead
+of using the typical __path__/__name__ test).
+
+"""
+import unittest
+import warnings
+from .. import util
+
+
+class Using__package__:
+
+    """Use of __package__ supersedes the use of __name__/__path__ to calculate
+    what package a module belongs to. The basic algorithm is [__package__]::
+
+      def resolve_name(name, package, level):
+          level -= 1
+          base = package.rsplit('.', level)[0]
+          return '{0}.{1}'.format(base, name)
+
+    But since there is no guarantee that __package__ has been set (or not been
+    set to None [None]), there has to be a way to calculate the attribute's value
+    [__name__]::
+
+      def calc_package(caller_name, has___path__):
+          if has__path__:
+              return caller_name
+          else:
+              return caller_name.rsplit('.', 1)[0]
+
+    Then the normal algorithm for relative name imports can proceed as if
+    __package__ had been set.
+
+    """
+
+    def import_module(self, globals_):
+        with self.mock_modules('pkg.__init__', 'pkg.fake') as importer:
+            with util.import_state(meta_path=[importer]):
+                self.__import__('pkg.fake')
+                module = self.__import__('',
+                                         globals=globals_,
+                                         fromlist=['attr'], level=2)
+        return module
+
+    def test_using___package__(self):
+        # [__package__]
+        module = self.import_module({'__package__': 'pkg.fake'})
+        self.assertEqual(module.__name__, 'pkg')
+
+    def test_using___name__(self):
+        # [__name__]
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            module = self.import_module({'__name__': 'pkg.fake',
+                                         '__path__': []})
+        self.assertEqual(module.__name__, 'pkg')
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_warn_when_using___name__(self):
+        with self.assertWarns(ImportWarning):
+            self.import_module({'__name__': 'pkg.fake', '__path__': []})
+
+    def test_None_as___package__(self):
+        # [None]
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            module = self.import_module({
+                '__name__': 'pkg.fake', '__path__': [], '__package__': None })
+        self.assertEqual(module.__name__, 'pkg')
+
+    def test_spec_fallback(self):
+        # If __package__ isn't defined, fall back on __spec__.parent.
+        module = self.import_module({'__spec__': FakeSpec('pkg.fake')})
+        self.assertEqual(module.__name__, 'pkg')
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_warn_when_package_and_spec_disagree(self):
+        # Raise an ImportWarning if __package__ != __spec__.parent.
+        with self.assertWarns(ImportWarning):
+            self.import_module({'__package__': 'pkg.fake',
+                                '__spec__': FakeSpec('pkg.fakefake')})
+
+    def test_bad__package__(self):
+        globals = {'__package__': '<not real>'}
+        with self.assertRaises(ModuleNotFoundError):
+            self.__import__('', globals, {}, ['relimport'], 1)
+
+    def test_bunk__package__(self):
+        globals = {'__package__': 42}
+        with self.assertRaises(TypeError):
+            self.__import__('', globals, {}, ['relimport'], 1)
+
+
+class FakeSpec:
+    def __init__(self, parent):
+        self.parent = parent
+
+
+class Using__package__PEP302(Using__package__):
+    mock_modules = util.mock_modules
+
+
+(Frozen_UsingPackagePEP302,
+ Source_UsingPackagePEP302
+ ) = util.test_both(Using__package__PEP302, __import__=util.__import__)
+
+
+class Using__package__PEP451(Using__package__):
+    mock_modules = util.mock_spec
+
+
+(Frozen_UsingPackagePEP451,
+ Source_UsingPackagePEP451
+ ) = util.test_both(Using__package__PEP451, __import__=util.__import__)
+
+
+class Setting__package__:
+
+    """Because __package__ is a new feature, it is not always set by a loader.
+    Import will set it as needed to help with the transition to relying on
+    __package__.
+
+    For a top-level module, __package__ is set to None [top-level]. For a
+    package __name__ is used for __package__ [package]. For submodules the
+    value is __name__.rsplit('.', 1)[0] [submodule].
+
+    """
+
+    __import__ = util.__import__['Source']
+
+    # [top-level]
+    def test_top_level(self):
+        with self.mock_modules('top_level') as mock:
+            with util.import_state(meta_path=[mock]):
+                del mock['top_level'].__package__
+                module = self.__import__('top_level')
+                self.assertEqual(module.__package__, '')
+
+    # [package]
+    def test_package(self):
+        with self.mock_modules('pkg.__init__') as mock:
+            with util.import_state(meta_path=[mock]):
+                del mock['pkg'].__package__
+                module = self.__import__('pkg')
+                self.assertEqual(module.__package__, 'pkg')
+
+    # [submodule]
+    def test_submodule(self):
+        with self.mock_modules('pkg.__init__', 'pkg.mod') as mock:
+            with util.import_state(meta_path=[mock]):
+                del mock['pkg.mod'].__package__
+                pkg = self.__import__('pkg.mod')
+                module = getattr(pkg, 'mod')
+                self.assertEqual(module.__package__, 'pkg')
+
+class Setting__package__PEP302(Setting__package__, unittest.TestCase):
+    mock_modules = util.mock_modules
+
+class Setting__package__PEP451(Setting__package__, unittest.TestCase):
+    mock_modules = util.mock_spec
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/import_/test_api.py

@@ -0,0 +1,119 @@
+from .. import util
+
+from importlib import machinery
+import sys
+import types
+import unittest
+
+PKG_NAME = 'fine'
+SUBMOD_NAME = 'fine.bogus'
+
+
+class BadSpecFinderLoader:
+    @classmethod
+    def find_spec(cls, fullname, path=None, target=None):
+        if fullname == SUBMOD_NAME:
+            spec = machinery.ModuleSpec(fullname, cls)
+            return spec
+
+    @staticmethod
+    def create_module(spec):
+        return None
+
+    @staticmethod
+    def exec_module(module):
+        if module.__name__ == SUBMOD_NAME:
+            raise ImportError('I cannot be loaded!')
+
+
+class BadLoaderFinder:
+    @classmethod
+    def find_module(cls, fullname, path):
+        if fullname == SUBMOD_NAME:
+            return cls
+
+    @classmethod
+    def load_module(cls, fullname):
+        if fullname == SUBMOD_NAME:
+            raise ImportError('I cannot be loaded!')
+
+
+class APITest:
+
+    """Test API-specific details for __import__ (e.g. raising the right
+    exception when passing in an int for the module name)."""
+
+    def test_raises_ModuleNotFoundError(self):
+        with self.assertRaises(ModuleNotFoundError):
+            util.import_importlib('some module that does not exist')
+
+    def test_name_requires_rparition(self):
+        # Raise TypeError if a non-string is passed in for the module name.
+        with self.assertRaises(TypeError):
+            self.__import__(42)
+
+    def test_negative_level(self):
+        # Raise ValueError when a negative level is specified.
+        # PEP 328 did away with sys.module None entries and the ambiguity of
+        # absolute/relative imports.
+        with self.assertRaises(ValueError):
+            self.__import__('os', globals(), level=-1)
+
+    def test_nonexistent_fromlist_entry(self):
+        # If something in fromlist doesn't exist, that's okay.
+        # issue15715
+        mod = types.ModuleType(PKG_NAME)
+        mod.__path__ = ['XXX']
+        with util.import_state(meta_path=[self.bad_finder_loader]):
+            with util.uncache(PKG_NAME):
+                sys.modules[PKG_NAME] = mod
+                self.__import__(PKG_NAME, fromlist=['not here'])
+
+    def test_fromlist_load_error_propagates(self):
+        # If something in fromlist triggers an exception not related to not
+        # existing, let that exception propagate.
+        # issue15316
+        mod = types.ModuleType(PKG_NAME)
+        mod.__path__ = ['XXX']
+        with util.import_state(meta_path=[self.bad_finder_loader]):
+            with util.uncache(PKG_NAME):
+                sys.modules[PKG_NAME] = mod
+                with self.assertRaises(ImportError):
+                    self.__import__(PKG_NAME,
+                                    fromlist=[SUBMOD_NAME.rpartition('.')[-1]])
+
+    def test_blocked_fromlist(self):
+        # If fromlist entry is None, let a ModuleNotFoundError propagate.
+        # issue31642
+        mod = types.ModuleType(PKG_NAME)
+        mod.__path__ = []
+        with util.import_state(meta_path=[self.bad_finder_loader]):
+            with util.uncache(PKG_NAME, SUBMOD_NAME):
+                sys.modules[PKG_NAME] = mod
+                sys.modules[SUBMOD_NAME] = None
+                with self.assertRaises(ModuleNotFoundError) as cm:
+                    self.__import__(PKG_NAME,
+                                    fromlist=[SUBMOD_NAME.rpartition('.')[-1]])
+                self.assertEqual(cm.exception.name, SUBMOD_NAME)
+
+
+class OldAPITests(APITest):
+    bad_finder_loader = BadLoaderFinder
+
+
+(Frozen_OldAPITests,
+ Source_OldAPITests
+ ) = util.test_both(OldAPITests, __import__=util.__import__)
+
+
+class SpecAPITests(APITest):
+    bad_finder_loader = BadSpecFinderLoader
+
+
+(Frozen_SpecAPITests,
+ Source_SpecAPITests
+ ) = util.test_both(SpecAPITests, __import__=util.__import__)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/import_/test_caching.py

@@ -0,0 +1,94 @@
+"""Test that sys.modules is used properly by import."""
+from .. import util
+import sys
+from types import MethodType
+import unittest
+
+
+class UseCache:
+
+    """When it comes to sys.modules, import prefers it over anything else.
+
+    Once a name has been resolved, sys.modules is checked to see if it contains
+    the module desired. If so, then it is returned [use cache]. If it is not
+    found, then the proper steps are taken to perform the import, but
+    sys.modules is still used to return the imported module (e.g., not what a
+    loader returns) [from cache on return]. This also applies to imports of
+    things contained within a package and thus get assigned as an attribute
+    [from cache to attribute] or pulled in thanks to a fromlist import
+    [from cache for fromlist]. But if sys.modules contains None then
+    ImportError is raised [None in cache].
+
+    """
+
+    def test_using_cache(self):
+        # [use cache]
+        module_to_use = "some module found!"
+        with util.uncache('some_module'):
+            sys.modules['some_module'] = module_to_use
+            module = self.__import__('some_module')
+            self.assertEqual(id(module_to_use), id(module))
+
+    def test_None_in_cache(self):
+        #[None in cache]
+        name = 'using_None'
+        with util.uncache(name):
+            sys.modules[name] = None
+            with self.assertRaises(ImportError) as cm:
+                self.__import__(name)
+            self.assertEqual(cm.exception.name, name)
+
+
+(Frozen_UseCache,
+ Source_UseCache
+ ) = util.test_both(UseCache, __import__=util.__import__)
+
+
+@unittest.skip("TODO: RUSTPYTHON")
+class ImportlibUseCache(UseCache, unittest.TestCase):
+
+    # Pertinent only to PEP 302; exec_module() doesn't return a module.
+
+    __import__ = util.__import__['Source']
+
+    def create_mock(self, *names, return_=None):
+        mock = util.mock_modules(*names)
+        original_load = mock.load_module
+        def load_module(self, fullname):
+            original_load(fullname)
+            return return_
+        mock.load_module = MethodType(load_module, mock)
+        return mock
+
+    # __import__ inconsistent between loaders and built-in import when it comes
+    #   to when to use the module in sys.modules and when not to.
+    def test_using_cache_after_loader(self):
+        # [from cache on return]
+        with self.create_mock('module') as mock:
+            with util.import_state(meta_path=[mock]):
+                module = self.__import__('module')
+                self.assertEqual(id(module), id(sys.modules['module']))
+
+    # See test_using_cache_after_loader() for reasoning.
+    def test_using_cache_for_assigning_to_attribute(self):
+        # [from cache to attribute]
+        with self.create_mock('pkg.__init__', 'pkg.module') as importer:
+            with util.import_state(meta_path=[importer]):
+                module = self.__import__('pkg.module')
+                self.assertTrue(hasattr(module, 'module'))
+                self.assertEqual(id(module.module),
+                                 id(sys.modules['pkg.module']))
+
+    # See test_using_cache_after_loader() for reasoning.
+    def test_using_cache_for_fromlist(self):
+        # [from cache for fromlist]
+        with self.create_mock('pkg.__init__', 'pkg.module') as importer:
+            with util.import_state(meta_path=[importer]):
+                module = self.__import__('pkg', fromlist=['module'])
+                self.assertTrue(hasattr(module, 'module'))
+                self.assertEqual(id(module.module),
+                                 id(sys.modules['pkg.module']))
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/import_/test_fromlist.py

@@ -0,0 +1,175 @@
+"""Test that the semantics relating to the 'fromlist' argument are correct."""
+from .. import util
+import warnings
+import unittest
+
+
+class ReturnValue:
+
+    """The use of fromlist influences what import returns.
+
+    If direct ``import ...`` statement is used, the root module or package is
+    returned [import return]. But if fromlist is set, then the specified module
+    is actually returned (whether it is a relative import or not)
+    [from return].
+
+    """
+
+    def test_return_from_import(self):
+        # [import return]
+        with util.mock_spec('pkg.__init__', 'pkg.module') as importer:
+            with util.import_state(meta_path=[importer]):
+                module = self.__import__('pkg.module')
+                self.assertEqual(module.__name__, 'pkg')
+
+    def test_return_from_from_import(self):
+        # [from return]
+        with util.mock_modules('pkg.__init__', 'pkg.module')as importer:
+            with util.import_state(meta_path=[importer]):
+                module = self.__import__('pkg.module', fromlist=['attr'])
+                self.assertEqual(module.__name__, 'pkg.module')
+
+
+(Frozen_ReturnValue,
+ Source_ReturnValue
+ ) = util.test_both(ReturnValue, __import__=util.__import__)
+
+
+class HandlingFromlist:
+
+    """Using fromlist triggers different actions based on what is being asked
+    of it.
+
+    If fromlist specifies an object on a module, nothing special happens
+    [object case]. This is even true if the object does not exist [bad object].
+
+    If a package is being imported, then what is listed in fromlist may be
+    treated as a module to be imported [module]. And this extends to what is
+    contained in __all__ when '*' is imported [using *]. And '*' does not need
+    to be the only name in the fromlist [using * with others].
+
+    """
+
+    def test_object(self):
+        # [object case]
+        with util.mock_modules('module') as importer:
+            with util.import_state(meta_path=[importer]):
+                module = self.__import__('module', fromlist=['attr'])
+                self.assertEqual(module.__name__, 'module')
+
+    def test_nonexistent_object(self):
+        # [bad object]
+        with util.mock_modules('module') as importer:
+            with util.import_state(meta_path=[importer]):
+                module = self.__import__('module', fromlist=['non_existent'])
+                self.assertEqual(module.__name__, 'module')
+                self.assertFalse(hasattr(module, 'non_existent'))
+
+    def test_module_from_package(self):
+        # [module]
+        with util.mock_modules('pkg.__init__', 'pkg.module') as importer:
+            with util.import_state(meta_path=[importer]):
+                module = self.__import__('pkg', fromlist=['module'])
+                self.assertEqual(module.__name__, 'pkg')
+                self.assertTrue(hasattr(module, 'module'))
+                self.assertEqual(module.module.__name__, 'pkg.module')
+
+    def test_nonexistent_from_package(self):
+        with util.mock_modules('pkg.__init__') as importer:
+            with util.import_state(meta_path=[importer]):
+                module = self.__import__('pkg', fromlist=['non_existent'])
+                self.assertEqual(module.__name__, 'pkg')
+                self.assertFalse(hasattr(module, 'non_existent'))
+
+    def test_module_from_package_triggers_ModuleNotFoundError(self):
+        # If a submodule causes an ModuleNotFoundError because it tries
+        # to import a module which doesn't exist, that should let the
+        # ModuleNotFoundError propagate.
+        def module_code():
+            import i_do_not_exist
+        with util.mock_modules('pkg.__init__', 'pkg.mod',
+                               module_code={'pkg.mod': module_code}) as importer:
+            with util.import_state(meta_path=[importer]):
+                with self.assertRaises(ModuleNotFoundError) as exc:
+                    self.__import__('pkg', fromlist=['mod'])
+                self.assertEqual('i_do_not_exist', exc.exception.name)
+
+    def test_empty_string(self):
+        with util.mock_modules('pkg.__init__', 'pkg.mod') as importer:
+            with util.import_state(meta_path=[importer]):
+                module = self.__import__('pkg.mod', fromlist=[''])
+                self.assertEqual(module.__name__, 'pkg.mod')
+
+    def basic_star_test(self, fromlist=['*']):
+        # [using *]
+        with util.mock_modules('pkg.__init__', 'pkg.module') as mock:
+            with util.import_state(meta_path=[mock]):
+                mock['pkg'].__all__ = ['module']
+                module = self.__import__('pkg', fromlist=fromlist)
+                self.assertEqual(module.__name__, 'pkg')
+                self.assertTrue(hasattr(module, 'module'))
+                self.assertEqual(module.module.__name__, 'pkg.module')
+
+    def test_using_star(self):
+        # [using *]
+        self.basic_star_test()
+
+    def test_fromlist_as_tuple(self):
+        self.basic_star_test(('*',))
+
+    def test_star_with_others(self):
+        # [using * with others]
+        context = util.mock_modules('pkg.__init__', 'pkg.module1', 'pkg.module2')
+        with context as mock:
+            with util.import_state(meta_path=[mock]):
+                mock['pkg'].__all__ = ['module1']
+                module = self.__import__('pkg', fromlist=['module2', '*'])
+                self.assertEqual(module.__name__, 'pkg')
+                self.assertTrue(hasattr(module, 'module1'))
+                self.assertTrue(hasattr(module, 'module2'))
+                self.assertEqual(module.module1.__name__, 'pkg.module1')
+                self.assertEqual(module.module2.__name__, 'pkg.module2')
+
+    def test_nonexistent_in_all(self):
+        with util.mock_modules('pkg.__init__') as importer:
+            with util.import_state(meta_path=[importer]):
+                importer['pkg'].__all__ = ['non_existent']
+                module = self.__import__('pkg', fromlist=['*'])
+                self.assertEqual(module.__name__, 'pkg')
+                self.assertFalse(hasattr(module, 'non_existent'))
+
+    def test_star_in_all(self):
+        with util.mock_modules('pkg.__init__') as importer:
+            with util.import_state(meta_path=[importer]):
+                importer['pkg'].__all__ = ['*']
+                module = self.__import__('pkg', fromlist=['*'])
+                self.assertEqual(module.__name__, 'pkg')
+                self.assertFalse(hasattr(module, '*'))
+
+    def test_invalid_type(self):
+        with util.mock_modules('pkg.__init__') as importer:
+            with util.import_state(meta_path=[importer]), \
+                 warnings.catch_warnings():
+                warnings.simplefilter('error', BytesWarning)
+                with self.assertRaisesRegex(TypeError, r'\bfrom\b'):
+                    self.__import__('pkg', fromlist=[b'attr'])
+                with self.assertRaisesRegex(TypeError, r'\bfrom\b'):
+                    self.__import__('pkg', fromlist=iter([b'attr']))
+
+    def test_invalid_type_in_all(self):
+        with util.mock_modules('pkg.__init__') as importer:
+            with util.import_state(meta_path=[importer]), \
+                 warnings.catch_warnings():
+                warnings.simplefilter('error', BytesWarning)
+                importer['pkg'].__all__ = [b'attr']
+                with self.assertRaisesRegex(TypeError, r'\bpkg\.__all__\b'):
+                    self.__import__('pkg', fromlist=['*'])
+
+
+(Frozen_FromList,
+ Source_FromList
+ ) = util.test_both(HandlingFromlist, __import__=util.__import__)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/import_/test_meta_path.py

@@ -0,0 +1,128 @@
+from .. import util
+import importlib._bootstrap
+import sys
+from types import MethodType
+import unittest
+import warnings
+
+
+class CallingOrder:
+
+    """Calls to the importers on sys.meta_path happen in order that they are
+    specified in the sequence, starting with the first importer
+    [first called], and then continuing on down until one is found that doesn't
+    return None [continuing]."""
+
+
+    def test_first_called(self):
+        # [first called]
+        mod = 'top_level'
+        with util.mock_spec(mod) as first, util.mock_spec(mod) as second:
+            with util.import_state(meta_path=[first, second]):
+                self.assertIs(self.__import__(mod), first.modules[mod])
+
+    def test_continuing(self):
+        # [continuing]
+        mod_name = 'for_real'
+        with util.mock_spec('nonexistent') as first, \
+             util.mock_spec(mod_name) as second:
+            first.find_spec = lambda self, fullname, path=None, parent=None: None
+            with util.import_state(meta_path=[first, second]):
+                self.assertIs(self.__import__(mod_name), second.modules[mod_name])
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_empty(self):
+        # Raise an ImportWarning if sys.meta_path is empty.
+        module_name = 'nothing'
+        try:
+            del sys.modules[module_name]
+        except KeyError:
+            pass
+        with util.import_state(meta_path=[]):
+            with warnings.catch_warnings(record=True) as w:
+                warnings.simplefilter('always')
+                self.assertIsNone(importlib._bootstrap._find_spec('nothing',
+                                                                  None))
+                self.assertEqual(len(w), 1)
+                self.assertTrue(issubclass(w[-1].category, ImportWarning))
+
+
+(Frozen_CallingOrder,
+ Source_CallingOrder
+ ) = util.test_both(CallingOrder, __import__=util.__import__)
+
+
+@unittest.skip("TODO: RUSTPYTHON")
+class CallSignature:
+
+    """If there is no __path__ entry on the parent module, then 'path' is None
+    [no path]. Otherwise, the value for __path__ is passed in for the 'path'
+    argument [path set]."""
+
+    def log_finder(self, importer):
+        fxn = getattr(importer, self.finder_name)
+        log = []
+        def wrapper(self, *args, **kwargs):
+            log.append([args, kwargs])
+            return fxn(*args, **kwargs)
+        return log, wrapper
+
+    def test_no_path(self):
+        # [no path]
+        mod_name = 'top_level'
+        assert '.' not in mod_name
+        with self.mock_modules(mod_name) as importer:
+            log, wrapped_call = self.log_finder(importer)
+            setattr(importer, self.finder_name, MethodType(wrapped_call, importer))
+            with util.import_state(meta_path=[importer]):
+                self.__import__(mod_name)
+                assert len(log) == 1
+                args = log[0][0]
+                # Assuming all arguments are positional.
+                self.assertEqual(args[0], mod_name)
+                self.assertIsNone(args[1])
+
+    def test_with_path(self):
+        # [path set]
+        pkg_name = 'pkg'
+        mod_name = pkg_name + '.module'
+        path = [42]
+        assert '.' in mod_name
+        with self.mock_modules(pkg_name+'.__init__', mod_name) as importer:
+            importer.modules[pkg_name].__path__ = path
+            log, wrapped_call = self.log_finder(importer)
+            setattr(importer, self.finder_name, MethodType(wrapped_call, importer))
+            with util.import_state(meta_path=[importer]):
+                self.__import__(mod_name)
+                assert len(log) == 2
+                args = log[1][0]
+                kwargs = log[1][1]
+                # Assuming all arguments are positional.
+                self.assertFalse(kwargs)
+                self.assertEqual(args[0], mod_name)
+                self.assertIs(args[1], path)
+
+
+class CallSignaturePEP302(CallSignature):
+    mock_modules = util.mock_modules
+    finder_name = 'find_module'
+
+
+(Frozen_CallSignaturePEP302,
+ Source_CallSignaturePEP302
+ ) = util.test_both(CallSignaturePEP302, __import__=util.__import__)
+
+
+class CallSignaturePEP451(CallSignature):
+    mock_modules = util.mock_spec
+    finder_name = 'find_spec'
+
+
+(Frozen_CallSignaturePEP451,
+ Source_CallSignaturePEP451
+ ) = util.test_both(CallSignaturePEP451, __import__=util.__import__)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/import_/test_packages.py

@@ -0,0 +1,110 @@
+from .. import util
+import sys
+import unittest
+from test import support
+
+
+class ParentModuleTests:
+
+    """Importing a submodule should import the parent modules."""
+
+    def test_import_parent(self):
+        with util.mock_spec('pkg.__init__', 'pkg.module') as mock:
+            with util.import_state(meta_path=[mock]):
+                module = self.__import__('pkg.module')
+                self.assertIn('pkg', sys.modules)
+
+    def test_bad_parent(self):
+        with util.mock_spec('pkg.module') as mock:
+            with util.import_state(meta_path=[mock]):
+                with self.assertRaises(ImportError) as cm:
+                    self.__import__('pkg.module')
+                self.assertEqual(cm.exception.name, 'pkg')
+
+    def test_raising_parent_after_importing_child(self):
+        def __init__():
+            import pkg.module
+            1/0
+        mock = util.mock_spec('pkg.__init__', 'pkg.module',
+                                 module_code={'pkg': __init__})
+        with mock:
+            with util.import_state(meta_path=[mock]):
+                with self.assertRaises(ZeroDivisionError):
+                    self.__import__('pkg')
+                self.assertNotIn('pkg', sys.modules)
+                self.assertIn('pkg.module', sys.modules)
+                with self.assertRaises(ZeroDivisionError):
+                    self.__import__('pkg.module')
+                self.assertNotIn('pkg', sys.modules)
+                self.assertIn('pkg.module', sys.modules)
+
+    def test_raising_parent_after_relative_importing_child(self):
+        def __init__():
+            from . import module
+            1/0
+        mock = util.mock_spec('pkg.__init__', 'pkg.module',
+                                 module_code={'pkg': __init__})
+        with mock:
+            with util.import_state(meta_path=[mock]):
+                with self.assertRaises((ZeroDivisionError, ImportError)):
+                    # This raises ImportError on the "from . import module"
+                    # line, not sure why.
+                    self.__import__('pkg')
+                self.assertNotIn('pkg', sys.modules)
+                with self.assertRaises((ZeroDivisionError, ImportError)):
+                    self.__import__('pkg.module')
+                self.assertNotIn('pkg', sys.modules)
+                # XXX False
+                #self.assertIn('pkg.module', sys.modules)
+
+    def test_raising_parent_after_double_relative_importing_child(self):
+        def __init__():
+            from ..subpkg import module
+            1/0
+        mock = util.mock_spec('pkg.__init__', 'pkg.subpkg.__init__',
+                                 'pkg.subpkg.module',
+                                 module_code={'pkg.subpkg': __init__})
+        with mock:
+            with util.import_state(meta_path=[mock]):
+                with self.assertRaises((ZeroDivisionError, ImportError)):
+                    # This raises ImportError on the "from ..subpkg import module"
+                    # line, not sure why.
+                    self.__import__('pkg.subpkg')
+                self.assertNotIn('pkg.subpkg', sys.modules)
+                with self.assertRaises((ZeroDivisionError, ImportError)):
+                    self.__import__('pkg.subpkg.module')
+                self.assertNotIn('pkg.subpkg', sys.modules)
+                # XXX False
+                #self.assertIn('pkg.subpkg.module', sys.modules)
+
+    def test_module_not_package(self):
+        # Try to import a submodule from a non-package should raise ImportError.
+        assert not hasattr(sys, '__path__')
+        with self.assertRaises(ImportError) as cm:
+            self.__import__('sys.no_submodules_here')
+        self.assertEqual(cm.exception.name, 'sys.no_submodules_here')
+
+    def test_module_not_package_but_side_effects(self):
+        # If a module injects something into sys.modules as a side-effect, then
+        # pick up on that fact.
+        name = 'mod'
+        subname = name + '.b'
+        def module_injection():
+            sys.modules[subname] = 'total bunk'
+        mock_spec = util.mock_spec('mod',
+                                         module_code={'mod': module_injection})
+        with mock_spec as mock:
+            with util.import_state(meta_path=[mock]):
+                try:
+                    submodule = self.__import__(subname)
+                finally:
+                    support.unload(subname)
+
+
+(Frozen_ParentTests,
+ Source_ParentTests
+ ) = util.test_both(ParentModuleTests, __import__=util.__import__)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/import_/test_path.py

@@ -0,0 +1,280 @@
+from .. import util
+
+importlib = util.import_importlib('importlib')
+machinery = util.import_importlib('importlib.machinery')
+
+import os
+import sys
+import tempfile
+from types import ModuleType
+import unittest
+import warnings
+import zipimport
+
+
+class FinderTests:
+
+    """Tests for PathFinder."""
+
+    find = None
+    check_found = None
+
+    def test_failure(self):
+        # Test None returned upon not finding a suitable loader.
+        module = '<test module>'
+        with util.import_state():
+            self.assertIsNone(self.find(module))
+
+    def test_sys_path(self):
+        # Test that sys.path is used when 'path' is None.
+        # Implicitly tests that sys.path_importer_cache is used.
+        module = '<test module>'
+        path = '<test path>'
+        importer = util.mock_spec(module)
+        with util.import_state(path_importer_cache={path: importer},
+                               path=[path]):
+            found = self.find(module)
+            self.check_found(found, importer)
+
+    def test_path(self):
+        # Test that 'path' is used when set.
+        # Implicitly tests that sys.path_importer_cache is used.
+        module = '<test module>'
+        path = '<test path>'
+        importer = util.mock_spec(module)
+        with util.import_state(path_importer_cache={path: importer}):
+            found = self.find(module, [path])
+            self.check_found(found, importer)
+
+    def test_empty_list(self):
+        # An empty list should not count as asking for sys.path.
+        module = 'module'
+        path = '<test path>'
+        importer = util.mock_spec(module)
+        with util.import_state(path_importer_cache={path: importer},
+                               path=[path]):
+            self.assertIsNone(self.find('module', []))
+
+    def test_path_hooks(self):
+        # Test that sys.path_hooks is used.
+        # Test that sys.path_importer_cache is set.
+        module = '<test module>'
+        path = '<test path>'
+        importer = util.mock_spec(module)
+        hook = util.mock_path_hook(path, importer=importer)
+        with util.import_state(path_hooks=[hook]):
+            found = self.find(module, [path])
+            self.check_found(found, importer)
+            self.assertIn(path, sys.path_importer_cache)
+            self.assertIs(sys.path_importer_cache[path], importer)
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_empty_path_hooks(self):
+        # Test that if sys.path_hooks is empty a warning is raised,
+        # sys.path_importer_cache gets None set, and PathFinder returns None.
+        path_entry = 'bogus_path'
+        with util.import_state(path_importer_cache={}, path_hooks=[],
+                               path=[path_entry]):
+            with warnings.catch_warnings(record=True) as w:
+                warnings.simplefilter('always')
+                self.assertIsNone(self.find('os'))
+                self.assertIsNone(sys.path_importer_cache[path_entry])
+                self.assertEqual(len(w), 1)
+                self.assertTrue(issubclass(w[-1].category, ImportWarning))
+
+    def test_path_importer_cache_empty_string(self):
+        # The empty string should create a finder using the cwd.
+        path = ''
+        module = '<test module>'
+        importer = util.mock_spec(module)
+        hook = util.mock_path_hook(os.getcwd(), importer=importer)
+        with util.import_state(path=[path], path_hooks=[hook]):
+            found = self.find(module)
+            self.check_found(found, importer)
+            self.assertIn(os.getcwd(), sys.path_importer_cache)
+
+    def test_None_on_sys_path(self):
+        # Putting None in sys.path[0] caused an import regression from Python
+        # 3.2: http://bugs.python.org/issue16514
+        new_path = sys.path[:]
+        new_path.insert(0, None)
+        new_path_importer_cache = sys.path_importer_cache.copy()
+        new_path_importer_cache.pop(None, None)
+        new_path_hooks = [zipimport.zipimporter,
+                          self.machinery.FileFinder.path_hook(
+                              *self.importlib._bootstrap_external._get_supported_file_loaders())]
+        missing = object()
+        email = sys.modules.pop('email', missing)
+        try:
+            with util.import_state(meta_path=sys.meta_path[:],
+                                   path=new_path,
+                                   path_importer_cache=new_path_importer_cache,
+                                   path_hooks=new_path_hooks):
+                module = self.importlib.import_module('email')
+                self.assertIsInstance(module, ModuleType)
+        finally:
+            if email is not missing:
+                sys.modules['email'] = email
+
+    def test_finder_with_find_module(self):
+        class TestFinder:
+            def find_module(self, fullname):
+                return self.to_return
+        failing_finder = TestFinder()
+        failing_finder.to_return = None
+        path = 'testing path'
+        with util.import_state(path_importer_cache={path: failing_finder}):
+            self.assertIsNone(
+                    self.machinery.PathFinder.find_spec('whatever', [path]))
+        success_finder = TestFinder()
+        success_finder.to_return = __loader__
+        with util.import_state(path_importer_cache={path: success_finder}):
+            spec = self.machinery.PathFinder.find_spec('whatever', [path])
+        self.assertEqual(spec.loader, __loader__)
+
+    def test_finder_with_find_loader(self):
+        class TestFinder:
+            loader = None
+            portions = []
+            def find_loader(self, fullname):
+                return self.loader, self.portions
+        path = 'testing path'
+        with util.import_state(path_importer_cache={path: TestFinder()}):
+            self.assertIsNone(
+                    self.machinery.PathFinder.find_spec('whatever', [path]))
+        success_finder = TestFinder()
+        success_finder.loader = __loader__
+        with util.import_state(path_importer_cache={path: success_finder}):
+            spec = self.machinery.PathFinder.find_spec('whatever', [path])
+        self.assertEqual(spec.loader, __loader__)
+
+    def test_finder_with_find_spec(self):
+        class TestFinder:
+            spec = None
+            def find_spec(self, fullname, target=None):
+                return self.spec
+        path = 'testing path'
+        with util.import_state(path_importer_cache={path: TestFinder()}):
+            self.assertIsNone(
+                    self.machinery.PathFinder.find_spec('whatever', [path]))
+        success_finder = TestFinder()
+        success_finder.spec = self.machinery.ModuleSpec('whatever', __loader__)
+        with util.import_state(path_importer_cache={path: success_finder}):
+            got = self.machinery.PathFinder.find_spec('whatever', [path])
+        self.assertEqual(got, success_finder.spec)
+
+    def test_deleted_cwd(self):
+        # Issue #22834
+        old_dir = os.getcwd()
+        self.addCleanup(os.chdir, old_dir)
+        new_dir = tempfile.mkdtemp()
+        try:
+            os.chdir(new_dir)
+            try:
+                os.rmdir(new_dir)
+            except OSError:
+                # EINVAL on Solaris, EBUSY on AIX, ENOTEMPTY on Windows
+                self.skipTest("platform does not allow "
+                              "the deletion of the cwd")
+        except:
+            os.chdir(old_dir)
+            os.rmdir(new_dir)
+            raise
+
+        with util.import_state(path=['']):
+            # Do not want FileNotFoundError raised.
+            self.assertIsNone(self.machinery.PathFinder.find_spec('whatever'))
+
+    def test_invalidate_caches_finders(self):
+        # Finders with an invalidate_caches() method have it called.
+        class FakeFinder:
+            def __init__(self):
+                self.called = False
+
+            def invalidate_caches(self):
+                self.called = True
+
+        cache = {'leave_alone': object(), 'finder_to_invalidate': FakeFinder()}
+        with util.import_state(path_importer_cache=cache):
+            self.machinery.PathFinder.invalidate_caches()
+        self.assertTrue(cache['finder_to_invalidate'].called)
+
+    def test_invalidate_caches_clear_out_None(self):
+        # Clear out None in sys.path_importer_cache() when invalidating caches.
+        cache = {'clear_out': None}
+        with util.import_state(path_importer_cache=cache):
+            self.machinery.PathFinder.invalidate_caches()
+        self.assertEqual(len(cache), 0)
+
+
+class FindModuleTests(FinderTests):
+    def find(self, *args, **kwargs):
+        return self.machinery.PathFinder.find_module(*args, **kwargs)
+    def check_found(self, found, importer):
+        self.assertIs(found, importer)
+
+
+(Frozen_FindModuleTests,
+ Source_FindModuleTests
+) = util.test_both(FindModuleTests, importlib=importlib, machinery=machinery)
+
+
+class FindSpecTests(FinderTests):
+    def find(self, *args, **kwargs):
+        return self.machinery.PathFinder.find_spec(*args, **kwargs)
+    def check_found(self, found, importer):
+        self.assertIs(found.loader, importer)
+
+
+(Frozen_FindSpecTests,
+ Source_FindSpecTests
+ ) = util.test_both(FindSpecTests, importlib=importlib, machinery=machinery)
+
+
+class PathEntryFinderTests:
+
+    def test_finder_with_failing_find_spec(self):
+        # PathEntryFinder with find_module() defined should work.
+        # Issue #20763.
+        class Finder:
+            path_location = 'test_finder_with_find_module'
+            def __init__(self, path):
+                if path != self.path_location:
+                    raise ImportError
+
+            @staticmethod
+            def find_module(fullname):
+                return None
+
+
+        with util.import_state(path=[Finder.path_location]+sys.path[:],
+                               path_hooks=[Finder]):
+            self.machinery.PathFinder.find_spec('importlib')
+
+    def test_finder_with_failing_find_module(self):
+        # PathEntryFinder with find_module() defined should work.
+        # Issue #20763.
+        class Finder:
+            path_location = 'test_finder_with_find_module'
+            def __init__(self, path):
+                if path != self.path_location:
+                    raise ImportError
+
+            @staticmethod
+            def find_module(fullname):
+                return None
+
+
+        with util.import_state(path=[Finder.path_location]+sys.path[:],
+                               path_hooks=[Finder]):
+            self.machinery.PathFinder.find_module('importlib')
+
+
+(Frozen_PEFTests,
+ Source_PEFTests
+ ) = util.test_both(PathEntryFinderTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/import_/test_relative_imports.py

@@ -0,0 +1,232 @@
+"""Test relative imports (PEP 328)."""
+from .. import util
+import unittest
+import warnings
+
+
+class RelativeImports:
+
+    """PEP 328 introduced relative imports. This allows for imports to occur
+    from within a package without having to specify the actual package name.
+
+    A simple example is to import another module within the same package
+    [module from module]::
+
+      # From pkg.mod1 with pkg.mod2 being a module.
+      from . import mod2
+
+    This also works for getting an attribute from a module that is specified
+    in a relative fashion [attr from module]::
+
+      # From pkg.mod1.
+      from .mod2 import attr
+
+    But this is in no way restricted to working between modules; it works
+    from [package to module],::
+
+      # From pkg, importing pkg.module which is a module.
+      from . import module
+
+    [module to package],::
+
+      # Pull attr from pkg, called from pkg.module which is a module.
+      from . import attr
+
+    and [package to package]::
+
+      # From pkg.subpkg1 (both pkg.subpkg[1,2] are packages).
+      from .. import subpkg2
+
+    The number of dots used is in no way restricted [deep import]::
+
+      # Import pkg.attr from pkg.pkg1.pkg2.pkg3.pkg4.pkg5.
+      from ...... import attr
+
+    To prevent someone from accessing code that is outside of a package, one
+    cannot reach the location containing the root package itself::
+
+      # From pkg.__init__ [too high from package]
+      from .. import top_level
+
+      # From pkg.module [too high from module]
+      from .. import top_level
+
+     Relative imports are the only type of import that allow for an empty
+     module name for an import [empty name].
+
+    """
+
+    def relative_import_test(self, create, globals_, callback):
+        """Abstract out boilerplace for setting up for an import test."""
+        uncache_names = []
+        for name in create:
+            if not name.endswith('.__init__'):
+                uncache_names.append(name)
+            else:
+                uncache_names.append(name[:-len('.__init__')])
+        with util.mock_spec(*create) as importer:
+            with util.import_state(meta_path=[importer]):
+                with warnings.catch_warnings():
+                    warnings.simplefilter("ignore")
+                    for global_ in globals_:
+                        with util.uncache(*uncache_names):
+                            callback(global_)
+
+
+    def test_module_from_module(self):
+        # [module from module]
+        create = 'pkg.__init__', 'pkg.mod2'
+        globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.mod1'}
+        def callback(global_):
+            self.__import__('pkg')  # For __import__().
+            module = self.__import__('', global_, fromlist=['mod2'], level=1)
+            self.assertEqual(module.__name__, 'pkg')
+            self.assertTrue(hasattr(module, 'mod2'))
+            self.assertEqual(module.mod2.attr, 'pkg.mod2')
+        self.relative_import_test(create, globals_, callback)
+
+    def test_attr_from_module(self):
+        # [attr from module]
+        create = 'pkg.__init__', 'pkg.mod2'
+        globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.mod1'}
+        def callback(global_):
+            self.__import__('pkg')  # For __import__().
+            module = self.__import__('mod2', global_, fromlist=['attr'],
+                                            level=1)
+            self.assertEqual(module.__name__, 'pkg.mod2')
+            self.assertEqual(module.attr, 'pkg.mod2')
+        self.relative_import_test(create, globals_, callback)
+
+    def test_package_to_module(self):
+        # [package to module]
+        create = 'pkg.__init__', 'pkg.module'
+        globals_ = ({'__package__': 'pkg'},
+                    {'__name__': 'pkg', '__path__': ['blah']})
+        def callback(global_):
+            self.__import__('pkg')  # For __import__().
+            module = self.__import__('', global_, fromlist=['module'],
+                             level=1)
+            self.assertEqual(module.__name__, 'pkg')
+            self.assertTrue(hasattr(module, 'module'))
+            self.assertEqual(module.module.attr, 'pkg.module')
+        self.relative_import_test(create, globals_, callback)
+
+    def test_module_to_package(self):
+        # [module to package]
+        create = 'pkg.__init__', 'pkg.module'
+        globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.module'}
+        def callback(global_):
+            self.__import__('pkg')  # For __import__().
+            module = self.__import__('', global_, fromlist=['attr'], level=1)
+            self.assertEqual(module.__name__, 'pkg')
+        self.relative_import_test(create, globals_, callback)
+
+    def test_package_to_package(self):
+        # [package to package]
+        create = ('pkg.__init__', 'pkg.subpkg1.__init__',
+                    'pkg.subpkg2.__init__')
+        globals_ =  ({'__package__': 'pkg.subpkg1'},
+                     {'__name__': 'pkg.subpkg1', '__path__': ['blah']})
+        def callback(global_):
+            module = self.__import__('', global_, fromlist=['subpkg2'],
+                                            level=2)
+            self.assertEqual(module.__name__, 'pkg')
+            self.assertTrue(hasattr(module, 'subpkg2'))
+            self.assertEqual(module.subpkg2.attr, 'pkg.subpkg2.__init__')
+
+    def test_deep_import(self):
+        # [deep import]
+        create = ['pkg.__init__']
+        for count in range(1,6):
+            create.append('{0}.pkg{1}.__init__'.format(
+                            create[-1][:-len('.__init__')], count))
+        globals_ = ({'__package__': 'pkg.pkg1.pkg2.pkg3.pkg4.pkg5'},
+                    {'__name__': 'pkg.pkg1.pkg2.pkg3.pkg4.pkg5',
+                        '__path__': ['blah']})
+        def callback(global_):
+            self.__import__(globals_[0]['__package__'])
+            module = self.__import__('', global_, fromlist=['attr'], level=6)
+            self.assertEqual(module.__name__, 'pkg')
+        self.relative_import_test(create, globals_, callback)
+
+    def test_too_high_from_package(self):
+        # [too high from package]
+        create = ['top_level', 'pkg.__init__']
+        globals_ = ({'__package__': 'pkg'},
+                    {'__name__': 'pkg', '__path__': ['blah']})
+        def callback(global_):
+            self.__import__('pkg')
+            with self.assertRaises(ValueError):
+                self.__import__('', global_, fromlist=['top_level'],
+                                    level=2)
+        self.relative_import_test(create, globals_, callback)
+
+    def test_too_high_from_module(self):
+        # [too high from module]
+        create = ['top_level', 'pkg.__init__', 'pkg.module']
+        globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.module'}
+        def callback(global_):
+            self.__import__('pkg')
+            with self.assertRaises(ValueError):
+                self.__import__('', global_, fromlist=['top_level'],
+                                    level=2)
+        self.relative_import_test(create, globals_, callback)
+
+    def test_empty_name_w_level_0(self):
+        # [empty name]
+        with self.assertRaises(ValueError):
+            self.__import__('')
+
+    def test_import_from_different_package(self):
+        # Test importing from a different package than the caller.
+        # in pkg.subpkg1.mod
+        # from ..subpkg2 import mod
+        create = ['__runpy_pkg__.__init__',
+                    '__runpy_pkg__.__runpy_pkg__.__init__',
+                    '__runpy_pkg__.uncle.__init__',
+                    '__runpy_pkg__.uncle.cousin.__init__',
+                    '__runpy_pkg__.uncle.cousin.nephew']
+        globals_ = {'__package__': '__runpy_pkg__.__runpy_pkg__'}
+        def callback(global_):
+            self.__import__('__runpy_pkg__.__runpy_pkg__')
+            module = self.__import__('uncle.cousin', globals_, {},
+                                    fromlist=['nephew'],
+                                level=2)
+            self.assertEqual(module.__name__, '__runpy_pkg__.uncle.cousin')
+        self.relative_import_test(create, globals_, callback)
+
+    def test_import_relative_import_no_fromlist(self):
+        # Import a relative module w/ no fromlist.
+        create = ['crash.__init__', 'crash.mod']
+        globals_ = [{'__package__': 'crash', '__name__': 'crash'}]
+        def callback(global_):
+            self.__import__('crash')
+            mod = self.__import__('mod', global_, {}, [], 1)
+            self.assertEqual(mod.__name__, 'crash.mod')
+        self.relative_import_test(create, globals_, callback)
+
+    def test_relative_import_no_globals(self):
+        # No globals for a relative import is an error.
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            with self.assertRaises(KeyError):
+                self.__import__('sys', level=1)
+
+    def test_relative_import_no_package(self):
+        with self.assertRaises(ImportError):
+            self.__import__('a', {'__package__': '', '__spec__': None},
+                            level=1)
+
+    def test_relative_import_no_package_exists_absolute(self):
+        with self.assertRaises(ImportError):
+            self.__import__('sys', {'__package__': '', '__spec__': None},
+                            level=1)
+
+
+(Frozen_RelativeImports,
+ Source_RelativeImports
+ ) = util.test_both(RelativeImports, __import__=util.__import__)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/namespace_pkgs/both_portions/foo/one.py

@@ -0,0 +1 @@
+attr = 'both_portions foo one'

Lib/test/test_importlib/namespace_pkgs/both_portions/foo/two.py

@@ -0,0 +1 @@
+attr = 'both_portions foo two'

Lib/test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test.py

@@ -0,0 +1 @@
+attr = 'in module'

Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/one.py

@@ -0,0 +1 @@
+attr = 'portion1 foo one'

Lib/test/test_importlib/namespace_pkgs/portion1/foo/one.py

@@ -0,0 +1 @@
+attr = 'portion1 foo one'

Lib/test/test_importlib/namespace_pkgs/portion2/foo/two.py

@@ -0,0 +1 @@
+attr = 'portion2 foo two'

Lib/test/test_importlib/namespace_pkgs/project1/parent/child/one.py

@@ -0,0 +1 @@
+attr = 'parent child one'

Lib/test/test_importlib/namespace_pkgs/project2/parent/child/two.py

@@ -0,0 +1 @@
+attr = 'parent child two'

Lib/test/test_importlib/namespace_pkgs/project3/parent/child/three.py

@@ -0,0 +1 @@
+attr = 'parent child three'

Lib/test/test_importlib/source/__init__.py

@@ -0,0 +1,5 @@
+import os
+from test.support import load_package_tests
+
+def load_tests(*args):
+    return load_package_tests(os.path.dirname(__file__), *args)

Lib/test/test_importlib/source/__main__.py

@@ -0,0 +1,4 @@
+from . import load_tests
+import unittest
+
+unittest.main()

Lib/test/test_importlib/source/test_case_sensitivity.py

@@ -0,0 +1,85 @@
+"""Test case-sensitivity (PEP 235)."""
+from .. import util
+
+importlib = util.import_importlib('importlib')
+machinery = util.import_importlib('importlib.machinery')
+
+import os
+from test import support as test_support
+import unittest
+
+
+@util.case_insensitive_tests
+class CaseSensitivityTest(util.CASEOKTestBase):
+
+    """PEP 235 dictates that on case-preserving, case-insensitive file systems
+    that imports are case-sensitive unless the PYTHONCASEOK environment
+    variable is set."""
+
+    name = 'MoDuLe'
+    assert name != name.lower()
+
+    def finder(self, path):
+        return self.machinery.FileFinder(path,
+                                      (self.machinery.SourceFileLoader,
+                                            self.machinery.SOURCE_SUFFIXES),
+                                        (self.machinery.SourcelessFileLoader,
+                                            self.machinery.BYTECODE_SUFFIXES))
+
+    def sensitivity_test(self):
+        """Look for a module with matching and non-matching sensitivity."""
+        sensitive_pkg = 'sensitive.{0}'.format(self.name)
+        insensitive_pkg = 'insensitive.{0}'.format(self.name.lower())
+        context = util.create_modules(insensitive_pkg, sensitive_pkg)
+        with context as mapping:
+            sensitive_path = os.path.join(mapping['.root'], 'sensitive')
+            insensitive_path = os.path.join(mapping['.root'], 'insensitive')
+            sensitive_finder = self.finder(sensitive_path)
+            insensitive_finder = self.finder(insensitive_path)
+            return self.find(sensitive_finder), self.find(insensitive_finder)
+
+    def test_sensitive(self):
+        with test_support.EnvironmentVarGuard() as env:
+            env.unset('PYTHONCASEOK')
+            self.caseok_env_changed(should_exist=False)
+            sensitive, insensitive = self.sensitivity_test()
+            self.assertIsNotNone(sensitive)
+            self.assertIn(self.name, sensitive.get_filename(self.name))
+            self.assertIsNone(insensitive)
+
+    def test_insensitive(self):
+        with test_support.EnvironmentVarGuard() as env:
+            env.set('PYTHONCASEOK', '1')
+            self.caseok_env_changed(should_exist=True)
+            sensitive, insensitive = self.sensitivity_test()
+            self.assertIsNotNone(sensitive)
+            self.assertIn(self.name, sensitive.get_filename(self.name))
+            self.assertIsNotNone(insensitive)
+            self.assertIn(self.name, insensitive.get_filename(self.name))
+
+
+class CaseSensitivityTestPEP302(CaseSensitivityTest):
+    def find(self, finder):
+        return finder.find_module(self.name)
+
+
+(Frozen_CaseSensitivityTestPEP302,
+ Source_CaseSensitivityTestPEP302
+ ) = util.test_both(CaseSensitivityTestPEP302, importlib=importlib,
+                    machinery=machinery)
+
+
+class CaseSensitivityTestPEP451(CaseSensitivityTest):
+    def find(self, finder):
+        found = finder.find_spec(self.name)
+        return found.loader if found is not None else found
+
+
+(Frozen_CaseSensitivityTestPEP451,
+ Source_CaseSensitivityTestPEP451
+ ) = util.test_both(CaseSensitivityTestPEP451, importlib=importlib,
+                    machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/source/test_file_loader.py

@@ -0,0 +1,804 @@
+from .. import abc
+from .. import util
+
+importlib = util.import_importlib('importlib')
+importlib_abc = util.import_importlib('importlib.abc')
+machinery = util.import_importlib('importlib.machinery')
+importlib_util = util.import_importlib('importlib.util')
+
+import errno
+import marshal
+import os
+import py_compile
+import shutil
+import stat
+import sys
+import types
+import unittest
+import warnings
+
+from test.support import make_legacy_pyc, unload
+
+from test.test_py_compile import without_source_date_epoch
+from test.test_py_compile import SourceDateEpochTestMeta
+
+
+class SimpleTest(abc.LoaderTests):
+
+    """Should have no issue importing a source module [basic]. And if there is
+    a syntax error, it should raise a SyntaxError [syntax error].
+
+    """
+
+    def setUp(self):
+        self.name = 'spam'
+        self.filepath = os.path.join('ham', self.name + '.py')
+        self.loader = self.machinery.SourceFileLoader(self.name, self.filepath)
+
+    def test_load_module_API(self):
+        class Tester(self.abc.FileLoader):
+            def get_source(self, _): return 'attr = 42'
+            def is_package(self, _): return False
+
+        loader = Tester('blah', 'blah.py')
+        self.addCleanup(unload, 'blah')
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', DeprecationWarning)
+            module = loader.load_module()  # Should not raise an exception.
+
+    def test_get_filename_API(self):
+        # If fullname is not set then assume self.path is desired.
+        class Tester(self.abc.FileLoader):
+            def get_code(self, _): pass
+            def get_source(self, _): pass
+            def is_package(self, _): pass
+            def module_repr(self, _): pass
+
+        path = 'some_path'
+        name = 'some_name'
+        loader = Tester(name, path)
+        self.assertEqual(path, loader.get_filename(name))
+        self.assertEqual(path, loader.get_filename())
+        self.assertEqual(path, loader.get_filename(None))
+        with self.assertRaises(ImportError):
+            loader.get_filename(name + 'XXX')
+
+    def test_equality(self):
+        other = self.machinery.SourceFileLoader(self.name, self.filepath)
+        self.assertEqual(self.loader, other)
+
+    def test_inequality(self):
+        other = self.machinery.SourceFileLoader('_' + self.name, self.filepath)
+        self.assertNotEqual(self.loader, other)
+
+    # [basic]
+    def test_module(self):
+        with util.create_modules('_temp') as mapping:
+            loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                module = loader.load_module('_temp')
+            self.assertIn('_temp', sys.modules)
+            check = {'__name__': '_temp', '__file__': mapping['_temp'],
+                     '__package__': ''}
+            for attr, value in check.items():
+                self.assertEqual(getattr(module, attr), value)
+
+    def test_package(self):
+        with util.create_modules('_pkg.__init__') as mapping:
+            loader = self.machinery.SourceFileLoader('_pkg',
+                                                 mapping['_pkg.__init__'])
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                module = loader.load_module('_pkg')
+            self.assertIn('_pkg', sys.modules)
+            check = {'__name__': '_pkg', '__file__': mapping['_pkg.__init__'],
+                     '__path__': [os.path.dirname(mapping['_pkg.__init__'])],
+                     '__package__': '_pkg'}
+            for attr, value in check.items():
+                self.assertEqual(getattr(module, attr), value)
+
+
+    def test_lacking_parent(self):
+        with util.create_modules('_pkg.__init__', '_pkg.mod')as mapping:
+            loader = self.machinery.SourceFileLoader('_pkg.mod',
+                                                    mapping['_pkg.mod'])
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                module = loader.load_module('_pkg.mod')
+            self.assertIn('_pkg.mod', sys.modules)
+            check = {'__name__': '_pkg.mod', '__file__': mapping['_pkg.mod'],
+                     '__package__': '_pkg'}
+            for attr, value in check.items():
+                self.assertEqual(getattr(module, attr), value)
+
+    def fake_mtime(self, fxn):
+        """Fake mtime to always be higher than expected."""
+        return lambda name: fxn(name) + 1
+
+    def test_module_reuse(self):
+        with util.create_modules('_temp') as mapping:
+            loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                module = loader.load_module('_temp')
+            module_id = id(module)
+            module_dict_id = id(module.__dict__)
+            with open(mapping['_temp'], 'w') as file:
+                file.write("testing_var = 42\n")
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                module = loader.load_module('_temp')
+            self.assertIn('testing_var', module.__dict__,
+                         "'testing_var' not in "
+                            "{0}".format(list(module.__dict__.keys())))
+            self.assertEqual(module, sys.modules['_temp'])
+            self.assertEqual(id(module), module_id)
+            self.assertEqual(id(module.__dict__), module_dict_id)
+
+    def test_state_after_failure(self):
+        # A failed reload should leave the original module intact.
+        attributes = ('__file__', '__path__', '__package__')
+        value = '<test>'
+        name = '_temp'
+        with util.create_modules(name) as mapping:
+            orig_module = types.ModuleType(name)
+            for attr in attributes:
+                setattr(orig_module, attr, value)
+            with open(mapping[name], 'w') as file:
+                file.write('+++ bad syntax +++')
+            loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
+            with self.assertRaises(SyntaxError):
+                loader.exec_module(orig_module)
+            for attr in attributes:
+                self.assertEqual(getattr(orig_module, attr), value)
+            with self.assertRaises(SyntaxError):
+                with warnings.catch_warnings():
+                    warnings.simplefilter('ignore', DeprecationWarning)
+                    loader.load_module(name)
+            for attr in attributes:
+                self.assertEqual(getattr(orig_module, attr), value)
+
+    # [syntax error]
+    def test_bad_syntax(self):
+        with util.create_modules('_temp') as mapping:
+            with open(mapping['_temp'], 'w') as file:
+                file.write('=')
+            loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
+            with self.assertRaises(SyntaxError):
+                with warnings.catch_warnings():
+                    warnings.simplefilter('ignore', DeprecationWarning)
+                    loader.load_module('_temp')
+            self.assertNotIn('_temp', sys.modules)
+
+    def test_file_from_empty_string_dir(self):
+        # Loading a module found from an empty string entry on sys.path should
+        # not only work, but keep all attributes relative.
+        file_path = '_temp.py'
+        with open(file_path, 'w') as file:
+            file.write("# test file for importlib")
+        try:
+            with util.uncache('_temp'):
+                loader = self.machinery.SourceFileLoader('_temp', file_path)
+                with warnings.catch_warnings():
+                    warnings.simplefilter('ignore', DeprecationWarning)
+                    mod = loader.load_module('_temp')
+                self.assertEqual(file_path, mod.__file__)
+                self.assertEqual(self.util.cache_from_source(file_path),
+                                 mod.__cached__)
+        finally:
+            os.unlink(file_path)
+            pycache = os.path.dirname(self.util.cache_from_source(file_path))
+            if os.path.exists(pycache):
+                shutil.rmtree(pycache)
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    @util.writes_bytecode_files
+    def test_timestamp_overflow(self):
+        # When a modification timestamp is larger than 2**32, it should be
+        # truncated rather than raise an OverflowError.
+        with util.create_modules('_temp') as mapping:
+            source = mapping['_temp']
+            compiled = self.util.cache_from_source(source)
+            with open(source, 'w') as f:
+                f.write("x = 5")
+            try:
+                os.utime(source, (2 ** 33 - 5, 2 ** 33 - 5))
+            except OverflowError:
+                self.skipTest("cannot set modification time to large integer")
+            except OSError as e:
+                if e.errno != getattr(errno, 'EOVERFLOW', None):
+                    raise
+                self.skipTest("cannot set modification time to large integer ({})".format(e))
+            loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
+            # PEP 451
+            module = types.ModuleType('_temp')
+            module.__spec__ = self.util.spec_from_loader('_temp', loader)
+            loader.exec_module(module)
+            self.assertEqual(module.x, 5)
+            self.assertTrue(os.path.exists(compiled))
+            os.unlink(compiled)
+            # PEP 302
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                mod = loader.load_module('_temp')
+            # Sanity checks.
+            self.assertEqual(mod.__cached__, compiled)
+            self.assertEqual(mod.x, 5)
+            # The pyc file was created.
+            self.assertTrue(os.path.exists(compiled))
+
+    def test_unloadable(self):
+        loader = self.machinery.SourceFileLoader('good name', {})
+        module = types.ModuleType('bad name')
+        module.__spec__ = self.machinery.ModuleSpec('bad name', loader)
+        with self.assertRaises(ImportError):
+            loader.exec_module(module)
+        with self.assertRaises(ImportError):
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                loader.load_module('bad name')
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    @util.writes_bytecode_files
+    def test_checked_hash_based_pyc(self):
+        with util.create_modules('_temp') as mapping:
+            source = mapping['_temp']
+            pyc = self.util.cache_from_source(source)
+            with open(source, 'wb') as fp:
+                fp.write(b'state = "old"')
+            os.utime(source, (50, 50))
+            py_compile.compile(
+                source,
+                invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+            )
+            loader = self.machinery.SourceFileLoader('_temp', source)
+            mod = types.ModuleType('_temp')
+            mod.__spec__ = self.util.spec_from_loader('_temp', loader)
+            loader.exec_module(mod)
+            self.assertEqual(mod.state, 'old')
+            # Write a new source with the same mtime and size as before.
+            with open(source, 'wb') as fp:
+                fp.write(b'state = "new"')
+            os.utime(source, (50, 50))
+            loader.exec_module(mod)
+            self.assertEqual(mod.state, 'new')
+            with open(pyc, 'rb') as fp:
+                data = fp.read()
+            self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b11)
+            self.assertEqual(
+                self.util.source_hash(b'state = "new"'),
+                data[8:16],
+            )
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    @util.writes_bytecode_files
+    def test_overridden_checked_hash_based_pyc(self):
+        with util.create_modules('_temp') as mapping, \
+             unittest.mock.patch('_imp.check_hash_based_pycs', 'never'):
+            source = mapping['_temp']
+            pyc = self.util.cache_from_source(source)
+            with open(source, 'wb') as fp:
+                fp.write(b'state = "old"')
+            os.utime(source, (50, 50))
+            py_compile.compile(
+                source,
+                invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+            )
+            loader = self.machinery.SourceFileLoader('_temp', source)
+            mod = types.ModuleType('_temp')
+            mod.__spec__ = self.util.spec_from_loader('_temp', loader)
+            loader.exec_module(mod)
+            self.assertEqual(mod.state, 'old')
+            # Write a new source with the same mtime and size as before.
+            with open(source, 'wb') as fp:
+                fp.write(b'state = "new"')
+            os.utime(source, (50, 50))
+            loader.exec_module(mod)
+            self.assertEqual(mod.state, 'old')
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    @util.writes_bytecode_files
+    def test_unchecked_hash_based_pyc(self):
+        with util.create_modules('_temp') as mapping:
+            source = mapping['_temp']
+            pyc = self.util.cache_from_source(source)
+            with open(source, 'wb') as fp:
+                fp.write(b'state = "old"')
+            os.utime(source, (50, 50))
+            py_compile.compile(
+                source,
+                invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH,
+            )
+            loader = self.machinery.SourceFileLoader('_temp', source)
+            mod = types.ModuleType('_temp')
+            mod.__spec__ = self.util.spec_from_loader('_temp', loader)
+            loader.exec_module(mod)
+            self.assertEqual(mod.state, 'old')
+            # Update the source file, which should be ignored.
+            with open(source, 'wb') as fp:
+                fp.write(b'state = "new"')
+            loader.exec_module(mod)
+            self.assertEqual(mod.state, 'old')
+            with open(pyc, 'rb') as fp:
+                data = fp.read()
+            self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b1)
+            self.assertEqual(
+                self.util.source_hash(b'state = "old"'),
+                data[8:16],
+            )
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    @util.writes_bytecode_files
+    def test_overiden_unchecked_hash_based_pyc(self):
+        with util.create_modules('_temp') as mapping, \
+             unittest.mock.patch('_imp.check_hash_based_pycs', 'always'):
+            source = mapping['_temp']
+            pyc = self.util.cache_from_source(source)
+            with open(source, 'wb') as fp:
+                fp.write(b'state = "old"')
+            os.utime(source, (50, 50))
+            py_compile.compile(
+                source,
+                invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH,
+            )
+            loader = self.machinery.SourceFileLoader('_temp', source)
+            mod = types.ModuleType('_temp')
+            mod.__spec__ = self.util.spec_from_loader('_temp', loader)
+            loader.exec_module(mod)
+            self.assertEqual(mod.state, 'old')
+            # Update the source file, which should be ignored.
+            with open(source, 'wb') as fp:
+                fp.write(b'state = "new"')
+            loader.exec_module(mod)
+            self.assertEqual(mod.state, 'new')
+            with open(pyc, 'rb') as fp:
+                data = fp.read()
+            self.assertEqual(int.from_bytes(data[4:8], 'little'), 0b1)
+            self.assertEqual(
+                self.util.source_hash(b'state = "new"'),
+                data[8:16],
+            )
+
+
+(Frozen_SimpleTest,
+ Source_SimpleTest
+ ) = util.test_both(SimpleTest, importlib=importlib, machinery=machinery,
+                    abc=importlib_abc, util=importlib_util)
+
+
+class SourceDateEpochTestMeta(SourceDateEpochTestMeta,
+                              type(Source_SimpleTest)):
+    pass
+
+
+class SourceDateEpoch_SimpleTest(Source_SimpleTest,
+                                 metaclass=SourceDateEpochTestMeta,
+                                 source_date_epoch=True):
+    pass
+
+
+class BadBytecodeTest:
+
+    def import_(self, file, module_name):
+        raise NotImplementedError
+
+    def manipulate_bytecode(self,
+                            name, mapping, manipulator, *,
+                            del_source=False,
+                            invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP):
+        """Manipulate the bytecode of a module by passing it into a callable
+        that returns what to use as the new bytecode."""
+        try:
+            del sys.modules['_temp']
+        except KeyError:
+            pass
+        py_compile.compile(mapping[name], invalidation_mode=invalidation_mode)
+        if not del_source:
+            bytecode_path = self.util.cache_from_source(mapping[name])
+        else:
+            os.unlink(mapping[name])
+            bytecode_path = make_legacy_pyc(mapping[name])
+        if manipulator:
+            with open(bytecode_path, 'rb') as file:
+                bc = file.read()
+                new_bc = manipulator(bc)
+            with open(bytecode_path, 'wb') as file:
+                if new_bc is not None:
+                    file.write(new_bc)
+        return bytecode_path
+
+    def _test_empty_file(self, test, *, del_source=False):
+        with util.create_modules('_temp') as mapping:
+            bc_path = self.manipulate_bytecode('_temp', mapping,
+                                                lambda bc: b'',
+                                                del_source=del_source)
+            test('_temp', mapping, bc_path)
+
+    @util.writes_bytecode_files
+    def _test_partial_magic(self, test, *, del_source=False):
+        # When their are less than 4 bytes to a .pyc, regenerate it if
+        # possible, else raise ImportError.
+        with util.create_modules('_temp') as mapping:
+            bc_path = self.manipulate_bytecode('_temp', mapping,
+                                                lambda bc: bc[:3],
+                                                del_source=del_source)
+            test('_temp', mapping, bc_path)
+
+    def _test_magic_only(self, test, *, del_source=False):
+        with util.create_modules('_temp') as mapping:
+            bc_path = self.manipulate_bytecode('_temp', mapping,
+                                                lambda bc: bc[:4],
+                                                del_source=del_source)
+            test('_temp', mapping, bc_path)
+
+    def _test_partial_flags(self, test, *, del_source=False):
+        with util.create_modules('_temp') as mapping:
+            bc_path = self.manipulate_bytecode('_temp', mapping,
+                                               lambda bc: bc[:7],
+                                               del_source=del_source)
+            test('_temp', mapping, bc_path)
+
+    def _test_partial_hash(self, test, *, del_source=False):
+        with util.create_modules('_temp') as mapping:
+            bc_path = self.manipulate_bytecode(
+                '_temp',
+                mapping,
+                lambda bc: bc[:13],
+                del_source=del_source,
+                invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
+            )
+            test('_temp', mapping, bc_path)
+        with util.create_modules('_temp') as mapping:
+            bc_path = self.manipulate_bytecode(
+                '_temp',
+                mapping,
+                lambda bc: bc[:13],
+                del_source=del_source,
+                invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH,
+            )
+            test('_temp', mapping, bc_path)
+
+    def _test_partial_timestamp(self, test, *, del_source=False):
+        with util.create_modules('_temp') as mapping:
+            bc_path = self.manipulate_bytecode('_temp', mapping,
+                                                lambda bc: bc[:11],
+                                                del_source=del_source)
+            test('_temp', mapping, bc_path)
+
+    def _test_partial_size(self, test, *, del_source=False):
+        with util.create_modules('_temp') as mapping:
+            bc_path = self.manipulate_bytecode('_temp', mapping,
+                                                lambda bc: bc[:15],
+                                                del_source=del_source)
+            test('_temp', mapping, bc_path)
+
+    def _test_no_marshal(self, *, del_source=False):
+        with util.create_modules('_temp') as mapping:
+            bc_path = self.manipulate_bytecode('_temp', mapping,
+                                                lambda bc: bc[:16],
+                                                del_source=del_source)
+            file_path = mapping['_temp'] if not del_source else bc_path
+            with self.assertRaises(EOFError):
+                self.import_(file_path, '_temp')
+
+    def _test_non_code_marshal(self, *, del_source=False):
+        with util.create_modules('_temp') as mapping:
+            bytecode_path = self.manipulate_bytecode('_temp', mapping,
+                                    lambda bc: bc[:16] + marshal.dumps(b'abcd'),
+                                    del_source=del_source)
+            file_path = mapping['_temp'] if not del_source else bytecode_path
+            with self.assertRaises(ImportError) as cm:
+                self.import_(file_path, '_temp')
+            self.assertEqual(cm.exception.name, '_temp')
+            self.assertEqual(cm.exception.path, bytecode_path)
+
+    def _test_bad_marshal(self, *, del_source=False):
+        with util.create_modules('_temp') as mapping:
+            bytecode_path = self.manipulate_bytecode('_temp', mapping,
+                                                lambda bc: bc[:16] + b'<test>',
+                                                del_source=del_source)
+            file_path = mapping['_temp'] if not del_source else bytecode_path
+            with self.assertRaises(EOFError):
+                self.import_(file_path, '_temp')
+
+    def _test_bad_magic(self, test, *, del_source=False):
+        with util.create_modules('_temp') as mapping:
+            bc_path = self.manipulate_bytecode('_temp', mapping,
+                                    lambda bc: b'\x00\x00\x00\x00' + bc[4:])
+            test('_temp', mapping, bc_path)
+
+
+class BadBytecodeTestPEP451(BadBytecodeTest):
+
+    def import_(self, file, module_name):
+        loader = self.loader(module_name, file)
+        module = types.ModuleType(module_name)
+        module.__spec__ = self.util.spec_from_loader(module_name, loader)
+        loader.exec_module(module)
+
+
+class BadBytecodeTestPEP302(BadBytecodeTest):
+
+    def import_(self, file, module_name):
+        loader = self.loader(module_name, file)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', DeprecationWarning)
+            module = loader.load_module(module_name)
+        self.assertIn(module_name, sys.modules)
+
+
+class SourceLoaderBadBytecodeTest:
+
+    @classmethod
+    def setUpClass(cls):
+        cls.loader = cls.machinery.SourceFileLoader
+
+    @util.writes_bytecode_files
+    def test_empty_file(self):
+        # When a .pyc is empty, regenerate it if possible, else raise
+        # ImportError.
+        def test(name, mapping, bytecode_path):
+            self.import_(mapping[name], name)
+            with open(bytecode_path, 'rb') as file:
+                self.assertGreater(len(file.read()), 16)
+
+        self._test_empty_file(test)
+
+    def test_partial_magic(self):
+        def test(name, mapping, bytecode_path):
+            self.import_(mapping[name], name)
+            with open(bytecode_path, 'rb') as file:
+                self.assertGreater(len(file.read()), 16)
+
+        self._test_partial_magic(test)
+
+    @util.writes_bytecode_files
+    def test_magic_only(self):
+        # When there is only the magic number, regenerate the .pyc if possible,
+        # else raise EOFError.
+        def test(name, mapping, bytecode_path):
+            self.import_(mapping[name], name)
+            with open(bytecode_path, 'rb') as file:
+                self.assertGreater(len(file.read()), 16)
+
+        self._test_magic_only(test)
+
+    @util.writes_bytecode_files
+    def test_bad_magic(self):
+        # When the magic number is different, the bytecode should be
+        # regenerated.
+        def test(name, mapping, bytecode_path):
+            self.import_(mapping[name], name)
+            with open(bytecode_path, 'rb') as bytecode_file:
+                self.assertEqual(bytecode_file.read(4),
+                                 self.util.MAGIC_NUMBER)
+
+        self._test_bad_magic(test)
+
+    @util.writes_bytecode_files
+    def test_partial_timestamp(self):
+        # When the timestamp is partial, regenerate the .pyc, else
+        # raise EOFError.
+        def test(name, mapping, bc_path):
+            self.import_(mapping[name], name)
+            with open(bc_path, 'rb') as file:
+                self.assertGreater(len(file.read()), 16)
+
+        self._test_partial_timestamp(test)
+
+    @util.writes_bytecode_files
+    def test_partial_flags(self):
+        # When the flags is partial, regenerate the .pyc, else raise EOFError.
+        def test(name, mapping, bc_path):
+            self.import_(mapping[name], name)
+            with open(bc_path, 'rb') as file:
+                self.assertGreater(len(file.read()), 16)
+
+        self._test_partial_flags(test)
+
+    @util.writes_bytecode_files
+    def test_partial_hash(self):
+        # When the hash is partial, regenerate the .pyc, else raise EOFError.
+        def test(name, mapping, bc_path):
+            self.import_(mapping[name], name)
+            with open(bc_path, 'rb') as file:
+                self.assertGreater(len(file.read()), 16)
+
+        self._test_partial_hash(test)
+
+    @util.writes_bytecode_files
+    def test_partial_size(self):
+        # When the size is partial, regenerate the .pyc, else
+        # raise EOFError.
+        def test(name, mapping, bc_path):
+            self.import_(mapping[name], name)
+            with open(bc_path, 'rb') as file:
+                self.assertGreater(len(file.read()), 16)
+
+        self._test_partial_size(test)
+
+    @util.writes_bytecode_files
+    def test_no_marshal(self):
+        # When there is only the magic number and timestamp, raise EOFError.
+        self._test_no_marshal()
+
+    @util.writes_bytecode_files
+    def test_non_code_marshal(self):
+        self._test_non_code_marshal()
+        # XXX ImportError when sourceless
+
+    # [bad marshal]
+    @util.writes_bytecode_files
+    def test_bad_marshal(self):
+        # Bad marshal data should raise a ValueError.
+        self._test_bad_marshal()
+
+    # [bad timestamp]
+    @util.writes_bytecode_files
+    @without_source_date_epoch
+    def test_old_timestamp(self):
+        # When the timestamp is older than the source, bytecode should be
+        # regenerated.
+        zeros = b'\x00\x00\x00\x00'
+        with util.create_modules('_temp') as mapping:
+            py_compile.compile(mapping['_temp'])
+            bytecode_path = self.util.cache_from_source(mapping['_temp'])
+            with open(bytecode_path, 'r+b') as bytecode_file:
+                bytecode_file.seek(8)
+                bytecode_file.write(zeros)
+            self.import_(mapping['_temp'], '_temp')
+            source_mtime = os.path.getmtime(mapping['_temp'])
+            source_timestamp = self.importlib._pack_uint32(source_mtime)
+            with open(bytecode_path, 'rb') as bytecode_file:
+                bytecode_file.seek(8)
+                self.assertEqual(bytecode_file.read(4), source_timestamp)
+
+    # [bytecode read-only]
+    @util.writes_bytecode_files
+    def test_read_only_bytecode(self):
+        # When bytecode is read-only but should be rewritten, fail silently.
+        with util.create_modules('_temp') as mapping:
+            # Create bytecode that will need to be re-created.
+            py_compile.compile(mapping['_temp'])
+            bytecode_path = self.util.cache_from_source(mapping['_temp'])
+            with open(bytecode_path, 'r+b') as bytecode_file:
+                bytecode_file.seek(0)
+                bytecode_file.write(b'\x00\x00\x00\x00')
+            # Make the bytecode read-only.
+            os.chmod(bytecode_path,
+                        stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
+            try:
+                # Should not raise OSError!
+                self.import_(mapping['_temp'], '_temp')
+            finally:
+                # Make writable for eventual clean-up.
+                os.chmod(bytecode_path, stat.S_IWUSR)
+
+
+# TODO: RustPython
+# class SourceLoaderBadBytecodeTestPEP451(
+#         SourceLoaderBadBytecodeTest, BadBytecodeTestPEP451):
+#     pass
+
+
+# (Frozen_SourceBadBytecodePEP451,
+#  Source_SourceBadBytecodePEP451
+#  ) = util.test_both(SourceLoaderBadBytecodeTestPEP451, importlib=importlib,
+#                     machinery=machinery, abc=importlib_abc,
+#                     util=importlib_util)
+
+
+# class SourceLoaderBadBytecodeTestPEP302(
+#         SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302):
+#     pass
+
+
+# (Frozen_SourceBadBytecodePEP302,
+#  Source_SourceBadBytecodePEP302
+#  ) = util.test_both(SourceLoaderBadBytecodeTestPEP302, importlib=importlib,
+#                     machinery=machinery, abc=importlib_abc,
+#                     util=importlib_util)
+
+
+class SourcelessLoaderBadBytecodeTest:
+
+    @classmethod
+    def setUpClass(cls):
+        cls.loader = cls.machinery.SourcelessFileLoader
+
+    def test_empty_file(self):
+        def test(name, mapping, bytecode_path):
+            with self.assertRaises(ImportError) as cm:
+                self.import_(bytecode_path, name)
+            self.assertEqual(cm.exception.name, name)
+            self.assertEqual(cm.exception.path, bytecode_path)
+
+        self._test_empty_file(test, del_source=True)
+
+    def test_partial_magic(self):
+        def test(name, mapping, bytecode_path):
+            with self.assertRaises(ImportError) as cm:
+                self.import_(bytecode_path, name)
+            self.assertEqual(cm.exception.name, name)
+            self.assertEqual(cm.exception.path, bytecode_path)
+        self._test_partial_magic(test, del_source=True)
+
+    def test_magic_only(self):
+        def test(name, mapping, bytecode_path):
+            with self.assertRaises(EOFError):
+                self.import_(bytecode_path, name)
+
+        self._test_magic_only(test, del_source=True)
+
+    def test_bad_magic(self):
+        def test(name, mapping, bytecode_path):
+            with self.assertRaises(ImportError) as cm:
+                self.import_(bytecode_path, name)
+            self.assertEqual(cm.exception.name, name)
+            self.assertEqual(cm.exception.path, bytecode_path)
+
+        self._test_bad_magic(test, del_source=True)
+
+    def test_partial_timestamp(self):
+        def test(name, mapping, bytecode_path):
+            with self.assertRaises(EOFError):
+                self.import_(bytecode_path, name)
+
+        self._test_partial_timestamp(test, del_source=True)
+
+    def test_partial_flags(self):
+        def test(name, mapping, bytecode_path):
+            with self.assertRaises(EOFError):
+                self.import_(bytecode_path, name)
+
+        self._test_partial_flags(test, del_source=True)
+
+    def test_partial_hash(self):
+        def test(name, mapping, bytecode_path):
+            with self.assertRaises(EOFError):
+                self.import_(bytecode_path, name)
+
+        self._test_partial_hash(test, del_source=True)
+
+    def test_partial_size(self):
+        def test(name, mapping, bytecode_path):
+            with self.assertRaises(EOFError):
+                self.import_(bytecode_path, name)
+
+        self._test_partial_size(test, del_source=True)
+
+    def test_no_marshal(self):
+        self._test_no_marshal(del_source=True)
+
+    def test_non_code_marshal(self):
+        self._test_non_code_marshal(del_source=True)
+
+
+# TODO: RustPython
+# class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest,
+#         BadBytecodeTestPEP451):
+#     pass
+
+
+# (Frozen_SourcelessBadBytecodePEP451,
+#  Source_SourcelessBadBytecodePEP451
+#  ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP451, importlib=importlib,
+#                     machinery=machinery, abc=importlib_abc,
+#                     util=importlib_util)
+
+
+# class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest,
+#         BadBytecodeTestPEP302):
+#     pass
+
+
+# (Frozen_SourcelessBadBytecodePEP302,
+#  Source_SourcelessBadBytecodePEP302
+#  ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib,
+#                     machinery=machinery, abc=importlib_abc,
+#                     util=importlib_util)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/source/test_finder.py

@@ -0,0 +1,237 @@
+from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import errno
+import os
+import py_compile
+import stat
+import sys
+import tempfile
+from test.support import make_legacy_pyc
+import unittest
+import warnings
+
+
+class FinderTests(abc.FinderTests):
+
+    """For a top-level module, it should just be found directly in the
+    directory being searched. This is true for a directory with source
+    [top-level source], bytecode [top-level bc], or both [top-level both].
+    There is also the possibility that it is a package [top-level package], in
+    which case there will be a directory with the module name and an
+    __init__.py file. If there is a directory without an __init__.py an
+    ImportWarning is returned [empty dir].
+
+    For sub-modules and sub-packages, the same happens as above but only use
+    the tail end of the name [sub module] [sub package] [sub empty].
+
+    When there is a conflict between a package and module having the same name
+    in the same directory, the package wins out [package over module]. This is
+    so that imports of modules within the package can occur rather than trigger
+    an import error.
+
+    When there is a package and module with the same name, always pick the
+    package over the module [package over module]. This is so that imports from
+    the package have the possibility of succeeding.
+
+    """
+
+    def get_finder(self, root):
+        loader_details = [(self.machinery.SourceFileLoader,
+                            self.machinery.SOURCE_SUFFIXES),
+                          (self.machinery.SourcelessFileLoader,
+                            self.machinery.BYTECODE_SUFFIXES)]
+        return self.machinery.FileFinder(root, *loader_details)
+
+    def import_(self, root, module):
+        finder = self.get_finder(root)
+        return self._find(finder, module, loader_only=True)
+
+    def run_test(self, test, create=None, *, compile_=None, unlink=None):
+        """Test the finding of 'test' with the creation of modules listed in
+        'create'.
+
+        Any names listed in 'compile_' are byte-compiled. Modules
+        listed in 'unlink' have their source files deleted.
+
+        """
+        if create is None:
+            create = {test}
+        with util.create_modules(*create) as mapping:
+            if compile_:
+                for name in compile_:
+                    py_compile.compile(mapping[name])
+            if unlink:
+                for name in unlink:
+                    os.unlink(mapping[name])
+                    try:
+                        make_legacy_pyc(mapping[name])
+                    except OSError as error:
+                        # Some tests do not set compile_=True so the source
+                        # module will not get compiled and there will be no
+                        # PEP 3147 pyc file to rename.
+                        if error.errno != errno.ENOENT:
+                            raise
+            loader = self.import_(mapping['.root'], test)
+            self.assertTrue(hasattr(loader, 'load_module'))
+            return loader
+
+    def test_module(self):
+        # [top-level source]
+        self.run_test('top_level')
+        # [top-level bc]
+        self.run_test('top_level', compile_={'top_level'},
+                      unlink={'top_level'})
+        # [top-level both]
+        self.run_test('top_level', compile_={'top_level'})
+
+    # [top-level package]
+    def test_package(self):
+        # Source.
+        self.run_test('pkg', {'pkg.__init__'})
+        # Bytecode.
+        self.run_test('pkg', {'pkg.__init__'}, compile_={'pkg.__init__'},
+                unlink={'pkg.__init__'})
+        # Both.
+        self.run_test('pkg', {'pkg.__init__'}, compile_={'pkg.__init__'})
+
+    # [sub module]
+    def test_module_in_package(self):
+        with util.create_modules('pkg.__init__', 'pkg.sub') as mapping:
+            pkg_dir = os.path.dirname(mapping['pkg.__init__'])
+            loader = self.import_(pkg_dir, 'pkg.sub')
+            self.assertTrue(hasattr(loader, 'load_module'))
+
+    # [sub package]
+    def test_package_in_package(self):
+        context = util.create_modules('pkg.__init__', 'pkg.sub.__init__')
+        with context as mapping:
+            pkg_dir = os.path.dirname(mapping['pkg.__init__'])
+            loader = self.import_(pkg_dir, 'pkg.sub')
+            self.assertTrue(hasattr(loader, 'load_module'))
+
+    # [package over modules]
+    def test_package_over_module(self):
+        name = '_temp'
+        loader = self.run_test(name, {'{0}.__init__'.format(name), name})
+        self.assertIn('__init__', loader.get_filename(name))
+
+    def test_failure(self):
+        with util.create_modules('blah') as mapping:
+            nothing = self.import_(mapping['.root'], 'sdfsadsadf')
+            self.assertIsNone(nothing)
+
+    def test_empty_string_for_dir(self):
+        # The empty string from sys.path means to search in the cwd.
+        finder = self.machinery.FileFinder('', (self.machinery.SourceFileLoader,
+            self.machinery.SOURCE_SUFFIXES))
+        with open('mod.py', 'w') as file:
+            file.write("# test file for importlib")
+        try:
+            loader = self._find(finder, 'mod', loader_only=True)
+            self.assertTrue(hasattr(loader, 'load_module'))
+        finally:
+            os.unlink('mod.py')
+
+    def test_invalidate_caches(self):
+        # invalidate_caches() should reset the mtime.
+        finder = self.machinery.FileFinder('', (self.machinery.SourceFileLoader,
+            self.machinery.SOURCE_SUFFIXES))
+        finder._path_mtime = 42
+        finder.invalidate_caches()
+        self.assertEqual(finder._path_mtime, -1)
+
+    # Regression test for http://bugs.python.org/issue14846
+    def test_dir_removal_handling(self):
+        mod = 'mod'
+        with util.create_modules(mod) as mapping:
+            finder = self.get_finder(mapping['.root'])
+            found = self._find(finder, 'mod', loader_only=True)
+            self.assertIsNotNone(found)
+        found = self._find(finder, 'mod', loader_only=True)
+        self.assertIsNone(found)
+
+    @unittest.skipUnless(sys.platform != 'win32',
+            'os.chmod() does not support the needed arguments under Windows')
+    def test_no_read_directory(self):
+        # Issue #16730
+        tempdir = tempfile.TemporaryDirectory()
+        original_mode = os.stat(tempdir.name).st_mode
+        def cleanup(tempdir):
+            """Cleanup function for the temporary directory.
+
+            Since we muck with the permissions, we want to set them back to
+            their original values to make sure the directory can be properly
+            cleaned up.
+
+            """
+            os.chmod(tempdir.name, original_mode)
+            # If this is not explicitly called then the __del__ method is used,
+            # but since already mucking around might as well explicitly clean
+            # up.
+            tempdir.__exit__(None, None, None)
+        self.addCleanup(cleanup, tempdir)
+        os.chmod(tempdir.name, stat.S_IWUSR | stat.S_IXUSR)
+        finder = self.get_finder(tempdir.name)
+        found = self._find(finder, 'doesnotexist')
+        self.assertEqual(found, self.NOT_FOUND)
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_ignore_file(self):
+        # If a directory got changed to a file from underneath us, then don't
+        # worry about looking for submodules.
+        with tempfile.NamedTemporaryFile() as file_obj:
+            finder = self.get_finder(file_obj.name)
+            found = self._find(finder, 'doesnotexist')
+            self.assertEqual(found, self.NOT_FOUND)
+
+
+class FinderTestsPEP451(FinderTests):
+
+    NOT_FOUND = None
+
+    def _find(self, finder, name, loader_only=False):
+        spec = finder.find_spec(name)
+        return spec.loader if spec is not None else spec
+
+
+(Frozen_FinderTestsPEP451,
+ Source_FinderTestsPEP451
+ ) = util.test_both(FinderTestsPEP451, machinery=machinery)
+
+
+class FinderTestsPEP420(FinderTests):
+
+    NOT_FOUND = (None, [])
+
+    def _find(self, finder, name, loader_only=False):
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            loader_portions = finder.find_loader(name)
+            return loader_portions[0] if loader_only else loader_portions
+
+
+(Frozen_FinderTestsPEP420,
+ Source_FinderTestsPEP420
+ ) = util.test_both(FinderTestsPEP420, machinery=machinery)
+
+
+class FinderTestsPEP302(FinderTests):
+
+    NOT_FOUND = None
+
+    def _find(self, finder, name, loader_only=False):
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            return finder.find_module(name)
+
+
+(Frozen_FinderTestsPEP302,
+ Source_FinderTestsPEP302
+ ) = util.test_both(FinderTestsPEP302, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/source/test_path_hook.py

@@ -0,0 +1,41 @@
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import unittest
+
+
+class PathHookTest:
+
+    """Test the path hook for source."""
+
+    def path_hook(self):
+        return self.machinery.FileFinder.path_hook((self.machinery.SourceFileLoader,
+            self.machinery.SOURCE_SUFFIXES))
+
+    def test_success(self):
+        with util.create_modules('dummy') as mapping:
+            self.assertTrue(hasattr(self.path_hook()(mapping['.root']),
+                                    'find_spec'))
+
+    def test_success_legacy(self):
+        with util.create_modules('dummy') as mapping:
+            self.assertTrue(hasattr(self.path_hook()(mapping['.root']),
+                                    'find_module'))
+
+    def test_empty_string(self):
+        # The empty string represents the cwd.
+        self.assertTrue(hasattr(self.path_hook()(''), 'find_spec'))
+
+    def test_empty_string_legacy(self):
+        # The empty string represents the cwd.
+        self.assertTrue(hasattr(self.path_hook()(''), 'find_module'))
+
+
+(Frozen_PathHookTest,
+ Source_PathHooktest
+ ) = util.test_both(PathHookTest, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/source/test_source_encoding.py

@@ -0,0 +1,181 @@
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+import codecs
+import importlib.util
+import re
+import types
+# Because sys.path gets essentially blanked, need to have unicodedata already
+# imported for the parser to use.
+import unicodedata
+import unittest
+import warnings
+
+
+CODING_RE = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII)
+
+
+class EncodingTest:
+
+    """PEP 3120 makes UTF-8 the default encoding for source code
+    [default encoding].
+
+    PEP 263 specifies how that can change on a per-file basis. Either the first
+    or second line can contain the encoding line [encoding first line]
+    encoding second line]. If the file has the BOM marker it is considered UTF-8
+    implicitly [BOM]. If any encoding is specified it must be UTF-8, else it is
+    an error [BOM and utf-8][BOM conflict].
+
+    """
+
+    variable = '\u00fc'
+    character = '\u00c9'
+    source_line = "{0} = '{1}'\n".format(variable, character)
+    module_name = '_temp'
+
+    def run_test(self, source):
+        with util.create_modules(self.module_name) as mapping:
+            with open(mapping[self.module_name], 'wb') as file:
+                file.write(source)
+            loader = self.machinery.SourceFileLoader(self.module_name,
+                                                  mapping[self.module_name])
+            return self.load(loader)
+
+    def create_source(self, encoding):
+        encoding_line = "# coding={0}".format(encoding)
+        assert CODING_RE.match(encoding_line)
+        source_lines = [encoding_line.encode('utf-8')]
+        source_lines.append(self.source_line.encode(encoding))
+        return b'\n'.join(source_lines)
+
+    def test_non_obvious_encoding(self):
+        # Make sure that an encoding that has never been a standard one for
+        # Python works.
+        encoding_line = "# coding=koi8-r"
+        assert CODING_RE.match(encoding_line)
+        source = "{0}\na=42\n".format(encoding_line).encode("koi8-r")
+        self.run_test(source)
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    # [default encoding]
+    def test_default_encoding(self):
+        self.run_test(self.source_line.encode('utf-8'))
+
+    # [encoding first line]
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_encoding_on_first_line(self):
+        encoding = 'Latin-1'
+        source = self.create_source(encoding)
+        self.run_test(source)
+
+    # [encoding second line]
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_encoding_on_second_line(self):
+        source = b"#/usr/bin/python\n" + self.create_source('Latin-1')
+        self.run_test(source)
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    # [BOM]
+    def test_bom(self):
+        self.run_test(codecs.BOM_UTF8 + self.source_line.encode('utf-8'))
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    # [BOM and utf-8]
+    def test_bom_and_utf_8(self):
+        source = codecs.BOM_UTF8 + self.create_source('utf-8')
+        self.run_test(source)
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    # [BOM conflict]
+    def test_bom_conflict(self):
+        source = codecs.BOM_UTF8 + self.create_source('latin-1')
+        with self.assertRaises(SyntaxError):
+            self.run_test(source)
+
+
+class EncodingTestPEP451(EncodingTest):
+
+    def load(self, loader):
+        module = types.ModuleType(self.module_name)
+        module.__spec__ = importlib.util.spec_from_loader(self.module_name, loader)
+        loader.exec_module(module)
+        return module
+
+
+(Frozen_EncodingTestPEP451,
+ Source_EncodingTestPEP451
+ ) = util.test_both(EncodingTestPEP451, machinery=machinery)
+
+
+class EncodingTestPEP302(EncodingTest):
+
+    def load(self, loader):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', DeprecationWarning)
+            return loader.load_module(self.module_name)
+
+
+(Frozen_EncodingTestPEP302,
+ Source_EncodingTestPEP302
+ ) = util.test_both(EncodingTestPEP302, machinery=machinery)
+
+
+class LineEndingTest:
+
+    r"""Source written with the three types of line endings (\n, \r\n, \r)
+    need to be readable [cr][crlf][lf]."""
+
+    def run_test(self, line_ending):
+        module_name = '_temp'
+        source_lines = [b"a = 42", b"b = -13", b'']
+        source = line_ending.join(source_lines)
+        with util.create_modules(module_name) as mapping:
+            with open(mapping[module_name], 'wb') as file:
+                file.write(source)
+            loader = self.machinery.SourceFileLoader(module_name,
+                                                     mapping[module_name])
+            return self.load(loader, module_name)
+
+    # [cr]
+    def test_cr(self):
+        self.run_test(b'\r')
+
+    # [crlf]
+    def test_crlf(self):
+        self.run_test(b'\r\n')
+
+    # [lf]
+    def test_lf(self):
+        self.run_test(b'\n')
+
+
+class LineEndingTestPEP451(LineEndingTest):
+
+    def load(self, loader, module_name):
+        module = types.ModuleType(module_name)
+        module.__spec__ = importlib.util.spec_from_loader(module_name, loader)
+        loader.exec_module(module)
+        return module
+
+
+(Frozen_LineEndingTestPEP451,
+ Source_LineEndingTestPEP451
+ ) = util.test_both(LineEndingTestPEP451, machinery=machinery)
+
+
+class LineEndingTestPEP302(LineEndingTest):
+
+    def load(self, loader, module_name):
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', DeprecationWarning)
+            return loader.load_module(module_name)
+
+
+(Frozen_LineEndingTestPEP302,
+ Source_LineEndingTestPEP302
+ ) = util.test_both(LineEndingTestPEP302, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/test_api.py

@@ -0,0 +1,465 @@
+from . import util as test_util
+
+init = test_util.import_importlib('importlib')
+util = test_util.import_importlib('importlib.util')
+machinery = test_util.import_importlib('importlib.machinery')
+
+import os.path
+import sys
+from test import support
+import types
+import unittest
+import warnings
+
+
+class ImportModuleTests:
+
+    """Test importlib.import_module."""
+
+    def test_module_import(self):
+        # Test importing a top-level module.
+        with test_util.mock_modules('top_level') as mock:
+            with test_util.import_state(meta_path=[mock]):
+                module = self.init.import_module('top_level')
+                self.assertEqual(module.__name__, 'top_level')
+
+    def test_absolute_package_import(self):
+        # Test importing a module from a package with an absolute name.
+        pkg_name = 'pkg'
+        pkg_long_name = '{0}.__init__'.format(pkg_name)
+        name = '{0}.mod'.format(pkg_name)
+        with test_util.mock_modules(pkg_long_name, name) as mock:
+            with test_util.import_state(meta_path=[mock]):
+                module = self.init.import_module(name)
+                self.assertEqual(module.__name__, name)
+
+    def test_shallow_relative_package_import(self):
+        # Test importing a module from a package through a relative import.
+        pkg_name = 'pkg'
+        pkg_long_name = '{0}.__init__'.format(pkg_name)
+        module_name = 'mod'
+        absolute_name = '{0}.{1}'.format(pkg_name, module_name)
+        relative_name = '.{0}'.format(module_name)
+        with test_util.mock_modules(pkg_long_name, absolute_name) as mock:
+            with test_util.import_state(meta_path=[mock]):
+                self.init.import_module(pkg_name)
+                module = self.init.import_module(relative_name, pkg_name)
+                self.assertEqual(module.__name__, absolute_name)
+
+    def test_deep_relative_package_import(self):
+        modules = ['a.__init__', 'a.b.__init__', 'a.c']
+        with test_util.mock_modules(*modules) as mock:
+            with test_util.import_state(meta_path=[mock]):
+                self.init.import_module('a')
+                self.init.import_module('a.b')
+                module = self.init.import_module('..c', 'a.b')
+                self.assertEqual(module.__name__, 'a.c')
+
+    def test_absolute_import_with_package(self):
+        # Test importing a module from a package with an absolute name with
+        # the 'package' argument given.
+        pkg_name = 'pkg'
+        pkg_long_name = '{0}.__init__'.format(pkg_name)
+        name = '{0}.mod'.format(pkg_name)
+        with test_util.mock_modules(pkg_long_name, name) as mock:
+            with test_util.import_state(meta_path=[mock]):
+                self.init.import_module(pkg_name)
+                module = self.init.import_module(name, pkg_name)
+                self.assertEqual(module.__name__, name)
+
+    def test_relative_import_wo_package(self):
+        # Relative imports cannot happen without the 'package' argument being
+        # set.
+        with self.assertRaises(TypeError):
+            self.init.import_module('.support')
+
+
+    def test_loaded_once(self):
+        # Issue #13591: Modules should only be loaded once when
+        # initializing the parent package attempts to import the
+        # module currently being imported.
+        b_load_count = 0
+        def load_a():
+            self.init.import_module('a.b')
+        def load_b():
+            nonlocal b_load_count
+            b_load_count += 1
+        code = {'a': load_a, 'a.b': load_b}
+        modules = ['a.__init__', 'a.b']
+        with test_util.mock_modules(*modules, module_code=code) as mock:
+            with test_util.import_state(meta_path=[mock]):
+                self.init.import_module('a.b')
+        self.assertEqual(b_load_count, 1)
+
+
+(Frozen_ImportModuleTests,
+ Source_ImportModuleTests
+ ) = test_util.test_both(ImportModuleTests, init=init)
+
+
+class FindLoaderTests:
+
+    FakeMetaFinder = None
+
+    def test_sys_modules(self):
+        # If a module with __loader__ is in sys.modules, then return it.
+        name = 'some_mod'
+        with test_util.uncache(name):
+            module = types.ModuleType(name)
+            loader = 'a loader!'
+            module.__loader__ = loader
+            sys.modules[name] = module
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                found = self.init.find_loader(name)
+            self.assertEqual(loader, found)
+
+    def test_sys_modules_loader_is_None(self):
+        # If sys.modules[name].__loader__ is None, raise ValueError.
+        name = 'some_mod'
+        with test_util.uncache(name):
+            module = types.ModuleType(name)
+            module.__loader__ = None
+            sys.modules[name] = module
+            with self.assertRaises(ValueError):
+                with warnings.catch_warnings():
+                    warnings.simplefilter('ignore', DeprecationWarning)
+                    self.init.find_loader(name)
+
+    def test_sys_modules_loader_is_not_set(self):
+        # Should raise ValueError
+        # Issue #17099
+        name = 'some_mod'
+        with test_util.uncache(name):
+            module = types.ModuleType(name)
+            try:
+                del module.__loader__
+            except AttributeError:
+                pass
+            sys.modules[name] = module
+            with self.assertRaises(ValueError):
+                with warnings.catch_warnings():
+                    warnings.simplefilter('ignore', DeprecationWarning)
+                    self.init.find_loader(name)
+
+    def test_success(self):
+        # Return the loader found on sys.meta_path.
+        name = 'some_mod'
+        with test_util.uncache(name):
+            with test_util.import_state(meta_path=[self.FakeMetaFinder]):
+                with warnings.catch_warnings():
+                    warnings.simplefilter('ignore', DeprecationWarning)
+                    self.assertEqual((name, None), self.init.find_loader(name))
+
+    def test_success_path(self):
+        # Searching on a path should work.
+        name = 'some_mod'
+        path = 'path to some place'
+        with test_util.uncache(name):
+            with test_util.import_state(meta_path=[self.FakeMetaFinder]):
+                with warnings.catch_warnings():
+                    warnings.simplefilter('ignore', DeprecationWarning)
+                    self.assertEqual((name, path),
+                                     self.init.find_loader(name, path))
+
+    def test_nothing(self):
+        # None is returned upon failure to find a loader.
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore', DeprecationWarning)
+            self.assertIsNone(self.init.find_loader('nevergoingtofindthismodule'))
+
+
+class FindLoaderPEP451Tests(FindLoaderTests):
+
+    class FakeMetaFinder:
+        @staticmethod
+        def find_spec(name, path=None, target=None):
+            return machinery['Source'].ModuleSpec(name, (name, path))
+
+
+(Frozen_FindLoaderPEP451Tests,
+ Source_FindLoaderPEP451Tests
+ ) = test_util.test_both(FindLoaderPEP451Tests, init=init)
+
+
+class FindLoaderPEP302Tests(FindLoaderTests):
+
+    class FakeMetaFinder:
+        @staticmethod
+        def find_module(name, path=None):
+            return name, path
+
+
+(Frozen_FindLoaderPEP302Tests,
+ Source_FindLoaderPEP302Tests
+ ) = test_util.test_both(FindLoaderPEP302Tests, init=init)
+
+
+class ReloadTests:
+
+    def test_reload_modules(self):
+        for mod in ('tokenize', 'time', 'marshal'):
+            with self.subTest(module=mod):
+                with support.CleanImport(mod):
+                    module = self.init.import_module(mod)
+                    self.init.reload(module)
+
+    def test_module_replaced(self):
+        def code():
+            import sys
+            module = type(sys)('top_level')
+            module.spam = 3
+            sys.modules['top_level'] = module
+        mock = test_util.mock_modules('top_level',
+                                      module_code={'top_level': code})
+        with mock:
+            with test_util.import_state(meta_path=[mock]):
+                module = self.init.import_module('top_level')
+                reloaded = self.init.reload(module)
+                actual = sys.modules['top_level']
+                self.assertEqual(actual.spam, 3)
+                self.assertEqual(reloaded.spam, 3)
+
+    def test_reload_missing_loader(self):
+        with support.CleanImport('types'):
+            import types
+            loader = types.__loader__
+            del types.__loader__
+            reloaded = self.init.reload(types)
+
+            self.assertIs(reloaded, types)
+            self.assertIs(sys.modules['types'], types)
+            self.assertEqual(reloaded.__loader__.path, loader.path)
+
+    def test_reload_loader_replaced(self):
+        with support.CleanImport('types'):
+            import types
+            types.__loader__ = None
+            self.init.invalidate_caches()
+            reloaded = self.init.reload(types)
+
+            self.assertIsNot(reloaded.__loader__, None)
+            self.assertIs(reloaded, types)
+            self.assertIs(sys.modules['types'], types)
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_reload_location_changed(self):
+        name = 'spam'
+        with support.temp_cwd(None) as cwd:
+            with test_util.uncache('spam'):
+                with support.DirsOnSysPath(cwd):
+                    # Start as a plain module.
+                    self.init.invalidate_caches()
+                    path = os.path.join(cwd, name + '.py')
+                    cached = self.util.cache_from_source(path)
+                    expected = {'__name__': name,
+                                '__package__': '',
+                                '__file__': path,
+                                '__cached__': cached,
+                                '__doc__': None,
+                                }
+                    support.create_empty_file(path)
+                    module = self.init.import_module(name)
+                    ns = vars(module).copy()
+                    loader = ns.pop('__loader__')
+                    spec = ns.pop('__spec__')
+                    ns.pop('__builtins__', None)  # An implementation detail.
+                    self.assertEqual(spec.name, name)
+                    self.assertEqual(spec.loader, loader)
+                    self.assertEqual(loader.path, path)
+                    self.assertEqual(ns, expected)
+
+                    # Change to a package.
+                    self.init.invalidate_caches()
+                    init_path = os.path.join(cwd, name, '__init__.py')
+                    cached = self.util.cache_from_source(init_path)
+                    expected = {'__name__': name,
+                                '__package__': name,
+                                '__file__': init_path,
+                                '__cached__': cached,
+                                '__path__': [os.path.dirname(init_path)],
+                                '__doc__': None,
+                                }
+                    os.mkdir(name)
+                    os.rename(path, init_path)
+                    reloaded = self.init.reload(module)
+                    ns = vars(reloaded).copy()
+                    loader = ns.pop('__loader__')
+                    spec = ns.pop('__spec__')
+                    ns.pop('__builtins__', None)  # An implementation detail.
+                    self.assertEqual(spec.name, name)
+                    self.assertEqual(spec.loader, loader)
+                    self.assertIs(reloaded, module)
+                    self.assertEqual(loader.path, init_path)
+                    self.maxDiff = None
+                    self.assertEqual(ns, expected)
+
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
+    def test_reload_namespace_changed(self):
+        name = 'spam'
+        with support.temp_cwd(None) as cwd:
+            with test_util.uncache('spam'):
+                with support.DirsOnSysPath(cwd):
+                    # Start as a namespace package.
+                    self.init.invalidate_caches()
+                    bad_path = os.path.join(cwd, name, '__init.py')
+                    cached = self.util.cache_from_source(bad_path)
+                    expected = {'__name__': name,
+                                '__package__': name,
+                                '__doc__': None,
+                                '__file__': None,
+                                }
+                    os.mkdir(name)
+                    with open(bad_path, 'w') as init_file:
+                        init_file.write('eggs = None')
+                    module = self.init.import_module(name)
+                    ns = vars(module).copy()
+                    loader = ns.pop('__loader__')
+                    path = ns.pop('__path__')
+                    spec = ns.pop('__spec__')
+                    ns.pop('__builtins__', None)  # An implementation detail.
+                    self.assertEqual(spec.name, name)
+                    self.assertIsNotNone(spec.loader)
+                    self.assertIsNotNone(loader)
+                    self.assertEqual(spec.loader, loader)
+                    self.assertEqual(set(path),
+                                     set([os.path.dirname(bad_path)]))
+                    with self.assertRaises(AttributeError):
+                        # a NamespaceLoader
+                        loader.path
+                    self.assertEqual(ns, expected)
+
+                    # Change to a regular package.
+                    self.init.invalidate_caches()
+                    init_path = os.path.join(cwd, name, '__init__.py')
+                    cached = self.util.cache_from_source(init_path)
+                    expected = {'__name__': name,
+                                '__package__': name,
+                                '__file__': init_path,
+                                '__cached__': cached,
+                                '__path__': [os.path.dirname(init_path)],
+                                '__doc__': None,
+                                'eggs': None,
+                                }
+                    os.rename(bad_path, init_path)
+                    reloaded = self.init.reload(module)
+                    ns = vars(reloaded).copy()
+                    loader = ns.pop('__loader__')
+                    spec = ns.pop('__spec__')
+                    ns.pop('__builtins__', None)  # An implementation detail.
+                    self.assertEqual(spec.name, name)
+                    self.assertEqual(spec.loader, loader)
+                    self.assertIs(reloaded, module)
+                    self.assertEqual(loader.path, init_path)
+                    self.assertEqual(ns, expected)
+
+    def test_reload_submodule(self):
+        # See #19851.
+        name = 'spam'
+        subname = 'ham'
+        with test_util.temp_module(name, pkg=True) as pkg_dir:
+            fullname, _ = test_util.submodule(name, subname, pkg_dir)
+            ham = self.init.import_module(fullname)
+            reloaded = self.init.reload(ham)
+            self.assertIs(reloaded, ham)
+
+    def test_module_missing_spec(self):
+        #Test that reload() throws ModuleNotFounderror when reloading
+        # a module who's missing a spec. (bpo-29851)
+        name = 'spam'
+        with test_util.uncache(name):
+            module = sys.modules[name] = types.ModuleType(name)
+            # Sanity check by attempting an import.
+            module = self.init.import_module(name)
+            self.assertIsNone(module.__spec__)
+            with self.assertRaises(ModuleNotFoundError):
+                self.init.reload(module)
+
+
+(Frozen_ReloadTests,
+ Source_ReloadTests
+ ) = test_util.test_both(ReloadTests, init=init, util=util)
+
+
+class InvalidateCacheTests:
+
+    def test_method_called(self):
+        # If defined the method should be called.
+        class InvalidatingNullFinder:
+            def __init__(self, *ignored):
+                self.called = False
+            def find_module(self, *args):
+                return None
+            def invalidate_caches(self):
+                self.called = True
+
+        key = 'gobledeegook'
+        meta_ins = InvalidatingNullFinder()
+        path_ins = InvalidatingNullFinder()
+        sys.meta_path.insert(0, meta_ins)
+        self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key))
+        sys.path_importer_cache[key] = path_ins
+        self.addCleanup(lambda: sys.meta_path.remove(meta_ins))
+        self.init.invalidate_caches()
+        self.assertTrue(meta_ins.called)
+        self.assertTrue(path_ins.called)
+
+    def test_method_lacking(self):
+        # There should be no issues if the method is not defined.
+        key = 'gobbledeegook'
+        sys.path_importer_cache[key] = None
+        self.addCleanup(lambda: sys.path_importer_cache.pop(key, None))
+        self.init.invalidate_caches()  # Shouldn't trigger an exception.
+
+
+(Frozen_InvalidateCacheTests,
+ Source_InvalidateCacheTests
+ ) = test_util.test_both(InvalidateCacheTests, init=init)
+
+
+class FrozenImportlibTests(unittest.TestCase):
+
+    def test_no_frozen_importlib(self):
+        # Should be able to import w/o _frozen_importlib being defined.
+        # Can't do an isinstance() check since separate copies of importlib
+        # may have been used for import, so just check the name is not for the
+        # frozen loader.
+        source_init = init['Source']
+        self.assertNotEqual(source_init.__loader__.__class__.__name__,
+                            'FrozenImporter')
+
+
+class StartupTests:
+
+    def test_everyone_has___loader__(self):
+        # Issue #17098: all modules should have __loader__ defined.
+        for name, module in sys.modules.items():
+            if isinstance(module, types.ModuleType):
+                with self.subTest(name=name):
+                    self.assertTrue(hasattr(module, '__loader__'),
+                                    '{!r} lacks a __loader__ attribute'.format(name))
+                    if self.machinery.BuiltinImporter.find_module(name):
+                        self.assertIsNot(module.__loader__, None)
+                    elif self.machinery.FrozenImporter.find_module(name):
+                        self.assertIsNot(module.__loader__, None)
+
+    def test_everyone_has___spec__(self):
+        for name, module in sys.modules.items():
+            if isinstance(module, types.ModuleType):
+                with self.subTest(name=name):
+                    self.assertTrue(hasattr(module, '__spec__'))
+                    if self.machinery.BuiltinImporter.find_module(name):
+                        self.assertIsNot(module.__spec__, None)
+                    elif self.machinery.FrozenImporter.find_module(name):
+                        self.assertIsNot(module.__spec__, None)
+
+
+(Frozen_StartupTests,
+ Source_StartupTests
+ ) = test_util.test_both(StartupTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/test_lazy.py

@@ -0,0 +1,146 @@
+import importlib
+from importlib import abc
+from importlib import util
+import sys
+import types
+import unittest
+
+from . import util as test_util
+
+
+class CollectInit:
+
+    def __init__(self, *args, **kwargs):
+        self.args = args
+        self.kwargs = kwargs
+
+    def exec_module(self, module):
+        return self
+
+
+class LazyLoaderFactoryTests(unittest.TestCase):
+
+    def test_init(self):
+        factory = util.LazyLoader.factory(CollectInit)
+        # E.g. what importlib.machinery.FileFinder instantiates loaders with
+        # plus keyword arguments.
+        lazy_loader = factory('module name', 'module path', kw='kw')
+        loader = lazy_loader.loader
+        self.assertEqual(('module name', 'module path'), loader.args)
+        self.assertEqual({'kw': 'kw'}, loader.kwargs)
+
+    def test_validation(self):
+        # No exec_module(), no lazy loading.
+        with self.assertRaises(TypeError):
+            util.LazyLoader.factory(object)
+
+
+class TestingImporter(abc.MetaPathFinder, abc.Loader):
+
+    module_name = 'lazy_loader_test'
+    mutated_name = 'changed'
+    loaded = None
+    source_code = 'attr = 42; __name__ = {!r}'.format(mutated_name)
+
+    def find_spec(self, name, path, target=None):
+        if name != self.module_name:
+            return None
+        return util.spec_from_loader(name, util.LazyLoader(self))
+
+    def exec_module(self, module):
+        exec(self.source_code, module.__dict__)
+        self.loaded = module
+
+
+@unittest.skip("TODO: RUSTPYTHON")
+class LazyLoaderTests(unittest.TestCase):
+
+    def test_init(self):
+        with self.assertRaises(TypeError):
+            # Classes that don't define exec_module() trigger TypeError.
+            util.LazyLoader(object)
+
+    def new_module(self, source_code=None):
+        loader = TestingImporter()
+        if source_code is not None:
+            loader.source_code = source_code
+        spec = util.spec_from_loader(TestingImporter.module_name,
+                                     util.LazyLoader(loader))
+        module = spec.loader.create_module(spec)
+        if module is None:
+            module = types.ModuleType(TestingImporter.module_name)
+        module.__spec__ = spec
+        module.__loader__ = spec.loader
+        spec.loader.exec_module(module)
+        # Module is now lazy.
+        self.assertIsNone(loader.loaded)
+        return module
+
+    def test_e2e(self):
+        # End-to-end test to verify the load is in fact lazy.
+        importer = TestingImporter()
+        assert importer.loaded is None
+        with test_util.uncache(importer.module_name):
+            with test_util.import_state(meta_path=[importer]):
+                module = importlib.import_module(importer.module_name)
+        self.assertIsNone(importer.loaded)
+        # Trigger load.
+        self.assertEqual(module.__loader__, importer)
+        self.assertIsNotNone(importer.loaded)
+        self.assertEqual(module, importer.loaded)
+
+    def test_attr_unchanged(self):
+        # An attribute only mutated as a side-effect of import should not be
+        # changed needlessly.
+        module = self.new_module()
+        self.assertEqual(TestingImporter.mutated_name, module.__name__)
+
+    def test_new_attr(self):
+        # A new attribute should persist.
+        module = self.new_module()
+        module.new_attr = 42
+        self.assertEqual(42, module.new_attr)
+
+    def test_mutated_preexisting_attr(self):
+        # Changing an attribute that already existed on the module --
+        # e.g. __name__ -- should persist.
+        module = self.new_module()
+        module.__name__ = 'bogus'
+        self.assertEqual('bogus', module.__name__)
+
+    def test_mutated_attr(self):
+        # Changing an attribute that comes into existence after an import
+        # should persist.
+        module = self.new_module()
+        module.attr = 6
+        self.assertEqual(6, module.attr)
+
+    def test_delete_eventual_attr(self):
+        # Deleting an attribute should stay deleted.
+        module = self.new_module()
+        del module.attr
+        self.assertFalse(hasattr(module, 'attr'))
+
+    def test_delete_preexisting_attr(self):
+        module = self.new_module()
+        del module.__name__
+        self.assertFalse(hasattr(module, '__name__'))
+
+    def test_module_substitution_error(self):
+        with test_util.uncache(TestingImporter.module_name):
+            fresh_module = types.ModuleType(TestingImporter.module_name)
+            sys.modules[TestingImporter.module_name] = fresh_module
+            module = self.new_module()
+            with self.assertRaisesRegex(ValueError, "substituted"):
+                module.__name__
+
+    def test_module_already_in_sys(self):
+        with test_util.uncache(TestingImporter.module_name):
+            module = self.new_module()
+            sys.modules[TestingImporter.module_name] = module
+            # Force the load; just care that no exception is raised.
+            module.__name__
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/test_locks.py

@@ -0,0 +1,153 @@
+from . import util as test_util
+
+init = test_util.import_importlib('importlib')
+
+import sys
+import threading
+import weakref
+
+from test import support
+from test import lock_tests
+
+
+class ModuleLockAsRLockTests:
+    locktype = classmethod(lambda cls: cls.LockType("some_lock"))
+
+    # _is_owned() unsupported
+    test__is_owned = None
+    # acquire(blocking=False) unsupported
+    test_try_acquire = None
+    test_try_acquire_contended = None
+    # `with` unsupported
+    test_with = None
+    # acquire(timeout=...) unsupported
+    test_timeout = None
+    # _release_save() unsupported
+    test_release_save_unacquired = None
+    # lock status in repr unsupported
+    test_repr = None
+    test_locked_repr = None
+
+LOCK_TYPES = {kind: splitinit._bootstrap._ModuleLock
+              for kind, splitinit in init.items()}
+
+(Frozen_ModuleLockAsRLockTests,
+ Source_ModuleLockAsRLockTests
+ ) = test_util.test_both(ModuleLockAsRLockTests, lock_tests.RLockTests,
+                         LockType=LOCK_TYPES)
+
+
+class DeadlockAvoidanceTests:
+
+    def setUp(self):
+        try:
+            self.old_switchinterval = sys.getswitchinterval()
+            support.setswitchinterval(0.000001)
+        except AttributeError:
+            self.old_switchinterval = None
+
+    def tearDown(self):
+        if self.old_switchinterval is not None:
+            sys.setswitchinterval(self.old_switchinterval)
+
+    def run_deadlock_avoidance_test(self, create_deadlock):
+        NLOCKS = 10
+        locks = [self.LockType(str(i)) for i in range(NLOCKS)]
+        pairs = [(locks[i], locks[(i+1)%NLOCKS]) for i in range(NLOCKS)]
+        if create_deadlock:
+            NTHREADS = NLOCKS
+        else:
+            NTHREADS = NLOCKS - 1
+        barrier = threading.Barrier(NTHREADS)
+        results = []
+
+        def _acquire(lock):
+            """Try to acquire the lock. Return True on success,
+            False on deadlock."""
+            try:
+                lock.acquire()
+            except self.DeadlockError:
+                return False
+            else:
+                return True
+
+        def f():
+            a, b = pairs.pop()
+            ra = _acquire(a)
+            barrier.wait()
+            rb = _acquire(b)
+            results.append((ra, rb))
+            if rb:
+                b.release()
+            if ra:
+                a.release()
+        lock_tests.Bunch(f, NTHREADS).wait_for_finished()
+        self.assertEqual(len(results), NTHREADS)
+        return results
+
+    def test_deadlock(self):
+        results = self.run_deadlock_avoidance_test(True)
+        # At least one of the threads detected a potential deadlock on its
+        # second acquire() call.  It may be several of them, because the
+        # deadlock avoidance mechanism is conservative.
+        nb_deadlocks = results.count((True, False))
+        self.assertGreaterEqual(nb_deadlocks, 1)
+        self.assertEqual(results.count((True, True)), len(results) - nb_deadlocks)
+
+    def test_no_deadlock(self):
+        results = self.run_deadlock_avoidance_test(False)
+        self.assertEqual(results.count((True, False)), 0)
+        self.assertEqual(results.count((True, True)), len(results))
+
+
+DEADLOCK_ERRORS = {kind: splitinit._bootstrap._DeadlockError
+                   for kind, splitinit in init.items()}
+
+(Frozen_DeadlockAvoidanceTests,
+ Source_DeadlockAvoidanceTests
+ ) = test_util.test_both(DeadlockAvoidanceTests,
+                         LockType=LOCK_TYPES,
+                         DeadlockError=DEADLOCK_ERRORS)
+
+
+class LifetimeTests:
+
+    @property
+    def bootstrap(self):
+        return self.init._bootstrap
+
+    def test_lock_lifetime(self):
+        name = "xyzzy"
+        self.assertNotIn(name, self.bootstrap._module_locks)
+        lock = self.bootstrap._get_module_lock(name)
+        self.assertIn(name, self.bootstrap._module_locks)
+        wr = weakref.ref(lock)
+        del lock
+        support.gc_collect()
+        self.assertNotIn(name, self.bootstrap._module_locks)
+        self.assertIsNone(wr())
+
+    def test_all_locks(self):
+        support.gc_collect()
+        self.assertEqual(0, len(self.bootstrap._module_locks),
+                         self.bootstrap._module_locks)
+
+# TODO: RustPython
+# (Frozen_LifetimeTests,
+#  Source_LifetimeTests
+#  ) = test_util.test_both(LifetimeTests, init=init)
+
+
+@support.reap_threads
+def test_main():
+    support.run_unittest(Frozen_ModuleLockAsRLockTests,
+                         Source_ModuleLockAsRLockTests,
+                         Frozen_DeadlockAvoidanceTests,
+                         Source_DeadlockAvoidanceTests,
+                         # Frozen_LifetimeTests,
+                         # Source_LifetimeTests
+                         )
+
+
+if __name__ == '__main__':
+    test_main()

Lib/test/test_importlib/test_namespace_pkgs.py

@@ -0,0 +1,343 @@
+import contextlib
+import importlib
+import os
+import sys
+import unittest
+
+from test.test_importlib import util
+
+# needed tests:
+#
+# need to test when nested, so that the top-level path isn't sys.path
+# need to test dynamic path detection, both at top-level and nested
+# with dynamic path, check when a loader is returned on path reload (that is,
+#  trying to switch from a namespace package to a regular package)
+
+
+@contextlib.contextmanager
+def sys_modules_context():
+    """
+    Make sure sys.modules is the same object and has the same content
+    when exiting the context as when entering.
+
+    Similar to importlib.test.util.uncache, but doesn't require explicit
+    names.
+    """
+    sys_modules_saved = sys.modules
+    sys_modules_copy = sys.modules.copy()
+    try:
+        yield
+    finally:
+        sys.modules = sys_modules_saved
+        sys.modules.clear()
+        sys.modules.update(sys_modules_copy)
+
+
+@contextlib.contextmanager
+def namespace_tree_context(**kwargs):
+    """
+    Save import state and sys.modules cache and restore it on exit.
+    Typical usage:
+
+    >>> with namespace_tree_context(path=['/tmp/xxyy/portion1',
+    ...         '/tmp/xxyy/portion2']):
+    ...     pass
+    """
+    # use default meta_path and path_hooks unless specified otherwise
+    kwargs.setdefault('meta_path', sys.meta_path)
+    kwargs.setdefault('path_hooks', sys.path_hooks)
+    import_context = util.import_state(**kwargs)
+    with import_context, sys_modules_context():
+        yield
+
+class NamespacePackageTest(unittest.TestCase):
+    """
+    Subclasses should define self.root and self.paths (under that root)
+    to be added to sys.path.
+    """
+    root = os.path.join(os.path.dirname(__file__), 'namespace_pkgs')
+
+    def setUp(self):
+        self.resolved_paths = [
+            os.path.join(self.root, path) for path in self.paths
+        ]
+        self.ctx = namespace_tree_context(path=self.resolved_paths)
+        self.ctx.__enter__()
+
+    def tearDown(self):
+        # TODO: will we ever want to pass exc_info to __exit__?
+        self.ctx.__exit__(None, None, None)
+
+
+class SingleNamespacePackage(NamespacePackageTest):
+    paths = ['portion1']
+
+    def test_simple_package(self):
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+    def test_cant_import_other(self):
+        with self.assertRaises(ImportError):
+            import foo.two
+
+    def test_module_repr(self):
+        import foo.one
+        self.assertEqual(repr(foo), "<module 'foo' (namespace)>")
+
+
+class DynamicPathNamespacePackage(NamespacePackageTest):
+    paths = ['portion1']
+
+    def test_dynamic_path(self):
+        # Make sure only 'foo.one' can be imported
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+        with self.assertRaises(ImportError):
+            import foo.two
+
+        # Now modify sys.path
+        sys.path.append(os.path.join(self.root, 'portion2'))
+
+        # And make sure foo.two is now importable
+        import foo.two
+        self.assertEqual(foo.two.attr, 'portion2 foo two')
+
+
+class CombinedNamespacePackages(NamespacePackageTest):
+    paths = ['both_portions']
+
+    def test_imports(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'both_portions foo one')
+        self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+
+class SeparatedNamespacePackages(NamespacePackageTest):
+    paths = ['portion1', 'portion2']
+
+    def test_imports(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'portion2 foo two')
+
+
+class SeparatedOverlappingNamespacePackages(NamespacePackageTest):
+    paths = ['portion1', 'both_portions']
+
+    def test_first_path_wins(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+    def test_first_path_wins_again(self):
+        sys.path.reverse()
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'both_portions foo one')
+        self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+    def test_first_path_wins_importing_second_first(self):
+        import foo.two
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'both_portions foo two')
+
+
+class SingleZipNamespacePackage(NamespacePackageTest):
+    paths = ['top_level_portion1.zip']
+
+    def test_simple_package(self):
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+    def test_cant_import_other(self):
+        with self.assertRaises(ImportError):
+            import foo.two
+
+
+class SeparatedZipNamespacePackages(NamespacePackageTest):
+    paths = ['top_level_portion1.zip', 'portion2']
+
+    def test_imports(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'portion2 foo two')
+        self.assertIn('top_level_portion1.zip', foo.one.__file__)
+        self.assertNotIn('.zip', foo.two.__file__)
+
+
+class SingleNestedZipNamespacePackage(NamespacePackageTest):
+    paths = ['nested_portion1.zip/nested_portion1']
+
+    def test_simple_package(self):
+        import foo.one
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+    def test_cant_import_other(self):
+        with self.assertRaises(ImportError):
+            import foo.two
+
+
+class SeparatedNestedZipNamespacePackages(NamespacePackageTest):
+    paths = ['nested_portion1.zip/nested_portion1', 'portion2']
+
+    def test_imports(self):
+        import foo.one
+        import foo.two
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+        self.assertEqual(foo.two.attr, 'portion2 foo two')
+        fn = os.path.join('nested_portion1.zip', 'nested_portion1')
+        self.assertIn(fn, foo.one.__file__)
+        self.assertNotIn('.zip', foo.two.__file__)
+
+
+class LegacySupport(NamespacePackageTest):
+    paths = ['not_a_namespace_pkg', 'portion1', 'portion2', 'both_portions']
+
+    def test_non_namespace_package_takes_precedence(self):
+        import foo.one
+        with self.assertRaises(ImportError):
+            import foo.two
+        self.assertIn('__init__', foo.__file__)
+        self.assertNotIn('namespace', str(foo.__loader__).lower())
+
+
+class DynamicPathCalculation(NamespacePackageTest):
+    paths = ['project1', 'project2']
+
+    def test_project3_fails(self):
+        import parent.child.one
+        self.assertEqual(len(parent.__path__), 2)
+        self.assertEqual(len(parent.child.__path__), 2)
+        import parent.child.two
+        self.assertEqual(len(parent.__path__), 2)
+        self.assertEqual(len(parent.child.__path__), 2)
+
+        self.assertEqual(parent.child.one.attr, 'parent child one')
+        self.assertEqual(parent.child.two.attr, 'parent child two')
+
+        with self.assertRaises(ImportError):
+            import parent.child.three
+
+        self.assertEqual(len(parent.__path__), 2)
+        self.assertEqual(len(parent.child.__path__), 2)
+
+    def test_project3_succeeds(self):
+        import parent.child.one
+        self.assertEqual(len(parent.__path__), 2)
+        self.assertEqual(len(parent.child.__path__), 2)
+        import parent.child.two
+        self.assertEqual(len(parent.__path__), 2)
+        self.assertEqual(len(parent.child.__path__), 2)
+
+        self.assertEqual(parent.child.one.attr, 'parent child one')
+        self.assertEqual(parent.child.two.attr, 'parent child two')
+
+        with self.assertRaises(ImportError):
+            import parent.child.three
+
+        # now add project3
+        sys.path.append(os.path.join(self.root, 'project3'))
+        import parent.child.three
+
+        # the paths dynamically get longer, to include the new directories
+        self.assertEqual(len(parent.__path__), 3)
+        self.assertEqual(len(parent.child.__path__), 3)
+
+        self.assertEqual(parent.child.three.attr, 'parent child three')
+
+
+class ZipWithMissingDirectory(NamespacePackageTest):
+    paths = ['missing_directory.zip']
+
+    @unittest.expectedFailure
+    def test_missing_directory(self):
+        # This will fail because missing_directory.zip contains:
+        #   Length      Date    Time    Name
+        # ---------  ---------- -----   ----
+        #        29  2012-05-03 18:13   foo/one.py
+        #         0  2012-05-03 20:57   bar/
+        #        38  2012-05-03 20:57   bar/two.py
+        # ---------                     -------
+        #        67                     3 files
+
+        # Because there is no 'foo/', the zipimporter currently doesn't
+        #  know that foo is a namespace package
+
+        import foo.one
+
+    def test_present_directory(self):
+        # This succeeds because there is a "bar/" in the zip file
+        import bar.two
+        self.assertEqual(bar.two.attr, 'missing_directory foo two')
+
+
+class ModuleAndNamespacePackageInSameDir(NamespacePackageTest):
+    paths = ['module_and_namespace_package']
+
+    def test_module_before_namespace_package(self):
+        # Make sure we find the module in preference to the
+        #  namespace package.
+        import a_test
+        self.assertEqual(a_test.attr, 'in module')
+
+
+class ReloadTests(NamespacePackageTest):
+    paths = ['portion1']
+
+    def test_simple_package(self):
+        import foo.one
+        foo = importlib.reload(foo)
+        self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+    def test_cant_import_other(self):
+        import foo
+        with self.assertRaises(ImportError):
+            import foo.two
+        foo = importlib.reload(foo)
+        with self.assertRaises(ImportError):
+            import foo.two
+
+    def test_dynamic_path(self):
+        import foo.one
+        with self.assertRaises(ImportError):
+            import foo.two
+
+        # Now modify sys.path and reload.
+        sys.path.append(os.path.join(self.root, 'portion2'))
+        foo = importlib.reload(foo)
+
+        # And make sure foo.two is now importable
+        import foo.two
+        self.assertEqual(foo.two.attr, 'portion2 foo two')
+
+
+class LoaderTests(NamespacePackageTest):
+    paths = ['portion1']
+
+    def test_namespace_loader_consistency(self):
+        # bpo-32303
+        import foo
+        self.assertEqual(foo.__loader__, foo.__spec__.loader)
+        self.assertIsNotNone(foo.__loader__)
+
+    def test_namespace_origin_consistency(self):
+        # bpo-32305
+        import foo
+        self.assertIsNone(foo.__spec__.origin)
+        self.assertIsNone(foo.__file__)
+
+    def test_path_indexable(self):
+        # bpo-35843
+        import foo
+        expected_path = os.path.join(self.root, 'portion1', 'foo')
+        self.assertEqual(foo.__path__[0], expected_path)
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_importlib/test_open.py

@@ -0,0 +1,75 @@
+import unittest
+
+from importlib import resources
+from . import data01
+from . import util
+
+
+class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase):
+    def execute(self, package, path):
+        with resources.open_binary(package, path):
+            pass
+
+
+class CommonTextTests(util.CommonResourceTests, unittest.TestCase):
+    def execute(self, package, path):
+        with resources.open_text(package, path):
+            pass
+
+
+class OpenTests:
+    def test_open_binary(self):
+        with resources.open_binary(self.data, 'binary.file') as fp:
+            result = fp.read()
+            self.assertEqual(result, b'\x00\x01\x02\x03')
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_open_text_default_encoding(self):
+        with resources.open_text(self.data, 'utf-8.file') as fp:
+            result = fp.read()
+            self.assertEqual(result, 'Hello, UTF-8 world!\n')
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_open_text_given_encoding(self):
+        with resources.open_text(
+                self.data, 'utf-16.file', 'utf-16', 'strict') as fp:
+            result = fp.read()
+        self.assertEqual(result, 'Hello, UTF-16 world!\n')
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_open_text_with_errors(self):
+        # Raises UnicodeError without the 'errors' argument.
+        with resources.open_text(
+                self.data, 'utf-16.file', 'utf-8', 'strict') as fp:
+            self.assertRaises(UnicodeError, fp.read)
+        with resources.open_text(
+                self.data, 'utf-16.file', 'utf-8', 'ignore') as fp:
+            result = fp.read()
+        self.assertEqual(
+            result,
+            'H\x00e\x00l\x00l\x00o\x00,\x00 '
+            '\x00U\x00T\x00F\x00-\x001\x006\x00 '
+            '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00')
+
+    def test_open_binary_FileNotFoundError(self):
+        self.assertRaises(
+            FileNotFoundError,
+            resources.open_binary, self.data, 'does-not-exist')
+
+    def test_open_text_FileNotFoundError(self):
+        self.assertRaises(
+            FileNotFoundError,
+            resources.open_text, self.data, 'does-not-exist')
+
+
+class OpenDiskTests(OpenTests, unittest.TestCase):
+    def setUp(self):
+        self.data = data01
+
+
+class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase):
+    pass
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/test_path.py

@@ -0,0 +1,40 @@
+import unittest
+
+from importlib import resources
+from . import data01
+from . import util
+
+
+class CommonTests(util.CommonResourceTests, unittest.TestCase):
+    def execute(self, package, path):
+        with resources.path(package, path):
+            pass
+
+
+class PathTests:
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_reading(self):
+        # Path should be readable.
+        # Test also implicitly verifies the returned object is a pathlib.Path
+        # instance.
+        with resources.path(self.data, 'utf-8.file') as path:
+            # pathlib.Path.read_text() was introduced in Python 3.5.
+            with path.open('r', encoding='utf-8') as file:
+                text = file.read()
+            self.assertEqual('Hello, UTF-8 world!\n', text)
+
+
+class PathDiskTests(PathTests, unittest.TestCase):
+    data = data01
+
+
+class PathZipTests(PathTests, util.ZipSetup, unittest.TestCase):
+    def test_remove_in_context_manager(self):
+        # It is not an error if the file that was temporarily stashed on the
+        # file system is removed inside the `with` stanza.
+        with resources.path(self.data, 'utf-8.file') as path:
+            path.unlink()
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/test_read.py

@@ -0,0 +1,65 @@
+import unittest
+
+from importlib import import_module, resources
+from . import data01
+from . import util
+
+
+class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase):
+    def execute(self, package, path):
+        resources.read_binary(package, path)
+
+
+class CommonTextTests(util.CommonResourceTests, unittest.TestCase):
+    def execute(self, package, path):
+        resources.read_text(package, path)
+
+
+class ReadTests:
+    def test_read_binary(self):
+        result = resources.read_binary(self.data, 'binary.file')
+        self.assertEqual(result, b'\0\1\2\3')
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_read_text_default_encoding(self):
+        result = resources.read_text(self.data, 'utf-8.file')
+        self.assertEqual(result, 'Hello, UTF-8 world!\n')
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_read_text_given_encoding(self):
+        result = resources.read_text(
+            self.data, 'utf-16.file', encoding='utf-16')
+        self.assertEqual(result, 'Hello, UTF-16 world!\n')
+
+    @unittest.skip("TODO: RUSTPYTHON")
+    def test_read_text_with_errors(self):
+        # Raises UnicodeError without the 'errors' argument.
+        self.assertRaises(
+            UnicodeError, resources.read_text, self.data, 'utf-16.file')
+        result = resources.read_text(self.data, 'utf-16.file', errors='ignore')
+        self.assertEqual(
+            result,
+            'H\x00e\x00l\x00l\x00o\x00,\x00 '
+            '\x00U\x00T\x00F\x00-\x001\x006\x00 '
+            '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00')
+
+
+class ReadDiskTests(ReadTests, unittest.TestCase):
+    data = data01
+
+
+class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase):
+    def test_read_submodule_resource(self):
+        submodule = import_module('ziptestdata.subdirectory')
+        result = resources.read_binary(
+            submodule, 'binary.file')
+        self.assertEqual(result, b'\0\1\2\3')
+
+    def test_read_submodule_resource_by_name(self):
+        result = resources.read_binary(
+            'ziptestdata.subdirectory', 'binary.file')
+        self.assertEqual(result, b'\0\1\2\3')
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/test_resource.py

@@ -0,0 +1,167 @@
+import sys
+import unittest
+
+from . import data01
+from . import zipdata01, zipdata02
+from . import util
+from importlib import resources, import_module
+
+
+class ResourceTests:
+    # Subclasses are expected to set the `data` attribute.
+
+    def test_is_resource_good_path(self):
+        self.assertTrue(resources.is_resource(self.data, 'binary.file'))
+
+    def test_is_resource_missing(self):
+        self.assertFalse(resources.is_resource(self.data, 'not-a-file'))
+
+    def test_is_resource_subresource_directory(self):
+        # Directories are not resources.
+        self.assertFalse(resources.is_resource(self.data, 'subdirectory'))
+
+    def test_contents(self):
+        contents = set(resources.contents(self.data))
+        # There may be cruft in the directory listing of the data directory.
+        # Under Python 3 we could have a __pycache__ directory, and under
+        # Python 2 we could have .pyc files.  These are both artifacts of the
+        # test suite importing these modules and writing these caches.  They
+        # aren't germane to this test, so just filter them out.
+        contents.discard('__pycache__')
+        contents.discard('__init__.pyc')
+        contents.discard('__init__.pyo')
+        self.assertEqual(contents, {
+            '__init__.py',
+            'subdirectory',
+            'utf-8.file',
+            'binary.file',
+            'utf-16.file',
+            })
+
+
+class ResourceDiskTests(ResourceTests, unittest.TestCase):
+    def setUp(self):
+        self.data = data01
+
+
+class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase):
+    pass
+
+
+@unittest.skip("TODO: RUSTPYTHON")
+class ResourceLoaderTests(unittest.TestCase):
+    def test_resource_contents(self):
+        package = util.create_package(
+            file=data01, path=data01.__file__, contents=['A', 'B', 'C'])
+        self.assertEqual(
+            set(resources.contents(package)),
+            {'A', 'B', 'C'})
+
+    def test_resource_is_resource(self):
+        package = util.create_package(
+            file=data01, path=data01.__file__,
+            contents=['A', 'B', 'C', 'D/E', 'D/F'])
+        self.assertTrue(resources.is_resource(package, 'B'))
+
+    def test_resource_directory_is_not_resource(self):
+        package = util.create_package(
+            file=data01, path=data01.__file__,
+            contents=['A', 'B', 'C', 'D/E', 'D/F'])
+        self.assertFalse(resources.is_resource(package, 'D'))
+
+    def test_resource_missing_is_not_resource(self):
+        package = util.create_package(
+            file=data01, path=data01.__file__,
+            contents=['A', 'B', 'C', 'D/E', 'D/F'])
+        self.assertFalse(resources.is_resource(package, 'Z'))
+
+
+class ResourceCornerCaseTests(unittest.TestCase):
+    def test_package_has_no_reader_fallback(self):
+        # Test odd ball packages which:
+        # 1. Do not have a ResourceReader as a loader
+        # 2. Are not on the file system
+        # 3. Are not in a zip file
+        module = util.create_package(
+            file=data01, path=data01.__file__, contents=['A', 'B', 'C'])
+        # Give the module a dummy loader.
+        module.__loader__ = object()
+        # Give the module a dummy origin.
+        module.__file__ = '/path/which/shall/not/be/named'
+        if sys.version_info >= (3,):
+            module.__spec__.loader = module.__loader__
+            module.__spec__.origin = module.__file__
+        self.assertFalse(resources.is_resource(module, 'A'))
+
+
+class ResourceFromZipsTest(util.ZipSetupBase, unittest.TestCase):
+    ZIP_MODULE = zipdata02                          # type: ignore
+
+    def test_unrelated_contents(self):
+        # https://gitlab.com/python-devs/importlib_resources/issues/44
+        #
+        # Here we have a zip file with two unrelated subpackages.  The bug
+        # reports that getting the contents of a resource returns unrelated
+        # files.
+        self.assertEqual(
+            set(resources.contents('ziptestdata.one')),
+            {'__init__.py', 'resource1.txt'})
+        self.assertEqual(
+            set(resources.contents('ziptestdata.two')),
+            {'__init__.py', 'resource2.txt'})
+
+
+class SubdirectoryResourceFromZipsTest(util.ZipSetupBase, unittest.TestCase):
+    ZIP_MODULE = zipdata01                          # type: ignore
+
+    def test_is_submodule_resource(self):
+        submodule = import_module('ziptestdata.subdirectory')
+        self.assertTrue(
+            resources.is_resource(submodule, 'binary.file'))
+
+    def test_read_submodule_resource_by_name(self):
+        self.assertTrue(
+            resources.is_resource('ziptestdata.subdirectory', 'binary.file'))
+
+    def test_submodule_contents(self):
+        submodule = import_module('ziptestdata.subdirectory')
+        self.assertEqual(
+            set(resources.contents(submodule)),
+            {'__init__.py', 'binary.file'})
+
+    def test_submodule_contents_by_name(self):
+        self.assertEqual(
+            set(resources.contents('ziptestdata.subdirectory')),
+            {'__init__.py', 'binary.file'})
+
+
+class NamespaceTest(unittest.TestCase):
+    def test_namespaces_cannot_have_resources(self):
+        contents = resources.contents('test.test_importlib.data03.namespace')
+        self.assertFalse(list(contents))
+        # Even though there is a file in the namespace directory, it is not
+        # considered a resource, since namespace packages can't have them.
+        self.assertFalse(resources.is_resource(
+            'test.test_importlib.data03.namespace',
+            'resource1.txt'))
+        # We should get an exception if we try to read it or open it.
+        self.assertRaises(
+            FileNotFoundError,
+            resources.open_text,
+            'test.test_importlib.data03.namespace', 'resource1.txt')
+        self.assertRaises(
+            FileNotFoundError,
+            resources.open_binary,
+            'test.test_importlib.data03.namespace', 'resource1.txt')
+        self.assertRaises(
+            FileNotFoundError,
+            resources.read_text,
+            'test.test_importlib.data03.namespace', 'resource1.txt')
+        self.assertRaises(
+            FileNotFoundError,
+            resources.read_binary,
+            'test.test_importlib.data03.namespace', 'resource1.txt')
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/test_spec.py

@@ -0,0 +1,819 @@
+from . import util as test_util
+
+init = test_util.import_importlib('importlib')
+machinery = test_util.import_importlib('importlib.machinery')
+util = test_util.import_importlib('importlib.util')
+
+import os.path
+import pathlib
+from test.support import CleanImport
+import unittest
+import sys
+import warnings
+
+
+
+class TestLoader:
+
+    def __init__(self, path=None, is_package=None):
+        self.path = path
+        self.package = is_package
+
+    def __repr__(self):
+        return '<TestLoader object>'
+
+    def __getattr__(self, name):
+        if name == 'get_filename' and self.path is not None:
+            return self._get_filename
+        if name == 'is_package':
+            return self._is_package
+        raise AttributeError(name)
+
+    def _get_filename(self, name):
+        return self.path
+
+    def _is_package(self, name):
+        return self.package
+
+    def create_module(self, spec):
+        return None
+
+
+class NewLoader(TestLoader):
+
+    EGGS = 1
+
+    def exec_module(self, module):
+        module.eggs = self.EGGS
+
+
+class LegacyLoader(TestLoader):
+
+    HAM = -1
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", DeprecationWarning)
+
+        frozen_util = util['Frozen']
+
+        @frozen_util.module_for_loader
+        def load_module(self, module):
+            module.ham = self.HAM
+            return module
+
+
+class ModuleSpecTests:
+
+    def setUp(self):
+        self.name = 'spam'
+        self.path = 'spam.py'
+        self.cached = self.util.cache_from_source(self.path)
+        self.loader = TestLoader()
+        self.spec = self.machinery.ModuleSpec(self.name, self.loader)
+        self.loc_spec = self.machinery.ModuleSpec(self.name, self.loader,
+                                                  origin=self.path)
+        self.loc_spec._set_fileattr = True
+
+    def test_default(self):
+        spec = self.machinery.ModuleSpec(self.name, self.loader)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.loader)
+        self.assertIs(spec.origin, None)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertIs(spec.cached, None)
+        self.assertFalse(spec.has_location)
+
+    def test_default_no_loader(self):
+        spec = self.machinery.ModuleSpec(self.name, None)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertIs(spec.loader, None)
+        self.assertIs(spec.origin, None)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertIs(spec.cached, None)
+        self.assertFalse(spec.has_location)
+
+    def test_default_is_package_false(self):
+        spec = self.machinery.ModuleSpec(self.name, self.loader,
+                                         is_package=False)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.loader)
+        self.assertIs(spec.origin, None)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertIs(spec.cached, None)
+        self.assertFalse(spec.has_location)
+
+    def test_default_is_package_true(self):
+        spec = self.machinery.ModuleSpec(self.name, self.loader,
+                                         is_package=True)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.loader)
+        self.assertIs(spec.origin, None)
+        self.assertIs(spec.loader_state, None)
+        self.assertEqual(spec.submodule_search_locations, [])
+        self.assertIs(spec.cached, None)
+        self.assertFalse(spec.has_location)
+
+    def test_has_location_setter(self):
+        spec = self.machinery.ModuleSpec(self.name, self.loader,
+                                         origin='somewhere')
+        self.assertFalse(spec.has_location)
+        spec.has_location = True
+        self.assertTrue(spec.has_location)
+
+    def test_equality(self):
+        other = type(sys.implementation)(name=self.name,
+                                         loader=self.loader,
+                                         origin=None,
+                                         submodule_search_locations=None,
+                                         has_location=False,
+                                         cached=None,
+                                         )
+
+        self.assertTrue(self.spec == other)
+
+    def test_equality_location(self):
+        other = type(sys.implementation)(name=self.name,
+                                         loader=self.loader,
+                                         origin=self.path,
+                                         submodule_search_locations=None,
+                                         has_location=True,
+                                         cached=self.cached,
+                                         )
+
+        self.assertEqual(self.loc_spec, other)
+
+    def test_inequality(self):
+        other = type(sys.implementation)(name='ham',
+                                         loader=self.loader,
+                                         origin=None,
+                                         submodule_search_locations=None,
+                                         has_location=False,
+                                         cached=None,
+                                         )
+
+        self.assertNotEqual(self.spec, other)
+
+    def test_inequality_incomplete(self):
+        other = type(sys.implementation)(name=self.name,
+                                         loader=self.loader,
+                                         )
+
+        self.assertNotEqual(self.spec, other)
+
+    def test_package(self):
+        spec = self.machinery.ModuleSpec('spam.eggs', self.loader)
+
+        self.assertEqual(spec.parent, 'spam')
+
+    def test_package_is_package(self):
+        spec = self.machinery.ModuleSpec('spam.eggs', self.loader,
+                                         is_package=True)
+
+        self.assertEqual(spec.parent, 'spam.eggs')
+
+    # cached
+
+    def test_cached_set(self):
+        before = self.spec.cached
+        self.spec.cached = 'there'
+        after = self.spec.cached
+
+        self.assertIs(before, None)
+        self.assertEqual(after, 'there')
+
+    def test_cached_no_origin(self):
+        spec = self.machinery.ModuleSpec(self.name, self.loader)
+
+        self.assertIs(spec.cached, None)
+
+    def test_cached_with_origin_not_location(self):
+        spec = self.machinery.ModuleSpec(self.name, self.loader,
+                                         origin=self.path)
+
+        self.assertIs(spec.cached, None)
+
+    def test_cached_source(self):
+        expected = self.util.cache_from_source(self.path)
+
+        self.assertEqual(self.loc_spec.cached, expected)
+
+    def test_cached_source_unknown_suffix(self):
+        self.loc_spec.origin = 'spam.spamspamspam'
+
+        self.assertIs(self.loc_spec.cached, None)
+
+    def test_cached_source_missing_cache_tag(self):
+        original = sys.implementation.cache_tag
+        sys.implementation.cache_tag = None
+        try:
+            cached = self.loc_spec.cached
+        finally:
+            sys.implementation.cache_tag = original
+
+        self.assertIs(cached, None)
+
+    def test_cached_sourceless(self):
+        self.loc_spec.origin = 'spam.pyc'
+
+        self.assertEqual(self.loc_spec.cached, 'spam.pyc')
+
+
+(Frozen_ModuleSpecTests,
+ Source_ModuleSpecTests
+ ) = test_util.test_both(ModuleSpecTests, util=util, machinery=machinery)
+
+
+class ModuleSpecMethodsTests:
+
+    @property
+    def bootstrap(self):
+        return self.init._bootstrap
+
+    def setUp(self):
+        self.name = 'spam'
+        self.path = 'spam.py'
+        self.cached = self.util.cache_from_source(self.path)
+        self.loader = TestLoader()
+        self.spec = self.machinery.ModuleSpec(self.name, self.loader)
+        self.loc_spec = self.machinery.ModuleSpec(self.name, self.loader,
+                                                  origin=self.path)
+        self.loc_spec._set_fileattr = True
+
+    # exec()
+
+    def test_exec(self):
+        self.spec.loader = NewLoader()
+        module = self.util.module_from_spec(self.spec)
+        sys.modules[self.name] = module
+        self.assertFalse(hasattr(module, 'eggs'))
+        self.bootstrap._exec(self.spec, module)
+
+        self.assertEqual(module.eggs, 1)
+
+    # load()
+
+    def test_load(self):
+        self.spec.loader = NewLoader()
+        with CleanImport(self.spec.name):
+            loaded = self.bootstrap._load(self.spec)
+            installed = sys.modules[self.spec.name]
+
+        self.assertEqual(loaded.eggs, 1)
+        self.assertIs(loaded, installed)
+
+    def test_load_replaced(self):
+        replacement = object()
+        class ReplacingLoader(TestLoader):
+            def exec_module(self, module):
+                sys.modules[module.__name__] = replacement
+        self.spec.loader = ReplacingLoader()
+        with CleanImport(self.spec.name):
+            loaded = self.bootstrap._load(self.spec)
+            installed = sys.modules[self.spec.name]
+
+        self.assertIs(loaded, replacement)
+        self.assertIs(installed, replacement)
+
+    def test_load_failed(self):
+        class FailedLoader(TestLoader):
+            def exec_module(self, module):
+                raise RuntimeError
+        self.spec.loader = FailedLoader()
+        with CleanImport(self.spec.name):
+            with self.assertRaises(RuntimeError):
+                loaded = self.bootstrap._load(self.spec)
+            self.assertNotIn(self.spec.name, sys.modules)
+
+    def test_load_failed_removed(self):
+        class FailedLoader(TestLoader):
+            def exec_module(self, module):
+                del sys.modules[module.__name__]
+                raise RuntimeError
+        self.spec.loader = FailedLoader()
+        with CleanImport(self.spec.name):
+            with self.assertRaises(RuntimeError):
+                loaded = self.bootstrap._load(self.spec)
+            self.assertNotIn(self.spec.name, sys.modules)
+
+    def test_load_legacy(self):
+        self.spec.loader = LegacyLoader()
+        with CleanImport(self.spec.name):
+            loaded = self.bootstrap._load(self.spec)
+
+        self.assertEqual(loaded.ham, -1)
+
+    def test_load_legacy_attributes(self):
+        self.spec.loader = LegacyLoader()
+        with CleanImport(self.spec.name):
+            loaded = self.bootstrap._load(self.spec)
+
+        self.assertIs(loaded.__loader__, self.spec.loader)
+        self.assertEqual(loaded.__package__, self.spec.parent)
+        self.assertIs(loaded.__spec__, self.spec)
+
+    def test_load_legacy_attributes_immutable(self):
+        module = object()
+        class ImmutableLoader(TestLoader):
+            def load_module(self, name):
+                sys.modules[name] = module
+                return module
+        self.spec.loader = ImmutableLoader()
+        with CleanImport(self.spec.name):
+            loaded = self.bootstrap._load(self.spec)
+
+            self.assertIs(sys.modules[self.spec.name], module)
+
+    # reload()
+
+    def test_reload(self):
+        self.spec.loader = NewLoader()
+        with CleanImport(self.spec.name):
+            loaded = self.bootstrap._load(self.spec)
+            reloaded = self.bootstrap._exec(self.spec, loaded)
+            installed = sys.modules[self.spec.name]
+
+        self.assertEqual(loaded.eggs, 1)
+        self.assertIs(reloaded, loaded)
+        self.assertIs(installed, loaded)
+
+    def test_reload_modified(self):
+        self.spec.loader = NewLoader()
+        with CleanImport(self.spec.name):
+            loaded = self.bootstrap._load(self.spec)
+            loaded.eggs = 2
+            reloaded = self.bootstrap._exec(self.spec, loaded)
+
+        self.assertEqual(loaded.eggs, 1)
+        self.assertIs(reloaded, loaded)
+
+    def test_reload_extra_attributes(self):
+        self.spec.loader = NewLoader()
+        with CleanImport(self.spec.name):
+            loaded = self.bootstrap._load(self.spec)
+            loaded.available = False
+            reloaded = self.bootstrap._exec(self.spec, loaded)
+
+        self.assertFalse(loaded.available)
+        self.assertIs(reloaded, loaded)
+
+    def test_reload_init_module_attrs(self):
+        self.spec.loader = NewLoader()
+        with CleanImport(self.spec.name):
+            loaded = self.bootstrap._load(self.spec)
+            loaded.__name__ = 'ham'
+            del loaded.__loader__
+            del loaded.__package__
+            del loaded.__spec__
+            self.bootstrap._exec(self.spec, loaded)
+
+        self.assertEqual(loaded.__name__, self.spec.name)
+        self.assertIs(loaded.__loader__, self.spec.loader)
+        self.assertEqual(loaded.__package__, self.spec.parent)
+        self.assertIs(loaded.__spec__, self.spec)
+        self.assertFalse(hasattr(loaded, '__path__'))
+        self.assertFalse(hasattr(loaded, '__file__'))
+        self.assertFalse(hasattr(loaded, '__cached__'))
+
+    def test_reload_legacy(self):
+        self.spec.loader = LegacyLoader()
+        with CleanImport(self.spec.name):
+            loaded = self.bootstrap._load(self.spec)
+            reloaded = self.bootstrap._exec(self.spec, loaded)
+            installed = sys.modules[self.spec.name]
+
+        self.assertEqual(loaded.ham, -1)
+        self.assertIs(reloaded, loaded)
+        self.assertIs(installed, loaded)
+
+
+(Frozen_ModuleSpecMethodsTests,
+ Source_ModuleSpecMethodsTests
+ ) = test_util.test_both(ModuleSpecMethodsTests, init=init, util=util,
+                         machinery=machinery)
+
+
+class ModuleReprTests:
+
+    @property
+    def bootstrap(self):
+        return self.init._bootstrap
+
+    def setUp(self):
+        self.module = type(os)('spam')
+        self.spec = self.machinery.ModuleSpec('spam', TestLoader())
+
+    def test_module___loader___module_repr(self):
+        class Loader:
+            def module_repr(self, module):
+                return '<delicious {}>'.format(module.__name__)
+        self.module.__loader__ = Loader()
+        modrepr = self.bootstrap._module_repr(self.module)
+
+        self.assertEqual(modrepr, '<delicious spam>')
+
+    def test_module___loader___module_repr_bad(self):
+        class Loader(TestLoader):
+            def module_repr(self, module):
+                raise Exception
+        self.module.__loader__ = Loader()
+        modrepr = self.bootstrap._module_repr(self.module)
+
+        self.assertEqual(modrepr,
+                         '<module {!r} (<TestLoader object>)>'.format('spam'))
+
+    def test_module___spec__(self):
+        origin = 'in a hole, in the ground'
+        self.spec.origin = origin
+        self.module.__spec__ = self.spec
+        modrepr = self.bootstrap._module_repr(self.module)
+
+        self.assertEqual(modrepr, '<module {!r} ({})>'.format('spam', origin))
+
+    def test_module___spec___location(self):
+        location = 'in_a_galaxy_far_far_away.py'
+        self.spec.origin = location
+        self.spec._set_fileattr = True
+        self.module.__spec__ = self.spec
+        modrepr = self.bootstrap._module_repr(self.module)
+
+        self.assertEqual(modrepr,
+                         '<module {!r} from {!r}>'.format('spam', location))
+
+    def test_module___spec___no_origin(self):
+        self.spec.loader = TestLoader()
+        self.module.__spec__ = self.spec
+        modrepr = self.bootstrap._module_repr(self.module)
+
+        self.assertEqual(modrepr,
+                         '<module {!r} (<TestLoader object>)>'.format('spam'))
+
+    def test_module___spec___no_origin_no_loader(self):
+        self.spec.loader = None
+        self.module.__spec__ = self.spec
+        modrepr = self.bootstrap._module_repr(self.module)
+
+        self.assertEqual(modrepr, '<module {!r}>'.format('spam'))
+
+    def test_module_no_name(self):
+        del self.module.__name__
+        modrepr = self.bootstrap._module_repr(self.module)
+
+        self.assertEqual(modrepr, '<module {!r}>'.format('?'))
+
+    def test_module_with_file(self):
+        filename = 'e/i/e/i/o/spam.py'
+        self.module.__file__ = filename
+        modrepr = self.bootstrap._module_repr(self.module)
+
+        self.assertEqual(modrepr,
+                         '<module {!r} from {!r}>'.format('spam', filename))
+
+    def test_module_no_file(self):
+        self.module.__loader__ = TestLoader()
+        modrepr = self.bootstrap._module_repr(self.module)
+
+        self.assertEqual(modrepr,
+                         '<module {!r} (<TestLoader object>)>'.format('spam'))
+
+    def test_module_no_file_no_loader(self):
+        modrepr = self.bootstrap._module_repr(self.module)
+
+        self.assertEqual(modrepr, '<module {!r}>'.format('spam'))
+
+
+(Frozen_ModuleReprTests,
+ Source_ModuleReprTests
+ ) = test_util.test_both(ModuleReprTests, init=init, util=util,
+                         machinery=machinery)
+
+
+class FactoryTests:
+
+    def setUp(self):
+        self.name = 'spam'
+        self.path = 'spam.py'
+        self.cached = self.util.cache_from_source(self.path)
+        self.loader = TestLoader()
+        self.fileloader = TestLoader(self.path)
+        self.pkgloader = TestLoader(self.path, True)
+
+    # spec_from_loader()
+
+    def test_spec_from_loader_default(self):
+        spec = self.util.spec_from_loader(self.name, self.loader)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.loader)
+        self.assertIs(spec.origin, None)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertIs(spec.cached, None)
+        self.assertFalse(spec.has_location)
+
+    def test_spec_from_loader_default_with_bad_is_package(self):
+        class Loader:
+            def is_package(self, name):
+                raise ImportError
+        loader = Loader()
+        spec = self.util.spec_from_loader(self.name, loader)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, loader)
+        self.assertIs(spec.origin, None)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertIs(spec.cached, None)
+        self.assertFalse(spec.has_location)
+
+    def test_spec_from_loader_origin(self):
+        origin = 'somewhere over the rainbow'
+        spec = self.util.spec_from_loader(self.name, self.loader,
+                                          origin=origin)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.loader)
+        self.assertIs(spec.origin, origin)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertIs(spec.cached, None)
+        self.assertFalse(spec.has_location)
+
+    def test_spec_from_loader_is_package_false(self):
+        spec = self.util.spec_from_loader(self.name, self.loader,
+                                          is_package=False)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.loader)
+        self.assertIs(spec.origin, None)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertIs(spec.cached, None)
+        self.assertFalse(spec.has_location)
+
+    def test_spec_from_loader_is_package_true(self):
+        spec = self.util.spec_from_loader(self.name, self.loader,
+                                          is_package=True)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.loader)
+        self.assertIs(spec.origin, None)
+        self.assertIs(spec.loader_state, None)
+        self.assertEqual(spec.submodule_search_locations, [])
+        self.assertIs(spec.cached, None)
+        self.assertFalse(spec.has_location)
+
+    def test_spec_from_loader_origin_and_is_package(self):
+        origin = 'where the streets have no name'
+        spec = self.util.spec_from_loader(self.name, self.loader,
+                                          origin=origin, is_package=True)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.loader)
+        self.assertIs(spec.origin, origin)
+        self.assertIs(spec.loader_state, None)
+        self.assertEqual(spec.submodule_search_locations, [])
+        self.assertIs(spec.cached, None)
+        self.assertFalse(spec.has_location)
+
+    def test_spec_from_loader_is_package_with_loader_false(self):
+        loader = TestLoader(is_package=False)
+        spec = self.util.spec_from_loader(self.name, loader)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, loader)
+        self.assertIs(spec.origin, None)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertIs(spec.cached, None)
+        self.assertFalse(spec.has_location)
+
+    def test_spec_from_loader_is_package_with_loader_true(self):
+        loader = TestLoader(is_package=True)
+        spec = self.util.spec_from_loader(self.name, loader)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, loader)
+        self.assertIs(spec.origin, None)
+        self.assertIs(spec.loader_state, None)
+        self.assertEqual(spec.submodule_search_locations, [])
+        self.assertIs(spec.cached, None)
+        self.assertFalse(spec.has_location)
+
+    def test_spec_from_loader_default_with_file_loader(self):
+        spec = self.util.spec_from_loader(self.name, self.fileloader)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.fileloader)
+        self.assertEqual(spec.origin, self.path)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertEqual(spec.cached, self.cached)
+        self.assertTrue(spec.has_location)
+
+    def test_spec_from_loader_is_package_false_with_fileloader(self):
+        spec = self.util.spec_from_loader(self.name, self.fileloader,
+                                          is_package=False)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.fileloader)
+        self.assertEqual(spec.origin, self.path)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertEqual(spec.cached, self.cached)
+        self.assertTrue(spec.has_location)
+
+    def test_spec_from_loader_is_package_true_with_fileloader(self):
+        spec = self.util.spec_from_loader(self.name, self.fileloader,
+                                          is_package=True)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.fileloader)
+        self.assertEqual(spec.origin, self.path)
+        self.assertIs(spec.loader_state, None)
+        self.assertEqual(spec.submodule_search_locations, [''])
+        self.assertEqual(spec.cached, self.cached)
+        self.assertTrue(spec.has_location)
+
+    # spec_from_file_location()
+
+    def test_spec_from_file_location_default(self):
+        spec = self.util.spec_from_file_location(self.name, self.path)
+
+        self.assertEqual(spec.name, self.name)
+        # Need to use a circuitous route to get at importlib.machinery to make
+        # sure the same class object is used in the isinstance() check as
+        # would have been used to create the loader.
+        self.assertIsInstance(spec.loader,
+                              self.util.abc.machinery.SourceFileLoader)
+        self.assertEqual(spec.loader.name, self.name)
+        self.assertEqual(spec.loader.path, self.path)
+        self.assertEqual(spec.origin, self.path)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertEqual(spec.cached, self.cached)
+        self.assertTrue(spec.has_location)
+
+    def test_spec_from_file_location_path_like_arg(self):
+        spec = self.util.spec_from_file_location(self.name,
+                                                 pathlib.PurePath(self.path))
+        self.assertEqual(spec.origin, self.path)
+
+    def test_spec_from_file_location_default_without_location(self):
+        spec = self.util.spec_from_file_location(self.name)
+
+        self.assertIs(spec, None)
+
+    def test_spec_from_file_location_default_bad_suffix(self):
+        spec = self.util.spec_from_file_location(self.name, 'spam.eggs')
+
+        self.assertIs(spec, None)
+
+    def test_spec_from_file_location_loader_no_location(self):
+        spec = self.util.spec_from_file_location(self.name,
+                                                 loader=self.fileloader)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.fileloader)
+        self.assertEqual(spec.origin, self.path)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertEqual(spec.cached, self.cached)
+        self.assertTrue(spec.has_location)
+
+    def test_spec_from_file_location_loader_no_location_no_get_filename(self):
+        spec = self.util.spec_from_file_location(self.name,
+                                                 loader=self.loader)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.loader)
+        self.assertEqual(spec.origin, '<unknown>')
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertIs(spec.cached, None)
+        self.assertTrue(spec.has_location)
+
+    def test_spec_from_file_location_loader_no_location_bad_get_filename(self):
+        class Loader:
+            def get_filename(self, name):
+                raise ImportError
+        loader = Loader()
+        spec = self.util.spec_from_file_location(self.name, loader=loader)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, loader)
+        self.assertEqual(spec.origin, '<unknown>')
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertIs(spec.cached, None)
+        self.assertTrue(spec.has_location)
+
+    def test_spec_from_file_location_smsl_none(self):
+        spec = self.util.spec_from_file_location(self.name, self.path,
+                                       loader=self.fileloader,
+                                       submodule_search_locations=None)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.fileloader)
+        self.assertEqual(spec.origin, self.path)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertEqual(spec.cached, self.cached)
+        self.assertTrue(spec.has_location)
+
+    def test_spec_from_file_location_smsl_empty(self):
+        spec = self.util.spec_from_file_location(self.name, self.path,
+                                       loader=self.fileloader,
+                                       submodule_search_locations=[])
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.fileloader)
+        self.assertEqual(spec.origin, self.path)
+        self.assertIs(spec.loader_state, None)
+        self.assertEqual(spec.submodule_search_locations, [''])
+        self.assertEqual(spec.cached, self.cached)
+        self.assertTrue(spec.has_location)
+
+    def test_spec_from_file_location_smsl_not_empty(self):
+        spec = self.util.spec_from_file_location(self.name, self.path,
+                                       loader=self.fileloader,
+                                       submodule_search_locations=['eggs'])
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.fileloader)
+        self.assertEqual(spec.origin, self.path)
+        self.assertIs(spec.loader_state, None)
+        self.assertEqual(spec.submodule_search_locations, ['eggs'])
+        self.assertEqual(spec.cached, self.cached)
+        self.assertTrue(spec.has_location)
+
+    def test_spec_from_file_location_smsl_default(self):
+        spec = self.util.spec_from_file_location(self.name, self.path,
+                                       loader=self.pkgloader)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.pkgloader)
+        self.assertEqual(spec.origin, self.path)
+        self.assertIs(spec.loader_state, None)
+        self.assertEqual(spec.submodule_search_locations, [''])
+        self.assertEqual(spec.cached, self.cached)
+        self.assertTrue(spec.has_location)
+
+    def test_spec_from_file_location_smsl_default_not_package(self):
+        class Loader:
+            def is_package(self, name):
+                return False
+        loader = Loader()
+        spec = self.util.spec_from_file_location(self.name, self.path,
+                                                 loader=loader)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, loader)
+        self.assertEqual(spec.origin, self.path)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertEqual(spec.cached, self.cached)
+        self.assertTrue(spec.has_location)
+
+    def test_spec_from_file_location_smsl_default_no_is_package(self):
+        spec = self.util.spec_from_file_location(self.name, self.path,
+                                       loader=self.fileloader)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, self.fileloader)
+        self.assertEqual(spec.origin, self.path)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertEqual(spec.cached, self.cached)
+        self.assertTrue(spec.has_location)
+
+    def test_spec_from_file_location_smsl_default_bad_is_package(self):
+        class Loader:
+            def is_package(self, name):
+                raise ImportError
+        loader = Loader()
+        spec = self.util.spec_from_file_location(self.name, self.path,
+                                                 loader=loader)
+
+        self.assertEqual(spec.name, self.name)
+        self.assertEqual(spec.loader, loader)
+        self.assertEqual(spec.origin, self.path)
+        self.assertIs(spec.loader_state, None)
+        self.assertIs(spec.submodule_search_locations, None)
+        self.assertEqual(spec.cached, self.cached)
+        self.assertTrue(spec.has_location)
+
+
+(Frozen_FactoryTests,
+ Source_FactoryTests
+ ) = test_util.test_both(FactoryTests, util=util, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()