mirror of
https://github.com/RustPython/RustPython.git
synced 2026-10-28 18:59:04 +08:00
Update modulefinder from v3.14.3
This commit is contained in:
CPython Developers authored and Lee Dogeon committed
1 parent
23c8eb80fb
commit
6255752cfb
2 files changed
+1109
No files matched your search
Vendored
+671
@@ -0,0 +1,671 @@
|
||||
"""Find modules used by a script, using introspection."""
|
||||
|
||||
import dis
|
||||
import importlib._bootstrap_external
|
||||
import importlib.machinery
|
||||
import marshal
|
||||
import os
|
||||
import io
|
||||
import sys
|
||||
|
||||
# Old imp constants:
|
||||
|
||||
_SEARCH_ERROR = 0
|
||||
_PY_SOURCE = 1
|
||||
_PY_COMPILED = 2
|
||||
_C_EXTENSION = 3
|
||||
_PKG_DIRECTORY = 5
|
||||
_C_BUILTIN = 6
|
||||
_PY_FROZEN = 7
|
||||
|
||||
# Modulefinder does a good job at simulating Python's, but it can not
|
||||
# handle __path__ modifications packages make at runtime. Therefore there
|
||||
# is a mechanism whereby you can register extra paths in this map for a
|
||||
# package, and it will be honored.
|
||||
|
||||
# Note this is a mapping is lists of paths.
|
||||
packagePathMap = {}
|
||||
|
||||
# A Public interface
|
||||
def AddPackagePath(packagename, path):
|
||||
packagePathMap.setdefault(packagename, []).append(path)
|
||||
|
||||
replacePackageMap = {}
|
||||
|
||||
# This ReplacePackage mechanism allows modulefinder to work around
|
||||
# situations in which a package injects itself under the name
|
||||
# of another package into sys.modules at runtime by calling
|
||||
# ReplacePackage("real_package_name", "faked_package_name")
|
||||
# before running ModuleFinder.
|
||||
|
||||
def ReplacePackage(oldname, newname):
|
||||
replacePackageMap[oldname] = newname
|
||||
|
||||
|
||||
def _find_module(name, path=None):
|
||||
"""An importlib reimplementation of imp.find_module (for our purposes)."""
|
||||
|
||||
# It's necessary to clear the caches for our Finder first, in case any
|
||||
# modules are being added/deleted/modified at runtime. In particular,
|
||||
# test_modulefinder.py changes file tree contents in a cache-breaking way:
|
||||
|
||||
importlib.machinery.PathFinder.invalidate_caches()
|
||||
|
||||
spec = importlib.machinery.PathFinder.find_spec(name, path)
|
||||
|
||||
if spec is None:
|
||||
raise ImportError("No module named {name!r}".format(name=name), name=name)
|
||||
|
||||
# Some special cases:
|
||||
|
||||
if spec.loader is importlib.machinery.BuiltinImporter:
|
||||
return None, None, ("", "", _C_BUILTIN)
|
||||
|
||||
if spec.loader is importlib.machinery.FrozenImporter:
|
||||
return None, None, ("", "", _PY_FROZEN)
|
||||
|
||||
file_path = spec.origin
|
||||
|
||||
if spec.loader.is_package(name):
|
||||
return None, os.path.dirname(file_path), ("", "", _PKG_DIRECTORY)
|
||||
|
||||
if isinstance(spec.loader, importlib.machinery.SourceFileLoader):
|
||||
kind = _PY_SOURCE
|
||||
|
||||
elif isinstance(
|
||||
spec.loader, (
|
||||
importlib.machinery.ExtensionFileLoader,
|
||||
importlib.machinery.AppleFrameworkLoader,
|
||||
)
|
||||
):
|
||||
kind = _C_EXTENSION
|
||||
|
||||
elif isinstance(spec.loader, importlib.machinery.SourcelessFileLoader):
|
||||
kind = _PY_COMPILED
|
||||
|
||||
else: # Should never happen.
|
||||
return None, None, ("", "", _SEARCH_ERROR)
|
||||
|
||||
file = io.open_code(file_path)
|
||||
suffix = os.path.splitext(file_path)[-1]
|
||||
|
||||
return file, file_path, (suffix, "rb", kind)
|
||||
|
||||
|
||||
class Module:
|
||||
|
||||
def __init__(self, name, file=None, path=None):
|
||||
self.__name__ = name
|
||||
self.__file__ = file
|
||||
self.__path__ = path
|
||||
self.__code__ = None
|
||||
# The set of global names that are assigned to in the module.
|
||||
# This includes those names imported through starimports of
|
||||
# Python modules.
|
||||
self.globalnames = {}
|
||||
# The set of starimports this module did that could not be
|
||||
# resolved, ie. a starimport from a non-Python module.
|
||||
self.starimports = {}
|
||||
|
||||
def __repr__(self):
|
||||
s = "Module(%r" % (self.__name__,)
|
||||
if self.__file__ is not None:
|
||||
s = s + ", %r" % (self.__file__,)
|
||||
if self.__path__ is not None:
|
||||
s = s + ", %r" % (self.__path__,)
|
||||
s = s + ")"
|
||||
return s
|
||||
|
||||
class ModuleFinder:
|
||||
|
||||
def __init__(self, path=None, debug=0, excludes=None, replace_paths=None):
|
||||
if path is None:
|
||||
path = sys.path
|
||||
self.path = path
|
||||
self.modules = {}
|
||||
self.badmodules = {}
|
||||
self.debug = debug
|
||||
self.indent = 0
|
||||
self.excludes = excludes if excludes is not None else []
|
||||
self.replace_paths = replace_paths if replace_paths is not None else []
|
||||
self.processed_paths = [] # Used in debugging only
|
||||
|
||||
def msg(self, level, str, *args):
|
||||
if level <= self.debug:
|
||||
for i in range(self.indent):
|
||||
print(" ", end=' ')
|
||||
print(str, end=' ')
|
||||
for arg in args:
|
||||
print(repr(arg), end=' ')
|
||||
print()
|
||||
|
||||
def msgin(self, *args):
|
||||
level = args[0]
|
||||
if level <= self.debug:
|
||||
self.indent = self.indent + 1
|
||||
self.msg(*args)
|
||||
|
||||
def msgout(self, *args):
|
||||
level = args[0]
|
||||
if level <= self.debug:
|
||||
self.indent = self.indent - 1
|
||||
self.msg(*args)
|
||||
|
||||
def run_script(self, pathname):
|
||||
self.msg(2, "run_script", pathname)
|
||||
with io.open_code(pathname) as fp:
|
||||
stuff = ("", "rb", _PY_SOURCE)
|
||||
self.load_module('__main__', fp, pathname, stuff)
|
||||
|
||||
def load_file(self, pathname):
|
||||
dir, name = os.path.split(pathname)
|
||||
name, ext = os.path.splitext(name)
|
||||
with io.open_code(pathname) as fp:
|
||||
stuff = (ext, "rb", _PY_SOURCE)
|
||||
self.load_module(name, fp, pathname, stuff)
|
||||
|
||||
def import_hook(self, name, caller=None, fromlist=None, level=-1):
|
||||
self.msg(3, "import_hook", name, caller, fromlist, level)
|
||||
parent = self.determine_parent(caller, level=level)
|
||||
q, tail = self.find_head_package(parent, name)
|
||||
m = self.load_tail(q, tail)
|
||||
if not fromlist:
|
||||
return q
|
||||
if m.__path__:
|
||||
self.ensure_fromlist(m, fromlist)
|
||||
return None
|
||||
|
||||
def determine_parent(self, caller, level=-1):
|
||||
self.msgin(4, "determine_parent", caller, level)
|
||||
if not caller or level == 0:
|
||||
self.msgout(4, "determine_parent -> None")
|
||||
return None
|
||||
pname = caller.__name__
|
||||
if level >= 1: # relative import
|
||||
if caller.__path__:
|
||||
level -= 1
|
||||
if level == 0:
|
||||
parent = self.modules[pname]
|
||||
assert parent is caller
|
||||
self.msgout(4, "determine_parent ->", parent)
|
||||
return parent
|
||||
if pname.count(".") < level:
|
||||
raise ImportError("relative importpath too deep")
|
||||
pname = ".".join(pname.split(".")[:-level])
|
||||
parent = self.modules[pname]
|
||||
self.msgout(4, "determine_parent ->", parent)
|
||||
return parent
|
||||
if caller.__path__:
|
||||
parent = self.modules[pname]
|
||||
assert caller is parent
|
||||
self.msgout(4, "determine_parent ->", parent)
|
||||
return parent
|
||||
if '.' in pname:
|
||||
i = pname.rfind('.')
|
||||
pname = pname[:i]
|
||||
parent = self.modules[pname]
|
||||
assert parent.__name__ == pname
|
||||
self.msgout(4, "determine_parent ->", parent)
|
||||
return parent
|
||||
self.msgout(4, "determine_parent -> None")
|
||||
return None
|
||||
|
||||
def find_head_package(self, parent, name):
|
||||
self.msgin(4, "find_head_package", parent, name)
|
||||
if '.' in name:
|
||||
i = name.find('.')
|
||||
head = name[:i]
|
||||
tail = name[i+1:]
|
||||
else:
|
||||
head = name
|
||||
tail = ""
|
||||
if parent:
|
||||
qname = "%s.%s" % (parent.__name__, head)
|
||||
else:
|
||||
qname = head
|
||||
q = self.import_module(head, qname, parent)
|
||||
if q:
|
||||
self.msgout(4, "find_head_package ->", (q, tail))
|
||||
return q, tail
|
||||
if parent:
|
||||
qname = head
|
||||
parent = None
|
||||
q = self.import_module(head, qname, parent)
|
||||
if q:
|
||||
self.msgout(4, "find_head_package ->", (q, tail))
|
||||
return q, tail
|
||||
self.msgout(4, "raise ImportError: No module named", qname)
|
||||
raise ImportError("No module named " + qname)
|
||||
|
||||
def load_tail(self, q, tail):
|
||||
self.msgin(4, "load_tail", q, tail)
|
||||
m = q
|
||||
while tail:
|
||||
i = tail.find('.')
|
||||
if i < 0: i = len(tail)
|
||||
head, tail = tail[:i], tail[i+1:]
|
||||
mname = "%s.%s" % (m.__name__, head)
|
||||
m = self.import_module(head, mname, m)
|
||||
if not m:
|
||||
self.msgout(4, "raise ImportError: No module named", mname)
|
||||
raise ImportError("No module named " + mname)
|
||||
self.msgout(4, "load_tail ->", m)
|
||||
return m
|
||||
|
||||
def ensure_fromlist(self, m, fromlist, recursive=0):
|
||||
self.msg(4, "ensure_fromlist", m, fromlist, recursive)
|
||||
for sub in fromlist:
|
||||
if sub == "*":
|
||||
if not recursive:
|
||||
all = self.find_all_submodules(m)
|
||||
if all:
|
||||
self.ensure_fromlist(m, all, 1)
|
||||
elif not hasattr(m, sub):
|
||||
subname = "%s.%s" % (m.__name__, sub)
|
||||
submod = self.import_module(sub, subname, m)
|
||||
if not submod:
|
||||
raise ImportError("No module named " + subname)
|
||||
|
||||
def find_all_submodules(self, m):
|
||||
if not m.__path__:
|
||||
return
|
||||
modules = {}
|
||||
# 'suffixes' used to be a list hardcoded to [".py", ".pyc"].
|
||||
# But we must also collect Python extension modules - although
|
||||
# we cannot separate normal dlls from Python extensions.
|
||||
suffixes = []
|
||||
suffixes += importlib.machinery.EXTENSION_SUFFIXES[:]
|
||||
suffixes += importlib.machinery.SOURCE_SUFFIXES[:]
|
||||
suffixes += importlib.machinery.BYTECODE_SUFFIXES[:]
|
||||
for dir in m.__path__:
|
||||
try:
|
||||
names = os.listdir(dir)
|
||||
except OSError:
|
||||
self.msg(2, "can't list directory", dir)
|
||||
continue
|
||||
for name in names:
|
||||
mod = None
|
||||
for suff in suffixes:
|
||||
n = len(suff)
|
||||
if name[-n:] == suff:
|
||||
mod = name[:-n]
|
||||
break
|
||||
if mod and mod != "__init__":
|
||||
modules[mod] = mod
|
||||
return modules.keys()
|
||||
|
||||
def import_module(self, partname, fqname, parent):
|
||||
self.msgin(3, "import_module", partname, fqname, parent)
|
||||
try:
|
||||
m = self.modules[fqname]
|
||||
except KeyError:
|
||||
pass
|
||||
else:
|
||||
self.msgout(3, "import_module ->", m)
|
||||
return m
|
||||
if fqname in self.badmodules:
|
||||
self.msgout(3, "import_module -> None")
|
||||
return None
|
||||
if parent and parent.__path__ is None:
|
||||
self.msgout(3, "import_module -> None")
|
||||
return None
|
||||
try:
|
||||
fp, pathname, stuff = self.find_module(partname,
|
||||
parent and parent.__path__, parent)
|
||||
except ImportError:
|
||||
self.msgout(3, "import_module ->", None)
|
||||
return None
|
||||
|
||||
try:
|
||||
m = self.load_module(fqname, fp, pathname, stuff)
|
||||
finally:
|
||||
if fp:
|
||||
fp.close()
|
||||
if parent:
|
||||
setattr(parent, partname, m)
|
||||
self.msgout(3, "import_module ->", m)
|
||||
return m
|
||||
|
||||
def load_module(self, fqname, fp, pathname, file_info):
|
||||
suffix, mode, type = file_info
|
||||
self.msgin(2, "load_module", fqname, fp and "fp", pathname)
|
||||
if type == _PKG_DIRECTORY:
|
||||
m = self.load_package(fqname, pathname)
|
||||
self.msgout(2, "load_module ->", m)
|
||||
return m
|
||||
if type == _PY_SOURCE:
|
||||
co = compile(fp.read(), pathname, 'exec')
|
||||
elif type == _PY_COMPILED:
|
||||
try:
|
||||
data = fp.read()
|
||||
importlib._bootstrap_external._classify_pyc(data, fqname, {})
|
||||
except ImportError as exc:
|
||||
self.msgout(2, "raise ImportError: " + str(exc), pathname)
|
||||
raise
|
||||
co = marshal.loads(memoryview(data)[16:])
|
||||
else:
|
||||
co = None
|
||||
m = self.add_module(fqname)
|
||||
m.__file__ = pathname
|
||||
if co:
|
||||
if self.replace_paths:
|
||||
co = self.replace_paths_in_code(co)
|
||||
m.__code__ = co
|
||||
self.scan_code(co, m)
|
||||
self.msgout(2, "load_module ->", m)
|
||||
return m
|
||||
|
||||
def _add_badmodule(self, name, caller):
|
||||
if name not in self.badmodules:
|
||||
self.badmodules[name] = {}
|
||||
if caller:
|
||||
self.badmodules[name][caller.__name__] = 1
|
||||
else:
|
||||
self.badmodules[name]["-"] = 1
|
||||
|
||||
def _safe_import_hook(self, name, caller, fromlist, level=-1):
|
||||
# wrapper for self.import_hook() that won't raise ImportError
|
||||
if name in self.badmodules:
|
||||
self._add_badmodule(name, caller)
|
||||
return
|
||||
try:
|
||||
self.import_hook(name, caller, level=level)
|
||||
except ImportError as msg:
|
||||
self.msg(2, "ImportError:", str(msg))
|
||||
self._add_badmodule(name, caller)
|
||||
except SyntaxError as msg:
|
||||
self.msg(2, "SyntaxError:", str(msg))
|
||||
self._add_badmodule(name, caller)
|
||||
else:
|
||||
if fromlist:
|
||||
for sub in fromlist:
|
||||
fullname = name + "." + sub
|
||||
if fullname in self.badmodules:
|
||||
self._add_badmodule(fullname, caller)
|
||||
continue
|
||||
try:
|
||||
self.import_hook(name, caller, [sub], level=level)
|
||||
except ImportError as msg:
|
||||
self.msg(2, "ImportError:", str(msg))
|
||||
self._add_badmodule(fullname, caller)
|
||||
|
||||
def scan_opcodes(self, co):
|
||||
# Scan the code, and yield 'interesting' opcode combinations
|
||||
for name in dis._find_store_names(co):
|
||||
yield "store", (name,)
|
||||
for name, level, fromlist in dis._find_imports(co):
|
||||
if level == 0: # absolute import
|
||||
yield "absolute_import", (fromlist, name)
|
||||
else: # relative import
|
||||
yield "relative_import", (level, fromlist, name)
|
||||
|
||||
def scan_code(self, co, m):
|
||||
code = co.co_code
|
||||
scanner = self.scan_opcodes
|
||||
for what, args in scanner(co):
|
||||
if what == "store":
|
||||
name, = args
|
||||
m.globalnames[name] = 1
|
||||
elif what == "absolute_import":
|
||||
fromlist, name = args
|
||||
have_star = 0
|
||||
if fromlist is not None:
|
||||
if "*" in fromlist:
|
||||
have_star = 1
|
||||
fromlist = [f for f in fromlist if f != "*"]
|
||||
self._safe_import_hook(name, m, fromlist, level=0)
|
||||
if have_star:
|
||||
# We've encountered an "import *". If it is a Python module,
|
||||
# the code has already been parsed and we can suck out the
|
||||
# global names.
|
||||
mm = None
|
||||
if m.__path__:
|
||||
# At this point we don't know whether 'name' is a
|
||||
# submodule of 'm' or a global module. Let's just try
|
||||
# the full name first.
|
||||
mm = self.modules.get(m.__name__ + "." + name)
|
||||
if mm is None:
|
||||
mm = self.modules.get(name)
|
||||
if mm is not None:
|
||||
m.globalnames.update(mm.globalnames)
|
||||
m.starimports.update(mm.starimports)
|
||||
if mm.__code__ is None:
|
||||
m.starimports[name] = 1
|
||||
else:
|
||||
m.starimports[name] = 1
|
||||
elif what == "relative_import":
|
||||
level, fromlist, name = args
|
||||
if name:
|
||||
self._safe_import_hook(name, m, fromlist, level=level)
|
||||
else:
|
||||
parent = self.determine_parent(m, level=level)
|
||||
self._safe_import_hook(parent.__name__, None, fromlist, level=0)
|
||||
else:
|
||||
# We don't expect anything else from the generator.
|
||||
raise RuntimeError(what)
|
||||
|
||||
for c in co.co_consts:
|
||||
if isinstance(c, type(co)):
|
||||
self.scan_code(c, m)
|
||||
|
||||
def load_package(self, fqname, pathname):
|
||||
self.msgin(2, "load_package", fqname, pathname)
|
||||
newname = replacePackageMap.get(fqname)
|
||||
if newname:
|
||||
fqname = newname
|
||||
m = self.add_module(fqname)
|
||||
m.__file__ = pathname
|
||||
m.__path__ = [pathname]
|
||||
|
||||
# As per comment at top of file, simulate runtime __path__ additions.
|
||||
m.__path__ = m.__path__ + packagePathMap.get(fqname, [])
|
||||
|
||||
fp, buf, stuff = self.find_module("__init__", m.__path__)
|
||||
try:
|
||||
self.load_module(fqname, fp, buf, stuff)
|
||||
self.msgout(2, "load_package ->", m)
|
||||
return m
|
||||
finally:
|
||||
if fp:
|
||||
fp.close()
|
||||
|
||||
def add_module(self, fqname):
|
||||
if fqname in self.modules:
|
||||
return self.modules[fqname]
|
||||
self.modules[fqname] = m = Module(fqname)
|
||||
return m
|
||||
|
||||
def find_module(self, name, path, parent=None):
|
||||
if parent is not None:
|
||||
# assert path is not None
|
||||
fullname = parent.__name__+'.'+name
|
||||
else:
|
||||
fullname = name
|
||||
if fullname in self.excludes:
|
||||
self.msgout(3, "find_module -> Excluded", fullname)
|
||||
raise ImportError(name)
|
||||
|
||||
if path is None:
|
||||
if name in sys.builtin_module_names:
|
||||
return (None, None, ("", "", _C_BUILTIN))
|
||||
|
||||
path = self.path
|
||||
|
||||
return _find_module(name, path)
|
||||
|
||||
def report(self):
|
||||
"""Print a report to stdout, listing the found modules with their
|
||||
paths, as well as modules that are missing, or seem to be missing.
|
||||
"""
|
||||
print()
|
||||
print(" %-25s %s" % ("Name", "File"))
|
||||
print(" %-25s %s" % ("----", "----"))
|
||||
# Print modules found
|
||||
keys = sorted(self.modules.keys())
|
||||
for key in keys:
|
||||
m = self.modules[key]
|
||||
if m.__path__:
|
||||
print("P", end=' ')
|
||||
else:
|
||||
print("m", end=' ')
|
||||
print("%-25s" % key, m.__file__ or "")
|
||||
|
||||
# Print missing modules
|
||||
missing, maybe = self.any_missing_maybe()
|
||||
if missing:
|
||||
print()
|
||||
print("Missing modules:")
|
||||
for name in missing:
|
||||
mods = sorted(self.badmodules[name].keys())
|
||||
print("?", name, "imported from", ', '.join(mods))
|
||||
# Print modules that may be missing, but then again, maybe not...
|
||||
if maybe:
|
||||
print()
|
||||
print("Submodules that appear to be missing, but could also be", end=' ')
|
||||
print("global names in the parent package:")
|
||||
for name in maybe:
|
||||
mods = sorted(self.badmodules[name].keys())
|
||||
print("?", name, "imported from", ', '.join(mods))
|
||||
|
||||
def any_missing(self):
|
||||
"""Return a list of modules that appear to be missing. Use
|
||||
any_missing_maybe() if you want to know which modules are
|
||||
certain to be missing, and which *may* be missing.
|
||||
"""
|
||||
missing, maybe = self.any_missing_maybe()
|
||||
return missing + maybe
|
||||
|
||||
def any_missing_maybe(self):
|
||||
"""Return two lists, one with modules that are certainly missing
|
||||
and one with modules that *may* be missing. The latter names could
|
||||
either be submodules *or* just global names in the package.
|
||||
|
||||
The reason it can't always be determined is that it's impossible to
|
||||
tell which names are imported when "from module import *" is done
|
||||
with an extension module, short of actually importing it.
|
||||
"""
|
||||
missing = []
|
||||
maybe = []
|
||||
for name in self.badmodules:
|
||||
if name in self.excludes:
|
||||
continue
|
||||
i = name.rfind(".")
|
||||
if i < 0:
|
||||
missing.append(name)
|
||||
continue
|
||||
subname = name[i+1:]
|
||||
pkgname = name[:i]
|
||||
pkg = self.modules.get(pkgname)
|
||||
if pkg is not None:
|
||||
if pkgname in self.badmodules[name]:
|
||||
# The package tried to import this module itself and
|
||||
# failed. It's definitely missing.
|
||||
missing.append(name)
|
||||
elif subname in pkg.globalnames:
|
||||
# It's a global in the package: definitely not missing.
|
||||
pass
|
||||
elif pkg.starimports:
|
||||
# It could be missing, but the package did an "import *"
|
||||
# from a non-Python module, so we simply can't be sure.
|
||||
maybe.append(name)
|
||||
else:
|
||||
# It's not a global in the package, the package didn't
|
||||
# do funny star imports, it's very likely to be missing.
|
||||
# The symbol could be inserted into the package from the
|
||||
# outside, but since that's not good style we simply list
|
||||
# it missing.
|
||||
missing.append(name)
|
||||
else:
|
||||
missing.append(name)
|
||||
missing.sort()
|
||||
maybe.sort()
|
||||
return missing, maybe
|
||||
|
||||
def replace_paths_in_code(self, co):
|
||||
new_filename = original_filename = os.path.normpath(co.co_filename)
|
||||
for f, r in self.replace_paths:
|
||||
if original_filename.startswith(f):
|
||||
new_filename = r + original_filename[len(f):]
|
||||
break
|
||||
|
||||
if self.debug and original_filename not in self.processed_paths:
|
||||
if new_filename != original_filename:
|
||||
self.msgout(2, "co_filename %r changed to %r" \
|
||||
% (original_filename,new_filename,))
|
||||
else:
|
||||
self.msgout(2, "co_filename %r remains unchanged" \
|
||||
% (original_filename,))
|
||||
self.processed_paths.append(original_filename)
|
||||
|
||||
consts = list(co.co_consts)
|
||||
for i in range(len(consts)):
|
||||
if isinstance(consts[i], type(co)):
|
||||
consts[i] = self.replace_paths_in_code(consts[i])
|
||||
|
||||
return co.replace(co_consts=tuple(consts), co_filename=new_filename)
|
||||
|
||||
|
||||
def test():
|
||||
# Parse command line
|
||||
import getopt
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:")
|
||||
except getopt.error as msg:
|
||||
print(msg)
|
||||
return
|
||||
|
||||
# Process options
|
||||
debug = 1
|
||||
domods = 0
|
||||
addpath = []
|
||||
exclude = []
|
||||
for o, a in opts:
|
||||
if o == '-d':
|
||||
debug = debug + 1
|
||||
if o == '-m':
|
||||
domods = 1
|
||||
if o == '-p':
|
||||
addpath = addpath + a.split(os.pathsep)
|
||||
if o == '-q':
|
||||
debug = 0
|
||||
if o == '-x':
|
||||
exclude.append(a)
|
||||
|
||||
# Provide default arguments
|
||||
if not args:
|
||||
script = "hello.py"
|
||||
else:
|
||||
script = args[0]
|
||||
|
||||
# Set the path based on sys.path and the script directory
|
||||
path = sys.path[:]
|
||||
path[0] = os.path.dirname(script)
|
||||
path = addpath + path
|
||||
if debug > 1:
|
||||
print("path:")
|
||||
for item in path:
|
||||
print(" ", repr(item))
|
||||
|
||||
# Create the module finder and turn its crank
|
||||
mf = ModuleFinder(path, debug, exclude)
|
||||
for arg in args[1:]:
|
||||
if arg == '-m':
|
||||
domods = 1
|
||||
continue
|
||||
if domods:
|
||||
if arg[-2:] == '.*':
|
||||
mf.import_hook(arg[:-2], None, ["*"])
|
||||
else:
|
||||
mf.import_hook(arg)
|
||||
else:
|
||||
mf.load_file(arg)
|
||||
mf.run_script(script)
|
||||
mf.report()
|
||||
return mf # for -i debugging
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
mf = test()
|
||||
except KeyboardInterrupt:
|
||||
print("\n[interrupted]")
|
||||
Vendored
+438
@@ -0,0 +1,438 @@
|
||||
import os
|
||||
import errno
|
||||
import importlib.machinery
|
||||
import py_compile
|
||||
import shutil
|
||||
import unittest
|
||||
import tempfile
|
||||
|
||||
from test import support
|
||||
|
||||
import modulefinder
|
||||
|
||||
# Each test description is a list of 5 items:
|
||||
#
|
||||
# 1. a module name that will be imported by modulefinder
|
||||
# 2. a list of module names that modulefinder is required to find
|
||||
# 3. a list of module names that modulefinder should complain
|
||||
# about because they are not found
|
||||
# 4. a list of module names that modulefinder should complain
|
||||
# about because they MAY be not found
|
||||
# 5. a string specifying packages to create; the format is obvious imo.
|
||||
#
|
||||
# Each package will be created in test_dir, and test_dir will be
|
||||
# removed after the tests again.
|
||||
# Modulefinder searches in a path that contains test_dir, plus
|
||||
# the standard Lib directory.
|
||||
|
||||
maybe_test = [
|
||||
"a.module",
|
||||
["a", "a.module", "sys",
|
||||
"b"],
|
||||
["c"], ["b.something"],
|
||||
"""\
|
||||
a/__init__.py
|
||||
a/module.py
|
||||
from b import something
|
||||
from c import something
|
||||
b/__init__.py
|
||||
from sys import *
|
||||
""",
|
||||
]
|
||||
|
||||
maybe_test_new = [
|
||||
"a.module",
|
||||
["a", "a.module", "sys",
|
||||
"b", "__future__"],
|
||||
["c"], ["b.something"],
|
||||
"""\
|
||||
a/__init__.py
|
||||
a/module.py
|
||||
from b import something
|
||||
from c import something
|
||||
b/__init__.py
|
||||
from __future__ import absolute_import
|
||||
from sys import *
|
||||
"""]
|
||||
|
||||
package_test = [
|
||||
"a.module",
|
||||
["a", "a.b", "a.c", "a.module", "mymodule", "sys"],
|
||||
["blahblah", "c"], [],
|
||||
"""\
|
||||
mymodule.py
|
||||
a/__init__.py
|
||||
import blahblah
|
||||
from a import b
|
||||
import c
|
||||
a/module.py
|
||||
import sys
|
||||
from a import b as x
|
||||
from a.c import sillyname
|
||||
a/b.py
|
||||
a/c.py
|
||||
from a.module import x
|
||||
import mymodule as sillyname
|
||||
from sys import version_info
|
||||
"""]
|
||||
|
||||
absolute_import_test = [
|
||||
"a.module",
|
||||
["a", "a.module",
|
||||
"b", "b.x", "b.y", "b.z",
|
||||
"__future__", "sys", "gc"],
|
||||
["blahblah", "z"], [],
|
||||
"""\
|
||||
mymodule.py
|
||||
a/__init__.py
|
||||
a/module.py
|
||||
from __future__ import absolute_import
|
||||
import sys # sys
|
||||
import blahblah # fails
|
||||
import gc # gc
|
||||
import b.x # b.x
|
||||
from b import y # b.y
|
||||
from b.z import * # b.z.*
|
||||
a/gc.py
|
||||
a/sys.py
|
||||
import mymodule
|
||||
a/b/__init__.py
|
||||
a/b/x.py
|
||||
a/b/y.py
|
||||
a/b/z.py
|
||||
b/__init__.py
|
||||
import z
|
||||
b/unused.py
|
||||
b/x.py
|
||||
b/y.py
|
||||
b/z.py
|
||||
"""]
|
||||
|
||||
relative_import_test = [
|
||||
"a.module",
|
||||
["__future__",
|
||||
"a", "a.module",
|
||||
"a.b", "a.b.y", "a.b.z",
|
||||
"a.b.c", "a.b.c.moduleC",
|
||||
"a.b.c.d", "a.b.c.e",
|
||||
"a.b.x",
|
||||
"gc"],
|
||||
[], [],
|
||||
"""\
|
||||
mymodule.py
|
||||
a/__init__.py
|
||||
from .b import y, z # a.b.y, a.b.z
|
||||
a/module.py
|
||||
from __future__ import absolute_import # __future__
|
||||
import gc # gc
|
||||
a/gc.py
|
||||
a/sys.py
|
||||
a/b/__init__.py
|
||||
from ..b import x # a.b.x
|
||||
#from a.b.c import moduleC
|
||||
from .c import moduleC # a.b.moduleC
|
||||
a/b/x.py
|
||||
a/b/y.py
|
||||
a/b/z.py
|
||||
a/b/g.py
|
||||
a/b/c/__init__.py
|
||||
from ..c import e # a.b.c.e
|
||||
a/b/c/moduleC.py
|
||||
from ..c import d # a.b.c.d
|
||||
a/b/c/d.py
|
||||
a/b/c/e.py
|
||||
a/b/c/x.py
|
||||
"""]
|
||||
|
||||
relative_import_test_2 = [
|
||||
"a.module",
|
||||
["a", "a.module",
|
||||
"a.sys",
|
||||
"a.b", "a.b.y", "a.b.z",
|
||||
"a.b.c", "a.b.c.d",
|
||||
"a.b.c.e",
|
||||
"a.b.c.moduleC",
|
||||
"a.b.c.f",
|
||||
"a.b.x",
|
||||
"a.another"],
|
||||
[], [],
|
||||
"""\
|
||||
mymodule.py
|
||||
a/__init__.py
|
||||
from . import sys # a.sys
|
||||
a/another.py
|
||||
a/module.py
|
||||
from .b import y, z # a.b.y, a.b.z
|
||||
a/gc.py
|
||||
a/sys.py
|
||||
a/b/__init__.py
|
||||
from .c import moduleC # a.b.c.moduleC
|
||||
from .c import d # a.b.c.d
|
||||
a/b/x.py
|
||||
a/b/y.py
|
||||
a/b/z.py
|
||||
a/b/c/__init__.py
|
||||
from . import e # a.b.c.e
|
||||
a/b/c/moduleC.py
|
||||
#
|
||||
from . import f # a.b.c.f
|
||||
from .. import x # a.b.x
|
||||
from ... import another # a.another
|
||||
a/b/c/d.py
|
||||
a/b/c/e.py
|
||||
a/b/c/f.py
|
||||
"""]
|
||||
|
||||
relative_import_test_3 = [
|
||||
"a.module",
|
||||
["a", "a.module"],
|
||||
["a.bar"],
|
||||
[],
|
||||
"""\
|
||||
a/__init__.py
|
||||
def foo(): pass
|
||||
a/module.py
|
||||
from . import foo
|
||||
from . import bar
|
||||
"""]
|
||||
|
||||
relative_import_test_4 = [
|
||||
"a.module",
|
||||
["a", "a.module"],
|
||||
[],
|
||||
[],
|
||||
"""\
|
||||
a/__init__.py
|
||||
def foo(): pass
|
||||
a/module.py
|
||||
from . import *
|
||||
"""]
|
||||
|
||||
bytecode_test = [
|
||||
"a",
|
||||
["a"],
|
||||
[],
|
||||
[],
|
||||
""
|
||||
]
|
||||
|
||||
syntax_error_test = [
|
||||
"a.module",
|
||||
["a", "a.module", "b"],
|
||||
["b.module"], [],
|
||||
"""\
|
||||
a/__init__.py
|
||||
a/module.py
|
||||
import b.module
|
||||
b/__init__.py
|
||||
b/module.py
|
||||
? # SyntaxError: invalid syntax
|
||||
"""]
|
||||
|
||||
|
||||
same_name_as_bad_test = [
|
||||
"a.module",
|
||||
["a", "a.module", "b", "b.c"],
|
||||
["c"], [],
|
||||
"""\
|
||||
a/__init__.py
|
||||
a/module.py
|
||||
import c
|
||||
from b import c
|
||||
b/__init__.py
|
||||
b/c.py
|
||||
"""]
|
||||
|
||||
coding_default_utf8_test = [
|
||||
"a_utf8",
|
||||
["a_utf8", "b_utf8"],
|
||||
[], [],
|
||||
"""\
|
||||
a_utf8.py
|
||||
# use the default of utf8
|
||||
print('Unicode test A code point 2090 \u2090 that is not valid in cp1252')
|
||||
import b_utf8
|
||||
b_utf8.py
|
||||
# use the default of utf8
|
||||
print('Unicode test B code point 2090 \u2090 that is not valid in cp1252')
|
||||
"""]
|
||||
|
||||
coding_explicit_utf8_test = [
|
||||
"a_utf8",
|
||||
["a_utf8", "b_utf8"],
|
||||
[], [],
|
||||
"""\
|
||||
a_utf8.py
|
||||
# coding=utf8
|
||||
print('Unicode test A code point 2090 \u2090 that is not valid in cp1252')
|
||||
import b_utf8
|
||||
b_utf8.py
|
||||
# use the default of utf8
|
||||
print('Unicode test B code point 2090 \u2090 that is not valid in cp1252')
|
||||
"""]
|
||||
|
||||
coding_explicit_cp1252_test = [
|
||||
"a_cp1252",
|
||||
["a_cp1252", "b_utf8"],
|
||||
[], [],
|
||||
b"""\
|
||||
a_cp1252.py
|
||||
# coding=cp1252
|
||||
# 0xe2 is not allowed in utf8
|
||||
print('CP1252 test P\xe2t\xe9')
|
||||
import b_utf8
|
||||
""" + """\
|
||||
b_utf8.py
|
||||
# use the default of utf8
|
||||
print('Unicode test A code point 2090 \u2090 that is not valid in cp1252')
|
||||
""".encode('utf-8')]
|
||||
|
||||
def open_file(path):
|
||||
dirname = os.path.dirname(path)
|
||||
try:
|
||||
os.makedirs(dirname)
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
return open(path, 'wb')
|
||||
|
||||
|
||||
def create_package(test_dir, source):
|
||||
ofi = None
|
||||
try:
|
||||
for line in source.splitlines():
|
||||
if type(line) != bytes:
|
||||
line = line.encode('utf-8')
|
||||
if line.startswith(b' ') or line.startswith(b'\t'):
|
||||
ofi.write(line.strip() + b'\n')
|
||||
else:
|
||||
if ofi:
|
||||
ofi.close()
|
||||
if type(line) == bytes:
|
||||
line = line.decode('utf-8')
|
||||
ofi = open_file(os.path.join(test_dir, line.strip()))
|
||||
finally:
|
||||
if ofi:
|
||||
ofi.close()
|
||||
|
||||
class ModuleFinderTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.test_dir = tempfile.mkdtemp()
|
||||
self.test_path = [self.test_dir, os.path.dirname(tempfile.__file__)]
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.test_dir)
|
||||
|
||||
def _do_test(self, info, report=False, debug=0, replace_paths=[], modulefinder_class=modulefinder.ModuleFinder):
|
||||
import_this, modules, missing, maybe_missing, source = info
|
||||
create_package(self.test_dir, source)
|
||||
mf = modulefinder_class(path=self.test_path, debug=debug,
|
||||
replace_paths=replace_paths)
|
||||
mf.import_hook(import_this)
|
||||
if report:
|
||||
mf.report()
|
||||
## # This wouldn't work in general when executed several times:
|
||||
## opath = sys.path[:]
|
||||
## sys.path = self.test_path
|
||||
## try:
|
||||
## __import__(import_this)
|
||||
## except:
|
||||
## import traceback; traceback.print_exc()
|
||||
## sys.path = opath
|
||||
## return
|
||||
modules = sorted(set(modules))
|
||||
found = sorted(mf.modules)
|
||||
# check if we found what we expected, not more, not less
|
||||
self.assertEqual(found, modules)
|
||||
|
||||
# check for missing and maybe missing modules
|
||||
bad, maybe = mf.any_missing_maybe()
|
||||
self.assertEqual(bad, missing)
|
||||
self.assertEqual(maybe, maybe_missing)
|
||||
|
||||
def test_package(self):
|
||||
self._do_test(package_test)
|
||||
|
||||
def test_maybe(self):
|
||||
self._do_test(maybe_test)
|
||||
|
||||
def test_maybe_new(self):
|
||||
self._do_test(maybe_test_new)
|
||||
|
||||
def test_absolute_imports(self):
|
||||
self._do_test(absolute_import_test)
|
||||
|
||||
def test_relative_imports(self):
|
||||
self._do_test(relative_import_test)
|
||||
|
||||
def test_relative_imports_2(self):
|
||||
self._do_test(relative_import_test_2)
|
||||
|
||||
def test_relative_imports_3(self):
|
||||
self._do_test(relative_import_test_3)
|
||||
|
||||
def test_relative_imports_4(self):
|
||||
self._do_test(relative_import_test_4)
|
||||
|
||||
def test_syntax_error(self):
|
||||
self._do_test(syntax_error_test)
|
||||
|
||||
def test_same_name_as_bad(self):
|
||||
self._do_test(same_name_as_bad_test)
|
||||
|
||||
def test_bytecode(self):
|
||||
base_path = os.path.join(self.test_dir, 'a')
|
||||
source_path = base_path + importlib.machinery.SOURCE_SUFFIXES[0]
|
||||
bytecode_path = base_path + importlib.machinery.BYTECODE_SUFFIXES[0]
|
||||
with open_file(source_path) as file:
|
||||
file.write('testing_modulefinder = True\n'.encode('utf-8'))
|
||||
py_compile.compile(source_path, cfile=bytecode_path)
|
||||
os.remove(source_path)
|
||||
self._do_test(bytecode_test)
|
||||
|
||||
def test_replace_paths(self):
|
||||
old_path = os.path.join(self.test_dir, 'a', 'module.py')
|
||||
new_path = os.path.join(self.test_dir, 'a', 'spam.py')
|
||||
with support.captured_stdout() as output:
|
||||
self._do_test(maybe_test, debug=2,
|
||||
replace_paths=[(old_path, new_path)])
|
||||
output = output.getvalue()
|
||||
expected = "co_filename %r changed to %r" % (old_path, new_path)
|
||||
self.assertIn(expected, output)
|
||||
|
||||
def test_extended_opargs(self):
|
||||
extended_opargs_test = [
|
||||
"a",
|
||||
["a", "b"],
|
||||
[], [],
|
||||
"""\
|
||||
a.py
|
||||
%r
|
||||
import b
|
||||
b.py
|
||||
""" % list(range(2**16))] # 2**16 constants
|
||||
self._do_test(extended_opargs_test)
|
||||
|
||||
def test_coding_default_utf8(self):
|
||||
self._do_test(coding_default_utf8_test)
|
||||
|
||||
def test_coding_explicit_utf8(self):
|
||||
self._do_test(coding_explicit_utf8_test)
|
||||
|
||||
def test_coding_explicit_cp1252(self):
|
||||
self._do_test(coding_explicit_cp1252_test)
|
||||
|
||||
def test_load_module_api(self):
|
||||
class CheckLoadModuleApi(modulefinder.ModuleFinder):
|
||||
def __init__(self, *args, **kwds):
|
||||
super().__init__(*args, **kwds)
|
||||
|
||||
def load_module(self, fqname, fp, pathname, file_info):
|
||||
# confirm that the fileinfo is a tuple of 3 elements
|
||||
suffix, mode, type = file_info
|
||||
return super().load_module(fqname, fp, pathname, file_info)
|
||||
|
||||
self._do_test(absolute_import_test, modulefinder_class=CheckLoadModuleApi)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user