mirror of
https://github.com/RustPython/RustPython.git
synced 2026-10-28 18:59:04 +08:00
Improve deps output (#6874)
* Improve deps output: [x]/[ ] sync status, TODO counts - Replace ambiguous [+] with [x] (synced) / [ ] (not synced) - Add (TODO: n) suffix for test files with expectedFailure/skip markers * Refactor update_lib: extract shared utilities
This commit is contained in:
Jeong, YunWon authored and GitHub committed
1 parent
93b26bf595
commit
6a3643cdde
17 files changed
+528
-572
No files matched your search
@@ -63,39 +63,39 @@ def main(argv: list[str] | None = None) -> int:
|
||||
args, remaining = parser.parse_known_args(argv)
|
||||
|
||||
if args.command == "quick":
|
||||
from update_lib.quick import main as quick_main
|
||||
from update_lib.cmd_quick import main as quick_main
|
||||
|
||||
return quick_main(remaining)
|
||||
|
||||
if args.command == "copy-lib":
|
||||
from update_lib.copy_lib import main as copy_lib_main
|
||||
from update_lib.cmd_copy_lib import main as copy_lib_main
|
||||
|
||||
return copy_lib_main(remaining)
|
||||
|
||||
if args.command == "migrate":
|
||||
from update_lib.migrate import main as migrate_main
|
||||
from update_lib.cmd_migrate import main as migrate_main
|
||||
|
||||
return migrate_main(remaining)
|
||||
|
||||
if args.command == "patches":
|
||||
from update_lib.patches import main as patches_main
|
||||
from update_lib.cmd_patches import main as patches_main
|
||||
|
||||
return patches_main(remaining)
|
||||
|
||||
if args.command == "auto-mark":
|
||||
from update_lib.auto_mark import main as auto_mark_main
|
||||
from update_lib.cmd_auto_mark import main as cmd_auto_mark_main
|
||||
|
||||
return auto_mark_main(remaining)
|
||||
return cmd_auto_mark_main(remaining)
|
||||
|
||||
if args.command == "deps":
|
||||
from update_lib.show_deps import main as show_deps_main
|
||||
from update_lib.cmd_deps import main as cmd_deps_main
|
||||
|
||||
return show_deps_main(remaining)
|
||||
return cmd_deps_main(remaining)
|
||||
|
||||
if args.command == "todo":
|
||||
from update_lib.show_todo import main as show_todo_main
|
||||
from update_lib.cmd_todo import main as cmd_todo_main
|
||||
|
||||
return show_todo_main(remaining)
|
||||
return cmd_todo_main(remaining)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ from dataclasses import dataclass, field
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))
|
||||
|
||||
from update_lib import COMMENT, PatchSpec, UtMethod, apply_patches
|
||||
from update_lib.path import test_name_from_path
|
||||
from update_lib.file_utils import get_test_module_name
|
||||
|
||||
|
||||
class TestRunError(Exception):
|
||||
@@ -455,7 +455,7 @@ def extract_test_methods(contents: str) -> set[tuple[str, str]]:
|
||||
Returns:
|
||||
Set of (class_name, method_name) tuples
|
||||
"""
|
||||
from update_lib.io_utils import safe_parse_ast
|
||||
from update_lib.file_utils import safe_parse_ast
|
||||
from update_lib.patch_spec import iter_tests
|
||||
|
||||
tree = safe_parse_ast(contents)
|
||||
@@ -490,7 +490,7 @@ def auto_mark_file(
|
||||
if not test_path.exists():
|
||||
raise FileNotFoundError(f"File not found: {test_path}")
|
||||
|
||||
test_name = test_name_from_path(test_path)
|
||||
test_name = get_test_module_name(test_path)
|
||||
if verbose:
|
||||
print(f"Running test: {test_name}")
|
||||
|
||||
@@ -587,7 +587,7 @@ def auto_mark_directory(
|
||||
if not test_dir.is_dir():
|
||||
raise ValueError(f"Not a directory: {test_dir}")
|
||||
|
||||
test_name = test_name_from_path(test_dir)
|
||||
test_name = get_test_module_name(test_dir)
|
||||
if verbose:
|
||||
print(f"Running test: {test_name}")
|
||||
|
||||
@@ -610,7 +610,7 @@ def auto_mark_directory(
|
||||
|
||||
for test_file in test_files:
|
||||
# Get module prefix for this file (e.g., "test_inspect.test_inspect")
|
||||
module_prefix = test_name_from_path(test_file)
|
||||
module_prefix = get_test_module_name(test_file)
|
||||
# For __init__.py, the test path doesn't include "__init__"
|
||||
if module_prefix.endswith(".__init__"):
|
||||
module_prefix = module_prefix[:-9] # Remove ".__init__"
|
||||
@@ -60,7 +60,7 @@ def copy_lib(
|
||||
verbose: Print progress messages
|
||||
"""
|
||||
from update_lib.deps import get_lib_paths
|
||||
from update_lib.path import parse_lib_path
|
||||
from update_lib.file_utils import parse_lib_path
|
||||
|
||||
# Extract module name and cpython prefix from path
|
||||
path_str = str(src_path).replace("\\", "/")
|
||||
@@ -189,9 +189,12 @@ def format_deps(
|
||||
"""
|
||||
from update_lib.deps import (
|
||||
DEPENDENCIES,
|
||||
count_test_todos,
|
||||
find_dependent_tests_tree,
|
||||
get_lib_paths,
|
||||
get_test_paths,
|
||||
is_path_synced,
|
||||
is_test_up_to_date,
|
||||
resolve_hard_dep_parent,
|
||||
)
|
||||
|
||||
@@ -216,13 +219,20 @@ def format_deps(
|
||||
lib_paths = get_lib_paths(name, cpython_prefix)
|
||||
existing_lib_paths = [p for p in lib_paths if p.exists()]
|
||||
for p in existing_lib_paths:
|
||||
lines.append(f"[+] lib: {p}")
|
||||
synced = is_path_synced(p, cpython_prefix, lib_prefix)
|
||||
marker = "[x]" if synced else "[ ]"
|
||||
lines.append(f"{marker} lib: {p}")
|
||||
|
||||
# test paths (only show existing)
|
||||
test_paths = get_test_paths(name, cpython_prefix)
|
||||
existing_test_paths = [p for p in test_paths if p.exists()]
|
||||
for p in existing_test_paths:
|
||||
lines.append(f"[+] test: {p}")
|
||||
test_name = p.stem if p.is_file() else p.name
|
||||
synced = is_test_up_to_date(test_name, cpython_prefix, lib_prefix)
|
||||
marker = "[x]" if synced else "[ ]"
|
||||
todo_count = count_test_todos(test_name, lib_prefix)
|
||||
todo_suffix = f" (TODO: {todo_count})" if todo_count > 0 else ""
|
||||
lines.append(f"{marker} test: {p}{todo_suffix}")
|
||||
|
||||
# If no lib or test paths exist, module doesn't exist
|
||||
if not existing_lib_paths and not existing_test_paths:
|
||||
@@ -17,7 +17,7 @@ import sys
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))
|
||||
|
||||
from update_lib.path import parse_lib_path
|
||||
from update_lib.file_utils import parse_lib_path
|
||||
|
||||
|
||||
def patch_single_content(
|
||||
File renamed without changes.
@@ -32,9 +32,9 @@ import sys
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))
|
||||
|
||||
from update_lib.deps import get_test_paths
|
||||
from update_lib.io_utils import safe_read_text
|
||||
from update_lib.path import (
|
||||
from update_lib.file_utils import (
|
||||
construct_lib_path,
|
||||
get_cpython_dir,
|
||||
get_module_name,
|
||||
get_test_files,
|
||||
is_lib_path,
|
||||
@@ -42,6 +42,7 @@ from update_lib.path import (
|
||||
lib_to_test_path,
|
||||
parse_lib_path,
|
||||
resolve_module_path,
|
||||
safe_read_text,
|
||||
)
|
||||
|
||||
|
||||
@@ -55,7 +56,7 @@ def collect_original_methods(
|
||||
- For file: set of (class_name, method_name) or None if file doesn't exist
|
||||
- For directory: dict mapping file path to set of methods, or None if dir doesn't exist
|
||||
"""
|
||||
from update_lib.auto_mark import extract_test_methods
|
||||
from update_lib.cmd_auto_mark import extract_test_methods
|
||||
|
||||
if not lib_path.exists():
|
||||
return None
|
||||
@@ -91,8 +92,8 @@ def quick(
|
||||
verbose: Print progress messages
|
||||
skip_build: Skip cargo build, use pre-built binary
|
||||
"""
|
||||
from update_lib.auto_mark import auto_mark_directory, auto_mark_file
|
||||
from update_lib.migrate import patch_directory, patch_file
|
||||
from update_lib.cmd_auto_mark import auto_mark_directory, auto_mark_file
|
||||
from update_lib.cmd_migrate import patch_directory, patch_file
|
||||
|
||||
# Determine lib_path and whether to migrate
|
||||
if is_lib_path(src_path):
|
||||
@@ -174,22 +175,6 @@ def quick(
|
||||
print(f"Removed expectedFailure from {num_removed} tests")
|
||||
|
||||
|
||||
def get_cpython_dir(src_path: pathlib.Path) -> pathlib.Path:
|
||||
"""Extract cpython directory from source path.
|
||||
|
||||
Example:
|
||||
cpython/Lib/dataclasses.py -> cpython
|
||||
/some/path/cpython/Lib/foo.py -> /some/path/cpython
|
||||
"""
|
||||
path_str = str(src_path).replace("\\", "/")
|
||||
lib_marker = "/Lib/"
|
||||
if lib_marker in path_str:
|
||||
idx = path_str.index(lib_marker)
|
||||
return pathlib.Path(path_str[:idx])
|
||||
# Shortcut case: assume "cpython"
|
||||
return pathlib.Path("cpython")
|
||||
|
||||
|
||||
def get_cpython_version(cpython_dir: pathlib.Path) -> str:
|
||||
"""Get CPython version from git tag."""
|
||||
import subprocess
|
||||
@@ -384,7 +369,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
lib_file_path = parse_lib_path(src_path)
|
||||
|
||||
if args.copy:
|
||||
from update_lib.copy_lib import copy_lib
|
||||
from update_lib.cmd_copy_lib import copy_lib
|
||||
|
||||
copy_lib(src_path)
|
||||
|
||||
@@ -449,7 +434,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return 1
|
||||
except Exception as e:
|
||||
# Handle TestRunError with a clean message
|
||||
from update_lib.auto_mark import TestRunError
|
||||
from update_lib.cmd_auto_mark import TestRunError
|
||||
|
||||
if isinstance(e, TestRunError):
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
@@ -13,6 +13,12 @@ import sys
|
||||
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))
|
||||
|
||||
from update_lib.deps import (
|
||||
count_test_todos,
|
||||
is_test_tracked,
|
||||
is_test_up_to_date,
|
||||
)
|
||||
|
||||
|
||||
def compute_todo_list(
|
||||
cpython_prefix: str,
|
||||
@@ -32,13 +38,13 @@ def compute_todo_list(
|
||||
Returns:
|
||||
List of dicts with module info, sorted by priority
|
||||
"""
|
||||
from update_lib.cmd_deps import get_all_modules
|
||||
from update_lib.deps import (
|
||||
get_all_hard_deps,
|
||||
get_rust_deps,
|
||||
get_soft_deps,
|
||||
is_up_to_date,
|
||||
)
|
||||
from update_lib.show_deps import get_all_modules
|
||||
|
||||
all_modules = get_all_modules(cpython_prefix)
|
||||
|
||||
@@ -164,8 +170,8 @@ def get_untracked_files(
|
||||
Returns:
|
||||
Sorted list of relative paths (e.g., ["foo.py", "data/file.txt"])
|
||||
"""
|
||||
from update_lib.cmd_deps import get_all_modules
|
||||
from update_lib.deps import resolve_hard_dep_parent
|
||||
from update_lib.show_deps import get_all_modules
|
||||
|
||||
cpython_lib = pathlib.Path(cpython_prefix) / "Lib"
|
||||
local_lib = pathlib.Path(lib_prefix)
|
||||
@@ -260,127 +266,6 @@ def get_original_files(
|
||||
return sorted(original)
|
||||
|
||||
|
||||
def _filter_rustpython_todo(content: str) -> str:
|
||||
"""Remove lines containing 'TODO: RUSTPYTHON' from content."""
|
||||
lines = content.splitlines(keepends=True)
|
||||
filtered = [line for line in lines if "TODO: RUSTPYTHON" not in line]
|
||||
return "".join(filtered)
|
||||
|
||||
|
||||
def _count_rustpython_todo(content: str) -> int:
|
||||
"""Count lines containing 'TODO: RUSTPYTHON' in content."""
|
||||
return sum(1 for line in content.splitlines() if "TODO: RUSTPYTHON" in line)
|
||||
|
||||
|
||||
def _compare_file_ignoring_todo(
|
||||
cpython_path: pathlib.Path, local_path: pathlib.Path
|
||||
) -> bool:
|
||||
"""Compare two files, ignoring TODO: RUSTPYTHON lines in local file."""
|
||||
try:
|
||||
cpython_content = cpython_path.read_text(encoding="utf-8")
|
||||
local_content = local_path.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return False
|
||||
|
||||
local_filtered = _filter_rustpython_todo(local_content)
|
||||
return cpython_content == local_filtered
|
||||
|
||||
|
||||
def _compare_dir_ignoring_todo(
|
||||
cpython_path: pathlib.Path, local_path: pathlib.Path
|
||||
) -> bool:
|
||||
"""Compare two directories, ignoring TODO: RUSTPYTHON lines in local files."""
|
||||
# Get all .py files in both directories
|
||||
cpython_files = {f.relative_to(cpython_path) for f in cpython_path.rglob("*.py")}
|
||||
local_files = {f.relative_to(local_path) for f in local_path.rglob("*.py")}
|
||||
|
||||
# Check for missing or extra files
|
||||
if cpython_files != local_files:
|
||||
return False
|
||||
|
||||
# Compare each file
|
||||
for rel_path in cpython_files:
|
||||
if not _compare_file_ignoring_todo(
|
||||
cpython_path / rel_path, local_path / rel_path
|
||||
):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def count_test_todos(test_name: str, lib_prefix: str) -> int:
|
||||
"""Count TODO: RUSTPYTHON lines in a test file/directory."""
|
||||
local_dir = pathlib.Path(lib_prefix) / "test" / test_name
|
||||
local_file = pathlib.Path(lib_prefix) / "test" / f"{test_name}.py"
|
||||
|
||||
if local_dir.exists():
|
||||
local_path = local_dir
|
||||
elif local_file.exists():
|
||||
local_path = local_file
|
||||
else:
|
||||
return 0
|
||||
|
||||
total = 0
|
||||
if local_path.is_file():
|
||||
try:
|
||||
content = local_path.read_text(encoding="utf-8")
|
||||
total = _count_rustpython_todo(content)
|
||||
except (OSError, UnicodeDecodeError):
|
||||
pass
|
||||
else:
|
||||
for py_file in local_path.rglob("*.py"):
|
||||
try:
|
||||
content = py_file.read_text(encoding="utf-8")
|
||||
total += _count_rustpython_todo(content)
|
||||
except (OSError, UnicodeDecodeError):
|
||||
pass
|
||||
|
||||
return total
|
||||
|
||||
|
||||
def is_test_tracked(test_name: str, cpython_prefix: str, lib_prefix: str) -> bool:
|
||||
"""Check if a test exists in our local Lib/test."""
|
||||
cpython_dir = pathlib.Path(cpython_prefix) / "Lib" / "test" / test_name
|
||||
cpython_file = pathlib.Path(cpython_prefix) / "Lib" / "test" / f"{test_name}.py"
|
||||
|
||||
if cpython_dir.exists():
|
||||
cpython_path = cpython_dir
|
||||
elif cpython_file.exists():
|
||||
cpython_path = cpython_file
|
||||
else:
|
||||
return True # No cpython test
|
||||
|
||||
local_path = pathlib.Path(lib_prefix) / "test" / cpython_path.name
|
||||
return local_path.exists()
|
||||
|
||||
|
||||
def is_test_up_to_date(test_name: str, cpython_prefix: str, lib_prefix: str) -> bool:
|
||||
"""Check if a test is up-to-date by comparing files.
|
||||
|
||||
Ignores lines containing 'TODO: RUSTPYTHON' in local files.
|
||||
"""
|
||||
# Try directory first, then file
|
||||
cpython_dir = pathlib.Path(cpython_prefix) / "Lib" / "test" / test_name
|
||||
cpython_file = pathlib.Path(cpython_prefix) / "Lib" / "test" / f"{test_name}.py"
|
||||
|
||||
if cpython_dir.exists():
|
||||
cpython_path = cpython_dir
|
||||
elif cpython_file.exists():
|
||||
cpython_path = cpython_file
|
||||
else:
|
||||
return True # No cpython test, consider up-to-date
|
||||
|
||||
local_path = pathlib.Path(lib_prefix) / "test" / cpython_path.name
|
||||
|
||||
if not local_path.exists():
|
||||
return False
|
||||
|
||||
if cpython_path.is_file():
|
||||
return _compare_file_ignoring_todo(cpython_path, local_path)
|
||||
else:
|
||||
return _compare_dir_ignoring_todo(cpython_path, local_path)
|
||||
|
||||
|
||||
def _build_test_to_lib_map(
|
||||
cpython_prefix: str,
|
||||
) -> tuple[dict[str, str], dict[str, list[str]]]:
|
||||
@@ -623,8 +508,8 @@ def format_all_todo(
|
||||
Returns:
|
||||
List of formatted lines
|
||||
"""
|
||||
from update_lib.cmd_deps import get_all_modules
|
||||
from update_lib.deps import is_up_to_date
|
||||
from update_lib.show_deps import get_all_modules
|
||||
|
||||
lines = []
|
||||
|
||||
+174
-119
@@ -14,7 +14,166 @@ import re
|
||||
import shelve
|
||||
import subprocess
|
||||
|
||||
from update_lib.io_utils import read_python_files, safe_parse_ast, safe_read_text
|
||||
from update_lib.file_utils import (
|
||||
_dircmp_is_same,
|
||||
compare_dir_contents,
|
||||
compare_file_contents,
|
||||
compare_paths,
|
||||
construct_lib_path,
|
||||
cpython_to_local_path,
|
||||
read_python_files,
|
||||
resolve_module_path,
|
||||
resolve_test_path,
|
||||
safe_parse_ast,
|
||||
safe_read_text,
|
||||
)
|
||||
|
||||
# === Import parsing utilities ===
|
||||
|
||||
|
||||
def _extract_top_level_code(content: str) -> str:
|
||||
"""Extract only top-level code from Python content for faster parsing."""
|
||||
def_idx = content.find("\ndef ")
|
||||
class_idx = content.find("\nclass ")
|
||||
|
||||
indices = [i for i in (def_idx, class_idx) if i != -1]
|
||||
if indices:
|
||||
content = content[: min(indices)]
|
||||
return content.rstrip("\n")
|
||||
|
||||
|
||||
_FROM_TEST_IMPORT_RE = re.compile(r"^from test import (.+)", re.MULTILINE)
|
||||
_FROM_TEST_DOT_RE = re.compile(r"^from test\.(\w+)", re.MULTILINE)
|
||||
_IMPORT_TEST_DOT_RE = re.compile(r"^import test\.(\w+)", re.MULTILINE)
|
||||
|
||||
|
||||
def parse_test_imports(content: str) -> set[str]:
|
||||
"""Parse test file content and extract test package dependencies."""
|
||||
content = _extract_top_level_code(content)
|
||||
imports = set()
|
||||
|
||||
for match in _FROM_TEST_IMPORT_RE.finditer(content):
|
||||
import_list = match.group(1)
|
||||
for part in import_list.split(","):
|
||||
name = part.split()[0].strip()
|
||||
if name and name not in ("support", "__init__"):
|
||||
imports.add(name)
|
||||
|
||||
for match in _FROM_TEST_DOT_RE.finditer(content):
|
||||
dep = match.group(1)
|
||||
if dep not in ("support", "__init__"):
|
||||
imports.add(dep)
|
||||
|
||||
for match in _IMPORT_TEST_DOT_RE.finditer(content):
|
||||
dep = match.group(1)
|
||||
if dep not in ("support", "__init__"):
|
||||
imports.add(dep)
|
||||
|
||||
return imports
|
||||
|
||||
|
||||
_IMPORT_RE = re.compile(r"^import\s+(\w[\w.]*)", re.MULTILINE)
|
||||
_FROM_IMPORT_RE = re.compile(r"^from\s+(\w[\w.]*)\s+import", re.MULTILINE)
|
||||
|
||||
|
||||
def parse_lib_imports(content: str) -> set[str]:
|
||||
"""Parse library file and extract all imported module names."""
|
||||
imports = set()
|
||||
|
||||
for match in _IMPORT_RE.finditer(content):
|
||||
imports.add(match.group(1))
|
||||
|
||||
for match in _FROM_IMPORT_RE.finditer(content):
|
||||
imports.add(match.group(1))
|
||||
|
||||
return imports
|
||||
|
||||
|
||||
# === TODO marker utilities ===
|
||||
|
||||
TODO_MARKER = "TODO: RUSTPYTHON"
|
||||
|
||||
|
||||
def filter_rustpython_todo(content: str) -> str:
|
||||
"""Remove lines containing RustPython TODO markers."""
|
||||
lines = content.splitlines(keepends=True)
|
||||
filtered = [line for line in lines if TODO_MARKER not in line]
|
||||
return "".join(filtered)
|
||||
|
||||
|
||||
def count_rustpython_todo(content: str) -> int:
|
||||
"""Count lines containing RustPython TODO markers."""
|
||||
return sum(1 for line in content.splitlines() if TODO_MARKER in line)
|
||||
|
||||
|
||||
def count_todo_in_path(path: pathlib.Path) -> int:
|
||||
"""Count RustPython TODO markers in a file or directory of .py files."""
|
||||
if path.is_file():
|
||||
content = safe_read_text(path)
|
||||
return count_rustpython_todo(content) if content else 0
|
||||
|
||||
total = 0
|
||||
for _, content in read_python_files(path):
|
||||
total += count_rustpython_todo(content)
|
||||
return total
|
||||
|
||||
|
||||
# === Test utilities ===
|
||||
|
||||
|
||||
def _get_cpython_test_path(test_name: str, cpython_prefix: str) -> pathlib.Path | None:
|
||||
"""Return the CPython test path for a test name, or None if missing."""
|
||||
cpython_path = resolve_test_path(test_name, cpython_prefix, prefer="dir")
|
||||
return cpython_path if cpython_path.exists() else None
|
||||
|
||||
|
||||
def _get_local_test_path(
|
||||
cpython_test_path: pathlib.Path, lib_prefix: str
|
||||
) -> pathlib.Path:
|
||||
"""Return the local Lib/test path matching a CPython test path."""
|
||||
return pathlib.Path(lib_prefix) / "test" / cpython_test_path.name
|
||||
|
||||
|
||||
def is_test_tracked(test_name: str, cpython_prefix: str, lib_prefix: str) -> bool:
|
||||
"""Check if a test exists in the local Lib/test."""
|
||||
cpython_path = _get_cpython_test_path(test_name, cpython_prefix)
|
||||
if cpython_path is None:
|
||||
return True
|
||||
local_path = _get_local_test_path(cpython_path, lib_prefix)
|
||||
return local_path.exists()
|
||||
|
||||
|
||||
def is_test_up_to_date(test_name: str, cpython_prefix: str, lib_prefix: str) -> bool:
|
||||
"""Check if a test is up-to-date, ignoring RustPython TODO markers."""
|
||||
cpython_path = _get_cpython_test_path(test_name, cpython_prefix)
|
||||
if cpython_path is None:
|
||||
return True
|
||||
|
||||
local_path = _get_local_test_path(cpython_path, lib_prefix)
|
||||
if not local_path.exists():
|
||||
return False
|
||||
|
||||
if cpython_path.is_file():
|
||||
return compare_file_contents(
|
||||
cpython_path, local_path, local_filter=filter_rustpython_todo
|
||||
)
|
||||
|
||||
return compare_dir_contents(
|
||||
cpython_path, local_path, local_filter=filter_rustpython_todo
|
||||
)
|
||||
|
||||
|
||||
def count_test_todos(test_name: str, lib_prefix: str) -> int:
|
||||
"""Count RustPython TODO markers in a test file/directory."""
|
||||
local_dir = pathlib.Path(lib_prefix) / "test" / test_name
|
||||
local_file = pathlib.Path(lib_prefix) / "test" / f"{test_name}.py"
|
||||
|
||||
if local_dir.exists():
|
||||
return count_todo_in_path(local_dir)
|
||||
if local_file.exists():
|
||||
return count_todo_in_path(local_file)
|
||||
return 0
|
||||
|
||||
|
||||
# === Cross-process cache using shelve ===
|
||||
|
||||
@@ -50,8 +209,6 @@ def clear_import_graph_caches() -> None:
|
||||
globals()["_lib_import_graph_cache"].clear()
|
||||
|
||||
|
||||
from update_lib.path import construct_lib_path, resolve_module_path
|
||||
|
||||
# Manual dependency table for irregular cases
|
||||
# Format: "name" -> {"lib": [...], "test": [...], "data": [...], "hard_deps": [...]}
|
||||
# - lib: override default path (default: name.py or name/)
|
||||
@@ -634,98 +791,6 @@ def get_test_paths(name: str, cpython_prefix: str) -> tuple[pathlib.Path, ...]:
|
||||
return (resolve_module_path(f"test/test_{name}", cpython_prefix, prefer="dir"),)
|
||||
|
||||
|
||||
def _extract_top_level_code(content: str) -> str:
|
||||
"""Extract only top-level code from Python content for faster parsing.
|
||||
|
||||
Cuts at first function/class definition since imports come before them.
|
||||
"""
|
||||
# Find first function or class definition
|
||||
def_idx = content.find("\ndef ")
|
||||
class_idx = content.find("\nclass ")
|
||||
|
||||
# Use the earlier of the two (if found)
|
||||
indices = [i for i in (def_idx, class_idx) if i != -1]
|
||||
if indices:
|
||||
content = content[: min(indices)]
|
||||
return content.rstrip("\n")
|
||||
|
||||
|
||||
_FROM_TEST_IMPORT_RE = re.compile(r"^from test import (.+)", re.MULTILINE)
|
||||
_FROM_TEST_DOT_RE = re.compile(r"^from test\.(\w+)", re.MULTILINE)
|
||||
_IMPORT_TEST_DOT_RE = re.compile(r"^import test\.(\w+)", re.MULTILINE)
|
||||
|
||||
|
||||
def parse_test_imports(content: str) -> set[str]:
|
||||
"""Parse test file content and extract test package dependencies.
|
||||
|
||||
Uses regex for speed - only matches top-level imports.
|
||||
|
||||
Args:
|
||||
content: Python file content
|
||||
|
||||
Returns:
|
||||
Set of module names imported from test package
|
||||
"""
|
||||
content = _extract_top_level_code(content)
|
||||
imports = set()
|
||||
|
||||
# Match "from test import foo, bar, baz"
|
||||
for match in _FROM_TEST_IMPORT_RE.finditer(content):
|
||||
import_list = match.group(1)
|
||||
# Parse "foo, bar as b, baz" -> ["foo", "bar", "baz"]
|
||||
for part in import_list.split(","):
|
||||
name = part.split()[0].strip() # Handle "foo as f"
|
||||
if name and name not in ("support", "__init__"):
|
||||
imports.add(name)
|
||||
|
||||
# Match "from test.foo import ..." -> depends on foo
|
||||
for match in _FROM_TEST_DOT_RE.finditer(content):
|
||||
dep = match.group(1)
|
||||
if dep not in ("support", "__init__"):
|
||||
imports.add(dep)
|
||||
|
||||
# Match "import test.foo" -> depends on foo
|
||||
for match in _IMPORT_TEST_DOT_RE.finditer(content):
|
||||
dep = match.group(1)
|
||||
if dep not in ("support", "__init__"):
|
||||
imports.add(dep)
|
||||
|
||||
return imports
|
||||
|
||||
|
||||
# Match "import foo.bar" - module name must start with word char (not dot)
|
||||
_IMPORT_RE = re.compile(r"^import\s+(\w[\w.]*)", re.MULTILINE)
|
||||
# Match "from foo.bar import" - exclude relative imports (from . or from ..)
|
||||
_FROM_IMPORT_RE = re.compile(r"^from\s+(\w[\w.]*)\s+import", re.MULTILINE)
|
||||
|
||||
|
||||
def parse_lib_imports(content: str) -> set[str]:
|
||||
"""Parse library file and extract all imported module names.
|
||||
|
||||
Uses regex for speed - only matches top-level imports (no leading whitespace).
|
||||
Returns full module paths (e.g., "collections.abc" not just "collections").
|
||||
|
||||
Args:
|
||||
content: Python file content
|
||||
|
||||
Returns:
|
||||
Set of imported module names (full paths)
|
||||
"""
|
||||
# Note: Don't truncate content here - some stdlib files have imports after
|
||||
# the first def/class (e.g., _pydecimal.py has `import contextvars` at line 343)
|
||||
imports = set()
|
||||
|
||||
# Match "import foo.bar" at line start
|
||||
for match in _IMPORT_RE.finditer(content):
|
||||
imports.add(match.group(1))
|
||||
|
||||
# Match "from foo.bar import ..." at line start
|
||||
for match in _FROM_IMPORT_RE.finditer(content):
|
||||
imports.add(match.group(1))
|
||||
|
||||
return imports
|
||||
|
||||
|
||||
@functools.cache
|
||||
def get_all_imports(name: str, cpython_prefix: str) -> frozenset[str]:
|
||||
"""Get all imports from a library file.
|
||||
@@ -787,24 +852,25 @@ def get_rust_deps(name: str, cpython_prefix: str) -> frozenset[str]:
|
||||
return frozenset(all_imports - soft_deps)
|
||||
|
||||
|
||||
def _dircmp_is_same(dcmp) -> bool:
|
||||
"""Recursively check if two directories are identical.
|
||||
def is_path_synced(
|
||||
cpython_path: pathlib.Path,
|
||||
cpython_prefix: str,
|
||||
lib_prefix: str,
|
||||
) -> bool:
|
||||
"""Check if a CPython path is synced with local.
|
||||
|
||||
Args:
|
||||
dcmp: filecmp.dircmp object
|
||||
cpython_path: Path in CPython directory
|
||||
cpython_prefix: CPython directory prefix
|
||||
lib_prefix: Local Lib directory prefix
|
||||
|
||||
Returns:
|
||||
True if directories are identical (including subdirectories)
|
||||
True if synced, False otherwise
|
||||
"""
|
||||
if dcmp.diff_files or dcmp.left_only or dcmp.right_only:
|
||||
local_path = cpython_to_local_path(cpython_path, cpython_prefix, lib_prefix)
|
||||
if local_path is None:
|
||||
return False
|
||||
|
||||
# Recursively check subdirectories
|
||||
for subdir in dcmp.subdirs.values():
|
||||
if not _dircmp_is_same(subdir):
|
||||
return False
|
||||
|
||||
return True
|
||||
return compare_paths(cpython_path, local_path)
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -819,8 +885,6 @@ def is_up_to_date(name: str, cpython_prefix: str, lib_prefix: str) -> bool:
|
||||
Returns:
|
||||
True if all files match, False otherwise
|
||||
"""
|
||||
import filecmp
|
||||
|
||||
lib_paths = get_lib_paths(name, cpython_prefix)
|
||||
|
||||
found_any = False
|
||||
@@ -835,18 +899,9 @@ def is_up_to_date(name: str, cpython_prefix: str, lib_prefix: str) -> bool:
|
||||
rel_path = cpython_path.relative_to(cpython_prefix)
|
||||
local_path = pathlib.Path(lib_prefix) / rel_path.relative_to("Lib")
|
||||
|
||||
if not local_path.exists():
|
||||
if not compare_paths(cpython_path, local_path):
|
||||
return False
|
||||
|
||||
if cpython_path.is_file():
|
||||
if not filecmp.cmp(cpython_path, local_path, shallow=False):
|
||||
return False
|
||||
else:
|
||||
# Directory comparison (recursive)
|
||||
dcmp = filecmp.dircmp(cpython_path, local_path)
|
||||
if not _dircmp_is_same(dcmp):
|
||||
return False
|
||||
|
||||
if not found_any:
|
||||
dep_info = DEPENDENCIES.get(name, {})
|
||||
if dep_info.get("lib") == []:
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
"""
|
||||
File utilities for update_lib.
|
||||
|
||||
This module provides functions for:
|
||||
- Safe file reading with error handling
|
||||
- Safe AST parsing with error handling
|
||||
- Iterating over Python files
|
||||
- Parsing and converting library paths
|
||||
- Detecting test paths vs library paths
|
||||
- Comparing files or directories for equality
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import filecmp
|
||||
import pathlib
|
||||
from collections.abc import Callable, Iterator
|
||||
|
||||
# === I/O utilities ===
|
||||
|
||||
|
||||
def safe_read_text(path: pathlib.Path) -> str | None:
|
||||
"""Read file content with UTF-8 encoding, returning None on error."""
|
||||
try:
|
||||
return path.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def safe_parse_ast(content: str) -> ast.Module | None:
|
||||
"""Parse Python content into AST, returning None on syntax error."""
|
||||
try:
|
||||
return ast.parse(content)
|
||||
except SyntaxError:
|
||||
return None
|
||||
|
||||
|
||||
def iter_python_files(path: pathlib.Path) -> Iterator[pathlib.Path]:
|
||||
"""Yield Python files from a file or directory."""
|
||||
if path.is_file():
|
||||
yield path
|
||||
else:
|
||||
yield from path.glob("**/*.py")
|
||||
|
||||
|
||||
def read_python_files(path: pathlib.Path) -> Iterator[tuple[pathlib.Path, str]]:
|
||||
"""Read all Python files from a path, yielding (path, content) pairs."""
|
||||
for py_file in iter_python_files(path):
|
||||
content = safe_read_text(py_file)
|
||||
if content is not None:
|
||||
yield py_file, content
|
||||
|
||||
|
||||
# === Path utilities ===
|
||||
|
||||
|
||||
def parse_lib_path(path: pathlib.Path | str) -> pathlib.Path:
|
||||
"""
|
||||
Extract the Lib/... portion from a path containing /Lib/.
|
||||
|
||||
Example:
|
||||
parse_lib_path("cpython/Lib/test/foo.py") -> Path("Lib/test/foo.py")
|
||||
"""
|
||||
path_str = str(path).replace("\\", "/")
|
||||
lib_marker = "/Lib/"
|
||||
|
||||
if lib_marker not in path_str:
|
||||
raise ValueError(f"Path must contain '/Lib/' or '\\Lib\\' (got: {path})")
|
||||
|
||||
idx = path_str.index(lib_marker)
|
||||
return pathlib.Path(path_str[idx + 1 :])
|
||||
|
||||
|
||||
def is_lib_path(path: pathlib.Path) -> bool:
|
||||
"""Check if path starts with Lib/"""
|
||||
path_str = str(path).replace("\\", "/")
|
||||
return path_str.startswith("Lib/") or path_str.startswith("./Lib/")
|
||||
|
||||
|
||||
def is_test_path(path: pathlib.Path) -> bool:
|
||||
"""Check if path is a test path (contains /Lib/test/ or starts with Lib/test/)"""
|
||||
path_str = str(path).replace("\\", "/")
|
||||
return "/Lib/test/" in path_str or path_str.startswith("Lib/test/")
|
||||
|
||||
|
||||
def lib_to_test_path(src_path: pathlib.Path) -> pathlib.Path:
|
||||
"""
|
||||
Convert library path to test path.
|
||||
|
||||
Examples:
|
||||
cpython/Lib/dataclasses.py -> cpython/Lib/test/test_dataclasses/
|
||||
cpython/Lib/json/__init__.py -> cpython/Lib/test/test_json/
|
||||
"""
|
||||
path_str = str(src_path).replace("\\", "/")
|
||||
lib_marker = "/Lib/"
|
||||
|
||||
if lib_marker in path_str:
|
||||
lib_path = parse_lib_path(src_path)
|
||||
lib_name = lib_path.stem if lib_path.suffix == ".py" else lib_path.name
|
||||
if lib_name == "__init__":
|
||||
lib_name = lib_path.parent.name
|
||||
prefix = path_str[: path_str.index(lib_marker)]
|
||||
dir_path = pathlib.Path(f"{prefix}/Lib/test/test_{lib_name}/")
|
||||
if dir_path.exists():
|
||||
return dir_path
|
||||
file_path = pathlib.Path(f"{prefix}/Lib/test/test_{lib_name}.py")
|
||||
if file_path.exists():
|
||||
return file_path
|
||||
return dir_path
|
||||
else:
|
||||
lib_name = src_path.stem if src_path.suffix == ".py" else src_path.name
|
||||
if lib_name == "__init__":
|
||||
lib_name = src_path.parent.name
|
||||
dir_path = pathlib.Path(f"Lib/test/test_{lib_name}/")
|
||||
if dir_path.exists():
|
||||
return dir_path
|
||||
file_path = pathlib.Path(f"Lib/test/test_{lib_name}.py")
|
||||
if file_path.exists():
|
||||
return file_path
|
||||
return dir_path
|
||||
|
||||
|
||||
def get_test_files(path: pathlib.Path) -> list[pathlib.Path]:
|
||||
"""Get all .py test files in a path (file or directory)."""
|
||||
if path.is_file():
|
||||
return [path]
|
||||
return sorted(path.glob("**/*.py"))
|
||||
|
||||
|
||||
def get_test_module_name(test_path: pathlib.Path) -> str:
|
||||
"""
|
||||
Extract test module name from a test file path.
|
||||
|
||||
Examples:
|
||||
Lib/test/test_foo.py -> test_foo
|
||||
Lib/test/test_ctypes/test_bar.py -> test_ctypes.test_bar
|
||||
"""
|
||||
test_path = pathlib.Path(test_path)
|
||||
if test_path.parent.name.startswith("test_"):
|
||||
return f"{test_path.parent.name}.{test_path.stem}"
|
||||
return test_path.stem
|
||||
|
||||
|
||||
def resolve_module_path(
|
||||
name: str, prefix: str = "cpython", prefer: str = "file"
|
||||
) -> pathlib.Path:
|
||||
"""
|
||||
Resolve module path, trying file or directory.
|
||||
|
||||
Args:
|
||||
name: Module name (e.g., "dataclasses", "json")
|
||||
prefix: CPython directory prefix
|
||||
prefer: "file" to try .py first, "dir" to try directory first
|
||||
"""
|
||||
file_path = pathlib.Path(f"{prefix}/Lib/{name}.py")
|
||||
dir_path = pathlib.Path(f"{prefix}/Lib/{name}")
|
||||
|
||||
if prefer == "file":
|
||||
if file_path.exists():
|
||||
return file_path
|
||||
if dir_path.exists():
|
||||
return dir_path
|
||||
return file_path
|
||||
else:
|
||||
if dir_path.exists():
|
||||
return dir_path
|
||||
if file_path.exists():
|
||||
return file_path
|
||||
return dir_path
|
||||
|
||||
|
||||
def construct_lib_path(prefix: str, *parts: str) -> pathlib.Path:
|
||||
"""Build a path under prefix/Lib/."""
|
||||
return pathlib.Path(prefix) / "Lib" / pathlib.Path(*parts)
|
||||
|
||||
|
||||
def resolve_test_path(
|
||||
test_name: str, prefix: str = "cpython", prefer: str = "dir"
|
||||
) -> pathlib.Path:
|
||||
"""Resolve a test module path under Lib/test/."""
|
||||
return resolve_module_path(f"test/{test_name}", prefix, prefer=prefer)
|
||||
|
||||
|
||||
def cpython_to_local_path(
|
||||
cpython_path: pathlib.Path,
|
||||
cpython_prefix: str,
|
||||
lib_prefix: str,
|
||||
) -> pathlib.Path | None:
|
||||
"""Convert CPython path to local Lib path."""
|
||||
try:
|
||||
rel_path = cpython_path.relative_to(cpython_prefix)
|
||||
return pathlib.Path(lib_prefix) / rel_path.relative_to("Lib")
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def get_module_name(path: pathlib.Path) -> str:
|
||||
"""Extract module name from path, handling __init__.py."""
|
||||
if path.suffix == ".py":
|
||||
name = path.stem
|
||||
if name == "__init__":
|
||||
return path.parent.name
|
||||
return name
|
||||
return path.name
|
||||
|
||||
|
||||
def get_cpython_dir(src_path: pathlib.Path) -> pathlib.Path:
|
||||
"""Extract CPython directory from a path containing /Lib/."""
|
||||
path_str = str(src_path).replace("\\", "/")
|
||||
lib_marker = "/Lib/"
|
||||
if lib_marker in path_str:
|
||||
idx = path_str.index(lib_marker)
|
||||
return pathlib.Path(path_str[:idx])
|
||||
return pathlib.Path("cpython")
|
||||
|
||||
|
||||
# === Comparison utilities ===
|
||||
|
||||
|
||||
def _dircmp_is_same(dcmp: filecmp.dircmp) -> bool:
|
||||
"""Recursively check if two directories are identical."""
|
||||
if dcmp.diff_files or dcmp.left_only or dcmp.right_only:
|
||||
return False
|
||||
|
||||
for subdir in dcmp.subdirs.values():
|
||||
if not _dircmp_is_same(subdir):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def compare_paths(cpython_path: pathlib.Path, local_path: pathlib.Path) -> bool:
|
||||
"""Compare a CPython path with a local path (file or directory)."""
|
||||
if not local_path.exists():
|
||||
return False
|
||||
|
||||
if cpython_path.is_file():
|
||||
return filecmp.cmp(cpython_path, local_path, shallow=False)
|
||||
|
||||
dcmp = filecmp.dircmp(cpython_path, local_path)
|
||||
return _dircmp_is_same(dcmp)
|
||||
|
||||
|
||||
def compare_file_contents(
|
||||
cpython_path: pathlib.Path,
|
||||
local_path: pathlib.Path,
|
||||
*,
|
||||
local_filter: Callable[[str], str] | None = None,
|
||||
encoding: str = "utf-8",
|
||||
) -> bool:
|
||||
"""Compare two files as text, optionally filtering local content."""
|
||||
try:
|
||||
cpython_content = cpython_path.read_text(encoding=encoding)
|
||||
local_content = local_path.read_text(encoding=encoding)
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return False
|
||||
|
||||
if local_filter is not None:
|
||||
local_content = local_filter(local_content)
|
||||
|
||||
return cpython_content == local_content
|
||||
|
||||
|
||||
def compare_dir_contents(
|
||||
cpython_dir: pathlib.Path,
|
||||
local_dir: pathlib.Path,
|
||||
*,
|
||||
pattern: str = "*.py",
|
||||
local_filter: Callable[[str], str] | None = None,
|
||||
encoding: str = "utf-8",
|
||||
) -> bool:
|
||||
"""Compare directory contents for matching files and text."""
|
||||
cpython_files = {f.relative_to(cpython_dir) for f in cpython_dir.rglob(pattern)}
|
||||
local_files = {f.relative_to(local_dir) for f in local_dir.rglob(pattern)}
|
||||
|
||||
if cpython_files != local_files:
|
||||
return False
|
||||
|
||||
for rel_path in cpython_files:
|
||||
if not compare_file_contents(
|
||||
cpython_dir / rel_path,
|
||||
local_dir / rel_path,
|
||||
local_filter=local_filter,
|
||||
encoding=encoding,
|
||||
):
|
||||
return False
|
||||
|
||||
return True
|
||||
@@ -1,81 +0,0 @@
|
||||
"""
|
||||
I/O utilities for update_lib.
|
||||
|
||||
This module provides functions for:
|
||||
- Safe file reading with error handling
|
||||
- Safe AST parsing with error handling
|
||||
- Iterating over Python files
|
||||
"""
|
||||
|
||||
import ast
|
||||
import pathlib
|
||||
from collections.abc import Iterator
|
||||
|
||||
|
||||
def safe_read_text(path: pathlib.Path) -> str | None:
|
||||
"""
|
||||
Read file content with UTF-8 encoding, returning None on error.
|
||||
|
||||
Args:
|
||||
path: Path to the file
|
||||
|
||||
Returns:
|
||||
File content as string, or None if reading fails
|
||||
"""
|
||||
try:
|
||||
return path.read_text(encoding="utf-8")
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def safe_parse_ast(content: str) -> ast.Module | None:
|
||||
"""
|
||||
Parse Python content into AST, returning None on syntax error.
|
||||
|
||||
Args:
|
||||
content: Python source code
|
||||
|
||||
Returns:
|
||||
AST module, or None if parsing fails
|
||||
"""
|
||||
try:
|
||||
return ast.parse(content)
|
||||
except SyntaxError:
|
||||
return None
|
||||
|
||||
|
||||
def iter_python_files(path: pathlib.Path) -> Iterator[pathlib.Path]:
|
||||
"""
|
||||
Yield Python files from a file or directory.
|
||||
|
||||
If path is a file, yields just that file.
|
||||
If path is a directory, yields all .py files recursively.
|
||||
|
||||
Args:
|
||||
path: Path to a file or directory
|
||||
|
||||
Yields:
|
||||
Paths to Python files
|
||||
"""
|
||||
if path.is_file():
|
||||
yield path
|
||||
else:
|
||||
yield from path.glob("**/*.py")
|
||||
|
||||
|
||||
def read_python_files(path: pathlib.Path) -> Iterator[tuple[pathlib.Path, str]]:
|
||||
"""
|
||||
Read all Python files from a path, yielding (path, content) pairs.
|
||||
|
||||
Skips files that cannot be read.
|
||||
|
||||
Args:
|
||||
path: Path to a file or directory
|
||||
|
||||
Yields:
|
||||
Tuples of (file_path, file_content)
|
||||
"""
|
||||
for py_file in iter_python_files(path):
|
||||
content = safe_read_text(py_file)
|
||||
if content is not None:
|
||||
yield py_file, content
|
||||
@@ -1,187 +0,0 @@
|
||||
"""
|
||||
Path utilities for update_lib.
|
||||
|
||||
This module provides functions for:
|
||||
- Parsing and converting library paths
|
||||
- Detecting test paths vs library paths
|
||||
- Extracting test module names from paths
|
||||
"""
|
||||
|
||||
import pathlib
|
||||
|
||||
|
||||
def parse_lib_path(path: pathlib.Path | str) -> pathlib.Path:
|
||||
"""
|
||||
Extract the Lib/... portion from a path containing /Lib/.
|
||||
|
||||
Example:
|
||||
parse_lib_path("cpython/Lib/test/foo.py") -> Path("Lib/test/foo.py")
|
||||
"""
|
||||
path_str = str(path).replace("\\", "/")
|
||||
lib_marker = "/Lib/"
|
||||
|
||||
if lib_marker not in path_str:
|
||||
raise ValueError(f"Path must contain '/Lib/' or '\\Lib\\' (got: {path})")
|
||||
|
||||
idx = path_str.index(lib_marker)
|
||||
return pathlib.Path(path_str[idx + 1 :])
|
||||
|
||||
|
||||
def is_lib_path(path: pathlib.Path) -> bool:
|
||||
"""Check if path starts with Lib/"""
|
||||
path_str = str(path).replace("\\", "/")
|
||||
return path_str.startswith("Lib/") or path_str.startswith("./Lib/")
|
||||
|
||||
|
||||
def is_test_path(path: pathlib.Path) -> bool:
|
||||
"""Check if path is a test path (contains /Lib/test/ or starts with Lib/test/)"""
|
||||
path_str = str(path).replace("\\", "/")
|
||||
return "/Lib/test/" in path_str or path_str.startswith("Lib/test/")
|
||||
|
||||
|
||||
def lib_to_test_path(src_path: pathlib.Path) -> pathlib.Path:
|
||||
"""
|
||||
Convert library path to test path.
|
||||
|
||||
Examples:
|
||||
cpython/Lib/dataclasses.py -> cpython/Lib/test/test_dataclasses/ (if dir exists)
|
||||
cpython/Lib/typing.py -> cpython/Lib/test/test_typing.py (if file exists)
|
||||
cpython/Lib/json/ -> cpython/Lib/test/test_json/
|
||||
cpython/Lib/json/__init__.py -> cpython/Lib/test/test_json/
|
||||
Lib/dataclasses.py -> Lib/test/test_dataclasses/
|
||||
"""
|
||||
path_str = str(src_path).replace("\\", "/")
|
||||
lib_marker = "/Lib/"
|
||||
|
||||
if lib_marker in path_str:
|
||||
lib_path = parse_lib_path(src_path)
|
||||
lib_name = lib_path.stem if lib_path.suffix == ".py" else lib_path.name
|
||||
# Handle __init__.py: use parent directory name
|
||||
if lib_name == "__init__":
|
||||
lib_name = lib_path.parent.name
|
||||
prefix = path_str[: path_str.index(lib_marker)]
|
||||
# Try directory first, then file
|
||||
dir_path = pathlib.Path(f"{prefix}/Lib/test/test_{lib_name}/")
|
||||
if dir_path.exists():
|
||||
return dir_path
|
||||
file_path = pathlib.Path(f"{prefix}/Lib/test/test_{lib_name}.py")
|
||||
if file_path.exists():
|
||||
return file_path
|
||||
# Default to directory (caller will handle non-existence)
|
||||
return dir_path
|
||||
else:
|
||||
# Path starts with Lib/ - extract name directly
|
||||
lib_name = src_path.stem if src_path.suffix == ".py" else src_path.name
|
||||
# Handle __init__.py: use parent directory name
|
||||
if lib_name == "__init__":
|
||||
lib_name = src_path.parent.name
|
||||
# Try directory first, then file
|
||||
dir_path = pathlib.Path(f"Lib/test/test_{lib_name}/")
|
||||
if dir_path.exists():
|
||||
return dir_path
|
||||
file_path = pathlib.Path(f"Lib/test/test_{lib_name}.py")
|
||||
if file_path.exists():
|
||||
return file_path
|
||||
return dir_path
|
||||
|
||||
|
||||
def get_test_files(path: pathlib.Path) -> list[pathlib.Path]:
|
||||
"""Get all .py test files in a path (file or directory)."""
|
||||
if path.is_file():
|
||||
return [path]
|
||||
return sorted(path.glob("**/*.py"))
|
||||
|
||||
|
||||
def test_name_from_path(test_path: pathlib.Path) -> str:
|
||||
"""
|
||||
Extract test module name from a test file path.
|
||||
|
||||
Examples:
|
||||
Lib/test/test_foo.py -> test_foo
|
||||
Lib/test/test_ctypes/test_bar.py -> test_ctypes.test_bar
|
||||
"""
|
||||
test_path = pathlib.Path(test_path)
|
||||
if test_path.parent.name.startswith("test_"):
|
||||
return f"{test_path.parent.name}.{test_path.stem}"
|
||||
return test_path.stem
|
||||
|
||||
|
||||
# --- Utility functions for reducing duplication ---
|
||||
|
||||
|
||||
def resolve_module_path(
|
||||
name: str, prefix: str = "cpython", prefer: str = "file"
|
||||
) -> pathlib.Path:
|
||||
"""
|
||||
Resolve module path, trying file or directory.
|
||||
|
||||
Args:
|
||||
name: Module name (e.g., "dataclasses", "json")
|
||||
prefix: CPython directory prefix
|
||||
prefer: "file" to try .py first, "dir" to try directory first
|
||||
|
||||
Returns:
|
||||
Path to the module (file or directory)
|
||||
|
||||
Examples:
|
||||
resolve_module_path("dataclasses") -> cpython/Lib/dataclasses.py
|
||||
resolve_module_path("json") -> cpython/Lib/json/
|
||||
"""
|
||||
file_path = pathlib.Path(f"{prefix}/Lib/{name}.py")
|
||||
dir_path = pathlib.Path(f"{prefix}/Lib/{name}")
|
||||
|
||||
if prefer == "file":
|
||||
if file_path.exists():
|
||||
return file_path
|
||||
if dir_path.exists():
|
||||
return dir_path
|
||||
return file_path # Default to file
|
||||
else:
|
||||
if dir_path.exists():
|
||||
return dir_path
|
||||
if file_path.exists():
|
||||
return file_path
|
||||
return dir_path # Default to dir
|
||||
|
||||
|
||||
def construct_lib_path(prefix: str, *parts: str) -> pathlib.Path:
|
||||
"""
|
||||
Build a path under prefix/Lib/.
|
||||
|
||||
Args:
|
||||
prefix: Directory prefix (e.g., "cpython")
|
||||
*parts: Path components after Lib/
|
||||
|
||||
Returns:
|
||||
Combined path
|
||||
|
||||
Examples:
|
||||
construct_lib_path("cpython", "test", "test_foo.py")
|
||||
-> cpython/Lib/test/test_foo.py
|
||||
construct_lib_path("cpython", "dataclasses.py")
|
||||
-> cpython/Lib/dataclasses.py
|
||||
"""
|
||||
return pathlib.Path(prefix) / "Lib" / pathlib.Path(*parts)
|
||||
|
||||
|
||||
def get_module_name(path: pathlib.Path) -> str:
|
||||
"""
|
||||
Extract module name from path, handling __init__.py.
|
||||
|
||||
Args:
|
||||
path: Path to a Python file or directory
|
||||
|
||||
Returns:
|
||||
Module name
|
||||
|
||||
Examples:
|
||||
get_module_name(Path("cpython/Lib/dataclasses.py")) -> "dataclasses"
|
||||
get_module_name(Path("cpython/Lib/json/__init__.py")) -> "json"
|
||||
get_module_name(Path("cpython/Lib/json/")) -> "json"
|
||||
"""
|
||||
if path.suffix == ".py":
|
||||
name = path.stem
|
||||
if name == "__init__":
|
||||
return path.parent.name
|
||||
return name
|
||||
return path.name
|
||||
@@ -3,7 +3,7 @@
|
||||
import subprocess
|
||||
import unittest
|
||||
|
||||
from update_lib.auto_mark import (
|
||||
from update_lib.cmd_auto_mark import (
|
||||
Test,
|
||||
TestResult,
|
||||
_is_super_call_only,
|
||||
@@ -277,7 +277,7 @@ class TestCollectTestChanges(unittest.TestCase):
|
||||
When test results come from a package's __init__.py, the path is like:
|
||||
'test.test_dataclasses.TestCase.test_foo' (no __init__)
|
||||
|
||||
But module_prefix from test_name_from_path would be:
|
||||
But module_prefix from get_test_module_name would be:
|
||||
'test_dataclasses.__init__'
|
||||
|
||||
So we need to strip '.__init__' and add 'test.' prefix.
|
||||
|
||||
@@ -10,7 +10,7 @@ class TestCopySingle(unittest.TestCase):
|
||||
|
||||
def test_copies_file(self):
|
||||
"""Test copying a single file."""
|
||||
from update_lib.copy_lib import _copy_single
|
||||
from update_lib.cmd_copy_lib import _copy_single
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmpdir = pathlib.Path(tmpdir)
|
||||
@@ -26,7 +26,7 @@ class TestCopySingle(unittest.TestCase):
|
||||
|
||||
def test_copies_directory(self):
|
||||
"""Test copying a directory."""
|
||||
from update_lib.copy_lib import _copy_single
|
||||
from update_lib.cmd_copy_lib import _copy_single
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmpdir = pathlib.Path(tmpdir)
|
||||
@@ -43,7 +43,7 @@ class TestCopySingle(unittest.TestCase):
|
||||
|
||||
def test_removes_existing_before_copy(self):
|
||||
"""Test that existing destination is removed before copy."""
|
||||
from update_lib.copy_lib import _copy_single
|
||||
from update_lib.cmd_copy_lib import _copy_single
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmpdir = pathlib.Path(tmpdir)
|
||||
@@ -63,7 +63,7 @@ class TestCopyLib(unittest.TestCase):
|
||||
|
||||
def test_raises_on_path_without_lib(self):
|
||||
"""Test that copy_lib raises ValueError when path doesn't contain /Lib/."""
|
||||
from update_lib.copy_lib import copy_lib
|
||||
from update_lib.cmd_copy_lib import copy_lib
|
||||
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
copy_lib(pathlib.Path("some/path/without/lib.py"))
|
||||
|
||||
@@ -4,7 +4,7 @@ import pathlib
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from update_lib.migrate import (
|
||||
from update_lib.cmd_migrate import (
|
||||
patch_directory,
|
||||
patch_file,
|
||||
patch_single_content,
|
||||
|
||||
@@ -4,13 +4,13 @@ import pathlib
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from update_lib.path import (
|
||||
from update_lib.file_utils import (
|
||||
get_test_files,
|
||||
get_test_module_name,
|
||||
is_lib_path,
|
||||
is_test_path,
|
||||
lib_to_test_path,
|
||||
parse_lib_path,
|
||||
test_name_from_path,
|
||||
)
|
||||
|
||||
|
||||
@@ -202,22 +202,22 @@ class TestGetTestFiles(unittest.TestCase):
|
||||
|
||||
|
||||
class TestTestNameFromPath(unittest.TestCase):
|
||||
"""Tests for test_name_from_path function."""
|
||||
"""Tests for get_test_module_name function."""
|
||||
|
||||
def test_simple_test_file(self):
|
||||
"""Test extracting name from simple test file."""
|
||||
path = pathlib.Path("Lib/test/test_foo.py")
|
||||
self.assertEqual(test_name_from_path(path), "test_foo")
|
||||
self.assertEqual(get_test_module_name(path), "test_foo")
|
||||
|
||||
def test_nested_test_file(self):
|
||||
"""Test extracting name from nested test directory."""
|
||||
path = pathlib.Path("Lib/test/test_ctypes/test_bar.py")
|
||||
self.assertEqual(test_name_from_path(path), "test_ctypes.test_bar")
|
||||
self.assertEqual(get_test_module_name(path), "test_ctypes.test_bar")
|
||||
|
||||
def test_test_directory(self):
|
||||
"""Test extracting name from test directory."""
|
||||
path = pathlib.Path("Lib/test/test_json")
|
||||
self.assertEqual(test_name_from_path(path), "test_json")
|
||||
self.assertEqual(get_test_module_name(path), "test_json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -5,13 +5,13 @@ import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from update_lib.path import lib_to_test_path
|
||||
from update_lib.quick import (
|
||||
from update_lib.cmd_quick import (
|
||||
_expand_shortcut,
|
||||
collect_original_methods,
|
||||
get_cpython_dir,
|
||||
git_commit,
|
||||
)
|
||||
from update_lib.file_utils import lib_to_test_path
|
||||
|
||||
|
||||
class TestGetCpythonDir(unittest.TestCase):
|
||||
@@ -159,7 +159,7 @@ class TestGitCommit(unittest.TestCase):
|
||||
"""Tests for git_commit function."""
|
||||
|
||||
@patch("subprocess.run")
|
||||
@patch("update_lib.quick.get_cpython_version")
|
||||
@patch("update_lib.cmd_quick.get_cpython_version")
|
||||
def test_none_lib_path_not_added(self, mock_version, mock_run):
|
||||
"""Test that None lib_path doesn't add '.' to git."""
|
||||
mock_version.return_value = "v3.14.0"
|
||||
@@ -177,7 +177,7 @@ class TestGitCommit(unittest.TestCase):
|
||||
self.assertNotIn(".", add_call[0][0][2:]) # Skip "git" and "add"
|
||||
|
||||
@patch("subprocess.run")
|
||||
@patch("update_lib.quick.get_cpython_version")
|
||||
@patch("update_lib.cmd_quick.get_cpython_version")
|
||||
def test_none_test_path_not_added(self, mock_version, mock_run):
|
||||
"""Test that None test_path doesn't add '.' to git."""
|
||||
mock_version.return_value = "v3.14.0"
|
||||
@@ -203,10 +203,10 @@ class TestGitCommit(unittest.TestCase):
|
||||
class TestQuickTestRunFailure(unittest.TestCase):
|
||||
"""Tests for quick() behavior when test run fails."""
|
||||
|
||||
@patch("update_lib.auto_mark.run_test")
|
||||
@patch("update_lib.cmd_auto_mark.run_test")
|
||||
def test_auto_mark_raises_on_test_run_failure(self, mock_run_test):
|
||||
"""Test that auto_mark_file raises when test run fails entirely."""
|
||||
from update_lib.auto_mark import TestResult, TestRunError, auto_mark_file
|
||||
from update_lib.cmd_auto_mark import TestResult, TestRunError, auto_mark_file
|
||||
|
||||
# Simulate test runner crash (empty tests_result)
|
||||
mock_run_test.return_value = TestResult(
|
||||
|
||||
Reference in new issue
Block a user