diff --git a/benches/README.md b/benches/README.md index c48b3c4a6..46ca3f449 100644 --- a/benches/README.md +++ b/benches/README.md @@ -54,6 +54,9 @@ in two ways: 1. The time to parse the file to AST 2. The time it takes to execute the file +Parsing stops at an AST in both implementations. The CPython reference uses +`compile(..., flags=ast.PyCF_ONLY_AST)` without compiling bytecode. + Execution includes compilation and runs each iteration with a fresh namespace whose `__name__` is `"__main__"`, so script entry points run in both RustPython and CPython. diff --git a/benches/execution.rs b/benches/execution.rs index cbfdd12ad..5b8c1524a 100644 --- a/benches/execution.rs +++ b/benches/execution.rs @@ -95,9 +95,14 @@ pub fn benchmark_file_parsing( let builtins = pyo3::types::PyModule::import(py, "builtins") .expect("Failed to import builtins"); let compile = builtins.getattr("compile").expect("no compile in builtins"); + let ast = pyo3::types::PyModule::import(py, "ast").expect("Failed to import ast"); + let only_ast: i32 = ast + .getattr("PyCF_ONLY_AST") + .and_then(|flag| flag.extract()) + .expect("no PyCF_ONLY_AST in ast"); b.iter(|| { let x = compile - .call1((contents, name, "exec")) + .call1((contents, name, "exec", only_ast)) .expect("Failed to parse code"); black_box(x); }) diff --git a/scripts/update_lib/file_utils.py b/scripts/update_lib/file_utils.py index 444c08864..3abdeb8f5 100644 --- a/scripts/update_lib/file_utils.py +++ b/scripts/update_lib/file_utils.py @@ -222,11 +222,29 @@ def get_cpython_dir(src_path: pathlib.Path) -> pathlib.Path: # === Comparison utilities === +def _files_are_same(left: pathlib.Path, right: pathlib.Path) -> bool: + # Recheck contents even when size and timestamps match, including after a + # previous comparison. filecmp caches results by those metadata alone. + with left.open("rb") as left_file, right.open("rb") as right_file: + while True: + chunk = left_file.read(65536) + if chunk != right_file.read(65536): + return False + if not chunk: + return True + + def _dircmp_is_same(dcmp: filecmp.dircmp) -> bool: """Recursively check if two directories are identical.""" - if dcmp.diff_files or dcmp.left_only or dcmp.right_only: + if dcmp.left_only or dcmp.right_only or dcmp.common_funny: return False + for name in dcmp.common_files: + if not _files_are_same( + pathlib.Path(dcmp.left) / name, pathlib.Path(dcmp.right) / name + ): + return False + for subdir in dcmp.subdirs.values(): if not _dircmp_is_same(subdir): return False @@ -236,14 +254,17 @@ def _dircmp_is_same(dcmp: filecmp.dircmp) -> bool: def compare_paths(cpython_path: pathlib.Path, local_path: pathlib.Path) -> bool: """Compare a CPython path with a local path (file or directory).""" - if not local_path.exists(): + if not cpython_path.exists() or not local_path.exists(): return False - if cpython_path.is_file(): - return filecmp.cmp(cpython_path, local_path, shallow=False) - - dcmp = filecmp.dircmp(cpython_path, local_path) - return _dircmp_is_same(dcmp) + try: + if cpython_path.is_file(): + return local_path.is_file() and _files_are_same(cpython_path, local_path) + if not cpython_path.is_dir() or not local_path.is_dir(): + return False + return _dircmp_is_same(filecmp.dircmp(cpython_path, local_path)) + except OSError: + return False def compare_file_contents( diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index 4d5d6c0a4..3895a323b 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -325,6 +325,26 @@ import nonexistent class TestDircmpIsSame(unittest.TestCase): """Tests for _dircmp_is_same function.""" + def test_matching_metadata(self): + import os + + from update_lib.file_utils import compare_paths + + with tempfile.TemporaryDirectory() as tmpdir: + left, right = (pathlib.Path(tmpdir) / name for name in ("left", "right")) + for directory in (left, right): + (directory / "nested").mkdir(parents=True) + (directory / "nested/module.py").write_bytes(b"x = 1\n") + source, destination = (p / "nested/module.py" for p in (left, right)) + timestamp = source.stat().st_mtime_ns + os.utime(destination, ns=(timestamp, timestamp)) + self.assertTrue(compare_paths(source, destination)) + self.assertTrue(compare_paths(left, right)) + destination.write_bytes(b"x = 2\n") + os.utime(destination, ns=(timestamp, timestamp)) + self.assertFalse(compare_paths(source, destination)) + self.assertFalse(compare_paths(left, right)) + def test_identical_directories(self): """Test that identical directories return True.""" import filecmp