From 831a9c675ad968e4dd043dfaefbd1d80aee489d3 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 25 Jan 2026 21:56:51 +0900 Subject: [PATCH 1/3] Improve deps output: [x]/[ ] sync status, TODO counts - Replace ambiguous [+] with [x] (synced) / [ ] (not synced) - Add (TODO: n) suffix for test files with expectedFailure/skip markers --- scripts/update_lib/deps.py | 78 ++++++++++++++++++++++++++++----- scripts/update_lib/show_deps.py | 13 +++++- 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 878ad3a12a8..30c0b8555c8 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -807,6 +807,71 @@ def _dircmp_is_same(dcmp) -> bool: return True +def compare_paths(cpython_path: pathlib.Path, local_path: pathlib.Path) -> bool: + """Compare a CPython path with a local path. + + Args: + cpython_path: Path in CPython directory (file or directory) + local_path: Corresponding path in local Lib directory + + Returns: + True if paths are identical, False otherwise + """ + import filecmp + + if not local_path.exists(): + return False + + if cpython_path.is_file(): + return filecmp.cmp(cpython_path, local_path, shallow=False) + else: + dcmp = filecmp.dircmp(cpython_path, local_path) + return _dircmp_is_same(dcmp) + + +def cpython_to_local_path( + cpython_path: pathlib.Path, + cpython_prefix: str, + lib_prefix: str, +) -> pathlib.Path | None: + """Convert CPython path to local Lib path. + + Args: + cpython_path: Path like cpython/Lib/foo.py + cpython_prefix: CPython directory prefix + lib_prefix: Local Lib directory prefix + + Returns: + Local path like Lib/foo.py, or None if conversion fails + """ + try: + rel_path = cpython_path.relative_to(cpython_prefix) + return pathlib.Path(lib_prefix) / rel_path.relative_to("Lib") + except ValueError: + return None + + +def is_path_synced( + cpython_path: pathlib.Path, + cpython_prefix: str, + lib_prefix: str, +) -> bool: + """Check if a CPython path is synced with local. + + Args: + cpython_path: Path in CPython directory + cpython_prefix: CPython directory prefix + lib_prefix: Local Lib directory prefix + + Returns: + True if synced, False otherwise + """ + local_path = cpython_to_local_path(cpython_path, cpython_prefix, lib_prefix) + if local_path is None: + return False + return compare_paths(cpython_path, local_path) + + @functools.cache def is_up_to_date(name: str, cpython_prefix: str, lib_prefix: str) -> bool: """Check if a module is up-to-date by comparing files. @@ -819,8 +884,6 @@ def is_up_to_date(name: str, cpython_prefix: str, lib_prefix: str) -> bool: Returns: True if all files match, False otherwise """ - import filecmp - lib_paths = get_lib_paths(name, cpython_prefix) found_any = False @@ -835,18 +898,9 @@ def is_up_to_date(name: str, cpython_prefix: str, lib_prefix: str) -> bool: rel_path = cpython_path.relative_to(cpython_prefix) local_path = pathlib.Path(lib_prefix) / rel_path.relative_to("Lib") - if not local_path.exists(): + if not compare_paths(cpython_path, local_path): return False - if cpython_path.is_file(): - if not filecmp.cmp(cpython_path, local_path, shallow=False): - return False - else: - # Directory comparison (recursive) - dcmp = filecmp.dircmp(cpython_path, local_path) - if not _dircmp_is_same(dcmp): - return False - if not found_any: dep_info = DEPENDENCIES.get(name, {}) if dep_info.get("lib") == []: diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py index fb9ea1089e9..3b62ac468a1 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/show_deps.py @@ -192,8 +192,10 @@ def format_deps( find_dependent_tests_tree, get_lib_paths, get_test_paths, + is_path_synced, resolve_hard_dep_parent, ) + from update_lib.show_todo import count_test_todos, is_test_up_to_date if _visited is None: _visited = set() @@ -216,13 +218,20 @@ def format_deps( lib_paths = get_lib_paths(name, cpython_prefix) existing_lib_paths = [p for p in lib_paths if p.exists()] for p in existing_lib_paths: - lines.append(f"[+] lib: {p}") + synced = is_path_synced(p, cpython_prefix, lib_prefix) + marker = "[x]" if synced else "[ ]" + lines.append(f"{marker} lib: {p}") # test paths (only show existing) test_paths = get_test_paths(name, cpython_prefix) existing_test_paths = [p for p in test_paths if p.exists()] for p in existing_test_paths: - lines.append(f"[+] test: {p}") + test_name = p.stem if p.is_file() else p.name + synced = is_test_up_to_date(test_name, cpython_prefix, lib_prefix) + marker = "[x]" if synced else "[ ]" + todo_count = count_test_todos(test_name, lib_prefix) + todo_suffix = f" (TODO: {todo_count})" if todo_count > 0 else "" + lines.append(f"{marker} test: {p}{todo_suffix}") # If no lib or test paths exist, module doesn't exist if not existing_lib_paths and not existing_test_paths: From e17f0ef04622caad44e7ae061b064657b406aa2c Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 25 Jan 2026 22:25:08 +0900 Subject: [PATCH 2/3] Refactor update_lib: extract shared utilities --- scripts/update_lib/__main__.py | 20 +- .../{auto_mark.py => cmd_auto_mark.py} | 10 +- .../{copy_lib.py => cmd_copy_lib.py} | 2 +- .../update_lib/{show_deps.py => cmd_deps.py} | 2 +- .../update_lib/{migrate.py => cmd_migrate.py} | 2 +- .../update_lib/{patches.py => cmd_patches.py} | 0 scripts/update_lib/{quick.py => cmd_quick.py} | 31 +- .../update_lib/{show_todo.py => cmd_todo.py} | 133 +------- scripts/update_lib/deps.py | 319 +++++++++--------- scripts/update_lib/file_utils.py | 290 ++++++++++++++++ scripts/update_lib/io_utils.py | 81 ----- scripts/update_lib/path.py | 187 ---------- scripts/update_lib/tests/test_auto_mark.py | 4 +- scripts/update_lib/tests/test_copy_lib.py | 8 +- scripts/update_lib/tests/test_migrate.py | 2 +- scripts/update_lib/tests/test_path.py | 12 +- scripts/update_lib/tests/test_quick.py | 12 +- 17 files changed, 504 insertions(+), 611 deletions(-) rename scripts/update_lib/{auto_mark.py => cmd_auto_mark.py} (99%) rename scripts/update_lib/{copy_lib.py => cmd_copy_lib.py} (98%) rename scripts/update_lib/{show_deps.py => cmd_deps.py} (99%) rename scripts/update_lib/{migrate.py => cmd_migrate.py} (98%) rename scripts/update_lib/{patches.py => cmd_patches.py} (100%) rename scripts/update_lib/{quick.py => cmd_quick.py} (93%) rename scripts/update_lib/{show_todo.py => cmd_todo.py} (82%) create mode 100644 scripts/update_lib/file_utils.py delete mode 100644 scripts/update_lib/io_utils.py delete mode 100644 scripts/update_lib/path.py diff --git a/scripts/update_lib/__main__.py b/scripts/update_lib/__main__.py index 9bbd849c534..49399db6f43 100644 --- a/scripts/update_lib/__main__.py +++ b/scripts/update_lib/__main__.py @@ -63,39 +63,39 @@ def main(argv: list[str] | None = None) -> int: args, remaining = parser.parse_known_args(argv) if args.command == "quick": - from update_lib.quick import main as quick_main + from update_lib.cmd_quick import main as quick_main return quick_main(remaining) if args.command == "copy-lib": - from update_lib.copy_lib import main as copy_lib_main + from update_lib.cmd_copy_lib import main as copy_lib_main return copy_lib_main(remaining) if args.command == "migrate": - from update_lib.migrate import main as migrate_main + from update_lib.cmd_migrate import main as migrate_main return migrate_main(remaining) if args.command == "patches": - from update_lib.patches import main as patches_main + from update_lib.cmd_patches import main as patches_main return patches_main(remaining) if args.command == "auto-mark": - from update_lib.auto_mark import main as auto_mark_main + from update_lib.cmd_auto_mark import main as cmd_auto_mark_main - return auto_mark_main(remaining) + return cmd_auto_mark_main(remaining) if args.command == "deps": - from update_lib.show_deps import main as show_deps_main + from update_lib.cmd_deps import main as cmd_deps_main - return show_deps_main(remaining) + return cmd_deps_main(remaining) if args.command == "todo": - from update_lib.show_todo import main as show_todo_main + from update_lib.cmd_todo import main as cmd_todo_main - return show_todo_main(remaining) + return cmd_todo_main(remaining) return 0 diff --git a/scripts/update_lib/auto_mark.py b/scripts/update_lib/cmd_auto_mark.py similarity index 99% rename from scripts/update_lib/auto_mark.py rename to scripts/update_lib/cmd_auto_mark.py index bfc80f0d9fa..0714f2a67dd 100644 --- a/scripts/update_lib/auto_mark.py +++ b/scripts/update_lib/cmd_auto_mark.py @@ -19,7 +19,7 @@ sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) from update_lib import COMMENT, PatchSpec, UtMethod, apply_patches -from update_lib.path import test_name_from_path +from update_lib.file_utils import get_test_module_name class TestRunError(Exception): @@ -455,7 +455,7 @@ def extract_test_methods(contents: str) -> set[tuple[str, str]]: Returns: Set of (class_name, method_name) tuples """ - from update_lib.io_utils import safe_parse_ast + from update_lib.file_utils import safe_parse_ast from update_lib.patch_spec import iter_tests tree = safe_parse_ast(contents) @@ -490,7 +490,7 @@ def auto_mark_file( if not test_path.exists(): raise FileNotFoundError(f"File not found: {test_path}") - test_name = test_name_from_path(test_path) + test_name = get_test_module_name(test_path) if verbose: print(f"Running test: {test_name}") @@ -587,7 +587,7 @@ def auto_mark_directory( if not test_dir.is_dir(): raise ValueError(f"Not a directory: {test_dir}") - test_name = test_name_from_path(test_dir) + test_name = get_test_module_name(test_dir) if verbose: print(f"Running test: {test_name}") @@ -610,7 +610,7 @@ def auto_mark_directory( for test_file in test_files: # Get module prefix for this file (e.g., "test_inspect.test_inspect") - module_prefix = test_name_from_path(test_file) + module_prefix = get_test_module_name(test_file) # For __init__.py, the test path doesn't include "__init__" if module_prefix.endswith(".__init__"): module_prefix = module_prefix[:-9] # Remove ".__init__" diff --git a/scripts/update_lib/copy_lib.py b/scripts/update_lib/cmd_copy_lib.py similarity index 98% rename from scripts/update_lib/copy_lib.py rename to scripts/update_lib/cmd_copy_lib.py index 2788f2ccc83..1b16497fc83 100644 --- a/scripts/update_lib/copy_lib.py +++ b/scripts/update_lib/cmd_copy_lib.py @@ -60,7 +60,7 @@ def copy_lib( verbose: Print progress messages """ from update_lib.deps import get_lib_paths - from update_lib.path import parse_lib_path + from update_lib.file_utils import parse_lib_path # Extract module name and cpython prefix from path path_str = str(src_path).replace("\\", "/") diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/cmd_deps.py similarity index 99% rename from scripts/update_lib/show_deps.py rename to scripts/update_lib/cmd_deps.py index 3b62ac468a1..26976bf486a 100644 --- a/scripts/update_lib/show_deps.py +++ b/scripts/update_lib/cmd_deps.py @@ -195,7 +195,7 @@ def format_deps( is_path_synced, resolve_hard_dep_parent, ) - from update_lib.show_todo import count_test_todos, is_test_up_to_date + from update_lib.deps import count_test_todos, is_test_up_to_date if _visited is None: _visited = set() diff --git a/scripts/update_lib/migrate.py b/scripts/update_lib/cmd_migrate.py similarity index 98% rename from scripts/update_lib/migrate.py rename to scripts/update_lib/cmd_migrate.py index 22ec9517fcc..77292831cea 100644 --- a/scripts/update_lib/migrate.py +++ b/scripts/update_lib/cmd_migrate.py @@ -17,7 +17,7 @@ sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) -from update_lib.path import parse_lib_path +from update_lib.file_utils import parse_lib_path def patch_single_content( diff --git a/scripts/update_lib/patches.py b/scripts/update_lib/cmd_patches.py similarity index 100% rename from scripts/update_lib/patches.py rename to scripts/update_lib/cmd_patches.py diff --git a/scripts/update_lib/quick.py b/scripts/update_lib/cmd_quick.py similarity index 93% rename from scripts/update_lib/quick.py rename to scripts/update_lib/cmd_quick.py index 19c5714c8f5..d23cf7a2e89 100644 --- a/scripts/update_lib/quick.py +++ b/scripts/update_lib/cmd_quick.py @@ -32,9 +32,10 @@ sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) from update_lib.deps import get_test_paths -from update_lib.io_utils import safe_read_text -from update_lib.path import ( +from update_lib.file_utils import safe_read_text +from update_lib.file_utils import ( construct_lib_path, + get_cpython_dir, get_module_name, get_test_files, is_lib_path, @@ -55,7 +56,7 @@ def collect_original_methods( - For file: set of (class_name, method_name) or None if file doesn't exist - For directory: dict mapping file path to set of methods, or None if dir doesn't exist """ - from update_lib.auto_mark import extract_test_methods + from update_lib.cmd_auto_mark import extract_test_methods if not lib_path.exists(): return None @@ -91,8 +92,8 @@ def quick( verbose: Print progress messages skip_build: Skip cargo build, use pre-built binary """ - from update_lib.auto_mark import auto_mark_directory, auto_mark_file - from update_lib.migrate import patch_directory, patch_file + from update_lib.cmd_auto_mark import auto_mark_directory, auto_mark_file + from update_lib.cmd_migrate import patch_directory, patch_file # Determine lib_path and whether to migrate if is_lib_path(src_path): @@ -174,22 +175,6 @@ def quick( print(f"Removed expectedFailure from {num_removed} tests") -def get_cpython_dir(src_path: pathlib.Path) -> pathlib.Path: - """Extract cpython directory from source path. - - Example: - cpython/Lib/dataclasses.py -> cpython - /some/path/cpython/Lib/foo.py -> /some/path/cpython - """ - path_str = str(src_path).replace("\\", "/") - lib_marker = "/Lib/" - if lib_marker in path_str: - idx = path_str.index(lib_marker) - return pathlib.Path(path_str[:idx]) - # Shortcut case: assume "cpython" - return pathlib.Path("cpython") - - def get_cpython_version(cpython_dir: pathlib.Path) -> str: """Get CPython version from git tag.""" import subprocess @@ -384,7 +369,7 @@ def main(argv: list[str] | None = None) -> int: lib_file_path = parse_lib_path(src_path) if args.copy: - from update_lib.copy_lib import copy_lib + from update_lib.cmd_copy_lib import copy_lib copy_lib(src_path) @@ -449,7 +434,7 @@ def main(argv: list[str] | None = None) -> int: return 1 except Exception as e: # Handle TestRunError with a clean message - from update_lib.auto_mark import TestRunError + from update_lib.cmd_auto_mark import TestRunError if isinstance(e, TestRunError): print(f"Error: {e}", file=sys.stderr) diff --git a/scripts/update_lib/show_todo.py b/scripts/update_lib/cmd_todo.py similarity index 82% rename from scripts/update_lib/show_todo.py rename to scripts/update_lib/cmd_todo.py index 352454ee4e9..7590bac987e 100644 --- a/scripts/update_lib/show_todo.py +++ b/scripts/update_lib/cmd_todo.py @@ -13,6 +13,12 @@ sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) +from update_lib.deps import ( + count_test_todos, + is_test_tracked, + is_test_up_to_date, +) + def compute_todo_list( cpython_prefix: str, @@ -38,7 +44,7 @@ def compute_todo_list( get_soft_deps, is_up_to_date, ) - from update_lib.show_deps import get_all_modules + from update_lib.cmd_deps import get_all_modules all_modules = get_all_modules(cpython_prefix) @@ -165,7 +171,7 @@ def get_untracked_files( Sorted list of relative paths (e.g., ["foo.py", "data/file.txt"]) """ from update_lib.deps import resolve_hard_dep_parent - from update_lib.show_deps import get_all_modules + from update_lib.cmd_deps import get_all_modules cpython_lib = pathlib.Path(cpython_prefix) / "Lib" local_lib = pathlib.Path(lib_prefix) @@ -260,127 +266,6 @@ def get_original_files( return sorted(original) -def _filter_rustpython_todo(content: str) -> str: - """Remove lines containing 'TODO: RUSTPYTHON' from content.""" - lines = content.splitlines(keepends=True) - filtered = [line for line in lines if "TODO: RUSTPYTHON" not in line] - return "".join(filtered) - - -def _count_rustpython_todo(content: str) -> int: - """Count lines containing 'TODO: RUSTPYTHON' in content.""" - return sum(1 for line in content.splitlines() if "TODO: RUSTPYTHON" in line) - - -def _compare_file_ignoring_todo( - cpython_path: pathlib.Path, local_path: pathlib.Path -) -> bool: - """Compare two files, ignoring TODO: RUSTPYTHON lines in local file.""" - try: - cpython_content = cpython_path.read_text(encoding="utf-8") - local_content = local_path.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - return False - - local_filtered = _filter_rustpython_todo(local_content) - return cpython_content == local_filtered - - -def _compare_dir_ignoring_todo( - cpython_path: pathlib.Path, local_path: pathlib.Path -) -> bool: - """Compare two directories, ignoring TODO: RUSTPYTHON lines in local files.""" - # Get all .py files in both directories - cpython_files = {f.relative_to(cpython_path) for f in cpython_path.rglob("*.py")} - local_files = {f.relative_to(local_path) for f in local_path.rglob("*.py")} - - # Check for missing or extra files - if cpython_files != local_files: - return False - - # Compare each file - for rel_path in cpython_files: - if not _compare_file_ignoring_todo( - cpython_path / rel_path, local_path / rel_path - ): - return False - - return True - - -def count_test_todos(test_name: str, lib_prefix: str) -> int: - """Count TODO: RUSTPYTHON lines in a test file/directory.""" - local_dir = pathlib.Path(lib_prefix) / "test" / test_name - local_file = pathlib.Path(lib_prefix) / "test" / f"{test_name}.py" - - if local_dir.exists(): - local_path = local_dir - elif local_file.exists(): - local_path = local_file - else: - return 0 - - total = 0 - if local_path.is_file(): - try: - content = local_path.read_text(encoding="utf-8") - total = _count_rustpython_todo(content) - except (OSError, UnicodeDecodeError): - pass - else: - for py_file in local_path.rglob("*.py"): - try: - content = py_file.read_text(encoding="utf-8") - total += _count_rustpython_todo(content) - except (OSError, UnicodeDecodeError): - pass - - return total - - -def is_test_tracked(test_name: str, cpython_prefix: str, lib_prefix: str) -> bool: - """Check if a test exists in our local Lib/test.""" - cpython_dir = pathlib.Path(cpython_prefix) / "Lib" / "test" / test_name - cpython_file = pathlib.Path(cpython_prefix) / "Lib" / "test" / f"{test_name}.py" - - if cpython_dir.exists(): - cpython_path = cpython_dir - elif cpython_file.exists(): - cpython_path = cpython_file - else: - return True # No cpython test - - local_path = pathlib.Path(lib_prefix) / "test" / cpython_path.name - return local_path.exists() - - -def is_test_up_to_date(test_name: str, cpython_prefix: str, lib_prefix: str) -> bool: - """Check if a test is up-to-date by comparing files. - - Ignores lines containing 'TODO: RUSTPYTHON' in local files. - """ - # Try directory first, then file - cpython_dir = pathlib.Path(cpython_prefix) / "Lib" / "test" / test_name - cpython_file = pathlib.Path(cpython_prefix) / "Lib" / "test" / f"{test_name}.py" - - if cpython_dir.exists(): - cpython_path = cpython_dir - elif cpython_file.exists(): - cpython_path = cpython_file - else: - return True # No cpython test, consider up-to-date - - local_path = pathlib.Path(lib_prefix) / "test" / cpython_path.name - - if not local_path.exists(): - return False - - if cpython_path.is_file(): - return _compare_file_ignoring_todo(cpython_path, local_path) - else: - return _compare_dir_ignoring_todo(cpython_path, local_path) - - def _build_test_to_lib_map( cpython_prefix: str, ) -> tuple[dict[str, str], dict[str, list[str]]]: @@ -624,7 +509,7 @@ def format_all_todo( List of formatted lines """ from update_lib.deps import is_up_to_date - from update_lib.show_deps import get_all_modules + from update_lib.cmd_deps import get_all_modules lines = [] diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 30c0b8555c8..4dd0a9d2ee8 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -14,7 +14,166 @@ import shelve import subprocess -from update_lib.io_utils import read_python_files, safe_parse_ast, safe_read_text +from update_lib.file_utils import ( + _dircmp_is_same, + compare_dir_contents, + compare_file_contents, + compare_paths, + construct_lib_path, + cpython_to_local_path, + read_python_files, + resolve_module_path, + resolve_test_path, + safe_parse_ast, + safe_read_text, +) + +# === Import parsing utilities === + + +def _extract_top_level_code(content: str) -> str: + """Extract only top-level code from Python content for faster parsing.""" + def_idx = content.find("\ndef ") + class_idx = content.find("\nclass ") + + indices = [i for i in (def_idx, class_idx) if i != -1] + if indices: + content = content[: min(indices)] + return content.rstrip("\n") + + +_FROM_TEST_IMPORT_RE = re.compile(r"^from test import (.+)", re.MULTILINE) +_FROM_TEST_DOT_RE = re.compile(r"^from test\.(\w+)", re.MULTILINE) +_IMPORT_TEST_DOT_RE = re.compile(r"^import test\.(\w+)", re.MULTILINE) + + +def parse_test_imports(content: str) -> set[str]: + """Parse test file content and extract test package dependencies.""" + content = _extract_top_level_code(content) + imports = set() + + for match in _FROM_TEST_IMPORT_RE.finditer(content): + import_list = match.group(1) + for part in import_list.split(","): + name = part.split()[0].strip() + if name and name not in ("support", "__init__"): + imports.add(name) + + for match in _FROM_TEST_DOT_RE.finditer(content): + dep = match.group(1) + if dep not in ("support", "__init__"): + imports.add(dep) + + for match in _IMPORT_TEST_DOT_RE.finditer(content): + dep = match.group(1) + if dep not in ("support", "__init__"): + imports.add(dep) + + return imports + + +_IMPORT_RE = re.compile(r"^import\s+(\w[\w.]*)", re.MULTILINE) +_FROM_IMPORT_RE = re.compile(r"^from\s+(\w[\w.]*)\s+import", re.MULTILINE) + + +def parse_lib_imports(content: str) -> set[str]: + """Parse library file and extract all imported module names.""" + imports = set() + + for match in _IMPORT_RE.finditer(content): + imports.add(match.group(1)) + + for match in _FROM_IMPORT_RE.finditer(content): + imports.add(match.group(1)) + + return imports + + +# === TODO marker utilities === + +TODO_MARKER = "TODO: RUSTPYTHON" + + +def filter_rustpython_todo(content: str) -> str: + """Remove lines containing RustPython TODO markers.""" + lines = content.splitlines(keepends=True) + filtered = [line for line in lines if TODO_MARKER not in line] + return "".join(filtered) + + +def count_rustpython_todo(content: str) -> int: + """Count lines containing RustPython TODO markers.""" + return sum(1 for line in content.splitlines() if TODO_MARKER in line) + + +def count_todo_in_path(path: pathlib.Path) -> int: + """Count RustPython TODO markers in a file or directory of .py files.""" + if path.is_file(): + content = safe_read_text(path) + return count_rustpython_todo(content) if content else 0 + + total = 0 + for _, content in read_python_files(path): + total += count_rustpython_todo(content) + return total + + +# === Test utilities === + + +def _get_cpython_test_path(test_name: str, cpython_prefix: str) -> pathlib.Path | None: + """Return the CPython test path for a test name, or None if missing.""" + cpython_path = resolve_test_path(test_name, cpython_prefix, prefer="dir") + return cpython_path if cpython_path.exists() else None + + +def _get_local_test_path( + cpython_test_path: pathlib.Path, lib_prefix: str +) -> pathlib.Path: + """Return the local Lib/test path matching a CPython test path.""" + return pathlib.Path(lib_prefix) / "test" / cpython_test_path.name + + +def is_test_tracked(test_name: str, cpython_prefix: str, lib_prefix: str) -> bool: + """Check if a test exists in the local Lib/test.""" + cpython_path = _get_cpython_test_path(test_name, cpython_prefix) + if cpython_path is None: + return True + local_path = _get_local_test_path(cpython_path, lib_prefix) + return local_path.exists() + + +def is_test_up_to_date(test_name: str, cpython_prefix: str, lib_prefix: str) -> bool: + """Check if a test is up-to-date, ignoring RustPython TODO markers.""" + cpython_path = _get_cpython_test_path(test_name, cpython_prefix) + if cpython_path is None: + return True + + local_path = _get_local_test_path(cpython_path, lib_prefix) + if not local_path.exists(): + return False + + if cpython_path.is_file(): + return compare_file_contents( + cpython_path, local_path, local_filter=filter_rustpython_todo + ) + + return compare_dir_contents( + cpython_path, local_path, local_filter=filter_rustpython_todo + ) + + +def count_test_todos(test_name: str, lib_prefix: str) -> int: + """Count RustPython TODO markers in a test file/directory.""" + local_dir = pathlib.Path(lib_prefix) / "test" / test_name + local_file = pathlib.Path(lib_prefix) / "test" / f"{test_name}.py" + + if local_dir.exists(): + return count_todo_in_path(local_dir) + if local_file.exists(): + return count_todo_in_path(local_file) + return 0 + # === Cross-process cache using shelve === @@ -50,8 +209,6 @@ def clear_import_graph_caches() -> None: globals()["_lib_import_graph_cache"].clear() -from update_lib.path import construct_lib_path, resolve_module_path - # Manual dependency table for irregular cases # Format: "name" -> {"lib": [...], "test": [...], "data": [...], "hard_deps": [...]} # - lib: override default path (default: name.py or name/) @@ -634,98 +791,6 @@ def get_test_paths(name: str, cpython_prefix: str) -> tuple[pathlib.Path, ...]: return (resolve_module_path(f"test/test_{name}", cpython_prefix, prefer="dir"),) -def _extract_top_level_code(content: str) -> str: - """Extract only top-level code from Python content for faster parsing. - - Cuts at first function/class definition since imports come before them. - """ - # Find first function or class definition - def_idx = content.find("\ndef ") - class_idx = content.find("\nclass ") - - # Use the earlier of the two (if found) - indices = [i for i in (def_idx, class_idx) if i != -1] - if indices: - content = content[: min(indices)] - return content.rstrip("\n") - - -_FROM_TEST_IMPORT_RE = re.compile(r"^from test import (.+)", re.MULTILINE) -_FROM_TEST_DOT_RE = re.compile(r"^from test\.(\w+)", re.MULTILINE) -_IMPORT_TEST_DOT_RE = re.compile(r"^import test\.(\w+)", re.MULTILINE) - - -def parse_test_imports(content: str) -> set[str]: - """Parse test file content and extract test package dependencies. - - Uses regex for speed - only matches top-level imports. - - Args: - content: Python file content - - Returns: - Set of module names imported from test package - """ - content = _extract_top_level_code(content) - imports = set() - - # Match "from test import foo, bar, baz" - for match in _FROM_TEST_IMPORT_RE.finditer(content): - import_list = match.group(1) - # Parse "foo, bar as b, baz" -> ["foo", "bar", "baz"] - for part in import_list.split(","): - name = part.split()[0].strip() # Handle "foo as f" - if name and name not in ("support", "__init__"): - imports.add(name) - - # Match "from test.foo import ..." -> depends on foo - for match in _FROM_TEST_DOT_RE.finditer(content): - dep = match.group(1) - if dep not in ("support", "__init__"): - imports.add(dep) - - # Match "import test.foo" -> depends on foo - for match in _IMPORT_TEST_DOT_RE.finditer(content): - dep = match.group(1) - if dep not in ("support", "__init__"): - imports.add(dep) - - return imports - - -# Match "import foo.bar" - module name must start with word char (not dot) -_IMPORT_RE = re.compile(r"^import\s+(\w[\w.]*)", re.MULTILINE) -# Match "from foo.bar import" - exclude relative imports (from . or from ..) -_FROM_IMPORT_RE = re.compile(r"^from\s+(\w[\w.]*)\s+import", re.MULTILINE) - - -def parse_lib_imports(content: str) -> set[str]: - """Parse library file and extract all imported module names. - - Uses regex for speed - only matches top-level imports (no leading whitespace). - Returns full module paths (e.g., "collections.abc" not just "collections"). - - Args: - content: Python file content - - Returns: - Set of imported module names (full paths) - """ - # Note: Don't truncate content here - some stdlib files have imports after - # the first def/class (e.g., _pydecimal.py has `import contextvars` at line 343) - imports = set() - - # Match "import foo.bar" at line start - for match in _IMPORT_RE.finditer(content): - imports.add(match.group(1)) - - # Match "from foo.bar import ..." at line start - for match in _FROM_IMPORT_RE.finditer(content): - imports.add(match.group(1)) - - return imports - - @functools.cache def get_all_imports(name: str, cpython_prefix: str) -> frozenset[str]: """Get all imports from a library file. @@ -787,70 +852,6 @@ def get_rust_deps(name: str, cpython_prefix: str) -> frozenset[str]: return frozenset(all_imports - soft_deps) -def _dircmp_is_same(dcmp) -> bool: - """Recursively check if two directories are identical. - - Args: - dcmp: filecmp.dircmp object - - Returns: - True if directories are identical (including subdirectories) - """ - if dcmp.diff_files or dcmp.left_only or dcmp.right_only: - return False - - # Recursively check subdirectories - for subdir in dcmp.subdirs.values(): - if not _dircmp_is_same(subdir): - return False - - return True - - -def compare_paths(cpython_path: pathlib.Path, local_path: pathlib.Path) -> bool: - """Compare a CPython path with a local path. - - Args: - cpython_path: Path in CPython directory (file or directory) - local_path: Corresponding path in local Lib directory - - Returns: - True if paths are identical, False otherwise - """ - import filecmp - - if not local_path.exists(): - return False - - if cpython_path.is_file(): - return filecmp.cmp(cpython_path, local_path, shallow=False) - else: - dcmp = filecmp.dircmp(cpython_path, local_path) - return _dircmp_is_same(dcmp) - - -def cpython_to_local_path( - cpython_path: pathlib.Path, - cpython_prefix: str, - lib_prefix: str, -) -> pathlib.Path | None: - """Convert CPython path to local Lib path. - - Args: - cpython_path: Path like cpython/Lib/foo.py - cpython_prefix: CPython directory prefix - lib_prefix: Local Lib directory prefix - - Returns: - Local path like Lib/foo.py, or None if conversion fails - """ - try: - rel_path = cpython_path.relative_to(cpython_prefix) - return pathlib.Path(lib_prefix) / rel_path.relative_to("Lib") - except ValueError: - return None - - def is_path_synced( cpython_path: pathlib.Path, cpython_prefix: str, diff --git a/scripts/update_lib/file_utils.py b/scripts/update_lib/file_utils.py new file mode 100644 index 00000000000..784a8aba5c8 --- /dev/null +++ b/scripts/update_lib/file_utils.py @@ -0,0 +1,290 @@ +""" +File utilities for update_lib. + +This module provides functions for: +- Safe file reading with error handling +- Safe AST parsing with error handling +- Iterating over Python files +- Parsing and converting library paths +- Detecting test paths vs library paths +- Comparing files or directories for equality +""" + +from __future__ import annotations + +import ast +import filecmp +import pathlib +from collections.abc import Callable, Iterator + + +# === I/O utilities === + + +def safe_read_text(path: pathlib.Path) -> str | None: + """Read file content with UTF-8 encoding, returning None on error.""" + try: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return None + + +def safe_parse_ast(content: str) -> ast.Module | None: + """Parse Python content into AST, returning None on syntax error.""" + try: + return ast.parse(content) + except SyntaxError: + return None + + +def iter_python_files(path: pathlib.Path) -> Iterator[pathlib.Path]: + """Yield Python files from a file or directory.""" + if path.is_file(): + yield path + else: + yield from path.glob("**/*.py") + + +def read_python_files(path: pathlib.Path) -> Iterator[tuple[pathlib.Path, str]]: + """Read all Python files from a path, yielding (path, content) pairs.""" + for py_file in iter_python_files(path): + content = safe_read_text(py_file) + if content is not None: + yield py_file, content + + +# === Path utilities === + + +def parse_lib_path(path: pathlib.Path | str) -> pathlib.Path: + """ + Extract the Lib/... portion from a path containing /Lib/. + + Example: + parse_lib_path("cpython/Lib/test/foo.py") -> Path("Lib/test/foo.py") + """ + path_str = str(path).replace("\\", "/") + lib_marker = "/Lib/" + + if lib_marker not in path_str: + raise ValueError(f"Path must contain '/Lib/' or '\\Lib\\' (got: {path})") + + idx = path_str.index(lib_marker) + return pathlib.Path(path_str[idx + 1 :]) + + +def is_lib_path(path: pathlib.Path) -> bool: + """Check if path starts with Lib/""" + path_str = str(path).replace("\\", "/") + return path_str.startswith("Lib/") or path_str.startswith("./Lib/") + + +def is_test_path(path: pathlib.Path) -> bool: + """Check if path is a test path (contains /Lib/test/ or starts with Lib/test/)""" + path_str = str(path).replace("\\", "/") + return "/Lib/test/" in path_str or path_str.startswith("Lib/test/") + + +def lib_to_test_path(src_path: pathlib.Path) -> pathlib.Path: + """ + Convert library path to test path. + + Examples: + cpython/Lib/dataclasses.py -> cpython/Lib/test/test_dataclasses/ + cpython/Lib/json/__init__.py -> cpython/Lib/test/test_json/ + """ + path_str = str(src_path).replace("\\", "/") + lib_marker = "/Lib/" + + if lib_marker in path_str: + lib_path = parse_lib_path(src_path) + lib_name = lib_path.stem if lib_path.suffix == ".py" else lib_path.name + if lib_name == "__init__": + lib_name = lib_path.parent.name + prefix = path_str[: path_str.index(lib_marker)] + dir_path = pathlib.Path(f"{prefix}/Lib/test/test_{lib_name}/") + if dir_path.exists(): + return dir_path + file_path = pathlib.Path(f"{prefix}/Lib/test/test_{lib_name}.py") + if file_path.exists(): + return file_path + return dir_path + else: + lib_name = src_path.stem if src_path.suffix == ".py" else src_path.name + if lib_name == "__init__": + lib_name = src_path.parent.name + dir_path = pathlib.Path(f"Lib/test/test_{lib_name}/") + if dir_path.exists(): + return dir_path + file_path = pathlib.Path(f"Lib/test/test_{lib_name}.py") + if file_path.exists(): + return file_path + return dir_path + + +def get_test_files(path: pathlib.Path) -> list[pathlib.Path]: + """Get all .py test files in a path (file or directory).""" + if path.is_file(): + return [path] + return sorted(path.glob("**/*.py")) + + +def get_test_module_name(test_path: pathlib.Path) -> str: + """ + Extract test module name from a test file path. + + Examples: + Lib/test/test_foo.py -> test_foo + Lib/test/test_ctypes/test_bar.py -> test_ctypes.test_bar + """ + test_path = pathlib.Path(test_path) + if test_path.parent.name.startswith("test_"): + return f"{test_path.parent.name}.{test_path.stem}" + return test_path.stem + + +def resolve_module_path( + name: str, prefix: str = "cpython", prefer: str = "file" +) -> pathlib.Path: + """ + Resolve module path, trying file or directory. + + Args: + name: Module name (e.g., "dataclasses", "json") + prefix: CPython directory prefix + prefer: "file" to try .py first, "dir" to try directory first + """ + file_path = pathlib.Path(f"{prefix}/Lib/{name}.py") + dir_path = pathlib.Path(f"{prefix}/Lib/{name}") + + if prefer == "file": + if file_path.exists(): + return file_path + if dir_path.exists(): + return dir_path + return file_path + else: + if dir_path.exists(): + return dir_path + if file_path.exists(): + return file_path + return dir_path + + +def construct_lib_path(prefix: str, *parts: str) -> pathlib.Path: + """Build a path under prefix/Lib/.""" + return pathlib.Path(prefix) / "Lib" / pathlib.Path(*parts) + + +def resolve_test_path( + test_name: str, prefix: str = "cpython", prefer: str = "dir" +) -> pathlib.Path: + """Resolve a test module path under Lib/test/.""" + return resolve_module_path(f"test/{test_name}", prefix, prefer=prefer) + + +def cpython_to_local_path( + cpython_path: pathlib.Path, + cpython_prefix: str, + lib_prefix: str, +) -> pathlib.Path | None: + """Convert CPython path to local Lib path.""" + try: + rel_path = cpython_path.relative_to(cpython_prefix) + return pathlib.Path(lib_prefix) / rel_path.relative_to("Lib") + except ValueError: + return None + + +def get_module_name(path: pathlib.Path) -> str: + """Extract module name from path, handling __init__.py.""" + if path.suffix == ".py": + name = path.stem + if name == "__init__": + return path.parent.name + return name + return path.name + + +def get_cpython_dir(src_path: pathlib.Path) -> pathlib.Path: + """Extract CPython directory from a path containing /Lib/.""" + path_str = str(src_path).replace("\\", "/") + lib_marker = "/Lib/" + if lib_marker in path_str: + idx = path_str.index(lib_marker) + return pathlib.Path(path_str[:idx]) + return pathlib.Path("cpython") + + +# === Comparison utilities === + + +def _dircmp_is_same(dcmp: filecmp.dircmp) -> bool: + """Recursively check if two directories are identical.""" + if dcmp.diff_files or dcmp.left_only or dcmp.right_only: + return False + + for subdir in dcmp.subdirs.values(): + if not _dircmp_is_same(subdir): + return False + + return True + + +def compare_paths(cpython_path: pathlib.Path, local_path: pathlib.Path) -> bool: + """Compare a CPython path with a local path (file or directory).""" + if not local_path.exists(): + return False + + if cpython_path.is_file(): + return filecmp.cmp(cpython_path, local_path, shallow=False) + + dcmp = filecmp.dircmp(cpython_path, local_path) + return _dircmp_is_same(dcmp) + + +def compare_file_contents( + cpython_path: pathlib.Path, + local_path: pathlib.Path, + *, + local_filter: Callable[[str], str] | None = None, + encoding: str = "utf-8", +) -> bool: + """Compare two files as text, optionally filtering local content.""" + try: + cpython_content = cpython_path.read_text(encoding=encoding) + local_content = local_path.read_text(encoding=encoding) + except (OSError, UnicodeDecodeError): + return False + + if local_filter is not None: + local_content = local_filter(local_content) + + return cpython_content == local_content + + +def compare_dir_contents( + cpython_dir: pathlib.Path, + local_dir: pathlib.Path, + *, + pattern: str = "*.py", + local_filter: Callable[[str], str] | None = None, + encoding: str = "utf-8", +) -> bool: + """Compare directory contents for matching files and text.""" + cpython_files = {f.relative_to(cpython_dir) for f in cpython_dir.rglob(pattern)} + local_files = {f.relative_to(local_dir) for f in local_dir.rglob(pattern)} + + if cpython_files != local_files: + return False + + for rel_path in cpython_files: + if not compare_file_contents( + cpython_dir / rel_path, + local_dir / rel_path, + local_filter=local_filter, + encoding=encoding, + ): + return False + + return True diff --git a/scripts/update_lib/io_utils.py b/scripts/update_lib/io_utils.py deleted file mode 100644 index 8bf0083211f..00000000000 --- a/scripts/update_lib/io_utils.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -I/O utilities for update_lib. - -This module provides functions for: -- Safe file reading with error handling -- Safe AST parsing with error handling -- Iterating over Python files -""" - -import ast -import pathlib -from collections.abc import Iterator - - -def safe_read_text(path: pathlib.Path) -> str | None: - """ - Read file content with UTF-8 encoding, returning None on error. - - Args: - path: Path to the file - - Returns: - File content as string, or None if reading fails - """ - try: - return path.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - return None - - -def safe_parse_ast(content: str) -> ast.Module | None: - """ - Parse Python content into AST, returning None on syntax error. - - Args: - content: Python source code - - Returns: - AST module, or None if parsing fails - """ - try: - return ast.parse(content) - except SyntaxError: - return None - - -def iter_python_files(path: pathlib.Path) -> Iterator[pathlib.Path]: - """ - Yield Python files from a file or directory. - - If path is a file, yields just that file. - If path is a directory, yields all .py files recursively. - - Args: - path: Path to a file or directory - - Yields: - Paths to Python files - """ - if path.is_file(): - yield path - else: - yield from path.glob("**/*.py") - - -def read_python_files(path: pathlib.Path) -> Iterator[tuple[pathlib.Path, str]]: - """ - Read all Python files from a path, yielding (path, content) pairs. - - Skips files that cannot be read. - - Args: - path: Path to a file or directory - - Yields: - Tuples of (file_path, file_content) - """ - for py_file in iter_python_files(path): - content = safe_read_text(py_file) - if content is not None: - yield py_file, content diff --git a/scripts/update_lib/path.py b/scripts/update_lib/path.py deleted file mode 100644 index d2360e21cd6..00000000000 --- a/scripts/update_lib/path.py +++ /dev/null @@ -1,187 +0,0 @@ -""" -Path utilities for update_lib. - -This module provides functions for: -- Parsing and converting library paths -- Detecting test paths vs library paths -- Extracting test module names from paths -""" - -import pathlib - - -def parse_lib_path(path: pathlib.Path | str) -> pathlib.Path: - """ - Extract the Lib/... portion from a path containing /Lib/. - - Example: - parse_lib_path("cpython/Lib/test/foo.py") -> Path("Lib/test/foo.py") - """ - path_str = str(path).replace("\\", "/") - lib_marker = "/Lib/" - - if lib_marker not in path_str: - raise ValueError(f"Path must contain '/Lib/' or '\\Lib\\' (got: {path})") - - idx = path_str.index(lib_marker) - return pathlib.Path(path_str[idx + 1 :]) - - -def is_lib_path(path: pathlib.Path) -> bool: - """Check if path starts with Lib/""" - path_str = str(path).replace("\\", "/") - return path_str.startswith("Lib/") or path_str.startswith("./Lib/") - - -def is_test_path(path: pathlib.Path) -> bool: - """Check if path is a test path (contains /Lib/test/ or starts with Lib/test/)""" - path_str = str(path).replace("\\", "/") - return "/Lib/test/" in path_str or path_str.startswith("Lib/test/") - - -def lib_to_test_path(src_path: pathlib.Path) -> pathlib.Path: - """ - Convert library path to test path. - - Examples: - cpython/Lib/dataclasses.py -> cpython/Lib/test/test_dataclasses/ (if dir exists) - cpython/Lib/typing.py -> cpython/Lib/test/test_typing.py (if file exists) - cpython/Lib/json/ -> cpython/Lib/test/test_json/ - cpython/Lib/json/__init__.py -> cpython/Lib/test/test_json/ - Lib/dataclasses.py -> Lib/test/test_dataclasses/ - """ - path_str = str(src_path).replace("\\", "/") - lib_marker = "/Lib/" - - if lib_marker in path_str: - lib_path = parse_lib_path(src_path) - lib_name = lib_path.stem if lib_path.suffix == ".py" else lib_path.name - # Handle __init__.py: use parent directory name - if lib_name == "__init__": - lib_name = lib_path.parent.name - prefix = path_str[: path_str.index(lib_marker)] - # Try directory first, then file - dir_path = pathlib.Path(f"{prefix}/Lib/test/test_{lib_name}/") - if dir_path.exists(): - return dir_path - file_path = pathlib.Path(f"{prefix}/Lib/test/test_{lib_name}.py") - if file_path.exists(): - return file_path - # Default to directory (caller will handle non-existence) - return dir_path - else: - # Path starts with Lib/ - extract name directly - lib_name = src_path.stem if src_path.suffix == ".py" else src_path.name - # Handle __init__.py: use parent directory name - if lib_name == "__init__": - lib_name = src_path.parent.name - # Try directory first, then file - dir_path = pathlib.Path(f"Lib/test/test_{lib_name}/") - if dir_path.exists(): - return dir_path - file_path = pathlib.Path(f"Lib/test/test_{lib_name}.py") - if file_path.exists(): - return file_path - return dir_path - - -def get_test_files(path: pathlib.Path) -> list[pathlib.Path]: - """Get all .py test files in a path (file or directory).""" - if path.is_file(): - return [path] - return sorted(path.glob("**/*.py")) - - -def test_name_from_path(test_path: pathlib.Path) -> str: - """ - Extract test module name from a test file path. - - Examples: - Lib/test/test_foo.py -> test_foo - Lib/test/test_ctypes/test_bar.py -> test_ctypes.test_bar - """ - test_path = pathlib.Path(test_path) - if test_path.parent.name.startswith("test_"): - return f"{test_path.parent.name}.{test_path.stem}" - return test_path.stem - - -# --- Utility functions for reducing duplication --- - - -def resolve_module_path( - name: str, prefix: str = "cpython", prefer: str = "file" -) -> pathlib.Path: - """ - Resolve module path, trying file or directory. - - Args: - name: Module name (e.g., "dataclasses", "json") - prefix: CPython directory prefix - prefer: "file" to try .py first, "dir" to try directory first - - Returns: - Path to the module (file or directory) - - Examples: - resolve_module_path("dataclasses") -> cpython/Lib/dataclasses.py - resolve_module_path("json") -> cpython/Lib/json/ - """ - file_path = pathlib.Path(f"{prefix}/Lib/{name}.py") - dir_path = pathlib.Path(f"{prefix}/Lib/{name}") - - if prefer == "file": - if file_path.exists(): - return file_path - if dir_path.exists(): - return dir_path - return file_path # Default to file - else: - if dir_path.exists(): - return dir_path - if file_path.exists(): - return file_path - return dir_path # Default to dir - - -def construct_lib_path(prefix: str, *parts: str) -> pathlib.Path: - """ - Build a path under prefix/Lib/. - - Args: - prefix: Directory prefix (e.g., "cpython") - *parts: Path components after Lib/ - - Returns: - Combined path - - Examples: - construct_lib_path("cpython", "test", "test_foo.py") - -> cpython/Lib/test/test_foo.py - construct_lib_path("cpython", "dataclasses.py") - -> cpython/Lib/dataclasses.py - """ - return pathlib.Path(prefix) / "Lib" / pathlib.Path(*parts) - - -def get_module_name(path: pathlib.Path) -> str: - """ - Extract module name from path, handling __init__.py. - - Args: - path: Path to a Python file or directory - - Returns: - Module name - - Examples: - get_module_name(Path("cpython/Lib/dataclasses.py")) -> "dataclasses" - get_module_name(Path("cpython/Lib/json/__init__.py")) -> "json" - get_module_name(Path("cpython/Lib/json/")) -> "json" - """ - if path.suffix == ".py": - name = path.stem - if name == "__init__": - return path.parent.name - return name - return path.name diff --git a/scripts/update_lib/tests/test_auto_mark.py b/scripts/update_lib/tests/test_auto_mark.py index c62a8af888e..d919601a0e9 100644 --- a/scripts/update_lib/tests/test_auto_mark.py +++ b/scripts/update_lib/tests/test_auto_mark.py @@ -3,7 +3,7 @@ import subprocess import unittest -from update_lib.auto_mark import ( +from update_lib.cmd_auto_mark import ( Test, TestResult, _is_super_call_only, @@ -277,7 +277,7 @@ def test_collect_init_module_matching(self): When test results come from a package's __init__.py, the path is like: 'test.test_dataclasses.TestCase.test_foo' (no __init__) - But module_prefix from test_name_from_path would be: + But module_prefix from get_test_module_name would be: 'test_dataclasses.__init__' So we need to strip '.__init__' and add 'test.' prefix. diff --git a/scripts/update_lib/tests/test_copy_lib.py b/scripts/update_lib/tests/test_copy_lib.py index 81ca73b5310..aca00cb18f3 100644 --- a/scripts/update_lib/tests/test_copy_lib.py +++ b/scripts/update_lib/tests/test_copy_lib.py @@ -10,7 +10,7 @@ class TestCopySingle(unittest.TestCase): def test_copies_file(self): """Test copying a single file.""" - from update_lib.copy_lib import _copy_single + from update_lib.cmd_copy_lib import _copy_single with tempfile.TemporaryDirectory() as tmpdir: tmpdir = pathlib.Path(tmpdir) @@ -26,7 +26,7 @@ def test_copies_file(self): def test_copies_directory(self): """Test copying a directory.""" - from update_lib.copy_lib import _copy_single + from update_lib.cmd_copy_lib import _copy_single with tempfile.TemporaryDirectory() as tmpdir: tmpdir = pathlib.Path(tmpdir) @@ -43,7 +43,7 @@ def test_copies_directory(self): def test_removes_existing_before_copy(self): """Test that existing destination is removed before copy.""" - from update_lib.copy_lib import _copy_single + from update_lib.cmd_copy_lib import _copy_single with tempfile.TemporaryDirectory() as tmpdir: tmpdir = pathlib.Path(tmpdir) @@ -63,7 +63,7 @@ class TestCopyLib(unittest.TestCase): def test_raises_on_path_without_lib(self): """Test that copy_lib raises ValueError when path doesn't contain /Lib/.""" - from update_lib.copy_lib import copy_lib + from update_lib.cmd_copy_lib import copy_lib with self.assertRaises(ValueError) as ctx: copy_lib(pathlib.Path("some/path/without/lib.py")) diff --git a/scripts/update_lib/tests/test_migrate.py b/scripts/update_lib/tests/test_migrate.py index ff93052f6af..0cc247ba841 100644 --- a/scripts/update_lib/tests/test_migrate.py +++ b/scripts/update_lib/tests/test_migrate.py @@ -4,7 +4,7 @@ import tempfile import unittest -from update_lib.migrate import ( +from update_lib.cmd_migrate import ( patch_directory, patch_file, patch_single_content, diff --git a/scripts/update_lib/tests/test_path.py b/scripts/update_lib/tests/test_path.py index affa1c8913a..5ca02ac693d 100644 --- a/scripts/update_lib/tests/test_path.py +++ b/scripts/update_lib/tests/test_path.py @@ -4,13 +4,13 @@ import tempfile import unittest -from update_lib.path import ( +from update_lib.file_utils import ( get_test_files, is_lib_path, is_test_path, lib_to_test_path, parse_lib_path, - test_name_from_path, + get_test_module_name, ) @@ -202,22 +202,22 @@ def test_nested_directory(self): class TestTestNameFromPath(unittest.TestCase): - """Tests for test_name_from_path function.""" + """Tests for get_test_module_name function.""" def test_simple_test_file(self): """Test extracting name from simple test file.""" path = pathlib.Path("Lib/test/test_foo.py") - self.assertEqual(test_name_from_path(path), "test_foo") + self.assertEqual(get_test_module_name(path), "test_foo") def test_nested_test_file(self): """Test extracting name from nested test directory.""" path = pathlib.Path("Lib/test/test_ctypes/test_bar.py") - self.assertEqual(test_name_from_path(path), "test_ctypes.test_bar") + self.assertEqual(get_test_module_name(path), "test_ctypes.test_bar") def test_test_directory(self): """Test extracting name from test directory.""" path = pathlib.Path("Lib/test/test_json") - self.assertEqual(test_name_from_path(path), "test_json") + self.assertEqual(get_test_module_name(path), "test_json") if __name__ == "__main__": diff --git a/scripts/update_lib/tests/test_quick.py b/scripts/update_lib/tests/test_quick.py index f02ca21e186..70c9ad0d844 100644 --- a/scripts/update_lib/tests/test_quick.py +++ b/scripts/update_lib/tests/test_quick.py @@ -5,8 +5,8 @@ import unittest from unittest.mock import patch -from update_lib.path import lib_to_test_path -from update_lib.quick import ( +from update_lib.file_utils import lib_to_test_path +from update_lib.cmd_quick import ( _expand_shortcut, collect_original_methods, get_cpython_dir, @@ -159,7 +159,7 @@ class TestGitCommit(unittest.TestCase): """Tests for git_commit function.""" @patch("subprocess.run") - @patch("update_lib.quick.get_cpython_version") + @patch("update_lib.cmd_quick.get_cpython_version") def test_none_lib_path_not_added(self, mock_version, mock_run): """Test that None lib_path doesn't add '.' to git.""" mock_version.return_value = "v3.14.0" @@ -177,7 +177,7 @@ def test_none_lib_path_not_added(self, mock_version, mock_run): self.assertNotIn(".", add_call[0][0][2:]) # Skip "git" and "add" @patch("subprocess.run") - @patch("update_lib.quick.get_cpython_version") + @patch("update_lib.cmd_quick.get_cpython_version") def test_none_test_path_not_added(self, mock_version, mock_run): """Test that None test_path doesn't add '.' to git.""" mock_version.return_value = "v3.14.0" @@ -203,10 +203,10 @@ def test_both_none_returns_false(self): class TestQuickTestRunFailure(unittest.TestCase): """Tests for quick() behavior when test run fails.""" - @patch("update_lib.auto_mark.run_test") + @patch("update_lib.cmd_auto_mark.run_test") def test_auto_mark_raises_on_test_run_failure(self, mock_run_test): """Test that auto_mark_file raises when test run fails entirely.""" - from update_lib.auto_mark import TestResult, TestRunError, auto_mark_file + from update_lib.cmd_auto_mark import TestResult, TestRunError, auto_mark_file # Simulate test runner crash (empty tests_result) mock_run_test.return_value = TestResult( From 2f53dbbeeb4d7070997ca68411f064d5b29e1612 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 26 Jan 2026 02:43:46 +0000 Subject: [PATCH 3/3] Auto-format: ruff check --select I --fix --- scripts/update_lib/cmd_deps.py | 3 ++- scripts/update_lib/cmd_quick.py | 2 +- scripts/update_lib/cmd_todo.py | 6 +++--- scripts/update_lib/file_utils.py | 1 - scripts/update_lib/tests/test_path.py | 2 +- scripts/update_lib/tests/test_quick.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/update_lib/cmd_deps.py b/scripts/update_lib/cmd_deps.py index 26976bf486a..affb4b3609c 100644 --- a/scripts/update_lib/cmd_deps.py +++ b/scripts/update_lib/cmd_deps.py @@ -189,13 +189,14 @@ def format_deps( """ from update_lib.deps import ( DEPENDENCIES, + count_test_todos, find_dependent_tests_tree, get_lib_paths, get_test_paths, is_path_synced, + is_test_up_to_date, resolve_hard_dep_parent, ) - from update_lib.deps import count_test_todos, is_test_up_to_date if _visited is None: _visited = set() diff --git a/scripts/update_lib/cmd_quick.py b/scripts/update_lib/cmd_quick.py index d23cf7a2e89..939ad50b780 100644 --- a/scripts/update_lib/cmd_quick.py +++ b/scripts/update_lib/cmd_quick.py @@ -32,7 +32,6 @@ sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) from update_lib.deps import get_test_paths -from update_lib.file_utils import safe_read_text from update_lib.file_utils import ( construct_lib_path, get_cpython_dir, @@ -43,6 +42,7 @@ lib_to_test_path, parse_lib_path, resolve_module_path, + safe_read_text, ) diff --git a/scripts/update_lib/cmd_todo.py b/scripts/update_lib/cmd_todo.py index 7590bac987e..87099aa5422 100644 --- a/scripts/update_lib/cmd_todo.py +++ b/scripts/update_lib/cmd_todo.py @@ -38,13 +38,13 @@ def compute_todo_list( Returns: List of dicts with module info, sorted by priority """ + from update_lib.cmd_deps import get_all_modules from update_lib.deps import ( get_all_hard_deps, get_rust_deps, get_soft_deps, is_up_to_date, ) - from update_lib.cmd_deps import get_all_modules all_modules = get_all_modules(cpython_prefix) @@ -170,8 +170,8 @@ def get_untracked_files( Returns: Sorted list of relative paths (e.g., ["foo.py", "data/file.txt"]) """ - from update_lib.deps import resolve_hard_dep_parent from update_lib.cmd_deps import get_all_modules + from update_lib.deps import resolve_hard_dep_parent cpython_lib = pathlib.Path(cpython_prefix) / "Lib" local_lib = pathlib.Path(lib_prefix) @@ -508,8 +508,8 @@ def format_all_todo( Returns: List of formatted lines """ - from update_lib.deps import is_up_to_date from update_lib.cmd_deps import get_all_modules + from update_lib.deps import is_up_to_date lines = [] diff --git a/scripts/update_lib/file_utils.py b/scripts/update_lib/file_utils.py index 784a8aba5c8..cb86ee2e664 100644 --- a/scripts/update_lib/file_utils.py +++ b/scripts/update_lib/file_utils.py @@ -17,7 +17,6 @@ import pathlib from collections.abc import Callable, Iterator - # === I/O utilities === diff --git a/scripts/update_lib/tests/test_path.py b/scripts/update_lib/tests/test_path.py index 5ca02ac693d..f2dcdcf8f05 100644 --- a/scripts/update_lib/tests/test_path.py +++ b/scripts/update_lib/tests/test_path.py @@ -6,11 +6,11 @@ from update_lib.file_utils import ( get_test_files, + get_test_module_name, is_lib_path, is_test_path, lib_to_test_path, parse_lib_path, - get_test_module_name, ) diff --git a/scripts/update_lib/tests/test_quick.py b/scripts/update_lib/tests/test_quick.py index 70c9ad0d844..b5354436e97 100644 --- a/scripts/update_lib/tests/test_quick.py +++ b/scripts/update_lib/tests/test_quick.py @@ -5,13 +5,13 @@ import unittest from unittest.mock import patch -from update_lib.file_utils import lib_to_test_path from update_lib.cmd_quick import ( _expand_shortcut, collect_original_methods, get_cpython_dir, git_commit, ) +from update_lib.file_utils import lib_to_test_path class TestGetCpythonDir(unittest.TestCase):