Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(ingestion): share traverse and parse stages between full and resume ingestion (#4001)
Raghav Chamadiya5fa2dec3ca90
diff --git a/packages/core/src/repowise/core/pipeline/phases/ingestion.py b/packages/core/src/repowise/core/pipeline/phases/ingestion.pyindex 993cb97a3..969c2eebc 100644--- a/packages/core/src/repowise/core/pipeline/phases/ingestion.py+++ b/packages/core/src/repowise/core/pipeline/phases/ingestion.py@@ -12,6 +12,7 @@ import multiprocessingimport osimport timefrom concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor+from dataclasses import dataclassfrom pathlib import Pathfrom typing import Any@@ -404,46 +405,39 @@ def _read_sources(return [r for r in results if r is not None]-async def _run_ingestion(- repo_path: Path,- *,- exclude_patterns: list[str] | None,- include_submodules: bool = False,- include_nested_repos: bool = False,- skip_tests: bool,- skip_infra: bool,- progress: ProgressCallback | None,- derive_environment_facts: bool = False,-) -> tuple[list[Any], list[Any], Any, dict[str, bytes], Any, Any]:- """Traverse, parse, and build the dependency graph.+# Languages ``skip_infra`` drops from the parse set.+_INFRA_LANGUAGES = ("dockerfile", "makefile", "terraform", "shell")- Returns (parsed_files, file_infos, repo_structure, source_map,- graph_builder, traversal_stats, tech_items).- """- from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder+# TS/JS-family languages whose presence wires the tsconfig path-alias resolver.+_TS_LANGUAGES = frozenset({"typescript", "javascript", "svelte", "vue", "astro"})- traverser = FileTraverser(- repo_path,- extra_exclude_patterns=exclude_patterns or None,- include_submodules=include_submodules,- include_nested_repos=include_nested_repos,- )- # Walk directory tree+@dataclass(frozen=True)+class _WalkOptions:+ """What to walk and keep, shared by the full and the resume ingestion."""++ exclude_patterns: list[str] | None+ include_submodules: bool+ include_nested_repos: bool+ skip_tests: bool+ skip_infra: bool+ derive_environment_facts: bool+++async def _build_file_infos(traverser: Any, progress: ProgressCallback | None) -> list[Any]:+ """Walk the tree and build a FileInfo per path, in walk order.++ Stat + header reads run in a thread pool; ``asyncio.wrap_future`` keeps the+ event loop responsive while they run. Results stay in walk order, matching+ the update path's rebuild (pipeline/incremental.py): files reach+ GraphBuilder in this order and ambiguous reference resolution tie-breaks on+ insertion order. Progress still ticks per completion.+ """all_paths = list(traverser._walk())if progress:- # Use indeterminate progress (spinner) to avoid showing a misleading- # pre-filter total like "2132/83601".+ # Indeterminate (spinner): a pre-filter total like "2132/83601" misleads.progress.on_phase_start("traverse", None)- # Parallel stat + header reads (I/O bound).- # Use asyncio.wrap_future so the event loop stays responsive while waiting.- # Results are kept in walk order, matching the update path's rebuild- # (pipeline/incremental.py): files are fed to GraphBuilder in this order,- # and ambiguous reference resolution tie-breaks on insertion order — a- # completion-ordered list made the init-built graph differ from the- # update-built one, so the first post-init update could never hit the- # centrality cache. Progress still ticks per completion via callbacks.io_pool = ThreadPoolExecutor(max_workers=8)try:aws = [@@ -454,193 +448,260 @@ async def _run_ingestion(for fut in aws:fut.add_done_callback(_traverse_tick)ordered = await asyncio.gather(*aws, return_exceptions=True)- file_infos: list[Any] = [- r for r in ordered if r is not None and not isinstance(r, BaseException)- ]+ return [r for r in ordered if r is not None and not isinstance(r, BaseException)]finally:- # shutdown(wait=True) is blocking — run in a thread to keep the- # event loop responsive. All submitted futures have already- # completed by the time we reach here (gather awaited them).+ # shutdown(wait=True) blocks; every future is already done (gather+ # awaited them), so this only keeps the event loop responsive.await asyncio.to_thread(io_pool.shutdown, wait=True)- repo_structure = traverser.get_repo_structure(file_infos)-- # Structural episodes ride the walk that just finished — nested-repo names- # and console scripts are by-products of it. The one check that describes- # the machine rather than the repository is opt-in: this function is the- # *full pipeline*, not ``init``, and the workspace updater and the hosted- # job executor both reach it.- await _record_structural_episodes(- repo_path, traverser, allow_formatter_check=derive_environment_facts- )- _phase_done(progress, "traverse")- # Filter- if skip_tests:+def _filter_file_infos(file_infos: list[Any], walk: _WalkOptions) -> list[Any]:+ if walk.skip_tests:file_infos = [fi for fi in file_infos if not fi.is_test]- if skip_infra:- file_infos = [- fi- for fi in file_infos- if fi.language not in ("dockerfile", "makefile", "terraform", "shell")- ]+ if walk.skip_infra:+ file_infos = [fi for fi in file_infos if fi.language not in _INFRA_LANGUAGES]+ return file_infos- # Report the traversal here, not at the end of the phase: the graph build,- # dynamic hints, PageRank and community detection all run below, and on a- # large repo "Scanned N files" used to land several minutes after the- # traverse bar had finished and the user had moved on.- _emit_traversal_summary(progress, traverser.stats, len(file_infos))- # ---- Parse phase: CPU-bound, run in ProcessPoolExecutor ----------------- # Say once, here, what each spawned worker would otherwise discover- # independently and log at a level no default run renders: this- # environment cannot parse some of the languages this repo contains.- # Those files still index (paths, git history, regex-tier imports) but- # carry no symbols, which is not something to find out from an empty- # symbol list weeks later.- _report_missing_grammars(getattr(traverser, "stats", None), progress)- _report_query_compilation_failures(getattr(traverser, "stats", None), progress)-- if progress:- progress.on_phase_start("parse", len(file_infos))+async def _traverse_stage(+ repo_path: Path,+ walk: _WalkOptions,+ progress: ProgressCallback | None,+) -> tuple[Any, list[Any], Any]:+ """Walk, describe the repo, record structural episodes, then filter.- # Read source bytes up front in a thread pool (I/O-bound; keeps worker- # args small: FileInfo + bytes, both picklable plain dataclasses/bytes).- fi_and_bytes: list[tuple] = _read_sources(file_infos, progress)+ Returns ``(traverser, file_infos, repo_structure)``; *file_infos* is in+ walk order with ``skip_tests`` / ``skip_infra`` applied, while+ *repo_structure* describes the unfiltered walk.+ """+ from repowise.core.ingestion import FileTraverser- parsed_files: list[Any] = []- source_map: dict[str, bytes] = {}- from repowise.core.workspace.update import get_head_commit+ traverser = FileTraverser(+ repo_path,+ extra_exclude_patterns=walk.exclude_patterns or None,+ include_submodules=walk.include_submodules,+ include_nested_repos=walk.include_nested_repos,+ )+ file_infos = await _build_file_infos(traverser, progress)+ repo_structure = traverser.get_repo_structure(file_infos)- graph_builder = GraphBuilder(- repo_path=repo_path,- exclude_patterns=exclude_patterns,- centrality_cache_dir=repo_path / ".repowise",- head_commit=get_head_commit(repo_path),- include_submodules=include_submodules,- include_nested_repos=include_nested_repos,+ # Structural episodes ride the walk that just finished: nested-repo names+ # and console scripts are by-products of it. The one check that describes+ # the machine rather than the repository is opt-in, because the full+ # pipeline is also reached by the workspace updater and hosted jobs.+ await _record_structural_episodes(+ repo_path, traverser, allow_formatter_check=walk.derive_environment_facts)- graph_builder.console_scripts = traverser.console_script_names+ _phase_done(progress, "traverse")+ return traverser, _filter_file_infos(file_infos, walk), repo_structure++async def _parse_in_pool(+ to_parse: list[tuple], workers: int, progress: ProgressCallback | None+) -> list[Any]:+ """Parse cache misses in the spawn process pool; results in *to_parse* order."""loop = asyncio.get_running_loop()+ with ProcessPoolExecutor(max_workers=workers, mp_context=_MP_SPAWN) as pool:+ tasks = [loop.run_in_executor(pool, _parse_one, item) for _idx, item, _h in to_parse]+ # Tick per finished worker: ``asyncio.gather`` alone holds every event+ # until the last file is done, which on large repos looked like a hang+ # at ``0/N``. Done-callbacks keep gather's ordered results intact.+ if progress is not None:+ _parse_tick = lambda _fut: progress.on_item_done("parse") # noqa: E731+ for fut in tasks:+ fut.add_done_callback(_parse_tick)+ return await asyncio.gather(*tasks, return_exceptions=True)- # Content-hash parse cache: unchanged files skip the worker pool- # entirely (an incremental update re-ingests the whole repo for one- # changed file). Misses parse exactly as before.- parse_cache, cached_hits, to_parse = _split_cached(repo_path, fi_and_bytes, progress)- workers = parse_pool_workers(len(to_parse))- parse_results: list[Any] = []+async def _parse_in_process(to_parse: list[tuple], progress: ProgressCallback | None) -> list[Any]:+ """Sequential in-process parse, the fallback when the pool is unavailable."""+ from repowise.core.ingestion import ASTParser- if to_parse:+ parse_results: list[Any] = []+ fallback_parser = ASTParser()+ for i, (_idx, (fi, source), _h) in enumerate(to_parse):try:- with ProcessPoolExecutor(max_workers=workers, mp_context=_MP_SPAWN) as pool:- tasks = [- loop.run_in_executor(pool, _parse_one, item) for _idx, item, _h in to_parse- ]- # Tick the parse-progress bar as each worker finishes —- # ``asyncio.gather`` would otherwise hold every event back- # until the last file is done, which on PowerToys-scale- # repos looked like a hang at ``0/N`` for many minutes.- # Per-task done-callbacks fire on the event loop thread and- # preserve gather's ordered results, so the aggregation- # loop below still indexes ``to_parse`` correctly.- if progress is not None:- _parse_tick = lambda _fut: progress.on_item_done("parse") # noqa: E731- for fut in tasks:- fut.add_done_callback(_parse_tick)- parse_results = await asyncio.gather(*tasks, return_exceptions=True)- except Exception as pool_exc:- logger.warning(- "process_pool_parse_failed_falling_back",- error=str(pool_exc),- )- # Worth saying out loud rather than only logging: the run still- # produces a correct index, but every file is now parsed in one- # process instead of up to eight, so a large repo takes several- # times longer for a reason nothing on screen explained.+ parse_results.append(fallback_parser.parse_file(fi, source))+ except Exception as exc:+ parse_results.append((fi.abs_path, str(exc)))+ if progress:+ progress.on_item_done("parse")+ if i % 50 == 49:+ await asyncio.sleep(0)+ return parse_results+++async def _parse_misses(+ to_parse: list[tuple],+ progress: ProgressCallback | None,+ *,+ fallback_event: str,+ warn_on_fallback: bool,+) -> list[Any]:+ workers = parse_pool_workers(len(to_parse))+ if not to_parse:+ return []+ try:+ return await _parse_in_pool(to_parse, workers, progress)+ except Exception as pool_exc:+ logger.warning(fallback_event, error=str(pool_exc))+ if warn_on_fallback:+ # Said out loud, not only logged: the index is still correct, but+ # one process instead of up to eight makes a large repo several+ # times slower for a reason nothing on screen explained.emit_warning(progress,f"Parallel parsing unavailable ({pool_exc}); ""falling back to a single process, which is much slower.",)- # Fallback: in-process sequential parse- parse_results = []- _fallback_parser = ASTParser()- for i, (_idx, (fi, source), _h) in enumerate(to_parse):- try:- result = _fallback_parser.parse_file(fi, source)- parse_results.append(result)- except Exception as exc:- parse_results.append((fi.abs_path, str(exc)))- if progress:- progress.on_item_done("parse")- if i % 50 == 49:- await asyncio.sleep(0)-- # Merge fresh parses back into traversal order alongside cache hits.+ return await _parse_in_process(to_parse, progress)+++def _merge_parse_results(+ cached_hits: dict[int, Any],+ to_parse: list[tuple],+ parse_results: list[Any],+ parse_cache: Any,+) -> dict[int, Any]:+ """Key fresh parses by traversal position next to the cache hits."""merged: dict[int, Any] = dict(cached_hits)for (idx, _item, content_hash), result in zip(to_parse, parse_results, strict=True):merged[idx] = resultif not isinstance(result, tuple | Exception):_cache_parsed(parse_cache, result, content_hash)+ return merged+- # Aggregate results into GraphBuilder on the main loop (not thread-safe).- for idx in range(len(fi_and_bytes)):- fi, source = fi_and_bytes[idx]+def _collect_parsed(+ fi_and_bytes: list[tuple],+ merged: dict[int, Any],+) -> tuple[list[Any], dict[str, bytes]]:+ """Successful parses and their source bytes, in traversal order."""+ parsed_files: list[Any] = []+ source_map: dict[str, bytes] = {}+ for idx, (fi, source) in enumerate(fi_and_bytes):result = merged.get(idx)if isinstance(result, tuple) and len(result) == 2 and isinstance(result[1], str):- # Error tuple: (abs_path_str, error_str)+ # Error tuple from a worker: (abs_path_str, error_str)logger.debug("parse_error_in_worker", path=result[0], error=result[1])elif isinstance(result, Exception):logger.debug("parse_exception_in_worker", path=fi.abs_path, error=str(result))elif result is not None:parsed_files.append(result)source_map[fi.path] = source- graph_builder.add_file(result)- # Process-pool path already ticked per-file via the done-callback- # attached above; only the fallback path ticks here (handled in- # its own loop). No tick needed in aggregation.+ return parsed_files, source_map++async def _parse_stage(+ repo_path: Path,+ file_infos: list[Any],+ progress: ProgressCallback | None,+ *,+ fallback_event: str,+ warn_on_fallback: bool,+) -> tuple[list[Any], dict[str, bytes]]:+ """Read, cache-split and parse *file_infos*; results keep their order.++ Source bytes are read up front in a thread pool so worker args stay small+ (FileInfo + bytes). Content-hash cache hits skip the worker pool; misses+ are parsed in the process pool. Does not mark the parse phase done.+ """+ if progress:+ progress.on_phase_start("parse", len(file_infos))+ fi_and_bytes: list[tuple] = _read_sources(file_infos, progress)+ parse_cache, cached_hits, to_parse = _split_cached(repo_path, fi_and_bytes, progress)+ parse_results = await _parse_misses(+ to_parse, progress, fallback_event=fallback_event, warn_on_fallback=warn_on_fallback+ )+ merged = _merge_parse_results(cached_hits, to_parse, parse_results, parse_cache)+ parsed_files, source_map = _collect_parsed(fi_and_bytes, merged)if parse_cache is not None:parse_cache.save()- # Lend the just-read bytes to the builder so the language warmups that- # scan file text during build() don't re-read the repo.+ return parsed_files, source_map+++def _detect_tech_items(+ repo_path: Path, progress: ProgressCallback | None, *, consequence: str+) -> list:+ """Detect the repo's tech stack; a failure warns and yields ``[]``."""+ try:+ from repowise.core.generation.editor_files.tech_stack import detect_tech_stack++ return detect_tech_stack(repo_path)+ except Exception as tech_exc:+ logger.warning("tech_stack_detection_failed", error=str(tech_exc))+ emit_warning(+ progress,+ f"Tech stack detection skipped ({tech_exc}); the repo's frameworks will be "+ f"missing from the index{consequence}",+ )+ return []+++def _new_graph_builder(+ repo_path: Path,+ traverser: Any,+ parsed_files: list[Any],+ source_map: dict[str, bytes],+ walk: _WalkOptions,+) -> Any:+ """A GraphBuilder fed *parsed_files* in order (insertion order breaks ties)."""+ from repowise.core.ingestion import GraphBuilder+ from repowise.core.workspace.update import get_head_commit++ graph_builder = GraphBuilder(+ repo_path=repo_path,+ exclude_patterns=walk.exclude_patterns,+ centrality_cache_dir=repo_path / ".repowise",+ head_commit=get_head_commit(repo_path),+ include_submodules=walk.include_submodules,+ include_nested_repos=walk.include_nested_repos,+ )+ graph_builder.console_scripts = traverser.console_script_names+ for parsed in parsed_files:+ graph_builder.add_file(parsed)+ # Lend the just-read bytes so the language warmups that scan file text+ # during build() don't re-read the repo.graph_builder.set_source_map(source_map)graph_builder.set_pytest_roots(traverser.pytest_roots)- _phase_done(progress, "parse")+ return graph_builder- # ---- tsconfig path-alias resolver (before graph build) ------------------- # Only runs when the repo has TS/JS files. On large TS monorepos the- # resolver indexes hundreds of tsconfig files up-front; without a phase- # label this shows up as a silent gap right after parsing.- _ts_langs = {"typescript", "javascript", "svelte", "vue", "astro"}- if any(pf.file_info.language in _ts_langs for pf in parsed_files):- if progress:- progress.on_phase_start("tsconfig", None)- from repowise.core.ingestion import wire_tsconfig_resolver- wire_tsconfig_resolver(- graph_builder,- repo_path,- include_submodules=include_submodules,- include_nested_repos=include_nested_repos,- )- _phase_done(progress, "tsconfig")-- # ---- Graph build phase -------------------------------------------------- # Sub-phases (graph.imports / graph.heritage / graph.calls) are emitted- # from inside GraphBuilder.build(); the orchestrator drives metrics/- # communities/flows below so the longest-running step is no longer an- # opaque "graph 0/1" spinner.- if progress and len(file_infos) >= _SLOW_GRAPH_BUILD_FILES:- # Reassurance before the longest silent stretch of the run, so it is- # rendered warning-weight rather than another dim stat (a "notice", not- # a "warning": it is not a degradation and must not be persisted into- # state.json["degraded"]) — which is also why it is- # gated: on a small repo the build is over in under a second, and a- # yellow "this may take several minutes" there is a false alarm made- # louder.+def _wire_tsconfig(+ graph_builder: Any,+ repo_path: Path,+ parsed_files: list[Any],+ walk: _WalkOptions,+ progress: ProgressCallback | None,+) -> None:+ """Wire the tsconfig path-alias resolver when the repo has TS/JS files.++ Its own phase label: on large TS monorepos it indexes hundreds of tsconfig+ files, which otherwise shows as a silent gap right after parsing.+ """+ if not any(pf.file_info.language in _TS_LANGUAGES for pf in parsed_files):+ return+ if progress:+ progress.on_phase_start("tsconfig", None)+ from repowise.core.ingestion import wire_tsconfig_resolver++ wire_tsconfig_resolver(+ graph_builder,+ repo_path,+ include_submodules=walk.include_submodules,+ include_nested_repos=walk.include_nested_repos,+ )+ _phase_done(progress, "tsconfig")+++async def _build_graph(+ graph_builder: Any, file_count: int, progress: ProgressCallback | None+) -> None:+ """Run ``GraphBuilder.build`` (it emits graph.imports/heritage/calls itself)."""+ if progress and file_count >= _SLOW_GRAPH_BUILD_FILES:+ # Reassurance before the longest silent stretch, as a "notice": not a+ # degradation, so never persisted into state.json["degraded"]. Gated+ # because on a small repo the build is over in under a second.progress.on_message("notice","Graph build can take several minutes on a first run. Safe to Ctrl-C — "@@ -648,25 +709,15 @@ async def _run_ingestion()await asyncio.to_thread(graph_builder.build, progress)- # Add framework-aware synthetic edges (conftest, Django, FastAPI, Flask).- # Two failures with two different costs, reported separately rather than as- # one silent pass: the stack also feeds the persisted tech list and the- # editor files, so losing it costs more than losing the edges. Keeping the- # edge call outside its try is deliberate — the handlers that read no stack- # (conftest, gtest, Flutter, Android) still run when detection failed.- tech_items: list = []- try:- from repowise.core.generation.editor_files.tech_stack import detect_tech_stack- tech_items = detect_tech_stack(repo_path)- except Exception as tech_exc:- logger.warning("tech_stack_detection_failed", error=str(tech_exc))- emit_warning(- progress,- f"Tech stack detection skipped ({tech_exc}); the repo's frameworks will be "- "missing from the index, along with every edge that depends on them.",- )+def _add_framework_edges(+ graph_builder: Any, tech_items: list, progress: ProgressCallback | None+) -> None:+ """Framework-aware synthetic edges (conftest, Django, FastAPI, Flask).+ Runs even when stack detection failed: the handlers that read no stack+ (conftest, gtest, Flutter, Android) still apply.+ """try:graph_builder.add_framework_edges([item.name for item in tech_items])except Exception as fw_exc:@@ -677,17 +728,25 @@ async def _run_ingestion("symbols will have no callers in the graph and may read as dead.",)- # ---- Dynamic hints wiring (after static graph is fully built) ----------++async def _add_dynamic_hints(+ graph_builder: Any,+ repo_path: Path,+ file_infos: list[Any],+ source_map: dict[str, bytes],+ progress: ProgressCallback | None,+) -> None:+ """Dynamic hint edges, after the static graph is fully built."""if progress:progress.on_phase_start("dynamic_hints", None)try:from repowise.core.ingestion.dynamic_hints import HintRegistryregistry = HintRegistry()- # Feed the traversed file list so hints query the indexed set- # directly instead of re-walking the tree (keeps gitignored and- # generated files out of hint edges, and skips a full walk).+ # The traversed file list keeps gitignored and generated files out of+ # hint edges and skips a second walk.hint_paths = [fi.path for fi in file_infos]+ loop = asyncio.get_running_loop()dynamic_edges = await loop.run_in_executor(None,lambda: registry.extract_all(@@ -708,19 +767,15 @@ async def _run_ingestion()_phase_done(progress, "dynamic_hints")- # ---- Graph metrics: prime caches with live progress ---------------------- # pagerank/betweenness/community/symbol_communities/execution_flows are- # otherwise computed lazily during persist + generation, where they hide- # behind opaque progress bars. Pre-compute them here so each is its own- # visible sub-phase, and fan the within-phase work out via- # asyncio.gather so betweenness (the dominant cost) overlaps with- # PageRank / community detection rather than running serially.- #- # Each algorithm is wrapped in ``_timed_step`` so we emit a per-algo- # completion line (` ↳ PageRank ✓ (Xs)`) as it finishes. Without these,- # the whole gather looks like one opaque several-minute void where- # betweenness centrality dominates — splitting the timing makes it- # obvious which step is the bottleneck on a given repo.++async def _prime_graph_metrics(graph_builder: Any, progress: ProgressCallback | None) -> None:+ """Compute centrality and communities now, each as a visible sub-phase.++ Otherwise they run lazily inside persist and generation behind opaque+ bars. Within a phase the kernels overlap via ``asyncio.gather`` so+ betweenness (the dominant cost) runs alongside the rest, and each is+ wrapped in ``_timed_step`` so its completion line names the bottleneck.+ """if progress:progress.on_phase_start("graph.metrics", None)await asyncio.gather(@@ -739,6 +794,63 @@ async def _run_ingestion()_phase_done(progress, "graph.communities")++async def _run_ingestion(+ repo_path: Path,+ *,+ exclude_patterns: list[str] | None,+ include_submodules: bool = False,+ include_nested_repos: bool = False,+ skip_tests: bool,+ skip_infra: bool,+ progress: ProgressCallback | None,+ derive_environment_facts: bool = False,+) -> tuple[list[Any], list[Any], Any, dict[str, bytes], Any, Any]:+ """Traverse, parse, and build the dependency graph.++ Returns (parsed_files, file_infos, repo_structure, source_map,+ graph_builder, traversal_stats, tech_items).+ """+ walk = _WalkOptions(+ exclude_patterns=exclude_patterns,+ include_submodules=include_submodules,+ include_nested_repos=include_nested_repos,+ skip_tests=skip_tests,+ skip_infra=skip_infra,+ derive_environment_facts=derive_environment_facts,+ )+ traverser, file_infos, repo_structure = await _traverse_stage(repo_path, walk, progress)++ # Reported now rather than at the end of the phase: the graph build and+ # metrics below can take minutes on a large repo.+ _emit_traversal_summary(progress, traverser.stats, len(file_infos))++ # Said once here rather than by every worker at a level no default run+ # renders: files in these languages index without symbols.+ _report_missing_grammars(getattr(traverser, "stats", None), progress)+ _report_query_compilation_failures(getattr(traverser, "stats", None), progress)++ parsed_files, source_map = await _parse_stage(+ repo_path,+ file_infos,+ progress,+ fallback_event="process_pool_parse_failed_falling_back",+ warn_on_fallback=True,+ )+ graph_builder = _new_graph_builder(repo_path, traverser, parsed_files, source_map, walk)+ _phase_done(progress, "parse")++ _wire_tsconfig(graph_builder, repo_path, parsed_files, walk, progress)+ await _build_graph(graph_builder, len(file_infos), progress)+ # Stack detection and framework edges fail separately: the stack also+ # feeds the persisted tech list and the editor files.+ tech_items = _detect_tech_items(+ repo_path, progress, consequence=", along with every edge that depends on them."+ )+ _add_framework_edges(graph_builder, tech_items, progress)+ await _add_dynamic_hints(graph_builder, repo_path, file_infos, source_map, progress)+ await _prime_graph_metrics(graph_builder, progress)+return (parsed_files,file_infos,@@ -775,125 +887,22 @@ async def reparse_for_resume(tech_items)`` — mirrors the prefix of :func:`_run_ingestion` minus thegraph builder and traversal stats."""- from repowise.core.ingestion import ASTParser, FileTraverser-- traverser = FileTraverser(- repo_path,- extra_exclude_patterns=exclude_patterns or None,+ walk = _WalkOptions(+ exclude_patterns=exclude_patterns,include_submodules=include_submodules,include_nested_repos=include_nested_repos,+ skip_tests=skip_tests,+ skip_infra=skip_infra,+ derive_environment_facts=derive_environment_facts,)-- all_paths = list(traverser._walk())- if progress:- progress.on_phase_start("traverse", None)-- file_infos: list[Any] = []- io_pool = ThreadPoolExecutor(max_workers=8)- try:- aws = [- asyncio.wrap_future(io_pool.submit(traverser._build_file_info, p)) for p in all_paths- ]- for coro in asyncio.as_completed(aws):- try:- result = await coro- except Exception:- result = None- if result is not None:- file_infos.append(result)- if progress:- progress.on_item_done("traverse")- finally:- await asyncio.to_thread(io_pool.shutdown, wait=True)-- repo_structure = traverser.get_repo_structure(file_infos)- # A resumed run is still the same command: the interrupted one may never- # have reached the derivation, so it runs here too rather than leaving a- # resumed repo without the facts a fresh one gets.- await _record_structural_episodes(- repo_path, traverser, allow_formatter_check=derive_environment_facts+ _traverser, file_infos, repo_structure = await _traverse_stage(repo_path, walk, progress)+ parsed_files, source_map = await _parse_stage(+ repo_path,+ file_infos,+ progress,+ fallback_event="resume_reparse_pool_failed_falling_back",+ warn_on_fallback=False,)- _phase_done(progress, "traverse")-- if skip_tests:- file_infos = [fi for fi in file_infos if not fi.is_test]- if skip_infra:- file_infos = [- fi- for fi in file_infos- if fi.language not in ("dockerfile", "makefile", "terraform", "shell")- ]-- if progress:- progress.on_phase_start("parse", len(file_infos))-- fi_and_bytes: list[tuple] = _read_sources(file_infos, progress)-- parsed_files: list[Any] = []- source_map: dict[str, bytes] = {}- loop = asyncio.get_running_loop()-- parse_cache, cached_hits, to_parse = _split_cached(repo_path, fi_and_bytes, progress)- workers = parse_pool_workers(len(to_parse))- parse_results: list[Any] = []-- if to_parse:- try:- with ProcessPoolExecutor(max_workers=workers, mp_context=_MP_SPAWN) as pool:- tasks = [- loop.run_in_executor(pool, _parse_one, item) for _idx, item, _h in to_parse- ]- if progress is not None:- _tick = lambda _fut: progress.on_item_done("parse") # noqa: E731- for fut in tasks:- fut.add_done_callback(_tick)- parse_results = await asyncio.gather(*tasks, return_exceptions=True)- except Exception as pool_exc:- logger.warning("resume_reparse_pool_failed_falling_back", error=str(pool_exc))- parse_results = []- _fallback_parser = ASTParser()- for i, (_idx, (fi, source), _h) in enumerate(to_parse):- try:- parse_results.append(_fallback_parser.parse_file(fi, source))- except Exception as exc:- parse_results.append((fi.abs_path, str(exc)))- if progress:- progress.on_item_done("parse")- if i % 50 == 49:- await asyncio.sleep(0)-- merged: dict[int, Any] = dict(cached_hits)- for (idx, _item, content_hash), result in zip(to_parse, parse_results, strict=True):- merged[idx] = result- if not isinstance(result, tuple | Exception):- _cache_parsed(parse_cache, result, content_hash)-- for idx in range(len(fi_and_bytes)):- fi, source = fi_and_bytes[idx]- result = merged.get(idx)- if isinstance(result, tuple) and len(result) == 2 and isinstance(result[1], str):- logger.debug("parse_error_in_worker", path=result[0], error=result[1])- elif isinstance(result, Exception):- logger.debug("parse_exception_in_worker", path=fi.abs_path, error=str(result))- elif result is not None:- parsed_files.append(result)- source_map[fi.path] = source-- if parse_cache is not None:- parse_cache.save()_phase_done(progress, "parse")-- tech_items: list = []- try:- from repowise.core.generation.editor_files.tech_stack import detect_tech_stack-- tech_items = detect_tech_stack(repo_path)- except Exception as tech_exc:- logger.warning("tech_stack_detection_failed", error=str(tech_exc))- emit_warning(- progress,- f"Tech stack detection skipped ({tech_exc}); the repo's frameworks will be "- "missing from the index.",- )-+ tech_items = _detect_tech_items(repo_path, progress, consequence=".")return parsed_files, file_infos, repo_structure, source_map, tech_itemsdiff --git a/tests/unit/pipeline/test_ingestion_file_order.py b/tests/unit/pipeline/test_ingestion_file_order.pynew file mode 100644index 000000000..1974fae07--- /dev/null+++ b/tests/unit/pipeline/test_ingestion_file_order.py@@ -0,0 +1,109 @@+"""Full and resume ingestion share their stages and return files in walk order.++Insertion order breaks ties in reference resolution, so the order files reach+GraphBuilder is part of the graph's identity, and resumed and fresh runs must+hand analysis the same order. FileInfo builds are slowed so they finish in+reverse walk order: a completion-ordered collection fails here.+"""++from __future__ import annotations++import time++import pytest++from repowise.core.ingestion import FileTraverser+from repowise.core.pipeline.phases import ingestion as ing++_FILES = 6++KW = {"exclude_patterns": None, "skip_tests": False, "skip_infra": False, "progress": None}+++def repo(tmp_path):+ for i in range(_FILES):+ (tmp_path / f"mod_{i}.py").write_text(f"def f_{i}(x):\n return x + {i}\n")+ sub = tmp_path / "pkg"+ sub.mkdir()+ (sub / "inner.py").write_text("from mod_0 import f_0\n\n\ndef g():\n return f_0(1)\n")+ return tmp_path+++def walk_order(monkeypatch, repo):+ """Walk order as relative paths; earlier paths finish their build last."""+ walk = list(FileTraverser(repo)._walk())+ rank = {p: i for i, p in enumerate(walk)}+ real = FileTraverser._build_file_info++ def slow_first(self, path):+ time.sleep((len(walk) - rank.get(path, 0)) * 0.05)+ return real(self, path)++ monkeypatch.setattr(FileTraverser, "_build_file_info", slow_first)+ return [p.relative_to(repo).as_posix() for p in walk]+++def single_worker_pool(monkeypatch):+ real = ing.ProcessPoolExecutor++ def one(*args, **kwargs):+ kwargs["max_workers"] = 1+ return real(*args, **kwargs)++ monkeypatch.setattr(ing, "ProcessPoolExecutor", one)+++async def test_full_ingestion_keeps_walk_order(repo, walk_order, single_worker_pool):+ parsed, file_infos, _s, source_map, gb, _stats, _tech = await ing._run_ingestion(repo, **KW)++ expected = [p for p in walk_order if p in source_map]+ assert len(expected) == _FILES + 1+ assert [fi.path for fi in file_infos if fi.path in source_map] == expected+ assert [p.file_info.path for p in parsed] == expected+ assert list(source_map) == expected+ assert list(gb._parsed_files) == expected+++async def test_resume_reparse_keeps_walk_order(repo, walk_order, single_worker_pool):+ parsed, file_infos, _s, source_map, _tech = await ing.reparse_for_resume(repo, **KW)++ expected = [p for p in walk_order if p in source_map]+ assert len(expected) == _FILES + 1+ assert [fi.path for fi in file_infos if fi.path in source_map] == expected+ assert [p.file_info.path for p in parsed] == expected+ assert list(source_map) == expected+++async def test_both_entry_points_use_the_shared_stages(repo, monkeypatch, single_worker_pool):+ calls: list[tuple[str, str]] = []+ real_traverse, real_parse = ing._traverse_stage, ing._parse_stage+ current = ""++ async def traverse(*args, **kwargs):+ calls.append(("traverse", current))+ return await real_traverse(*args, **kwargs)++ async def parse(*args, **kwargs):+ calls.append(("parse", current))+ return await real_parse(*args, **kwargs)++ monkeypatch.setattr(ing, "_traverse_stage", traverse)+ monkeypatch.setattr(ing, "_parse_stage", parse)++ current = "full"+ full = await ing._run_ingestion(repo, **KW)+ current = "resume"+ resume = await ing.reparse_for_resume(repo, **KW)++ assert calls == [+ ("traverse", "full"),+ ("parse", "full"),+ ("traverse", "resume"),+ ("parse", "resume"),+ ]+ # Same files, same order, same bytes from either entry point.+ assert [p.file_info.path for p in full[0]] == [p.file_info.path for p in resume[0]]+ assert full[3] == resume[3]diff --git a/tests/unit/pipeline/test_parse_pool_workers.py b/tests/unit/pipeline/test_parse_pool_workers.pyindex 99d89825d..0c2851bd4 100644--- a/tests/unit/pipeline/test_parse_pool_workers.py+++ b/tests/unit/pipeline/test_parse_pool_workers.py@@ -122,18 +122,24 @@ class TestParsePoolWorkers:class TestBothCallSitesAreBounded:"""The bound has to reach the executor on the init *and* resume paths.- Grepping the source is what this guards: the two paths carried a duplicated- ``min(os.cpu_count(), ...)`` expression and only one would have been noticed- if a future edit reverted it.+ Grepping the source is what this guards: both paths parse through+ ``_parse_stage``, and its pool is sized by the shared helper, never by+ ``os.cpu_count()``."""+ def test_pool_uses_the_shared_helper(self):+ import inspect++ src = inspect.getsource(ing._parse_misses)+ assert "parse_pool_workers(" in src, "the parse pool must be sized via the shared helper"+ assert "os.cpu_count()" not in src, "the pool must not be sized from the host core count"+@pytest.mark.parametrize("func", ["_run_ingestion", "reparse_for_resume"])- def test_call_site_uses_the_shared_helper(self, func):+ def test_call_site_parses_through_the_shared_stage(self, func):import inspect- target = getattr(ing, func)- src = inspect.getsource(target)- assert "parse_pool_workers(" in src, f"{func} must size its pool via the shared helper"+ src = inspect.getsource(getattr(ing, func))+ assert "_parse_stage(" in src, f"{func} must parse via the shared stage"assert "os.cpu_count()" not in src, f"{func} must not size its pool from the host core count"async def test_the_bound_reaches_the_real_executor(self, tmp_path, monkeypatch):