Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(workspace): stop contract extraction scanning nested repos (1M-file hang) (#516)
Raghav Chamadiyac0bc1184a876
diff --git a/packages/core/src/repowise/core/workspace/extractors/base.py b/packages/core/src/repowise/core/workspace/extractors/base.pyindex aad8049a6..bcbc04dad 100644--- a/packages/core/src/repowise/core/workspace/extractors/base.py+++ b/packages/core/src/repowise/core/workspace/extractors/base.py@@ -1,9 +1,12 @@"""Shared file-scanning primitives for contract extractors.-Both the HTTP and gRPC extractors walk a repository the same way: skip a fixed-set of build/vendor directories, ignore files above a size cap, and read each+Both the HTTP and gRPC extractors walk a repository the same way and read eachcandidate file as UTF-8. That traversal lives here once so the per-contract-type-orchestrators only declare *which* extensions they care about.+orchestrators only declare *which* extensions they care about. Discovery is+delegated to the ingestion :class:`~repowise.core.ingestion.traverser.FileTraverser`+so the scanned file set respects ``.gitignore`` and nested-repo boundaries+(identical to the repo's actual index) instead of a raw ``os.walk`` that would+descend into sibling/vendored repos rooted under the workspace."""from __future__ import annotations@@ -13,31 +16,6 @@ from collections.abc import Iteratorfrom dataclasses import dataclassfrom pathlib import Path-# Directories that never contain first-party source worth scanning.-BLOCKED_DIRS = frozenset(- {- ".git",- "node_modules",- "__pycache__",- ".venv",- "venv",- "dist",- "build",- "target",- "vendor",- ".next",- ".nuxt",- ".tox",- ".mypy_cache",- ".gradle",- ".mvn",- "out",- "bin",- }-)--MAX_FILE_SIZE = 512 * 1024 # 512 KB-@dataclass(frozen=True)class ScanContext:@@ -59,26 +37,35 @@ def iter_source_files() -> Iterator[tuple[str, str, str]]:"""Yield ``(rel_path, suffix, content)`` for each scannable source file.- Walks *repo_root*, pruning :data:`BLOCKED_DIRS` and dotfile directories,- skipping files whose extension is not in *extensions* or that exceed- :data:`MAX_FILE_SIZE`, and reading the rest as UTF-8 (replacing undecodable- bytes). Unreadable files are silently skipped.+ File discovery is delegated to the ingestion :class:`FileTraverser`, the+ same gitignore- and nested-repo-aware walker the main index uses, rather+ than a raw ``os.walk``. This matters whenever a workspace repo's root+ physically contains *other* git repositories (sibling/vendored repos,+ benchmark clones, build worktrees) or large gitignored trees: a naive walk+ descends into all of them and scans hundreds of thousands of foreign files.+ That is not hypothetical: on a developer checkout whose root held nested+ repos, the old walk yielded **1.05M files** for a repo whose real source is+ ~2k, wedging contract extraction for tens of minutes. FileTraverser prunes+ nested git repos and honours ``.gitignore``, so the scanned set matches the+ repo's actual index.++ Only files whose lower-cased suffix is in *extensions* are yielded.+ Oversized, binary, generated, and ignored files are already excluded by the+ traverser. Unreadable files are silently skipped."""- root = repo_root.resolve()- for dirpath, dirnames, filenames in os.walk(root):- dirnames[:] = [d for d in dirnames if d not in BLOCKED_DIRS and not d.startswith(".")]- for fname in filenames:- fpath = Path(dirpath) / fname- suffix = fpath.suffix.lower()- if suffix not in extensions:- continue- try:- if fpath.stat().st_size > MAX_FILE_SIZE:- continue- except OSError:- continue- try:- content = fpath.read_text(encoding="utf-8", errors="replace")- except OSError:- continue- yield fpath.relative_to(root).as_posix(), suffix, content+ from repowise.core.ingestion.traverser import FileTraverser++ root = Path(repo_root).resolve()+ if not root.is_dir():+ return++ traverser = FileTraverser(root)+ for info in traverser.traverse():+ suffix = os.path.splitext(info.path)[1].lower()+ if suffix not in extensions:+ continue+ try:+ content = Path(info.abs_path).read_text(encoding="utf-8", errors="replace")+ except OSError:+ continue+ yield info.path, suffix, contentdiff --git a/packages/core/src/repowise/core/workspace/extractors/service_boundary.py b/packages/core/src/repowise/core/workspace/extractors/service_boundary.pyindex b260f43df..5f01b7a1b 100644--- a/packages/core/src/repowise/core/workspace/extractors/service_boundary.py+++ b/packages/core/src/repowise/core/workspace/extractors/service_boundary.py@@ -4,6 +4,12 @@ Walks a repo directory tree looking for marker files (package.json, go.mod,Dockerfile, etc.) that indicate independent service boundaries. Used todistinguish intra-service calls from inter-service calls when matchingcontracts.++The directory walk goes through :func:`~repowise.core.fs_walk.walk_repo`, which+prunes junk directories and, critically, *nested git repositories*. A workspace+repo whose root physically contains sibling/vendored repos (or benchmark clones)+would otherwise be walked end-to-end, turning a sub-second scan into a+multi-minute stall over hundreds of thousands of foreign files."""from __future__ import annotations@@ -12,6 +18,7 @@ import osfrom dataclasses import dataclass, fieldfrom pathlib import Path+from repowise.core.fs_walk import walk_repofrom repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY# ---------------------------------------------------------------------------@@ -116,8 +123,10 @@ def detect_service_boundaries(repo_path: Path) -> list[ServiceBoundary]:repo_root = repo_path.resolve()boundaries: list[ServiceBoundary] = []- for dirpath, dirnames, filenames in os.walk(repo_root):- # Prune blocked dirs in-place+ for dirpath, dirnames, filenames in walk_repo(repo_root):+ # ``walk_repo`` already prunes junk dirs and nested git repos; layer on+ # the derived-output names (dist/build/target/...) it intentionally+ # leaves ambiguous, plus dotfile dirs, that are never service roots.dirnames[:] = [d for d in dirnames if d not in _BLOCKED_DIRS and not d.startswith(".")]current = Path(dirpath)diff --git a/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py b/packages/core/src/repowise/core/workspace/extractors/topic_extractor.pyindex fd2f61724..4d6c447cc 100644--- a/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py+++ b/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py@@ -7,7 +7,6 @@ consumer patterns.from __future__ import annotationsimport logging-import osimport refrom dataclasses import dataclassfrom pathlib import Path@@ -15,6 +14,8 @@ from typing import TYPE_CHECKINGfrom repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY+from .base import iter_source_files+if TYPE_CHECKING:from repowise.core.workspace.contracts import Contract@@ -24,30 +25,6 @@ _log = logging.getLogger("repowise.workspace.extractors.topic")# Constants# ----------------------------------------------------------------------------_BLOCKED_DIRS = frozenset(- {- ".git",- "node_modules",- "__pycache__",- ".venv",- "venv",- "dist",- "build",- "target",- "vendor",- ".next",- ".nuxt",- ".tox",- ".mypy_cache",- ".gradle",- ".mvn",- "out",- "bin",- }-)--_MAX_FILE_SIZE = 512 * 1024-_EXTENSIONS = _LANG_REGISTRY.extensions_for(["python", "typescript", "javascript", "java", "go"])@@ -247,58 +224,41 @@ class TopicExtractor:from repowise.core.workspace.contracts import Contractcontracts: list[Contract] = []- repo_root = repo_path.resolve()seen: set[tuple[str, str, str]] = set() # (file, contract_id, role) dedup- for dirpath, dirnames, filenames in os.walk(repo_root):- dirnames[:] = [d for d in dirnames if d not in _BLOCKED_DIRS and not d.startswith(".")]- for fname in filenames:- fpath = Path(dirpath) / fname- suffix = fpath.suffix.lower()- if suffix not in _EXTENSIONS:- continue- try:- if fpath.stat().st_size > _MAX_FILE_SIZE:+ # File discovery is gitignore- and nested-repo-aware (see+ # ``iter_source_files``) so a workspace repo whose root contains nested+ # repos or large ignored trees is not scanned end-to-end.+ for rel_path, _suffix, content in iter_source_files(repo_path, _EXTENSIONS):+ for pdef in _ALL_PATTERNS:+ for match in pdef.regex.finditer(content):+ topic_name = match.group(pdef.topic_group).strip()+ if not topic_name:+ continue++ contract_id = f"topic::{topic_name.lower()}"++ # Deduplicate within the same file+ dedup_key = (rel_path, contract_id, pdef.role)+ if dedup_key in seen:continue- except OSError:- continue-- try:- content = fpath.read_text(encoding="utf-8", errors="replace")- except OSError:- continue-- rel_path = fpath.relative_to(repo_root).as_posix()-- for pdef in _ALL_PATTERNS:- for match in pdef.regex.finditer(content):- topic_name = match.group(pdef.topic_group).strip()- if not topic_name:- continue-- contract_id = f"topic::{topic_name.lower()}"-- # Deduplicate within the same file- dedup_key = (rel_path, contract_id, pdef.role)- if dedup_key in seen:- continue- seen.add(dedup_key)-- contracts.append(- Contract(- repo=repo_alias,- contract_id=contract_id,- contract_type="topic",- role=pdef.role,- file_path=rel_path,- symbol_name=f"{pdef.label}('{topic_name}')",- confidence=pdef.confidence,- service=None,- meta={- "topic": topic_name,- "broker": pdef.broker,- },- )+ seen.add(dedup_key)++ contracts.append(+ Contract(+ repo=repo_alias,+ contract_id=contract_id,+ contract_type="topic",+ role=pdef.role,+ file_path=rel_path,+ symbol_name=f"{pdef.label}('{topic_name}')",+ confidence=pdef.confidence,+ service=None,+ meta={+ "topic": topic_name,+ "broker": pdef.broker,+ },)+ )return contractsdiff --git a/packages/server/src/repowise/server/routers/jobs.py b/packages/server/src/repowise/server/routers/jobs.pyindex 81f90e9b0..98f99c675 100644--- a/packages/server/src/repowise/server/routers/jobs.py+++ b/packages/server/src/repowise/server/routers/jobs.py@@ -4,6 +4,7 @@ from __future__ import annotationsimport asyncioimport json+from datetime import UTC, datetimefrom fastapi import APIRouter, Depends, HTTPException, Query, Requestfrom sqlalchemy import func, select@@ -50,6 +51,14 @@ async def _find_job_factory(app_state, job_id: str):return None, None+def _created_sort_key(job: GenerationJob) -> datetime:+ """Timezone-safe sort key for ``created_at`` across merged repo databases."""+ dt = job.created_at+ if dt is None:+ return datetime.min.replace(tzinfo=UTC)+ return dt if dt.tzinfo is not None else dt.replace(tzinfo=UTC)++@router.get("", response_model=list[JobResponse])async def list_jobs(request: Request,@@ -59,6 +68,7 @@ async def list_jobs(offset: int = Query(0, ge=0),) -> list[JobResponse]:"""List generation jobs, optionally filtered by repository or status."""+async def _query_jobs(factory) -> list[GenerationJob]:q = select(GenerationJob)if repo_id:@@ -89,7 +99,11 @@ async def list_jobs(except Exception:continue- jobs.sort(key=lambda j: j.created_at, reverse=True)+ # Jobs are merged from several repo databases in workspace mode, and SQLite+ # hands back a mix of timezone-aware and naive ``created_at`` values, which+ # cannot be compared directly. Coerce naive timestamps to UTC for the sort+ # (they are stored as UTC) so the merge never raises.+ jobs.sort(key=_created_sort_key, reverse=True)return [JobResponse.from_orm(j) for j in jobs[offset : offset + limit]]diff --git a/tests/unit/server/test_jobs.py b/tests/unit/server/test_jobs.pyindex 9acbd6523..94b996886 100644--- a/tests/unit/server/test_jobs.py+++ b/tests/unit/server/test_jobs.py@@ -14,6 +14,7 @@ from repowise.core.persistence import crudfrom repowise.core.persistence.database import get_session, init_dbfrom repowise.core.persistence.models import GenerationJobfrom repowise.server.app import reset_workspace_stale_jobs+from repowise.server.routers.jobs import _created_sort_keyfrom tests.unit.server.conftest import create_test_repo@@ -203,3 +204,28 @@ async def test_job_stream_completed(client: AsyncClient, app) -> None:assert "text/event-stream" in resp.headers["content-type"]# For a completed job, the stream should contain 'done' eventassert "event: done" in resp.text or "event: progress" in resp.text+++def test_created_sort_key_handles_mixed_tz_awareness() -> None:+ """Jobs merged from several repo DBs mix naive and aware ``created_at``.++ In workspace mode the per-repo SQLite databases hand back a mix of+ timezone-aware and naive timestamps; sorting them directly raised+ ``TypeError: can't compare offset-naive and offset-aware datetimes`` and+ 500'd ``GET /api/jobs``. The sort key must coerce them onto common ground.+ """+ from datetime import UTC, datetime++ aware_new = SimpleNamespace(created_at=datetime(2026, 1, 2, tzinfo=UTC))+ naive_newest = SimpleNamespace(created_at=datetime(2026, 1, 3)) # naive, newest+ aware_old = SimpleNamespace(created_at=datetime(2026, 1, 1, tzinfo=UTC))+ missing = SimpleNamespace(created_at=None)++ jobs = [aware_old, missing, naive_newest, aware_new]+ # Must not raise despite the naive/aware mix.+ jobs.sort(key=_created_sort_key, reverse=True)++ assert jobs[0] is naive_newest # 2026-01-03 is newest+ assert jobs[1] is aware_new # then 2026-01-02+ assert jobs[2] is aware_old # then 2026-01-01+ assert jobs[-1] is missing # None sorts oldestdiff --git a/tests/unit/workspace/test_extractor_traversal.py b/tests/unit/workspace/test_extractor_traversal.pynew file mode 100644index 000000000..cdbfacbe8--- /dev/null+++ b/tests/unit/workspace/test_extractor_traversal.py@@ -0,0 +1,96 @@+"""Regression tests for workspace extractor file discovery.++The contract/topic/service-boundary extractors must scan only the repo's own+source — respecting ``.gitignore`` and nested-repo boundaries — exactly like the+main index. A naive ``os.walk`` from the repo root descends into nested git+repos (sibling/vendored repos, benchmark clones) physically located under a+workspace root, which once turned a sub-second scan into a multi-minute stall+over ~1M foreign files. These tests pin that behaviour.+"""++from __future__ import annotations++from pathlib import Path++from repowise.core.workspace.extractors.base import iter_source_files+from repowise.core.workspace.extractors.service_boundary import detect_service_boundaries+from repowise.core.workspace.extractors.topic_extractor import TopicExtractor++_PY = frozenset({".py"})+++def _rel_paths(repo: Path) -> set[str]:+ return {rel for rel, _suffix, _content in iter_source_files(repo, _PY)}+++def test_iter_source_files_skips_nested_git_repo(tmp_path: Path) -> None:+ (tmp_path / "app.py").write_text("print('main')", encoding="utf-8")+ nested = tmp_path / "vendored"+ nested.mkdir()+ (nested / ".git").mkdir() # makes ``vendored`` look like its own repo+ (nested / "foreign.py").write_text("x = 1", encoding="utf-8")++ found = _rel_paths(tmp_path)++ assert "app.py" in found+ assert "vendored/foreign.py" not in found+++def test_iter_source_files_skips_nested_git_worktree_file(tmp_path: Path) -> None:+ # Worktrees and submodules use a ``.git`` *file* (not dir) — also a boundary.+ (tmp_path / "app.py").write_text("print('main')", encoding="utf-8")+ nested = tmp_path / "linked"+ nested.mkdir()+ (nested / ".git").write_text("gitdir: /somewhere/else", encoding="utf-8")+ (nested / "foreign.py").write_text("x = 1", encoding="utf-8")++ found = _rel_paths(tmp_path)++ assert "app.py" in found+ assert "linked/foreign.py" not in found+++def test_iter_source_files_respects_gitignore(tmp_path: Path) -> None:+ (tmp_path / ".gitignore").write_text("ignored/\n", encoding="utf-8")+ (tmp_path / "app.py").write_text("print('main')", encoding="utf-8")+ ignored = tmp_path / "ignored"+ ignored.mkdir()+ (ignored / "skip.py").write_text("y = 2", encoding="utf-8")++ found = _rel_paths(tmp_path)++ assert "app.py" in found+ assert "ignored/skip.py" not in found+++def test_topic_extractor_skips_nested_git_repo(tmp_path: Path) -> None:+ # A NATS producer in the repo is found; an identical one inside a nested+ # repo is not (the nested repo is a hard boundary).+ (tmp_path / "producer.py").write_text(+ 'nc.publish("events.user")', encoding="utf-8"+ )+ nested = tmp_path / "bench"+ nested.mkdir()+ (nested / ".git").mkdir()+ (nested / "other.py").write_text('nc.publish("events.foreign")', encoding="utf-8")++ contracts = TopicExtractor().extract(tmp_path, "primary")+ topics = {c.meta["topic"] for c in contracts}++ assert "events.user" in topics+ assert "events.foreign" not in topics+++def test_service_boundary_skips_nested_git_repo(tmp_path: Path) -> None:+ # A nested repo with service markers must not be reported as a sub-service+ # of the parent — it is an independent repo.+ nested = tmp_path / "sibling-service"+ nested.mkdir()+ (nested / ".git").mkdir()+ (nested / "pyproject.toml").write_text("[project]\nname='x'\n", encoding="utf-8")+ (nested / "main.py").write_text("print('svc')", encoding="utf-8")++ boundaries = detect_service_boundaries(tmp_path)+ names = {b.service_name for b in boundaries}++ assert "sibling-service" not in names