Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(workspace): stop contract extraction scanning nested repos (1M-file hang) (#516)

Raghav Chamadiyac0bc1184a876
diff --git a/packages/core/src/repowise/core/workspace/extractors/base.py b/packages/core/src/repowise/core/workspace/extractors/base.py
index aad8049a6..bcbc04dad 100644
--- a/packages/core/src/repowise/core/workspace/extractors/base.py
+++ b/packages/core/src/repowise/core/workspace/extractors/base.py
@@ -1,9 +1,12 @@
"""Shared file-scanning primitives for contract extractors.
-Both the HTTP and gRPC extractors walk a repository the same way: skip a fixed
-set of build/vendor directories, ignore files above a size cap, and read each
+Both the HTTP and gRPC extractors walk a repository the same way and read each
candidate file as UTF-8. That traversal lives here once so the per-contract-type
-orchestrators only declare *which* extensions they care about.
+orchestrators only declare *which* extensions they care about. Discovery is
+delegated to the ingestion :class:`~repowise.core.ingestion.traverser.FileTraverser`
+so the scanned file set respects ``.gitignore`` and nested-repo boundaries
+(identical to the repo's actual index) instead of a raw ``os.walk`` that would
+descend into sibling/vendored repos rooted under the workspace.
"""
from __future__ import annotations
@@ -13,31 +16,6 @@ from collections.abc import Iterator
from dataclasses import dataclass
from pathlib import Path
-# Directories that never contain first-party source worth scanning.
-BLOCKED_DIRS = frozenset(
- {
- ".git",
- "node_modules",
- "__pycache__",
- ".venv",
- "venv",
- "dist",
- "build",
- "target",
- "vendor",
- ".next",
- ".nuxt",
- ".tox",
- ".mypy_cache",
- ".gradle",
- ".mvn",
- "out",
- "bin",
- }
-)
-
-MAX_FILE_SIZE = 512 * 1024 # 512 KB
-
@dataclass(frozen=True)
class ScanContext:
@@ -59,26 +37,35 @@ def iter_source_files(
) -> Iterator[tuple[str, str, str]]:
"""Yield ``(rel_path, suffix, content)`` for each scannable source file.
- Walks *repo_root*, pruning :data:`BLOCKED_DIRS` and dotfile directories,
- skipping files whose extension is not in *extensions* or that exceed
- :data:`MAX_FILE_SIZE`, and reading the rest as UTF-8 (replacing undecodable
- bytes). Unreadable files are silently skipped.
+ File discovery is delegated to the ingestion :class:`FileTraverser`, the
+ same gitignore- and nested-repo-aware walker the main index uses, rather
+ than a raw ``os.walk``. This matters whenever a workspace repo's root
+ physically contains *other* git repositories (sibling/vendored repos,
+ benchmark clones, build worktrees) or large gitignored trees: a naive walk
+ descends into all of them and scans hundreds of thousands of foreign files.
+ That is not hypothetical: on a developer checkout whose root held nested
+ repos, the old walk yielded **1.05M files** for a repo whose real source is
+ ~2k, wedging contract extraction for tens of minutes. FileTraverser prunes
+ nested git repos and honours ``.gitignore``, so the scanned set matches the
+ repo's actual index.
+
+ Only files whose lower-cased suffix is in *extensions* are yielded.
+ Oversized, binary, generated, and ignored files are already excluded by the
+ traverser. Unreadable files are silently skipped.
"""
- root = repo_root.resolve()
- for dirpath, dirnames, filenames in os.walk(root):
- dirnames[:] = [d for d in dirnames if d not in BLOCKED_DIRS and not d.startswith(".")]
- for fname in filenames:
- fpath = Path(dirpath) / fname
- suffix = fpath.suffix.lower()
- if suffix not in extensions:
- continue
- try:
- if fpath.stat().st_size > MAX_FILE_SIZE:
- continue
- except OSError:
- continue
- try:
- content = fpath.read_text(encoding="utf-8", errors="replace")
- except OSError:
- continue
- yield fpath.relative_to(root).as_posix(), suffix, content
+ from repowise.core.ingestion.traverser import FileTraverser
+
+ root = Path(repo_root).resolve()
+ if not root.is_dir():
+ return
+
+ traverser = FileTraverser(root)
+ for info in traverser.traverse():
+ suffix = os.path.splitext(info.path)[1].lower()
+ if suffix not in extensions:
+ continue
+ try:
+ content = Path(info.abs_path).read_text(encoding="utf-8", errors="replace")
+ except OSError:
+ continue
+ yield info.path, suffix, content
diff --git a/packages/core/src/repowise/core/workspace/extractors/service_boundary.py b/packages/core/src/repowise/core/workspace/extractors/service_boundary.py
index b260f43df..5f01b7a1b 100644
--- a/packages/core/src/repowise/core/workspace/extractors/service_boundary.py
+++ b/packages/core/src/repowise/core/workspace/extractors/service_boundary.py
@@ -4,6 +4,12 @@ Walks a repo directory tree looking for marker files (package.json, go.mod,
Dockerfile, etc.) that indicate independent service boundaries. Used to
distinguish intra-service calls from inter-service calls when matching
contracts.
+
+The directory walk goes through :func:`~repowise.core.fs_walk.walk_repo`, which
+prunes junk directories and, critically, *nested git repositories*. A workspace
+repo whose root physically contains sibling/vendored repos (or benchmark clones)
+would otherwise be walked end-to-end, turning a sub-second scan into a
+multi-minute stall over hundreds of thousands of foreign files.
"""
from __future__ import annotations
@@ -12,6 +18,7 @@ import os
from dataclasses import dataclass, field
from pathlib import Path
+from repowise.core.fs_walk import walk_repo
from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
# ---------------------------------------------------------------------------
@@ -116,8 +123,10 @@ def detect_service_boundaries(repo_path: Path) -> list[ServiceBoundary]:
repo_root = repo_path.resolve()
boundaries: list[ServiceBoundary] = []
- for dirpath, dirnames, filenames in os.walk(repo_root):
- # Prune blocked dirs in-place
+ for dirpath, dirnames, filenames in walk_repo(repo_root):
+ # ``walk_repo`` already prunes junk dirs and nested git repos; layer on
+ # the derived-output names (dist/build/target/...) it intentionally
+ # leaves ambiguous, plus dotfile dirs, that are never service roots.
dirnames[:] = [d for d in dirnames if d not in _BLOCKED_DIRS and not d.startswith(".")]
current = Path(dirpath)
diff --git a/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py b/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py
index fd2f61724..4d6c447cc 100644
--- a/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py
+++ b/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py
@@ -7,7 +7,6 @@ consumer patterns.
from __future__ import annotations
import logging
-import os
import re
from dataclasses import dataclass
from pathlib import Path
@@ -15,6 +14,8 @@ from typing import TYPE_CHECKING
from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
+from .base import iter_source_files
+
if TYPE_CHECKING:
from repowise.core.workspace.contracts import Contract
@@ -24,30 +25,6 @@ _log = logging.getLogger("repowise.workspace.extractors.topic")
# Constants
# ---------------------------------------------------------------------------
-_BLOCKED_DIRS = frozenset(
- {
- ".git",
- "node_modules",
- "__pycache__",
- ".venv",
- "venv",
- "dist",
- "build",
- "target",
- "vendor",
- ".next",
- ".nuxt",
- ".tox",
- ".mypy_cache",
- ".gradle",
- ".mvn",
- "out",
- "bin",
- }
-)
-
-_MAX_FILE_SIZE = 512 * 1024
-
_EXTENSIONS = _LANG_REGISTRY.extensions_for(["python", "typescript", "javascript", "java", "go"])
@@ -247,58 +224,41 @@ class TopicExtractor:
from repowise.core.workspace.contracts import Contract
contracts: list[Contract] = []
- repo_root = repo_path.resolve()
seen: set[tuple[str, str, str]] = set() # (file, contract_id, role) dedup
- for dirpath, dirnames, filenames in os.walk(repo_root):
- dirnames[:] = [d for d in dirnames if d not in _BLOCKED_DIRS and not d.startswith(".")]
- for fname in filenames:
- fpath = Path(dirpath) / fname
- suffix = fpath.suffix.lower()
- if suffix not in _EXTENSIONS:
- continue
- try:
- if fpath.stat().st_size > _MAX_FILE_SIZE:
+ # File discovery is gitignore- and nested-repo-aware (see
+ # ``iter_source_files``) so a workspace repo whose root contains nested
+ # repos or large ignored trees is not scanned end-to-end.
+ for rel_path, _suffix, content in iter_source_files(repo_path, _EXTENSIONS):
+ for pdef in _ALL_PATTERNS:
+ for match in pdef.regex.finditer(content):
+ topic_name = match.group(pdef.topic_group).strip()
+ if not topic_name:
+ continue
+
+ contract_id = f"topic::{topic_name.lower()}"
+
+ # Deduplicate within the same file
+ dedup_key = (rel_path, contract_id, pdef.role)
+ if dedup_key in seen:
continue
- except OSError:
- continue
-
- try:
- content = fpath.read_text(encoding="utf-8", errors="replace")
- except OSError:
- continue
-
- rel_path = fpath.relative_to(repo_root).as_posix()
-
- for pdef in _ALL_PATTERNS:
- for match in pdef.regex.finditer(content):
- topic_name = match.group(pdef.topic_group).strip()
- if not topic_name:
- continue
-
- contract_id = f"topic::{topic_name.lower()}"
-
- # Deduplicate within the same file
- dedup_key = (rel_path, contract_id, pdef.role)
- if dedup_key in seen:
- continue
- seen.add(dedup_key)
-
- contracts.append(
- Contract(
- repo=repo_alias,
- contract_id=contract_id,
- contract_type="topic",
- role=pdef.role,
- file_path=rel_path,
- symbol_name=f"{pdef.label}('{topic_name}')",
- confidence=pdef.confidence,
- service=None,
- meta={
- "topic": topic_name,
- "broker": pdef.broker,
- },
- )
+ seen.add(dedup_key)
+
+ contracts.append(
+ Contract(
+ repo=repo_alias,
+ contract_id=contract_id,
+ contract_type="topic",
+ role=pdef.role,
+ file_path=rel_path,
+ symbol_name=f"{pdef.label}('{topic_name}')",
+ confidence=pdef.confidence,
+ service=None,
+ meta={
+ "topic": topic_name,
+ "broker": pdef.broker,
+ },
)
+ )
return contracts
diff --git a/packages/server/src/repowise/server/routers/jobs.py b/packages/server/src/repowise/server/routers/jobs.py
index 81f90e9b0..98f99c675 100644
--- a/packages/server/src/repowise/server/routers/jobs.py
+++ b/packages/server/src/repowise/server/routers/jobs.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
import json
+from datetime import UTC, datetime
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from sqlalchemy import func, select
@@ -50,6 +51,14 @@ async def _find_job_factory(app_state, job_id: str):
return None, None
+def _created_sort_key(job: GenerationJob) -> datetime:
+ """Timezone-safe sort key for ``created_at`` across merged repo databases."""
+ dt = job.created_at
+ if dt is None:
+ return datetime.min.replace(tzinfo=UTC)
+ return dt if dt.tzinfo is not None else dt.replace(tzinfo=UTC)
+
+
@router.get("", response_model=list[JobResponse])
async def list_jobs(
request: Request,
@@ -59,6 +68,7 @@ async def list_jobs(
offset: int = Query(0, ge=0),
) -> list[JobResponse]:
"""List generation jobs, optionally filtered by repository or status."""
+
async def _query_jobs(factory) -> list[GenerationJob]:
q = select(GenerationJob)
if repo_id:
@@ -89,7 +99,11 @@ async def list_jobs(
except Exception:
continue
- jobs.sort(key=lambda j: j.created_at, reverse=True)
+ # Jobs are merged from several repo databases in workspace mode, and SQLite
+ # hands back a mix of timezone-aware and naive ``created_at`` values, which
+ # cannot be compared directly. Coerce naive timestamps to UTC for the sort
+ # (they are stored as UTC) so the merge never raises.
+ jobs.sort(key=_created_sort_key, reverse=True)
return [JobResponse.from_orm(j) for j in jobs[offset : offset + limit]]
diff --git a/tests/unit/server/test_jobs.py b/tests/unit/server/test_jobs.py
index 9acbd6523..94b996886 100644
--- a/tests/unit/server/test_jobs.py
+++ b/tests/unit/server/test_jobs.py
@@ -14,6 +14,7 @@ from repowise.core.persistence import crud
from repowise.core.persistence.database import get_session, init_db
from repowise.core.persistence.models import GenerationJob
from repowise.server.app import reset_workspace_stale_jobs
+from repowise.server.routers.jobs import _created_sort_key
from tests.unit.server.conftest import create_test_repo
@@ -203,3 +204,28 @@ async def test_job_stream_completed(client: AsyncClient, app) -> None:
assert "text/event-stream" in resp.headers["content-type"]
# For a completed job, the stream should contain 'done' event
assert "event: done" in resp.text or "event: progress" in resp.text
+
+
+def test_created_sort_key_handles_mixed_tz_awareness() -> None:
+ """Jobs merged from several repo DBs mix naive and aware ``created_at``.
+
+ In workspace mode the per-repo SQLite databases hand back a mix of
+ timezone-aware and naive timestamps; sorting them directly raised
+ ``TypeError: can't compare offset-naive and offset-aware datetimes`` and
+ 500'd ``GET /api/jobs``. The sort key must coerce them onto common ground.
+ """
+ from datetime import UTC, datetime
+
+ aware_new = SimpleNamespace(created_at=datetime(2026, 1, 2, tzinfo=UTC))
+ naive_newest = SimpleNamespace(created_at=datetime(2026, 1, 3)) # naive, newest
+ aware_old = SimpleNamespace(created_at=datetime(2026, 1, 1, tzinfo=UTC))
+ missing = SimpleNamespace(created_at=None)
+
+ jobs = [aware_old, missing, naive_newest, aware_new]
+ # Must not raise despite the naive/aware mix.
+ jobs.sort(key=_created_sort_key, reverse=True)
+
+ assert jobs[0] is naive_newest # 2026-01-03 is newest
+ assert jobs[1] is aware_new # then 2026-01-02
+ assert jobs[2] is aware_old # then 2026-01-01
+ assert jobs[-1] is missing # None sorts oldest
diff --git a/tests/unit/workspace/test_extractor_traversal.py b/tests/unit/workspace/test_extractor_traversal.py
new file mode 100644
index 000000000..cdbfacbe8
--- /dev/null
+++ b/tests/unit/workspace/test_extractor_traversal.py
@@ -0,0 +1,96 @@
+"""Regression tests for workspace extractor file discovery.
+
+The contract/topic/service-boundary extractors must scan only the repo's own
+source — respecting ``.gitignore`` and nested-repo boundaries — exactly like the
+main index. A naive ``os.walk`` from the repo root descends into nested git
+repos (sibling/vendored repos, benchmark clones) physically located under a
+workspace root, which once turned a sub-second scan into a multi-minute stall
+over ~1M foreign files. These tests pin that behaviour.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from repowise.core.workspace.extractors.base import iter_source_files
+from repowise.core.workspace.extractors.service_boundary import detect_service_boundaries
+from repowise.core.workspace.extractors.topic_extractor import TopicExtractor
+
+_PY = frozenset({".py"})
+
+
+def _rel_paths(repo: Path) -> set[str]:
+ return {rel for rel, _suffix, _content in iter_source_files(repo, _PY)}
+
+
+def test_iter_source_files_skips_nested_git_repo(tmp_path: Path) -> None:
+ (tmp_path / "app.py").write_text("print('main')", encoding="utf-8")
+ nested = tmp_path / "vendored"
+ nested.mkdir()
+ (nested / ".git").mkdir() # makes ``vendored`` look like its own repo
+ (nested / "foreign.py").write_text("x = 1", encoding="utf-8")
+
+ found = _rel_paths(tmp_path)
+
+ assert "app.py" in found
+ assert "vendored/foreign.py" not in found
+
+
+def test_iter_source_files_skips_nested_git_worktree_file(tmp_path: Path) -> None:
+ # Worktrees and submodules use a ``.git`` *file* (not dir) — also a boundary.
+ (tmp_path / "app.py").write_text("print('main')", encoding="utf-8")
+ nested = tmp_path / "linked"
+ nested.mkdir()
+ (nested / ".git").write_text("gitdir: /somewhere/else", encoding="utf-8")
+ (nested / "foreign.py").write_text("x = 1", encoding="utf-8")
+
+ found = _rel_paths(tmp_path)
+
+ assert "app.py" in found
+ assert "linked/foreign.py" not in found
+
+
+def test_iter_source_files_respects_gitignore(tmp_path: Path) -> None:
+ (tmp_path / ".gitignore").write_text("ignored/\n", encoding="utf-8")
+ (tmp_path / "app.py").write_text("print('main')", encoding="utf-8")
+ ignored = tmp_path / "ignored"
+ ignored.mkdir()
+ (ignored / "skip.py").write_text("y = 2", encoding="utf-8")
+
+ found = _rel_paths(tmp_path)
+
+ assert "app.py" in found
+ assert "ignored/skip.py" not in found
+
+
+def test_topic_extractor_skips_nested_git_repo(tmp_path: Path) -> None:
+ # A NATS producer in the repo is found; an identical one inside a nested
+ # repo is not (the nested repo is a hard boundary).
+ (tmp_path / "producer.py").write_text(
+ 'nc.publish("events.user")', encoding="utf-8"
+ )
+ nested = tmp_path / "bench"
+ nested.mkdir()
+ (nested / ".git").mkdir()
+ (nested / "other.py").write_text('nc.publish("events.foreign")', encoding="utf-8")
+
+ contracts = TopicExtractor().extract(tmp_path, "primary")
+ topics = {c.meta["topic"] for c in contracts}
+
+ assert "events.user" in topics
+ assert "events.foreign" not in topics
+
+
+def test_service_boundary_skips_nested_git_repo(tmp_path: Path) -> None:
+ # A nested repo with service markers must not be reported as a sub-service
+ # of the parent — it is an independent repo.
+ nested = tmp_path / "sibling-service"
+ nested.mkdir()
+ (nested / ".git").mkdir()
+ (nested / "pyproject.toml").write_text("[project]\nname='x'\n", encoding="utf-8")
+ (nested / "main.py").write_text("print('svc')", encoding="utf-8")
+
+ boundaries = detect_service_boundaries(tmp_path)
+ names = {b.service_name for b in boundaries}
+
+ assert "sibling-service" not in names