Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(workspace): extract only the repos that changed, and time every cross-repo phase (#1596)

Raghav Chamadiyaba2dfed084bd
diff --git a/packages/core/src/repowise/core/workspace/contracts.py b/packages/core/src/repowise/core/workspace/contracts.py
index 82b2d8324..e238d18cb 100644
--- a/packages/core/src/repowise/core/workspace/contracts.py
+++ b/packages/core/src/repowise/core/workspace/contracts.py
@@ -10,6 +10,7 @@ from ``cross_repo_edges.json`` so Phase 3 and Phase 4 fail independently.
from __future__ import annotations
import asyncio
+import hashlib
import json
import logging
import re
@@ -145,6 +146,12 @@ class ContractStore:
#: contract count has no denominator, and 26 consumers looks like 26
#: consumers rather than 26 out of 130.
extraction_stats: dict[str, dict[str, int]] = field(default_factory=dict)
+ #: Per-repo-alias provenance: ``{"head": <sha>, "extracted_at": <iso>}``.
+ #: What makes an incremental run auditable — ``generated_at`` says when the
+ #: artifact was written, this says which commit each repo's rows describe.
+ #: A repo absent from here has never been extracted by a provenance-aware
+ #: run and must be re-extracted rather than trusted.
+ repo_provenance: dict[str, dict[str, str]] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
return {
@@ -153,6 +160,7 @@ class ContractStore:
"contracts": [c.to_dict() for c in self.contracts],
"contract_links": [lk.to_dict() for lk in self.contract_links],
"extraction_stats": self.extraction_stats,
+ "repo_provenance": self.repo_provenance,
}
@classmethod
@@ -163,8 +171,21 @@ class ContractStore:
contracts=[Contract.from_dict(c) for c in data.get("contracts", [])],
contract_links=[ContractLink.from_dict(lk) for lk in data.get("contract_links", [])],
extraction_stats=data.get("extraction_stats", {}),
+ # Absent on any artifact written before incremental extraction
+ # existed. Defaulting to empty means "no repo can prove its
+ # freshness", so the next run re-extracts everything — the safe
+ # direction for a one-off cost.
+ repo_provenance=data.get("repo_provenance", {}),
)
+ def rows_for_repo(self, alias: str) -> list[Contract]:
+ """Every contract this store holds for *alias*.
+
+ The unit of incremental reuse. Contracts carry their repo inline rather
+ than being grouped by it, so selecting one repo's rows is a filter.
+ """
+ return [c for c in self.contracts if c.repo == alias]
+
# ---------------------------------------------------------------------------
# Contract ID normalization
@@ -453,12 +474,22 @@ def annotate_consumer_targets(
Mutates the contracts in place so both :func:`match_contracts` and the
diagnostics builder read one resolution. ``service_bases`` maps a base token
or host (case-insensitive) to a repo alias.
+
+ The two keys this owns are cleared before being recomputed, so the result is
+ a function of *contracts* and *service_bases* alone and not of whatever a
+ previous run stamped. That matters now that incremental extraction carries
+ contracts forward: a consumer resolved to ``gamma`` keeps that ``meta`` when
+ it is reused, and if ``gamma`` has since left the workspace the resolution
+ no longer fires — so without the clear, the stale target survives and the
+ contract claims to call a repo that is not in the workspace any more.
"""
repo_aliases = {c.repo for c in contracts}
sb = {k.lower(): v for k, v in (service_bases or {}).items()}
for c in contracts:
if c.role != "consumer" or c.contract_type != "http":
continue
+ c.meta.pop("target_repo", None)
+ c.meta.pop("external", None)
target, external = _resolve_consumer_target(c, repo_aliases, sb)
if external:
c.meta["external"] = True
@@ -685,21 +716,66 @@ async def run_contract_extraction(
workspace_root: Path,
changed_repos: list[str],
boundaries_by_repo: dict[str, list[ServiceBoundary]] | None = None,
+ previous_store: ContractStore | None = None,
) -> ContractStore:
"""Full contract extraction pipeline.
Called from :func:`run_cross_repo_hooks` during ``repowise update --workspace``.
- 1. For each repo: walk once, then run every extractor over that file list
- 2. Assign service to each contract from the repo's boundaries
- 3. Run matching engine
- 4. Merge manual links from ``WorkspaceConfig``
- 5. Save ``contracts.json``
+ 1. Decide which repos need re-extraction; carry the rest forward verbatim
+ 2. For each re-extracted repo: walk once, then run every extractor over it
+ 3. Assign service to each contract from the repo's boundaries
+ 4. Run matching engine over the merged set
+ 5. Merge manual links from ``WorkspaceConfig``
+ 6. Save ``contracts.json``
*boundaries_by_repo* is the workspace-wide boundary map computed once by
:func:`run_cross_repo_hooks` and shared with the system-graph build, which
needs the same answer. When None (a direct call, or a test), boundaries are
detected here instead.
+
+ *previous_store* is the artifact as it stands on disk, the source of any
+ carried-forward rows. When None nothing is reused and every repo is
+ extracted — the behaviour before incremental extraction existed, and what a
+ caller gets by default.
+
+ Incremental reuse, and why it is shaped this way
+ ------------------------------------------------
+ The unit of reuse is the **repo alias**, and a repo's contracts, its
+ ``extraction_stats`` row and its ``repo_provenance`` row move together or
+ not at all. That atomicity is what keeps the coverage figure honest: the
+ numerator (HTTP consumers, counted from ``contracts``) and the denominator's
+ other half (``http_consumer_unresolved``, counted from ``extraction_stats``)
+ always describe the same extraction of the same commit. The workspace-wide
+ ratio is a sum over repos, each term internally consistent — an aggregate
+ over repos that may sit at different commits, which is what a workspace
+ artifact honestly is. ``repo_provenance`` records which commit each term
+ came from so that is auditable rather than implied.
+
+ Reuse is **validated, never trusted**. A repo is carried forward only when
+ all four hold: its alias is absent from *changed_repos*; the provenance we
+ persisted names the same HEAD it is at now; that provenance was written
+ under the same contract config; and its working tree is clean.
+ ``changed_repos`` is computed upstream and can be wrong (a crashed run, a
+ branch switch, a hook that fired on a partial set), so it is treated as a
+ hint and each of the other three is checked here against live state. HEAD
+ alone would not be enough: extraction reads the working tree, so uncommitted
+ edits change the answer without moving the commit the stamp names. Every way
+ of disagreeing resolves toward re-extraction, so the failure mode is a
+ slower run rather than a stale artifact.
+
+ Deletion is by construction. The merged set is assembled by iterating the
+ *current* ``repo_paths``, never the persisted store, so a repo dropped from
+ the workspace config or one that lost its ``.repowise/`` index contributes
+ nothing to contracts, stats or provenance. There is no prune step to forget,
+ and a removed repo cannot leave a phantom provider behind.
+
+ Links are always recomputed over the whole merged set, never merged.
+ :func:`annotate_consumer_targets` derives its set of internal repos from the
+ full contract list, and :func:`match_contracts` decides ``exact`` vs
+ ``candidate`` by counting providers workspace-wide — a subset pass would
+ mislabel both. Matching is dict-indexed and costs milliseconds against
+ seconds of extraction, so scoping it would buy nothing and risk everything.
"""
from .extractors import (
DataExtractor,
@@ -733,6 +809,62 @@ async def run_contract_extraction(
if len(repo_paths) < 2:
return ContractStore()
+ # Which repos can be carried forward, and which must be re-extracted.
+ # Probing a repo's state costs a `git rev-parse` and a dirty check; the
+ # alternative is trusting changed_repos, which is exactly the trust a stale
+ # artifact would exploit. Skipped entirely when there is nothing to reuse.
+ from .update import get_head_commit
+
+ prior = previous_store or ContractStore()
+ changed = set(changed_repos)
+ reusable: set[str] = set()
+ heads: dict[str, str | None] = {}
+ # Fingerprints the settings that decide what extraction even looks for. A
+ # repo extracted under `detect_data: false`, or before a `service_bases`
+ # entry existed, holds rows that answer a question no longer being asked —
+ # and its HEAD is unchanged, so nothing else here would notice.
+ config_fp = hashlib.sha256(
+ json.dumps(contract_config.to_dict(), sort_keys=True).encode()
+ ).hexdigest()[:16]
+
+ if previous_store is not None:
+ from ..ingestion.change_detector import has_working_tree_changes
+
+ aliases = list(repo_paths)
+ probes = await asyncio.gather(
+ *[
+ asyncio.gather(
+ asyncio.to_thread(get_head_commit, repo_paths[a]),
+ asyncio.to_thread(has_working_tree_changes, repo_paths[a]),
+ )
+ for a in aliases
+ ]
+ )
+ for alias, (head, dirty) in zip(aliases, probes, strict=True):
+ heads[alias] = head
+ stamped = prior.repo_provenance.get(alias, {})
+ # Four independent ways to fail, all resolving toward re-extraction:
+ # upstream said it changed; HEAD is unreadable (not a git checkout,
+ # or git failed) so nothing can be proven; HEAD moved; or the config
+ # that shaped the persisted rows is not the config in force now.
+ # The dirty check is the one HEAD cannot cover: extraction reads the
+ # working tree, so uncommitted edits change the answer without
+ # moving the commit that the stamp names.
+ if (
+ alias not in changed
+ and head is not None
+ and head == stamped.get("head")
+ and stamped.get("config_fp") == config_fp
+ and not dirty
+ ):
+ reusable.add(alias)
+ to_extract = {alias: path for alias, path in repo_paths.items() if alias not in reusable}
+ # Only the repos being extracted need a stamp, and only they were probed
+ # above when there was no previous store to compare against.
+ for alias in to_extract:
+ if alias not in heads:
+ heads[alias] = await asyncio.to_thread(get_head_commit, repo_paths[alias])
+
# Per-repo extraction. Returns the contracts plus the counters the
# extractors filled in, which are the denominator half of any coverage
# figure and so must survive past this function.
@@ -787,6 +919,11 @@ async def run_contract_extraction(
# count is the honest half of any recall figure: it says how many real
# client calls were located but could not be resolved to an endpoint.
stats: dict[str, int] = {}
+ # Recorded per repo so a walk regression is visible in the artifact and
+ # assertable in a test, not just in a wall-clock number that varies by
+ # machine. One walk per repo is the invariant this counts.
+ stats["files_walked"] = len(files)
+ stats["walks"] = 1
for extractor in extractors:
kwargs = (
@@ -815,11 +952,35 @@ async def run_contract_extraction(
return contracts, stats
results = await asyncio.gather(
- *[_extract_one_repo(alias, path) for alias, path in repo_paths.items()]
+ *[_extract_one_repo(alias, path) for alias, path in to_extract.items()]
)
+ fresh = dict(zip(to_extract, results, strict=True))
+
+ # Assemble the merged set by walking the CURRENT repo set. A repo that left
+ # the workspace, or lost its index, is simply never visited here — that is
+ # the deletion, and it cannot be forgotten because there is no separate
+ # prune step to forget.
all_contracts: list[Contract] = []
extraction_stats: dict[str, dict[str, int]] = {}
- for alias, (repo_contracts, repo_stats) in zip(repo_paths, results, strict=True):
+ provenance: dict[str, dict[str, str]] = {}
+ now_iso = datetime.now(UTC).isoformat()
+ for alias in repo_paths:
+ if alias in fresh:
+ repo_contracts, repo_stats = fresh[alias]
+ head = heads[alias]
+ provenance[alias] = {
+ "head": head or "",
+ "extracted_at": now_iso,
+ "config_fp": config_fp,
+ }
+ else:
+ # Carried forward as one unit: rows, counters and the stamp saying
+ # which commit they describe. Splitting these is what would let the
+ # coverage ratio mix a numerator and a denominator from different
+ # commits, so they are only ever read together.
+ repo_contracts = prior.rows_for_repo(alias)
+ repo_stats = prior.extraction_stats.get(alias, {})
+ provenance[alias] = dict(prior.repo_provenance.get(alias, {}))
all_contracts.extend(repo_contracts)
if repo_stats:
extraction_stats[alias] = repo_stats
@@ -834,17 +995,27 @@ async def run_contract_extraction(
store = ContractStore(
version=1,
- generated_at=datetime.now(UTC).isoformat(),
+ generated_at=now_iso,
contracts=all_contracts,
contract_links=links,
extraction_stats=extraction_stats,
+ repo_provenance=provenance,
)
out_path = save_contract_store(store, workspace_root)
+ # files_walked is summed over the repos extracted THIS RUN, not over
+ # extraction_stats — that dict also holds the counters carried forward for
+ # skipped repos, so summing it would report the artifact's lifetime total
+ # under a name that reads as work just done, and an incremental run would
+ # look exactly like a full rescan.
_log.info(
- "Contract extraction complete: %d contracts, %d links → %s",
+ "Contract extraction complete: %d contracts, %d links, "
+ "%d repo(s) extracted, %d reused, %d file(s) walked this run → %s",
len(all_contracts),
len(links),
+ len(to_extract),
+ len(reusable),
+ sum(stats.get("files_walked", 0) for _, stats in fresh.values()),
out_path,
)
diff --git a/packages/core/src/repowise/core/workspace/update.py b/packages/core/src/repowise/core/workspace/update.py
index a3376f563..8309561c5 100644
--- a/packages/core/src/repowise/core/workspace/update.py
+++ b/packages/core/src/repowise/core/workspace/update.py
@@ -11,7 +11,7 @@ import json as _json
import logging
import sqlite3
import subprocess
-from collections.abc import Callable
+from collections.abc import Awaitable, Callable
from contextlib import suppress
from dataclasses import dataclass
from datetime import UTC, datetime
@@ -38,6 +38,7 @@ from repowise.core.update_lock import (
from ..docs_mode import docs_mode_state_fields
from ..ingestion.change_detector import has_working_tree_changes
+from ..pipeline.phase_timing import PhaseTimingRecorder
from .config import WorkspaceConfig
_log = logging.getLogger("repowise.workspace.update")
@@ -884,10 +885,27 @@ async def update_workspace(
# ---------------------------------------------------------------------------
-# Cross-repo hooks (Phase 3/4 placeholder)
+# Cross-repo hooks
# ---------------------------------------------------------------------------
+def _log_phase(timings: PhaseTimingRecorder, phase: str, **counts: int) -> None:
+ """Emit one structured line for a finished cross-repo phase.
+
+ Duration *and* the counts the phase produced. A timer alone cannot tell a
+ genuine speed-up from a phase that got faster by doing less work, which is
+ the specific regression workspace mode has already had once — so the counts
+ are not optional decoration.
+ """
+ detail = ", ".join(f"{k}={v}" for k, v in counts.items())
+ _log.info(
+ "workspace phase %s: %.2fs%s",
+ phase,
+ timings.timings.get(phase, 0.0),
+ f" ({detail})" if detail else "",
+ )
+
+
async def run_cross_repo_hooks(
ws_config: WorkspaceConfig,
workspace_root: Path,
@@ -913,58 +931,178 @@ async def run_cross_repo_hooks(
run_system_graph_build,
)
+ timings = PhaseTimingRecorder()
+
# Service boundaries, detected once for the whole workspace. Contract
# extraction and the system-graph build both need them and each used to walk
# every repo for its own copy.
boundaries_by_repo: dict[str, list[ServiceBoundary]] = {}
+ timings.on_phase_start("boundaries", None)
try:
boundaries_by_repo = await asyncio.to_thread(
_detect_boundaries_by_repo, ws_config, workspace_root
)
except Exception:
_log.warning("Service boundary detection failed", exc_info=True)
-
- overlay = CrossRepoOverlay()
- try:
- overlay = await run_cross_repo_analysis(ws_config, workspace_root, changed_repos)
- except Exception:
- _log.warning("Cross-repo analysis failed", exc_info=True)
+ timings.on_phase_done("boundaries")
+ _log_phase(
+ timings,
+ "boundaries",
+ repos=len(boundaries_by_repo),
+ boundaries=sum(len(b) for b in boundaries_by_repo.values()),
+ )
# Snapshot the contracts as they stand on disk BEFORE extraction overwrites
- # them — this is the cheapest honest "previous" state for the breaking-change
- # diff (no contract history needed; the last index is the baseline).
+ # them. Two jobs, one read: the baseline for the breaking-change diff, and
+ # the source of any rows extraction carries forward instead of recomputing.
+ # Loaded here rather than inside extraction so the ordering against the
+ # write is explicit and the 360 KB parse happens once.
previous_store = load_contract_store(workspace_root) or ContractStore()
- # Contract extraction (overwrites contracts.json).
+ # Phases 1 and 2 are independent: they read disjoint inputs (git history and
+ # manifests vs source files and the parse cache), write different artifacts
+ # (cross_repo_edges.json vs contracts.json), and share no mutable state —
+ # boundaries_by_repo is computed above and passed in read-only. Both push
+ # their blocking work through asyncio.to_thread, so gather genuinely
+ # overlaps them rather than interleaving two synchronous bodies.
+ # Each coroutine times itself, so running them together does not cost the
+ # ability to say which one dominates — the question this instrumentation
+ # exists to answer, and one a single duration for the pair would destroy.
+ async def _timed(phase: str, coro: Awaitable[Any]) -> Any:
+ timings.on_phase_start(phase, None)
+ try:
+ return await coro
+ finally:
+ timings.on_phase_done(phase)
+
+ overlay_result, store_result = await asyncio.gather(
+ _timed("cross_repo_analysis", run_cross_repo_analysis(ws_config, workspace_root, changed_repos)),
+ _timed(
+ "contract_extraction",
+ run_contract_extraction(
+ ws_config,
+ workspace_root,
+ changed_repos,
+ boundaries_by_repo or None,
+ previous_store,
+ ),
+ ),
+ return_exceptions=True,
+ )
+
+ # Cancellation and interpreter-level exits are not phase failures — the old
+ # `except Exception` let them propagate by construction, and gather's
+ # return_exceptions=True does not. Re-raise them before anything downstream
+ # treats a cancelled run as a completed one.
+ for result in (overlay_result, store_result):
+ if isinstance(result, BaseException) and not isinstance(result, Exception):
+ raise result
+
+ overlay = CrossRepoOverlay()
+ if isinstance(overlay_result, Exception):
+ _log.warning("Cross-repo analysis failed", exc_info=overlay_result)
+ else:
+ overlay = overlay_result
+
store = ContractStore()
- try:
- store = await run_contract_extraction(
- ws_config, workspace_root, changed_repos, boundaries_by_repo or None
- )
- except Exception:
- _log.warning("Contract extraction failed", exc_info=True)
+ extraction_ok = not isinstance(store_result, Exception)
+ if isinstance(store_result, Exception):
+ _log.warning("Contract extraction failed", exc_info=store_result)
+ else:
+ store = store_result
+
+ _log_phase(
+ timings,
+ "cross_repo_analysis",
+ co_changes=len(overlay.co_changes),
+ package_deps=len(overlay.package_deps),
+ )
+ _log_phase(
+ timings,
+ "contract_extraction",
+ contracts=len(store.contracts),
+ links=len(store.contract_links),
+ # Repos actually walked this run — the number that separates a real
+ # speed-up from a phase that got faster by doing less than it claims.
+ # Derived from the stamp rather than summing extraction_stats, which
+ # also carries the counters of the repos this run skipped.
+ repos_extracted=sum(
+ 1 for p in store.repo_provenance.values() if p.get("extracted_at") == store.generated_at
+ ),
+ repos_reused=sum(
+ 1 for p in store.repo_provenance.values() if p.get("extracted_at") != store.generated_at
+ ),
+ )
# System graph — the normalized service-granular structure every workspace
# view reads. Built last so it folds in the contracts and overlay above.
system_graph: SystemGraph | None = None
+ timings.on_phase_start("system_graph", None)
try:
system_graph = await run_system_graph_build(
ws_config, workspace_root, store, overlay, boundaries_by_repo or None
)
except Exception:
_log.warning("System graph build failed", exc_info=True)
+ timings.on_phase_done("system_graph")
+ _log_phase(
+ timings,
+ "system_graph",
+ nodes=len(system_graph.nodes) if system_graph else 0,
+ edges=len(system_graph.edges) if system_graph else 0,
+ )
# Breaking-change guard — diff the previous (on-disk) contracts against the
# freshly extracted set and persist the impacted-consumer report.
- try:
- run_breaking_change_detection(workspace_root, previous_store, store)
- except Exception:
- _log.warning("Breaking-change detection failed", exc_info=True)
+ #
+ # Gated on the SHAPE of the result, not on whether an exception escaped.
+ # Diffing a populated baseline against an empty set reports every contract
+ # in the workspace as removed, and extraction has more than one way to hand
+ # back nothing: it raises, or it returns an empty store early because fewer
+ # than two repos are currently indexed (a repo unmounted, renamed, or having
+ # just lost its .repowise/). Both would publish themselves as the largest
+ # breaking change the workspace has ever seen. An empty result is only
+ # trustworthy when the baseline is empty too.
+ if extraction_ok and (store.contracts or not previous_store.contracts):
+ report = None
+ timings.on_phase_start("breaking_change", None)
+ try:
+ report = run_breaking_change_detection(workspace_root, previous_store, store)
+ except Exception:
+ _log.warning("Breaking-change detection failed", exc_info=True)
+ timings.on_phase_done("breaking_change")
+ _log_phase(
+ timings,
+ "breaking_change",
+ breaking=len(report.changes) if report else 0,
+ )
+ else:
+ _log.warning(
+ "Skipping breaking-change detection: extraction produced no contracts "
+ "(ok=%s) while the previous artifact holds %d — diffing these would "
+ "report the whole workspace as removed",
+ extraction_ok,
+ len(previous_store.contracts),
+ )
# Conformance + cycles — check declared dependency rules and detect circular
# service dependencies over the freshly-built system graph.
if system_graph is not None:
+ conformance = None
+ timings.on_phase_start("conformance", None)
try:
- run_conformance_check(ws_config, workspace_root, system_graph)
+ conformance = run_conformance_check(ws_config, workspace_root, system_graph)
except Exception:
_log.warning("Conformance check failed", exc_info=True)
+ timings.on_phase_done("conformance")
+ _log_phase(
+ timings,
+ "conformance",
+ rules=conformance.rules_evaluated if conformance else 0,
+ violations=len(conformance.violations) if conformance else 0,
+ # The true total, not the capped list — a workspace with 500 cycles
+ # must not report the same number as one with 50.
+ cycles=conformance.total_cycles if conformance else 0,
+ )
+
+ _log.info("Cross-repo phase timings (seconds): %s", timings.timings)
diff --git a/tests/unit/workspace/test_incremental_extraction.py b/tests/unit/workspace/test_incremental_extraction.py
new file mode 100644
index 000000000..4e1a6b416
--- /dev/null
+++ b/tests/unit/workspace/test_incremental_extraction.py
@@ -0,0 +1,497 @@
+"""Incremental contract extraction: what gets reused, and what must not.
+
+A one-line commit in one repo used to re-extract every repo in the workspace,
+because ``changed_repos`` was accepted and never read. Scoping that is easy; the
+part that needs tests is what happens to the *skipped* repos' rows.
+
+The invariant these tests defend is that a repo's contracts, its
+``extraction_stats`` row and its ``repo_provenance`` row move together or not at
+all. Splitting them is what would let the coverage ratio take its numerator from
+one commit and its denominator from another — a wrong number that still looks
+like a number.
+
+The other half is that reuse is validated rather than trusted. ``changed_repos``
+is a hint from upstream; what is checked against live state is the repo's HEAD,
+its working tree, and the contract config the persisted rows were produced
+under. Every way of disagreeing resolves toward re-extraction.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from repowise.core.workspace import contracts as contracts_mod
+from repowise.core.workspace.config import ContractConfig, RepoEntry, WorkspaceConfig
+from repowise.core.workspace.contracts import ContractStore, run_contract_extraction
+
+REPO_SOURCE = {
+ "svc/api.py": ('router = APIRouter(prefix="/api")\n\n\[email protected]("/users")\ndef users(): ...\n'),
+ "svc/db.py": 'Q = "SELECT id FROM users WHERE id = :id"\n',
+ "svc/pyproject.toml": '[project]\nname = "svc"\n',
+ # An HTTP consumer, so the annotation the carry-forward path has to
+ # recompute actually exists in the fixture.
+ "svc/client.ts": "await fetch('/api/users');\n",
+}
+
+
+def _git(repo: Path, *args: str) -> str:
+ result = subprocess.run(
+ ["git", *args], cwd=str(repo), capture_output=True, text=True, check=True
+ )
+ return result.stdout.strip()
+
+
+def _make_git_repo(root: Path, alias: str) -> Path:
+ """A real git checkout — these tests turn on HEAD actually moving."""
+ root.mkdir(parents=True, exist_ok=True)
+ for rel, content in REPO_SOURCE.items():
+ path = root / rel
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(content.replace("/api", f"/{alias}"), encoding="utf-8")
+ (root / ".repowise").mkdir(exist_ok=True)
+ _git(root, "init", "-q")
+ _git(root, "config", "user.email", "[email protected]")
+ _git(root, "config", "user.name", "Test")
+ _git(root, "add", "-A")
+ _git(root, "commit", "-q", "-m", "initial")
+ return root
+
+
+def workspace(tmp_path: Path) -> WorkspaceConfig:
+ for alias in ("alpha", "beta"):
+ _make_git_repo(tmp_path / alias, alias)
+ return WorkspaceConfig(
+ repos=[RepoEntry(path="alpha", alias="alpha"), RepoEntry(path="beta", alias="beta")],
+ contracts=ContractConfig(),
+ )
+
+
+def no_save(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)
+
+
+def extracted(monkeypatch: pytest.MonkeyPatch) -> list[str]:
+ """Record which repo aliases actually ran extraction.
+
+ ``run_contract_extraction`` imports ``iter_source_files`` at call time, so
+ patching it on the defining module is what the running code sees.
+ """
+ seen: list[str] = []
+
+ from repowise.core.workspace.extractors import base
+
+ original_iter = base.iter_source_files
+
+ def recording_iter(repo_path, wanted, exclude, content_hashes=None):
+ seen.append(Path(repo_path).name)
+ return original_iter(repo_path, wanted, exclude, content_hashes)
+
+ monkeypatch.setattr(base, "iter_source_files", recording_iter)
+ return seen
+
+
+async def _first_run(workspace: WorkspaceConfig, root: Path) -> ContractStore:
+ """A full extraction, producing the store a later run may carry forward."""
+ return await run_contract_extraction(workspace, root, [])
+
+
+# ---------------------------------------------------------------------------
+# Scoping actually happens
+# ---------------------------------------------------------------------------
+
+
+async def test_unchanged_repo_is_not_re_extracted(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None, extracted: list[str]
+) -> None:
+ first = await _first_run(workspace, tmp_path)
+ extracted.clear()
+
+ await run_contract_extraction(workspace, tmp_path, ["alpha"], None, first)
+
+ assert extracted == ["alpha"], "beta was unchanged and should not have been walked"
+
+
+async def test_carried_forward_contracts_are_identical(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None
+) -> None:
+ first = await _first_run(workspace, tmp_path)
+ second = await run_contract_extraction(workspace, tmp_path, ["alpha"], None, first)
+
+ def ids(store: ContractStore, alias: str) -> set[str]:
+ return {c.contract_id for c in store.rows_for_repo(alias)}
+
+ assert ids(second, "beta") == ids(first, "beta")
+ assert ids(second, "alpha") == ids(first, "alpha")
+
+
+async def test_no_previous_store_means_everything_is_extracted(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None, extracted: list[str]
+) -> None:
+ """The default, and what every existing caller and test relies on."""
+ await run_contract_extraction(workspace, tmp_path, ["alpha"])
+ assert sorted(extracted) == ["alpha", "beta"]
+
+
+# ---------------------------------------------------------------------------
+# Reuse is validated, not trusted
+# ---------------------------------------------------------------------------
+
+
+async def test_moved_head_is_re_extracted_even_when_not_in_changed_repos(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None, extracted: list[str]
+) -> None:
+ """The whole point of the HEAD check.
+
+ ``changed_repos`` says beta is clean. Beta has in fact moved — a crashed
+ previous run, a branch switch, a hook that fired on a partial set. The
+ persisted sha disagrees, so beta is re-extracted rather than carried
+ forward stale.
+ """
+ first = await _first_run(workspace, tmp_path)
+ beta = tmp_path / "beta"
+ (beta / "svc" / "extra.py").write_text('@router.get("/added")\ndef added(): ...\n', encoding="utf-8")
+ _git(beta, "add", "-A")
+ _git(beta, "commit", "-q", "-m", "second")
+ extracted.clear()
+
+ await run_contract_extraction(workspace, tmp_path, ["alpha"], None, first)
+
+ assert sorted(extracted) == ["alpha", "beta"]
+
+
+async def test_missing_provenance_forces_re_extraction(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None, extracted: list[str]
+) -> None:
+ """An artifact written before provenance existed cannot prove freshness."""
+ first = await _first_run(workspace, tmp_path)
+ first.repo_provenance.pop("beta")
+ extracted.clear()
+
+ await run_contract_extraction(workspace, tmp_path, ["alpha"], None, first)
+
+ assert sorted(extracted) == ["alpha", "beta"]
+
+
+async def test_unreadable_head_forces_re_extraction(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None, extracted: list[str]
+) -> None:
+ """A repo that is not a git checkout can never be proven unchanged."""
+ first = await _first_run(workspace, tmp_path)
+ extracted.clear()
+
+ import repowise.core.workspace.update as update_mod
+
+ original = update_mod.get_head_commit
+
+ def no_head(repo_path):
+ return None if Path(repo_path).name == "beta" else original(repo_path)
+
+ with pytest.MonkeyPatch.context() as mp:
+ mp.setattr(update_mod, "get_head_commit", no_head)
+ await run_contract_extraction(workspace, tmp_path, ["alpha"], None, first)
+
+ assert sorted(extracted) == ["alpha", "beta"]
+
+
+async def test_dirty_working_tree_forces_re_extraction(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None, extracted: list[str]
+) -> None:
+ """HEAD is not enough on its own.
+
+ Extraction reads the working tree, so an uncommitted route change alters
+ what it would find while leaving HEAD exactly where the stamp says it is.
+ Trusting the sha alone would hide that edit until something else happened
+ to commit.
+ """
+ first = await _first_run(workspace, tmp_path)
+ (tmp_path / "beta" / "svc" / "api.py").write_text(
+ 'router = APIRouter()\n\n\[email protected]("/uncommitted")\ndef added(): ...\n',
+ encoding="utf-8",
+ )
+ extracted.clear()
+
+ second = await run_contract_extraction(workspace, tmp_path, ["alpha"], None, first)
+
+ assert sorted(extracted) == ["alpha", "beta"]
+ assert any("uncommitted" in c.contract_id for c in second.rows_for_repo("beta"))
+
+
+async def test_changed_contract_config_forces_re_extraction(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None, extracted: list[str]
+) -> None:
+ """Rows extracted under a different question must not be reused.
+
+ Turning a detector off, or adding a ``service_bases`` mapping, changes what
+ extraction looks for and how consumers resolve — but it moves no commit, so
+ the HEAD check alone would happily carry the old answers forward and the
+ config change would appear to do nothing until someone committed.
+ """
+ first = await _first_run(workspace, tmp_path)
+ extracted.clear()
+
+ narrowed = WorkspaceConfig(
+ repos=list(workspace.repos),
+ contracts=ContractConfig(detect_data=False),
+ )
+ second = await run_contract_extraction(narrowed, tmp_path, [], None, first)
+
+ assert sorted(extracted) == ["alpha", "beta"]
+ assert not [c for c in second.contracts if c.contract_type == "data"]
+
+
+async def test_same_config_still_reuses(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None, extracted: list[str]
+) -> None:
+ """The fingerprint must be stable across runs, or nothing is ever reused."""
+ first = await _first_run(workspace, tmp_path)
+ extracted.clear()
+
+ await run_contract_extraction(workspace, tmp_path, [], None, first)
+
+ assert extracted == []
+
+
+# ---------------------------------------------------------------------------
+# Deletion — the silent-staleness half
+# ---------------------------------------------------------------------------
+
+
+async def test_repo_removed_from_config_leaves_no_phantom_rows(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None
+) -> None:
+ """A dropped repo takes its contracts, stats and provenance with it.
+
+ Carrying them forward is how a workspace grows providers that no code
+ declares. The merged set is built from the current repo list, so a repo that
+ is gone is never visited and cannot contribute.
+ """
+ for alias in ("gamma",):
+ _make_git_repo(tmp_path / alias, alias)
+ three = WorkspaceConfig(
+ repos=[
+ RepoEntry(path="alpha", alias="alpha"),
+ RepoEntry(path="beta", alias="beta"),
+ RepoEntry(path="gamma", alias="gamma"),
+ ],
+ contracts=ContractConfig(),
+ )
+ first = await run_contract_extraction(three, tmp_path, [])
+ assert first.rows_for_repo("gamma"), "fixture should produce gamma contracts"
+
+ second = await run_contract_extraction(workspace, tmp_path, ["alpha"], None, first)
+
+ assert second.rows_for_repo("gamma") == []
+ assert "gamma" not in second.extraction_stats
+ assert "gamma" not in second.repo_provenance
+
+
+async def test_unindexed_repo_leaves_no_phantom_rows(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None
+) -> None:
+ """Losing ``.repowise/`` is the same deletion by a different route."""
+ first = await _first_run(workspace, tmp_path)
+ assert first.rows_for_repo("beta")
+
+ for alias in ("gamma", "delta"):
+ _make_git_repo(tmp_path / alias, alias)
+ (tmp_path / "beta" / ".repowise").rmdir()
+ wider = WorkspaceConfig(
+ repos=[
+ RepoEntry(path="alpha", alias="alpha"),
+ RepoEntry(path="beta", alias="beta"),
+ RepoEntry(path="gamma", alias="gamma"),
+ RepoEntry(path="delta", alias="delta"),
+ ],
+ contracts=ContractConfig(),
+ )
+
+ second = await run_contract_extraction(wider, tmp_path, ["alpha"], None, first)
+
+ assert second.rows_for_repo("beta") == []
+ assert "beta" not in second.repo_provenance
+
+
+# ---------------------------------------------------------------------------
+# Stats and contracts move as one unit
+# ---------------------------------------------------------------------------
+
+
+async def test_stats_and_contracts_are_carried_forward_together(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None
+) -> None:
+ """The coverage ratio's numerator and denominator stay commit-aligned.
+
+ If a repo's contracts were reused while its stats were dropped (or vice
+ versa), ``build_diagnostics`` would divide a count of consumers from one
+ commit by an unresolved count from another and report a coverage percentage
+ that describes no run that ever happened.
+ """
+ first = await _first_run(workspace, tmp_path)
+ first.extraction_stats["beta"] = {"http_consumer_unresolved": 7, "files_walked": 3, "walks": 1}
+
+ second = await run_contract_extraction(workspace, tmp_path, ["alpha"], None, first)
+
+ assert second.extraction_stats["beta"]["http_consumer_unresolved"] == 7
+ assert second.rows_for_repo("beta") == first.rows_for_repo("beta")
+ assert second.repo_provenance["beta"] == first.repo_provenance["beta"]
+
+
+async def test_re_extracted_repo_gets_fresh_stats_and_stamp(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None
+) -> None:
+ first = await _first_run(workspace, tmp_path)
+ first.extraction_stats["alpha"] = {"http_consumer_unresolved": 99}
+
+ second = await run_contract_extraction(workspace, tmp_path, ["alpha"], None, first)
+
+ assert second.extraction_stats["alpha"].get("http_consumer_unresolved", 0) != 99
+ assert second.repo_provenance["alpha"]["extracted_at"] == second.generated_at
+ assert second.repo_provenance["beta"]["extracted_at"] != second.generated_at
+
+
+async def test_provenance_records_the_commit_each_repo_describes(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None
+) -> None:
+ first = await _first_run(workspace, tmp_path)
+ for alias in ("alpha", "beta"):
+ assert first.repo_provenance[alias]["head"] == _git(tmp_path / alias, "rev-parse", "HEAD")
+
+
+async def test_provenance_round_trips_through_json(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None
+) -> None:
+ first = await _first_run(workspace, tmp_path)
+ revived = ContractStore.from_dict(first.to_dict())
+ assert revived.repo_provenance == first.repo_provenance
+ assert revived.extraction_stats == first.extraction_stats
+
+
+async def test_legacy_artifact_without_provenance_loads(tmp_path: Path) -> None:
+ """Reading an artifact from before this field existed must not raise."""
+ store = ContractStore.from_dict(
+ {"version": 1, "generated_at": "2026-01-01T00:00:00Z", "contracts": [], "contract_links": []}
+ )
+ assert store.repo_provenance == {}
+
+
+# ---------------------------------------------------------------------------
+# Links are recomputed globally, never merged
+# ---------------------------------------------------------------------------
+
+
+async def test_a_new_provider_links_a_carried_forward_consumer(
+ tmp_path: Path, no_save: None
+) -> None:
+ """The reason matching is never scoped.
+
+ Beta's consumer is carried forward untouched. Alpha gains the provider that
+ satisfies it. If matching ran only over the changed repo's contracts, or if
+ beta's old (unmatched) links were merged forward, this link would not exist.
+ """
+ alpha = tmp_path / "alpha"
+ beta = tmp_path / "beta"
+ alpha.mkdir()
+ (alpha / ".repowise").mkdir()
+ (alpha / "placeholder.py").write_text("x = 1\n", encoding="utf-8")
+ beta.mkdir()
+ (beta / ".repowise").mkdir()
+ (beta / "client.ts").write_text("await fetch('/api/orders');\n", encoding="utf-8")
+ for repo in (alpha, beta):
+ _git(repo, "init", "-q")
+ _git(repo, "config", "user.email", "[email protected]")
+ _git(repo, "config", "user.name", "Test")
+ _git(repo, "add", "-A")
+ _git(repo, "commit", "-q", "-m", "initial")
+
+ config = WorkspaceConfig(
+ repos=[RepoEntry(path="alpha", alias="alpha"), RepoEntry(path="beta", alias="beta")],
+ contracts=ContractConfig(),
+ )
+ first = await run_contract_extraction(config, tmp_path, [])
+ assert not [lk for lk in first.contract_links if lk.contract_id.startswith("http::GET::/api/orders")]
+
+ (alpha / "api.py").write_text(
+ 'router = APIRouter()\n\n\[email protected]("/api/orders")\ndef orders(): ...\n', encoding="utf-8"
+ )
+ _git(alpha, "add", "-A")
+ _git(alpha, "commit", "-q", "-m", "add provider")
+
+ second = await run_contract_extraction(config, tmp_path, ["alpha"], None, first)
+
+ linked = [lk for lk in second.contract_links if "orders" in lk.contract_id]
+ assert linked, "a carried-forward consumer must match a newly extracted provider"
+ assert linked[0].consumer_repo == "beta"
+ assert linked[0].provider_repo == "alpha"
+
+
+async def test_carried_consumer_drops_a_target_that_left_the_workspace(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None
+) -> None:
+ """Annotation is recomputed for carried rows, not inherited.
+
+ ``annotate_consumer_targets`` only ever wrote its keys; nothing cleared
+ them. That was invisible while every contract was re-extracted from source
+ each run, because ``meta`` started fresh. Once a consumer is carried
+ forward, an old ``target_repo`` naming a repo that has since left the
+ workspace would survive and the contract would keep claiming a call into it.
+ """
+ first = await _first_run(workspace, tmp_path)
+ # Must be a consumer of the repo that gets CARRIED FORWARD. Staleness on a
+ # re-extracted repo is impossible by construction — its contracts are new
+ # objects — so injecting there would assert nothing.
+ carried = [
+ c
+ for c in first.rows_for_repo("beta")
+ if c.role == "consumer" and c.contract_type == "http"
+ ]
+ assert carried, "fixture must give beta an HTTP consumer or this asserts nothing"
+ carried[0].meta["target_repo"] = "gamma_that_is_gone"
+
+ second = await run_contract_extraction(workspace, tmp_path, ["alpha"], None, first)
+
+ stale = [
+ c
+ for c in second.contracts
+ if c.meta.get("target_repo") == "gamma_that_is_gone"
+ ]
+ assert not stale, "a carried-forward consumer kept a target repo that no longer exists"
+
+
+# ---------------------------------------------------------------------------
+# Walk budget under incremental extraction
+# ---------------------------------------------------------------------------
+
+
+async def test_walk_count_scales_with_changed_repos_not_workspace_size(
+ workspace: WorkspaceConfig, tmp_path: Path, no_save: None, extracted: list[str]
+) -> None:
+ """The counter that makes the win visible in the artifact.
+
+ ``files_walked`` is per repo, so the workspace total after a one-repo commit
+ is that repo's file count — not the sum over every repo. A regression that
+ quietly reintroduced the full rescan would move this number.
+ """
+ first = await _first_run(workspace, tmp_path)
+ extracted.clear()
+
+ second = await run_contract_extraction(workspace, tmp_path, ["alpha"], None, first)
+
+ # The oracle is what was actually walked, not a number the store carries:
+ # every per-repo counter in extraction_stats survives carry-forward, so a
+ # full rescan and an incremental run leave the same totals behind and no
+ # assertion over them can tell the two apart.
+ assert extracted == ["alpha"]
+ walked_this_run = sum(
+ second.extraction_stats[a].get("files_walked", 0)
+ for a, p in second.repo_provenance.items()
+ if p.get("extracted_at") == second.generated_at
+ )
+ full_total = sum(s.get("files_walked", 0) for s in first.extraction_stats.values())
+ assert 0 < walked_this_run < full_total
+ assert second.extraction_stats["alpha"]["walks"] == 1 # still one walk per repo