Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(workspace): one changed repo stops rewriting every member's store and editor files (#4569)

Raghav Chamadiya6d95a2629bd8
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/member_refresh.py b/packages/cli/src/repowise/cli/commands/update_cmd/member_refresh.py
index 22829fb80..4786fae0c 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/member_refresh.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/member_refresh.py
@@ -86,10 +86,16 @@ def member_fingerprint(repo_path: Path) -> str | None:
for name in _MEMBER_FILES:
try:
st = (repo_path / name).stat()
- # Size as well as mtime: NTFS can give two quick writes one mtime.
- parts.append(f"{name}:{st.st_mtime_ns}:{st.st_size}")
except OSError:
parts.append(f"{name}:-")
+ continue
+ # An empty WAL holds no writes: a read-only reader (the workspace's
+ # sync-time probe) leaves one behind, which must not read as a write.
+ if name.endswith("-wal") and st.st_size == 0:
+ parts.append(f"{name}:-")
+ else:
+ # Size as well as mtime: NTFS can give two quick writes one mtime.
+ parts.append(f"{name}:{st.st_mtime_ns}:{st.st_size}")
return hashlib.sha256("\n".join(parts).encode()).hexdigest()
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/workspace.py b/packages/cli/src/repowise/cli/commands/update_cmd/workspace.py
index 3e75211b2..56c3851b2 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/workspace.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/workspace.py
@@ -344,6 +344,8 @@ def _workspace_index_only_update(
repo_filter=repo_alias,
dry_run=False,
include_working_tree=include_working_tree,
+ # _reconcile_up_to_date restamped the members that moved.
+ restamp_up_to_date=False,
on_repo_start=_on_start,
on_repo_done=_print_repo_result,
)
@@ -611,6 +613,8 @@ def _update_core_members(
run_hooks=False,
dry_run=False,
include_working_tree=include_working_tree,
+ # _reconcile_up_to_date restamped the members that moved.
+ restamp_up_to_date=False,
on_repo_start=_on_start,
on_repo_done=_print_repo_result,
)
diff --git a/packages/core/src/repowise/core/events/snapshot.py b/packages/core/src/repowise/core/events/snapshot.py
index 8a29043a5..01f86cfc2 100644
--- a/packages/core/src/repowise/core/events/snapshot.py
+++ b/packages/core/src/repowise/core/events/snapshot.py
@@ -267,15 +267,25 @@ def provided_contracts(repo_root: Path, head_commit: str | None) -> dict[str, di
store = load_contract_store(ws_root) if alias else None
if store is None or store.repo_provenance.get(alias, {}).get("head") != head_commit:
return None
+ return contract_consumer_counts(store).get(alias, {})
+
+
+def contract_consumer_counts(store: Any) -> dict[str, dict[str, dict[str, int]]]:
+ """``{repo: {file: {contract_id: consumer links}}}`` for every provided contract in *store*.
+
+ Links from other repos only: what another member changes can move them.
+ """
consumers = Counter(
- (lk.provider_file, lk.contract_id)
+ (lk.provider_repo, lk.provider_file, lk.contract_id)
for lk in store.contract_links
- if lk.provider_repo == alias and lk.consumer_repo != alias
+ if lk.consumer_repo != lk.provider_repo
)
- out: dict[str, dict[str, int]] = {}
- for c in store.rows_for_repo(alias):
+ out: dict[str, dict[str, dict[str, int]]] = {}
+ for c in store.contracts:
if c.role == "provider":
- out.setdefault(c.file_path, {})[c.contract_id] = consumers[(c.file_path, c.contract_id)]
+ out.setdefault(c.repo, {}).setdefault(c.file_path, {})[c.contract_id] = consumers[
+ (c.repo, c.file_path, c.contract_id)
+ ]
return out
diff --git a/packages/core/src/repowise/core/workspace/update/phases.py b/packages/core/src/repowise/core/workspace/update/phases.py
index 197f34531..2c587004f 100644
--- a/packages/core/src/repowise/core/workspace/update/phases.py
+++ b/packages/core/src/repowise/core/workspace/update/phases.py
@@ -63,17 +63,18 @@ async def run_cross_repo_hooks(
timings = PhaseTimingRecorder()
- # One read connection per repo serves all five phases; only the HTTP
- # extractor reads it, so nothing is opened when that is off. Deferred:
+ # One read connection per repo serves all five phases; the HTTP and code
+ # extractors read it, so nothing is opened when both are off. Deferred:
# extraction opens only the repos it re-extracts and the packages they
# import, so a run with one repo changed loads one repo's symbols.
+ contracts = ws_config.contracts
workspace_index = (
await open_workspace_index(ws_config, workspace_root, deferred=True)
- if ws_config.contracts.detect_http
+ if contracts.detect_http or contracts.detect_code_api
else WorkspaceIndex({})
)
try:
- settled = await _run_phases(
+ settled, consumers_moved = await _run_phases(
ws_config, workspace_root, changed_repos, timings, workspace_index
)
finally:
@@ -96,8 +97,10 @@ async def run_cross_repo_hooks(
stamp_analysis_config(ws_config, workspace_root)
# Each updated repo snapshotted its components before contracts were
- # extracted at its new commit; measure them now.
- for alias in changed_repos:
+ # extracted at its new commit; measure them now. An unchanged provider
+ # whose cross-repo consumers moved is measured too; no other member's
+ # store is opened.
+ for alias in dict.fromkeys([*changed_repos, *sorted(consumers_moved)]):
entry = ws_config.get_repo(alias)
if entry is not None:
await _refresh_contract_facts((workspace_root / entry.path).resolve())
@@ -167,10 +170,12 @@ async def _run_phases(
changed_repos: list[str],
timings: PhaseTimingRecorder,
workspace_index: WorkspaceIndex,
-) -> bool:
+) -> tuple[bool, set[str]]:
"""The five cross-repo phases, over an already-open workspace index.
- True when extraction and the system graph both succeeded.
+ Returns whether extraction and the system graph both succeeded, and the
+ repos whose provided contracts or their cross-repo consumer links differ
+ from the previous extraction.
"""
from ..contracts import ContractStore, load_contract_store
@@ -203,7 +208,21 @@ async def _run_phases(
_conformance_phase(ws_config, workspace_root, timings, system_graph)
_log.info("Cross-repo phase timings (seconds): %s", timings.timings)
- return extraction_ok and system_graph is not None
+ moved = _contract_counts_moved(previous_store, store) if extraction_ok else set()
+ return extraction_ok and system_graph is not None, moved
+
+
+def _contract_counts_moved(previous: ContractStore, store: ContractStore) -> set[str]:
+ """Repos whose provided contracts or consumer-link counts differ between the stores.
+
+ Exactly what each repo's component snapshot measures
+ (:func:`~repowise.core.events.snapshot.contract_consumer_counts`), so a
+ member outside this set has nothing to re-measure.
+ """
+ from ...events.snapshot import contract_consumer_counts
+
+ before, after = contract_consumer_counts(previous), contract_consumer_counts(store)
+ return {alias for alias in {*before, *after} if before.get(alias) != after.get(alias)}
async def _boundaries_phase(
diff --git a/packages/core/src/repowise/core/workspace/update/workspace_pass.py b/packages/core/src/repowise/core/workspace/update/workspace_pass.py
index 00e9b3f96..020c25ef4 100644
--- a/packages/core/src/repowise/core/workspace/update/workspace_pass.py
+++ b/packages/core/src/repowise/core/workspace/update/workspace_pass.py
@@ -90,6 +90,7 @@ async def update_workspace(
commit_depth: int = _DEFAULT_COMMIT_LIMIT,
exclude_patterns: list[str] | None = None,
include_working_tree: bool = False,
+ restamp_up_to_date: bool = True,
on_repo_start: Callable[[str], None] | None = None,
on_repo_done: Callable[[RepoUpdateResult], None] | None = None,
) -> list[RepoUpdateResult]:
@@ -114,6 +115,10 @@ async def update_workspace(
it. Off for every commit-anchored caller (hooks, webhooks, a
manual sync); ``repowise watch --workspace`` sets it, because a
repo it is watching normally has no new commits at all.
+ restamp_up_to_date: Advance each up-to-date member's DB freshness
+ stamp. The CLI turns it off: it has already restamped the members
+ whose fingerprint moved, and a restamp here would rewrite every
+ member's DB, and so its editor files, on every change.
on_repo_start: Called with alias when a repo update begins.
on_repo_done: Called with result when a repo update finishes.
@@ -126,7 +131,9 @@ async def update_workspace(
# A child repo may have been updated on its own; read its state.json first.
sync_workspace_state_from_disk(workspace_root, ws_config)
- stale_repos = await _select_stale_repos(workspace_root, entries, include_working_tree, results)
+ stale_repos = await _select_stale_repos(
+ workspace_root, entries, include_working_tree, results, restamp_up_to_date
+ )
# An edited contracts: or conformance: block is work too, with no repo stale.
if dry_run or not (
stale_repos or (run_hooks and analysis_config_changed(ws_config, workspace_root))
@@ -188,6 +195,7 @@ async def _select_stale_repos(
entries: list[RepoEntry],
include_working_tree: bool,
results: list[RepoUpdateResult],
+ restamp_up_to_date: bool = True,
) -> list[_StaleRepo]:
"""Return the stale entries; append a skipped result for every other one."""
stale_repos: list[_StaleRepo] = []
@@ -207,7 +215,8 @@ async def _select_stale_repos(
if not is_stale:
# Nothing to regenerate, but the DB freshness stamp can still be
# behind; reconcile it so the server stops reporting "index behind checkout".
- await reconcile_repo_head_commit(abs_path, current_head)
+ if restamp_up_to_date:
+ await reconcile_repo_head_commit(abs_path, current_head)
results.append(
RepoUpdateResult(
alias=entry.alias,
diff --git a/tests/unit/cli/test_workspace_member_refresh.py b/tests/unit/cli/test_workspace_member_refresh.py
index e2f538ade..9e2979a70 100644
--- a/tests/unit/cli/test_workspace_member_refresh.py
+++ b/tests/unit/cli/test_workspace_member_refresh.py
@@ -68,6 +68,23 @@ def test_a_write_to_one_store_refreshes_only_that_member(tmp_path, refreshed):
assert refreshed == ["web"]
+def test_an_empty_wal_left_by_a_read_only_reader_is_not_a_write(tmp_path, refreshed):
+ """A read-only probe of a WAL store leaves an empty ``-wal`` behind; the
+ member is unchanged. A WAL holding writes still refreshes it."""
+ ws_config = _ws(tmp_path, "api", "web")
+ _refresh(tmp_path, ws_config)
+ refreshed.clear()
+
+ for alias in ("api", "web"):
+ (tmp_path / alias / ".repowise" / "wiki.db-wal").write_bytes(b"")
+ _refresh(tmp_path, ws_config)
+ assert refreshed == []
+
+ (tmp_path / "web" / ".repowise" / "wiki.db-wal").write_bytes(b"frame")
+ _refresh(tmp_path, ws_config)
+ assert refreshed == ["web"]
+
+
def test_a_deleted_editor_file_is_written_again(tmp_path, refreshed):
ws_config = _ws(tmp_path, "api")
claude_md = tmp_path / "api" / ".claude" / "CLAUDE.md"
diff --git a/tests/unit/workspace/test_update.py b/tests/unit/workspace/test_update.py
index 05a64ac3b..b1e3f0ff5 100644
--- a/tests/unit/workspace/test_update.py
+++ b/tests/unit/workspace/test_update.py
@@ -255,6 +255,54 @@ class TestUpdateWorkspace:
assert results[0].updated is False
assert results[0].skipped_reason == "up_to_date"
+ def test_a_caller_that_restamped_leaves_up_to_date_members_untouched(
+ self, tmp_path: Path
+ ) -> None:
+ """The CLI restamps the up-to-date members whose fingerprint moved
+ before the pass; the pass must not reopen every other member's store,
+ which would rewrite each one (and its editor files) on every change.
+ The member that changed is still stamped."""
+ import asyncio
+
+ backend = _make_git_repo(tmp_path, "backend")
+ backend_head = get_head_commit(backend)
+ _write_state(backend, backend_head)
+ frontend = _make_git_repo(tmp_path, "frontend")
+ old_head = get_head_commit(frontend)
+ _write_state(frontend, old_head)
+ new_head = _add_commit(frontend, "page.txt")
+ ws_config = WorkspaceConfig(
+ repos=[
+ RepoEntry(path="backend", alias="backend", last_commit_at_index=backend_head),
+ RepoEntry(path="frontend", alias="frontend", last_commit_at_index=old_head),
+ ],
+ )
+ ws_config.save(tmp_path)
+ stamp = AsyncMock()
+
+ async def _run():
+ with (
+ patch(
+ "repowise.core.workspace.update.workspace_pass.update_single_repo_index",
+ new_callable=AsyncMock,
+ return_value=RepoUpdateResult(alias="frontend", updated=True),
+ ),
+ patch(
+ "repowise.core.workspace.update.workspace_pass.reconcile_repo_head_commit",
+ stamp,
+ ),
+ ):
+ return await update_workspace(
+ tmp_path, ws_config, run_hooks=False, restamp_up_to_date=False
+ )
+
+ results = asyncio.run(_run())
+ assert {r.alias: r.skipped_reason for r in results} == {
+ "backend": "up_to_date",
+ "frontend": None,
+ }
+ assert [c.args for c in stamp.await_args_list] == [(frontend.resolve(), new_head)]
+
def test_up_to_date_repo_reconciles_freshness_stamp(self, tmp_path: Path) -> None:
"""An up-to-date workspace sync still refreshes the repo row timestamp."""
from repowise.core.persistence import (
@@ -1076,6 +1124,7 @@ class TestCrossRepoHooks:
async def _phases(*_a, **_k):
order.append("phases")
+ return True, set()
async def _refresh(path):
order.append(path.name)
@@ -1090,6 +1139,99 @@ class TestCrossRepoHooks:
asyncio.run(run_cross_repo_hooks(ws_config, tmp_path, ["api"]))
assert order == ["phases", "api"]
+ def test_an_unchanged_provider_whose_consumers_moved_is_measured(self, tmp_path: Path) -> None:
+ """A change in one repo that adds or drops a call to another repo's
+ contract moves that provider's consumer counts: its facts refresh
+ too, and no other member's store is opened."""
+ ws_config = WorkspaceConfig(
+ repos=[
+ RepoEntry(path="web", alias="web"),
+ RepoEntry(path="api", alias="api"),
+ RepoEntry(path="mail", alias="mail"),
+ ]
+ )
+ refreshed: list[str] = []
+
+ async def _phases(*_a, **_k):
+ return True, {"api"}
+
+ async def _refresh(path):
+ refreshed.append(path.name)
+
+ import asyncio
+
+ with (
+ patch("repowise.core.workspace.update.phases._run_phases", _phases),
+ patch("repowise.core.workspace.store.refresh_workspace_store", AsyncMock()),
+ patch("repowise.core.workspace.update.phases._refresh_contract_facts", _refresh),
+ ):
+ asyncio.run(run_cross_repo_hooks(ws_config, tmp_path, ["web"]))
+ assert refreshed == ["web", "api"]
+
+ def test_consumer_counts_moved_names_only_the_affected_provider(self) -> None:
+ from repowise.core.workspace.contracts import Contract, ContractLink, ContractStore
+ from repowise.core.workspace.update.phases import _contract_counts_moved
+
+ def _provider(repo: str, cid: str) -> Contract:
+ return Contract(repo, cid, "http", "provider", "app.py", "h", 1.0)
+
+ def _link(consumer: str, provider: str, cid: str) -> ContractLink:
+ return ContractLink(
+ cid, "http", "exact", 1.0, provider, "app.py", "h", None,
+ consumer, "client.py", "c", None,
+ )
+
+ contracts = [_provider("api", "http::GET::/a"), _provider("mail", "http::POST::/m")]
+ before = ContractStore(
+ contracts=list(contracts),
+ contract_links=[_link("web", "api", "http::GET::/a"), _link("web", "mail", "http::POST::/m")],
+ )
+ # web drops its call to api; mail's consumers and every contract stay.
+ after = ContractStore(
+ contracts=list(contracts),
+ contract_links=[_link("web", "mail", "http::POST::/m")],
+ )
+ assert _contract_counts_moved(before, after) == {"api"}
+ assert _contract_counts_moved(after, after) == set()
+ # A provided contract that appears is a move of its own repo.
+ grown = ContractStore(
+ contracts=[*contracts, _provider("mail", "http::GET::/n")],
+ contract_links=list(after.contract_links),
+ )
+ assert _contract_counts_moved(after, grown) == {"mail"}
+
+ def test_detect_http_off_still_opens_the_index_for_code_contracts(
+ self, tmp_path: Path
+ ) -> None:
+ """``code`` contracts read the symbol index too, so turning HTTP off
+ must not hand extraction an empty index."""
+ from repowise.core.workspace.repo_index import WorkspaceIndex
+
+ ws_config = WorkspaceConfig(
+ repos=[RepoEntry(path="api", alias="api"), RepoEntry(path="web", alias="web")]
+ )
+ ws_config.contracts.detect_http = False
+ seen: list[WorkspaceIndex] = []
+
+ async def _phases(_cfg, _root, _changed, _timings, index):
+ seen.append(index)
+ return True, set()
+
+ opened = WorkspaceIndex({}, {"api": tmp_path / "api"})
+
+ import asyncio
+
+ with (
+ patch("repowise.core.workspace.update.phases._run_phases", _phases),
+ patch("repowise.core.workspace.store.refresh_workspace_store", AsyncMock()),
+ patch(
+ "repowise.core.workspace.repo_index.open_workspace_index",
+ AsyncMock(return_value=opened),
+ ),
+ ):
+ asyncio.run(run_cross_repo_hooks(ws_config, tmp_path, []))
+ assert seen == [opened]
+
# ---------------------------------------------------------------------------
# RepoUpdateResult
@@ -1208,7 +1350,10 @@ class TestAnalysisConfigChange:
)
assert analysis_config_changed(ws_config, tmp_path)
with (
- patch("repowise.core.workspace.update.phases._run_phases", AsyncMock()),
+ patch(
+ "repowise.core.workspace.update.phases._run_phases",
+ AsyncMock(return_value=(True, set())),
+ ),
patch("repowise.core.workspace.store.refresh_workspace_store", AsyncMock()),
):
asyncio.run(run_cross_repo_hooks(ws_config, tmp_path, []))
@@ -1226,7 +1371,7 @@ class TestAnalysisConfigChange:
with (
patch(
"repowise.core.workspace.update.phases._run_phases",
- AsyncMock(return_value=False),
+ AsyncMock(return_value=(False, set())),
),
patch("repowise.core.workspace.store.refresh_workspace_store", AsyncMock()),
):