Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(health): rebuild the derived queues whenever the findings are replaced (#2000)

Raghav Chamadiya234f34adfeed
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
index 2adb77fd9..aa7dcb905 100644
--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py
@@ -1098,13 +1098,13 @@ async def _rescore_health_from_db(
init_db,
upsert_repository,
)
- from repowise.core.persistence.crud import (
- save_coverage_files,
- save_health_findings,
- save_health_metrics,
- )
+ from repowise.core.persistence.crud import save_coverage_files
from repowise.core.persistence.models import GitMetadata
- from repowise.core.pipeline.persist import persist_graph_nodes
+ from repowise.core.pipeline.persist import (
+ persist_graph_nodes,
+ save_full_health_report,
+ )
+ from repowise.core.workspace.update import get_head_commit
url = get_db_url_for_repo(repo_path)
engine = create_engine(url)
@@ -1169,8 +1169,14 @@ async def _rescore_health_from_db(
f"[yellow]{len(report.findings)} findings[/yellow]"
)
- await save_health_metrics(session, repo_id, report.metrics or [])
- await save_health_findings(session, repo_id, list(report.findings or []))
+ # Same writer the index's analysis phase uses, so a re-score leaves
+ # the derived queues agreeing with the findings it just replaced
+ # rather than describing the set it deleted. Read HEAD off disk:
+ # this pass just scored the working tree, and the stored column is
+ # written by a different step whose ordering is not guaranteed.
+ await save_full_health_report(
+ session, repo_id, report, analyzed_commit=get_head_commit(Path(repo_path))
+ )
if coverage_files:
await save_coverage_files(
session,
diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.py
index 71ad58180..5816bf8ed 100644
--- a/packages/core/src/repowise/core/pipeline/incremental.py
+++ b/packages/core/src/repowise/core/pipeline/incremental.py
@@ -1654,6 +1654,7 @@ async def persist_incremental_index(
except Exception as exc:
_skip("Decision purge", exc)
+ pruned = 0
# Drop file-scoped rows for files that have actually been deleted.
# Without this an incremental update tombstones the deleted file's
# page and leaves everything else: graph nodes, edges, metrics,
@@ -1709,6 +1710,34 @@ async def persist_incremental_index(
except Exception as exc:
_skip("Deleted-file prune", exc)
+ # The prune deletes the health findings of files that are gone, and
+ # the performance and refactoring queues are materialized from those
+ # findings. Without this they keep serving a cause whose evidence was
+ # removed a moment ago, on a file the store no longer claims exists.
+ # After the prune for the same reason the prune runs last: only now
+ # is the surviving finding set final.
+ if pruned:
+ try:
+ from repowise.core.persistence.crud import (
+ finalize_performance_opportunities,
+ finalize_refactoring_opportunities,
+ )
+
+ analyzed_commit = await _analyzed_commit(session, repo_id)
+ await finalize_performance_opportunities(
+ session,
+ repo_id,
+ analyzed_commit=analyzed_commit,
+ plan_policy=getattr(
+ partial_health_report, "performance_plan_policy", None
+ ),
+ )
+ await finalize_refactoring_opportunities(
+ session, repo_id, analyzed_commit=analyzed_commit
+ )
+ except Exception as exc:
+ _skip("Queue rebuild after prune", exc)
+
# After the session closes: on SQLite the full-text index shares the
# database file, so writing to it while the session holds a write lock
# raises "database is locked".
diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.py
index a09fac56a..97a131c98 100644
--- a/packages/core/src/repowise/core/pipeline/persist.py
+++ b/packages/core/src/repowise/core/pipeline/persist.py
@@ -1501,6 +1501,84 @@ async def persist_git(result: Any, session: Any, repo_id: str) -> None:
)
+async def save_full_health_report(
+ session: Any,
+ repo_id: str,
+ health_report: Any,
+ *,
+ analyzed_commit: str | None,
+) -> None:
+ """Full-replace a repository's health rows and everything derived from them.
+
+ Both callers that re-score every file land here: the analysis phase of an
+ index, and the update command's periodic/config-triggered re-score. The
+ metrics, the findings, the non-performance refactoring suggestions and the
+ two materialized read models move together, so a re-score cannot leave the
+ queues describing findings that were deleted out from under them.
+
+ ``analyzed_commit`` stamps the metric rows and the read models with the
+ commit these scores were computed against; ``None`` reads as "not
+ recorded", which is honest, while a wrong sha would not be.
+ """
+ from repowise.core.persistence.crud import (
+ finalize_performance_opportunities,
+ finalize_refactoring_opportunities,
+ save_health_findings,
+ save_health_metrics,
+ save_refactoring_suggestions,
+ )
+
+ hr = health_report
+ metrics = list(getattr(hr, "metrics", None) or [])
+ findings = list(getattr(hr, "findings", None) or [])
+ if not metrics and not findings:
+ # A pass that scored nothing is a broken pass, not a clean repository.
+ # Every write below replaces the repository's rows wholesale, so
+ # honouring an empty report would delete every metric, resolve every
+ # open plan a person has triaged, and empty both queues on the strength
+ # of a parse failure or an exclude pattern that matched everything.
+ return
+ await save_health_metrics(
+ session, repo_id, metrics, analyzed_commit=analyzed_commit
+ )
+ # One savepoint over the findings and everything derived from them, so a
+ # rebuild that failed halfway cannot leave the queue describing findings
+ # that were never stored. It runs on an empty finding set too: the queue
+ # has to be reconciled to nothing just the same.
+ async with session.begin_nested():
+ if findings:
+ await save_health_findings(session, repo_id, findings)
+ # Structured refactoring suggestions (Extract Class, ...). Repo-wide
+ # reconciliation: an unchanged plan keeps its id and its triage state,
+ # and one nobody detects any more resolves rather than disappearing.
+ # Performance plans are excluded: the finalizer below owns them, so
+ # they are generated once, from the merged stored findings, by the same
+ # writer on every path.
+ await save_refactoring_suggestions(
+ session,
+ repo_id,
+ [
+ suggestion
+ for suggestion in (getattr(hr, "refactoring_suggestions", None) or [])
+ if getattr(suggestion, "refactoring_type", None) != "performance_fix"
+ ],
+ )
+ # Group the stored performance findings into the materialized causal
+ # read model, then reconcile lifecycle and plans.
+ await finalize_performance_opportunities(
+ session,
+ repo_id,
+ analyzed_commit=analyzed_commit,
+ plan_policy=getattr(hr, "performance_plan_policy", None),
+ )
+ # Fold the plans just written into per-file opportunities. Last in the
+ # savepoint because it composes over the stored rows, so it has to see
+ # the reconciliation above rather than the detector output.
+ await finalize_refactoring_opportunities(
+ session, repo_id, analyzed_commit=analyzed_commit
+ )
+
+
async def persist_analysis(result: Any, session: Any, repo_id: str) -> None:
"""Persist analysis-phase outputs: dead code, health, decisions, governance.
@@ -1512,15 +1590,10 @@ async def persist_analysis(result: Any, session: Any, repo_id: str) -> None:
from repowise.core.analysis.health.trends import snapshot_file_maps
from repowise.core.persistence.crud import (
bulk_upsert_decisions,
- finalize_performance_opportunities,
- finalize_refactoring_opportunities,
recompute_decision_staleness,
save_coverage_files,
save_dead_code_findings,
- save_health_findings,
- save_health_metrics,
save_health_snapshot,
- save_refactoring_suggestions,
upsert_git_function_blame_bulk,
)
@@ -1535,43 +1608,7 @@ async def persist_analysis(result: Any, session: Any, repo_id: str) -> None:
# metric rows, so a reader can tell how far the health pass lags the
# index instead of assuming the two moved together.
head_sha = getattr(result, "head_commit", None) or getattr(result, "commit_sha", None)
- await save_health_metrics(session, repo_id, hr.metrics or [], analyzed_commit=head_sha)
- # One savepoint over the findings and everything derived from them, so
- # a rebuild that failed halfway cannot leave the queue describing
- # findings that were never stored. It runs on an empty finding set too:
- # the queue has to be reconciled to nothing just the same.
- async with session.begin_nested():
- if hr.findings:
- await save_health_findings(session, repo_id, hr.findings)
- # Structured refactoring suggestions (Extract Class, ...). Repo-wide
- # reconciliation: an unchanged plan keeps its id and its triage
- # state, and one nobody detects any more resolves rather than
- # disappearing. Performance plans are excluded: the finalizer below
- # owns them, so they are generated once, from the merged stored
- # findings, by the same writer on the full and the incremental path.
- await save_refactoring_suggestions(
- session,
- repo_id,
- [
- suggestion
- for suggestion in (getattr(hr, "refactoring_suggestions", None) or [])
- if getattr(suggestion, "refactoring_type", None) != "performance_fix"
- ],
- )
- # Group the stored performance findings into the materialized causal
- # read model, then reconcile lifecycle and plans.
- await finalize_performance_opportunities(
- session,
- repo_id,
- analyzed_commit=head_sha,
- plan_policy=getattr(hr, "performance_plan_policy", None),
- )
- # Fold the plans just written into per-file opportunities. Last in
- # the savepoint because it composes over the stored rows, so it has
- # to see the reconciliation above rather than the detector output.
- await finalize_refactoring_opportunities(
- session, repo_id, analyzed_commit=head_sha
- )
+ await save_full_health_report(session, repo_id, hr, analyzed_commit=head_sha)
# Resolved coverage rows, when a report was ingested this run.
coverage_files = getattr(hr, "coverage_files", None)
if coverage_files:
diff --git a/tests/unit/cli/test_rescore_rebuilds_read_models.py b/tests/unit/cli/test_rescore_rebuilds_read_models.py
new file mode 100644
index 000000000..10b16dc3f
--- /dev/null
+++ b/tests/unit/cli/test_rescore_rebuilds_read_models.py
@@ -0,0 +1,183 @@
+"""What the update command's full health re-score must leave behind.
+
+The re-score deletes and re-inserts every finding in the repository. The
+performance and refactoring queues are materialized *from* those findings, so
+a re-score that does not rebuild them leaves them serving rows derived from a
+finding set that no longer exists. This drives the real re-score entry point
+and pins that it reconciles them.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+from types import SimpleNamespace
+
+import networkx as nx
+import pytest
+from sqlalchemy import select
+
+from repowise.cli.commands.update_cmd.persistence import _rescore_health_from_db
+from repowise.cli.helpers import get_db_url_for_repo
+from repowise.core.persistence import (
+ create_engine,
+ create_session_factory,
+ get_session,
+ init_db,
+ upsert_repository,
+)
+from repowise.core.persistence.models import (
+ HealthFileMetric,
+ PerformanceOpportunity,
+ PerformanceSummary,
+)
+
+
+class _EmptyGraphBuilder:
+ """The re-score reuses the update's builder; none of this run needs one."""
+
+ def graph(self):
+ return nx.DiGraph()
+
+ def pagerank(self):
+ return {}
+
+ betweenness_centrality = pagerank
+ symbol_pagerank = pagerank
+ symbol_betweenness_centrality = pagerank
+ community_detection = pagerank
+ symbol_communities = pagerank
+
+ def community_info(self):
+ return {}
+
+
+_SCORED = "src/app.py"
+_SOURCE = "def run(items):\n return [i for i in items if i]\n"
+
+
+def _parsed_file(repo_path: Path) -> SimpleNamespace:
+ """The re-score scores the caller's parsed files; one real one is enough."""
+ return SimpleNamespace(
+ file_info=SimpleNamespace(
+ path=_SCORED,
+ language="python",
+ abs_path=str(repo_path / _SCORED),
+ is_test=False,
+ ),
+ symbols=[],
+ )
+
+
+def git_repo(tmp_path: Path) -> Path:
+ (tmp_path / "README.md").write_text("parity\n", encoding="utf-8")
+ (tmp_path / "src").mkdir()
+ (tmp_path / _SCORED).write_text(_SOURCE, encoding="utf-8")
+ for args in (
+ ["init", "-q"],
+ ["config", "user.email", "[email protected]"],
+ ["config", "user.name", "t"],
+ ["add", "."],
+ ["commit", "-q", "-m", "init"],
+ ):
+ subprocess.run(["git", *args], cwd=tmp_path, check=True, capture_output=True)
+ return tmp_path
+
+
+async def _seed_orphan_opportunity(repo_path: Path) -> tuple[str, str]:
+ """An open queue row with no finding behind it, as a re-score would leave."""
+ engine = create_engine(get_db_url_for_repo(repo_path))
+ await init_db(engine)
+ factory = create_session_factory(engine)
+ async with get_session(factory) as session:
+ repo = await upsert_repository(
+ session, name=repo_path.name, local_path=str(repo_path)
+ )
+ session.add(
+ PerformanceOpportunity(
+ id="orphan-row",
+ repository_id=repo.id,
+ opportunity_id="perf2_orphaned_cause",
+ performance_model_version=2,
+ status="open",
+ rank_position=0,
+ rank_score=1.0,
+ file_path="src/app.py",
+ observations_total=1,
+ affected_call_sites_total=1,
+ affected_files_total=1,
+ analyzed_commit="0" * 40,
+ )
+ )
+ await session.commit()
+ repo_id = repo.id
+ await engine.dispose()
+ return repo_id, "perf2_orphaned_cause"
+
+
+async def _read_back(repo_path: Path, repo_id: str):
+ engine = create_engine(get_db_url_for_repo(repo_path))
+ factory = create_session_factory(engine)
+ async with get_session(factory) as session:
+ rows = (
+ (
+ await session.execute(
+ select(PerformanceOpportunity).where(
+ PerformanceOpportunity.repository_id == repo_id
+ )
+ )
+ )
+ .scalars()
+ .all()
+ )
+ statuses = {row.opportunity_id: row.status for row in rows}
+ summary = await session.get(PerformanceSummary, repo_id)
+ summary_commit = summary.analyzed_commit if summary else None
+ metrics = {
+ row.file_path: row.analyzed_commit
+ for row in (
+ (
+ await session.execute(
+ select(HealthFileMetric).where(
+ HealthFileMetric.repository_id == repo_id
+ )
+ )
+ )
+ .scalars()
+ .all()
+ )
+ }
+ await engine.dispose()
+ return statuses, summary_commit, metrics
+
+
+class TestRescoreRebuildsTheReadModels:
+ async def test_a_cause_with_no_surviving_finding_is_resolved(self, git_repo: Path):
+ repo_id, opportunity_id = await _seed_orphan_opportunity(git_repo)
+
+ await _rescore_health_from_db(
+ git_repo, _EmptyGraphBuilder(), [_parsed_file(git_repo)], []
+ )
+
+ statuses, _, _ = await _read_back(git_repo, repo_id)
+ assert statuses[opportunity_id] == "resolved"
+
+ async def test_it_stamps_the_commit_it_scored_against(self, git_repo: Path):
+ repo_id, _ = await _seed_orphan_opportunity(git_repo)
+ head = subprocess.run(
+ ["git", "rev-parse", "HEAD"],
+ cwd=git_repo,
+ check=True,
+ capture_output=True,
+ text=True,
+ ).stdout.strip()
+
+ await _rescore_health_from_db(
+ git_repo, _EmptyGraphBuilder(), [_parsed_file(git_repo)], []
+ )
+
+ _, summary_commit, metrics = await _read_back(git_repo, repo_id)
+ assert summary_commit == head
+ # The column the index-freshness signal is read from.
+ assert metrics[_SCORED] == head
diff --git a/tests/unit/persistence/test_full_health_report_writer.py b/tests/unit/persistence/test_full_health_report_writer.py
new file mode 100644
index 000000000..9eb553b2b
--- /dev/null
+++ b/tests/unit/persistence/test_full_health_report_writer.py
@@ -0,0 +1,208 @@
+"""The writer every full re-score goes through, and what it must rebuild.
+
+A re-score deletes and re-inserts every finding in the repository. The
+materialized queues are derived from those findings, so unless they are
+rebuilt in the same transaction they are left describing rows that no longer
+exist. These pin that they move together, and that the metric rows carry the
+commit the scores were computed against.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from sqlalchemy import select
+
+from repowise.core.analysis.health.models import HealthFindingData, Severity
+from repowise.core.analysis.health.perf.opportunities import link_performance_findings
+from repowise.core.persistence.models import (
+ HealthFileMetric,
+ PerformanceOpportunity,
+)
+from repowise.core.pipeline.persist import save_full_health_report
+from tests.unit.persistence.helpers import insert_repo
+
+
+def _perf_finding(
+ line: int, *, path: str = "src/app.py", sink: str = "src/db.py::fetch"
+) -> HealthFindingData:
+ return HealthFindingData(
+ biomarker_type="serial_await_in_loop",
+ severity=Severity.MEDIUM,
+ file_path=path,
+ function_name="run",
+ line_start=line,
+ line_end=line,
+ details={
+ "boundary_kind": "db",
+ "cross_function": True,
+ "path": [f"{path}::run", "src/shared.py::load", sink],
+ "resolution_basis": "call-site",
+ "dataflow_verified": True,
+ },
+ health_impact=0.0,
+ reason="Awaited database work repeats for every loop iteration.",
+ dimension="performance",
+ )
+
+
+def _report(findings: list, *, metrics: list | None = None) -> SimpleNamespace:
+ link_performance_findings(findings)
+ return SimpleNamespace(
+ metrics=metrics if metrics is not None else [_metric()],
+ findings=findings,
+ refactoring_suggestions=[],
+ performance_plan_policy=None,
+ )
+
+
+def _metric(path: str = "src/app.py") -> dict:
+ return {
+ "file_path": path,
+ "score": 5.0,
+ "max_ccn": 3,
+ "max_nesting": 1,
+ "nloc": 40,
+ "duplication_pct": 0.0,
+ "has_test_file": False,
+ "line_coverage_pct": None,
+ "branch_coverage_pct": None,
+ "module": "src",
+ }
+
+
+async def _open_perf_ids(session, repo_id: str) -> set[str]:
+ rows = (
+ (
+ await session.execute(
+ select(PerformanceOpportunity).where(
+ PerformanceOpportunity.repository_id == repo_id,
+ PerformanceOpportunity.status == "open",
+ )
+ )
+ )
+ .scalars()
+ .all()
+ )
+ return {row.opportunity_id for row in rows}
+
+
+class TestSaveFullHealthReport:
+ async def test_it_materializes_the_queue_from_the_findings_it_wrote(self, async_session):
+ repo_id = (await insert_repo(async_session)).id
+ await save_full_health_report(
+ async_session, repo_id, _report([_perf_finding(10)]), analyzed_commit="a" * 40
+ )
+ assert len(await _open_perf_ids(async_session, repo_id)) == 1
+
+ async def test_a_cause_the_rescore_no_longer_sees_is_resolved(self, async_session):
+ """The defect this exists for: a re-score replaces every finding.
+
+ Without the rebuild in the same transaction the queue keeps serving the
+ opportunity, pointing at a finding row the re-score deleted.
+ """
+ repo_id = (await insert_repo(async_session)).id
+ await save_full_health_report(
+ async_session, repo_id, _report([_perf_finding(10)]), analyzed_commit="a" * 40
+ )
+ before = await _open_perf_ids(async_session, repo_id)
+ assert len(before) == 1
+
+ # The repeated work now lands on a different sink, which is a
+ # different cause. The re-score deleted the finding the first one was
+ # built from, so the queue must stop serving it.
+ await save_full_health_report(
+ async_session,
+ repo_id,
+ _report(
+ [_perf_finding(10, path="src/other.py", sink="src/cache.py::lookup")],
+ metrics=[_metric("src/other.py")],
+ ),
+ analyzed_commit="b" * 40,
+ )
+ after = await _open_perf_ids(async_session, repo_id)
+ assert len(after) == 1
+ assert after != before
+
+ stored = {row.opportunity_id: row.status for row in await session_rows(async_session, repo_id)}
+ assert stored[next(iter(before))] == "resolved"
+ assert stored[next(iter(after))] == "open"
+
+ async def test_it_stamps_the_metric_rows_with_the_analyzed_commit(self, async_session):
+ repo_id = (await insert_repo(async_session)).id
+ await save_full_health_report(
+ async_session, repo_id, _report([]), analyzed_commit="c" * 40
+ )
+ rows = (
+ (
+ await async_session.execute(
+ select(HealthFileMetric).where(HealthFileMetric.repository_id == repo_id)
+ )
+ )
+ .scalars()
+ .all()
+ )
+ assert rows
+ assert {row.analyzed_commit for row in rows} == {"c" * 40}
+
+ async def test_a_pass_that_scored_nothing_writes_nothing(self, async_session):
+ """A parse failure must not read as "this repository became clean".
+
+ Every write here replaces the repository's rows wholesale, so honouring
+ an empty report would delete the metrics, resolve plans a person had
+ triaged, and empty both queues on the strength of a broken run.
+ """
+ repo_id = (await insert_repo(async_session)).id
+ await save_full_health_report(
+ async_session, repo_id, _report([_perf_finding(10)]), analyzed_commit="a" * 40
+ )
+ before = await _open_perf_ids(async_session, repo_id)
+ assert before
+
+ empty = SimpleNamespace(
+ metrics=[], findings=[], refactoring_suggestions=[], performance_plan_policy=None
+ )
+ await save_full_health_report(
+ async_session, repo_id, empty, analyzed_commit="b" * 40
+ )
+
+ assert await _open_perf_ids(async_session, repo_id) == before
+ metrics = (
+ (
+ await async_session.execute(
+ select(HealthFileMetric).where(HealthFileMetric.repository_id == repo_id)
+ )
+ )
+ .scalars()
+ .all()
+ )
+ assert metrics, "an empty report must not delete the stored metrics"
+
+ async def test_an_unstamped_pass_leaves_the_column_null(self, async_session):
+ """``None`` reads as "not recorded"; it must not borrow a stale sha."""
+ repo_id = (await insert_repo(async_session)).id
+ await save_full_health_report(async_session, repo_id, _report([]), analyzed_commit=None)
+ rows = (
+ (
+ await async_session.execute(
+ select(HealthFileMetric).where(HealthFileMetric.repository_id == repo_id)
+ )
+ )
+ .scalars()
+ .all()
+ )
+ assert {row.analyzed_commit for row in rows} == {None}
+
+
+async def session_rows(session, repo_id: str):
+ return (
+ (
+ await session.execute(
+ select(PerformanceOpportunity).where(
+ PerformanceOpportunity.repository_id == repo_id
+ )
+ )
+ )
+ .scalars()
+ .all()
+ )