Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(health): rebuild the derived queues whenever the findings are replaced (#2000)
Raghav Chamadiya234f34adfeed
diff --git a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.pyindex 2adb77fd9..aa7dcb905 100644--- a/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py+++ b/packages/cli/src/repowise/cli/commands/update_cmd/persistence.py@@ -1098,13 +1098,13 @@ async def _rescore_health_from_db(init_db,upsert_repository,)- from repowise.core.persistence.crud import (- save_coverage_files,- save_health_findings,- save_health_metrics,- )+ from repowise.core.persistence.crud import save_coverage_filesfrom repowise.core.persistence.models import GitMetadata- from repowise.core.pipeline.persist import persist_graph_nodes+ from repowise.core.pipeline.persist import (+ persist_graph_nodes,+ save_full_health_report,+ )+ from repowise.core.workspace.update import get_head_commiturl = get_db_url_for_repo(repo_path)engine = create_engine(url)@@ -1169,8 +1169,14 @@ async def _rescore_health_from_db(f"[yellow]{len(report.findings)} findings[/yellow]")- await save_health_metrics(session, repo_id, report.metrics or [])- await save_health_findings(session, repo_id, list(report.findings or []))+ # Same writer the index's analysis phase uses, so a re-score leaves+ # the derived queues agreeing with the findings it just replaced+ # rather than describing the set it deleted. Read HEAD off disk:+ # this pass just scored the working tree, and the stored column is+ # written by a different step whose ordering is not guaranteed.+ await save_full_health_report(+ session, repo_id, report, analyzed_commit=get_head_commit(Path(repo_path))+ )if coverage_files:await save_coverage_files(session,diff --git a/packages/core/src/repowise/core/pipeline/incremental.py b/packages/core/src/repowise/core/pipeline/incremental.pyindex 71ad58180..5816bf8ed 100644--- a/packages/core/src/repowise/core/pipeline/incremental.py+++ b/packages/core/src/repowise/core/pipeline/incremental.py@@ -1654,6 +1654,7 @@ async def persist_incremental_index(except Exception as exc:_skip("Decision purge", exc)+ pruned = 0# Drop file-scoped rows for files that have actually been deleted.# Without this an incremental update tombstones the deleted file's# page and leaves everything else: graph nodes, edges, metrics,@@ -1709,6 +1710,34 @@ async def persist_incremental_index(except Exception as exc:_skip("Deleted-file prune", exc)+ # The prune deletes the health findings of files that are gone, and+ # the performance and refactoring queues are materialized from those+ # findings. Without this they keep serving a cause whose evidence was+ # removed a moment ago, on a file the store no longer claims exists.+ # After the prune for the same reason the prune runs last: only now+ # is the surviving finding set final.+ if pruned:+ try:+ from repowise.core.persistence.crud import (+ finalize_performance_opportunities,+ finalize_refactoring_opportunities,+ )++ analyzed_commit = await _analyzed_commit(session, repo_id)+ await finalize_performance_opportunities(+ session,+ repo_id,+ analyzed_commit=analyzed_commit,+ plan_policy=getattr(+ partial_health_report, "performance_plan_policy", None+ ),+ )+ await finalize_refactoring_opportunities(+ session, repo_id, analyzed_commit=analyzed_commit+ )+ except Exception as exc:+ _skip("Queue rebuild after prune", exc)+# After the session closes: on SQLite the full-text index shares the# database file, so writing to it while the session holds a write lock# raises "database is locked".diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.pyindex a09fac56a..97a131c98 100644--- a/packages/core/src/repowise/core/pipeline/persist.py+++ b/packages/core/src/repowise/core/pipeline/persist.py@@ -1501,6 +1501,84 @@ async def persist_git(result: Any, session: Any, repo_id: str) -> None:)+async def save_full_health_report(+ session: Any,+ repo_id: str,+ health_report: Any,+ *,+ analyzed_commit: str | None,+) -> None:+ """Full-replace a repository's health rows and everything derived from them.++ Both callers that re-score every file land here: the analysis phase of an+ index, and the update command's periodic/config-triggered re-score. The+ metrics, the findings, the non-performance refactoring suggestions and the+ two materialized read models move together, so a re-score cannot leave the+ queues describing findings that were deleted out from under them.++ ``analyzed_commit`` stamps the metric rows and the read models with the+ commit these scores were computed against; ``None`` reads as "not+ recorded", which is honest, while a wrong sha would not be.+ """+ from repowise.core.persistence.crud import (+ finalize_performance_opportunities,+ finalize_refactoring_opportunities,+ save_health_findings,+ save_health_metrics,+ save_refactoring_suggestions,+ )++ hr = health_report+ metrics = list(getattr(hr, "metrics", None) or [])+ findings = list(getattr(hr, "findings", None) or [])+ if not metrics and not findings:+ # A pass that scored nothing is a broken pass, not a clean repository.+ # Every write below replaces the repository's rows wholesale, so+ # honouring an empty report would delete every metric, resolve every+ # open plan a person has triaged, and empty both queues on the strength+ # of a parse failure or an exclude pattern that matched everything.+ return+ await save_health_metrics(+ session, repo_id, metrics, analyzed_commit=analyzed_commit+ )+ # One savepoint over the findings and everything derived from them, so a+ # rebuild that failed halfway cannot leave the queue describing findings+ # that were never stored. It runs on an empty finding set too: the queue+ # has to be reconciled to nothing just the same.+ async with session.begin_nested():+ if findings:+ await save_health_findings(session, repo_id, findings)+ # Structured refactoring suggestions (Extract Class, ...). Repo-wide+ # reconciliation: an unchanged plan keeps its id and its triage state,+ # and one nobody detects any more resolves rather than disappearing.+ # Performance plans are excluded: the finalizer below owns them, so+ # they are generated once, from the merged stored findings, by the same+ # writer on every path.+ await save_refactoring_suggestions(+ session,+ repo_id,+ [+ suggestion+ for suggestion in (getattr(hr, "refactoring_suggestions", None) or [])+ if getattr(suggestion, "refactoring_type", None) != "performance_fix"+ ],+ )+ # Group the stored performance findings into the materialized causal+ # read model, then reconcile lifecycle and plans.+ await finalize_performance_opportunities(+ session,+ repo_id,+ analyzed_commit=analyzed_commit,+ plan_policy=getattr(hr, "performance_plan_policy", None),+ )+ # Fold the plans just written into per-file opportunities. Last in the+ # savepoint because it composes over the stored rows, so it has to see+ # the reconciliation above rather than the detector output.+ await finalize_refactoring_opportunities(+ session, repo_id, analyzed_commit=analyzed_commit+ )++async def persist_analysis(result: Any, session: Any, repo_id: str) -> None:"""Persist analysis-phase outputs: dead code, health, decisions, governance.@@ -1512,15 +1590,10 @@ async def persist_analysis(result: Any, session: Any, repo_id: str) -> None:from repowise.core.analysis.health.trends import snapshot_file_mapsfrom repowise.core.persistence.crud import (bulk_upsert_decisions,- finalize_performance_opportunities,- finalize_refactoring_opportunities,recompute_decision_staleness,save_coverage_files,save_dead_code_findings,- save_health_findings,- save_health_metrics,save_health_snapshot,- save_refactoring_suggestions,upsert_git_function_blame_bulk,)@@ -1535,43 +1608,7 @@ async def persist_analysis(result: Any, session: Any, repo_id: str) -> None:# metric rows, so a reader can tell how far the health pass lags the# index instead of assuming the two moved together.head_sha = getattr(result, "head_commit", None) or getattr(result, "commit_sha", None)- await save_health_metrics(session, repo_id, hr.metrics or [], analyzed_commit=head_sha)- # One savepoint over the findings and everything derived from them, so- # a rebuild that failed halfway cannot leave the queue describing- # findings that were never stored. It runs on an empty finding set too:- # the queue has to be reconciled to nothing just the same.- async with session.begin_nested():- if hr.findings:- await save_health_findings(session, repo_id, hr.findings)- # Structured refactoring suggestions (Extract Class, ...). Repo-wide- # reconciliation: an unchanged plan keeps its id and its triage- # state, and one nobody detects any more resolves rather than- # disappearing. Performance plans are excluded: the finalizer below- # owns them, so they are generated once, from the merged stored- # findings, by the same writer on the full and the incremental path.- await save_refactoring_suggestions(- session,- repo_id,- [- suggestion- for suggestion in (getattr(hr, "refactoring_suggestions", None) or [])- if getattr(suggestion, "refactoring_type", None) != "performance_fix"- ],- )- # Group the stored performance findings into the materialized causal- # read model, then reconcile lifecycle and plans.- await finalize_performance_opportunities(- session,- repo_id,- analyzed_commit=head_sha,- plan_policy=getattr(hr, "performance_plan_policy", None),- )- # Fold the plans just written into per-file opportunities. Last in- # the savepoint because it composes over the stored rows, so it has- # to see the reconciliation above rather than the detector output.- await finalize_refactoring_opportunities(- session, repo_id, analyzed_commit=head_sha- )+ await save_full_health_report(session, repo_id, hr, analyzed_commit=head_sha)# Resolved coverage rows, when a report was ingested this run.coverage_files = getattr(hr, "coverage_files", None)if coverage_files:diff --git a/tests/unit/cli/test_rescore_rebuilds_read_models.py b/tests/unit/cli/test_rescore_rebuilds_read_models.pynew file mode 100644index 000000000..10b16dc3f--- /dev/null+++ b/tests/unit/cli/test_rescore_rebuilds_read_models.py@@ -0,0 +1,183 @@+"""What the update command's full health re-score must leave behind.++The re-score deletes and re-inserts every finding in the repository. The+performance and refactoring queues are materialized *from* those findings, so+a re-score that does not rebuild them leaves them serving rows derived from a+finding set that no longer exists. This drives the real re-score entry point+and pins that it reconciles them.+"""++from __future__ import annotations++import subprocess+from pathlib import Path+from types import SimpleNamespace++import networkx as nx+import pytest+from sqlalchemy import select++from repowise.cli.commands.update_cmd.persistence import _rescore_health_from_db+from repowise.cli.helpers import get_db_url_for_repo+from repowise.core.persistence import (+ create_engine,+ create_session_factory,+ get_session,+ init_db,+ upsert_repository,+)+from repowise.core.persistence.models import (+ HealthFileMetric,+ PerformanceOpportunity,+ PerformanceSummary,+)+++class _EmptyGraphBuilder:+ """The re-score reuses the update's builder; none of this run needs one."""++ def graph(self):+ return nx.DiGraph()++ def pagerank(self):+ return {}++ betweenness_centrality = pagerank+ symbol_pagerank = pagerank+ symbol_betweenness_centrality = pagerank+ community_detection = pagerank+ symbol_communities = pagerank++ def community_info(self):+ return {}+++_SCORED = "src/app.py"+_SOURCE = "def run(items):\n return [i for i in items if i]\n"+++def _parsed_file(repo_path: Path) -> SimpleNamespace:+ """The re-score scores the caller's parsed files; one real one is enough."""+ return SimpleNamespace(+ file_info=SimpleNamespace(+ path=_SCORED,+ language="python",+ abs_path=str(repo_path / _SCORED),+ is_test=False,+ ),+ symbols=[],+ )+++def git_repo(tmp_path: Path) -> Path:+ (tmp_path / "README.md").write_text("parity\n", encoding="utf-8")+ (tmp_path / "src").mkdir()+ (tmp_path / _SCORED).write_text(_SOURCE, encoding="utf-8")+ for args in (+ ["init", "-q"],+ ["config", "user.email", "[email protected]"],+ ["config", "user.name", "t"],+ ["add", "."],+ ["commit", "-q", "-m", "init"],+ ):+ subprocess.run(["git", *args], cwd=tmp_path, check=True, capture_output=True)+ return tmp_path+++async def _seed_orphan_opportunity(repo_path: Path) -> tuple[str, str]:+ """An open queue row with no finding behind it, as a re-score would leave."""+ engine = create_engine(get_db_url_for_repo(repo_path))+ await init_db(engine)+ factory = create_session_factory(engine)+ async with get_session(factory) as session:+ repo = await upsert_repository(+ session, name=repo_path.name, local_path=str(repo_path)+ )+ session.add(+ PerformanceOpportunity(+ id="orphan-row",+ repository_id=repo.id,+ opportunity_id="perf2_orphaned_cause",+ performance_model_version=2,+ status="open",+ rank_position=0,+ rank_score=1.0,+ file_path="src/app.py",+ observations_total=1,+ affected_call_sites_total=1,+ affected_files_total=1,+ analyzed_commit="0" * 40,+ )+ )+ await session.commit()+ repo_id = repo.id+ await engine.dispose()+ return repo_id, "perf2_orphaned_cause"+++async def _read_back(repo_path: Path, repo_id: str):+ engine = create_engine(get_db_url_for_repo(repo_path))+ factory = create_session_factory(engine)+ async with get_session(factory) as session:+ rows = (+ (+ await session.execute(+ select(PerformanceOpportunity).where(+ PerformanceOpportunity.repository_id == repo_id+ )+ )+ )+ .scalars()+ .all()+ )+ statuses = {row.opportunity_id: row.status for row in rows}+ summary = await session.get(PerformanceSummary, repo_id)+ summary_commit = summary.analyzed_commit if summary else None+ metrics = {+ row.file_path: row.analyzed_commit+ for row in (+ (+ await session.execute(+ select(HealthFileMetric).where(+ HealthFileMetric.repository_id == repo_id+ )+ )+ )+ .scalars()+ .all()+ )+ }+ await engine.dispose()+ return statuses, summary_commit, metrics+++class TestRescoreRebuildsTheReadModels:+ async def test_a_cause_with_no_surviving_finding_is_resolved(self, git_repo: Path):+ repo_id, opportunity_id = await _seed_orphan_opportunity(git_repo)++ await _rescore_health_from_db(+ git_repo, _EmptyGraphBuilder(), [_parsed_file(git_repo)], []+ )++ statuses, _, _ = await _read_back(git_repo, repo_id)+ assert statuses[opportunity_id] == "resolved"++ async def test_it_stamps_the_commit_it_scored_against(self, git_repo: Path):+ repo_id, _ = await _seed_orphan_opportunity(git_repo)+ head = subprocess.run(+ ["git", "rev-parse", "HEAD"],+ cwd=git_repo,+ check=True,+ capture_output=True,+ text=True,+ ).stdout.strip()++ await _rescore_health_from_db(+ git_repo, _EmptyGraphBuilder(), [_parsed_file(git_repo)], []+ )++ _, summary_commit, metrics = await _read_back(git_repo, repo_id)+ assert summary_commit == head+ # The column the index-freshness signal is read from.+ assert metrics[_SCORED] == headdiff --git a/tests/unit/persistence/test_full_health_report_writer.py b/tests/unit/persistence/test_full_health_report_writer.pynew file mode 100644index 000000000..9eb553b2b--- /dev/null+++ b/tests/unit/persistence/test_full_health_report_writer.py@@ -0,0 +1,208 @@+"""The writer every full re-score goes through, and what it must rebuild.++A re-score deletes and re-inserts every finding in the repository. The+materialized queues are derived from those findings, so unless they are+rebuilt in the same transaction they are left describing rows that no longer+exist. These pin that they move together, and that the metric rows carry the+commit the scores were computed against.+"""++from __future__ import annotations++from types import SimpleNamespace++from sqlalchemy import select++from repowise.core.analysis.health.models import HealthFindingData, Severity+from repowise.core.analysis.health.perf.opportunities import link_performance_findings+from repowise.core.persistence.models import (+ HealthFileMetric,+ PerformanceOpportunity,+)+from repowise.core.pipeline.persist import save_full_health_report+from tests.unit.persistence.helpers import insert_repo+++def _perf_finding(+ line: int, *, path: str = "src/app.py", sink: str = "src/db.py::fetch"+) -> HealthFindingData:+ return HealthFindingData(+ biomarker_type="serial_await_in_loop",+ severity=Severity.MEDIUM,+ file_path=path,+ function_name="run",+ line_start=line,+ line_end=line,+ details={+ "boundary_kind": "db",+ "cross_function": True,+ "path": [f"{path}::run", "src/shared.py::load", sink],+ "resolution_basis": "call-site",+ "dataflow_verified": True,+ },+ health_impact=0.0,+ reason="Awaited database work repeats for every loop iteration.",+ dimension="performance",+ )+++def _report(findings: list, *, metrics: list | None = None) -> SimpleNamespace:+ link_performance_findings(findings)+ return SimpleNamespace(+ metrics=metrics if metrics is not None else [_metric()],+ findings=findings,+ refactoring_suggestions=[],+ performance_plan_policy=None,+ )+++def _metric(path: str = "src/app.py") -> dict:+ return {+ "file_path": path,+ "score": 5.0,+ "max_ccn": 3,+ "max_nesting": 1,+ "nloc": 40,+ "duplication_pct": 0.0,+ "has_test_file": False,+ "line_coverage_pct": None,+ "branch_coverage_pct": None,+ "module": "src",+ }+++async def _open_perf_ids(session, repo_id: str) -> set[str]:+ rows = (+ (+ await session.execute(+ select(PerformanceOpportunity).where(+ PerformanceOpportunity.repository_id == repo_id,+ PerformanceOpportunity.status == "open",+ )+ )+ )+ .scalars()+ .all()+ )+ return {row.opportunity_id for row in rows}+++class TestSaveFullHealthReport:+ async def test_it_materializes_the_queue_from_the_findings_it_wrote(self, async_session):+ repo_id = (await insert_repo(async_session)).id+ await save_full_health_report(+ async_session, repo_id, _report([_perf_finding(10)]), analyzed_commit="a" * 40+ )+ assert len(await _open_perf_ids(async_session, repo_id)) == 1++ async def test_a_cause_the_rescore_no_longer_sees_is_resolved(self, async_session):+ """The defect this exists for: a re-score replaces every finding.++ Without the rebuild in the same transaction the queue keeps serving the+ opportunity, pointing at a finding row the re-score deleted.+ """+ repo_id = (await insert_repo(async_session)).id+ await save_full_health_report(+ async_session, repo_id, _report([_perf_finding(10)]), analyzed_commit="a" * 40+ )+ before = await _open_perf_ids(async_session, repo_id)+ assert len(before) == 1++ # The repeated work now lands on a different sink, which is a+ # different cause. The re-score deleted the finding the first one was+ # built from, so the queue must stop serving it.+ await save_full_health_report(+ async_session,+ repo_id,+ _report(+ [_perf_finding(10, path="src/other.py", sink="src/cache.py::lookup")],+ metrics=[_metric("src/other.py")],+ ),+ analyzed_commit="b" * 40,+ )+ after = await _open_perf_ids(async_session, repo_id)+ assert len(after) == 1+ assert after != before++ stored = {row.opportunity_id: row.status for row in await session_rows(async_session, repo_id)}+ assert stored[next(iter(before))] == "resolved"+ assert stored[next(iter(after))] == "open"++ async def test_it_stamps_the_metric_rows_with_the_analyzed_commit(self, async_session):+ repo_id = (await insert_repo(async_session)).id+ await save_full_health_report(+ async_session, repo_id, _report([]), analyzed_commit="c" * 40+ )+ rows = (+ (+ await async_session.execute(+ select(HealthFileMetric).where(HealthFileMetric.repository_id == repo_id)+ )+ )+ .scalars()+ .all()+ )+ assert rows+ assert {row.analyzed_commit for row in rows} == {"c" * 40}++ async def test_a_pass_that_scored_nothing_writes_nothing(self, async_session):+ """A parse failure must not read as "this repository became clean".++ Every write here replaces the repository's rows wholesale, so honouring+ an empty report would delete the metrics, resolve plans a person had+ triaged, and empty both queues on the strength of a broken run.+ """+ repo_id = (await insert_repo(async_session)).id+ await save_full_health_report(+ async_session, repo_id, _report([_perf_finding(10)]), analyzed_commit="a" * 40+ )+ before = await _open_perf_ids(async_session, repo_id)+ assert before++ empty = SimpleNamespace(+ metrics=[], findings=[], refactoring_suggestions=[], performance_plan_policy=None+ )+ await save_full_health_report(+ async_session, repo_id, empty, analyzed_commit="b" * 40+ )++ assert await _open_perf_ids(async_session, repo_id) == before+ metrics = (+ (+ await async_session.execute(+ select(HealthFileMetric).where(HealthFileMetric.repository_id == repo_id)+ )+ )+ .scalars()+ .all()+ )+ assert metrics, "an empty report must not delete the stored metrics"++ async def test_an_unstamped_pass_leaves_the_column_null(self, async_session):+ """``None`` reads as "not recorded"; it must not borrow a stale sha."""+ repo_id = (await insert_repo(async_session)).id+ await save_full_health_report(async_session, repo_id, _report([]), analyzed_commit=None)+ rows = (+ (+ await async_session.execute(+ select(HealthFileMetric).where(HealthFileMetric.repository_id == repo_id)+ )+ )+ .scalars()+ .all()+ )+ assert {row.analyzed_commit for row in rows} == {None}+++async def session_rows(session, repo_id: str):+ return (+ (+ await session.execute(+ select(PerformanceOpportunity).where(+ PerformanceOpportunity.repository_id == repo_id+ )+ )+ )+ .scalars()+ .all()+ )