Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(health): stop the overview route reading both health tables twice (#1350)
Raghav Chamadiya698e9d7b0c22
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py b/packages/core/src/repowise/core/persistence/crud/analysis/__init__.pyindex 7a8eb1da4..54a3b35e5 100644--- a/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py+++ b/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py@@ -30,10 +30,12 @@ from .dead_code import ()from .health import (HEALTH_SNAPSHOT_RETENTION,+ HealthSnapshotHeadline,get_deduction_by_path,get_file_language_map,get_health_findings,get_health_metrics,+ get_health_snapshot_headline,get_health_summary,get_perf_coverage,list_health_snapshots,@@ -55,6 +57,7 @@ from .refactoring import (__all__ = ["HEALTH_SNAPSHOT_RETENTION",+ "HealthSnapshotHeadline","covered_source_files","files_covered_by","get_coverage_summary",@@ -64,6 +67,7 @@ __all__ = ["get_file_language_map","get_health_findings","get_health_metrics",+ "get_health_snapshot_headline","get_health_summary","get_perf_coverage","get_refactoring_suggestion",diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/health.py b/packages/core/src/repowise/core/persistence/crud/analysis/health.pyindex bf3726c5a..17190b970 100644--- a/packages/core/src/repowise/core/persistence/crud/analysis/health.py+++ b/packages/core/src/repowise/core/persistence/crud/analysis/health.py@@ -6,7 +6,7 @@ from __future__ import annotationsimport jsonfrom datetime import datetimefrom pathlib import Path-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, NamedTuplefrom sqlalchemy import func, or_, selectfrom sqlalchemy.ext.asyncio import AsyncSession@@ -430,14 +430,25 @@ async def get_perf_coverage(session: AsyncSession, repository_id: str) -> PerfCoasync def get_health_summary(- session: AsyncSession, repository_id: str, *, metrics: list | None = None+ session: AsyncSession,+ repository_id: str,+ *,+ metrics: list | None = None,+ findings: list | None = None,) -> dict:"""Aggregate KPIs over the per-file metrics table.- *metrics* lets a caller that has already loaded the per-file rows hand them- over instead of paying for a second full read. Callers doing more than one- thing with them — the overview payload computes both these KPIs and the- defect-accuracy stat — would otherwise pull the whole table twice.+ *metrics* and *findings* let a caller that has already loaded those rows+ hand them over instead of paying for a second full read. Callers doing more+ than one thing with them — the overview payload builds these KPIs, the+ severity breakdown and the defect-accuracy stat off the same two tables —+ would otherwise pull each table twice per request.++ Both are expected to be exactly what this function would have read itself:+ ``get_health_metrics(repository_id)`` and+ ``get_health_findings(repository_id)``, i.e. every open row for the repo,+ post-exclusion. Handing over a *filtered* subset would silently skew+ ``open_findings`` and the per-dimension counts, which are repo headlines."""if metrics is None:metrics = await get_health_metrics(session, repository_id)@@ -510,7 +521,8 @@ async def get_health_summary(worst_performance_path = perf_worst.file_pathworst_performance_score = round(perf_worst.performance_score, 2)- findings = await get_health_findings(session, repository_id)+ if findings is None:+ findings = await get_health_findings(session, repository_id)by_dim: dict[str, int] = {}for finding in findings:dim = finding.dimension or "defect"@@ -642,6 +654,41 @@ async def list_health_snapshots(return rows+class HealthSnapshotHeadline(NamedTuple):+ """The three snapshot scalars a dashboard header actually reads."""++ hotspot_health: float | None+ taken_at: datetime | None+ snapshot_count: int+++async def get_health_snapshot_headline(+ session: AsyncSession, repository_id: str+) -> HealthSnapshotHeadline:+ """Latest snapshot's ``hotspot_health`` / ``taken_at``, plus how many exist.++ ``list_health_snapshots`` hydrates whole entities, and every row carries a+ ``per_file_scores_json`` blob sized by the repo's file count — on this+ codebase that is ~186 KB each, 2.2 MB across the retained history. A caller+ that only needs the header scalars was reading all of it to use two floats.++ Two columns, no JSON. Callers that genuinely need the per-file maps (the+ trend routes, the file drawer's sparkline) should keep using+ ``list_health_snapshots``.+ """+ rows = (+ await session.execute(+ select(HealthSnapshot.hotspot_health, HealthSnapshot.taken_at)+ .where(HealthSnapshot.repository_id == repository_id)+ .order_by(HealthSnapshot.taken_at.asc(), HealthSnapshot.id.asc())+ )+ ).all()+ if not rows:+ return HealthSnapshotHeadline(None, None, 0)+ hotspot_health, taken_at = rows[-1]+ return HealthSnapshotHeadline(float(hotspot_health), taken_at, len(rows))++async def upsert_health_findings(session: AsyncSession,repository_id: str,diff --git a/packages/server/src/repowise/server/routers/code_health/overview_routes.py b/packages/server/src/repowise/server/routers/code_health/overview_routes.pyindex 0bbaabff7..9b7ffab1b 100644--- a/packages/server/src/repowise/server/routers/code_health/overview_routes.py+++ b/packages/server/src/repowise/server/routers/code_health/overview_routes.py@@ -49,25 +49,27 @@ async def health_overview(repo = await crud.get_repository(session, repo_id)if repo is None:raise HTTPException(status_code=404, detail="Repository not found")- # ``metrics=`` exists for exactly this caller (see the parameter's- # docstring): without it the summary pulls the whole metrics table a second- # time, and since that read now also aggregates the deduction column it- # would pay for the ranking twice.+ # ``metrics=`` / ``findings=`` exist for exactly this caller (see the+ # parameter docstring): without them the summary pulls both whole tables a+ # second time. The metrics read also aggregates the deduction column for the+ # worst-first ranking, and the findings read is the most expensive of the+ # two — so the route was paying for each of them twice per request.metrics = await crud.get_health_metrics(session, repo_id)- summary = await crud.get_health_summary(session, repo_id, metrics=metrics)findings = await crud.get_health_findings(session, repo_id)- snapshots = await crud.list_health_snapshots(session, repo_id)+ summary = await crud.get_health_summary(+ session, repo_id, metrics=metrics, findings=findings+ )# Pull hotspot_health from the latest snapshot (KPIs aren't recomputed- # on every overview hit — the snapshot is authoritative).- hotspot_health: float | None = None- snapshot_taken_at = None- if snapshots:- latest = snapshots[-1]- hotspot_health = round(float(latest.hotspot_health), 2)- snapshot_taken_at = latest.taken_at+ # on every overview hit — the snapshot is authoritative). Scalars only:+ # a full snapshot entity drags its per-file score map, and this route reads+ # none of it.+ snapshot = await crud.get_health_snapshot_headline(session, repo_id)+ hotspot_health = (+ round(snapshot.hotspot_health, 2) if snapshot.hotspot_health is not None else None+ )- last_indexed_at = _resolve_last_indexed_at(snapshot_taken_at, repo.updated_at)+ last_indexed_at = _resolve_last_indexed_at(snapshot.taken_at, repo.updated_at)leads = _leads_by_file(findings)metric_dicts = [_metric_to_dict(m, leads.get(m.file_path)) for m in metrics]@@ -86,9 +88,14 @@ async def health_overview(# "Does the score find the bugs?" self-validation, derived from the same# metrics + findings (prior_defect biomarker) already loaded above. ``None``# when the repo lacks enough files / defect history to be honest.+ #+ # Fed the rows it actually reads. It reads ``file_path`` / ``score`` off the+ # metrics — so ``metric_dicts`` above serves, no second conversion — and+ # only the ``prior_defect`` findings, so converting the other ~90% (which+ # means a ``json.loads`` of every ``details_json``) was pure waste.defect_accuracy = compute_defect_accuracy(- [_metric_to_dict(m) for m in metrics],- [_finding_to_dict(f) for f in findings],+ metric_dicts,+ [_finding_to_dict(f) for f in findings if f.biomarker_type == "prior_defect"],)top_findings = await _attach_symbol_ids(@@ -109,7 +116,7 @@ async def health_overview(# so the freshness signal self-heals on read (see the /api/repos# overlay). This is the extension's primary indexed-commit source."head_commit": resolve_indexed_commit(repo.head_commit, repo.local_path),- "snapshot_count": len(snapshots),+ "snapshot_count": snapshot.snapshot_count,},}diff --git a/packages/server/src/repowise/server/routers/overview.py b/packages/server/src/repowise/server/routers/overview.pyindex c2231c15f..7c78b5f0e 100644--- a/packages/server/src/repowise/server/routers/overview.py+++ b/packages/server/src/repowise/server/routers/overview.py@@ -369,7 +369,10 @@ async def overview_summary(# defect-accuracy stat both want every per-file row, and letting each fetch# its own pulled the whole table twice per page load.health_metrics = await crud.get_health_metrics(session, repo_id)- health_summary = await crud.get_health_summary(session, repo_id, metrics=health_metrics)+ findings = await crud.get_health_findings(session, repo_id)+ health_summary = await crud.get_health_summary(+ session, repo_id, metrics=health_metrics, findings=findings+ )snapshots = await crud.list_health_snapshots(session, repo_id)hotspot_health: float | None = Nonelast_indexed_at: str | None = None@@ -394,7 +397,6 @@ async def overview_summary(except Exception:pass- findings = await crud.get_health_findings(session, repo_id)severity_breakdown = {"critical": 0, "high": 0, "medium": 0, "low": 0}for f in findings:s = (f.severity or "").lower()@@ -405,6 +407,31 @@ async def overview_summary(# ranking, shown on the health card. Sourced here rather than from the stats# payload: this is a health number, and having the overview reach across# into another page's payload for it is what broke when that payload changed.+ #+ # ``prior_defect`` rows only, each carrying its parsed ``details``. Two+ # problems in one shape: the stat reads no other biomarker, so converting+ # the other ~90% of findings was waste; and it reads the fix count and the+ # window *out of* ``details``, which this call site never supplied — so+ # every flagged file reported ``recent_fixes: 1`` (333 of 999 have more, up+ # to 19) and ``window_days`` echoed the default instead of the indexed+ # value. The health dashboard computes the same stat from the real numbers,+ # so the two surfaces disagreed on one figure.+ # Parsed per row: one malformed blob must not cost the whole panel.+ prior_defect_rows: list[dict] = []+ for f in findings:+ if f.biomarker_type != "prior_defect":+ continue+ details: Any = {}+ with contextlib.suppress(Exception):+ details = json.loads(f.details_json) if f.details_json else {}+ prior_defect_rows.append(+ {+ "file_path": f.file_path,+ "biomarker_type": f.biomarker_type,+ "details": details,+ }+ )+defect_accuracy = Nonetry:from repowise.core.analysis.health.defect_accuracy import compute_defect_accuracy@@ -420,14 +447,7 @@ async def overview_summary(}for m in health_metrics],- [- {- "file_path": f.file_path,- "biomarker_type": f.biomarker_type,- "severity": f.severity,- }- for f in findings- ],+ prior_defect_rows,)except Exception:# Best-effort: the card omits the panel rather than failing the page.diff --git a/tests/unit/health/test_defect_accuracy.py b/tests/unit/health/test_defect_accuracy.pyindex d458c5074..637b84b80 100644--- a/tests/unit/health/test_defect_accuracy.py+++ b/tests/unit/health/test_defect_accuracy.py@@ -62,6 +62,27 @@ def test_lift_none_when_no_baseline_is_handled() -> None:assert stat["lift"] is not None+def test_non_prior_defect_findings_are_ignored() -> None:+ """Only ``prior_defect`` rows carry the label, so callers may pre-filter.++ The overview route hands over just those rows rather than converting all+ ~10k findings (each a ``json.loads`` of its details blob) for this function+ to discard 90% of them. That is only safe while feeding the other+ biomarkers in changes nothing — including ones carrying a details key that+ looks like a label.+ """+ metrics = _metrics(30)+ labels = [_fix(f"f{i}.py") for i in range(6)]+ noise = [+ {"biomarker_type": b, "file_path": f"f{i}.py", "details": {"prior_defect_count": 99}}+ for i, b in enumerate(("complex_method", "io_in_loop", "coverage_gradient"))+ ]++ assert compute_defect_accuracy(metrics, labels) == compute_defect_accuracy(+ metrics, labels + noise+ )++def test_window_days_from_finding_details() -> None:metrics = _metrics(30)findings = [_fix(f"f{i}.py", window_days=90) for i in range(6)]diff --git a/tests/unit/persistence/test_health_overview_reads.py b/tests/unit/persistence/test_health_overview_reads.pynew file mode 100644index 000000000..2d7540095--- /dev/null+++ b/tests/unit/persistence/test_health_overview_reads.py@@ -0,0 +1,138 @@+"""The two reads ``/health/overview`` was paying for twice.++``get_health_summary`` internally loads both the metrics and the findings+tables. The overview route had already loaded both for its own payload, so+every request scanned each table twice. ``metrics=`` closed half of that;+these pin the ``findings=`` half and the snapshot-scalar read that replaced a+full-entity load of every retained per-file score map.+"""++from __future__ import annotations++from datetime import UTC, datetime, timedelta++from repowise.core.persistence.crud import (+ get_health_snapshot_headline,+ get_health_summary,+ save_health_findings,+ save_health_metrics,+ save_health_snapshot,+ upsert_repository,+)+++def _metric(path: str, score: float, nloc: int = 10) -> dict:+ return {+ "file_path": path,+ "score": score,+ "max_ccn": 1,+ "max_nesting": 1,+ "nloc": nloc,+ "has_test_file": False,+ "module": "src",+ }+++def _finding(path: str, impact: float, *, name: str = "f", dimension: str = "defect") -> dict:+ return {+ "file_path": path,+ "biomarker_type": "complex_method",+ "severity": "high",+ "function_name": name,+ "line_start": 1,+ "line_end": 2,+ "details": {},+ "health_impact": impact,+ "reason": "reason",+ "dimension": dimension,+ }+++async def _seed(async_session, tmp_path, metrics, findings):+ repo = await upsert_repository(async_session, name="repo", local_path=str(tmp_path))+ await save_health_metrics(async_session, repo.id, metrics)+ await save_health_findings(async_session, repo.id, findings)+ return repo+++async def test_handing_over_findings_matches_loading_them(async_session, tmp_path) -> None:+ """The kwarg is a pure read-elision: same repo, same summary."""+ repo = await _seed(+ async_session,+ tmp_path,+ [_metric("a.py", 3.0), _metric("b.py", 7.0)],+ [+ _finding("a.py", 2.0),+ _finding("a.py", 1.0, name="g", dimension="performance"),+ _finding("b.py", 0.5, dimension="maintainability"),+ ],+ )++ from repowise.core.persistence.crud import get_health_findings, get_health_metrics++ metrics = await get_health_metrics(async_session, repo.id)+ findings = await get_health_findings(async_session, repo.id)++ assert await get_health_summary(async_session, repo.id) == await get_health_summary(+ async_session, repo.id, metrics=metrics, findings=findings+ )+++async def test_findings_kwarg_is_actually_used_not_re_read(async_session, tmp_path) -> None:+ """Hand over a deliberately short list; the counts must follow it.++ If the kwarg were ignored the function would re-query and report 3. This is+ the only way to prove the second scan is gone without instrumenting SQL —+ and it is also why the docstring insists callers pass the *unfiltered* set.+ """+ repo = await _seed(+ async_session,+ tmp_path,+ [_metric("a.py", 3.0)],+ [+ _finding("a.py", 2.0),+ _finding("a.py", 1.0, name="g", dimension="performance"),+ _finding("a.py", 0.5, name="h", dimension="maintainability"),+ ],+ )++ from repowise.core.persistence.crud import get_health_findings++ findings = await get_health_findings(async_session, repo.id)+ assert len(findings) == 3++ summary = await get_health_summary(async_session, repo.id, findings=findings[:1])+ assert summary["open_findings"] == 1+ assert summary["performance_findings"] == 0+++async def test_snapshot_headline_reads_the_latest_and_counts_the_rest(+ async_session, tmp_path+) -> None:+ repo = await upsert_repository(async_session, name="repo", local_path=str(tmp_path))+ base = datetime(2026, 1, 1, tzinfo=UTC)+ # Inserted newest-first so a headline that trusted insertion order fails.+ for offset, hotspot in ((2, 4.0), (0, 6.0), (1, 5.0)):+ await save_health_snapshot(+ async_session,+ repo.id,+ hotspot_health=hotspot,+ average_health=7.0,+ worst_performer_path="a.py",+ worst_performer_score=1.0,+ per_file_scores={"a.py": 1.0},+ taken_at=base + timedelta(days=offset),+ )++ headline = await get_health_snapshot_headline(async_session, repo.id)++ assert headline.hotspot_health == 4.0+ assert headline.taken_at.replace(tzinfo=UTC) == base + timedelta(days=2)+ assert headline.snapshot_count == 3+++async def test_snapshot_headline_on_a_repo_with_no_history(async_session, tmp_path) -> None:+ """No snapshots is not an error: the header renders "not measured"."""+ repo = await upsert_repository(async_session, name="repo", local_path=str(tmp_path))++ assert await get_health_snapshot_headline(async_session, repo.id) == (None, None, 0)diff --git a/tests/unit/server/test_overview_defect_accuracy.py b/tests/unit/server/test_overview_defect_accuracy.pynew file mode 100644index 000000000..fdaaeaf84--- /dev/null+++ b/tests/unit/server/test_overview_defect_accuracy.py@@ -0,0 +1,93 @@+"""The ``defect_accuracy`` block of /api/repos/{id}/overview-summary.++The stat reads its per-file fix count and its window out of each finding's+``details``. This route built its finding dicts by hand and left ``details``+out, so it silently fell back to the defaults for both — reporting one recent+fix per flagged file and a 180-day window regardless of what was indexed, while+the health dashboard's copy of the same stat reported the real numbers.+"""++from __future__ import annotations++import pytest+from httpx import AsyncClient++from repowise.core.persistence import crud+from repowise.core.persistence.database import get_session+from tests.unit.server.conftest import create_test_repo++# compute_defect_accuracy stays silent below 25 scored files / 5 defect files.+_FILES = 30+_DEFECT_FILES = 6+_WINDOW_DAYS = 90+++def _metric(i: int) -> dict:+ return {+ "file_path": f"src/f{i:02d}.py",+ "score": float(i) / 3.0,+ "max_ccn": 1,+ "max_nesting": 1,+ "nloc": 10,+ "has_test_file": False,+ "module": "src",+ }+++def _prior_defect(i: int, count: int) -> dict:+ return {+ "file_path": f"src/f{i:02d}.py",+ "biomarker_type": "prior_defect",+ "severity": "medium",+ "function_name": None,+ "line_start": None,+ "line_end": None,+ "details": {"prior_defect_count": count, "window_days": _WINDOW_DAYS},+ "health_impact": 0.3,+ "reason": "recently bug-fixed",+ }+++async def test_defect_accuracy_reports_the_indexed_fix_counts_and_window(+ client: AsyncClient, app+) -> None:+ repo = await create_test_repo(client)++ async with get_session(app.state.session_factory) as session:+ await crud.save_health_metrics(session, repo["id"], [_metric(i) for i in range(_FILES)])+ await crud.save_health_findings(+ session,+ repo["id"],+ # Deliberately not 1: the old hand-built dicts dropped ``details``,+ # so every one of these came back as a single fix.+ [_prior_defect(i, count=i + 2) for i in range(_DEFECT_FILES)]+ # Noise the stat must ignore — it reads no other biomarker.+ + [+ {+ "file_path": "src/f29.py",+ "biomarker_type": "complex_method",+ "severity": "high",+ "function_name": "big",+ "line_start": 1,+ "line_end": 40,+ "details": {"prior_defect_count": 99, "window_days": 7},+ "health_impact": 2.0,+ "reason": "complex",+ }+ ],+ )++ resp = await client.get(f"/api/repos/{repo['id']}/overview-summary")+ assert resp.status_code == 200+ stat = resp.json()["health"]["defect_accuracy"]+ assert stat is not None++ assert stat["window_days"] == _WINDOW_DAYS+ assert stat["defect_files"] == _DEFECT_FILES+ flagged = {row["file_path"]: row["recent_fixes"] for row in stat["flagged_files"]}+ for i in range(_DEFECT_FILES):+ assert flagged[f"src/f{i:02d}.py"] == i + 2+ # The complex_method row carries a details blob that looks like a label;+ # counting it would make f29 a defect file and push defect_files to 7.+ assert flagged.get("src/f29.py", 0) == 0