Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(health): stop the overview route reading both health tables twice (#1350)

Raghav Chamadiya698e9d7b0c22
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py b/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py
index 7a8eb1da4..54a3b35e5 100644
--- a/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py
+++ b/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py
@@ -30,10 +30,12 @@ from .dead_code import (
)
from .health import (
HEALTH_SNAPSHOT_RETENTION,
+ HealthSnapshotHeadline,
get_deduction_by_path,
get_file_language_map,
get_health_findings,
get_health_metrics,
+ get_health_snapshot_headline,
get_health_summary,
get_perf_coverage,
list_health_snapshots,
@@ -55,6 +57,7 @@ from .refactoring import (
__all__ = [
"HEALTH_SNAPSHOT_RETENTION",
+ "HealthSnapshotHeadline",
"covered_source_files",
"files_covered_by",
"get_coverage_summary",
@@ -64,6 +67,7 @@ __all__ = [
"get_file_language_map",
"get_health_findings",
"get_health_metrics",
+ "get_health_snapshot_headline",
"get_health_summary",
"get_perf_coverage",
"get_refactoring_suggestion",
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/health.py b/packages/core/src/repowise/core/persistence/crud/analysis/health.py
index bf3726c5a..17190b970 100644
--- a/packages/core/src/repowise/core/persistence/crud/analysis/health.py
+++ b/packages/core/src/repowise/core/persistence/crud/analysis/health.py
@@ -6,7 +6,7 @@ from __future__ import annotations
import json
from datetime import datetime
from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, NamedTuple
from sqlalchemy import func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -430,14 +430,25 @@ async def get_perf_coverage(session: AsyncSession, repository_id: str) -> PerfCo
async def get_health_summary(
- session: AsyncSession, repository_id: str, *, metrics: list | None = None
+ session: AsyncSession,
+ repository_id: str,
+ *,
+ metrics: list | None = None,
+ findings: list | None = None,
) -> dict:
"""Aggregate KPIs over the per-file metrics table.
- *metrics* lets a caller that has already loaded the per-file rows hand them
- over instead of paying for a second full read. Callers doing more than one
- thing with them — the overview payload computes both these KPIs and the
- defect-accuracy stat — would otherwise pull the whole table twice.
+ *metrics* and *findings* let a caller that has already loaded those rows
+ hand them over instead of paying for a second full read. Callers doing more
+ than one thing with them — the overview payload builds these KPIs, the
+ severity breakdown and the defect-accuracy stat off the same two tables —
+ would otherwise pull each table twice per request.
+
+ Both are expected to be exactly what this function would have read itself:
+ ``get_health_metrics(repository_id)`` and
+ ``get_health_findings(repository_id)``, i.e. every open row for the repo,
+ post-exclusion. Handing over a *filtered* subset would silently skew
+ ``open_findings`` and the per-dimension counts, which are repo headlines.
"""
if metrics is None:
metrics = await get_health_metrics(session, repository_id)
@@ -510,7 +521,8 @@ async def get_health_summary(
worst_performance_path = perf_worst.file_path
worst_performance_score = round(perf_worst.performance_score, 2)
- findings = await get_health_findings(session, repository_id)
+ if findings is None:
+ findings = await get_health_findings(session, repository_id)
by_dim: dict[str, int] = {}
for finding in findings:
dim = finding.dimension or "defect"
@@ -642,6 +654,41 @@ async def list_health_snapshots(
return rows
+class HealthSnapshotHeadline(NamedTuple):
+ """The three snapshot scalars a dashboard header actually reads."""
+
+ hotspot_health: float | None
+ taken_at: datetime | None
+ snapshot_count: int
+
+
+async def get_health_snapshot_headline(
+ session: AsyncSession, repository_id: str
+) -> HealthSnapshotHeadline:
+ """Latest snapshot's ``hotspot_health`` / ``taken_at``, plus how many exist.
+
+ ``list_health_snapshots`` hydrates whole entities, and every row carries a
+ ``per_file_scores_json`` blob sized by the repo's file count — on this
+ codebase that is ~186 KB each, 2.2 MB across the retained history. A caller
+ that only needs the header scalars was reading all of it to use two floats.
+
+ Two columns, no JSON. Callers that genuinely need the per-file maps (the
+ trend routes, the file drawer's sparkline) should keep using
+ ``list_health_snapshots``.
+ """
+ rows = (
+ await session.execute(
+ select(HealthSnapshot.hotspot_health, HealthSnapshot.taken_at)
+ .where(HealthSnapshot.repository_id == repository_id)
+ .order_by(HealthSnapshot.taken_at.asc(), HealthSnapshot.id.asc())
+ )
+ ).all()
+ if not rows:
+ return HealthSnapshotHeadline(None, None, 0)
+ hotspot_health, taken_at = rows[-1]
+ return HealthSnapshotHeadline(float(hotspot_health), taken_at, len(rows))
+
+
async def upsert_health_findings(
session: AsyncSession,
repository_id: str,
diff --git a/packages/server/src/repowise/server/routers/code_health/overview_routes.py b/packages/server/src/repowise/server/routers/code_health/overview_routes.py
index 0bbaabff7..9b7ffab1b 100644
--- a/packages/server/src/repowise/server/routers/code_health/overview_routes.py
+++ b/packages/server/src/repowise/server/routers/code_health/overview_routes.py
@@ -49,25 +49,27 @@ async def health_overview(
repo = await crud.get_repository(session, repo_id)
if repo is None:
raise HTTPException(status_code=404, detail="Repository not found")
- # ``metrics=`` exists for exactly this caller (see the parameter's
- # docstring): without it the summary pulls the whole metrics table a second
- # time, and since that read now also aggregates the deduction column it
- # would pay for the ranking twice.
+ # ``metrics=`` / ``findings=`` exist for exactly this caller (see the
+ # parameter docstring): without them the summary pulls both whole tables a
+ # second time. The metrics read also aggregates the deduction column for the
+ # worst-first ranking, and the findings read is the most expensive of the
+ # two — so the route was paying for each of them twice per request.
metrics = await crud.get_health_metrics(session, repo_id)
- summary = await crud.get_health_summary(session, repo_id, metrics=metrics)
findings = await crud.get_health_findings(session, repo_id)
- snapshots = await crud.list_health_snapshots(session, repo_id)
+ summary = await crud.get_health_summary(
+ session, repo_id, metrics=metrics, findings=findings
+ )
# Pull hotspot_health from the latest snapshot (KPIs aren't recomputed
- # on every overview hit — the snapshot is authoritative).
- hotspot_health: float | None = None
- snapshot_taken_at = None
- if snapshots:
- latest = snapshots[-1]
- hotspot_health = round(float(latest.hotspot_health), 2)
- snapshot_taken_at = latest.taken_at
+ # on every overview hit — the snapshot is authoritative). Scalars only:
+ # a full snapshot entity drags its per-file score map, and this route reads
+ # none of it.
+ snapshot = await crud.get_health_snapshot_headline(session, repo_id)
+ hotspot_health = (
+ round(snapshot.hotspot_health, 2) if snapshot.hotspot_health is not None else None
+ )
- last_indexed_at = _resolve_last_indexed_at(snapshot_taken_at, repo.updated_at)
+ last_indexed_at = _resolve_last_indexed_at(snapshot.taken_at, repo.updated_at)
leads = _leads_by_file(findings)
metric_dicts = [_metric_to_dict(m, leads.get(m.file_path)) for m in metrics]
@@ -86,9 +88,14 @@ async def health_overview(
# "Does the score find the bugs?" self-validation, derived from the same
# metrics + findings (prior_defect biomarker) already loaded above. ``None``
# when the repo lacks enough files / defect history to be honest.
+ #
+ # Fed the rows it actually reads. It reads ``file_path`` / ``score`` off the
+ # metrics — so ``metric_dicts`` above serves, no second conversion — and
+ # only the ``prior_defect`` findings, so converting the other ~90% (which
+ # means a ``json.loads`` of every ``details_json``) was pure waste.
defect_accuracy = compute_defect_accuracy(
- [_metric_to_dict(m) for m in metrics],
- [_finding_to_dict(f) for f in findings],
+ metric_dicts,
+ [_finding_to_dict(f) for f in findings if f.biomarker_type == "prior_defect"],
)
top_findings = await _attach_symbol_ids(
@@ -109,7 +116,7 @@ async def health_overview(
# so the freshness signal self-heals on read (see the /api/repos
# overlay). This is the extension's primary indexed-commit source.
"head_commit": resolve_indexed_commit(repo.head_commit, repo.local_path),
- "snapshot_count": len(snapshots),
+ "snapshot_count": snapshot.snapshot_count,
},
}
diff --git a/packages/server/src/repowise/server/routers/overview.py b/packages/server/src/repowise/server/routers/overview.py
index c2231c15f..7c78b5f0e 100644
--- a/packages/server/src/repowise/server/routers/overview.py
+++ b/packages/server/src/repowise/server/routers/overview.py
@@ -369,7 +369,10 @@ async def overview_summary(
# defect-accuracy stat both want every per-file row, and letting each fetch
# its own pulled the whole table twice per page load.
health_metrics = await crud.get_health_metrics(session, repo_id)
- health_summary = await crud.get_health_summary(session, repo_id, metrics=health_metrics)
+ findings = await crud.get_health_findings(session, repo_id)
+ health_summary = await crud.get_health_summary(
+ session, repo_id, metrics=health_metrics, findings=findings
+ )
snapshots = await crud.list_health_snapshots(session, repo_id)
hotspot_health: float | None = None
last_indexed_at: str | None = None
@@ -394,7 +397,6 @@ async def overview_summary(
except Exception:
pass
- findings = await crud.get_health_findings(session, repo_id)
severity_breakdown = {"critical": 0, "high": 0, "medium": 0, "low": 0}
for f in findings:
s = (f.severity or "").lower()
@@ -405,6 +407,31 @@ async def overview_summary(
# ranking, shown on the health card. Sourced here rather than from the stats
# payload: this is a health number, and having the overview reach across
# into another page's payload for it is what broke when that payload changed.
+ #
+ # ``prior_defect`` rows only, each carrying its parsed ``details``. Two
+ # problems in one shape: the stat reads no other biomarker, so converting
+ # the other ~90% of findings was waste; and it reads the fix count and the
+ # window *out of* ``details``, which this call site never supplied — so
+ # every flagged file reported ``recent_fixes: 1`` (333 of 999 have more, up
+ # to 19) and ``window_days`` echoed the default instead of the indexed
+ # value. The health dashboard computes the same stat from the real numbers,
+ # so the two surfaces disagreed on one figure.
+ # Parsed per row: one malformed blob must not cost the whole panel.
+ prior_defect_rows: list[dict] = []
+ for f in findings:
+ if f.biomarker_type != "prior_defect":
+ continue
+ details: Any = {}
+ with contextlib.suppress(Exception):
+ details = json.loads(f.details_json) if f.details_json else {}
+ prior_defect_rows.append(
+ {
+ "file_path": f.file_path,
+ "biomarker_type": f.biomarker_type,
+ "details": details,
+ }
+ )
+
defect_accuracy = None
try:
from repowise.core.analysis.health.defect_accuracy import compute_defect_accuracy
@@ -420,14 +447,7 @@ async def overview_summary(
}
for m in health_metrics
],
- [
- {
- "file_path": f.file_path,
- "biomarker_type": f.biomarker_type,
- "severity": f.severity,
- }
- for f in findings
- ],
+ prior_defect_rows,
)
except Exception:
# Best-effort: the card omits the panel rather than failing the page.
diff --git a/tests/unit/health/test_defect_accuracy.py b/tests/unit/health/test_defect_accuracy.py
index d458c5074..637b84b80 100644
--- a/tests/unit/health/test_defect_accuracy.py
+++ b/tests/unit/health/test_defect_accuracy.py
@@ -62,6 +62,27 @@ def test_lift_none_when_no_baseline_is_handled() -> None:
assert stat["lift"] is not None
+def test_non_prior_defect_findings_are_ignored() -> None:
+ """Only ``prior_defect`` rows carry the label, so callers may pre-filter.
+
+ The overview route hands over just those rows rather than converting all
+ ~10k findings (each a ``json.loads`` of its details blob) for this function
+ to discard 90% of them. That is only safe while feeding the other
+ biomarkers in changes nothing — including ones carrying a details key that
+ looks like a label.
+ """
+ metrics = _metrics(30)
+ labels = [_fix(f"f{i}.py") for i in range(6)]
+ noise = [
+ {"biomarker_type": b, "file_path": f"f{i}.py", "details": {"prior_defect_count": 99}}
+ for i, b in enumerate(("complex_method", "io_in_loop", "coverage_gradient"))
+ ]
+
+ assert compute_defect_accuracy(metrics, labels) == compute_defect_accuracy(
+ metrics, labels + noise
+ )
+
+
def test_window_days_from_finding_details() -> None:
metrics = _metrics(30)
findings = [_fix(f"f{i}.py", window_days=90) for i in range(6)]
diff --git a/tests/unit/persistence/test_health_overview_reads.py b/tests/unit/persistence/test_health_overview_reads.py
new file mode 100644
index 000000000..2d7540095
--- /dev/null
+++ b/tests/unit/persistence/test_health_overview_reads.py
@@ -0,0 +1,138 @@
+"""The two reads ``/health/overview`` was paying for twice.
+
+``get_health_summary`` internally loads both the metrics and the findings
+tables. The overview route had already loaded both for its own payload, so
+every request scanned each table twice. ``metrics=`` closed half of that;
+these pin the ``findings=`` half and the snapshot-scalar read that replaced a
+full-entity load of every retained per-file score map.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+from repowise.core.persistence.crud import (
+ get_health_snapshot_headline,
+ get_health_summary,
+ save_health_findings,
+ save_health_metrics,
+ save_health_snapshot,
+ upsert_repository,
+)
+
+
+def _metric(path: str, score: float, nloc: int = 10) -> dict:
+ return {
+ "file_path": path,
+ "score": score,
+ "max_ccn": 1,
+ "max_nesting": 1,
+ "nloc": nloc,
+ "has_test_file": False,
+ "module": "src",
+ }
+
+
+def _finding(path: str, impact: float, *, name: str = "f", dimension: str = "defect") -> dict:
+ return {
+ "file_path": path,
+ "biomarker_type": "complex_method",
+ "severity": "high",
+ "function_name": name,
+ "line_start": 1,
+ "line_end": 2,
+ "details": {},
+ "health_impact": impact,
+ "reason": "reason",
+ "dimension": dimension,
+ }
+
+
+async def _seed(async_session, tmp_path, metrics, findings):
+ repo = await upsert_repository(async_session, name="repo", local_path=str(tmp_path))
+ await save_health_metrics(async_session, repo.id, metrics)
+ await save_health_findings(async_session, repo.id, findings)
+ return repo
+
+
+async def test_handing_over_findings_matches_loading_them(async_session, tmp_path) -> None:
+ """The kwarg is a pure read-elision: same repo, same summary."""
+ repo = await _seed(
+ async_session,
+ tmp_path,
+ [_metric("a.py", 3.0), _metric("b.py", 7.0)],
+ [
+ _finding("a.py", 2.0),
+ _finding("a.py", 1.0, name="g", dimension="performance"),
+ _finding("b.py", 0.5, dimension="maintainability"),
+ ],
+ )
+
+ from repowise.core.persistence.crud import get_health_findings, get_health_metrics
+
+ metrics = await get_health_metrics(async_session, repo.id)
+ findings = await get_health_findings(async_session, repo.id)
+
+ assert await get_health_summary(async_session, repo.id) == await get_health_summary(
+ async_session, repo.id, metrics=metrics, findings=findings
+ )
+
+
+async def test_findings_kwarg_is_actually_used_not_re_read(async_session, tmp_path) -> None:
+ """Hand over a deliberately short list; the counts must follow it.
+
+ If the kwarg were ignored the function would re-query and report 3. This is
+ the only way to prove the second scan is gone without instrumenting SQL —
+ and it is also why the docstring insists callers pass the *unfiltered* set.
+ """
+ repo = await _seed(
+ async_session,
+ tmp_path,
+ [_metric("a.py", 3.0)],
+ [
+ _finding("a.py", 2.0),
+ _finding("a.py", 1.0, name="g", dimension="performance"),
+ _finding("a.py", 0.5, name="h", dimension="maintainability"),
+ ],
+ )
+
+ from repowise.core.persistence.crud import get_health_findings
+
+ findings = await get_health_findings(async_session, repo.id)
+ assert len(findings) == 3
+
+ summary = await get_health_summary(async_session, repo.id, findings=findings[:1])
+ assert summary["open_findings"] == 1
+ assert summary["performance_findings"] == 0
+
+
+async def test_snapshot_headline_reads_the_latest_and_counts_the_rest(
+ async_session, tmp_path
+) -> None:
+ repo = await upsert_repository(async_session, name="repo", local_path=str(tmp_path))
+ base = datetime(2026, 1, 1, tzinfo=UTC)
+ # Inserted newest-first so a headline that trusted insertion order fails.
+ for offset, hotspot in ((2, 4.0), (0, 6.0), (1, 5.0)):
+ await save_health_snapshot(
+ async_session,
+ repo.id,
+ hotspot_health=hotspot,
+ average_health=7.0,
+ worst_performer_path="a.py",
+ worst_performer_score=1.0,
+ per_file_scores={"a.py": 1.0},
+ taken_at=base + timedelta(days=offset),
+ )
+
+ headline = await get_health_snapshot_headline(async_session, repo.id)
+
+ assert headline.hotspot_health == 4.0
+ assert headline.taken_at.replace(tzinfo=UTC) == base + timedelta(days=2)
+ assert headline.snapshot_count == 3
+
+
+async def test_snapshot_headline_on_a_repo_with_no_history(async_session, tmp_path) -> None:
+ """No snapshots is not an error: the header renders "not measured"."""
+ repo = await upsert_repository(async_session, name="repo", local_path=str(tmp_path))
+
+ assert await get_health_snapshot_headline(async_session, repo.id) == (None, None, 0)
diff --git a/tests/unit/server/test_overview_defect_accuracy.py b/tests/unit/server/test_overview_defect_accuracy.py
new file mode 100644
index 000000000..fdaaeaf84
--- /dev/null
+++ b/tests/unit/server/test_overview_defect_accuracy.py
@@ -0,0 +1,93 @@
+"""The ``defect_accuracy`` block of /api/repos/{id}/overview-summary.
+
+The stat reads its per-file fix count and its window out of each finding's
+``details``. This route built its finding dicts by hand and left ``details``
+out, so it silently fell back to the defaults for both — reporting one recent
+fix per flagged file and a 180-day window regardless of what was indexed, while
+the health dashboard's copy of the same stat reported the real numbers.
+"""
+
+from __future__ import annotations
+
+import pytest
+from httpx import AsyncClient
+
+from repowise.core.persistence import crud
+from repowise.core.persistence.database import get_session
+from tests.unit.server.conftest import create_test_repo
+
+# compute_defect_accuracy stays silent below 25 scored files / 5 defect files.
+_FILES = 30
+_DEFECT_FILES = 6
+_WINDOW_DAYS = 90
+
+
+def _metric(i: int) -> dict:
+ return {
+ "file_path": f"src/f{i:02d}.py",
+ "score": float(i) / 3.0,
+ "max_ccn": 1,
+ "max_nesting": 1,
+ "nloc": 10,
+ "has_test_file": False,
+ "module": "src",
+ }
+
+
+def _prior_defect(i: int, count: int) -> dict:
+ return {
+ "file_path": f"src/f{i:02d}.py",
+ "biomarker_type": "prior_defect",
+ "severity": "medium",
+ "function_name": None,
+ "line_start": None,
+ "line_end": None,
+ "details": {"prior_defect_count": count, "window_days": _WINDOW_DAYS},
+ "health_impact": 0.3,
+ "reason": "recently bug-fixed",
+ }
+
+
+async def test_defect_accuracy_reports_the_indexed_fix_counts_and_window(
+ client: AsyncClient, app
+) -> None:
+ repo = await create_test_repo(client)
+
+ async with get_session(app.state.session_factory) as session:
+ await crud.save_health_metrics(session, repo["id"], [_metric(i) for i in range(_FILES)])
+ await crud.save_health_findings(
+ session,
+ repo["id"],
+ # Deliberately not 1: the old hand-built dicts dropped ``details``,
+ # so every one of these came back as a single fix.
+ [_prior_defect(i, count=i + 2) for i in range(_DEFECT_FILES)]
+ # Noise the stat must ignore — it reads no other biomarker.
+ + [
+ {
+ "file_path": "src/f29.py",
+ "biomarker_type": "complex_method",
+ "severity": "high",
+ "function_name": "big",
+ "line_start": 1,
+ "line_end": 40,
+ "details": {"prior_defect_count": 99, "window_days": 7},
+ "health_impact": 2.0,
+ "reason": "complex",
+ }
+ ],
+ )
+
+ resp = await client.get(f"/api/repos/{repo['id']}/overview-summary")
+ assert resp.status_code == 200
+ stat = resp.json()["health"]["defect_accuracy"]
+ assert stat is not None
+
+ assert stat["window_days"] == _WINDOW_DAYS
+ assert stat["defect_files"] == _DEFECT_FILES
+ flagged = {row["file_path"]: row["recent_fixes"] for row in stat["flagged_files"]}
+ for i in range(_DEFECT_FILES):
+ assert flagged[f"src/f{i:02d}.py"] == i + 2
+ # The complex_method row carries a details blob that looks like a label;
+ # counting it would make f29 a defect file and push defect_files to 7.
+ assert flagged.get("src/f29.py", 0) == 0