Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(health): stop the badge reading the whole dataset, and index symbols by file (#1351)

Raghav Chamadiya718af981d0eb
diff --git a/packages/cli/src/repowise/cli/commands/augment_cmd/fast_lookup.py b/packages/cli/src/repowise/cli/commands/augment_cmd/fast_lookup.py
index d1d12190c..f3539fa70 100644
--- a/packages/cli/src/repowise/cli/commands/augment_cmd/fast_lookup.py
+++ b/packages/cli/src/repowise/cli/commands/augment_cmd/fast_lookup.py
@@ -160,12 +160,21 @@ def symbols_named(
BINARY collation unless a column says otherwise, and neither the ORM path
nor this one asks for NOCASE. The widened rescue depends on that, since it
generates its case variants explicitly.
+
+ Ordered because the ``LIMIT`` truncates. Unordered, the rows arrive in
+ whatever order the chosen index happens to walk, so *which* rows survive
+ the cut was the planner's choice — and adding an unrelated index to this
+ table silently changed the answer. ``(file_path, name)`` is the order the
+ ``uq_wiki_symbol`` autoindex gave for free, its key being
+ ``"<path>::<name>"``, so this pins the long-standing result rather than
+ picking a new one.
"""
if not names:
return []
placeholders = ",".join("?" * len(names))
return conn.execute(
"SELECT name, kind, file_path, start_line FROM wiki_symbols "
- f"WHERE repository_id = ? AND name IN ({placeholders}) LIMIT ?",
+ f"WHERE repository_id = ? AND name IN ({placeholders}) "
+ "ORDER BY file_path, name LIMIT ?",
(repository_id, *names, limit),
).fetchall()
diff --git a/packages/cli/src/repowise/cli/commands/augment_cmd/search.py b/packages/cli/src/repowise/cli/commands/augment_cmd/search.py
index 635484cc8..8f5676f8c 100644
--- a/packages/cli/src/repowise/cli/commands/augment_cmd/search.py
+++ b/packages/cli/src/repowise/cli/commands/augment_cmd/search.py
@@ -793,9 +793,16 @@ async def _rescue(
# sit well outside the first two rows, and post-filtering them would
# have made the widened rescue almost never fire, silently.
name_clause = WikiSymbol.name.in_(sorted(variants))
+ # Ordered because the LIMIT truncates: unordered, which rows survive the cut
+ # is whatever order the chosen index walks, so an unrelated index added to
+ # wiki_symbols changes the result. (file_path, name) is what the
+ # uq_wiki_symbol autoindex gave for free — its key is "<path>::<name>" — so
+ # this pins the long-standing behaviour. Mirrored in
+ # ``fast_lookup.symbols_named``, which serves the same rescue.
sym_stmt = (
select(WikiSymbol.name, WikiSymbol.kind, WikiSymbol.file_path, WikiSymbol.start_line)
.where(WikiSymbol.repository_id == repo_id, name_clause)
+ .order_by(WikiSymbol.file_path, WikiSymbol.name)
.limit(_RESCUE_TOP_N if matched is None else _RESCUE_EXACT_FETCH)
)
rows = (await session.execute(sym_stmt)).all()
diff --git a/packages/core/alembic/versions/0046_wiki_symbols_repo_path_index.py b/packages/core/alembic/versions/0046_wiki_symbols_repo_path_index.py
new file mode 100644
index 000000000..7f1810444
--- /dev/null
+++ b/packages/core/alembic/versions/0046_wiki_symbols_repo_path_index.py
@@ -0,0 +1,56 @@
+"""Index ``wiki_symbols`` on ``(repository_id, file_path)``.
+
+Every file-scoped symbol lookup joins on this pair, but the only index covering
+it was the one behind ``uq_wiki_symbol``, keyed ``(repository_id, symbol_id)``.
+A query filtering by ``file_path`` could therefore seek on ``repository_id``
+alone and then filter the rest in memory — on the repowise index that is 28,175
+rows scanned to return 6,937.
+
+Measured there for a 400-path ``IN`` (the shape ``_attach_symbol_ids`` issues
+when the health dashboard resolves findings to symbols): 33.3ms -> 11.7ms, with
+the plan changing from ``SEARCH ... USING INDEX sqlite_autoindex_wiki_symbols_2
+(repository_id=?)`` to a keyed seek on the new index, returning the same rows.
+
+One caveat worth recording, because it is a property of adding *any* index
+here. A query with a ``LIMIT`` and no ``ORDER BY`` returns whichever rows the
+chosen index reaches first, so changing which index the planner picks changes
+the answer. ``augment_cmd``'s symbol rescue had two such queries; both now
+order by ``(file_path, name)``, which is what the ``uq_wiki_symbol`` autoindex
+was giving them implicitly.
+
+Declared in ``WikiSymbol.__table_args__`` too. Local stores are created by
+``init_db`` and never run Alembic, while hosted only ever sees Alembic, so both
+declarations are needed and they converge on the same index. Same split as the
+``health_findings`` indexes in 0045.
+
+Revision ID: 0046
+Revises: 0045
+Create Date: 2026-08-07
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from alembic import op
+
+# revision identifiers
+revision: str = "0046"
+down_revision: str | None = "0045"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+_INDEX_NAME = "ix_wiki_symbols_repo_path"
+
+
+def upgrade() -> None:
+ op.create_index(
+ _INDEX_NAME,
+ "wiki_symbols",
+ ["repository_id", "file_path"],
+ if_not_exists=True,
+ )
+
+
+def downgrade() -> None:
+ op.drop_index(_INDEX_NAME, table_name="wiki_symbols", if_exists=True)
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py b/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py
index 54a3b35e5..950f05fd1 100644
--- a/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py
+++ b/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py
@@ -31,6 +31,7 @@ from .dead_code import (
from .health import (
HEALTH_SNAPSHOT_RETENTION,
HealthSnapshotHeadline,
+ get_average_health,
get_deduction_by_path,
get_file_language_map,
get_health_findings,
@@ -60,6 +61,7 @@ __all__ = [
"HealthSnapshotHeadline",
"covered_source_files",
"files_covered_by",
+ "get_average_health",
"get_coverage_summary",
"get_dead_code_findings",
"get_dead_code_summary",
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/health.py b/packages/core/src/repowise/core/persistence/crud/analysis/health.py
index 17190b970..6349f30c0 100644
--- a/packages/core/src/repowise/core/persistence/crud/analysis/health.py
+++ b/packages/core/src/repowise/core/persistence/crud/analysis/health.py
@@ -409,6 +409,47 @@ async def get_health_metrics(
)
+async def get_average_health(session: AsyncSession, repository_id: str) -> float | None:
+ """The repo's NLOC-weighted average health score, or ``None`` if unmeasured.
+
+ Numerically identical to ``get_health_summary()["average_health"]`` — same
+ weighting, same ``max(nloc, 1)`` floor, same 2dp rounding — for callers that
+ want *only* that number. The summary computes it as a by-product of building
+ twenty-odd KPIs, which costs a full ORM hydration of the metrics table, the
+ grouped deduction aggregate behind the worst-first ranking, the whole
+ findings table for the per-dimension counts, and the graph's language map
+ for perf coverage. The public badge endpoints want one float.
+
+ Three columns and the same exclusion filter every other health read applies.
+ That filter is a compiled ``pathspec``, not something SQL can express, which
+ is why this is a narrow select reduced in Python rather than an ``AVG()``:
+ a SQL average would quietly include files the dashboard excludes, and the
+ badge would then disagree with the page it links to.
+ """
+ rows = _filter_excluded_paths(
+ list(
+ (
+ await session.execute(
+ select(
+ HealthFileMetric.file_path,
+ HealthFileMetric.score,
+ HealthFileMetric.nloc,
+ ).where(HealthFileMetric.repository_id == repository_id)
+ )
+ ).all()
+ ),
+ await _health_exclude_spec(session, repository_id),
+ )
+ if not rows:
+ return None
+ total_nloc = sum(max(r.nloc, 1) for r in rows)
+ if total_nloc:
+ avg = sum(r.score * max(r.nloc, 1) for r in rows) / total_nloc
+ else:
+ avg = sum(r.score for r in rows) / len(rows)
+ return round(avg, 2)
+
+
async def get_file_language_map(session: AsyncSession, repository_id: str) -> dict[str, str]:
"""``{file_path: language_tag}`` for every file node in the graph."""
q = select(GraphNode.node_id, GraphNode.language).where(
diff --git a/packages/core/src/repowise/core/persistence/models.py b/packages/core/src/repowise/core/persistence/models.py
index 57c6230c1..0ce6d40f4 100644
--- a/packages/core/src/repowise/core/persistence/models.py
+++ b/packages/core/src/repowise/core/persistence/models.py
@@ -413,7 +413,22 @@ class WikiSymbol(Base):
DateTime(timezone=True), nullable=False, default=_now_utc, onupdate=_now_utc
)
- __table_args__ = (UniqueConstraint("repository_id", "symbol_id", name="uq_wiki_symbol"),)
+ __table_args__ = (
+ UniqueConstraint("repository_id", "symbol_id", name="uq_wiki_symbol"),
+ # The unique constraint's implicit index is keyed on ``symbol_id``, so a
+ # lookup by *file* could only seek on ``repository_id`` and then filter
+ # the repo's symbols in memory. That is the shape behind every
+ # file-scoped symbol join (health findings -> symbol ids, the file
+ # drawer, the symbol panel), not just one caller. Measured on a real
+ # 28,175-symbol index, a 400-path lookup returning 6,937 rows went
+ # 33.3ms -> 11.7ms, the plan flipping to a keyed seek, same rows.
+ #
+ # Adding this changed which index the planner picks for *other* queries
+ # on this table, and an unordered ``LIMIT`` there is decided by whatever
+ # order the chosen index walks. ``augment_cmd``'s symbol rescue had two
+ # such queries and now orders explicitly — see ``symbols_named``.
+ Index("ix_wiki_symbols_repo_path", "repository_id", "file_path"),
+ )
class GitMetadata(Base):
diff --git a/packages/server/src/repowise/server/routers/code_health/badge.py b/packages/server/src/repowise/server/routers/code_health/badge.py
index 3223006d1..a078ff24b 100644
--- a/packages/server/src/repowise/server/routers/code_health/badge.py
+++ b/packages/server/src/repowise/server/routers/code_health/badge.py
@@ -73,12 +73,25 @@ def _render_badge_svg(label: str, message: str, color_name: str) -> str:
async def _badge_average_health(session: AsyncSession, repo_id: str) -> float | None:
+ """The one number these endpoints render.
+
+ These are README-embedded and public, so they get hit by anyone loading the
+ README — and they were reading the entire health dataset to print ``7.5/10``.
+ ``get_health_summary`` hydrates every per-file metric row, runs the grouped
+ deduction aggregate for a ranking no badge shows, scans the whole findings
+ table for per-dimension counts, and loads the graph's language map, all to
+ have one float taken off the result. ``get_average_health`` computes that
+ float and nothing else, from the same rows with the same weighting.
+ """
repo = await crud.get_repository(session, repo_id)
if repo is None:
raise HTTPException(status_code=404, detail="Repository not found")
- summary = await crud.get_health_summary(session, repo_id)
- avg = summary.get("average_health")
- return float(avg) if avg is not None else None
+ avg = await crud.get_average_health(session, repo_id)
+ # A repo with no metrics has always badged as a perfect 10.0 here, because
+ # ``get_health_summary`` returns 10.0 for an empty table. Preserved rather
+ # than quietly corrected: the crud helper reports "unmeasured" honestly as
+ # ``None``, and mapping that to 10.0 is this endpoint's existing contract.
+ return 10.0 if avg is None else avg
@router.get("/api/repos/{repo_id}/health/badge.json")
diff --git a/tests/unit/persistence/test_average_health.py b/tests/unit/persistence/test_average_health.py
new file mode 100644
index 000000000..831efabe7
--- /dev/null
+++ b/tests/unit/persistence/test_average_health.py
@@ -0,0 +1,100 @@
+"""``get_average_health`` — the badge's one number, without the dataset.
+
+The public badge endpoints render a single float and were reading the whole
+health dataset to get it. The narrow read has to agree with
+``get_health_summary`` exactly, or the badge in a README disagrees with the
+dashboard it links to.
+"""
+
+from __future__ import annotations
+
+from repowise.core.persistence.crud import (
+ get_average_health,
+ get_health_summary,
+ save_health_metrics,
+ upsert_repository,
+)
+
+
+def _metric(path: str, score: float, nloc: int) -> dict:
+ return {
+ "file_path": path,
+ "score": score,
+ "max_ccn": 1,
+ "max_nesting": 1,
+ "nloc": nloc,
+ "has_test_file": False,
+ "module": path.split("/")[0],
+ }
+
+
+async def test_matches_the_summary_and_is_nloc_weighted(async_session, tmp_path) -> None:
+ """Deliberately lopsided NLOC: a plain mean would give 5.5, not 1.9."""
+ repo = await upsert_repository(async_session, name="repo", local_path=str(tmp_path))
+ await save_health_metrics(
+ async_session,
+ repo.id,
+ [_metric("src/big.py", 1.0, 900), _metric("src/small.py", 10.0, 100)],
+ )
+
+ avg = await get_average_health(async_session, repo.id)
+
+ assert avg == 1.9 # (1.0*900 + 10.0*100) / 1000
+ assert avg == (await get_health_summary(async_session, repo.id))["average_health"]
+
+
+async def test_zero_nloc_rows_fall_back_to_a_plain_mean(async_session, tmp_path) -> None:
+ """``max(nloc, 1)`` floors the weight, matching the summary's arithmetic."""
+ repo = await upsert_repository(async_session, name="repo", local_path=str(tmp_path))
+ await save_health_metrics(
+ async_session, repo.id, [_metric("a.py", 2.0, 0), _metric("b.py", 8.0, 0)]
+ )
+
+ assert await get_average_health(async_session, repo.id) == 5.0
+ assert (
+ await get_average_health(async_session, repo.id)
+ == (await get_health_summary(async_session, repo.id))["average_health"]
+ )
+
+
+async def test_honors_the_exclusion_spec(async_session, tmp_path) -> None:
+ """The reason this is a narrow select reduced in Python, not a SQL ``AVG``.
+
+ Exclusion is a compiled pathspec, so SQL cannot apply it. An ``AVG()`` would
+ have averaged the excluded rows in and reported a different number than the
+ dashboard — silently, and only on repos that configure excludes.
+ """
+ repo = await upsert_repository(
+ async_session,
+ name="repo",
+ local_path=str(tmp_path),
+ settings={"exclude_patterns": ["generated/"]},
+ )
+ await save_health_metrics(
+ async_session,
+ repo.id,
+ [_metric("src/app.py", 8.0, 100), _metric("generated/pb.py", 1.0, 900)],
+ )
+
+ # Unfiltered this would be 1.7; the excluded file must not count at all.
+ assert await get_average_health(async_session, repo.id) == 8.0
+ assert (
+ await get_average_health(async_session, repo.id)
+ == (await get_health_summary(async_session, repo.id))["average_health"]
+ )
+
+
+async def test_unmeasured_repo_reports_none_rather_than_a_perfect_score(
+ async_session, tmp_path
+) -> None:
+ """No rows is "not measured", not 10.0.
+
+ ``get_health_summary`` returns 10.0 here, which the badge endpoint has
+ always rendered and still does — it maps ``None`` back to 10.0 itself. The
+ crud helper stays honest so a new caller is not handed a perfect score for
+ a repo nobody has analysed.
+ """
+ repo = await upsert_repository(async_session, name="repo", local_path=str(tmp_path))
+
+ assert await get_average_health(async_session, repo.id) is None
+ assert (await get_health_summary(async_session, repo.id))["average_health"] == 10.0
diff --git a/tests/unit/persistence/test_wiki_symbols_index.py b/tests/unit/persistence/test_wiki_symbols_index.py
new file mode 100644
index 000000000..e73b4e4ac
--- /dev/null
+++ b/tests/unit/persistence/test_wiki_symbols_index.py
@@ -0,0 +1,95 @@
+"""``wiki_symbols`` must be seekable by ``(repository_id, file_path)``.
+
+The unique constraint's implicit index is keyed on ``symbol_id``, so a lookup
+by file could only seek on ``repository_id`` and filter the rest in memory —
+every symbol the repo has, to return the handful belonging to one file.
+
+The index is declared on the model *and* shipped as Alembic 0046. Local stores
+come from ``init_db`` and never run Alembic; hosted only ever runs Alembic. This
+pins the model half, which is what ``init_db`` builds from.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+
+from sqlalchemy import text
+
+from repowise.cli.commands.augment_cmd.fast_lookup import symbols_named
+from repowise.core.persistence.models import WikiSymbol
+
+_INDEX_NAME = "ix_wiki_symbols_repo_path"
+
+
+def test_index_is_declared_on_the_model() -> None:
+ by_name = {ix.name: ix for ix in WikiSymbol.__table__.indexes}
+ assert _INDEX_NAME in by_name, "init_db-created stores would lack the index"
+ assert [c.name for c in by_name[_INDEX_NAME].columns] == ["repository_id", "file_path"]
+
+
+async def test_a_file_scoped_lookup_seeks_instead_of_scanning(async_session) -> None:
+ """The plan, not the timing — a wall-clock assert would be flaky in CI."""
+ plan = (
+ await async_session.execute(
+ text(
+ "EXPLAIN QUERY PLAN SELECT symbol_id FROM wiki_symbols "
+ "WHERE repository_id = :r AND file_path IN (:a, :b)"
+ ),
+ {"r": "repo", "a": "x.py", "b": "y.py"},
+ )
+ ).all()
+
+ detail = " ".join(str(row[-1]) for row in plan)
+ assert _INDEX_NAME in detail, f"expected a seek on {_INDEX_NAME}, got: {detail}"
+ assert "file_path" in detail, f"index used but not keyed on file_path: {detail}"
+
+
+_SYMBOL_COUNT = 40
+_NAMES = [f"s{i:02d}" for i in range(_SYMBOL_COUNT)]
+
+
+def _seed_symbols(conn: sqlite3.Connection, *, with_index: bool) -> sqlite3.Connection:
+ """One file, many symbols, inserted in reverse name order.
+
+ That reversal is the whole point: rowid order and name order disagree, so a
+ plan that walks ``(repository_id, file_path)`` (then rowid) reaches
+ different rows first than one walking ``uq_wiki_symbol`` (whose key is
+ ``"<path>::<name>"``, i.e. name order within a file).
+ """
+ conn.execute(
+ "CREATE TABLE wiki_symbols (id TEXT PRIMARY KEY, repository_id TEXT, "
+ "file_path TEXT, symbol_id TEXT, name TEXT, kind TEXT, start_line INT)"
+ )
+ conn.execute("CREATE UNIQUE INDEX uq_wiki_symbol ON wiki_symbols (repository_id, symbol_id)")
+ for i, name in enumerate(reversed(_NAMES)):
+ conn.execute(
+ "INSERT INTO wiki_symbols VALUES (?,?,?,?,?,?,?)",
+ (f"id{i}", "repo", "a.py", f"a.py::{name}", name, "function", i),
+ )
+ if with_index:
+ conn.execute(f"CREATE INDEX {_INDEX_NAME} ON wiki_symbols (repository_id, file_path)")
+ conn.commit()
+ return conn
+
+
+def test_symbol_rescue_page_does_not_depend_on_which_index_exists() -> None:
+ """A ``LIMIT`` with no ``ORDER BY`` is decided by the planner's index choice.
+
+ Adding ``ix_wiki_symbols_repo_path`` changed which rows ``symbols_named``
+ returned — on this fixture the page moved from ``s00..s07`` to ``s39..s32``,
+ a disjoint set. Its caller reduces the page to a single hint, so the widened
+ rescue could name a different symbol purely because an unrelated index
+ appeared on the table. Ordering makes the page a property of the data.
+ """
+ without = _seed_symbols(sqlite3.connect(":memory:"), with_index=False)
+ with_index = _seed_symbols(sqlite3.connect(":memory:"), with_index=True)
+ try:
+ before = symbols_named(without, "repo", _NAMES, 8)
+ after = symbols_named(with_index, "repo", _NAMES, 8)
+
+ assert before == after, "the returned page moved when an index was added"
+ # And it is the page the name-keyed autoindex was giving all along.
+ assert [r[0] for r in before] == _NAMES[:8]
+ finally:
+ without.close()
+ with_index.close()