Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(health): stop the badge reading the whole dataset, and index symbols by file (#1351)
Raghav Chamadiya718af981d0eb
diff --git a/packages/cli/src/repowise/cli/commands/augment_cmd/fast_lookup.py b/packages/cli/src/repowise/cli/commands/augment_cmd/fast_lookup.pyindex d1d12190c..f3539fa70 100644--- a/packages/cli/src/repowise/cli/commands/augment_cmd/fast_lookup.py+++ b/packages/cli/src/repowise/cli/commands/augment_cmd/fast_lookup.py@@ -160,12 +160,21 @@ def symbols_named(BINARY collation unless a column says otherwise, and neither the ORM pathnor this one asks for NOCASE. The widened rescue depends on that, since itgenerates its case variants explicitly.++ Ordered because the ``LIMIT`` truncates. Unordered, the rows arrive in+ whatever order the chosen index happens to walk, so *which* rows survive+ the cut was the planner's choice — and adding an unrelated index to this+ table silently changed the answer. ``(file_path, name)`` is the order the+ ``uq_wiki_symbol`` autoindex gave for free, its key being+ ``"<path>::<name>"``, so this pins the long-standing result rather than+ picking a new one."""if not names:return []placeholders = ",".join("?" * len(names))return conn.execute("SELECT name, kind, file_path, start_line FROM wiki_symbols "- f"WHERE repository_id = ? AND name IN ({placeholders}) LIMIT ?",+ f"WHERE repository_id = ? AND name IN ({placeholders}) "+ "ORDER BY file_path, name LIMIT ?",(repository_id, *names, limit),).fetchall()diff --git a/packages/cli/src/repowise/cli/commands/augment_cmd/search.py b/packages/cli/src/repowise/cli/commands/augment_cmd/search.pyindex 635484cc8..8f5676f8c 100644--- a/packages/cli/src/repowise/cli/commands/augment_cmd/search.py+++ b/packages/cli/src/repowise/cli/commands/augment_cmd/search.py@@ -793,9 +793,16 @@ async def _rescue(# sit well outside the first two rows, and post-filtering them would# have made the widened rescue almost never fire, silently.name_clause = WikiSymbol.name.in_(sorted(variants))+ # Ordered because the LIMIT truncates: unordered, which rows survive the cut+ # is whatever order the chosen index walks, so an unrelated index added to+ # wiki_symbols changes the result. (file_path, name) is what the+ # uq_wiki_symbol autoindex gave for free — its key is "<path>::<name>" — so+ # this pins the long-standing behaviour. Mirrored in+ # ``fast_lookup.symbols_named``, which serves the same rescue.sym_stmt = (select(WikiSymbol.name, WikiSymbol.kind, WikiSymbol.file_path, WikiSymbol.start_line).where(WikiSymbol.repository_id == repo_id, name_clause)+ .order_by(WikiSymbol.file_path, WikiSymbol.name).limit(_RESCUE_TOP_N if matched is None else _RESCUE_EXACT_FETCH))rows = (await session.execute(sym_stmt)).all()diff --git a/packages/core/alembic/versions/0046_wiki_symbols_repo_path_index.py b/packages/core/alembic/versions/0046_wiki_symbols_repo_path_index.pynew file mode 100644index 000000000..7f1810444--- /dev/null+++ b/packages/core/alembic/versions/0046_wiki_symbols_repo_path_index.py@@ -0,0 +1,56 @@+"""Index ``wiki_symbols`` on ``(repository_id, file_path)``.++Every file-scoped symbol lookup joins on this pair, but the only index covering+it was the one behind ``uq_wiki_symbol``, keyed ``(repository_id, symbol_id)``.+A query filtering by ``file_path`` could therefore seek on ``repository_id``+alone and then filter the rest in memory — on the repowise index that is 28,175+rows scanned to return 6,937.++Measured there for a 400-path ``IN`` (the shape ``_attach_symbol_ids`` issues+when the health dashboard resolves findings to symbols): 33.3ms -> 11.7ms, with+the plan changing from ``SEARCH ... USING INDEX sqlite_autoindex_wiki_symbols_2+(repository_id=?)`` to a keyed seek on the new index, returning the same rows.++One caveat worth recording, because it is a property of adding *any* index+here. A query with a ``LIMIT`` and no ``ORDER BY`` returns whichever rows the+chosen index reaches first, so changing which index the planner picks changes+the answer. ``augment_cmd``'s symbol rescue had two such queries; both now+order by ``(file_path, name)``, which is what the ``uq_wiki_symbol`` autoindex+was giving them implicitly.++Declared in ``WikiSymbol.__table_args__`` too. Local stores are created by+``init_db`` and never run Alembic, while hosted only ever sees Alembic, so both+declarations are needed and they converge on the same index. Same split as the+``health_findings`` indexes in 0045.++Revision ID: 0046+Revises: 0045+Create Date: 2026-08-07+"""++from __future__ import annotations++from collections.abc import Sequence++from alembic import op++# revision identifiers+revision: str = "0046"+down_revision: str | None = "0045"+branch_labels: str | Sequence[str] | None = None+depends_on: str | Sequence[str] | None = None++_INDEX_NAME = "ix_wiki_symbols_repo_path"+++def upgrade() -> None:+ op.create_index(+ _INDEX_NAME,+ "wiki_symbols",+ ["repository_id", "file_path"],+ if_not_exists=True,+ )+++def downgrade() -> None:+ op.drop_index(_INDEX_NAME, table_name="wiki_symbols", if_exists=True)diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py b/packages/core/src/repowise/core/persistence/crud/analysis/__init__.pyindex 54a3b35e5..950f05fd1 100644--- a/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py+++ b/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py@@ -31,6 +31,7 @@ from .dead_code import (from .health import (HEALTH_SNAPSHOT_RETENTION,HealthSnapshotHeadline,+ get_average_health,get_deduction_by_path,get_file_language_map,get_health_findings,@@ -60,6 +61,7 @@ __all__ = ["HealthSnapshotHeadline","covered_source_files","files_covered_by",+ "get_average_health","get_coverage_summary","get_dead_code_findings","get_dead_code_summary",diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/health.py b/packages/core/src/repowise/core/persistence/crud/analysis/health.pyindex 17190b970..6349f30c0 100644--- a/packages/core/src/repowise/core/persistence/crud/analysis/health.py+++ b/packages/core/src/repowise/core/persistence/crud/analysis/health.py@@ -409,6 +409,47 @@ async def get_health_metrics()+async def get_average_health(session: AsyncSession, repository_id: str) -> float | None:+ """The repo's NLOC-weighted average health score, or ``None`` if unmeasured.++ Numerically identical to ``get_health_summary()["average_health"]`` — same+ weighting, same ``max(nloc, 1)`` floor, same 2dp rounding — for callers that+ want *only* that number. The summary computes it as a by-product of building+ twenty-odd KPIs, which costs a full ORM hydration of the metrics table, the+ grouped deduction aggregate behind the worst-first ranking, the whole+ findings table for the per-dimension counts, and the graph's language map+ for perf coverage. The public badge endpoints want one float.++ Three columns and the same exclusion filter every other health read applies.+ That filter is a compiled ``pathspec``, not something SQL can express, which+ is why this is a narrow select reduced in Python rather than an ``AVG()``:+ a SQL average would quietly include files the dashboard excludes, and the+ badge would then disagree with the page it links to.+ """+ rows = _filter_excluded_paths(+ list(+ (+ await session.execute(+ select(+ HealthFileMetric.file_path,+ HealthFileMetric.score,+ HealthFileMetric.nloc,+ ).where(HealthFileMetric.repository_id == repository_id)+ )+ ).all()+ ),+ await _health_exclude_spec(session, repository_id),+ )+ if not rows:+ return None+ total_nloc = sum(max(r.nloc, 1) for r in rows)+ if total_nloc:+ avg = sum(r.score * max(r.nloc, 1) for r in rows) / total_nloc+ else:+ avg = sum(r.score for r in rows) / len(rows)+ return round(avg, 2)++async def get_file_language_map(session: AsyncSession, repository_id: str) -> dict[str, str]:"""``{file_path: language_tag}`` for every file node in the graph."""q = select(GraphNode.node_id, GraphNode.language).where(diff --git a/packages/core/src/repowise/core/persistence/models.py b/packages/core/src/repowise/core/persistence/models.pyindex 57c6230c1..0ce6d40f4 100644--- a/packages/core/src/repowise/core/persistence/models.py+++ b/packages/core/src/repowise/core/persistence/models.py@@ -413,7 +413,22 @@ class WikiSymbol(Base):DateTime(timezone=True), nullable=False, default=_now_utc, onupdate=_now_utc)- __table_args__ = (UniqueConstraint("repository_id", "symbol_id", name="uq_wiki_symbol"),)+ __table_args__ = (+ UniqueConstraint("repository_id", "symbol_id", name="uq_wiki_symbol"),+ # The unique constraint's implicit index is keyed on ``symbol_id``, so a+ # lookup by *file* could only seek on ``repository_id`` and then filter+ # the repo's symbols in memory. That is the shape behind every+ # file-scoped symbol join (health findings -> symbol ids, the file+ # drawer, the symbol panel), not just one caller. Measured on a real+ # 28,175-symbol index, a 400-path lookup returning 6,937 rows went+ # 33.3ms -> 11.7ms, the plan flipping to a keyed seek, same rows.+ #+ # Adding this changed which index the planner picks for *other* queries+ # on this table, and an unordered ``LIMIT`` there is decided by whatever+ # order the chosen index walks. ``augment_cmd``'s symbol rescue had two+ # such queries and now orders explicitly — see ``symbols_named``.+ Index("ix_wiki_symbols_repo_path", "repository_id", "file_path"),+ )class GitMetadata(Base):diff --git a/packages/server/src/repowise/server/routers/code_health/badge.py b/packages/server/src/repowise/server/routers/code_health/badge.pyindex 3223006d1..a078ff24b 100644--- a/packages/server/src/repowise/server/routers/code_health/badge.py+++ b/packages/server/src/repowise/server/routers/code_health/badge.py@@ -73,12 +73,25 @@ def _render_badge_svg(label: str, message: str, color_name: str) -> str:async def _badge_average_health(session: AsyncSession, repo_id: str) -> float | None:+ """The one number these endpoints render.++ These are README-embedded and public, so they get hit by anyone loading the+ README — and they were reading the entire health dataset to print ``7.5/10``.+ ``get_health_summary`` hydrates every per-file metric row, runs the grouped+ deduction aggregate for a ranking no badge shows, scans the whole findings+ table for per-dimension counts, and loads the graph's language map, all to+ have one float taken off the result. ``get_average_health`` computes that+ float and nothing else, from the same rows with the same weighting.+ """repo = await crud.get_repository(session, repo_id)if repo is None:raise HTTPException(status_code=404, detail="Repository not found")- summary = await crud.get_health_summary(session, repo_id)- avg = summary.get("average_health")- return float(avg) if avg is not None else None+ avg = await crud.get_average_health(session, repo_id)+ # A repo with no metrics has always badged as a perfect 10.0 here, because+ # ``get_health_summary`` returns 10.0 for an empty table. Preserved rather+ # than quietly corrected: the crud helper reports "unmeasured" honestly as+ # ``None``, and mapping that to 10.0 is this endpoint's existing contract.+ return 10.0 if avg is None else avg@router.get("/api/repos/{repo_id}/health/badge.json")diff --git a/tests/unit/persistence/test_average_health.py b/tests/unit/persistence/test_average_health.pynew file mode 100644index 000000000..831efabe7--- /dev/null+++ b/tests/unit/persistence/test_average_health.py@@ -0,0 +1,100 @@+"""``get_average_health`` — the badge's one number, without the dataset.++The public badge endpoints render a single float and were reading the whole+health dataset to get it. The narrow read has to agree with+``get_health_summary`` exactly, or the badge in a README disagrees with the+dashboard it links to.+"""++from __future__ import annotations++from repowise.core.persistence.crud import (+ get_average_health,+ get_health_summary,+ save_health_metrics,+ upsert_repository,+)+++def _metric(path: str, score: float, nloc: int) -> dict:+ return {+ "file_path": path,+ "score": score,+ "max_ccn": 1,+ "max_nesting": 1,+ "nloc": nloc,+ "has_test_file": False,+ "module": path.split("/")[0],+ }+++async def test_matches_the_summary_and_is_nloc_weighted(async_session, tmp_path) -> None:+ """Deliberately lopsided NLOC: a plain mean would give 5.5, not 1.9."""+ repo = await upsert_repository(async_session, name="repo", local_path=str(tmp_path))+ await save_health_metrics(+ async_session,+ repo.id,+ [_metric("src/big.py", 1.0, 900), _metric("src/small.py", 10.0, 100)],+ )++ avg = await get_average_health(async_session, repo.id)++ assert avg == 1.9 # (1.0*900 + 10.0*100) / 1000+ assert avg == (await get_health_summary(async_session, repo.id))["average_health"]+++async def test_zero_nloc_rows_fall_back_to_a_plain_mean(async_session, tmp_path) -> None:+ """``max(nloc, 1)`` floors the weight, matching the summary's arithmetic."""+ repo = await upsert_repository(async_session, name="repo", local_path=str(tmp_path))+ await save_health_metrics(+ async_session, repo.id, [_metric("a.py", 2.0, 0), _metric("b.py", 8.0, 0)]+ )++ assert await get_average_health(async_session, repo.id) == 5.0+ assert (+ await get_average_health(async_session, repo.id)+ == (await get_health_summary(async_session, repo.id))["average_health"]+ )+++async def test_honors_the_exclusion_spec(async_session, tmp_path) -> None:+ """The reason this is a narrow select reduced in Python, not a SQL ``AVG``.++ Exclusion is a compiled pathspec, so SQL cannot apply it. An ``AVG()`` would+ have averaged the excluded rows in and reported a different number than the+ dashboard — silently, and only on repos that configure excludes.+ """+ repo = await upsert_repository(+ async_session,+ name="repo",+ local_path=str(tmp_path),+ settings={"exclude_patterns": ["generated/"]},+ )+ await save_health_metrics(+ async_session,+ repo.id,+ [_metric("src/app.py", 8.0, 100), _metric("generated/pb.py", 1.0, 900)],+ )++ # Unfiltered this would be 1.7; the excluded file must not count at all.+ assert await get_average_health(async_session, repo.id) == 8.0+ assert (+ await get_average_health(async_session, repo.id)+ == (await get_health_summary(async_session, repo.id))["average_health"]+ )+++async def test_unmeasured_repo_reports_none_rather_than_a_perfect_score(+ async_session, tmp_path+) -> None:+ """No rows is "not measured", not 10.0.++ ``get_health_summary`` returns 10.0 here, which the badge endpoint has+ always rendered and still does — it maps ``None`` back to 10.0 itself. The+ crud helper stays honest so a new caller is not handed a perfect score for+ a repo nobody has analysed.+ """+ repo = await upsert_repository(async_session, name="repo", local_path=str(tmp_path))++ assert await get_average_health(async_session, repo.id) is None+ assert (await get_health_summary(async_session, repo.id))["average_health"] == 10.0diff --git a/tests/unit/persistence/test_wiki_symbols_index.py b/tests/unit/persistence/test_wiki_symbols_index.pynew file mode 100644index 000000000..e73b4e4ac--- /dev/null+++ b/tests/unit/persistence/test_wiki_symbols_index.py@@ -0,0 +1,95 @@+"""``wiki_symbols`` must be seekable by ``(repository_id, file_path)``.++The unique constraint's implicit index is keyed on ``symbol_id``, so a lookup+by file could only seek on ``repository_id`` and filter the rest in memory —+every symbol the repo has, to return the handful belonging to one file.++The index is declared on the model *and* shipped as Alembic 0046. Local stores+come from ``init_db`` and never run Alembic; hosted only ever runs Alembic. This+pins the model half, which is what ``init_db`` builds from.+"""++from __future__ import annotations++import sqlite3++from sqlalchemy import text++from repowise.cli.commands.augment_cmd.fast_lookup import symbols_named+from repowise.core.persistence.models import WikiSymbol++_INDEX_NAME = "ix_wiki_symbols_repo_path"+++def test_index_is_declared_on_the_model() -> None:+ by_name = {ix.name: ix for ix in WikiSymbol.__table__.indexes}+ assert _INDEX_NAME in by_name, "init_db-created stores would lack the index"+ assert [c.name for c in by_name[_INDEX_NAME].columns] == ["repository_id", "file_path"]+++async def test_a_file_scoped_lookup_seeks_instead_of_scanning(async_session) -> None:+ """The plan, not the timing — a wall-clock assert would be flaky in CI."""+ plan = (+ await async_session.execute(+ text(+ "EXPLAIN QUERY PLAN SELECT symbol_id FROM wiki_symbols "+ "WHERE repository_id = :r AND file_path IN (:a, :b)"+ ),+ {"r": "repo", "a": "x.py", "b": "y.py"},+ )+ ).all()++ detail = " ".join(str(row[-1]) for row in plan)+ assert _INDEX_NAME in detail, f"expected a seek on {_INDEX_NAME}, got: {detail}"+ assert "file_path" in detail, f"index used but not keyed on file_path: {detail}"+++_SYMBOL_COUNT = 40+_NAMES = [f"s{i:02d}" for i in range(_SYMBOL_COUNT)]+++def _seed_symbols(conn: sqlite3.Connection, *, with_index: bool) -> sqlite3.Connection:+ """One file, many symbols, inserted in reverse name order.++ That reversal is the whole point: rowid order and name order disagree, so a+ plan that walks ``(repository_id, file_path)`` (then rowid) reaches+ different rows first than one walking ``uq_wiki_symbol`` (whose key is+ ``"<path>::<name>"``, i.e. name order within a file).+ """+ conn.execute(+ "CREATE TABLE wiki_symbols (id TEXT PRIMARY KEY, repository_id TEXT, "+ "file_path TEXT, symbol_id TEXT, name TEXT, kind TEXT, start_line INT)"+ )+ conn.execute("CREATE UNIQUE INDEX uq_wiki_symbol ON wiki_symbols (repository_id, symbol_id)")+ for i, name in enumerate(reversed(_NAMES)):+ conn.execute(+ "INSERT INTO wiki_symbols VALUES (?,?,?,?,?,?,?)",+ (f"id{i}", "repo", "a.py", f"a.py::{name}", name, "function", i),+ )+ if with_index:+ conn.execute(f"CREATE INDEX {_INDEX_NAME} ON wiki_symbols (repository_id, file_path)")+ conn.commit()+ return conn+++def test_symbol_rescue_page_does_not_depend_on_which_index_exists() -> None:+ """A ``LIMIT`` with no ``ORDER BY`` is decided by the planner's index choice.++ Adding ``ix_wiki_symbols_repo_path`` changed which rows ``symbols_named``+ returned — on this fixture the page moved from ``s00..s07`` to ``s39..s32``,+ a disjoint set. Its caller reduces the page to a single hint, so the widened+ rescue could name a different symbol purely because an unrelated index+ appeared on the table. Ordering makes the page a property of the data.+ """+ without = _seed_symbols(sqlite3.connect(":memory:"), with_index=False)+ with_index = _seed_symbols(sqlite3.connect(":memory:"), with_index=True)+ try:+ before = symbols_named(without, "repo", _NAMES, 8)+ after = symbols_named(with_index, "repo", _NAMES, 8)++ assert before == after, "the returned page moved when an index was added"+ # And it is the page the name-keyed autoindex was giving all along.+ assert [r[0] for r in before] == _NAMES[:8]+ finally:+ without.close()+ with_index.close()