Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(stats): one language split and one module count on every surface (#2710)

Swati Ahujaff948ac73960
diff --git a/packages/core/src/repowise/core/analysis/health/aggregation.py b/packages/core/src/repowise/core/analysis/health/aggregation.py
index 1def03d33..f436cb0a2 100644
--- a/packages/core/src/repowise/core/analysis/health/aggregation.py
+++ b/packages/core/src/repowise/core/analysis/health/aggregation.py
@@ -31,11 +31,13 @@ from .scoring import (
__all__ = [
"MODULE_ROOT_LABEL",
+ "NLOC_NULL_REASON",
"SEVERITY_ORDER",
"biomarker_breakdown",
"finding_base_deduction",
"finding_raw_deduction",
"module_label",
+ "module_labels",
"module_rollups",
"primary_and_magnitude",
"primary_and_magnitude_by_file",
@@ -46,6 +48,9 @@ __all__ = [
#: Label for a file that sits directly in the repository root.
MODULE_ROOT_LABEL = "root"
+#: Why a file has no NLOC, for surfaces that show the gap as null.
+NLOC_NULL_REASON = "not measured: health analysis counts NLOC only in files it parses as code"
+
#: The buckets a severity breakdown always declares, worst first. Fixed rather
#: than derived from the data so an absent severity reads as 0, not as missing.
SEVERITY_ORDER: tuple[str, ...] = ("critical", "high", "medium", "low")
@@ -66,6 +71,15 @@ def module_label(file_path: str | None) -> str:
return head if separator else MODULE_ROOT_LABEL
+def module_labels(file_paths: Iterable[str | None]) -> set[str]:
+ """The modules *file_paths* span: the one axis every module count uses.
+
+ Module health buckets on :func:`module_label`, so a count taken any other way
+ (package boundaries, top two segments) names a number no module list shows.
+ """
+ return {module_label(path) for path in file_paths if path}
+
+
def _nloc(row: Any) -> int:
"""A row's NLOC as a non-negative int; a missing or null value is 0."""
try:
diff --git a/packages/core/src/repowise/core/stats_highlights.py b/packages/core/src/repowise/core/stats_highlights.py
index 384d12737..3e1bc3955 100644
--- a/packages/core/src/repowise/core/stats_highlights.py
+++ b/packages/core/src/repowise/core/stats_highlights.py
@@ -108,49 +108,74 @@ def _is_external(node: Row) -> bool:
# ---------------------------------------------------------------------------
+def file_mix(file_nodes: Iterable[Row]) -> dict[str, Any]:
+ """The language split and module count every surface reports for a file set.
+
+ ``languages`` holds code: any ``is_code`` language, parsed or not, so a
+ Haskell repo still lists Haskell. ``docs_config_languages`` holds config,
+ markup, data and unregistered tags, which a developer would not list as what
+ a project is written in. ``module_count`` is the code files' modules on the
+ :func:`module_label` axis, the one module health buckets on.
+ """
+ from repowise.core.analysis.health.aggregation import module_labels
+ from repowise.core.ingestion.languages.registry import REGISTRY
+
+ code: Counter[str] = Counter()
+ other: Counter[str] = Counter()
+ code_paths: list[str] = []
+ for node in file_nodes:
+ lang = node.get("language")
+ if not lang or _is_external(node):
+ continue
+ spec = REGISTRY.get(lang)
+ if spec is not None and spec.is_code:
+ code[lang] += 1
+ code_paths.append(str(node.get("node_id") or node.get("id") or ""))
+ else:
+ other[lang] += 1
+ return {
+ "languages": _ranked_languages(code),
+ "docs_config_languages": _ranked_languages(other),
+ "language_count": len(code),
+ "module_count": len(module_labels(code_paths)),
+ }
+
+
+def _ranked_languages(counts: Counter[str]) -> list[dict[str, Any]]:
+ """Busiest first; ties on the name so the order is stable across requests."""
+ ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
+ return [{"language": lang, "file_count": n} for lang, n in ranked]
+
+
def build_scale(file_nodes: Iterable[Row], metrics: Iterable[Row]) -> dict[str, Any]:
- """Counts, NLOC and the language mix.
+ """Counts and NLOC, plus the :func:`file_mix` languages and module count.
``file_nodes`` are graph file nodes; external dependency nodes are dropped
- so the file count describes the repo. Only code languages are counted:
- JSON, YAML and Markdown are formats a developer would not list as the
- languages a project is written in.
+ so the file count describes the repo.
"""
- from repowise.core.ingestion.languages.registry import REGISTRY
-
- code = REGISTRY.code_languages()
+ file_nodes = list(file_nodes)
file_count = 0
symbol_count = 0
- langs: Counter[str] = Counter()
for node in file_nodes:
if _is_external(node):
continue
file_count += 1
symbol_count += _int(node.get("symbol_count"))
- lang = node.get("language")
- if lang in code:
- langs[lang] += 1
total_nloc = 0
test_nloc = 0
- modules: set[str] = set()
for m in metrics:
nloc = _int(m.get("nloc"))
total_nloc += nloc
if m.get("is_test"):
test_nloc += nloc
- if m.get("module"):
- modules.add(m["module"])
- languages = [{"language": k, "file_count": v} for k, v in langs.most_common()]
return {
"file_count": file_count,
"symbol_count": symbol_count,
- "module_count": len(modules),
"total_nloc": total_nloc,
"test_nloc": test_nloc,
- "language_count": len(languages),
- "languages": languages,
+ **file_mix(file_nodes),
"size_class": size_class(total_nloc),
}
diff --git a/packages/server/src/repowise/server/mcp_server/tool_overview/history.py b/packages/server/src/repowise/server/mcp_server/tool_overview/history.py
index e5147c486..1593b6608 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_overview/history.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_overview/history.py
@@ -5,6 +5,8 @@ from __future__ import annotations
from collections import Counter, defaultdict
from typing import Any
+from repowise.core.analysis.health.aggregation import module_label
+
def _build_git_health(all_git: list) -> dict[str, Any]:
"""Repo-wide git health summary (hotspots, bus factor, churn trend, top modules)."""
@@ -37,10 +39,10 @@ def _churn_trend(all_git: list) -> str:
def _top_churn_modules(all_git: list) -> list[str]:
- """Five busiest modules by 90-day commits, keyed on the first two path segments."""
+ """Five busiest modules by 90-day commits, on the module axis every count uses."""
module_churn: Counter = Counter()
for g in all_git:
- module_churn["/".join(g.file_path.split("/")[:2])] += g.commit_count_90d or 0
+ module_churn[module_label(g.file_path)] += g.commit_count_90d or 0
return [m for m, _ in module_churn.most_common(5) if module_churn[m] > 0]
diff --git a/packages/server/src/repowise/server/routers/files.py b/packages/server/src/repowise/server/routers/files.py
index 938cf77ee..848ce8e4f 100644
--- a/packages/server/src/repowise/server/routers/files.py
+++ b/packages/server/src/repowise/server/routers/files.py
@@ -18,6 +18,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
+from repowise.core.analysis.health.aggregation import NLOC_NULL_REASON
from repowise.core.analysis.health.signals import file_signals
from repowise.core.analysis.health.trends import file_trend
from repowise.core.ids import is_external
@@ -220,6 +221,8 @@ async def files_index(
"files": files,
"total": len(files),
"languages": languages,
+ "loc_unit": "nloc",
+ "loc_null_reason": NLOC_NULL_REASON,
}
diff --git a/packages/server/src/repowise/server/routers/overview.py b/packages/server/src/repowise/server/routers/overview.py
index 9cecf2c6f..1905ad500 100644
--- a/packages/server/src/repowise/server/routers/overview.py
+++ b/packages/server/src/repowise/server/routers/overview.py
@@ -32,6 +32,7 @@ from repowise.core.persistence.models import (
GraphNode,
Page,
)
+from repowise.core.stats_highlights import file_mix
from repowise.server.deps import get_db_session, verify_api_key
from repowise.server.routers.git import _hotspot_from_row
from repowise.server.services.attention import build_attention
@@ -297,23 +298,24 @@ async def overview_summary(
if owner:
bucket = module_owner_files.setdefault(module, {})
bucket[owner] = bucket.get(owner, 0) + 1
- module_count = len(module_file_totals)
silo_count = 0
for module, owners in module_owner_files.items():
top = max(owners.values(), default=0)
if module_file_totals.get(module) and top / module_file_totals[module] > 0.8:
silo_count += 1
- # --- Language distribution (server-side; replaces the full graph export)
- lang_rows = await session.execute(
- select(GraphNode.language, func.count(GraphNode.id))
- .where(GraphNode.repository_id == repo_id, GraphNode.node_type == "file")
- .group_by(GraphNode.language)
- )
- languages = sorted(
- ({"language": lang or "other", "file_count": n} for lang, n in lang_rows),
- key=lambda r: -r["file_count"],
+ # Languages and module count come from the one rule Stats uses, so the two
+ # pages cannot disagree on either number.
+ # One narrow row per file node, not a GROUP BY: module_count needs paths. Fine to
+ # ~100k files; past that, count languages in SQL and fetch only code paths.
+ node_rows = await session.execute(
+ select(GraphNode.node_id, GraphNode.language, GraphNode.external_system_id).where(
+ GraphNode.repository_id == repo_id, GraphNode.node_type == "file"
+ )
)
+ mix = file_mix(row._mapping for row in node_rows)
+ module_count = mix["module_count"]
+ languages = mix["languages"]
# --- Health KPIs + deltas vs previous snapshot ------------------------
# Loaded once and handed to both consumers below: the KPI rollup and the
@@ -574,6 +576,7 @@ async def overview_summary(
],
},
"languages": languages,
+ "docs_config_languages": mix["docs_config_languages"],
"attention": attention,
# Counts for everything the sources hold, beside the capped list. The
# page shows five rows out of these, and "5 open" would be a lie that
diff --git a/packages/types/src/files.ts b/packages/types/src/files.ts
index 8de6962bf..6fb6549a4 100644
--- a/packages/types/src/files.ts
+++ b/packages/types/src/files.ts
@@ -217,6 +217,7 @@ export interface FileDeadCodeFinding {
export interface FileRow {
file_path: string;
language: string;
+ /** NLOC, not raw lines; null where health measured none (see `loc_null_reason`). */
loc: number | null;
symbol_count: number;
pagerank_pct: number;
@@ -246,6 +247,10 @@ export interface FilesIndexResponse {
files: FileRow[];
total: number;
languages: FileLanguageCount[];
+ /** What `loc` counts. Optional so older servers parse. */
+ loc_unit?: "nloc";
+ /** Why a row's `loc` is null. */
+ loc_null_reason?: string;
}
export interface FileDetailResponse {
diff --git a/packages/types/src/overview.ts b/packages/types/src/overview.ts
index eaf2a5c1e..b671750c2 100644
--- a/packages/types/src/overview.ts
+++ b/packages/types/src/overview.ts
@@ -250,7 +250,10 @@ export interface OverviewSummaryResponse {
repo: OverviewRepoMeta;
stats: OverviewStats;
health: OverviewHealth;
+ /** Code languages only. */
languages: OverviewLanguage[];
+ /** Config, markup and data formats, kept apart from `languages`. */
+ docs_config_languages?: OverviewLanguage[];
attention: OverviewAttentionItem[];
/** Optional: a server predating the merged attention list omits it, and the
* UI then falls back to counting the rows it was given. */
diff --git a/packages/types/src/stats.ts b/packages/types/src/stats.ts
index 3ca19e225..e80bae3da 100644
--- a/packages/types/src/stats.ts
+++ b/packages/types/src/stats.ts
@@ -34,6 +34,8 @@ export interface StatsScale {
/** Code languages only: data and markup formats are not counted. */
language_count: number;
languages: StatsLanguage[];
+ /** Config, markup and data formats, kept apart from `languages`. */
+ docs_config_languages?: StatsLanguage[];
size_class: StatsSizeClass;
}
diff --git a/tests/unit/server/test_count_parity.py b/tests/unit/server/test_count_parity.py
new file mode 100644
index 000000000..48ed1a9e5
--- /dev/null
+++ b/tests/unit/server/test_count_parity.py
@@ -0,0 +1,114 @@
+"""One number per concept: overview, stats and module health agree.
+
+Languages come from one split (code apart from docs/config) and module counts
+from one axis (the top-level directory module health buckets on), so no two
+surfaces can print different figures for the same repository.
+"""
+
+from __future__ import annotations
+
+import pytest
+from httpx import AsyncClient
+
+from repowise.core.persistence import crud
+from repowise.core.persistence.database import get_session
+from repowise.core.persistence.models import GraphNode
+from repowise.core.stats_highlights import file_mix
+from repowise.server.routers.stats import stats_highlights
+from tests.unit.server.conftest import create_test_repo
+
+# Two code modules plus root, a docs-only directory and a config file.
+_FILES = (
+ ("src/app.py", "python"),
+ ("src/util.py", "python"),
+ ("web/index.ts", "typescript"),
+ ("setup.py", "python"),
+ ("docs/guide.md", "markdown"),
+ ("config.yaml", "yaml"),
+)
+
+
+async def _seed(session_factory, repo_id: str) -> None:
+ async with get_session(session_factory) as session:
+ for path, language in _FILES:
+ session.add(
+ GraphNode(
+ repository_id=repo_id,
+ node_id=path,
+ node_type="file",
+ language=language,
+ is_test=False,
+ )
+ )
+ is_code = language in {"python", "typescript"}
+ await crud.upsert_git_metadata(
+ session,
+ repository_id=repo_id,
+ file_path=path,
+ commit_count_total=3,
+ primary_owner_name="Alice",
+ primary_owner_email="[email protected]",
+ history_only=not is_code,
+ )
+ await session.commit()
+
+
+async def _surfaces(client: AsyncClient, app) -> tuple[dict, dict, dict]:
+ """The overview payload, the Stats scale block and the module health list."""
+ repo = await create_test_repo(client)
+ await _seed(app.state.session_factory, repo["id"])
+ overview = (await client.get(f"/api/repos/{repo['id']}/overview-summary")).json()
+ modules = (await client.get(f"/api/repos/{repo['id']}/modules/health")).json()
+ async with get_session(app.state.session_factory) as session:
+ scale = (await stats_highlights(repo["id"], session))["scale"]
+ return overview, scale, modules
+
+
+async def test_overview_stats_language_parity(client: AsyncClient, app) -> None:
+ overview, scale, _ = await _surfaces(client, app)
+
+ assert overview["languages"] == scale["languages"]
+ assert overview["docs_config_languages"] == scale["docs_config_languages"]
+ assert [row["language"] for row in scale["languages"]] == ["python", "typescript"]
+ assert {row["language"] for row in scale["docs_config_languages"]} == {"markdown", "yaml"}
+ assert scale["language_count"] == 2
+
+
+async def test_module_count_matches_modules_health(client: AsyncClient, app) -> None:
+ overview, scale, modules = await _surfaces(client, app)
+
+ assert {m["module_path"] for m in modules["items"]} == {"src", "web", "root"}
+ assert overview["stats"]["module_count"] == scale["module_count"] == modules["total"] == 3
+
+
+def test_file_mix_splits_code_from_docs_config() -> None:
+ mix = file_mix(
+ [
+ {"node_id": "lib/Main.hs", "language": "haskell"},
+ {"node_id": "lib/Util.hs", "language": "haskell"},
+ {"node_id": "README.md", "language": "markdown"},
+ {"node_id": "data.weird", "language": "not-a-language"},
+ {"node_id": "external:base", "language": "external"},
+ {"node_id": "unparsed", "language": None},
+ ]
+ )
+ # A language with no AST parser is still what the project is written in.
+ assert mix["languages"] == [{"language": "haskell", "file_count": 2}]
+ assert mix["docs_config_languages"] == [
+ {"language": "markdown", "file_count": 1},
+ {"language": "not-a-language", "file_count": 1},
+ ]
+ assert mix["module_count"] == 1
+
+
+async def test_files_index_labels_loc_as_nloc(client: AsyncClient, app) -> None:
+ repo = await create_test_repo(client)
+ await _seed(app.state.session_factory, repo["id"])
+
+ body = (await client.get(f"/api/repos/{repo['id']}/files")).json()
+ assert body["loc_unit"] == "nloc"
+ assert body["loc_null_reason"]
+ assert all(row["loc"] is None for row in body["files"])