Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(chat): build chart dataset rows from plain records (#3951)

Raghav Chamadiya5dc62ea69ef6
diff --git a/packages/server/src/repowise/server/chat_datasets/architecture.py b/packages/server/src/repowise/server/chat_datasets/architecture.py
index 2c7b582fb..86e16b094 100644
--- a/packages/server/src/repowise/server/chat_datasets/architecture.py
+++ b/packages/server/src/repowise/server/chat_datasets/architecture.py
@@ -8,8 +8,10 @@ the architecture layers get_overview lists, in stack order.
from __future__ import annotations
import json
+from collections.abc import Iterable
from typing import Any
+from repowise.core.analysis.health.rows import field
from repowise.server.chat_datasets import (
ARROW,
SUMS,
@@ -40,18 +42,31 @@ async def load_dependencies(ctx: LoadContext) -> list[dict[str, Any]]:
containers, relations = await container_dependencies(
ctx.session, ctx.repository.id, exclude_spec=_get_exclude_spec(ctx.repo_path)
)
- path_of = {c.id: c.path or "." for c in containers}
- return [
- {
- "edge": f"{path_of[r.source_id]} {ARROW} {path_of[r.target_id]}",
- "from_package": path_of[r.source_id],
- "to_package": path_of[r.target_id],
- "weight": r.edge_count,
- "coupling": r.coupling,
- }
- for r in relations
- if r.source_id in path_of and r.target_id in path_of
- ]
+ return dependencies_rows(containers, relations)
+
+
+def dependencies_rows(containers: Iterable[Any], relations: Iterable[Any]) -> list[dict[str, Any]]:
+ """The ``dependencies`` dataset's rows over the container view's boxes and edges.
+
+ *containers* hold ``id`` and ``path``; *relations* hold ``source_id``,
+ ``target_id``, ``edge_count`` and ``coupling``. An edge to a box not in
+ *containers* is left out.
+ """
+ path_of = {field(c, "id"): field(c, "path") or "." for c in containers}
+ rows = []
+ for r in relations:
+ source, target = field(r, "source_id"), field(r, "target_id")
+ if source in path_of and target in path_of:
+ rows.append(
+ {
+ "edge": f"{path_of[source]} {ARROW} {path_of[target]}",
+ "from_package": path_of[source],
+ "to_package": path_of[target],
+ "weight": field(r, "edge_count"),
+ "coupling": field(r, "coupling"),
+ }
+ )
+ return rows
@register_dataset(
@@ -67,19 +82,28 @@ async def load_dependencies(ctx: LoadContext) -> list[dict[str, Any]]:
noun="layers",
)
async def load_layers(ctx: LoadContext) -> list[dict[str, Any]]:
- from repowise.core.generation.layers import is_adjacent_layer
from repowise.core.persistence.crud.knowledge_graph import get_kg_layers
- layers = [
- layer
- for layer in await get_kg_layers(ctx.session, ctx.repository.id)
- if not is_adjacent_layer(layer.layer_id)
- ]
- return [
- {
- "layer": layer.name,
- "files": len(json.loads(layer.node_ids_json) if layer.node_ids_json else []),
- "order": i + 1,
- }
- for i, layer in enumerate(layers)
- ]
+ return layers_rows(await get_kg_layers(ctx.session, ctx.repository.id))
+
+
+def layers_rows(layers: Iterable[Any]) -> list[dict[str, Any]]:
+ """The ``layers`` dataset's rows over knowledge-graph layers, in the order given.
+
+ Each record holds ``layer_id``, ``name`` and ``node_ids_json``; adjacent
+ layers (outside the stack) are left out.
+ """
+ from repowise.core.generation.layers import is_adjacent_layer
+
+ stack = [layer for layer in layers if not is_adjacent_layer(field(layer, "layer_id"))]
+ rows = []
+ for i, layer in enumerate(stack):
+ node_ids = field(layer, "node_ids_json")
+ rows.append(
+ {
+ "layer": field(layer, "name"),
+ "files": len(json.loads(node_ids) if node_ids else []),
+ "order": i + 1,
+ }
+ )
+ return rows
diff --git a/packages/server/src/repowise/server/chat_datasets/files.py b/packages/server/src/repowise/server/chat_datasets/files.py
index 9030a0f36..dd6a0f61b 100644
--- a/packages/server/src/repowise/server/chat_datasets/files.py
+++ b/packages/server/src/repowise/server/chat_datasets/files.py
@@ -10,11 +10,12 @@ columns, and the coverage report's per-file rows.
from __future__ import annotations
from collections import Counter, defaultdict
-from collections.abc import Iterable
+from collections.abc import Iterable, Mapping
from typing import Any
from sqlalchemy import select
+from repowise.core.analysis.health.rows import field
from repowise.server.chat_datasets import (
AVERAGES,
SUMS,
@@ -143,9 +144,105 @@ def _top_level(path: str) -> str:
return head if sep else ROOT_PACKAGE
-async def files_index(ctx: LoadContext) -> dict[str, dict[str, Any]]:
- """The files dataset by path, for datasets that place rows by file."""
- return {row["path"]: row for row in await ctx.rows("files")}
+def by_path(files: Iterable[Mapping[str, Any]]) -> dict[str, Mapping[str, Any]]:
+ """The files dataset's rows by path, for datasets that place rows by file."""
+ return {row["path"]: row for row in files}
+
+
+# ---------------------------------------------------------------------------
+# Rows over records already read
+# ---------------------------------------------------------------------------
+
+
+def files_rows(
+ metrics: Iterable[Any],
+ git: Iterable[Any],
+ coverage: Iterable[Any],
+ languages: Mapping[str, str | None],
+) -> list[dict[str, Any]]:
+ """The ``files`` dataset's rows over records a caller already holds.
+
+ *metrics* are the per-file health metrics (``file_path``, ``score``,
+ ``nloc``, ``max_ccn``, ``module``, ``is_test``, ``code_origin``) the
+ health exclude rule kept; *git* the code files' git metadata
+ (``file_path``, ``commit_count_90d``, ``prior_defect_count``,
+ ``is_hotspot``); *coverage* the coverage report's rows (``file_path``,
+ ``line_coverage_pct``, ``total_coverable_lines``); *languages* maps a path
+ to its language. A record is a dataclass, an ORM or SQL row, or a dict.
+ """
+ from repowise.core.analysis.health.grading import band_for
+ from repowise.core.analysis.health.ranking import scope_reason
+
+ metrics = list(metrics)
+ git_of = {field(g, "file_path"): g for g in git}
+ coverage_of = {field(c, "file_path"): c for c in coverage}
+ paths = [field(m, "file_path") for m in metrics]
+ packages = {field(m, "file_path"): field(m, "module") or ROOT_PACKAGE for m in metrics}
+ dirs = directories(packages)
+ labels = unique_suffixes(paths)
+ rows = []
+ for m in metrics:
+ path = field(m, "file_path")
+ g = git_of.get(path)
+ c = coverage_of.get(path)
+ # Unrounded, so an average here is exactly the engine's; views round for display.
+ score = field(m, "score")
+ rows.append(
+ {
+ "path": path,
+ "file": labels[path],
+ "package": packages[path],
+ "directory": dirs[path],
+ "language": languages.get(path),
+ "health_score": score,
+ "health_band": band_for(score).replace("_", " ") if score is not None else None,
+ "nloc": field(m, "nloc"),
+ "max_ccn": field(m, "max_ccn"),
+ "commits_90d": field(g, "commit_count_90d") if g is not None else 0,
+ "fix_count": field(g, "prior_defect_count") if g is not None else 0,
+ "coverage_pct": round(field(c, "line_coverage_pct"), 2) if c is not None else None,
+ "coverable_lines": field(c, "total_coverable_lines") if c is not None else None,
+ "is_hotspot": bool(g is not None and field(g, "is_hotspot")),
+ "is_test": bool(field(m, "is_test")),
+ "is_production": scope_reason(m) is None,
+ }
+ )
+ return rows
+
+
+def coverage_rows(
+ report: Iterable[Any], files: Iterable[Mapping[str, Any]]
+) -> list[dict[str, Any]]:
+ """The ``coverage`` dataset's rows over the report's per-file records.
+
+ *report* holds ``file_path``, ``line_coverage_pct``,
+ ``total_coverable_lines`` and ``covered_line_count``; *files* is the
+ ``files`` dataset's rows, which place each path in its package.
+ """
+ report = list(report)
+ placed = place([field(c, "file_path") for c in report], by_path(files))
+ labels = unique_suffixes(placed)
+ rows = []
+ for c in report:
+ path = field(c, "file_path")
+ pct = field(c, "line_coverage_pct")
+ coverable = field(c, "total_coverable_lines")
+ package, directory = placed[path]
+ covered = field(c, "covered_line_count")
+ if covered is None:
+ covered = round(pct / 100 * (coverable or 0))
+ rows.append(
+ {
+ "path": path,
+ "file": labels[path],
+ "package": package,
+ "directory": directory,
+ "coverage_pct": round(pct, 2),
+ "covered_lines": covered,
+ "coverable_lines": coverable,
+ }
+ )
+ return rows
# ---------------------------------------------------------------------------
@@ -176,8 +273,6 @@ async def _coverage_rows(session: Any, repo_id: str) -> list[Any]:
noun="files",
)
async def load_files(ctx: LoadContext) -> list[dict[str, Any]]:
- from repowise.core.analysis.health.grading import band_for
- from repowise.core.analysis.health.ranking import scope_reason
from repowise.core.persistence.crud.analysis.health import (
get_file_language_map,
health_exclude_spec,
@@ -207,41 +302,10 @@ async def load_files(ctx: LoadContext) -> list[dict[str, Any]]:
GitMetadata.prior_defect_count,
GitMetadata.is_hotspot,
).where(code_file_rows(repo_id))
- git = {g.file_path: g for g in plain_rows(await session.execute(git_q))}
- coverage = {c.file_path: c for c in await _coverage_rows(session, repo_id)}
+ git = plain_rows(await session.execute(git_q))
+ coverage = await _coverage_rows(session, repo_id)
languages = await get_file_language_map(session, repo_id)
-
- paths = [m.file_path for m in metrics]
- packages = {m.file_path: m.module or ROOT_PACKAGE for m in metrics}
- dirs = directories(packages)
- labels = unique_suffixes(paths)
- rows = []
- for m in metrics:
- g = git.get(m.file_path)
- c = coverage.get(m.file_path)
- # Unrounded, so an average here is exactly the engine's; views round for display.
- score = m.score
- rows.append(
- {
- "path": m.file_path,
- "file": labels[m.file_path],
- "package": packages[m.file_path],
- "directory": dirs[m.file_path],
- "language": languages.get(m.file_path),
- "health_score": score,
- "health_band": band_for(score).replace("_", " ") if score is not None else None,
- "nloc": m.nloc,
- "max_ccn": m.max_ccn,
- "commits_90d": g.commit_count_90d if g else 0,
- "fix_count": g.prior_defect_count if g else 0,
- "coverage_pct": round(c.line_coverage_pct, 2) if c else None,
- "coverable_lines": c.total_coverable_lines if c else None,
- "is_hotspot": bool(g and g.is_hotspot),
- "is_test": bool(m.is_test),
- "is_production": scope_reason(m) is None,
- }
- )
- return rows
+ return files_rows(metrics, git, coverage, languages)
@register_dataset(
@@ -255,24 +319,4 @@ async def load_files(ctx: LoadContext) -> list[dict[str, Any]]:
)
async def load_coverage(ctx: LoadContext) -> list[dict[str, Any]]:
report = await _coverage_rows(ctx.session, ctx.repository.id)
- files = await files_index(ctx)
- placed = place([c.file_path for c in report], files)
- labels = unique_suffixes(placed)
- rows = []
- for c in report:
- package, directory = placed[c.file_path]
- covered = c.covered_line_count
- if covered is None:
- covered = round(c.line_coverage_pct / 100 * (c.total_coverable_lines or 0))
- rows.append(
- {
- "path": c.file_path,
- "file": labels[c.file_path],
- "package": package,
- "directory": directory,
- "coverage_pct": round(c.line_coverage_pct, 2),
- "covered_lines": covered,
- "coverable_lines": c.total_coverable_lines,
- }
- )
- return rows
+ return coverage_rows(report, await ctx.rows("files"))
diff --git a/packages/server/src/repowise/server/chat_datasets/findings.py b/packages/server/src/repowise/server/chat_datasets/findings.py
index 863ac2c01..038cfef6b 100644
--- a/packages/server/src/repowise/server/chat_datasets/findings.py
+++ b/packages/server/src/repowise/server/chat_datasets/findings.py
@@ -8,12 +8,14 @@ confidence floors.
from __future__ import annotations
+from collections.abc import Iterable, Mapping
from typing import Any
from sqlalchemy import select
+from repowise.core.analysis.health.rows import field
from repowise.server.chat_datasets import SUMS, DatasetField, LoadContext, register_dataset
-from repowise.server.chat_datasets.files import files_index, place, unique_suffixes
+from repowise.server.chat_datasets.files import by_path, place, unique_suffixes
_PLACE_FIELDS = {
"path": DatasetField("text", "the file's full repository path"),
@@ -52,22 +54,34 @@ async def load_findings(ctx: LoadContext) -> list[dict[str, Any]]:
HealthFinding.status == "open",
shown("health", HealthFinding.biomarker_type, HealthFinding.file_path),
)
- files = await files_index(ctx)
+ return findings_rows(plain_rows(await ctx.session.execute(q)), await ctx.rows("files"))
+
+
+def findings_rows(found: Iterable[Any], files: Iterable[Mapping[str, Any]]) -> list[dict[str, Any]]:
+ """The ``findings`` dataset's rows over open, shown health findings.
+
+ *found* holds ``file_path``, ``biomarker_type``, ``severity`` and
+ ``dimension``; *files* is the ``files`` dataset's rows.
+ """
+ index = by_path(files)
# Files the health exclude rule drops carry no listed finding either.
- found = [f for f in plain_rows(await ctx.session.execute(q)) if f.file_path in files]
- placed = place([f.file_path for f in found], files)
- return [
- {
- "kind": f.biomarker_type,
- "severity": f.severity,
- "dimension": f.dimension or "defect",
- "path": f.file_path,
- "file": files[f.file_path]["file"],
- "package": placed[f.file_path][0],
- "directory": placed[f.file_path][1],
- }
- for f in found
- ]
+ kept = [f for f in found if field(f, "file_path") in index]
+ placed = place([field(f, "file_path") for f in kept], index)
+ rows = []
+ for f in kept:
+ path = field(f, "file_path")
+ rows.append(
+ {
+ "kind": field(f, "biomarker_type"),
+ "severity": field(f, "severity"),
+ "dimension": field(f, "dimension") or "defect",
+ "path": path,
+ "file": index[path]["file"],
+ "package": placed[path][0],
+ "directory": placed[path][1],
+ }
+ )
+ return rows
def _tier(confidence: float) -> str:
@@ -99,21 +113,35 @@ async def load_dead_code(ctx: LoadContext) -> list[dict[str, Any]]:
from repowise.core.persistence.crud.analysis.dead_code import get_dead_code_findings
found = await get_dead_code_findings(ctx.session, ctx.repository.id)
- files = await files_index(ctx)
- placed = place([f.file_path for f in found], files)
+ return dead_code_rows(found, await ctx.rows("files"))
+
+
+def dead_code_rows(found: Iterable[Any], files: Iterable[Mapping[str, Any]]) -> list[dict[str, Any]]:
+ """The ``dead_code`` dataset's rows over open dead-code findings.
+
+ *found* holds ``file_path``, ``kind``, ``confidence``, ``symbol_name``,
+ ``lines`` and ``safe_to_delete``; *files* is the ``files`` dataset's rows.
+ """
+ found = list(found)
+ index = by_path(files)
+ placed = place([field(f, "file_path") for f in found], index)
labels = unique_suffixes(placed)
- return [
- {
- "kind": f.kind,
- "tier": _tier(f.confidence or 0.0),
- "confidence": f.confidence,
- "symbol": f.symbol_name,
- "lines": f.lines,
- "safe_to_delete": bool(f.safe_to_delete),
- "path": f.file_path,
- "file": files[f.file_path]["file"] if f.file_path in files else labels[f.file_path],
- "package": placed[f.file_path][0],
- "directory": placed[f.file_path][1],
- }
- for f in found
- ]
+ rows = []
+ for f in found:
+ path = field(f, "file_path")
+ confidence = field(f, "confidence")
+ rows.append(
+ {
+ "kind": field(f, "kind"),
+ "tier": _tier(confidence or 0.0),
+ "confidence": confidence,
+ "symbol": field(f, "symbol_name"),
+ "lines": field(f, "lines"),
+ "safe_to_delete": bool(field(f, "safe_to_delete")),
+ "path": path,
+ "file": index[path]["file"] if path in index else labels[path],
+ "package": placed[path][0],
+ "directory": placed[path][1],
+ }
+ )
+ return rows
diff --git a/packages/server/src/repowise/server/chat_datasets/history.py b/packages/server/src/repowise/server/chat_datasets/history.py
index 1ca9ee8e0..71722b7aa 100644
--- a/packages/server/src/repowise/server/chat_datasets/history.py
+++ b/packages/server/src/repowise/server/chat_datasets/history.py
@@ -7,11 +7,13 @@ snapshot, the scalars get_health's trend reads.
from __future__ import annotations
+from collections.abc import Iterable
from datetime import UTC
from typing import Any
from sqlalchemy import select
+from repowise.core.analysis.health.rows import field
from repowise.server.chat_datasets import (
AVERAGES,
SUMS,
@@ -44,7 +46,6 @@ from repowise.server.chat_datasets import (
)
async def load_changes(ctx: LoadContext) -> list[dict[str, Any]]:
from repowise.core.persistence.models import Change
- from repowise.server.mcp_server._changes import _week
q = select(
Change.merged_at,
@@ -54,8 +55,21 @@ async def load_changes(ctx: LoadContext) -> list[dict[str, Any]]:
Change.lines_added,
Change.lines_deleted,
).where(Change.repository_id == ctx.repository.id)
+ return changes_rows(await ctx.session.execute(q))
+
+
+def changes_rows(changes: Iterable[Any]) -> list[dict[str, Any]]:
+ """The ``changes`` dataset's rows over merged-change records, oldest day first.
+
+ Each record holds ``merged_at``, ``category``, ``shape``,
+ ``files_changed``, ``lines_added`` and ``lines_deleted``; one never
+ merged is left out.
+ """
+ from repowise.server.mcp_server._changes import _week
+
rows = []
- for merged_at, category, shape, files, added, deleted in await ctx.session.execute(q):
+ for c in changes:
+ merged_at = field(c, "merged_at")
if merged_at is None:
continue
moment = merged_at if merged_at.tzinfo else merged_at.replace(tzinfo=UTC)
@@ -64,11 +78,11 @@ async def load_changes(ctx: LoadContext) -> list[dict[str, Any]]:
"date": moment.date().isoformat(),
"week": _week(moment),
"month": moment.strftime("%Y-%m"),
- "category": category,
- "shape": shape,
- "files_changed": files,
- "lines_added": added,
- "lines_deleted": deleted,
+ "category": field(c, "category"),
+ "shape": field(c, "shape"),
+ "files_changed": field(c, "files_changed"),
+ "lines_added": field(c, "lines_added"),
+ "lines_deleted": field(c, "lines_deleted"),
}
)
rows.sort(key=lambda r: r["date"])
@@ -94,11 +108,22 @@ async def load_health_trend(ctx: LoadContext) -> list[dict[str, Any]]:
# Scalars only: whole snapshot rows carry every file's score.
headline = await get_health_snapshot_headline(ctx.session, ctx.repository.id, recent=50)
- return [
- {
- "taken_at": s.taken_at.isoformat(timespec="seconds"),
- "average_health": round(s.average_health, 2),
- "hotspot_health": round(s.hotspot_health, 2) if s.hotspot_health is not None else None,
- }
- for s in headline.recent
- ]
+ return health_trend_rows(headline.recent)
+
+
+def health_trend_rows(snapshots: Iterable[Any]) -> list[dict[str, Any]]:
+ """The ``health_trend`` dataset's rows over snapshot scalars, in the order given.
+
+ Each record holds ``taken_at``, ``average_health`` and ``hotspot_health``.
+ """
+ rows = []
+ for s in snapshots:
+ hotspot = field(s, "hotspot_health")
+ rows.append(
+ {
+ "taken_at": field(s, "taken_at").isoformat(timespec="seconds"),
+ "average_health": round(field(s, "average_health"), 2),
+ "hotspot_health": round(hotspot, 2) if hotspot is not None else None,
+ }
+ )
+ return rows
diff --git a/tests/unit/server/mcp/test_chat_datasets.py b/tests/unit/server/mcp/test_chat_datasets.py
index 0a3f55842..b2ec13d40 100644
--- a/tests/unit/server/mcp/test_chat_datasets.py
+++ b/tests/unit/server/mcp/test_chat_datasets.py
@@ -358,3 +358,85 @@ async def test_a_failed_load_is_not_cached_and_the_tool_says_so(chart_index, pro
state["fail"] = False
assert "error" not in await run_chart({"dataset": "probe", "x": "k"}, None, TurnViews())
assert len(calls) == 2
+
+
+# ---------------------------------------------------------------------------
+# Row builders over plain records
+# ---------------------------------------------------------------------------
+
+
+def _record(obj) -> dict:
+ """An ORM entity or named tuple as a plain dict, the shape a consumer may hold."""
+ from sqlalchemy import inspect
+
+ if hasattr(obj, "_asdict"):
+ return obj._asdict()
+ return {a.key: getattr(obj, a.key) for a in inspect(obj).mapper.column_attrs}
+
+
+async def _entities(session, model, *where) -> list[dict]:
+ from sqlalchemy import select
+
+ return [_record(o) for o in (await session.execute(select(model).where(*where))).scalars()]
+
+
+async def test_row_builders_match_the_loaders_over_the_same_records(chart_index, session):
+ from repowise.core.persistence.crud.analysis.dead_code import get_dead_code_findings
+ from repowise.core.persistence.crud.analysis.health import (
+ get_file_language_map,
+ get_health_snapshot_headline,
+ )
+ from repowise.core.persistence.crud.analysis.shown import shown
+ from repowise.core.persistence.crud.git import code_file_rows
+ from repowise.core.persistence.crud.knowledge_graph import get_kg_layers
+ from repowise.core.persistence.models import GitMetadata, HealthFinding
+ from repowise.server.chat_datasets.architecture import layers_rows
+ from repowise.server.chat_datasets.files import coverage_rows, files_rows
+ from repowise.server.chat_datasets.findings import dead_code_rows, findings_rows
+ from repowise.server.chat_datasets.history import changes_rows, health_trend_rows
+
+ rid = chart_index
+ coverage = await _entities(session, CoverageFile, CoverageFile.repository_id == rid)
+ files = files_rows(
+ await _entities(session, HealthFileMetric, HealthFileMetric.repository_id == rid),
+ await _entities(session, GitMetadata, code_file_rows(rid)),
+ coverage,
+ await get_file_language_map(session, rid),
+ )
+ assert files and files == await _rows("files")
+ assert coverage_rows(coverage, files) == await _rows("coverage")
+ found = await _entities(
+ session,
+ HealthFinding,
+ HealthFinding.repository_id == rid,
+ HealthFinding.status == "open",
+ shown("health", HealthFinding.biomarker_type, HealthFinding.file_path),
+ )
+ assert findings_rows(found, files) == await _rows("findings")
+ dead = [_record(f) for f in await get_dead_code_findings(session, rid)]
+ assert dead_code_rows(dead, files) == await _rows("dead_code")
+ changes = await _entities(session, Change, Change.repository_id == rid)
+ assert changes_rows(changes) == await _rows("changes")
+ headline = await get_health_snapshot_headline(session, rid, recent=50)
+ assert health_trend_rows([_record(s) for s in headline.recent]) == await _rows("health_trend")
+ layers = [_record(layer) for layer in await get_kg_layers(session, rid)]
+ assert layers_rows(layers) == await _rows("layers")
+
+
+def test_dependency_rows_label_edges_and_drop_unknown_boxes():
+ from repowise.server.chat_datasets.architecture import dependencies_rows
+
+ containers = [{"id": "a", "path": "packages/a"}, {"id": "root", "path": ""}]
+ relations = [
+ {"source_id": "a", "target_id": "root", "edge_count": 3, "coupling": "loose"},
+ {"source_id": "a", "target_id": "gone", "edge_count": 9, "coupling": "tight"},
+ ]
+ assert dependencies_rows(containers, relations) == [
+ {
+ "edge": f"packages/a {chat_datasets.ARROW} .",
+ "from_package": "packages/a",
+ "to_package": ".",
+ "weight": 3,
+ "coupling": "loose",
+ }
+ ]