Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(chat): build chart dataset rows from plain records (#3951)
Raghav Chamadiya5dc62ea69ef6
diff --git a/packages/server/src/repowise/server/chat_datasets/architecture.py b/packages/server/src/repowise/server/chat_datasets/architecture.pyindex 2c7b582fb..86e16b094 100644--- a/packages/server/src/repowise/server/chat_datasets/architecture.py+++ b/packages/server/src/repowise/server/chat_datasets/architecture.py@@ -8,8 +8,10 @@ the architecture layers get_overview lists, in stack order.from __future__ import annotationsimport json+from collections.abc import Iterablefrom typing import Any+from repowise.core.analysis.health.rows import fieldfrom repowise.server.chat_datasets import (ARROW,SUMS,@@ -40,18 +42,31 @@ async def load_dependencies(ctx: LoadContext) -> list[dict[str, Any]]:containers, relations = await container_dependencies(ctx.session, ctx.repository.id, exclude_spec=_get_exclude_spec(ctx.repo_path))- path_of = {c.id: c.path or "." for c in containers}- return [- {- "edge": f"{path_of[r.source_id]} {ARROW} {path_of[r.target_id]}",- "from_package": path_of[r.source_id],- "to_package": path_of[r.target_id],- "weight": r.edge_count,- "coupling": r.coupling,- }- for r in relations- if r.source_id in path_of and r.target_id in path_of- ]+ return dependencies_rows(containers, relations)+++def dependencies_rows(containers: Iterable[Any], relations: Iterable[Any]) -> list[dict[str, Any]]:+ """The ``dependencies`` dataset's rows over the container view's boxes and edges.++ *containers* hold ``id`` and ``path``; *relations* hold ``source_id``,+ ``target_id``, ``edge_count`` and ``coupling``. An edge to a box not in+ *containers* is left out.+ """+ path_of = {field(c, "id"): field(c, "path") or "." for c in containers}+ rows = []+ for r in relations:+ source, target = field(r, "source_id"), field(r, "target_id")+ if source in path_of and target in path_of:+ rows.append(+ {+ "edge": f"{path_of[source]} {ARROW} {path_of[target]}",+ "from_package": path_of[source],+ "to_package": path_of[target],+ "weight": field(r, "edge_count"),+ "coupling": field(r, "coupling"),+ }+ )+ return rows@register_dataset(@@ -67,19 +82,28 @@ async def load_dependencies(ctx: LoadContext) -> list[dict[str, Any]]:noun="layers",)async def load_layers(ctx: LoadContext) -> list[dict[str, Any]]:- from repowise.core.generation.layers import is_adjacent_layerfrom repowise.core.persistence.crud.knowledge_graph import get_kg_layers- layers = [- layer- for layer in await get_kg_layers(ctx.session, ctx.repository.id)- if not is_adjacent_layer(layer.layer_id)- ]- return [- {- "layer": layer.name,- "files": len(json.loads(layer.node_ids_json) if layer.node_ids_json else []),- "order": i + 1,- }- for i, layer in enumerate(layers)- ]+ return layers_rows(await get_kg_layers(ctx.session, ctx.repository.id))+++def layers_rows(layers: Iterable[Any]) -> list[dict[str, Any]]:+ """The ``layers`` dataset's rows over knowledge-graph layers, in the order given.++ Each record holds ``layer_id``, ``name`` and ``node_ids_json``; adjacent+ layers (outside the stack) are left out.+ """+ from repowise.core.generation.layers import is_adjacent_layer++ stack = [layer for layer in layers if not is_adjacent_layer(field(layer, "layer_id"))]+ rows = []+ for i, layer in enumerate(stack):+ node_ids = field(layer, "node_ids_json")+ rows.append(+ {+ "layer": field(layer, "name"),+ "files": len(json.loads(node_ids) if node_ids else []),+ "order": i + 1,+ }+ )+ return rowsdiff --git a/packages/server/src/repowise/server/chat_datasets/files.py b/packages/server/src/repowise/server/chat_datasets/files.pyindex 9030a0f36..dd6a0f61b 100644--- a/packages/server/src/repowise/server/chat_datasets/files.py+++ b/packages/server/src/repowise/server/chat_datasets/files.py@@ -10,11 +10,12 @@ columns, and the coverage report's per-file rows.from __future__ import annotationsfrom collections import Counter, defaultdict-from collections.abc import Iterable+from collections.abc import Iterable, Mappingfrom typing import Anyfrom sqlalchemy import select+from repowise.core.analysis.health.rows import fieldfrom repowise.server.chat_datasets import (AVERAGES,SUMS,@@ -143,9 +144,105 @@ def _top_level(path: str) -> str:return head if sep else ROOT_PACKAGE-async def files_index(ctx: LoadContext) -> dict[str, dict[str, Any]]:- """The files dataset by path, for datasets that place rows by file."""- return {row["path"]: row for row in await ctx.rows("files")}+def by_path(files: Iterable[Mapping[str, Any]]) -> dict[str, Mapping[str, Any]]:+ """The files dataset's rows by path, for datasets that place rows by file."""+ return {row["path"]: row for row in files}+++# ---------------------------------------------------------------------------+# Rows over records already read+# ---------------------------------------------------------------------------+++def files_rows(+ metrics: Iterable[Any],+ git: Iterable[Any],+ coverage: Iterable[Any],+ languages: Mapping[str, str | None],+) -> list[dict[str, Any]]:+ """The ``files`` dataset's rows over records a caller already holds.++ *metrics* are the per-file health metrics (``file_path``, ``score``,+ ``nloc``, ``max_ccn``, ``module``, ``is_test``, ``code_origin``) the+ health exclude rule kept; *git* the code files' git metadata+ (``file_path``, ``commit_count_90d``, ``prior_defect_count``,+ ``is_hotspot``); *coverage* the coverage report's rows (``file_path``,+ ``line_coverage_pct``, ``total_coverable_lines``); *languages* maps a path+ to its language. A record is a dataclass, an ORM or SQL row, or a dict.+ """+ from repowise.core.analysis.health.grading import band_for+ from repowise.core.analysis.health.ranking import scope_reason++ metrics = list(metrics)+ git_of = {field(g, "file_path"): g for g in git}+ coverage_of = {field(c, "file_path"): c for c in coverage}+ paths = [field(m, "file_path") for m in metrics]+ packages = {field(m, "file_path"): field(m, "module") or ROOT_PACKAGE for m in metrics}+ dirs = directories(packages)+ labels = unique_suffixes(paths)+ rows = []+ for m in metrics:+ path = field(m, "file_path")+ g = git_of.get(path)+ c = coverage_of.get(path)+ # Unrounded, so an average here is exactly the engine's; views round for display.+ score = field(m, "score")+ rows.append(+ {+ "path": path,+ "file": labels[path],+ "package": packages[path],+ "directory": dirs[path],+ "language": languages.get(path),+ "health_score": score,+ "health_band": band_for(score).replace("_", " ") if score is not None else None,+ "nloc": field(m, "nloc"),+ "max_ccn": field(m, "max_ccn"),+ "commits_90d": field(g, "commit_count_90d") if g is not None else 0,+ "fix_count": field(g, "prior_defect_count") if g is not None else 0,+ "coverage_pct": round(field(c, "line_coverage_pct"), 2) if c is not None else None,+ "coverable_lines": field(c, "total_coverable_lines") if c is not None else None,+ "is_hotspot": bool(g is not None and field(g, "is_hotspot")),+ "is_test": bool(field(m, "is_test")),+ "is_production": scope_reason(m) is None,+ }+ )+ return rows+++def coverage_rows(+ report: Iterable[Any], files: Iterable[Mapping[str, Any]]+) -> list[dict[str, Any]]:+ """The ``coverage`` dataset's rows over the report's per-file records.++ *report* holds ``file_path``, ``line_coverage_pct``,+ ``total_coverable_lines`` and ``covered_line_count``; *files* is the+ ``files`` dataset's rows, which place each path in its package.+ """+ report = list(report)+ placed = place([field(c, "file_path") for c in report], by_path(files))+ labels = unique_suffixes(placed)+ rows = []+ for c in report:+ path = field(c, "file_path")+ pct = field(c, "line_coverage_pct")+ coverable = field(c, "total_coverable_lines")+ package, directory = placed[path]+ covered = field(c, "covered_line_count")+ if covered is None:+ covered = round(pct / 100 * (coverable or 0))+ rows.append(+ {+ "path": path,+ "file": labels[path],+ "package": package,+ "directory": directory,+ "coverage_pct": round(pct, 2),+ "covered_lines": covered,+ "coverable_lines": coverable,+ }+ )+ return rows# ---------------------------------------------------------------------------@@ -176,8 +273,6 @@ async def _coverage_rows(session: Any, repo_id: str) -> list[Any]:noun="files",)async def load_files(ctx: LoadContext) -> list[dict[str, Any]]:- from repowise.core.analysis.health.grading import band_for- from repowise.core.analysis.health.ranking import scope_reasonfrom repowise.core.persistence.crud.analysis.health import (get_file_language_map,health_exclude_spec,@@ -207,41 +302,10 @@ async def load_files(ctx: LoadContext) -> list[dict[str, Any]]:GitMetadata.prior_defect_count,GitMetadata.is_hotspot,).where(code_file_rows(repo_id))- git = {g.file_path: g for g in plain_rows(await session.execute(git_q))}- coverage = {c.file_path: c for c in await _coverage_rows(session, repo_id)}+ git = plain_rows(await session.execute(git_q))+ coverage = await _coverage_rows(session, repo_id)languages = await get_file_language_map(session, repo_id)-- paths = [m.file_path for m in metrics]- packages = {m.file_path: m.module or ROOT_PACKAGE for m in metrics}- dirs = directories(packages)- labels = unique_suffixes(paths)- rows = []- for m in metrics:- g = git.get(m.file_path)- c = coverage.get(m.file_path)- # Unrounded, so an average here is exactly the engine's; views round for display.- score = m.score- rows.append(- {- "path": m.file_path,- "file": labels[m.file_path],- "package": packages[m.file_path],- "directory": dirs[m.file_path],- "language": languages.get(m.file_path),- "health_score": score,- "health_band": band_for(score).replace("_", " ") if score is not None else None,- "nloc": m.nloc,- "max_ccn": m.max_ccn,- "commits_90d": g.commit_count_90d if g else 0,- "fix_count": g.prior_defect_count if g else 0,- "coverage_pct": round(c.line_coverage_pct, 2) if c else None,- "coverable_lines": c.total_coverable_lines if c else None,- "is_hotspot": bool(g and g.is_hotspot),- "is_test": bool(m.is_test),- "is_production": scope_reason(m) is None,- }- )- return rows+ return files_rows(metrics, git, coverage, languages)@register_dataset(@@ -255,24 +319,4 @@ async def load_files(ctx: LoadContext) -> list[dict[str, Any]]:)async def load_coverage(ctx: LoadContext) -> list[dict[str, Any]]:report = await _coverage_rows(ctx.session, ctx.repository.id)- files = await files_index(ctx)- placed = place([c.file_path for c in report], files)- labels = unique_suffixes(placed)- rows = []- for c in report:- package, directory = placed[c.file_path]- covered = c.covered_line_count- if covered is None:- covered = round(c.line_coverage_pct / 100 * (c.total_coverable_lines or 0))- rows.append(- {- "path": c.file_path,- "file": labels[c.file_path],- "package": package,- "directory": directory,- "coverage_pct": round(c.line_coverage_pct, 2),- "covered_lines": covered,- "coverable_lines": c.total_coverable_lines,- }- )- return rows+ return coverage_rows(report, await ctx.rows("files"))diff --git a/packages/server/src/repowise/server/chat_datasets/findings.py b/packages/server/src/repowise/server/chat_datasets/findings.pyindex 863ac2c01..038cfef6b 100644--- a/packages/server/src/repowise/server/chat_datasets/findings.py+++ b/packages/server/src/repowise/server/chat_datasets/findings.py@@ -8,12 +8,14 @@ confidence floors.from __future__ import annotations+from collections.abc import Iterable, Mappingfrom typing import Anyfrom sqlalchemy import select+from repowise.core.analysis.health.rows import fieldfrom repowise.server.chat_datasets import SUMS, DatasetField, LoadContext, register_dataset-from repowise.server.chat_datasets.files import files_index, place, unique_suffixes+from repowise.server.chat_datasets.files import by_path, place, unique_suffixes_PLACE_FIELDS = {"path": DatasetField("text", "the file's full repository path"),@@ -52,22 +54,34 @@ async def load_findings(ctx: LoadContext) -> list[dict[str, Any]]:HealthFinding.status == "open",shown("health", HealthFinding.biomarker_type, HealthFinding.file_path),)- files = await files_index(ctx)+ return findings_rows(plain_rows(await ctx.session.execute(q)), await ctx.rows("files"))+++def findings_rows(found: Iterable[Any], files: Iterable[Mapping[str, Any]]) -> list[dict[str, Any]]:+ """The ``findings`` dataset's rows over open, shown health findings.++ *found* holds ``file_path``, ``biomarker_type``, ``severity`` and+ ``dimension``; *files* is the ``files`` dataset's rows.+ """+ index = by_path(files)# Files the health exclude rule drops carry no listed finding either.- found = [f for f in plain_rows(await ctx.session.execute(q)) if f.file_path in files]- placed = place([f.file_path for f in found], files)- return [- {- "kind": f.biomarker_type,- "severity": f.severity,- "dimension": f.dimension or "defect",- "path": f.file_path,- "file": files[f.file_path]["file"],- "package": placed[f.file_path][0],- "directory": placed[f.file_path][1],- }- for f in found- ]+ kept = [f for f in found if field(f, "file_path") in index]+ placed = place([field(f, "file_path") for f in kept], index)+ rows = []+ for f in kept:+ path = field(f, "file_path")+ rows.append(+ {+ "kind": field(f, "biomarker_type"),+ "severity": field(f, "severity"),+ "dimension": field(f, "dimension") or "defect",+ "path": path,+ "file": index[path]["file"],+ "package": placed[path][0],+ "directory": placed[path][1],+ }+ )+ return rowsdef _tier(confidence: float) -> str:@@ -99,21 +113,35 @@ async def load_dead_code(ctx: LoadContext) -> list[dict[str, Any]]:from repowise.core.persistence.crud.analysis.dead_code import get_dead_code_findingsfound = await get_dead_code_findings(ctx.session, ctx.repository.id)- files = await files_index(ctx)- placed = place([f.file_path for f in found], files)+ return dead_code_rows(found, await ctx.rows("files"))+++def dead_code_rows(found: Iterable[Any], files: Iterable[Mapping[str, Any]]) -> list[dict[str, Any]]:+ """The ``dead_code`` dataset's rows over open dead-code findings.++ *found* holds ``file_path``, ``kind``, ``confidence``, ``symbol_name``,+ ``lines`` and ``safe_to_delete``; *files* is the ``files`` dataset's rows.+ """+ found = list(found)+ index = by_path(files)+ placed = place([field(f, "file_path") for f in found], index)labels = unique_suffixes(placed)- return [- {- "kind": f.kind,- "tier": _tier(f.confidence or 0.0),- "confidence": f.confidence,- "symbol": f.symbol_name,- "lines": f.lines,- "safe_to_delete": bool(f.safe_to_delete),- "path": f.file_path,- "file": files[f.file_path]["file"] if f.file_path in files else labels[f.file_path],- "package": placed[f.file_path][0],- "directory": placed[f.file_path][1],- }- for f in found- ]+ rows = []+ for f in found:+ path = field(f, "file_path")+ confidence = field(f, "confidence")+ rows.append(+ {+ "kind": field(f, "kind"),+ "tier": _tier(confidence or 0.0),+ "confidence": confidence,+ "symbol": field(f, "symbol_name"),+ "lines": field(f, "lines"),+ "safe_to_delete": bool(field(f, "safe_to_delete")),+ "path": path,+ "file": index[path]["file"] if path in index else labels[path],+ "package": placed[path][0],+ "directory": placed[path][1],+ }+ )+ return rowsdiff --git a/packages/server/src/repowise/server/chat_datasets/history.py b/packages/server/src/repowise/server/chat_datasets/history.pyindex 1ca9ee8e0..71722b7aa 100644--- a/packages/server/src/repowise/server/chat_datasets/history.py+++ b/packages/server/src/repowise/server/chat_datasets/history.py@@ -7,11 +7,13 @@ snapshot, the scalars get_health's trend reads.from __future__ import annotations+from collections.abc import Iterablefrom datetime import UTCfrom typing import Anyfrom sqlalchemy import select+from repowise.core.analysis.health.rows import fieldfrom repowise.server.chat_datasets import (AVERAGES,SUMS,@@ -44,7 +46,6 @@ from repowise.server.chat_datasets import ()async def load_changes(ctx: LoadContext) -> list[dict[str, Any]]:from repowise.core.persistence.models import Change- from repowise.server.mcp_server._changes import _weekq = select(Change.merged_at,@@ -54,8 +55,21 @@ async def load_changes(ctx: LoadContext) -> list[dict[str, Any]]:Change.lines_added,Change.lines_deleted,).where(Change.repository_id == ctx.repository.id)+ return changes_rows(await ctx.session.execute(q))+++def changes_rows(changes: Iterable[Any]) -> list[dict[str, Any]]:+ """The ``changes`` dataset's rows over merged-change records, oldest day first.++ Each record holds ``merged_at``, ``category``, ``shape``,+ ``files_changed``, ``lines_added`` and ``lines_deleted``; one never+ merged is left out.+ """+ from repowise.server.mcp_server._changes import _week+rows = []- for merged_at, category, shape, files, added, deleted in await ctx.session.execute(q):+ for c in changes:+ merged_at = field(c, "merged_at")if merged_at is None:continuemoment = merged_at if merged_at.tzinfo else merged_at.replace(tzinfo=UTC)@@ -64,11 +78,11 @@ async def load_changes(ctx: LoadContext) -> list[dict[str, Any]]:"date": moment.date().isoformat(),"week": _week(moment),"month": moment.strftime("%Y-%m"),- "category": category,- "shape": shape,- "files_changed": files,- "lines_added": added,- "lines_deleted": deleted,+ "category": field(c, "category"),+ "shape": field(c, "shape"),+ "files_changed": field(c, "files_changed"),+ "lines_added": field(c, "lines_added"),+ "lines_deleted": field(c, "lines_deleted"),})rows.sort(key=lambda r: r["date"])@@ -94,11 +108,22 @@ async def load_health_trend(ctx: LoadContext) -> list[dict[str, Any]]:# Scalars only: whole snapshot rows carry every file's score.headline = await get_health_snapshot_headline(ctx.session, ctx.repository.id, recent=50)- return [- {- "taken_at": s.taken_at.isoformat(timespec="seconds"),- "average_health": round(s.average_health, 2),- "hotspot_health": round(s.hotspot_health, 2) if s.hotspot_health is not None else None,- }- for s in headline.recent- ]+ return health_trend_rows(headline.recent)+++def health_trend_rows(snapshots: Iterable[Any]) -> list[dict[str, Any]]:+ """The ``health_trend`` dataset's rows over snapshot scalars, in the order given.++ Each record holds ``taken_at``, ``average_health`` and ``hotspot_health``.+ """+ rows = []+ for s in snapshots:+ hotspot = field(s, "hotspot_health")+ rows.append(+ {+ "taken_at": field(s, "taken_at").isoformat(timespec="seconds"),+ "average_health": round(field(s, "average_health"), 2),+ "hotspot_health": round(hotspot, 2) if hotspot is not None else None,+ }+ )+ return rowsdiff --git a/tests/unit/server/mcp/test_chat_datasets.py b/tests/unit/server/mcp/test_chat_datasets.pyindex 0a3f55842..b2ec13d40 100644--- a/tests/unit/server/mcp/test_chat_datasets.py+++ b/tests/unit/server/mcp/test_chat_datasets.py@@ -358,3 +358,85 @@ async def test_a_failed_load_is_not_cached_and_the_tool_says_so(chart_index, prostate["fail"] = Falseassert "error" not in await run_chart({"dataset": "probe", "x": "k"}, None, TurnViews())assert len(calls) == 2+++# ---------------------------------------------------------------------------+# Row builders over plain records+# ---------------------------------------------------------------------------+++def _record(obj) -> dict:+ """An ORM entity or named tuple as a plain dict, the shape a consumer may hold."""+ from sqlalchemy import inspect++ if hasattr(obj, "_asdict"):+ return obj._asdict()+ return {a.key: getattr(obj, a.key) for a in inspect(obj).mapper.column_attrs}+++async def _entities(session, model, *where) -> list[dict]:+ from sqlalchemy import select++ return [_record(o) for o in (await session.execute(select(model).where(*where))).scalars()]+++async def test_row_builders_match_the_loaders_over_the_same_records(chart_index, session):+ from repowise.core.persistence.crud.analysis.dead_code import get_dead_code_findings+ from repowise.core.persistence.crud.analysis.health import (+ get_file_language_map,+ get_health_snapshot_headline,+ )+ from repowise.core.persistence.crud.analysis.shown import shown+ from repowise.core.persistence.crud.git import code_file_rows+ from repowise.core.persistence.crud.knowledge_graph import get_kg_layers+ from repowise.core.persistence.models import GitMetadata, HealthFinding+ from repowise.server.chat_datasets.architecture import layers_rows+ from repowise.server.chat_datasets.files import coverage_rows, files_rows+ from repowise.server.chat_datasets.findings import dead_code_rows, findings_rows+ from repowise.server.chat_datasets.history import changes_rows, health_trend_rows++ rid = chart_index+ coverage = await _entities(session, CoverageFile, CoverageFile.repository_id == rid)+ files = files_rows(+ await _entities(session, HealthFileMetric, HealthFileMetric.repository_id == rid),+ await _entities(session, GitMetadata, code_file_rows(rid)),+ coverage,+ await get_file_language_map(session, rid),+ )+ assert files and files == await _rows("files")+ assert coverage_rows(coverage, files) == await _rows("coverage")+ found = await _entities(+ session,+ HealthFinding,+ HealthFinding.repository_id == rid,+ HealthFinding.status == "open",+ shown("health", HealthFinding.biomarker_type, HealthFinding.file_path),+ )+ assert findings_rows(found, files) == await _rows("findings")+ dead = [_record(f) for f in await get_dead_code_findings(session, rid)]+ assert dead_code_rows(dead, files) == await _rows("dead_code")+ changes = await _entities(session, Change, Change.repository_id == rid)+ assert changes_rows(changes) == await _rows("changes")+ headline = await get_health_snapshot_headline(session, rid, recent=50)+ assert health_trend_rows([_record(s) for s in headline.recent]) == await _rows("health_trend")+ layers = [_record(layer) for layer in await get_kg_layers(session, rid)]+ assert layers_rows(layers) == await _rows("layers")+++def test_dependency_rows_label_edges_and_drop_unknown_boxes():+ from repowise.server.chat_datasets.architecture import dependencies_rows++ containers = [{"id": "a", "path": "packages/a"}, {"id": "root", "path": ""}]+ relations = [+ {"source_id": "a", "target_id": "root", "edge_count": 3, "coupling": "loose"},+ {"source_id": "a", "target_id": "gone", "edge_count": 9, "coupling": "tight"},+ ]+ assert dependencies_rows(containers, relations) == [+ {+ "edge": f"packages/a {chat_datasets.ARROW} .",+ "from_package": "packages/a",+ "to_package": ".",+ "weight": 3,+ "coupling": "loose",+ }+ ]