Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(actions): build RepoFacts from plain rows in a pure core builder (#2752)

Raghav Chamadiya1441e894f1be
diff --git a/packages/core/src/repowise/core/analysis/actions/__init__.py b/packages/core/src/repowise/core/analysis/actions/__init__.py
index 34259fde7..f6e5a792d 100644
--- a/packages/core/src/repowise/core/analysis/actions/__init__.py
+++ b/packages/core/src/repowise/core/analysis/actions/__init__.py
@@ -1,6 +1,7 @@
"""Next actions: stored evidence turned into a short list of things to do.
-The fold is pure (``engine.compose_actions`` over ``facts.RepoFacts``); the
+The fold is pure (``engine.compose_actions`` over ``facts.RepoFacts``), and so
+is ``build.build_repo_facts``, which turns plain rows into those facts. The
loader that reads the stores lives in
``repowise.core.persistence.crud.analysis.actions``.
"""
diff --git a/packages/core/src/repowise/core/analysis/actions/build.py b/packages/core/src/repowise/core/analysis/actions/build.py
new file mode 100644
index 000000000..cbbc72423
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/actions/build.py
@@ -0,0 +1,522 @@
+"""Build :class:`RepoFacts` from plain rows, with no store behind them.
+
+Every input is a sequence of rows, and a row is a mapping or any object with
+the named attributes (a SQL row, an ORM object, a dict decoded from JSON). The
+SQL loader in ``repowise.core.persistence.crud.analysis.actions`` narrows its
+reads and hands the rows here; a caller holding the same rows in memory calls
+:func:`build_repo_facts` directly. Each builder applies its store's whole rule,
+so a caller may pass unfiltered rows.
+
+Timestamps may be datetimes or ISO strings. They are compared with the anchor,
+so keep them all naive or all aware.
+
+Row shapes (the field names are the SQL columns):
+
+``files``
+ ``file_path``, ``commit_count_90d``, ``last_commit_at``, ``bug_magnet``,
+ ``bus_factor``, ``primary_owner_name``, ``primary_owner_email``,
+ ``primary_owner_commit_pct``, and from the file's health metric
+ ``is_test``, ``score``, ``max_ccn``, ``nloc``, ``line_coverage_pct``.
+``fix_events``
+ ``file_path``, ``fix_sha``, ``committed_at``, ``shape_kind``, ``attribution``.
+``health_findings``
+ ``file_path``, ``biomarker_type``, ``severity``, ``function_name``,
+ ``line_start``, ``reason``, ``health_impact``, ``status`` (absent = open).
+``authors``
+ ``author_name``, ``author_email``, ``committed_at``: one row per commit, or
+ per identity with its latest commit.
+``commit_health``
+ ``sha``, ``subject``, ``committed_at`` (the commit's), ``file_path``,
+ ``symbol``, ``biomarker_type``, ``severity``, ``change_kind``,
+ ``line_start``, ``reason``, ``attribution_basis``.
+``performance``
+ ``opportunity_id``, ``biomarker_type``, ``boundary_kind``, ``file_path``,
+ ``intervention_symbol``, ``affected_call_sites_total``,
+ ``affected_files_total``, ``actionability_state``, ``execution_context``,
+ ``status`` (absent = open), and ``details`` (a dict) or ``details_json``
+ carrying ``facets`` and ``plan``.
+``security``
+ ``file_path``, ``kind``, ``line_number``, ``snippet``, ``severity``,
+ ``commit_sha`` (empty for the working tree).
+``doc_drift``
+ ``file_path``, ``kind``, ``target``, ``raw``, ``line_number``, ``reason``,
+ ``confidence``; ``known_paths`` is every path the repository's history held.
+``dead_code``
+ ``id``, ``file_path``, ``symbol_name``, ``lines``, ``kind``,
+ ``safe_to_delete`` (the stored verdict), ``status`` (absent = open).
+``decisions``
+ One mapping: ``stale_decisions`` (rows with ``id``, ``title``) and
+ ``summary`` (``proposed``, ``active`` counts).
+``coverage``
+ One mapping: ``files_measured``, ``ingested_at``, ``ingested_commit_sha``,
+ ``partial``.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections import defaultdict
+from collections.abc import Callable, Iterable, Mapping
+from dataclasses import replace
+from datetime import datetime, timedelta
+from typing import Any
+
+from repowise.core.analysis.dead_code.risk_factors import REVIEW_ONLY_KINDS
+from repowise.core.analysis.finding_registry import excluded_types
+from repowise.core.analysis.health.models import primary_finding, split_by_origin
+from repowise.core.analysis.health.rows import detail_map, field
+from repowise.core.analysis.health.scoring import HISTORY_CATEGORY, biomarker_category
+from repowise.core.author_identity import author_identity_key
+
+from .context import WEEK
+from .facts import (
+ CoverageState,
+ DeadFacts,
+ DecisionFacts,
+ DriftFacts,
+ FileFacts,
+ LeadFinding,
+ PerfFacts,
+ RecentFinding,
+ RepoFacts,
+ SecretFacts,
+)
+from .rules.hygiene import PUBLIC_ENV_KIND, SECRET_KINDS
+
+logger = logging.getLogger(__name__)
+
+QUARTER = timedelta(days=90)
+
+#: Attribution bases that tie a finding to lines the commit wrote. A
+#: ``file_change`` basis only says the commit touched the file, which is not
+#: enough to tell someone they made it worse.
+AUTHORED_BASES = ("added_lines", "new_file", "changed_symbol")
+
+#: Files whose lead finding is looked up: busy bug magnets only, so the lookup
+#: stays a keyed read whatever the repository's size.
+LEAD_LOOKUP_MIN_COMMITS = 5
+
+#: Why a store with no input is not evaluated.
+ABSENT = "Not in this index yet; run `repowise update`."
+
+Rows = Iterable[Any]
+
+
+def _when(value: Any) -> datetime | None:
+ if isinstance(value, str) and value:
+ try:
+ return datetime.fromisoformat(value)
+ except ValueError:
+ return None
+ return value if isinstance(value, datetime) else None
+
+
+def _open(row: Any) -> bool:
+ return (field(row, "status") or "open") == "open"
+
+
+def _is_test(path: str, files: Mapping[str, FileFacts]) -> bool:
+ f = files.get(path)
+ return f is not None and f.is_test
+
+
+def _test_path(path: str | None) -> bool:
+ lowered = (path or "").lower()
+ return any(
+ seg in lowered
+ for seg in ("/tests/", "/test/", "__tests__", "/fixtures/", ".test.", ".spec.", "_test.")
+ ) or lowered.startswith(("tests/", "test/"))
+
+
+# --- files --------------------------------------------------------------------
+
+
+def build_files(
+ rows: Rows,
+ *,
+ since: datetime | None,
+ fix_events: Rows = (),
+ dependents: Mapping[str, int] | None = None,
+) -> dict[str, Any]:
+ """Per-file facts, fix counts in the window from ``since``; no leads yet."""
+ fix_shas: dict[str, set[str]] = defaultdict(set)
+ all_fix: set[str] = set()
+ if since is not None:
+ for e in fix_events:
+ at = _when(field(e, "committed_at"))
+ if (
+ at is not None
+ and at >= since
+ and field(e, "shape_kind") == "code_fix"
+ and field(e, "attribution") != "none"
+ ):
+ fix_shas[field(e, "file_path")].add(field(e, "fix_sha"))
+ all_fix.add(field(e, "fix_sha"))
+ dependents = dependents or {}
+ files: dict[str, FileFacts] = {}
+ for r in rows:
+ path = field(r, "file_path")
+ name, email = field(r, "primary_owner_name"), field(r, "primary_owner_email")
+ files[path] = FileFacts(
+ path=path,
+ is_test=bool(field(r, "is_test")),
+ score=field(r, "score"),
+ max_ccn=field(r, "max_ccn"),
+ nloc=field(r, "nloc"),
+ line_coverage_pct=field(r, "line_coverage_pct"),
+ commits_90d=field(r, "commit_count_90d") or 0,
+ last_commit_at=_when(field(r, "last_commit_at")),
+ bug_magnet=bool(field(r, "bug_magnet")),
+ fix_commits_90d=len(fix_shas.get(path, ())),
+ bus_factor=field(r, "bus_factor"),
+ owner_key=author_identity_key(name, email) if (name or email) else None,
+ owner_name=name,
+ owner_pct=field(r, "primary_owner_commit_pct"),
+ dependents=dependents.get(path),
+ )
+ return {
+ "files": files,
+ "fix_shas_by_file": {p: frozenset(s) for p, s in fix_shas.items()},
+ "fix_commits_90d": len(all_fix),
+ }
+
+
+def lead_paths(files: Mapping[str, FileFacts]) -> list[str]:
+ """The files whose lead finding is worth a lookup."""
+ return [
+ p
+ for p, f in files.items()
+ if f.bug_magnet
+ and not f.is_test
+ and f.commits_90d >= LEAD_LOOKUP_MIN_COMMITS
+ and f.fix_commits_90d >= 3
+ ]
+
+
+def with_leads(files: Mapping[str, FileFacts], findings: Rows) -> dict[str, FileFacts]:
+ """``files`` with each :func:`lead_paths` file's lead code-shape finding.
+
+ History markers are context, never the thing to do.
+ """
+ wanted = set(lead_paths(files))
+ hidden = excluded_types()
+ by_path: dict[str, list] = defaultdict(list)
+ for f in findings:
+ path = field(f, "file_path")
+ if path in wanted and _open(f) and field(f, "biomarker_type") not in hidden:
+ by_path[path].append(f)
+ out = dict(files)
+ for path, found in by_path.items():
+ shape, _history = split_by_origin(found)
+ lead = primary_finding(shape)
+ if lead is not None:
+ out[path] = replace(
+ out[path],
+ lead=LeadFinding(
+ biomarker=field(lead, "biomarker_type"),
+ severity=field(lead, "severity"),
+ function=field(lead, "function_name"),
+ line=field(lead, "line_start"),
+ reason=field(lead, "reason") or "",
+ ),
+ )
+ return out
+
+
+# --- the other stores ---------------------------------------------------------
+
+
+def build_authors(rows: Rows, *, since: datetime | None) -> dict[str, Any]:
+ last: dict[str, datetime] = {}
+ active: set[str] = set()
+ for r in rows:
+ latest = _when(field(r, "committed_at"))
+ if latest is None:
+ continue
+ key = author_identity_key(field(r, "author_name"), field(r, "author_email"))
+ if key not in last or latest > last[key]:
+ last[key] = latest
+ if since is not None and latest >= since:
+ active.add(key)
+ return {"author_last_commit": last, "active_authors_90d": len(active)}
+
+
+def _recent_candidates(
+ rows: Rows, *, week: datetime | None, files: Mapping[str, FileFacts]
+) -> list[tuple[Any, datetime]]:
+ """Authored critical/high code-shape findings from commits since ``week``."""
+ if week is None:
+ return []
+ hidden = excluded_types()
+ out = []
+ for f in rows:
+ at = _when(field(f, "committed_at"))
+ biomarker = field(f, "biomarker_type")
+ if (
+ at is not None
+ and at >= week
+ and field(f, "change_kind") in ("introduced", "worsened")
+ and field(f, "severity") in ("critical", "high")
+ and field(f, "attribution_basis") in AUTHORED_BASES
+ and not _is_test(field(f, "file_path"), files)
+ and biomarker_category(biomarker) != HISTORY_CATEGORY
+ and biomarker not in hidden
+ ):
+ out.append((f, at))
+ return out
+
+
+def build_recent(
+ rows: Rows,
+ *,
+ week: datetime | None,
+ open_findings: Rows,
+ files: Mapping[str, FileFacts],
+) -> dict[str, Any]:
+ """Findings a commit in the last week wrote that are still open."""
+ if week is None:
+ return {}
+ candidates = _recent_candidates(rows, week=week, files=files)
+ if not candidates:
+ return {"recent_findings": ()}
+ open_keys = {
+ (field(h, "file_path"), field(h, "biomarker_type"), field(h, "function_name"))
+ for h in open_findings
+ if _open(h)
+ }
+ recent = []
+ seen: set[tuple[str, str, str | None]] = set()
+ for f, at in sorted(candidates, key=lambda c: c[1], reverse=True):
+ key = (field(f, "file_path"), field(f, "biomarker_type"), field(f, "symbol"))
+ # Still open, and counted once however many commits touched it.
+ if key not in open_keys or key in seen:
+ continue
+ seen.add(key)
+ recent.append(
+ RecentFinding(
+ sha=field(f, "sha"),
+ subject=field(f, "subject") or "",
+ committed_at=at,
+ file_path=field(f, "file_path"),
+ symbol=field(f, "symbol"),
+ biomarker=field(f, "biomarker_type"),
+ severity=field(f, "severity"),
+ change_kind=field(f, "change_kind"),
+ line=field(f, "line_start"),
+ reason=field(f, "reason") or "",
+ )
+ )
+ return {"recent_findings": tuple(recent)}
+
+
+def build_perf(rows: Rows) -> dict[str, Any]:
+ """Open production opportunities that are ready to plan or advise on."""
+ out = []
+ for r in rows:
+ if not (
+ _open(r)
+ and field(r, "execution_context") == "production"
+ and field(r, "actionability_state") in ("plan_ready", "advisory")
+ ):
+ continue
+ details = detail_map(r)
+ facets = details.get("facets") or {}
+ plan = details.get("plan") or {}
+ out.append(
+ PerfFacts(
+ opportunity_id=field(r, "opportunity_id"),
+ biomarker=field(r, "biomarker_type"),
+ boundary=field(r, "boundary_kind"),
+ file_path=field(r, "file_path"),
+ symbol=field(r, "intervention_symbol"),
+ call_sites=field(r, "affected_call_sites_total") or 0,
+ files=field(r, "affected_files_total") or 0,
+ actionability=field(r, "actionability_state"),
+ exposure=facets.get("exposure"),
+ loop_magnitude=facets.get("loop_magnitude"),
+ effort=plan.get("effort_bucket"),
+ )
+ )
+ return {"perf": tuple(out)}
+
+
+def build_secrets(rows: Rows, files: Mapping[str, FileFacts]) -> dict[str, Any]:
+ """High-severity secrets in the working tree, outside tests."""
+ kinds = {*SECRET_KINDS, PUBLIC_ENV_KIND}
+ return {
+ "secrets": tuple(
+ SecretFacts(
+ field(r, "file_path"),
+ field(r, "kind"),
+ field(r, "line_number"),
+ field(r, "snippet") or "",
+ )
+ for r in rows
+ if not field(r, "commit_sha")
+ and field(r, "severity") == "high"
+ and field(r, "kind") in kinds
+ and not _is_test(field(r, "file_path"), files)
+ and not _test_path(field(r, "file_path"))
+ )
+ }
+
+
+def build_drift(rows: Rows, known_paths: Iterable[str]) -> dict[str, Any]:
+ known = set(known_paths)
+ return {
+ "drift": tuple(
+ DriftFacts(
+ document=field(r, "file_path"),
+ kind=field(r, "kind"),
+ target=field(r, "target") or "",
+ raw=field(r, "raw") or "",
+ line=field(r, "line_number"),
+ reason=field(r, "reason") or "",
+ confidence=float(field(r, "confidence") or 0.0),
+ target_known=(field(r, "target") in known),
+ )
+ for r in rows
+ )
+ }
+
+
+def build_dead(rows: Rows, files: Mapping[str, FileFacts]) -> dict[str, Any]:
+ """Open, deletion-ready findings outside tests."""
+ skipped = excluded_types() | REVIEW_ONLY_KINDS
+ return {
+ "dead": tuple(
+ DeadFacts(
+ field(r, "id"),
+ field(r, "file_path"),
+ field(r, "symbol_name"),
+ field(r, "lines") or 0,
+ )
+ for r in rows
+ if _open(r)
+ and field(r, "safe_to_delete")
+ and field(r, "kind") not in skipped
+ and not _is_test(field(r, "file_path"), files)
+ and not _test_path(field(r, "file_path"))
+ )
+ }
+
+
+def build_decisions(summary: Mapping[str, Any]) -> dict[str, Any]:
+ counts = summary.get("summary") or {}
+ return {
+ "stale_decisions": tuple(
+ DecisionFacts(field(d, "id"), field(d, "title"))
+ for d in summary.get("stale_decisions") or []
+ ),
+ "proposed_decisions": int(counts.get("proposed") or 0),
+ "accepted_decisions": int(counts.get("active") or 0),
+ }
+
+
+def build_coverage(row: Mapping[str, Any], head_sha: str | None) -> dict[str, Any]:
+ count = row.get("files_measured") or 0
+ if not count:
+ return {"coverage": CoverageState("unknown")}
+ sha = row.get("ingested_commit_sha")
+ # Stale only when both commits are known and differ; a report without a
+ # commit cannot be called out of date.
+ status = "stale" if sha and head_sha and sha != head_sha else "measured"
+ return {
+ "coverage": CoverageState(
+ status, _when(row.get("ingested_at")), int(count), bool(row.get("partial"))
+ )
+ }
+
+
+# --- all of it ----------------------------------------------------------------
+
+
+def build_repo_facts(
+ *,
+ anchor: datetime | str | None,
+ head_sha: str | None = None,
+ files: Rows | None = None,
+ fix_events: Rows = (),
+ dependents: Mapping[str, int] | None = None,
+ health_findings: Rows = (),
+ authors: Rows | None = None,
+ commit_health: Rows | None = None,
+ performance: Rows | None = None,
+ security: Rows | None = None,
+ doc_drift: Rows | None = None,
+ known_paths: Iterable[str] = (),
+ dead_code: Rows | None = None,
+ decisions: Mapping[str, Any] | None = None,
+ coverage: Mapping[str, Any] | None = None,
+ unavailable: Mapping[str, str] | None = None,
+ absent_reason: str = ABSENT,
+) -> RepoFacts:
+ """`RepoFacts` from rows (shapes in the module docstring).
+
+ ``anchor`` is the newest commit's time and ``head_sha`` its sha. A store
+ passed as ``None`` is reported unavailable with ``absent_reason``; one that
+ fails to build is reported the same way, so it costs only its own rules.
+ ``unavailable`` names stores the caller withholds, with the reason; they
+ are still built, so their facts inform the rest.
+ """
+ anchor = _when(anchor)
+ since = (anchor - QUARTER) if anchor else None
+ week = (anchor - WEEK) if anchor else None
+ values: dict[str, Any] = {"anchor": anchor}
+ missing: dict[str, str] = {}
+ health_findings = list(health_findings)
+
+ def build(store: str, rows: Any, fn: Callable[[Any], dict[str, Any]]) -> None:
+ if rows is None:
+ missing[store] = absent_reason
+ return
+ try:
+ values.update(fn(rows))
+ except Exception as exc: # one store must not cost the list
+ logger.warning("actions: %s unavailable: %s", store, exc)
+ missing[store] = absent_reason
+
+ def file_store(rows: Rows) -> dict[str, Any]:
+ out = build_files(rows, since=since, fix_events=fix_events, dependents=dependents)
+ out["files"] = with_leads(out["files"], health_findings)
+ return out
+
+ build("files", files, file_store)
+ known_files: Mapping[str, FileFacts] = values.get("files") or {}
+ build("authors", authors, lambda rows: build_authors(rows, since=since))
+ build(
+ "commit_health",
+ commit_health,
+ lambda rows: build_recent(
+ rows, week=week, open_findings=health_findings, files=known_files
+ ),
+ )
+ build("performance", performance, build_perf)
+ build("security", security, lambda rows: build_secrets(rows, known_files))
+ build("doc_drift", doc_drift, lambda rows: build_drift(rows, known_paths))
+ build("dead_code", dead_code, lambda rows: build_dead(rows, known_files))
+ build("decisions", decisions, build_decisions)
+ build("coverage", coverage, lambda row: build_coverage(row, head_sha))
+ values["unavailable"] = {**missing, **(unavailable or {})}
+ return RepoFacts(**values)
+
+
+__all__ = [
+ "ABSENT",
+ "AUTHORED_BASES",
+ "LEAD_LOOKUP_MIN_COMMITS",
+ "QUARTER",
+ "WEEK",
+ "build_authors",
+ "build_coverage",
+ "build_dead",
+ "build_decisions",
+ "build_drift",
+ "build_files",
+ "build_perf",
+ "build_recent",
+ "build_repo_facts",
+ "build_secrets",
+ "lead_paths",
+ "with_leads",
+]
diff --git a/packages/core/src/repowise/core/analysis/health/models.py b/packages/core/src/repowise/core/analysis/health/models.py
index 4a25848b6..5b5c1a594 100644
--- a/packages/core/src/repowise/core/analysis/health/models.py
+++ b/packages/core/src/repowise/core/analysis/health/models.py
@@ -105,6 +105,7 @@ def split_by_origin(findings: Iterable[Any]) -> tuple[list[Any], list[Any]]:
report the second as context.
"""
# Deferred: ``scoring`` imports this module for its data classes.
+ from .rows import field as row_field
from .scoring import HISTORY_CATEGORY, biomarker_category
code_shape: list[Any] = []
@@ -112,7 +113,7 @@ def split_by_origin(findings: Iterable[Any]) -> tuple[list[Any], list[Any]]:
for f in findings:
target = (
history
- if biomarker_category(getattr(f, "biomarker_type", "")) == HISTORY_CATEGORY
+ if biomarker_category(row_field(f, "biomarker_type", "")) == HISTORY_CATEGORY
else code_shape
)
target.append(f)
diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/actions.py b/packages/core/src/repowise/core/persistence/crud/analysis/actions.py
index 9a04006c8..407aa2cc2 100644
--- a/packages/core/src/repowise/core/persistence/crud/analysis/actions.py
+++ b/packages/core/src/repowise/core/persistence/crud/analysis/actions.py
@@ -3,39 +3,43 @@
Every store is read in its own savepoint and may fail on its own: an index
built before a detector shipped, or an older schema, costs that store's rules
(reported as unavailable, with the reason) rather than the whole list.
+
+The reads here only narrow; the row-to-fact rule is the pure builder in
+``repowise.core.analysis.actions.build``.
"""
from __future__ import annotations
-import json
import logging
-from collections import defaultdict
from collections.abc import Awaitable, Callable
-from dataclasses import replace
-from datetime import UTC, datetime, timedelta
+from datetime import UTC, datetime
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from repowise.core.analysis.actions import ActionStateRecord, RepoFacts, compose_actions
-from repowise.core.analysis.actions.facts import (
- CoverageState,
- DeadFacts,
- DecisionFacts,
- DriftFacts,
- FileFacts,
- LeadFinding,
- PerfFacts,
- RecentFinding,
- SecretFacts,
+from repowise.core.analysis.actions.build import (
+ ABSENT,
+ AUTHORED_BASES,
+ QUARTER,
+ WEEK,
+ build_authors,
+ build_coverage,
+ build_dead,
+ build_decisions,
+ build_drift,
+ build_files,
+ build_perf,
+ build_recent,
+ build_secrets,
+ lead_paths,
+ with_leads,
)
+from repowise.core.analysis.actions.facts import FileFacts
from repowise.core.analysis.actions.rules.hygiene import PUBLIC_ENV_KIND, SECRET_KINDS
from repowise.core.analysis.dead_code.risk_factors import REVIEW_ONLY_KINDS
from repowise.core.analysis.finding_registry import excluded_types
-from repowise.core.analysis.health.models import primary_finding, split_by_origin
-from repowise.core.analysis.health.scoring import HISTORY_CATEGORY, biomarker_category
-from repowise.core.author_identity import author_identity_key
from ...models import (
ActionState,
@@ -57,17 +61,15 @@ from ...models import (
logger = logging.getLogger(__name__)
-QUARTER = timedelta(days=90)
-WEEK = timedelta(days=7)
-
-#: Attribution bases that tie a finding to lines the commit wrote. A
-#: ``file_change`` basis only says the commit touched the file, which is not
-#: enough to tell someone they made it worse.
-AUTHORED_BASES = ("added_lines", "new_file", "changed_symbol")
-
-#: Files whose lead finding is looked up: busy bug magnets only, so the lookup
-#: stays a keyed read whatever the repository's size.
-LEAD_LOOKUP_MIN_COMMITS = 5
+_HEALTH_FINDING_COLUMNS = (
+ HealthFinding.file_path,
+ HealthFinding.biomarker_type,
+ HealthFinding.severity,
+ HealthFinding.function_name,
+ HealthFinding.line_start,
+ HealthFinding.reason,
+ HealthFinding.health_impact,
+)
async def _anchor(session: AsyncSession, repo_id: str) -> tuple[datetime | None, str | None]:
@@ -117,121 +119,55 @@ async def _files(session: AsyncSession, repo_id: str, since: datetime | None) ->
)
).all()
)
- fix_shas: dict[str, set[str]] = defaultdict(set)
- all_fix: set[str] = set()
+ fix_events: list[Any] = []
if since is not None:
- for path, sha in (
+ fix_events = (
await session.execute(
- select(FixEvent.file_path, FixEvent.fix_sha).where(
+ select(
+ FixEvent.file_path,
+ FixEvent.fix_sha,
+ FixEvent.committed_at,
+ FixEvent.shape_kind,
+ FixEvent.attribution,
+ ).where(
FixEvent.repository_id == repo_id,
FixEvent.committed_at >= since,
FixEvent.shape_kind == "code_fix",
FixEvent.attribution != "none",
)
)
- ).all():
- fix_shas[path].add(sha)
- all_fix.add(sha)
-
- files: dict[str, FileFacts] = {}
- for r in rows:
- files[r.file_path] = FileFacts(
- path=r.file_path,
- is_test=bool(r.is_test),
- score=r.score,
- max_ccn=r.max_ccn,
- nloc=r.nloc,
- line_coverage_pct=r.line_coverage_pct,
- commits_90d=r.commit_count_90d or 0,
- last_commit_at=r.last_commit_at,
- bug_magnet=bool(r.bug_magnet),
- fix_commits_90d=len(fix_shas.get(r.file_path, ())),
- bus_factor=r.bus_factor,
- owner_key=(
- author_identity_key(r.primary_owner_name, r.primary_owner_email)
- if (r.primary_owner_name or r.primary_owner_email)
- else None
- ),
- owner_name=r.primary_owner_name,
- owner_pct=r.primary_owner_commit_pct,
- dependents=dependents.get(r.file_path),
- )
-
- # Lead finding for the busy bug magnets: the code-shape finding an edit can
- # fix. History markers are context, never the thing to do.
- lead_paths = [
- p
- for p, f in files.items()
- if f.bug_magnet
- and not f.is_test
- and f.commits_90d >= LEAD_LOOKUP_MIN_COMMITS
- and f.fix_commits_90d >= 3
- ]
- if lead_paths:
+ ).all()
+ out = build_files(rows, since=since, fix_events=fix_events, dependents=dependents)
+ # The lead lookup stays keyed to the few files that get one.
+ paths = lead_paths(out["files"])
+ if paths:
findings = (
await session.execute(
- select(
- HealthFinding.file_path,
- HealthFinding.biomarker_type,
- HealthFinding.severity,
- HealthFinding.function_name,
- HealthFinding.line_start,
- HealthFinding.reason,
- HealthFinding.health_impact,
- ).where(
+ select(*_HEALTH_FINDING_COLUMNS).where(
HealthFinding.repository_id == repo_id,
HealthFinding.status == "open",
- HealthFinding.file_path.in_(lead_paths),
+ HealthFinding.file_path.in_(paths),
HealthFinding.biomarker_type.not_in(excluded_types()),
)
)
).all()
- by_path: dict[str, list] = defaultdict(list)
- for f in findings:
- by_path[f.file_path].append(f)
- for path, found in by_path.items():
- shape, _history = split_by_origin(found)
- lead = primary_finding(shape)
- if lead is not None:
- files[path] = replace(
- files[path],
- lead=LeadFinding(
- biomarker=lead.biomarker_type,
- severity=lead.severity,
- function=lead.function_name,
- line=lead.line_start,
- reason=lead.reason or "",
- ),
- )
- return {
- "files": files,
- "fix_shas_by_file": {p: frozenset(s) for p, s in fix_shas.items()},
- "fix_commits_90d": len(all_fix),
- }
+ out["files"] = with_leads(out["files"], findings)
+ return out
async def _authors(session: AsyncSession, repo_id: str, since: datetime | None) -> dict[str, Any]:
- last: dict[str, datetime] = {}
- active: set[str] = set()
- for name, email, latest in (
+ rows = (
await session.execute(
select(
GitCommit.author_name,
GitCommit.author_email,
- func.max(GitCommit.committed_at),
+ func.max(GitCommit.committed_at).label("committed_at"),
)
.where(GitCommit.repository_id == repo_id)
.group_by(GitCommit.author_name, GitCommit.author_email)
)
- ).all():
- if latest is None:
- continue
- key = author_identity_key(name, email)
- if key not in last or latest > last[key]:
- last[key] = latest
- if since is not None and latest >= since:
- active.add(key)
- return {"author_last_commit": last, "active_authors_90d": len(active)}
+ ).all()
+ return build_authors(rows, since=since)
async def _recent(
@@ -241,7 +177,19 @@ async def _recent(
return {}
rows = (
await session.execute(
- select(GitCommitHealthFinding, GitCommit.subject, GitCommit.committed_at)
+ select(
+ GitCommitHealthFinding.sha,
+ GitCommitHealthFinding.file_path,
+ GitCommitHealthFinding.symbol,
+ GitCommitHealthFinding.biomarker_type,
+ GitCommitHealthFinding.severity,
+ GitCommitHealthFinding.change_kind,
+ GitCommitHealthFinding.line_start,
+ GitCommitHealthFinding.reason,
+ GitCommitHealthFinding.attribution_basis,
+ GitCommit.subject,
+ GitCommit.committed_at,
+ )
.join(
GitCommit,
(GitCommit.repository_id == GitCommitHealthFinding.repository_id)
@@ -256,55 +204,18 @@ async def _recent(
)
)
).all()
- candidates = [
- (f, subject, at)
- for f, subject, at in rows
- if not (files.get(f.file_path) and files[f.file_path].is_test)
- and biomarker_category(f.biomarker_type) != HISTORY_CATEGORY
- and f.biomarker_type not in excluded_types()
- ]
- if not candidates:
- return {"recent_findings": ()}
- paths = {f.file_path for f, _, _ in candidates}
- open_keys = {
- (p, b, fn)
- for p, b, fn in (
+ open_findings: list[Any] = []
+ if rows:
+ open_findings = (
await session.execute(
- select(
- HealthFinding.file_path,
- HealthFinding.biomarker_type,
- HealthFinding.function_name,
- ).where(
+ select(*_HEALTH_FINDING_COLUMNS).where(
HealthFinding.repository_id == repo_id,
HealthFinding.status == "open",
- HealthFinding.file_path.in_(paths),
+ HealthFinding.file_path.in_({r.file_path for r in rows}),
)
)
).all()
- }
- recent = []
- seen: set[tuple[str, str, str | None]] = set()
- for f, subject, at in sorted(candidates, key=lambda r: r[2] or datetime.min, reverse=True):
- key = (f.file_path, f.biomarker_type, f.symbol)
- # Still open, and counted once however many commits touched it.
- if key not in open_keys or key in seen:
- continue
- seen.add(key)
- recent.append(
- RecentFinding(
- sha=f.sha,
- subject=subject or "",
- committed_at=at,
- file_path=f.file_path,
- symbol=f.symbol,
- biomarker=f.biomarker_type,
- severity=f.severity,
- change_kind=f.change_kind,
- line=f.line_start,
- reason=f.reason or "",
- )
- )
- return {"recent_findings": tuple(recent)}
+ return build_recent(rows, week=since, open_findings=open_findings, files=files)
async def _perf(session: AsyncSession, repo_id: str) -> dict[str, Any]:
@@ -322,30 +233,7 @@ async def _perf(session: AsyncSession, repo_id: str) -> dict[str, Any]:
.scalars()
.all()
)
- out = []
- for r in rows:
- try:
- details = json.loads(r.details_json or "{}")
- except ValueError:
- details = {}
- facets = details.get("facets") or {}
- plan = details.get("plan") or {}
- out.append(
- PerfFacts(
- opportunity_id=r.opportunity_id,
- biomarker=r.biomarker_type,
- boundary=r.boundary_kind,
- file_path=r.file_path,
- symbol=r.intervention_symbol,
- call_sites=r.affected_call_sites_total or 0,
- files=r.affected_files_total or 0,
- actionability=r.actionability_state,
- exposure=facets.get("exposure"),
- loop_magnitude=facets.get("loop_magnitude"),
- effort=plan.get("effort_bucket"),
- )
- )
- return {"perf": tuple(out)}
+ return build_perf(rows)
async def _secrets(session: AsyncSession, repo_id: str, files: dict[str, FileFacts]) -> dict:
@@ -356,6 +244,8 @@ async def _secrets(session: AsyncSession, repo_id: str, files: dict[str, FileFac
SecurityFinding.kind,
SecurityFinding.line_number,
SecurityFinding.snippet,
+ SecurityFinding.severity,
+ SecurityFinding.commit_sha,
).where(
SecurityFinding.repository_id == repo_id,
SecurityFinding.commit_sha == "",
@@ -364,22 +254,7 @@ async def _secrets(session: AsyncSession, repo_id: str, files: dict[str, FileFac
)
)
).all()
- return {
- "secrets": tuple(
- SecretFacts(r.file_path, r.kind, r.line_number, r.snippet or "")
- for r in rows
- if not (files.get(r.file_path) and files[r.file_path].is_test)
- and not _test_path(r.file_path)
- )
- }
-
-
-def _test_path(path: str) -> bool:
- lowered = path.lower()
- return any(
- seg in lowered
- for seg in ("/tests/", "/test/", "__tests__", "/fixtures/", ".test.", ".spec.", "_test.")
- ) or lowered.startswith(("tests/", "test/"))
+ return build_secrets(rows, files)
async def _drift(session: AsyncSession, repo_id: str) -> dict[str, Any]:
@@ -425,21 +300,7 @@ async def _drift(session: AsyncSession, repo_id: str) -> dict[str, Any]:
.scalars()
.all()
)
- return {
- "drift": tuple(
- DriftFacts(
- document=r.file_path,
- kind=r.kind,
- target=r.target or "",
- raw=r.raw or "",
- line=r.line_number,
- reason=r.reason or "",
- confidence=float(r.confidence or 0.0),
- target_known=(r.target in known),
- )
- for r in rows
- )
- }
+ return build_drift(rows, known)
async def _dead(session: AsyncSession, repo_id: str, files: dict[str, FileFacts]) -> dict:
@@ -450,6 +311,9 @@ async def _dead(session: AsyncSession, repo_id: str, files: dict[str, FileFacts]
DeadCodeFinding.file_path,
DeadCodeFinding.symbol_name,
DeadCodeFinding.lines,
+ DeadCodeFinding.kind,
+ DeadCodeFinding.safe_to_delete,
+ DeadCodeFinding.status,
).where(
DeadCodeFinding.repository_id == repo_id,
DeadCodeFinding.status == "open",
@@ -458,28 +322,13 @@ async def _dead(session: AsyncSession, repo_id: str, files: dict[str, FileFacts]
)
)
).all()
- return {
- "dead": tuple(
- DeadFacts(r.id, r.file_path, r.symbol_name, r.lines or 0)
- for r in rows
- if not (files.get(r.file_path) and files[r.file_path].is_test)
- and not _test_path(r.file_path)
- )
- }
+ return build_dead(rows, files)
async def _decisions(session: AsyncSession, repo_id: str) -> dict[str, Any]:
from ..decision_health import get_decision_health_summary
- summary = await get_decision_health_summary(session, repo_id)
- counts = summary.get("summary") or {}
- return {
- "stale_decisions": tuple(
- DecisionFacts(d.id, d.title) for d in summary.get("stale_decisions") or []
- ),
- "proposed_decisions": int(counts.get("proposed") or 0),
- "accepted_decisions": int(counts.get("active") or 0),
- }
+ return build_decisions(await get_decision_health_summary(session, repo_id))
async def _coverage(session: AsyncSession, repo_id: str, head_sha: str | None) -> dict:
@@ -492,38 +341,42 @@ async def _coverage(session: AsyncSession, repo_id: str, head_sha: str | None) -
).where(CoverageFile.repository_id == repo_id)
)
).one()
- if not count:
- return {"coverage": CoverageState("unknown")}
- # The ingest record is authoritative for when, at which commit, and whether
- # the report's paths all mapped. Indexes that predate it carry the same
- # commit on every row, because one ingest replaces them all.
partial = False
- try:
- async with session.begin_nested():
- ingest = (
- await session.execute(
- select(
- CoverageIngest.ingested_at,
- CoverageIngest.ingested_commit_sha,
- CoverageIngest.mapping_partial,
+ if count:
+ # The ingest record is authoritative for when, at which commit, and
+ # whether the report's paths all mapped. Indexes that predate it carry
+ # the same commit on every row, because one ingest replaces them all.
+ try:
+ async with session.begin_nested():
+ ingest = (
+ await session.execute(
+ select(
+ CoverageIngest.ingested_at,
+ CoverageIngest.ingested_commit_sha,
+ CoverageIngest.mapping_partial,
+ )
+ .where(CoverageIngest.repository_id == repo_id)
+ .order_by(CoverageIngest.ingested_at.desc())
+ .limit(1)
)
- .where(CoverageIngest.repository_id == repo_id)
- .order_by(CoverageIngest.ingested_at.desc())
- .limit(1)
+ ).first()
+ if ingest is not None:
+ latest_at, latest_sha, partial = (
+ ingest.ingested_at,
+ ingest.ingested_commit_sha,
+ bool(ingest.mapping_partial),
)
- ).first()
- if ingest is not None:
- latest_at, latest_sha, partial = (
- ingest.ingested_at,
- ingest.ingested_commit_sha,
- bool(ingest.mapping_partial),
- )
- except Exception: # an index from before ingests were recorded
- pass
- # Stale only when both commits are known and differ; a report without a
- # commit cannot be called out of date.
- status = "stale" if latest_sha and head_sha and latest_sha != head_sha else "measured"
- return {"coverage": CoverageState(status, latest_at, int(count), partial)}
+ except Exception: # an index from before ingests were recorded
+ pass
+ return build_coverage(
+ {
+ "files_measured": count,
+ "ingested_at": latest_at,
+ "ingested_commit_sha": latest_sha,
+ "partial": partial,
+ },
+ head_sha,
+ )
_Reader = Callable[[], Awaitable[dict[str, Any]]]
@@ -542,7 +395,7 @@ async def load_repo_facts(session: AsyncSession, repo_id: str) -> RepoFacts:
values.update(await reader())
except Exception as exc: # one store must not cost the list
logger.warning("actions: %s unavailable: %s", store, exc)
- unavailable[store] = "Not in this index yet; run `repowise update`."
+ unavailable[store] = ABSENT
await read("files", lambda: _files(session, repo_id, since))
files: dict[str, FileFacts] = values.get("files") or {}
diff --git a/tests/unit/persistence/test_actions_build.py b/tests/unit/persistence/test_actions_build.py
new file mode 100644
index 000000000..7d96e4c37
--- /dev/null
+++ b/tests/unit/persistence/test_actions_build.py
@@ -0,0 +1,199 @@
+"""The pure facts builder over plain rows agrees with the SQL loader.
+
+The rows below restate the loader test's seed as dicts, unfiltered and with ISO
+timestamps, the way a caller holding its rows in memory would pass them.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import timedelta
+
+from sqlalchemy import select
+
+from repowise.core.analysis.actions.build import ABSENT, build_repo_facts
+from repowise.core.persistence.crud.analysis.actions import load_repo_facts
+from repowise.core.persistence.models import DeadCodeFinding
+from tests.unit.persistence.test_actions_loader import ANCHOR, _seed
+
+# SQLite hands datetimes back naive; the rows match it, and the column defaults.
+_T = ANCHOR.replace(tzinfo=None)
+
+
+def _iso(delta: timedelta = timedelta()) -> str:
+ return (_T - delta).isoformat()
+
+
+COMMITS = [
+ {"sha": "head", "author_name": "Ada", "author_email": "[email protected]",
+ "committed_at": _iso(), "subject": "chore: head"},
+ {"sha": "recent", "author_name": "Bob", "author_email": "[email protected]",
+ "committed_at": _iso(timedelta(days=2)), "subject": "feat: build step"},
+ {"sha": "older", "author_name": "Cy", "author_email": "[email protected]",
+ "committed_at": _iso(timedelta(days=20)), "subject": "fix: things"},
+]
+_BY_SHA = {c["sha"]: c for c in COMMITS}
+
+FILES = [
+ {"file_path": "src/core.py", "commit_count_90d": 12, "bug_magnet": True,
+ "last_commit_at": _iso(timedelta(days=1)), "bus_factor": 0,
+ "score": 10.0, "max_ccn": 12, "nloc": 300, "is_test": False},
+]
+
+FIX_EVENTS = [
+ {"file_path": "src/core.py", "fix_sha": f"fix{i}", "shape_kind": "code_fix",
+ "attribution": "exact", "committed_at": _iso(timedelta(days=10 + i))}
+ for i in range(4)
+]
+
+HEALTH_FINDINGS = [
+ {"file_path": "src/core.py", "biomarker_type": "change_entropy", "severity": "high",
+ "health_impact": 3.0},
+ {"file_path": "src/core.py", "biomarker_type": "complex_method", "severity": "high",
+ "function_name": "run", "health_impact": 1.0},
+ {"file_path": "src/new.py", "biomarker_type": "complex_method", "severity": "high",
+ "function_name": "build", "health_impact": 1.0},
+ {"file_path": "src/touched.py", "biomarker_type": "long_method", "severity": "high",
+ "function_name": "x", "health_impact": 1.0},
+]
+
+COMMIT_HEALTH = [
+ {**_BY_SHA[sha], "sha": sha, "change_kind": "introduced", "biomarker_type": biomarker,
+ "severity": "high", "file_path": path, "symbol": symbol, "attribution_basis": basis}
+ for sha, biomarker, path, symbol, basis in [
+ ("recent", "complex_method", "src/new.py", "build", "added_lines"),
+ ("recent", "long_method", "src/touched.py", "x", "file_change"),
+ ("recent", "change_entropy", "src/hist.py", None, "added_lines"),
+ ]
+]
+
+PERFORMANCE = [
+ {"opportunity_id": "perf1", "execution_context": "production",
+ "actionability_state": "plan_ready", "boundary_kind": "db", "biomarker_type": "n_plus_one",
+ "file_path": "src/repo.py", "intervention_symbol": "src/repo.py::Repo.load",
+ "affected_call_sites_total": 4, "affected_files_total": 2,
+ "details": {"facets": {"exposure": "entry_reachable", "loop_magnitude": "grows_with_data"},
+ "plan": {"effort_bucket": "S"}}},
+ {"opportunity_id": "perf2", "execution_context": "production",
+ "actionability_state": "plan_ready", "boundary_kind": "db", "biomarker_type": "",
+ "file_path": "src/other.py",
+ "affected_call_sites_total": 9,
+ "details_json": json.dumps({"facets": {"exposure": "internal"}})},
+]
+
+SECURITY = [
+ {"file_path": "src/settings.py", "kind": "hardcoded_secret", "severity": "high",
+ "snippet": 'TOKEN = "sk_live_123"', "line_number": 4, "commit_sha": ""},
+ {"file_path": "tests/test_x.py", "kind": "hardcoded_secret", "severity": "high",
+ "snippet": 'TOKEN = "fake"', "line_number": 4, "commit_sha": ""},
+]
+
+DOC_DRIFT = [
+ {"file_path": "README.md", "kind": "anchor", "line_number": 3, "target": "README.md#setup",
+ "raw": "#setup", "confidence": 0.9},
+ {"file_path": "docs/guide.md", "kind": "path", "line_number": 5, "target": "src/old.py",
+ "raw": "src/old.py", "confidence": 0.9},
+ {"file_path": "docs/other.md", "kind": "path", "line_number": 5,
+ "target": "elsewhere/never.py", "raw": "elsewhere/never.py", "confidence": 0.9},
+]
+
+
+def _rows(dead_ids: list[str], **overrides):
+ rows = {
+ "anchor": _iso(),
+ "head_sha": "head",
+ "files": FILES,
+ "fix_events": FIX_EVENTS,
+ "health_findings": HEALTH_FINDINGS,
+ "authors": COMMITS,
+ "commit_health": COMMIT_HEALTH,
+ "performance": PERFORMANCE,
+ "security": SECURITY,
+ "doc_drift": DOC_DRIFT,
+ "known_paths": ["src/old.py"],
+ "dead_code": [
+ {"id": i, "kind": "unused_export", "file_path": "src/dead.py",
+ "symbol_name": f"f{n}", "lines": 10, "safe_to_delete": True}
+ for n, i in enumerate(dead_ids)
+ ],
+ "decisions": {"stale_decisions": [], "summary": {}},
+ "coverage": {"files_measured": 0},
+ }
+ return {**rows, **overrides}
+
+
+async def _dead_ids(session, rid: str) -> list[str]:
+ return list(
+ (
+ await session.execute(
+ select(DeadCodeFinding.id)
+ .where(DeadCodeFinding.repository_id == rid)
+ .order_by(DeadCodeFinding.symbol_name)
+ )
+ ).scalars()
+ )
+
+
+async def test_builder_on_rows_matches_the_sql_loader(async_session) -> None:
+ rid = await _seed(async_session)
+ loaded = await load_repo_facts(async_session, rid)
+ rows = _rows(await _dead_ids(async_session, rid))
+ # Rows the loader's reads never return: each builder drops them itself.
+ rows["fix_events"] = [
+ *FIX_EVENTS,
+ {"file_path": "src/core.py", "fix_sha": "doc", "shape_kind": "docs",
+ "attribution": "exact", "committed_at": _iso(timedelta(days=1))},
+ {"file_path": "src/core.py", "fix_sha": "old", "shape_kind": "code_fix",
+ "attribution": "exact", "committed_at": _iso(timedelta(days=200))},
+ ]
+ rows["health_findings"] = [
+ *HEALTH_FINDINGS,
+ {"file_path": "src/core.py", "biomarker_type": "long_method", "severity": "critical",
+ "function_name": "gone", "health_impact": 9.0, "status": "resolved"},
+ ]
+ rows["performance"] = [
+ *PERFORMANCE,
+ {**PERFORMANCE[0], "opportunity_id": "perf3", "execution_context": "test"},
+ {**PERFORMANCE[0], "opportunity_id": "perf4", "status": "resolved"},
+ ]
+ rows["security"] = [*SECURITY, {**SECURITY[0], "commit_sha": "abc"},
+ {**SECURITY[0], "severity": "medium"}]
+ rows["dead_code"] = [
+ *rows["dead_code"],
+ {"id": "unsafe", "kind": "unused_export", "file_path": "src/dead.py",
+ "symbol_name": "g", "lines": 5, "safe_to_delete": False},
+ {"id": "resolved", "kind": "unused_export", "file_path": "src/dead.py",
+ "symbol_name": "h", "lines": 5, "safe_to_delete": True, "status": "resolved"},
+ {"id": "in_tests", "kind": "unused_export", "file_path": "tests/helpers.py",
+ "symbol_name": "k", "lines": 5, "safe_to_delete": True},
+ ]
+ built = build_repo_facts(**rows)
+
+ assert built == loaded
+ # The seed exercises every store, so equality is not over empty facts.
+ assert built.files["src/core.py"].lead.biomarker == "complex_method"
+ assert [f.file_path for f in built.recent_findings] == ["src/new.py"]
+ assert [s.file_path for s in built.secrets] == ["src/settings.py"]
+ assert (built.fix_commits_90d, built.active_authors_90d, len(built.dead)) == (4, 3, 3)
+
+
+def test_a_store_passed_as_none_is_unavailable() -> None:
+ facts = build_repo_facts(**_rows([], performance=None, security=None))
+ assert facts.unavailable == {"performance": ABSENT, "security": ABSENT}
+ assert facts.perf == () and facts.secrets == ()
+ assert facts.drift
+
+
+def test_a_withheld_store_is_still_built_and_named() -> None:
+ facts = build_repo_facts(**_rows([], unavailable={"files": "No fix history here."}))
+ assert facts.unavailable == {"files": "No fix history here."}
+ # Its facts still keep a test file out of the other stores.
+ assert "src/core.py" in facts.files
+
+
+def test_a_store_that_fails_to_build_costs_only_itself() -> None:
+ facts = build_repo_facts(
+ **_rows([], doc_drift=[{"confidence": "not a number"}]), absent_reason="Not here."
+ )
+ assert facts.unavailable == {"doc_drift": "Not here."}
+ assert facts.secrets