Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(core): move decision records, answer parsing, staleness and commit helpers out of the extractor (#2589)

Raghav Chamadiyace433c0fb84a
diff --git a/packages/core/src/repowise/core/analysis/decisions/commit_mining.py b/packages/core/src/repowise/core/analysis/decisions/commit_mining.py
new file mode 100644
index 000000000..f4544bfc5
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/decisions/commit_mining.py
@@ -0,0 +1,238 @@
+"""Shared pieces of the two commit miners: git archaeology and PR bodies."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from typing import Any
+
+from repowise.core.analysis.decisions.scope import (
+ SCOPE_BASIS_SELECTED,
+ commit_scope_basis,
+ commit_scope_files,
+ selected_scope_files,
+)
+
+from .commit_signals import count_decision_signals
+from .records import ExtractedDecision
+
+#: The output budget for one batch of either commit prompt. Reasoning tokens
+#: are charged to it, so a budget that only fits the answer buys an empty
+#: body rather than a short one; see decision ``1c228ed8``.
+_BATCH_MAX_TOKENS = 8000
+
+#: How many of a commit's files either commit prompt will show. Sorted before
+#: truncating, so the files offered depend on the commit, not on map order.
+_MAX_PROMPT_FILES = 20
+
+# PR/squash body markers: a body containing any of these reads like a PR
+# description worth mining (vs an incidental multi-line commit message).
+_PR_BODY_MARKERS = (
+ "## why",
+ "## motivation",
+ "## what",
+ "## changes",
+ "## context",
+ "## summary",
+ "closes #",
+ "fixes #",
+ "resolves #",
+ "before:",
+ "after:",
+)
+_MAX_PR_BODIES = 25
+
+
+def _scope_from_selection(
+ decision: ExtractedDecision,
+ commit_files: Sequence[str] | None,
+) -> tuple[list[str], str]:
+ """The files and basis for one decision mined out of one commit.
+
+ Returns the model's own selection, validated against the commit's file
+ list, under :data:`SCOPE_BASIS_SELECTED`. An empty selection stays empty:
+ the record keeps its commit and evidence but governs no file. Only a
+ *missing* answer falls back to the commit-wide scope.
+ """
+ if decision.proposed_files is None:
+ return (commit_scope_files(commit_files), commit_scope_basis(commit_files))
+ chosen = selected_scope_files(decision.proposed_files, commit_files)
+ decision.proposed_files = None
+ return (chosen, SCOPE_BASIS_SELECTED)
+
+
+def _signal_commit_info(commit: dict) -> dict | None:
+ """The prompt-ready record of a commit with decision signals, else None."""
+ msg = commit.get("message", "")
+ body = commit.get("body", "")
+ # Scan subject and body: squash-merge repos carry the rationale in the body.
+ signal_count = count_decision_signals(f"{msg}\n{body}".lower())
+ if signal_count <= 0:
+ return None
+ return {
+ "sha": commit.get("sha", ""),
+ "message": msg,
+ "body": body,
+ "author": commit.get("author", ""),
+ "date": commit.get("date", ""),
+ "signal_count": signal_count,
+ }
+
+
+def _attribute_to_commit(
+ decision: ExtractedDecision,
+ batch: list[dict],
+ subject_key: str,
+ files_by_sha: dict[str, list[str]],
+ source_by_sha: dict[str, str],
+) -> str:
+ """Bind a decision mined from a batch of commits to the commit it came from.
+
+ Prefers the sha the model reported, else the first commit whose subject
+ (``batch[i][subject_key]``, first 40 chars) appears in the title. Scopes
+ the decision to that commit's files and returns the sha, or ``""`` when
+ nothing matched.
+ """
+ sha = decision.evidence_commits[0] if decision.evidence_commits else ""
+ if not sha:
+ # Try to match back to a commit
+ for c in batch:
+ if c[subject_key][:40].lower() in decision.title.lower():
+ sha = c["sha"]
+ break
+ if sha:
+ decision.evidence_commits = [sha]
+ decision.affected_files, decision.scope_basis = _scope_from_selection(
+ decision, files_by_sha.get(sha)
+ )
+ decision.source_text = source_by_sha.get(sha, "")
+ # Cleared either way: an unattributed decision has no commit to validate
+ # the model's paths against.
+ decision.proposed_files = None
+ return sha
+
+
+def _significant_commits(meta: dict) -> Any:
+ """A file's ``significant_commits_json`` as the miner reads it.
+
+ A string that does not decode yields no commits; anything else passes
+ through as-is.
+ """
+ commits_json = meta.get("significant_commits_json", "[]")
+ if not isinstance(commits_json, str):
+ return commits_json
+ try:
+ return json.loads(commits_json)
+ except (json.JSONDecodeError, TypeError):
+ return []
+
+
+def _admit_signal_commit(commit: dict, commit_map: dict[str, dict]) -> bool:
+ """Whether *commit*'s file should be counted, adding it to *commit_map* on first sight.
+
+ A new sha is admitted only when it carries decision signals.
+ """
+ sha = commit.get("sha", "")
+ if not sha or sha in commit_map:
+ return True
+ info = _signal_commit_info(commit)
+ if info is None:
+ return False
+ commit_map[sha] = info
+ return True
+
+
+def signal_commits(
+ git_meta_map: dict[str, dict],
+) -> tuple[dict[str, dict], dict[str, list[str]]]:
+ """Unique significant commits carrying decision signals, and each one's files.
+
+ Returns ``(sha -> commit info, sha -> files it touched)``.
+ """
+ commit_map: dict[str, dict] = {}
+ commit_files: dict[str, list[str]] = {}
+ for file_path, meta in git_meta_map.items():
+ for commit in _significant_commits(meta):
+ if _admit_signal_commit(commit, commit_map):
+ commit_files.setdefault(commit.get("sha", ""), []).append(file_path)
+ return commit_map, commit_files
+
+
+def _loads_commits(value: Any) -> list[dict]:
+ """Parse a ``significant_commits_json`` blob into a list of dicts."""
+ if isinstance(value, list):
+ return value
+ if isinstance(value, str):
+ try:
+ data = json.loads(value)
+ except (json.JSONDecodeError, TypeError):
+ return []
+ return data if isinstance(data, list) else []
+ return []
+
+
+def _pr_candidate(commit: dict, body: str) -> dict | None:
+ """The prompt-ready record of a PR-shaped body with decision signals, else None."""
+ low = body.lower()
+ is_prish = commit.get("pr_number") is not None or any(m in low for m in _PR_BODY_MARKERS)
+ if not is_prish or count_decision_signals(low) <= 0:
+ return None
+ return {
+ "sha": commit["sha"],
+ "subject": commit.get("message", ""),
+ "body": body,
+ "pr": commit.get("pr_number"),
+ }
+
+
+def pr_candidates(
+ git_meta_map: dict[str, dict],
+) -> tuple[dict[str, dict], dict[str, list[str]]]:
+ """PR / squash bodies worth mining, and every commit's files.
+
+ Returns ``(sha -> candidate, sha -> files it touched)``.
+ """
+ candidates: dict[str, dict] = {}
+ files_by_sha: dict[str, list[str]] = {}
+ for fp, meta in git_meta_map.items():
+ for c in _loads_commits(meta.get("significant_commits_json")):
+ sha = c.get("sha", "")
+ if not sha:
+ continue
+ files_by_sha.setdefault(sha, []).append(fp)
+ body = (c.get("body") or "").strip()
+ if sha in candidates or not body:
+ continue
+ candidate = _pr_candidate(c, body)
+ if candidate is not None:
+ candidates[sha] = candidate
+ return candidates, files_by_sha
+
+
+def _prompt_files(files: list[str]) -> str:
+ return ", ".join(sorted(files)[:_MAX_PROMPT_FILES])
+
+
+def git_commit_block(commit: dict, body: str, files: list[str]) -> str:
+ """One commit's entry in the git archaeology prompt."""
+ body_block = f"Body: {body[:1500]}\n" if body else ""
+ return (
+ f"\n--- Commit {commit['sha'][:8]} ---\n"
+ f"Message: {commit['message']}\n"
+ f"{body_block}"
+ f"Author: {commit['author']}\n"
+ f"Date: {commit['date']}\n"
+ f"Files changed: "
+ f"{_prompt_files(files)}\n"
+ )
+
+
+def pr_commit_block(candidate: dict, files: list[str]) -> str:
+ """One PR body's entry in the PR mining prompt."""
+ pr_label = f" (PR #{candidate['pr']})" if candidate.get("pr") else ""
+ return (
+ f"\n--- Commit {candidate['sha'][:8]}{pr_label} ---\n"
+ f"Subject: {candidate['subject']}\n"
+ f"Body:\n{candidate['body'][:2000]}\n"
+ f"Files changed: {_prompt_files(files)}\n"
+ )
diff --git a/packages/core/src/repowise/core/analysis/decisions/extractor.py b/packages/core/src/repowise/core/analysis/decisions/extractor.py
index caf1ebc59..7760b3460 100644
--- a/packages/core/src/repowise/core/analysis/decisions/extractor.py
+++ b/packages/core/src/repowise/core/analysis/decisions/extractor.py
@@ -1,73 +1,59 @@
-"""Architectural Decision Intelligence - extraction from multiple sources.
-
-Capture sources (see ``decision_provenance.SOURCE_RANK`` for the trust ladder):
- 1. Inline markers (# WHY:, # DECISION:, etc.)
- 2. Git archaeology (significant commit messages)
- 3. ADR auto-discovery (Nygard/MADR records — deterministic parse first)
- 4. PR / squash-body mining (commit bodies captured in git indexing)
- 5. Comment archaeology (LLM rationale prose on high-centrality code)
- + CLI capture (manual entry)
-
-Sources can be disabled per-repo via ``decisions.sources`` in
-``.repowise/config.yaml`` (see :data:`SOURCE_NAMES` /
-:meth:`DecisionExtractor.extract_all`).
-
-Three sources have been retired, all for the same reason: they mined prose that
-describes a repo rather than evidence of a choice made in it, and the records
-they produced were never acted on. ``code_comment`` went first (#751) — the
-query-time live-grep miner serves the same comments fresh, so persisting them
-only flooded the proposed queue. ``readme_mining`` and ``changelog`` follow:
-between them they produced 153 records in this project's own store and **zero**
-that ever became active, while accounting for most of a 214-deep review queue.
-Retired source names are kept in ``SOURCE_RANK`` so rows written before the
-removal still rank, and :data:`RETIRED_SOURCES` drives the one-shot purge on the
-persist path. The ADR miner still borrows ``README_MINING_PROMPT`` for its
-unstructured-file fallback; that is the prompt, not the source.
-
-Determinism-first: ADRs are parsed structurally before any LLM call.
-Every extracted decision passes an anti-hallucination substring gate
-(:meth:`DecisionExtractor._apply_substring_gate`) — fields not grounded in the
-verbatim source span are dropped, and evidence-less decisions are rejected.
-
-All LLM calls are wrapped in try/except - failures never propagate.
+"""Architectural decision extraction from every index-time source.
+
+Sources run in :data:`SOURCE_NAMES` order (inline markers, git archaeology,
+ADRs, PR / squash bodies, comment archaeology, conventions) and can be disabled
+per repo via ``decisions.sources`` in ``.repowise/config.yaml``. ADRs are parsed
+structurally before any LLM call.
+
+``code_comment``, ``readme_mining`` and ``changelog`` are retired: they mined
+prose that describes a repo rather than evidence of a choice made in it. Their
+names stay in ``SOURCE_RANK`` so older rows still rank, and
+:data:`RETIRED_SOURCES` drives the purge on the persist path.
+
+Every decision passes the anti-hallucination substring gate before it is
+returned, and a source that fails is reported in the result, never raised.
"""
from __future__ import annotations
import asyncio
import contextlib
-import json
import re
from collections.abc import Collection, Iterator, Sequence
-from dataclasses import dataclass, field
-from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import structlog
from repowise.core.analysis.decisions.gate import apply_substring_gate
-from repowise.core.analysis.decisions.lifecycle import ARCHITECTURAL_KIND
from repowise.core.analysis.decisions.policy import (
INDEX_SOURCE_KEYS,
DecisionPolicy,
resolve_policy,
)
-from repowise.core.analysis.decisions.scope import (
- SCOPE_BASIS_SELECTED,
- commit_scope_basis,
- commit_scope_files,
- resolve_module_nodes,
- selected_scope_files,
-)
+from repowise.core.analysis.decisions.scope import resolve_module_nodes
from .adr import _ADR_STATUS_MAP, bullets, find_adr_files, read_front_matter, split_headings
-from .commit_signals import count_decision_signals
+from .commit_mining import (
+ _BATCH_MAX_TOKENS,
+ _MAX_PR_BODIES,
+ _attribute_to_commit,
+ git_commit_block,
+ pr_candidates,
+ pr_commit_block,
+ signal_commits,
+)
from .markers import ( # noqa: F401 (MARKER_RE re-exported)
MARKER_RE,
find_markers,
strip_code_blocks,
)
+from .model_answers import ( # noqa: F401 (_coerce_paths, _collect_batches re-exported)
+ _coerce_paths,
+ _collect_batches,
+ _run_batches,
+ parse_decisions_json,
+)
from .prompts import (
_SYSTEM_PROMPT,
COMMENT_ARCHAEOLOGY_PROMPT,
@@ -76,11 +62,18 @@ from .prompts import (
PR_BODY_MINING_PROMPT,
README_MINING_PROMPT,
)
+from .records import ( # noqa: F401 (the two errors are re-exported)
+ DecisionExtractionReport,
+ DecisionSourceError,
+ EmptyModelResponseError,
+ ExtractedDecision,
+)
from .source_files import (
_BINARY_EXTENSIONS,
extract_leading_prose,
iter_source_files,
)
+from .staleness import _as_aware_utc, compute_staleness, last_code_change # noqa: F401
logger = structlog.get_logger(__name__)
@@ -98,210 +91,11 @@ def _truncate_title(text: str, limit: int) -> str:
window = text[:limit]
cut = window.rfind(" ")
if cut <= 0:
- # Single over-long word — hard cut, still signal truncation.
+ # Single over-long word: hard cut, still signal truncation.
return window.rstrip() + "…"
return window[:cut].rstrip() + "…"
-def _coerce_line(value: object) -> int | None:
- """Read an LLM-reported line number, or ``None`` if it isn't one.
-
- Models answer ``12``, ``"12"`` and ``"line 12"`` interchangeably; anything
- that isn't a positive integer is no attribution at all and must not be
- guessed at.
- """
- if isinstance(value, bool):
- return None
- if isinstance(value, int):
- return value if value > 0 else None
- if isinstance(value, str):
- digits = re.search(r"\d+", value)
- if digits:
- line = int(digits.group())
- return line if line > 0 else None
- return None
-
-
-def _as_aware_utc(value: datetime) -> datetime:
- """Return ``value`` as a timezone-aware UTC datetime.
-
- SQLite drops timezone information from ``DateTime(timezone=True)`` columns,
- but Repowise writes those values as UTC. Treat naive values as UTC so they
- can be compared with git metadata timestamps, which are already aware UTC.
- """
- if value.tzinfo is None or value.utcoffset() is None:
- return value.replace(tzinfo=UTC)
- return value.astimezone(UTC)
-
-
-def _coerce_dt(value: datetime | str) -> datetime:
- """Parse an ISO string into a datetime, passing datetimes through.
-
- Both shapes reach staleness: the ORM hands back datetimes, while the git
- metadata map carries whatever was persisted, which for SQLite is a string.
- """
- if isinstance(value, str):
- return datetime.fromisoformat(value.replace("Z", "+00:00"))
- return value
-
-
-# ---------------------------------------------------------------------------
-# Data models
-# ---------------------------------------------------------------------------
-
-
-#: The output budget for one batch of either commit prompt. Reasoning tokens
-#: are charged to it, so a budget that only fits the answer buys an empty
-#: body rather than a short one. Roughly twice the largest completion measured
-#: on this repository; see decision ``1c228ed8``.
-_BATCH_MAX_TOKENS = 8000
-
-#: How many of a commit's files either commit prompt will show. The model has
-#: to read the list to pick from it, and a commit that touched ninety files is
-#: not one whose decisions can be assigned by reading the list anyway.
-#:
-#: Sorted before truncating at both call sites, so which files the model is
-#: allowed to choose from is a property of the commit rather than of the order
-#: ``_git_meta_map`` happened to be built in.
-_MAX_PROMPT_FILES = 20
-
-
-def _coerce_paths(value: object) -> list[str] | None:
- """A list of path-ish strings out of whatever the model returned.
-
- ``None`` for an absent *or malformed* key, so that a model which answered
- ``[]`` is told apart from one that never usefully answered: the first is a
- decision about none of the commit's files, which binds the record to
- nothing, and the second is a provider that has not seen the new prompt,
- which falls back to the commit list. A list of dicts or numbers is the
- second case, not the first -- reading it as "chose nothing" would silently
- make the record govern nothing forever on a shape error.
-
- Models return a bare string for a one-element list often enough to be
- worth handling.
- """
- if value is None:
- return None
- if isinstance(value, str):
- value = [value]
- if not isinstance(value, list):
- return None
- kept = [v.strip() for v in value if isinstance(v, str) and v.strip()]
- if value and not kept:
- return None
- return kept
-
-
-@dataclass
-class ExtractedDecision:
- title: str
- context: str = ""
- decision: str = ""
- rationale: str = ""
- alternatives: list[str] = field(default_factory=list)
- consequences: list[str] = field(default_factory=list)
- affected_files: list[str] = field(default_factory=list)
- #: What the mining model named, before the commit's own file list has
- #: validated it. Mining-time only: the two commit miners intersect it into
- #: :attr:`affected_files` and clear it, so nothing downstream ever reads a
- #: path the model produced and the commit does not list.
- #:
- #: ``None`` means the model was never asked or did not answer, which is
- #: not the same as an empty list. An empty list is the model saying this
- #: decision is about none of the commit's files, and that answer binds the
- #: record to nothing; ``None`` falls back to the old breadth rule.
- proposed_files: list[str] | None = None
- affected_modules: list[str] = field(default_factory=list)
- tags: list[str] = field(default_factory=list)
- source: str = "inline_marker"
- #: See :class:`~repowise.core.persistence.models.DecisionRecord.scope_basis`.
- #: Set by the commit-derived miners, which are the only ones that take a
- #: file list they did not choose per file.
- scope_basis: str = ""
- evidence_commits: list[str] = field(default_factory=list)
- evidence_file: str | None = None
- evidence_line: int | None = None
- confidence: float = 0.5
- status: str = "proposed"
- # Which of the two nouns this is. Defaulted rather than classified per
- # extractor: every other source reads an artifact already written about the
- # code, and only the session lane mines the prose where an agreement about
- # conducting the work gets stated.
- kind: str = ARCHITECTURAL_KIND
- # The verbatim claimed quote (LLM/parser output) and the verdict from the
- # anti-hallucination substring gate (Phase 1D).
- source_quote: str = ""
- verification: str = "unverified" # exact | fuzzy | unverified
- # Transient: the verbatim source span this decision was drawn from. Set by
- # each extractor, consumed by the substring gate, then cleared before
- # persistence (the persistence layer ignores unknown dict keys anyway).
- source_text: str = ""
- # ``source`` alone cannot separate the two session lanes. Blank and False
- # mean the lane said nothing, not that it said no.
- lane: str = ""
- needs_split: bool = False
- # A source that measures its own conformance writes it; None means the
- # git-diff recompute owns it.
- staleness_score: float | None = None
-
-
-class DecisionSourceError(RuntimeError):
- """Every batch of a decision source failed.
-
- Raised so :meth:`DecisionExtractor.extract_all` records the source as
- failed rather than empty. A source that loses *some* batches still
- returns what it has and only logs, because partial supply beats none.
- """
-
-
-class EmptyModelResponseError(DecisionSourceError):
- """The model returned no body at all for one batch.
-
- Distinct from ``[]``, which is a real answer and the common one. Raised so
- the batch lands in :func:`_collect_batches` as a failure rather than as a
- source with nothing in it.
- """
-
-
-def _collect_batches(
- source: str,
- results: list[Any],
-) -> list[ExtractedDecision]:
- """Flatten ``asyncio.gather(..., return_exceptions=True)`` output.
-
- Exceptions are logged per batch. If nothing survived and there was work
- to do, the source failed outright and says so instead of returning a
- zero that reads like an empty repository.
- """
- decisions: list[ExtractedDecision] = []
- errors: list[str] = []
- for result in results:
- if isinstance(result, list):
- decisions.extend(result)
- else:
- errors.append(f"{type(result).__name__}: {result}")
- logger.warning(
- "decision_extractor.batch_failed",
- source=source,
- error=str(result),
- )
- if errors and len(errors) == len(results):
- raise DecisionSourceError(f"all {len(errors)} batch(es) failed — {errors[0]}")
- return decisions
-
-
-@dataclass
-class DecisionExtractionReport:
- total_found: int
- decisions: list[ExtractedDecision]
- by_source: dict[str, int]
- # Sources that raised, {source name: error text}. A source that fails
- # returns an empty list, so without this a total outage and an honestly
- # empty repo are the same number on screen. Callers render it; nothing
- # else can tell the two apart.
- failures: dict[str, str] = field(default_factory=dict)
-
-
# Every index-time capture source, in progress order. The CLI derives its
# progress-bar step count from this. Re-exported from the source registry so
# capabilities and run order cannot drift apart.
@@ -317,31 +111,6 @@ def enabled_source_names(repo_config: dict[str, Any] | None) -> tuple[str, ...]:
return resolve_policy(repo_config).policy.enabled_index_sources()
-# ---------------------------------------------------------------------------
-# Comment marker detection
-# ---------------------------------------------------------------------------
-
-# ---------------------------------------------------------------------------
-# ADR / PR / comment source configuration
-# ---------------------------------------------------------------------------
-
-# PR/squash body markers — a body containing any of these reads like a PR
-# description worth mining (vs an incidental multi-line commit message).
-_PR_BODY_MARKERS = (
- "## why",
- "## motivation",
- "## what",
- "## changes",
- "## context",
- "## summary",
- "closes #",
- "fixes #",
- "resolves #",
- "before:",
- "after:",
-)
-_MAX_PR_BODIES = 25
-
# Prose that signals rationale in a block comment / docstring (beyond the
# explicit WHY:/DECISION: markers already covered by inline_marker).
_COMMENT_RATIONALE_CUES = (
@@ -365,149 +134,99 @@ _MAX_COMMENT_NODES = 30
# with more markers is sent in several calls rather than truncated.
_MARKERS_PER_CALL = 5
-# ---------------------------------------------------------------------------
-# ---------------------------------------------------------------------------
-# Main extractor
-# ---------------------------------------------------------------------------
-
-
-def _scope_from_selection(
- decision: ExtractedDecision,
- commit_files: Sequence[str] | None,
-) -> tuple[list[str], str]:
- """The files and basis for one decision mined out of one commit.
-
- Returns the model's own selection, validated against the commit's file
- list, under :data:`SCOPE_BASIS_SELECTED`. Falling back to the commit's
- whole footprint when the model selected nothing would reinstate exactly
- what this replaces, so an empty selection stays empty: the record keeps
- its commit, its evidence and its place in repository-wide answers, and
- stops answering "what governs this file". Roughly one record in six lands
- here, and every one of them measured as a record whose subject was not in
- the commit's list to begin with.
-
- The old breadth rule is the fallback for a *missing* answer rather than an
- empty one -- a provider that ignored the new key, or a cached response
- written before it existed. There the record is scoped as it always was and
- the legacy basis says so.
- """
- if decision.proposed_files is None:
- return (commit_scope_files(commit_files), commit_scope_basis(commit_files))
- chosen = selected_scope_files(decision.proposed_files, commit_files)
- decision.proposed_files = None
- return (chosen, SCOPE_BASIS_SELECTED)
-
-
-
-def _signal_commit_info(commit: dict) -> dict | None:
- """The prompt-ready record of a commit with decision signals, else None."""
- msg = commit.get("message", "")
- body = commit.get("body", "")
- # Scan subject + body for signals — squash-merge repos carry the
- # decision rationale in the body, not the one-line subject.
- signal_count = count_decision_signals(f"{msg}\n{body}".lower())
- if signal_count <= 0:
- return None
- return {
- "sha": commit.get("sha", ""),
- "message": msg,
- "body": body,
- "author": commit.get("author", ""),
- "date": commit.get("date", ""),
- "signal_count": signal_count,
- }
-
-async def _run_batches(
- source: str,
- items: Sequence[Any],
- size: int,
- process: Any,
-) -> list[ExtractedDecision]:
- """Run *process* over *items* in batches of *size*, concurrently.
-
- Each batch's failure is contained and counted by :func:`_collect_batches`,
- which raises only when every batch failed.
+
+def _adr_title(front_matter_title: str, body: str) -> str:
+ """The ADR's title: front matter, else its first H1, less any ``ADR-0007:`` prefix."""
+ title = front_matter_title
+ if not title:
+ m = re.search(r"^#\s+(.+)$", body, re.MULTILINE)
+ if m:
+ title = m.group(1).strip()
+ return re.sub(r"^ADR[-\s]*\d+[:\s-]*", "", title, flags=re.IGNORECASE).strip() or title
+
+
+def _first_section(sections: dict[str, str], *headings: str) -> str:
+ """The body of the first of *headings* the ADR fills in, else ``""``."""
+ for heading in headings:
+ if sections.get(heading):
+ return sections[heading]
+ return ""
+
+
+def _adr_status(declared: str) -> str:
+ """The record status for an ADR's declared status.
+
+ An undeclared status stays ``proposed``: a committed ADR is the one
+ artifact allowed to accept its own decision, so a draft must not.
"""
- batches = [items[i : i + size] for i in range(0, len(items), size)]
- results = await asyncio.gather(*[process(b) for b in batches], return_exceptions=True)
- return _collect_batches(source, list(results))
-
-
-def _attribute_to_commit(
- decision: ExtractedDecision,
- batch: list[dict],
- subject_key: str,
- files_by_sha: dict[str, list[str]],
- source_by_sha: dict[str, str],
-) -> str:
- """Bind a decision mined from a batch of commits to the commit it came from.
-
- Prefers the sha the model reported, else the first commit whose subject
- (``batch[i][subject_key]``, first 40 chars) appears in the title. Scopes
- the decision to that commit's files and returns the sha, or ``""`` when
- nothing matched.
+ status_key = declared.strip().lower().split()[0] if declared.strip() else ""
+ return _ADR_STATUS_MAP.get(status_key, "proposed")
+
+
+_TAG_KEYWORDS = {
+ "auth": ["auth", "jwt", "oauth", "token", "session", "login"],
+ "database": ["database", "sql", "postgres", "sqlite", "redis", "mongo", "db"],
+ "api": ["api", "rest", "graphql", "endpoint", "route"],
+ "performance": ["performance", "cache", "speed", "latency", "optimize"],
+ "security": ["security", "encrypt", "hash", "cors", "csrf", "xss"],
+ "infra": ["docker", "kubernetes", "deploy", "ci", "cd", "terraform"],
+ "testing": ["test", "mock", "fixture", "assert"],
+}
+
+
+def _infer_tags(text: str) -> list[str]:
+ """Infer tags from decision text."""
+ text_lower = text.lower()
+ return [
+ tag for tag, keywords in _TAG_KEYWORDS.items() if any(kw in text_lower for kw in keywords)
+ ]
+
+
+def _snippet_for(
+ decision: ExtractedDecision, batch: Sequence[tuple[str, str]]
+) -> tuple[str, str]:
+ """The ``(file, prose)`` a comment decision came from, by file stem in its text.
+
+ Best effort: the batch's first snippet when no stem matches.
"""
- sha = decision.evidence_commits[0] if decision.evidence_commits else ""
- if not sha:
- # Try to match back to a commit
- for c in batch:
- if c[subject_key][:40].lower() in decision.title.lower():
- sha = c["sha"]
- break
- if sha:
- decision.evidence_commits = [sha]
- decision.affected_files, decision.scope_basis = _scope_from_selection(
- decision, files_by_sha.get(sha)
- )
- decision.source_text = source_by_sha.get(sha, "")
- # Cleared whether or not the sha resolved: an unattributed decision has no
- # commit to validate paths against, so the model's list is unusable rather
- # than merely unused.
- decision.proposed_files = None
- return sha
-
-
-def _load_json_payload(content: str) -> Any | None:
- """Decode a model's JSON answer, tolerating code fences and surrounding prose."""
- if content.startswith("```"):
- # Remove markdown code fences
- lines = content.split("\n")
- content = "\n".join(line for line in lines if not line.strip().startswith("```"))
+ hay = (decision.title + " " + decision.decision).lower()
+ for fp, prose in batch:
+ stem = Path(fp).stem.lower()
+ if stem and stem in hay:
+ return fp, prose
+ return batch[0]
+
+
+# Above this node count, rank by degree (O(nodes)) instead of solving PageRank:
+# comment archaeology only needs a rough "most depended-on files" ranking.
+_PAGERANK_NODE_CEILING = 20_000
+
+
+def _pagerank(g: Any) -> dict[str, float]:
+ """PageRank for a graph small enough to solve, else empty."""
try:
- return json.loads(content)
- except json.JSONDecodeError:
- pass
- # Try to find JSON array in the response
- match = re.search(r"\[.*\]", content, re.DOTALL)
- if not match:
- return None
+ node_count = g.number_of_nodes()
+ except Exception:
+ node_count = 0
+ if not 0 < node_count <= _PAGERANK_NODE_CEILING:
+ return {}
+ try:
+ import networkx as nx
+
+ return nx.pagerank(g, max_iter=50, tol=1e-4)
+ except Exception:
+ return {}
+
+
+def _centrality_scores(g: Any) -> dict[str, float]:
+ """Node centrality: PageRank where affordable, else degree; empty when unreadable."""
+ scores = _pagerank(g)
+ if scores:
+ return scores
try:
- return json.loads(match.group())
- except json.JSONDecodeError:
- return None
-
-
-def _decision_from_item(item: dict) -> ExtractedDecision:
- """One decision from one object of a model's JSON answer."""
- return ExtractedDecision(
- title=item.get("title", ""),
- context=item.get("context", ""),
- decision=item.get("decision", ""),
- rationale=item.get("rationale", ""),
- alternatives=item.get("alternatives", []),
- consequences=item.get("consequences", []),
- tags=item.get("tags", []),
- # Only the two commit prompts ask for this. Every other prompt omits
- # the key, so this stays None and the miner that owns those decisions
- # keeps scoping them its own way.
- proposed_files=_coerce_paths(item.get("affected_files")),
- evidence_commits=[item["commit_sha"]] if "commit_sha" in item else [],
- # Which marker this came from, for the inline-marker miner's per-marker
- # attribution. Absent (and left None) for every other prompt; they
- # scope by sha or by file instead.
- evidence_line=_coerce_line(item.get("marker_line")),
- source_quote=item.get("source_quote", ""),
- )
+ return {node: float(g.degree(node)) for node in g.nodes}
+ except Exception:
+ return {}
class DecisionExtractor:
@@ -525,17 +244,13 @@ class DecisionExtractor:
) -> None:
self._repo_path = Path(repo_path)
self._provider = provider
- # Per-source model gate. ``None`` means "no policy supplied", which
- # keeps the provider available to every source, as before this existed.
+ # Per-source model gate; ``None`` lets every source use the provider.
self._policy = policy
self._graph = graph
self._git_meta_map = git_meta_map or {}
self._parsed_files = parsed_files or []
- # ``source_map`` is ingestion's already-computed {rel_path: bytes} for
- # the indexed file set. When present, the inline-marker scan reuses it
- # for both discovery and reads instead of re-walking the tree and
- # re-reading every file from disk (redundant with ingestion). ``None``
- # keeps the legacy self-walk fallback for callers that don't thread it.
+ # Ingestion's {rel_path: bytes} for the indexed set. When given, the
+ # marker scan reads it instead of walking and re-reading the tree.
self._source_map = source_map
def _llm(self, source: str) -> Any | None:
@@ -589,10 +304,8 @@ class DecisionExtractor:
self, file_path: str, markers: list[dict], marker_llm: Any | None
) -> list[ExtractedDecision]:
"""One file's markers as decisions: model-structured when possible, else raw."""
- # Get 1-hop graph neighbors for affected_files
affected = self._get_neighbors(file_path)
if not marker_llm:
- # No LLM — create minimal decisions from raw marker text
return [self._raw_decision_from_marker(file_path, m, affected) for m in markers]
try:
llm_decisions = await self._structure_markers_via_llm(file_path, markers)
@@ -602,7 +315,6 @@ class DecisionExtractor:
"decision_extractor.llm_structuring_failed",
file=file_path,
)
- # Fall through to raw extraction
return [self._raw_decision_from_marker(file_path, m, affected) for m in markers]
return llm_decisions
@@ -613,18 +325,15 @@ class DecisionExtractor:
markers: list[dict],
affected: list[str],
) -> None:
- """Bind each model-structured decision to the marker it was drawn from."""
+ """Bind each model-structured decision to the marker it was drawn from.
+
+ The model reports ``marker_line``, parsed into ``evidence_line``. A
+ decision that names no known marker gets no source span, so the gate
+ leaves it ``unverified`` instead of verifying it against another
+ marker's text.
+ """
markers_by_line = {m["line"]: m for m in markers}
for d in decisions:
- # Attribute the decision to the one marker it was drawn from (the
- # prompt asks for `marker_line`, which the parser lands in
- # `evidence_line`). Joining every marker's context into one span
- # and handing it to all of them let the substring gate stamp a
- # decision `exact` against a *different* marker's text, and put
- # marker 1's line number on marker 3's decision. A decision we
- # cannot attribute gets no source span at all: the gate then leaves
- # it `unverified`, which is the honest verdict, rather than
- # verifying it against a neighbour.
marker = markers_by_line.get(d.evidence_line)
if marker is None and len(markers) == 1:
marker = markers[0] # unambiguous without the hint
@@ -647,10 +356,8 @@ class DecisionExtractor:
return ExtractedDecision(
title=_truncate_title(marker["text"], 100),
decision=marker["text"],
- # No context. This lane has only the marker's own text, and
- # `evidence_file`/`evidence_line` below already say where it was
- # found, so the location string it used to carry was a restatement
- # that made a record with no stated reason read as having one.
+ # No context: the marker's text is all this lane has, and its
+ # location is already in evidence_file / evidence_line.
source="inline_marker",
status="active",
confidence=0.7,
@@ -658,7 +365,7 @@ class DecisionExtractor:
evidence_line=marker["line"],
affected_files=list({file_path} | set(affected)),
affected_modules=self._infer_modules([file_path, *affected]),
- tags=self._infer_tags(marker["text"]),
+ tags=_infer_tags(marker["text"]),
source_quote=marker["text"],
source_text=marker.get("context", marker["text"]),
)
@@ -668,12 +375,7 @@ class DecisionExtractor:
) -> list[ExtractedDecision]:
"""Use LLM to structure inline markers into decision records.
- Markers are sent five per call. The batch size caps prompt length;
- it is not a cap on how many markers a file may have. ``markers[:5]``
- alone dropped every marker past the fifth: the caller invokes this
- once per file, and the raw-marker fallback below only runs on an
- exception, so those markers were never structured and never fell
- back — they left no record and no log line.
+ Every marker is sent, ``_MARKERS_PER_CALL`` per call.
"""
provider = self._llm("inline_marker")
decisions: list[ExtractedDecision] = []
@@ -695,7 +397,7 @@ class DecisionExtractor:
response = await provider.generate(
_SYSTEM_PROMPT, prompt, max_tokens=2000, temperature=0.2
)
- decisions.extend(self._parse_decisions_json(response.content))
+ decisions.extend(parse_decisions_json(response.content))
return decisions
# ------------------------------------------------------------------
@@ -708,7 +410,7 @@ class DecisionExtractor:
if not provider or not self._git_meta_map:
return []
- commit_map, commit_files = self._signal_commits()
+ commit_map, commit_files = signal_commits(self._git_meta_map)
if not commit_map:
return []
@@ -724,25 +426,15 @@ class DecisionExtractor:
commits_block = ""
source_by_sha: dict[str, str] = {}
for c in batch:
- files = commit_files.get(c["sha"], [])
body = (c.get("body") or "").strip()
- body_block = f"Body: {body[:1500]}\n" if body else ""
- commits_block += (
- f"\n--- Commit {c['sha'][:8]} ---\n"
- f"Message: {c['message']}\n"
- f"{body_block}"
- f"Author: {c['author']}\n"
- f"Date: {c['date']}\n"
- f"Files changed: "
- f"{', '.join(sorted(files)[:_MAX_PROMPT_FILES])}\n"
- )
+ commits_block += git_commit_block(c, body, commit_files.get(c["sha"], []))
source_by_sha[c["sha"]] = f"{c['message']}\n{body}".strip()
prompt = GIT_ARCHAEOLOGY_PROMPT.format(commits_block=commits_block)
response = await provider.generate(
_SYSTEM_PROMPT, prompt, max_tokens=_BATCH_MAX_TOKENS, temperature=0.2
)
- extracted = self._parse_decisions_json(response.content)
+ extracted = parse_decisions_json(response.content)
# Enrich with commit metadata
for d in extracted:
@@ -760,34 +452,6 @@ class DecisionExtractor:
return await _run_batches("git_archaeology", ranked, 5, _process_batch)
- def _signal_commits(self) -> tuple[dict[str, dict], dict[str, list[str]]]:
- """Unique significant commits carrying decision signals, and each one's files.
-
- Returns ``(sha -> commit info, sha -> files it touched)``.
- """
- commit_map: dict[str, dict] = {} # sha → commit info
- commit_files: dict[str, list[str]] = {} # sha → files
-
- for file_path, meta in self._git_meta_map.items():
- commits_json = meta.get("significant_commits_json", "[]")
- if isinstance(commits_json, str):
- try:
- commits = json.loads(commits_json)
- except (json.JSONDecodeError, TypeError):
- continue
- else:
- commits = commits_json
-
- for commit in commits:
- sha = commit.get("sha", "")
- if sha and sha not in commit_map:
- info = _signal_commit_info(commit)
- if info is None:
- continue
- commit_map[sha] = info
- commit_files.setdefault(sha, []).append(file_path)
- return commit_map, commit_files
-
# ------------------------------------------------------------------
# Source 3: ADR auto-discovery (deterministic-first)
# ------------------------------------------------------------------
@@ -808,40 +472,47 @@ class DecisionExtractor:
decisions: list[ExtractedDecision] = []
for path in adr_paths:
- try:
- content = path.read_text(encoding="utf-8", errors="replace")
- except (OSError, UnicodeDecodeError):
+ loaded = self._read_adr(path)
+ if loaded is None:
continue
- if len(content) > 50_000:
- content = content[:50_000]
- try:
- rel = str(path.relative_to(self._repo_path))
- except ValueError:
- rel = str(path)
-
+ rel, content = loaded
parsed = self._parse_adr(content, rel)
if parsed is not None:
decisions.append(parsed)
elif provider:
- try:
- stripped = strip_code_blocks(content)
- prompt = README_MINING_PROMPT.format(file_path=rel, content=stripped[:15_000])
- response = await provider.generate(
- _SYSTEM_PROMPT, prompt, max_tokens=2000, temperature=0.2
- )
- for d in self._parse_decisions_json(response.content):
- d.source = "adr"
- d.status = "proposed"
- d.confidence = 0.80
- d.evidence_file = rel
- d.source_text = stripped
- d.affected_modules = self._infer_modules_from_text(
- d.title + " " + d.decision
- )
- decisions.append(d)
- except Exception:
- logger.warning("decision_extractor.adr_llm_failed", file=rel)
+ decisions.extend(await self._mine_unstructured_adr(provider, rel, content))
+
+ return decisions
+
+ def _read_adr(self, path: Path) -> tuple[str, str] | None:
+ """An ADR's repo-relative path and its first 50k chars, or None when unreadable."""
+ try:
+ content = path.read_text(encoding="utf-8", errors="replace")
+ except (OSError, UnicodeDecodeError):
+ return None
+ return self._rel_path(path), content[:50_000]
+ async def _mine_unstructured_adr(
+ self, provider: Any, rel: str, content: str
+ ) -> list[ExtractedDecision]:
+ """The LLM fallback for a file named like an ADR that has no ADR structure."""
+ decisions: list[ExtractedDecision] = []
+ try:
+ stripped = strip_code_blocks(content)
+ prompt = README_MINING_PROMPT.format(file_path=rel, content=stripped[:15_000])
+ response = await provider.generate(
+ _SYSTEM_PROMPT, prompt, max_tokens=2000, temperature=0.2
+ )
+ for d in parse_decisions_json(response.content):
+ d.source = "adr"
+ d.status = "proposed"
+ d.confidence = 0.80
+ d.evidence_file = rel
+ d.source_text = stripped
+ d.affected_modules = self._infer_modules_from_text(d.title + " " + d.decision)
+ decisions.append(d)
+ except Exception:
+ logger.warning("decision_extractor.adr_llm_failed", file=rel)
return decisions
def _parse_adr(self, content: str, rel_path: str) -> ExtractedDecision | None:
@@ -853,50 +524,35 @@ class DecisionExtractor:
"""
status, title, body = read_front_matter(content)
sections = split_headings(body)
- if not title:
- m = re.search(r"^#\s+(.+)$", body, re.MULTILINE)
- if m:
- title = m.group(1).strip()
- # Strip an "ADR-0007:" style prefix from the title.
- title = re.sub(r"^ADR[-\s]*\d+[:\s-]*", "", title, flags=re.IGNORECASE).strip() or title
-
- context = sections.get("context") or sections.get("context and problem statement", "")
- decision_txt = sections.get("decision") or sections.get("decision outcome", "")
- rationale = sections.get("rationale") or sections.get("decision drivers", "")
- consequences = sections.get("consequences", "")
- if not status:
- status = sections.get("status", "")
-
- # Require recognizable ADR structure — at minimum a Decision or Context
- # section — otherwise let the LLM fallback handle it.
+ title = _adr_title(title, body)
+
+ context = _first_section(sections, "context", "context and problem statement")
+ decision_txt = _first_section(sections, "decision", "decision outcome")
+ rationale = _first_section(sections, "rationale", "decision drivers")
+
+ # Require a Decision or Context section; anything less goes to the LLM
+ # fallback.
if not (decision_txt or context):
return None
- # A document with no Status section has not said it is accepted, and a
- # committed ADR is the one artifact allowed to accept its own decision.
- # Defaulting to ``active`` therefore let any draft under docs/adr/ grant
- # itself authority.
- status_key = status.strip().lower().split()[0] if status.strip() else ""
- mapped_status = _ADR_STATUS_MAP.get(status_key, "proposed")
-
return ExtractedDecision(
title=_truncate_title(title or rel_path, 200),
context=context.strip(),
decision=decision_txt.strip(),
rationale=rationale.strip(),
- consequences=bullets(consequences),
+ consequences=bullets(sections.get("consequences", "")),
source="adr",
- status=mapped_status,
+ status=_adr_status(status or sections.get("status", "")),
confidence=0.90,
evidence_file=rel_path,
source_quote=(decision_txt or context).strip()[:500],
source_text=content,
- tags=self._infer_tags(f"{title} {decision_txt}"),
+ tags=_infer_tags(f"{title} {decision_txt}"),
affected_modules=self._infer_modules_from_text(f"{title} {decision_txt}"),
)
# ------------------------------------------------------------------
- # Source 4: PR / squash-body mining (consumes commit bodies from 1A)
+ # Source 4: PR / squash-body mining
# ------------------------------------------------------------------
async def mine_pr_bodies(self) -> list[ExtractedDecision]:
@@ -905,28 +561,7 @@ class DecisionExtractor:
if not provider or not self._git_meta_map:
return []
- candidates: dict[str, dict] = {}
- files_by_sha: dict[str, list[str]] = {}
- for fp, meta in self._git_meta_map.items():
- for c in self._loads_commits(meta.get("significant_commits_json")):
- sha = c.get("sha", "")
- if not sha:
- continue
- files_by_sha.setdefault(sha, []).append(fp)
- body = (c.get("body") or "").strip()
- if sha in candidates or not body:
- continue
- low = body.lower()
- is_prish = c.get("pr_number") is not None or any(m in low for m in _PR_BODY_MARKERS)
- has_signal = count_decision_signals(low) > 0
- if is_prish and has_signal:
- candidates[sha] = {
- "sha": sha,
- "subject": c.get("message", ""),
- "body": body,
- "pr": c.get("pr_number"),
- }
-
+ candidates, files_by_sha = pr_candidates(self._git_meta_map)
if not candidates:
return []
@@ -936,29 +571,16 @@ class DecisionExtractor:
bodies_block = ""
source_by_sha: dict[str, str] = {}
for c in batch:
- pr_label = f" (PR #{c['pr']})" if c.get("pr") else ""
- # The file list is what makes "affected_files" answerable.
- # This miner asked for a decision without ever showing which
- # files the commit touched, and scored 20% on topic against
- # git archaeology's 45% on the same store.
- files = files_by_sha.get(c["sha"], [])
- bodies_block += (
- f"\n--- Commit {c['sha'][:8]}{pr_label} ---\n"
- f"Subject: {c['subject']}\n"
- f"Body:\n{c['body'][:2000]}\n"
- f"Files changed: {', '.join(sorted(files)[:_MAX_PROMPT_FILES])}\n"
- )
+ # The file list is what lets the model answer "affected_files".
+ bodies_block += pr_commit_block(c, files_by_sha.get(c["sha"], []))
source_by_sha[c["sha"]] = f"{c['subject']}\n{c['body']}"
prompt = PR_BODY_MINING_PROMPT.format(bodies_block=bodies_block)
- # Let this propagate to the gather below. Swallowed here, a total
- # provider outage returned five empty lists, so ``_collect_batches``
- # saw no errors and the run reported "Nothing found in: pull
- # requests" — the exact zero-that-means-failure this change exists
- # to remove.
+ # Not caught: _run_batches counts a failed batch, so an outage is
+ # reported rather than read as nothing found.
response = await provider.generate(
_SYSTEM_PROMPT, prompt, max_tokens=_BATCH_MAX_TOKENS, temperature=0.2
)
- extracted = self._parse_decisions_json(response.content)
+ extracted = parse_decisions_json(response.content)
for d in extracted:
_attribute_to_commit(d, batch, "subject", files_by_sha, source_by_sha)
d.source = "pr"
@@ -1002,23 +624,13 @@ class DecisionExtractor:
for fp, prose in batch:
comments_block += f"\n--- {fp} ---\n{prose[:1500]}\n"
prompt = COMMENT_ARCHAEOLOGY_PROMPT.format(comments_block=comments_block)
- # Was a bare ``except Exception: return []`` here with no log at
- # all — the quietest of the three swallows, and the one that made
- # "comment: 0" unfalsifiable. Let it propagate to the gather
- # below, which counts it.
+ # Not caught: _run_batches counts a failed batch.
response = await provider.generate(
_SYSTEM_PROMPT, prompt, max_tokens=2500, temperature=0.2
)
- extracted = self._parse_decisions_json(response.content)
- # Best-effort attribution to the originating file by token overlap.
+ extracted = parse_decisions_json(response.content)
for d in extracted:
- best_fp, best_prose = batch[0]
- hay = (d.title + " " + d.decision).lower()
- for fp, prose in batch:
- stem = Path(fp).stem.lower()
- if stem and stem in hay:
- best_fp, best_prose = fp, prose
- break
+ best_fp, best_prose = _snippet_for(d, batch)
d.source = "comment"
d.status = "proposed"
d.confidence = 0.55
@@ -1044,12 +656,6 @@ class DecisionExtractor:
self._graph, self._parsed_files, self._source_map, self._repo_path
)
- # Above this node count, skip the iterative PageRank solve and use degree
- # centrality (O(nodes)) instead — comment archaeology only needs a rough
- # "most depended-on files" ranking, not exact PageRank, and the iterative
- # solve would otherwise add seconds on very large graphs.
- _PAGERANK_NODE_CEILING = 20_000
-
def _top_central_files(self, n: int) -> list[str]:
"""Top-*n* file nodes by centrality (existing on disk).
@@ -1057,26 +663,9 @@ class DecisionExtractor:
centrality on very large graphs (or if networkx is unavailable) so this
never becomes an ingestion bottleneck.
"""
- g = self._graph
- if g is None:
+ if self._graph is None:
return []
- scores: dict[str, float] = {}
- try:
- node_count = g.number_of_nodes()
- except Exception:
- node_count = 0
- if 0 < node_count <= self._PAGERANK_NODE_CEILING:
- try:
- import networkx as nx
-
- scores = nx.pagerank(g, max_iter=50, tol=1e-4)
- except Exception:
- scores = {}
- if not scores:
- try:
- scores = {node: float(g.degree(node)) for node in g.nodes}
- except Exception:
- return []
+ scores = _centrality_scores(self._graph)
ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
out: list[str] = []
for node, _score in ranked:
@@ -1086,22 +675,8 @@ class DecisionExtractor:
if p.is_file() and p.suffix.lower() not in _BINARY_EXTENSIONS:
out.append(node)
return out
-
- @staticmethod
- def _loads_commits(value: Any) -> list[dict]:
- """Parse a ``significant_commits_json`` blob into a list of dicts."""
- if isinstance(value, list):
- return value
- if isinstance(value, str):
- try:
- data = json.loads(value)
- except (json.JSONDecodeError, TypeError):
- return []
- return data if isinstance(data, list) else []
- return []
-
# ------------------------------------------------------------------
- # Anti-hallucination substring gate (Phase 1D)
+ # Anti-hallucination substring gate
# ------------------------------------------------------------------
def _apply_substring_gate(
@@ -1109,84 +684,14 @@ class DecisionExtractor:
) -> tuple[list[ExtractedDecision], int]:
"""Run the shared anti-hallucination gate over extracted decisions.
- Thin wrapper around :func:`decision_gate.apply_substring_gate` — the
- gate orchestration is factored out so the Phase-2 LLM-docs harvest path
- enforces the *same* grounding rules. See that module for the contract.
+ Thin wrapper around :func:`decision_gate.apply_substring_gate`, which
+ the docs harvest path shares so both enforce the same grounding rules.
"""
return apply_substring_gate(decisions)
- # ------------------------------------------------------------------
- # Staleness computation (static method)
- # ------------------------------------------------------------------
-
- @staticmethod
- def compute_staleness(
- decision_created_at: datetime,
- affected_files: list[str],
- git_meta_map: dict[str, dict],
- decision_text: str = "",
- ) -> float:
- """Fraction of *affected_files* that have changed since the record's birth.
-
- A fact about the code, not a judgement about the record: 0.0 means
- nothing it governs has moved, 1.0 means all of it has. There is no
- tuned constant left in it, which is the point — the previous formula
- was ``commit_count / 15 * 0.7 + age_days / 365 * 0.3``, whose divisors
- were fitted to one repository's history and produced ~0 for almost
- every record here regardless of whether the code had moved.
-
- Also gone: a keyword boost that read recent commit *messages* for words
- like "migrate away". That inferred intent from English prose, which
- does not travel, and it mixed a guess into a value other surfaces
- store and compare.
-
- *decision_text* is accepted and unused, so the two call sites keep
- working; it goes when they do.
-
- A file with no git metadata **after** the caller's gap fill counts as
- changed: the record names something the repository does not track, so
- it cannot be shown to still hold.
- """
- if not affected_files:
- # No scope, so the question cannot be asked. Callers distinguish
- # this from a genuine 0.0 by the empty file list, and
- # `decision health` reports it as unscoped rather than fresh.
- return 0.0
-
- created = _as_aware_utc(_coerce_dt(decision_created_at)) if decision_created_at else None
- changed = 0
- for fp in affected_files:
- meta = git_meta_map.get(fp)
- if meta is None:
- changed += 1 # named but not tracked — cannot be shown to hold
- continue
- last_commit = meta.get("last_commit_at")
- if not last_commit or created is None:
- continue
- if _as_aware_utc(_coerce_dt(last_commit)) > created:
- changed += 1
-
- return round(changed / len(affected_files), 3)
-
- @staticmethod
- def last_code_change(
- affected_files: list[str],
- git_meta_map: dict[str, dict],
- ) -> datetime | None:
- """When the code a decision governs last moved, or None.
-
- Reduces the same inputs :meth:`compute_staleness` reads: that counts
- how many files moved since the record was born, this reports when the
- most recent of them moved. None where the question cannot be answered,
- meaning no scope or no git metadata for anything the record names.
- """
- dates: list[datetime] = []
- for file_path in affected_files:
- meta = git_meta_map.get(file_path)
- last_commit = meta.get("last_commit_at") if meta else None
- if last_commit:
- dates.append(_as_aware_utc(_coerce_dt(last_commit)))
- return max(dates) if dates else None
+ # Kept on the class: callers read staleness as ``DecisionExtractor.compute_staleness``.
+ compute_staleness = staticmethod(compute_staleness)
+ last_code_change = staticmethod(last_code_change)
# ------------------------------------------------------------------
# Main entry point
@@ -1223,7 +728,7 @@ class DecisionExtractor:
none.
Every extracted decision is then put through the anti-hallucination
- substring gate (:meth:`_apply_substring_gate`) before being returned —
+ substring gate (:meth:`_apply_substring_gate`) before being returned:
ungrounded LLM fields are dropped and evidence-less decisions rejected.
"""
@@ -1236,9 +741,8 @@ class DecisionExtractor:
logger.info("decision_extractor.finished", source=name, count=len(result))
return result
except Exception as exc:
- # Recorded as well as logged: the CLI pins core to ERROR, so
- # a source that dies here used to reach the user as a plain
- # zero indistinguishable from "this repo has no ADRs".
+ # Recorded as well as logged: the CLI pins core logging to
+ # ERROR, so the report is where a failed source shows up.
failures[name] = f"{type(exc).__name__}: {exc}"
logger.warning("decision_extractor.source_failed", source=name, error=str(exc))
return []
@@ -1246,7 +750,7 @@ class DecisionExtractor:
if on_step:
on_step(name)
- # (source name, bound coroutine factory) — order is the progress order.
+ # (source name, bound coroutine factory), in progress order.
all_sources: list[tuple[str, Any]] = [
("inline_marker", self.scan_inline_markers),
("git_archaeology", self.mine_git_archaeology),
@@ -1305,12 +809,9 @@ class DecisionExtractor:
Text comes from ``source_map`` when the file was just ingested, else a
targeted disk read; deleted / unreadable paths are skipped.
* ``source_map`` (init path): ingestion's already-decoded indexed set.
- Discovery AND reads are free — no tree walk, no per-file ``read_text``.
- Paths are POSIX (``FileInfo.path``), matching the graph node keys the
- neighbour lookup joins against.
- * legacy self-walk (``source_map is None``): the original ``os.walk`` +
- git-tracked filter, kept so callers that don't thread ``source_map``
- behave exactly as before.
+ No tree walk and no per-file read. Paths are POSIX (``FileInfo.path``),
+ matching the graph node keys the neighbour lookup joins against.
+ * the tree walk, for callers that pass no ``source_map``.
"""
if restrict_to_files:
for rel_path in restrict_to_files:
@@ -1324,6 +825,10 @@ class DecisionExtractor:
yield rel_path, source.decode("utf-8", errors="replace")
return
+ yield from self._iter_walked_files()
+
+ def _iter_walked_files(self) -> Iterator[tuple[str, str]]:
+ """``(rel_path, text)`` for every readable file the legacy tree walk finds."""
for file_path in iter_source_files(self._repo_path):
if not file_path.is_file():
continue
@@ -1331,11 +836,14 @@ class DecisionExtractor:
text = file_path.read_text(encoding="utf-8", errors="replace")
except (OSError, UnicodeDecodeError):
continue
- try:
- rel_path = str(file_path.relative_to(self._repo_path))
- except ValueError:
- rel_path = str(file_path)
- yield rel_path, text
+ yield self._rel_path(file_path), text
+
+ def _rel_path(self, path: Path) -> str:
+ """*path* relative to the repo root, or as given when it lies outside it."""
+ try:
+ return str(path.relative_to(self._repo_path))
+ except ValueError:
+ return str(path)
def _read_source_text(self, rel_path: str) -> str | None:
"""Decode one file's text, preferring ingestion's in-memory bytes.
@@ -1393,44 +901,3 @@ class DecisionExtractor:
# ``packages/core/.../decisions`` should not also claim every ancestor.
deepest = {d for d in matched if not any(o != d and o.startswith(d + "/") for o in matched)}
return sorted(deepest, key=lambda d: (-d.count("/"), d))[:5]
-
- def _infer_tags(self, text: str) -> list[str]:
- """Infer tags from decision text."""
- tag_keywords = {
- "auth": ["auth", "jwt", "oauth", "token", "session", "login"],
- "database": ["database", "sql", "postgres", "sqlite", "redis", "mongo", "db"],
- "api": ["api", "rest", "graphql", "endpoint", "route"],
- "performance": ["performance", "cache", "speed", "latency", "optimize"],
- "security": ["security", "encrypt", "hash", "cors", "csrf", "xss"],
- "infra": ["docker", "kubernetes", "deploy", "ci", "cd", "terraform"],
- "testing": ["test", "mock", "fixture", "assert"],
- }
- text_lower = text.lower()
- tags = []
- for tag, keywords in tag_keywords.items():
- if any(kw in text_lower for kw in keywords):
- tags.append(tag)
- return tags
-
- def _parse_decisions_json(self, content: str) -> list[ExtractedDecision]:
- """Parse LLM response as JSON array of decisions.
-
- A blank body raises :class:`EmptyModelResponseError`; every caller
- sits inside a gather or a fallback that counts that as a lost batch.
- """
- # Extract JSON from response (may be wrapped in markdown code blocks)
- content = content.strip()
- if not content:
- raise EmptyModelResponseError(
- "the model returned no content for this batch"
- )
- data = _load_json_payload(content)
- if isinstance(data, dict):
- data = [data]
- if not isinstance(data, list):
- return []
- return [
- _decision_from_item(item)
- for item in data
- if isinstance(item, dict) and item.get("title", "")
- ]
diff --git a/packages/core/src/repowise/core/analysis/decisions/model_answers.py b/packages/core/src/repowise/core/analysis/decisions/model_answers.py
new file mode 100644
index 000000000..e8e178969
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/decisions/model_answers.py
@@ -0,0 +1,162 @@
+"""Turning a model's JSON answer into decisions, and counting batches that fail."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import re
+from collections.abc import Sequence
+from typing import Any
+
+import structlog
+
+from .records import DecisionSourceError, EmptyModelResponseError, ExtractedDecision
+
+logger = structlog.get_logger(__name__)
+
+
+def _coerce_line(value: object) -> int | None:
+ """Read an LLM-reported line number, or ``None`` if it isn't one.
+
+ Models answer ``12``, ``"12"`` and ``"line 12"`` interchangeably; anything
+ that isn't a positive integer is no attribution at all and must not be
+ guessed at.
+ """
+ if isinstance(value, bool):
+ return None
+ if isinstance(value, int):
+ return value if value > 0 else None
+ if isinstance(value, str):
+ digits = re.search(r"\d+", value)
+ if digits:
+ line = int(digits.group())
+ return line if line > 0 else None
+ return None
+
+
+def _coerce_paths(value: object) -> list[str] | None:
+ """A list of path-ish strings out of whatever the model returned.
+
+ ``None`` for an absent *or malformed* key, which falls back to the
+ commit's file list; ``[]`` is a real answer that binds the record to
+ nothing, so a shape error must never read as one. A bare string is taken
+ as a one-element list.
+ """
+ if value is None:
+ return None
+ if isinstance(value, str):
+ value = [value]
+ if not isinstance(value, list):
+ return None
+ kept = [v.strip() for v in value if isinstance(v, str) and v.strip()]
+ if value and not kept:
+ return None
+ return kept
+
+
+def _collect_batches(
+ source: str,
+ results: list[Any],
+) -> list[ExtractedDecision]:
+ """Flatten ``asyncio.gather(..., return_exceptions=True)`` output.
+
+ Exceptions are logged per batch. If nothing survived and there was work
+ to do, the source failed outright and says so instead of returning a
+ zero that reads like an empty repository.
+ """
+ decisions: list[ExtractedDecision] = []
+ errors: list[str] = []
+ for result in results:
+ if isinstance(result, list):
+ decisions.extend(result)
+ else:
+ errors.append(f"{type(result).__name__}: {result}")
+ logger.warning(
+ "decision_extractor.batch_failed",
+ source=source,
+ error=str(result),
+ )
+ if errors and len(errors) == len(results):
+ raise DecisionSourceError(f"all {len(errors)} batch(es) failed — {errors[0]}")
+ return decisions
+
+
+async def _run_batches(
+ source: str,
+ items: Sequence[Any],
+ size: int,
+ process: Any,
+) -> list[ExtractedDecision]:
+ """Run *process* over *items* in batches of *size*, concurrently.
+
+ Each batch's failure is contained and counted by :func:`_collect_batches`,
+ which raises only when every batch failed.
+ """
+ batches = [items[i : i + size] for i in range(0, len(items), size)]
+ results = await asyncio.gather(*[process(b) for b in batches], return_exceptions=True)
+ return _collect_batches(source, list(results))
+
+
+def _load_json_payload(content: str) -> Any | None:
+ """Decode a model's JSON answer, tolerating code fences and surrounding prose."""
+ if content.startswith("```"):
+ # Remove markdown code fences
+ lines = content.split("\n")
+ content = "\n".join(line for line in lines if not line.strip().startswith("```"))
+ try:
+ return json.loads(content)
+ except json.JSONDecodeError:
+ return _embedded_json_array(content)
+
+
+def _embedded_json_array(content: str) -> Any | None:
+ """The outermost ``[...]`` span inside surrounding prose, decoded, else None."""
+ match = re.search(r"\[.*\]", content, re.DOTALL)
+ if not match:
+ return None
+ try:
+ return json.loads(match.group())
+ except json.JSONDecodeError:
+ return None
+
+
+def _decision_from_item(item: dict) -> ExtractedDecision:
+ """One decision from one object of a model's JSON answer."""
+ return ExtractedDecision(
+ title=item.get("title", ""),
+ context=item.get("context", ""),
+ decision=item.get("decision", ""),
+ rationale=item.get("rationale", ""),
+ alternatives=item.get("alternatives", []),
+ consequences=item.get("consequences", []),
+ tags=item.get("tags", []),
+ # Only the two commit prompts ask for this; elsewhere it stays None.
+ proposed_files=_coerce_paths(item.get("affected_files")),
+ evidence_commits=[item["commit_sha"]] if "commit_sha" in item else [],
+ # The marker this came from, for inline-marker attribution; None elsewhere.
+ evidence_line=_coerce_line(item.get("marker_line")),
+ source_quote=item.get("source_quote", ""),
+ )
+
+
+def parse_decisions_json(content: str) -> list[ExtractedDecision]:
+ """Parse LLM response as JSON array of decisions.
+
+ A blank body raises :class:`EmptyModelResponseError`; every caller
+ sits inside a gather or a fallback that counts that as a lost batch.
+ """
+ content = content.strip()
+ if not content:
+ raise EmptyModelResponseError(
+ "the model returned no content for this batch"
+ )
+ data = _load_json_payload(content)
+ if isinstance(data, dict):
+ data = [data]
+ if not isinstance(data, list):
+ return []
+ return [
+ _decision_from_item(item)
+ for item in data
+ if isinstance(item, dict) and item.get("title", "")
+ ]
diff --git a/packages/core/src/repowise/core/analysis/decisions/records.py b/packages/core/src/repowise/core/analysis/decisions/records.py
new file mode 100644
index 000000000..cca06c31e
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/decisions/records.py
@@ -0,0 +1,80 @@
+"""The records every decision source produces, and the errors a source raises."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from repowise.core.analysis.decisions.lifecycle import ARCHITECTURAL_KIND
+
+
+@dataclass
+class ExtractedDecision:
+ title: str
+ context: str = ""
+ decision: str = ""
+ rationale: str = ""
+ alternatives: list[str] = field(default_factory=list)
+ consequences: list[str] = field(default_factory=list)
+ affected_files: list[str] = field(default_factory=list)
+ #: The files the mining model named, before the commit's own file list
+ #: validates them. Mining-time only: the commit miners intersect it into
+ #: :attr:`affected_files` and clear it. ``[]`` binds the record to nothing;
+ #: ``None`` (never asked, or no answer) falls back to the commit-wide scope.
+ proposed_files: list[str] | None = None
+ affected_modules: list[str] = field(default_factory=list)
+ tags: list[str] = field(default_factory=list)
+ source: str = "inline_marker"
+ #: See :class:`~repowise.core.persistence.models.DecisionRecord.scope_basis`.
+ #: Set by the commit-derived miners, which are the only ones that take a
+ #: file list they did not choose per file.
+ scope_basis: str = ""
+ evidence_commits: list[str] = field(default_factory=list)
+ evidence_file: str | None = None
+ evidence_line: int | None = None
+ confidence: float = 0.5
+ status: str = "proposed"
+ # Defaulted: only the session lane mines prose where an agreement about
+ # conducting the work is stated.
+ kind: str = ARCHITECTURAL_KIND
+ # The claimed verbatim quote and the substring gate's verdict on it.
+ source_quote: str = ""
+ verification: str = "unverified" # exact | fuzzy | unverified
+ # Transient: the verbatim span this decision was drawn from, consumed by
+ # the substring gate and cleared before persistence.
+ source_text: str = ""
+ # ``source`` alone cannot separate the two session lanes. Blank and False
+ # mean the lane said nothing, not that it said no.
+ lane: str = ""
+ needs_split: bool = False
+ # A source that measures its own conformance writes it; None means the
+ # git-diff recompute owns it.
+ staleness_score: float | None = None
+
+
+class DecisionSourceError(RuntimeError):
+ """Every batch of a decision source failed.
+
+ Raised so :meth:`DecisionExtractor.extract_all` records the source as
+ failed rather than empty. A source that loses *some* batches still
+ returns what it has and only logs, because partial supply beats none.
+ """
+
+
+class EmptyModelResponseError(DecisionSourceError):
+ """The model returned no body at all for one batch.
+
+ Distinct from ``[]``, which is a real answer and the common one. Raised so
+ the batch lands in :func:`_collect_batches` as a failure rather than as a
+ source with nothing in it.
+ """
+
+
+@dataclass
+class DecisionExtractionReport:
+ total_found: int
+ decisions: list[ExtractedDecision]
+ by_source: dict[str, int]
+ # {source name: error text} for sources that raised. A failed source
+ # contributes zero decisions, so this is what tells an outage from an
+ # empty repo.
+ failures: dict[str, str] = field(default_factory=dict)
diff --git a/packages/core/src/repowise/core/analysis/decisions/staleness.py b/packages/core/src/repowise/core/analysis/decisions/staleness.py
new file mode 100644
index 000000000..b8c35e207
--- /dev/null
+++ b/packages/core/src/repowise/core/analysis/decisions/staleness.py
@@ -0,0 +1,87 @@
+"""How far the code a decision governs has moved since the decision was recorded."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+
+def _as_aware_utc(value: datetime) -> datetime:
+ """Return ``value`` as a timezone-aware UTC datetime.
+
+ SQLite drops timezone information from ``DateTime(timezone=True)`` columns,
+ but Repowise writes those values as UTC. Treat naive values as UTC so they
+ can be compared with git metadata timestamps, which are already aware UTC.
+ """
+ if value.tzinfo is None or value.utcoffset() is None:
+ return value.replace(tzinfo=UTC)
+ return value.astimezone(UTC)
+
+
+def _coerce_dt(value: datetime | str) -> datetime:
+ """Parse an ISO string into a datetime, passing datetimes through.
+
+ Both shapes reach staleness: the ORM hands back datetimes, while the git
+ metadata map carries whatever was persisted, which for SQLite is a string.
+ """
+ if isinstance(value, str):
+ return datetime.fromisoformat(value.replace("Z", "+00:00"))
+ return value
+
+
+def compute_staleness(
+ decision_created_at: datetime,
+ affected_files: list[str],
+ git_meta_map: dict[str, dict],
+ decision_text: str = "",
+) -> float:
+ """Fraction of *affected_files* that have changed since the record's birth.
+
+ A fact about the code, not a judgement about the record: 0.0 means
+ nothing it governs has moved, 1.0 means all of it has. No tuned
+ constants and no reading of commit messages. *decision_text* is
+ accepted and unused so existing call sites keep working.
+
+ A file with no git metadata **after** the caller's gap fill counts as
+ changed: the record names something the repository does not track, so
+ it cannot be shown to still hold.
+ """
+ if not affected_files:
+ # No scope, so the question cannot be asked. Callers distinguish
+ # this from a genuine 0.0 by the empty file list, and
+ # `decision health` reports it as unscoped rather than fresh.
+ return 0.0
+
+ created = _as_aware_utc(_coerce_dt(decision_created_at)) if decision_created_at else None
+ changed = 0
+ for fp in affected_files:
+ meta = git_meta_map.get(fp)
+ if meta is None:
+ changed += 1 # named but not tracked, so cannot be shown to hold
+ continue
+ last_commit = meta.get("last_commit_at")
+ if not last_commit or created is None:
+ continue
+ if _as_aware_utc(_coerce_dt(last_commit)) > created:
+ changed += 1
+
+ return round(changed / len(affected_files), 3)
+
+
+def last_code_change(
+ affected_files: list[str],
+ git_meta_map: dict[str, dict],
+) -> datetime | None:
+ """When the code a decision governs last moved, or None.
+
+ Reduces the same inputs :meth:`compute_staleness` reads: that counts
+ how many files moved since the record was born, this reports when the
+ most recent of them moved. None where the question cannot be answered,
+ meaning no scope or no git metadata for anything the record names.
+ """
+ dates: list[datetime] = []
+ for file_path in affected_files:
+ meta = git_meta_map.get(file_path)
+ last_commit = meta.get("last_commit_at") if meta else None
+ if last_commit:
+ dates.append(_as_aware_utc(_coerce_dt(last_commit)))
+ return max(dates) if dates else None
diff --git a/tests/unit/analysis/test_decision_batch_budget.py b/tests/unit/analysis/test_decision_batch_budget.py
index 9d249c599..e4b4aaf98 100644
--- a/tests/unit/analysis/test_decision_batch_budget.py
+++ b/tests/unit/analysis/test_decision_batch_budget.py
@@ -1,7 +1,7 @@
"""An empty model response is a lost batch, not an empty repository.
The defect these pin: a miner whose output budget was too small got an empty
-body back, ``_parse_decisions_json`` read that as "no decisions here", and a
+body back, ``parse_decisions_json`` read that as "no decisions here", and a
lane that had lost every batch reported nothing found. The budget that caused
it and the measurement behind its replacement are recorded in decision
``decision-batch-token-budget``.
@@ -20,6 +20,7 @@ from repowise.core.analysis.decisions.extractor import (
DecisionSourceError,
EmptyModelResponseError,
_collect_batches,
+ parse_decisions_json,
)
_SHA = "9a0b27a7"
@@ -70,37 +71,33 @@ def _extractor(tmp_path, content: str) -> DecisionExtractor:
# --- the guard -------------------------------------------------------------
-def test_a_blank_body_raises_rather_than_reading_as_no_decisions(tmp_path):
- ex = _extractor(tmp_path, "")
+def test_a_blank_body_raises_rather_than_reading_as_no_decisions():
with pytest.raises(EmptyModelResponseError):
- ex._parse_decisions_json("")
+ parse_decisions_json("")
-def test_whitespace_is_blank_too(tmp_path):
- ex = _extractor(tmp_path, "")
+def test_whitespace_is_blank_too():
with pytest.raises(EmptyModelResponseError):
- ex._parse_decisions_json(" \n\t ")
+ parse_decisions_json(" \n\t ")
-def test_an_empty_array_is_a_real_answer_and_does_not_raise(tmp_path):
+def test_an_empty_array_is_a_real_answer_and_does_not_raise():
"""The distinction the whole guard rests on.
``[]`` is the model saying these commits hold no architectural decision,
which is correct for most commits. Only a missing body is a failure.
"""
- ex = _extractor(tmp_path, "[]")
- assert ex._parse_decisions_json("[]") == []
+ assert parse_decisions_json("[]") == []
-def test_unparseable_content_still_reads_as_nothing_found(tmp_path):
+def test_unparseable_content_still_reads_as_nothing_found():
"""Not every parse failure is a lost batch.
Prose where JSON was asked for is the model declining in its own words,
and that has always read as nothing found. Widening the guard to cover it
would turn a working lane into a failing one.
"""
- ex = _extractor(tmp_path, "x")
- assert ex._parse_decisions_json("I could not find any decisions.") == []
+ assert parse_decisions_json("I could not find any decisions.") == []
# --- what the miners do with it -------------------------------------------