Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(core): split crud/decisions.py by concern (#2558)

Raghav Chamadiyaf0bdf3145fb7
diff --git a/packages/core/src/repowise/core/persistence/crud/decision_evidence.py b/packages/core/src/repowise/core/persistence/crud/decision_evidence.py
new file mode 100644
index 000000000..15b35c1a4
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/crud/decision_evidence.py
@@ -0,0 +1,153 @@
+"""Decision evidence rows and the headline score derived from them."""
+
+from __future__ import annotations
+
+import json
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from repowise.core.analysis.decisions.provenance import completeness, compute_confidence
+
+from ..models import DecisionEvidence, DecisionRecord
+
+
+def _evidence_quote(d: dict) -> str:
+ """Pick the verbatim span recorded as this evidence row's source quote.
+
+ Prefers the LLM/parser-supplied ``source_quote``; falls back to the
+ decision/rationale text so an evidence row is never empty.
+ """
+ return (d.get("source_quote") or d.get("decision") or d.get("rationale") or "").strip()
+
+
+async def _upsert_decision_evidence(
+ session: AsyncSession,
+ decision_id: str,
+ *,
+ source: str,
+ source_rank: int,
+ evidence_file: str | None,
+ evidence_line: int | None,
+ evidence_commit: str | None,
+ source_quote: str,
+ confidence: float,
+ verification: str,
+) -> None:
+ """Insert or update one evidence row, idempotent on its natural key.
+
+ Natural key is ``(decision_id, source, evidence_file, evidence_commit)``.
+ NULLs are matched explicitly (SQLite treats NULLs as distinct in a unique
+ constraint), so re-indexing the same source converges instead of
+ duplicating, while an incremental update adds genuinely new evidence.
+ """
+ q = select(DecisionEvidence).where(
+ DecisionEvidence.decision_id == decision_id,
+ DecisionEvidence.source == source,
+ )
+ q = (
+ q.where(DecisionEvidence.evidence_file == evidence_file)
+ if evidence_file is not None
+ else q.where(DecisionEvidence.evidence_file.is_(None))
+ )
+ q = (
+ q.where(DecisionEvidence.evidence_commit == evidence_commit)
+ if evidence_commit is not None
+ else q.where(DecisionEvidence.evidence_commit.is_(None))
+ )
+ existing = (await session.execute(q)).scalar_one_or_none()
+
+ if existing is not None:
+ existing.source_rank = source_rank
+ existing.evidence_line = evidence_line
+ existing.source_quote = source_quote
+ existing.confidence = confidence
+ existing.verification = verification
+ return
+
+ session.add(
+ DecisionEvidence(
+ decision_id=decision_id,
+ source=source,
+ source_rank=source_rank,
+ evidence_file=evidence_file,
+ evidence_line=evidence_line,
+ evidence_commit=evidence_commit,
+ source_quote=source_quote,
+ confidence=confidence,
+ verification=verification,
+ )
+ )
+
+
+def _json_list(raw: str | None) -> list[str]:
+ """A JSON array column as a list; anything unparsable reads as empty."""
+ try:
+ value = json.loads(raw or "[]")
+ except (TypeError, ValueError):
+ return []
+ return [str(v) for v in value] if isinstance(value, list) else []
+
+
+def _best_verification(values: list[str]) -> str:
+ """Reduce per-evidence verdicts to the strongest: exact > fuzzy > unverified."""
+ if "exact" in values:
+ return "exact"
+ if "fuzzy" in values:
+ return "fuzzy"
+ return "unverified"
+
+
+def record_completeness(rec: DecisionRecord) -> int:
+ """:func:`completeness` for a stored row, parsing its two JSON columns."""
+ return completeness(
+ decision=rec.decision,
+ rationale=rec.rationale,
+ context=rec.context,
+ consequences=_json_list(rec.consequences_json),
+ alternatives=_json_list(rec.alternatives_json),
+ )
+
+
+def _rederive_headline(rec: DecisionRecord, evidence: list[DecisionEvidence]) -> None:
+ """Set a record's confidence + verification from its full evidence set.
+
+ The single definition of how a headline is scored. Every writer uses it:
+ the upsert path after accreting a run's evidence, ``reconcile_source_ranks``
+ after a ladder edit, and ``reconcile_decision_confidence`` after a formula
+ edit. Kept as one function because two of them were briefly copy-pasted and
+ nothing would have forced the copies to stay equal.
+
+ Confidence rises with the best source rank and with the number of
+ *independent* corroborating sources, and with how much of its body the
+ record fills, so it is derived from the whole set plus the row rather than
+ from whichever evidence row happened to arrive last. No-op on empty
+ evidence: a record with nothing behind it keeps whatever it had.
+ """
+ if not evidence:
+ return
+ best_ver = _best_verification([e.verification for e in evidence])
+ rec.confidence = compute_confidence(
+ max(e.source_rank for e in evidence),
+ len({e.source for e in evidence}),
+ best_ver,
+ filled_fields=record_completeness(rec),
+ )
+ rec.verification = best_ver
+
+async def list_decision_evidence(
+ session: AsyncSession,
+ decision_id: str,
+) -> list[DecisionEvidence]:
+ """Return all evidence rows for a decision, highest source rank first."""
+ result = await session.execute(
+ select(DecisionEvidence)
+ .where(DecisionEvidence.decision_id == decision_id)
+ .order_by(DecisionEvidence.source_rank.desc(), DecisionEvidence.created_at.asc())
+ )
+ return list(result.scalars().all())
+
+
+def _first_commit(d: dict) -> str | None:
+ commits = d.get("evidence_commits") or []
+ return commits[0] if commits else None
diff --git a/packages/core/src/repowise/core/persistence/crud/decision_health.py b/packages/core/src/repowise/core/persistence/crud/decision_health.py
new file mode 100644
index 000000000..f789c698b
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/crud/decision_health.py
@@ -0,0 +1,193 @@
+"""The decision health summary: lane counts, stale records, ungoverned hotspots."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from repowise.core.analysis.decisions.lifecycle import status_rank
+from repowise.core.analysis.decisions.scope import binds_to_paths
+
+from ..models import DecisionRecord, GitMetadata
+
+
+async def get_decision_health_summary(
+ session: AsyncSession,
+ repository_id: str,
+) -> dict:
+ """Return decision health: counts by lane, stale decisions, ungoverned hotspots.
+
+ The five list fields are returned ranked worst-first: stale by staleness,
+ proposed by confidence, ungoverned hotspots by temporal hotspot score,
+ retired by lane (history before tombstone) and unscoped by confidence. A
+ caller that shows only the first few shows the few that matter.
+ Callers may truncate; they must not re-order.
+
+ ``retired_decisions`` holds ``(lane, record)`` pairs: the lane is derived
+ from the acceptance, which the ``status`` column may disagree with.
+
+ Counts the acceptance, not the status column. The key names are the ones
+ every caller already renders, and they keep their product meaning:
+ ``active`` is what governs, ``proposed`` is what nobody has accepted. What
+ changed is that a record reaches ``active`` here by having an acceptance,
+ the same way it does on every other governance read. Counting the column
+ made this the one surface still reporting a hundred governing decisions on
+ a store whose acceptances were empty, and it fed ``ungoverned_hotspots``,
+ so unaccepted records were also suppressing the files nothing governs.
+ """
+ result = await session.execute(
+ select(DecisionRecord).where(
+ DecisionRecord.repository_id == repository_id,
+ )
+ )
+ all_decisions = list(result.scalars().all())
+ from .authority import decision_currencies
+
+ currencies = await decision_currencies(session, repository_id, all_decisions)
+
+ lanes = _Lanes()
+ for d in all_decisions:
+ lanes.add(d, currencies.get(d.id))
+ ungoverned = await _ungoverned_hotspots(session, repository_id, lanes.governed_files)
+ lanes.rank()
+ conflicts = await _conflict_summaries(session, repository_id, all_decisions)
+ lanes.counts["conflicts"] = len(conflicts)
+
+ return {
+ "summary": lanes.counts,
+ "stale_decisions": lanes.stale,
+ "proposed_awaiting_review": lanes.proposed,
+ "ungoverned_hotspots": ungoverned,
+ "conflicts": conflicts,
+ "retired_decisions": lanes.retired,
+ "unscoped_decisions": lanes.unscoped,
+ }
+
+
+@dataclass
+class _Lanes:
+ """Every record sorted into the lane its acceptance puts it in."""
+
+ counts: dict[str, int] = field(
+ default_factory=lambda: {
+ "active": 0,
+ "proposed": 0,
+ "deprecated": 0,
+ "superseded": 0,
+ "dismissed": 0,
+ "stale": 0,
+ # Accepted records naming no file. They score 0.0 because the
+ # staleness question cannot be asked of them, which renders
+ # identically to a record whose code genuinely has not moved, so
+ # they are counted separately rather than banked as fresh.
+ "unscoped": 0,
+ }
+ )
+ stale: list[DecisionRecord] = field(default_factory=list)
+ proposed: list[DecisionRecord] = field(default_factory=list)
+ # Counted-only lanes. The record is in hand at the point that drops it, so
+ # naming it costs no query. ``retired`` carries its lane because that lane
+ # is derived from the acceptance where there is one, and a record's
+ # ``status`` column may disagree with it.
+ retired: list[tuple[str, DecisionRecord]] = field(default_factory=list)
+ unscoped: list[DecisionRecord] = field(default_factory=list)
+ # Files an *accepted* decision names. A candidate naming a hotspot does not
+ # make it governed, and counting one did: it removed the file from
+ # ``ungoverned_hotspots``, which is the list whose whole job is to say
+ # where nobody has decided anything.
+ governed_files: set[str] = field(default_factory=set)
+
+ def add(self, d: DecisionRecord, currency: str | None) -> None:
+ if currency is None:
+ # No acceptance, so it does not govern. A retired one is not
+ # awaiting review either: it keeps the status that retired it and
+ # stays out of the queue, which is the one thing a tombstone must
+ # never be counted as.
+ if d.status in ("dismissed", "deprecated", "superseded"):
+ self.counts[d.status] = self.counts.get(d.status, 0) + 1
+ self.retired.append((d.status, d))
+ else:
+ self.counts["proposed"] += 1
+ self.proposed.append(d)
+ return
+ if currency in ("superseded", "dismissed"):
+ self.counts[currency] += 1
+ self.retired.append((currency, d))
+ return
+ self.counts["active"] += 1
+ if currency == "needs_review":
+ self.counts["stale"] += 1
+ self.stale.append(d)
+ if currency == "uncheckable":
+ self.counts["unscoped"] += 1
+ self.unscoped.append(d)
+ # ``governed_files`` is the denominator for "ungoverned hotspots",
+ # so a footprint would suppress every file its commit touched.
+ if binds_to_paths(d.scope_basis):
+ self.governed_files.update(json.loads(d.affected_files_json))
+
+ def rank(self) -> None:
+ """Order every list worst-first, with an id tiebreak so runs agree.
+
+ Callers truncate these lists and must not re-order them.
+ """
+ self.stale.sort(key=lambda d: (-(d.staleness_score or 0.0), d.id))
+ self.proposed.sort(key=lambda d: (-(d.confidence or 0.0), d.id))
+ # Retired by lane, history before tombstone. Not by ``updated_at``: it
+ # moves on any write, so it does not say when a record was retired.
+ # ``unscoped`` by confidence, the key ``proposed`` already uses.
+ self.retired.sort(key=lambda pair: (status_rank(pair[0]), pair[1].id))
+ self.unscoped.sort(key=lambda d: (-(d.confidence or 0.0), d.id))
+
+
+async def _ungoverned_hotspots(
+ session: AsyncSession, repository_id: str, governed_files: set[str]
+) -> list[str]:
+ """Hotspot files no accepted decision names, hottest first.
+
+ Same key as ``routers/overview.py``: score descending with NULLs last, then churn.
+ """
+ hotspot_result = await session.execute(
+ select(
+ GitMetadata.file_path,
+ GitMetadata.temporal_hotspot_score,
+ GitMetadata.churn_percentile,
+ ).where(
+ GitMetadata.repository_id == repository_id,
+ GitMetadata.is_hotspot == True, # noqa: E712
+ )
+ )
+ hotspot_rows = {row[0]: (row[1], row[2]) for row in hotspot_result.all()}
+
+ def _hotspot_rank(file_path: str) -> tuple[bool, float, float, str]:
+ score, churn = hotspot_rows[file_path]
+ return (score is None, -(score or 0.0), -(churn or 0.0), file_path)
+
+ return sorted(hotspot_rows.keys() - governed_files, key=_hotspot_rank)
+
+
+async def _conflict_summaries(
+ session: AsyncSession, repository_id: str, all_decisions: list[DecisionRecord]
+) -> list[dict]:
+ """Phase 3B: contradictory active decisions (``conflicts_with`` edges)."""
+ from ..decision_graph import list_conflict_edges
+
+ by_id = {d.id: d for d in all_decisions}
+ conflicts: list[dict] = []
+ for edge in await list_conflict_edges(session, repository_id):
+ src = by_id.get(edge.src_decision_id)
+ dst = by_id.get(edge.dst_decision_id)
+ if src is None or dst is None:
+ continue
+ conflicts.append(
+ {
+ "src": {"id": src.id, "title": src.title, "status": src.status},
+ "dst": {"id": dst.id, "title": dst.title, "status": dst.status},
+ "confidence": edge.confidence,
+ "evidence": edge.evidence,
+ }
+ )
+ return conflicts
diff --git a/packages/core/src/repowise/core/persistence/crud/decision_identity.py b/packages/core/src/repowise/core/persistence/crud/decision_identity.py
new file mode 100644
index 000000000..a6e7b5056
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/crud/decision_identity.py
@@ -0,0 +1,161 @@
+"""A decision's derived id, and the status/kind rules re-extraction must respect."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Iterable
+
+from repowise.core.analysis.decisions.lifecycle import ARCHITECTURAL_KIND, DECISION_KINDS
+from repowise.core.analysis.decisions.provenance import normalize_text
+
+# Namespaced so a decision id can never collide with another kind of derived
+# id, and versioned so a future change of the recipe is a visible new value
+# rather than a silent reshuffle of every id in every store.
+_ID_NAMESPACE = "repowise.decision.id.v2"
+
+# The identity fields are joined with a separator none of them can contain,
+# so no pair of distinct records can flatten to the same string.
+_FIELD_SEP = "\x00"
+
+# ``evidence_file`` is nullable and NULL is not the empty string here: the
+# dedupe query treats them as different records, so the derivation must too.
+_NULL_EVIDENCE_FILE = "\x01"
+
+# The pinned quote is truncated before it is hashed. A quote long enough to
+# reach this is a paragraph, and its opening already separates two decisions
+# that share a file set; carrying the rest only widens the surface on which a
+# re-mine can reword the identity out from under a reference somebody wrote.
+_IDENTITY_QUOTE_CHARS = 200
+
+# Statuses a human (or the evolution judge) set deliberately. Re-extraction
+# must never walk one of these back to ``proposed``: a re-harvested source
+# ties or beats the stored rank, so without this guard a reindex would flip
+# confirmed decisions back into the review queue and resurrect dismissed ones
+# (#751). ``dismissed`` is terminal: the row is kept purely as a tombstone so
+# the same content never re-proposes.
+_PROTECTED_STATUSES = frozenset({"active", "deprecated", "superseded", "dismissed"})
+
+
+def _extraction_status(incoming: str) -> str:
+ """The status extraction is allowed to write.
+
+ Extraction proposes; acceptance promotes. An incoming ``active`` is a
+ parsed ADR heading or an old code path's optimism, and neither is an
+ acceptance event, so it lands as a proposal. A tracked ADR still reaches
+ ``active`` — via :func:`_accept_from_tracked_artifact` below, which records
+ the document as the accepter instead of leaving the promotion anonymous.
+ """
+ return "proposed" if incoming == "active" else incoming
+
+
+def _merge_status(existing: str, incoming: str) -> str:
+ """Resolve a re-extracted status against the stored one.
+
+ A protected stored status wins over an incoming ``proposed``; anything else
+ follows the incoming value, after :func:`_extraction_status` has taken
+ authority out of extraction's hands.
+ """
+ incoming = _extraction_status(incoming)
+ if incoming == "proposed" and existing in _PROTECTED_STATUSES:
+ return existing
+ return incoming
+
+
+def _extraction_kind(incoming: str | None) -> str:
+ """The kind extraction is allowed to write.
+
+ Unrecognised falls to ``architectural`` rather than being stored: the
+ vocabulary has no CHECK behind it, every reader tests ``== AGREEMENT_KIND``,
+ and a junk value that merely happens to read as architectural everywhere is
+ safe by luck. Normalising here makes it safe by construction.
+ """
+ return incoming if incoming in DECISION_KINDS else ARCHITECTURAL_KIND
+
+def identity_quote_for(text: str | None) -> str:
+ """The pinned form of a decision's quote, for identity only.
+
+ Whitespace-collapsed, lowercased and truncated, so the identity survives a
+ re-mine that reflows or recases the same sentence. The stored column keeps
+ the verbatim text; only this derived form is hashed.
+ """
+ return normalize_text(text)[:_IDENTITY_QUOTE_CHARS]
+
+
+def derive_decision_id(
+ repository_id: str,
+ title: str,
+ *,
+ source: str,
+ evidence_file: str | None,
+ affected_files: Iterable[str] = (),
+ evidence_line: int | None = None,
+ identity_quote: str = "",
+ needs_split: bool = False,
+) -> str:
+ """The id a decision with this identity has, in every store, on every run.
+
+ Keyed on the evidence rather than on the title. A title is the most
+ volatile thing a decision carries: two extractions of the same choice word
+ it differently and produce two records, and rewording one moves its id.
+ Over the dogfood store, title identity collapses **zero** duplicate pairs
+ while the files-plus-evidence-plus-quote key below collapses **120 pairs
+ across 42 classes**, and loses none of what title identity caught.
+
+ The quote is load-bearing, and is why the file set alone is not the key:
+ two unrelated classes in that store share a file pair and are separated
+ only by what was said. It is also the fragile part, because most records
+ key on mined prose rather than on a committed span, so it is **pinned in
+ the row at first capture and never re-derived**. A later extraction that
+ rewords the same sentence then leaves the id where it is.
+
+ A record with no evidence at all keeps its title, for the same reason the
+ rest of them lose it: identity is the evidence, and a record that has
+ none has no evidence identity to key on. Without this every scopeless,
+ quoteless record in a repository derives one id and the second one to be
+ written collides with the first. Two of the 357 records in the dogfood
+ store are in that state, and the construction paths that supply nothing
+ but a title are all in that state.
+
+ A record flagged ``needs_split`` keeps its title in the key. A bundled
+ claim shares its files and its evidence with the separate decisions it
+ bundles, so evidence identity would fold all of them together and file two
+ decisions under a third one's name. Holding the flagged record on title
+ identity is what keeps it apart, and it is why the flag has to be set
+ before this key ships. The guard is partial: it reaches the bundles a
+ marker can see and no others, measured at 28 of 357 records held out for
+ 20 of 120 merges given up.
+
+ ``source`` is deliberately **not** in the key. Two lanes that mined the
+ same sentence out of the same files recorded one decision, not two, and
+ the quote already separates two that merely share a scope.
+
+ 32 lowercase hex, because ``DecisionRecord.id`` is ``String(32)`` and so
+ is every foreign key to it, so a truncated digest fits without a column
+ change. ``evidence_file`` is NULL far more often than not, so it gets a
+ sentinel no path can contain rather than collapsing into the empty string.
+
+ Note the id follows the identity, so editing the scope or the pinned quote
+ moves it. :mod:`decision_id_migration`, which runs at the head of every
+ index, is what settles that, and it leaves an alias behind so an id
+ already written down keeps resolving.
+ """
+ files = sorted({str(f) for f in affected_files if f})
+ quote = identity_quote_for(identity_quote)
+ grounded = bool(files) or evidence_file is not None or bool(quote)
+ parts = (
+ _ID_NAMESPACE,
+ repository_id,
+ json.dumps(files),
+ json.dumps(
+ [
+ _NULL_EVIDENCE_FILE if evidence_file is None else evidence_file,
+ evidence_line,
+ ]
+ ),
+ quote,
+ title if needs_split or not grounded else "",
+ source if not grounded else "",
+ )
+ digest = hashlib.sha256(_FIELD_SEP.join(parts).encode("utf-8")).hexdigest()
+ return digest[:32]
diff --git a/packages/core/src/repowise/core/persistence/crud/decision_ingest.py b/packages/core/src/repowise/core/persistence/crud/decision_ingest.py
new file mode 100644
index 000000000..fe32d9b2d
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/crud/decision_ingest.py
@@ -0,0 +1,558 @@
+"""Batch ingest of extracted decisions: grouping, dedupe and provenance."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any
+
+import structlog
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from repowise.core.analysis.decisions.provenance import rank_for_source
+
+from ..decision_graph import sync_links_from_record
+from ..models import DecisionRecord, _now_utc
+from .decision_evidence import (
+ _evidence_quote,
+ _first_commit,
+ _rederive_headline,
+ _upsert_decision_evidence,
+ list_decision_evidence,
+)
+from .decision_identity import (
+ _extraction_kind,
+ _extraction_status,
+ _merge_status,
+ derive_decision_id,
+)
+from .decision_review_meta import _write_candidate_meta
+
+
+def _normalize_title(title: str) -> str:
+ """Normalize a decision title for cross-source dedup comparison."""
+ import re as _re
+
+ t = title.lower().strip()
+ t = _re.sub(r"[^a-z0-9\s]", "", t)
+ t = _re.sub(r"\s+", " ", t)
+ return t
+
+#: Sources whose evidence file is itself a reviewable, version-controlled
+#: statement of the decision. Only these can accept without a person present.
+_TRACKED_ARTIFACT_SOURCES: frozenset[str] = frozenset({"adr"})
+
+
+async def _accept_from_tracked_artifact(
+ session: AsyncSession, rec: DecisionRecord, headline: dict
+) -> None:
+ """Grant acceptance to an ADR the repository has committed as accepted.
+
+ Silent on anything that is not one: a missing rationale or scope means the
+ document did not say enough to bind future work, and a refusal there is
+ correct rather than an error to report. Re-running is a no-op because the
+ acceptance already exists.
+ """
+ if headline.get("status") != "active":
+ return
+ if headline.get("source") not in _TRACKED_ARTIFACT_SOURCES:
+ return
+ artifact = headline.get("evidence_file") or ""
+ if not artifact:
+ return
+
+ from repowise.core.analysis.decisions.accepter import is_tracked
+
+ from ..models import Repository
+ from .authority import AcceptanceRefusedError, is_accepted, record_acceptance
+
+ if await is_accepted(session, rec.id):
+ return
+ repo = await session.get(Repository, rec.repository_id)
+ if repo is None or not is_tracked(repo.local_path, artifact):
+ # An uncommitted document is one machine's file. Letting it accept would
+ # replace the human acceptance event with a file an agent can write.
+ return
+ try:
+ await record_acceptance(
+ session,
+ rec,
+ action="accepted",
+ currency="active",
+ artifact=artifact,
+ kind="import",
+ note="accepted by a tracked decision record",
+ )
+ except AcceptanceRefusedError as exc:
+ structlog.get_logger(__name__).debug(
+ "adr_acceptance_refused", decision_id=rec.id, reason=str(exc)
+ )
+
+
+async def bulk_upsert_decisions(
+ session: AsyncSession,
+ repository_id: str,
+ decisions: list[dict],
+ *,
+ vector_store: Any | None = None,
+) -> list[str]:
+ """Upsert decisions, accreting provenance instead of discarding losers.
+
+ Decisions with near-identical normalized titles are merged into a single
+ :class:`DecisionRecord`; every contributing source becomes a
+ :class:`DecisionEvidence` row. The record's headline fields come from the
+ highest-``source_rank`` contributor, and its confidence is recomputed from
+ the best rank + the number of independently corroborating sources + the
+ strongest surviving verification verdict.
+
+ Idempotent: a full re-index converges (evidence rows upsert on their
+ natural key); an incremental update adds new evidence and re-derives the
+ headline + confidence from the union.
+
+ When *vector_store* (the shared page-generator store) is supplied, a
+ Phase-2C semantic pass augments the cheap normalized-title match: an
+ incoming group that matched no existing record by title is looked up in the
+ store, and if its nearest ``decision:`` neighbour clears the cosine
+ threshold it is folded into that record as additional evidence — so
+ paraphrases ("Use Redis" vs "Adopt Redis cache") collapse into one record.
+ Every touched record is (re-)embedded into the store, so decisions are
+ matchable next run *and* discoverable via ``search_codebase``.
+
+ Returns the ids of every record touched (created or updated) this call, so
+ a caller can run the Phase-3 supersession/conflict detection over just the
+ records that changed.
+ """
+ groups = _group_by_normalized_title(decisions)
+ if not groups:
+ return []
+
+ existing_by_norm = await _existing_records_by_title(session, repository_id)
+ # A store hit (which returns a decision id) resolves back to the live
+ # record through this map, and it is grown as records are created so
+ # paraphrases *within* one batch also collapse.
+ id_to_rec: dict[str, DecisionRecord] = {rec.id: rec for rec in existing_by_norm.values()}
+
+ # Headline candidate per group: highest source rank, tie-break by
+ # confidence. Hoisted so the batched embedding can see every group's
+ # match text before the loop runs.
+ headline_by_norm: dict[str, dict] = {
+ norm: max(
+ members,
+ key=lambda d: (rank_for_source(d.get("source", "")), d.get("confidence", 0.0)),
+ )
+ for norm, members in groups.items()
+ }
+ dedup = await _SemanticDedup.prepare(vector_store, headline_by_norm, group_count=len(groups))
+
+ touched_ids: list[str] = []
+ captured: dict[str, tuple[str, bool]] = {}
+
+ for norm, members in groups.items():
+ headline = headline_by_norm[norm]
+ rec = existing_by_norm.get(norm)
+
+ # No title match → ask the store whether a semantically-equivalent
+ # decision already exists, and fold into it if so (cheap title dedup
+ # stays the first pass; this only runs on the residual).
+ # A computed record must not fold into a prose record that merely
+ # sounds alike; title dedup still applies to it.
+ if rec is None and headline.get("source") != "conventions":
+ match_id = await dedup.find_match(norm, headline)
+ if match_id is not None and match_id in id_to_rec:
+ rec = id_to_rec[match_id]
+ existing_by_norm[norm] = rec
+
+ # A dismissed record is a tombstone: leave it untouched (no headline
+ # promotion, no new evidence, not re-embedded, not in touched_ids) so
+ # dismissals survive every reindex.
+ if rec is not None and rec.status == "dismissed":
+ continue
+
+ created = False
+ if rec is None:
+ identity = _headline_identity(repository_id, headline, members)
+ # Identity is evidence, so two titles for one decision derive one
+ # id. Folding into the record that id already names is the point of
+ # the key: inserting instead would collide on the primary key, and
+ # skipping would leave the duplicate this is meant to collapse.
+ folded = id_to_rec.get(identity.id) or await session.get(DecisionRecord, identity.id)
+ if folded is not None and folded.repository_id == repository_id:
+ if folded.status == "dismissed":
+ continue
+ rec = folded
+ else:
+ rec = _new_record(repository_id, identity, headline)
+ session.add(rec)
+ await session.flush()
+ created = True
+ existing_by_norm[norm] = rec
+ id_to_rec[rec.id] = rec
+ if not created:
+ _maybe_promote_headline(rec, headline)
+
+ await _accrete_evidence(session, rec.id, members)
+
+ # A committed ADR that says "accepted" is the one non-human acceptance
+ # the contract allows, because the document is version controlled and
+ # reviewable. Recorded here, where the parsed heading is still in hand.
+ await _accept_from_tracked_artifact(session, rec, headline)
+
+ # Re-derive headline confidence + verification from the FULL evidence
+ # set (existing + just-added), so corroboration accrues across runs.
+ _rederive_headline(rec, await list_decision_evidence(session, rec.id))
+ rec.updated_at = _now_utc()
+ touched_ids.append(rec.id)
+ _record_capture(captured, rec.id, headline, members)
+
+ await sync_links_from_record(session, rec)
+ await dedup.index_record(rec, norm)
+
+ await dedup.flush(touched_ids, id_to_rec)
+ await _write_candidate_meta(session, repository_id, captured)
+
+ await session.flush()
+ return touched_ids
+
+
+def _group_by_normalized_title(decisions: list[dict]) -> dict[str, list[dict]]:
+ """Incoming decisions keyed by normalized title; untitled ones are dropped."""
+ groups: dict[str, list[dict]] = {}
+ for d in decisions:
+ norm = _normalize_title(d.get("title", ""))
+ if not norm:
+ continue
+ groups.setdefault(norm, []).append(d)
+ return groups
+
+
+async def _existing_records_by_title(
+ session: AsyncSession, repository_id: str
+) -> dict[str, DecisionRecord]:
+ """This repository's records by normalized title, so cross-run merges land on one row.
+
+ On a title collision the most authoritative existing row is canonical.
+ """
+ existing_rows = await session.execute(
+ select(DecisionRecord).where(DecisionRecord.repository_id == repository_id)
+ )
+ existing_by_norm: dict[str, DecisionRecord] = {}
+ for rec in existing_rows.scalars().all():
+ norm = _normalize_title(rec.title)
+ prior = existing_by_norm.get(norm)
+ if prior is None or rank_for_source(rec.source) > rank_for_source(prior.source):
+ existing_by_norm[norm] = rec
+ return existing_by_norm
+
+
+@dataclass(frozen=True)
+class _HeadlineIdentity:
+ """The derived id of a group that matched no record, and the quote it pins."""
+
+ id: str
+ quote: str
+
+
+def _headline_identity(
+ repository_id: str, headline: dict, members: list[dict]
+) -> _HeadlineIdentity:
+ # Pinned now and never revised: the identity is keyed on it, and a later
+ # extraction rewording the same sentence must not move the id.
+ quote = _evidence_quote(headline)
+ derived = derive_decision_id(
+ repository_id,
+ headline.get("title", ""),
+ source=headline.get("source", "cli"),
+ evidence_file=headline.get("evidence_file"),
+ affected_files=headline.get("affected_files") or [],
+ evidence_line=headline.get("evidence_line"),
+ identity_quote=quote,
+ needs_split=any(bool(d.get("needs_split")) for d in members),
+ )
+ return _HeadlineIdentity(id=derived, quote=quote)
+
+
+def _new_record(
+ repository_id: str, identity: _HeadlineIdentity, headline: dict
+) -> DecisionRecord:
+ """A new record whose every field comes from the group's headline."""
+ # A source that measures its own conformance supplies the score;
+ # otherwise the git-diff recompute fills it in later.
+ headline_staleness = headline.get("staleness_score")
+ return DecisionRecord(
+ id=identity.id,
+ identity_quote=identity.quote,
+ repository_id=repository_id,
+ title=headline.get("title", ""),
+ status=_extraction_status(headline.get("status", "proposed")),
+ kind=_extraction_kind(headline.get("kind")),
+ context=headline.get("context") or "",
+ decision=headline.get("decision") or "",
+ rationale=headline.get("rationale") or "",
+ alternatives_json=json.dumps(headline.get("alternatives") or []),
+ consequences_json=json.dumps(headline.get("consequences") or []),
+ affected_files_json=json.dumps(headline.get("affected_files") or []),
+ # Rides with the file list it describes.
+ scope_basis=headline.get("scope_basis") or "",
+ affected_modules_json=json.dumps(headline.get("affected_modules") or []),
+ tags_json=json.dumps(headline.get("tags") or []),
+ evidence_commits_json=json.dumps(headline.get("evidence_commits") or []),
+ source=headline.get("source", "cli"),
+ evidence_file=headline.get("evidence_file"),
+ evidence_line=headline.get("evidence_line"),
+ confidence=headline.get("confidence", 0.5),
+ staleness_score=(float(headline_staleness) if headline_staleness is not None else 0.0),
+ )
+
+
+def _maybe_promote_headline(rec: DecisionRecord, headline: dict) -> None:
+ """Promote *headline*'s fields onto *rec* if its source is at least as authoritative.
+
+ Provenance still accretes either way; this only decides whose wording the
+ record shows.
+ """
+ if rank_for_source(headline.get("source", "")) < rank_for_source(rec.source):
+ return
+ rec.title = headline.get("title", rec.title)
+ rec.status = _merge_status(rec.status, headline.get("status", rec.status))
+ # ``kind`` is deliberately absent: the noun is decided when the
+ # record is created and re-extraction does not revisit it. Most
+ # sources default the field rather than deciding it, so an incoming
+ # value is usually the absence of a judgement; and this branch runs
+ # on accepted records too, where flipping the noun would change what
+ # a record governs behind the person who accepted it. The migration
+ # refuses that for the same reason, and the manifest is where a
+ # person changes it deliberately.
+ rec.context = headline.get("context") or rec.context
+ rec.decision = headline.get("decision") or rec.decision
+ rec.rationale = headline.get("rationale") or rec.rationale
+ rec.alternatives_json = json.dumps(headline.get("alternatives") or [])
+ rec.consequences_json = json.dumps(headline.get("consequences") or [])
+ rec.affected_files_json = json.dumps(headline.get("affected_files") or [])
+ rec.scope_basis = headline.get("scope_basis") or ""
+ rec.affected_modules_json = json.dumps(headline.get("affected_modules") or [])
+ rec.tags_json = json.dumps(headline.get("tags") or [])
+ rec.evidence_commits_json = json.dumps(headline.get("evidence_commits") or [])
+ rec.source = headline.get("source", rec.source)
+ rec.evidence_file = headline.get("evidence_file")
+ rec.evidence_line = headline.get("evidence_line")
+ # Only a source that measures its own conformance supplies this.
+ if headline.get("staleness_score") is not None:
+ rec.staleness_score = float(headline["staleness_score"])
+
+
+async def _accrete_evidence(session: AsyncSession, decision_id: str, members: list[dict]) -> None:
+ """Upsert one evidence row per contributing source occurrence."""
+ for d in members:
+ src = d.get("source", "cli")
+ await _upsert_decision_evidence(
+ session,
+ decision_id,
+ source=src,
+ source_rank=rank_for_source(src),
+ evidence_file=d.get("evidence_file"),
+ evidence_line=d.get("evidence_line"),
+ evidence_commit=_first_commit(d),
+ source_quote=_evidence_quote(d),
+ confidence=d.get("confidence", 0.5),
+ verification=d.get("verification", "unverified"),
+ )
+
+
+def _record_capture(
+ captured: dict[str, tuple[str, bool]],
+ decision_id: str,
+ headline: dict,
+ members: list[dict],
+) -> None:
+ """Accumulate the lane and split flag this run's candidate-meta row carries.
+
+ Accumulates because two title groups can fold onto one record. The split
+ flag is read from every member: any contributor raising it flags the record.
+ """
+ prior_lane, prior_split = captured.get(decision_id, ("", False))
+ captured[decision_id] = (
+ prior_lane or headline.get("lane") or "",
+ prior_split or any(bool(d.get("needs_split")) for d in members),
+ )
+
+
+class _SemanticDedup:
+ """Phase-2C semantic dedup against the shared vector store; inert without one.
+
+ A group with no title match folds into its nearest stored neighbour above
+ the cosine threshold. Batch mode embeds all groups up front, dedupes within
+ the batch against a pending index and defers store writes to one upsert;
+ stores that cannot search by vector use the per-item flow.
+ """
+
+ def __init__(
+ self,
+ vector_store: Any | None,
+ *,
+ group_vecs: dict[str, list[float]] | None = None,
+ group_texts: dict[str, str] | None = None,
+ ) -> None:
+ self._store = vector_store
+ self._group_vecs = group_vecs or {}
+ self._group_texts = group_texts or {}
+ self._batch_mode = group_vecs is not None
+ # Vectors of records touched this batch but not yet written to the
+ # store: the in-batch side of dedup in batch mode. A record is only
+ # registered when its final text equals the group text its vector came
+ # from, so the index never matches against text the record doesn't
+ # actually contain.
+ self._pending: Any | None = None
+ if self._batch_mode:
+ from repowise.core.analysis.decision_semantic_match import PendingDecisionIndex
+
+ self._pending = PendingDecisionIndex()
+
+ @classmethod
+ async def prepare(
+ cls,
+ vector_store: Any | None,
+ headline_by_norm: dict[str, dict],
+ *,
+ group_count: int,
+ ) -> _SemanticDedup:
+ if vector_store is None:
+ return cls(None)
+ from repowise.core.analysis.decision_semantic_match import decision_match_text
+
+ ordered = [
+ (norm, decision_match_text(h.get("title", ""), h.get("decision") or ""))
+ for norm, h in headline_by_norm.items()
+ ]
+ ordered = [(norm, text) for norm, text in ordered if text]
+ vecs = None
+ if ordered and _supports_vector_search(vector_store):
+ try:
+ vecs = await vector_store.embed_texts([text for _, text in ordered])
+ except Exception:
+ vecs = None
+ batched = vecs is not None and len(vecs) == len(ordered)
+
+ structlog.get_logger(__name__).info(
+ "decision_dedup_embedding",
+ groups=group_count,
+ batched=batched,
+ )
+ if not batched:
+ return cls(vector_store)
+ return cls(
+ vector_store,
+ group_vecs=dict(zip((n for n, _ in ordered), vecs, strict=True)),
+ group_texts=dict(ordered),
+ )
+
+ async def find_match(self, norm: str, headline: dict) -> str | None:
+ """The id of a stored decision this group paraphrases, if any."""
+ if self._store is None:
+ return None
+ from repowise.core.analysis.decision_semantic_match import (
+ find_duplicate_decision,
+ find_duplicate_decision_by_vector,
+ )
+
+ q_vec = self._group_vecs.get(norm)
+ if self._batch_mode and q_vec is not None:
+ return await find_duplicate_decision_by_vector(
+ self._store, q_vec, pending=self._pending
+ )
+ return await find_duplicate_decision(
+ self._store,
+ title=headline.get("title", ""),
+ decision=headline.get("decision") or "",
+ )
+
+ async def index_record(self, rec: DecisionRecord, norm: str) -> None:
+ """Make *rec* matchable by later groups in this batch and by future runs.
+
+ Best-effort, never blocks the SQL upsert. In batch mode the store write
+ is deferred to :meth:`flush`; until then the group vector stands in for
+ the record in the pending index, but only when the record's final text
+ IS the group text (created, or promoted from this headline). A fold that
+ kept the record's own fields must not be matchable under the incoming
+ group's text.
+ """
+ if self._store is None:
+ return
+ from repowise.core.analysis.decision_semantic_match import (
+ decision_match_text,
+ upsert_decision_vector,
+ )
+
+ if not self._batch_mode:
+ await upsert_decision_vector(
+ self._store,
+ rec.id,
+ title=rec.title,
+ decision=rec.decision or "",
+ evidence_file=rec.evidence_file,
+ )
+ return
+ q_vec = self._group_vecs.get(norm)
+ final_text = decision_match_text(rec.title, rec.decision or "")
+ if (
+ q_vec is not None
+ and self._pending is not None
+ and rec.id not in self._pending.ids
+ and final_text == self._group_texts.get(norm)
+ ):
+ self._pending.add(rec.id, q_vec)
+
+ async def flush(self, touched_ids: list[str], id_to_rec: dict[str, DecisionRecord]) -> None:
+ """Batched store write of every touched record's *final* text.
+
+ Post merge/promotion, reusing the vectors already computed for
+ unchanged texts and embedding only the delta.
+ """
+ if self._store is None or not self._batch_mode or not touched_ids:
+ return
+ from repowise.core.analysis.decision_semantic_match import (
+ decision_vector_item,
+ upsert_decision_vectors,
+ )
+
+ items: list[tuple[str, str, dict]] = []
+ for rid in dict.fromkeys(touched_ids):
+ rec = id_to_rec.get(rid)
+ if rec is None:
+ continue
+ item = decision_vector_item(
+ rec.id,
+ title=rec.title,
+ decision=rec.decision or "",
+ evidence_file=rec.evidence_file,
+ )
+ if item is not None:
+ items.append(item)
+ vectors_by_text = {
+ text: self._group_vecs[norm]
+ for norm, text in self._group_texts.items()
+ if norm in self._group_vecs
+ }
+ await upsert_decision_vectors(self._store, items, vectors_by_text=vectors_by_text)
+
+
+def _supports_vector_search(vector_store: Any) -> bool:
+ """Whether *vector_store* can be searched by a raw vector.
+
+ Static capability check: the base-class ``search_by_vector`` is the
+ "unsupported" sentinel. A runtime probe would conflate a transient store
+ error with lack of support and silently fall back to the O(N)-round-trip
+ path batch mode exists to avoid.
+ """
+ from repowise.core.persistence.vector_store import VectorStore
+ from repowise.core.providers.embedding import store_has_semantic_vectors
+
+ store_cls_search = getattr(type(vector_store), "search_by_vector", None)
+ return (
+ callable(store_cls_search)
+ and store_cls_search is not VectorStore.search_by_vector
+ # A keyless store supports the call and cannot answer it: every
+ # matcher refuses on it. Without this the batch pays for a full embed
+ # round whose vectors are then never read.
+ and store_has_semantic_vectors(vector_store)
+ )
diff --git a/packages/core/src/repowise/core/persistence/crud/decision_repairs.py b/packages/core/src/repowise/core/persistence/crud/decision_repairs.py
new file mode 100644
index 000000000..efb0ed304
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/crud/decision_repairs.py
@@ -0,0 +1,313 @@
+"""Idempotent store-wide decision repairs and purges; each is a no-op once applied."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import structlog
+from sqlalchemy import delete, or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from repowise.core.analysis.decisions.provenance import SOURCE_RANK, rank_for_source
+
+from ..models import (
+ DecisionEdge,
+ DecisionEvidence,
+ DecisionNodeLink,
+ DecisionRecord,
+ _now_utc,
+)
+from .authority import accepted_predicate
+from .decision_evidence import _rederive_headline, list_decision_evidence
+
+
+def _stale_rank_filter() -> Any:
+ """SQL matching evidence rows whose stored rank disagrees with the ladder.
+
+ Derived from ``SOURCE_RANK`` rather than hardcoded, so a future ladder edit
+ is picked up here automatically instead of needing a second place updated.
+ Bounded by the size of the ladder (a dozen terms), and it keeps the common
+ case — a store already on the current ladder — to an indexed match on
+ ``source`` that returns nothing, rather than hydrating the whole table.
+ """
+ return or_(
+ *[
+ (DecisionEvidence.source == name) & (DecisionEvidence.source_rank != rank)
+ for name, rank in SOURCE_RANK.items()
+ ]
+ )
+
+
+async def reconcile_source_ranks(session: AsyncSession) -> int:
+ """Re-stamp evidence rows whose stored rank predates a ``SOURCE_RANK`` edit.
+
+ ``DecisionEvidence.source_rank`` is written once at insert time, and two
+ ``ORDER BY`` clauses read it straight from the column, so it cannot simply be
+ derived on read. That makes the ladder a value copied into every row: editing
+ ``SOURCE_RANK`` leaves existing rows on the previous ladder, and because
+ headline confidence comes from ``max(source_rank)`` across a decision's
+ evidence, a store would end up scoring headlines off a mixture of two ladders
+ without anything looking wrong.
+
+ **This is the only repair path, deliberately.** An Alembic data migration was
+ written first and removed: hosted runs the same persist pipeline as a local
+ store, so the migration was redundant, and worse, it fixed the ranks *without*
+ re-deriving confidence — which made this function's own no-op check pass on
+ the next run and left hosted confidences on the old ladder permanently. One
+ path cannot disagree with itself.
+
+ Not repo-scoped, on purpose: the ladder is global, so in a workspace store
+ holding several repositories every one of them is on the same ladder and all
+ of them need the same repair.
+
+ Idempotent. Returns the number of evidence rows re-stamped (0 when already
+ reconciled, which is the steady state after the first run).
+ """
+ moved = (
+ (await session.execute(select(DecisionEvidence).where(_stale_rank_filter())))
+ .scalars()
+ .all()
+ )
+ if not moved:
+ return 0
+
+ for row in moved:
+ row.source_rank = rank_for_source(row.source)
+ await session.flush()
+
+ now = _now_utc()
+ for decision_id in {row.decision_id for row in moved}:
+ rec = await session.get(DecisionRecord, decision_id)
+ if rec is None:
+ continue
+ _rederive_headline(rec, await list_decision_evidence(session, decision_id))
+ rec.updated_at = now
+
+ # Flush the re-scored headlines too, not just the ranks. Without this the
+ # function returns with confidence still pending in the session, so whether
+ # the repair survives depends on what the caller does next.
+ await session.flush()
+
+ structlog.get_logger(__name__).info(
+ "decisions.source_ranks_reconciled", evidence_rows=len(moved)
+ )
+ return len(moved)
+
+
+async def reconcile_decision_confidence(session: AsyncSession) -> int:
+ """Re-score headlines whose stored confidence predates a formula edit.
+
+ The sibling of :func:`reconcile_source_ranks`, for the other half of the
+ same problem. That one repairs a stale *input*, a rank copied into a row
+ before the ladder moved, so it can find its work with an indexed
+ filter. A change to the formula itself leaves every input valid and every
+ stored score wrong, and nothing in a row marks which formula produced it,
+ so the only way to find the work is to recompute and compare.
+
+ Ceiling: two selects that hydrate the whole decision table, which is
+ small; a store large enough for that to hurt would want the comparison
+ pushed into SQL.
+
+ Not repo-scoped, for the reason the ladder is not: the formula is global.
+ Idempotent: 0 once reconciled, which is the steady state. Returns the
+ number of records re-scored, and re-derives verification alongside it.
+
+ A record with no evidence rows is left alone, deliberately: this pass
+ re-derives, it does not invent. The rows in that state are the ones no
+ extractor wrote, manual entries and manifest imports, and both score
+ themselves where they are created.
+ """
+ evidence_by_id: dict[str, list[DecisionEvidence]] = {}
+ for row in (await session.execute(select(DecisionEvidence))).scalars().all():
+ evidence_by_id.setdefault(row.decision_id, []).append(row)
+ if not evidence_by_id:
+ return 0
+
+ now = _now_utc()
+ rescored = 0
+ for rec in (await session.execute(select(DecisionRecord))).scalars().all():
+ evidence = evidence_by_id.get(rec.id)
+ if not evidence:
+ continue # see the docstring: re-derive, never invent
+ before = (rec.confidence, rec.verification)
+ _rederive_headline(rec, evidence)
+ if (rec.confidence, rec.verification) != before:
+ rec.updated_at = now
+ rescored += 1
+
+ if rescored:
+ await session.flush()
+ structlog.get_logger(__name__).info(
+ "decisions.confidence_reconciled", records=rescored
+ )
+ return rescored
+
+
+#: Prefix ``detect_supersessions_and_conflicts`` stamps on every edge it writes
+#: (``auto-detected: <signal> (sim=0.81)``). It is the only marker that
+#: separates a machine retirement from a human one, so the repair below keys on
+#: it rather than on "has a supersedes edge".
+_AUTO_EDGE_EVIDENCE_PREFIX = "auto-detected:"
+
+
+async def unretire_auto_superseded(session: AsyncSession) -> int:
+ """Undo retirements made by the semantic supersession detector (3B).
+
+ That detector scoped a conflict by cosine similarity and matched unrelated
+ records; it is now off (``SEMANTIC_SUPERSESSION_ENABLED``). Turning it off
+ only stops the *next* bad retirement — the rows it already flipped stay
+ ``superseded``, which is a protected status, so re-extraction will never
+ walk one back and a store would go on reporting a quarter of its corpus as
+ retired-by-nothing. This is the other half of the same change.
+
+ A row is repaired only when all three hold: status ``superseded``,
+ ``superseded_by`` set, and an ``auto-detected:`` supersedes edge from that
+ same successor. ``run_update_evolution`` sets the status without the
+ pairing, the CLI's ``decision deprecate`` writes ``deprecated``, and
+ ``upsert_decision_edge`` has no caller outside 3B — so nothing else in the
+ codebase produces all three.
+
+ One case is knowingly in range: a human can set both fields through
+ ``PATCH /api/repos/{id}/decisions/{id}``, and if they did so by accepting
+ one of this detector's own proposals in the UI, the auto edge is still
+ there and the row is restored to ``proposed``. Accepted rather than
+ guarded: the retirement they confirmed rests on the same bad match as the
+ 74, ``proposed`` keeps the record readable, and ``decision confirm`` or the
+ same PATCH puts it back in one step. The inverse — leaving a wrongly
+ retired record hidden because a human once clicked through — is not
+ recoverable at all.
+
+ Restored to ``proposed``, not ``active``: the flip overwrote the previous
+ status without recording it, and 3B fired on both. ``proposed`` is the
+ lower claim of the two and is recoverable by ``decision confirm``; guessing
+ ``active`` would mint governance a human never granted.
+
+ The edges go too — both kinds it wrote. They are the same artifact from the
+ same detector: ``supersedes`` is what ``build_lineage_chain`` walks, so
+ leaving those would hand ``get_why`` a lineage that still presents the
+ un-retired record as replaced, and ``conflicts_with`` is what the health
+ dashboard counts as a governance smell.
+
+ Not repo-scoped, like ``reconcile_source_ranks``: a workspace store holds
+ several repositories and the detector ran over all of them.
+
+ Idempotent — once repaired the edges are gone, so the next scan matches
+ nothing. Returns the number of records restored.
+ """
+ auto_edges = (
+ (
+ await session.execute(
+ select(DecisionEdge).where(
+ DecisionEdge.kind.in_(("supersedes", "conflicts_with")),
+ DecisionEdge.evidence.startswith(_AUTO_EDGE_EVIDENCE_PREFIX),
+ )
+ )
+ )
+ .scalars()
+ .all()
+ )
+ if not auto_edges:
+ return 0
+
+ successors_by_target: dict[str, set[str]] = {}
+ for edge in auto_edges:
+ if edge.kind == "supersedes":
+ successors_by_target.setdefault(edge.dst_decision_id, set()).add(edge.src_decision_id)
+
+ now = _now_utc()
+ restored = 0
+ for target_id, successor_ids in successors_by_target.items():
+ rec = await session.get(DecisionRecord, target_id)
+ if rec is None or rec.status != "superseded" or rec.superseded_by is None:
+ continue
+ if rec.superseded_by not in successor_ids:
+ # Retired by something else. Leave the *status* alone — the edge
+ # still goes, below, because it is this detector's noise either way.
+ continue
+ rec.status = "proposed"
+ rec.superseded_by = None
+ rec.updated_at = now
+ restored += 1
+
+ for edge in auto_edges:
+ await session.delete(edge)
+ await session.flush()
+
+ structlog.get_logger(__name__).info(
+ "decisions.auto_supersessions_reverted",
+ records_restored=restored,
+ edges_deleted=len(auto_edges),
+ )
+ return restored
+
+
+async def purge_proposed_decisions_by_source(
+ session: AsyncSession,
+ repository_id: str,
+ source: str,
+) -> int:
+ """Delete still-``proposed`` records of *source* plus their child rows.
+
+ One-shot cleanup for retired extraction sources (today: the removed
+ ``code_comment`` harvest, #751). Only ``proposed`` rows go — anything the
+ user confirmed, deprecated, or dismissed is kept. Child rows are deleted
+ explicitly rather than trusting the FK cascade, which on SQLite depends on
+ the ``foreign_keys`` pragma. Returns the number of records deleted.
+ """
+ result = await session.execute(
+ select(DecisionRecord.id).where(
+ DecisionRecord.repository_id == repository_id,
+ DecisionRecord.source == source,
+ DecisionRecord.status == "proposed",
+ ~accepted_predicate(),
+ )
+ )
+ ids = [row[0] for row in result.all()]
+ if not ids:
+ return 0
+
+ await _delete_decisions(session, ids)
+ structlog.get_logger(__name__).info("decision_purge_by_source", source=source, deleted=len(ids))
+ return len(ids)
+
+
+async def purge_proposed_decisions_outside_files(
+ session: AsyncSession,
+ repository_id: str,
+ current_file_paths: set[str],
+) -> int:
+ """Delete unreviewed extracted decisions whose evidence file left scope."""
+ result = await session.execute(
+ select(DecisionRecord.id, DecisionRecord.evidence_file).where(
+ DecisionRecord.repository_id == repository_id,
+ DecisionRecord.status == "proposed",
+ DecisionRecord.evidence_file.is_not(None),
+ ~accepted_predicate(),
+ )
+ )
+ ids = [row[0] for row in result.all() if row[1] not in current_file_paths]
+ if not ids:
+ return 0
+
+ await _delete_decisions(session, ids)
+ return len(ids)
+
+
+async def _delete_decisions(session: AsyncSession, ids: list[str]) -> None:
+ """Delete decision records and their child rows.
+
+ Child rows are deleted explicitly rather than trusting the FK cascade,
+ which on SQLite depends on the ``foreign_keys`` pragma.
+ """
+ await session.execute(delete(DecisionEvidence).where(DecisionEvidence.decision_id.in_(ids)))
+ await session.execute(
+ delete(DecisionEdge).where(
+ or_(
+ DecisionEdge.src_decision_id.in_(ids),
+ DecisionEdge.dst_decision_id.in_(ids),
+ )
+ )
+ )
+ await session.execute(delete(DecisionNodeLink).where(DecisionNodeLink.decision_id.in_(ids)))
+ await session.execute(delete(DecisionRecord).where(DecisionRecord.id.in_(ids)))
+ await session.flush()
diff --git a/packages/core/src/repowise/core/persistence/crud/decision_review_meta.py b/packages/core/src/repowise/core/persistence/crud/decision_review_meta.py
new file mode 100644
index 000000000..85d7318e6
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/crud/decision_review_meta.py
@@ -0,0 +1,77 @@
+"""The review-queue row every open decision candidate is owed."""
+
+from __future__ import annotations
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from repowise.core import __version__
+
+from ..models import DecisionCandidateMeta, DecisionRecord
+from .authority import candidate_predicate, candidate_review_signals, upsert_candidate_meta
+
+
+async def _write_candidate_meta(
+ session: AsyncSession,
+ repository_id: str,
+ captured: dict[str, tuple[str, bool]],
+ *,
+ only: set[str] | None = None,
+) -> None:
+ """Refresh the review row every open candidate in the repository is owed.
+
+ Not only the ones this run touched: the staging store does not re-emit an
+ already-promoted decision, so a backlog would otherwise stay unjudged until
+ something happened to re-extract each record individually. The signals come
+ from the record, so an unchanged candidate costs a comparison and no write.
+ """
+ rows = (
+ await session.execute(
+ select(DecisionRecord, DecisionCandidateMeta)
+ .outerjoin(
+ DecisionCandidateMeta,
+ DecisionCandidateMeta.decision_id == DecisionRecord.id,
+ )
+ .where(
+ DecisionRecord.repository_id == repository_id,
+ candidate_predicate(),
+ *([DecisionRecord.id.in_(only)] if only is not None else []),
+ )
+ )
+ ).all()
+
+ for rec, meta in rows:
+ priority, scope_unresolved = candidate_review_signals(rec)
+ capture = captured.get(rec.id)
+ if capture is None:
+ # Not extracted this run, so the provenance is not ours to write:
+ # only the contract-derived signals are refreshed, and a row with
+ # no lane yet takes the record's own source.
+ if (
+ meta is not None
+ and meta.lane
+ and meta.review_priority == priority
+ and meta.scope_unresolved == scope_unresolved
+ ):
+ continue
+ await upsert_candidate_meta(
+ session,
+ rec,
+ lane=meta.lane if meta is not None and meta.lane else rec.source,
+ review_priority=priority,
+ scope_unresolved=scope_unresolved,
+ )
+ continue
+
+ lane, needs_split = capture
+ await upsert_candidate_meta(
+ session,
+ rec,
+ lane=lane or rec.source,
+ extractor_version=__version__,
+ review_priority=priority,
+ # Raised, never cleared: a re-extraction must not walk back a split
+ # somebody asked for.
+ needs_split=True if needs_split else None,
+ scope_unresolved=scope_unresolved,
+ )
diff --git a/packages/core/src/repowise/core/persistence/crud/decision_staleness.py b/packages/core/src/repowise/core/persistence/crud/decision_staleness.py
new file mode 100644
index 000000000..fb6938052
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/crud/decision_staleness.py
@@ -0,0 +1,213 @@
+"""Staleness scoring for decisions, from the git history of the files they name."""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..decision_graph import sync_links_from_record
+from ..models import DecisionRecord, GitMetadata, _now_utc
+
+
+async def _fill_git_meta_gaps(
+ session: AsyncSession,
+ repository_id: str,
+ git_meta_map: dict[str, dict],
+ wanted: set[str],
+) -> dict[str, dict]:
+ """``git_meta_map`` widened with persisted rows for the *wanted* paths.
+
+ The caller's map is one run's git metadata, and on an incremental update
+ that is only the files that changed in it. Staleness scoring reads a path
+ with no entry as a file that is gone and scores it 1.00, so every decision
+ over an untouched file went maximally stale for not having been touched.
+ The persisted rows cover every file the index has seen, which is the set
+ that answers "is this file still here"; a path missing from those too is
+ genuinely untracked and still scores 1.00. The run's own entries win —
+ they are this run's fresher numbers for the files it re-read.
+ """
+ missing = wanted - git_meta_map.keys()
+ if not missing:
+ return git_meta_map
+
+ filled: dict[str, dict] = {}
+ # Chunked so the IN clause stays under SQLite's bind-parameter ceiling.
+ batch = sorted(missing)
+ for start in range(0, len(batch), 500):
+ rows = await session.execute(
+ select(GitMetadata).where(
+ GitMetadata.repository_id == repository_id,
+ GitMetadata.file_path.in_(batch[start : start + 500]),
+ )
+ )
+ for row in rows.scalars().all():
+ filled[row.file_path] = {
+ col.name: getattr(row, col.name)
+ for col in row.__table__.columns
+ if col.name not in ("id", "repository_id")
+ }
+ filled.update(git_meta_map)
+ return filled
+
+
+async def recompute_decision_staleness(
+ session: AsyncSession,
+ repository_id: str,
+ git_meta_map: dict[str, dict],
+) -> int:
+ """Recompute staleness_score for all active decisions. Returns update count.
+
+ Also fills ``last_code_change`` from the same per-file dates the score is
+ counted from, and re-derives ``affected_modules_json`` from the files each
+ record names.
+ The two belong in one pass because they are one repair: a record's module
+ linkage used to be the first path segment, which in a ``packages/`` layout
+ made almost every record claim ``packages`` or ``tests``, and the rows that
+ predate the fix carry it. Deriving here rather than in a data migration
+ keeps the single-repair-path rule — the one this repo already learned when
+ an alembic migration and a runtime repair disagreed about confidence.
+ """
+ result = await session.execute(
+ select(DecisionRecord).where(
+ DecisionRecord.repository_id == repository_id,
+ DecisionRecord.status.in_(["active", "proposed"]),
+ )
+ )
+ decisions = list(result.scalars().all())
+
+ affected_by_id: dict[str, list[str]] = {}
+ for dec in decisions:
+ affected = json.loads(dec.affected_files_json)
+ if affected:
+ affected_by_id[dec.id] = affected
+
+ remodelled = _backfill_module_nodes(decisions, affected_by_id)
+ modules_updated = len(remodelled)
+ # This repair runs after ``backfill_decision_node_links`` on both index
+ # paths, so a module array it rewrites here would otherwise keep the link
+ # the backfill had already judged correct.
+ for dec in remodelled:
+ await sync_links_from_record(session, dec)
+ if not affected_by_id:
+ if modules_updated:
+ await session.flush()
+ return 0
+
+ git_meta_map = await _fill_git_meta_gaps(
+ session,
+ repository_id,
+ git_meta_map,
+ {fp for paths in affected_by_id.values() for fp in paths},
+ )
+
+ now = _now_utc()
+ updated = 0
+ dated = 0
+ for dec in decisions:
+ affected = affected_by_id.get(dec.id)
+ if not affected:
+ continue
+
+ # Before the conventions skip below: a date is a fact about the code,
+ # not a score, so it is filled for every scoped record.
+ if _refresh_last_code_change(dec, affected, git_meta_map, now):
+ dated += 1
+
+ # The source writes its own conformance share, and a git-diff score
+ # would overwrite it. Only the score is skipped; the repairs above
+ # still apply.
+ if dec.source == "conventions":
+ continue
+
+ if _rescore_staleness(dec, affected, git_meta_map, now):
+ updated += 1
+
+ if updated or modules_updated or dated:
+ await session.flush()
+ # Deliberately the staleness count alone. The callers print this as
+ # "N decisions rescored"; folding the module repair or the date fill into
+ # it would report a rescore that did not happen.
+ return updated
+
+
+def _refresh_last_code_change(
+ dec: DecisionRecord, affected: list[str], git_meta_map: dict[str, dict], now: datetime
+) -> bool:
+ """Fill ``last_code_change`` from the files' git dates; True when it moved."""
+ # Function-local, like every other analysis import here: the persistence
+ # layer cannot depend on analysis at module scope.
+ from repowise.core.analysis.decisions.extractor import DecisionExtractor, _as_aware_utc
+
+ last_change = DecisionExtractor.last_code_change(affected, git_meta_map)
+ stored = dec.last_code_change
+ # SQLite drops tzinfo, so an aware value would differ from the naive
+ # one it just wrote and rewrite ``updated_at`` on every run.
+ if last_change == (_as_aware_utc(stored) if stored else None):
+ return False
+ dec.last_code_change = last_change
+ dec.updated_at = now
+ return True
+
+
+def _rescore_staleness(
+ dec: DecisionRecord, affected: list[str], git_meta_map: dict[str, dict], now: datetime
+) -> bool:
+ """Recompute the git-diff staleness score; True when it moved by more than 0.01."""
+ from repowise.core.analysis.decisions.extractor import DecisionExtractor
+
+ new_score = DecisionExtractor.compute_staleness(
+ dec.created_at,
+ affected,
+ git_meta_map,
+ )
+ if abs(new_score - dec.staleness_score) > 0.01:
+ dec.staleness_score = round(new_score, 3)
+ dec.updated_at = now
+ return True
+ return False
+
+
+def _backfill_module_nodes(
+ decisions: list[DecisionRecord],
+ affected_by_id: dict[str, list[str]],
+) -> list[DecisionRecord]:
+ """Re-derive each record's module linkage from its files.
+
+ Returns the records whose array moved, because the caller has to relink
+ each one: this rewrites half a record's scope and the graph holds the
+ other half.
+
+ Records naming no file are left alone: there is nothing to derive from, and
+ an invented scope is worse than an absent one.
+ """
+ from repowise.core.analysis.decisions.scope import resolve_module_nodes
+
+ moved = []
+ for dec in decisions:
+ affected = affected_by_id.get(dec.id)
+ if not affected:
+ continue
+ derived = resolve_module_nodes(affected)
+ if derived != json.loads(dec.affected_modules_json or "[]"):
+ dec.affected_modules_json = json.dumps(derived)
+ moved.append(dec)
+ return moved
+
+
+async def get_stale_decisions(
+ session: AsyncSession,
+ repository_id: str,
+ threshold: float = 0.5,
+) -> list[DecisionRecord]:
+ """Return active decisions with staleness_score >= threshold."""
+ result = await session.execute(
+ select(DecisionRecord).where(
+ DecisionRecord.repository_id == repository_id,
+ DecisionRecord.status.in_(["active"]),
+ DecisionRecord.staleness_score >= threshold,
+ )
+ )
+ return list(result.scalars().all())
diff --git a/packages/core/src/repowise/core/persistence/crud/decisions.py b/packages/core/src/repowise/core/persistence/crud/decisions.py
index d08fffb44..62edca3d5 100644
--- a/packages/core/src/repowise/core/persistence/crud/decisions.py
+++ b/packages/core/src/repowise/core/persistence/crud/decisions.py
@@ -1,126 +1,61 @@
"""CRUD operations for the decisions domain (repowise persistence layer).
Split out of the former monolithic ``crud.py``; ``crud/__init__.py`` re-exports
-every public name, so existing imports are unaffected.
+every public name, so existing imports are unaffected. Single-record reads and
+writes live here; the ``decision_*`` siblings hold the rest and are re-exported.
"""
from __future__ import annotations
-import hashlib
import json
-from collections.abc import Iterable
from datetime import datetime
from typing import Any
-import structlog
-from sqlalchemy import case, delete, func, or_, select
+from sqlalchemy import case, func, select
from sqlalchemy.ext.asyncio import AsyncSession
-from repowise.core import __version__
-from repowise.core.analysis.decisions.lifecycle import (
- ARCHITECTURAL_KIND,
- DECISION_KINDS,
- DECISION_STATUS_ORDER,
- RETIRED_STATUSES,
- status_rank,
-)
-from repowise.core.analysis.decisions.provenance import (
- SOURCE_RANK,
- completeness,
- compute_confidence,
- normalize_text,
- rank_for_source,
-)
-from repowise.core.analysis.decisions.scope import SCOPE_BASIS_STATED, binds_to_paths
+from repowise.core.analysis.decisions.lifecycle import DECISION_STATUS_ORDER, RETIRED_STATUSES
+from repowise.core.analysis.decisions.provenance import compute_confidence, rank_for_source
+from repowise.core.analysis.decisions.scope import SCOPE_BASIS_STATED
from ..decision_graph import scope_modules, sync_links_from_record
-from ..models import (
- DecisionCandidateMeta,
- DecisionEdge,
- DecisionEvidence,
- DecisionNodeLink,
- DecisionRecord,
- GitMetadata,
- _now_utc,
+from ..models import DecisionCandidateMeta, DecisionRecord, _now_utc
+from .authority import accepted_predicate
+
+# Re-exported: these moved to sibling modules and are still imported from here.
+from .decision_evidence import ( # noqa: F401
+ _json_list,
+ _rederive_headline,
+ list_decision_evidence,
+ record_completeness,
)
-from .authority import (
- accepted_predicate,
- candidate_predicate,
- candidate_review_signals,
- upsert_candidate_meta,
+from .decision_health import get_decision_health_summary # noqa: F401
+from .decision_identity import ( # noqa: F401
+ _extraction_kind,
+ _extraction_status,
+ _merge_status,
+ derive_decision_id,
+ identity_quote_for,
)
+from .decision_ingest import bulk_upsert_decisions # noqa: F401
+from .decision_repairs import ( # noqa: F401
+ purge_proposed_decisions_by_source,
+ purge_proposed_decisions_outside_files,
+ reconcile_decision_confidence,
+ reconcile_source_ranks,
+ unretire_auto_superseded,
+)
+from .decision_review_meta import _write_candidate_meta
+from .decision_staleness import get_stale_decisions, recompute_decision_staleness # noqa: F401
# ---------------------------------------------------------------------------
# DecisionRecord CRUD
# ---------------------------------------------------------------------------
-# Namespaced so a decision id can never collide with another kind of derived
-# id, and versioned so a future change of the recipe is a visible new value
-# rather than a silent reshuffle of every id in every store.
-_ID_NAMESPACE = "repowise.decision.id.v2"
-
-# The identity fields are joined with a separator none of them can contain,
-# so no pair of distinct records can flatten to the same string.
-_FIELD_SEP = "\x00"
-
-# ``evidence_file`` is nullable and NULL is not the empty string here: the
-# dedupe query treats them as different records, so the derivation must too.
-_NULL_EVIDENCE_FILE = "\x01"
-
-# The pinned quote is truncated before it is hashed. A quote long enough to
-# reach this is a paragraph, and its opening already separates two decisions
-# that share a file set; carrying the rest only widens the surface on which a
-# re-mine can reword the identity out from under a reference somebody wrote.
-_IDENTITY_QUOTE_CHARS = 200
-
_VALID_DECISION_STATUSES = frozenset(
{"proposed", "active", "deprecated", "superseded", "dismissed"}
)
-# Statuses a human (or the evolution judge) set deliberately. Re-extraction
-# must never walk one of these back to ``proposed``: a re-harvested source
-# ties or beats the stored rank, so without this guard a reindex would flip
-# confirmed decisions back into the review queue and resurrect dismissed ones
-# (#751). ``dismissed`` is terminal: the row is kept purely as a tombstone so
-# the same content never re-proposes.
-_PROTECTED_STATUSES = frozenset({"active", "deprecated", "superseded", "dismissed"})
-
-
-def _extraction_status(incoming: str) -> str:
- """The status extraction is allowed to write.
-
- Extraction proposes; acceptance promotes. An incoming ``active`` is a
- parsed ADR heading or an old code path's optimism, and neither is an
- acceptance event, so it lands as a proposal. A tracked ADR still reaches
- ``active`` — via :func:`_accept_from_tracked_artifact` below, which records
- the document as the accepter instead of leaving the promotion anonymous.
- """
- return "proposed" if incoming == "active" else incoming
-
-
-def _merge_status(existing: str, incoming: str) -> str:
- """Resolve a re-extracted status against the stored one.
-
- A protected stored status wins over an incoming ``proposed``; anything else
- follows the incoming value, after :func:`_extraction_status` has taken
- authority out of extraction's hands.
- """
- incoming = _extraction_status(incoming)
- if incoming == "proposed" and existing in _PROTECTED_STATUSES:
- return existing
- return incoming
-
-
-def _extraction_kind(incoming: str | None) -> str:
- """The kind extraction is allowed to write.
-
- Unrecognised falls to ``architectural`` rather than being stored: the
- vocabulary has no CHECK behind it, every reader tests ``== AGREEMENT_KIND``,
- and a junk value that merely happens to read as architectural everywhere is
- safe by luck. Normalising here makes it safe by construction.
- """
- return incoming if incoming in DECISION_KINDS else ARCHITECTURAL_KIND
-
def _dedup_query(
repository_id: str,
@@ -145,95 +80,6 @@ def _dedup_query(
return q.where(DecisionRecord.evidence_file.is_(None))
-def identity_quote_for(text: str | None) -> str:
- """The pinned form of a decision's quote, for identity only.
-
- Whitespace-collapsed, lowercased and truncated, so the identity survives a
- re-mine that reflows or recases the same sentence. The stored column keeps
- the verbatim text; only this derived form is hashed.
- """
- return normalize_text(text)[:_IDENTITY_QUOTE_CHARS]
-
-
-def derive_decision_id(
- repository_id: str,
- title: str,
- *,
- source: str,
- evidence_file: str | None,
- affected_files: Iterable[str] = (),
- evidence_line: int | None = None,
- identity_quote: str = "",
- needs_split: bool = False,
-) -> str:
- """The id a decision with this identity has, in every store, on every run.
-
- Keyed on the evidence rather than on the title. A title is the most
- volatile thing a decision carries: two extractions of the same choice word
- it differently and produce two records, and rewording one moves its id.
- Over the dogfood store, title identity collapses **zero** duplicate pairs
- while the files-plus-evidence-plus-quote key below collapses **120 pairs
- across 42 classes**, and loses none of what title identity caught.
-
- The quote is load-bearing, and is why the file set alone is not the key:
- two unrelated classes in that store share a file pair and are separated
- only by what was said. It is also the fragile part, because most records
- key on mined prose rather than on a committed span, so it is **pinned in
- the row at first capture and never re-derived**. A later extraction that
- rewords the same sentence then leaves the id where it is.
-
- A record with no evidence at all keeps its title, for the same reason the
- rest of them lose it: identity is the evidence, and a record that has
- none has no evidence identity to key on. Without this every scopeless,
- quoteless record in a repository derives one id and the second one to be
- written collides with the first. Two of the 357 records in the dogfood
- store are in that state, and the construction paths that supply nothing
- but a title are all in that state.
-
- A record flagged ``needs_split`` keeps its title in the key. A bundled
- claim shares its files and its evidence with the separate decisions it
- bundles, so evidence identity would fold all of them together and file two
- decisions under a third one's name. Holding the flagged record on title
- identity is what keeps it apart, and it is why the flag has to be set
- before this key ships. The guard is partial: it reaches the bundles a
- marker can see and no others, measured at 28 of 357 records held out for
- 20 of 120 merges given up.
-
- ``source`` is deliberately **not** in the key. Two lanes that mined the
- same sentence out of the same files recorded one decision, not two, and
- the quote already separates two that merely share a scope.
-
- 32 lowercase hex, because ``DecisionRecord.id`` is ``String(32)`` and so
- is every foreign key to it, so a truncated digest fits without a column
- change. ``evidence_file`` is NULL far more often than not, so it gets a
- sentinel no path can contain rather than collapsing into the empty string.
-
- Note the id follows the identity, so editing the scope or the pinned quote
- moves it. :mod:`decision_id_migration`, which runs at the head of every
- index, is what settles that, and it leaves an alias behind so an id
- already written down keeps resolving.
- """
- files = sorted({str(f) for f in affected_files if f})
- quote = identity_quote_for(identity_quote)
- grounded = bool(files) or evidence_file is not None or bool(quote)
- parts = (
- _ID_NAMESPACE,
- repository_id,
- json.dumps(files),
- json.dumps(
- [
- _NULL_EVIDENCE_FILE if evidence_file is None else evidence_file,
- evidence_line,
- ]
- ),
- quote,
- title if needs_split or not grounded else "",
- source if not grounded else "",
- )
- digest = hashlib.sha256(_FIELD_SEP.join(parts).encode("utf-8")).hexdigest()
- return digest[:32]
-
-
async def find_decision_by_title(
session: AsyncSession,
repository_id: str,
@@ -744,1288 +590,3 @@ async def delete_decision(session: AsyncSession, decision_id: str) -> bool:
await session.delete(rec)
await session.flush()
return True
-
-
-def _normalize_title(title: str) -> str:
- """Normalize a decision title for cross-source dedup comparison."""
- import re as _re
-
- t = title.lower().strip()
- t = _re.sub(r"[^a-z0-9\s]", "", t)
- t = _re.sub(r"\s+", " ", t)
- return t
-
-
-def _evidence_quote(d: dict) -> str:
- """Pick the verbatim span recorded as this evidence row's source quote.
-
- Prefers the LLM/parser-supplied ``source_quote``; falls back to the
- decision/rationale text so an evidence row is never empty.
- """
- return (d.get("source_quote") or d.get("decision") or d.get("rationale") or "").strip()
-
-
-async def _upsert_decision_evidence(
- session: AsyncSession,
- decision_id: str,
- *,
- source: str,
- source_rank: int,
- evidence_file: str | None,
- evidence_line: int | None,
- evidence_commit: str | None,
- source_quote: str,
- confidence: float,
- verification: str,
-) -> None:
- """Insert or update one evidence row, idempotent on its natural key.
-
- Natural key is ``(decision_id, source, evidence_file, evidence_commit)``.
- NULLs are matched explicitly (SQLite treats NULLs as distinct in a unique
- constraint), so re-indexing the same source converges instead of
- duplicating, while an incremental update adds genuinely new evidence.
- """
- q = select(DecisionEvidence).where(
- DecisionEvidence.decision_id == decision_id,
- DecisionEvidence.source == source,
- )
- q = (
- q.where(DecisionEvidence.evidence_file == evidence_file)
- if evidence_file is not None
- else q.where(DecisionEvidence.evidence_file.is_(None))
- )
- q = (
- q.where(DecisionEvidence.evidence_commit == evidence_commit)
- if evidence_commit is not None
- else q.where(DecisionEvidence.evidence_commit.is_(None))
- )
- existing = (await session.execute(q)).scalar_one_or_none()
-
- if existing is not None:
- existing.source_rank = source_rank
- existing.evidence_line = evidence_line
- existing.source_quote = source_quote
- existing.confidence = confidence
- existing.verification = verification
- return
-
- session.add(
- DecisionEvidence(
- decision_id=decision_id,
- source=source,
- source_rank=source_rank,
- evidence_file=evidence_file,
- evidence_line=evidence_line,
- evidence_commit=evidence_commit,
- source_quote=source_quote,
- confidence=confidence,
- verification=verification,
- )
- )
-
-
-def _json_list(raw: str | None) -> list[str]:
- """A JSON array column as a list; anything unparsable reads as empty."""
- try:
- value = json.loads(raw or "[]")
- except (TypeError, ValueError):
- return []
- return [str(v) for v in value] if isinstance(value, list) else []
-
-
-def _best_verification(values: list[str]) -> str:
- """Reduce per-evidence verdicts to the strongest: exact > fuzzy > unverified."""
- if "exact" in values:
- return "exact"
- if "fuzzy" in values:
- return "fuzzy"
- return "unverified"
-
-
-def record_completeness(rec: DecisionRecord) -> int:
- """:func:`completeness` for a stored row, parsing its two JSON columns."""
- return completeness(
- decision=rec.decision,
- rationale=rec.rationale,
- context=rec.context,
- consequences=_json_list(rec.consequences_json),
- alternatives=_json_list(rec.alternatives_json),
- )
-
-
-def _rederive_headline(rec: DecisionRecord, evidence: list[DecisionEvidence]) -> None:
- """Set a record's confidence + verification from its full evidence set.
-
- The single definition of how a headline is scored. Every writer uses it:
- the upsert path after accreting a run's evidence, ``reconcile_source_ranks``
- after a ladder edit, and ``reconcile_decision_confidence`` after a formula
- edit. Kept as one function because two of them were briefly copy-pasted and
- nothing would have forced the copies to stay equal.
-
- Confidence rises with the best source rank and with the number of
- *independent* corroborating sources, and with how much of its body the
- record fills, so it is derived from the whole set plus the row rather than
- from whichever evidence row happened to arrive last. No-op on empty
- evidence: a record with nothing behind it keeps whatever it had.
- """
- if not evidence:
- return
- best_ver = _best_verification([e.verification for e in evidence])
- rec.confidence = compute_confidence(
- max(e.source_rank for e in evidence),
- len({e.source for e in evidence}),
- best_ver,
- filled_fields=record_completeness(rec),
- )
- rec.verification = best_ver
-
-
-def _stale_rank_filter() -> Any:
- """SQL matching evidence rows whose stored rank disagrees with the ladder.
-
- Derived from ``SOURCE_RANK`` rather than hardcoded, so a future ladder edit
- is picked up here automatically instead of needing a second place updated.
- Bounded by the size of the ladder (a dozen terms), and it keeps the common
- case — a store already on the current ladder — to an indexed match on
- ``source`` that returns nothing, rather than hydrating the whole table.
- """
- return or_(
- *[
- (DecisionEvidence.source == name) & (DecisionEvidence.source_rank != rank)
- for name, rank in SOURCE_RANK.items()
- ]
- )
-
-
-async def reconcile_source_ranks(session: AsyncSession) -> int:
- """Re-stamp evidence rows whose stored rank predates a ``SOURCE_RANK`` edit.
-
- ``DecisionEvidence.source_rank`` is written once at insert time, and two
- ``ORDER BY`` clauses read it straight from the column, so it cannot simply be
- derived on read. That makes the ladder a value copied into every row: editing
- ``SOURCE_RANK`` leaves existing rows on the previous ladder, and because
- headline confidence comes from ``max(source_rank)`` across a decision's
- evidence, a store would end up scoring headlines off a mixture of two ladders
- without anything looking wrong.
-
- **This is the only repair path, deliberately.** An Alembic data migration was
- written first and removed: hosted runs the same persist pipeline as a local
- store, so the migration was redundant, and worse, it fixed the ranks *without*
- re-deriving confidence — which made this function's own no-op check pass on
- the next run and left hosted confidences on the old ladder permanently. One
- path cannot disagree with itself.
-
- Not repo-scoped, on purpose: the ladder is global, so in a workspace store
- holding several repositories every one of them is on the same ladder and all
- of them need the same repair.
-
- Idempotent. Returns the number of evidence rows re-stamped (0 when already
- reconciled, which is the steady state after the first run).
- """
- moved = (
- (await session.execute(select(DecisionEvidence).where(_stale_rank_filter())))
- .scalars()
- .all()
- )
- if not moved:
- return 0
-
- for row in moved:
- row.source_rank = rank_for_source(row.source)
- await session.flush()
-
- now = _now_utc()
- for decision_id in {row.decision_id for row in moved}:
- rec = await session.get(DecisionRecord, decision_id)
- if rec is None:
- continue
- _rederive_headline(rec, await list_decision_evidence(session, decision_id))
- rec.updated_at = now
-
- # Flush the re-scored headlines too, not just the ranks. Without this the
- # function returns with confidence still pending in the session, so whether
- # the repair survives depends on what the caller does next.
- await session.flush()
-
- structlog.get_logger(__name__).info(
- "decisions.source_ranks_reconciled", evidence_rows=len(moved)
- )
- return len(moved)
-
-
-async def reconcile_decision_confidence(session: AsyncSession) -> int:
- """Re-score headlines whose stored confidence predates a formula edit.
-
- The sibling of :func:`reconcile_source_ranks`, for the other half of the
- same problem. That one repairs a stale *input*, a rank copied into a row
- before the ladder moved, so it can find its work with an indexed
- filter. A change to the formula itself leaves every input valid and every
- stored score wrong, and nothing in a row marks which formula produced it,
- so the only way to find the work is to recompute and compare.
-
- Ceiling: two selects that hydrate the whole decision table, which is
- small; a store large enough for that to hurt would want the comparison
- pushed into SQL.
-
- Not repo-scoped, for the reason the ladder is not: the formula is global.
- Idempotent: 0 once reconciled, which is the steady state. Returns the
- number of records re-scored, and re-derives verification alongside it.
-
- A record with no evidence rows is left alone, deliberately: this pass
- re-derives, it does not invent. The rows in that state are the ones no
- extractor wrote, manual entries and manifest imports, and both score
- themselves where they are created.
- """
- evidence_by_id: dict[str, list[DecisionEvidence]] = {}
- for row in (await session.execute(select(DecisionEvidence))).scalars().all():
- evidence_by_id.setdefault(row.decision_id, []).append(row)
- if not evidence_by_id:
- return 0
-
- now = _now_utc()
- rescored = 0
- for rec in (await session.execute(select(DecisionRecord))).scalars().all():
- evidence = evidence_by_id.get(rec.id)
- if not evidence:
- continue # see the docstring: re-derive, never invent
- before = (rec.confidence, rec.verification)
- _rederive_headline(rec, evidence)
- if (rec.confidence, rec.verification) != before:
- rec.updated_at = now
- rescored += 1
-
- if rescored:
- await session.flush()
- structlog.get_logger(__name__).info(
- "decisions.confidence_reconciled", records=rescored
- )
- return rescored
-
-
-#: Prefix ``detect_supersessions_and_conflicts`` stamps on every edge it writes
-#: (``auto-detected: <signal> (sim=0.81)``). It is the only marker that
-#: separates a machine retirement from a human one, so the repair below keys on
-#: it rather than on "has a supersedes edge".
-_AUTO_EDGE_EVIDENCE_PREFIX = "auto-detected:"
-
-
-async def unretire_auto_superseded(session: AsyncSession) -> int:
- """Undo retirements made by the semantic supersession detector (3B).
-
- That detector scoped a conflict by cosine similarity and matched unrelated
- records; it is now off (``SEMANTIC_SUPERSESSION_ENABLED``). Turning it off
- only stops the *next* bad retirement — the rows it already flipped stay
- ``superseded``, which is a protected status, so re-extraction will never
- walk one back and a store would go on reporting a quarter of its corpus as
- retired-by-nothing. This is the other half of the same change.
-
- A row is repaired only when all three hold: status ``superseded``,
- ``superseded_by`` set, and an ``auto-detected:`` supersedes edge from that
- same successor. ``run_update_evolution`` sets the status without the
- pairing, the CLI's ``decision deprecate`` writes ``deprecated``, and
- ``upsert_decision_edge`` has no caller outside 3B — so nothing else in the
- codebase produces all three.
-
- One case is knowingly in range: a human can set both fields through
- ``PATCH /api/repos/{id}/decisions/{id}``, and if they did so by accepting
- one of this detector's own proposals in the UI, the auto edge is still
- there and the row is restored to ``proposed``. Accepted rather than
- guarded: the retirement they confirmed rests on the same bad match as the
- 74, ``proposed`` keeps the record readable, and ``decision confirm`` or the
- same PATCH puts it back in one step. The inverse — leaving a wrongly
- retired record hidden because a human once clicked through — is not
- recoverable at all.
-
- Restored to ``proposed``, not ``active``: the flip overwrote the previous
- status without recording it, and 3B fired on both. ``proposed`` is the
- lower claim of the two and is recoverable by ``decision confirm``; guessing
- ``active`` would mint governance a human never granted.
-
- The edges go too — both kinds it wrote. They are the same artifact from the
- same detector: ``supersedes`` is what ``build_lineage_chain`` walks, so
- leaving those would hand ``get_why`` a lineage that still presents the
- un-retired record as replaced, and ``conflicts_with`` is what the health
- dashboard counts as a governance smell.
-
- Not repo-scoped, like ``reconcile_source_ranks``: a workspace store holds
- several repositories and the detector ran over all of them.
-
- Idempotent — once repaired the edges are gone, so the next scan matches
- nothing. Returns the number of records restored.
- """
- auto_edges = (
- (
- await session.execute(
- select(DecisionEdge).where(
- DecisionEdge.kind.in_(("supersedes", "conflicts_with")),
- DecisionEdge.evidence.startswith(_AUTO_EDGE_EVIDENCE_PREFIX),
- )
- )
- )
- .scalars()
- .all()
- )
- if not auto_edges:
- return 0
-
- successors_by_target: dict[str, set[str]] = {}
- for edge in auto_edges:
- if edge.kind == "supersedes":
- successors_by_target.setdefault(edge.dst_decision_id, set()).add(edge.src_decision_id)
-
- now = _now_utc()
- restored = 0
- for target_id, successor_ids in successors_by_target.items():
- rec = await session.get(DecisionRecord, target_id)
- if rec is None or rec.status != "superseded" or rec.superseded_by is None:
- continue
- if rec.superseded_by not in successor_ids:
- # Retired by something else. Leave the *status* alone — the edge
- # still goes, below, because it is this detector's noise either way.
- continue
- rec.status = "proposed"
- rec.superseded_by = None
- rec.updated_at = now
- restored += 1
-
- for edge in auto_edges:
- await session.delete(edge)
- await session.flush()
-
- structlog.get_logger(__name__).info(
- "decisions.auto_supersessions_reverted",
- records_restored=restored,
- edges_deleted=len(auto_edges),
- )
- return restored
-
-
-async def list_decision_evidence(
- session: AsyncSession,
- decision_id: str,
-) -> list[DecisionEvidence]:
- """Return all evidence rows for a decision, highest source rank first."""
- result = await session.execute(
- select(DecisionEvidence)
- .where(DecisionEvidence.decision_id == decision_id)
- .order_by(DecisionEvidence.source_rank.desc(), DecisionEvidence.created_at.asc())
- )
- return list(result.scalars().all())
-
-
-def _first_commit(d: dict) -> str | None:
- commits = d.get("evidence_commits") or []
- return commits[0] if commits else None
-
-
-#: Sources whose evidence file is itself a reviewable, version-controlled
-#: statement of the decision. Only these can accept without a person present.
-_TRACKED_ARTIFACT_SOURCES: frozenset[str] = frozenset({"adr"})
-
-
-async def _accept_from_tracked_artifact(
- session: AsyncSession, rec: DecisionRecord, headline: dict
-) -> None:
- """Grant acceptance to an ADR the repository has committed as accepted.
-
- Silent on anything that is not one: a missing rationale or scope means the
- document did not say enough to bind future work, and a refusal there is
- correct rather than an error to report. Re-running is a no-op because the
- acceptance already exists.
- """
- if headline.get("status") != "active":
- return
- if headline.get("source") not in _TRACKED_ARTIFACT_SOURCES:
- return
- artifact = headline.get("evidence_file") or ""
- if not artifact:
- return
-
- from repowise.core.analysis.decisions.accepter import is_tracked
-
- from ..models import Repository
- from .authority import AcceptanceRefusedError, is_accepted, record_acceptance
-
- if await is_accepted(session, rec.id):
- return
- repo = await session.get(Repository, rec.repository_id)
- if repo is None or not is_tracked(repo.local_path, artifact):
- # An uncommitted document is one machine's file. Letting it accept would
- # replace the human acceptance event with a file an agent can write.
- return
- try:
- await record_acceptance(
- session,
- rec,
- action="accepted",
- currency="active",
- artifact=artifact,
- kind="import",
- note="accepted by a tracked decision record",
- )
- except AcceptanceRefusedError as exc:
- structlog.get_logger(__name__).debug(
- "adr_acceptance_refused", decision_id=rec.id, reason=str(exc)
- )
-
-
-async def bulk_upsert_decisions(
- session: AsyncSession,
- repository_id: str,
- decisions: list[dict],
- *,
- vector_store: Any | None = None,
-) -> list[str]:
- """Upsert decisions, accreting provenance instead of discarding losers.
-
- Decisions with near-identical normalized titles are merged into a single
- :class:`DecisionRecord`; every contributing source becomes a
- :class:`DecisionEvidence` row. The record's headline fields come from the
- highest-``source_rank`` contributor, and its confidence is recomputed from
- the best rank + the number of independently corroborating sources + the
- strongest surviving verification verdict.
-
- Idempotent: a full re-index converges (evidence rows upsert on their
- natural key); an incremental update adds new evidence and re-derives the
- headline + confidence from the union.
-
- When *vector_store* (the shared page-generator store) is supplied, a
- Phase-2C semantic pass augments the cheap normalized-title match: an
- incoming group that matched no existing record by title is looked up in the
- store, and if its nearest ``decision:`` neighbour clears the cosine
- threshold it is folded into that record as additional evidence — so
- paraphrases ("Use Redis" vs "Adopt Redis cache") collapse into one record.
- Every touched record is (re-)embedded into the store, so decisions are
- matchable next run *and* discoverable via ``search_codebase``.
-
- Returns the ids of every record touched (created or updated) this call, so
- a caller can run the Phase-3 supersession/conflict detection over just the
- records that changed.
- """
- # Group incoming decisions by normalized title.
- groups: dict[str, list[dict]] = {}
- for d in decisions:
- norm = _normalize_title(d.get("title", ""))
- if not norm:
- continue
- groups.setdefault(norm, []).append(d)
-
- if not groups:
- return []
-
- # Map existing records (this repo) by normalized title so cross-run merges
- # land on the same row. On a title collision keep the most authoritative
- # existing row as canonical.
- existing_rows = await session.execute(
- select(DecisionRecord).where(DecisionRecord.repository_id == repository_id)
- )
- existing_by_norm: dict[str, DecisionRecord] = {}
- for rec in existing_rows.scalars().all():
- norm = _normalize_title(rec.title)
- prior = existing_by_norm.get(norm)
- if prior is None or rank_for_source(rec.source) > rank_for_source(prior.source):
- existing_by_norm[norm] = rec
-
- # Phase 2C semantic dedup runs against the shared vector store. ``id_to_rec``
- # lets a store hit (which returns a decision id) resolve back to the live
- # record, and is grown as records are created so paraphrases *within* one
- # batch also collapse.
- id_to_rec: dict[str, DecisionRecord] = {rec.id: rec for rec in existing_by_norm.values()}
-
- # Headline candidate per group: highest source rank, tie-break by
- # confidence. Hoisted so the batched embedding below can see every
- # group's match text before the loop runs.
- headline_by_norm: dict[str, dict] = {
- norm: max(
- members,
- key=lambda d: (rank_for_source(d.get("source", "")), d.get("confidence", 0.0)),
- )
- for norm, members in groups.items()
- }
-
- # Batched embedding for dedup. The per-item path embeds one query per
- # residual group and one upsert per touched record — thousands of serial
- # network round-trips on a first index. Instead: embed every group's
- # match text in a few chunked requests up front, search the store by raw
- # vector, dedup *within* the batch against a local pending index, and
- # defer the store writes to one batched upsert after the loop. Stores
- # that can't hand back vectors (no embedder, or search_by_vector left at
- # the base-class default) keep the per-item flow unchanged.
- batch_mode = False
- group_vecs: dict[str, list[float]] = {}
- group_texts: dict[str, str] = {}
- if vector_store is not None:
- from repowise.core.analysis.decision_semantic_match import (
- PendingDecisionIndex,
- decision_match_text,
- decision_vector_item,
- find_duplicate_decision,
- find_duplicate_decision_by_vector,
- upsert_decision_vector,
- upsert_decision_vectors,
- )
- from repowise.core.persistence.vector_store import VectorStore
- from repowise.core.providers.embedding import store_has_semantic_vectors
-
- # Static capability check: the base-class search_by_vector is the
- # "unsupported" sentinel. A runtime probe would conflate a transient
- # store error with lack of support and silently fall back to the
- # O(N)-round-trip path this branch exists to avoid.
- store_cls_search = getattr(type(vector_store), "search_by_vector", None)
- supports_vector_search = (
- callable(store_cls_search)
- and store_cls_search is not VectorStore.search_by_vector
- # A keyless store supports the call and cannot answer it: every
- # matcher below refuses on it. Without this the batch pays for a
- # full embed round whose vectors are then never read.
- and store_has_semantic_vectors(vector_store)
- )
-
- ordered = [
- (norm, decision_match_text(h.get("title", ""), h.get("decision") or ""))
- for norm, h in headline_by_norm.items()
- ]
- ordered = [(norm, text) for norm, text in ordered if text]
- if ordered and supports_vector_search:
- try:
- vecs = await vector_store.embed_texts([text for _, text in ordered])
- except Exception:
- vecs = None
- if vecs is not None and len(vecs) == len(ordered):
- group_vecs = dict(zip((n for n, _ in ordered), vecs, strict=True))
- group_texts = dict(ordered)
- batch_mode = True
-
- structlog.get_logger(__name__).info(
- "decision_dedup_embedding",
- groups=len(groups),
- batched=batch_mode,
- )
-
- # Vectors of records touched this batch but not yet written to the store —
- # the in-batch side of dedup in batch mode. A record is only registered
- # when its final text equals the group text its vector came from, so the
- # index never matches against text the record doesn't actually contain.
- pending = PendingDecisionIndex() if batch_mode else None
-
- touched_ids: list[str] = []
- captured: dict[str, tuple[str, bool]] = {}
-
- for norm, members in groups.items():
- headline = headline_by_norm[norm]
- rec = existing_by_norm.get(norm)
-
- # No title match → ask the store whether a semantically-equivalent
- # decision already exists, and fold into it if so (cheap title dedup
- # stays the first pass; this only runs on the residual).
- # A computed record must not fold into a prose record that merely
- # sounds alike; title dedup still applies to it.
- if rec is None and vector_store is not None and headline.get("source") != "conventions":
- q_vec = group_vecs.get(norm)
- if batch_mode and q_vec is not None:
- match_id = await find_duplicate_decision_by_vector(
- vector_store, q_vec, pending=pending
- )
- else:
- match_id = await find_duplicate_decision(
- vector_store,
- title=headline.get("title", ""),
- decision=headline.get("decision") or "",
- )
- if match_id is not None and match_id in id_to_rec:
- rec = id_to_rec[match_id]
- existing_by_norm[norm] = rec
-
- # A dismissed record is a tombstone: leave it untouched (no headline
- # promotion, no new evidence, not re-embedded, not in touched_ids) so
- # dismissals survive every reindex.
- if rec is not None and rec.status == "dismissed":
- continue
-
- if rec is None:
- headline_title = headline.get("title", "")
- headline_source = headline.get("source", "cli")
- headline_evidence_file = headline.get("evidence_file")
- headline_files = headline.get("affected_files") or []
- # Pinned now and never revised: the identity is keyed on it, and a
- # later extraction rewording the same sentence must not move the id.
- headline_quote = _evidence_quote(headline)
- derived = derive_decision_id(
- repository_id,
- headline_title,
- source=headline_source,
- evidence_file=headline_evidence_file,
- affected_files=headline_files,
- evidence_line=headline.get("evidence_line"),
- identity_quote=headline_quote,
- needs_split=any(bool(d.get("needs_split")) for d in members),
- )
- # Identity is evidence, so two titles for one decision derive one
- # id. Folding into the record that id already names is the point of
- # the key: inserting instead would collide on the primary key, and
- # skipping would leave the duplicate this is meant to collapse.
- folded = id_to_rec.get(derived) or await session.get(DecisionRecord, derived)
- if folded is not None and folded.repository_id == repository_id:
- if folded.status == "dismissed":
- continue
- rec = folded
- existing_by_norm[norm] = rec
- id_to_rec[rec.id] = rec
- if rec is None:
- # A source that measures its own conformance supplies the score;
- # otherwise the git-diff recompute fills it in later.
- headline_staleness = headline.get("staleness_score")
- rec = DecisionRecord(
- id=derived,
- identity_quote=headline_quote,
- repository_id=repository_id,
- title=headline_title,
- status=_extraction_status(headline.get("status", "proposed")),
- kind=_extraction_kind(headline.get("kind")),
- context=headline.get("context") or "",
- decision=headline.get("decision") or "",
- rationale=headline.get("rationale") or "",
- alternatives_json=json.dumps(headline.get("alternatives") or []),
- consequences_json=json.dumps(headline.get("consequences") or []),
- affected_files_json=json.dumps(headline_files),
- # Rides with the file list it describes.
- scope_basis=headline.get("scope_basis") or "",
- affected_modules_json=json.dumps(headline.get("affected_modules") or []),
- tags_json=json.dumps(headline.get("tags") or []),
- evidence_commits_json=json.dumps(headline.get("evidence_commits") or []),
- source=headline_source,
- evidence_file=headline_evidence_file,
- evidence_line=headline.get("evidence_line"),
- confidence=headline.get("confidence", 0.5),
- staleness_score=(
- float(headline_staleness) if headline_staleness is not None else 0.0
- ),
- )
- session.add(rec)
- await session.flush()
- existing_by_norm[norm] = rec
- id_to_rec[rec.id] = rec
- elif rank_for_source(headline.get("source", "")) >= rank_for_source(rec.source):
- # A new contributor at least as authoritative as the current
- # headline → promote its fields (provenance still accretes below).
- rec.title = headline.get("title", rec.title)
- rec.status = _merge_status(rec.status, headline.get("status", rec.status))
- # ``kind`` is deliberately absent: the noun is decided when the
- # record is created and re-extraction does not revisit it. Most
- # sources default the field rather than deciding it, so an incoming
- # value is usually the absence of a judgement; and this branch runs
- # on accepted records too, where flipping the noun would change what
- # a record governs behind the person who accepted it. The migration
- # refuses that for the same reason, and the manifest is where a
- # person changes it deliberately.
- rec.context = headline.get("context") or rec.context
- rec.decision = headline.get("decision") or rec.decision
- rec.rationale = headline.get("rationale") or rec.rationale
- rec.alternatives_json = json.dumps(headline.get("alternatives") or [])
- rec.consequences_json = json.dumps(headline.get("consequences") or [])
- rec.affected_files_json = json.dumps(headline.get("affected_files") or [])
- rec.scope_basis = headline.get("scope_basis") or ""
- rec.affected_modules_json = json.dumps(headline.get("affected_modules") or [])
- rec.tags_json = json.dumps(headline.get("tags") or [])
- rec.evidence_commits_json = json.dumps(headline.get("evidence_commits") or [])
- rec.source = headline.get("source", rec.source)
- rec.evidence_file = headline.get("evidence_file")
- rec.evidence_line = headline.get("evidence_line")
- # Only a source that measures its own conformance supplies this.
- if headline.get("staleness_score") is not None:
- rec.staleness_score = float(headline["staleness_score"])
-
- # Accrete one evidence row per contributing source occurrence.
- for d in members:
- src = d.get("source", "cli")
- await _upsert_decision_evidence(
- session,
- rec.id,
- source=src,
- source_rank=rank_for_source(src),
- evidence_file=d.get("evidence_file"),
- evidence_line=d.get("evidence_line"),
- evidence_commit=_first_commit(d),
- source_quote=_evidence_quote(d),
- confidence=d.get("confidence", 0.5),
- verification=d.get("verification", "unverified"),
- )
-
- # A committed ADR that says "accepted" is the one non-human acceptance
- # the contract allows, because the document is version controlled and
- # reviewable. Recorded here, where the parsed heading is still in hand.
- await _accept_from_tracked_artifact(session, rec, headline)
-
- # Re-derive headline confidence + verification from the FULL evidence
- # set (existing + just-added), so corroboration accrues across runs.
- _rederive_headline(rec, await list_decision_evidence(session, rec.id))
- rec.updated_at = _now_utc()
- touched_ids.append(rec.id)
- # Two title groups can fold onto one record, so this accumulates:
- # the later group must not drop what the earlier one raised.
- #
- # The split flag is read from every member, not from the headline
- # alone. The headline is the highest-ranked source in the group, and
- # the lane that notices a claim bundles two decisions is usually not
- # the highest-ranked one: a session-mined candidate that flags itself
- # loses its flag the moment a CLI-authored record shares its title.
- # A flag raised by any contributor is a flag on the record.
- prior_lane, prior_split = captured.get(rec.id, ("", False))
- captured[rec.id] = (
- prior_lane or headline.get("lane") or "",
- prior_split or any(bool(d.get("needs_split")) for d in members),
- )
-
- await sync_links_from_record(session, rec)
-
- # (Re-)embed the record into the shared store so it's matchable by
- # later groups in this batch + future runs, and discoverable via
- # search_codebase. Best-effort — never blocks the SQL upsert. In
- # batch mode the write is deferred to one batched upsert after the
- # loop; until then the group vector stands in for the record in the
- # pending index — but only when the record's final text IS the group
- # text (created, or promoted from this headline). A fold that kept
- # the record's own fields must not be matchable under the incoming
- # group's text.
- if vector_store is not None:
- if batch_mode:
- q_vec = group_vecs.get(norm)
- final_text = decision_match_text(rec.title, rec.decision or "")
- if (
- q_vec is not None
- and pending is not None
- and rec.id not in pending.ids
- and final_text == group_texts.get(norm)
- ):
- pending.add(rec.id, q_vec)
- else:
- await upsert_decision_vector(
- vector_store,
- rec.id,
- title=rec.title,
- decision=rec.decision or "",
- evidence_file=rec.evidence_file,
- )
-
- # Batched store write: upsert every touched record's *final* text (post
- # merge/promotion) in one go, reusing the vectors already computed for
- # unchanged texts and embedding only the delta.
- if vector_store is not None and batch_mode and touched_ids:
- items: list[tuple[str, str, dict]] = []
- for rid in dict.fromkeys(touched_ids):
- rec = id_to_rec.get(rid)
- if rec is None:
- continue
- item = decision_vector_item(
- rec.id,
- title=rec.title,
- decision=rec.decision or "",
- evidence_file=rec.evidence_file,
- )
- if item is not None:
- items.append(item)
- vectors_by_text = {
- text: group_vecs[norm] for norm, text in group_texts.items() if norm in group_vecs
- }
- await upsert_decision_vectors(vector_store, items, vectors_by_text=vectors_by_text)
-
- await _write_candidate_meta(session, repository_id, captured)
-
- await session.flush()
- return touched_ids
-
-
-async def _write_candidate_meta(
- session: AsyncSession,
- repository_id: str,
- captured: dict[str, tuple[str, bool]],
- *,
- only: set[str] | None = None,
-) -> None:
- """Refresh the review row every open candidate in the repository is owed.
-
- Not only the ones this run touched: the staging store does not re-emit an
- already-promoted decision, so a backlog would otherwise stay unjudged until
- something happened to re-extract each record individually. The signals come
- from the record, so an unchanged candidate costs a comparison and no write.
- """
- rows = (
- await session.execute(
- select(DecisionRecord, DecisionCandidateMeta)
- .outerjoin(
- DecisionCandidateMeta,
- DecisionCandidateMeta.decision_id == DecisionRecord.id,
- )
- .where(
- DecisionRecord.repository_id == repository_id,
- candidate_predicate(),
- *([DecisionRecord.id.in_(only)] if only is not None else []),
- )
- )
- ).all()
-
- for rec, meta in rows:
- priority, scope_unresolved = candidate_review_signals(rec)
- capture = captured.get(rec.id)
- if capture is None:
- # Not extracted this run, so the provenance is not ours to write:
- # only the contract-derived signals are refreshed, and a row with
- # no lane yet takes the record's own source.
- if (
- meta is not None
- and meta.lane
- and meta.review_priority == priority
- and meta.scope_unresolved == scope_unresolved
- ):
- continue
- await upsert_candidate_meta(
- session,
- rec,
- lane=meta.lane if meta is not None and meta.lane else rec.source,
- review_priority=priority,
- scope_unresolved=scope_unresolved,
- )
- continue
-
- lane, needs_split = capture
- await upsert_candidate_meta(
- session,
- rec,
- lane=lane or rec.source,
- extractor_version=__version__,
- review_priority=priority,
- # Raised, never cleared: a re-extraction must not walk back a split
- # somebody asked for.
- needs_split=True if needs_split else None,
- scope_unresolved=scope_unresolved,
- )
-
-
-async def purge_proposed_decisions_by_source(
- session: AsyncSession,
- repository_id: str,
- source: str,
-) -> int:
- """Delete still-``proposed`` records of *source* plus their child rows.
-
- One-shot cleanup for retired extraction sources (today: the removed
- ``code_comment`` harvest, #751). Only ``proposed`` rows go — anything the
- user confirmed, deprecated, or dismissed is kept. Child rows are deleted
- explicitly rather than trusting the FK cascade, which on SQLite depends on
- the ``foreign_keys`` pragma. Returns the number of records deleted.
- """
- result = await session.execute(
- select(DecisionRecord.id).where(
- DecisionRecord.repository_id == repository_id,
- DecisionRecord.source == source,
- DecisionRecord.status == "proposed",
- ~accepted_predicate(),
- )
- )
- ids = [row[0] for row in result.all()]
- if not ids:
- return 0
-
- await session.execute(delete(DecisionEvidence).where(DecisionEvidence.decision_id.in_(ids)))
- await session.execute(
- delete(DecisionEdge).where(
- or_(
- DecisionEdge.src_decision_id.in_(ids),
- DecisionEdge.dst_decision_id.in_(ids),
- )
- )
- )
- await session.execute(delete(DecisionNodeLink).where(DecisionNodeLink.decision_id.in_(ids)))
- await session.execute(delete(DecisionRecord).where(DecisionRecord.id.in_(ids)))
- await session.flush()
- structlog.get_logger(__name__).info("decision_purge_by_source", source=source, deleted=len(ids))
- return len(ids)
-
-
-async def purge_proposed_decisions_outside_files(
- session: AsyncSession,
- repository_id: str,
- current_file_paths: set[str],
-) -> int:
- """Delete unreviewed extracted decisions whose evidence file left scope."""
- result = await session.execute(
- select(DecisionRecord.id, DecisionRecord.evidence_file).where(
- DecisionRecord.repository_id == repository_id,
- DecisionRecord.status == "proposed",
- DecisionRecord.evidence_file.is_not(None),
- ~accepted_predicate(),
- )
- )
- ids = [row[0] for row in result.all() if row[1] not in current_file_paths]
- if not ids:
- return 0
-
- await session.execute(delete(DecisionEvidence).where(DecisionEvidence.decision_id.in_(ids)))
- await session.execute(
- delete(DecisionEdge).where(
- or_(
- DecisionEdge.src_decision_id.in_(ids),
- DecisionEdge.dst_decision_id.in_(ids),
- )
- )
- )
- await session.execute(delete(DecisionNodeLink).where(DecisionNodeLink.decision_id.in_(ids)))
- await session.execute(delete(DecisionRecord).where(DecisionRecord.id.in_(ids)))
- await session.flush()
- return len(ids)
-
-
-async def _fill_git_meta_gaps(
- session: AsyncSession,
- repository_id: str,
- git_meta_map: dict[str, dict],
- wanted: set[str],
-) -> dict[str, dict]:
- """``git_meta_map`` widened with persisted rows for the *wanted* paths.
-
- The caller's map is one run's git metadata, and on an incremental update
- that is only the files that changed in it. Staleness scoring reads a path
- with no entry as a file that is gone and scores it 1.00, so every decision
- over an untouched file went maximally stale for not having been touched.
- The persisted rows cover every file the index has seen, which is the set
- that answers "is this file still here"; a path missing from those too is
- genuinely untracked and still scores 1.00. The run's own entries win —
- they are this run's fresher numbers for the files it re-read.
- """
- missing = wanted - git_meta_map.keys()
- if not missing:
- return git_meta_map
-
- filled: dict[str, dict] = {}
- # Chunked so the IN clause stays under SQLite's bind-parameter ceiling.
- batch = sorted(missing)
- for start in range(0, len(batch), 500):
- rows = await session.execute(
- select(GitMetadata).where(
- GitMetadata.repository_id == repository_id,
- GitMetadata.file_path.in_(batch[start : start + 500]),
- )
- )
- for row in rows.scalars().all():
- filled[row.file_path] = {
- col.name: getattr(row, col.name)
- for col in row.__table__.columns
- if col.name not in ("id", "repository_id")
- }
- filled.update(git_meta_map)
- return filled
-
-
-async def recompute_decision_staleness(
- session: AsyncSession,
- repository_id: str,
- git_meta_map: dict[str, dict],
-) -> int:
- """Recompute staleness_score for all active decisions. Returns update count.
-
- Also fills ``last_code_change`` from the same per-file dates the score is
- counted from, and re-derives ``affected_modules_json`` from the files each
- record names.
- The two belong in one pass because they are one repair: a record's module
- linkage used to be the first path segment, which in a ``packages/`` layout
- made almost every record claim ``packages`` or ``tests``, and the rows that
- predate the fix carry it. Deriving here rather than in a data migration
- keeps the single-repair-path rule — the one this repo already learned when
- an alembic migration and a runtime repair disagreed about confidence.
- """
- result = await session.execute(
- select(DecisionRecord).where(
- DecisionRecord.repository_id == repository_id,
- DecisionRecord.status.in_(["active", "proposed"]),
- )
- )
- decisions = list(result.scalars().all())
-
- affected_by_id: dict[str, list[str]] = {}
- for dec in decisions:
- affected = json.loads(dec.affected_files_json)
- if affected:
- affected_by_id[dec.id] = affected
-
- remodelled = _backfill_module_nodes(decisions, affected_by_id)
- modules_updated = len(remodelled)
- # This repair runs after ``backfill_decision_node_links`` on both index
- # paths, so a module array it rewrites here would otherwise keep the link
- # the backfill had already judged correct.
- for dec in remodelled:
- await sync_links_from_record(session, dec)
- if not affected_by_id:
- if modules_updated:
- await session.flush()
- return 0
-
- git_meta_map = await _fill_git_meta_gaps(
- session,
- repository_id,
- git_meta_map,
- {fp for paths in affected_by_id.values() for fp in paths},
- )
-
- now = _now_utc()
- updated = 0
- dated = 0
- # Function-local, like every other analysis import here: the persistence
- # layer cannot depend on analysis at module scope.
- from repowise.core.analysis.decisions.extractor import (
- DecisionExtractor,
- _as_aware_utc,
- )
-
- for dec in decisions:
- affected = affected_by_id.get(dec.id)
- if not affected:
- continue
-
- # Before the conventions skip below: a date is a fact about the code,
- # not a score, so it is filled for every scoped record.
- last_change = DecisionExtractor.last_code_change(affected, git_meta_map)
- stored = dec.last_code_change
- # SQLite drops tzinfo, so an aware value would differ from the naive
- # one it just wrote and rewrite ``updated_at`` on every run.
- if last_change != (_as_aware_utc(stored) if stored else None):
- dec.last_code_change = last_change
- dec.updated_at = now
- dated += 1
-
- # The source writes its own conformance share, and a git-diff score
- # would overwrite it. Only the score is skipped; the repairs above
- # still apply.
- if dec.source == "conventions":
- continue
-
- new_score = DecisionExtractor.compute_staleness(
- dec.created_at,
- affected,
- git_meta_map,
- )
- if abs(new_score - dec.staleness_score) > 0.01:
- dec.staleness_score = round(new_score, 3)
- dec.updated_at = now
- updated += 1
-
- if updated or modules_updated or dated:
- await session.flush()
- # Deliberately the staleness count alone. The callers print this as
- # "N decisions rescored"; folding the module repair or the date fill into
- # it would report a rescore that did not happen.
- return updated
-
-
-def _backfill_module_nodes(
- decisions: list[DecisionRecord],
- affected_by_id: dict[str, list[str]],
-) -> list[DecisionRecord]:
- """Re-derive each record's module linkage from its files.
-
- Returns the records whose array moved, because the caller has to relink
- each one: this rewrites half a record's scope and the graph holds the
- other half.
-
- Records naming no file are left alone: there is nothing to derive from, and
- an invented scope is worse than an absent one.
- """
- from repowise.core.analysis.decisions.scope import resolve_module_nodes
-
- moved = []
- for dec in decisions:
- affected = affected_by_id.get(dec.id)
- if not affected:
- continue
- derived = resolve_module_nodes(affected)
- if derived != json.loads(dec.affected_modules_json or "[]"):
- dec.affected_modules_json = json.dumps(derived)
- moved.append(dec)
- return moved
-
-
-async def get_stale_decisions(
- session: AsyncSession,
- repository_id: str,
- threshold: float = 0.5,
-) -> list[DecisionRecord]:
- """Return active decisions with staleness_score >= threshold."""
- result = await session.execute(
- select(DecisionRecord).where(
- DecisionRecord.repository_id == repository_id,
- DecisionRecord.status.in_(["active"]),
- DecisionRecord.staleness_score >= threshold,
- )
- )
- return list(result.scalars().all())
-
-
-async def get_decision_health_summary(
- session: AsyncSession,
- repository_id: str,
-) -> dict:
- """Return decision health: counts by lane, stale decisions, ungoverned hotspots.
-
- The five list fields are returned ranked worst-first: stale by staleness,
- proposed by confidence, ungoverned hotspots by temporal hotspot score,
- retired by lane (history before tombstone) and unscoped by confidence. A
- caller that shows only the first few shows the few that matter.
- Callers may truncate; they must not re-order.
-
- ``retired_decisions`` holds ``(lane, record)`` pairs: the lane is derived
- from the acceptance, which the ``status`` column may disagree with.
-
- Counts the acceptance, not the status column. The key names are the ones
- every caller already renders, and they keep their product meaning:
- ``active`` is what governs, ``proposed`` is what nobody has accepted. What
- changed is that a record reaches ``active`` here by having an acceptance,
- the same way it does on every other governance read. Counting the column
- made this the one surface still reporting a hundred governing decisions on
- a store whose acceptances were empty, and it fed ``ungoverned_hotspots``,
- so unaccepted records were also suppressing the files nothing governs.
- """
- result = await session.execute(
- select(DecisionRecord).where(
- DecisionRecord.repository_id == repository_id,
- )
- )
- all_decisions = list(result.scalars().all())
- from .authority import decision_currencies
-
- currencies = await decision_currencies(session, repository_id, all_decisions)
-
- counts = {
- "active": 0,
- "proposed": 0,
- "deprecated": 0,
- "superseded": 0,
- "dismissed": 0,
- "stale": 0,
- # Accepted records naming no file. They score 0.0 because the staleness
- # question cannot be asked of them, which renders identically to a
- # record whose code genuinely has not moved — so they are counted
- # separately rather than banked as fresh.
- "unscoped": 0,
- }
- stale_decisions: list[DecisionRecord] = []
- proposed_decisions: list[DecisionRecord] = []
- # Counted-only lanes. The record is in hand at the ``continue`` that drops
- # it, so naming it costs no query. ``retired`` carries its lane because
- # that lane is derived from the acceptance where there is one, and a
- # record's ``status`` column may disagree with it.
- retired_decisions: list[tuple[str, DecisionRecord]] = []
- unscoped_decisions: list[DecisionRecord] = []
-
- # Files an *accepted* decision names. A candidate naming a hotspot does not
- # make it governed, and counting one did: it removed the file from
- # ``ungoverned_hotspots``, which is the list whose whole job is to say
- # where nobody has decided anything.
- governed_files: set[str] = set()
- for d in all_decisions:
- currency = currencies.get(d.id)
- if currency is None:
- # No acceptance, so it does not govern. A retired one is not
- # awaiting review either: it keeps the status that retired it and
- # stays out of the queue, which is the one thing a tombstone must
- # never be counted as.
- if d.status in ("dismissed", "deprecated", "superseded"):
- counts[d.status] = counts.get(d.status, 0) + 1
- retired_decisions.append((d.status, d))
- else:
- counts["proposed"] += 1
- proposed_decisions.append(d)
- continue
- if currency == "superseded":
- counts["superseded"] += 1
- retired_decisions.append(("superseded", d))
- continue
- if currency == "dismissed":
- counts["dismissed"] += 1
- retired_decisions.append(("dismissed", d))
- continue
- counts["active"] += 1
- if currency == "needs_review":
- counts["stale"] += 1
- stale_decisions.append(d)
- if currency == "uncheckable":
- counts["unscoped"] += 1
- unscoped_decisions.append(d)
- # ``governed_files`` is the denominator for "ungoverned hotspots",
- # so a footprint would suppress every file its commit touched.
- if not binds_to_paths(d.scope_basis):
- continue
- for fp in json.loads(d.affected_files_json):
- governed_files.add(fp)
-
- # Find ungoverned hotspots, hottest first. Sorting these by path put the file
- # most in need of a decision behind whatever sorts alphabetically first,
- # with the score that answers the question sitting unread one column over.
- # The key is the one ``routers/overview.py`` already applies to these same
- # rows in SQL (score descending with NULLs last, then churn) rather than a
- # second answer to "which hotspot matters most". A NULL score is genuinely
- # unknown and is not the same as a measured zero.
- hotspot_result = await session.execute(
- select(
- GitMetadata.file_path,
- GitMetadata.temporal_hotspot_score,
- GitMetadata.churn_percentile,
- ).where(
- GitMetadata.repository_id == repository_id,
- GitMetadata.is_hotspot == True, # noqa: E712
- )
- )
- hotspot_rows = {row[0]: (row[1], row[2]) for row in hotspot_result.all()}
-
- def _hotspot_rank(file_path: str) -> tuple[bool, float, float, str]:
- score, churn = hotspot_rows[file_path]
- return (score is None, -(score or 0.0), -(churn or 0.0), file_path)
-
- ungoverned = sorted(hotspot_rows.keys() - governed_files, key=_hotspot_rank)
-
- # Rank here rather than at the five call sites, none of which does. The MCP
- # health dashboard and ``repowise decision health`` cut all three lists; the
- # overview attention panel cuts the hotspots and renders the other two in the
- # order it is handed; the decisions route serves them whole in that order;
- # and ``health/governance.py`` walks them to *write* one finding row per
- # entry. So the callers that truncate were showing whichever rows the scan
- # returned first, and the ones that do not were still listing them by it,
- # while the score answering "which of these first" rides on every record,
- # unread. The ``or 0.0`` guards match how every other reader of these two
- # fields spells it rather than trusting a column default to have been
- # back-filled; the id tiebreak makes the key total, so two runs agree.
- stale_decisions.sort(key=lambda d: (-(d.staleness_score or 0.0), d.id))
- proposed_decisions.sort(key=lambda d: (-(d.confidence or 0.0), d.id))
- # Retired by lane, history before tombstone. Not by ``updated_at``: it
- # moves on any write, so it does not say when a record was retired.
- # ``unscoped`` by confidence, the key ``proposed`` already uses.
- retired_decisions.sort(key=lambda pair: (status_rank(pair[0]), pair[1].id))
- unscoped_decisions.sort(key=lambda d: (-(d.confidence or 0.0), d.id))
-
- # Phase 3B: surface contradictory active decisions (conflicts_with edges).
- from ..decision_graph import list_conflict_edges
-
- by_id = {d.id: d for d in all_decisions}
- conflicts: list[dict] = []
- for edge in await list_conflict_edges(session, repository_id):
- src = by_id.get(edge.src_decision_id)
- dst = by_id.get(edge.dst_decision_id)
- if src is None or dst is None:
- continue
- conflicts.append(
- {
- "src": {"id": src.id, "title": src.title, "status": src.status},
- "dst": {"id": dst.id, "title": dst.title, "status": dst.status},
- "confidence": edge.confidence,
- "evidence": edge.evidence,
- }
- )
- counts["conflicts"] = len(conflicts)
-
- return {
- "summary": counts,
- "stale_decisions": stale_decisions,
- "proposed_awaiting_review": proposed_decisions,
- "ungoverned_hotspots": ungoverned,
- "conflicts": conflicts,
- "retired_decisions": retired_decisions,
- "unscoped_decisions": unscoped_decisions,
- }