Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(core): split crud/decisions.py by concern (#2558)
Raghav Chamadiyaf0bdf3145fb7
diff --git a/packages/core/src/repowise/core/persistence/crud/decision_evidence.py b/packages/core/src/repowise/core/persistence/crud/decision_evidence.pynew file mode 100644index 000000000..15b35c1a4--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/decision_evidence.py@@ -0,0 +1,153 @@+"""Decision evidence rows and the headline score derived from them."""++from __future__ import annotations++import json++from sqlalchemy import select+from sqlalchemy.ext.asyncio import AsyncSession++from repowise.core.analysis.decisions.provenance import completeness, compute_confidence++from ..models import DecisionEvidence, DecisionRecord+++def _evidence_quote(d: dict) -> str:+ """Pick the verbatim span recorded as this evidence row's source quote.++ Prefers the LLM/parser-supplied ``source_quote``; falls back to the+ decision/rationale text so an evidence row is never empty.+ """+ return (d.get("source_quote") or d.get("decision") or d.get("rationale") or "").strip()+++async def _upsert_decision_evidence(+ session: AsyncSession,+ decision_id: str,+ *,+ source: str,+ source_rank: int,+ evidence_file: str | None,+ evidence_line: int | None,+ evidence_commit: str | None,+ source_quote: str,+ confidence: float,+ verification: str,+) -> None:+ """Insert or update one evidence row, idempotent on its natural key.++ Natural key is ``(decision_id, source, evidence_file, evidence_commit)``.+ NULLs are matched explicitly (SQLite treats NULLs as distinct in a unique+ constraint), so re-indexing the same source converges instead of+ duplicating, while an incremental update adds genuinely new evidence.+ """+ q = select(DecisionEvidence).where(+ DecisionEvidence.decision_id == decision_id,+ DecisionEvidence.source == source,+ )+ q = (+ q.where(DecisionEvidence.evidence_file == evidence_file)+ if evidence_file is not None+ else q.where(DecisionEvidence.evidence_file.is_(None))+ )+ q = (+ q.where(DecisionEvidence.evidence_commit == evidence_commit)+ if evidence_commit is not None+ else q.where(DecisionEvidence.evidence_commit.is_(None))+ )+ existing = (await session.execute(q)).scalar_one_or_none()++ if existing is not None:+ existing.source_rank = source_rank+ existing.evidence_line = evidence_line+ existing.source_quote = source_quote+ existing.confidence = confidence+ existing.verification = verification+ return++ session.add(+ DecisionEvidence(+ decision_id=decision_id,+ source=source,+ source_rank=source_rank,+ evidence_file=evidence_file,+ evidence_line=evidence_line,+ evidence_commit=evidence_commit,+ source_quote=source_quote,+ confidence=confidence,+ verification=verification,+ )+ )+++def _json_list(raw: str | None) -> list[str]:+ """A JSON array column as a list; anything unparsable reads as empty."""+ try:+ value = json.loads(raw or "[]")+ except (TypeError, ValueError):+ return []+ return [str(v) for v in value] if isinstance(value, list) else []+++def _best_verification(values: list[str]) -> str:+ """Reduce per-evidence verdicts to the strongest: exact > fuzzy > unverified."""+ if "exact" in values:+ return "exact"+ if "fuzzy" in values:+ return "fuzzy"+ return "unverified"+++def record_completeness(rec: DecisionRecord) -> int:+ """:func:`completeness` for a stored row, parsing its two JSON columns."""+ return completeness(+ decision=rec.decision,+ rationale=rec.rationale,+ context=rec.context,+ consequences=_json_list(rec.consequences_json),+ alternatives=_json_list(rec.alternatives_json),+ )+++def _rederive_headline(rec: DecisionRecord, evidence: list[DecisionEvidence]) -> None:+ """Set a record's confidence + verification from its full evidence set.++ The single definition of how a headline is scored. Every writer uses it:+ the upsert path after accreting a run's evidence, ``reconcile_source_ranks``+ after a ladder edit, and ``reconcile_decision_confidence`` after a formula+ edit. Kept as one function because two of them were briefly copy-pasted and+ nothing would have forced the copies to stay equal.++ Confidence rises with the best source rank and with the number of+ *independent* corroborating sources, and with how much of its body the+ record fills, so it is derived from the whole set plus the row rather than+ from whichever evidence row happened to arrive last. No-op on empty+ evidence: a record with nothing behind it keeps whatever it had.+ """+ if not evidence:+ return+ best_ver = _best_verification([e.verification for e in evidence])+ rec.confidence = compute_confidence(+ max(e.source_rank for e in evidence),+ len({e.source for e in evidence}),+ best_ver,+ filled_fields=record_completeness(rec),+ )+ rec.verification = best_ver++async def list_decision_evidence(+ session: AsyncSession,+ decision_id: str,+) -> list[DecisionEvidence]:+ """Return all evidence rows for a decision, highest source rank first."""+ result = await session.execute(+ select(DecisionEvidence)+ .where(DecisionEvidence.decision_id == decision_id)+ .order_by(DecisionEvidence.source_rank.desc(), DecisionEvidence.created_at.asc())+ )+ return list(result.scalars().all())+++def _first_commit(d: dict) -> str | None:+ commits = d.get("evidence_commits") or []+ return commits[0] if commits else Nonediff --git a/packages/core/src/repowise/core/persistence/crud/decision_health.py b/packages/core/src/repowise/core/persistence/crud/decision_health.pynew file mode 100644index 000000000..f789c698b--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/decision_health.py@@ -0,0 +1,193 @@+"""The decision health summary: lane counts, stale records, ungoverned hotspots."""++from __future__ import annotations++import json+from dataclasses import dataclass, field++from sqlalchemy import select+from sqlalchemy.ext.asyncio import AsyncSession++from repowise.core.analysis.decisions.lifecycle import status_rank+from repowise.core.analysis.decisions.scope import binds_to_paths++from ..models import DecisionRecord, GitMetadata+++async def get_decision_health_summary(+ session: AsyncSession,+ repository_id: str,+) -> dict:+ """Return decision health: counts by lane, stale decisions, ungoverned hotspots.++ The five list fields are returned ranked worst-first: stale by staleness,+ proposed by confidence, ungoverned hotspots by temporal hotspot score,+ retired by lane (history before tombstone) and unscoped by confidence. A+ caller that shows only the first few shows the few that matter.+ Callers may truncate; they must not re-order.++ ``retired_decisions`` holds ``(lane, record)`` pairs: the lane is derived+ from the acceptance, which the ``status`` column may disagree with.++ Counts the acceptance, not the status column. The key names are the ones+ every caller already renders, and they keep their product meaning:+ ``active`` is what governs, ``proposed`` is what nobody has accepted. What+ changed is that a record reaches ``active`` here by having an acceptance,+ the same way it does on every other governance read. Counting the column+ made this the one surface still reporting a hundred governing decisions on+ a store whose acceptances were empty, and it fed ``ungoverned_hotspots``,+ so unaccepted records were also suppressing the files nothing governs.+ """+ result = await session.execute(+ select(DecisionRecord).where(+ DecisionRecord.repository_id == repository_id,+ )+ )+ all_decisions = list(result.scalars().all())+ from .authority import decision_currencies++ currencies = await decision_currencies(session, repository_id, all_decisions)++ lanes = _Lanes()+ for d in all_decisions:+ lanes.add(d, currencies.get(d.id))+ ungoverned = await _ungoverned_hotspots(session, repository_id, lanes.governed_files)+ lanes.rank()+ conflicts = await _conflict_summaries(session, repository_id, all_decisions)+ lanes.counts["conflicts"] = len(conflicts)++ return {+ "summary": lanes.counts,+ "stale_decisions": lanes.stale,+ "proposed_awaiting_review": lanes.proposed,+ "ungoverned_hotspots": ungoverned,+ "conflicts": conflicts,+ "retired_decisions": lanes.retired,+ "unscoped_decisions": lanes.unscoped,+ }+++@dataclass+class _Lanes:+ """Every record sorted into the lane its acceptance puts it in."""++ counts: dict[str, int] = field(+ default_factory=lambda: {+ "active": 0,+ "proposed": 0,+ "deprecated": 0,+ "superseded": 0,+ "dismissed": 0,+ "stale": 0,+ # Accepted records naming no file. They score 0.0 because the+ # staleness question cannot be asked of them, which renders+ # identically to a record whose code genuinely has not moved, so+ # they are counted separately rather than banked as fresh.+ "unscoped": 0,+ }+ )+ stale: list[DecisionRecord] = field(default_factory=list)+ proposed: list[DecisionRecord] = field(default_factory=list)+ # Counted-only lanes. The record is in hand at the point that drops it, so+ # naming it costs no query. ``retired`` carries its lane because that lane+ # is derived from the acceptance where there is one, and a record's+ # ``status`` column may disagree with it.+ retired: list[tuple[str, DecisionRecord]] = field(default_factory=list)+ unscoped: list[DecisionRecord] = field(default_factory=list)+ # Files an *accepted* decision names. A candidate naming a hotspot does not+ # make it governed, and counting one did: it removed the file from+ # ``ungoverned_hotspots``, which is the list whose whole job is to say+ # where nobody has decided anything.+ governed_files: set[str] = field(default_factory=set)++ def add(self, d: DecisionRecord, currency: str | None) -> None:+ if currency is None:+ # No acceptance, so it does not govern. A retired one is not+ # awaiting review either: it keeps the status that retired it and+ # stays out of the queue, which is the one thing a tombstone must+ # never be counted as.+ if d.status in ("dismissed", "deprecated", "superseded"):+ self.counts[d.status] = self.counts.get(d.status, 0) + 1+ self.retired.append((d.status, d))+ else:+ self.counts["proposed"] += 1+ self.proposed.append(d)+ return+ if currency in ("superseded", "dismissed"):+ self.counts[currency] += 1+ self.retired.append((currency, d))+ return+ self.counts["active"] += 1+ if currency == "needs_review":+ self.counts["stale"] += 1+ self.stale.append(d)+ if currency == "uncheckable":+ self.counts["unscoped"] += 1+ self.unscoped.append(d)+ # ``governed_files`` is the denominator for "ungoverned hotspots",+ # so a footprint would suppress every file its commit touched.+ if binds_to_paths(d.scope_basis):+ self.governed_files.update(json.loads(d.affected_files_json))++ def rank(self) -> None:+ """Order every list worst-first, with an id tiebreak so runs agree.++ Callers truncate these lists and must not re-order them.+ """+ self.stale.sort(key=lambda d: (-(d.staleness_score or 0.0), d.id))+ self.proposed.sort(key=lambda d: (-(d.confidence or 0.0), d.id))+ # Retired by lane, history before tombstone. Not by ``updated_at``: it+ # moves on any write, so it does not say when a record was retired.+ # ``unscoped`` by confidence, the key ``proposed`` already uses.+ self.retired.sort(key=lambda pair: (status_rank(pair[0]), pair[1].id))+ self.unscoped.sort(key=lambda d: (-(d.confidence or 0.0), d.id))+++async def _ungoverned_hotspots(+ session: AsyncSession, repository_id: str, governed_files: set[str]+) -> list[str]:+ """Hotspot files no accepted decision names, hottest first.++ Same key as ``routers/overview.py``: score descending with NULLs last, then churn.+ """+ hotspot_result = await session.execute(+ select(+ GitMetadata.file_path,+ GitMetadata.temporal_hotspot_score,+ GitMetadata.churn_percentile,+ ).where(+ GitMetadata.repository_id == repository_id,+ GitMetadata.is_hotspot == True, # noqa: E712+ )+ )+ hotspot_rows = {row[0]: (row[1], row[2]) for row in hotspot_result.all()}++ def _hotspot_rank(file_path: str) -> tuple[bool, float, float, str]:+ score, churn = hotspot_rows[file_path]+ return (score is None, -(score or 0.0), -(churn or 0.0), file_path)++ return sorted(hotspot_rows.keys() - governed_files, key=_hotspot_rank)+++async def _conflict_summaries(+ session: AsyncSession, repository_id: str, all_decisions: list[DecisionRecord]+) -> list[dict]:+ """Phase 3B: contradictory active decisions (``conflicts_with`` edges)."""+ from ..decision_graph import list_conflict_edges++ by_id = {d.id: d for d in all_decisions}+ conflicts: list[dict] = []+ for edge in await list_conflict_edges(session, repository_id):+ src = by_id.get(edge.src_decision_id)+ dst = by_id.get(edge.dst_decision_id)+ if src is None or dst is None:+ continue+ conflicts.append(+ {+ "src": {"id": src.id, "title": src.title, "status": src.status},+ "dst": {"id": dst.id, "title": dst.title, "status": dst.status},+ "confidence": edge.confidence,+ "evidence": edge.evidence,+ }+ )+ return conflictsdiff --git a/packages/core/src/repowise/core/persistence/crud/decision_identity.py b/packages/core/src/repowise/core/persistence/crud/decision_identity.pynew file mode 100644index 000000000..a6e7b5056--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/decision_identity.py@@ -0,0 +1,161 @@+"""A decision's derived id, and the status/kind rules re-extraction must respect."""++from __future__ import annotations++import hashlib+import json+from collections.abc import Iterable++from repowise.core.analysis.decisions.lifecycle import ARCHITECTURAL_KIND, DECISION_KINDS+from repowise.core.analysis.decisions.provenance import normalize_text++# Namespaced so a decision id can never collide with another kind of derived+# id, and versioned so a future change of the recipe is a visible new value+# rather than a silent reshuffle of every id in every store.+_ID_NAMESPACE = "repowise.decision.id.v2"++# The identity fields are joined with a separator none of them can contain,+# so no pair of distinct records can flatten to the same string.+_FIELD_SEP = "\x00"++# ``evidence_file`` is nullable and NULL is not the empty string here: the+# dedupe query treats them as different records, so the derivation must too.+_NULL_EVIDENCE_FILE = "\x01"++# The pinned quote is truncated before it is hashed. A quote long enough to+# reach this is a paragraph, and its opening already separates two decisions+# that share a file set; carrying the rest only widens the surface on which a+# re-mine can reword the identity out from under a reference somebody wrote.+_IDENTITY_QUOTE_CHARS = 200++# Statuses a human (or the evolution judge) set deliberately. Re-extraction+# must never walk one of these back to ``proposed``: a re-harvested source+# ties or beats the stored rank, so without this guard a reindex would flip+# confirmed decisions back into the review queue and resurrect dismissed ones+# (#751). ``dismissed`` is terminal: the row is kept purely as a tombstone so+# the same content never re-proposes.+_PROTECTED_STATUSES = frozenset({"active", "deprecated", "superseded", "dismissed"})+++def _extraction_status(incoming: str) -> str:+ """The status extraction is allowed to write.++ Extraction proposes; acceptance promotes. An incoming ``active`` is a+ parsed ADR heading or an old code path's optimism, and neither is an+ acceptance event, so it lands as a proposal. A tracked ADR still reaches+ ``active`` — via :func:`_accept_from_tracked_artifact` below, which records+ the document as the accepter instead of leaving the promotion anonymous.+ """+ return "proposed" if incoming == "active" else incoming+++def _merge_status(existing: str, incoming: str) -> str:+ """Resolve a re-extracted status against the stored one.++ A protected stored status wins over an incoming ``proposed``; anything else+ follows the incoming value, after :func:`_extraction_status` has taken+ authority out of extraction's hands.+ """+ incoming = _extraction_status(incoming)+ if incoming == "proposed" and existing in _PROTECTED_STATUSES:+ return existing+ return incoming+++def _extraction_kind(incoming: str | None) -> str:+ """The kind extraction is allowed to write.++ Unrecognised falls to ``architectural`` rather than being stored: the+ vocabulary has no CHECK behind it, every reader tests ``== AGREEMENT_KIND``,+ and a junk value that merely happens to read as architectural everywhere is+ safe by luck. Normalising here makes it safe by construction.+ """+ return incoming if incoming in DECISION_KINDS else ARCHITECTURAL_KIND++def identity_quote_for(text: str | None) -> str:+ """The pinned form of a decision's quote, for identity only.++ Whitespace-collapsed, lowercased and truncated, so the identity survives a+ re-mine that reflows or recases the same sentence. The stored column keeps+ the verbatim text; only this derived form is hashed.+ """+ return normalize_text(text)[:_IDENTITY_QUOTE_CHARS]+++def derive_decision_id(+ repository_id: str,+ title: str,+ *,+ source: str,+ evidence_file: str | None,+ affected_files: Iterable[str] = (),+ evidence_line: int | None = None,+ identity_quote: str = "",+ needs_split: bool = False,+) -> str:+ """The id a decision with this identity has, in every store, on every run.++ Keyed on the evidence rather than on the title. A title is the most+ volatile thing a decision carries: two extractions of the same choice word+ it differently and produce two records, and rewording one moves its id.+ Over the dogfood store, title identity collapses **zero** duplicate pairs+ while the files-plus-evidence-plus-quote key below collapses **120 pairs+ across 42 classes**, and loses none of what title identity caught.++ The quote is load-bearing, and is why the file set alone is not the key:+ two unrelated classes in that store share a file pair and are separated+ only by what was said. It is also the fragile part, because most records+ key on mined prose rather than on a committed span, so it is **pinned in+ the row at first capture and never re-derived**. A later extraction that+ rewords the same sentence then leaves the id where it is.++ A record with no evidence at all keeps its title, for the same reason the+ rest of them lose it: identity is the evidence, and a record that has+ none has no evidence identity to key on. Without this every scopeless,+ quoteless record in a repository derives one id and the second one to be+ written collides with the first. Two of the 357 records in the dogfood+ store are in that state, and the construction paths that supply nothing+ but a title are all in that state.++ A record flagged ``needs_split`` keeps its title in the key. A bundled+ claim shares its files and its evidence with the separate decisions it+ bundles, so evidence identity would fold all of them together and file two+ decisions under a third one's name. Holding the flagged record on title+ identity is what keeps it apart, and it is why the flag has to be set+ before this key ships. The guard is partial: it reaches the bundles a+ marker can see and no others, measured at 28 of 357 records held out for+ 20 of 120 merges given up.++ ``source`` is deliberately **not** in the key. Two lanes that mined the+ same sentence out of the same files recorded one decision, not two, and+ the quote already separates two that merely share a scope.++ 32 lowercase hex, because ``DecisionRecord.id`` is ``String(32)`` and so+ is every foreign key to it, so a truncated digest fits without a column+ change. ``evidence_file`` is NULL far more often than not, so it gets a+ sentinel no path can contain rather than collapsing into the empty string.++ Note the id follows the identity, so editing the scope or the pinned quote+ moves it. :mod:`decision_id_migration`, which runs at the head of every+ index, is what settles that, and it leaves an alias behind so an id+ already written down keeps resolving.+ """+ files = sorted({str(f) for f in affected_files if f})+ quote = identity_quote_for(identity_quote)+ grounded = bool(files) or evidence_file is not None or bool(quote)+ parts = (+ _ID_NAMESPACE,+ repository_id,+ json.dumps(files),+ json.dumps(+ [+ _NULL_EVIDENCE_FILE if evidence_file is None else evidence_file,+ evidence_line,+ ]+ ),+ quote,+ title if needs_split or not grounded else "",+ source if not grounded else "",+ )+ digest = hashlib.sha256(_FIELD_SEP.join(parts).encode("utf-8")).hexdigest()+ return digest[:32]diff --git a/packages/core/src/repowise/core/persistence/crud/decision_ingest.py b/packages/core/src/repowise/core/persistence/crud/decision_ingest.pynew file mode 100644index 000000000..fe32d9b2d--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/decision_ingest.py@@ -0,0 +1,558 @@+"""Batch ingest of extracted decisions: grouping, dedupe and provenance."""++from __future__ import annotations++import json+from dataclasses import dataclass+from typing import Any++import structlog+from sqlalchemy import select+from sqlalchemy.ext.asyncio import AsyncSession++from repowise.core.analysis.decisions.provenance import rank_for_source++from ..decision_graph import sync_links_from_record+from ..models import DecisionRecord, _now_utc+from .decision_evidence import (+ _evidence_quote,+ _first_commit,+ _rederive_headline,+ _upsert_decision_evidence,+ list_decision_evidence,+)+from .decision_identity import (+ _extraction_kind,+ _extraction_status,+ _merge_status,+ derive_decision_id,+)+from .decision_review_meta import _write_candidate_meta+++def _normalize_title(title: str) -> str:+ """Normalize a decision title for cross-source dedup comparison."""+ import re as _re++ t = title.lower().strip()+ t = _re.sub(r"[^a-z0-9\s]", "", t)+ t = _re.sub(r"\s+", " ", t)+ return t++#: Sources whose evidence file is itself a reviewable, version-controlled+#: statement of the decision. Only these can accept without a person present.+_TRACKED_ARTIFACT_SOURCES: frozenset[str] = frozenset({"adr"})+++async def _accept_from_tracked_artifact(+ session: AsyncSession, rec: DecisionRecord, headline: dict+) -> None:+ """Grant acceptance to an ADR the repository has committed as accepted.++ Silent on anything that is not one: a missing rationale or scope means the+ document did not say enough to bind future work, and a refusal there is+ correct rather than an error to report. Re-running is a no-op because the+ acceptance already exists.+ """+ if headline.get("status") != "active":+ return+ if headline.get("source") not in _TRACKED_ARTIFACT_SOURCES:+ return+ artifact = headline.get("evidence_file") or ""+ if not artifact:+ return++ from repowise.core.analysis.decisions.accepter import is_tracked++ from ..models import Repository+ from .authority import AcceptanceRefusedError, is_accepted, record_acceptance++ if await is_accepted(session, rec.id):+ return+ repo = await session.get(Repository, rec.repository_id)+ if repo is None or not is_tracked(repo.local_path, artifact):+ # An uncommitted document is one machine's file. Letting it accept would+ # replace the human acceptance event with a file an agent can write.+ return+ try:+ await record_acceptance(+ session,+ rec,+ action="accepted",+ currency="active",+ artifact=artifact,+ kind="import",+ note="accepted by a tracked decision record",+ )+ except AcceptanceRefusedError as exc:+ structlog.get_logger(__name__).debug(+ "adr_acceptance_refused", decision_id=rec.id, reason=str(exc)+ )+++async def bulk_upsert_decisions(+ session: AsyncSession,+ repository_id: str,+ decisions: list[dict],+ *,+ vector_store: Any | None = None,+) -> list[str]:+ """Upsert decisions, accreting provenance instead of discarding losers.++ Decisions with near-identical normalized titles are merged into a single+ :class:`DecisionRecord`; every contributing source becomes a+ :class:`DecisionEvidence` row. The record's headline fields come from the+ highest-``source_rank`` contributor, and its confidence is recomputed from+ the best rank + the number of independently corroborating sources + the+ strongest surviving verification verdict.++ Idempotent: a full re-index converges (evidence rows upsert on their+ natural key); an incremental update adds new evidence and re-derives the+ headline + confidence from the union.++ When *vector_store* (the shared page-generator store) is supplied, a+ Phase-2C semantic pass augments the cheap normalized-title match: an+ incoming group that matched no existing record by title is looked up in the+ store, and if its nearest ``decision:`` neighbour clears the cosine+ threshold it is folded into that record as additional evidence — so+ paraphrases ("Use Redis" vs "Adopt Redis cache") collapse into one record.+ Every touched record is (re-)embedded into the store, so decisions are+ matchable next run *and* discoverable via ``search_codebase``.++ Returns the ids of every record touched (created or updated) this call, so+ a caller can run the Phase-3 supersession/conflict detection over just the+ records that changed.+ """+ groups = _group_by_normalized_title(decisions)+ if not groups:+ return []++ existing_by_norm = await _existing_records_by_title(session, repository_id)+ # A store hit (which returns a decision id) resolves back to the live+ # record through this map, and it is grown as records are created so+ # paraphrases *within* one batch also collapse.+ id_to_rec: dict[str, DecisionRecord] = {rec.id: rec for rec in existing_by_norm.values()}++ # Headline candidate per group: highest source rank, tie-break by+ # confidence. Hoisted so the batched embedding can see every group's+ # match text before the loop runs.+ headline_by_norm: dict[str, dict] = {+ norm: max(+ members,+ key=lambda d: (rank_for_source(d.get("source", "")), d.get("confidence", 0.0)),+ )+ for norm, members in groups.items()+ }+ dedup = await _SemanticDedup.prepare(vector_store, headline_by_norm, group_count=len(groups))++ touched_ids: list[str] = []+ captured: dict[str, tuple[str, bool]] = {}++ for norm, members in groups.items():+ headline = headline_by_norm[norm]+ rec = existing_by_norm.get(norm)++ # No title match → ask the store whether a semantically-equivalent+ # decision already exists, and fold into it if so (cheap title dedup+ # stays the first pass; this only runs on the residual).+ # A computed record must not fold into a prose record that merely+ # sounds alike; title dedup still applies to it.+ if rec is None and headline.get("source") != "conventions":+ match_id = await dedup.find_match(norm, headline)+ if match_id is not None and match_id in id_to_rec:+ rec = id_to_rec[match_id]+ existing_by_norm[norm] = rec++ # A dismissed record is a tombstone: leave it untouched (no headline+ # promotion, no new evidence, not re-embedded, not in touched_ids) so+ # dismissals survive every reindex.+ if rec is not None and rec.status == "dismissed":+ continue++ created = False+ if rec is None:+ identity = _headline_identity(repository_id, headline, members)+ # Identity is evidence, so two titles for one decision derive one+ # id. Folding into the record that id already names is the point of+ # the key: inserting instead would collide on the primary key, and+ # skipping would leave the duplicate this is meant to collapse.+ folded = id_to_rec.get(identity.id) or await session.get(DecisionRecord, identity.id)+ if folded is not None and folded.repository_id == repository_id:+ if folded.status == "dismissed":+ continue+ rec = folded+ else:+ rec = _new_record(repository_id, identity, headline)+ session.add(rec)+ await session.flush()+ created = True+ existing_by_norm[norm] = rec+ id_to_rec[rec.id] = rec+ if not created:+ _maybe_promote_headline(rec, headline)++ await _accrete_evidence(session, rec.id, members)++ # A committed ADR that says "accepted" is the one non-human acceptance+ # the contract allows, because the document is version controlled and+ # reviewable. Recorded here, where the parsed heading is still in hand.+ await _accept_from_tracked_artifact(session, rec, headline)++ # Re-derive headline confidence + verification from the FULL evidence+ # set (existing + just-added), so corroboration accrues across runs.+ _rederive_headline(rec, await list_decision_evidence(session, rec.id))+ rec.updated_at = _now_utc()+ touched_ids.append(rec.id)+ _record_capture(captured, rec.id, headline, members)++ await sync_links_from_record(session, rec)+ await dedup.index_record(rec, norm)++ await dedup.flush(touched_ids, id_to_rec)+ await _write_candidate_meta(session, repository_id, captured)++ await session.flush()+ return touched_ids+++def _group_by_normalized_title(decisions: list[dict]) -> dict[str, list[dict]]:+ """Incoming decisions keyed by normalized title; untitled ones are dropped."""+ groups: dict[str, list[dict]] = {}+ for d in decisions:+ norm = _normalize_title(d.get("title", ""))+ if not norm:+ continue+ groups.setdefault(norm, []).append(d)+ return groups+++async def _existing_records_by_title(+ session: AsyncSession, repository_id: str+) -> dict[str, DecisionRecord]:+ """This repository's records by normalized title, so cross-run merges land on one row.++ On a title collision the most authoritative existing row is canonical.+ """+ existing_rows = await session.execute(+ select(DecisionRecord).where(DecisionRecord.repository_id == repository_id)+ )+ existing_by_norm: dict[str, DecisionRecord] = {}+ for rec in existing_rows.scalars().all():+ norm = _normalize_title(rec.title)+ prior = existing_by_norm.get(norm)+ if prior is None or rank_for_source(rec.source) > rank_for_source(prior.source):+ existing_by_norm[norm] = rec+ return existing_by_norm+++@dataclass(frozen=True)+class _HeadlineIdentity:+ """The derived id of a group that matched no record, and the quote it pins."""++ id: str+ quote: str+++def _headline_identity(+ repository_id: str, headline: dict, members: list[dict]+) -> _HeadlineIdentity:+ # Pinned now and never revised: the identity is keyed on it, and a later+ # extraction rewording the same sentence must not move the id.+ quote = _evidence_quote(headline)+ derived = derive_decision_id(+ repository_id,+ headline.get("title", ""),+ source=headline.get("source", "cli"),+ evidence_file=headline.get("evidence_file"),+ affected_files=headline.get("affected_files") or [],+ evidence_line=headline.get("evidence_line"),+ identity_quote=quote,+ needs_split=any(bool(d.get("needs_split")) for d in members),+ )+ return _HeadlineIdentity(id=derived, quote=quote)+++def _new_record(+ repository_id: str, identity: _HeadlineIdentity, headline: dict+) -> DecisionRecord:+ """A new record whose every field comes from the group's headline."""+ # A source that measures its own conformance supplies the score;+ # otherwise the git-diff recompute fills it in later.+ headline_staleness = headline.get("staleness_score")+ return DecisionRecord(+ id=identity.id,+ identity_quote=identity.quote,+ repository_id=repository_id,+ title=headline.get("title", ""),+ status=_extraction_status(headline.get("status", "proposed")),+ kind=_extraction_kind(headline.get("kind")),+ context=headline.get("context") or "",+ decision=headline.get("decision") or "",+ rationale=headline.get("rationale") or "",+ alternatives_json=json.dumps(headline.get("alternatives") or []),+ consequences_json=json.dumps(headline.get("consequences") or []),+ affected_files_json=json.dumps(headline.get("affected_files") or []),+ # Rides with the file list it describes.+ scope_basis=headline.get("scope_basis") or "",+ affected_modules_json=json.dumps(headline.get("affected_modules") or []),+ tags_json=json.dumps(headline.get("tags") or []),+ evidence_commits_json=json.dumps(headline.get("evidence_commits") or []),+ source=headline.get("source", "cli"),+ evidence_file=headline.get("evidence_file"),+ evidence_line=headline.get("evidence_line"),+ confidence=headline.get("confidence", 0.5),+ staleness_score=(float(headline_staleness) if headline_staleness is not None else 0.0),+ )+++def _maybe_promote_headline(rec: DecisionRecord, headline: dict) -> None:+ """Promote *headline*'s fields onto *rec* if its source is at least as authoritative.++ Provenance still accretes either way; this only decides whose wording the+ record shows.+ """+ if rank_for_source(headline.get("source", "")) < rank_for_source(rec.source):+ return+ rec.title = headline.get("title", rec.title)+ rec.status = _merge_status(rec.status, headline.get("status", rec.status))+ # ``kind`` is deliberately absent: the noun is decided when the+ # record is created and re-extraction does not revisit it. Most+ # sources default the field rather than deciding it, so an incoming+ # value is usually the absence of a judgement; and this branch runs+ # on accepted records too, where flipping the noun would change what+ # a record governs behind the person who accepted it. The migration+ # refuses that for the same reason, and the manifest is where a+ # person changes it deliberately.+ rec.context = headline.get("context") or rec.context+ rec.decision = headline.get("decision") or rec.decision+ rec.rationale = headline.get("rationale") or rec.rationale+ rec.alternatives_json = json.dumps(headline.get("alternatives") or [])+ rec.consequences_json = json.dumps(headline.get("consequences") or [])+ rec.affected_files_json = json.dumps(headline.get("affected_files") or [])+ rec.scope_basis = headline.get("scope_basis") or ""+ rec.affected_modules_json = json.dumps(headline.get("affected_modules") or [])+ rec.tags_json = json.dumps(headline.get("tags") or [])+ rec.evidence_commits_json = json.dumps(headline.get("evidence_commits") or [])+ rec.source = headline.get("source", rec.source)+ rec.evidence_file = headline.get("evidence_file")+ rec.evidence_line = headline.get("evidence_line")+ # Only a source that measures its own conformance supplies this.+ if headline.get("staleness_score") is not None:+ rec.staleness_score = float(headline["staleness_score"])+++async def _accrete_evidence(session: AsyncSession, decision_id: str, members: list[dict]) -> None:+ """Upsert one evidence row per contributing source occurrence."""+ for d in members:+ src = d.get("source", "cli")+ await _upsert_decision_evidence(+ session,+ decision_id,+ source=src,+ source_rank=rank_for_source(src),+ evidence_file=d.get("evidence_file"),+ evidence_line=d.get("evidence_line"),+ evidence_commit=_first_commit(d),+ source_quote=_evidence_quote(d),+ confidence=d.get("confidence", 0.5),+ verification=d.get("verification", "unverified"),+ )+++def _record_capture(+ captured: dict[str, tuple[str, bool]],+ decision_id: str,+ headline: dict,+ members: list[dict],+) -> None:+ """Accumulate the lane and split flag this run's candidate-meta row carries.++ Accumulates because two title groups can fold onto one record. The split+ flag is read from every member: any contributor raising it flags the record.+ """+ prior_lane, prior_split = captured.get(decision_id, ("", False))+ captured[decision_id] = (+ prior_lane or headline.get("lane") or "",+ prior_split or any(bool(d.get("needs_split")) for d in members),+ )+++class _SemanticDedup:+ """Phase-2C semantic dedup against the shared vector store; inert without one.++ A group with no title match folds into its nearest stored neighbour above+ the cosine threshold. Batch mode embeds all groups up front, dedupes within+ the batch against a pending index and defers store writes to one upsert;+ stores that cannot search by vector use the per-item flow.+ """++ def __init__(+ self,+ vector_store: Any | None,+ *,+ group_vecs: dict[str, list[float]] | None = None,+ group_texts: dict[str, str] | None = None,+ ) -> None:+ self._store = vector_store+ self._group_vecs = group_vecs or {}+ self._group_texts = group_texts or {}+ self._batch_mode = group_vecs is not None+ # Vectors of records touched this batch but not yet written to the+ # store: the in-batch side of dedup in batch mode. A record is only+ # registered when its final text equals the group text its vector came+ # from, so the index never matches against text the record doesn't+ # actually contain.+ self._pending: Any | None = None+ if self._batch_mode:+ from repowise.core.analysis.decision_semantic_match import PendingDecisionIndex++ self._pending = PendingDecisionIndex()++ @classmethod+ async def prepare(+ cls,+ vector_store: Any | None,+ headline_by_norm: dict[str, dict],+ *,+ group_count: int,+ ) -> _SemanticDedup:+ if vector_store is None:+ return cls(None)+ from repowise.core.analysis.decision_semantic_match import decision_match_text++ ordered = [+ (norm, decision_match_text(h.get("title", ""), h.get("decision") or ""))+ for norm, h in headline_by_norm.items()+ ]+ ordered = [(norm, text) for norm, text in ordered if text]+ vecs = None+ if ordered and _supports_vector_search(vector_store):+ try:+ vecs = await vector_store.embed_texts([text for _, text in ordered])+ except Exception:+ vecs = None+ batched = vecs is not None and len(vecs) == len(ordered)++ structlog.get_logger(__name__).info(+ "decision_dedup_embedding",+ groups=group_count,+ batched=batched,+ )+ if not batched:+ return cls(vector_store)+ return cls(+ vector_store,+ group_vecs=dict(zip((n for n, _ in ordered), vecs, strict=True)),+ group_texts=dict(ordered),+ )++ async def find_match(self, norm: str, headline: dict) -> str | None:+ """The id of a stored decision this group paraphrases, if any."""+ if self._store is None:+ return None+ from repowise.core.analysis.decision_semantic_match import (+ find_duplicate_decision,+ find_duplicate_decision_by_vector,+ )++ q_vec = self._group_vecs.get(norm)+ if self._batch_mode and q_vec is not None:+ return await find_duplicate_decision_by_vector(+ self._store, q_vec, pending=self._pending+ )+ return await find_duplicate_decision(+ self._store,+ title=headline.get("title", ""),+ decision=headline.get("decision") or "",+ )++ async def index_record(self, rec: DecisionRecord, norm: str) -> None:+ """Make *rec* matchable by later groups in this batch and by future runs.++ Best-effort, never blocks the SQL upsert. In batch mode the store write+ is deferred to :meth:`flush`; until then the group vector stands in for+ the record in the pending index, but only when the record's final text+ IS the group text (created, or promoted from this headline). A fold that+ kept the record's own fields must not be matchable under the incoming+ group's text.+ """+ if self._store is None:+ return+ from repowise.core.analysis.decision_semantic_match import (+ decision_match_text,+ upsert_decision_vector,+ )++ if not self._batch_mode:+ await upsert_decision_vector(+ self._store,+ rec.id,+ title=rec.title,+ decision=rec.decision or "",+ evidence_file=rec.evidence_file,+ )+ return+ q_vec = self._group_vecs.get(norm)+ final_text = decision_match_text(rec.title, rec.decision or "")+ if (+ q_vec is not None+ and self._pending is not None+ and rec.id not in self._pending.ids+ and final_text == self._group_texts.get(norm)+ ):+ self._pending.add(rec.id, q_vec)++ async def flush(self, touched_ids: list[str], id_to_rec: dict[str, DecisionRecord]) -> None:+ """Batched store write of every touched record's *final* text.++ Post merge/promotion, reusing the vectors already computed for+ unchanged texts and embedding only the delta.+ """+ if self._store is None or not self._batch_mode or not touched_ids:+ return+ from repowise.core.analysis.decision_semantic_match import (+ decision_vector_item,+ upsert_decision_vectors,+ )++ items: list[tuple[str, str, dict]] = []+ for rid in dict.fromkeys(touched_ids):+ rec = id_to_rec.get(rid)+ if rec is None:+ continue+ item = decision_vector_item(+ rec.id,+ title=rec.title,+ decision=rec.decision or "",+ evidence_file=rec.evidence_file,+ )+ if item is not None:+ items.append(item)+ vectors_by_text = {+ text: self._group_vecs[norm]+ for norm, text in self._group_texts.items()+ if norm in self._group_vecs+ }+ await upsert_decision_vectors(self._store, items, vectors_by_text=vectors_by_text)+++def _supports_vector_search(vector_store: Any) -> bool:+ """Whether *vector_store* can be searched by a raw vector.++ Static capability check: the base-class ``search_by_vector`` is the+ "unsupported" sentinel. A runtime probe would conflate a transient store+ error with lack of support and silently fall back to the O(N)-round-trip+ path batch mode exists to avoid.+ """+ from repowise.core.persistence.vector_store import VectorStore+ from repowise.core.providers.embedding import store_has_semantic_vectors++ store_cls_search = getattr(type(vector_store), "search_by_vector", None)+ return (+ callable(store_cls_search)+ and store_cls_search is not VectorStore.search_by_vector+ # A keyless store supports the call and cannot answer it: every+ # matcher refuses on it. Without this the batch pays for a full embed+ # round whose vectors are then never read.+ and store_has_semantic_vectors(vector_store)+ )diff --git a/packages/core/src/repowise/core/persistence/crud/decision_repairs.py b/packages/core/src/repowise/core/persistence/crud/decision_repairs.pynew file mode 100644index 000000000..efb0ed304--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/decision_repairs.py@@ -0,0 +1,313 @@+"""Idempotent store-wide decision repairs and purges; each is a no-op once applied."""++from __future__ import annotations++from typing import Any++import structlog+from sqlalchemy import delete, or_, select+from sqlalchemy.ext.asyncio import AsyncSession++from repowise.core.analysis.decisions.provenance import SOURCE_RANK, rank_for_source++from ..models import (+ DecisionEdge,+ DecisionEvidence,+ DecisionNodeLink,+ DecisionRecord,+ _now_utc,+)+from .authority import accepted_predicate+from .decision_evidence import _rederive_headline, list_decision_evidence+++def _stale_rank_filter() -> Any:+ """SQL matching evidence rows whose stored rank disagrees with the ladder.++ Derived from ``SOURCE_RANK`` rather than hardcoded, so a future ladder edit+ is picked up here automatically instead of needing a second place updated.+ Bounded by the size of the ladder (a dozen terms), and it keeps the common+ case — a store already on the current ladder — to an indexed match on+ ``source`` that returns nothing, rather than hydrating the whole table.+ """+ return or_(+ *[+ (DecisionEvidence.source == name) & (DecisionEvidence.source_rank != rank)+ for name, rank in SOURCE_RANK.items()+ ]+ )+++async def reconcile_source_ranks(session: AsyncSession) -> int:+ """Re-stamp evidence rows whose stored rank predates a ``SOURCE_RANK`` edit.++ ``DecisionEvidence.source_rank`` is written once at insert time, and two+ ``ORDER BY`` clauses read it straight from the column, so it cannot simply be+ derived on read. That makes the ladder a value copied into every row: editing+ ``SOURCE_RANK`` leaves existing rows on the previous ladder, and because+ headline confidence comes from ``max(source_rank)`` across a decision's+ evidence, a store would end up scoring headlines off a mixture of two ladders+ without anything looking wrong.++ **This is the only repair path, deliberately.** An Alembic data migration was+ written first and removed: hosted runs the same persist pipeline as a local+ store, so the migration was redundant, and worse, it fixed the ranks *without*+ re-deriving confidence — which made this function's own no-op check pass on+ the next run and left hosted confidences on the old ladder permanently. One+ path cannot disagree with itself.++ Not repo-scoped, on purpose: the ladder is global, so in a workspace store+ holding several repositories every one of them is on the same ladder and all+ of them need the same repair.++ Idempotent. Returns the number of evidence rows re-stamped (0 when already+ reconciled, which is the steady state after the first run).+ """+ moved = (+ (await session.execute(select(DecisionEvidence).where(_stale_rank_filter())))+ .scalars()+ .all()+ )+ if not moved:+ return 0++ for row in moved:+ row.source_rank = rank_for_source(row.source)+ await session.flush()++ now = _now_utc()+ for decision_id in {row.decision_id for row in moved}:+ rec = await session.get(DecisionRecord, decision_id)+ if rec is None:+ continue+ _rederive_headline(rec, await list_decision_evidence(session, decision_id))+ rec.updated_at = now++ # Flush the re-scored headlines too, not just the ranks. Without this the+ # function returns with confidence still pending in the session, so whether+ # the repair survives depends on what the caller does next.+ await session.flush()++ structlog.get_logger(__name__).info(+ "decisions.source_ranks_reconciled", evidence_rows=len(moved)+ )+ return len(moved)+++async def reconcile_decision_confidence(session: AsyncSession) -> int:+ """Re-score headlines whose stored confidence predates a formula edit.++ The sibling of :func:`reconcile_source_ranks`, for the other half of the+ same problem. That one repairs a stale *input*, a rank copied into a row+ before the ladder moved, so it can find its work with an indexed+ filter. A change to the formula itself leaves every input valid and every+ stored score wrong, and nothing in a row marks which formula produced it,+ so the only way to find the work is to recompute and compare.++ Ceiling: two selects that hydrate the whole decision table, which is+ small; a store large enough for that to hurt would want the comparison+ pushed into SQL.++ Not repo-scoped, for the reason the ladder is not: the formula is global.+ Idempotent: 0 once reconciled, which is the steady state. Returns the+ number of records re-scored, and re-derives verification alongside it.++ A record with no evidence rows is left alone, deliberately: this pass+ re-derives, it does not invent. The rows in that state are the ones no+ extractor wrote, manual entries and manifest imports, and both score+ themselves where they are created.+ """+ evidence_by_id: dict[str, list[DecisionEvidence]] = {}+ for row in (await session.execute(select(DecisionEvidence))).scalars().all():+ evidence_by_id.setdefault(row.decision_id, []).append(row)+ if not evidence_by_id:+ return 0++ now = _now_utc()+ rescored = 0+ for rec in (await session.execute(select(DecisionRecord))).scalars().all():+ evidence = evidence_by_id.get(rec.id)+ if not evidence:+ continue # see the docstring: re-derive, never invent+ before = (rec.confidence, rec.verification)+ _rederive_headline(rec, evidence)+ if (rec.confidence, rec.verification) != before:+ rec.updated_at = now+ rescored += 1++ if rescored:+ await session.flush()+ structlog.get_logger(__name__).info(+ "decisions.confidence_reconciled", records=rescored+ )+ return rescored+++#: Prefix ``detect_supersessions_and_conflicts`` stamps on every edge it writes+#: (``auto-detected: <signal> (sim=0.81)``). It is the only marker that+#: separates a machine retirement from a human one, so the repair below keys on+#: it rather than on "has a supersedes edge".+_AUTO_EDGE_EVIDENCE_PREFIX = "auto-detected:"+++async def unretire_auto_superseded(session: AsyncSession) -> int:+ """Undo retirements made by the semantic supersession detector (3B).++ That detector scoped a conflict by cosine similarity and matched unrelated+ records; it is now off (``SEMANTIC_SUPERSESSION_ENABLED``). Turning it off+ only stops the *next* bad retirement — the rows it already flipped stay+ ``superseded``, which is a protected status, so re-extraction will never+ walk one back and a store would go on reporting a quarter of its corpus as+ retired-by-nothing. This is the other half of the same change.++ A row is repaired only when all three hold: status ``superseded``,+ ``superseded_by`` set, and an ``auto-detected:`` supersedes edge from that+ same successor. ``run_update_evolution`` sets the status without the+ pairing, the CLI's ``decision deprecate`` writes ``deprecated``, and+ ``upsert_decision_edge`` has no caller outside 3B — so nothing else in the+ codebase produces all three.++ One case is knowingly in range: a human can set both fields through+ ``PATCH /api/repos/{id}/decisions/{id}``, and if they did so by accepting+ one of this detector's own proposals in the UI, the auto edge is still+ there and the row is restored to ``proposed``. Accepted rather than+ guarded: the retirement they confirmed rests on the same bad match as the+ 74, ``proposed`` keeps the record readable, and ``decision confirm`` or the+ same PATCH puts it back in one step. The inverse — leaving a wrongly+ retired record hidden because a human once clicked through — is not+ recoverable at all.++ Restored to ``proposed``, not ``active``: the flip overwrote the previous+ status without recording it, and 3B fired on both. ``proposed`` is the+ lower claim of the two and is recoverable by ``decision confirm``; guessing+ ``active`` would mint governance a human never granted.++ The edges go too — both kinds it wrote. They are the same artifact from the+ same detector: ``supersedes`` is what ``build_lineage_chain`` walks, so+ leaving those would hand ``get_why`` a lineage that still presents the+ un-retired record as replaced, and ``conflicts_with`` is what the health+ dashboard counts as a governance smell.++ Not repo-scoped, like ``reconcile_source_ranks``: a workspace store holds+ several repositories and the detector ran over all of them.++ Idempotent — once repaired the edges are gone, so the next scan matches+ nothing. Returns the number of records restored.+ """+ auto_edges = (+ (+ await session.execute(+ select(DecisionEdge).where(+ DecisionEdge.kind.in_(("supersedes", "conflicts_with")),+ DecisionEdge.evidence.startswith(_AUTO_EDGE_EVIDENCE_PREFIX),+ )+ )+ )+ .scalars()+ .all()+ )+ if not auto_edges:+ return 0++ successors_by_target: dict[str, set[str]] = {}+ for edge in auto_edges:+ if edge.kind == "supersedes":+ successors_by_target.setdefault(edge.dst_decision_id, set()).add(edge.src_decision_id)++ now = _now_utc()+ restored = 0+ for target_id, successor_ids in successors_by_target.items():+ rec = await session.get(DecisionRecord, target_id)+ if rec is None or rec.status != "superseded" or rec.superseded_by is None:+ continue+ if rec.superseded_by not in successor_ids:+ # Retired by something else. Leave the *status* alone — the edge+ # still goes, below, because it is this detector's noise either way.+ continue+ rec.status = "proposed"+ rec.superseded_by = None+ rec.updated_at = now+ restored += 1++ for edge in auto_edges:+ await session.delete(edge)+ await session.flush()++ structlog.get_logger(__name__).info(+ "decisions.auto_supersessions_reverted",+ records_restored=restored,+ edges_deleted=len(auto_edges),+ )+ return restored+++async def purge_proposed_decisions_by_source(+ session: AsyncSession,+ repository_id: str,+ source: str,+) -> int:+ """Delete still-``proposed`` records of *source* plus their child rows.++ One-shot cleanup for retired extraction sources (today: the removed+ ``code_comment`` harvest, #751). Only ``proposed`` rows go — anything the+ user confirmed, deprecated, or dismissed is kept. Child rows are deleted+ explicitly rather than trusting the FK cascade, which on SQLite depends on+ the ``foreign_keys`` pragma. Returns the number of records deleted.+ """+ result = await session.execute(+ select(DecisionRecord.id).where(+ DecisionRecord.repository_id == repository_id,+ DecisionRecord.source == source,+ DecisionRecord.status == "proposed",+ ~accepted_predicate(),+ )+ )+ ids = [row[0] for row in result.all()]+ if not ids:+ return 0++ await _delete_decisions(session, ids)+ structlog.get_logger(__name__).info("decision_purge_by_source", source=source, deleted=len(ids))+ return len(ids)+++async def purge_proposed_decisions_outside_files(+ session: AsyncSession,+ repository_id: str,+ current_file_paths: set[str],+) -> int:+ """Delete unreviewed extracted decisions whose evidence file left scope."""+ result = await session.execute(+ select(DecisionRecord.id, DecisionRecord.evidence_file).where(+ DecisionRecord.repository_id == repository_id,+ DecisionRecord.status == "proposed",+ DecisionRecord.evidence_file.is_not(None),+ ~accepted_predicate(),+ )+ )+ ids = [row[0] for row in result.all() if row[1] not in current_file_paths]+ if not ids:+ return 0++ await _delete_decisions(session, ids)+ return len(ids)+++async def _delete_decisions(session: AsyncSession, ids: list[str]) -> None:+ """Delete decision records and their child rows.++ Child rows are deleted explicitly rather than trusting the FK cascade,+ which on SQLite depends on the ``foreign_keys`` pragma.+ """+ await session.execute(delete(DecisionEvidence).where(DecisionEvidence.decision_id.in_(ids)))+ await session.execute(+ delete(DecisionEdge).where(+ or_(+ DecisionEdge.src_decision_id.in_(ids),+ DecisionEdge.dst_decision_id.in_(ids),+ )+ )+ )+ await session.execute(delete(DecisionNodeLink).where(DecisionNodeLink.decision_id.in_(ids)))+ await session.execute(delete(DecisionRecord).where(DecisionRecord.id.in_(ids)))+ await session.flush()diff --git a/packages/core/src/repowise/core/persistence/crud/decision_review_meta.py b/packages/core/src/repowise/core/persistence/crud/decision_review_meta.pynew file mode 100644index 000000000..85d7318e6--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/decision_review_meta.py@@ -0,0 +1,77 @@+"""The review-queue row every open decision candidate is owed."""++from __future__ import annotations++from sqlalchemy import select+from sqlalchemy.ext.asyncio import AsyncSession++from repowise.core import __version__++from ..models import DecisionCandidateMeta, DecisionRecord+from .authority import candidate_predicate, candidate_review_signals, upsert_candidate_meta+++async def _write_candidate_meta(+ session: AsyncSession,+ repository_id: str,+ captured: dict[str, tuple[str, bool]],+ *,+ only: set[str] | None = None,+) -> None:+ """Refresh the review row every open candidate in the repository is owed.++ Not only the ones this run touched: the staging store does not re-emit an+ already-promoted decision, so a backlog would otherwise stay unjudged until+ something happened to re-extract each record individually. The signals come+ from the record, so an unchanged candidate costs a comparison and no write.+ """+ rows = (+ await session.execute(+ select(DecisionRecord, DecisionCandidateMeta)+ .outerjoin(+ DecisionCandidateMeta,+ DecisionCandidateMeta.decision_id == DecisionRecord.id,+ )+ .where(+ DecisionRecord.repository_id == repository_id,+ candidate_predicate(),+ *([DecisionRecord.id.in_(only)] if only is not None else []),+ )+ )+ ).all()++ for rec, meta in rows:+ priority, scope_unresolved = candidate_review_signals(rec)+ capture = captured.get(rec.id)+ if capture is None:+ # Not extracted this run, so the provenance is not ours to write:+ # only the contract-derived signals are refreshed, and a row with+ # no lane yet takes the record's own source.+ if (+ meta is not None+ and meta.lane+ and meta.review_priority == priority+ and meta.scope_unresolved == scope_unresolved+ ):+ continue+ await upsert_candidate_meta(+ session,+ rec,+ lane=meta.lane if meta is not None and meta.lane else rec.source,+ review_priority=priority,+ scope_unresolved=scope_unresolved,+ )+ continue++ lane, needs_split = capture+ await upsert_candidate_meta(+ session,+ rec,+ lane=lane or rec.source,+ extractor_version=__version__,+ review_priority=priority,+ # Raised, never cleared: a re-extraction must not walk back a split+ # somebody asked for.+ needs_split=True if needs_split else None,+ scope_unresolved=scope_unresolved,+ )diff --git a/packages/core/src/repowise/core/persistence/crud/decision_staleness.py b/packages/core/src/repowise/core/persistence/crud/decision_staleness.pynew file mode 100644index 000000000..fb6938052--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/decision_staleness.py@@ -0,0 +1,213 @@+"""Staleness scoring for decisions, from the git history of the files they name."""++from __future__ import annotations++import json+from datetime import datetime++from sqlalchemy import select+from sqlalchemy.ext.asyncio import AsyncSession++from ..decision_graph import sync_links_from_record+from ..models import DecisionRecord, GitMetadata, _now_utc+++async def _fill_git_meta_gaps(+ session: AsyncSession,+ repository_id: str,+ git_meta_map: dict[str, dict],+ wanted: set[str],+) -> dict[str, dict]:+ """``git_meta_map`` widened with persisted rows for the *wanted* paths.++ The caller's map is one run's git metadata, and on an incremental update+ that is only the files that changed in it. Staleness scoring reads a path+ with no entry as a file that is gone and scores it 1.00, so every decision+ over an untouched file went maximally stale for not having been touched.+ The persisted rows cover every file the index has seen, which is the set+ that answers "is this file still here"; a path missing from those too is+ genuinely untracked and still scores 1.00. The run's own entries win —+ they are this run's fresher numbers for the files it re-read.+ """+ missing = wanted - git_meta_map.keys()+ if not missing:+ return git_meta_map++ filled: dict[str, dict] = {}+ # Chunked so the IN clause stays under SQLite's bind-parameter ceiling.+ batch = sorted(missing)+ for start in range(0, len(batch), 500):+ rows = await session.execute(+ select(GitMetadata).where(+ GitMetadata.repository_id == repository_id,+ GitMetadata.file_path.in_(batch[start : start + 500]),+ )+ )+ for row in rows.scalars().all():+ filled[row.file_path] = {+ col.name: getattr(row, col.name)+ for col in row.__table__.columns+ if col.name not in ("id", "repository_id")+ }+ filled.update(git_meta_map)+ return filled+++async def recompute_decision_staleness(+ session: AsyncSession,+ repository_id: str,+ git_meta_map: dict[str, dict],+) -> int:+ """Recompute staleness_score for all active decisions. Returns update count.++ Also fills ``last_code_change`` from the same per-file dates the score is+ counted from, and re-derives ``affected_modules_json`` from the files each+ record names.+ The two belong in one pass because they are one repair: a record's module+ linkage used to be the first path segment, which in a ``packages/`` layout+ made almost every record claim ``packages`` or ``tests``, and the rows that+ predate the fix carry it. Deriving here rather than in a data migration+ keeps the single-repair-path rule — the one this repo already learned when+ an alembic migration and a runtime repair disagreed about confidence.+ """+ result = await session.execute(+ select(DecisionRecord).where(+ DecisionRecord.repository_id == repository_id,+ DecisionRecord.status.in_(["active", "proposed"]),+ )+ )+ decisions = list(result.scalars().all())++ affected_by_id: dict[str, list[str]] = {}+ for dec in decisions:+ affected = json.loads(dec.affected_files_json)+ if affected:+ affected_by_id[dec.id] = affected++ remodelled = _backfill_module_nodes(decisions, affected_by_id)+ modules_updated = len(remodelled)+ # This repair runs after ``backfill_decision_node_links`` on both index+ # paths, so a module array it rewrites here would otherwise keep the link+ # the backfill had already judged correct.+ for dec in remodelled:+ await sync_links_from_record(session, dec)+ if not affected_by_id:+ if modules_updated:+ await session.flush()+ return 0++ git_meta_map = await _fill_git_meta_gaps(+ session,+ repository_id,+ git_meta_map,+ {fp for paths in affected_by_id.values() for fp in paths},+ )++ now = _now_utc()+ updated = 0+ dated = 0+ for dec in decisions:+ affected = affected_by_id.get(dec.id)+ if not affected:+ continue++ # Before the conventions skip below: a date is a fact about the code,+ # not a score, so it is filled for every scoped record.+ if _refresh_last_code_change(dec, affected, git_meta_map, now):+ dated += 1++ # The source writes its own conformance share, and a git-diff score+ # would overwrite it. Only the score is skipped; the repairs above+ # still apply.+ if dec.source == "conventions":+ continue++ if _rescore_staleness(dec, affected, git_meta_map, now):+ updated += 1++ if updated or modules_updated or dated:+ await session.flush()+ # Deliberately the staleness count alone. The callers print this as+ # "N decisions rescored"; folding the module repair or the date fill into+ # it would report a rescore that did not happen.+ return updated+++def _refresh_last_code_change(+ dec: DecisionRecord, affected: list[str], git_meta_map: dict[str, dict], now: datetime+) -> bool:+ """Fill ``last_code_change`` from the files' git dates; True when it moved."""+ # Function-local, like every other analysis import here: the persistence+ # layer cannot depend on analysis at module scope.+ from repowise.core.analysis.decisions.extractor import DecisionExtractor, _as_aware_utc++ last_change = DecisionExtractor.last_code_change(affected, git_meta_map)+ stored = dec.last_code_change+ # SQLite drops tzinfo, so an aware value would differ from the naive+ # one it just wrote and rewrite ``updated_at`` on every run.+ if last_change == (_as_aware_utc(stored) if stored else None):+ return False+ dec.last_code_change = last_change+ dec.updated_at = now+ return True+++def _rescore_staleness(+ dec: DecisionRecord, affected: list[str], git_meta_map: dict[str, dict], now: datetime+) -> bool:+ """Recompute the git-diff staleness score; True when it moved by more than 0.01."""+ from repowise.core.analysis.decisions.extractor import DecisionExtractor++ new_score = DecisionExtractor.compute_staleness(+ dec.created_at,+ affected,+ git_meta_map,+ )+ if abs(new_score - dec.staleness_score) > 0.01:+ dec.staleness_score = round(new_score, 3)+ dec.updated_at = now+ return True+ return False+++def _backfill_module_nodes(+ decisions: list[DecisionRecord],+ affected_by_id: dict[str, list[str]],+) -> list[DecisionRecord]:+ """Re-derive each record's module linkage from its files.++ Returns the records whose array moved, because the caller has to relink+ each one: this rewrites half a record's scope and the graph holds the+ other half.++ Records naming no file are left alone: there is nothing to derive from, and+ an invented scope is worse than an absent one.+ """+ from repowise.core.analysis.decisions.scope import resolve_module_nodes++ moved = []+ for dec in decisions:+ affected = affected_by_id.get(dec.id)+ if not affected:+ continue+ derived = resolve_module_nodes(affected)+ if derived != json.loads(dec.affected_modules_json or "[]"):+ dec.affected_modules_json = json.dumps(derived)+ moved.append(dec)+ return moved+++async def get_stale_decisions(+ session: AsyncSession,+ repository_id: str,+ threshold: float = 0.5,+) -> list[DecisionRecord]:+ """Return active decisions with staleness_score >= threshold."""+ result = await session.execute(+ select(DecisionRecord).where(+ DecisionRecord.repository_id == repository_id,+ DecisionRecord.status.in_(["active"]),+ DecisionRecord.staleness_score >= threshold,+ )+ )+ return list(result.scalars().all())diff --git a/packages/core/src/repowise/core/persistence/crud/decisions.py b/packages/core/src/repowise/core/persistence/crud/decisions.pyindex d08fffb44..62edca3d5 100644--- a/packages/core/src/repowise/core/persistence/crud/decisions.py+++ b/packages/core/src/repowise/core/persistence/crud/decisions.py@@ -1,126 +1,61 @@"""CRUD operations for the decisions domain (repowise persistence layer).Split out of the former monolithic ``crud.py``; ``crud/__init__.py`` re-exports-every public name, so existing imports are unaffected.+every public name, so existing imports are unaffected. Single-record reads and+writes live here; the ``decision_*`` siblings hold the rest and are re-exported."""from __future__ import annotations-import hashlibimport json-from collections.abc import Iterablefrom datetime import datetimefrom typing import Any-import structlog-from sqlalchemy import case, delete, func, or_, select+from sqlalchemy import case, func, selectfrom sqlalchemy.ext.asyncio import AsyncSession-from repowise.core import __version__-from repowise.core.analysis.decisions.lifecycle import (- ARCHITECTURAL_KIND,- DECISION_KINDS,- DECISION_STATUS_ORDER,- RETIRED_STATUSES,- status_rank,-)-from repowise.core.analysis.decisions.provenance import (- SOURCE_RANK,- completeness,- compute_confidence,- normalize_text,- rank_for_source,-)-from repowise.core.analysis.decisions.scope import SCOPE_BASIS_STATED, binds_to_paths+from repowise.core.analysis.decisions.lifecycle import DECISION_STATUS_ORDER, RETIRED_STATUSES+from repowise.core.analysis.decisions.provenance import compute_confidence, rank_for_source+from repowise.core.analysis.decisions.scope import SCOPE_BASIS_STATEDfrom ..decision_graph import scope_modules, sync_links_from_record-from ..models import (- DecisionCandidateMeta,- DecisionEdge,- DecisionEvidence,- DecisionNodeLink,- DecisionRecord,- GitMetadata,- _now_utc,+from ..models import DecisionCandidateMeta, DecisionRecord, _now_utc+from .authority import accepted_predicate++# Re-exported: these moved to sibling modules and are still imported from here.+from .decision_evidence import ( # noqa: F401+ _json_list,+ _rederive_headline,+ list_decision_evidence,+ record_completeness,)-from .authority import (- accepted_predicate,- candidate_predicate,- candidate_review_signals,- upsert_candidate_meta,+from .decision_health import get_decision_health_summary # noqa: F401+from .decision_identity import ( # noqa: F401+ _extraction_kind,+ _extraction_status,+ _merge_status,+ derive_decision_id,+ identity_quote_for,)+from .decision_ingest import bulk_upsert_decisions # noqa: F401+from .decision_repairs import ( # noqa: F401+ purge_proposed_decisions_by_source,+ purge_proposed_decisions_outside_files,+ reconcile_decision_confidence,+ reconcile_source_ranks,+ unretire_auto_superseded,+)+from .decision_review_meta import _write_candidate_meta+from .decision_staleness import get_stale_decisions, recompute_decision_staleness # noqa: F401# ---------------------------------------------------------------------------# DecisionRecord CRUD# ----------------------------------------------------------------------------# Namespaced so a decision id can never collide with another kind of derived-# id, and versioned so a future change of the recipe is a visible new value-# rather than a silent reshuffle of every id in every store.-_ID_NAMESPACE = "repowise.decision.id.v2"--# The identity fields are joined with a separator none of them can contain,-# so no pair of distinct records can flatten to the same string.-_FIELD_SEP = "\x00"--# ``evidence_file`` is nullable and NULL is not the empty string here: the-# dedupe query treats them as different records, so the derivation must too.-_NULL_EVIDENCE_FILE = "\x01"--# The pinned quote is truncated before it is hashed. A quote long enough to-# reach this is a paragraph, and its opening already separates two decisions-# that share a file set; carrying the rest only widens the surface on which a-# re-mine can reword the identity out from under a reference somebody wrote.-_IDENTITY_QUOTE_CHARS = 200-_VALID_DECISION_STATUSES = frozenset({"proposed", "active", "deprecated", "superseded", "dismissed"})-# Statuses a human (or the evolution judge) set deliberately. Re-extraction-# must never walk one of these back to ``proposed``: a re-harvested source-# ties or beats the stored rank, so without this guard a reindex would flip-# confirmed decisions back into the review queue and resurrect dismissed ones-# (#751). ``dismissed`` is terminal: the row is kept purely as a tombstone so-# the same content never re-proposes.-_PROTECTED_STATUSES = frozenset({"active", "deprecated", "superseded", "dismissed"})---def _extraction_status(incoming: str) -> str:- """The status extraction is allowed to write.-- Extraction proposes; acceptance promotes. An incoming ``active`` is a- parsed ADR heading or an old code path's optimism, and neither is an- acceptance event, so it lands as a proposal. A tracked ADR still reaches- ``active`` — via :func:`_accept_from_tracked_artifact` below, which records- the document as the accepter instead of leaving the promotion anonymous.- """- return "proposed" if incoming == "active" else incoming---def _merge_status(existing: str, incoming: str) -> str:- """Resolve a re-extracted status against the stored one.-- A protected stored status wins over an incoming ``proposed``; anything else- follows the incoming value, after :func:`_extraction_status` has taken- authority out of extraction's hands.- """- incoming = _extraction_status(incoming)- if incoming == "proposed" and existing in _PROTECTED_STATUSES:- return existing- return incoming---def _extraction_kind(incoming: str | None) -> str:- """The kind extraction is allowed to write.-- Unrecognised falls to ``architectural`` rather than being stored: the- vocabulary has no CHECK behind it, every reader tests ``== AGREEMENT_KIND``,- and a junk value that merely happens to read as architectural everywhere is- safe by luck. Normalising here makes it safe by construction.- """- return incoming if incoming in DECISION_KINDS else ARCHITECTURAL_KIND-def _dedup_query(repository_id: str,@@ -145,95 +80,6 @@ def _dedup_query(return q.where(DecisionRecord.evidence_file.is_(None))-def identity_quote_for(text: str | None) -> str:- """The pinned form of a decision's quote, for identity only.-- Whitespace-collapsed, lowercased and truncated, so the identity survives a- re-mine that reflows or recases the same sentence. The stored column keeps- the verbatim text; only this derived form is hashed.- """- return normalize_text(text)[:_IDENTITY_QUOTE_CHARS]---def derive_decision_id(- repository_id: str,- title: str,- *,- source: str,- evidence_file: str | None,- affected_files: Iterable[str] = (),- evidence_line: int | None = None,- identity_quote: str = "",- needs_split: bool = False,-) -> str:- """The id a decision with this identity has, in every store, on every run.-- Keyed on the evidence rather than on the title. A title is the most- volatile thing a decision carries: two extractions of the same choice word- it differently and produce two records, and rewording one moves its id.- Over the dogfood store, title identity collapses **zero** duplicate pairs- while the files-plus-evidence-plus-quote key below collapses **120 pairs- across 42 classes**, and loses none of what title identity caught.-- The quote is load-bearing, and is why the file set alone is not the key:- two unrelated classes in that store share a file pair and are separated- only by what was said. It is also the fragile part, because most records- key on mined prose rather than on a committed span, so it is **pinned in- the row at first capture and never re-derived**. A later extraction that- rewords the same sentence then leaves the id where it is.-- A record with no evidence at all keeps its title, for the same reason the- rest of them lose it: identity is the evidence, and a record that has- none has no evidence identity to key on. Without this every scopeless,- quoteless record in a repository derives one id and the second one to be- written collides with the first. Two of the 357 records in the dogfood- store are in that state, and the construction paths that supply nothing- but a title are all in that state.-- A record flagged ``needs_split`` keeps its title in the key. A bundled- claim shares its files and its evidence with the separate decisions it- bundles, so evidence identity would fold all of them together and file two- decisions under a third one's name. Holding the flagged record on title- identity is what keeps it apart, and it is why the flag has to be set- before this key ships. The guard is partial: it reaches the bundles a- marker can see and no others, measured at 28 of 357 records held out for- 20 of 120 merges given up.-- ``source`` is deliberately **not** in the key. Two lanes that mined the- same sentence out of the same files recorded one decision, not two, and- the quote already separates two that merely share a scope.-- 32 lowercase hex, because ``DecisionRecord.id`` is ``String(32)`` and so- is every foreign key to it, so a truncated digest fits without a column- change. ``evidence_file`` is NULL far more often than not, so it gets a- sentinel no path can contain rather than collapsing into the empty string.-- Note the id follows the identity, so editing the scope or the pinned quote- moves it. :mod:`decision_id_migration`, which runs at the head of every- index, is what settles that, and it leaves an alias behind so an id- already written down keeps resolving.- """- files = sorted({str(f) for f in affected_files if f})- quote = identity_quote_for(identity_quote)- grounded = bool(files) or evidence_file is not None or bool(quote)- parts = (- _ID_NAMESPACE,- repository_id,- json.dumps(files),- json.dumps(- [- _NULL_EVIDENCE_FILE if evidence_file is None else evidence_file,- evidence_line,- ]- ),- quote,- title if needs_split or not grounded else "",- source if not grounded else "",- )- digest = hashlib.sha256(_FIELD_SEP.join(parts).encode("utf-8")).hexdigest()- return digest[:32]--async def find_decision_by_title(session: AsyncSession,repository_id: str,@@ -744,1288 +590,3 @@ async def delete_decision(session: AsyncSession, decision_id: str) -> bool:await session.delete(rec)await session.flush()return True---def _normalize_title(title: str) -> str:- """Normalize a decision title for cross-source dedup comparison."""- import re as _re-- t = title.lower().strip()- t = _re.sub(r"[^a-z0-9\s]", "", t)- t = _re.sub(r"\s+", " ", t)- return t---def _evidence_quote(d: dict) -> str:- """Pick the verbatim span recorded as this evidence row's source quote.-- Prefers the LLM/parser-supplied ``source_quote``; falls back to the- decision/rationale text so an evidence row is never empty.- """- return (d.get("source_quote") or d.get("decision") or d.get("rationale") or "").strip()---async def _upsert_decision_evidence(- session: AsyncSession,- decision_id: str,- *,- source: str,- source_rank: int,- evidence_file: str | None,- evidence_line: int | None,- evidence_commit: str | None,- source_quote: str,- confidence: float,- verification: str,-) -> None:- """Insert or update one evidence row, idempotent on its natural key.-- Natural key is ``(decision_id, source, evidence_file, evidence_commit)``.- NULLs are matched explicitly (SQLite treats NULLs as distinct in a unique- constraint), so re-indexing the same source converges instead of- duplicating, while an incremental update adds genuinely new evidence.- """- q = select(DecisionEvidence).where(- DecisionEvidence.decision_id == decision_id,- DecisionEvidence.source == source,- )- q = (- q.where(DecisionEvidence.evidence_file == evidence_file)- if evidence_file is not None- else q.where(DecisionEvidence.evidence_file.is_(None))- )- q = (- q.where(DecisionEvidence.evidence_commit == evidence_commit)- if evidence_commit is not None- else q.where(DecisionEvidence.evidence_commit.is_(None))- )- existing = (await session.execute(q)).scalar_one_or_none()-- if existing is not None:- existing.source_rank = source_rank- existing.evidence_line = evidence_line- existing.source_quote = source_quote- existing.confidence = confidence- existing.verification = verification- return-- session.add(- DecisionEvidence(- decision_id=decision_id,- source=source,- source_rank=source_rank,- evidence_file=evidence_file,- evidence_line=evidence_line,- evidence_commit=evidence_commit,- source_quote=source_quote,- confidence=confidence,- verification=verification,- )- )---def _json_list(raw: str | None) -> list[str]:- """A JSON array column as a list; anything unparsable reads as empty."""- try:- value = json.loads(raw or "[]")- except (TypeError, ValueError):- return []- return [str(v) for v in value] if isinstance(value, list) else []---def _best_verification(values: list[str]) -> str:- """Reduce per-evidence verdicts to the strongest: exact > fuzzy > unverified."""- if "exact" in values:- return "exact"- if "fuzzy" in values:- return "fuzzy"- return "unverified"---def record_completeness(rec: DecisionRecord) -> int:- """:func:`completeness` for a stored row, parsing its two JSON columns."""- return completeness(- decision=rec.decision,- rationale=rec.rationale,- context=rec.context,- consequences=_json_list(rec.consequences_json),- alternatives=_json_list(rec.alternatives_json),- )---def _rederive_headline(rec: DecisionRecord, evidence: list[DecisionEvidence]) -> None:- """Set a record's confidence + verification from its full evidence set.-- The single definition of how a headline is scored. Every writer uses it:- the upsert path after accreting a run's evidence, ``reconcile_source_ranks``- after a ladder edit, and ``reconcile_decision_confidence`` after a formula- edit. Kept as one function because two of them were briefly copy-pasted and- nothing would have forced the copies to stay equal.-- Confidence rises with the best source rank and with the number of- *independent* corroborating sources, and with how much of its body the- record fills, so it is derived from the whole set plus the row rather than- from whichever evidence row happened to arrive last. No-op on empty- evidence: a record with nothing behind it keeps whatever it had.- """- if not evidence:- return- best_ver = _best_verification([e.verification for e in evidence])- rec.confidence = compute_confidence(- max(e.source_rank for e in evidence),- len({e.source for e in evidence}),- best_ver,- filled_fields=record_completeness(rec),- )- rec.verification = best_ver---def _stale_rank_filter() -> Any:- """SQL matching evidence rows whose stored rank disagrees with the ladder.-- Derived from ``SOURCE_RANK`` rather than hardcoded, so a future ladder edit- is picked up here automatically instead of needing a second place updated.- Bounded by the size of the ladder (a dozen terms), and it keeps the common- case — a store already on the current ladder — to an indexed match on- ``source`` that returns nothing, rather than hydrating the whole table.- """- return or_(- *[- (DecisionEvidence.source == name) & (DecisionEvidence.source_rank != rank)- for name, rank in SOURCE_RANK.items()- ]- )---async def reconcile_source_ranks(session: AsyncSession) -> int:- """Re-stamp evidence rows whose stored rank predates a ``SOURCE_RANK`` edit.-- ``DecisionEvidence.source_rank`` is written once at insert time, and two- ``ORDER BY`` clauses read it straight from the column, so it cannot simply be- derived on read. That makes the ladder a value copied into every row: editing- ``SOURCE_RANK`` leaves existing rows on the previous ladder, and because- headline confidence comes from ``max(source_rank)`` across a decision's- evidence, a store would end up scoring headlines off a mixture of two ladders- without anything looking wrong.-- **This is the only repair path, deliberately.** An Alembic data migration was- written first and removed: hosted runs the same persist pipeline as a local- store, so the migration was redundant, and worse, it fixed the ranks *without*- re-deriving confidence — which made this function's own no-op check pass on- the next run and left hosted confidences on the old ladder permanently. One- path cannot disagree with itself.-- Not repo-scoped, on purpose: the ladder is global, so in a workspace store- holding several repositories every one of them is on the same ladder and all- of them need the same repair.-- Idempotent. Returns the number of evidence rows re-stamped (0 when already- reconciled, which is the steady state after the first run).- """- moved = (- (await session.execute(select(DecisionEvidence).where(_stale_rank_filter())))- .scalars()- .all()- )- if not moved:- return 0-- for row in moved:- row.source_rank = rank_for_source(row.source)- await session.flush()-- now = _now_utc()- for decision_id in {row.decision_id for row in moved}:- rec = await session.get(DecisionRecord, decision_id)- if rec is None:- continue- _rederive_headline(rec, await list_decision_evidence(session, decision_id))- rec.updated_at = now-- # Flush the re-scored headlines too, not just the ranks. Without this the- # function returns with confidence still pending in the session, so whether- # the repair survives depends on what the caller does next.- await session.flush()-- structlog.get_logger(__name__).info(- "decisions.source_ranks_reconciled", evidence_rows=len(moved)- )- return len(moved)---async def reconcile_decision_confidence(session: AsyncSession) -> int:- """Re-score headlines whose stored confidence predates a formula edit.-- The sibling of :func:`reconcile_source_ranks`, for the other half of the- same problem. That one repairs a stale *input*, a rank copied into a row- before the ladder moved, so it can find its work with an indexed- filter. A change to the formula itself leaves every input valid and every- stored score wrong, and nothing in a row marks which formula produced it,- so the only way to find the work is to recompute and compare.-- Ceiling: two selects that hydrate the whole decision table, which is- small; a store large enough for that to hurt would want the comparison- pushed into SQL.-- Not repo-scoped, for the reason the ladder is not: the formula is global.- Idempotent: 0 once reconciled, which is the steady state. Returns the- number of records re-scored, and re-derives verification alongside it.-- A record with no evidence rows is left alone, deliberately: this pass- re-derives, it does not invent. The rows in that state are the ones no- extractor wrote, manual entries and manifest imports, and both score- themselves where they are created.- """- evidence_by_id: dict[str, list[DecisionEvidence]] = {}- for row in (await session.execute(select(DecisionEvidence))).scalars().all():- evidence_by_id.setdefault(row.decision_id, []).append(row)- if not evidence_by_id:- return 0-- now = _now_utc()- rescored = 0- for rec in (await session.execute(select(DecisionRecord))).scalars().all():- evidence = evidence_by_id.get(rec.id)- if not evidence:- continue # see the docstring: re-derive, never invent- before = (rec.confidence, rec.verification)- _rederive_headline(rec, evidence)- if (rec.confidence, rec.verification) != before:- rec.updated_at = now- rescored += 1-- if rescored:- await session.flush()- structlog.get_logger(__name__).info(- "decisions.confidence_reconciled", records=rescored- )- return rescored---#: Prefix ``detect_supersessions_and_conflicts`` stamps on every edge it writes-#: (``auto-detected: <signal> (sim=0.81)``). It is the only marker that-#: separates a machine retirement from a human one, so the repair below keys on-#: it rather than on "has a supersedes edge".-_AUTO_EDGE_EVIDENCE_PREFIX = "auto-detected:"---async def unretire_auto_superseded(session: AsyncSession) -> int:- """Undo retirements made by the semantic supersession detector (3B).-- That detector scoped a conflict by cosine similarity and matched unrelated- records; it is now off (``SEMANTIC_SUPERSESSION_ENABLED``). Turning it off- only stops the *next* bad retirement — the rows it already flipped stay- ``superseded``, which is a protected status, so re-extraction will never- walk one back and a store would go on reporting a quarter of its corpus as- retired-by-nothing. This is the other half of the same change.-- A row is repaired only when all three hold: status ``superseded``,- ``superseded_by`` set, and an ``auto-detected:`` supersedes edge from that- same successor. ``run_update_evolution`` sets the status without the- pairing, the CLI's ``decision deprecate`` writes ``deprecated``, and- ``upsert_decision_edge`` has no caller outside 3B — so nothing else in the- codebase produces all three.-- One case is knowingly in range: a human can set both fields through- ``PATCH /api/repos/{id}/decisions/{id}``, and if they did so by accepting- one of this detector's own proposals in the UI, the auto edge is still- there and the row is restored to ``proposed``. Accepted rather than- guarded: the retirement they confirmed rests on the same bad match as the- 74, ``proposed`` keeps the record readable, and ``decision confirm`` or the- same PATCH puts it back in one step. The inverse — leaving a wrongly- retired record hidden because a human once clicked through — is not- recoverable at all.-- Restored to ``proposed``, not ``active``: the flip overwrote the previous- status without recording it, and 3B fired on both. ``proposed`` is the- lower claim of the two and is recoverable by ``decision confirm``; guessing- ``active`` would mint governance a human never granted.-- The edges go too — both kinds it wrote. They are the same artifact from the- same detector: ``supersedes`` is what ``build_lineage_chain`` walks, so- leaving those would hand ``get_why`` a lineage that still presents the- un-retired record as replaced, and ``conflicts_with`` is what the health- dashboard counts as a governance smell.-- Not repo-scoped, like ``reconcile_source_ranks``: a workspace store holds- several repositories and the detector ran over all of them.-- Idempotent — once repaired the edges are gone, so the next scan matches- nothing. Returns the number of records restored.- """- auto_edges = (- (- await session.execute(- select(DecisionEdge).where(- DecisionEdge.kind.in_(("supersedes", "conflicts_with")),- DecisionEdge.evidence.startswith(_AUTO_EDGE_EVIDENCE_PREFIX),- )- )- )- .scalars()- .all()- )- if not auto_edges:- return 0-- successors_by_target: dict[str, set[str]] = {}- for edge in auto_edges:- if edge.kind == "supersedes":- successors_by_target.setdefault(edge.dst_decision_id, set()).add(edge.src_decision_id)-- now = _now_utc()- restored = 0- for target_id, successor_ids in successors_by_target.items():- rec = await session.get(DecisionRecord, target_id)- if rec is None or rec.status != "superseded" or rec.superseded_by is None:- continue- if rec.superseded_by not in successor_ids:- # Retired by something else. Leave the *status* alone — the edge- # still goes, below, because it is this detector's noise either way.- continue- rec.status = "proposed"- rec.superseded_by = None- rec.updated_at = now- restored += 1-- for edge in auto_edges:- await session.delete(edge)- await session.flush()-- structlog.get_logger(__name__).info(- "decisions.auto_supersessions_reverted",- records_restored=restored,- edges_deleted=len(auto_edges),- )- return restored---async def list_decision_evidence(- session: AsyncSession,- decision_id: str,-) -> list[DecisionEvidence]:- """Return all evidence rows for a decision, highest source rank first."""- result = await session.execute(- select(DecisionEvidence)- .where(DecisionEvidence.decision_id == decision_id)- .order_by(DecisionEvidence.source_rank.desc(), DecisionEvidence.created_at.asc())- )- return list(result.scalars().all())---def _first_commit(d: dict) -> str | None:- commits = d.get("evidence_commits") or []- return commits[0] if commits else None---#: Sources whose evidence file is itself a reviewable, version-controlled-#: statement of the decision. Only these can accept without a person present.-_TRACKED_ARTIFACT_SOURCES: frozenset[str] = frozenset({"adr"})---async def _accept_from_tracked_artifact(- session: AsyncSession, rec: DecisionRecord, headline: dict-) -> None:- """Grant acceptance to an ADR the repository has committed as accepted.-- Silent on anything that is not one: a missing rationale or scope means the- document did not say enough to bind future work, and a refusal there is- correct rather than an error to report. Re-running is a no-op because the- acceptance already exists.- """- if headline.get("status") != "active":- return- if headline.get("source") not in _TRACKED_ARTIFACT_SOURCES:- return- artifact = headline.get("evidence_file") or ""- if not artifact:- return-- from repowise.core.analysis.decisions.accepter import is_tracked-- from ..models import Repository- from .authority import AcceptanceRefusedError, is_accepted, record_acceptance-- if await is_accepted(session, rec.id):- return- repo = await session.get(Repository, rec.repository_id)- if repo is None or not is_tracked(repo.local_path, artifact):- # An uncommitted document is one machine's file. Letting it accept would- # replace the human acceptance event with a file an agent can write.- return- try:- await record_acceptance(- session,- rec,- action="accepted",- currency="active",- artifact=artifact,- kind="import",- note="accepted by a tracked decision record",- )- except AcceptanceRefusedError as exc:- structlog.get_logger(__name__).debug(- "adr_acceptance_refused", decision_id=rec.id, reason=str(exc)- )---async def bulk_upsert_decisions(- session: AsyncSession,- repository_id: str,- decisions: list[dict],- *,- vector_store: Any | None = None,-) -> list[str]:- """Upsert decisions, accreting provenance instead of discarding losers.-- Decisions with near-identical normalized titles are merged into a single- :class:`DecisionRecord`; every contributing source becomes a- :class:`DecisionEvidence` row. The record's headline fields come from the- highest-``source_rank`` contributor, and its confidence is recomputed from- the best rank + the number of independently corroborating sources + the- strongest surviving verification verdict.-- Idempotent: a full re-index converges (evidence rows upsert on their- natural key); an incremental update adds new evidence and re-derives the- headline + confidence from the union.-- When *vector_store* (the shared page-generator store) is supplied, a- Phase-2C semantic pass augments the cheap normalized-title match: an- incoming group that matched no existing record by title is looked up in the- store, and if its nearest ``decision:`` neighbour clears the cosine- threshold it is folded into that record as additional evidence — so- paraphrases ("Use Redis" vs "Adopt Redis cache") collapse into one record.- Every touched record is (re-)embedded into the store, so decisions are- matchable next run *and* discoverable via ``search_codebase``.-- Returns the ids of every record touched (created or updated) this call, so- a caller can run the Phase-3 supersession/conflict detection over just the- records that changed.- """- # Group incoming decisions by normalized title.- groups: dict[str, list[dict]] = {}- for d in decisions:- norm = _normalize_title(d.get("title", ""))- if not norm:- continue- groups.setdefault(norm, []).append(d)-- if not groups:- return []-- # Map existing records (this repo) by normalized title so cross-run merges- # land on the same row. On a title collision keep the most authoritative- # existing row as canonical.- existing_rows = await session.execute(- select(DecisionRecord).where(DecisionRecord.repository_id == repository_id)- )- existing_by_norm: dict[str, DecisionRecord] = {}- for rec in existing_rows.scalars().all():- norm = _normalize_title(rec.title)- prior = existing_by_norm.get(norm)- if prior is None or rank_for_source(rec.source) > rank_for_source(prior.source):- existing_by_norm[norm] = rec-- # Phase 2C semantic dedup runs against the shared vector store. ``id_to_rec``- # lets a store hit (which returns a decision id) resolve back to the live- # record, and is grown as records are created so paraphrases *within* one- # batch also collapse.- id_to_rec: dict[str, DecisionRecord] = {rec.id: rec for rec in existing_by_norm.values()}-- # Headline candidate per group: highest source rank, tie-break by- # confidence. Hoisted so the batched embedding below can see every- # group's match text before the loop runs.- headline_by_norm: dict[str, dict] = {- norm: max(- members,- key=lambda d: (rank_for_source(d.get("source", "")), d.get("confidence", 0.0)),- )- for norm, members in groups.items()- }-- # Batched embedding for dedup. The per-item path embeds one query per- # residual group and one upsert per touched record — thousands of serial- # network round-trips on a first index. Instead: embed every group's- # match text in a few chunked requests up front, search the store by raw- # vector, dedup *within* the batch against a local pending index, and- # defer the store writes to one batched upsert after the loop. Stores- # that can't hand back vectors (no embedder, or search_by_vector left at- # the base-class default) keep the per-item flow unchanged.- batch_mode = False- group_vecs: dict[str, list[float]] = {}- group_texts: dict[str, str] = {}- if vector_store is not None:- from repowise.core.analysis.decision_semantic_match import (- PendingDecisionIndex,- decision_match_text,- decision_vector_item,- find_duplicate_decision,- find_duplicate_decision_by_vector,- upsert_decision_vector,- upsert_decision_vectors,- )- from repowise.core.persistence.vector_store import VectorStore- from repowise.core.providers.embedding import store_has_semantic_vectors-- # Static capability check: the base-class search_by_vector is the- # "unsupported" sentinel. A runtime probe would conflate a transient- # store error with lack of support and silently fall back to the- # O(N)-round-trip path this branch exists to avoid.- store_cls_search = getattr(type(vector_store), "search_by_vector", None)- supports_vector_search = (- callable(store_cls_search)- and store_cls_search is not VectorStore.search_by_vector- # A keyless store supports the call and cannot answer it: every- # matcher below refuses on it. Without this the batch pays for a- # full embed round whose vectors are then never read.- and store_has_semantic_vectors(vector_store)- )-- ordered = [- (norm, decision_match_text(h.get("title", ""), h.get("decision") or ""))- for norm, h in headline_by_norm.items()- ]- ordered = [(norm, text) for norm, text in ordered if text]- if ordered and supports_vector_search:- try:- vecs = await vector_store.embed_texts([text for _, text in ordered])- except Exception:- vecs = None- if vecs is not None and len(vecs) == len(ordered):- group_vecs = dict(zip((n for n, _ in ordered), vecs, strict=True))- group_texts = dict(ordered)- batch_mode = True-- structlog.get_logger(__name__).info(- "decision_dedup_embedding",- groups=len(groups),- batched=batch_mode,- )-- # Vectors of records touched this batch but not yet written to the store —- # the in-batch side of dedup in batch mode. A record is only registered- # when its final text equals the group text its vector came from, so the- # index never matches against text the record doesn't actually contain.- pending = PendingDecisionIndex() if batch_mode else None-- touched_ids: list[str] = []- captured: dict[str, tuple[str, bool]] = {}-- for norm, members in groups.items():- headline = headline_by_norm[norm]- rec = existing_by_norm.get(norm)-- # No title match → ask the store whether a semantically-equivalent- # decision already exists, and fold into it if so (cheap title dedup- # stays the first pass; this only runs on the residual).- # A computed record must not fold into a prose record that merely- # sounds alike; title dedup still applies to it.- if rec is None and vector_store is not None and headline.get("source") != "conventions":- q_vec = group_vecs.get(norm)- if batch_mode and q_vec is not None:- match_id = await find_duplicate_decision_by_vector(- vector_store, q_vec, pending=pending- )- else:- match_id = await find_duplicate_decision(- vector_store,- title=headline.get("title", ""),- decision=headline.get("decision") or "",- )- if match_id is not None and match_id in id_to_rec:- rec = id_to_rec[match_id]- existing_by_norm[norm] = rec-- # A dismissed record is a tombstone: leave it untouched (no headline- # promotion, no new evidence, not re-embedded, not in touched_ids) so- # dismissals survive every reindex.- if rec is not None and rec.status == "dismissed":- continue-- if rec is None:- headline_title = headline.get("title", "")- headline_source = headline.get("source", "cli")- headline_evidence_file = headline.get("evidence_file")- headline_files = headline.get("affected_files") or []- # Pinned now and never revised: the identity is keyed on it, and a- # later extraction rewording the same sentence must not move the id.- headline_quote = _evidence_quote(headline)- derived = derive_decision_id(- repository_id,- headline_title,- source=headline_source,- evidence_file=headline_evidence_file,- affected_files=headline_files,- evidence_line=headline.get("evidence_line"),- identity_quote=headline_quote,- needs_split=any(bool(d.get("needs_split")) for d in members),- )- # Identity is evidence, so two titles for one decision derive one- # id. Folding into the record that id already names is the point of- # the key: inserting instead would collide on the primary key, and- # skipping would leave the duplicate this is meant to collapse.- folded = id_to_rec.get(derived) or await session.get(DecisionRecord, derived)- if folded is not None and folded.repository_id == repository_id:- if folded.status == "dismissed":- continue- rec = folded- existing_by_norm[norm] = rec- id_to_rec[rec.id] = rec- if rec is None:- # A source that measures its own conformance supplies the score;- # otherwise the git-diff recompute fills it in later.- headline_staleness = headline.get("staleness_score")- rec = DecisionRecord(- id=derived,- identity_quote=headline_quote,- repository_id=repository_id,- title=headline_title,- status=_extraction_status(headline.get("status", "proposed")),- kind=_extraction_kind(headline.get("kind")),- context=headline.get("context") or "",- decision=headline.get("decision") or "",- rationale=headline.get("rationale") or "",- alternatives_json=json.dumps(headline.get("alternatives") or []),- consequences_json=json.dumps(headline.get("consequences") or []),- affected_files_json=json.dumps(headline_files),- # Rides with the file list it describes.- scope_basis=headline.get("scope_basis") or "",- affected_modules_json=json.dumps(headline.get("affected_modules") or []),- tags_json=json.dumps(headline.get("tags") or []),- evidence_commits_json=json.dumps(headline.get("evidence_commits") or []),- source=headline_source,- evidence_file=headline_evidence_file,- evidence_line=headline.get("evidence_line"),- confidence=headline.get("confidence", 0.5),- staleness_score=(- float(headline_staleness) if headline_staleness is not None else 0.0- ),- )- session.add(rec)- await session.flush()- existing_by_norm[norm] = rec- id_to_rec[rec.id] = rec- elif rank_for_source(headline.get("source", "")) >= rank_for_source(rec.source):- # A new contributor at least as authoritative as the current- # headline → promote its fields (provenance still accretes below).- rec.title = headline.get("title", rec.title)- rec.status = _merge_status(rec.status, headline.get("status", rec.status))- # ``kind`` is deliberately absent: the noun is decided when the- # record is created and re-extraction does not revisit it. Most- # sources default the field rather than deciding it, so an incoming- # value is usually the absence of a judgement; and this branch runs- # on accepted records too, where flipping the noun would change what- # a record governs behind the person who accepted it. The migration- # refuses that for the same reason, and the manifest is where a- # person changes it deliberately.- rec.context = headline.get("context") or rec.context- rec.decision = headline.get("decision") or rec.decision- rec.rationale = headline.get("rationale") or rec.rationale- rec.alternatives_json = json.dumps(headline.get("alternatives") or [])- rec.consequences_json = json.dumps(headline.get("consequences") or [])- rec.affected_files_json = json.dumps(headline.get("affected_files") or [])- rec.scope_basis = headline.get("scope_basis") or ""- rec.affected_modules_json = json.dumps(headline.get("affected_modules") or [])- rec.tags_json = json.dumps(headline.get("tags") or [])- rec.evidence_commits_json = json.dumps(headline.get("evidence_commits") or [])- rec.source = headline.get("source", rec.source)- rec.evidence_file = headline.get("evidence_file")- rec.evidence_line = headline.get("evidence_line")- # Only a source that measures its own conformance supplies this.- if headline.get("staleness_score") is not None:- rec.staleness_score = float(headline["staleness_score"])-- # Accrete one evidence row per contributing source occurrence.- for d in members:- src = d.get("source", "cli")- await _upsert_decision_evidence(- session,- rec.id,- source=src,- source_rank=rank_for_source(src),- evidence_file=d.get("evidence_file"),- evidence_line=d.get("evidence_line"),- evidence_commit=_first_commit(d),- source_quote=_evidence_quote(d),- confidence=d.get("confidence", 0.5),- verification=d.get("verification", "unverified"),- )-- # A committed ADR that says "accepted" is the one non-human acceptance- # the contract allows, because the document is version controlled and- # reviewable. Recorded here, where the parsed heading is still in hand.- await _accept_from_tracked_artifact(session, rec, headline)-- # Re-derive headline confidence + verification from the FULL evidence- # set (existing + just-added), so corroboration accrues across runs.- _rederive_headline(rec, await list_decision_evidence(session, rec.id))- rec.updated_at = _now_utc()- touched_ids.append(rec.id)- # Two title groups can fold onto one record, so this accumulates:- # the later group must not drop what the earlier one raised.- #- # The split flag is read from every member, not from the headline- # alone. The headline is the highest-ranked source in the group, and- # the lane that notices a claim bundles two decisions is usually not- # the highest-ranked one: a session-mined candidate that flags itself- # loses its flag the moment a CLI-authored record shares its title.- # A flag raised by any contributor is a flag on the record.- prior_lane, prior_split = captured.get(rec.id, ("", False))- captured[rec.id] = (- prior_lane or headline.get("lane") or "",- prior_split or any(bool(d.get("needs_split")) for d in members),- )-- await sync_links_from_record(session, rec)-- # (Re-)embed the record into the shared store so it's matchable by- # later groups in this batch + future runs, and discoverable via- # search_codebase. Best-effort — never blocks the SQL upsert. In- # batch mode the write is deferred to one batched upsert after the- # loop; until then the group vector stands in for the record in the- # pending index — but only when the record's final text IS the group- # text (created, or promoted from this headline). A fold that kept- # the record's own fields must not be matchable under the incoming- # group's text.- if vector_store is not None:- if batch_mode:- q_vec = group_vecs.get(norm)- final_text = decision_match_text(rec.title, rec.decision or "")- if (- q_vec is not None- and pending is not None- and rec.id not in pending.ids- and final_text == group_texts.get(norm)- ):- pending.add(rec.id, q_vec)- else:- await upsert_decision_vector(- vector_store,- rec.id,- title=rec.title,- decision=rec.decision or "",- evidence_file=rec.evidence_file,- )-- # Batched store write: upsert every touched record's *final* text (post- # merge/promotion) in one go, reusing the vectors already computed for- # unchanged texts and embedding only the delta.- if vector_store is not None and batch_mode and touched_ids:- items: list[tuple[str, str, dict]] = []- for rid in dict.fromkeys(touched_ids):- rec = id_to_rec.get(rid)- if rec is None:- continue- item = decision_vector_item(- rec.id,- title=rec.title,- decision=rec.decision or "",- evidence_file=rec.evidence_file,- )- if item is not None:- items.append(item)- vectors_by_text = {- text: group_vecs[norm] for norm, text in group_texts.items() if norm in group_vecs- }- await upsert_decision_vectors(vector_store, items, vectors_by_text=vectors_by_text)-- await _write_candidate_meta(session, repository_id, captured)-- await session.flush()- return touched_ids---async def _write_candidate_meta(- session: AsyncSession,- repository_id: str,- captured: dict[str, tuple[str, bool]],- *,- only: set[str] | None = None,-) -> None:- """Refresh the review row every open candidate in the repository is owed.-- Not only the ones this run touched: the staging store does not re-emit an- already-promoted decision, so a backlog would otherwise stay unjudged until- something happened to re-extract each record individually. The signals come- from the record, so an unchanged candidate costs a comparison and no write.- """- rows = (- await session.execute(- select(DecisionRecord, DecisionCandidateMeta)- .outerjoin(- DecisionCandidateMeta,- DecisionCandidateMeta.decision_id == DecisionRecord.id,- )- .where(- DecisionRecord.repository_id == repository_id,- candidate_predicate(),- *([DecisionRecord.id.in_(only)] if only is not None else []),- )- )- ).all()-- for rec, meta in rows:- priority, scope_unresolved = candidate_review_signals(rec)- capture = captured.get(rec.id)- if capture is None:- # Not extracted this run, so the provenance is not ours to write:- # only the contract-derived signals are refreshed, and a row with- # no lane yet takes the record's own source.- if (- meta is not None- and meta.lane- and meta.review_priority == priority- and meta.scope_unresolved == scope_unresolved- ):- continue- await upsert_candidate_meta(- session,- rec,- lane=meta.lane if meta is not None and meta.lane else rec.source,- review_priority=priority,- scope_unresolved=scope_unresolved,- )- continue-- lane, needs_split = capture- await upsert_candidate_meta(- session,- rec,- lane=lane or rec.source,- extractor_version=__version__,- review_priority=priority,- # Raised, never cleared: a re-extraction must not walk back a split- # somebody asked for.- needs_split=True if needs_split else None,- scope_unresolved=scope_unresolved,- )---async def purge_proposed_decisions_by_source(- session: AsyncSession,- repository_id: str,- source: str,-) -> int:- """Delete still-``proposed`` records of *source* plus their child rows.-- One-shot cleanup for retired extraction sources (today: the removed- ``code_comment`` harvest, #751). Only ``proposed`` rows go — anything the- user confirmed, deprecated, or dismissed is kept. Child rows are deleted- explicitly rather than trusting the FK cascade, which on SQLite depends on- the ``foreign_keys`` pragma. Returns the number of records deleted.- """- result = await session.execute(- select(DecisionRecord.id).where(- DecisionRecord.repository_id == repository_id,- DecisionRecord.source == source,- DecisionRecord.status == "proposed",- ~accepted_predicate(),- )- )- ids = [row[0] for row in result.all()]- if not ids:- return 0-- await session.execute(delete(DecisionEvidence).where(DecisionEvidence.decision_id.in_(ids)))- await session.execute(- delete(DecisionEdge).where(- or_(- DecisionEdge.src_decision_id.in_(ids),- DecisionEdge.dst_decision_id.in_(ids),- )- )- )- await session.execute(delete(DecisionNodeLink).where(DecisionNodeLink.decision_id.in_(ids)))- await session.execute(delete(DecisionRecord).where(DecisionRecord.id.in_(ids)))- await session.flush()- structlog.get_logger(__name__).info("decision_purge_by_source", source=source, deleted=len(ids))- return len(ids)---async def purge_proposed_decisions_outside_files(- session: AsyncSession,- repository_id: str,- current_file_paths: set[str],-) -> int:- """Delete unreviewed extracted decisions whose evidence file left scope."""- result = await session.execute(- select(DecisionRecord.id, DecisionRecord.evidence_file).where(- DecisionRecord.repository_id == repository_id,- DecisionRecord.status == "proposed",- DecisionRecord.evidence_file.is_not(None),- ~accepted_predicate(),- )- )- ids = [row[0] for row in result.all() if row[1] not in current_file_paths]- if not ids:- return 0-- await session.execute(delete(DecisionEvidence).where(DecisionEvidence.decision_id.in_(ids)))- await session.execute(- delete(DecisionEdge).where(- or_(- DecisionEdge.src_decision_id.in_(ids),- DecisionEdge.dst_decision_id.in_(ids),- )- )- )- await session.execute(delete(DecisionNodeLink).where(DecisionNodeLink.decision_id.in_(ids)))- await session.execute(delete(DecisionRecord).where(DecisionRecord.id.in_(ids)))- await session.flush()- return len(ids)---async def _fill_git_meta_gaps(- session: AsyncSession,- repository_id: str,- git_meta_map: dict[str, dict],- wanted: set[str],-) -> dict[str, dict]:- """``git_meta_map`` widened with persisted rows for the *wanted* paths.-- The caller's map is one run's git metadata, and on an incremental update- that is only the files that changed in it. Staleness scoring reads a path- with no entry as a file that is gone and scores it 1.00, so every decision- over an untouched file went maximally stale for not having been touched.- The persisted rows cover every file the index has seen, which is the set- that answers "is this file still here"; a path missing from those too is- genuinely untracked and still scores 1.00. The run's own entries win —- they are this run's fresher numbers for the files it re-read.- """- missing = wanted - git_meta_map.keys()- if not missing:- return git_meta_map-- filled: dict[str, dict] = {}- # Chunked so the IN clause stays under SQLite's bind-parameter ceiling.- batch = sorted(missing)- for start in range(0, len(batch), 500):- rows = await session.execute(- select(GitMetadata).where(- GitMetadata.repository_id == repository_id,- GitMetadata.file_path.in_(batch[start : start + 500]),- )- )- for row in rows.scalars().all():- filled[row.file_path] = {- col.name: getattr(row, col.name)- for col in row.__table__.columns- if col.name not in ("id", "repository_id")- }- filled.update(git_meta_map)- return filled---async def recompute_decision_staleness(- session: AsyncSession,- repository_id: str,- git_meta_map: dict[str, dict],-) -> int:- """Recompute staleness_score for all active decisions. Returns update count.-- Also fills ``last_code_change`` from the same per-file dates the score is- counted from, and re-derives ``affected_modules_json`` from the files each- record names.- The two belong in one pass because they are one repair: a record's module- linkage used to be the first path segment, which in a ``packages/`` layout- made almost every record claim ``packages`` or ``tests``, and the rows that- predate the fix carry it. Deriving here rather than in a data migration- keeps the single-repair-path rule — the one this repo already learned when- an alembic migration and a runtime repair disagreed about confidence.- """- result = await session.execute(- select(DecisionRecord).where(- DecisionRecord.repository_id == repository_id,- DecisionRecord.status.in_(["active", "proposed"]),- )- )- decisions = list(result.scalars().all())-- affected_by_id: dict[str, list[str]] = {}- for dec in decisions:- affected = json.loads(dec.affected_files_json)- if affected:- affected_by_id[dec.id] = affected-- remodelled = _backfill_module_nodes(decisions, affected_by_id)- modules_updated = len(remodelled)- # This repair runs after ``backfill_decision_node_links`` on both index- # paths, so a module array it rewrites here would otherwise keep the link- # the backfill had already judged correct.- for dec in remodelled:- await sync_links_from_record(session, dec)- if not affected_by_id:- if modules_updated:- await session.flush()- return 0-- git_meta_map = await _fill_git_meta_gaps(- session,- repository_id,- git_meta_map,- {fp for paths in affected_by_id.values() for fp in paths},- )-- now = _now_utc()- updated = 0- dated = 0- # Function-local, like every other analysis import here: the persistence- # layer cannot depend on analysis at module scope.- from repowise.core.analysis.decisions.extractor import (- DecisionExtractor,- _as_aware_utc,- )-- for dec in decisions:- affected = affected_by_id.get(dec.id)- if not affected:- continue-- # Before the conventions skip below: a date is a fact about the code,- # not a score, so it is filled for every scoped record.- last_change = DecisionExtractor.last_code_change(affected, git_meta_map)- stored = dec.last_code_change- # SQLite drops tzinfo, so an aware value would differ from the naive- # one it just wrote and rewrite ``updated_at`` on every run.- if last_change != (_as_aware_utc(stored) if stored else None):- dec.last_code_change = last_change- dec.updated_at = now- dated += 1-- # The source writes its own conformance share, and a git-diff score- # would overwrite it. Only the score is skipped; the repairs above- # still apply.- if dec.source == "conventions":- continue-- new_score = DecisionExtractor.compute_staleness(- dec.created_at,- affected,- git_meta_map,- )- if abs(new_score - dec.staleness_score) > 0.01:- dec.staleness_score = round(new_score, 3)- dec.updated_at = now- updated += 1-- if updated or modules_updated or dated:- await session.flush()- # Deliberately the staleness count alone. The callers print this as- # "N decisions rescored"; folding the module repair or the date fill into- # it would report a rescore that did not happen.- return updated---def _backfill_module_nodes(- decisions: list[DecisionRecord],- affected_by_id: dict[str, list[str]],-) -> list[DecisionRecord]:- """Re-derive each record's module linkage from its files.-- Returns the records whose array moved, because the caller has to relink- each one: this rewrites half a record's scope and the graph holds the- other half.-- Records naming no file are left alone: there is nothing to derive from, and- an invented scope is worse than an absent one.- """- from repowise.core.analysis.decisions.scope import resolve_module_nodes-- moved = []- for dec in decisions:- affected = affected_by_id.get(dec.id)- if not affected:- continue- derived = resolve_module_nodes(affected)- if derived != json.loads(dec.affected_modules_json or "[]"):- dec.affected_modules_json = json.dumps(derived)- moved.append(dec)- return moved---async def get_stale_decisions(- session: AsyncSession,- repository_id: str,- threshold: float = 0.5,-) -> list[DecisionRecord]:- """Return active decisions with staleness_score >= threshold."""- result = await session.execute(- select(DecisionRecord).where(- DecisionRecord.repository_id == repository_id,- DecisionRecord.status.in_(["active"]),- DecisionRecord.staleness_score >= threshold,- )- )- return list(result.scalars().all())---async def get_decision_health_summary(- session: AsyncSession,- repository_id: str,-) -> dict:- """Return decision health: counts by lane, stale decisions, ungoverned hotspots.-- The five list fields are returned ranked worst-first: stale by staleness,- proposed by confidence, ungoverned hotspots by temporal hotspot score,- retired by lane (history before tombstone) and unscoped by confidence. A- caller that shows only the first few shows the few that matter.- Callers may truncate; they must not re-order.-- ``retired_decisions`` holds ``(lane, record)`` pairs: the lane is derived- from the acceptance, which the ``status`` column may disagree with.-- Counts the acceptance, not the status column. The key names are the ones- every caller already renders, and they keep their product meaning:- ``active`` is what governs, ``proposed`` is what nobody has accepted. What- changed is that a record reaches ``active`` here by having an acceptance,- the same way it does on every other governance read. Counting the column- made this the one surface still reporting a hundred governing decisions on- a store whose acceptances were empty, and it fed ``ungoverned_hotspots``,- so unaccepted records were also suppressing the files nothing governs.- """- result = await session.execute(- select(DecisionRecord).where(- DecisionRecord.repository_id == repository_id,- )- )- all_decisions = list(result.scalars().all())- from .authority import decision_currencies-- currencies = await decision_currencies(session, repository_id, all_decisions)-- counts = {- "active": 0,- "proposed": 0,- "deprecated": 0,- "superseded": 0,- "dismissed": 0,- "stale": 0,- # Accepted records naming no file. They score 0.0 because the staleness- # question cannot be asked of them, which renders identically to a- # record whose code genuinely has not moved — so they are counted- # separately rather than banked as fresh.- "unscoped": 0,- }- stale_decisions: list[DecisionRecord] = []- proposed_decisions: list[DecisionRecord] = []- # Counted-only lanes. The record is in hand at the ``continue`` that drops- # it, so naming it costs no query. ``retired`` carries its lane because- # that lane is derived from the acceptance where there is one, and a- # record's ``status`` column may disagree with it.- retired_decisions: list[tuple[str, DecisionRecord]] = []- unscoped_decisions: list[DecisionRecord] = []-- # Files an *accepted* decision names. A candidate naming a hotspot does not- # make it governed, and counting one did: it removed the file from- # ``ungoverned_hotspots``, which is the list whose whole job is to say- # where nobody has decided anything.- governed_files: set[str] = set()- for d in all_decisions:- currency = currencies.get(d.id)- if currency is None:- # No acceptance, so it does not govern. A retired one is not- # awaiting review either: it keeps the status that retired it and- # stays out of the queue, which is the one thing a tombstone must- # never be counted as.- if d.status in ("dismissed", "deprecated", "superseded"):- counts[d.status] = counts.get(d.status, 0) + 1- retired_decisions.append((d.status, d))- else:- counts["proposed"] += 1- proposed_decisions.append(d)- continue- if currency == "superseded":- counts["superseded"] += 1- retired_decisions.append(("superseded", d))- continue- if currency == "dismissed":- counts["dismissed"] += 1- retired_decisions.append(("dismissed", d))- continue- counts["active"] += 1- if currency == "needs_review":- counts["stale"] += 1- stale_decisions.append(d)- if currency == "uncheckable":- counts["unscoped"] += 1- unscoped_decisions.append(d)- # ``governed_files`` is the denominator for "ungoverned hotspots",- # so a footprint would suppress every file its commit touched.- if not binds_to_paths(d.scope_basis):- continue- for fp in json.loads(d.affected_files_json):- governed_files.add(fp)-- # Find ungoverned hotspots, hottest first. Sorting these by path put the file- # most in need of a decision behind whatever sorts alphabetically first,- # with the score that answers the question sitting unread one column over.- # The key is the one ``routers/overview.py`` already applies to these same- # rows in SQL (score descending with NULLs last, then churn) rather than a- # second answer to "which hotspot matters most". A NULL score is genuinely- # unknown and is not the same as a measured zero.- hotspot_result = await session.execute(- select(- GitMetadata.file_path,- GitMetadata.temporal_hotspot_score,- GitMetadata.churn_percentile,- ).where(- GitMetadata.repository_id == repository_id,- GitMetadata.is_hotspot == True, # noqa: E712- )- )- hotspot_rows = {row[0]: (row[1], row[2]) for row in hotspot_result.all()}-- def _hotspot_rank(file_path: str) -> tuple[bool, float, float, str]:- score, churn = hotspot_rows[file_path]- return (score is None, -(score or 0.0), -(churn or 0.0), file_path)-- ungoverned = sorted(hotspot_rows.keys() - governed_files, key=_hotspot_rank)-- # Rank here rather than at the five call sites, none of which does. The MCP- # health dashboard and ``repowise decision health`` cut all three lists; the- # overview attention panel cuts the hotspots and renders the other two in the- # order it is handed; the decisions route serves them whole in that order;- # and ``health/governance.py`` walks them to *write* one finding row per- # entry. So the callers that truncate were showing whichever rows the scan- # returned first, and the ones that do not were still listing them by it,- # while the score answering "which of these first" rides on every record,- # unread. The ``or 0.0`` guards match how every other reader of these two- # fields spells it rather than trusting a column default to have been- # back-filled; the id tiebreak makes the key total, so two runs agree.- stale_decisions.sort(key=lambda d: (-(d.staleness_score or 0.0), d.id))- proposed_decisions.sort(key=lambda d: (-(d.confidence or 0.0), d.id))- # Retired by lane, history before tombstone. Not by ``updated_at``: it- # moves on any write, so it does not say when a record was retired.- # ``unscoped`` by confidence, the key ``proposed`` already uses.- retired_decisions.sort(key=lambda pair: (status_rank(pair[0]), pair[1].id))- unscoped_decisions.sort(key=lambda d: (-(d.confidence or 0.0), d.id))-- # Phase 3B: surface contradictory active decisions (conflicts_with edges).- from ..decision_graph import list_conflict_edges-- by_id = {d.id: d for d in all_decisions}- conflicts: list[dict] = []- for edge in await list_conflict_edges(session, repository_id):- src = by_id.get(edge.src_decision_id)- dst = by_id.get(edge.dst_decision_id)- if src is None or dst is None:- continue- conflicts.append(- {- "src": {"id": src.id, "title": src.title, "status": src.status},- "dst": {"id": dst.id, "title": dst.title, "status": dst.status},- "confidence": edge.confidence,- "evidence": edge.evidence,- }- )- counts["conflicts"] = len(conflicts)-- return {- "summary": counts,- "stale_decisions": stale_decisions,- "proposed_awaiting_review": proposed_decisions,- "ungoverned_hotspots": ungoverned,- "conflicts": conflicts,- "retired_decisions": retired_decisions,- "unscoped_decisions": unscoped_decisions,- }