Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(decisions): fold stored duplicate candidates in one sweep (#2314)
Raghav Chamadiyadbf7ed0e78a0
diff --git a/docs/layers/DECISIONS.md b/docs/layers/DECISIONS.mdindex 788b47235..5fa582c81 100644--- a/docs/layers/DECISIONS.md+++ b/docs/layers/DECISIONS.md@@ -141,10 +141,20 @@ review as though they were the team's.| `decision confirm ID...` | Accept, optionally editing the reason and scope on the way. Takes many ids. || `decision confirm ID` on a decision | Reaffirm it after review. || `decision merge ID INTO_ID` | Fold a candidate into an existing decision. The old id resolves to the target. |+| `decision dedupe` | Fold candidates that duplicate another candidate, in one sweep. Dry run until `--apply`. || `decision split ID` | Flag a candidate as bundling two choices. Never splits it for you. || `decision dismiss ID...` | Tombstone it. On an accepted decision this also withdraws its authority. Takes many ids. || `decision deprecate ID --superseded-by ID2` | Retire it with an explicit lineage edge. |+`merge` folds one pair and needs the target to be an accepted decision, because+that merge changes what governs. `dedupe` is the other case: a store that has+been indexed many times accumulates the same decision written several ways, and+none of those records governs anything yet. It folds each duplicate into the+candidate it duplicates — evidence, governed files and all — and only ever into+a record it is directly measured against, so a chain of near-neighbours is not+collapsed into one decision. Run it without `--apply` first; it prints exactly+what it would fold.+`confirm` and `dismiss` take a list because a repository accumulates candidatesfaster than anyone reviews them one command at a time. Every id in a batch goesthrough the same acceptance contract as a single one, and each is applied on itsdiff --git a/docs/reference/CLI_REFERENCE.md b/docs/reference/CLI_REFERENCE.mdindex c8cc81731..365b9fbee 100644--- a/docs/reference/CLI_REFERENCE.md+++ b/docs/reference/CLI_REFERENCE.md@@ -1012,6 +1012,7 @@ repowise decision candidates [PATH] # what is awaiting review; these governrepowise decision confirm ID... [PATH] # accept candidates: this is what makes them governrepowise decision dismiss ID... [PATH] # tombstone them (sticky; never re-proposed)repowise decision merge ID INTO_ID # fold a candidate into an existing decision+repowise decision dedupe [PATH] # fold candidates that duplicate another candidate (dry run by default)repowise decision split ID [PATH] # flag a candidate as bundling two choicesrepowise decision deprecate ID [PATH] # retire a decision, optionally naming its successorrepowise decision health [PATH] # health dashboarddiff --git a/packages/cli/src/repowise/cli/commands/decision_cmd.py b/packages/cli/src/repowise/cli/commands/decision_cmd.pyindex 59838e849..df75232ef 100644--- a/packages/cli/src/repowise/cli/commands/decision_cmd.py+++ b/packages/cli/src/repowise/cli/commands/decision_cmd.py@@ -70,6 +70,7 @@ def _register_config_commands() -> None:from repowise.cli.commands.decision_review_cmd import (candidates_command,+ dedupe_command,export_command,import_command,merge_command,@@ -82,6 +83,7 @@ def _register_config_commands() -> None:migrate_command,candidates_command,merge_command,+ dedupe_command,split_command,export_command,import_command,diff --git a/packages/cli/src/repowise/cli/commands/decision_review_cmd.py b/packages/cli/src/repowise/cli/commands/decision_review_cmd.pyindex 11fd9b493..5ab23ca61 100644--- a/packages/cli/src/repowise/cli/commands/decision_review_cmd.py+++ b/packages/cli/src/repowise/cli/commands/decision_review_cmd.py@@ -11,10 +11,11 @@ import clickfrom rich.table import Tablefrom repowise.cli.helpers import console, get_db_url_for_repo, run_async-from repowise.cli.output import emit_json, emit_refusal, format_option+from repowise.cli.output import emit_json, emit_refusal, format_option, notice_console__all__ = ["candidates_command",+ "dedupe_command","export_command","import_command","merge_command",@@ -246,6 +247,116 @@ def merge_command(decision_id: str, into_id: str, path: str | None, fmt: str) ->)+# ---------------------------------------------------------------------------+# decision dedupe+# ---------------------------------------------------------------------------++[email protected]("dedupe")[email protected]("path", required=False, default=None)[email protected]("--apply", "apply_", is_flag=True, default=False, help="Write the plan.")+ "--tau",+ default=None,+ # Floored well below the 0.83 default but not at zero: this command deletes+ # what it groups, and a threshold near zero groups the whole corpus.+ type=click.FloatRange(0.5, 1.0),+ help="Cosine threshold to fold at [default: 0.83]",+)+ "--limit",+ default=10,+ type=click.IntRange(min=1),+ show_default=True,+ help="Clusters to list in the report.",+)+@format_option()+def dedupe_command(+ path: str | None, apply_: bool, tau: float | None, limit: int, fmt: str+) -> None:+ """Fold candidates that duplicate another candidate into one record.++ Dry run by default. Folds only into a record the duplicate is itself+ measured against, so a chain of near-neighbours is never collapsed into+ one decision. Evidence and governed files move to the surviving record,+ and every folded id keeps resolving.+ """+ from repowise.cli.commands.decision_cmd import _resolve_decision_repo+ from repowise.cli.providers.embedders import build_embedder, resolve_embedder_for_repo+ from repowise.cli.providers.vector_store import build_vector_store+ from repowise.core.analysis.decisions.semantic_match import DEFAULT_DEDUP_TAU+ from repowise.core.providers.embedding.base import MockEmbedder++ repo_path = _resolve_decision_repo(path, fmt)+ threshold = DEFAULT_DEDUP_TAU if tau is None else tau++ embedder = build_embedder(resolve_embedder_for_repo(repo_path), repo_path)+ # Test vectors are hashes, so they would group decisions that have nothing+ # to do with each other, and this command deletes what it groups.+ if isinstance(embedder, MockEmbedder):+ emit_refusal(+ "no_real_embedder",+ "Decisions can only be compared with a real embedder, not test vectors.",+ fmt,+ remedy="Set an embedder key and run 'repowise reindex'.",+ )+ vector_store = build_vector_store(repo_path, embedder)+ if vector_store is None:+ emit_refusal(+ "no_vector_store",+ "This repo has no vector store to compare decisions with.",+ fmt,+ remedy="Set an embedder key and run 'repowise reindex'.",+ )++ async def _run():+ from repowise.core.persistence import get_session+ from repowise.core.persistence.decision_dedupe import (+ apply_dedupe,+ plan_dedupe,+ render_plan,+ )++ engine, sf = await _open(repo_path)+ try:+ async with get_session(sf) as session:+ repo_id = await _repository_id(session, repo_path)+ plan = await plan_dedupe(+ session, repo_id, vector_store=vector_store, tau=threshold+ )+ if apply_:+ try:+ plan = await apply_dedupe(+ session, repo_id, vector_store=vector_store, plan=plan+ )+ except Exception:+ # A fold that dies after its vectors are dropped leaves+ # records the matcher can no longer see; reindex is what+ # puts them back, and nothing else says so.+ notice_console(fmt).print(+ "[yellow]The sweep failed partway. No records were "+ "folded, but run [cyan]repowise reindex[/cyan] to "+ "restore any decision vectors it had removed.[/yellow]"+ )+ raise+ return plan, render_plan(plan, limit=limit)+ finally:+ await engine.dispose()+ await vector_store.close()++ plan, report = run_async(_run())+ if fmt == "json":+ payload = plan.as_dict()+ payload["applied"] = apply_+ emit_json(payload)+ return+ if apply_ and not plan.degraded:+ report = report.replace("(dry run)", "(applied)")+ console.print(report)+ if not apply_ and plan.clusters:+ console.print("\n[dim]Nothing was written. Re-run with --apply.[/dim]")++@click.command("split")@click.argument("decision_id")@click.argument("path", required=False, default=None)diff --git a/packages/core/src/repowise/core/persistence/decision_dedupe.py b/packages/core/src/repowise/core/persistence/decision_dedupe.pynew file mode 100644index 000000000..36e37db20--- /dev/null+++ b/packages/core/src/repowise/core/persistence/decision_dedupe.py@@ -0,0 +1,487 @@+"""Folding duplicate candidates that are already stored.++Cross-run dedup stops new duplicates at the pass that creates the record; it+cannot reach the ones written before it existed. This is the retroactive half:+it produces the store the write path would have produced, by folding each+duplicate into the record it duplicates and moving its evidence there.++Stars, not components+---------------------+A record is folded only into a record it is itself measured at or above *tau*+against. Similarity is not transitive: A~B and B~C at the threshold leaves A~C+free to be anything, and a chain of near-neighbours can span two genuinely+different decisions bridged by records that bundle both. So a connected+component is covered by one or more *stars*: take its most authoritative member+as canonical, fold the members holding a direct edge to it, and repeat on+whatever is left. Every fold performed is a measured pair, never an inferred+one.++What is eligible+----------------+Both ends must be candidates nobody has touched. Folding into an accepted+decision changes what governs, and :func:`merge_candidate` already owns that+with the review trail it deserves; a retired record carries lineage a fold+would erase; and a record a reviewer has flagged, split or dismissed holds a+judgement that its deletion would throw away. This sweep grants no authority+and overrules no person.++What a fold keeps, and what it drops+------------------------------------+Governed files and modules, evidence commits and tags are unioned onto the+canonical, so no file loses the decision that governs it. Of the duplicate's+evidence, every row citing a source, file and commit the canonical does not+already hold moves across; a row matching one the canonical has corroborates+nothing new and goes with its record, matching the convergence+``_upsert_decision_evidence`` performs on the same key. So the set of distinct+evidence keys is conserved and the row count falls.++Three things are dropped rather than merged, each because merging them is+worse than losing them. ``consequences`` and ``alternatives`` are prose kept+in a list, and unioning them across a cluster yields mostly restatements of+each other; the canonical is chosen partly on completeness, so the+best-stated version survives. The duplicate's review row goes with it, taking+the lane and grounding that recorded how *that* record was extracted — facts+about a row that no longer exists. Nothing here is reconstructible, so a+caller that needs it should read the plan before applying.++One thing this cannot promise: the unioned scope is as durable as any other+record's, which is to say a later re-extraction at equal or higher source rank+replaces it, the same way it replaces the canonical's own.+"""++from __future__ import annotations++import json+from collections import defaultdict+from dataclasses import dataclass, field++from sqlalchemy import delete, select, update+from sqlalchemy.ext.asyncio import AsyncSession++from repowise.core.analysis.decisions.provenance import rank_for_source+from repowise.core.analysis.decisions.semantic_match import (+ DECISION_VECTOR_PREFIX,+ DEFAULT_DEDUP_TAU,+ SEARCH_FETCH,+ decision_match_text,+ find_related_decisions_many,+)+from repowise.core.providers.embedding import store_has_semantic_vectors++# ``_add_alias`` is private but its chain repointing is the load-bearing part:+# without it a prior alias of a folded record points at an id this sweep deletes.+from .crud.authority import _add_alias, accepted_decision_ids+from .crud.decisions import _rederive_headline, list_decision_evidence+from .decision_graph import sync_decision_node_links+from .models import (+ DecisionCandidateMeta,+ DecisionEdge,+ DecisionEvidence,+ DecisionRecord,+)++__all__ = [+ "DedupePlan",+ "FoldPlan",+ "apply_dedupe",+ "plan_dedupe",+ "render_plan",+]++#: Set-valued provenance the canonical absorbs from what it folds. See the+#: module docstring for what is deliberately left out.+_UNION_FIELDS: tuple[tuple[str, str], ...] = (+ ("affected_files_json", "files"),+ ("affected_modules_json", "modules"),+ ("evidence_commits_json", "commits"),+ ("tags_json", "tags"),+)++#: Statuses a duplicate may be folded from. A record with no acceptance is a+#: candidate whatever this column says, so ``active`` belongs here; the retired+#: statuses do not, because their lineage would go with them.+_FOLDABLE_STATUSES: frozenset[str] = frozenset({"proposed", "active"})+++@dataclass+class FoldPlan:+ """One star: the record that survives, and what folds into it."""++ canonical_id: str+ canonical_title: str+ #: ``(decision_id, title, cosine against the canonical)``, closest first.+ folded: list[tuple[str, str, float]] = field(default_factory=list)++ @property+ def size(self) -> int:+ return len(self.folded) + 1++ def as_dict(self) -> dict:+ return {+ "canonical_id": self.canonical_id,+ "canonical_title": self.canonical_title,+ "folded": [+ {"id": i, "title": t, "similarity": round(s, 4)} for i, t, s in self.folded+ ],+ }+++@dataclass+class DedupePlan:+ """What the sweep will fold, and the evidence it rests on."""++ tau: float = DEFAULT_DEDUP_TAU+ #: Foldable candidates carrying match text.+ eligible: int = 0+ #: Pairs at or above *tau* stored as separate records.+ pairs: int = 0+ clusters: list[FoldPlan] = field(default_factory=list)+ #: True when the store could not be asked, so the zeros above mean nothing.+ degraded: bool = False++ @property+ def folded_count(self) -> int:+ return sum(len(c.folded) for c in self.clusters)++ def counts(self) -> dict[str, int]:+ return {+ "eligible": self.eligible,+ "pairs": self.pairs,+ "clusters": len(self.clusters),+ "folded": self.folded_count,+ }++ def as_dict(self) -> dict:+ return {+ "tau": self.tau,+ "degraded": self.degraded,+ **self.counts(),+ "clusters": [c.as_dict() for c in self.clusters],+ }+++def _json_list(raw: str | None) -> list[str]:+ try:+ value = json.loads(raw or "[]")+ except (TypeError, ValueError):+ return []+ return [str(v) for v in value] if isinstance(value, list) else []+++def _completeness(rec: DecisionRecord) -> int:+ """How many of a record's five body fields say anything."""+ filled = sum(+ 1 for text in (rec.decision, rec.rationale, rec.context) if (text or "").strip()+ )+ filled += sum(1 for raw in (rec.consequences_json, rec.alternatives_json) if _json_list(raw))+ return filled+++def _canonical_key(rec: DecisionRecord) -> tuple:+ """Rank the members of a cluster; the maximum survives.++ Source rank first, because that is what the write path promotes a headline+ on. Completeness breaks the rank ties, so a blank record cannot outrank one+ that says what was decided; confidence does not read completeness, so it+ cannot stand in for it. The last two only make the pick deterministic.+ """+ return (+ rank_for_source(rec.source),+ _completeness(rec),+ rec.confidence or 0.0,+ rec.created_at.isoformat() if rec.created_at else "",+ rec.id,+ )+++async def _foldable(session: AsyncSession, repository_id: str) -> list[DecisionRecord]:+ """The records this sweep is allowed to fold. See the module docstring."""+ result = await session.execute(+ select(DecisionRecord).where(DecisionRecord.repository_id == repository_id)+ )+ accepted = await accepted_decision_ids(session, repository_id)+ reviewed = {+ decision_id+ for decision_id, in (+ await session.execute(+ select(DecisionCandidateMeta.decision_id).where(+ DecisionCandidateMeta.repository_id == repository_id,+ DecisionCandidateMeta.review_state != "open",+ )+ )+ ).all()+ }+ return [+ rec+ for rec in result.scalars().all()+ if rec.status in _FOLDABLE_STATUSES+ and rec.id not in accepted+ and rec.id not in reviewed+ and decision_match_text(rec.title, rec.decision or "")+ ]+++async def plan_dedupe(+ session: AsyncSession,+ repository_id: str,+ *,+ vector_store,+ tau: float = DEFAULT_DEDUP_TAU,+) -> DedupePlan:+ """Work out what folds into what. Writes nothing.++ Sets ``degraded`` on a store that cannot answer semantically, so a keyless+ run reads as "could not look" rather than "found nothing".+ """+ plan = DedupePlan(tau=tau)+ if not store_has_semantic_vectors(vector_store):+ plan.degraded = True+ return plan++ records = await _foldable(session, repository_id)+ plan.eligible = len(records)+ if len(records) < 2:+ return plan++ by_id = {rec.id: rec for rec in records}+ # The neighbour window is the store's own, so a record with more near-twins+ # than it holds keeps the closest; a second sweep picks up any remainder.+ neighbours = await find_related_decisions_many(+ vector_store,+ [(rec.title, rec.decision or "", {rec.id}) for rec in records],+ lo=tau,+ limit=SEARCH_FETCH,+ )++ # Symmetrise: one record's window can be crowded while the other's is not,+ # and a pair measured from either side is the same pair.+ edges: dict[str, dict[str, float]] = defaultdict(dict)+ for rec, related in zip(records, neighbours, strict=True):+ for other_id, score in related:+ if other_id not in by_id or other_id == rec.id:+ continue+ best = max(score, edges[rec.id].get(other_id, 0.0))+ edges[rec.id][other_id] = best+ edges[other_id][rec.id] = best+ plan.pairs = sum(len(v) for v in edges.values()) // 2+ plan.clusters = _star_cover(records, edges)+ return plan+++def _star_cover(+ records: list[DecisionRecord], edges: dict[str, dict[str, float]]+) -> list[FoldPlan]:+ """Cover each component with stars around its most authoritative members."""+ by_id = {rec.id: rec for rec in records}+ unassigned = {rec.id for rec in records if edges.get(rec.id)}+ clusters: list[FoldPlan] = []+ while unassigned:+ canonical = max(unassigned, key=lambda i: _canonical_key(by_id[i]))+ unassigned.discard(canonical)+ folded = sorted(+ (i for i in edges[canonical] if i in unassigned),+ key=lambda i: -edges[canonical][i],+ )+ unassigned -= set(folded)+ if folded:+ clusters.append(+ FoldPlan(+ canonical_id=canonical,+ canonical_title=by_id[canonical].title,+ folded=[(i, by_id[i].title, edges[canonical][i]) for i in folded],+ )+ )+ # Largest first, so the report's head is the corpus's worst duplication.+ clusters.sort(key=lambda c: (-c.size, c.canonical_id))+ return clusters+++async def apply_dedupe(+ session: AsyncSession,+ repository_id: str,+ *,+ vector_store,+ plan: DedupePlan | None = None,+ tau: float = DEFAULT_DEDUP_TAU,+) -> DedupePlan:+ """Fold the plan, and return what was actually folded.++ Idempotent: a second run finds nothing left to fold. The caller owns the+ transaction, and the vectors go *before* it commits. The other order leaves+ a vector whose record is gone, which the matcher drops with no log line+ while silently re-creating the duplicate this removed; this order can only+ leave a record whose vector is gone, which the next ``reindex`` restores.++ A supplied *plan* is re-checked against the store rather than trusted: it+ may have been built before someone accepted one of its members.+ """+ if plan is None:+ plan = await plan_dedupe(session, repository_id, vector_store=vector_store, tau=tau)+ if not plan.clusters:+ return plan++ eligible = {rec.id for rec in await _foldable(session, repository_id)}+ applied = DedupePlan(+ tau=plan.tau, eligible=plan.eligible, pairs=plan.pairs, degraded=plan.degraded+ )+ retired: list[str] = []+ for cluster in plan.clusters:+ if cluster.canonical_id not in eligible:+ continue+ canonical = await session.get(DecisionRecord, cluster.canonical_id)+ if canonical is None:+ continue+ union = {+ name: set(_json_list(getattr(canonical, column))) for column, name in _UNION_FIELDS+ }+ done = FoldPlan(canonical_id=canonical.id, canonical_title=canonical.title)+ for folded_id, title, score in cluster.folded:+ folded = await session.get(DecisionRecord, folded_id)+ if folded is None or folded_id not in eligible:+ continue+ for column, name in _UNION_FIELDS:+ union[name] |= set(_json_list(getattr(folded, column)))+ await _absorb_evidence(session, canonical.id, folded.id)+ await _repoint_references(session, canonical.id, folded.id)+ await _add_alias(session, folded.id, canonical.id, reason="merged")+ await session.delete(folded)+ await session.flush()+ retired.append(folded_id)+ done.folded.append((folded_id, title, score))+ if not done.folded:+ continue++ for column, name in _UNION_FIELDS:+ setattr(canonical, column, json.dumps(sorted(union[name])))+ # Sync replaces rather than accretes, so it must see the union.+ await sync_decision_node_links(+ session,+ repository_id,+ canonical.id,+ files=sorted(union["files"]),+ modules=sorted(union["modules"]),+ )+ _rederive_headline(canonical, await list_decision_evidence(session, canonical.id))+ applied.clusters.append(done)++ await session.flush()+ if retired and vector_store is not None:+ await vector_store.delete_many(+ [f"{DECISION_VECTOR_PREFIX}{decision_id}" for decision_id in sorted(retired)]+ )+ return applied+++async def _absorb_evidence(session: AsyncSession, canonical_id: str, folded_id: str) -> None:+ """Move the folded record's evidence across, keeping what is already there.++ A row whose natural key the canonical already holds cites the same source,+ file and commit, so it corroborates nothing the canonical does not have and+ is left to go with its record; re-upserting it instead would overwrite the+ canonical's quote, line and verdict with the duplicate's. ``held`` compares+ NULLs equal, which is what ``_upsert_decision_evidence`` does explicitly and+ not what the unique constraint does, so a fold converges on that key the+ same way a re-index of the same source does.+ """+ held = {+ (source, file, commit)+ for source, file, commit in (+ await session.execute(+ select(+ DecisionEvidence.source,+ DecisionEvidence.evidence_file,+ DecisionEvidence.evidence_commit,+ ).where(DecisionEvidence.decision_id == canonical_id)+ )+ ).all()+ }+ rows = (+ (+ await session.execute(+ select(DecisionEvidence).where(DecisionEvidence.decision_id == folded_id)+ )+ )+ .scalars()+ .all()+ )+ for row in rows:+ key = (row.source, row.evidence_file, row.evidence_commit)+ if key in held:+ continue+ row.decision_id = canonical_id+ held.add(key)+ await session.flush()+++async def _repoint_references(session: AsyncSession, canonical_id: str, folded_id: str) -> None:+ """Send every pointer at the folded record to the canonical instead.++ ``superseded_by`` and the edge endpoints carry no foreign key, so nothing+ would catch them dangling.+ """+ await session.execute(+ update(DecisionRecord)+ .where(DecisionRecord.superseded_by == folded_id)+ .values(superseded_by=canonical_id)+ )+ # An edge between the two ends would become a self-edge; drop it first, so+ # the rewrite below only ever sees edges pointing outside the pair.+ await session.execute(+ delete(DecisionEdge).where(+ DecisionEdge.src_decision_id.in_([canonical_id, folded_id]),+ DecisionEdge.dst_decision_id.in_([canonical_id, folded_id]),+ )+ )+ for column in (DecisionEdge.src_decision_id, DecisionEdge.dst_decision_id):+ edges = (+ (await session.execute(select(DecisionEdge).where(column == folded_id)))+ .scalars()+ .all()+ )+ for edge in edges:+ src = canonical_id if edge.src_decision_id == folded_id else edge.src_decision_id+ dst = canonical_id if edge.dst_decision_id == folded_id else edge.dst_decision_id+ clash = (+ await session.execute(+ select(DecisionEdge.id).where(+ DecisionEdge.src_decision_id == src,+ DecisionEdge.dst_decision_id == dst,+ DecisionEdge.kind == edge.kind,+ )+ )+ ).first()+ if clash is not None:+ await session.delete(edge)+ else:+ edge.src_decision_id, edge.dst_decision_id = src, dst+ await session.flush()+++def render_plan(plan: DedupePlan, *, limit: int = 10) -> str:+ """Report the plan for a person. Same shape dry run and applied."""+ if plan.degraded:+ return (+ "\n Decision dedupe (dry run)\n\n"+ " This store has no semantic vectors to compare decisions with, so\n"+ " nothing could be examined. Set an embedder key and run\n"+ " 'repowise reindex' first.\n"+ )+ counts = plan.counts()+ remain = counts["eligible"] - counts["folded"]+ lines = [+ "",+ " Decision dedupe (dry run)",+ "",+ f" Candidates scanned {counts['eligible']}",+ f" Duplicate pairs >= {plan.tau:.2f} {counts['pairs']}",+ f" Merge clusters {counts['clusters']}",+ f" Records folded {counts['folded']} -> {remain} candidates remain",+ ]+ if not plan.clusters:+ return "\n".join([*lines, "", " Nothing to fold."]) + "\n"+ lines += ["", f" Largest clusters (showing {min(limit, len(plan.clusters))}):"]+ for cluster in plan.clusters[:limit]:+ lines.append(f" {cluster.canonical_id[:8]} {cluster.canonical_title}")+ for folded_id, title, score in cluster.folded:+ lines.append(f" <- {folded_id[:8]} {score:.3f} {title}")+ return "\n".join(lines) + "\n"diff --git a/tests/unit/persistence/test_decision_dedupe.py b/tests/unit/persistence/test_decision_dedupe.pynew file mode 100644index 000000000..149cd939a--- /dev/null+++ b/tests/unit/persistence/test_decision_dedupe.py@@ -0,0 +1,469 @@+"""The retroactive dedupe sweep: what it folds, and what it refuses to fold.++The rule under test is that a record folds only into a record it is itself+measured against. The angles below are chosen so A-B and B-C clear the+threshold while A-C does not, which is the case a connected-component sweep+gets wrong.+"""++from __future__ import annotations++import json+import math++import pytest+from sqlalchemy import select++from repowise.core.analysis.decisions.semantic_match import DECISION_VECTOR_PREFIX+from repowise.core.persistence.crud.authority import (+ accept_decision,+ resolve_decision_id,+)+from repowise.core.persistence.decision_dedupe import (+ apply_dedupe,+ plan_dedupe,+ render_plan,+)+from repowise.core.persistence.models import (+ DecisionEvidence,+ DecisionNodeLink,+ DecisionRecord,+)+from repowise.core.persistence.vector_store import InMemoryVectorStore+from tests.unit.persistence.helpers import insert_repo++TAU = 0.8++#: Angle per marker, in degrees. 35 apart clears TAU (cos 35 = 0.819); 70+#: apart does not (cos 70 = 0.342), so "a" and "c" are neighbours of "b"+#: without being neighbours of each other.+_ANGLES = {"a": 0.0, "b": 35.0, "c": 70.0, "far": 89.0}+++class _AngleEmbedder:+ """Places each text on a circle by the ``@<marker>`` token it carries.++ Lets a test state the cosine between two decisions directly instead of+ hoping a keyword heuristic lands on the intended side of the threshold.+ """++ dimensions = 2++ async def embed(self, texts: list[str]) -> list[list[float]]:+ out = []+ for text in texts:+ marker = next((m for m in _ANGLES if f"@{m}" in text), "far")+ theta = math.radians(_ANGLES[marker])+ out.append([math.cos(theta), math.sin(theta)])+ return out+++def _store() -> InMemoryVectorStore:+ return InMemoryVectorStore(_AngleEmbedder())+++async def _seed(session, store, repo_id, specs: list[dict]) -> list[DecisionRecord]:+ """Write records and their vectors the way the index leaves them."""+ from repowise.core.analysis.decisions.semantic_match import upsert_decision_vector++ records = []+ for i, spec in enumerate(specs):+ rec = DecisionRecord(+ id=f"rec{i:04d}" + "0" * 22,+ repository_id=repo_id,+ title=spec["title"],+ status=spec.get("status", "proposed"),+ decision=spec.get("decision", ""),+ rationale=spec.get("rationale", ""),+ context="",+ source=spec.get("source", "pr"),+ confidence=spec.get("confidence", 0.5),+ affected_files_json=json.dumps(spec.get("files", [])),+ affected_modules_json=json.dumps([]),+ evidence_commits_json=json.dumps(spec.get("commits", [])),+ tags_json=json.dumps([]),+ alternatives_json=json.dumps([]),+ consequences_json=json.dumps([]),+ )+ session.add(rec)+ records.append(rec)+ await session.flush()+ for rec in records:+ await upsert_decision_vector(+ store, rec.id, title=rec.title, decision=rec.decision or ""+ )+ return records+++async def repo_id(async_session):+ repo = await insert_repo(async_session)+ return repo.id+++async def test_a_c_are_not_merged_through_b(async_session, repo_id):+ """The transitivity case: B bridges A and C, and A and C stay apart."""+ store = _store()+ a, b, c = await _seed(+ async_session,+ store,+ repo_id,+ [+ {"title": "Alpha @a", "source": "pr"},+ {"title": "Bravo @b", "source": "comment"},+ {"title": "Charlie @c", "source": "comment"},+ ],+ )+ plan = await plan_dedupe(async_session, repo_id, vector_store=store, tau=TAU)++ folded_into = {+ folded_id: cluster.canonical_id+ for cluster in plan.clusters+ for folded_id, _t, _s in cluster.folded+ }+ # B folds into exactly one of its two neighbours, never both, and the+ # neighbour it did not fold into is left as its own record.+ assert len(folded_into) == 1+ assert b.id in folded_into+ assert folded_into[b.id] in {a.id, c.id}+ assert a.id not in folded_into and c.id not in folded_into+++async def test_source_rank_then_completeness_picks_the_canonical(async_session, repo_id):+ store = _store()+ _thin, full, low = await _seed(+ async_session,+ store,+ repo_id,+ [+ {"title": "Thin @a", "source": "pr"},+ {"title": "Full @a", "source": "pr", "decision": "Do the thing", "rationale": "Why"},+ {"title": "Low @a", "source": "comment", "decision": "Do the thing too"},+ ],+ )+ plan = await plan_dedupe(async_session, repo_id, vector_store=store, tau=TAU)++ assert len(plan.clusters) == 1+ # ``pr`` outranks ``comment``, so the comment record cannot be canonical+ # even though it has a body; between the two ``pr`` records the one that+ # says something wins.+ assert plan.clusters[0].canonical_id == full.id+ assert low.id in {i for i, _t, _s in plan.clusters[0].folded}+++async def test_fold_moves_evidence_without_downgrading_the_canonical(async_session, repo_id):+ store = _store()+ canonical, duplicate = await _seed(+ async_session,+ store,+ repo_id,+ [+ {"title": "Keep @a", "source": "pr", "decision": "Body"},+ {"title": "Drop @a", "source": "comment"},+ ],+ )+ # Same natural key on both, stronger on the canonical.+ async_session.add_all(+ [+ DecisionEvidence(+ decision_id=canonical.id,+ source="pr",+ source_rank=7,+ evidence_file="a.py",+ evidence_commit="c1",+ source_quote="q",+ confidence=0.8,+ verification="exact",+ ),+ DecisionEvidence(+ decision_id=duplicate.id,+ source="pr",+ source_rank=7,+ evidence_file="a.py",+ evidence_commit="c1",+ source_quote="q",+ confidence=0.2,+ verification="fuzzy",+ ),+ DecisionEvidence(+ decision_id=duplicate.id,+ source="comment",+ source_rank=3,+ evidence_file="b.py",+ evidence_commit="c2",+ source_quote="q2",+ confidence=0.4,+ verification="exact",+ ),+ ]+ )+ await async_session.flush()++ await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)++ rows = (+ (+ await async_session.execute(+ select(DecisionEvidence).where(DecisionEvidence.decision_id == canonical.id)+ )+ )+ .scalars()+ .all()+ )+ by_key = {(r.source, r.evidence_file): r for r in rows}+ # The genuinely new source moved; the colliding one did not overwrite.+ assert ("comment", "b.py") in by_key+ assert by_key[("pr", "a.py")].verification == "exact"+ assert by_key[("pr", "a.py")].confidence == pytest.approx(0.8)+++async def test_folded_id_still_resolves_and_its_files_survive(async_session, repo_id):+ store = _store()+ canonical, duplicate = await _seed(+ async_session,+ store,+ repo_id,+ [+ {"title": "Keep @a", "source": "pr", "decision": "Body", "files": ["kept.py"]},+ {"title": "Drop @a", "source": "comment", "files": ["only_on_dup.py"]},+ ],+ )+ await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)++ assert await async_session.get(DecisionRecord, duplicate.id) is None+ assert await resolve_decision_id(async_session, duplicate.id) == canonical.id++ survivor = await async_session.get(DecisionRecord, canonical.id)+ assert json.loads(survivor.affected_files_json) == ["kept.py", "only_on_dup.py"]+ nodes = {+ row.node_id+ for row in (+ (+ await async_session.execute(+ select(DecisionNodeLink).where(+ DecisionNodeLink.decision_id == canonical.id+ )+ )+ )+ .scalars()+ .all()+ )+ }+ # The duplicate's governed file keeps a decision, rather than losing all+ # coverage when its record goes.+ assert "only_on_dup.py" in nodes+++async def test_the_folded_vector_goes_and_the_canonical_stays(async_session, repo_id):+ store = _store()+ canonical, duplicate = await _seed(+ async_session,+ store,+ repo_id,+ [+ {"title": "Keep @a", "source": "pr", "decision": "Body"},+ {"title": "Drop @a", "source": "comment"},+ ],+ )+ await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)++ present = await store.list_page_ids()+ assert f"{DECISION_VECTOR_PREFIX}{canonical.id}" in present+ assert f"{DECISION_VECTOR_PREFIX}{duplicate.id}" not in present+++async def test_second_run_folds_nothing(async_session, repo_id):+ store = _store()+ await _seed(+ async_session,+ store,+ repo_id,+ [+ {"title": "Keep @a", "source": "pr", "decision": "Body"},+ {"title": "Drop @a", "source": "comment"},+ ],+ )+ first = await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)+ assert first.folded_count == 1++ second = await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)+ assert second.folded_count == 0+ assert second.clusters == []+++async def test_an_accepted_decision_is_left_alone(async_session, repo_id):+ store = _store()+ accepted, duplicate = await _seed(+ async_session,+ store,+ repo_id,+ [+ {"title": "Governs @a", "source": "pr", "decision": "Body"},+ {"title": "Echo @a", "source": "pr", "decision": "Body"},+ ],+ )+ await accept_decision(+ async_session,+ accepted,+ accepter="someone",+ reason="because",+ scope=["x.py"],+ evidence=["e"],+ )+ await async_session.flush()++ plan = await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)++ # Folding into something that governs is a merge, and merge_candidate owns+ # that; neither record may be touched here.+ assert plan.folded_count == 0+ assert await async_session.get(DecisionRecord, duplicate.id) is not None+++async def test_a_keyless_store_says_so_instead_of_saying_clean(async_session, repo_id):+ from repowise.core.providers.embedding.base import KeylessEmbedder++ store = _store()+ await _seed(+ async_session,+ store,+ repo_id,+ [+ {"title": "Keep @a", "source": "pr", "decision": "Body"},+ {"title": "Drop @a", "source": "comment"},+ ],+ )+ keyless = InMemoryVectorStore(KeylessEmbedder())+ plan = await plan_dedupe(async_session, repo_id, vector_store=keyless, tau=TAU)++ # A store that could not be asked must not read as a store with nothing+ # to fold: the report turns on this flag, not on the zero.+ assert plan.degraded is True+ assert plan.clusters == []+ assert "nothing could be examined" in render_plan(plan)+++async def test_a_record_a_reviewer_flagged_is_not_folded(async_session, repo_id):+ from repowise.core.persistence.crud.authority import request_split++ store = _store()+ _canonical, flagged = await _seed(+ async_session,+ store,+ repo_id,+ [+ {"title": "Keep @a", "source": "pr", "decision": "Body"},+ {"title": "Bundles two @a", "source": "pr", "decision": "Body and more"},+ ],+ )+ # request_split leaves status at "proposed", so only the review row says a+ # person has judged this record.+ await request_split(async_session, flagged, reason="two choices in one")++ plan = await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)++ assert plan.folded_count == 0+ assert await async_session.get(DecisionRecord, flagged.id) is not None+++async def test_an_unaccepted_active_record_is_still_a_candidate(async_session, repo_id):+ store = _store()+ _canonical, legacy = await _seed(+ async_session,+ store,+ repo_id,+ [+ {"title": "Keep @a", "source": "pr", "decision": "Body"},+ {"title": "Legacy @a", "source": "comment", "status": "active"},+ ],+ )+ plan = await plan_dedupe(async_session, repo_id, vector_store=store, tau=TAU)++ # Authority lives in the acceptance log, not the status column, so an+ # "active" row nobody accepted is a duplicate candidate like any other.+ assert plan.eligible == 2+ assert legacy.id in {i for c in plan.clusters for i, _t, _s in c.folded}+++async def test_a_stale_plan_cannot_fold_a_record_since_accepted(async_session, repo_id):+ store = _store()+ canonical, duplicate = await _seed(+ async_session,+ store,+ repo_id,+ [+ {"title": "Keep @a", "source": "pr", "decision": "Body"},+ {"title": "Drop @a", "source": "pr", "decision": "Body too"},+ ],+ )+ plan = await plan_dedupe(async_session, repo_id, vector_store=store, tau=TAU)+ assert plan.folded_count == 1++ # Someone accepts the record the stale plan wants to delete.+ target = duplicate if plan.clusters[0].canonical_id == canonical.id else canonical+ await accept_decision(+ async_session,+ target,+ accepter="someone",+ reason="because",+ scope=["x.py"],+ evidence=["e"],+ )+ await async_session.flush()++ applied = await apply_dedupe(async_session, repo_id, vector_store=store, plan=plan)++ assert applied.folded_count == 0+ assert await async_session.get(DecisionRecord, target.id) is not None+++async def test_an_ineligible_record_is_not_dragged_into_a_cluster(async_session, repo_id):+ """Three mutually-close records, the highest-ranking one flagged.++ The exclusion tests each leave fewer than two foldable records, so they+ short-circuit before a cluster is formed. This one forms a real cluster+ around the flagged record and pins that it still stays out — and because+ ``cli`` outranks ``pr``, it would have been the canonical if it were+ eligible, which is the strongest form of the check.+ """+ from repowise.core.persistence.crud.authority import request_split++ store = _store()+ first, flagged, third = await _seed(+ async_session,+ store,+ repo_id,+ [+ {"title": "One @a", "source": "pr", "decision": "Body"},+ {"title": "Two @a", "source": "cli", "decision": "Body and more"},+ {"title": "Three @a", "source": "pr", "decision": "Body again"},+ ],+ )+ await request_split(async_session, flagged, reason="two choices in one")++ applied = await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)++ assert applied.folded_count == 1+ assert await async_session.get(DecisionRecord, flagged.id) is not None+ touched = {c.canonical_id for c in applied.clusters} | {+ i for c in applied.clusters for i, _t, _s in c.folded+ }+ assert flagged.id not in touched+ assert touched == {first.id, third.id}+++async def test_a_superseded_record_keeps_its_lineage(async_session, repo_id):+ store = _store()+ _canonical, retired = await _seed(+ async_session,+ store,+ repo_id,+ [+ {"title": "Keep @a", "source": "pr", "decision": "Body"},+ {"title": "Retired @a", "source": "pr", "status": "superseded"},+ ],+ )+ plan = await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)++ assert plan.folded_count == 0+ assert await async_session.get(DecisionRecord, retired.id) is not None