Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(decisions): fold stored duplicate candidates in one sweep (#2314)

Raghav Chamadiyadbf7ed0e78a0
diff --git a/docs/layers/DECISIONS.md b/docs/layers/DECISIONS.md
index 788b47235..5fa582c81 100644
--- a/docs/layers/DECISIONS.md
+++ b/docs/layers/DECISIONS.md
@@ -141,10 +141,20 @@ review as though they were the team's.
| `decision confirm ID...` | Accept, optionally editing the reason and scope on the way. Takes many ids. |
| `decision confirm ID` on a decision | Reaffirm it after review. |
| `decision merge ID INTO_ID` | Fold a candidate into an existing decision. The old id resolves to the target. |
+| `decision dedupe` | Fold candidates that duplicate another candidate, in one sweep. Dry run until `--apply`. |
| `decision split ID` | Flag a candidate as bundling two choices. Never splits it for you. |
| `decision dismiss ID...` | Tombstone it. On an accepted decision this also withdraws its authority. Takes many ids. |
| `decision deprecate ID --superseded-by ID2` | Retire it with an explicit lineage edge. |
+`merge` folds one pair and needs the target to be an accepted decision, because
+that merge changes what governs. `dedupe` is the other case: a store that has
+been indexed many times accumulates the same decision written several ways, and
+none of those records governs anything yet. It folds each duplicate into the
+candidate it duplicates — evidence, governed files and all — and only ever into
+a record it is directly measured against, so a chain of near-neighbours is not
+collapsed into one decision. Run it without `--apply` first; it prints exactly
+what it would fold.
+
`confirm` and `dismiss` take a list because a repository accumulates candidates
faster than anyone reviews them one command at a time. Every id in a batch goes
through the same acceptance contract as a single one, and each is applied on its
diff --git a/docs/reference/CLI_REFERENCE.md b/docs/reference/CLI_REFERENCE.md
index c8cc81731..365b9fbee 100644
--- a/docs/reference/CLI_REFERENCE.md
+++ b/docs/reference/CLI_REFERENCE.md
@@ -1012,6 +1012,7 @@ repowise decision candidates [PATH] # what is awaiting review; these govern
repowise decision confirm ID... [PATH] # accept candidates: this is what makes them govern
repowise decision dismiss ID... [PATH] # tombstone them (sticky; never re-proposed)
repowise decision merge ID INTO_ID # fold a candidate into an existing decision
+repowise decision dedupe [PATH] # fold candidates that duplicate another candidate (dry run by default)
repowise decision split ID [PATH] # flag a candidate as bundling two choices
repowise decision deprecate ID [PATH] # retire a decision, optionally naming its successor
repowise decision health [PATH] # health dashboard
diff --git a/packages/cli/src/repowise/cli/commands/decision_cmd.py b/packages/cli/src/repowise/cli/commands/decision_cmd.py
index 59838e849..df75232ef 100644
--- a/packages/cli/src/repowise/cli/commands/decision_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/decision_cmd.py
@@ -70,6 +70,7 @@ def _register_config_commands() -> None:
from repowise.cli.commands.decision_review_cmd import (
candidates_command,
+ dedupe_command,
export_command,
import_command,
merge_command,
@@ -82,6 +83,7 @@ def _register_config_commands() -> None:
migrate_command,
candidates_command,
merge_command,
+ dedupe_command,
split_command,
export_command,
import_command,
diff --git a/packages/cli/src/repowise/cli/commands/decision_review_cmd.py b/packages/cli/src/repowise/cli/commands/decision_review_cmd.py
index 11fd9b493..5ab23ca61 100644
--- a/packages/cli/src/repowise/cli/commands/decision_review_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/decision_review_cmd.py
@@ -11,10 +11,11 @@ import click
from rich.table import Table
from repowise.cli.helpers import console, get_db_url_for_repo, run_async
-from repowise.cli.output import emit_json, emit_refusal, format_option
+from repowise.cli.output import emit_json, emit_refusal, format_option, notice_console
__all__ = [
"candidates_command",
+ "dedupe_command",
"export_command",
"import_command",
"merge_command",
@@ -246,6 +247,116 @@ def merge_command(decision_id: str, into_id: str, path: str | None, fmt: str) ->
)
+# ---------------------------------------------------------------------------
+# decision dedupe
+# ---------------------------------------------------------------------------
+
+
[email protected]("path", required=False, default=None)
[email protected]("--apply", "apply_", is_flag=True, default=False, help="Write the plan.")
+ "--tau",
+ default=None,
+ # Floored well below the 0.83 default but not at zero: this command deletes
+ # what it groups, and a threshold near zero groups the whole corpus.
+ type=click.FloatRange(0.5, 1.0),
+ help="Cosine threshold to fold at [default: 0.83]",
+)
+ "--limit",
+ default=10,
+ type=click.IntRange(min=1),
+ show_default=True,
+ help="Clusters to list in the report.",
+)
+@format_option()
+def dedupe_command(
+ path: str | None, apply_: bool, tau: float | None, limit: int, fmt: str
+) -> None:
+ """Fold candidates that duplicate another candidate into one record.
+
+ Dry run by default. Folds only into a record the duplicate is itself
+ measured against, so a chain of near-neighbours is never collapsed into
+ one decision. Evidence and governed files move to the surviving record,
+ and every folded id keeps resolving.
+ """
+ from repowise.cli.commands.decision_cmd import _resolve_decision_repo
+ from repowise.cli.providers.embedders import build_embedder, resolve_embedder_for_repo
+ from repowise.cli.providers.vector_store import build_vector_store
+ from repowise.core.analysis.decisions.semantic_match import DEFAULT_DEDUP_TAU
+ from repowise.core.providers.embedding.base import MockEmbedder
+
+ repo_path = _resolve_decision_repo(path, fmt)
+ threshold = DEFAULT_DEDUP_TAU if tau is None else tau
+
+ embedder = build_embedder(resolve_embedder_for_repo(repo_path), repo_path)
+ # Test vectors are hashes, so they would group decisions that have nothing
+ # to do with each other, and this command deletes what it groups.
+ if isinstance(embedder, MockEmbedder):
+ emit_refusal(
+ "no_real_embedder",
+ "Decisions can only be compared with a real embedder, not test vectors.",
+ fmt,
+ remedy="Set an embedder key and run 'repowise reindex'.",
+ )
+ vector_store = build_vector_store(repo_path, embedder)
+ if vector_store is None:
+ emit_refusal(
+ "no_vector_store",
+ "This repo has no vector store to compare decisions with.",
+ fmt,
+ remedy="Set an embedder key and run 'repowise reindex'.",
+ )
+
+ async def _run():
+ from repowise.core.persistence import get_session
+ from repowise.core.persistence.decision_dedupe import (
+ apply_dedupe,
+ plan_dedupe,
+ render_plan,
+ )
+
+ engine, sf = await _open(repo_path)
+ try:
+ async with get_session(sf) as session:
+ repo_id = await _repository_id(session, repo_path)
+ plan = await plan_dedupe(
+ session, repo_id, vector_store=vector_store, tau=threshold
+ )
+ if apply_:
+ try:
+ plan = await apply_dedupe(
+ session, repo_id, vector_store=vector_store, plan=plan
+ )
+ except Exception:
+ # A fold that dies after its vectors are dropped leaves
+ # records the matcher can no longer see; reindex is what
+ # puts them back, and nothing else says so.
+ notice_console(fmt).print(
+ "[yellow]The sweep failed partway. No records were "
+ "folded, but run [cyan]repowise reindex[/cyan] to "
+ "restore any decision vectors it had removed.[/yellow]"
+ )
+ raise
+ return plan, render_plan(plan, limit=limit)
+ finally:
+ await engine.dispose()
+ await vector_store.close()
+
+ plan, report = run_async(_run())
+ if fmt == "json":
+ payload = plan.as_dict()
+ payload["applied"] = apply_
+ emit_json(payload)
+ return
+ if apply_ and not plan.degraded:
+ report = report.replace("(dry run)", "(applied)")
+ console.print(report)
+ if not apply_ and plan.clusters:
+ console.print("\n[dim]Nothing was written. Re-run with --apply.[/dim]")
+
+
@click.command("split")
@click.argument("decision_id")
@click.argument("path", required=False, default=None)
diff --git a/packages/core/src/repowise/core/persistence/decision_dedupe.py b/packages/core/src/repowise/core/persistence/decision_dedupe.py
new file mode 100644
index 000000000..36e37db20
--- /dev/null
+++ b/packages/core/src/repowise/core/persistence/decision_dedupe.py
@@ -0,0 +1,487 @@
+"""Folding duplicate candidates that are already stored.
+
+Cross-run dedup stops new duplicates at the pass that creates the record; it
+cannot reach the ones written before it existed. This is the retroactive half:
+it produces the store the write path would have produced, by folding each
+duplicate into the record it duplicates and moving its evidence there.
+
+Stars, not components
+---------------------
+A record is folded only into a record it is itself measured at or above *tau*
+against. Similarity is not transitive: A~B and B~C at the threshold leaves A~C
+free to be anything, and a chain of near-neighbours can span two genuinely
+different decisions bridged by records that bundle both. So a connected
+component is covered by one or more *stars*: take its most authoritative member
+as canonical, fold the members holding a direct edge to it, and repeat on
+whatever is left. Every fold performed is a measured pair, never an inferred
+one.
+
+What is eligible
+----------------
+Both ends must be candidates nobody has touched. Folding into an accepted
+decision changes what governs, and :func:`merge_candidate` already owns that
+with the review trail it deserves; a retired record carries lineage a fold
+would erase; and a record a reviewer has flagged, split or dismissed holds a
+judgement that its deletion would throw away. This sweep grants no authority
+and overrules no person.
+
+What a fold keeps, and what it drops
+------------------------------------
+Governed files and modules, evidence commits and tags are unioned onto the
+canonical, so no file loses the decision that governs it. Of the duplicate's
+evidence, every row citing a source, file and commit the canonical does not
+already hold moves across; a row matching one the canonical has corroborates
+nothing new and goes with its record, matching the convergence
+``_upsert_decision_evidence`` performs on the same key. So the set of distinct
+evidence keys is conserved and the row count falls.
+
+Three things are dropped rather than merged, each because merging them is
+worse than losing them. ``consequences`` and ``alternatives`` are prose kept
+in a list, and unioning them across a cluster yields mostly restatements of
+each other; the canonical is chosen partly on completeness, so the
+best-stated version survives. The duplicate's review row goes with it, taking
+the lane and grounding that recorded how *that* record was extracted — facts
+about a row that no longer exists. Nothing here is reconstructible, so a
+caller that needs it should read the plan before applying.
+
+One thing this cannot promise: the unioned scope is as durable as any other
+record's, which is to say a later re-extraction at equal or higher source rank
+replaces it, the same way it replaces the canonical's own.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from dataclasses import dataclass, field
+
+from sqlalchemy import delete, select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from repowise.core.analysis.decisions.provenance import rank_for_source
+from repowise.core.analysis.decisions.semantic_match import (
+ DECISION_VECTOR_PREFIX,
+ DEFAULT_DEDUP_TAU,
+ SEARCH_FETCH,
+ decision_match_text,
+ find_related_decisions_many,
+)
+from repowise.core.providers.embedding import store_has_semantic_vectors
+
+# ``_add_alias`` is private but its chain repointing is the load-bearing part:
+# without it a prior alias of a folded record points at an id this sweep deletes.
+from .crud.authority import _add_alias, accepted_decision_ids
+from .crud.decisions import _rederive_headline, list_decision_evidence
+from .decision_graph import sync_decision_node_links
+from .models import (
+ DecisionCandidateMeta,
+ DecisionEdge,
+ DecisionEvidence,
+ DecisionRecord,
+)
+
+__all__ = [
+ "DedupePlan",
+ "FoldPlan",
+ "apply_dedupe",
+ "plan_dedupe",
+ "render_plan",
+]
+
+#: Set-valued provenance the canonical absorbs from what it folds. See the
+#: module docstring for what is deliberately left out.
+_UNION_FIELDS: tuple[tuple[str, str], ...] = (
+ ("affected_files_json", "files"),
+ ("affected_modules_json", "modules"),
+ ("evidence_commits_json", "commits"),
+ ("tags_json", "tags"),
+)
+
+#: Statuses a duplicate may be folded from. A record with no acceptance is a
+#: candidate whatever this column says, so ``active`` belongs here; the retired
+#: statuses do not, because their lineage would go with them.
+_FOLDABLE_STATUSES: frozenset[str] = frozenset({"proposed", "active"})
+
+
+@dataclass
+class FoldPlan:
+ """One star: the record that survives, and what folds into it."""
+
+ canonical_id: str
+ canonical_title: str
+ #: ``(decision_id, title, cosine against the canonical)``, closest first.
+ folded: list[tuple[str, str, float]] = field(default_factory=list)
+
+ @property
+ def size(self) -> int:
+ return len(self.folded) + 1
+
+ def as_dict(self) -> dict:
+ return {
+ "canonical_id": self.canonical_id,
+ "canonical_title": self.canonical_title,
+ "folded": [
+ {"id": i, "title": t, "similarity": round(s, 4)} for i, t, s in self.folded
+ ],
+ }
+
+
+@dataclass
+class DedupePlan:
+ """What the sweep will fold, and the evidence it rests on."""
+
+ tau: float = DEFAULT_DEDUP_TAU
+ #: Foldable candidates carrying match text.
+ eligible: int = 0
+ #: Pairs at or above *tau* stored as separate records.
+ pairs: int = 0
+ clusters: list[FoldPlan] = field(default_factory=list)
+ #: True when the store could not be asked, so the zeros above mean nothing.
+ degraded: bool = False
+
+ @property
+ def folded_count(self) -> int:
+ return sum(len(c.folded) for c in self.clusters)
+
+ def counts(self) -> dict[str, int]:
+ return {
+ "eligible": self.eligible,
+ "pairs": self.pairs,
+ "clusters": len(self.clusters),
+ "folded": self.folded_count,
+ }
+
+ def as_dict(self) -> dict:
+ return {
+ "tau": self.tau,
+ "degraded": self.degraded,
+ **self.counts(),
+ "clusters": [c.as_dict() for c in self.clusters],
+ }
+
+
+def _json_list(raw: str | None) -> list[str]:
+ try:
+ value = json.loads(raw or "[]")
+ except (TypeError, ValueError):
+ return []
+ return [str(v) for v in value] if isinstance(value, list) else []
+
+
+def _completeness(rec: DecisionRecord) -> int:
+ """How many of a record's five body fields say anything."""
+ filled = sum(
+ 1 for text in (rec.decision, rec.rationale, rec.context) if (text or "").strip()
+ )
+ filled += sum(1 for raw in (rec.consequences_json, rec.alternatives_json) if _json_list(raw))
+ return filled
+
+
+def _canonical_key(rec: DecisionRecord) -> tuple:
+ """Rank the members of a cluster; the maximum survives.
+
+ Source rank first, because that is what the write path promotes a headline
+ on. Completeness breaks the rank ties, so a blank record cannot outrank one
+ that says what was decided; confidence does not read completeness, so it
+ cannot stand in for it. The last two only make the pick deterministic.
+ """
+ return (
+ rank_for_source(rec.source),
+ _completeness(rec),
+ rec.confidence or 0.0,
+ rec.created_at.isoformat() if rec.created_at else "",
+ rec.id,
+ )
+
+
+async def _foldable(session: AsyncSession, repository_id: str) -> list[DecisionRecord]:
+ """The records this sweep is allowed to fold. See the module docstring."""
+ result = await session.execute(
+ select(DecisionRecord).where(DecisionRecord.repository_id == repository_id)
+ )
+ accepted = await accepted_decision_ids(session, repository_id)
+ reviewed = {
+ decision_id
+ for decision_id, in (
+ await session.execute(
+ select(DecisionCandidateMeta.decision_id).where(
+ DecisionCandidateMeta.repository_id == repository_id,
+ DecisionCandidateMeta.review_state != "open",
+ )
+ )
+ ).all()
+ }
+ return [
+ rec
+ for rec in result.scalars().all()
+ if rec.status in _FOLDABLE_STATUSES
+ and rec.id not in accepted
+ and rec.id not in reviewed
+ and decision_match_text(rec.title, rec.decision or "")
+ ]
+
+
+async def plan_dedupe(
+ session: AsyncSession,
+ repository_id: str,
+ *,
+ vector_store,
+ tau: float = DEFAULT_DEDUP_TAU,
+) -> DedupePlan:
+ """Work out what folds into what. Writes nothing.
+
+ Sets ``degraded`` on a store that cannot answer semantically, so a keyless
+ run reads as "could not look" rather than "found nothing".
+ """
+ plan = DedupePlan(tau=tau)
+ if not store_has_semantic_vectors(vector_store):
+ plan.degraded = True
+ return plan
+
+ records = await _foldable(session, repository_id)
+ plan.eligible = len(records)
+ if len(records) < 2:
+ return plan
+
+ by_id = {rec.id: rec for rec in records}
+ # The neighbour window is the store's own, so a record with more near-twins
+ # than it holds keeps the closest; a second sweep picks up any remainder.
+ neighbours = await find_related_decisions_many(
+ vector_store,
+ [(rec.title, rec.decision or "", {rec.id}) for rec in records],
+ lo=tau,
+ limit=SEARCH_FETCH,
+ )
+
+ # Symmetrise: one record's window can be crowded while the other's is not,
+ # and a pair measured from either side is the same pair.
+ edges: dict[str, dict[str, float]] = defaultdict(dict)
+ for rec, related in zip(records, neighbours, strict=True):
+ for other_id, score in related:
+ if other_id not in by_id or other_id == rec.id:
+ continue
+ best = max(score, edges[rec.id].get(other_id, 0.0))
+ edges[rec.id][other_id] = best
+ edges[other_id][rec.id] = best
+ plan.pairs = sum(len(v) for v in edges.values()) // 2
+ plan.clusters = _star_cover(records, edges)
+ return plan
+
+
+def _star_cover(
+ records: list[DecisionRecord], edges: dict[str, dict[str, float]]
+) -> list[FoldPlan]:
+ """Cover each component with stars around its most authoritative members."""
+ by_id = {rec.id: rec for rec in records}
+ unassigned = {rec.id for rec in records if edges.get(rec.id)}
+ clusters: list[FoldPlan] = []
+ while unassigned:
+ canonical = max(unassigned, key=lambda i: _canonical_key(by_id[i]))
+ unassigned.discard(canonical)
+ folded = sorted(
+ (i for i in edges[canonical] if i in unassigned),
+ key=lambda i: -edges[canonical][i],
+ )
+ unassigned -= set(folded)
+ if folded:
+ clusters.append(
+ FoldPlan(
+ canonical_id=canonical,
+ canonical_title=by_id[canonical].title,
+ folded=[(i, by_id[i].title, edges[canonical][i]) for i in folded],
+ )
+ )
+ # Largest first, so the report's head is the corpus's worst duplication.
+ clusters.sort(key=lambda c: (-c.size, c.canonical_id))
+ return clusters
+
+
+async def apply_dedupe(
+ session: AsyncSession,
+ repository_id: str,
+ *,
+ vector_store,
+ plan: DedupePlan | None = None,
+ tau: float = DEFAULT_DEDUP_TAU,
+) -> DedupePlan:
+ """Fold the plan, and return what was actually folded.
+
+ Idempotent: a second run finds nothing left to fold. The caller owns the
+ transaction, and the vectors go *before* it commits. The other order leaves
+ a vector whose record is gone, which the matcher drops with no log line
+ while silently re-creating the duplicate this removed; this order can only
+ leave a record whose vector is gone, which the next ``reindex`` restores.
+
+ A supplied *plan* is re-checked against the store rather than trusted: it
+ may have been built before someone accepted one of its members.
+ """
+ if plan is None:
+ plan = await plan_dedupe(session, repository_id, vector_store=vector_store, tau=tau)
+ if not plan.clusters:
+ return plan
+
+ eligible = {rec.id for rec in await _foldable(session, repository_id)}
+ applied = DedupePlan(
+ tau=plan.tau, eligible=plan.eligible, pairs=plan.pairs, degraded=plan.degraded
+ )
+ retired: list[str] = []
+ for cluster in plan.clusters:
+ if cluster.canonical_id not in eligible:
+ continue
+ canonical = await session.get(DecisionRecord, cluster.canonical_id)
+ if canonical is None:
+ continue
+ union = {
+ name: set(_json_list(getattr(canonical, column))) for column, name in _UNION_FIELDS
+ }
+ done = FoldPlan(canonical_id=canonical.id, canonical_title=canonical.title)
+ for folded_id, title, score in cluster.folded:
+ folded = await session.get(DecisionRecord, folded_id)
+ if folded is None or folded_id not in eligible:
+ continue
+ for column, name in _UNION_FIELDS:
+ union[name] |= set(_json_list(getattr(folded, column)))
+ await _absorb_evidence(session, canonical.id, folded.id)
+ await _repoint_references(session, canonical.id, folded.id)
+ await _add_alias(session, folded.id, canonical.id, reason="merged")
+ await session.delete(folded)
+ await session.flush()
+ retired.append(folded_id)
+ done.folded.append((folded_id, title, score))
+ if not done.folded:
+ continue
+
+ for column, name in _UNION_FIELDS:
+ setattr(canonical, column, json.dumps(sorted(union[name])))
+ # Sync replaces rather than accretes, so it must see the union.
+ await sync_decision_node_links(
+ session,
+ repository_id,
+ canonical.id,
+ files=sorted(union["files"]),
+ modules=sorted(union["modules"]),
+ )
+ _rederive_headline(canonical, await list_decision_evidence(session, canonical.id))
+ applied.clusters.append(done)
+
+ await session.flush()
+ if retired and vector_store is not None:
+ await vector_store.delete_many(
+ [f"{DECISION_VECTOR_PREFIX}{decision_id}" for decision_id in sorted(retired)]
+ )
+ return applied
+
+
+async def _absorb_evidence(session: AsyncSession, canonical_id: str, folded_id: str) -> None:
+ """Move the folded record's evidence across, keeping what is already there.
+
+ A row whose natural key the canonical already holds cites the same source,
+ file and commit, so it corroborates nothing the canonical does not have and
+ is left to go with its record; re-upserting it instead would overwrite the
+ canonical's quote, line and verdict with the duplicate's. ``held`` compares
+ NULLs equal, which is what ``_upsert_decision_evidence`` does explicitly and
+ not what the unique constraint does, so a fold converges on that key the
+ same way a re-index of the same source does.
+ """
+ held = {
+ (source, file, commit)
+ for source, file, commit in (
+ await session.execute(
+ select(
+ DecisionEvidence.source,
+ DecisionEvidence.evidence_file,
+ DecisionEvidence.evidence_commit,
+ ).where(DecisionEvidence.decision_id == canonical_id)
+ )
+ ).all()
+ }
+ rows = (
+ (
+ await session.execute(
+ select(DecisionEvidence).where(DecisionEvidence.decision_id == folded_id)
+ )
+ )
+ .scalars()
+ .all()
+ )
+ for row in rows:
+ key = (row.source, row.evidence_file, row.evidence_commit)
+ if key in held:
+ continue
+ row.decision_id = canonical_id
+ held.add(key)
+ await session.flush()
+
+
+async def _repoint_references(session: AsyncSession, canonical_id: str, folded_id: str) -> None:
+ """Send every pointer at the folded record to the canonical instead.
+
+ ``superseded_by`` and the edge endpoints carry no foreign key, so nothing
+ would catch them dangling.
+ """
+ await session.execute(
+ update(DecisionRecord)
+ .where(DecisionRecord.superseded_by == folded_id)
+ .values(superseded_by=canonical_id)
+ )
+ # An edge between the two ends would become a self-edge; drop it first, so
+ # the rewrite below only ever sees edges pointing outside the pair.
+ await session.execute(
+ delete(DecisionEdge).where(
+ DecisionEdge.src_decision_id.in_([canonical_id, folded_id]),
+ DecisionEdge.dst_decision_id.in_([canonical_id, folded_id]),
+ )
+ )
+ for column in (DecisionEdge.src_decision_id, DecisionEdge.dst_decision_id):
+ edges = (
+ (await session.execute(select(DecisionEdge).where(column == folded_id)))
+ .scalars()
+ .all()
+ )
+ for edge in edges:
+ src = canonical_id if edge.src_decision_id == folded_id else edge.src_decision_id
+ dst = canonical_id if edge.dst_decision_id == folded_id else edge.dst_decision_id
+ clash = (
+ await session.execute(
+ select(DecisionEdge.id).where(
+ DecisionEdge.src_decision_id == src,
+ DecisionEdge.dst_decision_id == dst,
+ DecisionEdge.kind == edge.kind,
+ )
+ )
+ ).first()
+ if clash is not None:
+ await session.delete(edge)
+ else:
+ edge.src_decision_id, edge.dst_decision_id = src, dst
+ await session.flush()
+
+
+def render_plan(plan: DedupePlan, *, limit: int = 10) -> str:
+ """Report the plan for a person. Same shape dry run and applied."""
+ if plan.degraded:
+ return (
+ "\n Decision dedupe (dry run)\n\n"
+ " This store has no semantic vectors to compare decisions with, so\n"
+ " nothing could be examined. Set an embedder key and run\n"
+ " 'repowise reindex' first.\n"
+ )
+ counts = plan.counts()
+ remain = counts["eligible"] - counts["folded"]
+ lines = [
+ "",
+ " Decision dedupe (dry run)",
+ "",
+ f" Candidates scanned {counts['eligible']}",
+ f" Duplicate pairs >= {plan.tau:.2f} {counts['pairs']}",
+ f" Merge clusters {counts['clusters']}",
+ f" Records folded {counts['folded']} -> {remain} candidates remain",
+ ]
+ if not plan.clusters:
+ return "\n".join([*lines, "", " Nothing to fold."]) + "\n"
+ lines += ["", f" Largest clusters (showing {min(limit, len(plan.clusters))}):"]
+ for cluster in plan.clusters[:limit]:
+ lines.append(f" {cluster.canonical_id[:8]} {cluster.canonical_title}")
+ for folded_id, title, score in cluster.folded:
+ lines.append(f" <- {folded_id[:8]} {score:.3f} {title}")
+ return "\n".join(lines) + "\n"
diff --git a/tests/unit/persistence/test_decision_dedupe.py b/tests/unit/persistence/test_decision_dedupe.py
new file mode 100644
index 000000000..149cd939a
--- /dev/null
+++ b/tests/unit/persistence/test_decision_dedupe.py
@@ -0,0 +1,469 @@
+"""The retroactive dedupe sweep: what it folds, and what it refuses to fold.
+
+The rule under test is that a record folds only into a record it is itself
+measured against. The angles below are chosen so A-B and B-C clear the
+threshold while A-C does not, which is the case a connected-component sweep
+gets wrong.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+
+import pytest
+from sqlalchemy import select
+
+from repowise.core.analysis.decisions.semantic_match import DECISION_VECTOR_PREFIX
+from repowise.core.persistence.crud.authority import (
+ accept_decision,
+ resolve_decision_id,
+)
+from repowise.core.persistence.decision_dedupe import (
+ apply_dedupe,
+ plan_dedupe,
+ render_plan,
+)
+from repowise.core.persistence.models import (
+ DecisionEvidence,
+ DecisionNodeLink,
+ DecisionRecord,
+)
+from repowise.core.persistence.vector_store import InMemoryVectorStore
+from tests.unit.persistence.helpers import insert_repo
+
+TAU = 0.8
+
+#: Angle per marker, in degrees. 35 apart clears TAU (cos 35 = 0.819); 70
+#: apart does not (cos 70 = 0.342), so "a" and "c" are neighbours of "b"
+#: without being neighbours of each other.
+_ANGLES = {"a": 0.0, "b": 35.0, "c": 70.0, "far": 89.0}
+
+
+class _AngleEmbedder:
+ """Places each text on a circle by the ``@<marker>`` token it carries.
+
+ Lets a test state the cosine between two decisions directly instead of
+ hoping a keyword heuristic lands on the intended side of the threshold.
+ """
+
+ dimensions = 2
+
+ async def embed(self, texts: list[str]) -> list[list[float]]:
+ out = []
+ for text in texts:
+ marker = next((m for m in _ANGLES if f"@{m}" in text), "far")
+ theta = math.radians(_ANGLES[marker])
+ out.append([math.cos(theta), math.sin(theta)])
+ return out
+
+
+def _store() -> InMemoryVectorStore:
+ return InMemoryVectorStore(_AngleEmbedder())
+
+
+async def _seed(session, store, repo_id, specs: list[dict]) -> list[DecisionRecord]:
+ """Write records and their vectors the way the index leaves them."""
+ from repowise.core.analysis.decisions.semantic_match import upsert_decision_vector
+
+ records = []
+ for i, spec in enumerate(specs):
+ rec = DecisionRecord(
+ id=f"rec{i:04d}" + "0" * 22,
+ repository_id=repo_id,
+ title=spec["title"],
+ status=spec.get("status", "proposed"),
+ decision=spec.get("decision", ""),
+ rationale=spec.get("rationale", ""),
+ context="",
+ source=spec.get("source", "pr"),
+ confidence=spec.get("confidence", 0.5),
+ affected_files_json=json.dumps(spec.get("files", [])),
+ affected_modules_json=json.dumps([]),
+ evidence_commits_json=json.dumps(spec.get("commits", [])),
+ tags_json=json.dumps([]),
+ alternatives_json=json.dumps([]),
+ consequences_json=json.dumps([]),
+ )
+ session.add(rec)
+ records.append(rec)
+ await session.flush()
+ for rec in records:
+ await upsert_decision_vector(
+ store, rec.id, title=rec.title, decision=rec.decision or ""
+ )
+ return records
+
+
+async def repo_id(async_session):
+ repo = await insert_repo(async_session)
+ return repo.id
+
+
+async def test_a_c_are_not_merged_through_b(async_session, repo_id):
+ """The transitivity case: B bridges A and C, and A and C stay apart."""
+ store = _store()
+ a, b, c = await _seed(
+ async_session,
+ store,
+ repo_id,
+ [
+ {"title": "Alpha @a", "source": "pr"},
+ {"title": "Bravo @b", "source": "comment"},
+ {"title": "Charlie @c", "source": "comment"},
+ ],
+ )
+ plan = await plan_dedupe(async_session, repo_id, vector_store=store, tau=TAU)
+
+ folded_into = {
+ folded_id: cluster.canonical_id
+ for cluster in plan.clusters
+ for folded_id, _t, _s in cluster.folded
+ }
+ # B folds into exactly one of its two neighbours, never both, and the
+ # neighbour it did not fold into is left as its own record.
+ assert len(folded_into) == 1
+ assert b.id in folded_into
+ assert folded_into[b.id] in {a.id, c.id}
+ assert a.id not in folded_into and c.id not in folded_into
+
+
+async def test_source_rank_then_completeness_picks_the_canonical(async_session, repo_id):
+ store = _store()
+ _thin, full, low = await _seed(
+ async_session,
+ store,
+ repo_id,
+ [
+ {"title": "Thin @a", "source": "pr"},
+ {"title": "Full @a", "source": "pr", "decision": "Do the thing", "rationale": "Why"},
+ {"title": "Low @a", "source": "comment", "decision": "Do the thing too"},
+ ],
+ )
+ plan = await plan_dedupe(async_session, repo_id, vector_store=store, tau=TAU)
+
+ assert len(plan.clusters) == 1
+ # ``pr`` outranks ``comment``, so the comment record cannot be canonical
+ # even though it has a body; between the two ``pr`` records the one that
+ # says something wins.
+ assert plan.clusters[0].canonical_id == full.id
+ assert low.id in {i for i, _t, _s in plan.clusters[0].folded}
+
+
+async def test_fold_moves_evidence_without_downgrading_the_canonical(async_session, repo_id):
+ store = _store()
+ canonical, duplicate = await _seed(
+ async_session,
+ store,
+ repo_id,
+ [
+ {"title": "Keep @a", "source": "pr", "decision": "Body"},
+ {"title": "Drop @a", "source": "comment"},
+ ],
+ )
+ # Same natural key on both, stronger on the canonical.
+ async_session.add_all(
+ [
+ DecisionEvidence(
+ decision_id=canonical.id,
+ source="pr",
+ source_rank=7,
+ evidence_file="a.py",
+ evidence_commit="c1",
+ source_quote="q",
+ confidence=0.8,
+ verification="exact",
+ ),
+ DecisionEvidence(
+ decision_id=duplicate.id,
+ source="pr",
+ source_rank=7,
+ evidence_file="a.py",
+ evidence_commit="c1",
+ source_quote="q",
+ confidence=0.2,
+ verification="fuzzy",
+ ),
+ DecisionEvidence(
+ decision_id=duplicate.id,
+ source="comment",
+ source_rank=3,
+ evidence_file="b.py",
+ evidence_commit="c2",
+ source_quote="q2",
+ confidence=0.4,
+ verification="exact",
+ ),
+ ]
+ )
+ await async_session.flush()
+
+ await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)
+
+ rows = (
+ (
+ await async_session.execute(
+ select(DecisionEvidence).where(DecisionEvidence.decision_id == canonical.id)
+ )
+ )
+ .scalars()
+ .all()
+ )
+ by_key = {(r.source, r.evidence_file): r for r in rows}
+ # The genuinely new source moved; the colliding one did not overwrite.
+ assert ("comment", "b.py") in by_key
+ assert by_key[("pr", "a.py")].verification == "exact"
+ assert by_key[("pr", "a.py")].confidence == pytest.approx(0.8)
+
+
+async def test_folded_id_still_resolves_and_its_files_survive(async_session, repo_id):
+ store = _store()
+ canonical, duplicate = await _seed(
+ async_session,
+ store,
+ repo_id,
+ [
+ {"title": "Keep @a", "source": "pr", "decision": "Body", "files": ["kept.py"]},
+ {"title": "Drop @a", "source": "comment", "files": ["only_on_dup.py"]},
+ ],
+ )
+ await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)
+
+ assert await async_session.get(DecisionRecord, duplicate.id) is None
+ assert await resolve_decision_id(async_session, duplicate.id) == canonical.id
+
+ survivor = await async_session.get(DecisionRecord, canonical.id)
+ assert json.loads(survivor.affected_files_json) == ["kept.py", "only_on_dup.py"]
+ nodes = {
+ row.node_id
+ for row in (
+ (
+ await async_session.execute(
+ select(DecisionNodeLink).where(
+ DecisionNodeLink.decision_id == canonical.id
+ )
+ )
+ )
+ .scalars()
+ .all()
+ )
+ }
+ # The duplicate's governed file keeps a decision, rather than losing all
+ # coverage when its record goes.
+ assert "only_on_dup.py" in nodes
+
+
+async def test_the_folded_vector_goes_and_the_canonical_stays(async_session, repo_id):
+ store = _store()
+ canonical, duplicate = await _seed(
+ async_session,
+ store,
+ repo_id,
+ [
+ {"title": "Keep @a", "source": "pr", "decision": "Body"},
+ {"title": "Drop @a", "source": "comment"},
+ ],
+ )
+ await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)
+
+ present = await store.list_page_ids()
+ assert f"{DECISION_VECTOR_PREFIX}{canonical.id}" in present
+ assert f"{DECISION_VECTOR_PREFIX}{duplicate.id}" not in present
+
+
+async def test_second_run_folds_nothing(async_session, repo_id):
+ store = _store()
+ await _seed(
+ async_session,
+ store,
+ repo_id,
+ [
+ {"title": "Keep @a", "source": "pr", "decision": "Body"},
+ {"title": "Drop @a", "source": "comment"},
+ ],
+ )
+ first = await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)
+ assert first.folded_count == 1
+
+ second = await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)
+ assert second.folded_count == 0
+ assert second.clusters == []
+
+
+async def test_an_accepted_decision_is_left_alone(async_session, repo_id):
+ store = _store()
+ accepted, duplicate = await _seed(
+ async_session,
+ store,
+ repo_id,
+ [
+ {"title": "Governs @a", "source": "pr", "decision": "Body"},
+ {"title": "Echo @a", "source": "pr", "decision": "Body"},
+ ],
+ )
+ await accept_decision(
+ async_session,
+ accepted,
+ accepter="someone",
+ reason="because",
+ scope=["x.py"],
+ evidence=["e"],
+ )
+ await async_session.flush()
+
+ plan = await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)
+
+ # Folding into something that governs is a merge, and merge_candidate owns
+ # that; neither record may be touched here.
+ assert plan.folded_count == 0
+ assert await async_session.get(DecisionRecord, duplicate.id) is not None
+
+
+async def test_a_keyless_store_says_so_instead_of_saying_clean(async_session, repo_id):
+ from repowise.core.providers.embedding.base import KeylessEmbedder
+
+ store = _store()
+ await _seed(
+ async_session,
+ store,
+ repo_id,
+ [
+ {"title": "Keep @a", "source": "pr", "decision": "Body"},
+ {"title": "Drop @a", "source": "comment"},
+ ],
+ )
+ keyless = InMemoryVectorStore(KeylessEmbedder())
+ plan = await plan_dedupe(async_session, repo_id, vector_store=keyless, tau=TAU)
+
+ # A store that could not be asked must not read as a store with nothing
+ # to fold: the report turns on this flag, not on the zero.
+ assert plan.degraded is True
+ assert plan.clusters == []
+ assert "nothing could be examined" in render_plan(plan)
+
+
+async def test_a_record_a_reviewer_flagged_is_not_folded(async_session, repo_id):
+ from repowise.core.persistence.crud.authority import request_split
+
+ store = _store()
+ _canonical, flagged = await _seed(
+ async_session,
+ store,
+ repo_id,
+ [
+ {"title": "Keep @a", "source": "pr", "decision": "Body"},
+ {"title": "Bundles two @a", "source": "pr", "decision": "Body and more"},
+ ],
+ )
+ # request_split leaves status at "proposed", so only the review row says a
+ # person has judged this record.
+ await request_split(async_session, flagged, reason="two choices in one")
+
+ plan = await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)
+
+ assert plan.folded_count == 0
+ assert await async_session.get(DecisionRecord, flagged.id) is not None
+
+
+async def test_an_unaccepted_active_record_is_still_a_candidate(async_session, repo_id):
+ store = _store()
+ _canonical, legacy = await _seed(
+ async_session,
+ store,
+ repo_id,
+ [
+ {"title": "Keep @a", "source": "pr", "decision": "Body"},
+ {"title": "Legacy @a", "source": "comment", "status": "active"},
+ ],
+ )
+ plan = await plan_dedupe(async_session, repo_id, vector_store=store, tau=TAU)
+
+ # Authority lives in the acceptance log, not the status column, so an
+ # "active" row nobody accepted is a duplicate candidate like any other.
+ assert plan.eligible == 2
+ assert legacy.id in {i for c in plan.clusters for i, _t, _s in c.folded}
+
+
+async def test_a_stale_plan_cannot_fold_a_record_since_accepted(async_session, repo_id):
+ store = _store()
+ canonical, duplicate = await _seed(
+ async_session,
+ store,
+ repo_id,
+ [
+ {"title": "Keep @a", "source": "pr", "decision": "Body"},
+ {"title": "Drop @a", "source": "pr", "decision": "Body too"},
+ ],
+ )
+ plan = await plan_dedupe(async_session, repo_id, vector_store=store, tau=TAU)
+ assert plan.folded_count == 1
+
+ # Someone accepts the record the stale plan wants to delete.
+ target = duplicate if plan.clusters[0].canonical_id == canonical.id else canonical
+ await accept_decision(
+ async_session,
+ target,
+ accepter="someone",
+ reason="because",
+ scope=["x.py"],
+ evidence=["e"],
+ )
+ await async_session.flush()
+
+ applied = await apply_dedupe(async_session, repo_id, vector_store=store, plan=plan)
+
+ assert applied.folded_count == 0
+ assert await async_session.get(DecisionRecord, target.id) is not None
+
+
+async def test_an_ineligible_record_is_not_dragged_into_a_cluster(async_session, repo_id):
+ """Three mutually-close records, the highest-ranking one flagged.
+
+ The exclusion tests each leave fewer than two foldable records, so they
+ short-circuit before a cluster is formed. This one forms a real cluster
+ around the flagged record and pins that it still stays out — and because
+ ``cli`` outranks ``pr``, it would have been the canonical if it were
+ eligible, which is the strongest form of the check.
+ """
+ from repowise.core.persistence.crud.authority import request_split
+
+ store = _store()
+ first, flagged, third = await _seed(
+ async_session,
+ store,
+ repo_id,
+ [
+ {"title": "One @a", "source": "pr", "decision": "Body"},
+ {"title": "Two @a", "source": "cli", "decision": "Body and more"},
+ {"title": "Three @a", "source": "pr", "decision": "Body again"},
+ ],
+ )
+ await request_split(async_session, flagged, reason="two choices in one")
+
+ applied = await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)
+
+ assert applied.folded_count == 1
+ assert await async_session.get(DecisionRecord, flagged.id) is not None
+ touched = {c.canonical_id for c in applied.clusters} | {
+ i for c in applied.clusters for i, _t, _s in c.folded
+ }
+ assert flagged.id not in touched
+ assert touched == {first.id, third.id}
+
+
+async def test_a_superseded_record_keeps_its_lineage(async_session, repo_id):
+ store = _store()
+ _canonical, retired = await _seed(
+ async_session,
+ store,
+ repo_id,
+ [
+ {"title": "Keep @a", "source": "pr", "decision": "Body"},
+ {"title": "Retired @a", "source": "pr", "status": "superseded"},
+ ],
+ )
+ plan = await apply_dedupe(async_session, repo_id, vector_store=store, tau=TAU)
+
+ assert plan.folded_count == 0
+ assert await async_session.get(DecisionRecord, retired.id) is not None