Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(core): batch decision embeddings in persistence and reindex (#641)

Swati Ahuja3f4592061740
diff --git a/packages/cli/src/repowise/cli/commands/reindex_cmd.py b/packages/cli/src/repowise/cli/commands/reindex_cmd.py
index 30c794bc2..c2e8a7d4b 100644
--- a/packages/cli/src/repowise/cli/commands/reindex_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/reindex_cmd.py
@@ -23,7 +23,7 @@ from repowise.cli.ui import BRAND_STYLE, OWL_SPINNER
default="auto",
help="Embedder to use. 'auto' detects from env vars / config.",
)
[email protected]("--batch-size", type=int, default=20, help="Pages per embedding batch.")
[email protected]("--batch-size", type=int, default=32, help="Pages per embedding batch.")
def reindex_command(path: str | None, embedder: str, batch_size: int) -> None:
"""Rebuild vector search index from existing wiki pages.
@@ -94,7 +94,7 @@ async def _reindex(repo_path, embedder_name: str, batch_size: int) -> None:
pages = list(result.scalars().all())
# --- Load decision records ---
- from repowise.core.analysis.decision_semantic_match import upsert_decision_vector
+ from repowise.core.analysis.decision_semantic_match import decision_vector_item
from repowise.core.persistence.models import DecisionRecord
async with factory() as session:
@@ -125,13 +125,46 @@ async def _reindex(repo_path, embedder_name: str, batch_size: int) -> None:
) as progress:
task = progress.add_task("Indexing pages...", total=total)
- # Pages
+ warned = 0
+
+ async def _embed_slice(items: list[tuple[str, str, dict]]) -> None:
+ """Embed one slice batched; on failure retry per item.
+
+ The batched call is the fast path (one embedder request per
+ chunk). A raised error falls back to per-item embedding so one
+ poison item can't sink its neighbours, and the indexed/failed
+ counters stay per-item accurate.
+ """
+ nonlocal indexed, failed, warned
+ try:
+ await vector_store.embed_batch(items)
+ indexed += len(items)
+ return
+ except Exception:
+ pass
+ for page_id, text, meta in items:
+ try:
+ await vector_store.embed_and_upsert(page_id, text, meta)
+ indexed += 1
+ except Exception as exc:
+ failed += 1
+ warned += 1
+ if warned <= 3:
+ console.print(
+ f"[yellow] Warning: failed to embed {page_id}: {exc}[/yellow]"
+ )
+
+ # Pages — one batched embed per slice instead of one embedder
+ # round-trip per page (a large wiki paid thousands of serial calls).
for i in range(0, len(pages), batch_size):
batch = pages[i : i + batch_size]
+ items = []
for page in batch:
- try:
- text = f"{page.title}\n{page.content}" if page.content else page.title or ""
- await vector_store.embed_and_upsert(
+ text = f"{page.title}\n{page.content}" if page.content else page.title or ""
+ if not text.strip():
+ continue # embedders reject empty input; nothing to index
+ items.append(
+ (
page.id,
text,
{
@@ -140,36 +173,32 @@ async def _reindex(repo_path, embedder_name: str, batch_size: int) -> None:
"target_path": page.target_path or "",
},
)
- indexed += 1
- except Exception as exc:
- failed += 1
- if failed <= 3:
- console.print(
- f"[yellow] Warning: failed to embed {page.id}: {exc}[/yellow]"
- )
- progress.advance(task)
-
- # Decision records — embedded into the shared page store under the decision: namespace
+ )
+ await _embed_slice(items)
+ progress.advance(task, advance=len(batch))
+
+ # Decision records — embedded into the shared page store under the
+ # decision: namespace, batched like the pages. Uses embed_batch
+ # directly (which raises on failure) rather than the ingest-side
+ # best-effort wrapper, so the indexed/failed counters stay honest.
progress.update(task, description="Indexing decisions...")
for i in range(0, len(decisions), batch_size):
batch = decisions[i : i + batch_size]
- for d in batch:
- try:
- await upsert_decision_vector(
- vector_store,
+ items = [
+ item
+ for d in batch
+ if (
+ item := decision_vector_item(
d.id,
title=d.title or "",
decision=d.decision or "",
evidence_file=getattr(d, "evidence_file", None),
)
- indexed += 1
- except Exception as exc:
- failed += 1
- if failed <= 3:
- console.print(
- f"[yellow] Warning: failed to embed decision {d.id}: {exc}[/yellow]"
- )
- progress.advance(task)
+ )
+ is not None
+ ]
+ await _embed_slice(items)
+ progress.advance(task, advance=len(batch))
await vector_store.close()
await engine.dispose()
diff --git a/packages/core/src/repowise/core/analysis/decisions/evolution.py b/packages/core/src/repowise/core/analysis/decisions/evolution.py
index 25b6d3867..8ef523b04 100644
--- a/packages/core/src/repowise/core/analysis/decisions/evolution.py
+++ b/packages/core/src/repowise/core/analysis/decisions/evolution.py
@@ -41,7 +41,7 @@ from repowise.core.analysis.decisions.gate import apply_substring_gate
from repowise.core.analysis.decisions.provenance import normalize_text
from repowise.core.analysis.decisions.semantic_match import (
DEFAULT_DEDUP_TAU,
- find_related_decisions,
+ find_related_decisions_many,
)
logger = structlog.get_logger(__name__)
@@ -328,18 +328,29 @@ async def detect_supersessions_and_conflicts(
# Avoid recording both (A supersedes B) and (B supersedes A) within one run.
handled_pairs: set[frozenset[str]] = set()
- for tid in touched_ids:
- rec = await session.get(DecisionRecord, tid)
- if rec is None or rec.repository_id != repository_id:
- continue
- related = await find_related_decisions(
- vector_store,
- title=rec.title,
- decision=rec.decision or "",
- lo=RELATED_TAU,
- hi=DEFAULT_DEDUP_TAU,
- exclude_ids={rec.id},
+ # Load the touched records first (one SELECT, not one session.get per id),
+ # then resolve every "same topic" band in one batched store call — the
+ # per-record search embedded its query over the network, so N touched
+ # decisions cost N serial round-trips.
+ from sqlalchemy import select
+
+ unique_ids = list(dict.fromkeys(touched_ids))
+ rows = await session.execute(
+ select(DecisionRecord).where(
+ DecisionRecord.id.in_(unique_ids),
+ DecisionRecord.repository_id == repository_id,
)
+ )
+ by_id = {rec.id: rec for rec in rows.scalars().all()}
+ recs: list[Any] = [by_id[tid] for tid in unique_ids if tid in by_id]
+ related_lists = await find_related_decisions_many(
+ vector_store,
+ [(rec.title, rec.decision or "", {rec.id}) for rec in recs],
+ lo=RELATED_TAU,
+ hi=DEFAULT_DEDUP_TAU,
+ )
+
+ for rec, related in zip(recs, related_lists, strict=True):
for other_id, sim in related:
pair = frozenset({rec.id, other_id})
if pair in handled_pairs:
diff --git a/packages/core/src/repowise/core/analysis/decisions/semantic_match.py b/packages/core/src/repowise/core/analysis/decisions/semantic_match.py
index 0ba3b3340..9740a34c2 100644
--- a/packages/core/src/repowise/core/analysis/decisions/semantic_match.py
+++ b/packages/core/src/repowise/core/analysis/decisions/semantic_match.py
@@ -31,10 +31,17 @@ __all__ = [
"DECISION_PAGE_TYPE",
"DECISION_VECTOR_PREFIX",
"DEFAULT_DEDUP_TAU",
+ "SEARCH_FETCH",
+ "PendingDecisionIndex",
"decision_match_text",
+ "decision_vector_item",
"find_duplicate_decision",
+ "find_duplicate_decision_by_vector",
"find_related_decisions",
+ "find_related_decisions_many",
+ "nearest_decision_hit",
"upsert_decision_vector",
+ "upsert_decision_vectors",
]
# Cosine threshold above which two decision texts are treated as the same
@@ -54,7 +61,9 @@ DECISION_PAGE_TYPE = "decision_record"
# namespace. Decisions are a small slice of a store dominated by page vectors,
# so we over-fetch to make sure a genuine paraphrase (which sits at very high
# similarity) isn't crowded out of the window by unrelated-but-near pages.
-_SEARCH_FETCH = 50
+# Public: the batched dedup path in ``crud.bulk_upsert_decisions`` issues its
+# own vector searches and must use the same window.
+SEARCH_FETCH = 50
def decision_match_text(title: str, decision: str = "") -> str:
@@ -73,6 +82,211 @@ def _decision_page_id(decision_id: str) -> str:
return f"{DECISION_VECTOR_PREFIX}{decision_id}"
+def decision_vector_item(
+ decision_id: str,
+ *,
+ title: str,
+ decision: str = "",
+ evidence_file: str | None = None,
+) -> tuple[str, str, dict] | None:
+ """The ``(page_id, text, metadata)`` item that embeds a decision.
+
+ Single source of truth for the store payload shared by the per-item
+ :func:`upsert_decision_vector` and the batched write paths, so both always
+ persist the same shape. Returns None for an empty decision text.
+ """
+ text = decision_match_text(title, decision)
+ if not text:
+ return None
+ return (
+ _decision_page_id(decision_id),
+ text,
+ {
+ "title": title or "",
+ "page_type": DECISION_PAGE_TYPE,
+ "target_path": evidence_file or "",
+ "content": text,
+ },
+ )
+
+
+def nearest_decision_hit(
+ results: list[Any],
+ exclude_ids: set[str] | None = None,
+) -> tuple[str, float] | None:
+ """The nearest ``decision:``-namespace hit in best-first *results*.
+
+ Shared filter for the text and vector search paths: returns
+ ``(decision_id, score)`` of the first decision hit not in *exclude_ids*,
+ or None when no decision appears in the window. Thresholding is the
+ caller's job — the nearest hit decides, so a caller that finds it below
+ its ``tau`` knows nothing closer exists.
+ """
+ exclude = exclude_ids or set()
+ for r in results:
+ page_id = getattr(r, "page_id", "")
+ if not page_id.startswith(DECISION_VECTOR_PREFIX):
+ continue
+ decision_id = page_id[len(DECISION_VECTOR_PREFIX) :]
+ if decision_id in exclude:
+ continue
+ return decision_id, float(getattr(r, "score", 0.0))
+ return None
+
+
+class PendingDecisionIndex:
+ """Nearest-neighbour index over decision vectors not yet written to the store.
+
+ Backs the in-batch side of dedup in ``bulk_upsert_decisions``: records
+ touched this batch are only matchable here until the deferred store write
+ lands. Scoring uses numpy when available (it ships with every bundled
+ persistent backend) so the per-group scan doesn't burn O(groups² · dim)
+ pure-Python CPU on a large first index; without numpy it falls back to a
+ norm-free dot loop over pre-normalised vectors.
+ """
+
+ def __init__(self) -> None:
+ self._ids: list[str] = []
+ self._id_set: set[str] = set()
+ try:
+ import numpy as np
+ except ImportError:
+ np = None # type: ignore[assignment]
+ self._np = np
+ self._buf: Any = None # numpy (capacity, dim) buffer of normalised rows
+ self._vecs: list[list[float]] = [] # pure-Python fallback, normalised
+ self._n = 0
+
+ @property
+ def ids(self) -> set[str]:
+ return self._id_set
+
+ def _normalise(self, vector: list[float]) -> Any:
+ if self._np is not None:
+ v = self._np.asarray(vector, dtype=self._np.float64)
+ norm = float(self._np.linalg.norm(v))
+ return v / norm if norm > 0 else None
+ norm = sum(x * x for x in vector) ** 0.5
+ return [x / norm for x in vector] if norm > 0 else None
+
+ def add(self, decision_id: str, vector: list[float]) -> None:
+ v = self._normalise(vector)
+ if v is None:
+ return
+ if self._np is not None:
+ if self._buf is None:
+ self._buf = self._np.empty((64, len(vector)), dtype=self._np.float64)
+ elif self._n == len(self._buf):
+ grown = self._np.empty((len(self._buf) * 2, self._buf.shape[1]))
+ grown[: self._n] = self._buf[: self._n]
+ self._buf = grown
+ self._buf[self._n] = v
+ else:
+ self._vecs.append(v)
+ self._ids.append(decision_id)
+ self._id_set.add(decision_id)
+ self._n += 1
+
+ def best(self, vector: list[float]) -> tuple[str, float] | None:
+ """The ``(decision_id, cosine)`` of the nearest pending vector, or None."""
+ if self._n == 0:
+ return None
+ q = self._normalise(vector)
+ if q is None:
+ return None
+ if self._np is not None:
+ scores = self._buf[: self._n] @ q
+ i = int(scores.argmax())
+ return self._ids[i], float(scores[i])
+ best_i, best_score = 0, float("-inf")
+ for i, v in enumerate(self._vecs):
+ score = sum(x * y for x, y in zip(v, q, strict=True))
+ if score > best_score:
+ best_i, best_score = i, score
+ return self._ids[best_i], best_score
+
+
+async def find_duplicate_decision_by_vector(
+ store: Any,
+ vector: list[float],
+ *,
+ pending: PendingDecisionIndex | None = None,
+ tau: float = DEFAULT_DEDUP_TAU,
+ exclude_ids: set[str] | None = None,
+) -> str | None:
+ """Vector-side twin of :func:`find_duplicate_decision` for batched dedup.
+
+ Searches the store by a precomputed query *vector* and, when *pending* is
+ given, also scores against vectors touched earlier in the same batch that
+ haven't been written to the store yet. Store hits whose ids appear in
+ *pending* are skipped — the pending index holds a fresher vector for them
+ than the store's stale row. Same policy as the text twin: the overall
+ nearest decision decides, thresholded at *tau*; store errors degrade to
+ the pending-only comparison.
+ """
+ best: tuple[str, float] | None = None
+ try:
+ results = await store.search_by_vector(vector, limit=SEARCH_FETCH)
+ except Exception:
+ results = None
+ if results:
+ exclude = set(exclude_ids or set())
+ if pending is not None:
+ exclude |= pending.ids
+ best = nearest_decision_hit(results, exclude)
+ if pending is not None:
+ local = pending.best(vector)
+ if local is not None and (best is None or local[1] > best[1]):
+ best = local
+ if best is None:
+ return None
+ decision_id, score = best
+ return decision_id if score >= tau else None
+
+
+async def upsert_decision_vectors(
+ store: Any,
+ items: list[tuple[str, str, dict]],
+ *,
+ vectors_by_text: dict[str, list[float]] | None = None,
+) -> None:
+ """Batched, best-effort embed+upsert of decision items into the store.
+
+ One :meth:`embed_texts` round instead of one embedder call per decision,
+ skipping texts whose vectors the caller already computed
+ (*vectors_by_text* — the dedup pass embeds the same texts minutes
+ earlier). Falls back to :meth:`embed_batch` when the store can't take raw
+ vectors OR when the vector path raises, so a transient store error can't
+ silently drop the whole batch. Failures of the fallback itself are
+ swallowed like the per-item path: search visibility is an enhancement,
+ never a reason to abort the SQL upsert that already happened.
+ """
+ if not items:
+ return
+ known = vectors_by_text or {}
+ wrote = False
+ try:
+ to_embed = [(pid, text, meta) for pid, text, meta in items if text not in known]
+ vectors = await store.embed_texts([text for _pid, text, _meta in to_embed])
+ if vectors is not None and len(vectors) == len(to_embed):
+ embedded = {
+ text: vec for (_pid, text, _meta), vec in zip(to_embed, vectors, strict=True)
+ }
+ wrote = bool(
+ await store.upsert_vectors(
+ [
+ (pid, known[text] if text in known else embedded[text], meta)
+ for pid, text, meta in items
+ ]
+ )
+ )
+ except Exception:
+ wrote = False
+ if not wrote:
+ with contextlib.suppress(Exception):
+ await store.embed_batch(items)
+
+
async def upsert_decision_vector(
store: Any,
decision_id: str,
@@ -87,22 +301,15 @@ async def upsert_decision_vector(
``search_codebase``. Failures are swallowed: a vector-store hiccup must
never abort the SQL upsert that already happened.
"""
- text = decision_match_text(title, decision)
- if not text:
+ item = decision_vector_item(
+ decision_id, title=title, decision=decision, evidence_file=evidence_file
+ )
+ if item is None:
return
# Best-effort: search visibility / dedup is an enhancement, not a
# correctness requirement for the SQL record, so a store hiccup is swallowed.
with contextlib.suppress(Exception):
- await store.embed_and_upsert(
- _decision_page_id(decision_id),
- text,
- {
- "title": title or "",
- "page_type": DECISION_PAGE_TYPE,
- "target_path": evidence_file or "",
- "content": text,
- },
- )
+ await store.embed_and_upsert(*item)
async def find_duplicate_decision(
@@ -123,23 +330,18 @@ async def find_duplicate_decision(
query = decision_match_text(title, decision)
if not query:
return None
- exclude = exclude_ids or set()
try:
- results = await store.search(query, limit=_SEARCH_FETCH)
+ results = await store.search(query, limit=SEARCH_FETCH)
except Exception:
return None
- for r in results: # results are ordered best-first
- page_id = getattr(r, "page_id", "")
- if not page_id.startswith(DECISION_VECTOR_PREFIX):
- continue
- decision_id = page_id[len(DECISION_VECTOR_PREFIX) :]
- if decision_id in exclude:
- continue
- # First (nearest) decision hit decides: if it clears the bar it's the
- # match; if not, nothing closer exists, so there is no duplicate.
- return decision_id if getattr(r, "score", 0.0) >= tau else None
- return None
+ # First (nearest) decision hit decides: if it clears the bar it's the
+ # match; if not, nothing closer exists, so there is no duplicate.
+ hit = nearest_decision_hit(results, exclude_ids)
+ if hit is None:
+ return None
+ decision_id, score = hit
+ return decision_id if score >= tau else None
async def find_related_decisions(
@@ -166,12 +368,22 @@ async def find_related_decisions(
query = decision_match_text(title, decision)
if not query:
return []
- exclude = exclude_ids or set()
try:
- results = await store.search(query, limit=_SEARCH_FETCH)
+ results = await store.search(query, limit=SEARCH_FETCH)
except Exception:
return []
+ return _related_from_results(results, lo=lo, hi=hi, exclude_ids=exclude_ids, limit=limit)
+
+def _related_from_results(
+ results: list[Any],
+ *,
+ lo: float,
+ hi: float,
+ exclude_ids: set[str] | None,
+ limit: int,
+) -> list[tuple[str, float]]:
+ exclude = exclude_ids or set()
out: list[tuple[str, float]] = []
for r in results:
page_id = getattr(r, "page_id", "")
@@ -186,3 +398,40 @@ async def find_related_decisions(
if len(out) >= limit:
break
return out
+
+
+async def find_related_decisions_many(
+ store: Any,
+ items: list[tuple[str, str, set[str]]],
+ *,
+ lo: float,
+ hi: float = 1.01,
+ limit: int = 25,
+) -> list[list[tuple[str, float]]]:
+ """Batched :func:`find_related_decisions` — one embedding round, not N.
+
+ *items* is ``(title, decision, exclude_ids)`` per query; the result list
+ is aligned by index. Uses the store's ``search_many`` so every query is
+ embedded in a single embedder call — the per-query network round-trip is
+ what made the supersession pass scale with decision count. Store errors
+ degrade to empty lists, matching the per-item helper.
+ """
+ if not items:
+ return []
+ queries = [decision_match_text(title, decision) for title, decision, _ in items]
+ try:
+ # Empty query texts still occupy their slot (keeps results aligned)
+ # but must not reach the embedder — some providers reject "".
+ all_results = await store.search_many([q or " " for q in queries], limit=SEARCH_FETCH)
+ except Exception:
+ return [[] for _ in items]
+ if len(all_results) != len(items):
+ # A store that drops or pads result slots would desync the zip below;
+ # degrade the whole call to empty like any other store error.
+ return [[] for _ in items]
+ return [
+ _related_from_results(results, lo=lo, hi=hi, exclude_ids=exclude, limit=limit)
+ if query
+ else []
+ for results, query, (_t, _d, exclude) in zip(all_results, queries, items, strict=True)
+ ]
diff --git a/packages/core/src/repowise/core/persistence/crud/decisions.py b/packages/core/src/repowise/core/persistence/crud/decisions.py
index 2c5460f09..bda263bb6 100644
--- a/packages/core/src/repowise/core/persistence/crud/decisions.py
+++ b/packages/core/src/repowise/core/persistence/crud/decisions.py
@@ -10,6 +10,7 @@ import json
from datetime import datetime
from typing import Any
+import structlog
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -441,31 +442,98 @@ async def bulk_upsert_decisions(
# record, and is grown as records are created so paraphrases *within* one
# batch also collapse.
id_to_rec: dict[str, DecisionRecord] = {rec.id: rec for rec in existing_by_norm.values()}
+
+ # Headline candidate per group: highest source rank, tie-break by
+ # confidence. Hoisted so the batched embedding below can see every
+ # group's match text before the loop runs.
+ headline_by_norm: dict[str, dict] = {
+ norm: max(
+ members,
+ key=lambda d: (rank_for_source(d.get("source", "")), d.get("confidence", 0.0)),
+ )
+ for norm, members in groups.items()
+ }
+
+ # Batched embedding for dedup. The per-item path embeds one query per
+ # residual group and one upsert per touched record — thousands of serial
+ # network round-trips on a first index. Instead: embed every group's
+ # match text in a few chunked requests up front, search the store by raw
+ # vector, dedup *within* the batch against a local pending index, and
+ # defer the store writes to one batched upsert after the loop. Stores
+ # that can't hand back vectors (no embedder, or search_by_vector left at
+ # the base-class default) keep the per-item flow unchanged.
+ batch_mode = False
+ group_vecs: dict[str, list[float]] = {}
+ group_texts: dict[str, str] = {}
if vector_store is not None:
from repowise.core.analysis.decision_semantic_match import (
+ PendingDecisionIndex,
+ decision_match_text,
+ decision_vector_item,
find_duplicate_decision,
+ find_duplicate_decision_by_vector,
upsert_decision_vector,
+ upsert_decision_vectors,
+ )
+ from repowise.core.persistence.vector_store import VectorStore
+
+ # Static capability check: the base-class search_by_vector is the
+ # "unsupported" sentinel. A runtime probe would conflate a transient
+ # store error with lack of support and silently fall back to the
+ # O(N)-round-trip path this branch exists to avoid.
+ store_cls_search = getattr(type(vector_store), "search_by_vector", None)
+ supports_vector_search = (
+ callable(store_cls_search) and store_cls_search is not VectorStore.search_by_vector
+ )
+
+ ordered = [
+ (norm, decision_match_text(h.get("title", ""), h.get("decision") or ""))
+ for norm, h in headline_by_norm.items()
+ ]
+ ordered = [(norm, text) for norm, text in ordered if text]
+ if ordered and supports_vector_search:
+ try:
+ vecs = await vector_store.embed_texts([text for _, text in ordered])
+ except Exception:
+ vecs = None
+ if vecs is not None and len(vecs) == len(ordered):
+ group_vecs = dict(zip((n for n, _ in ordered), vecs, strict=True))
+ group_texts = dict(ordered)
+ batch_mode = True
+
+ structlog.get_logger(__name__).info(
+ "decision_dedup_embedding",
+ groups=len(groups),
+ batched=batch_mode,
)
+ # Vectors of records touched this batch but not yet written to the store —
+ # the in-batch side of dedup in batch mode. A record is only registered
+ # when its final text equals the group text its vector came from, so the
+ # index never matches against text the record doesn't actually contain.
+ pending = PendingDecisionIndex() if batch_mode else None
+
touched_ids: list[str] = []
for norm, members in groups.items():
- # Headline candidate: highest source rank, tie-break by confidence.
- headline = max(
- members,
- key=lambda d: (rank_for_source(d.get("source", "")), d.get("confidence", 0.0)),
- )
+ headline = headline_by_norm[norm]
rec = existing_by_norm.get(norm)
# No title match → ask the store whether a semantically-equivalent
# decision already exists, and fold into it if so (cheap title dedup
# stays the first pass; this only runs on the residual).
if rec is None and vector_store is not None:
- match_id = await find_duplicate_decision(
- vector_store,
- title=headline.get("title", ""),
- decision=headline.get("decision") or "",
- )
+ q_vec = group_vecs.get(norm)
+ if batch_mode and q_vec is not None:
+ match_id = await find_duplicate_decision_by_vector(
+ vector_store, q_vec, pending=pending
+ )
+ else:
+ match_id = await find_duplicate_decision(
+ vector_store,
+ title=headline.get("title", ""),
+ decision=headline.get("decision") or "",
+ )
if match_id is not None and match_id in id_to_rec:
rec = id_to_rec[match_id]
existing_by_norm[norm] = rec
@@ -553,15 +621,54 @@ async def bulk_upsert_decisions(
# (Re-)embed the record into the shared store so it's matchable by
# later groups in this batch + future runs, and discoverable via
- # search_codebase. Best-effort — never blocks the SQL upsert.
+ # search_codebase. Best-effort — never blocks the SQL upsert. In
+ # batch mode the write is deferred to one batched upsert after the
+ # loop; until then the group vector stands in for the record in the
+ # pending index — but only when the record's final text IS the group
+ # text (created, or promoted from this headline). A fold that kept
+ # the record's own fields must not be matchable under the incoming
+ # group's text.
if vector_store is not None:
- await upsert_decision_vector(
- vector_store,
+ if batch_mode:
+ q_vec = group_vecs.get(norm)
+ final_text = decision_match_text(rec.title, rec.decision or "")
+ if (
+ q_vec is not None
+ and pending is not None
+ and rec.id not in pending.ids
+ and final_text == group_texts.get(norm)
+ ):
+ pending.add(rec.id, q_vec)
+ else:
+ await upsert_decision_vector(
+ vector_store,
+ rec.id,
+ title=rec.title,
+ decision=rec.decision or "",
+ evidence_file=rec.evidence_file,
+ )
+
+ # Batched store write: upsert every touched record's *final* text (post
+ # merge/promotion) in one go, reusing the vectors already computed for
+ # unchanged texts and embedding only the delta.
+ if vector_store is not None and batch_mode and touched_ids:
+ items: list[tuple[str, str, dict]] = []
+ for rid in dict.fromkeys(touched_ids):
+ rec = id_to_rec.get(rid)
+ if rec is None:
+ continue
+ item = decision_vector_item(
rec.id,
title=rec.title,
decision=rec.decision or "",
evidence_file=rec.evidence_file,
)
+ if item is not None:
+ items.append(item)
+ vectors_by_text = {
+ text: group_vecs[norm] for norm, text in group_texts.items() if norm in group_vecs
+ }
+ await upsert_decision_vectors(vector_store, items, vectors_by_text=vectors_by_text)
await session.flush()
return touched_ids
diff --git a/packages/core/src/repowise/core/persistence/vector_store/_base.py b/packages/core/src/repowise/core/persistence/vector_store/_base.py
index 7a2c83459..80187c121 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/_base.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/_base.py
@@ -76,6 +76,49 @@ class VectorStore(ABC):
for page_id, text, metadata in items:
await self.embed_and_upsert(page_id, text, metadata)
+ async def embed_texts(self, texts: list[str]) -> list[list[float]] | None:
+ """Embed *texts* in batched embedder requests, without upserting.
+
+ Lets a caller that needs the raw vectors (e.g. decision dedup, which
+ searches *and* upserts the same text) pay for one batched embedding
+ instead of one round-trip per item. Returns ``None`` when the backend
+ holds no embedder — callers must fall back to the per-item text APIs.
+ Chunked so a large input can't blow the embedder's per-request token
+ cap; each text is capped at :data:`EMBED_TEXT_MAX_CHARS`.
+ """
+ embedder = getattr(self, "_embedder", None)
+ if embedder is None:
+ return None # backend can't embed directly — caller falls back
+ if not texts:
+ return []
+ out: list[list[float]] = []
+ for _chunk, capped_texts in iter_embed_chunks([("", t, {}) for t in texts]):
+ out.extend(await embedder.embed(capped_texts))
+ return out
+
+ async def search_by_vector(
+ self, vector: list[float], limit: int = 10
+ ) -> list[SearchResult] | None:
+ """Return the *limit* nearest pages to a precomputed query *vector*.
+
+ Batching hook for callers that already embedded their queries via
+ :meth:`embed_texts`. Returns ``None`` when the backend can't search by
+ raw vector (callers fall back to :meth:`search`), never raises for
+ that reason.
+ """
+ return None
+
+ async def upsert_vectors(self, items: list[tuple[str, list[float], dict]]) -> bool:
+ """Upsert many ``(page_id, vector, metadata)`` items without embedding.
+
+ The write-side counterpart of :meth:`search_by_vector`: callers that
+ computed vectors once via :meth:`embed_texts` can persist them without
+ a second embedder round-trip per item. Returns ``False`` when the
+ backend doesn't support raw-vector writes (callers fall back to
+ :meth:`embed_batch`), ``True`` after a successful write.
+ """
+ return False
+
@abstractmethod
async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
"""Embed *query* and return the *limit* nearest pages."""
diff --git a/packages/core/src/repowise/core/persistence/vector_store/in_memory.py b/packages/core/src/repowise/core/persistence/vector_store/in_memory.py
index 6cf1b49aa..559693417 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/in_memory.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/in_memory.py
@@ -61,12 +61,22 @@ class InMemoryVectorStore(VectorStore):
)
return results
+ async def upsert_vectors(self, items: list[tuple[str, list[float], dict]]) -> bool:
+ for page_id, vector, metadata in items:
+ self._store[page_id] = (list(vector), dict(metadata))
+ return True
+
async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
if not self._store:
return []
q_vecs = await self._embedder.embed([query])
return self._search_by_vector(q_vecs[0], limit)
+ async def search_by_vector(self, vector: list[float], limit: int = 10) -> list[SearchResult]:
+ if not self._store:
+ return []
+ return self._search_by_vector(vector, limit)
+
async def search_many(self, queries: list[str], limit: int = 10) -> list[list[SearchResult]]:
"""One embedder call for all queries, then local scoring per query."""
if not queries:
diff --git a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
index 582f91f4d..1e721edd1 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/lancedb_store.py
@@ -217,6 +217,18 @@ class LanceDBVectorStore(VectorStore):
for r in raw
]
+ async def upsert_vectors(self, items: list[tuple[str, list[float], dict]]) -> bool:
+ if not items:
+ return True
+ await self._ensure_connected()
+ await self._ensure_table([float(v) for v in items[0][1]])
+ rows = [
+ self._row(page_id, [float(v) for v in vector], metadata)
+ for page_id, vector, metadata in items
+ ]
+ await self._upsert_rows(rows)
+ return True
+
async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
await self._ensure_connected()
if self._table is None:
@@ -225,6 +237,12 @@ class LanceDBVectorStore(VectorStore):
q_vecs = await self._embedder.embed([query])
return await self._search_by_vector([float(v) for v in q_vecs[0]], limit)
+ async def search_by_vector(self, vector: list[float], limit: int = 10) -> list[SearchResult]:
+ await self._ensure_connected()
+ if self._table is None:
+ return []
+ return await self._search_by_vector([float(v) for v in vector], limit)
+
async def search_many(self, queries: list[str], limit: int = 10) -> list[list[SearchResult]]:
"""One embedder call for all queries; the vector lookups are local."""
if not queries:
diff --git a/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py b/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py
index 325e21a96..61f132488 100644
--- a/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py
+++ b/packages/core/src/repowise/core/persistence/vector_store/pgvector_store.py
@@ -90,9 +90,34 @@ class PgVectorStore(VectorStore):
await session.execute(stmt, params)
await session.commit()
+ async def upsert_vectors(self, items: list[tuple[str, list[float], dict]]) -> bool:
+ """Raw-vector counterpart of :meth:`embed_and_upsert`, same semantics:
+ UPDATE-only against ``wiki_pages``, so ids without a row (e.g. the
+ synthetic ``decision:`` namespace) are silently skipped — this backend
+ stores embeddings on page rows, not as standalone vectors. Returning
+ True means the statement ran, not that every id matched a row.
+ """
+ if not items:
+ return True
+
+ from sqlalchemy.sql import text as sa_text
+
+ stmt = sa_text("UPDATE wiki_pages SET embedding = CAST(:emb AS vector) WHERE id = :pid")
+ params = [
+ {"emb": _encode([float(v) for v in vector]), "pid": page_id}
+ for page_id, vector, _meta in items
+ ]
+ async with self._session_factory() as session:
+ await session.execute(stmt, params)
+ await session.commit()
+ return True
+
async def search(self, query: str, limit: int = 10) -> list[SearchResult]:
q_vecs = await self._embedder.embed([query])
- vec_str = _encode(q_vecs[0])
+ return await self.search_by_vector(q_vecs[0], limit)
+
+ async def search_by_vector(self, vector: list[float], limit: int = 10) -> list[SearchResult]:
+ vec_str = _encode([float(v) for v in vector])
from sqlalchemy.sql import text as sa_text
diff --git a/tests/unit/persistence/test_decision_semantic_dedup.py b/tests/unit/persistence/test_decision_semantic_dedup.py
index 0222e9dd1..e174144de 100644
--- a/tests/unit/persistence/test_decision_semantic_dedup.py
+++ b/tests/unit/persistence/test_decision_semantic_dedup.py
@@ -184,3 +184,61 @@ async def test_intra_batch_paraphrases_collapse(async_session):
assert len(rows) == 1
evidence = await list_decision_evidence(async_session, rows[0].id)
assert {e.source for e in evidence} == {"inline_marker", "llm_inferred"}
+
+
+class _CountingOrthogonalEmbedder:
+ """Counts embed() round-trips; every distinct text embeds orthogonally.
+
+ Each ``embed()`` call models one network request to a real provider — the
+ regression under guard is per-decision round-trips, not per-text cost.
+ Unlike :class:`_KeywordEmbedder` (6 hash buckets, so unrelated texts can
+ collide at cosine 1.0), each distinct text gets its own axis so no
+ decision merges and the residual set stays at full size.
+ """
+
+ dimensions = 64
+
+ def __init__(self) -> None:
+ self.calls = 0
+ self._axis_by_text: dict[str, int] = {}
+
+ async def embed(self, texts: list[str]) -> list[list[float]]:
+ self.calls += 1
+ out: list[list[float]] = []
+ for t in texts:
+ axis = self._axis_by_text.setdefault(t, len(self._axis_by_text))
+ v = [0.0] * self.dimensions
+ v[axis] = 1.0
+ out.append(v)
+ return out
+
+
+async def test_bulk_upsert_embedder_roundtrips_do_not_scale_with_decisions(async_session):
+ """One first-index batch of N decisions must cost O(N/chunk) embedder
+ requests, not O(N).
+
+ The pre-batching implementation issued one search embedding per residual
+ decision plus one upsert embedding per touched record — ~2N serial network
+ round-trips inside the persistence write transaction, which stalled
+ ``repowise init`` Phase 4 for tens of minutes on repos with hundreds of
+ harvested code-comment decisions.
+ """
+ embedder = _CountingOrthogonalEmbedder()
+ store = InMemoryVectorStore(embedder)
+ repo = await insert_repo(async_session)
+
+ n = 48 # distinct topics → every decision is residual (worst case)
+ decisions = [
+ _decision(f"Adopt component {i}", source="code_comment", quote=f"adopt component {i}")
+ for i in range(n)
+ ]
+ await bulk_upsert_decisions(async_session, repo.id, decisions, vector_store=store)
+
+ rows = await _rows(async_session, repo.id)
+ assert len(rows) == n # nothing wrongly collapsed
+ # Two batched rounds (dedup queries + final upserts), chunked at 16 items:
+ # ceil(48/16) * 2 = 6. The old per-item path needed ~2 * 48 = 96.
+ assert embedder.calls <= 10, (
+ f"embedder called {embedder.calls} times for {n} decisions — "
+ "per-decision embedding round-trips have regressed"
+ )