Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(keyless): close two unguarded vector reads and the confidence hole (#1380)

Raghav Chamadiya8f1431bdf431
diff --git a/packages/cli/src/repowise/cli/commands/search_cmd.py b/packages/cli/src/repowise/cli/commands/search_cmd.py
index ea9c439cd..9d3280087 100644
--- a/packages/cli/src/repowise/cli/commands/search_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/search_cmd.py
@@ -13,6 +13,12 @@ from repowise.cli.helpers import (
run_async,
)
+# Rank-fusion damping for the workspace fan-out, matching the value the server's
+# retrieval fusion uses. Only reached when repos in one workspace answered on
+# different score scales (some semantic, some full-text), where the raw scores
+# are not comparable and the rank is the only shared quantity.
+_WORKSPACE_RRF_K = 60
+
@click.command("search")
@click.argument("query")
@@ -101,33 +107,58 @@ def search_command(
_search_symbol(repo_path, query, limit)
return
- # Multi-repo fan-out — gather, sort by score, render once.
+ # Multi-repo fan-out — gather, rank, render once.
all_results: list = []
+ mixed_scales = False
+ keyless_repos: list[str] = []
for rp in repo_paths:
try:
if mode == "fulltext":
results = _collect_fulltext(rp, query, limit)
elif mode == "semantic":
- results = _collect_semantic(rp, query, limit)
+ results, served_fulltext = _collect_semantic(rp, query, limit)
+ if served_fulltext:
+ keyless_repos.append(rp.name)
+ else:
+ mixed_scales = mixed_scales or bool(results)
else: # symbol
results = _collect_symbol(rp, query, limit)
except Exception as exc:
console.print(f"[yellow]search failed for {rp.name}: {exc}[/yellow]")
continue
- for r in results:
- all_results.append((rp.name, r))
+ for rank, r in enumerate(results):
+ all_results.append((rp.name, r, rank))
if not all_results:
console.print("[yellow]No results across the workspace.[/yellow]")
return
+ if keyless_repos:
+ console.print(
+ f"[yellow]No embedder configured for: {', '.join(sorted(keyless_repos))}."
+ "[/yellow]\nThose repos contributed full-text results."
+ )
+
if mode == "symbol":
- _render_symbol_rows(all_results, query, limit, multi=True)
+ _render_symbol_rows([(n, r) for n, r, _ in all_results], query, limit, multi=True)
else:
- # Sort by score desc, slice to limit
- all_results.sort(key=lambda pair: getattr(pair[1], "score", 0.0), reverse=True)
+ if mode == "semantic" and mixed_scales and keyless_repos:
+ # Full-text and vector scores are not the same quantity: FTS returns
+ # a negated FTS5 rank (typically >1) and LanceDB returns 1 - cosine
+ # distance (roughly 0..1). Sorting them together lets one keyless
+ # repo's full-text rows evict every semantic hit in the workspace.
+ # Fuse on RANK instead, which is what this codebase already does
+ # wherever it combines independently-scored retrievers.
+ all_results.sort(key=lambda t: 1.0 / (t[2] + _WORKSPACE_RRF_K), reverse=True)
+ else:
+ # One scale throughout: the raw score is meaningful, so keep the
+ # existing ordering rather than perturbing it with a rank fusion.
+ all_results.sort(key=lambda t: getattr(t[1], "score", 0.0), reverse=True)
all_results = all_results[:limit]
- _display_results_multi(all_results, f"{mode.capitalize()} search: '{query}' (workspace)")
+ _display_results_multi(
+ [(n, r) for n, r, _ in all_results],
+ f"{mode.capitalize()} search: '{query}' (workspace)",
+ )
def _search_fulltext(repo_path, query: str, limit: int) -> None:
@@ -146,7 +177,10 @@ def _search_fulltext(repo_path, query: str, limit: int) -> None:
def _search_semantic(repo_path, query: str, limit: int) -> None:
+ served_fulltext = False
+
async def _run():
+ nonlocal served_fulltext
from pathlib import Path
# Try LanceDB first (populated during repowise init)
@@ -158,12 +192,22 @@ def _search_semantic(repo_path, query: str, limit: int) -> None:
resolve_embedder_for_repo,
)
from repowise.core.persistence.vector_store import LanceDBVectorStore
+ from repowise.core.providers.embedding import store_has_semantic_vectors
embedder = build_embedder(resolve_embedder_for_repo(repo_path))
store = LanceDBVectorStore(str(lance_dir), embedder=embedder)
- results = await store.search(query, limit=limit)
- await store.close()
- return results
+ try:
+ # A keyless index's own vectors are not discriminative, so
+ # ranking on them serves noise as if it were meaning. Fall
+ # through to full text instead: `--mode semantic` has to
+ # answer with something, and full text is what a keyless
+ # index actually offers. Same predicate every other vector
+ # read uses; this site was simply never wired to it.
+ if store_has_semantic_vectors(store):
+ return await store.search(query, limit=limit)
+ served_fulltext = True
+ finally:
+ await store.close()
except Exception:
pass
@@ -178,7 +222,35 @@ def _search_semantic(repo_path, query: str, limit: int) -> None:
return results
results = run_async(_run())
- _display_results(results, f"Semantic search: '{query}'")
+ if served_fulltext:
+ # Say which mode actually answered. Labelling full-text results
+ # "Semantic search" is how someone concludes semantic retrieval is bad
+ # when what they have is no embedder. This is the common case, not the
+ # rare one: a genuinely keyless repo never reaches build_embedder's
+ # failure warning, because resolving to the keyless embedder is not a
+ # failure.
+ #
+ # Worded for both states. A repo pinned to a real embedder whose key has
+ # since gone away also lands here, having already printed build_embedder's
+ # warning; telling that user "no embedder configured" would contradict
+ # both their config and the line above it.
+ from repowise.cli.providers.embedders import resolve_embedder_for_repo
+
+ pinned = resolve_embedder_for_repo(repo_path)
+ why = (
+ "No embedder configured for this repo"
+ if pinned == "mock"
+ else f"The '{pinned}' embedder could not be used"
+ )
+ console.print(
+ f"[yellow]{why}, so there is no semantic index to search.[/yellow]\n"
+ "Showing full-text results instead. Set an embedder key and run "
+ "[cyan]repowise reindex[/cyan] for semantic search."
+ )
+ _display_results(
+ results,
+ f"Full-text search: '{query}'" if served_fulltext else f"Semantic search: '{query}'",
+ )
def _search_symbol(repo_path, query: str, limit: int) -> None:
@@ -265,7 +337,15 @@ def _collect_fulltext(repo_path, query: str, limit: int):
def _collect_semantic(repo_path, query: str, limit: int):
+ """Return ``(results, served_fulltext)`` for one repo in the fan-out.
+
+ The flag is what lets the caller avoid sorting full-text scores against
+ vector scores, and what lets it say which repos answered lexically.
+ """
+ served_fulltext = False
+
async def _run():
+ nonlocal served_fulltext
from pathlib import Path
from repowise.core.persistence import FullTextSearch, create_engine
@@ -278,12 +358,19 @@ def _collect_semantic(repo_path, query: str, limit: int):
resolve_embedder_for_repo,
)
from repowise.core.persistence.vector_store import LanceDBVectorStore
+ from repowise.core.providers.embedding import store_has_semantic_vectors
embedder = build_embedder(resolve_embedder_for_repo(repo_path))
store = LanceDBVectorStore(str(lance_dir), embedder=embedder)
- results = await store.search(query, limit=limit)
- await store.close()
- return results
+ try:
+ # See _search_semantic: a keyless store ranks on noise, so
+ # this repo falls through to its full-text results rather
+ # than contributing a window of them to the workspace fan-out.
+ if store_has_semantic_vectors(store):
+ return await store.search(query, limit=limit)
+ served_fulltext = True
+ finally:
+ await store.close()
except Exception:
pass
@@ -294,7 +381,7 @@ def _collect_semantic(repo_path, query: str, limit: int):
await engine.dispose()
return results
- return run_async(_run())
+ return run_async(_run()), served_fulltext
def _collect_symbol(repo_path, query: str, limit: int):
diff --git a/packages/cli/src/repowise/cli/providers/embedders.py b/packages/cli/src/repowise/cli/providers/embedders.py
index 179e040f9..644c02a08 100644
--- a/packages/cli/src/repowise/cli/providers/embedders.py
+++ b/packages/cli/src/repowise/cli/providers/embedders.py
@@ -127,11 +127,22 @@ def resolve_embedder_for_repo(repo_path: Any) -> str:
def build_embedder(embedder_name_resolved: str) -> Any:
- """Construct the configured embedder, falling back to MockEmbedder.
+ """Construct the configured embedder, falling back to KeylessEmbedder.
Shared by the generation flows and the decision semantic-dedup wiring so
the same backend selection logic isn't duplicated. Real providers fall
- back to the deterministic mock when their SDK/credentials are unavailable.
+ back to the keyless embedder when their SDK/credentials are unavailable.
+
+ That fallback is **said out loud**. It used to be a bare ``except`` that
+ returned the keyless embedder with nothing printed anywhere, so
+ ``--embedder ollama`` against a stopped Ollama produced a repo that indexes
+ and searches without complaint and has no semantic retrieval at all: the
+ 8-wide vectors carry no signal, and the read path now declines to rank on
+ them. ``doctor`` reports healthy throughout. The server path already
+ records this state as ``degraded: True``; the CLI path recorded nothing.
+
+ Warn only when someone named a real backend. ``mock`` is the keyless
+ default and reaching it is not a failure, so it stays silent.
"""
from repowise.core.providers.embedding.base import KeylessEmbedder
from repowise.core.providers.embedding.registry import get_embedder
@@ -140,5 +151,17 @@ def build_embedder(embedder_name_resolved: str) -> Any:
return KeylessEmbedder()
try:
return get_embedder(embedder_name_resolved, **_embedder_kwargs(embedder_name_resolved))
- except Exception:
+ except Exception as exc:
+ # Said here, once, rather than in each of the thirteen call sites, for
+ # the same reason build_vector_store says its own refusal here: a
+ # caller that forgets is exactly how this became invisible.
+ from repowise.cli.helpers import console
+
+ console.print(
+ f"[yellow]Embedder '{embedder_name_resolved}' could not be built:[/yellow] "
+ f"{type(exc).__name__}: {exc}\n"
+ "Falling back to keyless embeddings, which means [bold]no semantic search[/bold] "
+ "for this run.\nFull-text and symbol search are unaffected. Fix the key or "
+ "endpoint and run [cyan]repowise reindex[/cyan]\nto restore semantic search."
+ )
return KeylessEmbedder()
diff --git a/packages/server/src/repowise/server/mcp_server/_watchdog.py b/packages/server/src/repowise/server/mcp_server/_watchdog.py
index 3398220be..11f995513 100644
--- a/packages/server/src/repowise/server/mcp_server/_watchdog.py
+++ b/packages/server/src/repowise/server/mcp_server/_watchdog.py
@@ -24,8 +24,10 @@ entirely.
from __future__ import annotations
+import contextlib
import logging
import os
+import sys
import threading
from repowise.core.procutils import ProcInfo, ancestor_chain, pid_alive, process_create_token
@@ -127,6 +129,23 @@ def start_parent_watchdog() -> threading.Thread | None:
info.name or "?",
info.pid,
)
+ # os._exit skips interpreter shutdown, which includes
+ # flushing, so the line above can sit in a buffer and
+ # never reach the log. That makes a watchdog kill
+ # indistinguishable from the server simply answering
+ # nothing: an in-flight batch of tool calls returns
+ # instantly and empty from then on with no recorded
+ # cause. Flush so the reason survives.
+ #
+ # Deliberately NOT logging.shutdown(): it acquires every
+ # handler's lock, and the trigger condition here is a
+ # dead client, which is exactly when nobody is draining
+ # our stderr pipe. A main thread blocked inside emit()
+ # holds that lock, so acquiring it would hang the
+ # watchdog forever and leak the process this exists to
+ # kill. Flushing the stream takes no logging lock.
+ with contextlib.suppress(Exception):
+ sys.stderr.flush()
os._exit(0)
except Exception:
# Probe hiccup — never let the watchdog kill or crash the
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py
index 79389b501..1bd2e96c1 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py
@@ -133,6 +133,7 @@ from repowise.server.mcp_server.tool_answer.config import (
_INLINE_BODY_MAX_LINES,
_INLINE_BODY_MAX_SYMBOLS,
_PAGE_EXCERPT_HITS,
+ _SYMBOL_AGREEMENT_TOP_RANK_MAX,
_SYSTEM_PROMPT,
_USER_TEMPLATE,
)
@@ -194,6 +195,12 @@ if _MAX_CHARS_PER_HIT_EXCERPT < _GATED_EXCERPT_CHARS:
# value (0/off/false/no) to restore the legacy abstain-on-ambiguous behaviour.
_ALWAYS_SYNTHESIZE_ENV = "REPOWISE_ANSWER_ALWAYS_SYNTHESIZE"
_AGREEMENT_CONFIDENCE_ENV = "REPOWISE_ANSWER_AGREEMENT_CONFIDENCE"
+# The keyless-only half of that signal (FTS + symbol in place of FTS + vector,
+# which a keyless index can never produce). Its own flag on purpose: the pair
+# above was tuned against the 99-question eval, whereas this substitutes a leg
+# the fusion already prices at a third weight, so it must be reversible and
+# A/B-able WITHOUT also switching off the measured half.
+_SYMBOL_AGREEMENT_ENV = "REPOWISE_ANSWER_SYMBOL_AGREEMENT"
# Keep the exact_symbol union fast path from hijacking a "how does X work"
# mechanism question, whose real answer often lives in a different file than the
# named symbol's body.
@@ -287,15 +294,30 @@ def _agreement_confidence_enabled() -> bool:
}
-def _agreement_dominant(hits: list[dict]) -> bool:
+def _symbol_agreement_enabled() -> bool:
+ """Whether a keyless index may read agreement from FTS + the symbol leg.
+
+ Independently reversible from :func:`_agreement_confidence_enabled`, which
+ governs the measured FTS + vector pair. Turning this off restores the state
+ where a keyless index simply cannot earn the agreement lift.
+ """
+ return os.environ.get(_SYMBOL_AGREEMENT_ENV, "").strip().lower() not in {
+ "0",
+ "false",
+ "no",
+ "off",
+ }
+
+
+def _agreement_dominant(hits: list[dict], *, vector_leg_keyless: bool = False) -> bool:
"""True when the top hit is the confident pick by retriever AGREEMENT.
RRF fusion compresses scores: a page both retrievers rank #1 barely
outscores one they rank #2 (ratio ~1.017), so the numeric dominance ratio
calls the *most* confident retrieval "non-dominant" and demotes it. This
- reads the per-source ranks (``_fts_rank`` / ``_vec_rank``) instead: when FTS
- and vector independently put the SAME page at (or within a rank of) the top,
- that consensus is a stronger ground-truth signal than any RRF score margin.
+ reads the per-source ranks instead: when two retrievers put the SAME page at
+ (or within a rank of) the top, that consensus is a stronger ground-truth
+ signal than any RRF score margin.
Conservative. Requires the top hit to be found by BOTH retrievers near the
top of each, to rank no lower than the runner-up in either source, and the
@@ -303,30 +325,66 @@ def _agreement_dominant(hits: list[dict]) -> bool:
lower-ranked in a source). Otherwise returns False and the caller falls back
to the pure ratio/gap gate. Agreement can only LIFT — the demotion gates
still apply.
+
+ ``vector_leg_keyless`` swaps the vector leg for the symbol leg. On an index
+ with no semantic vectors the vector leg is skipped outright, so ``_vec_rank``
+ is never written for any question and a fixed FTS+vector pair makes this
+ signal permanently unreachable — every keyless answer is then graded by the
+ pure ratio gate, which is exactly the gate this function exists because it
+ mis-reads. The symbol leg runs on every index and records ``_sym_rank``.
+
+ **The caller must pass the retrieval leg's own status, not infer it from
+ the hits.** By the time this runs, ``hits`` is capped to the top 5 out of a
+ much larger fused pool, so "no hit carries a ``_vec_rank``" is *not*
+ evidence the leg was skipped: a keyed index whose vector leg timed out,
+ errored, was scope-filtered, or was simply outranked by five FTS-and-symbol
+ hits presents identically. Substituting on that inference would fire exactly
+ when evidence is weakest and manufacture "high" confidence from it.
+
+ The symbol pair is held to a stricter rank than the vector pair. FTS and the
+ symbol leg are not independent — the wiki page FTS indexes contains the
+ public symbol table the symbol leg matches on — so their agreeing is closer
+ to one lexical match observed twice than to two retrievers concurring. The
+ fusion beside this already prices that leg at a third of the others'
+ (``_SYMBOL_LEG_RRF_K`` 180 vs ``_RRF_K`` 60, after a same-named React
+ component displaced the right file on the 99-question eval). Requiring an
+ exact rank-0 tie keeps the weaker signal from carrying the stronger claim.
+
+ Note the consequence: at ``top_rank_max = 0`` the runner-up comparison below
+ can no longer reject anything, because two hits cannot share rank 0 within
+ one leg. The symbol pair therefore reduces exactly to "the top hit is #1 in
+ FTS and #1 in the symbol leg", which is the intended rule; the shared gap
+ check is retained for the vector pair, where it does constrain.
"""
if len(hits) < 2:
return False
+ if vector_leg_keyless:
+ second_field = "_sym_rank"
+ top_rank_max = _SYMBOL_AGREEMENT_TOP_RANK_MAX
+ else:
+ second_field = "_vec_rank"
+ top_rank_max = _AGREEMENT_TOP_RANK_MAX
top = hits[0]
- top_fts = top.get("_fts_rank")
- top_vec = top.get("_vec_rank")
+ top_a = top.get("_fts_rank")
+ top_b = top.get(second_field)
# Top must be a consensus pick: found by both retrievers, near the top of
# each. A one-retriever top hit is exactly the ambiguous case we must NOT
# lift.
- if top_fts is None or top_vec is None:
+ if top_a is None or top_b is None:
return False
- if top_fts > _AGREEMENT_TOP_RANK_MAX or top_vec > _AGREEMENT_TOP_RANK_MAX:
+ if top_a > top_rank_max or top_b > top_rank_max:
return False
second = hits[1]
- sec_fts = second.get("_fts_rank")
- sec_vec = second.get("_vec_rank")
+ sec_a = second.get("_fts_rank")
+ sec_b = second.get(second_field)
# Runner-up found by only one retriever → the consensus top clearly wins.
- if sec_fts is None or sec_vec is None:
+ if sec_a is None or sec_b is None:
return True
# Runner-up found by both: the top must rank at least as high in BOTH
# sources (no source disagrees) and strictly ahead in at least one.
- if top_fts <= sec_fts and top_vec <= sec_vec:
- return (sec_fts - top_fts) >= _AGREEMENT_RANK_GAP or (
- sec_vec - top_vec
+ if top_a <= sec_a and top_b <= sec_b:
+ return (sec_a - top_a) >= _AGREEMENT_RANK_GAP or (
+ sec_b - top_b
) >= _AGREEMENT_RANK_GAP
return False
@@ -1160,7 +1218,20 @@ async def get_answer(
# that RRF fusion compresses out of the numeric score. Computed once and
# OR'd into every place the ratio/gap gate decides dominance, so it can
# only LIFT a retrieval — never demote one the ratio already trusts.
- agreement_dominant = _agreement_dominant(hits) if _agreement_confidence_enabled() else False
+ # Read the vector leg's own recorded status rather than inferring it from
+ # `hits`, which is capped to 5 by here: "no _vec_rank in the top 5" is also
+ # what a timed-out, errored, scope-filtered or simply outranked vector leg
+ # looks like, and those must NOT fall back to the symbol leg.
+ agreement_dominant = (
+ _agreement_dominant(
+ hits,
+ vector_leg_keyless=(
+ _symbol_agreement_enabled() and _retrieval_legs().get("vector") == "keyless"
+ ),
+ )
+ if _agreement_confidence_enabled()
+ else False
+ )
dominant = True
if len(hits) >= 2:
top_score = hits[0].get("score", 0.0)
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/config.py b/packages/server/src/repowise/server/mcp_server/tool_answer/config.py
index 8773465e2..f330d087c 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_answer/config.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/config.py
@@ -163,6 +163,18 @@ _COVERAGE_THRESHOLD = 0.66
# The top hit must rank no lower than this in EACH retriever (0-indexed, so
# 1 == "#1 or #2 in both").
_AGREEMENT_TOP_RANK_MAX = 1
+# The same ceiling for the FTS+symbol pair a keyless index has to use instead:
+# no vector leg runs there, so it can never produce a rank, and a fixed
+# FTS+vector pair would make agreement permanently unreachable for those users.
+# Stricter than the vector pair on purpose — an exact rank-0 tie in both, not
+# "#1 or #2". FTS and the symbol leg read overlapping text (the indexed wiki
+# page carries the public symbol table the symbol leg matches on), so the two
+# agreeing is nearer to one lexical match counted twice than to two independent
+# retrievers concurring. `_SYMBOL_LEG_RRF_K` (180 vs `_RRF_K` 60) already prices
+# that leg at a third of the others' in ranking, after a same-named React
+# component displaced the right file on the 99-question eval; this stops the
+# confidence gate from handing the same signal a full vote.
+_SYMBOL_AGREEMENT_TOP_RANK_MAX = 0
# The runner-up must trail the top by at least this many ranks in at least one
# source (when it was found by both). 1 == "top is strictly ahead somewhere".
_AGREEMENT_RANK_GAP = 1
diff --git a/tests/unit/cli/test_embedder_resolution.py b/tests/unit/cli/test_embedder_resolution.py
index 3f6f92c2f..30611696f 100644
--- a/tests/unit/cli/test_embedder_resolution.py
+++ b/tests/unit/cli/test_embedder_resolution.py
@@ -594,3 +594,57 @@ def test_duplicate_reembed_actions_run_once() -> None:
)
asyncio.run(apply_auto(verdict, _Ctx()))
assert calls == 1
+
+
+# --- build_embedder must say when it degrades -----------------------------
+#
+# The fallback to keyless embeddings used to be a bare ``except`` that returned
+# a KeylessEmbedder with nothing printed anywhere. ``--embedder ollama`` against
+# a stopped Ollama therefore produced a repo that indexed and searched without
+# complaint and had no semantic retrieval at all, while ``doctor`` reported
+# healthy. The server path already recorded this as ``degraded: True``; the CLI
+# path recorded nothing at all.
+
+
+def _capture_console(monkeypatch) -> list[str]:
+ """Collect what build_embedder prints, without a real terminal."""
+ from repowise.cli import helpers
+
+ printed: list[str] = []
+ monkeypatch.setattr(
+ helpers.console, "print", lambda *a, **k: printed.append(" ".join(str(x) for x in a))
+ )
+ return printed
+
+
+def test_build_embedder_reports_a_failed_real_backend(monkeypatch):
+ from repowise.core.providers.embedding.base import KeylessEmbedder
+
+ def _boom(name: str, **kwargs: object):
+ raise RuntimeError("connection refused")
+
+ monkeypatch.setattr("repowise.core.providers.embedding.registry.get_embedder", _boom)
+ printed = _capture_console(monkeypatch)
+
+ embedder = providers.build_embedder("ollama")
+
+ # Still falls back rather than crashing the run: full-text search is
+ # unaffected and a hard failure here would break indexing outright.
+ assert isinstance(embedder, KeylessEmbedder)
+ said = " ".join(printed)
+ # The three things a user needs: which backend, why, and what it costs them.
+ assert "ollama" in said
+ assert "connection refused" in said
+ assert "semantic search" in said.lower()
+
+
+def test_build_embedder_is_silent_for_the_keyless_default(monkeypatch):
+ """``mock`` is what a no-key run resolves to. Reaching it is not a failure
+ and must not print a warning, or every keyless run cries wolf."""
+ from repowise.core.providers.embedding.base import KeylessEmbedder
+
+ printed = _capture_console(monkeypatch)
+ embedder = providers.build_embedder("mock")
+
+ assert isinstance(embedder, KeylessEmbedder)
+ assert printed == []
diff --git a/tests/unit/server/mcp/test_answer_agreement.py b/tests/unit/server/mcp/test_answer_agreement.py
index 2223f18fc..7318c21a3 100644
--- a/tests/unit/server/mcp/test_answer_agreement.py
+++ b/tests/unit/server/mcp/test_answer_agreement.py
@@ -102,6 +102,14 @@ def _patch_agreement_pipeline(monkeypatch, answer_mod, *, top_both: bool):
"""
async def _fake_retrieve(question, ctx):
+ # The real hybrid_retrieve opens a fresh per-request leg record as its
+ # first act, and the confidence grade now reads that record. Mimic it,
+ # or this fake inherits whatever leg statuses an earlier test left in
+ # the ambient context and the grade gets computed from another test's
+ # run. (A test that calls begin_leg_record outside a task leaks it.)
+ import repowise.server.mcp_server._answer_pipeline as _pipeline
+
+ _pipeline.begin_leg_record()
top = {"page_id": "file_page:pkg/alpha/one.py", "score": 6.0, "_fts_rank": 0}
if top_both:
top["_vec_rank"] = 0
@@ -219,3 +227,197 @@ async def test_flag_off_restores_pure_ratio(setup_mcp, monkeypatch):
result = await get_answer("how does upload streaming chunk data")
assert result["confidence"] == "medium"
+
+
+# ---------------------------------------------------------------------------
+# Keyless indexes: the second opinion is the symbol leg
+# ---------------------------------------------------------------------------
+#
+# #1378 made a keyless index skip the vector leg outright, so `_vec_rank` is
+# never written on any hit for any question. A fixed FTS+vector pair therefore
+# made agreement permanently unreachable for every keyless user, and since
+# agreement can only LIFT, every keyless answer fell back to the pure RRF ratio
+# gate - the exact gate this signal exists because it mis-reads. The symbol leg
+# runs on every index and already records `_sym_rank`, which nothing else reads.
+
+
+class TestKeylessAgreementFallsBackToSymbol:
+ def test_fts_and_symbol_consensus_lifts_when_the_leg_is_keyless(self) -> None:
+ hits = [
+ {"_fts_rank": 0, "_sym_rank": 0},
+ {"_fts_rank": 1, "_sym_rank": 1},
+ ]
+ assert _agreement_dominant(hits, vector_leg_keyless=True) is True
+
+ def test_symbol_disagreement_still_does_not_lift(self) -> None:
+ """The fallback must be conservative, not a free pass.
+
+ Note this case is rejected by the rank-0 ceiling, not by the runner-up
+ disagreement logic — at that ceiling the gap check cannot reject
+ anything, since two hits cannot share rank 0 in one leg. Kept because
+ the property (sources disagreeing must not lift) is what matters, not
+ which clause enforces it.
+ """
+ hits = [
+ {"_fts_rank": 1, "_sym_rank": 0},
+ {"_fts_rank": 0, "_sym_rank": 2},
+ ]
+ assert _agreement_dominant(hits, vector_leg_keyless=True) is False
+
+ def test_fts_only_does_not_lift_on_a_keyless_index(self) -> None:
+ """One retriever is one retriever, whichever legs are available."""
+ hits = [{"_fts_rank": 0}, {"_fts_rank": 1}]
+ assert _agreement_dominant(hits, vector_leg_keyless=True) is False
+
+ def test_symbol_pair_needs_an_exact_rank_zero_tie(self) -> None:
+ """FTS and symbol read overlapping text — the indexed page carries the
+ symbol table the symbol leg matches on — so "#1 or #2" is too loose for
+ that pair even though it is the right ceiling for FTS+vector."""
+ hits = [
+ {"_fts_rank": 1, "_sym_rank": 1},
+ {"_fts_rank": 3, "_sym_rank": 3},
+ ]
+ assert _agreement_dominant(hits, vector_leg_keyless=True) is False
+ # The identical rank shape DOES lift under the vector pair's ceiling,
+ # which is what makes this a deliberate asymmetry rather than a typo.
+ assert (
+ _agreement_dominant(
+ [{"_fts_rank": 1, "_vec_rank": 1}, {"_fts_rank": 3, "_vec_rank": 3}]
+ )
+ is True
+ )
+
+ def test_a_keyed_index_never_substitutes_the_symbol_leg(self) -> None:
+ """The regression the explicit parameter exists to prevent.
+
+ `hits` is capped to the top 5 before the grade is computed, so "no
+ _vec_rank in this list" is ALSO what a keyed index looks like when its
+ vector leg timed out, errored, was scope-filtered, or was outranked by
+ five FTS-and-symbol hits. Those are precisely the states where evidence
+ is weakest, so inferring the substitution from the hits would
+ manufacture high confidence out of a retrieval failure.
+ """
+ hits = [
+ {"_fts_rank": 0, "_sym_rank": 0},
+ {"_fts_rank": 1, "_sym_rank": 1},
+ ]
+ assert _agreement_dominant(hits, vector_leg_keyless=False) is False
+ # The default is the safe one, for any caller that forgets to pass it.
+ assert _agreement_dominant(hits) is False
+
+ def test_a_live_vector_leg_that_disagrees_still_suppresses(self) -> None:
+ hits = [
+ {"_fts_rank": 0, "_sym_rank": 0, "_vec_rank": 7},
+ {"_fts_rank": 1, "_sym_rank": 1, "_vec_rank": 0},
+ ]
+ assert _agreement_dominant(hits) is False
+
+
+# ---------------------------------------------------------------------------
+# The call-site seam
+# ---------------------------------------------------------------------------
+#
+# `vector_leg_keyless` defaults to the safe value, which means dropping the
+# kwarg at the call site silently restores the keyless hole and every unit test
+# above still passes. This is the one seam the explicit-parameter design
+# created, so it gets an end-to-end test that goes through `retrieval_legs()`.
+
+
+def _patch_keyless_agreement_pipeline(monkeypatch, answer_mod, pipeline_mod, *, leg: str):
+ """FTS+symbol consensus, no vector ranks, and a declared vector leg status."""
+
+ async def _fake_retrieve(question, ctx):
+ # begin_leg_record() installs the per-request contextvar and hands back
+ # the dict the real legs mutate by reference; do the same here.
+ record = pipeline_mod.begin_leg_record()
+ record["vector"] = leg
+ record["fts"] = "ok"
+ return [
+ {
+ "page_id": "file_page:pkg/alpha/one.py",
+ "score": 6.0,
+ "_fts_rank": 0,
+ "_sym_rank": 0,
+ },
+ {
+ "page_id": "file_page:pkg/alpha/two.py",
+ "score": 5.9,
+ "_fts_rank": 1,
+ "_sym_rank": 1,
+ },
+ ]
+
+ async def _fake_hydrate(hits, ctx, *, scope=None):
+ for i, h in enumerate(hits):
+ h["target_path"] = h["page_id"].removeprefix("file_page:")
+ h["title"] = h["target_path"]
+ h["summary"] = "Upload module summary."
+ h["snippet"] = ""
+ h["page_type"] = "file_page"
+ if i == 0:
+ h["symbols"] = [dict(_MATCHED_SYMBOL)]
+ return hits
+
+ monkeypatch.setattr(answer_mod, "_hybrid_retrieve", _fake_retrieve)
+ monkeypatch.setattr(answer_mod, "_hydrate_hits", _fake_hydrate)
+
+
+async def test_keyless_leg_status_reaches_the_grade(setup_mcp, monkeypatch):
+ """A recorded `keyless` vector leg lets FTS+symbol consensus lift to high."""
+ import repowise.server.mcp_server._answer_pipeline as pipeline_mod
+ import repowise.server.mcp_server.tool_answer.answer as answer_mod
+ from repowise.server.mcp_server import get_answer
+
+ monkeypatch.setenv("REPOWISE_ANSWER_AGREEMENT_CONFIDENCE", "on")
+ monkeypatch.setenv("REPOWISE_ANSWER_SYMBOL_AGREEMENT", "on")
+ monkeypatch.setenv("REPOWISE_ANSWER_EARN_HIGH_GROUNDING", "off")
+ _patch_keyless_agreement_pipeline(monkeypatch, answer_mod, pipeline_mod, leg="keyless")
+ _patch_provider(monkeypatch, answer_mod, _GOOD_ANSWER)
+
+ result = await get_answer("how does upload chunking work")
+ assert result["confidence"] == "high"
+
+
+async def test_a_failed_vector_leg_is_not_treated_as_keyless(setup_mcp, monkeypatch):
+ """The regression that matters: identical hits, but the vector leg TIMED OUT.
+
+ A keyed index whose vector leg timed out presents exactly like a keyless one
+ once `hits` is capped to five, and it must NOT earn the lift — that is the
+ state where the evidence is weakest, not strongest.
+ """
+ import repowise.server.mcp_server._answer_pipeline as pipeline_mod
+ import repowise.server.mcp_server.tool_answer.answer as answer_mod
+ from repowise.server.mcp_server import get_answer
+
+ monkeypatch.setenv("REPOWISE_ANSWER_AGREEMENT_CONFIDENCE", "on")
+ monkeypatch.setenv("REPOWISE_ANSWER_SYMBOL_AGREEMENT", "on")
+ monkeypatch.setenv("REPOWISE_ANSWER_EARN_HIGH_GROUNDING", "off")
+ _patch_keyless_agreement_pipeline(monkeypatch, answer_mod, pipeline_mod, leg="timeout")
+ _patch_provider(monkeypatch, answer_mod, _GOOD_ANSWER)
+
+ result = await get_answer("how does upload chunking work")
+ assert result["confidence"] == "medium"
+
+
+async def test_symbol_agreement_flag_is_independently_reversible(setup_mcp, monkeypatch):
+ """Turning the keyless pair off must not turn the measured vector pair off."""
+ import repowise.server.mcp_server._answer_pipeline as pipeline_mod
+ import repowise.server.mcp_server.tool_answer.answer as answer_mod
+ from repowise.server.mcp_server import get_answer
+
+ monkeypatch.setenv("REPOWISE_ANSWER_AGREEMENT_CONFIDENCE", "on")
+ monkeypatch.setenv("REPOWISE_ANSWER_SYMBOL_AGREEMENT", "off")
+ monkeypatch.setenv("REPOWISE_ANSWER_EARN_HIGH_GROUNDING", "off")
+ _patch_keyless_agreement_pipeline(monkeypatch, answer_mod, pipeline_mod, leg="keyless")
+ _patch_provider(monkeypatch, answer_mod, _GOOD_ANSWER)
+
+ result = await get_answer("how does upload chunking work")
+ assert result["confidence"] == "medium"
+
+ # ...and the vector pair still lifts with the symbol flag off.
+ _patch_agreement_pipeline(monkeypatch, answer_mod, top_both=True)
+ _patch_provider(monkeypatch, answer_mod, _GOOD_ANSWER)
+ assert (await get_answer("how does upload chunking work again"))["confidence"] == "high"
diff --git a/tests/unit/server/mcp/test_keyless_vector_leg.py b/tests/unit/server/mcp/test_keyless_vector_leg.py
index ea1711350..2096ff61f 100644
--- a/tests/unit/server/mcp/test_keyless_vector_leg.py
+++ b/tests/unit/server/mcp/test_keyless_vector_leg.py
@@ -236,3 +236,174 @@ def test_dedup_by_vector_ignores_the_pending_index_too():
semantic_match.find_duplicate_decision_by_vector(store, vector, pending=_Pending())
)
assert got is None
+
+
+# --------------------------------------------------------------------------
+# `repowise search --mode semantic` (CLI)
+# --------------------------------------------------------------------------
+#
+# The two CLI search paths were never wired to the predicate at all. Everything
+# else that reads vectors was swept and guarded; these build their own store
+# inline, so the sweep did not reach them and a keyless user got a window of
+# nearest-neighbour noise rendered as semantic results, with the full-text
+# fallback sitting unused directly below.
+
+
+class _Row:
+ """The shape the CLI result renderer reads off a hit."""
+
+ def __init__(self, tag: str):
+ self.tag = tag
+ self.score = 0.9
+ self.page_id = tag
+ self.title = tag
+ self.snippet = tag
+ self.page_type = "file_page"
+ self.target_path = tag
+
+
+class _FakeLanceStore:
+ """Stands in for LanceDBVectorStore: records whether it was ranked on."""
+
+ def __init__(self, _path, embedder):
+ self._embedder = embedder
+ self.searched = False
+ self.closed = False
+
+ async def search(self, query, limit=10):
+ self.searched = True
+ return [_Row("vector-noise")]
+
+ async def close(self):
+ self.closed = True
+
+
+def _patch_cli_search(monkeypatch, tmp_path, embedder):
+ """Point both CLI search paths at a fake store and a known FTS result."""
+ from repowise.cli.commands import search_cmd
+
+ built: dict[str, _FakeLanceStore] = {}
+
+ def _make(path, embedder=None):
+ store = _FakeLanceStore(path, embedder)
+ built["store"] = store
+ return store
+
+ (tmp_path / ".repowise" / "lancedb").mkdir(parents=True, exist_ok=True)
+ monkeypatch.setattr(
+ "repowise.core.persistence.vector_store.LanceDBVectorStore", _make, raising=False
+ )
+ monkeypatch.setattr(
+ "repowise.cli.providers.embedders.build_embedder", lambda _n: embedder
+ )
+ # "mock" is what a genuinely keyless repo resolves to; individual tests
+ # override this to cover the "pinned a real embedder that then failed" case.
+ monkeypatch.setattr(
+ "repowise.cli.providers.embedders.resolve_embedder_for_repo", lambda _p: "mock"
+ )
+
+ class _FTS:
+ def __init__(self, _engine):
+ pass
+
+ async def search(self, query, limit=10):
+ return [_Row("fts-result")]
+
+ class _Engine:
+ async def dispose(self):
+ return None
+
+ monkeypatch.setattr("repowise.core.persistence.FullTextSearch", _FTS, raising=False)
+ monkeypatch.setattr("repowise.core.persistence.create_engine", lambda _u: _Engine(), raising=False)
+ monkeypatch.setattr(search_cmd, "get_db_url_for_repo", lambda _p: "sqlite+aiosqlite:///:memory:")
+ # _search_semantic renders rather than returns, so capture what it would
+ # show AND the title, which is itself part of the contract: full-text rows
+ # must not be labelled "Semantic search".
+ shown: dict = {"rows": [], "title": None, "notices": []}
+ monkeypatch.setattr(
+ search_cmd,
+ "_display_results",
+ lambda results, title: (
+ shown["rows"].extend(r.tag for r in results),
+ shown.__setitem__("title", title),
+ ),
+ )
+ monkeypatch.setattr(
+ search_cmd.console,
+ "print",
+ lambda *a, **k: shown["notices"].append(" ".join(str(x) for x in a)),
+ )
+ return search_cmd, built, shown
+
+
+def test_cli_semantic_search_does_not_rank_on_a_keyless_store(monkeypatch, tmp_path):
+ search_cmd, built, shown = _patch_cli_search(monkeypatch, tmp_path, KeylessEmbedder())
+
+ search_cmd._search_semantic(tmp_path, "how does auth work", 5)
+
+ assert shown["rows"] == ["fts-result"], "keyless must serve full text, not vector noise"
+ assert built["store"].searched is False
+ assert built["store"].closed is True, "the store is still closed on the guarded path"
+
+
+def test_cli_semantic_search_says_full_text_answered(monkeypatch, tmp_path):
+ """Rendering full-text rows under a "Semantic search" heading is how someone
+ concludes semantic retrieval is bad when what they have is no embedder."""
+ search_cmd, _built, shown = _patch_cli_search(monkeypatch, tmp_path, KeylessEmbedder())
+
+ search_cmd._search_semantic(tmp_path, "how does auth work", 5)
+
+ assert "Full-text" in shown["title"]
+ assert "Semantic" not in shown["title"]
+ said = " ".join(shown["notices"]).lower()
+ assert "no embedder configured" in said
+ assert "full-text" in said
+
+
+def test_cli_semantic_search_notice_names_a_configured_embedder(monkeypatch, tmp_path):
+ """A repo pinned to a real embedder whose key has gone away lands here too.
+ Telling that user "no embedder configured" contradicts their own config."""
+ search_cmd, _built, shown = _patch_cli_search(monkeypatch, tmp_path, KeylessEmbedder())
+ monkeypatch.setattr(
+ "repowise.cli.providers.embedders.resolve_embedder_for_repo", lambda _p: "ollama"
+ )
+
+ search_cmd._search_semantic(tmp_path, "how does auth work", 5)
+
+ said = " ".join(shown["notices"])
+ assert "ollama" in said
+ assert "No embedder configured" not in said
+
+
+def test_cli_semantic_search_still_ranks_on_a_real_store(monkeypatch, tmp_path):
+ """The guard must not over-fire: a real embedder still gets semantic search."""
+ search_cmd, built, shown = _patch_cli_search(monkeypatch, tmp_path, _RealisticEmbedder())
+
+ search_cmd._search_semantic(tmp_path, "how does auth work", 5)
+
+ assert shown["rows"] == ["vector-noise"]
+ assert shown["title"] is not None and "Semantic" in shown["title"]
+ assert built["store"].searched is True
+ assert shown["notices"] == [], "a working embedder must not be warned about"
+
+
+def test_cli_workspace_semantic_collect_skips_a_keyless_repo(monkeypatch, tmp_path):
+ search_cmd, built, _shown = _patch_cli_search(monkeypatch, tmp_path, KeylessEmbedder())
+
+ results, served_fulltext = search_cmd._collect_semantic(tmp_path, "how does auth work", 5)
+
+ assert [r.tag for r in results] == ["fts-result"]
+ assert built["store"].searched is False
+ # The flag is what stops the fan-out sorting these full-text scores against
+ # another repo's cosine scores, which are not the same quantity.
+ assert served_fulltext is True
+
+
+def test_cli_workspace_semantic_collect_reports_a_real_store_as_semantic(monkeypatch, tmp_path):
+ search_cmd, built, _shown = _patch_cli_search(monkeypatch, tmp_path, _RealisticEmbedder())
+
+ results, served_fulltext = search_cmd._collect_semantic(tmp_path, "how does auth work", 5)
+
+ assert [r.tag for r in results] == ["vector-noise"]
+ assert built["store"].searched is True
+ assert served_fulltext is False