Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(answer): always synthesize; grade confidence post-synthesis (#919)

Swati Ahujaa76f43fbe019
diff --git a/packages/server/src/repowise/server/mcp_server/_answer_context.py b/packages/server/src/repowise/server/mcp_server/_answer_context.py
index 32a530d14..e188ec05a 100644
--- a/packages/server/src/repowise/server/mcp_server/_answer_context.py
+++ b/packages/server/src/repowise/server/mcp_server/_answer_context.py
@@ -164,9 +164,7 @@ async def build_structured_prelude(
sections.append(f"Recent significant commits: {commits_line}")
if decisions:
- titles = "; ".join(
- f"{d['title']} ({d['status']})" for d in decisions
- )
+ titles = "; ".join(f"{d['title']} ({d['status']})" for d in decisions)
sections.append(f"Decision records touching these files: {titles}")
if not sections:
@@ -188,7 +186,7 @@ def _top_symbols_summary(hits: list[dict]) -> str:
name = s.get("name") or s.get("signature") or ""
kind = s.get("kind") or "?"
if name:
- matched.append(f"{name} ({kind}) in {h.get('target_path','')}")
+ matched.append(f"{name} ({kind}) in {h.get('target_path', '')}")
if len(matched) >= 4:
break
if len(matched) >= 4:
@@ -208,9 +206,7 @@ def _top_symbols_summary(hits: list[dict]) -> str:
return ""
-async def _recent_commits_summary(
- hits: list[dict], ctx: Any, repo_id: str
-) -> str:
+async def _recent_commits_summary(hits: list[dict], ctx: Any, repo_id: str) -> str:
"""Short summary of significant commits across the top hits.
GitMetadata.significant_commits_json is already filtered by the indexer
@@ -296,7 +292,11 @@ def _format_hits_block(hits: list[dict], max_chars_per_hit: int) -> str:
"""Format the per-hit excerpts. Mirrors the prior tool_answer behaviour."""
parts: list[str] = []
for i, h in enumerate(hits, start=1):
- body_src = h.get("summary") or h.get("snippet") or ""
+ # Prefer a real page excerpt when one was attached (the ambiguous-
+ # retrieval / always-synthesize path enriches non-dominant hits with
+ # actual page content so the LLM reads across the candidates, not just
+ # one-line summaries). Falls back to the summary/snippet otherwise.
+ body_src = h.get("excerpt") or h.get("summary") or h.get("snippet") or ""
body = body_src[:max_chars_per_hit]
# Tag expanded hits so the LLM knows they didn't surface in retrieval
# directly — useful context when deciding how much weight to put on
@@ -305,8 +305,8 @@ def _format_hits_block(hits: list[dict], max_chars_per_hit: int) -> str:
sources = h.get("_sources") or set()
src_tag = f" [{'+'.join(sorted(sources))}]" if sources else ""
block = [
- f"[{i}] {h.get('target_path','')} (score={h.get('score', 0.0):.3f}){tag}{src_tag}",
- f" title: {h.get('title','')}",
+ f"[{i}] {h.get('target_path', '')} (score={h.get('score', 0.0):.3f}){tag}{src_tag}",
+ f" title: {h.get('title', '')}",
f" summary: {body}",
]
symbols = h.get("symbols") or []
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py
index 34861a7f3..ad26f42ec 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py
@@ -44,6 +44,7 @@ import asyncio
import contextlib
import json as _json
import logging
+import os
import time
from pathlib import Path
@@ -137,6 +138,43 @@ from repowise.server.mcp_server.tool_answer.synthesis import (
_log = logging.getLogger("repowise.mcp.answer")
+# Always-synthesize flag. Default ON: synthesis runs for every retrieval and the
+# post-synthesis grading cascade demotes confidence instead of the tool
+# abstaining, so coverage matches a research assistant that answers every
+# question (the pre-synthesis dominance gate abstained on ~58%). Set to a falsey
+# value (0/off/false/no) to restore the legacy abstain-on-ambiguous behaviour.
+_ALWAYS_SYNTHESIZE_ENV = "REPOWISE_ANSWER_ALWAYS_SYNTHESIZE"
+
+
+def _always_synthesize() -> bool:
+ """Whether to always synthesize (default) or keep the legacy abstain gate."""
+ return os.environ.get(_ALWAYS_SYNTHESIZE_ENV, "").strip().lower() not in {
+ "0",
+ "false",
+ "no",
+ "off",
+ }
+
+
+def _build_best_guesses(hits: list[dict]) -> list[dict]:
+ """Decision-shaped candidate list: per-file justification, score, excerpt.
+
+ The evidence an ambiguous-retrieval reply carries so the agent can pick ONE
+ file to verify instead of skimming five. Shared by the legacy abstain path
+ and the always-synthesize low/medium fold-in.
+ """
+ return [
+ {
+ "file": h.get("target_path"),
+ "why_relevant": _candidate_justification(h),
+ "score": round(h.get("score", 0.0), 3),
+ "domain_penalty": h.get("_domain_penalty"),
+ **({"excerpt": h["excerpt"]} if h.get("excerpt") else {}),
+ }
+ for h in hits[:_GATED_RETURN_HITS]
+ if h.get("target_path")
+ ]
+
def _json_default(obj):
"""Serialize the non-JSON types retrieval hits carry (``_sources`` sets).
@@ -768,112 +806,95 @@ async def get_answer(
),
}
- # --- Confidence gate ---------------------------------------------------
- # Skip synthesis when retrieval is NOT clearly dominant. The dominance
- # ratio (top score / second score) is the sole gating criterion: above
- # the threshold the top hit is reliably the right answer; below it the
- # top-1 / top-2 ambiguity is large enough that we hand the agent ranked
- # excerpts and let it ground in source.
+ # --- Retrieval dominance -----------------------------------------------
+ # ``dominant`` = retrieval clearly pointed at ONE page (the top hit
+ # outscores the rest). It no longer decides WHETHER to synthesize — under
+ # the always-synthesize default, synthesis runs for every retrieval so
+ # coverage matches a research assistant that answers every question (the
+ # pre-synthesis gate abstained on ~58%). It now feeds the confidence grade
+ # as a CEILING (a non-dominant retrieval is "answered, but verify", never
+ # "high") and gates the ambiguous-retrieval evidence folded into the reply.
#
- # Coverage (fraction of query terms present in the top hit) is also
- # available via the re-ranker and is used to bias score-based ranking,
- # but is intentionally NOT used as a hard gate here. Natural-language
- # questions rarely have all their content terms co-occurring in a single
- # page (typical coverage is 0.15–0.25), so a coverage threshold over-
- # fires on confidently-dominant retrievals and degrades the cheap path.
+ # Two-tier test: at high retrieval quality (both scores excellent) close
+ # ratios are expected, so use an absolute gap; at lower quality the ratio
+ # gate flags genuinely ambiguous retrievals. Coverage (fraction of query
+ # terms in the top hit) biases ranking but is intentionally NOT a gate:
+ # natural-language questions rarely have all content terms in one page
+ # (typical 0.15-0.25), so a coverage threshold over-fires. Default dominant
+ # for a lone hit (nothing to be ambiguous against).
+ always_synthesize = _always_synthesize()
+ dominant = True
if len(hits) >= 2:
top_score = hits[0].get("score", 0.0)
second_score = hits[1].get("score", 0.0) or 1e-9
-
- # Two-tier gating: at high retrieval quality (both scores
- # excellent), close ratios are expected and normal — use an
- # absolute gap instead. At lower quality, the ratio-based
- # gate prevents synthesis on genuinely ambiguous retrievals.
if top_score >= 3.0:
dominant = (top_score - second_score) >= 0.5
else:
dominant = (top_score / second_score) >= _DOMINANCE_RATIO
- if not dominant:
- # Attach real page content to the top candidates before shaping
- # the reply. Agent-transcript evidence (context-tool bench,
- # 2026-07-17): a pointers-only gated payload sends the agent into
- # an 8-15 call Grep/Read spree that costs more than a bare agent —
- # it paid for the tool call and still had to acquire all content
- # natively. Excerpts turn the miss path into "pick one candidate,
- # verify with at most one Read". (The 2026-07-11 dogfood dropped
- # the per-hit key_symbols METADATA dump as unused; readable
- # content is the part that was worth keeping.)
- with contextlib.suppress(Exception):
- await _enrich_gated_excerpts(hits, ctx)
- # Structured candidate set: a decision-shaped list with a
- # one-line justification per file. Beats the prior flat
- # ``fallback_targets`` list because the agent can pick ONE file
- # to Read first instead of skimming five.
- best_guesses = [
- {
- "file": h.get("target_path"),
- "why_relevant": _candidate_justification(h),
- "score": round(h.get("score", 0.0), 3),
- "domain_penalty": h.get("_domain_penalty"),
- **({"excerpt": h["excerpt"]} if h.get("excerpt") else {}),
- }
- for h in hits[:_GATED_RETURN_HITS]
- if h.get("target_path")
- ]
- # Mine source comments for rationale the wiki/decision corpus
- # missed — turns "go Read these 5 files" into a cited why.
- code_rationale = _gather_code_rationale(ctx, hits, fallback_targets, question)
- has_excerpts = any("excerpt" in g for g in best_guesses)
- gated: dict = {
- "answer": "",
- "citations": [],
- "confidence": "low",
- "retrieval_quality": "weak",
- "best_guesses": best_guesses,
- "next_action_hint": (
- (
- f"Start from the excerpt of {best_guesses[0]['file']} — "
- "it scored highest; Read the file only to verify "
- "details the excerpt does not settle."
- if has_excerpts
- else f"Read {best_guesses[0]['file']} first — it scored "
- "highest but retrieval was ambiguous, so verify "
- "before answering."
- )
- if best_guesses
- else (
- 'Retry search_codebase with mode="symbol" or '
- 'mode="path" on the key terms; Grep only if those '
- "miss too."
- )
- ),
- "fallback_targets": fallback_targets,
- "retrieval": [],
- "note": (
- "Multiple plausible candidates — synthesis skipped to "
- "avoid anchoring on a wrong frame. Each best_guess entry "
- "names why that file is in the running"
- + (
- ", and its excerpt carries that page's actual content."
- if has_excerpts
- else "."
- )
- ),
- }
- if code_rationale:
- gated["code_rationale"] = code_rationale
- gated["note"] += (
- " code_rationale carries rationale comments mined from the "
- "candidate source — they may already answer the question."
+ if not always_synthesize and not dominant:
+ # Legacy abstain path (REPOWISE_ANSWER_ALWAYS_SYNTHESIZE=off): retrieval
+ # is ambiguous, so skip synthesis and hand back ranked excerpts +
+ # best_guesses for the agent to ground in.
+ #
+ # Attach real page content to the top candidates before shaping the
+ # reply. Agent-transcript evidence (context-tool bench, 2026-07-17): a
+ # pointers-only gated payload sends the agent into an 8-15 call Grep/Read
+ # spree that costs more than a bare agent — it paid for the tool call and
+ # still had to acquire all content natively. Excerpts turn the miss path
+ # into "pick one candidate, verify with at most one Read".
+ with contextlib.suppress(Exception):
+ await _enrich_gated_excerpts(hits, ctx)
+ best_guesses = _build_best_guesses(hits)
+ # Mine source comments for rationale the wiki/decision corpus missed —
+ # turns "go Read these 5 files" into a cited why.
+ code_rationale = _gather_code_rationale(ctx, hits, fallback_targets, question)
+ has_excerpts = any("excerpt" in g for g in best_guesses)
+ gated: dict = {
+ "answer": "",
+ "citations": [],
+ "confidence": "low",
+ "retrieval_quality": "weak",
+ "best_guesses": best_guesses,
+ "next_action_hint": (
+ (
+ f"Start from the excerpt of {best_guesses[0]['file']} — "
+ "it scored highest; Read the file only to verify "
+ "details the excerpt does not settle."
+ if has_excerpts
+ else f"Read {best_guesses[0]['file']} first — it scored "
+ "highest but retrieval was ambiguous, so verify "
+ "before answering."
)
- gated["_meta"] = _build_meta(
- timing_ms=(time.perf_counter() - t0) * 1000,
- hint=_answer_hint("low", len(hits)),
- repository=repository,
- targets=fallback_targets,
+ if best_guesses
+ else (
+ 'Retry search_codebase with mode="symbol" or '
+ 'mode="path" on the key terms; Grep only if those '
+ "miss too."
+ )
+ ),
+ "fallback_targets": fallback_targets,
+ "retrieval": [],
+ "note": (
+ "Multiple plausible candidates — synthesis skipped to "
+ "avoid anchoring on a wrong frame. Each best_guess entry "
+ "names why that file is in the running"
+ + (", and its excerpt carries that page's actual content." if has_excerpts else ".")
+ ),
+ }
+ if code_rationale:
+ gated["code_rationale"] = code_rationale
+ gated["note"] += (
+ " code_rationale carries rationale comments mined from the "
+ "candidate source — they may already answer the question."
)
- return gated
+ gated["_meta"] = _build_meta(
+ timing_ms=(time.perf_counter() - t0) * 1000,
+ hint=_answer_hint("low", len(hits)),
+ repository=repository,
+ targets=fallback_targets,
+ )
+ return gated
# Confidence is the only axis we gate on. We deliberately do NOT add a
# second gate keyed on question shape (e.g. relational questions
@@ -954,6 +975,15 @@ async def get_answer(
payload["_meta"]["degraded"] = "no-llm-provider"
return payload
+ # Ambiguous retrieval (always-synthesize): pull real page content across the
+ # non-dominant top pages so the LLM synthesizes over the actual candidate
+ # content, not one-line summaries — the same excerpts the legacy abstain path
+ # served the agent. No-op on dominant retrievals. (The excerpts also back the
+ # best_guesses folded into a low/medium reply below.)
+ if not dominant:
+ with contextlib.suppress(Exception):
+ await _enrich_gated_excerpts(hits, ctx)
+
# Decision fusion (why-shaped questions only) + structured prelude. Both
# layers are gated on signal: no ADRs for the top hits → no decisions
# block, no symbols / commits / decisions → no prelude. Empty layers are
@@ -1186,6 +1216,15 @@ async def get_answer(
else:
frame_unsupported = []
+ # Non-dominant ceiling: ambiguous retrieval is the calibration cost of
+ # always synthesizing — the answer may be right, but with no single dominant
+ # page it must never read "high" (cite without verifying). Cap at medium even
+ # if all six gates passed. (A non-dominant retrieval already scores <high via
+ # the ratio, so this is usually a no-op; it is explicit so the
+ # always-synthesize contract — "answered, but verify" — is self-documenting.)
+ if not dominant and confidence == "high":
+ confidence = "medium"
+
# retrieval_quality is a separate signal from confidence. Where confidence
# says "how much should you trust the synthesised text", retrieval_quality
# says "how good was the retrieval that fed it". The agent uses confidence
@@ -1334,6 +1373,34 @@ async def get_answer(
if concept_rationale:
payload["code_rationale"] = concept_rationale
+ # Ambiguous-retrieval evidence (always-synthesize). The questions that used
+ # to abstain (no dominant page) now carry synthesized PROSE — but the
+ # retrieval was genuinely ambiguous, so ship the same evidence the old
+ # abstain path did: best_guesses (per-file justification + excerpts) and
+ # mined code_rationale, plus an honest caveat. This is the "answered, but
+ # verify against these candidates" reply that replaced the empty pointer
+ # list. Guarded so it never touches the dominant / high-confidence paths.
+ if not dominant:
+ payload.setdefault("best_guesses", _build_best_guesses(hits))
+ if "code_rationale" not in payload:
+ _cr = _gather_code_rationale(ctx, hits, fallback_targets, question)
+ _cr = _drop_already_surfaced(_cr, symbol_bodies, quotes)
+ if _cr:
+ payload["code_rationale"] = _cr
+ _caveat = (
+ "Retrieval was ambiguous (no single dominant page), so this was "
+ f"synthesized across several candidates and held at {confidence} "
+ "confidence — verify against best_guesses"
+ + (" or the code_rationale comments." if payload.get("code_rationale") else ".")
+ )
+ payload["note"] = (payload["note"] + " " + _caveat) if payload.get("note") else _caveat
+ if payload.get("best_guesses"):
+ payload.setdefault(
+ "next_action_hint",
+ f"Verify against {payload['best_guesses'][0]['file']} — it scored "
+ "highest, but retrieval was ambiguous across the top candidates.",
+ )
+
# Flow-path lead: when the question anchored 2+ endpoints, surface the
# dependency/call chain the answer traverses so the agent sees the path in
# the same call instead of reconstructing it hop by hop.
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/config.py b/packages/server/src/repowise/server/mcp_server/tool_answer/config.py
index cce20805b..7f17f43d2 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_answer/config.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/config.py
@@ -291,7 +291,11 @@ _HIGH_CONFIDENCE_SCORE_FLOOR = 1.5
# v7: concept anchoring - number-bearing why/value questions anchor the file
# whose comment justifies the number and surface that comment as code_rationale
# even on the high path. Cached v6 payloads predate the anchor + surfacing.
-_ANSWER_SCHEMA_VERSION = 8
+# v8: always-synthesize — retrievals that used to abstain (no dominant page) now
+# carry synthesized prose + best_guesses/code_rationale evidence at medium/low
+# instead of an empty pointer list. Cached v8 gated (empty-answer) rows must
+# bypass so the new answered-with-evidence contract reaches callers.
+_ANSWER_SCHEMA_VERSION = 9
# Hard TTL on answer-cache rows. Commit-based invalidation (the payload's
# stamped ``_indexed_commit`` vs the repo's current head) is the primary
diff --git a/tests/unit/server/mcp/test_answer_always_synthesize.py b/tests/unit/server/mcp/test_answer_always_synthesize.py
new file mode 100644
index 000000000..1cb181d85
--- /dev/null
+++ b/tests/unit/server/mcp/test_answer_always_synthesize.py
@@ -0,0 +1,140 @@
+"""Always-synthesize behaviour for get_answer.
+
+The pre-synthesis dominance gate used to abstain (empty answer + pointer list)
+whenever retrieval was not clearly dominant — ~58% of questions. Synthesis now
+runs for every retrieval and the post-synthesis grading cascade demotes
+confidence instead of abstaining:
+
+ * a non-dominant retrieval returns synthesized PROSE at medium (never high),
+ with best_guesses folded in as ambiguous-retrieval evidence;
+ * a hedged / ungrounded synthesis still demotes to low, but carries the prose
+ and evidence — never an empty answer;
+ * REPOWISE_ANSWER_ALWAYS_SYNTHESIZE=off restores the legacy abstain path.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+
+def _patch_pipeline(monkeypatch, answer_mod, scores=(2.0, 1.9), *, with_symbols=False):
+ """Two hits at the given scores; scores default to a non-dominant pair
+ (ratio 1.05 < 1.2, top < 3.0)."""
+
+ async def _fake_retrieve(question, ctx):
+ return [
+ {"page_id": "file_page:pkg/alpha/one.py", "score": scores[0], "_sources": {"fts"}},
+ {"page_id": "file_page:pkg/alpha/two.py", "score": scores[1], "_sources": {"fts"}},
+ ]
+
+ async def _fake_hydrate(hits, ctx, *, scope=None):
+ for i, h in enumerate(hits):
+ h["target_path"] = h["page_id"].removeprefix("file_page:")
+ h["title"] = h["target_path"]
+ h["summary"] = "Module summary for the page. " * 4
+ h["snippet"] = ""
+ h["page_type"] = "file_page"
+ if with_symbols and i == 0:
+ h["symbols"] = [
+ {
+ "name": "go",
+ "kind": "function",
+ "signature": "def go()",
+ "docstring": "",
+ "start_line": 3,
+ "end_line": 9,
+ "_matched": True,
+ }
+ ]
+ return hits
+
+ monkeypatch.setattr(answer_mod, "_hybrid_retrieve", _fake_retrieve)
+ monkeypatch.setattr(answer_mod, "_hydrate_hits", _fake_hydrate)
+
+
+def _patch_provider(monkeypatch, answer_mod, content):
+ class _Provider:
+ provider_name = "mock"
+ model_name = "mock-1"
+
+ async def generate(self, **kwargs):
+ return SimpleNamespace(content=content)
+
+ monkeypatch.setattr(answer_mod, "_resolve_provider_for_answer", lambda _p: _Provider())
+
+
+async def test_non_dominant_returns_prose_at_medium_with_evidence(setup_mcp, monkeypatch):
+ """(a) A non-dominant retrieval now returns a non-empty answer at
+ confidence=medium with best_guesses present (was: empty abstain)."""
+ import repowise.server.mcp_server.tool_answer.answer as answer_mod
+ from repowise.server.mcp_server import get_answer
+
+ _patch_pipeline(monkeypatch, answer_mod, scores=(2.0, 1.9))
+ _patch_provider(monkeypatch, answer_mod, "The go() function drives it (pkg/alpha/one.py).")
+
+ result = await get_answer("how does the alpha module go function work")
+ assert result["answer"], "synthesis must run — no abstention"
+ assert result["confidence"] == "medium", "non-dominant retrieval caps at medium"
+ assert result["best_guesses"], "ambiguous-retrieval evidence folded in"
+ assert "ambiguous" in result["note"]
+
+
+async def test_hedged_non_dominant_demotes_low_but_keeps_prose(setup_mcp, monkeypatch):
+ """(b) A hedged synthesis on non-dominant retrieval still demotes to low,
+ but now carries the prose + best_guesses instead of an empty answer."""
+ import repowise.server.mcp_server.tool_answer.answer as answer_mod
+ from repowise.server.mcp_server import get_answer
+
+ _patch_pipeline(monkeypatch, answer_mod, scores=(2.0, 1.9))
+ _patch_provider(
+ monkeypatch,
+ answer_mod,
+ "The provided excerpts do not contain the implementation details.",
+ )
+
+ result = await get_answer("how does the alpha module go function work")
+ assert result["confidence"] == "low", "a hedge demotes to low"
+ assert result["answer"], "the hedged prose is still served, not an empty answer"
+ assert result["best_guesses"], "ambiguous-retrieval evidence folded in"
+
+
+async def test_flag_off_restores_legacy_abstain(setup_mcp, monkeypatch):
+ """(c) REPOWISE_ANSWER_ALWAYS_SYNTHESIZE=off restores the pre-synthesis
+ abstain: empty answer + best_guesses, and synthesis never runs."""
+ import repowise.server.mcp_server.tool_answer.answer as answer_mod
+ from repowise.server.mcp_server import get_answer
+
+ monkeypatch.setenv("REPOWISE_ANSWER_ALWAYS_SYNTHESIZE", "off")
+ _patch_pipeline(monkeypatch, answer_mod, scores=(2.0, 1.9))
+
+ def _no_provider(_p):
+ raise AssertionError("flag-off abstain path must not resolve a provider")
+
+ monkeypatch.setattr(answer_mod, "_resolve_provider_for_answer", _no_provider)
+
+ result = await get_answer("how does the alpha module go function work")
+ assert result["answer"] == "", "flag off abstains — no synthesized prose"
+ assert result["confidence"] == "low"
+ assert result["best_guesses"], "abstain path still hands back candidates"
+
+
+async def test_dominant_retrieval_unaffected_no_best_guesses(setup_mcp, monkeypatch):
+ """Negative control: a dominant retrieval keeps the high-confidence contract
+ (no best_guesses fold-in, no ambiguity caveat)."""
+ import repowise.server.mcp_server.tool_answer.answer as answer_mod
+ from repowise.server.mcp_server import get_answer
+
+ # 5.0 vs 4.0: top >= 3.0, gap 1.0 >= 0.5 → dominant; ratio 1.25 → high.
+ _patch_pipeline(monkeypatch, answer_mod, scores=(5.0, 4.0), with_symbols=True)
+ _patch_provider(monkeypatch, answer_mod, "The go() function drives it (pkg/alpha/one.py).")
+
+ result = await get_answer("how does the alpha module go function work")
+ assert result["confidence"] == "high"
+ assert "best_guesses" not in result
+ assert "ambiguous" not in (result.get("note") or "")
diff --git a/tests/unit/server/mcp/test_answer_calibration.py b/tests/unit/server/mcp/test_answer_calibration.py
index bfba83283..3b4fad339 100644
--- a/tests/unit/server/mcp/test_answer_calibration.py
+++ b/tests/unit/server/mcp/test_answer_calibration.py
@@ -837,13 +837,18 @@ async def test_hedged_answer_surfaces_code_rationale(setup_mcp, monkeypatch, tmp
@pytest.mark.asyncio
-async def test_gated_answer_surfaces_code_rationale(setup_mcp, monkeypatch, tmp_path):
- """Ambiguous retrieval (gated, no synthesis) → still mine source rationale."""
+async def test_non_dominant_answer_surfaces_code_rationale(setup_mcp, monkeypatch, tmp_path):
+ """Ambiguous retrieval → synthesize AND fold in mined source rationale.
+
+ Under the always-synthesize default a non-dominant retrieval no longer
+ abstains: synthesis runs (capped at medium), and the ambiguous-retrieval
+ fold-in still mines the candidate source for the rationale comment.
+ """
import repowise.server.mcp_server as mcp_mod
import repowise.server.mcp_server.tool_answer.answer as answer_mod
from repowise.server.mcp_server import get_answer
- # Two near-tied hits (4.0 vs 3.8) → dominance gate fails → best_guesses path.
+ # Two near-tied hits (4.0 vs 3.8) → non-dominant → medium ceiling + evidence.
async def _fake_retrieve(question, ctx):
return [
{"page_id": "file_page:pkg/alpha/one.py", "score": 4.0},
@@ -861,6 +866,7 @@ async def test_gated_answer_surfaces_code_rationale(setup_mcp, monkeypatch, tmp_
monkeypatch.setattr(answer_mod, "_hybrid_retrieve", _fake_retrieve)
monkeypatch.setattr(answer_mod, "_hydrate_hits", _fake_hydrate)
+ _patch_provider(monkeypatch, answer_mod, "Uploads chunk at 8MB (pkg/alpha/one.py).")
(tmp_path / "pkg" / "alpha").mkdir(parents=True)
(tmp_path / "pkg" / "alpha" / "one.py").write_text(
@@ -872,20 +878,21 @@ async def test_gated_answer_surfaces_code_rationale(setup_mcp, monkeypatch, tmp_
monkeypatch.setattr(mcp_mod, "_repo_path", str(tmp_path))
result = await get_answer("why do uploads chunk at 8mb")
- assert result["confidence"] == "low"
- assert "best_guesses" in result # confirms we hit the gated path
+ assert result["confidence"] == "medium" # non-dominant ceiling
+ assert result["answer"], "non-dominant retrieval now carries synthesized prose"
+ assert "best_guesses" in result # ambiguous-retrieval evidence folded in
assert "code_rationale" in result
assert any("OOMs" in r["comment"] for r in result["code_rationale"])
@pytest.mark.asyncio
-async def test_gated_answer_carries_candidate_excerpts(setup_mcp, monkeypatch, tmp_path):
- """Gated path serves page content inline, not just pointers.
+async def test_non_dominant_best_guesses_carry_candidate_excerpts(setup_mcp, monkeypatch, tmp_path):
+ """Ambiguous-retrieval evidence carries page content inline, not just pointers.
- A pointers-only miss payload makes the agent re-acquire all content
- natively (observed as an 8-15 call Grep/Read spree in agent transcripts),
- so the gated reply must carry the top candidates' actual page content and
- say so in its guidance text.
+ A pointers-only reply makes the agent re-acquire all content natively
+ (observed as an 8-15 call Grep/Read spree in agent transcripts), so the
+ best_guesses folded into a non-dominant reply carry the top candidates'
+ actual page content beside the synthesized prose.
"""
import repowise.server.mcp_server as mcp_mod
import repowise.server.mcp_server.tool_answer.answer as answer_mod
@@ -913,17 +920,19 @@ async def test_gated_answer_carries_candidate_excerpts(setup_mcp, monkeypatch, t
monkeypatch.setattr(answer_mod, "_hybrid_retrieve", _fake_retrieve)
monkeypatch.setattr(answer_mod, "_hydrate_hits", _fake_hydrate)
monkeypatch.setattr(answer_mod, "_enrich_gated_excerpts", _fake_excerpts)
+ _patch_provider(monkeypatch, answer_mod, "Uploads chunk at 8MB (pkg/alpha/one.py).")
(tmp_path / "pkg" / "alpha").mkdir(parents=True)
(tmp_path / "pkg" / "alpha" / "one.py").write_text("CHUNK = 8\n", encoding="utf-8")
monkeypatch.setattr(mcp_mod, "_repo_path", str(tmp_path))
result = await get_answer("why do uploads chunk at 8mb")
- assert result["confidence"] == "low"
+ assert result["confidence"] == "medium" # non-dominant ceiling
+ assert result["answer"]
top = result["best_guesses"][0]
assert top["excerpt"].startswith("Page content for pkg/alpha/one.py")
- # Guidance must direct the agent at the excerpt, not at a blind Read.
- assert "excerpt" in result["next_action_hint"]
- assert "excerpt" in result["note"]
+ # The reply names the ambiguity and points at best_guesses to verify.
+ assert "best_guesses" in result["note"]
+ assert "ambiguous" in result["note"]
# ---------------------------------------------------------------------------
diff --git a/tests/unit/server/mcp/test_answer_payload.py b/tests/unit/server/mcp/test_answer_payload.py
index 35672299e..f6ee5e3e2 100644
--- a/tests/unit/server/mcp/test_answer_payload.py
+++ b/tests/unit/server/mcp/test_answer_payload.py
@@ -251,16 +251,20 @@ async def test_medium_confidence_keeps_two_truncated_hits(setup_mcp, monkeypatch
@pytest.mark.asyncio
-async def test_gated_path_serves_clean_retrieval(setup_mcp, monkeypatch):
+async def test_non_dominant_synthesizes_with_evidence(setup_mcp, monkeypatch):
+ # 2.0 vs 1.9: top < 3.0 and ratio 1.05 < 1.2 → non-dominant. Under the
+ # always-synthesize default the tool no longer abstains: synthesis runs, the
+ # non-dominant ceiling caps confidence at medium, and the reply folds in the
+ # ambiguous-retrieval evidence (best_guesses) beside the prose.
import repowise.server.mcp_server.tool_answer.answer as answer_mod
from repowise.server.mcp_server import get_answer
- # 2.0 vs 1.9: top < 3.0 and ratio 1.05 < 1.2 → gated, no synthesis.
_patch_pipeline(monkeypatch, answer_mod, scores=(2.0, 1.9))
- _patch_provider(monkeypatch, answer_mod, "unused")
+ _patch_provider(monkeypatch, answer_mod, "The go() function drives it (pkg/beta/one.py).")
result = await get_answer("how does the beta module go function work")
- assert result["confidence"] == "low"
+ assert result["confidence"] == "medium"
+ assert result["answer"], "non-dominant retrieval now carries synthesized prose"
assert result["best_guesses"]
for entry in result["retrieval"]:
assert "page_id" not in entry