Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(mcp): stop grading a derived value as a fabrication (#1861)

Raghav Chamadiya5af72bb5e155
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/confidence.py b/packages/server/src/repowise/server/mcp_server/tool_answer/confidence.py
index 971c42c88..a29849f50 100644
--- a/packages/server/src/repowise/server/mcp_server/tool_answer/confidence.py
+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/confidence.py
@@ -89,6 +89,15 @@ def _numbers_in(text: str) -> set[str]:
return set(_NUMBER_RE.findall(_THOUSANDS_SEP_RE.sub("", text or "")))
+def _asserted_numbers(answer_text: str) -> set[str]:
+ """Standalone numbers the answer asserts, citation line refs removed.
+
+ Shared by :func:`_ungrounded_numbers` and the value-grounding gate so the
+ two never disagree about what counts as an asserted value.
+ """
+ return _numbers_in(_FILE_LINE_REF_RE.sub(" ", answer_text or ""))
+
+
def _is_value_question(question: str) -> bool:
"""True when the question asks for a concrete value."""
return bool(_VALUE_QUESTION_RE.search(question or ""))
@@ -142,8 +151,7 @@ def _ungrounded_numbers(answer_text: str, hits: list[dict]) -> list[str]:
everything the LLM was shown for the hits — titles, summaries, snippets,
and hydrated symbols (signatures, docstrings, source excerpts).
"""
- text = _FILE_LINE_REF_RE.sub(" ", answer_text or "")
- asserted = _numbers_in(text)
+ asserted = _asserted_numbers(answer_text)
if not asserted:
return []
@@ -756,12 +764,23 @@ def _grade_answer(
# appear somewhere in the material retrieval actually contained. A
# number synthesis produced from thin air is a factual error delivered
# with authority — the single worst calibration failure, because the
- # consumer was told not to verify. Cap at low and say why.
+ # consumer was told not to verify. Cap and say why.
ungrounded_values: list[str] = []
if not hedged and _is_value_question(question):
ungrounded_values = _ungrounded_numbers(answer_text, hits)
if ungrounded_values:
- confidence = "low"
+ # A value derived from grounded operands is not a fabrication, so
+ # soften one notch instead of capping — but only on a clean high over
+ # dominant retrieval (earn_high on weak retrieval stays capped), and
+ # only when some OTHER asserted number is grounded. Without that
+ # second clause a lone invented value would soften too, which is the
+ # case this gate exists to catch. The note and next_action_hint fire
+ # on ungrounded_values regardless of tier.
+ grounded_sibling = len(_asserted_numbers(answer_text)) > len(ungrounded_values)
+ if grounded_sibling and confidence == "high" and dominant:
+ confidence = "medium"
+ else:
+ confidence = "low"
# Fifth gate — citation-source gate: a high-confidence answer must cite
# at least one page that contributed actual source material (hydrated
diff --git a/tests/unit/server/mcp/test_answer_calibration.py b/tests/unit/server/mcp/test_answer_calibration.py
index e23139ca3..87eb3f0dc 100644
--- a/tests/unit/server/mcp/test_answer_calibration.py
+++ b/tests/unit/server/mcp/test_answer_calibration.py
@@ -357,6 +357,31 @@ async def test_grounded_value_keeps_high_confidence(setup_mcp, monkeypatch):
assert "High confidence" in result["note"]
+async def test_derived_value_softens_to_medium_not_low(setup_mcp, monkeypatch):
+ """Grounded 2 plus a computed 6 softens to medium.
+
+ ``test_ungrounded_value_caps_confidence_low`` is the negative control: a
+ lone invented value with no grounded sibling still caps at low.
+ """
+ import repowise.server.mcp_server.tool_answer.answer as answer_mod
+ from repowise.server.mcp_server import get_answer
+
+ _patch_pipeline(monkeypatch, answer_mod, with_symbols=True, symbol=_FN_SYMBOL)
+ _patch_provider(
+ monkeypatch,
+ answer_mod,
+ "min_count_policy returns 2, and the three retry tiers "
+ "multiply it to 6 (pkg/alpha/one.py).",
+ )
+
+ result = await get_answer("What is the default value of min_count_policy?")
+ assert result["confidence"] == "medium"
+ # Still told to verify: the note and hint key off ungrounded_values, not tier.
+ assert "6" in result["note"]
+ assert "next_action_hint" in result
+
+
@pytest.mark.asyncio
async def test_high_confidence_requires_source_backed_citation(setup_mcp, monkeypatch):
"""No cited page contributed symbols → high is downgraded to medium."""