Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(mcp): stop grading a derived value as a fabrication (#1861)
Raghav Chamadiya5af72bb5e155
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/confidence.py b/packages/server/src/repowise/server/mcp_server/tool_answer/confidence.pyindex 971c42c88..a29849f50 100644--- a/packages/server/src/repowise/server/mcp_server/tool_answer/confidence.py+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/confidence.py@@ -89,6 +89,15 @@ def _numbers_in(text: str) -> set[str]:return set(_NUMBER_RE.findall(_THOUSANDS_SEP_RE.sub("", text or "")))+def _asserted_numbers(answer_text: str) -> set[str]:+ """Standalone numbers the answer asserts, citation line refs removed.++ Shared by :func:`_ungrounded_numbers` and the value-grounding gate so the+ two never disagree about what counts as an asserted value.+ """+ return _numbers_in(_FILE_LINE_REF_RE.sub(" ", answer_text or ""))++def _is_value_question(question: str) -> bool:"""True when the question asks for a concrete value."""return bool(_VALUE_QUESTION_RE.search(question or ""))@@ -142,8 +151,7 @@ def _ungrounded_numbers(answer_text: str, hits: list[dict]) -> list[str]:everything the LLM was shown for the hits — titles, summaries, snippets,and hydrated symbols (signatures, docstrings, source excerpts)."""- text = _FILE_LINE_REF_RE.sub(" ", answer_text or "")- asserted = _numbers_in(text)+ asserted = _asserted_numbers(answer_text)if not asserted:return []@@ -756,12 +764,23 @@ def _grade_answer(# appear somewhere in the material retrieval actually contained. A# number synthesis produced from thin air is a factual error delivered# with authority — the single worst calibration failure, because the- # consumer was told not to verify. Cap at low and say why.+ # consumer was told not to verify. Cap and say why.ungrounded_values: list[str] = []if not hedged and _is_value_question(question):ungrounded_values = _ungrounded_numbers(answer_text, hits)if ungrounded_values:- confidence = "low"+ # A value derived from grounded operands is not a fabrication, so+ # soften one notch instead of capping — but only on a clean high over+ # dominant retrieval (earn_high on weak retrieval stays capped), and+ # only when some OTHER asserted number is grounded. Without that+ # second clause a lone invented value would soften too, which is the+ # case this gate exists to catch. The note and next_action_hint fire+ # on ungrounded_values regardless of tier.+ grounded_sibling = len(_asserted_numbers(answer_text)) > len(ungrounded_values)+ if grounded_sibling and confidence == "high" and dominant:+ confidence = "medium"+ else:+ confidence = "low"# Fifth gate — citation-source gate: a high-confidence answer must cite# at least one page that contributed actual source material (hydrateddiff --git a/tests/unit/server/mcp/test_answer_calibration.py b/tests/unit/server/mcp/test_answer_calibration.pyindex e23139ca3..87eb3f0dc 100644--- a/tests/unit/server/mcp/test_answer_calibration.py+++ b/tests/unit/server/mcp/test_answer_calibration.py@@ -357,6 +357,31 @@ async def test_grounded_value_keeps_high_confidence(setup_mcp, monkeypatch):assert "High confidence" in result["note"]+async def test_derived_value_softens_to_medium_not_low(setup_mcp, monkeypatch):+ """Grounded 2 plus a computed 6 softens to medium.++ ``test_ungrounded_value_caps_confidence_low`` is the negative control: a+ lone invented value with no grounded sibling still caps at low.+ """+ import repowise.server.mcp_server.tool_answer.answer as answer_mod+ from repowise.server.mcp_server import get_answer++ _patch_pipeline(monkeypatch, answer_mod, with_symbols=True, symbol=_FN_SYMBOL)+ _patch_provider(+ monkeypatch,+ answer_mod,+ "min_count_policy returns 2, and the three retry tiers "+ "multiply it to 6 (pkg/alpha/one.py).",+ )++ result = await get_answer("What is the default value of min_count_policy?")+ assert result["confidence"] == "medium"+ # Still told to verify: the note and hint key off ungrounded_values, not tier.+ assert "6" in result["note"]+ assert "next_action_hint" in result++@pytest.mark.asyncioasync def test_high_confidence_requires_source_backed_citation(setup_mcp, monkeypatch):"""No cited page contributed symbols → high is downgraded to medium."""