Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
"""The MCP instrument seam emits one anonymous ``mcp_tool_call`` per call.
Guards the two things that matter: the reported properties are coarse and
non-identifying (enums/booleans/buckets, never query text or paths), and the
emit is best-effort so it can never break or slow a tool response.
"""
from __future__ import annotations
import json
import pytest
from repowise.server.mcp_server._savings import wrapper
def _size(result) -> tuple[int, int]:
"""The size the wrapper should report, from the same formula the budget uses."""
chars = len(json.dumps(result, separators=(",", ":"), default=str))
return chars, chars // 4
class TestTelemetryProperties:
@pytest.fixture(autouse=True)
def _no_ambient_embedder(self, monkeypatch: pytest.MonkeyPatch):
"""Pin the install-level state so the exact-shape cases stay exact.
`semantic_search` is read from server state rather than the response, so
without this an unrelated test leaving an embedder resolved would add a
key here and the equality assertions would fail for the wrong reason.
"""
monkeypatch.setattr(wrapper, "_semantic_search_state", lambda: None)
# Likewise the host label, which reads the process environment.
monkeypatch.setattr(wrapper, "_client_label", lambda: "claude_code")
def test_answer_shape_extracts_enums_and_flags(self):
result = {
"answer": "…prose the agent reads…", # must NOT be reported
"confidence": "high",
"retrieval_quality": "strong",
"grounding": "exact_symbol",
"_meta": {"index_behind": True, "embedder_degraded": False, "timing_ms": 12.3},
}
props = wrapper._telemetry_properties("get_answer", result, 42)
chars, tokens = _size(result)
assert props == {
"tool": "get_answer",
"status": "ok",
"duration_ms": 42,
"client": "claude_code",
"confidence": "high",
"retrieval_quality": "strong",
"grounding": "exact_symbol",
"index_behind": True,
"embedder_degraded": False,
"response_chars": chars,
"response_tokens": tokens,
}
def test_degraded_answer_reports_why_synthesis_was_missing(self):
""""No provider" and "the provider failed" are different products.
Both return the same payload shape, so without this dimension a keyless
install working exactly as designed is indistinguishable from a broken
one, and neither can be sized.
"""
result = {
"answer": "…assembled boilerplate…",
"confidence": "medium",
"retrieval_quality": "high",
"degraded": "no-llm-provider",
"_meta": {"embedder_degraded": False},
}
props = wrapper._telemetry_properties("get_answer", result, 7)
assert props["degraded"] == "no-llm-provider"
assert props["embedder_degraded"] is False
# The reason is an enum; the prose it explains never travels.
assert "answer" not in props
def test_semantic_search_is_read_from_the_install_not_the_response(
self, monkeypatch: pytest.MonkeyPatch
):
"""The response shape is untouched; the dimension still gets all three states.
Putting this on the wire would have cost every caller tokens on every
call to say something `embedder_degraded` already implies. It is a fact
about the install, so it is read from the install.
"""
monkeypatch.setattr(wrapper, "_semantic_search_state", lambda: False)
props = wrapper._telemetry_properties("get_answer", {"confidence": "low"}, 1)
assert props["semantic_search"] is False
monkeypatch.setattr(wrapper, "_semantic_search_state", lambda: True)
assert wrapper._telemetry_properties("get_answer", {}, 1)["semantic_search"] is True
def test_an_unevaluated_embedder_reports_no_semantic_search(
self, monkeypatch: pytest.MonkeyPatch
):
"""Absent stays distinct from False: "never checked" is not "off"."""
monkeypatch.setattr(wrapper, "_semantic_search_state", lambda: None)
assert "semantic_search" not in wrapper._telemetry_properties("get_answer", {}, 1)
def test_error_result_is_status_error(self):
props = wrapper._telemetry_properties("get_symbol", {"error": "boom"}, 3)
assert props["status"] == "error"
# No site noted a reason, so it says so rather than guessing.
assert props["error_reason"] == "other"
def test_error_reason_comes_from_the_note_not_the_message(self):
from repowise.server.mcp_server._savings import interaction
with interaction.begin("get_symbol"):
interaction.note_error_reason("not_found")
interaction.note_error_reason("internal") # the first note wins
props = wrapper._telemetry_properties("get_symbol", {"error": "Symbol x"}, 3)
assert props["error_reason"] == "not_found"
def test_an_unknown_reason_is_never_recorded(self):
from repowise.server.mcp_server._savings import interaction
with interaction.begin("get_symbol"):
interaction.note_error_reason("Symbol not found: 'secret'")
props = wrapper._telemetry_properties("get_symbol", {"error": "x"}, 3)
assert props["error_reason"] == "other"
def test_ok_result_has_no_error_reason(self):
assert "error_reason" not in wrapper._telemetry_properties("get_answer", {}, 1)
def test_search_result_reports_count_bucket_not_content(self):
result = {"results": [{"path": "secret/a.py"}, {"path": "secret/b.py"}]}
props = wrapper._telemetry_properties("search_codebase", result, 5)
assert props["results_bucket"] == "1-3"
# Only coarse keys — no results/paths reach the wire.
assert set(props) == {
"tool",
"status",
"duration_ms",
"client",
"results_bucket",
"response_chars",
"response_tokens",
}
assert (props["response_chars"], props["response_tokens"]) == _size(result)
def test_non_dict_result_is_safe(self):
props = wrapper._telemetry_properties("get_overview", "unexpected", 1)
assert props == {
"tool": "get_overview",
"status": "ok",
"duration_ms": 1,
"client": "claude_code",
}
def test_response_size_is_the_serialised_payload(self):
"""Size is a number about the payload, not any of its content."""
result = {"answer": "x" * 100, "confidence": "high"}
props = wrapper._telemetry_properties("get_answer", result, 1)
chars, tokens = _size(result)
assert props["response_chars"] == chars
assert props["response_tokens"] == tokens == chars // 4
def test_unserialisable_result_reports_no_size(self, monkeypatch: pytest.MonkeyPatch):
"""A dump that blows up drops the size keys instead of failing the emit."""
class _Boom:
@staticmethod
def dumps(*a, **k):
raise TypeError("nope")
monkeypatch.setattr(wrapper, "json", _Boom)
props = wrapper._telemetry_properties("get_answer", {"confidence": "high"}, 1)
assert props == {
"tool": "get_answer",
"status": "ok",
"duration_ms": 1,
"client": "claude_code",
"confidence": "high",
}
@pytest.mark.asyncio
async def test_instrument_emits_one_event(monkeypatch: pytest.MonkeyPatch):
from repowise.core.platform import telemetry
calls: list[tuple[str, dict]] = []
monkeypatch.setattr(telemetry, "record_event", lambda event, props: calls.append((event, props)))
async def get_answer(question: str) -> dict:
return {"answer": "x", "confidence": "medium", "_meta": {}}
out = await wrapper.instrument(get_answer)("how does X work")
assert out["confidence"] == "medium" # response unchanged
assert len(calls) == 1
event, props = calls[0]
assert event == "mcp_tool_call"
assert props["tool"] == "get_answer"
assert props["confidence"] == "medium"
assert "duration_ms" in props
@pytest.mark.asyncio
async def test_telemetry_failure_never_breaks_the_tool(monkeypatch: pytest.MonkeyPatch):
from repowise.core.platform import telemetry
def boom(*a, **k):
raise RuntimeError("telemetry backend down")
monkeypatch.setattr(telemetry, "record_event", boom)
async def get_overview() -> dict:
return {"ok": True, "_meta": {}}
# The tool result must survive a telemetry emit that raises.
out = await wrapper.instrument(get_overview)()
assert out == {"ok": True, "_meta": {}}
def test_response_size_prefers_the_budget_stamp(monkeypatch: pytest.MonkeyPatch):
"""The budget pass already measured the payload; a second dump is waste."""
monkeypatch.setattr(wrapper, "_semantic_search_state", lambda: None)
result = {"answer": "x" * 50, "_meta": {"response_budget": {"serialized_chars": 4000}}}
props = wrapper._telemetry_properties("get_answer", result, 1)
assert props["response_chars"] == 4000
assert props["response_tokens"] == 1000
class TestClientLabel:
def test_announced_client_wins(self, monkeypatch: pytest.MonkeyPatch):
from repowise.server.mcp_server._savings import interaction
monkeypatch.setattr(interaction, "_attribution", lambda: ("codex", {}, None))
monkeypatch.setenv("CLAUDECODE", "1")
with interaction.begin("get_answer"):
assert wrapper._client_label() == "codex"
def test_falls_back_to_the_environment(self, monkeypatch: pytest.MonkeyPatch):
from repowise.core.platform import telemetry
monkeypatch.setattr(telemetry, "detect_client", lambda: "cursor")
assert wrapper._client_label() == "cursor"
def test_a_server_is_never_a_terminal(self, monkeypatch: pytest.MonkeyPatch):
from repowise.core.platform import telemetry
monkeypatch.setattr(telemetry, "detect_client", lambda: "terminal")
assert wrapper._client_label() == "unknown"