Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(answer): meter what one synthesis call costs (#1180)
Swati Ahuja97f6edda6df1
diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.pyindex f6dd6cffa..82cacd430 100644--- a/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/answer.py@@ -1228,7 +1228,13 @@ async def get_answer(# remote API answers in single-digit seconds; an agent-CLI subprocess or a# local model needs minutes, and the old flat 30s cancelled every one of# those before it could return (#1119).- answer_text, failure_note = await synthesize(provider, _SYSTEM_PROMPT, user_prompt)+ answer_text, failure_note = await synthesize(+ provider,+ _SYSTEM_PROMPT,+ user_prompt,+ session_factory=getattr(ctx, "session_factory", None),+ repo_id=repo_id,+ )if failure_note is not None:return _degraded_payload(reason="synthesis-failed",diff --git a/packages/server/src/repowise/server/mcp_server/tool_answer/synthesis.py b/packages/server/src/repowise/server/mcp_server/tool_answer/synthesis.pyindex b7fa1870e..588d0db2f 100644--- a/packages/server/src/repowise/server/mcp_server/tool_answer/synthesis.py+++ b/packages/server/src/repowise/server/mcp_server/tool_answer/synthesis.py@@ -35,6 +35,9 @@ _FALLBACK_TIMEOUT_S = 30.0# bare transport error. 600s also matches the agent-CLI providers' own# subprocess ceiling, past which raising this is inert anyway._MAX_TIMEOUT_S = 600.0+# Operation label for the ledger rows written here, so answer spend is its own+# bucket on the costs report rather than folded into doc generation.+_COST_OPERATION = "answer_synthesis"def _hash_question(question: str) -> str:@@ -306,7 +309,92 @@ def _empty_completion_note(provider, response) -> str:)-async def synthesize(provider, system_prompt: str, user_prompt: str) -> tuple[str, str | None]:+async def _record_synthesis_cost(+ provider,+ response,+ session_factory,+ repo_id: str | None,+) -> None:+ """Meter one completed synthesis call: a ledger row plus a telemetry line.++ Every published figure for what a question costs to answer has come from+ metering the provider *outside* the product, because the token counts the+ provider returns were read for nothing but the empty-completion check. That+ makes the cost of a change to this path unverifiable anywhere it actually+ runs.++ Two states are reported separately on purpose. A call with counts is priced+ with the same table the generation ledger uses and persisted under its own+ operation label. A call whose counts are both zero is a provider adapter+ that did not normalise ``usage`` — it is *not* a free call, so it warns and+ writes nothing; a zero row would total up as free forever.++ Never raises: metering a call must not be able to fail an answer that+ already succeeded.+ """+ input_tokens = int(getattr(response, "input_tokens", 0) or 0)+ output_tokens = int(getattr(response, "output_tokens", 0) or 0)+ cached_tokens = int(getattr(response, "cached_tokens", 0) or 0)+ model = getattr(provider, "model_name", None) or "?"+ name = getattr(provider, "provider_name", None) or "?"++ if not input_tokens and not output_tokens:+ _log.warning(+ "get_answer synthesis reported no usage (provider=%s, model=%s) — this call is "+ "unpriced, not free. The provider adapter is not normalising input/output tokens.",+ name,+ model,+ )+ return++ try:+ from repowise.core.generation.cost_tracker import CostTracker++ tracker = CostTracker(session_factory, repo_id)+ cost_usd = await tracker.record(+ model=model,+ input_tokens=input_tokens,+ output_tokens=output_tokens,+ operation=_COST_OPERATION,+ )+ except Exception:+ _log.warning(+ "get_answer synthesis cost not recorded (provider=%s, model=%s, "+ "input_tokens=%d, output_tokens=%d, cached_tokens=%d)",+ name,+ model,+ input_tokens,+ output_tokens,+ cached_tokens,+ exc_info=True,+ )+ return++ # One line per synthesis, logged whether or not there was a ledger to write+ # to — an MCP server pointed at a read-only or repo-less index still has to+ # be able to report what it spent. ``cached_tokens`` is on the line because a+ # cost comparison across two runs is meaningless without it.+ _log.info(+ "get_answer synthesis cost: provider=%s model=%s operation=%s "+ "input_tokens=%d output_tokens=%d cached_tokens=%d cost_usd=%.6f",+ name,+ model,+ _COST_OPERATION,+ input_tokens,+ output_tokens,+ cached_tokens,+ cost_usd,+ )+++async def synthesize(+ provider,+ system_prompt: str,+ user_prompt: str,+ *,+ session_factory=None,+ repo_id: str | None = None,+) -> tuple[str, str | None]:"""Run one synthesis call. Returns ``(answer_text, failure_note)``.``failure_note`` is None on success. Owning the budget, the call and the@@ -314,6 +402,10 @@ async def synthesize(provider, system_prompt: str, user_prompt: str) -> tuple[stcancelled the call is the number the note quotes, and a caller cannot wireup the call while forgetting the budget (which is how #1119 survived ahardcoded 30s for as long as it did).++ ``session_factory`` / ``repo_id`` are where the cost row lands. Both+ optional: with neither, the call is still priced and logged, just not+ persisted."""timeout_s = _synthesis_timeout(provider)@@ -346,6 +438,11 @@ async def synthesize(provider, system_prompt: str, user_prompt: str) -> tuple[sttimed_out = Trueresponse, failure = None, excif failure is None:+ # Metered before the empty-completion branch: a reasoning model that+ # spends its whole budget on hidden thinking returns no text and is the+ # most expensive call this path makes, so it is the last one that should+ # go unpriced.+ await _record_synthesis_cost(provider, response, session_factory, repo_id)text = (getattr(response, "content", None) or "").strip()return (text, None) if text else ("", _empty_completion_note(provider, response))diff --git a/tests/unit/server/mcp/test_answer_excerpt_cap.py b/tests/unit/server/mcp/test_answer_excerpt_cap.pyindex 80337a505..625488aa5 100644--- a/tests/unit/server/mcp/test_answer_excerpt_cap.py+++ b/tests/unit/server/mcp/test_answer_excerpt_cap.py@@ -80,7 +80,7 @@ def _capture_prompt(monkeypatch, answer_mod, answer_text: str) -> list[str]:async def generate(self, **kwargs):return SimpleNamespace(content=answer_text)- async def _fake_synthesize(provider, system_prompt, user_prompt):+ async def _fake_synthesize(provider, system_prompt, user_prompt, **_kwargs):seen.append(user_prompt)return answer_text, Nonediff --git a/tests/unit/server/mcp/test_answer_page_excerpts.py b/tests/unit/server/mcp/test_answer_page_excerpts.pyindex ff287f545..f1fd03ff5 100644--- a/tests/unit/server/mcp/test_answer_page_excerpts.py+++ b/tests/unit/server/mcp/test_answer_page_excerpts.py@@ -93,7 +93,7 @@ def _capture_prompt(monkeypatch, answer_mod, answer_text: str) -> list[str]:async def generate(self, **kwargs):return SimpleNamespace(content=answer_text)- async def _fake_synthesize(provider, system_prompt, user_prompt):+ async def _fake_synthesize(provider, system_prompt, user_prompt, **_kwargs):seen.append(user_prompt)return answer_text, Nonediff --git a/tests/unit/server/mcp/test_answer_synthesis_cost.py b/tests/unit/server/mcp/test_answer_synthesis_cost.pynew file mode 100644index 000000000..55547b34d--- /dev/null+++ b/tests/unit/server/mcp/test_answer_synthesis_cost.py@@ -0,0 +1,144 @@+"""get_answer meters what its own synthesis call costs.++The tool has never been able to report its own spend: ``synthesize`` threw the+provider's token counts away, so the only way to price a question was to run it+from outside and meter the provider. That makes every cost claim about the+answer path unverifiable in production, and it hides the two states that matter+— a provider that reports no usage at all, and a call that got expensive.++A completed call now writes one ``llm_costs`` row under its own operation+label, and a response carrying no usage warns instead of writing zeros (a zero+row reads as a free call, which is the more expensive mistake).+"""++from __future__ import annotations++import logging++import pytest+from sqlalchemy import select+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine+from sqlalchemy.pool import StaticPool++from repowise.core.persistence.database import init_db+from repowise.core.persistence.models import LlmCost, Repository+from repowise.core.providers.llm.base import GeneratedResponse+from repowise.server.mcp_server.tool_answer.synthesis import synthesize++# Asserted as a literal, not imported: the label is the bucket the costs report+# groups by, so renaming the constant must not quietly rename the bucket.+_COST_OPERATION = "answer_synthesis"+++class _Provider:+ """Provider that returns a fixed response, recording nothing else."""++ provider_name = "testprov"+ model_name = "gemini-3.1-flash-lite-preview"+ interactive_timeout_s = 30.0++ def __init__(self, response: GeneratedResponse) -> None:+ self._response = response++ async def generate(self, **_kwargs) -> GeneratedResponse:+ return self._response+++async def factory():+ engine = create_async_engine(+ "sqlite+aiosqlite:///:memory:",+ connect_args={"check_same_thread": False},+ poolclass=StaticPool,+ )+ await init_db(engine)+ sessionmaker = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)+ async with sessionmaker() as session:+ session.add(+ Repository(+ id="repo1",+ name="test-repo",+ url="https://github.com/example/test-repo",+ local_path="/tmp/test-repo",+ default_branch="main",+ settings_json="{}",+ )+ )+ await session.commit()+ yield sessionmaker+ await engine.dispose()+++async def _ledger_rows(factory) -> list[LlmCost]:+ async with factory() as session:+ rows = await session.execute(select(LlmCost))+ return list(rows.scalars().all())+++async def test_a_completed_synthesis_writes_one_cost_row(factory):+ provider = _Provider(+ GeneratedResponse(content="the answer", input_tokens=61_741, output_tokens=402)+ )++ text, note = await synthesize(+ provider, "system", "user", session_factory=factory, repo_id="repo1"+ )++ assert (text, note) == ("the answer", None)+ rows = await _ledger_rows(factory)+ assert len(rows) == 1+ row = rows[0]+ assert row.operation == _COST_OPERATION+ assert row.model == "gemini-3.1-flash-lite-preview"+ assert (row.input_tokens, row.output_tokens) == (61_741, 402)+ assert row.cost_usd > 0.0+++async def test_a_response_with_no_usage_warns_instead_of_writing_zeros(factory, caplog):+ """A provider adapter that forgets to normalise usage must not read as free."""+ provider = _Provider(GeneratedResponse(content="the answer", input_tokens=0, output_tokens=0))++ with caplog.at_level(logging.WARNING, logger="repowise.mcp.answer"):+ text, note = await synthesize(+ provider, "system", "user", session_factory=factory, repo_id="repo1"+ )++ assert (text, note) == ("the answer", None)+ assert await _ledger_rows(factory) == []+ assert any(+ "usage" in r.message.lower() and r.levelno == logging.WARNING for r in caplog.records+ ), caplog.text+++async def test_the_cost_row_is_logged_even_with_nowhere_to_persist_it(factory, caplog):+ """The MCP server can run against a repo it has no writable ledger for."""+ provider = _Provider(+ GeneratedResponse(+ content="the answer", input_tokens=1_200, output_tokens=90, cached_tokens=800+ )+ )++ with caplog.at_level(logging.INFO, logger="repowise.mcp.answer"):+ text, _ = await synthesize(provider, "system", "user")++ assert text == "the answer"+ assert await _ledger_rows(factory) == []+ logged = "\n".join(r.getMessage() for r in caplog.records)+ assert "cached_tokens=800" in logged, logged+ assert "input_tokens=1200" in logged, logged+++async def test_a_failed_synthesis_writes_no_cost_row(factory):+ class _Failing(_Provider):+ async def generate(self, **_kwargs):+ raise RuntimeError("rate limited")++ provider = _Failing(GeneratedResponse(content="", input_tokens=0, output_tokens=0))++ text, note = await synthesize(+ provider, "system", "user", session_factory=factory, repo_id="repo1"+ )++ assert text == ""+ assert note is not None and "DEGRADED" in note+ assert await _ledger_rows(factory) == []