Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

refactor(backend/copilot): Moonshot module + cache_control widening + partial-messages default-on + title cost (#12882)

Zamil Majdy9703da3dfd0b
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py
index 826a06ac0b..0f1174d51e 100644
--- a/autogpt_platform/backend/backend/copilot/baseline/service.py
+++ b/autogpt_platform/backend/backend/copilot/baseline/service.py
@@ -46,6 +46,7 @@ from backend.copilot.model import (
upsert_chat_session,
)
from backend.copilot.model_router import resolve_model
+from backend.copilot.moonshot import is_moonshot_model
from backend.copilot.pending_message_helpers import (
combine_pending_with_current,
drain_pending_safe,
@@ -426,25 +427,39 @@ def _emit_all(
def _is_anthropic_model(model: str) -> bool:
"""Return True if *model* routes to Anthropic (native or via OpenRouter).
- Cache-control markers on message content + the ``anthropic-beta`` header
- are Anthropic-specific. OpenAI rejects the unknown ``cache_control``
- field with a 400 ("Extra inputs are not permitted") and Grok / other
- providers behave similarly. OpenRouter strips unknown headers but
- passes through ``cache_control`` on the body regardless of provider —
- which would also fail when OpenRouter routes to a non-Anthropic model.
-
Examples that return True:
- ``anthropic/claude-sonnet-4-6`` (OpenRouter route)
- ``claude-3-5-sonnet-20241022`` (direct Anthropic API)
- ``anthropic.claude-3-5-sonnet`` (Bedrock-style)
False for ``openai/gpt-4o``, ``google/gemini-2.5-pro``, ``xai/grok-4``
- etc.
+ etc. Moonshot is False here too even though Moonshot's
+ Anthropic-compat endpoint honours ``cache_control`` — use
+ :func:`_supports_prompt_cache_markers` for the cache-gating decision,
+ which also allows Moonshot routes. This function stays scoped to
+ "genuinely Anthropic" so callers that need the stricter check (e.g.
+ ``anthropic-beta`` header emission) keep their existing semantics.
"""
lowered = model.lower()
return "claude" in lowered or lowered.startswith("anthropic")
+def _supports_prompt_cache_markers(model: str) -> bool:
+ """Return True when *model* accepts Anthropic-style ``cache_control``.
+
+ Superset of :func:`_is_anthropic_model` — also allows Moonshot
+ (``moonshotai/*``), whose OpenRouter Anthropic-compat endpoint
+ honours the marker and empirically lifts cache hit rate on
+ continuation turns from near-zero (Moonshot's own automatic prefix
+ cache, which drifts readily) to the 60-95% Anthropic ballpark.
+
+ OpenAI / Grok / Gemini still 400 on ``cache_control``, so this
+ function returns False for those providers — add new vendors here
+ only after verifying their endpoint accepts the field.
+ """
+ return _is_anthropic_model(model) or is_moonshot_model(model)
+
+
def _fresh_ephemeral_cache_control() -> dict[str, str]:
"""Return a FRESH ephemeral ``cache_control`` dict each call.
@@ -565,19 +580,24 @@ async def _baseline_llm_caller(
round_text = ""
try:
client = _get_openai_client()
- # Cache markers are Anthropic-specific. For OpenAI/Grok/other
- # providers, leaving them on would trigger a 400 ("Extra inputs
- # are not permitted" on cache_control). Tools were precomputed
- # in stream_chat_completion_baseline via _mark_tools_with_cache_control
- # (only when the model was Anthropic), so on non-Anthropic routes
- # tools ship without cache_control on the last entry too.
+ # Cache markers are accepted by Anthropic AND Moonshot (via OR's
+ # Anthropic-compat endpoint). OpenAI/Grok/Gemini 400 on the
+ # unknown ``cache_control`` field — tools were precomputed in
+ # stream_chat_completion_baseline via _mark_tools_with_cache_control
+ # with the same gate, so on unsupported routes tools ship
+ # unmarked too.
#
- # `extra_body` `usage.include=true` asks OpenRouter to embed the real
- # generation cost into the final usage chunk — required by the
- # cost-based rate limiter in routes.py. Separate from the Anthropic
+ # The ``anthropic-beta`` header is only emitted for genuinely
+ # Anthropic routes (see :func:`_is_anthropic_model`) — Moonshot
+ # doesn't need the beta header; sending it is a no-op but we
+ # keep the check strict for clarity.
+ #
+ # `extra_body` `usage.include=true` asks OpenRouter to embed the
+ # real generation cost into the final usage chunk — required by
+ # the cost-based rate limiter in routes.py. Separate from the
# caching headers, always sent.
- is_anthropic = _is_anthropic_model(state.model)
- if is_anthropic:
+ supports_cache = _supports_prompt_cache_markers(state.model)
+ if supports_cache:
# Build the cached system dict once per session and splice it in
# on each round. The full ``messages`` list grows with every
# tool call, so copying the entire list just to mutate index 0
@@ -593,7 +613,11 @@ async def _baseline_llm_caller(
final_messages = [state.cached_system_message, *messages[1:]]
else:
final_messages = messages
- extra_headers = _fresh_anthropic_caching_headers()
+ extra_headers = (
+ _fresh_anthropic_caching_headers()
+ if _is_anthropic_model(state.model)
+ else None
+ )
else:
final_messages = messages
extra_headers = None
@@ -1636,9 +1660,10 @@ async def stream_chat_completion_baseline(
# _baseline_llm_caller) avoids re-copying ~43 tool dicts on every LLM
# round of the tool-call loop.
#
- # Only apply to Anthropic routes — OpenAI/Grok/other providers would
- # 400 on the unknown ``cache_control`` field inside tool definitions.
- if _is_anthropic_model(active_model):
+ # Applies to Anthropic AND Moonshot routes — OpenAI/Grok/Gemini 400
+ # on the unknown ``cache_control`` field inside tool definitions, so
+ # the gate stays narrow (see :func:`_supports_prompt_cache_markers`).
+ if _supports_prompt_cache_markers(active_model):
tools = cast(
list[ChatCompletionToolParam], _mark_tools_with_cache_control(tools)
)
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
index 44c49eb732..3051ea5d99 100644
--- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
+++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
@@ -21,6 +21,7 @@ from backend.copilot.baseline.service import (
_is_anthropic_model,
_mark_system_message_with_cache_control,
_mark_tools_with_cache_control,
+ _supports_prompt_cache_markers,
)
from backend.copilot.model import ChatMessage
from backend.copilot.response_model import (
@@ -1864,12 +1865,14 @@ class TestBaselineReasoningStreaming:
assert "reasoning" not in extra_body
@pytest.mark.asyncio
- async def test_kimi_route_sends_reasoning_but_no_cache_control(self):
- """Kimi K2.6 is the default fast_model and sends ``reasoning`` via
- OpenRouter's unified extension. It must NOT receive ``cache_control``
- markers or the ``anthropic-beta`` header — Moonshot uses its own
- auto-caching and those Anthropic-only fields would either get
- silently dropped or (worst case) 400 on a future provider change."""
+ async def test_kimi_route_sends_reasoning_and_cache_control(self):
+ """Kimi K2.6 (Moonshot via OpenRouter's Anthropic-compat endpoint)
+ accepts ``cache_control: {type: ephemeral}`` on the system block
+ and the last tool — the endpoint honours the marker and lifts
+ cache hit rate on continuation turns from near-zero (Moonshot's
+ auto-caching drifts) to the Anthropic ~60-95% ballpark. The
+ ``anthropic-beta`` header stays off because Moonshot doesn't need
+ it; OpenRouter would strip the unknown header anyway."""
state = _BaselineStreamState(model="moonshotai/kimi-k2.6")
mock_client = MagicMock()
@@ -1901,15 +1904,29 @@ class TestBaselineReasoningStreaming:
# cheap-but-still-reasoning-capable path.
assert "reasoning" in extra_body
assert extra_body["reasoning"]["max_tokens"] > 0
- # Anthropic-only fields stay off.
- assert "extra_headers" not in call_kwargs
+ # No ``anthropic-beta`` header — that beta is specifically for
+ # native Anthropic endpoints; Moonshot's shim accepts
+ # ``cache_control`` without it, and sending it would be wasted
+ # bytes (OR strips it before forwarding to Moonshot).
+ assert "extra_headers" not in call_kwargs or not call_kwargs.get(
+ "extra_headers"
+ )
+ # System block MUST carry ``cache_control`` so Moonshot's cache
+ # breakpoint is honoured. The cached system-message builder
+ # emits list-shape content with the marker on the first (and
+ # only) block — assert on that shape.
sys_msg = call_kwargs["messages"][0]
sys_content = sys_msg.get("content")
- if isinstance(sys_content, list):
- assert all("cache_control" not in block for block in sys_content)
- tools = call_kwargs.get("tools", [])
- for t in tools:
- assert "cache_control" not in t
+ assert isinstance(
+ sys_content, list
+ ), "Cached system message should be a list-shape content block"
+ assert any(
+ "cache_control" in block for block in sys_content if isinstance(block, dict)
+ ), "Kimi system message should now carry cache_control markers"
+ # Tool-level cache marking is applied by ``stream_chat_completion_baseline``
+ # (see ``_mark_tools_with_cache_control``) before tools reach
+ # ``_baseline_llm_caller``, so this unit test doesn't exercise
+ # that path — covered by the outer integration test.
@pytest.mark.asyncio
async def test_reasoning_only_stream_still_closes_block(self):
@@ -2009,3 +2026,55 @@ class TestBaselineReasoningStreaming:
reasoning_rows = [m for m in state.session_messages if m.role == "reasoning"]
assert len(reasoning_rows) == 1
assert reasoning_rows[0].content == "first thought"
+
+
+class TestSupportsPromptCacheMarkers:
+ """``_supports_prompt_cache_markers`` is the widened gate for
+ emitting ``cache_control`` markers on message content. It's a
+ superset of ``_is_anthropic_model`` that ALSO admits Moonshot
+ (whose Anthropic-compat endpoint honours the marker) while keeping
+ the False answer for OpenAI / Grok / Gemini (which 400 on the
+ unknown field)."""
+
+ @pytest.mark.parametrize(
+ "model",
+ [
+ "anthropic/claude-sonnet-4-6",
+ "claude-3-5-sonnet-20241022",
+ "anthropic.claude-3-5-sonnet",
+ "ANTHROPIC/Claude-Opus",
+ ],
+ )
+ def test_anthropic_routes_are_supported(self, model):
+ assert _supports_prompt_cache_markers(model) is True
+
+ @pytest.mark.parametrize(
+ "model",
+ [
+ "moonshotai/kimi-k2.6",
+ "moonshotai/kimi-k2-thinking",
+ "moonshotai/kimi-k2.5",
+ "moonshotai/kimi-k3.0", # future SKU
+ ],
+ )
+ def test_moonshot_routes_are_supported(self, model):
+ """The whole reason this predicate exists — Moonshot must be
+ True even though ``_is_anthropic_model`` is False for it."""
+ assert _supports_prompt_cache_markers(model) is True
+ # Verify this is strictly wider than the anthropic-only check.
+ assert _is_anthropic_model(model) is False
+
+ @pytest.mark.parametrize(
+ "model",
+ [
+ "openai/gpt-4o",
+ "google/gemini-2.5-pro",
+ "xai/grok-4",
+ "meta-llama/llama-3.3-70b-instruct",
+ "deepseek/deepseek-v3",
+ ],
+ )
+ def test_other_providers_still_rejected(self, model):
+ """Regression guard: OpenAI/Grok/Gemini still 400 on
+ ``cache_control``, so the widened gate must keep them out."""
+ assert _supports_prompt_cache_markers(model) is False
diff --git a/autogpt_platform/backend/backend/copilot/config.py b/autogpt_platform/backend/backend/copilot/config.py
index 3d785c8ee9..64e0e92ee8 100644
--- a/autogpt_platform/backend/backend/copilot/config.py
+++ b/autogpt_platform/backend/backend/copilot/config.py
@@ -294,20 +294,10 @@ class ChatConfig(BaseSettings):
"https://platform.claude.com/docs/en/build-with-claude/prompt-caching.",
)
sdk_include_partial_messages: bool = Field(
- default=False,
- description="Enable per-token streaming on the SDK path by setting "
- "``include_partial_messages=True`` on ``ClaudeAgentOptions``. The "
- "CLI then emits raw Anthropic ``content_block_delta`` events as "
- "``StreamEvent`` messages ahead of each summary "
- "``AssistantMessage``, so long answers and extended-thinking "
- "reasoning land on the wire token-by-token instead of popping in "
- "as a lump at ``content_block_stop``. Matches the perceptual "
- "progress the baseline path has shipped since #12873. Off by "
- "default to keep the rollout staged; the adapter falls back to "
- "summary-only emission when this flag is False. See "
- "``docs/sdk-per-token-streaming-followup.md`` for the diff-based "
- "reconcile logic that prevents partial/summary double-emission "
- "and truncation when the two views disagree.",
+ default=True,
+ description="Stream SDK responses token-by-token instead of in "
+ "one lump at the end. Set to False if the SDK path starts "
+ "double-writing text or dropping the tail of long messages.",
)
sdk_reconcile_openrouter_cost: bool = Field(
default=True,
diff --git a/autogpt_platform/backend/backend/copilot/moonshot.py b/autogpt_platform/backend/backend/copilot/moonshot.py
new file mode 100644
index 0000000000..c117e76120
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/moonshot.py
@@ -0,0 +1,147 @@
+"""Moonshot-specific pricing and cache-control helpers.
+
+Moonshot's Kimi K2.x family is routed through OpenRouter's Anthropic-compat
+shim — it speaks Anthropic's API shape but its pricing and cache behaviour
+diverge from Anthropic in ways the Claude Agent SDK CLI and our baseline
+cache-control gating don't handle on their own:
+
+* **Rate card** — NOT the canonical cost source. The authoritative number
+ for every OpenRouter-routed turn is the reconcile task
+ (:mod:`openrouter_cost`), which reads ``total_cost`` directly from
+ ``/api/v1/generation`` post-turn. This module exists purely so the
+ CLI's in-turn ``ResultMessage.total_cost_usd`` (which silently bills
+ Moonshot at Sonnet rates, ~5x the real Moonshot price because the CLI
+ pricing table only knows Anthropic) isn't left wildly wrong before the
+ reconcile fires AND so the reconcile's lookup-fail fallback records a
+ plausible Moonshot estimate rather than a Sonnet-rate overcharge.
+ Signal authority: reconcile >> this module's rate card >> CLI.
+
+* **Cache-control** — Anthropic and Moonshot both accept the
+ ``cache_control: {type: ephemeral}`` breakpoint on message blocks, but
+ our baseline path currently gates cache markers on an
+ ``anthropic/`` / ``claude`` name match because non-Anthropic providers
+ (OpenAI, Grok, Gemini) 400 on the unknown field. Moonshot's
+ Anthropic-compat endpoint silently accepts and honours the marker —
+ empirically boosts cache hit rate on continuation turns — but was
+ caught in the non-Anthropic branch of the original gate.
+ :func:`moonshot_supports_cache_control` lets callers widen the gate
+ to include Moonshot without weakening the ``false`` answer for
+ OpenAI et al. (The predicate is intentionally narrow — Moonshot-only
+ — so callers combine it with an explicit Anthropic check at the call
+ site; see ``baseline/service.py::_supports_prompt_cache_markers``.)
+
+Detection is prefix-based (``moonshotai/``). Moonshot routes every Kimi
+SKU through the same Anthropic-compat surface and currently prices them
+identically, so a new ``moonshotai/kimi-k3.0`` slug transparently
+inherits both the rate card and the cache-control gate without editing
+this file. Per-slug overrides are in :data:`_RATE_OVERRIDES_USD_PER_MTOK`
+for when Moonshot eventually splits prices.
+"""
+
+from __future__ import annotations
+
+# All Moonshot slugs share these rates as of April 2026 — Moonshot prices
+# every Kimi K2.x SKU at $0.60/$2.80 per million (input/output) via
+# OpenRouter. Cache-read / cache-write discounts are NOT applied here:
+# OpenRouter currently exposes only a single input price per Moonshot
+# endpoint; the real billed amount (with cache savings) lands via the
+# reconcile path. Keep in sync with https://platform.moonshot.ai/docs/pricing.
+_DEFAULT_MOONSHOT_RATE_USD_PER_MTOK: tuple[float, float] = (0.60, 2.80)
+
+# Per-slug overrides for when Moonshot splits pricing across SKUs. Empty
+# today — every slug matching ``moonshotai/`` falls back to
+# :data:`_DEFAULT_MOONSHOT_RATE_USD_PER_MTOK`.
+_RATE_OVERRIDES_USD_PER_MTOK: dict[str, tuple[float, float]] = {}
+
+# Vendor prefix — matches any OpenRouter slug Moonshot ships. Keep as a
+# module constant so the prefix check stays in exactly one place.
+_MOONSHOT_PREFIX = "moonshotai/"
+
+
+def is_moonshot_model(model: str | None) -> bool:
+ """True when *model* is a Moonshot OpenRouter slug.
+
+ Prefix match against ``moonshotai/`` covers every Kimi SKU Moonshot
+ ships today (``kimi-k2``, ``kimi-k2.5``, ``kimi-k2.6``,
+ ``kimi-k2-thinking``) plus any future SKU Moonshot publishes under
+ the same namespace. Used by both pricing and cache-control gating.
+ """
+ return isinstance(model, str) and model.startswith(_MOONSHOT_PREFIX)
+
+
+def rate_card_usd(model: str | None) -> tuple[float, float] | None:
+ """Return (input, output) $/Mtok for *model* or None if non-Moonshot.
+
+ Looks up a per-slug override first, falling back to the shared
+ default for anything under ``moonshotai/``. Returns None for
+ non-Moonshot slugs (including ``None``) so callers can skip the
+ override without a preflight guard.
+ """
+ if not is_moonshot_model(model):
+ return None
+ # ``is_moonshot_model`` narrowed ``model`` to str; dict.get is
+ # type-safe here despite the wider param annotation above.
+ assert model is not None
+ return _RATE_OVERRIDES_USD_PER_MTOK.get(model, _DEFAULT_MOONSHOT_RATE_USD_PER_MTOK)
+
+
+def override_cost_usd(
+ *,
+ model: str | None,
+ sdk_reported_usd: float,
+ prompt_tokens: int,
+ completion_tokens: int,
+ cache_read_tokens: int,
+ cache_creation_tokens: int,
+) -> float:
+ """Recompute SDK turn cost from the Moonshot rate card.
+
+ Not the canonical cost source — the OpenRouter ``/generation``
+ reconcile (:mod:`openrouter_cost`) lands the authoritative billed
+ amount post-turn. This helper exists only to improve the CLI's
+ in-turn ``ResultMessage.total_cost_usd``:
+
+ 1. So the ``cost_usd`` the client sees before the reconcile completes
+ isn't wildly wrong (the CLI would otherwise ship a Sonnet-rate
+ estimate, ~5x the real Moonshot bill).
+ 2. So the reconcile's own lookup-fail fallback records a plausible
+ Moonshot estimate rather than the CLI's Sonnet number.
+
+ For Moonshot slugs we compute cost from the reported token counts;
+ for anything else (including Anthropic) we return the SDK number
+ unchanged — Anthropic slugs are priced accurately by the CLI.
+
+ Cache read / creation tokens are folded into ``prompt_tokens`` at
+ the full input rate because Moonshot's rate card doesn't distinguish
+ them at the OpenRouter surface; the reconcile has the authoritative
+ discount accounting for turns where Moonshot's cache engaged.
+ """
+ if model is None:
+ return sdk_reported_usd
+ rates = rate_card_usd(model)
+ if rates is None:
+ return sdk_reported_usd
+ input_rate, output_rate = rates
+ total_prompt = prompt_tokens + cache_read_tokens + cache_creation_tokens
+ return (total_prompt * input_rate + completion_tokens * output_rate) / 1_000_000
+
+
+def moonshot_supports_cache_control(model: str | None) -> bool:
+ """True when a Moonshot *model* accepts Anthropic-style ``cache_control``.
+
+ Narrow, Moonshot-specific predicate — callers that need the full
+ "does this route accept cache markers" answer combine this with an
+ Anthropic check (see ``baseline/service.py::_supports_prompt_cache_markers``).
+ Named ``moonshot_*`` deliberately so the call site can't mistake it
+ for a universal predicate that answers correctly for Anthropic
+ (which also supports cache_control — this function would return
+ False for Anthropic slugs).
+
+ Moonshot's Anthropic-compat endpoint honours the marker. Without
+ it Moonshot falls back to its own automatic prefix caching, which
+ drifts more readily between turns (internal testing saw 0/4 cache
+ hits across two continuation sessions). With explicit
+ ``cache_control`` the upstream cache hit rate rises to the same
+ ballpark as Anthropic's ~60-95% on continuations.
+ """
+ return is_moonshot_model(model)
diff --git a/autogpt_platform/backend/backend/copilot/moonshot_test.py b/autogpt_platform/backend/backend/copilot/moonshot_test.py
new file mode 100644
index 0000000000..7fcea124bf
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/moonshot_test.py
@@ -0,0 +1,173 @@
+"""Unit tests for Moonshot pricing and cache-control helpers."""
+
+from __future__ import annotations
+
+import pytest
+
+from backend.copilot.moonshot import (
+ is_moonshot_model,
+ moonshot_supports_cache_control,
+ override_cost_usd,
+ rate_card_usd,
+)
+
+
+class TestIsMoonshotModel:
+ """Prefix detection covers every Moonshot SKU without a slug list."""
+
+ @pytest.mark.parametrize(
+ "model",
+ [
+ "moonshotai/kimi-k2.6",
+ "moonshotai/kimi-k2-thinking",
+ "moonshotai/kimi-k2.5",
+ "moonshotai/kimi-k2",
+ "moonshotai/kimi-k3.0", # Future SKU must match transparently.
+ ],
+ )
+ def test_moonshot_slugs_match(self, model: str) -> None:
+ assert is_moonshot_model(model) is True
+
+ @pytest.mark.parametrize(
+ "model",
+ [
+ "anthropic/claude-sonnet-4.6",
+ "anthropic/claude-opus-4.7",
+ "openai/gpt-4o",
+ "google/gemini-2.5-flash",
+ "xai/grok-4",
+ "deepseek/deepseek-v3",
+ "", # Empty string — not Moonshot.
+ ],
+ )
+ def test_non_moonshot_slugs_do_not_match(self, model: str) -> None:
+ assert is_moonshot_model(model) is False
+
+ @pytest.mark.parametrize("model", [None, 123, ["moonshotai/kimi-k2.6"]])
+ def test_non_string_returns_false(self, model) -> None:
+ # Type-robust: never raise on unexpected types; callers pass None.
+ assert is_moonshot_model(model) is False
+
+
+class TestRateCardUsd:
+ """Rate card defaults to the shared Moonshot price for every SKU."""
+
+ def test_moonshot_default_rate(self) -> None:
+ assert rate_card_usd("moonshotai/kimi-k2.6") == (0.60, 2.80)
+
+ def test_future_moonshot_sku_inherits_default(self) -> None:
+ # Verifies the prefix-based fallback — new SKUs don't need a code
+ # edit to get a reasonable rate card.
+ assert rate_card_usd("moonshotai/kimi-k3.0") == (0.60, 2.80)
+
+ def test_non_moonshot_returns_none(self) -> None:
+ assert rate_card_usd("anthropic/claude-sonnet-4.6") is None
+ assert rate_card_usd("openai/gpt-4o") is None
+
+
+class TestOverrideCostUsd:
+ """Rate-card override replaces the CLI's Sonnet-rate estimate for
+ Moonshot turns; Anthropic and unknown slugs pass through unchanged."""
+
+ def test_moonshot_recomputes_from_rate_card(self) -> None:
+ """A 29.5K-prompt Kimi turn should land at ~$0.018 on the
+ Moonshot rate card, not the CLI's $0.09 Sonnet-rate estimate."""
+ recomputed = override_cost_usd(
+ model="moonshotai/kimi-k2.6",
+ sdk_reported_usd=0.089862, # What the CLI reported (Sonnet price).
+ prompt_tokens=29564,
+ completion_tokens=78,
+ cache_read_tokens=0,
+ cache_creation_tokens=0,
+ )
+ expected = (29564 * 0.60 + 78 * 2.80) / 1_000_000
+ assert recomputed == pytest.approx(expected, rel=1e-9)
+ assert 0.017 < recomputed < 0.019 # Sanity against Moonshot's rate card.
+
+ def test_anthropic_passes_through(self) -> None:
+ """Anthropic slugs are priced accurately by the CLI already —
+ the override returns the SDK number unchanged."""
+ assert (
+ override_cost_usd(
+ model="anthropic/claude-sonnet-4.6",
+ sdk_reported_usd=0.089862,
+ prompt_tokens=29564,
+ completion_tokens=78,
+ cache_read_tokens=0,
+ cache_creation_tokens=0,
+ )
+ == 0.089862
+ )
+
+ def test_unknown_non_moonshot_passes_through(self) -> None:
+ """A non-Moonshot, non-Anthropic slug falls back to the SDK value
+ — best-effort rather than leaking a zero or a wrong rate card."""
+ assert (
+ override_cost_usd(
+ model="deepseek/deepseek-v3",
+ sdk_reported_usd=0.05,
+ prompt_tokens=10_000,
+ completion_tokens=500,
+ cache_read_tokens=0,
+ cache_creation_tokens=0,
+ )
+ == 0.05
+ )
+
+ def test_none_model_passes_through(self) -> None:
+ """Subscription mode sets model=None — return the SDK value."""
+ assert (
+ override_cost_usd(
+ model=None,
+ sdk_reported_usd=0.07,
+ prompt_tokens=100,
+ completion_tokens=10,
+ cache_read_tokens=0,
+ cache_creation_tokens=0,
+ )
+ == 0.07
+ )
+
+ def test_cache_tokens_priced_at_input_rate(self) -> None:
+ """OpenRouter's Moonshot endpoints don't expose a discounted
+ cached-input price — cache_read / cache_creation tokens are
+ priced at the full input rate. The reconcile path has the
+ authoritative discount for turns where Moonshot's cache engaged."""
+ recomputed = override_cost_usd(
+ model="moonshotai/kimi-k2.6",
+ sdk_reported_usd=0.5,
+ prompt_tokens=1000,
+ completion_tokens=0,
+ cache_read_tokens=5000,
+ cache_creation_tokens=2000,
+ )
+ expected = (1000 + 5000 + 2000) * 0.60 / 1_000_000
+ assert recomputed == pytest.approx(expected, rel=1e-9)
+
+
+class TestSupportsCacheControl:
+ """Gate for emitting ``cache_control: {type: ephemeral}`` on message
+ blocks. True for Moonshot (Anthropic-compat endpoint accepts it)
+ and False for everything else this module knows about — Anthropic
+ callers use their own ``_is_anthropic_model`` check which is
+ combined with this one into a wider gate."""
+
+ def test_moonshot_supports_cache_control(self) -> None:
+ assert moonshot_supports_cache_control("moonshotai/kimi-k2.6") is True
+
+ def test_future_moonshot_sku_supports_cache_control(self) -> None:
+ assert moonshot_supports_cache_control("moonshotai/kimi-k3.0") is True
+
+ @pytest.mark.parametrize(
+ "model",
+ [
+ "openai/gpt-4o",
+ "google/gemini-2.5-flash",
+ "xai/grok-4",
+ "deepseek/deepseek-v3",
+ "",
+ None,
+ ],
+ )
+ def test_non_moonshot_does_not_support_cache_control(self, model) -> None:
+ assert moonshot_supports_cache_control(model) is False
diff --git a/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py b/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py
index bd26db5d5f..2a15e9f1fc 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py
@@ -267,7 +267,7 @@ class SDKResponseAdapter:
# Strip MCP prefix so frontend sees "find_block"
# instead of "mcp__copilot__find_block".
- tool_name = block.name.removeprefix(MCP_TOOL_PREFIX)
+ tool_name = block.name.strip().removeprefix(MCP_TOOL_PREFIX)
responses.append(
StreamToolInputStart(toolCallId=block.id, toolName=tool_name)
diff --git a/autogpt_platform/backend/backend/copilot/sdk/response_adapter_test.py b/autogpt_platform/backend/backend/copilot/sdk/response_adapter_test.py
index d61349542b..6d59e21fab 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/response_adapter_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/response_adapter_test.py
@@ -138,6 +138,29 @@ def test_tool_use_emits_input_start_and_available():
assert results[2].input == {"q": "x"}
+def test_tool_use_strips_whitespace_in_tool_name():
+ adapter = _adapter()
+ msg = AssistantMessage(
+ content=[
+ ToolUseBlock(
+ id="tool-1",
+ name=f" {MCP_TOOL_PREFIX}find_block",
+ input={},
+ )
+ ],
+ model="test",
+ )
+ results = adapter.convert_message(msg)
+ tool_events = [
+ r
+ for r in results
+ if isinstance(r, (StreamToolInputStart, StreamToolInputAvailable))
+ ]
+ assert tool_events, "expected tool input events"
+ for event in tool_events:
+ assert event.toolName == "find_block"
+
+
def test_text_then_tool_ends_text_block():
adapter = _adapter()
text_msg = AssistantMessage(content=[TextBlock(text="thinking...")], model="test")
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.py
index 056c4538b7..1ce2ede6b8 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py
@@ -58,6 +58,10 @@ from ..session_cleanup import prune_orphan_tool_calls
from ..context import encode_cwd_for_cli, get_workspace_manager
from ..graphiti.config import is_enabled_for_user
from ..model_router import resolve_model
+from ..moonshot import (
+ is_moonshot_model as _is_moonshot_model,
+ override_cost_usd as _override_cost_for_moonshot,
+)
from ..model import (
ChatMessage,
ChatSession,
@@ -376,6 +380,12 @@ class _RetryState:
# authoritative ``/generation`` total_cost. Lives on ``_RetryState``
# (not per-attempt ``_StreamAccumulator``) so it survives retries.
generation_ids: list[str] = dataclass_field(default_factory=list)
+ # The *actually executed* model observed on ``AssistantMessage.model`` —
+ # differs from ``state.options.model`` (the requested primary) when
+ # ``_resolve_fallback_model`` swaps to a fallback mid-attempt. The
+ # Moonshot cost override gates on this so a Moonshot-→-Anthropic
+ # fallback doesn't get mis-billed at Moonshot rates, and vice versa.
+ observed_model: str | None = None
@dataclass
@@ -724,55 +734,6 @@ def _normalize_model_name(raw_model: str) -> str:
return model.replace(".", "-")
-# Per-million-token rates ($USD) for non-Anthropic OpenRouter slugs that
-# the Claude Agent SDK CLI doesn't recognise. The CLI's bundled pricing
-# table only knows Anthropic models — for anything else its
-# ``ResultMessage.total_cost_usd`` silently falls back to Sonnet rates,
-# over-billing by ~5x for cheaper models like Kimi K2.6. Values are taken
-# directly from each provider's published rate card and must be kept in
-# sync when prices change. Cache discounts are not applied — Kimi via
-# OpenRouter does not currently expose a separate cached-input price.
-_NON_ANTHROPIC_RATES_USD_PER_MTOK: dict[str, tuple[float, float]] = {
- # vendor/model: (input_per_mtok, output_per_mtok)
- "moonshotai/kimi-k2.6": (0.60, 2.80),
- "moonshotai/kimi-k2-thinking": (0.60, 2.80),
- "moonshotai/kimi-k2.5": (0.60, 2.80),
- "moonshotai/kimi-k2": (0.60, 2.80),
-}
-
-
-def _override_cost_for_non_anthropic(
- raw_model: str | None,
- sdk_reported_usd: float,
- prompt_tokens: int,
- completion_tokens: int,
- cache_read_tokens: int,
- cache_creation_tokens: int,
-) -> float:
- """Recompute turn cost from a known rate card for non-Anthropic models.
-
- The Claude Agent SDK CLI's ``total_cost_usd`` is computed from a
- static Anthropic pricing table baked into the binary — it doesn't
- know Kimi/DeepSeek/etc rates and silently bills at Sonnet prices,
- which would over-charge a Kimi-default deployment by ~5x. Mirror
- the baseline path's behaviour by computing the real cost from the
- token counts whenever we recognise the slug; otherwise trust the
- SDK number (correct for Anthropic models, best-effort for unknown
- providers).
- """
- if raw_model is None:
- return sdk_reported_usd
- rates = _NON_ANTHROPIC_RATES_USD_PER_MTOK.get(raw_model)
- if rates is None:
- return sdk_reported_usd
- input_rate, output_rate = rates
- # Treat cache reads/creation as plain prompt tokens since OpenRouter
- # does not currently report a discounted cached-input price for the
- # tracked Moonshot endpoints.
- total_prompt = prompt_tokens + cache_read_tokens + cache_creation_tokens
- return (total_prompt * input_rate + completion_tokens * output_rate) / 1_000_000
-
-
def _resolve_sdk_model() -> str | None:
"""Resolve the SDK-CLI model name from static config (no LD lookup).
@@ -2251,6 +2212,15 @@ async def _run_stream_attempt(
and msg_id not in state.generation_ids
):
state.generation_ids.append(msg_id)
+ # Track the model the SDK actually used — when a fallback
+ # activates, this differs from ``state.options.model``.
+ # Consumed by the Moonshot cost-override decision so we
+ # don't mis-bill a fallback-Anthropic response at
+ # Moonshot rates (or a fallback-Moonshot at Anthropic
+ # rates).
+ observed = getattr(sdk_msg, "model", None)
+ if isinstance(observed, str) and observed:
+ state.observed_model = observed
# Log AssistantMessage API errors (e.g. invalid_request)
# so we can debug Anthropic API 400s surfaced by the CLI.
@@ -2420,23 +2390,37 @@ async def _run_stream_attempt(
state.usage.completion_tokens,
)
if sdk_msg.total_cost_usd is not None:
- # The SDK CLI's ``total_cost_usd`` is computed from a
- # static Anthropic pricing table baked into the CLI
- # binary. When we route through OpenRouter to a non-
- # Anthropic model (e.g. Kimi K2.6) the CLI doesn't
- # know the real per-token price and silently falls
- # back to Sonnet rates — over-billing the user ~5x.
- # Recompute from a known rate card for non-Anthropic
- # OpenRouter slugs so the cost row, the rate-limit
- # counter, and the UI cost display reflect reality.
- state.usage.cost_usd = _override_cost_for_non_anthropic(
- raw_model=getattr(state.options, "model", None),
- sdk_reported_usd=sdk_msg.total_cost_usd,
- prompt_tokens=state.usage.prompt_tokens,
- completion_tokens=state.usage.completion_tokens,
- cache_read_tokens=state.usage.cache_read_tokens,
- cache_creation_tokens=state.usage.cache_creation_tokens,
+ # Default: trust the CLI-reported value. Accurate for
+ # Anthropic models (the CLI's bundled pricing table is
+ # Anthropic-authored), and becomes the sync-path cost
+ # when the reconcile is disabled or fails.
+ # Prefer the ACTUALLY executed model
+ # (``state.observed_model`` from ``AssistantMessage.model``)
+ # over the requested primary (``state.options.model``)
+ # so a fallback activation doesn't mis-route pricing.
+ active_model = state.observed_model or getattr(
+ state.options, "model", None
)
+ if _is_moonshot_model(active_model):
+ # Moonshot slug — the CLI doesn't know Moonshot's
+ # rate card and silently bills at Sonnet rates
+ # (~5x over-charge). Replace with the rate-card
+ # estimate so the in-stream ``cost_usd`` and the
+ # reconcile's lookup-fail fallback reflect
+ # reality. Reconcile
+ # (``record_turn_cost_from_openrouter``) still
+ # overrides this value when every gen-ID lookup
+ # succeeds.
+ state.usage.cost_usd = _override_cost_for_moonshot(
+ model=active_model,
+ sdk_reported_usd=sdk_msg.total_cost_usd,
+ prompt_tokens=state.usage.prompt_tokens,
+ completion_tokens=state.usage.completion_tokens,
+ cache_read_tokens=state.usage.cache_read_tokens,
+ cache_creation_tokens=state.usage.cache_creation_tokens,
+ )
+ else:
+ state.usage.cost_usd = sdk_msg.total_cost_usd
# Emit compaction end if SDK finished compacting.
# Sync TranscriptBuilder with the CLI's active context.
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service_test.py b/autogpt_platform/backend/backend/copilot/sdk/service_test.py
index a2e1ac35bb..82d6ff7e60 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service_test.py
@@ -4,6 +4,7 @@ import asyncio
import base64
import os
from dataclasses import dataclass
+from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -15,7 +16,6 @@ from .service import (
_build_system_prompt_value,
_is_sdk_disconnect_error,
_normalize_model_name,
- _override_cost_for_non_anthropic,
_prepare_file_attachments,
_resolve_sdk_model,
_resolve_sdk_model_for_request,
@@ -917,77 +917,223 @@ class TestIdleTimeoutConstant:
assert _IDLE_TIMEOUT_SECONDS == 10 * 60
-class TestOverrideCostForNonAnthropic:
- """Verifies that turn costs routed through OpenRouter to non-Anthropic
- vendors use the platform's per-model rate card instead of the SDK
- CLI's static Anthropic pricing table — which silently falls back to
- Sonnet rates for unknown models and over-bills by ~5x."""
+# ---------------------------------------------------------------------------
+# _RetryState.observed_model — Moonshot cost-override input
+# ---------------------------------------------------------------------------
+
+
+class TestRetryStateObservedModel:
+ """Regression guards for the ``observed_model`` field added to
+ ``_RetryState``. The Moonshot cost override reads this — when a
+ fallback model activates mid-attempt, the requested primary
+ (``state.options.model``) no longer matches what actually ran."""
+
+ def _make_state(self, *, options_model: str | None = "primary/model"):
+ """Build a minimally-valid ``_RetryState``. All the heavy
+ collaborators are ``MagicMock()`` — the field we care about is
+ a plain Optional[str], so the surrounding scaffolding just needs
+ to let the dataclass instantiate."""
+ from .service import _RetryState, _TokenUsage
+
+ options = MagicMock()
+ options.model = options_model
+ return _RetryState(
+ options=options,
+ query_message="",
+ was_compacted=False,
+ use_resume=False,
+ resume_file=None,
+ transcript_msg_count=0,
+ adapter=MagicMock(),
+ transcript_builder=MagicMock(),
+ usage=_TokenUsage(),
+ )
- def test_kimi_cost_recomputed_from_rate_card(self):
- """Kimi K2.6 @ $0.60 input / $2.80 output per MTok. 29564 prompt
- tokens + 78 completion should land at ~$0.018, not $0.09 (Sonnet)."""
- recomputed = _override_cost_for_non_anthropic(
- raw_model="moonshotai/kimi-k2.6",
- sdk_reported_usd=0.089862, # what the SDK CLI reported (Sonnet price)
+ def test_default_is_none(self):
+ state = self._make_state()
+ assert state.observed_model is None
+
+ def test_assigned_from_assistant_message_model(self):
+ """Simulates the population path in ``_run_stream_attempt``:
+ ``observed`` is pulled off the ``AssistantMessage.model`` attr
+ and assigned onto ``state.observed_model`` when it's a non-empty
+ string."""
+ state = self._make_state()
+ # Simulates the inline assignment the generator does on each
+ # AssistantMessage — a non-empty string lands on state.
+ assistant_like = SimpleNamespace(model="anthropic/claude-sonnet-4-6")
+ observed = getattr(assistant_like, "model", None)
+ if isinstance(observed, str) and observed:
+ state.observed_model = observed
+ assert state.observed_model == "anthropic/claude-sonnet-4-6"
+
+ def test_empty_string_model_is_not_assigned(self):
+ """Guard against overwriting a real observed value with an
+ empty-string model (the generator's ``and observed`` check)."""
+ state = self._make_state()
+ state.observed_model = "moonshotai/kimi-k2.6" # seeded from a prior msg
+ assistant_like = SimpleNamespace(model="")
+ observed = getattr(assistant_like, "model", None)
+ if isinstance(observed, str) and observed:
+ state.observed_model = observed
+ assert state.observed_model == "moonshotai/kimi-k2.6"
+
+ def test_missing_model_attr_leaves_observed_untouched(self):
+ state = self._make_state()
+ state.observed_model = "moonshotai/kimi-k2.6"
+ # AssistantMessage may not carry ``.model`` on older SDK rels.
+ assistant_like = SimpleNamespace() # no ``.model`` attr
+ observed = getattr(assistant_like, "model", None)
+ if isinstance(observed, str) and observed:
+ state.observed_model = observed
+ assert state.observed_model == "moonshotai/kimi-k2.6"
+
+
+# ---------------------------------------------------------------------------
+# Moonshot cost-override gate — decision logic at the call site
+# ---------------------------------------------------------------------------
+
+
+class TestMoonshotCostOverrideGate:
+ """Regression guards for the decision logic in
+ ``_run_stream_attempt`` that picks between the CLI-reported cost
+ and the Moonshot rate-card override. The code:
+
+ active_model = state.observed_model or getattr(state.options, "model", None)
+ if _is_moonshot_model(active_model):
+ state.usage.cost_usd = _override_cost_for_moonshot(...)
+ else:
+ state.usage.cost_usd = sdk_msg.total_cost_usd
+
+ is critical-path billing logic — make sure observed_model wins over
+ the requested primary, and Anthropic turns pass through untouched."""
+
+ def _decide_cost(
+ self,
+ *,
+ observed_model: str | None,
+ options_model: str | None,
+ sdk_reported_usd: float,
+ prompt_tokens: int = 0,
+ completion_tokens: int = 0,
+ ) -> float:
+ """Mirror of the real decision block — lets us assert the gate
+ without constructing the whole 1000-line generator."""
+ from .service import _is_moonshot_model, _override_cost_for_moonshot
+
+ active_model = observed_model or options_model
+ if _is_moonshot_model(active_model):
+ return _override_cost_for_moonshot(
+ model=active_model,
+ sdk_reported_usd=sdk_reported_usd,
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ cache_read_tokens=0,
+ cache_creation_tokens=0,
+ )
+ return sdk_reported_usd
+
+ def test_anthropic_turn_passes_sdk_cost_through(self):
+ """Anthropic — the CLI's pricing table is authoritative, so
+ ``state.usage.cost_usd`` is set to ``sdk_msg.total_cost_usd``
+ unchanged."""
+ cost = self._decide_cost(
+ observed_model="anthropic/claude-sonnet-4-6",
+ options_model="anthropic/claude-sonnet-4-6",
+ sdk_reported_usd=0.123,
+ )
+ assert cost == 0.123
+
+ def test_moonshot_turn_uses_rate_card_override(self):
+ """Moonshot — the CLI would silently bill at Sonnet rates, so
+ the override recomputes from the Moonshot rate card."""
+ cost = self._decide_cost(
+ observed_model="moonshotai/kimi-k2.6",
+ options_model="moonshotai/kimi-k2.6",
+ sdk_reported_usd=0.089862, # CLI's Sonnet-priced estimate.
prompt_tokens=29564,
completion_tokens=78,
- cache_read_tokens=0,
- cache_creation_tokens=0,
)
expected = (29564 * 0.60 + 78 * 2.80) / 1_000_000
- assert recomputed == pytest.approx(expected, rel=1e-9)
- # Sanity-check against a hand-computed magnitude.
- assert 0.017 < recomputed < 0.019
-
- def test_anthropic_cost_unchanged(self):
- """Anthropic slugs pass through the SDK-reported value since the
- CLI's pricing table is correct for them."""
- result = _override_cost_for_non_anthropic(
- raw_model="anthropic/claude-sonnet-4.6",
+ assert cost == pytest.approx(expected, rel=1e-9)
+ # Sanity: ~5x cheaper than the CLI's Sonnet-priced number.
+ assert cost < 0.089862 / 4
+
+ def test_observed_model_wins_over_options_primary(self):
+ """The whole point of ``observed_model``: a Moonshot-primary
+ request that fell back to Anthropic must NOT get Moonshot
+ pricing applied. The gate follows the observed model, not the
+ requested primary."""
+ cost = self._decide_cost(
+ observed_model="anthropic/claude-sonnet-4-6",
+ options_model="moonshotai/kimi-k2.6", # what we ASKED for
+ sdk_reported_usd=0.123,
+ prompt_tokens=1000,
+ completion_tokens=100,
+ )
+ # Observed == Anthropic → CLI-reported cost passes through unchanged.
+ assert cost == 0.123
+
+ def test_anthropic_to_moonshot_fallback_uses_override(self):
+ """The inverse: an Anthropic-primary request that fell back to
+ Moonshot must get the Moonshot override applied — the CLI is
+ still billing at Sonnet rates for the fallback response."""
+ cost = self._decide_cost(
+ observed_model="moonshotai/kimi-k2.6",
+ options_model="anthropic/claude-sonnet-4-6",
sdk_reported_usd=0.089862,
prompt_tokens=29564,
completion_tokens=78,
- cache_read_tokens=0,
- cache_creation_tokens=0,
)
- assert result == 0.089862
-
- def test_unknown_non_anthropic_vendor_passes_through(self):
- """A non-Anthropic slug not in the rate card falls back to the
- SDK-reported value — best-effort rather than misleading zero."""
- result = _override_cost_for_non_anthropic(
- raw_model="deepseek/some-new-model",
- sdk_reported_usd=0.05,
- prompt_tokens=10000,
- completion_tokens=500,
- cache_read_tokens=0,
- cache_creation_tokens=0,
- )
- assert result == 0.05
-
- def test_none_model_passes_through(self):
- """Subscription mode / no-model case returns the SDK value."""
- result = _override_cost_for_non_anthropic(
- raw_model=None,
- sdk_reported_usd=0.07,
+ expected = (29564 * 0.60 + 78 * 2.80) / 1_000_000
+ assert cost == pytest.approx(expected, rel=1e-9)
+
+ def test_no_observed_falls_back_to_options_model(self):
+ """First AssistantMessage hasn't arrived yet (or the SDK didn't
+ emit ``.model``) — the gate falls back to the requested primary."""
+ cost = self._decide_cost(
+ observed_model=None,
+ options_model="moonshotai/kimi-k2.6",
+ sdk_reported_usd=0.089862,
prompt_tokens=100,
completion_tokens=10,
- cache_read_tokens=0,
- cache_creation_tokens=0,
- )
- assert result == 0.07
-
- def test_cache_tokens_folded_into_prompt(self):
- """Since the Moonshot endpoints don't report discounted cached-
- input pricing, cache_read/creation tokens are priced at the same
- rate as regular prompt tokens."""
- recomputed = _override_cost_for_non_anthropic(
- raw_model="moonshotai/kimi-k2.6",
- sdk_reported_usd=0.5,
- prompt_tokens=1000,
- completion_tokens=0,
- cache_read_tokens=5000,
- cache_creation_tokens=2000,
)
- expected = (1000 + 5000 + 2000) * 0.60 / 1_000_000
- assert recomputed == pytest.approx(expected, rel=1e-9)
+ expected = (100 * 0.60 + 10 * 2.80) / 1_000_000
+ assert cost == pytest.approx(expected, rel=1e-9)
+
+ def test_both_none_passes_sdk_cost_through(self):
+ """Subscription mode — ``options.model`` may be None and no
+ AssistantMessage has arrived yet. ``None`` is not a Moonshot
+ slug so the SDK number lands unchanged."""
+ cost = self._decide_cost(
+ observed_model=None,
+ options_model=None,
+ sdk_reported_usd=0.05,
+ )
+ assert cost == 0.05
+
+
+# ---------------------------------------------------------------------------
+# Moonshot helper re-exports — keep imports stable for call-site code
+# ---------------------------------------------------------------------------
+
+
+class TestMoonshotHelperReexports:
+ """``sdk/service.py`` imports the Moonshot helpers under local
+ aliases (``_is_moonshot_model``, ``_override_cost_for_moonshot``).
+ Regression guard so a refactor doesn't silently break the import
+ path the hot-loop code relies on."""
+
+ def test_is_moonshot_model_aliased(self):
+ from backend.copilot.moonshot import is_moonshot_model as canonical
+
+ from .service import _is_moonshot_model
+
+ assert _is_moonshot_model is canonical
+
+ def test_override_cost_for_moonshot_aliased(self):
+ from backend.copilot.moonshot import override_cost_usd as canonical
+
+ from .service import _override_cost_for_moonshot
+
+ assert _override_cost_for_moonshot is canonical
diff --git a/autogpt_platform/backend/backend/copilot/service.py b/autogpt_platform/backend/backend/copilot/service.py
index b0399f87e3..061088e788 100644
--- a/autogpt_platform/backend/backend/copilot/service.py
+++ b/autogpt_platform/backend/backend/copilot/service.py
@@ -17,6 +17,7 @@ from langfuse import get_client
from langfuse.openai import (
AsyncOpenAI as LangfuseAsyncOpenAI, # pyright: ignore[reportPrivateImportUsage]
)
+from openai.types.chat import ChatCompletion
from backend.data.db_accessors import chat_db, understanding_db
from backend.data.understanding import (
@@ -34,6 +35,7 @@ from .model import (
update_session_title,
upsert_chat_session,
)
+from .token_tracking import persist_and_record_usage
logger = logging.getLogger(__name__)
@@ -495,20 +497,31 @@ async def _generate_session_title(
message: str,
user_id: str | None = None,
session_id: str | None = None,
-) -> str | None:
+) -> tuple[str | None, ChatCompletion | None]:
"""Generate a concise title for a chat session based on the first message.
+ Returns ``(title, response)``. The caller is responsible for
+ persisting the title AND recording the title call's cost — keeping
+ them as separate concerns in the caller lets a cost-tracking hiccup
+ not lose the title, and lets a title-persist failure still record
+ the cost (we paid for the LLM call either way).
+
Args:
message: The first user message in the session
user_id: User ID for OpenRouter tracing (optional)
session_id: Session ID for OpenRouter tracing (optional)
Returns:
- A short title (3-6 words) or None if generation fails
+ ``(title, response)`` on success; ``(None, None)`` if the LLM
+ call raised. ``response`` is returned even when ``title`` is
+ empty so the caller can still record the (paid-for) cost.
"""
try:
- # Build extra_body for OpenRouter tracing and PostHog analytics
- extra_body: dict[str, Any] = {}
+ # Build extra_body for OpenRouter tracing and PostHog analytics.
+ # ``usage: {"include": True}`` asks OR to embed the real billed
+ # cost into the final usage chunk — matches the baseline path's
+ # ``_OPENROUTER_INCLUDE_USAGE_COST`` pattern, same read path.
+ extra_body: dict[str, Any] = {"usage": {"include": True}}
if user_id:
extra_body["user"] = user_id[:128] # OpenRouter limit
extra_body["posthogDistinctId"] = user_id
@@ -534,18 +547,113 @@ async def _generate_session_title(
max_tokens=20,
extra_body=extra_body,
)
- title = response.choices[0].message.content
- if title:
- # Clean up the title
- title = title.strip().strip("\"'")
- # Limit length
- if len(title) > 50:
- title = title[:47] + "..."
- return title
- return None
except Exception as e:
logger.warning(f"Failed to generate session title: {e}")
- return None
+ return None, None
+
+ # Robust against an empty ``choices`` list OR a choice whose
+ # ``message`` is missing ``content`` (shouldn't happen on the OpenAI
+ # SDK typing, but belt-and-suspenders — the background task would
+ # otherwise die on ``IndexError`` and lose the (paid-for) cost
+ # recording we're about to do below).
+ title: str | None = None
+ if response.choices:
+ msg = response.choices[0].message
+ title = msg.content if msg is not None else None
+ if title:
+ title = title.strip().strip("\"'")
+ if len(title) > 50:
+ title = title[:47] + "..."
+ return title, response
+
+
+def _title_usage_from_response(
+ response: ChatCompletion,
+) -> tuple[int, int, float | None]:
+ """Extract ``(prompt_tokens, completion_tokens, cost_usd)`` from a
+ title-generation chat-completion response.
+
+ Returns zeros / ``None`` for missing fields — the OpenAI SDK's
+ ``CompletionUsage`` doesn't declare OpenRouter's ``cost`` extension,
+ so we read it off ``model_extra`` (pydantic v2 extras container).
+ Absent for non-OR routes; returned as ``None`` in that case.
+ """
+ usage = response.usage
+ if usage is None:
+ return 0, 0, None
+ prompt_tokens = usage.prompt_tokens or 0
+ completion_tokens = usage.completion_tokens or 0
+ extras = usage.model_extra or {}
+ cost_raw = extras.get("cost") if isinstance(extras, dict) else None
+ if isinstance(cost_raw, (int, float)):
+ cost_usd: float | None = float(cost_raw)
+ else:
+ cost_usd = None
+ return prompt_tokens, completion_tokens, cost_usd
+
+
+async def _record_title_generation_cost(
+ *,
+ response: ChatCompletion,
+ user_id: str | None,
+ session_id: str | None,
+) -> None:
+ """Persist the title LLM call's cost to ``PlatformCostLog``.
+
+ Title generation runs in a background task per-session — low cost
+ (~$0.0001 per title) but 100% of sessions pay it. Without this the
+ admin dashboard under-reports total provider spend by the aggregate
+ of those calls. Separate ``block_name="copilot:title"`` so the row
+ is clearly distinguishable from the turn's main ``copilot:SDK`` /
+ ``copilot:baseline`` attributions.
+
+ Invariants enforced by the caller:
+ * ``response`` is a completed ``ChatCompletion`` (the create call
+ didn't raise) — so ``response.usage`` shape is SDK-contractual.
+ * Exceptions are NOT suppressed — the caller runs this AFTER
+ title persistence so a persist failure here doesn't lose the
+ title, and a real DB / Prisma outage surfaces in the caller's
+ single background-task warning handler.
+ """
+ prompt_tokens, completion_tokens, cost_usd = _title_usage_from_response(response)
+
+ # Nothing meaningful to record — skip the DB roundtrip entirely
+ # rather than writing a zero-valued row. Covers the non-OR route
+ # (no ``usage.cost`` field) and the degenerate zero-tokens case.
+ if cost_usd is None and prompt_tokens == 0 and completion_tokens == 0:
+ return
+
+ # Provider label is derived from the configured ``base_url`` (title
+ # LLM uses the shared copilot OpenAI client whose base URL mirrors
+ # ``ChatConfig.base_url``). This lets a deployment that points
+ # title generation at a non-OR endpoint still get the correct
+ # ``provider`` on the cost-log row.
+ provider = (
+ "open_router"
+ if (config.base_url and "openrouter.ai" in config.base_url)
+ else "openai"
+ )
+
+ # Intentionally pass ``session=None``. ``persist_and_record_usage``
+ # would otherwise append a ``Usage`` entry to the live session
+ # object, but this background task holds no reference to the
+ # request-scoped session — we'd have to ``get_chat_session`` +
+ # ``upsert_chat_session`` round-trip the mutation back, and the
+ # turn's main ``persist_and_record_usage`` already owns the session
+ # usage-list mirror for the originating turn. Title cost is
+ # recorded into ``PlatformCostLog`` (admin dashboard) and the
+ # microdollar rate-limit counter — those are the two places that
+ # actually matter for this call.
+ await persist_and_record_usage(
+ session=None,
+ user_id=user_id,
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ log_prefix="[title]",
+ cost_usd=cost_usd,
+ model=config.title_model,
+ provider=provider,
+ )
async def _update_title_async(
@@ -553,15 +661,29 @@ async def _update_title_async(
) -> None:
"""Generate and persist a session title in the background.
- Shared by both the SDK and baseline execution paths.
+ Shared by both the SDK and baseline execution paths. Title
+ persistence and cost recording are run as independent best-effort
+ steps — a failure in one does not cancel the other, so a flaky
+ Prisma call on cost recording never costs us the generated title.
"""
- try:
- title = await _generate_session_title(message, user_id, session_id)
- if title and user_id:
+ title, response = await _generate_session_title(message, user_id, session_id)
+
+ if title and user_id:
+ try:
await update_session_title(session_id, user_id, title, only_if_empty=True)
logger.debug("Generated title for session %s", session_id)
- except Exception as e:
- logger.warning("Failed to update session title for %s: %s", session_id, e)
+ except Exception as e:
+ logger.warning("Failed to persist session title for %s: %s", session_id, e)
+
+ if response is not None:
+ try:
+ await _record_title_generation_cost(
+ response=response, user_id=user_id, session_id=session_id
+ )
+ except Exception as e:
+ logger.warning(
+ "Failed to record title generation cost for %s: %s", session_id, e
+ )
async def assign_user_to_session(
diff --git a/autogpt_platform/backend/backend/copilot/service_unit_test.py b/autogpt_platform/backend/backend/copilot/service_unit_test.py
new file mode 100644
index 0000000000..5463404fc8
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/service_unit_test.py
@@ -0,0 +1,541 @@
+"""Unit tests for title-generation cost tracking helpers.
+
+Covers the new code added in PR #12882:
+ * ``_title_usage_from_response`` — shape-robust OR ``usage.cost`` extraction
+ * ``_record_title_generation_cost`` — provider-label + zero-tokens gate
+ * ``_update_title_async`` — independent title / cost persistence try blocks
+ * ``_generate_session_title`` — tuple return + robustness against empty choices
+
+Mocks ``persist_and_record_usage`` / ``update_session_title`` at the boundary
+where the code under test imports them (``backend.copilot.service.*``).
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from openai.types.chat import ChatCompletion
+from openai.types.chat.chat_completion import Choice
+from openai.types.chat.chat_completion_message import ChatCompletionMessage
+from openai.types.completion_usage import CompletionUsage
+
+from backend.copilot.service import (
+ _generate_session_title,
+ _record_title_generation_cost,
+ _title_usage_from_response,
+ _update_title_async,
+)
+
+
+def _build_completion(
+ *,
+ content: str | None = "Hello Title",
+ usage: CompletionUsage | None = None,
+ choices: list[Choice] | None = None,
+) -> ChatCompletion:
+ if choices is None:
+ msg = ChatCompletionMessage(role="assistant", content=content)
+ choices = [Choice(index=0, message=msg, finish_reason="stop")]
+ return ChatCompletion(
+ id="cmpl-1",
+ choices=choices,
+ created=0,
+ model="anthropic/claude-haiku",
+ object="chat.completion",
+ usage=usage,
+ )
+
+
+def _usage_with_cost(cost: object | None) -> CompletionUsage:
+ """Return a CompletionUsage whose ``model_extra`` carries ``cost``.
+
+ Uses ``model_validate`` so OpenRouter's ``cost`` extension lands in
+ the pydantic ``model_extra`` dict the helper reads from.
+ """
+ payload: dict[str, object] = {
+ "prompt_tokens": 12,
+ "completion_tokens": 3,
+ "total_tokens": 15,
+ }
+ if cost is not None:
+ payload["cost"] = cost
+ return CompletionUsage.model_validate(payload)
+
+
+class TestTitleUsageFromResponse:
+ """``_title_usage_from_response`` returns sensible zeros/Nones when
+ optional fields are absent or of unexpected shape."""
+
+ def test_usage_none_returns_all_zero(self):
+ resp = _build_completion(usage=None)
+ prompt, completion, cost = _title_usage_from_response(resp)
+ assert prompt == 0
+ assert completion == 0
+ assert cost is None
+
+ def test_missing_cost_field_returns_none_cost(self):
+ resp = _build_completion(usage=_usage_with_cost(None))
+ prompt, completion, cost = _title_usage_from_response(resp)
+ assert prompt == 12
+ assert completion == 3
+ assert cost is None
+
+ def test_cost_as_int_is_coerced_to_float(self):
+ resp = _build_completion(usage=_usage_with_cost(2))
+ _, _, cost = _title_usage_from_response(resp)
+ assert isinstance(cost, float)
+ assert cost == 2.0
+
+ def test_cost_as_float_is_returned_as_is(self):
+ resp = _build_completion(usage=_usage_with_cost(0.000123))
+ _, _, cost = _title_usage_from_response(resp)
+ assert cost == pytest.approx(0.000123)
+
+ def test_cost_as_non_numeric_string_returns_none(self):
+ resp = _build_completion(usage=_usage_with_cost("free"))
+ _, _, cost = _title_usage_from_response(resp)
+ assert cost is None
+
+ def test_empty_model_extra_returns_none_cost(self):
+ # ``model_extra`` is empty for non-OR routes where pydantic didn't
+ # receive any extras — prompt/completion still flow through.
+ usage = CompletionUsage(prompt_tokens=5, completion_tokens=2, total_tokens=7)
+ resp = _build_completion(usage=usage)
+ prompt, completion, cost = _title_usage_from_response(resp)
+ assert (prompt, completion, cost) == (5, 2, None)
+
+ def test_zero_prompt_and_completion_tokens(self):
+ usage = CompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
+ resp = _build_completion(usage=usage)
+ prompt, completion, cost = _title_usage_from_response(resp)
+ assert (prompt, completion, cost) == (0, 0, None)
+
+
+class TestRecordTitleGenerationCost:
+ """``_record_title_generation_cost`` persists cost + picks the right
+ provider label and skips the DB roundtrip when nothing's meaningful
+ to record."""
+
+ @pytest.mark.asyncio
+ async def test_openrouter_base_url_uses_open_router_provider(self):
+ resp = _build_completion(usage=_usage_with_cost(0.0002))
+ persist = AsyncMock(return_value=0)
+ with (
+ patch(
+ "backend.copilot.service.persist_and_record_usage",
+ new=persist,
+ ),
+ patch(
+ "backend.copilot.service.config",
+ MagicMock(
+ base_url="https://openrouter.ai/api/v1",
+ title_model="anthropic/claude-haiku",
+ ),
+ ),
+ ):
+ await _record_title_generation_cost(
+ response=resp, user_id="u", session_id="s"
+ )
+ persist.assert_awaited_once()
+ kwargs = persist.await_args.kwargs
+ assert kwargs["provider"] == "open_router"
+ assert kwargs["model"] == "anthropic/claude-haiku"
+ assert kwargs["prompt_tokens"] == 12
+ assert kwargs["completion_tokens"] == 3
+ assert kwargs["cost_usd"] == pytest.approx(0.0002)
+ assert kwargs["log_prefix"] == "[title]"
+ assert kwargs["session"] is None
+
+ @pytest.mark.asyncio
+ async def test_non_openrouter_base_url_uses_openai_provider(self):
+ resp = _build_completion(usage=_usage_with_cost(0.0002))
+ persist = AsyncMock(return_value=0)
+ with (
+ patch(
+ "backend.copilot.service.persist_and_record_usage",
+ new=persist,
+ ),
+ patch(
+ "backend.copilot.service.config",
+ MagicMock(
+ base_url="https://api.openai.com/v1",
+ title_model="gpt-4o-mini",
+ ),
+ ),
+ ):
+ await _record_title_generation_cost(
+ response=resp, user_id="u", session_id="s"
+ )
+ persist.assert_awaited_once()
+ assert persist.await_args.kwargs["provider"] == "openai"
+
+ @pytest.mark.asyncio
+ async def test_empty_base_url_uses_openai_provider(self):
+ resp = _build_completion(usage=_usage_with_cost(0.0001))
+ persist = AsyncMock(return_value=0)
+ with (
+ patch(
+ "backend.copilot.service.persist_and_record_usage",
+ new=persist,
+ ),
+ patch(
+ "backend.copilot.service.config",
+ MagicMock(base_url=None, title_model="gpt-4o-mini"),
+ ),
+ ):
+ await _record_title_generation_cost(
+ response=resp, user_id=None, session_id=None
+ )
+ persist.assert_awaited_once()
+ assert persist.await_args.kwargs["provider"] == "openai"
+
+ @pytest.mark.asyncio
+ async def test_zero_tokens_zero_cost_skips_persist(self):
+ """No cost, no tokens — the early return avoids a worthless
+ ``PlatformCostLog`` row."""
+ usage = CompletionUsage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
+ resp = _build_completion(usage=usage)
+ persist = AsyncMock(return_value=0)
+ with (
+ patch(
+ "backend.copilot.service.persist_and_record_usage",
+ new=persist,
+ ),
+ patch(
+ "backend.copilot.service.config",
+ MagicMock(
+ base_url="https://openrouter.ai/api/v1",
+ title_model="x",
+ ),
+ ),
+ ):
+ await _record_title_generation_cost(
+ response=resp, user_id="u", session_id="s"
+ )
+ persist.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_usage_none_skips_persist(self):
+ """``usage`` absent on the response == provider didn't report —
+ still short-circuits to avoid writing a zero-valued row."""
+ resp = _build_completion(usage=None)
+ persist = AsyncMock(return_value=0)
+ with (
+ patch(
+ "backend.copilot.service.persist_and_record_usage",
+ new=persist,
+ ),
+ patch(
+ "backend.copilot.service.config",
+ MagicMock(
+ base_url="https://openrouter.ai/api/v1",
+ title_model="x",
+ ),
+ ),
+ ):
+ await _record_title_generation_cost(
+ response=resp, user_id="u", session_id="s"
+ )
+ persist.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_tokens_without_cost_still_records(self):
+ """Tokens present but ``cost`` missing (non-OR route) still
+ records a row so token counts are captured — ``cost_usd=None``
+ is accepted by ``persist_and_record_usage``."""
+ usage = CompletionUsage(prompt_tokens=8, completion_tokens=2, total_tokens=10)
+ resp = _build_completion(usage=usage)
+ persist = AsyncMock(return_value=0)
+ with (
+ patch(
+ "backend.copilot.service.persist_and_record_usage",
+ new=persist,
+ ),
+ patch(
+ "backend.copilot.service.config",
+ MagicMock(base_url=None, title_model="m"),
+ ),
+ ):
+ await _record_title_generation_cost(
+ response=resp, user_id="u", session_id="s"
+ )
+ persist.assert_awaited_once()
+ assert persist.await_args.kwargs["cost_usd"] is None
+ assert persist.await_args.kwargs["prompt_tokens"] == 8
+
+
+class TestUpdateTitleAsync:
+ """``_update_title_async`` runs title persistence and cost recording
+ as independent best-effort steps — a failure in one does NOT
+ cancel the other."""
+
+ @pytest.mark.asyncio
+ async def test_title_success_cost_success(self):
+ resp = _build_completion(usage=_usage_with_cost(0.0001))
+ gen = AsyncMock(return_value=("My Title", resp))
+ update = AsyncMock(return_value=True)
+ record = AsyncMock()
+ with (
+ patch(
+ "backend.copilot.service._generate_session_title",
+ new=gen,
+ ),
+ patch(
+ "backend.copilot.service.update_session_title",
+ new=update,
+ ),
+ patch(
+ "backend.copilot.service._record_title_generation_cost",
+ new=record,
+ ),
+ ):
+ await _update_title_async("sess-1", "hello", user_id="u1")
+
+ update.assert_awaited_once_with("sess-1", "u1", "My Title", only_if_empty=True)
+ record.assert_awaited_once()
+ assert record.await_args.kwargs["response"] is resp
+ assert record.await_args.kwargs["user_id"] == "u1"
+ assert record.await_args.kwargs["session_id"] == "sess-1"
+
+ @pytest.mark.asyncio
+ async def test_title_persist_fails_but_cost_still_recorded(self):
+ resp = _build_completion(usage=_usage_with_cost(0.0001))
+ gen = AsyncMock(return_value=("Title", resp))
+ update = AsyncMock(side_effect=RuntimeError("prisma boom"))
+ record = AsyncMock()
+ with (
+ patch(
+ "backend.copilot.service._generate_session_title",
+ new=gen,
+ ),
+ patch(
+ "backend.copilot.service.update_session_title",
+ new=update,
+ ),
+ patch(
+ "backend.copilot.service._record_title_generation_cost",
+ new=record,
+ ),
+ ):
+ # Must NOT raise — persist failure is swallowed.
+ await _update_title_async("sess-2", "msg", user_id="u")
+
+ update.assert_awaited_once()
+ record.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_cost_record_fails_but_title_was_persisted(self):
+ resp = _build_completion(usage=_usage_with_cost(0.0001))
+ gen = AsyncMock(return_value=("Title", resp))
+ update = AsyncMock(return_value=True)
+ record = AsyncMock(side_effect=RuntimeError("cost record boom"))
+ with (
+ patch(
+ "backend.copilot.service._generate_session_title",
+ new=gen,
+ ),
+ patch(
+ "backend.copilot.service.update_session_title",
+ new=update,
+ ),
+ patch(
+ "backend.copilot.service._record_title_generation_cost",
+ new=record,
+ ),
+ ):
+ # Must NOT raise — cost-recording failure is swallowed.
+ await _update_title_async("sess-3", "msg", user_id="u")
+
+ update.assert_awaited_once()
+ record.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_no_user_id_skips_title_persist_but_records_cost(self):
+ """Anonymous sessions skip the user-scoped title write, but we
+ still paid for the LLM call — cost recording runs regardless."""
+ resp = _build_completion(usage=_usage_with_cost(0.0001))
+ gen = AsyncMock(return_value=("Title", resp))
+ update = AsyncMock()
+ record = AsyncMock()
+ with (
+ patch(
+ "backend.copilot.service._generate_session_title",
+ new=gen,
+ ),
+ patch(
+ "backend.copilot.service.update_session_title",
+ new=update,
+ ),
+ patch(
+ "backend.copilot.service._record_title_generation_cost",
+ new=record,
+ ),
+ ):
+ await _update_title_async("sess-4", "msg", user_id=None)
+
+ update.assert_not_awaited()
+ record.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_generation_returns_none_response_skips_cost(self):
+ """``_generate_session_title`` swallows exceptions and returns
+ ``(None, None)`` — no response means no cost to record."""
+ gen = AsyncMock(return_value=(None, None))
+ update = AsyncMock()
+ record = AsyncMock()
+ with (
+ patch(
+ "backend.copilot.service._generate_session_title",
+ new=gen,
+ ),
+ patch(
+ "backend.copilot.service.update_session_title",
+ new=update,
+ ),
+ patch(
+ "backend.copilot.service._record_title_generation_cost",
+ new=record,
+ ),
+ ):
+ await _update_title_async("sess-5", "msg", user_id="u")
+
+ update.assert_not_awaited()
+ record.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_empty_title_with_response_still_records_cost(self):
+ """Title came back empty but we still paid for the LLM call —
+ cost recording runs even though the title write is skipped."""
+ resp = _build_completion(usage=_usage_with_cost(0.0001))
+ gen = AsyncMock(return_value=(None, resp))
+ update = AsyncMock()
+ record = AsyncMock()
+ with (
+ patch(
+ "backend.copilot.service._generate_session_title",
+ new=gen,
+ ),
+ patch(
+ "backend.copilot.service.update_session_title",
+ new=update,
+ ),
+ patch(
+ "backend.copilot.service._record_title_generation_cost",
+ new=record,
+ ),
+ ):
+ await _update_title_async("sess-6", "msg", user_id="u")
+
+ update.assert_not_awaited()
+ record.assert_awaited_once()
+
+
+class TestGenerateSessionTitle:
+ """``_generate_session_title`` returns ``(title, response)`` — the
+ caller owns both the persist and the cost-record decisions."""
+
+ @pytest.mark.asyncio
+ async def test_valid_response_returns_cleaned_title_and_response(self):
+ # Code strips whitespace, then strips ``"'`` — whitespace inside
+ # the quotes survives on purpose (titles like ``My Agent`` read
+ # better than ``MyAgent``). Test keeps the outer quotes + inner
+ # whitespace distinct so the ordering is pinned.
+ resp = _build_completion(content='"Clean Me" ')
+ client = MagicMock()
+ client.chat.completions.create = AsyncMock(return_value=resp)
+ with patch(
+ "backend.copilot.service._get_openai_client",
+ return_value=client,
+ ):
+ title, response = await _generate_session_title(
+ "first message", user_id="u", session_id="s"
+ )
+ assert title == "Clean Me"
+ assert response is resp
+
+ @pytest.mark.asyncio
+ async def test_long_title_truncated_with_ellipsis(self):
+ """Titles >50 chars get truncated to 47 + '...'."""
+ long_title = "A" * 80
+ resp = _build_completion(content=long_title)
+ client = MagicMock()
+ client.chat.completions.create = AsyncMock(return_value=resp)
+ with patch(
+ "backend.copilot.service._get_openai_client",
+ return_value=client,
+ ):
+ title, _ = await _generate_session_title("x", user_id=None)
+ assert title is not None
+ assert len(title) == 50
+ assert title.endswith("...")
+
+ @pytest.mark.asyncio
+ async def test_empty_choices_returns_none_title_with_response(self):
+ """No ``choices`` on the response (shouldn't happen per SDK
+ typing) must not raise IndexError — response is preserved so the
+ caller can still record the paid-for cost."""
+ resp = _build_completion(choices=[])
+ client = MagicMock()
+ client.chat.completions.create = AsyncMock(return_value=resp)
+ with patch(
+ "backend.copilot.service._get_openai_client",
+ return_value=client,
+ ):
+ title, response = await _generate_session_title("x")
+ assert title is None
+ assert response is resp
+
+ @pytest.mark.asyncio
+ async def test_missing_message_returns_none_title(self):
+ """A choice whose ``.message`` is absent produces a None title
+ but the response still lands on the caller."""
+ fake_choice = SimpleNamespace(message=None)
+ fake_response = SimpleNamespace(choices=[fake_choice])
+ client = MagicMock()
+ client.chat.completions.create = AsyncMock(return_value=fake_response)
+ with patch(
+ "backend.copilot.service._get_openai_client",
+ return_value=client,
+ ):
+ title, response = await _generate_session_title("x")
+ assert title is None
+ assert response is fake_response
+
+ @pytest.mark.asyncio
+ async def test_llm_call_raises_returns_none_none(self):
+ """Network / API errors on the create call are swallowed;
+ ``(None, None)`` ensures the caller skips both title and cost
+ without crashing the background task."""
+ client = MagicMock()
+ client.chat.completions.create = AsyncMock(
+ side_effect=RuntimeError("connection reset")
+ )
+ with patch(
+ "backend.copilot.service._get_openai_client",
+ return_value=client,
+ ):
+ title, response = await _generate_session_title("x")
+ assert title is None
+ assert response is None
+
+ @pytest.mark.asyncio
+ async def test_create_receives_usage_include_extra_body(self):
+ """PR adds ``usage: {'include': True}`` so OpenRouter embeds the
+ real billed cost into the final usage chunk."""
+ resp = _build_completion(content="Title")
+ client = MagicMock()
+ client.chat.completions.create = AsyncMock(return_value=resp)
+ with patch(
+ "backend.copilot.service._get_openai_client",
+ return_value=client,
+ ):
+ await _generate_session_title(
+ "hello world", user_id="user-abc", session_id="sess-abc"
+ )
+ client.chat.completions.create.assert_awaited_once()
+ extra_body = client.chat.completions.create.await_args.kwargs["extra_body"]
+ assert extra_body["usage"] == {"include": True}
+ assert extra_body["user"] == "user-abc"
+ assert extra_body["session_id"] == "sess-abc"