AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/copilot): reduce SDK + baseline prompt cache waste (#12866)
Zamil Majdye17e9f13c4c6
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.pyindex 8a26002e25..4e495264c8 100644--- a/autogpt_platform/backend/backend/copilot/baseline/service.py+++ b/autogpt_platform/backend/backend/copilot/baseline/service.py@@ -15,7 +15,7 @@ import reimport shutilimport tempfileimport uuid-from collections.abc import AsyncGenerator, Sequence+from collections.abc import AsyncGenerator, Mapping, Sequencefrom dataclasses import dataclass, fieldfrom functools import partialfrom typing import TYPE_CHECKING, Any, cast@@ -47,7 +47,7 @@ from backend.copilot.pending_messages import (drain_pending_messages,format_pending_as_user_message,)-from backend.copilot.prompting import get_baseline_supplement, get_graphiti_supplement+from backend.copilot.prompting import SHARED_TOOL_NOTES, get_graphiti_supplementfrom backend.copilot.response_model import (StreamBaseResponse,StreamError,@@ -168,12 +168,37 @@ def _extract_usage_cost(usage: CompletionUsage) -> float | None:def _extract_cache_creation_tokens(ptd: PromptTokensDetails) -> int:- """Read Anthropic's ``cache_creation_input_tokens`` off an OpenAI- ``PromptTokensDetails`` — it's a provider-specific extra, not in the- typed model, so we read it via ``model_extra`` rather than- ``getattr``.+ """Return cache-write token count from an OpenAI-compatible+ ``PromptTokensDetails``, handling provider-specific field names and+ SDK-version shape differences.++ Two shapes we care about:++ - **OpenRouter** (our primary baseline provider) streams the cache-write+ count as ``cache_write_tokens``. Newer ``openai-python`` versions+ declare this as a typed attribute on ``PromptTokensDetails``; older+ versions expose it only in ``model_extra``. Verified empirically:+ cold-cache request returns ``cache_write_tokens`` > 0, warm-cache+ request returns ``cached_tokens`` > 0 and ``cache_write_tokens`` = 0.+ - **Direct Anthropic API** uses ``cache_creation_input_tokens`` —+ never a typed attribute on the OpenAI SDK, always lives in+ ``model_extra``.++ Lookup order: typed attr → ``model_extra`` (OpenRouter) → ``model_extra``+ (Anthropic-native). ``getattr`` handles both the typed-attr case+ (newer SDK) and the no-such-attr case (older SDK) — we can't only use+ ``model_extra`` because when the field is typed it's filtered out of+ ``model_extra``, leaving us at 0 on the modern happy path."""- return int((ptd.model_extra or {}).get("cache_creation_input_tokens") or 0)+ typed_val = getattr(ptd, "cache_write_tokens", None)+ if typed_val:+ return int(typed_val)+ extras = ptd.model_extra or {}+ return int(+ extras.get("cache_write_tokens")+ or extras.get("cache_creation_input_tokens")+ or 0+ )async def _prepare_baseline_attachments(@@ -327,6 +352,137 @@ class _BaselineStreamState:# block only appends the *new* assistant text (avoiding duplication of# round-1 text when round-1 entries were cleared from session_messages)._flushed_assistant_text_len: int = 0+ # Memoised system-message dict with cache_control applied. The system+ # prompt is static within a session, so we build it once on the first+ # LLM round and reuse the same dict on subsequent rounds — avoiding+ # an O(N) dict-copy of the growing ``messages`` list on every tool-call+ # iteration. ``None`` means "not yet computed" (or the first message+ # wasn't a system role, so no marking applies).+ cached_system_message: dict[str, Any] | None = None+++def _is_anthropic_model(model: str) -> bool:+ """Return True if *model* routes to Anthropic (native or via OpenRouter).++ Cache-control markers on message content + the ``anthropic-beta`` header+ are Anthropic-specific. OpenAI rejects the unknown ``cache_control``+ field with a 400 ("Extra inputs are not permitted") and Grok / other+ providers behave similarly. OpenRouter strips unknown headers but+ passes through ``cache_control`` on the body regardless of provider —+ which would also fail when OpenRouter routes to a non-Anthropic model.++ Examples that return True:+ - ``anthropic/claude-sonnet-4-6`` (OpenRouter route)+ - ``claude-3-5-sonnet-20241022`` (direct Anthropic API)+ - ``anthropic.claude-3-5-sonnet`` (Bedrock-style)++ False for ``openai/gpt-4o``, ``google/gemini-2.5-pro``, ``xai/grok-4``+ etc.+ """+ lowered = model.lower()+ return "claude" in lowered or lowered.startswith("anthropic")+++def _fresh_ephemeral_cache_control() -> dict[str, str]:+ """Return a FRESH ephemeral ``cache_control`` dict each call.++ The ``ttl`` is sourced from :attr:`ChatConfig.baseline_prompt_cache_ttl`+ (default ``1h``) so the static prefix stays warm across many users'+ requests in the same workspace cache. Anthropic caches are keyed+ per-workspace, so every copilot user reading the same system prompt+ hits the same cached entry.++ Using a shared module-level dict would let any downstream mutation+ (e.g. the OpenAI SDK normalising fields in-place) poison every future+ request's marker. Construction is O(1) so the safety margin is free.+ """+ return {"type": "ephemeral", "ttl": config.baseline_prompt_cache_ttl}+++def _fresh_anthropic_caching_headers() -> dict[str, str]:+ """Return a FRESH ``extra_headers`` dict requesting the Anthropic+ prompt-caching beta.++ Same reasoning as :func:`_fresh_ephemeral_cache_control`: never hand a+ shared module-level dict to third-party SDKs. OpenRouter auto-forwards+ cache_control for Anthropic routes without this header, but passing it+ makes the intent unambiguous on-wire and is a no-op for non-Anthropic+ providers (unknown headers are dropped).+ """+ return {"anthropic-beta": "prompt-caching-2024-07-31"}+++def _mark_tools_with_cache_control(+ tools: Sequence[Mapping[str, Any]],+) -> list[dict[str, Any]]:+ """Return a copy of *tools* with ``cache_control`` on the last entry.++ Marking the last tool is a cache breakpoint that covers the whole tool+ schema block as a cacheable prefix segment. Extracted from+ :func:`_mark_system_message_with_cache_control` so callers can precompute+ the marked tool list once per session — the tool set is static within a+ request and the ~43 dict-copies would otherwise run on every LLM round+ in the tool-call loop.++ **Only call this for Anthropic model routes.** Non-Anthropic providers+ (OpenAI, Grok, Gemini) reject the unknown ``cache_control`` field with+ a 400 schema validation error. Gate via :func:`_is_anthropic_model`.+ """+ cached: list[dict[str, Any]] = [dict(t) for t in tools]+ if cached:+ cached[-1] = {+ **cached[-1],+ "cache_control": _fresh_ephemeral_cache_control(),+ }+ return cached+++def _build_cached_system_message(+ system_message: Mapping[str, Any],+) -> dict[str, Any]:+ """Return a copy of *system_message* with ``cache_control`` applied.++ Anthropic's cache uses prefix-match with up to 4 explicit breakpoints.+ Combined with the last-tool marker this gives two cache segments — the+ system block alone, and system+all-tools — so requests that share only+ the system prefix still get a partial cache hit.++ The system message is rebuilt via spread (``{**original, ...}``) so any+ unknown fields the caller set (e.g. ``name``) survive the transformation.+ Non-Anthropic models silently ignore the markers.++ Returns the original dict (shallow-copied) unchanged when the content+ shape is unsupported (missing / non-string / empty) — callers should+ splice it into the message list as-is in that case.+ """+ sys_copy = dict(system_message)+ sys_content = sys_copy.get("content")+ if isinstance(sys_content, str) and sys_content:+ sys_copy["content"] = [+ {+ "type": "text",+ "text": sys_content,+ "cache_control": _fresh_ephemeral_cache_control(),+ }+ ]+ return sys_copy+++def _mark_system_message_with_cache_control(+ messages: Sequence[Mapping[str, Any]],+) -> list[dict[str, Any]]:+ """Return a copy of *messages* with ``cache_control`` on the system block.++ Thin wrapper around :func:`_build_cached_system_message` that preserves+ the original list shape. Prefer the memoised path in+ ``_baseline_llm_caller`` (which builds the cached system dict once per+ session) for hot-loop callers; this function is retained for call sites+ outside the tool-call loop where per-call copying is acceptable.+ """+ cached_messages: list[dict[str, Any]] = [dict(m) for m in messages]+ if cached_messages and cached_messages[0].get("role") == "system":+ cached_messages[0] = _build_cached_system_message(cached_messages[0])+ return cached_messagesasync def _baseline_llm_caller(@@ -347,28 +503,51 @@ async def _baseline_llm_caller(round_text = ""try:client = _get_openai_client()- typed_messages = cast(list[ChatCompletionMessageParam], messages)- # extra_body `usage.include=true` asks OpenRouter to embed the real- # generation cost into the final usage chunk. Without this we only get- # token counts and have no authoritative cost for rate limiting.- if tools:- typed_tools = cast(list[ChatCompletionToolParam], tools)- response = await client.chat.completions.create(- model=state.model,- messages=typed_messages,- tools=typed_tools,- stream=True,- stream_options={"include_usage": True},- extra_body=_OPENROUTER_INCLUDE_USAGE_COST,- )+ # Cache markers are Anthropic-specific. For OpenAI/Grok/other+ # providers, leaving them on would trigger a 400 ("Extra inputs+ # are not permitted" on cache_control). Tools were precomputed+ # in stream_chat_completion_baseline via _mark_tools_with_cache_control+ # (only when the model was Anthropic), so on non-Anthropic routes+ # tools ship without cache_control on the last entry too.+ #+ # `extra_body` `usage.include=true` asks OpenRouter to embed the real+ # generation cost into the final usage chunk — required by the+ # cost-based rate limiter in routes.py. Separate from the Anthropic+ # caching headers, always sent.+ is_anthropic = _is_anthropic_model(state.model)+ if is_anthropic:+ # Build the cached system dict once per session and splice it in+ # on each round. The full ``messages`` list grows with every+ # tool call, so copying the entire list just to mutate index 0+ # scales with conversation length (sentry flagged this); this+ # splice touches only list slots, not message contents.+ if (+ state.cached_system_message is None+ and messages+ and messages[0].get("role") == "system"+ ):+ state.cached_system_message = _build_cached_system_message(messages[0])+ if state.cached_system_message is not None and messages:+ final_messages = [state.cached_system_message, *messages[1:]]+ else:+ final_messages = messages+ extra_headers = _fresh_anthropic_caching_headers()else:- response = await client.chat.completions.create(- model=state.model,- messages=typed_messages,- stream=True,- stream_options={"include_usage": True},- extra_body=_OPENROUTER_INCLUDE_USAGE_COST,- )+ final_messages = messages+ extra_headers = None+ typed_messages = cast(list[ChatCompletionMessageParam], final_messages)+ create_kwargs: dict[str, Any] = {+ "model": state.model,+ "messages": typed_messages,+ "stream": True,+ "stream_options": {"include_usage": True},+ "extra_body": _OPENROUTER_INCLUDE_USAGE_COST,+ }+ if extra_headers:+ create_kwargs["extra_headers"] = extra_headers+ if tools:+ create_kwargs["tools"] = cast(list[ChatCompletionToolParam], list(tools))+ response = await client.chat.completions.create(**create_kwargs)tool_calls_by_index: dict[int, dict[str, str]] = {}# Iterate under an inner try/finally so early exits (cancel, tool-call@@ -1170,7 +1349,7 @@ async def stream_chat_completion_baseline(graphiti_enabled = await is_enabled_for_user(user_id)graphiti_supplement = get_graphiti_supplement() if graphiti_enabled else ""- system_prompt = base_system_prompt + get_baseline_supplement() + graphiti_supplement+ system_prompt = base_system_prompt + SHARED_TOOL_NOTES + graphiti_supplement# Warm context: pre-load relevant facts from Graphiti on first turn.# Use the pre-drain count so pending messages drained at turn start@@ -1320,6 +1499,18 @@ async def stream_chat_completion_baseline(if permissions is not None:tools = _filter_tools_by_permissions(tools, permissions)+ # Pre-mark cache_control on the last tool schema once per session. The+ # tool set is static within a request, so doing this here (instead of in+ # _baseline_llm_caller) avoids re-copying ~43 tool dicts on every LLM+ # round of the tool-call loop.+ #+ # Only apply to Anthropic routes — OpenAI/Grok/other providers would+ # 400 on the unknown ``cache_control`` field inside tool definitions.+ if _is_anthropic_model(active_model):+ tools = cast(+ list[ChatCompletionToolParam], _mark_tools_with_cache_control(tools)+ )+# Propagate execution context so tool handlers can read session-level flags.set_execution_context(user_id,@@ -1707,6 +1898,8 @@ async def stream_chat_completion_baseline(prompt_tokens=billed_prompt,completion_tokens=state.turn_completion_tokens,total_tokens=billed_prompt + state.turn_completion_tokens,+ cache_read_tokens=state.turn_cache_read_tokens,+ cache_creation_tokens=state.turn_cache_creation_tokens,)yield StreamFinish()diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.pyindex e21618c367..4e70767426 100644--- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py+++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py@@ -13,7 +13,14 @@ from backend.copilot.baseline.service import (_baseline_conversation_updater,_baseline_llm_caller,_BaselineStreamState,+ _build_cached_system_message,_compress_session_messages,+ _extract_cache_creation_tokens,+ _fresh_anthropic_caching_headers,+ _fresh_ephemeral_cache_control,+ _is_anthropic_model,+ _mark_system_message_with_cache_control,+ _mark_tools_with_cache_control,)from backend.copilot.model import ChatMessagefrom backend.copilot.transcript_builder import TranscriptBuilder@@ -605,11 +612,18 @@ def _make_usage_chunk(chunk.usage.model_extra = usage_extrasif cached_tokens is not None or cache_creation_input_tokens is not None:- ptd = MagicMock()- ptd.cached_tokens = cached_tokens or 0- ptd.model_extra = {- "cache_creation_input_tokens": cache_creation_input_tokens or 0- }+ # Build a real ``PromptTokensDetails`` so ``getattr(ptd,+ # "cache_write_tokens", None)`` returns ``None`` on this SDK version+ # (rather than a truthy MagicMock attribute) and the extraction+ # helper's typed-attr vs model_extra fallback resolves correctly.+ from openai.types.completion_usage import PromptTokensDetails++ ptd = PromptTokensDetails.model_validate({"cached_tokens": cached_tokens or 0})+ if cache_creation_input_tokens is not None:+ if ptd.model_extra is None:+ object.__setattr__(ptd, "__pydantic_extra__", {})+ assert ptd.model_extra is not None+ ptd.model_extra["cache_creation_input_tokens"] = cache_creation_input_tokenschunk.usage.prompt_tokens_details = ptdelse:chunk.usage.prompt_tokens_details = None@@ -1209,3 +1223,288 @@ class TestMidLoopPendingFlushOrdering:assert assistant_msgs[1].tool_calls is None# Crucially: only 2 assistant messages, not 3 (no duplicate)assert len(assistant_msgs) == 2+++class TestApplyPromptCacheMarkers:+ """Tests for _apply_prompt_cache_markers — Anthropic ephemeral+ cache_control markers on baseline OpenRouter requests."""++ def test_system_message_converted_to_content_blocks(self):+ messages = [+ {"role": "system", "content": "You are helpful."},+ {"role": "user", "content": "hello"},+ ]++ cached_messages = _mark_system_message_with_cache_control(messages)++ assert cached_messages[0]["role"] == "system"+ assert cached_messages[0]["content"] == [+ {+ "type": "text",+ "text": "You are helpful.",+ "cache_control": {"type": "ephemeral", "ttl": "1h"},+ }+ ]+ # User message must be untouched.+ assert cached_messages[1] == {"role": "user", "content": "hello"}++ def test_system_message_preserves_unknown_fields(self):+ # Future-proofing: a system message with extra keys (e.g. "name") must+ # keep them after the content-blocks conversion.+ messages = [+ {"role": "system", "content": "sys", "name": "developer"},+ ]++ cached_messages = _mark_system_message_with_cache_control(messages)++ assert cached_messages[0]["name"] == "developer"+ assert cached_messages[0]["role"] == "system"++ def test_last_tool_gets_cache_control(self):+ tools = [+ {"type": "function", "function": {"name": "a"}},+ {"type": "function", "function": {"name": "b"}},+ ]++ cached_tools = _mark_tools_with_cache_control(tools)++ assert "cache_control" not in cached_tools[0]+ assert cached_tools[-1]["cache_control"] == {+ "type": "ephemeral",+ "ttl": "1h",+ }+ # Last tool's other fields preserved.+ assert cached_tools[-1]["function"] == {"name": "b"}++ def test_does_not_mutate_input(self):+ messages = [{"role": "system", "content": "sys"}]+ tools = [{"type": "function", "function": {"name": "a"}}]++ _mark_system_message_with_cache_control(messages)+ _mark_tools_with_cache_control(tools)++ assert messages == [{"role": "system", "content": "sys"}]+ assert tools == [{"type": "function", "function": {"name": "a"}}]++ def test_no_system_message_safe(self):+ messages = [{"role": "user", "content": "hi"}]+ cached_messages = _mark_system_message_with_cache_control(messages)+ assert cached_messages == messages++ def test_empty_tools_safe(self):+ assert _mark_tools_with_cache_control([]) == []++ def test_non_string_system_content_left_untouched(self):+ # If the content is already a list of blocks (e.g. caller pre-marked),+ # the helper must not overwrite it.+ pre_marked = [+ {+ "type": "text",+ "text": "sys",+ "cache_control": {"type": "ephemeral", "ttl": "1h"},+ }+ ]+ messages = [{"role": "system", "content": pre_marked}]+ cached_messages = _mark_system_message_with_cache_control(messages)+ assert cached_messages[0]["content"] == pre_marked++ def test_is_anthropic_model_matches_claude_and_anthropic_prefix(self):+ assert _is_anthropic_model("anthropic/claude-sonnet-4-6")+ assert _is_anthropic_model("claude-3-5-sonnet-20241022")+ assert _is_anthropic_model("anthropic.claude-3-5-sonnet-20241022-v2:0")+ assert _is_anthropic_model("ANTHROPIC/Claude-Opus") # case insensitive++ def test_is_anthropic_model_rejects_other_providers(self):+ assert not _is_anthropic_model("openai/gpt-4o")+ assert not _is_anthropic_model("openai/gpt-5")+ assert not _is_anthropic_model("google/gemini-2.5-pro")+ assert not _is_anthropic_model("xai/grok-4")+ assert not _is_anthropic_model("meta-llama/llama-3.3-70b-instruct")++ def test_cache_control_uses_configured_ttl(self, monkeypatch):+ """TTL comes from ChatConfig.baseline_prompt_cache_ttl — defaults+ to 1h so the static prefix (system + tools) stays warm across+ workspace users past the 5-min default window."""+ from backend.copilot.baseline import service as bsvc++ assert bsvc.config.baseline_prompt_cache_ttl == "1h"+ cc = bsvc._fresh_ephemeral_cache_control()+ assert cc == {"type": "ephemeral", "ttl": "1h"}+ monkeypatch.setattr(bsvc.config, "baseline_prompt_cache_ttl", "5m")+ assert bsvc._fresh_ephemeral_cache_control() == {+ "type": "ephemeral",+ "ttl": "5m",+ }++ def test_fresh_helpers_return_distinct_objects(self):+ """Regression guard: the `_fresh_*` helpers must return a NEW dict+ on every call. A future refactor returning a module-level constant+ would silently reintroduce the shared-mutable-state bug flagged+ during earlier review cycles."""+ assert _fresh_ephemeral_cache_control() is not _fresh_ephemeral_cache_control()+ assert (+ _fresh_anthropic_caching_headers() is not _fresh_anthropic_caching_headers()+ )++ def test_extract_cache_creation_tokens_openrouter_typed_attr(self):+ """Newer ``openai-python`` declares ``cache_write_tokens`` as a+ typed attribute on ``PromptTokensDetails`` — it no longer lands in+ ``model_extra``. Verified empirically against the production+ openai==1.113 installed in this venv: OpenRouter streaming+ response populates ``ptd.cache_write_tokens`` directly while+ ``ptd.model_extra`` is ``{}``.+ """+ from openai.types.completion_usage import PromptTokensDetails++ ptd = PromptTokensDetails.model_validate(+ {+ "audio_tokens": 0,+ "cached_tokens": 0,+ "cache_write_tokens": 4432,+ "video_tokens": 0,+ }+ )+ assert getattr(ptd, "cache_write_tokens", None) == 4432+ assert _extract_cache_creation_tokens(ptd) == 4432++ def test_extract_cache_creation_tokens_openrouter_model_extra(self):+ """Older SDKs that don't yet declare ``cache_write_tokens`` as a+ typed field leave it in ``model_extra`` — the helper must still+ find it there."""+ from openai.types.completion_usage import PromptTokensDetails++ ptd = PromptTokensDetails.model_validate({"cached_tokens": 0})+ # Force the value into model_extra (simulates the old SDK shape+ # where the field wasn't typed yet).+ if ptd.model_extra is None:+ # Pydantic v2 sometimes exposes __pydantic_extra__ as None when+ # extras are disabled; initialise to a dict to mutate safely.+ object.__setattr__(ptd, "__pydantic_extra__", {})+ assert ptd.model_extra is not None+ ptd.model_extra["cache_write_tokens"] = 7777+ assert _extract_cache_creation_tokens(ptd) == 7777++ def test_extract_cache_creation_tokens_anthropic_native_field(self):+ """Direct Anthropic API uses ``cache_creation_input_tokens`` —+ falls through as the final path when neither+ ``cache_write_tokens`` typed attr nor model_extra entry exists."""+ from openai.types.completion_usage import PromptTokensDetails++ ptd = PromptTokensDetails.model_validate({"cached_tokens": 0})+ if ptd.model_extra is None:+ object.__setattr__(ptd, "__pydantic_extra__", {})+ assert ptd.model_extra is not None+ ptd.model_extra["cache_creation_input_tokens"] = 2048+ assert _extract_cache_creation_tokens(ptd) == 2048++ def test_extract_cache_creation_tokens_absent(self):+ """Neither provider field present → 0 (non-Anthropic routes or+ cache-miss responses)."""+ from openai.types.completion_usage import PromptTokensDetails++ ptd = PromptTokensDetails.model_validate({"cached_tokens": 0})+ assert _extract_cache_creation_tokens(ptd) == 0++ def test_build_cached_system_message_applies_cache_control(self):+ """The single-message helper wraps the string content in a text block+ with an ephemeral cache_control marker."""+ out = _build_cached_system_message({"role": "system", "content": "hi"})+ assert out["role"] == "system"+ assert out["content"] == [+ {+ "type": "text",+ "text": "hi",+ "cache_control": {"type": "ephemeral", "ttl": "1h"},+ }+ ]++ def test_build_cached_system_message_preserves_extra_fields(self):+ """Unknown keys (e.g. ``name``) survive the transformation."""+ out = _build_cached_system_message(+ {"role": "system", "content": "sys", "name": "dev"}+ )+ assert out["name"] == "dev"+ assert out["role"] == "system"++ def test_build_cached_system_message_non_string_passthrough(self):+ """Pre-marked list content is returned as-is (shallow-copied)."""+ pre_marked = [+ {+ "type": "text",+ "text": "sys",+ "cache_control": {"type": "ephemeral", "ttl": "1h"},+ }+ ]+ out = _build_cached_system_message({"role": "system", "content": pre_marked})+ assert out["content"] is pre_marked++ @pytest.mark.asyncio+ async def test_baseline_llm_caller_memoises_cached_system_message(self):+ """The cached system dict is built once and reused across rounds.++ Guards against the perf regression where the entire (growing)+ ``messages`` list was copied on every tool-call iteration just to+ mark the static system prompt.+ """+ state = _BaselineStreamState(model="anthropic/claude-sonnet-4")+ chunk = _make_usage_chunk(prompt_tokens=10, completion_tokens=5)++ mock_client = MagicMock()+ mock_client.chat.completions.create = AsyncMock(+ side_effect=[_make_stream_mock(chunk), _make_stream_mock(chunk)]+ )++ messages: list[dict] = [+ {"role": "system", "content": "You are helpful."},+ {"role": "user", "content": "hi"},+ ]+ with patch(+ "backend.copilot.baseline.service._get_openai_client",+ return_value=mock_client,+ ):+ await _baseline_llm_caller(messages=messages, tools=[], state=state)+ first_cached = state.cached_system_message+ assert first_cached is not None+ # Simulate the tool-call loop growing ``messages`` between rounds.+ messages.append({"role": "assistant", "content": "ok"})+ messages.append({"role": "user", "content": "follow up"})+ await _baseline_llm_caller(messages=messages, tools=[], state=state)++ # Same dict instance reused — not rebuilt per round.+ assert state.cached_system_message is first_cached++ # Second call's first message is the memoised system dict (not a new copy).+ second_call_messages = mock_client.chat.completions.create.call_args_list[1][1][+ "messages"+ ]+ assert second_call_messages[0] is first_cached+ # And the tail messages were spliced in, not re-copied.+ assert second_call_messages[1] is messages[1]+ assert second_call_messages[-1] is messages[-1]++ @pytest.mark.asyncio+ async def test_baseline_llm_caller_skips_memoisation_for_non_anthropic(self):+ """Non-Anthropic routes pass messages through unmodified — no cache+ dict is built, no list splicing happens."""+ state = _BaselineStreamState(model="openai/gpt-4o")+ chunk = _make_usage_chunk(prompt_tokens=10, completion_tokens=5)++ mock_client = MagicMock()+ mock_client.chat.completions.create = AsyncMock(+ return_value=_make_stream_mock(chunk)+ )++ messages: list[dict] = [+ {"role": "system", "content": "sys"},+ {"role": "user", "content": "hi"},+ ]+ with patch(+ "backend.copilot.baseline.service._get_openai_client",+ return_value=mock_client,+ ):+ await _baseline_llm_caller(messages=messages, tools=[], state=state)++ assert state.cached_system_message is None+ # The exact same list object reaches the provider (no copy needed).+ call_messages = mock_client.chat.completions.create.call_args[1]["messages"]+ assert call_messages is messagesdiff --git a/autogpt_platform/backend/backend/copilot/config.py b/autogpt_platform/backend/backend/copilot/config.pyindex 3277854172..1080921fd8 100644--- a/autogpt_platform/backend/backend/copilot/config.py+++ b/autogpt_platform/backend/backend/copilot/config.py@@ -225,6 +225,18 @@ class ChatConfig(BaseSettings):"from the prefix. Set to False to fall back to passing the system ""prompt as a raw string.",)+ baseline_prompt_cache_ttl: str = Field(+ default="1h",+ description="TTL for the ephemeral prompt-cache markers on the baseline "+ "OpenRouter path. Anthropic supports only `5m` (default, 1.25x input "+ "price for the write) or `1h` (2x input price for the write). 1h is "+ "strictly cheaper overall when the static prefix gets >7 reads per "+ "write-window; since the system prompt + tools array is identical "+ "across all users in our workspace, 1h is the default so cross-user "+ "reads amortise the higher write cost. Anthropic has no longer "+ "(24h, permanent) TTL option — see "+ "https://platform.claude.com/docs/en/build-with-claude/prompt-caching.",+ )claude_agent_cli_path: str | None = Field(default=None,description="Optional explicit path to a Claude Code CLI binary. "diff --git a/autogpt_platform/backend/backend/copilot/prompting.py b/autogpt_platform/backend/backend/copilot/prompting.pyindex 2f52bd460d..399d31c1cc 100644--- a/autogpt_platform/backend/backend/copilot/prompting.py+++ b/autogpt_platform/backend/backend/copilot/prompting.py@@ -8,10 +8,12 @@ handling the distinction between:from functools import cache-from backend.copilot.tools import TOOL_REGISTRY--# Shared technical notes that apply to both SDK and baseline modes-_SHARED_TOOL_NOTES = """\+# Workflow rules appended to the system prompt on every copilot turn+# (baseline appends directly; SDK appends via the storage-supplement+# template). These are cross-tool rules (file sharing, @@agptfile: refs,+# tool-discovery priority, sub-agent etiquette) that don't belong on any+# individual tool schema.+SHARED_TOOL_NOTES = """\### Sharing filesAfter `write_workspace_file`, embed the `download_url` in Markdown:@@ -261,7 +263,7 @@ When a tool output contains `<tool-output-truncated workspace_path="...">`, thefull output is in workspace storage (NOT on the local filesystem). To access it:- Use `read_workspace_file(path="...", offset=..., length=50000)` for reading sections.- To process in the sandbox, use `read_workspace_file(path="...", save_to_path="{working_dir}/file.json")` first, then use `bash_exec` on the local copy.-{_SHARED_TOOL_NOTES}{extra_notes}"""+{SHARED_TOOL_NOTES}{extra_notes}"""# Pre-built supplements for common environments@@ -312,35 +314,6 @@ def _get_cloud_sandbox_supplement() -> str:)-def _generate_tool_documentation() -> str:- """Auto-generate tool documentation from TOOL_REGISTRY.-- NOTE: This is ONLY used in baseline mode (direct OpenAI API).- SDK mode doesn't need it since Claude gets tool schemas automatically.-- This generates a complete list of available tools with their descriptions,- ensuring the documentation stays in sync with the actual tool implementations.- All workflow guidance is now embedded in individual tool descriptions.-- Only documents tools that are available in the current environment- (checked via tool.is_available property).- """- docs = "\n## AVAILABLE TOOLS\n\n"-- # Sort tools alphabetically for consistent output- # Filter by is_available to match get_available_tools() behavior- for name in sorted(TOOL_REGISTRY.keys()):- tool = TOOL_REGISTRY[name]- if not tool.is_available:- continue- schema = tool.as_openai_tool()- desc = schema["function"].get("description", "No description available")- # Format as bullet list with tool name in code style- docs += f"- **`{name}`**: {desc}\n"-- return docs--_USER_FOLLOW_UP_NOTE = """# `<user_follow_up>` blocks in tool output@@ -438,17 +411,3 @@ You have access to persistent temporal memory tools that remember facts across s- group_id is handled automatically by the system — never set it yourself.- When storing, be specific about operational rules and instructions (e.g., "CC Sarah on client communications" not just "Sarah is the assistant")."""---def get_baseline_supplement() -> str:- """Get the supplement for baseline mode (direct OpenAI API).-- Baseline mode INCLUDES auto-generated tool documentation because the- direct API doesn't automatically provide tool schemas to Claude.- Also includes shared technical notes (but NOT SDK-specific environment details).-- Returns:- The supplement string to append to the system prompt- """- tool_docs = _generate_tool_documentation()- return tool_docs + _SHARED_TOOL_NOTESdiff --git a/autogpt_platform/backend/backend/copilot/sdk/sdk_compat_test.py b/autogpt_platform/backend/backend/copilot/sdk/sdk_compat_test.pyindex 5d132aa94d..7cf8af3396 100644--- a/autogpt_platform/backend/backend/copilot/sdk/sdk_compat_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/sdk_compat_test.py@@ -94,21 +94,23 @@ def test_agent_options_accepts_required_fields():def test_agent_options_accepts_system_prompt_preset_with_exclude_dynamic_sections():"""Verify ClaudeAgentOptions accepts the exact preset dict _build_system_prompt_value produces.- The production code always includes ``exclude_dynamic_sections=True`` in the preset- dict. This compat test mirrors that exact shape so any SDK version that starts- rejecting unknown keys will be caught here rather than at runtime.+ The Turn 1 (non-resume) code path includes ``exclude_dynamic_sections=True`` in+ the preset dict for cross-user caching. This compat test mirrors that exact+ shape so any SDK version that starts rejecting unknown keys will be caught+ here rather than at runtime."""from claude_agent_sdk import ClaudeAgentOptionsfrom claude_agent_sdk.types import SystemPromptPresetfrom .service import _build_system_prompt_value- # Call the production helper directly so this test is tied to the real- # dict shape rather than a hand-rolled copy.preset = _build_system_prompt_value("custom system prompt", cross_user_cache=True)assert isinstance(preset, dict), "_build_system_prompt_value must return a dict when caching is on"+ assert preset.get("exclude_dynamic_sections") is True, (+ "Turn 1 must strip dynamic sections to keep the prefix cacheable " "cross-user"+ )sdk_preset = cast(SystemPromptPreset, preset)opts = ClaudeAgentOptions(system_prompt=sdk_preset)@@ -116,8 +118,9 @@ def test_agent_options_accepts_system_prompt_preset_with_exclude_dynamic_sectiondef test_build_system_prompt_value_returns_plain_string_when_cross_user_cache_off():- """When cross_user_cache=False (e.g. on --resume turns), the helper must return- a plain string so the preset+resume crash is avoided."""+ """When cross_user_cache=False (feature flag disabled globally), the+ helper returns a plain string; the CLI will receive --system-prompt+ (replace-mode) and skip the preset entirely."""from .service import _build_system_prompt_valueresult = _build_system_prompt_value("my prompt", cross_user_cache=False)@@ -262,6 +265,12 @@ _KNOWN_GOOD_BUNDLED_CLI_VERSIONS: frozenset[str] = frozenset("2.1.97", # claude-agent-sdk 0.1.58 -- OpenRouter-safe only with# CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 (injected by# build_sdk_env() in env.py).+ "2.1.116", # claude-agent-sdk 0.1.64 -- first bundled version that+ # fixes the --resume + excludeDynamicSections=True crash+ # (introduced in 2.1.98), unlocking cross-user prompt+ # cache reads on every resumed SDK turn. Still requires+ # CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1. Verified+ # OpenRouter-safe via cli_openrouter_compat_test.py.})diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.pyindex e4f29a2b65..8fe8aa12df 100644--- a/autogpt_platform/backend/backend/copilot/sdk/service.py+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py@@ -836,16 +836,25 @@ def _is_fallback_stderr(line: str) -> bool:def _build_system_prompt_value(system_prompt: str,+ *,cross_user_cache: bool,) -> str | SystemPromptPreset:"""Build the ``system_prompt`` argument for :class:`ClaudeAgentOptions`.When *cross_user_cache* is enabled, returns a :class:`SystemPromptPreset`- dict so the Claude Code default prompt becomes a cacheable prefix shared- across all users; our custom *system_prompt* is appended after it.+ with ``exclude_dynamic_sections=True`` so every turn — Turn 1 *and*+ resumed turns — shares the same static prefix and hits the cross-user+ prompt cache. Our custom *system_prompt* is appended after the preset.++ Requires CLI ≥ 2.1.98 (older CLIs crash when ``excludeDynamicSections``+ is combined with ``--resume``). The SDK bundles CLI 2.1.116 at+ ``claude-agent-sdk >= 0.1.64``, so the pin in ``pyproject.toml`` is+ the single source of truth — no external install needed.- When disabled (or if the SDK is too old to support ``SystemPromptPreset``),- the raw *system_prompt* string is returned unchanged.+ When *cross_user_cache* is disabled, the raw *system_prompt* string is+ returned. Note this causes the CLI to REPLACE its built-in prompt via+ ``--system-prompt`` (vs ``--append-system-prompt`` for the preset),+ which loses Claude Code's default prompt and its cache markers entirely.An empty *system_prompt* is accepted: the preset dict will have``append: ""`` which the SDK treats as no custom suffix.@@ -3036,15 +3045,17 @@ async def stream_chat_completion_sdk(sid,)- # Use SystemPromptPreset for cross-user prompt caching.- # WORKAROUND: CLI 2.1.97 (sdk 0.1.58) exits code 1 when- # excludeDynamicSections=True is in the initialize request AND- # --resume is active. Disable the preset on resumed turns.- # Turn 1 still gets the preset (no --resume).- _cross_user = config.claude_agent_cross_user_prompt_cache and not use_resume+ # Use SystemPromptPreset with exclude_dynamic_sections=True on+ # every turn — including resumed ones — so all turns share the+ # same static prefix and hit the cross-user prompt cache.+ #+ # Requires CLI ≥ 2.1.98 (older CLIs crash when excludeDynamicSections+ # is combined with --resume). claude-agent-sdk >= 0.1.64 bundles+ # CLI 2.1.116, so the pin in pyproject.toml is sufficient — no+ # external install or env-var override needed.system_prompt_value = _build_system_prompt_value(system_prompt,- cross_user_cache=_cross_user,+ cross_user_cache=config.claude_agent_cross_user_prompt_cache,)sdk_options_kwargs: dict[str, Any] = {@@ -3401,15 +3412,12 @@ async def stream_chat_completion_sdk(# fail with "Session ID already in use".sdk_options_kwargs_retry.pop("resume", None)sdk_options_kwargs_retry.pop("session_id", None)- # Recompute system_prompt for retry — ctx.use_resume may have- # changed (context reduction enabled --resume). CLI 2.1.97- # crashes when excludeDynamicSections=True is combined with- # --resume, so disable the cross-user preset on resumed turns.- _cross_user_retry = (- config.claude_agent_cross_user_prompt_cache and not ctx.use_resume- )+ # Recompute system_prompt for retry — the preset is safe on+ # every turn (requires CLI ≥ 2.1.98, installed in the Docker+ # image and configured via CHAT_CLAUDE_AGENT_CLI_PATH).sdk_options_kwargs_retry["system_prompt"] = _build_system_prompt_value(- system_prompt, cross_user_cache=_cross_user_retry+ system_prompt,+ cross_user_cache=config.claude_agent_cross_user_prompt_cache,)state.options = ClaudeAgentOptions(**sdk_options_kwargs_retry) # type: ignore[arg-type] # dynamic kwargs# Retry intentionally omits prior_messages (transcript+gap context) anddiff --git a/autogpt_platform/backend/backend/copilot/sdk/service_test.py b/autogpt_platform/backend/backend/copilot/sdk/service_test.pyindex f7ebe766f6..d47f67252a 100644--- a/autogpt_platform/backend/backend/copilot/sdk/service_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/service_test.py@@ -177,70 +177,18 @@ class TestPromptSupplement:assert "## Tool notes" in local_supplementassert "## Tool notes" in e2b_supplement- def test_baseline_supplement_includes_tool_docs(self):- """Baseline mode MUST include tool documentation (direct API needs it)."""- from backend.copilot.prompting import get_baseline_supplement-- supplement = get_baseline_supplement()-- # MUST have tool list section- assert "## AVAILABLE TOOLS" in supplement-- # Should NOT have environment-specific notes (SDK-only)- assert "## Tool notes" not in supplement-- def test_baseline_supplement_includes_key_tools(self):- """Baseline supplement should document all essential tools."""- from backend.copilot.prompting import get_baseline_supplement- from backend.copilot.tools import TOOL_REGISTRY-- docs = get_baseline_supplement()-- # Core agent workflow tools (always available)- assert "`create_agent`" in docs- assert "`run_agent`" in docs- assert "`find_library_agent`" in docs- assert "`edit_agent`" in docs-- # MCP integration (always available)- assert "`run_mcp_tool`" in docs-- # Folder management (always available)- assert "`create_folder`" in docs-- # Browser tools only if available (Playwright may not be installed in CI)- if (- TOOL_REGISTRY.get("browser_navigate")- and TOOL_REGISTRY["browser_navigate"].is_available- ):- assert "`browser_navigate`" in docs-- def test_baseline_supplement_includes_workflows(self):- """Baseline supplement should include workflow guidance in tool descriptions."""- from backend.copilot.prompting import get_baseline_supplement-- docs = get_baseline_supplement()-- # Workflows are now in individual tool descriptions (not separate sections)- # Check that key workflow concepts appear in tool descriptions- assert "agent_json" in docs or "find_block" in docs- assert "run_mcp_tool" in docs-- def test_baseline_supplement_completeness(self):- """All available tools from TOOL_REGISTRY should appear in baseline supplement."""- from backend.copilot.prompting import get_baseline_supplement- from backend.copilot.tools import TOOL_REGISTRY-- docs = get_baseline_supplement()-- # Verify each available registered tool is documented- # (matches _generate_tool_documentation which filters by is_available)- for tool_name, tool in TOOL_REGISTRY.items():- if not tool.is_available:- continue- assert (- f"`{tool_name}`" in docs- ), f"Tool '{tool_name}' missing from baseline supplement"+ def test_baseline_supplement_has_shared_notes_no_tool_list(self):+ """Baseline now relies on the OpenAI tools array for schemas and only+ appends SHARED_TOOL_NOTES (workflow rules not present in any schema).+ The old auto-generated ``## AVAILABLE TOOLS`` list is gone — it was+ ~4.3K tokens of pure duplication of the tools array."""+ from backend.copilot.prompting import SHARED_TOOL_NOTES++ assert "## AVAILABLE TOOLS" not in SHARED_TOOL_NOTES+ # Keep the high-value workflow rules that are NOT in any tool schema.+ assert "@@agptfile:" in SHARED_TOOL_NOTES+ assert "Tool Discovery Priority" in SHARED_TOOL_NOTES+ assert "run_sub_session" in SHARED_TOOL_NOTESdef test_pause_task_scheduled_before_transcript_upload(self):"""Pause is scheduled as a background task before transcript upload begins.@@ -284,21 +232,6 @@ class TestPromptSupplement:# concurrently during upload's first yield. The ordering guarantee is# that create_task is CALLED before upload is AWAITED (see source order).- def test_baseline_supplement_no_duplicate_tools(self):- """No tool should appear multiple times in baseline supplement."""- from backend.copilot.prompting import get_baseline_supplement- from backend.copilot.tools import TOOL_REGISTRY-- docs = get_baseline_supplement()-- # Count occurrences of each available tool in the entire supplement- for tool_name, tool in TOOL_REGISTRY.items():- if not tool.is_available:- continue- # Count how many times this tool appears as a bullet point- count = docs.count(f"- **`{tool_name}`**")- assert count == 1, f"Tool '{tool_name}' appears {count} times (should be 1)"-# ---------------------------------------------------------------------------# _cleanup_sdk_tool_results — orchestration + rate-limiting@@ -700,6 +633,17 @@ class TestSystemPromptPreset:assert result["append"] == ""assert result["exclude_dynamic_sections"] is True+ def test_resume_and_fresh_share_the_same_static_prefix(self):+ """Every turn (fresh + --resume) must emit the same preset dict+ so the cross-user cache prefix match works on all turns. This+ relies on CLI ≥ 2.1.98 (installed in the Docker image); older+ CLIs would crash on --resume + excludeDynamicSections=True."""+ fresh = _build_system_prompt_value("sys", cross_user_cache=True)+ resumed = _build_system_prompt_value("sys", cross_user_cache=True)+ assert fresh == resumed+ assert isinstance(fresh, dict)+ assert fresh.get("exclude_dynamic_sections") is True+def test_default_config_is_enabled(self, _clean_config_env):"""The default value for claude_agent_cross_user_prompt_cache is True."""cfg = cfg_mod.ChatConfig(diff --git a/autogpt_platform/backend/poetry.lock b/autogpt_platform/backend/poetry.lockindex 03c93c286a..a9aafef96f 100644--- a/autogpt_platform/backend/poetry.lock+++ b/autogpt_platform/backend/poetry.lock@@ -1,4 +1,4 @@-# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand.+# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand.[[package]]name = "agentmail"@@ -909,18 +909,18 @@ files = [[[package]]name = "claude-agent-sdk"-version = "0.1.58"+version = "0.1.64"description = "Python SDK for Claude Code"optional = falsepython-versions = ">=3.10"groups = ["main"]files = [- {file = "claude_agent_sdk-0.1.58-py3-none-macosx_11_0_arm64.whl", hash = "sha256:69197950809754c4f06bba8261f2d99c3f9605b6cc1c13d3409d0eb82fb4ee64"},- {file = "claude_agent_sdk-0.1.58-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:75d60883fc5e2070bccd8d9b19505fe16af8e049120c03821e9dc8c826cca434"},- {file = "claude_agent_sdk-0.1.58-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:7bf4eb0f00ec944a7b63eb94788f120dfb0460c348a525235c7d6641805acc1d"},- {file = "claude_agent_sdk-0.1.58-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:650d298a3d3c0dcdde4b5f1dbf52f472ff0b0ec82987b27ffa2a4e0e72928408"},- {file = "claude_agent_sdk-0.1.58-py3-none-win_amd64.whl", hash = "sha256:2c2130a7ffe06ed4f88d56b217a5091c91c9bcb1a69cfd94d5dcf0d2946d8c55"},- {file = "claude_agent_sdk-0.1.58.tar.gz", hash = "sha256:77bee8fd60be033cb870def46c2ab1625a512fa8a3de4ff8d766664ffb16d6a6"},+ {file = "claude_agent_sdk-0.1.64-py3-none-macosx_11_0_arm64.whl", hash = "sha256:4cf47a9e40c0a683a05afff4fac1e3d5ea7965b1e9f72a8e266c8d2efbf65904"},+ {file = "claude_agent_sdk-0.1.64-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:7fe765c6482c74bc6b0b4491ad3bddd1349c25f4cdf4483191c68ea9c1336825"},+ {file = "claude_agent_sdk-0.1.64-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:605eebf46e7590e4f878572c2743954fba3f3530dfd99e10ff3b8b41a9fee757"},+ {file = "claude_agent_sdk-0.1.64-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:bbb1373ee0b4494e2db24aa10d312d22b86895b4b8f18eb5b58f99f14d827237"},+ {file = "claude_agent_sdk-0.1.64-py3-none-win_amd64.whl", hash = "sha256:453fa251e2a4aeed580c72d4c7b2cb98fc8d8d26012798126f5cb11a9829cd71"},+ {file = "claude_agent_sdk-0.1.64.tar.gz", hash = "sha256:147e513cb45095b57c37d74b8d01dd41b5f3ec7f70e408edce43a6590159c27d"},][package.dependencies]@@ -930,6 +930,8 @@ typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""}[package.extras]dev = ["anyio[trio] (>=4.0.0)", "mypy (>=1.0.0)", "pytest (>=7.0.0)", "pytest-asyncio (>=0.20.0)", "pytest-cov (>=4.0.0)", "ruff (>=0.1.0)"]+examples = ["asyncpg (>=0.27.0)", "boto3 (>=1.28.0)", "fakeredis (>=2.20.0)", "moto[s3] (>=5.0.0)", "redis (>=4.2.0)"]+otel = ["opentelemetry-api (>=1.20.0)"][[package]]name = "cleo"@@ -8929,4 +8931,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt[metadata]lock-version = "2.1"python-versions = ">=3.10,<3.14"-content-hash = "c4cc6a0a26869a167ce182b178224554135d89d8ffa4605257d17b3f495cdf59"+content-hash = "529e1acbb1213421ef617f9dab309787cf81ea5d787eeffebc1bd38a42daf976"diff --git a/autogpt_platform/backend/pyproject.toml b/autogpt_platform/backend/pyproject.tomlindex ea81390d81..6e7003a65d 100644--- a/autogpt_platform/backend/pyproject.toml+++ b/autogpt_platform/backend/pyproject.toml@@ -18,7 +18,7 @@ apscheduler = "^3.11.1"autogpt-libs = { path = "../autogpt_libs", develop = true }bleach = { extras = ["css"], version = "^6.2.0" }cachetools = "^5.5.0"-claude-agent-sdk = "0.1.58" # latest stable; bundled CLI 2.1.97 -- CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 env var strips the broken context-management beta. See sdk_compat_test.py.+claude-agent-sdk = "^0.1.64" # bundled CLI 2.1.116 -- 2.1.98+ fixes the --resume + excludeDynamicSections crash that used to force a per-turn 33K cache write. CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 env var strips the broken context-management beta. See sdk_compat_test.py.click = "^8.2.0"cryptography = "^46.0"discord-py = "^2.5.2"