AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
perf(backend/copilot): enable LLM prompt caching to reduce token costs
Zamil Majdy815659d18877
diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.pyindex 298aeb9240..f61f6eb331 100644--- a/autogpt_platform/backend/backend/blocks/llm.py+++ b/autogpt_platform/backend/backend/blocks/llm.py@@ -836,6 +836,11 @@ async def llm_call(elif provider == "anthropic":an_tools = convert_openai_tool_fmt_to_anthropic(tools)+ # Cache tool definitions alongside the system prompt.+ # Placing cache_control on the last tool caches all tool schemas as a+ # single prefix — reads cost 10% of normal input tokens.+ if isinstance(an_tools, list) and an_tools:+ an_tools[-1] = {**an_tools[-1], "cache_control": {"type": "ephemeral"}}system_messages = [p["content"] for p in prompt if p["role"] == "system"]sysprompt = " ".join(system_messages)@@ -861,7 +866,13 @@ async def llm_call(try:resp = await client.messages.create(model=llm_model.value,- system=sysprompt,+ system=[+ {+ "type": "text",+ "text": sysprompt,+ "cache_control": {"type": "ephemeral"},+ }+ ],messages=messages,max_tokens=max_tokens,tools=an_tools,diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.pyindex 41eac0de23..490368c03a 100644--- a/autogpt_platform/backend/backend/copilot/baseline/service.py+++ b/autogpt_platform/backend/backend/copilot/baseline/service.py@@ -39,7 +39,7 @@ from backend.copilot.response_model import (StreamUsage,)from backend.copilot.service import (- _build_system_prompt,+ _build_cacheable_system_prompt,_generate_session_title,_get_openai_client,config,@@ -47,6 +47,7 @@ from backend.copilot.service import (from backend.copilot.token_tracking import persist_and_record_usagefrom backend.copilot.tools import execute_tool, get_available_toolsfrom backend.copilot.tracking import track_user_message+from backend.data.understanding import format_understanding_for_promptfrom backend.util.exceptions import NotFoundErrorfrom backend.util.prompt import compress_context@@ -175,17 +176,13 @@ async def stream_chat_completion_baseline(message_id = str(uuid.uuid4())- # Build system prompt only on the first turn to avoid mid-conversation- # changes from concurrent chats updating business understanding.+ # Build a fully static system prompt for token caching.+ # User-specific context is injected into the first user message instead+ # so the system prompt stays identical across all users and sessions.is_first_turn = len(session.messages) <= 1- if is_first_turn:- base_system_prompt, _ = await _build_system_prompt(- user_id, has_conversation_history=False- )- else:- base_system_prompt, _ = await _build_system_prompt(- user_id=None, has_conversation_history=True- )+ base_system_prompt, understanding = await _build_cacheable_system_prompt(+ user_id if is_first_turn else None+ )# Append tool documentation and technical notessystem_prompt = base_system_prompt + get_baseline_supplement()@@ -201,6 +198,16 @@ async def stream_chat_completion_baseline(if msg.role in ("user", "assistant") and msg.content:openai_messages.append({"role": msg.role, "content": msg.content})+ # Inject user context into the first user message on first turn+ if is_first_turn and understanding:+ user_ctx = format_understanding_for_prompt(understanding)+ for msg in openai_messages:+ if msg["role"] == "user":+ msg["content"] = (+ f"<user_context>\n{user_ctx}\n</user_context>\n\n{msg['content']}"+ )+ break+tools = get_available_tools()yield StreamStart(messageId=message_id, sessionId=session_id)diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.pyindex b8351e9b89..8ed5ca91b9 100644--- a/autogpt_platform/backend/backend/copilot/sdk/service.py+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py@@ -32,6 +32,7 @@ from pydantic import BaseModelfrom backend.copilot.context import get_workspace_managerfrom backend.data.redis_client import get_redis_async+from backend.data.understanding import format_understanding_for_promptfrom backend.executor.cluster_lock import AsyncClusterLockfrom backend.util.exceptions import NotFoundErrorfrom backend.util.settings import Settings@@ -66,7 +67,7 @@ from ..response_model import (StreamUsage,)from ..service import (- _build_system_prompt,+ _build_cacheable_system_prompt,_generate_session_title,_is_langfuse_configured,)@@ -1556,9 +1557,9 @@ async def stream_chat_completion_sdk()return None- e2b_sandbox, (base_system_prompt, _), dl = await asyncio.gather(+ e2b_sandbox, (base_system_prompt, understanding), dl = await asyncio.gather(_setup_e2b(),- _build_system_prompt(user_id, has_conversation_history=has_history),+ _build_cacheable_system_prompt(user_id),_fetch_transcript(),)@@ -1700,6 +1701,14 @@ async def stream_chat_completion_sdk(transcript_msg_count,session_id,)+ # On the first turn inject user context into the message instead of the+ # system prompt — the system prompt is now static (same for all users)+ # so the LLM can cache it across sessions.+ if not has_history and understanding:+ user_ctx = format_understanding_for_prompt(understanding)+ query_message = (+ f"<user_context>\n{user_ctx}\n</user_context>\n\n{query_message}"+ )# If files are attached, prepare them: images become vision# content blocks in the user message, other files go to sdk_cwd.attachments = await _prepare_file_attachments(diff --git a/autogpt_platform/backend/backend/copilot/service.py b/autogpt_platform/backend/backend/copilot/service.pyindex 5058341e68..b92b6f9064 100644--- a/autogpt_platform/backend/backend/copilot/service.py+++ b/autogpt_platform/backend/backend/copilot/service.py@@ -65,6 +65,21 @@ Your goal is to help users automate tasks by:Be concise, proactive, and action-oriented. Bias toward showing working solutions over lengthy explanations."""+# Static system prompt for token caching — identical for all users.+# User-specific context is injected into the first user message instead,+# so the system prompt never changes and can be cached across all sessions.+_CACHEABLE_SYSTEM_PROMPT = """You are an AI automation assistant helping users build and run automations.++Your goal is to help users automate tasks by:+- Understanding their needs and business context+- Building and running working automations+- Delivering tangible value through action, not just explanation++Be concise, proactive, and action-oriented. Bias toward showing working solutions over lengthy explanations.++When the user provides a <user_context> block in their message, use it to personalise your responses.+For users you are meeting for the first time with no context provided, greet them warmly and introduce them to the AutoGPT platform."""+# ---------------------------------------------------------------------------# Shared helpers (used by SDK service and baseline)@@ -145,6 +160,50 @@ async def _build_system_prompt(return compiled, understanding+async def _build_cacheable_system_prompt(+ user_id: str | None,+) -> tuple[str, Any]:+ """Build a fully static system prompt suitable for LLM token caching.++ Unlike _build_system_prompt, user-specific context is NOT embedded here.+ Callers must inject the returned understanding into the first user message+ via format_understanding_for_prompt() so the system prompt stays identical+ across all users and sessions, enabling cross-session cache hits.++ Returns:+ Tuple of (static_prompt, understanding_object_or_None)+ """+ understanding = None+ if user_id:+ try:+ understanding = await understanding_db().get_business_understanding(user_id)+ except Exception as e:+ logger.warning(f"Failed to fetch business understanding: {e}")++ if _is_langfuse_configured():+ try:+ label = (+ None+ if settings.config.app_env == AppEnvironment.PRODUCTION+ else "latest"+ )+ prompt = await asyncio.to_thread(+ _get_langfuse().get_prompt,+ config.langfuse_prompt_name,+ label=label,+ cache_ttl_seconds=config.langfuse_prompt_cache_ttl,+ )+ # Pass empty string so existing Langfuse templates stay static+ compiled = prompt.compile(users_information="")+ return compiled, understanding+ except Exception as e:+ logger.warning(+ f"Failed to fetch cacheable prompt from Langfuse, using default: {e}"+ )++ return _CACHEABLE_SYSTEM_PROMPT, understanding++async def _generate_session_title(message: str,user_id: str | None = None,