AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(platform/copilot): Reduce time to first output (#12828)
Reinier van der Leerda93d752a1dc
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.pyindex d8bd17f641..a94a90aa11 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py@@ -1148,6 +1148,11 @@ async def stream_chat_post(yield StreamFinish().to_sse()return+ # Flush a heartbeat immediately so the client knows the+ # connection is live — without this the first event arrives+ # only after the _stream_listener's first xread (up to 5 s).+ yield StreamHeartbeat().to_sse()+# Read from the subscriber queue and yield to SSElogger.info("[TIMING] Starting to read from subscriber_queue",diff --git a/autogpt_platform/backend/backend/copilot/config.py b/autogpt_platform/backend/backend/copilot/config.pyindex 2dad177e44..cf1d00536d 100644--- a/autogpt_platform/backend/backend/copilot/config.py+++ b/autogpt_platform/backend/backend/copilot/config.py@@ -244,6 +244,10 @@ class ChatConfig(BaseSettings):"compaction cascades. Skipped unconditionally for Moonshot routes.",)claude_agent_max_thinking_tokens: int = Field(+ deprecated=(+ "Setting a thinking token budget is not supported in Claude 4.7+. "+ "Use `claude_agent_thinking_effort` instead to steer thinking effort."+ ),default=8192,ge=0,le=128000,@@ -273,13 +277,15 @@ class ChatConfig(BaseSettings):description="Max Redis stream entries replayed on SSE reconnect.",)claude_agent_thinking_effort: Literal["low", "medium", "high", "max"] | None = (+ # TODO: add xhigh when SDK support catches upField(default=None,description="Thinking effort level: 'low', 'medium', 'high', 'max', or None. "- "Applies to models that emit a reasoning channel — Opus (extended "- "thinking) and Kimi K2.6 (OpenRouter ``reasoning`` extension lit "- "up by #12871). Sonnet does not have extended thinking — setting "- "effort on Sonnet can cause <internal_reasoning> tag leaks. "+ "Applies to models that emit a reasoning channel — Sonnet, Opus, "+ "and Mythos (adaptive thinking) and Kimi K2.6 "+ "(OpenRouter ``reasoning`` extension lit up by #12871). "+ "Check https://platform.claude.com/docs/en/build-with-claude/adaptive-thinking " # noqa+ "for model compatibility and guidance. ""None = let the model decide. Override via CHAT_CLAUDE_AGENT_THINKING_EFFORT.",))diff --git a/autogpt_platform/backend/backend/copilot/sdk/responsiveness_test.py b/autogpt_platform/backend/backend/copilot/sdk/responsiveness_test.pynew file mode 100644index 0000000000..e2921f8b8f--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/sdk/responsiveness_test.py@@ -0,0 +1,172 @@+"""Unit tests for the latency-reduction helpers introduced in SECRT-1912.++Covers ``_fetch_graphiti_context`` (extracted to module level so all four+parallel I/O paths in ``stream_chat_completion_sdk`` are independently+testable) and the adaptive thinking ``effort`` resolution that depends on+the per-turn ``mode`` argument.+"""++from __future__ import annotations++from unittest.mock import AsyncMock, patch++import pytest++from backend.copilot.model import ChatMessage, ChatSession++from .service import _fetch_graphiti_context+++def _make_session(user_id: str, *, n_messages: int = 0) -> ChatSession:+ """Build a minimal ChatSession with a controllable message count."""+ session = ChatSession.new(user_id, dry_run=False)+ for i in range(n_messages):+ session.messages.append(ChatMessage(role="user", content=f"prior message {i}"))+ return session+++# ---------------------------------------------------------------------------+# _fetch_graphiti_context+# ---------------------------------------------------------------------------+++async def test_fetch_graphiti_context_returns_disabled_when_flag_off():+ """When LD flag is off, return (False, "") without fetching warm context."""+ session = _make_session("user-1")+ with patch(+ "backend.copilot.sdk.service.is_enabled_for_user",+ new=AsyncMock(return_value=False),+ ):+ enabled, ctx = await _fetch_graphiti_context("user-1", session, "hello")+ assert enabled is False+ assert ctx == ""+++async def test_fetch_graphiti_context_skips_warm_for_anonymous_users():+ """Anonymous turns (no user_id) skip warm context even when enabled."""+ session = _make_session("user-1")+ with patch(+ "backend.copilot.sdk.service.is_enabled_for_user",+ new=AsyncMock(return_value=True),+ ):+ enabled, ctx = await _fetch_graphiti_context(None, session, "hello")+ assert enabled is True+ assert ctx == ""+++async def test_fetch_graphiti_context_skips_warm_on_followup_turns():+ """Sessions with prior history skip warm context — it's a turn-1 only feature."""+ session = _make_session("user-1", n_messages=3)+ with patch(+ "backend.copilot.sdk.service.is_enabled_for_user",+ new=AsyncMock(return_value=True),+ ):+ enabled, ctx = await _fetch_graphiti_context("user-1", session, "follow-up")+ assert enabled is True+ assert ctx == ""+++async def test_fetch_graphiti_context_loads_warm_context_on_first_turn():+ """Turn 1 (≤1 prior message) with an authenticated user loads warm context."""+ session = _make_session("user-1")+ with (+ patch(+ "backend.copilot.sdk.service.is_enabled_for_user",+ new=AsyncMock(return_value=True),+ ),+ patch(+ "backend.copilot.graphiti.context.fetch_warm_context",+ new=AsyncMock(return_value="<warm_facts>important fact</warm_facts>"),+ ),+ ):+ enabled, ctx = await _fetch_graphiti_context("user-1", session, "first prompt")+ assert enabled is True+ assert ctx == "<warm_facts>important fact</warm_facts>"+++async def test_fetch_graphiti_context_handles_empty_warm_context():+ """``fetch_warm_context`` returning ``None`` collapses to empty string."""+ session = _make_session("user-1")+ with (+ patch(+ "backend.copilot.sdk.service.is_enabled_for_user",+ new=AsyncMock(return_value=True),+ ),+ patch(+ "backend.copilot.graphiti.context.fetch_warm_context",+ new=AsyncMock(return_value=None),+ ),+ ):+ enabled, ctx = await _fetch_graphiti_context("user-1", session, "first prompt")+ assert enabled is True+ assert ctx == ""+++async def test_fetch_graphiti_context_handles_none_message():+ """A None message (e.g. resume turn with no fresh user input) is normalised+ to an empty search string before hitting ``fetch_warm_context``."""+ session = _make_session("user-1")+ fetch_mock = AsyncMock(return_value="ctx")+ with (+ patch(+ "backend.copilot.sdk.service.is_enabled_for_user",+ new=AsyncMock(return_value=True),+ ),+ patch(+ "backend.copilot.graphiti.context.fetch_warm_context",+ new=fetch_mock,+ ),+ ):+ enabled, ctx = await _fetch_graphiti_context("user-1", session, None)+ assert enabled is True+ assert ctx == "ctx"+ fetch_mock.assert_awaited_once_with("user-1", "")+++# ---------------------------------------------------------------------------+# Adaptive ``effort`` resolution (mirrors the inline expression in+# ``stream_chat_completion_sdk``). A regression here means the SDK either+# loses the fast-mode override or stops honouring the operator's static+# ``CHAT_CLAUDE_AGENT_THINKING_EFFORT`` override.+# ---------------------------------------------------------------------------+++def _resolve_effort(mode: str | None, configured: str | None) -> str:+ """Mirror the inline ``effort=`` expression so the branching is unit-tested+ without needing to construct ClaudeAgentOptions. Keep this in sync with+ [service.py:stream_chat_completion_sdk] when the formula changes."""+ return "medium" if mode == "fast" else (configured or "high")+++class TestAdaptiveEffortResolution:+ """Verify the ``effort`` value chosen for each mode/config combo.++ This tests the formula used at the ``ClaudeAgentOptions`` call site:+ ``"medium" if mode == "fast" else (config_value or "high")``+ """++ def test_fast_mode_pins_to_medium_regardless_of_config(self):+ """fast mode always uses 'medium' — the config override is ignored."""+ assert _resolve_effort("fast", configured=None) == "medium"+ assert _resolve_effort("fast", configured="low") == "medium"+ assert _resolve_effort("fast", configured="max") == "medium"++ def test_extended_thinking_mode_honours_config_override(self):+ """extended_thinking respects ``claude_agent_thinking_effort`` when set."""+ assert _resolve_effort("extended_thinking", configured="low") == "low"+ assert _resolve_effort("extended_thinking", configured="max") == "max"++ def test_extended_thinking_defaults_to_high_when_unconfigured(self):+ """No config override → 'high' (the responsive default for SDK turns)."""+ assert _resolve_effort("extended_thinking", configured=None) == "high"++ def test_default_mode_treats_none_as_extended_thinking(self):+ """``mode=None`` (server default) follows the same path as extended_thinking."""+ assert _resolve_effort(None, configured=None) == "high"+ assert _resolve_effort(None, configured="medium") == "medium"diff --git a/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py b/autogpt_platform/backend/backend/copilot/sdk/security_hooks.pyindex 6401dfade4..94e9f00ffb 100644--- a/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py+++ b/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py@@ -10,6 +10,8 @@ import refrom collections.abc import Callablefrom typing import Any, cast+from claude_agent_sdk.types import HookEvent, HookMatcher+from backend.copilot.context import (get_execution_context,is_allowed_local_path,@@ -184,7 +186,7 @@ def create_security_hooks(sdk_cwd: str | None = None,max_subtasks: int = 3,on_compact: Callable[[str], None] | None = None,-) -> dict[str, Any]:+) -> dict[HookEvent, list[HookMatcher]]:"""Create the security hooks configuration for Claude Agent SDK.Includes security validation and observability hooks:@@ -448,7 +450,7 @@ def create_security_hooks()return cast(SyncHookJSONOutput, {})- hooks: dict[str, Any] = {+ return {"PreToolUse": [HookMatcher(matcher="*", hooks=[pre_tool_use_hook])],"PostToolUse": [HookMatcher(matcher="*", hooks=[post_tool_use_hook])],"PostToolUseFailure": [@@ -458,8 +460,6 @@ def create_security_hooks("SubagentStart": [HookMatcher(matcher="*", hooks=[subagent_start_hook])],"SubagentStop": [HookMatcher(matcher="*", hooks=[subagent_stop_hook])],}-- return hooksexcept ImportError:# Fallback for when SDK isn't available - return empty hookslogger.warning("claude-agent-sdk not available, security hooks disabled")diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.pyindex f9427f2d80..29d14aa35e 100644--- a/autogpt_platform/backend/backend/copilot/sdk/service.py+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py@@ -5,6 +5,7 @@import asyncioimport base64import functools+from copy import copyimport jsonimport loggingimport os@@ -3618,8 +3619,6 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues'advanced' → Claude Opus; 'standard' → global config default.Takes priority over per-user LaunchDarkly targeting."""- _ = mode # SDK path ignores the requested mode.-if session is None:session = await get_chat_session(session_id, user_id)@@ -3806,9 +3805,9 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues)return# --- Run independent async I/O operations in parallel ---- # E2B sandbox setup, system prompt build (Langfuse + DB), and transcript- # download are independent network calls. Running them concurrently- # saves ~200-500ms compared to sequential execution.+ # E2B sandbox setup, system prompt build (Langfuse + DB), Graphiti+ # warm-context, and CLI session restore are all independent network+ # calls. Running them concurrently saves ~500-1000ms vs sequential.async def _setup_e2b():"""Set up E2B sandbox if configured, return sandbox or None."""@@ -3839,16 +3838,31 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssuesreturn sandbox- e2b_sandbox, (base_system_prompt, understanding) = await asyncio.gather(+ (+ e2b_sandbox,+ (base_system_prompt, understanding),+ (graphiti_enabled, warm_ctx),+ _restore,+ ) = await asyncio.gather(_setup_e2b(),_build_system_prompt(user_id if not has_history else None),+ _fetch_graphiti_context(user_id, session, message),+ # Restore CLI session — single GCS round-trip covers both+ # --resume and builder state. message_count watermark lives+ # in the companion .meta.json alongside the session file.+ _restore_cli_session_for_turn(+ user_id,+ session_id,+ session,+ sdk_cwd,+ transcript_builder,+ log_prefix,+ ),)use_e2b = e2b_sandbox is not None# Append appropriate supplement (Claude gets tool schemas automatically)- graphiti_enabled = await is_enabled_for_user(user_id)-graphiti_supplement = get_graphiti_supplement() if graphiti_enabled else ""# Append the builder-session block (graph id+name + full building# guide) AFTER the shared supplements so the system prompt is@@ -3863,21 +3877,6 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues+ builder_session_suffix)- # Warm context: pre-load relevant facts from Graphiti on first turn.- # Stored here and injected into the first user message (not the system- # prompt) so the system prompt stays identical across all users and- # sessions, enabling cross-session Anthropic prompt-cache hits.- warm_ctx = ""- if graphiti_enabled and user_id and len(session.messages) <= 1:- from ..graphiti.context import fetch_warm_context-- warm_ctx = await fetch_warm_context(user_id, message or "") or ""-- # Restore CLI session — single GCS round-trip covers both --resume and builder state.- # message_count watermark lives in the companion .meta.json alongside the session file.- _restore = await _restore_cli_session_for_turn(- user_id, session_id, session, sdk_cwd, transcript_builder, log_prefix- )transcript_content = _restore.transcript_contenttranscript_covers_prefix = _restore.transcript_covers_prefixuse_resume = _restore.use_resume@@ -3974,47 +3973,52 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssuescross_user_cache=config.claude_agent_cross_user_prompt_cache,)- sdk_options_kwargs: dict[str, Any] = {- "system_prompt": system_prompt_value,- "mcp_servers": {"copilot": mcp_server},- "allowed_tools": allowed,- "disallowed_tools": disallowed,- "hooks": security_hooks,- "cwd": sdk_cwd,- "max_buffer_size": config.claude_agent_max_buffer_size,- "stderr": _on_stderr,+ sdk_options = ClaudeAgentOptions(+ system_prompt=system_prompt_value,+ mcp_servers={"copilot": mcp_server},+ allowed_tools=allowed,+ disallowed_tools=disallowed,+ hooks=security_hooks,+ cwd=sdk_cwd,+ max_buffer_size=config.claude_agent_max_buffer_size,+ stderr=_on_stderr,# --- P0 guardrails ---# fallback_model: SDK auto-retries with this cheaper model on# 529 (overloaded) errors, avoiding user-visible failures.- "fallback_model": fallback_model,+ fallback_model=fallback_model,# max_turns: hard cap on agentic tool-use loops per query to# prevent runaway execution from burning budget.- "max_turns": config.agent_max_turns,+ max_turns=config.agent_max_turns,# max_budget_usd: per-query spend ceiling enforced by the CLI.# Sized to the smaller of the configured per-query default and# the user's *actual* remaining daily/weekly USD cap so the# CLI's "wrap up gracefully" reminder fires when they're close# to the real limit, not the static $10 default.- "max_budget_usd": await _resolve_dynamic_max_budget_usd(user_id),- }- # max_thinking_tokens: cap extended thinking output per LLM call.- # Thinking tokens are billed at output rate ($75/M for Opus) and- # account for ~54% of total cost. 8192 is the default.- # Intentionally sent for all models including Sonnet — the CLI- # silently ignores this field for non-Opus models (those without- # native extended thinking), so it is safe to pass unconditionally.- # Setting to 0 acts as the kill switch (same as baseline): omit the- # kwarg so the CLI falls back to its default (extended thinking off).+ max_budget_usd=await _resolve_dynamic_max_budget_usd(user_id),+ # thinking: specify extended thinking mode. Thinking tokens are+ # billed at output rate ($75/M for Opus) and account for ~54%+ # of total cost. The CLI silently ignores this field for+ # models without native extended thinking, so it is safe to+ # pass unconditionally.+ # NOTE: Claude 4.7+ does not support capped thinking token+ # budget: use `effort` instead to steer thinking effort.+ thinking={"type": "adaptive"},+ # effort: applies to models with extended thinking (Sonnet,+ # Opus, Mythos) and Kimi K2.6 via OpenRouter's ``reasoning``+ # extension (#12871).+ effort=(+ "medium"+ if mode == "fast"+ else (config.claude_agent_thinking_effort or "high")+ ),+ )+ # max_thinking_tokens: legacy cost cap retained for non-4.7 models+ # and Kimi K2.6. Setting to 0 acts as the kill switch (same as+ # baseline): omit the kwarg so the CLI falls back to its default.if config.claude_agent_max_thinking_tokens > 0:- sdk_options_kwargs["max_thinking_tokens"] = (- config.claude_agent_max_thinking_tokens- )- # effort: only set for models with extended thinking (Opus).- # Setting effort on Sonnet causes <internal_reasoning> tag leaks.- if config.claude_agent_thinking_effort:- sdk_options_kwargs["effort"] = config.claude_agent_thinking_effort+ sdk_options.max_thinking_tokens = config.claude_agent_max_thinking_tokensif sdk_model:- sdk_options_kwargs["model"] = sdk_model+ sdk_options.model = sdk_modelif config.sdk_include_partial_messages:# Opt into per-token streaming — the CLI emits raw Anthropic# ``content_block_delta`` events as ``StreamEvent`` messages@@ -4025,15 +4029,15 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues# only the tail diff from the subsequent summary, so content# never double-emits and a summary-only short block still# reaches the UI.- sdk_options_kwargs["include_partial_messages"] = True+ sdk_options.include_partial_messages = Trueif sdk_env:- sdk_options_kwargs["env"] = sdk_env+ sdk_options.env = sdk_envif use_resume and resume_file:# --resume {uuid} implies the session UUID — do NOT also pass# --session-id here. CLI >=2.1.97 rejects the combination of# --session-id + --resume unless --fork-session is also given.- sdk_options_kwargs["resume"] = resume_file+ sdk_options.resume = resume_fileelse:# Set session_id whenever NOT resuming so the CLI writes the# native session file to a predictable path for@@ -4047,15 +4051,13 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues# None), so no conflict with an existing file.# When --resume is active the session_id is already implied by# the resume file; passing it again would be rejected by the CLI.- sdk_options_kwargs["session_id"] = session_id+ sdk_options.session_id = session_id# Optional explicit Claude Code CLI binary path (decouples the# bundled SDK version from the CLI version we run — needed because# the CLI bundled in 0.1.46+ is broken against OpenRouter). Falls# back to the bundled binary when unset.if config.claude_agent_cli_path:- sdk_options_kwargs["cli_path"] = config.claude_agent_cli_path-- options = ClaudeAgentOptions(**sdk_options_kwargs) # type: ignore[arg-type] # dynamic kwargs+ sdk_options.cli_path = config.claude_agent_cli_pathadapter = SDKResponseAdapter(message_id=message_id,@@ -4309,7 +4311,7 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssuesfallback_notified_per_attempt = Falsestate = _RetryState(- options=options,+ options=sdk_options,query_message=query_message,was_compacted=was_compacted,use_resume=use_resume,@@ -4369,10 +4371,10 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssuesskip_transcript_upload = True# Rebuild SDK options and query for the reduced context- sdk_options_kwargs_retry = dict(sdk_options_kwargs)+ sdk_options_retry = copy(sdk_options)if ctx.use_resume and ctx.resume_file:- sdk_options_kwargs_retry["resume"] = ctx.resume_file- sdk_options_kwargs_retry.pop("session_id", None)+ sdk_options_retry.resume = ctx.resume_file+ sdk_options_retry.session_id = Noneelse:# No --resume on this retry. Whether we entered with# ``session_id`` (T1, mode-switch) or with ``--resume`` (T2+),@@ -4386,16 +4388,16 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues# guaranteeing the next turn re-trips prompt-too-long.if sdk_cwd:delete_stale_cli_session_file(sdk_cwd, session_id, log_prefix)- sdk_options_kwargs_retry.pop("resume", None)- sdk_options_kwargs_retry["session_id"] = session_id+ sdk_options_retry.resume = None+ sdk_options_retry.session_id = session_id# Recompute system_prompt for retry — the preset is safe on- # every turn (requires CLI ≥ 2.1.98, installed in the Docker- # image and configured via CHAT_CLAUDE_AGENT_CLI_PATH).- sdk_options_kwargs_retry["system_prompt"] = _build_system_prompt_value(+ # every turn (requires CLI ≥ 2.1.98, bundled in+ # claude-agent-sdk >= 0.1.64).+ sdk_options_retry.system_prompt = _build_system_prompt_value(system_prompt,cross_user_cache=config.claude_agent_cross_user_prompt_cache,)- state.options = ClaudeAgentOptions(**sdk_options_kwargs_retry) # type: ignore[arg-type] # dynamic kwargs+ state.options = sdk_options_retry# Retry intentionally omits prior_messages (transcript+gap context) and# falls back to full session.messages[:-1] from DB — the authoritative# source. transcript+gap is an optimisation for the first attempt only;@@ -5141,3 +5143,29 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssuesraiseif _auto_requeued:return+++async def _fetch_graphiti_context(+ user_id: str | None,+ session: ChatSession,+ message: str | None,+) -> tuple[bool, str]:+ """Check Graphiti flag and fetch warm context in one shot.++ Returns ``(graphiti_enabled, warm_ctx)`` where ``warm_ctx`` is a+ pre-loaded fact bundle injected into the first user message (not the+ system prompt) so the system prompt stays identical across users and+ sessions, enabling cross-session Anthropic prompt-cache hits. Skips+ the fetch on follow-up turns (history > 1 message) and when the user+ is anonymous.+ """+ enabled = await is_enabled_for_user(user_id)+ if not enabled:+ return False, ""+ if not (user_id and len(session.messages) <= 1):+ return True, ""++ from ..graphiti.context import fetch_warm_context++ ctx = await fetch_warm_context(user_id, message or "") or ""+ return True, ctx