Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(copilot): remove legacy copilot, add baseline non-SDK mode with tool calling (#12276)

Zamil Majdy0215332386b5
diff --git a/autogpt_platform/backend/backend/copilot/baseline/__init__.py b/autogpt_platform/backend/backend/copilot/baseline/__init__.py
new file mode 100644
index 0000000000..edfb30d1d6
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/baseline/__init__.py
@@ -0,0 +1,3 @@
+from .service import stream_chat_completion_baseline
+
+__all__ = ["stream_chat_completion_baseline"]
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py
new file mode 100644
index 0000000000..6855fa702f
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/baseline/service.py
@@ -0,0 +1,398 @@
+"""Baseline LLM fallback — OpenAI-compatible streaming with tool calling.
+
+Used when ``CHAT_USE_CLAUDE_AGENT_SDK=false``, e.g. as a fallback when the
+Claude Agent SDK / Anthropic API is unavailable. Routes through any
+OpenAI-compatible provider (OpenRouter by default) and reuses the same
+shared tool registry as the SDK path.
+"""
+
+import asyncio
+import logging
+import uuid
+from collections.abc import AsyncGenerator
+from typing import Any
+
+import orjson
+
+from backend.copilot.model import (
+ ChatMessage,
+ ChatSession,
+ get_chat_session,
+ update_session_title,
+ upsert_chat_session,
+)
+from backend.copilot.response_model import (
+ StreamBaseResponse,
+ StreamError,
+ StreamFinish,
+ StreamFinishStep,
+ StreamStart,
+ StreamStartStep,
+ StreamTextDelta,
+ StreamTextEnd,
+ StreamTextStart,
+ StreamToolInputAvailable,
+ StreamToolInputStart,
+ StreamToolOutputAvailable,
+)
+from backend.copilot.service import (
+ _build_system_prompt,
+ _generate_session_title,
+ client,
+ config,
+)
+from backend.copilot.tools import execute_tool, get_available_tools
+from backend.copilot.tracking import track_user_message
+from backend.util.exceptions import NotFoundError
+from backend.util.prompt import compress_context
+
+logger = logging.getLogger(__name__)
+
+# Set to hold background tasks to prevent garbage collection
+_background_tasks: set[asyncio.Task[Any]] = set()
+
+# Maximum number of tool-call rounds before forcing a text response.
+_MAX_TOOL_ROUNDS = 30
+
+
+async def _update_title_async(
+ session_id: str, message: str, user_id: str | None
+) -> None:
+ """Generate and persist a session title in the background."""
+ try:
+ title = await _generate_session_title(message, user_id, session_id)
+ if title:
+ await update_session_title(session_id, title)
+ except Exception as e:
+ logger.warning("[Baseline] Failed to update session title: %s", e)
+
+
+async def _compress_session_messages(
+ messages: list[ChatMessage],
+) -> list[ChatMessage]:
+ """Compress session messages if they exceed the model's token limit.
+
+ Uses the shared compress_context() utility which supports LLM-based
+ summarization of older messages while keeping recent ones intact,
+ with progressive truncation and middle-out deletion as fallbacks.
+ """
+ messages_dict = []
+ for msg in messages:
+ msg_dict: dict[str, Any] = {"role": msg.role}
+ if msg.content:
+ msg_dict["content"] = msg.content
+ messages_dict.append(msg_dict)
+
+ try:
+ result = await compress_context(
+ messages=messages_dict,
+ model=config.model,
+ client=client,
+ )
+ except Exception as e:
+ logger.warning("[Baseline] Context compression with LLM failed: %s", e)
+ result = await compress_context(
+ messages=messages_dict,
+ model=config.model,
+ client=None,
+ )
+
+ if result.was_compacted:
+ logger.info(
+ "[Baseline] Context compacted: %d -> %d tokens "
+ "(%d summarized, %d dropped)",
+ result.original_token_count,
+ result.token_count,
+ result.messages_summarized,
+ result.messages_dropped,
+ )
+ return [
+ ChatMessage(role=m["role"], content=m.get("content"))
+ for m in result.messages
+ ]
+
+ return messages
+
+
+async def stream_chat_completion_baseline(
+ session_id: str,
+ message: str | None = None,
+ is_user_message: bool = True,
+ user_id: str | None = None,
+ session: ChatSession | None = None,
+ **_kwargs: Any,
+) -> AsyncGenerator[StreamBaseResponse, None]:
+ """Baseline LLM with tool calling via OpenAI-compatible API.
+
+ Designed as a fallback when the Claude Agent SDK is unavailable.
+ Uses the same tool registry as the SDK path but routes through any
+ OpenAI-compatible provider (e.g. OpenRouter).
+
+ Flow: stream response -> if tool_calls, execute them -> feed results back -> repeat.
+ """
+ if session is None:
+ session = await get_chat_session(session_id, user_id)
+
+ if not session:
+ raise NotFoundError(
+ f"Session {session_id} not found. Please create a new session first."
+ )
+
+ # Append user message
+ new_role = "user" if is_user_message else "assistant"
+ if message and (
+ len(session.messages) == 0
+ or not (
+ session.messages[-1].role == new_role
+ and session.messages[-1].content == message
+ )
+ ):
+ session.messages.append(ChatMessage(role=new_role, content=message))
+ if is_user_message:
+ track_user_message(
+ user_id=user_id,
+ session_id=session_id,
+ message_length=len(message),
+ )
+
+ session = await upsert_chat_session(session)
+
+ # Generate title for new sessions
+ if is_user_message and not session.title:
+ user_messages = [m for m in session.messages if m.role == "user"]
+ if len(user_messages) == 1:
+ first_message = user_messages[0].content or message or ""
+ if first_message:
+ task = asyncio.create_task(
+ _update_title_async(session_id, first_message, user_id)
+ )
+ _background_tasks.add(task)
+ task.add_done_callback(_background_tasks.discard)
+
+ message_id = str(uuid.uuid4())
+
+ # Build system prompt only on the first turn to avoid mid-conversation
+ # changes from concurrent chats updating business understanding.
+ is_first_turn = len(session.messages) <= 1
+ if is_first_turn:
+ system_prompt, _ = await _build_system_prompt(
+ user_id, has_conversation_history=False
+ )
+ else:
+ system_prompt, _ = await _build_system_prompt(
+ user_id=None, has_conversation_history=True
+ )
+
+ # Compress context if approaching the model's token limit
+ messages_for_context = await _compress_session_messages(session.messages)
+
+ # Build OpenAI message list from session history
+ openai_messages: list[dict[str, Any]] = [
+ {"role": "system", "content": system_prompt}
+ ]
+ for msg in messages_for_context:
+ if msg.role in ("user", "assistant") and msg.content:
+ openai_messages.append({"role": msg.role, "content": msg.content})
+
+ tools = get_available_tools()
+
+ yield StreamStart(messageId=message_id, sessionId=session_id)
+
+ assistant_text = ""
+ text_block_id = str(uuid.uuid4())
+ text_started = False
+ step_open = False
+ try:
+ for _round in range(_MAX_TOOL_ROUNDS):
+ # Open a new step for each LLM round
+ yield StreamStartStep()
+ step_open = True
+
+ # Stream a response from the model
+ create_kwargs: dict[str, Any] = dict(
+ model=config.model,
+ messages=openai_messages,
+ stream=True,
+ )
+ if tools:
+ create_kwargs["tools"] = tools
+ response = await client.chat.completions.create(**create_kwargs) # type: ignore[arg-type] # dynamic kwargs
+
+ # Accumulate streamed response (text + tool calls)
+ round_text = ""
+ tool_calls_by_index: dict[int, dict[str, str]] = {}
+
+ async for chunk in response:
+ delta = chunk.choices[0].delta if chunk.choices else None
+ if not delta:
+ continue
+
+ # Text content
+ if delta.content:
+ if not text_started:
+ yield StreamTextStart(id=text_block_id)
+ text_started = True
+ round_text += delta.content
+ yield StreamTextDelta(id=text_block_id, delta=delta.content)
+
+ # Tool call fragments (streamed incrementally)
+ if delta.tool_calls:
+ for tc in delta.tool_calls:
+ idx = tc.index
+ if idx not in tool_calls_by_index:
+ tool_calls_by_index[idx] = {
+ "id": "",
+ "name": "",
+ "arguments": "",
+ }
+ entry = tool_calls_by_index[idx]
+ if tc.id:
+ entry["id"] = tc.id
+ if tc.function and tc.function.name:
+ entry["name"] = tc.function.name
+ if tc.function and tc.function.arguments:
+ entry["arguments"] += tc.function.arguments
+
+ # Close text block if we had one this round
+ if text_started:
+ yield StreamTextEnd(id=text_block_id)
+ text_started = False
+ text_block_id = str(uuid.uuid4())
+
+ # Accumulate text for session persistence
+ assistant_text += round_text
+
+ # No tool calls -> model is done
+ if not tool_calls_by_index:
+ yield StreamFinishStep()
+ step_open = False
+ break
+
+ # Close step before tool execution
+ yield StreamFinishStep()
+ step_open = False
+
+ # Append the assistant message with tool_calls to context
+ assistant_msg: dict[str, Any] = {"role": "assistant"}
+ if round_text:
+ assistant_msg["content"] = round_text
+ assistant_msg["tool_calls"] = [
+ {
+ "id": tc["id"],
+ "type": "function",
+ "function": {
+ "name": tc["name"],
+ "arguments": tc["arguments"],
+ },
+ }
+ for tc in tool_calls_by_index.values()
+ ]
+ openai_messages.append(assistant_msg)
+
+ # Execute each tool call and stream events
+ for tc in tool_calls_by_index.values():
+ tool_call_id = tc["id"]
+ tool_name = tc["name"]
+ raw_args = tc["arguments"] or "{}"
+ try:
+ tool_args = orjson.loads(raw_args)
+ except orjson.JSONDecodeError as parse_err:
+ parse_error = (
+ f"Invalid JSON arguments for tool '{tool_name}': {parse_err}"
+ )
+ logger.warning("[Baseline] %s", parse_error)
+ yield StreamToolOutputAvailable(
+ toolCallId=tool_call_id,
+ toolName=tool_name,
+ output=parse_error,
+ success=False,
+ )
+ openai_messages.append(
+ {
+ "role": "tool",
+ "tool_call_id": tool_call_id,
+ "content": parse_error,
+ }
+ )
+ continue
+
+ yield StreamToolInputStart(toolCallId=tool_call_id, toolName=tool_name)
+ yield StreamToolInputAvailable(
+ toolCallId=tool_call_id,
+ toolName=tool_name,
+ input=tool_args,
+ )
+
+ # Execute via shared tool registry
+ try:
+ result: StreamToolOutputAvailable = await execute_tool(
+ tool_name=tool_name,
+ parameters=tool_args,
+ user_id=user_id,
+ session=session,
+ tool_call_id=tool_call_id,
+ )
+ yield result
+ tool_output = (
+ result.output
+ if isinstance(result.output, str)
+ else str(result.output)
+ )
+ except Exception as e:
+ error_output = f"Tool execution error: {e}"
+ logger.error(
+ "[Baseline] Tool %s failed: %s",
+ tool_name,
+ error_output,
+ exc_info=True,
+ )
+ yield StreamToolOutputAvailable(
+ toolCallId=tool_call_id,
+ toolName=tool_name,
+ output=error_output,
+ success=False,
+ )
+ tool_output = error_output
+
+ # Append tool result to context for next round
+ openai_messages.append(
+ {
+ "role": "tool",
+ "tool_call_id": tool_call_id,
+ "content": tool_output,
+ }
+ )
+ else:
+ # for-loop exhausted without break -> tool-round limit hit
+ limit_msg = (
+ f"Exceeded {_MAX_TOOL_ROUNDS} tool-call rounds "
+ "without a final response."
+ )
+ logger.error("[Baseline] %s", limit_msg)
+ yield StreamError(
+ errorText=limit_msg,
+ code="baseline_tool_round_limit",
+ )
+
+ except Exception as e:
+ error_msg = str(e) or type(e).__name__
+ logger.error("[Baseline] Streaming error: %s", error_msg, exc_info=True)
+ # Close any open text/step before emitting error
+ if text_started:
+ yield StreamTextEnd(id=text_block_id)
+ if step_open:
+ yield StreamFinishStep()
+ yield StreamError(errorText=error_msg, code="baseline_error")
+ # Still persist whatever we got
+ finally:
+ # Persist assistant response
+ if assistant_text:
+ session.messages.append(
+ ChatMessage(role="assistant", content=assistant_text)
+ )
+ try:
+ await upsert_chat_session(session)
+ except Exception as persist_err:
+ logger.error("[Baseline] Failed to persist session: %s", persist_err)
+
+ yield StreamFinish()
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_test.py
new file mode 100644
index 0000000000..a68be3f909
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/baseline/service_test.py
@@ -0,0 +1,99 @@
+import logging
+from os import getenv
+
+import pytest
+
+from backend.copilot.baseline import stream_chat_completion_baseline
+from backend.copilot.model import (
+ create_chat_session,
+ get_chat_session,
+ upsert_chat_session,
+)
+from backend.copilot.response_model import (
+ StreamError,
+ StreamFinish,
+ StreamStart,
+ StreamTextDelta,
+)
+
+logger = logging.getLogger(__name__)
+
+
[email protected](loop_scope="session")
+async def test_baseline_multi_turn(setup_test_user, test_user_id):
+ """Test that the baseline LLM path streams responses and maintains history.
+
+ Turn 1: Send a message with a unique keyword.
+ Turn 2: Ask the model to recall the keyword — proving conversation history
+ is correctly passed to the single-call LLM.
+ """
+ api_key: str | None = getenv("OPEN_ROUTER_API_KEY")
+ if not api_key:
+ return pytest.skip("OPEN_ROUTER_API_KEY is not set, skipping test")
+
+ session = await create_chat_session(test_user_id)
+ session = await upsert_chat_session(session)
+
+ # --- Turn 1: send a message with a unique keyword ---
+ keyword = "QUASAR99"
+ turn1_msg = (
+ f"Please remember this special keyword: {keyword}. "
+ "Just confirm you've noted it, keep your response brief."
+ )
+ turn1_text = ""
+ turn1_errors: list[str] = []
+ got_start = False
+ got_finish = False
+
+ async for chunk in stream_chat_completion_baseline(
+ session.session_id,
+ turn1_msg,
+ user_id=test_user_id,
+ ):
+ if isinstance(chunk, StreamStart):
+ got_start = True
+ elif isinstance(chunk, StreamTextDelta):
+ turn1_text += chunk.delta
+ elif isinstance(chunk, StreamError):
+ turn1_errors.append(chunk.errorText)
+ elif isinstance(chunk, StreamFinish):
+ got_finish = True
+
+ assert got_start, "Turn 1 did not yield StreamStart"
+ assert got_finish, "Turn 1 did not yield StreamFinish"
+ assert not turn1_errors, f"Turn 1 errors: {turn1_errors}"
+ assert turn1_text, "Turn 1 produced no text"
+ logger.info(f"Turn 1 response: {turn1_text[:100]}")
+
+ # Reload session for turn 2
+ session = await get_chat_session(session.session_id, test_user_id)
+ assert session, "Session not found after turn 1"
+
+ # Verify messages were persisted (user + assistant)
+ assert (
+ len(session.messages) >= 2
+ ), f"Expected at least 2 messages after turn 1, got {len(session.messages)}"
+
+ # --- Turn 2: ask model to recall the keyword ---
+ turn2_msg = "What was the special keyword I asked you to remember?"
+ turn2_text = ""
+ turn2_errors: list[str] = []
+
+ async for chunk in stream_chat_completion_baseline(
+ session.session_id,
+ turn2_msg,
+ user_id=test_user_id,
+ session=session,
+ ):
+ if isinstance(chunk, StreamTextDelta):
+ turn2_text += chunk.delta
+ elif isinstance(chunk, StreamError):
+ turn2_errors.append(chunk.errorText)
+
+ assert not turn2_errors, f"Turn 2 errors: {turn2_errors}"
+ assert turn2_text, "Turn 2 produced no text"
+ assert keyword in turn2_text, (
+ f"Model did not recall keyword '{keyword}' in turn 2. "
+ f"Response: {turn2_text[:200]}"
+ )
+ logger.info(f"Turn 2 recalled keyword successfully: {turn2_text[:100]}")
diff --git a/autogpt_platform/backend/backend/copilot/config.py b/autogpt_platform/backend/backend/copilot/config.py
index 828307d085..b7174b0016 100644
--- a/autogpt_platform/backend/backend/copilot/config.py
+++ b/autogpt_platform/backend/backend/copilot/config.py
@@ -26,11 +26,6 @@ class ChatConfig(BaseSettings):
# Session TTL Configuration - 12 hours
session_ttl: int = Field(default=43200, description="Session TTL in seconds")
- # Streaming Configuration
- max_retries: int = Field(
- default=3,
- description="Max retries for fallback path (SDK handles retries internally)",
- )
max_agent_runs: int = Field(default=30, description="Maximum number of agent runs")
max_agent_schedules: int = Field(
default=30, description="Maximum number of agent schedules"
@@ -71,7 +66,7 @@ class ChatConfig(BaseSettings):
# Claude Agent SDK Configuration
use_claude_agent_sdk: bool = Field(
default=True,
- description="Use Claude Agent SDK for chat completions",
+ description="Use Claude Agent SDK (True) or OpenAI-compatible LLM baseline (False)",
)
claude_agent_model: str | None = Field(
default=None,
@@ -113,12 +108,6 @@ class ChatConfig(BaseSettings):
description="E2B sandbox keepalive timeout in seconds.",
)
- # Extended thinking configuration for Claude models
- thinking_enabled: bool = Field(
- default=True,
- description="Enable adaptive thinking for Claude models via OpenRouter",
- )
-
@field_validator("use_e2b_sandbox", mode="before")
@classmethod
def get_use_e2b_sandbox(cls, v):
diff --git a/autogpt_platform/backend/backend/copilot/executor/processor.py b/autogpt_platform/backend/backend/copilot/executor/processor.py
index c5ff8c0ae1..97c441b7f5 100644
--- a/autogpt_platform/backend/backend/copilot/executor/processor.py
+++ b/autogpt_platform/backend/backend/copilot/executor/processor.py
@@ -9,8 +9,8 @@ import logging
import threading
import time
-from backend.copilot import service as copilot_service
from backend.copilot import stream_registry
+from backend.copilot.baseline import stream_chat_completion_baseline
from backend.copilot.config import ChatConfig
from backend.copilot.response_model import StreamFinish
from backend.copilot.sdk import service as sdk_service
@@ -194,7 +194,7 @@ class CoPilotProcessor:
):
"""Async execution logic for a CoPilot turn.
- Calls the stream_chat_completion service function and publishes
+ Calls the chat completion service (SDK or baseline) and publishes
results to the stream registry.
Args:
@@ -218,9 +218,9 @@ class CoPilotProcessor:
stream_fn = (
sdk_service.stream_chat_completion_sdk
if use_sdk
- else copilot_service.stream_chat_completion
+ else stream_chat_completion_baseline
)
- log.info(f"Using {'SDK' if use_sdk else 'standard'} service")
+ log.info(f"Using {'SDK' if use_sdk else 'baseline'} service")
# Stream chat completion and publish chunks to Redis.
async for chunk in stream_fn(
@@ -228,7 +228,6 @@ class CoPilotProcessor:
message=entry.message if entry.message else None,
is_user_message=entry.is_user_message,
user_id=entry.user_id,
- context=entry.context,
):
if cancel.is_set():
log.info("Cancel requested, breaking stream")
diff --git a/autogpt_platform/backend/backend/copilot/parallel_tool_calls_test.py b/autogpt_platform/backend/backend/copilot/parallel_tool_calls_test.py
deleted file mode 100644
index 81aef29f3d..0000000000
--- a/autogpt_platform/backend/backend/copilot/parallel_tool_calls_test.py
+++ /dev/null
@@ -1,269 +0,0 @@
-"""Tests for parallel tool call execution in CoPilot.
-
-These tests mock _yield_tool_call to avoid importing the full copilot stack
-which requires Prisma, DB connections, etc.
-"""
-
-import asyncio
-import time
-from typing import Any, cast
-
-import pytest
-
-
-async def test_parallel_tool_calls_run_concurrently():
- """Multiple tool calls should complete in ~max(delays), not sum(delays)."""
- from backend.copilot.response_model import (
- StreamToolInputAvailable,
- StreamToolOutputAvailable,
- )
- from backend.copilot.service import _execute_tool_calls_parallel
-
- n_tools = 3
- delay_per_tool = 0.2
- tool_calls = [
- {
- "id": f"call_{i}",
- "type": "function",
- "function": {"name": f"tool_{i}", "arguments": "{}"},
- }
- for i in range(n_tools)
- ]
-
- class FakeSession:
- session_id = "test"
- user_id = "test"
-
- def __init__(self):
- self.messages = []
-
- original_yield = None
-
- async def fake_yield(tc_list, idx, sess):
- yield StreamToolInputAvailable(
- toolCallId=tc_list[idx]["id"],
- toolName=tc_list[idx]["function"]["name"],
- input={},
- )
- await asyncio.sleep(delay_per_tool)
- yield StreamToolOutputAvailable(
- toolCallId=tc_list[idx]["id"],
- toolName=tc_list[idx]["function"]["name"],
- output="{}",
- )
-
- import backend.copilot.service as svc
-
- original_yield = svc._yield_tool_call
- svc._yield_tool_call = fake_yield
- try:
- start = time.monotonic()
- events = []
- async for event in _execute_tool_calls_parallel(
- tool_calls, cast(Any, FakeSession())
- ):
- events.append(event)
- elapsed = time.monotonic() - start
- finally:
- svc._yield_tool_call = original_yield
-
- assert len(events) == n_tools * 2
- # Parallel: should take ~delay, not ~n*delay
- assert elapsed < delay_per_tool * (
- n_tools - 0.5
- ), f"Took {elapsed:.2f}s, expected parallel (~{delay_per_tool}s)"
-
-
-async def test_single_tool_call_works():
- """Single tool call should work identically."""
- from backend.copilot.response_model import (
- StreamToolInputAvailable,
- StreamToolOutputAvailable,
- )
- from backend.copilot.service import _execute_tool_calls_parallel
-
- tool_calls = [
- {
- "id": "call_0",
- "type": "function",
- "function": {"name": "t", "arguments": "{}"},
- }
- ]
-
- class FakeSession:
- session_id = "test"
- user_id = "test"
-
- def __init__(self):
- self.messages = []
-
- async def fake_yield(tc_list, idx, sess):
- yield StreamToolInputAvailable(toolCallId="call_0", toolName="t", input={})
- yield StreamToolOutputAvailable(toolCallId="call_0", toolName="t", output="{}")
-
- import backend.copilot.service as svc
-
- orig = svc._yield_tool_call
- svc._yield_tool_call = fake_yield
- try:
- events = [
- e
- async for e in _execute_tool_calls_parallel(
- tool_calls, cast(Any, FakeSession())
- )
- ]
- finally:
- svc._yield_tool_call = orig
-
- assert len(events) == 2
-
-
-async def test_retryable_error_propagates():
- """Retryable errors should be raised after all tools finish."""
- from backend.copilot.response_model import StreamToolOutputAvailable
- from backend.copilot.service import _execute_tool_calls_parallel
-
- tool_calls = [
- {
- "id": f"call_{i}",
- "type": "function",
- "function": {"name": f"t_{i}", "arguments": "{}"},
- }
- for i in range(2)
- ]
-
- class FakeSession:
- session_id = "test"
- user_id = "test"
-
- def __init__(self):
- self.messages = []
-
- async def fake_yield(tc_list, idx, sess):
- if idx == 1:
- raise KeyError("bad")
- from backend.copilot.response_model import StreamToolInputAvailable
-
- yield StreamToolInputAvailable(
- toolCallId=tc_list[idx]["id"], toolName="t_0", input={}
- )
- await asyncio.sleep(0.05)
- yield StreamToolOutputAvailable(
- toolCallId=tc_list[idx]["id"], toolName="t_0", output="{}"
- )
-
- import backend.copilot.service as svc
-
- orig = svc._yield_tool_call
- svc._yield_tool_call = fake_yield
- try:
- events = []
- with pytest.raises(KeyError):
- async for event in _execute_tool_calls_parallel(
- tool_calls, cast(Any, FakeSession())
- ):
- events.append(event)
- # First tool's events should still be yielded
- assert any(isinstance(e, StreamToolOutputAvailable) for e in events)
- finally:
- svc._yield_tool_call = orig
-
-
-async def test_session_shared_across_parallel_tools():
- """All parallel tools should receive the same session instance."""
- from backend.copilot.response_model import (
- StreamToolInputAvailable,
- StreamToolOutputAvailable,
- )
- from backend.copilot.service import _execute_tool_calls_parallel
-
- tool_calls = [
- {
- "id": f"call_{i}",
- "type": "function",
- "function": {"name": f"t_{i}", "arguments": "{}"},
- }
- for i in range(3)
- ]
-
- class FakeSession:
- session_id = "test"
- user_id = "test"
-
- def __init__(self):
- self.messages = []
-
- observed_sessions = []
-
- async def fake_yield(tc_list, idx, sess):
- observed_sessions.append(sess)
- yield StreamToolInputAvailable(
- toolCallId=tc_list[idx]["id"], toolName=f"t_{idx}", input={}
- )
- yield StreamToolOutputAvailable(
- toolCallId=tc_list[idx]["id"], toolName=f"t_{idx}", output="{}"
- )
-
- import backend.copilot.service as svc
-
- orig = svc._yield_tool_call
- svc._yield_tool_call = fake_yield
- try:
- async for _ in _execute_tool_calls_parallel(
- tool_calls, cast(Any, FakeSession())
- ):
- pass
- finally:
- svc._yield_tool_call = orig
-
- assert len(observed_sessions) == 3
- assert observed_sessions[0] is observed_sessions[1] is observed_sessions[2]
-
-
-async def test_cancellation_cleans_up():
- """Generator close should cancel in-flight tasks."""
- from backend.copilot.response_model import StreamToolInputAvailable
- from backend.copilot.service import _execute_tool_calls_parallel
-
- tool_calls = [
- {
- "id": f"call_{i}",
- "type": "function",
- "function": {"name": f"t_{i}", "arguments": "{}"},
- }
- for i in range(2)
- ]
-
- class FakeSession:
- session_id = "test"
- user_id = "test"
-
- def __init__(self):
- self.messages = []
-
- started = asyncio.Event()
-
- async def fake_yield(tc_list, idx, sess):
- yield StreamToolInputAvailable(
- toolCallId=tc_list[idx]["id"], toolName=f"t_{idx}", input={}
- )
- started.set()
- await asyncio.sleep(10) # simulate long-running
-
- import backend.copilot.service as svc
-
- orig = svc._yield_tool_call
- svc._yield_tool_call = fake_yield
- try:
- gen = _execute_tool_calls_parallel(tool_calls, cast(Any, FakeSession()))
- await gen.__anext__() # get first event
- await started.wait()
- await gen.aclose() # close generator
- finally:
- svc._yield_tool_call = orig
- # If we get here without hanging, cleanup worked
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.py
index 6891482098..0b4b4b423c 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py
@@ -571,17 +571,12 @@ async def _build_query_message(
async def stream_chat_completion_sdk(
session_id: str,
message: str | None = None,
- tool_call_response: str | None = None, # noqa: ARG001
is_user_message: bool = True,
user_id: str | None = None,
- retry_count: int = 0, # noqa: ARG001
session: ChatSession | None = None,
- context: dict[str, str] | None = None, # noqa: ARG001
+ **_kwargs: Any,
) -> AsyncGenerator[StreamBaseResponse, None]:
- """Stream chat completion using Claude Agent SDK.
-
- Drop-in replacement for stream_chat_completion with improved reliability.
- """
+ """Stream chat completion using Claude Agent SDK."""
if session is None:
session = await get_chat_session(session_id, user_id)
@@ -728,495 +723,491 @@ async def stream_chat_completion_sdk(
yield StreamStart(messageId=message_id, sessionId=session_id)
set_execution_context(user_id, session, sandbox=e2b_sandbox, sdk_cwd=sdk_cwd)
- try:
- # Fail fast when no API credentials are available at all
- sdk_env = _build_sdk_env(session_id=session_id, user_id=user_id)
- if not sdk_env and not os.environ.get("ANTHROPIC_API_KEY"):
- raise RuntimeError(
- "No API key configured. Set OPEN_ROUTER_API_KEY "
- "(or CHAT_API_KEY) for OpenRouter routing, "
- "or ANTHROPIC_API_KEY for direct Anthropic access."
- )
- mcp_server = create_copilot_mcp_server(use_e2b=use_e2b)
+ # Fail fast when no API credentials are available at all
+ sdk_env = _build_sdk_env(session_id=session_id, user_id=user_id)
+ if not sdk_env and not os.environ.get("ANTHROPIC_API_KEY"):
+ raise RuntimeError(
+ "No API key configured. Set OPEN_ROUTER_API_KEY "
+ "(or CHAT_API_KEY) for OpenRouter routing, "
+ "or ANTHROPIC_API_KEY for direct Anthropic access."
+ )
- sdk_model = _resolve_sdk_model()
+ mcp_server = create_copilot_mcp_server(use_e2b=use_e2b)
- # --- Transcript capture via Stop hook ---
- # Read the file content immediately — the SDK may clean up
- # the file before our finally block runs.
- def _on_stop(transcript_path: str, sdk_session_id: str) -> None:
- captured_transcript.path = transcript_path
- captured_transcript.sdk_session_id = sdk_session_id
- content = read_transcript_file(transcript_path)
- if content:
- captured_transcript.raw_content = content
- logger.info(
- f"[SDK] Stop hook: captured {len(content)}B from "
- f"{transcript_path}"
- )
- else:
- logger.warning(
- f"[SDK] Stop hook: transcript file empty/missing at "
- f"{transcript_path}"
- )
+ sdk_model = _resolve_sdk_model()
- # Track SDK-internal compaction (PreCompact hook → start, next msg → end)
- compaction = CompactionTracker()
+ # --- Transcript capture via Stop hook ---
+ # Read the file content immediately — the SDK may clean up
+ # the file before our finally block runs.
+ def _on_stop(transcript_path: str, sdk_session_id: str) -> None:
+ captured_transcript.path = transcript_path
+ captured_transcript.sdk_session_id = sdk_session_id
+ content = read_transcript_file(transcript_path)
+ if content:
+ captured_transcript.raw_content = content
+ logger.info(
+ f"[SDK] Stop hook: captured {len(content)}B from "
+ f"{transcript_path}"
+ )
+ else:
+ logger.warning(
+ f"[SDK] Stop hook: transcript file empty/missing at "
+ f"{transcript_path}"
+ )
- security_hooks = create_security_hooks(
- user_id,
- sdk_cwd=sdk_cwd,
- max_subtasks=config.claude_agent_max_subtasks,
- on_stop=_on_stop if config.claude_agent_use_resume else None,
- on_compact=compaction.on_compact,
- )
+ # Track SDK-internal compaction (PreCompact hook → start, next msg → end)
+ compaction = CompactionTracker()
- # --- Resume strategy: download transcript from bucket ---
- transcript_msg_count = 0 # watermark: session.messages length at upload
+ security_hooks = create_security_hooks(
+ user_id,
+ sdk_cwd=sdk_cwd,
+ max_subtasks=config.claude_agent_max_subtasks,
+ on_stop=_on_stop if config.claude_agent_use_resume else None,
+ on_compact=compaction.on_compact,
+ )
- if config.claude_agent_use_resume and user_id and len(session.messages) > 1:
- dl = await download_transcript(user_id, session_id)
- is_valid = bool(dl and validate_transcript(dl.content))
- if dl and is_valid:
- logger.info(
- f"[SDK] Transcript available for session {session_id}: "
- f"{len(dl.content)}B, msg_count={dl.message_count}"
- )
- resume_file = write_transcript_to_tempfile(
- dl.content, session_id, sdk_cwd
- )
- if resume_file:
- use_resume = True
- transcript_msg_count = dl.message_count
- logger.debug(
- f"[SDK] Using --resume ({len(dl.content)}B, "
- f"msg_count={transcript_msg_count})"
- )
- elif dl:
- logger.warning(
- f"[SDK] Transcript downloaded but invalid for {session_id}"
- )
- else:
- logger.warning(
- f"[SDK] No transcript available for {session_id} "
- f"({len(session.messages)} messages in session)"
- )
+ # --- Resume strategy: download transcript from bucket ---
+ transcript_msg_count = 0 # watermark: session.messages length at upload
- allowed = get_copilot_tool_names(use_e2b=use_e2b)
- disallowed = get_sdk_disallowed_tools(use_e2b=use_e2b)
- sdk_options_kwargs: dict[str, Any] = {
- "system_prompt": system_prompt,
- "mcp_servers": {"copilot": mcp_server},
- "allowed_tools": allowed,
- "disallowed_tools": disallowed,
- "hooks": security_hooks,
- "cwd": sdk_cwd,
- "max_buffer_size": config.claude_agent_max_buffer_size,
- }
- if sdk_env:
- sdk_options_kwargs["model"] = sdk_model
- sdk_options_kwargs["env"] = sdk_env
- if use_resume and resume_file:
- sdk_options_kwargs["resume"] = resume_file
-
- options = ClaudeAgentOptions(**sdk_options_kwargs) # type: ignore[arg-type]
-
- adapter = SDKResponseAdapter(message_id=message_id, session_id=session_id)
-
- # Propagate user_id/session_id as OTEL context attributes so the
- # langsmith tracing integration attaches them to every span. This
- # is what Langfuse (or any OTEL backend) maps to its native
- # user/session fields.
- _otel_ctx = propagate_attributes(
- user_id=user_id,
- session_id=session_id,
- trace_name="copilot-sdk",
- tags=["sdk"],
- metadata={"resume": str(use_resume)},
- )
- _otel_ctx.__enter__()
-
- async with ClaudeSDKClient(options=options) as client:
- current_message = message or ""
- if not current_message and session.messages:
- last_user = [m for m in session.messages if m.role == "user"]
- if last_user:
- current_message = last_user[-1].content or ""
-
- if not current_message.strip():
- yield StreamError(
- errorText="Message cannot be empty.",
- code="empty_prompt",
+ if config.claude_agent_use_resume and user_id and len(session.messages) > 1:
+ dl = await download_transcript(user_id, session_id)
+ is_valid = bool(dl and validate_transcript(dl.content))
+ if dl and is_valid:
+ logger.info(
+ f"[SDK] Transcript available for session {session_id}: "
+ f"{len(dl.content)}B, msg_count={dl.message_count}"
+ )
+ resume_file = write_transcript_to_tempfile(
+ dl.content, session_id, sdk_cwd
+ )
+ if resume_file:
+ use_resume = True
+ transcript_msg_count = dl.message_count
+ logger.debug(
+ f"[SDK] Using --resume ({len(dl.content)}B, "
+ f"msg_count={transcript_msg_count})"
)
- return
-
- query_message, was_compacted = await _build_query_message(
- current_message,
- session,
- use_resume,
- transcript_msg_count,
- session_id,
+ elif dl:
+ logger.warning(
+ f"[SDK] Transcript downloaded but invalid for {session_id}"
)
- logger.info(
- "[SDK] [%s] Sending query — resume=%s, total_msgs=%d, query_len=%d",
- session_id[:12],
- use_resume,
- len(session.messages),
- len(query_message),
+ else:
+ logger.warning(
+ f"[SDK] No transcript available for {session_id} "
+ f"({len(session.messages)} messages in session)"
)
- compaction.reset_for_query()
- if was_compacted:
- for ev in compaction.emit_pre_query(session):
- yield ev
- await client.query(query_message, session_id=session_id)
-
- assistant_response = ChatMessage(role="assistant", content="")
- accumulated_tool_calls: list[dict[str, Any]] = []
- has_appended_assistant = False
- has_tool_results = False
-
- # Use an explicit async iterator with non-cancelling heartbeats.
- # CRITICAL: we must NOT cancel __anext__() mid-flight — doing so
- # (via asyncio.timeout or wait_for) corrupts the SDK's internal
- # anyio memory stream, causing StopAsyncIteration on the next
- # call and silently dropping all in-flight tool results.
- # Instead, wrap __anext__() in a Task and use asyncio.wait()
- # with a timeout. On timeout we emit a heartbeat but keep the
- # Task alive so it can deliver the next message.
- msg_iter = client.receive_response().__aiter__()
- pending_task: asyncio.Task[Any] | None = None
- try:
- while not stream_completed:
- if pending_task is None:
-
- async def _next_msg() -> Any:
- return await msg_iter.__anext__()
-
- pending_task = asyncio.create_task(_next_msg())
-
- done, _ = await asyncio.wait(
- {pending_task}, timeout=_HEARTBEAT_INTERVAL
+ allowed = get_copilot_tool_names(use_e2b=use_e2b)
+ disallowed = get_sdk_disallowed_tools(use_e2b=use_e2b)
+ sdk_options_kwargs: dict[str, Any] = {
+ "system_prompt": system_prompt,
+ "mcp_servers": {"copilot": mcp_server},
+ "allowed_tools": allowed,
+ "disallowed_tools": disallowed,
+ "hooks": security_hooks,
+ "cwd": sdk_cwd,
+ "max_buffer_size": config.claude_agent_max_buffer_size,
+ }
+ if sdk_model:
+ sdk_options_kwargs["model"] = sdk_model
+ if sdk_env:
+ sdk_options_kwargs["env"] = sdk_env
+ if use_resume and resume_file:
+ sdk_options_kwargs["resume"] = resume_file
+
+ options = ClaudeAgentOptions(**sdk_options_kwargs) # type: ignore[arg-type] # dynamic kwargs
+
+ adapter = SDKResponseAdapter(message_id=message_id, session_id=session_id)
+
+ # Propagate user_id/session_id as OTEL context attributes so the
+ # langsmith tracing integration attaches them to every span. This
+ # is what Langfuse (or any OTEL backend) maps to its native
+ # user/session fields.
+ _otel_ctx = propagate_attributes(
+ user_id=user_id,
+ session_id=session_id,
+ trace_name="copilot-sdk",
+ tags=["sdk"],
+ metadata={"resume": str(use_resume)},
+ )
+ _otel_ctx.__enter__()
+
+ async with ClaudeSDKClient(options=options) as client:
+ current_message = message or ""
+ if not current_message and session.messages:
+ last_user = [m for m in session.messages if m.role == "user"]
+ if last_user:
+ current_message = last_user[-1].content or ""
+
+ if not current_message.strip():
+ yield StreamError(
+ errorText="Message cannot be empty.",
+ code="empty_prompt",
+ )
+ return
+
+ query_message, was_compacted = await _build_query_message(
+ current_message,
+ session,
+ use_resume,
+ transcript_msg_count,
+ session_id,
+ )
+ logger.info(
+ "[SDK] [%s] Sending query — resume=%s, total_msgs=%d, query_len=%d",
+ session_id[:12],
+ use_resume,
+ len(session.messages),
+ len(query_message),
+ )
+
+ compaction.reset_for_query()
+ if was_compacted:
+ for ev in compaction.emit_pre_query(session):
+ yield ev
+ await client.query(query_message, session_id=session_id)
+
+ assistant_response = ChatMessage(role="assistant", content="")
+ accumulated_tool_calls: list[dict[str, Any]] = []
+ has_appended_assistant = False
+ has_tool_results = False
+ ended_with_stream_error = False
+
+ # Use an explicit async iterator with non-cancelling heartbeats.
+ # CRITICAL: we must NOT cancel __anext__() mid-flight — doing so
+ # (via asyncio.timeout or wait_for) corrupts the SDK's internal
+ # anyio memory stream, causing StopAsyncIteration on the next
+ # call and silently dropping all in-flight tool results.
+ # Instead, wrap __anext__() in a Task and use asyncio.wait()
+ # with a timeout. On timeout we emit a heartbeat but keep the
+ # Task alive so it can deliver the next message.
+ msg_iter = client.receive_response().__aiter__()
+ pending_task: asyncio.Task[Any] | None = None
+ try:
+ while not stream_completed:
+ if pending_task is None:
+
+ async def _next_msg() -> Any:
+ return await msg_iter.__anext__()
+
+ pending_task = asyncio.create_task(_next_msg())
+
+ done, _ = await asyncio.wait(
+ {pending_task}, timeout=_HEARTBEAT_INTERVAL
+ )
+
+ if not done:
+ await lock.refresh()
+ for ev in compaction.emit_start_if_ready():
+ yield ev
+ yield StreamHeartbeat()
+ continue
+
+ # Task completed — get result
+ pending_task = None
+ try:
+ sdk_msg = done.pop().result()
+ except StopAsyncIteration:
+ logger.info(
+ "[SDK] [%s] Stream ended normally (StopAsyncIteration)",
+ session_id[:12],
)
+ break
+ except Exception as stream_err:
+ # SDK sends {"type": "error"} which raises
+ # Exception in receive_response() — capture it
+ # so the session can still be saved and the
+ # frontend gets a clean finish.
+ logger.error(
+ "[SDK] [%s] Stream error from SDK: %s",
+ session_id[:12],
+ stream_err,
+ exc_info=True,
+ )
+ ended_with_stream_error = True
+ yield StreamError(
+ errorText=f"SDK stream error: {stream_err}",
+ code="sdk_stream_error",
+ )
+ break
- if not done:
- await lock.refresh()
- for ev in compaction.emit_start_if_ready():
- yield ev
- yield StreamHeartbeat()
- continue
+ logger.info(
+ "[SDK] [%s] Received: %s %s "
+ "(unresolved=%d, current=%d, resolved=%d)",
+ session_id[:12],
+ type(sdk_msg).__name__,
+ getattr(sdk_msg, "subtype", ""),
+ len(adapter.current_tool_calls)
+ - len(adapter.resolved_tool_calls),
+ len(adapter.current_tool_calls),
+ len(adapter.resolved_tool_calls),
+ )
- # Task completed — get result
- pending_task = None
- try:
- sdk_msg = done.pop().result()
- except StopAsyncIteration:
- logger.info(
- "[SDK] [%s] Stream ended normally (StopAsyncIteration)",
+ # Race-condition fix: SDK hooks (PostToolUse) are
+ # executed asynchronously via start_soon() — the next
+ # message can arrive before the hook stashes output.
+ # wait_for_stash() awaits an asyncio.Event signaled by
+ # stash_pending_tool_output(), completing as soon as
+ # the hook finishes (typically <1ms). The sleep(0)
+ # after lets any remaining concurrent hooks complete.
+ #
+ # Skip for parallel tool continuations: when the SDK
+ # sends parallel tool calls as separate
+ # AssistantMessages (each containing only
+ # ToolUseBlocks), we must NOT wait/flush — the prior
+ # tools are still executing concurrently.
+ is_parallel_continuation = isinstance(
+ sdk_msg, AssistantMessage
+ ) and all(isinstance(b, ToolUseBlock) for b in sdk_msg.content)
+
+ if (
+ adapter.has_unresolved_tool_calls
+ and isinstance(sdk_msg, (AssistantMessage, ResultMessage))
+ and not is_parallel_continuation
+ ):
+ if await wait_for_stash(timeout=0.5):
+ await asyncio.sleep(0)
+ else:
+ logger.warning(
+ "[SDK] [%s] Timed out waiting for "
+ "PostToolUse hook stash "
+ "(%d unresolved tool calls)",
session_id[:12],
+ len(adapter.current_tool_calls)
+ - len(adapter.resolved_tool_calls),
)
- break
- except Exception as stream_err:
- # SDK sends {"type": "error"} which raises
- # Exception in receive_response() — capture it
- # so the session can still be saved and the
- # frontend gets a clean finish.
- logger.error(
- "[SDK] [%s] Stream error from SDK: %s",
- session_id[:12],
- stream_err,
- exc_info=True,
- )
- yield StreamError(
- errorText=f"SDK stream error: {stream_err}",
- code="sdk_stream_error",
- )
- break
+ # Log ResultMessage details for debugging
+ if isinstance(sdk_msg, ResultMessage):
logger.info(
- "[SDK] [%s] Received: %s %s "
+ "[SDK] [%s] Received: ResultMessage %s "
"(unresolved=%d, current=%d, resolved=%d)",
session_id[:12],
- type(sdk_msg).__name__,
- getattr(sdk_msg, "subtype", ""),
+ sdk_msg.subtype,
len(adapter.current_tool_calls)
- len(adapter.resolved_tool_calls),
len(adapter.current_tool_calls),
len(adapter.resolved_tool_calls),
)
+ if sdk_msg.subtype in ("error", "error_during_execution"):
+ logger.error(
+ "[SDK] [%s] SDK execution failed with error: %s",
+ session_id[:12],
+ sdk_msg.result or "(no error message provided)",
+ )
- # Race-condition fix: SDK hooks (PostToolUse) are
- # executed asynchronously via start_soon() — the next
- # message can arrive before the hook stashes output.
- # wait_for_stash() awaits an asyncio.Event signaled by
- # stash_pending_tool_output(), completing as soon as
- # the hook finishes (typically <1ms). The sleep(0)
- # after lets any remaining concurrent hooks complete.
- #
- # Skip for parallel tool continuations: when the SDK
- # sends parallel tool calls as separate
- # AssistantMessages (each containing only
- # ToolUseBlocks), we must NOT wait/flush — the prior
- # tools are still executing concurrently.
- is_parallel_continuation = isinstance(
- sdk_msg, AssistantMessage
- ) and all(isinstance(b, ToolUseBlock) for b in sdk_msg.content)
-
- if (
- adapter.has_unresolved_tool_calls
- and isinstance(sdk_msg, (AssistantMessage, ResultMessage))
- and not is_parallel_continuation
- ):
- if await wait_for_stash(timeout=0.5):
- await asyncio.sleep(0)
- else:
- logger.warning(
- "[SDK] [%s] Timed out waiting for "
- "PostToolUse hook stash "
- "(%d unresolved tool calls)",
- session_id[:12],
- len(adapter.current_tool_calls)
- - len(adapter.resolved_tool_calls),
- )
+ # Emit compaction end if SDK finished compacting
+ for ev in await compaction.emit_end_if_ready(session):
+ yield ev
- # Log ResultMessage details for debugging
- if isinstance(sdk_msg, ResultMessage):
+ for response in adapter.convert_message(sdk_msg):
+ if isinstance(response, StreamStart):
+ continue
+
+ # Log tool events for debugging
+ if isinstance(
+ response,
+ (
+ StreamToolInputAvailable,
+ StreamToolOutputAvailable,
+ ),
+ ):
+ extra = ""
+ if isinstance(response, StreamToolOutputAvailable):
+ out_len = len(str(response.output))
+ extra = f", output_len={out_len}"
logger.info(
- "[SDK] [%s] Received: ResultMessage %s "
- "(unresolved=%d, current=%d, resolved=%d)",
+ "[SDK] [%s] Tool event: %s, tool=%s%s",
session_id[:12],
- sdk_msg.subtype,
- len(adapter.current_tool_calls)
- - len(adapter.resolved_tool_calls),
- len(adapter.current_tool_calls),
- len(adapter.resolved_tool_calls),
+ type(response).__name__,
+ getattr(response, "toolName", "N/A"),
+ extra,
)
- if sdk_msg.subtype in ("error", "error_during_execution"):
- logger.error(
- "[SDK] [%s] SDK execution failed with error: %s",
- session_id[:12],
- sdk_msg.result or "(no error message provided)",
- )
-
- # Emit compaction end if SDK finished compacting
- for ev in await compaction.emit_end_if_ready(session):
- yield ev
- for response in adapter.convert_message(sdk_msg):
- if isinstance(response, StreamStart):
- continue
-
- # Log tool events for debugging
- if isinstance(
- response,
- (
- StreamToolInputAvailable,
- StreamToolOutputAvailable,
- ),
- ):
- extra = ""
- if isinstance(response, StreamToolOutputAvailable):
- out_len = len(str(response.output))
- extra = f", output_len={out_len}"
- logger.info(
- "[SDK] [%s] Tool event: %s, tool=%s%s",
- session_id[:12],
- type(response).__name__,
- getattr(response, "toolName", "N/A"),
- extra,
- )
+ # Log errors being sent to frontend
+ if isinstance(response, StreamError):
+ logger.error(
+ "[SDK] [%s] Sending error to frontend: %s (code=%s)",
+ session_id[:12],
+ response.errorText,
+ response.code,
+ )
- # Log errors being sent to frontend
- if isinstance(response, StreamError):
- logger.error(
- "[SDK] [%s] Sending error to frontend: %s (code=%s)",
- session_id[:12],
- response.errorText,
- response.code,
- )
+ yield response
- yield response
-
- if isinstance(response, StreamTextDelta):
- delta = response.delta or ""
- # After tool results, start a new assistant
- # message for the post-tool text.
- if has_tool_results and has_appended_assistant:
- assistant_response = ChatMessage(
- role="assistant", content=delta
- )
- accumulated_tool_calls = []
- has_appended_assistant = False
- has_tool_results = False
- session.messages.append(assistant_response)
- has_appended_assistant = True
- else:
- assistant_response.content = (
- assistant_response.content or ""
- ) + delta
- if not has_appended_assistant:
- session.messages.append(assistant_response)
- has_appended_assistant = True
-
- elif isinstance(response, StreamToolInputAvailable):
- accumulated_tool_calls.append(
- {
- "id": response.toolCallId,
- "type": "function",
- "function": {
- "name": response.toolName,
- "arguments": json.dumps(
- response.input or {}
- ),
- },
- }
+ if isinstance(response, StreamTextDelta):
+ delta = response.delta or ""
+ # After tool results, start a new assistant
+ # message for the post-tool text.
+ if has_tool_results and has_appended_assistant:
+ assistant_response = ChatMessage(
+ role="assistant", content=delta
)
- assistant_response.tool_calls = accumulated_tool_calls
+ accumulated_tool_calls = []
+ has_appended_assistant = False
+ has_tool_results = False
+ session.messages.append(assistant_response)
+ has_appended_assistant = True
+ else:
+ assistant_response.content = (
+ assistant_response.content or ""
+ ) + delta
if not has_appended_assistant:
session.messages.append(assistant_response)
has_appended_assistant = True
- elif isinstance(response, StreamToolOutputAvailable):
- session.messages.append(
- ChatMessage(
- role="tool",
- content=(
- response.output
- if isinstance(response.output, str)
- else str(response.output)
- ),
- tool_call_id=response.toolCallId,
- )
+ elif isinstance(response, StreamToolInputAvailable):
+ accumulated_tool_calls.append(
+ {
+ "id": response.toolCallId,
+ "type": "function",
+ "function": {
+ "name": response.toolName,
+ "arguments": json.dumps(response.input or {}),
+ },
+ }
+ )
+ assistant_response.tool_calls = accumulated_tool_calls
+ if not has_appended_assistant:
+ session.messages.append(assistant_response)
+ has_appended_assistant = True
+
+ elif isinstance(response, StreamToolOutputAvailable):
+ session.messages.append(
+ ChatMessage(
+ role="tool",
+ content=(
+ response.output
+ if isinstance(response.output, str)
+ else str(response.output)
+ ),
+ tool_call_id=response.toolCallId,
)
- has_tool_results = True
+ )
+ has_tool_results = True
- elif isinstance(response, StreamFinish):
- stream_completed = True
+ elif isinstance(response, StreamFinish):
+ stream_completed = True
- except asyncio.CancelledError:
- # Task/generator was cancelled (e.g. client disconnect,
- # server shutdown). Log and let the safety-net / finally
- # blocks handle cleanup.
- logger.warning(
- "[SDK] [%s] Streaming loop cancelled (asyncio.CancelledError)",
- session_id[:12],
- )
- raise
- finally:
- # Cancel the pending __anext__ task to avoid a leaked
- # coroutine. This is safe even if the task already
- # completed.
- if pending_task is not None and not pending_task.done():
- pending_task.cancel()
- try:
- await pending_task
- except (asyncio.CancelledError, StopAsyncIteration):
- pass
-
- # Safety net: if tools are still unresolved after the
- # streaming loop (e.g. StopAsyncIteration before ResultMessage,
- # or SDK not sending UserMessages for built-in tools), flush
- # them now so the frontend stops showing spinners.
- if adapter.has_unresolved_tool_calls:
- logger.warning(
- "[SDK] [%s] %d unresolved tool(s) after stream loop — "
- "flushing as safety net",
- session_id[:12],
- len(adapter.current_tool_calls)
- - len(adapter.resolved_tool_calls),
- )
- safety_responses: list[StreamBaseResponse] = []
- adapter._flush_unresolved_tool_calls(safety_responses)
- for response in safety_responses:
- if isinstance(
- response,
- (StreamToolInputAvailable, StreamToolOutputAvailable),
- ):
- logger.info(
- "[SDK] [%s] Safety flush: %s, tool=%s",
- session_id[:12],
- type(response).__name__,
- getattr(response, "toolName", "N/A"),
- )
- yield response
+ except asyncio.CancelledError:
+ # Task/generator was cancelled (e.g. client disconnect,
+ # server shutdown). Log and let the safety-net / finally
+ # blocks handle cleanup.
+ logger.warning(
+ "[SDK] [%s] Streaming loop cancelled (asyncio.CancelledError)",
+ session_id[:12],
+ )
+ raise
+ finally:
+ # Cancel the pending __anext__ task to avoid a leaked
+ # coroutine. This is safe even if the task already
+ # completed.
+ if pending_task is not None and not pending_task.done():
+ pending_task.cancel()
+ try:
+ await pending_task
+ except (asyncio.CancelledError, StopAsyncIteration):
+ # Expected: task was cancelled or exhausted during cleanup
+ logger.info(
+ "[SDK] Pending __anext__ task completed during cleanup"
+ )
- # If the stream ended without a ResultMessage, the SDK
- # CLI exited unexpectedly or the user stopped execution.
- # Close any open text/step so chunks are well-formed, and
- # append a cancellation message so users see feedback.
- # StreamFinish is published by mark_session_completed in the processor.
- if not stream_completed:
- logger.info(
- "[SDK] [%s] Stream ended without ResultMessage (stopped by user)",
- session_id[:12],
- )
- closing_responses: list[StreamBaseResponse] = []
- adapter._end_text_if_open(closing_responses)
- for r in closing_responses:
- yield r
-
- # Add "Stopped by user" message so it persists after refresh
- # Use COPILOT_SYSTEM_PREFIX so frontend renders it as system message, not assistant
- session.messages.append(
- ChatMessage(
- role="assistant",
- content=f"{COPILOT_SYSTEM_PREFIX} Execution stopped by user",
+ # Safety net: if tools are still unresolved after the
+ # streaming loop (e.g. StopAsyncIteration before ResultMessage,
+ # or SDK not sending UserMessages for built-in tools), flush
+ # them now so the frontend stops showing spinners.
+ if adapter.has_unresolved_tool_calls:
+ logger.warning(
+ "[SDK] [%s] %d unresolved tool(s) after stream loop — "
+ "flushing as safety net",
+ session_id[:12],
+ len(adapter.current_tool_calls) - len(adapter.resolved_tool_calls),
+ )
+ safety_responses: list[StreamBaseResponse] = []
+ adapter._flush_unresolved_tool_calls(safety_responses)
+ for response in safety_responses:
+ if isinstance(
+ response,
+ (StreamToolInputAvailable, StreamToolOutputAvailable),
+ ):
+ logger.info(
+ "[SDK] [%s] Safety flush: %s, tool=%s",
+ session_id[:12],
+ type(response).__name__,
+ getattr(response, "toolName", "N/A"),
)
+ yield response
+
+ # If the stream ended without a ResultMessage, the SDK
+ # CLI exited unexpectedly or the user stopped execution.
+ # Close any open text/step so chunks are well-formed, and
+ # append a cancellation message so users see feedback.
+ # StreamFinish is published by mark_session_completed in the processor.
+ if not stream_completed and not ended_with_stream_error:
+ logger.info(
+ "[SDK] [%s] Stream ended without ResultMessage (stopped by user)",
+ session_id[:12],
+ )
+ closing_responses: list[StreamBaseResponse] = []
+ adapter._end_text_if_open(closing_responses)
+ for r in closing_responses:
+ yield r
+
+ # Add "Stopped by user" message so it persists after refresh
+ # Use COPILOT_SYSTEM_PREFIX so frontend renders it as system message, not assistant
+ session.messages.append(
+ ChatMessage(
+ role="assistant",
+ content=f"{COPILOT_SYSTEM_PREFIX} Execution stopped by user",
)
+ )
- if (
- assistant_response.content or assistant_response.tool_calls
- ) and not has_appended_assistant:
- session.messages.append(assistant_response)
-
- # --- Upload transcript for next-turn --resume ---
- # After async with the SDK task group has exited, so the Stop
- # hook has already fired and the CLI has been SIGTERMed. The
- # CLI uses appendFileSync, so all writes are safely on disk.
- if config.claude_agent_use_resume and user_id:
- # With --resume the CLI appends to the resume file (most
- # complete). Otherwise use the Stop hook path.
- if use_resume and resume_file:
- raw_transcript = read_transcript_file(resume_file)
- logger.debug("[SDK] Transcript source: resume file")
- elif captured_transcript.path:
- raw_transcript = read_transcript_file(captured_transcript.path)
- logger.debug(
- "[SDK] Transcript source: stop hook (%s), read result: %s",
- captured_transcript.path,
- f"{len(raw_transcript)}B" if raw_transcript else "None",
- )
- else:
- raw_transcript = None
+ if (
+ assistant_response.content or assistant_response.tool_calls
+ ) and not has_appended_assistant:
+ session.messages.append(assistant_response)
- if not raw_transcript:
- logger.debug(
- "[SDK] No usable transcript — CLI file had no "
- "conversation entries (expected for first turn "
- "without --resume)"
- )
+ # --- Upload transcript for next-turn --resume ---
+ # After async with the SDK task group has exited, so the Stop
+ # hook has already fired and the CLI has been SIGTERMed. The
+ # CLI uses appendFileSync, so all writes are safely on disk.
+ if config.claude_agent_use_resume and user_id:
+ # With --resume the CLI appends to the resume file (most
+ # complete). Otherwise use the Stop hook path.
+ if use_resume and resume_file:
+ raw_transcript = read_transcript_file(resume_file)
+ logger.debug("[SDK] Transcript source: resume file")
+ elif captured_transcript.path:
+ raw_transcript = read_transcript_file(captured_transcript.path)
+ logger.debug(
+ "[SDK] Transcript source: stop hook (%s), read result: %s",
+ captured_transcript.path,
+ f"{len(raw_transcript)}B" if raw_transcript else "None",
+ )
+ else:
+ raw_transcript = None
- if raw_transcript:
- # Shield the upload from generator cancellation so a
- # client disconnect / page refresh doesn't lose the
- # transcript. The upload must finish even if the SSE
- # connection is torn down.
- await asyncio.shield(
- _try_upload_transcript(
- user_id,
- session_id,
- raw_transcript,
- message_count=len(session.messages),
- )
- )
+ if not raw_transcript:
+ logger.debug(
+ "[SDK] No usable transcript — CLI file had no "
+ "conversation entries (expected for first turn "
+ "without --resume)"
+ )
- except ImportError:
- raise RuntimeError(
- "claude-agent-sdk is not installed. "
- "Disable SDK mode (CHAT_USE_CLAUDE_AGENT_SDK=false) "
- "to use the OpenAI-compatible fallback."
- )
+ if raw_transcript:
+ # Shield the upload from generator cancellation so a
+ # client disconnect / page refresh doesn't lose the
+ # transcript. The upload must finish even if the SSE
+ # connection is torn down.
+ await asyncio.shield(
+ _try_upload_transcript(
+ user_id,
+ session_id,
+ raw_transcript,
+ message_count=len(session.messages),
+ )
+ )
logger.info(
"[SDK] [%s] Stream completed successfully with %d messages",
diff --git a/autogpt_platform/backend/backend/copilot/service.py b/autogpt_platform/backend/backend/copilot/service.py
index d2ccce1a6a..c97a803385 100644
--- a/autogpt_platform/backend/backend/copilot/service.py
+++ b/autogpt_platform/backend/backend/copilot/service.py
@@ -1,31 +1,18 @@
+"""CoPilot service — shared helpers used by both SDK and baseline paths.
+
+This module contains:
+- System prompt building (Langfuse + default fallback)
+- Session title generation
+- Session assignment
+- Shared config and client instances
+"""
+
import asyncio
import logging
-import time
-from asyncio import CancelledError
-from collections.abc import AsyncGenerator
-from typing import TYPE_CHECKING, Any, cast
+from typing import Any
import openai
-
-if TYPE_CHECKING:
- from backend.util.prompt import CompressResult
-
-import orjson
from langfuse import get_client
-from openai import (
- APIConnectionError,
- APIError,
- APIStatusError,
- PermissionDeniedError,
- RateLimitError,
-)
-from openai.types.chat import (
- ChatCompletionChunk,
- ChatCompletionMessageParam,
- ChatCompletionStreamOptionsParam,
- ChatCompletionSystemMessageParam,
- ChatCompletionToolParam,
-)
from backend.data.db_accessors import understanding_db
from backend.data.understanding import format_understanding_for_prompt
@@ -33,42 +20,10 @@ from backend.util.exceptions import NotFoundError
from backend.util.settings import AppEnvironment, Settings
from .config import ChatConfig
-from .constants import COMPACTION_TOOL_NAME
-from .model import (
- ChatMessage,
- ChatSession,
- ChatSessionInfo,
- Usage,
- cache_chat_session,
- get_chat_session,
- update_session_title,
- upsert_chat_session,
-)
-from .response_model import (
- StreamBaseResponse,
- StreamError,
- StreamFinish,
- StreamFinishStep,
- StreamHeartbeat,
- StreamStart,
- StreamStartStep,
- StreamTextDelta,
- StreamTextEnd,
- StreamTextStart,
- StreamToolInputAvailable,
- StreamToolInputStart,
- StreamToolOutputAvailable,
- StreamUsage,
-)
-from .tools import execute_tool, get_available_tools
-from .tools.models import ErrorResponse
-from .tracking import track_user_message
+from .model import ChatSessionInfo, get_chat_session, upsert_chat_session
logger = logging.getLogger(__name__)
-# Set to hold background tasks to prevent garbage collection
-_background_tasks: set[asyncio.Task[Any]] = set()
-
config = ChatConfig()
settings = Settings()
client = openai.AsyncOpenAI(api_key=config.api_key, base_url=config.base_url)
@@ -195,6 +150,11 @@ Adapt flexibly to the conversation context. Not every interaction requires all s
You are NOT a chatbot. You are NOT documentation. You are a partner who helps busy business owners get value quickly by showing proof through working automations. Bias toward action over explanation."""
+# ---------------------------------------------------------------------------
+# Shared helpers (used by SDK service and baseline)
+# ---------------------------------------------------------------------------
+
+
def _is_langfuse_configured() -> bool:
"""Check if Langfuse credentials are configured."""
return bool(
@@ -340,1190 +300,3 @@ async def assign_user_to_session(
session.user_id = user_id
session = await upsert_chat_session(session)
return session
-
-
-async def stream_chat_completion(
- session_id: str,
- message: str | None = None,
- tool_call_response: str | None = None,
- is_user_message: bool = True,
- user_id: str | None = None,
- retry_count: int = 0,
- session: ChatSession | None = None,
- context: dict[str, str] | None = None, # {url: str, content: str}
- _continuation_message_id: (
- str | None
- ) = None, # Internal: reuse message ID for tool call continuations
-) -> AsyncGenerator[StreamBaseResponse, None]:
- """Main entry point for streaming chat completions with database handling.
-
- This function handles all database operations and delegates streaming
- to the internal _stream_chat_chunks function.
-
- Args:
- session_id: Chat session ID
- user_message: User's input message
- user_id: User ID for authentication (None for anonymous)
- session: Optional pre-loaded session object (for recursive calls to avoid Redis refetch)
-
- Yields:
- StreamBaseResponse objects formatted as SSE
-
- Raises:
- NotFoundError: If session_id is invalid
-
- """
- completion_start = time.monotonic()
-
- # Build log metadata for structured logging
- log_meta = {"component": "ChatService", "session_id": session_id}
- if user_id:
- log_meta["user_id"] = user_id
-
- logger.info(
- f"[TIMING] stream_chat_completion STARTED, session={session_id}, user={user_id}, "
- f"message_len={len(message) if message else 0}, is_user={is_user_message}",
- extra={
- "json_fields": {
- **log_meta,
- "message_len": len(message) if message else 0,
- "is_user_message": is_user_message,
- }
- },
- )
-
- # Only fetch from Redis if session not provided (initial call)
- if session is None:
- fetch_start = time.monotonic()
- session = await get_chat_session(session_id, user_id)
- fetch_time = (time.monotonic() - fetch_start) * 1000
- logger.info(
- f"[TIMING] get_chat_session took {fetch_time:.1f}ms, "
- f"n_messages={len(session.messages) if session else 0}",
- extra={
- "json_fields": {
- **log_meta,
- "duration_ms": fetch_time,
- "n_messages": len(session.messages) if session else 0,
- }
- },
- )
- else:
- logger.info(
- f"[TIMING] Using provided session, messages={len(session.messages)}",
- extra={"json_fields": {**log_meta, "n_messages": len(session.messages)}},
- )
-
- if not session:
- raise NotFoundError(
- f"Session {session_id} not found. Please create a new session first."
- )
-
- # Append the new message to the session if it's not already there
- new_message_role = "user" if is_user_message else "assistant"
- if message and (
- len(session.messages) == 0
- or not (
- session.messages[-1].role == new_message_role
- and session.messages[-1].content == message
- )
- ):
- session.messages.append(ChatMessage(role=new_message_role, content=message))
-
- # Track user message in PostHog
- if is_user_message and message:
- posthog_start = time.monotonic()
- track_user_message(
- user_id=user_id,
- session_id=session_id,
- message_length=len(message),
- )
- posthog_time = (time.monotonic() - posthog_start) * 1000
- logger.info(
- f"[TIMING] track_user_message took {posthog_time:.1f}ms",
- extra={"json_fields": {**log_meta, "duration_ms": posthog_time}},
- )
-
- upsert_start = time.monotonic()
- session = await upsert_chat_session(session)
- upsert_time = (time.monotonic() - upsert_start) * 1000
- logger.info(
- f"[TIMING] upsert_chat_session took {upsert_time:.1f}ms",
- extra={"json_fields": {**log_meta, "duration_ms": upsert_time}},
- )
- assert session, "Session not found"
-
- # Generate title for new sessions on first user message (non-blocking)
- # Check: is_user_message, no title yet, and this is the first user message
- user_messages = [m for m in session.messages if m.role == "user"]
- first_user_msg = message or (user_messages[0].content if user_messages else None)
- if is_user_message and first_user_msg and not session.title:
- if len(user_messages) == 1:
- # First user message - generate title in background
- # Capture only the values we need (not the session object) to avoid
- # stale data issues when the main flow modifies the session
- captured_session_id = session_id
- captured_message = first_user_msg
- captured_user_id = user_id
-
- async def _update_title():
- try:
- title = await _generate_session_title(
- captured_message,
- user_id=captured_user_id,
- session_id=captured_session_id,
- )
- if title:
- # Use dedicated title update function that doesn't
- # touch messages, avoiding race conditions
- await update_session_title(captured_session_id, title)
- logger.info(
- f"Generated title for session {captured_session_id}: {title}"
- )
- except Exception as e:
- logger.warning(f"Failed to update session title: {e}")
-
- # Fire and forget - don't block the chat response
- task = asyncio.create_task(_update_title())
- _background_tasks.add(task)
- task.add_done_callback(_background_tasks.discard)
-
- # Build system prompt with business understanding
- prompt_start = time.monotonic()
- system_prompt, understanding = await _build_system_prompt(user_id)
- prompt_time = (time.monotonic() - prompt_start) * 1000
- logger.info(
- f"[TIMING] _build_system_prompt took {prompt_time:.1f}ms",
- extra={"json_fields": {**log_meta, "duration_ms": prompt_time}},
- )
-
- # Initialize variables for streaming
- assistant_response = ChatMessage(
- role="assistant",
- content="",
- )
- accumulated_tool_calls: list[dict[str, Any]] = []
- has_saved_assistant_message = False
- has_appended_streaming_message = False
- last_cache_time = 0.0
- last_cache_content_len = 0
-
- has_yielded_end = False
- has_yielded_error = False
- has_done_tool_call = False
- has_received_text = False
- text_streaming_ended = False
- tool_response_messages: list[ChatMessage] = []
- should_retry = False
-
- # Generate unique IDs for AI SDK protocol
- import uuid as uuid_module
-
- is_continuation = _continuation_message_id is not None
- message_id = _continuation_message_id or str(uuid_module.uuid4())
- text_block_id = str(uuid_module.uuid4())
-
- # Only yield message start for the initial call, not for continuations.
- setup_time = (time.monotonic() - completion_start) * 1000
- logger.info(
- f"[TIMING] Setup complete, yielding StreamStart at {setup_time:.1f}ms",
- extra={"json_fields": {**log_meta, "setup_time_ms": setup_time}},
- )
- if not is_continuation:
- yield StreamStart(messageId=message_id, sessionId=session.session_id)
-
- # Emit start-step before each LLM call (AI SDK uses this to add step boundaries)
- yield StreamStartStep()
-
- try:
- logger.info(
- "[TIMING] Calling _stream_chat_chunks",
- extra={"json_fields": log_meta},
- )
- async for chunk in _stream_chat_chunks(
- session=session,
- tools=get_available_tools(),
- system_prompt=system_prompt,
- text_block_id=text_block_id,
- ):
- # Pass through out-of-band events (e.g. compaction notices)
- # without contaminating the primary text stream state.
- if isinstance(chunk, (StreamStartStep, StreamFinishStep)):
- yield chunk
- continue
- if (
- isinstance(
- chunk,
- (
- StreamToolInputStart,
- StreamToolInputAvailable,
- StreamToolOutputAvailable,
- ),
- )
- and getattr(chunk, "toolName", None) == COMPACTION_TOOL_NAME
- ):
- yield chunk
- continue
- if (
- isinstance(chunk, (StreamTextStart, StreamTextDelta, StreamTextEnd))
- and getattr(chunk, "id", None) != text_block_id
- ):
- yield chunk
- continue
-
- if isinstance(chunk, StreamTextStart):
- # Emit text-start before first text delta
- if not has_received_text:
- yield chunk
- elif isinstance(chunk, StreamTextDelta):
- delta = chunk.delta or ""
- assert assistant_response.content is not None
- assistant_response.content += delta
- has_received_text = True
- if not has_appended_streaming_message:
- session.messages.append(assistant_response)
- has_appended_streaming_message = True
- current_time = time.monotonic()
- content_len = len(assistant_response.content)
- if (
- current_time - last_cache_time >= 1.0
- and content_len > last_cache_content_len
- ):
- try:
- await cache_chat_session(session)
- except Exception as e:
- logger.warning(
- f"Failed to cache partial session {session.session_id}: {e}"
- )
- last_cache_time = current_time
- last_cache_content_len = content_len
- yield chunk
- elif isinstance(chunk, StreamTextEnd):
- # Emit text-end after text completes
- if has_received_text and not text_streaming_ended:
- text_streaming_ended = True
- yield chunk
- elif isinstance(chunk, StreamToolInputStart):
- # Emit text-end before first tool call, but only if we've received text
- if has_received_text and not text_streaming_ended:
- yield StreamTextEnd(id=text_block_id)
- text_streaming_ended = True
- yield chunk
- elif isinstance(chunk, StreamToolInputAvailable):
- # Accumulate tool calls in OpenAI format
- accumulated_tool_calls.append(
- {
- "id": chunk.toolCallId,
- "type": "function",
- "function": {
- "name": chunk.toolName,
- "arguments": orjson.dumps(chunk.input).decode("utf-8"),
- },
- }
- )
- yield chunk
- elif isinstance(chunk, StreamToolOutputAvailable):
- result_content = (
- chunk.output
- if isinstance(chunk.output, str)
- else orjson.dumps(chunk.output).decode("utf-8")
- )
- tool_response_messages.append(
- ChatMessage(
- role="tool",
- content=result_content,
- tool_call_id=chunk.toolCallId,
- )
- )
- has_done_tool_call = True
- # Track if any tool execution failed
- if not chunk.success:
- logger.warning(
- f"Tool {chunk.toolName} (ID: {chunk.toolCallId}) execution failed"
- )
- yield chunk
- elif isinstance(chunk, StreamFinish):
- if has_done_tool_call:
- # Tool calls happened — close the step but don't send message-level finish.
- # The continuation will open a new step, and finish will come at the end.
- yield StreamFinishStep()
- if not has_done_tool_call:
- # Emit text-end before finish if we received text but haven't closed it
- if has_received_text and not text_streaming_ended:
- yield StreamTextEnd(id=text_block_id)
- text_streaming_ended = True
-
- # Save assistant message before yielding finish to ensure it's persisted
- # even if client disconnects immediately after receiving StreamFinish
- if not has_saved_assistant_message:
- messages_to_save_early: list[ChatMessage] = []
- if accumulated_tool_calls:
- assistant_response.tool_calls = accumulated_tool_calls
- if not has_appended_streaming_message and (
- assistant_response.content or assistant_response.tool_calls
- ):
- messages_to_save_early.append(assistant_response)
- messages_to_save_early.extend(tool_response_messages)
-
- if messages_to_save_early:
- session.messages.extend(messages_to_save_early)
- logger.info(
- f"Saving assistant message before StreamFinish: "
- f"content_len={len(assistant_response.content or '')}, "
- f"tool_calls={len(assistant_response.tool_calls or [])}, "
- f"tool_responses={len(tool_response_messages)}"
- )
- if messages_to_save_early or has_appended_streaming_message:
- await upsert_chat_session(session)
- has_saved_assistant_message = True
-
- has_yielded_end = True
- # Emit finish-step (resets AI SDK text/reasoning state).
- # StreamFinish is published by mark_session_completed.
- yield StreamFinishStep()
- elif isinstance(chunk, StreamError):
- has_yielded_error = True
- yield chunk
- elif isinstance(chunk, StreamUsage):
- session.usage.append(
- Usage(
- prompt_tokens=chunk.promptTokens,
- completion_tokens=chunk.completionTokens,
- total_tokens=chunk.totalTokens,
- )
- )
- elif isinstance(chunk, StreamHeartbeat):
- # Pass through heartbeat to keep SSE connection alive
- yield chunk
- else:
- logger.error(f"Unknown chunk type: {type(chunk)}", exc_info=True)
-
- except CancelledError:
- if not has_saved_assistant_message:
- if accumulated_tool_calls:
- assistant_response.tool_calls = accumulated_tool_calls
- if assistant_response.content:
- assistant_response.content = (
- f"{assistant_response.content}\n\n[interrupted]"
- )
- else:
- assistant_response.content = "[interrupted]"
- if not has_appended_streaming_message:
- session.messages.append(assistant_response)
- if tool_response_messages:
- session.messages.extend(tool_response_messages)
- try:
- await upsert_chat_session(session)
- except Exception as e:
- logger.warning(
- f"Failed to save interrupted session {session.session_id}: {e}"
- )
- raise
- except Exception as e:
- logger.error(f"Error during stream: {e!s}", exc_info=True)
-
- # Check if this is a retryable error (JSON parsing, incomplete tool calls, etc.)
- is_retryable = isinstance(e, (orjson.JSONDecodeError, KeyError, TypeError))
-
- if is_retryable and retry_count < config.max_retries:
- logger.info(
- f"Retryable error encountered. Attempt {retry_count + 1}/{config.max_retries}"
- )
- # Close the current step before retrying so the recursive call's
- # StreamStartStep doesn't produce unbalanced step events.
- if not has_yielded_end:
- yield StreamFinishStep()
- should_retry = True
- else:
- # Non-retryable error or max retries exceeded
- # Save any partial progress before reporting error
- messages_to_save: list[ChatMessage] = []
-
- # Add assistant message if it has content or tool calls
- if accumulated_tool_calls:
- assistant_response.tool_calls = accumulated_tool_calls
- if not has_appended_streaming_message and (
- assistant_response.content or assistant_response.tool_calls
- ):
- messages_to_save.append(assistant_response)
-
- # Add tool response messages after assistant message
- messages_to_save.extend(tool_response_messages)
-
- if not has_saved_assistant_message:
- if messages_to_save:
- session.messages.extend(messages_to_save)
- if messages_to_save or has_appended_streaming_message:
- await upsert_chat_session(session)
-
- if not has_yielded_error:
- error_message = str(e)
- if not is_retryable:
- error_message = f"Non-retryable error: {error_message}"
- elif retry_count >= config.max_retries:
- error_message = (
- f"Max retries ({config.max_retries}) exceeded: {error_message}"
- )
-
- error_response = StreamError(errorText=error_message)
- yield error_response
- if not has_yielded_end:
- yield StreamFinishStep()
- return
-
- # Handle retry outside of exception handler to avoid nesting
- if should_retry and retry_count < config.max_retries:
- logger.info(
- f"Retrying stream_chat_completion for session {session_id}, attempt {retry_count + 1}"
- )
- async for chunk in stream_chat_completion(
- session_id=session.session_id,
- user_id=user_id,
- retry_count=retry_count + 1,
- session=session,
- context=context,
- _continuation_message_id=message_id, # Reuse message ID since start was already sent
- ):
- yield chunk
- return # Exit after retry to avoid double-saving in finally block
-
- # Normal completion path - save session and handle tool call continuation
- # Only save if we haven't already saved when StreamFinish was received
- if not has_saved_assistant_message:
- logger.info(
- f"Normal completion path: session={session.session_id}, "
- f"current message_count={len(session.messages)}"
- )
-
- # Build the messages list in the correct order
- messages_to_save: list[ChatMessage] = []
-
- # Add assistant message with tool_calls if any.
- # Use extend (not assign) to preserve tool_calls already added by
- # _yield_tool_call for long-running tools.
- if accumulated_tool_calls:
- if not assistant_response.tool_calls:
- assistant_response.tool_calls = []
- assistant_response.tool_calls.extend(accumulated_tool_calls)
- logger.info(
- f"Added {len(accumulated_tool_calls)} tool calls to assistant message"
- )
- if not has_appended_streaming_message and (
- assistant_response.content or assistant_response.tool_calls
- ):
- messages_to_save.append(assistant_response)
- logger.info(
- f"Saving assistant message with content_len={len(assistant_response.content or '')}, tool_calls={len(assistant_response.tool_calls or [])}"
- )
-
- # Add tool response messages after assistant message
- messages_to_save.extend(tool_response_messages)
- logger.info(
- f"Saving {len(tool_response_messages)} tool response messages, "
- f"total_to_save={len(messages_to_save)}"
- )
-
- if messages_to_save:
- session.messages.extend(messages_to_save)
- logger.info(
- f"Extended session messages, new message_count={len(session.messages)}"
- )
- if messages_to_save or has_appended_streaming_message:
- await upsert_chat_session(session)
- else:
- logger.info(
- "Assistant message already saved when StreamFinish was received, "
- "skipping duplicate save"
- )
-
- # If we did a tool call, stream the chat completion again to get the next response
- if has_done_tool_call:
- logger.info(
- "Tool call executed, streaming chat completion again to get assistant response"
- )
- async for chunk in stream_chat_completion(
- session_id=session.session_id,
- user_id=user_id,
- session=session, # Pass session object to avoid Redis refetch
- context=context,
- tool_call_response=str(tool_response_messages),
- _continuation_message_id=message_id, # Reuse message ID to avoid duplicates
- ):
- yield chunk
-
-
-# Retry configuration for OpenAI API calls
-MAX_RETRIES = 3
-BASE_DELAY_SECONDS = 1.0
-MAX_DELAY_SECONDS = 30.0
-
-
-def _is_retryable_error(error: Exception) -> bool:
- """Determine if an error is retryable."""
- if isinstance(error, RateLimitError):
- return True
- if isinstance(error, APIConnectionError):
- return True
- if isinstance(error, APIStatusError):
- # APIStatusError has a response with status_code
- # Retry on 5xx status codes (server errors)
- if error.response.status_code >= 500:
- return True
- if isinstance(error, APIError):
- # Retry on overloaded errors or 500 errors (may not have status code)
- error_message = str(error).lower()
- if "overloaded" in error_message or "internal server error" in error_message:
- return True
- return False
-
-
-def _is_region_blocked_error(error: Exception) -> bool:
- if isinstance(error, PermissionDeniedError):
- return "not available in your region" in str(error).lower()
- return "not available in your region" in str(error).lower()
-
-
-async def _manage_context_window(
- messages: list,
- model: str,
- api_key: str | None = None,
- base_url: str | None = None,
-) -> "CompressResult":
- """
- Manage context window using the unified compress_context function.
-
- This is a thin wrapper that creates an OpenAI client for summarization
- and delegates to the shared compression logic in prompt.py.
-
- Args:
- messages: List of messages in OpenAI format
- model: Model name for token counting and summarization
- api_key: API key for summarization calls
- base_url: Base URL for summarization calls
-
- Returns:
- CompressResult with compacted messages and metadata
- """
- import openai
-
- from backend.util.prompt import compress_context
-
- # Convert messages to dict format
- messages_dict = []
- for msg in messages:
- if isinstance(msg, dict):
- msg_dict = {k: v for k, v in msg.items() if v is not None}
- else:
- msg_dict = dict(msg)
- messages_dict.append(msg_dict)
-
- # Only create client if api_key is provided (enables summarization)
- # Use context manager to avoid socket leaks
- if api_key:
- async with openai.AsyncOpenAI(
- api_key=api_key, base_url=base_url, timeout=30.0
- ) as client:
- return await compress_context(
- messages=messages_dict,
- model=model,
- client=client,
- )
- else:
- # No API key - use truncation-only mode
- return await compress_context(
- messages=messages_dict,
- model=model,
- client=None,
- )
-
-
-async def _stream_chat_chunks(
- session: ChatSession,
- tools: list[ChatCompletionToolParam],
- system_prompt: str | None = None,
- text_block_id: str | None = None,
-) -> AsyncGenerator[StreamBaseResponse, None]:
- """
- Pure streaming function for OpenAI chat completions with tool calling.
-
- This function is database-agnostic and focuses only on streaming logic.
- Implements exponential backoff retry for transient API errors.
-
- Args:
- session: Chat session with conversation history
- tools: Available tools for the model
- system_prompt: System prompt to prepend to messages
-
- Yields:
- SSE formatted JSON response objects
-
- """
- import time as time_module
-
- stream_chunks_start = time_module.perf_counter()
- model = config.model
-
- # Build log metadata for structured logging
- log_meta = {"component": "ChatService", "session_id": session.session_id}
- if session.user_id:
- log_meta["user_id"] = session.user_id
-
- logger.info(
- f"[TIMING] _stream_chat_chunks STARTED, session={session.session_id}, "
- f"user={session.user_id}, n_messages={len(session.messages)}",
- extra={"json_fields": {**log_meta, "n_messages": len(session.messages)}},
- )
-
- messages = session.to_openai_messages()
- if system_prompt:
- system_message = ChatCompletionSystemMessageParam(
- role="system",
- content=system_prompt,
- )
- messages = [system_message] + messages
-
- # Apply context window management
- context_start = time_module.perf_counter()
- context_result = await _manage_context_window(
- messages=messages,
- model=model,
- api_key=config.api_key,
- base_url=config.base_url,
- )
- context_time = (time_module.perf_counter() - context_start) * 1000
- logger.info(
- f"[TIMING] _manage_context_window took {context_time:.1f}ms",
- extra={"json_fields": {**log_meta, "duration_ms": context_time}},
- )
-
- if context_result.error:
- if "System prompt dropped" in context_result.error:
- # Warning only - continue with reduced context
- yield StreamError(
- errorText=(
- "Warning: System prompt dropped due to size constraints. "
- "Assistant behavior may be affected."
- )
- )
- else:
- # Any other error - abort to prevent failed LLM calls
- yield StreamError(
- errorText=(
- f"Context window management failed: {context_result.error}. "
- "Please start a new conversation."
- )
- )
- return
-
- messages = context_result.messages
- if context_result.was_compacted:
- logger.info(
- f"Context compacted for streaming: {context_result.token_count} tokens"
- )
- from .sdk.compaction import emit_compaction
-
- for ev in emit_compaction(session):
- yield ev
-
- # Loop to handle tool calls and continue conversation
- while True:
- retry_count = 0
- last_error: Exception | None = None
-
- while retry_count <= MAX_RETRIES:
- try:
- elapsed = (time_module.perf_counter() - stream_chunks_start) * 1000
- retry_info = (
- f" (retry {retry_count}/{MAX_RETRIES})" if retry_count > 0 else ""
- )
- logger.info(
- f"[TIMING] Creating OpenAI stream at {elapsed:.1f}ms{retry_info}",
- extra={
- "json_fields": {
- **log_meta,
- "elapsed_ms": elapsed,
- "retry_count": retry_count,
- }
- },
- )
-
- # Build extra_body for OpenRouter tracing and PostHog analytics
- extra_body: dict[str, Any] = {
- "posthogProperties": {
- "environment": settings.config.app_env.value,
- },
- }
- if session.user_id:
- extra_body["user"] = session.user_id[:128] # OpenRouter limit
- extra_body["posthogDistinctId"] = session.user_id
- if session.session_id:
- extra_body["session_id"] = session.session_id[
- :128
- ] # OpenRouter limit
-
- # Broadcast trace metadata — forwarded to Langfuse via
- # OpenRouter's org-level Broadcast integration.
- extra_body["trace"] = {
- "trace_name": "copilot-chat",
- "environment": settings.config.app_env.value,
- }
-
- # Enable adaptive thinking for Anthropic models via OpenRouter
- if config.thinking_enabled and "anthropic" in model.lower():
- extra_body["reasoning"] = {"enabled": True}
-
- api_call_start = time_module.perf_counter()
- stream = await client.chat.completions.create(
- model=model,
- messages=cast(list[ChatCompletionMessageParam], messages),
- tools=tools,
- tool_choice="auto",
- stream=True,
- stream_options=ChatCompletionStreamOptionsParam(include_usage=True),
- extra_body=extra_body,
- )
- api_init_time = (time_module.perf_counter() - api_call_start) * 1000
- logger.info(
- f"[TIMING] OpenAI stream object returned in {api_init_time:.1f}ms",
- extra={"json_fields": {**log_meta, "duration_ms": api_init_time}},
- )
-
- # Variables to accumulate tool calls
- tool_calls: list[dict[str, Any]] = []
- active_tool_call_idx: int | None = None
- finish_reason: str | None = None
- # Track which tool call indices have had their start event emitted
- emitted_start_for_idx: set[int] = set()
-
- # Track if we've started the text block
- text_started = False
- first_content_chunk = True
- chunk_count = 0
-
- # Process the stream
- chunk: ChatCompletionChunk
- async for chunk in stream:
- chunk_count += 1
- if chunk.usage:
- yield StreamUsage(
- promptTokens=chunk.usage.prompt_tokens,
- completionTokens=chunk.usage.completion_tokens,
- totalTokens=chunk.usage.total_tokens,
- )
-
- if chunk.choices:
- choice = chunk.choices[0]
- delta = choice.delta
-
- # Capture finish reason
- if choice.finish_reason:
- finish_reason = choice.finish_reason
- logger.info(f"Finish reason: {finish_reason}")
-
- # Handle content streaming
- if delta.content:
- # Emit text-start on first text content
- if not text_started and text_block_id:
- yield StreamTextStart(id=text_block_id)
- text_started = True
- # Log timing for first content chunk
- if first_content_chunk:
- first_content_chunk = False
- ttfc = (
- time_module.perf_counter() - api_call_start
- ) * 1000
- logger.info(
- f"[TIMING] FIRST CONTENT CHUNK at {ttfc:.1f}ms "
- f"(since API call), n_chunks={chunk_count}",
- extra={
- "json_fields": {
- **log_meta,
- "time_to_first_chunk_ms": ttfc,
- "n_chunks": chunk_count,
- }
- },
- )
- # Stream the text delta
- text_response = StreamTextDelta(
- id=text_block_id or "",
- delta=delta.content,
- )
- yield text_response
-
- # Handle tool calls
- if delta.tool_calls:
- for tc_chunk in delta.tool_calls:
- idx = tc_chunk.index
-
- # Update active tool call index if needed
- if (
- active_tool_call_idx is None
- or active_tool_call_idx != idx
- ):
- active_tool_call_idx = idx
-
- # Ensure we have a tool call object at this index
- while len(tool_calls) <= idx:
- tool_calls.append(
- {
- "id": "",
- "type": "function",
- "function": {
- "name": "",
- "arguments": "",
- },
- },
- )
-
- # Accumulate the tool call data
- if tc_chunk.id:
- tool_calls[idx]["id"] = tc_chunk.id
- if tc_chunk.function:
- if tc_chunk.function.name:
- tool_calls[idx]["function"][
- "name"
- ] = tc_chunk.function.name
- if tc_chunk.function.arguments:
- tool_calls[idx]["function"][
- "arguments"
- ] += tc_chunk.function.arguments
-
- # Emit StreamToolInputStart only after we have the tool call ID
- if (
- idx not in emitted_start_for_idx
- and tool_calls[idx]["id"]
- and tool_calls[idx]["function"]["name"]
- ):
- yield StreamToolInputStart(
- toolCallId=tool_calls[idx]["id"],
- toolName=tool_calls[idx]["function"]["name"],
- )
- emitted_start_for_idx.add(idx)
- stream_duration = time_module.perf_counter() - api_call_start
- logger.info(
- f"[TIMING] OpenAI stream COMPLETE, finish_reason={finish_reason}, "
- f"duration={stream_duration:.2f}s, "
- f"n_chunks={chunk_count}, n_tool_calls={len(tool_calls)}",
- extra={
- "json_fields": {
- **log_meta,
- "stream_duration_ms": stream_duration * 1000,
- "finish_reason": finish_reason,
- "n_chunks": chunk_count,
- "n_tool_calls": len(tool_calls),
- }
- },
- )
-
- # Execute all accumulated tool calls in parallel
- # Events are yielded as they arrive from each concurrent tool
- async for event in _execute_tool_calls_parallel(tool_calls, session):
- yield event
-
- total_time = (time_module.perf_counter() - stream_chunks_start) * 1000
- logger.info(
- f"[TIMING] _stream_chat_chunks COMPLETED in {total_time / 1000:.1f}s; "
- f"session={session.session_id}, user={session.user_id}",
- extra={"json_fields": {**log_meta, "total_time_ms": total_time}},
- )
- return
- except Exception as e:
- last_error = e
-
- if _is_retryable_error(e) and retry_count < MAX_RETRIES:
- retry_count += 1
- # Calculate delay with exponential backoff
- delay = min(
- BASE_DELAY_SECONDS * (2 ** (retry_count - 1)),
- MAX_DELAY_SECONDS,
- )
- logger.warning(
- f"Retryable error in stream: {e!s}. "
- f"Retrying in {delay:.1f}s (attempt {retry_count}/{MAX_RETRIES})"
- )
- await asyncio.sleep(delay)
- continue # Retry the stream
- else:
- # Non-retryable error or max retries exceeded
- _log_api_error(
- error=e,
- context="stream (not retrying)",
- session_id=session.session_id if session else None,
- message_count=len(messages) if messages else None,
- model=model,
- retry_count=retry_count,
- )
- error_code = None
- error_text = str(e)
-
- error_details = _extract_api_error_details(e)
- if error_details.get("response_body"):
- body = error_details["response_body"]
- if isinstance(body, dict):
- err = body.get("error")
- if isinstance(err, dict) and err.get("message"):
- error_text = err["message"]
- elif body.get("message"):
- error_text = body["message"]
-
- if _is_region_blocked_error(e):
- error_code = "MODEL_NOT_AVAILABLE_REGION"
- error_text = (
- "This model is not available in your region. "
- "Please connect via VPN and try again."
- )
- error_response = StreamError(
- errorText=error_text,
- code=error_code,
- )
- yield error_response
- return
-
- # If we exit the retry loop without returning, it means we exhausted retries
- if last_error:
- _log_api_error(
- error=last_error,
- context=f"stream (max retries {MAX_RETRIES} exceeded)",
- session_id=session.session_id if session else None,
- message_count=len(messages) if messages else None,
- model=model,
- retry_count=MAX_RETRIES,
- )
- yield StreamError(errorText=f"Max retries exceeded: {last_error!s}")
- return
-
-
-async def _execute_tool_calls_parallel(
- tool_calls: list[dict[str, Any]],
- session: ChatSession,
-) -> AsyncGenerator[StreamBaseResponse, None]:
- """Execute all tool calls concurrently, yielding stream events as they arrive.
-
- Each tool runs as an ``asyncio.Task``, pushing events into a shared queue.
- """
- queue: asyncio.Queue[StreamBaseResponse | None] = asyncio.Queue()
- n_tools = len(tool_calls)
- retryable_errors: list[Exception] = []
-
- async def _run_tool(idx: int) -> None:
- tool_name = tool_calls[idx].get("function", {}).get("name", "unknown")
- tool_call_id = tool_calls[idx].get("id", f"unknown_{idx}")
- try:
- async for event in _yield_tool_call(tool_calls, idx, session):
- await queue.put(event)
- except (orjson.JSONDecodeError, KeyError, TypeError) as e:
- logger.error(
- f"Failed to parse tool call {idx} ({tool_name}): {e}",
- exc_info=True,
- )
- retryable_errors.append(e)
- except Exception as e:
- # Infrastructure / setup errors — emit an error output so the
- # client always sees a terminal event and doesn't hang.
- logger.error(f"Tool call {idx} ({tool_name}) failed: {e}", exc_info=True)
- await queue.put(
- StreamToolOutputAvailable(
- toolCallId=tool_call_id,
- toolName=tool_name,
- output=ErrorResponse(
- message=f"Tool execution failed: {e!s}",
- error=type(e).__name__,
- session_id=session.session_id,
- ).model_dump_json(),
- success=False,
- )
- )
- finally:
- await queue.put(None) # sentinel
-
- tasks = [asyncio.create_task(_run_tool(idx)) for idx in range(n_tools)]
- try:
- finished = 0
- while finished < n_tools:
- event = await queue.get()
- if event is None:
- finished += 1
- else:
- yield event
- if retryable_errors:
- if len(retryable_errors) > 1:
- logger.warning(
- f"{len(retryable_errors)} tool calls had retryable errors; "
- f"re-raising first to trigger retry"
- )
- raise retryable_errors[0]
- finally:
- for t in tasks:
- if not t.done():
- t.cancel()
- await asyncio.gather(*tasks, return_exceptions=True)
-
-
-async def _yield_tool_call(
- tool_calls: list[dict[str, Any]],
- yield_idx: int,
- session: ChatSession,
-) -> AsyncGenerator[StreamBaseResponse, None]:
- """
- Yield a tool call and its execution result.
-
- Yields heartbeat events every 10 seconds to keep the SSE connection alive
- while the tool executes.
-
- Raises:
- orjson.JSONDecodeError: If tool call arguments cannot be parsed as JSON
- KeyError: If expected tool call fields are missing
- TypeError: If tool call structure is invalid
- """
- tool_name = tool_calls[yield_idx]["function"]["name"]
- tool_call_id = tool_calls[yield_idx]["id"]
-
- # Parse tool call arguments - handle empty arguments gracefully
- raw_arguments = tool_calls[yield_idx]["function"]["arguments"]
- if raw_arguments:
- arguments = orjson.loads(raw_arguments)
- else:
- arguments = {}
-
- yield StreamToolInputAvailable(
- toolCallId=tool_call_id,
- toolName=tool_name,
- input=arguments,
- )
-
- # Run tool execution in background task with heartbeats
- tool_task = asyncio.create_task(
- execute_tool(
- tool_name=tool_name,
- parameters=arguments,
- tool_call_id=tool_call_id,
- user_id=session.user_id,
- session=session,
- )
- )
-
- # Yield heartbeats every 10 seconds while waiting for tool to complete
- # IMPORTANT: Must be less than frontend timeout (12s in useCopilotPage.ts)
- heartbeat_interval = 10.0 # seconds
- while not tool_task.done():
- try:
- # Wait for either the task to complete or the heartbeat interval
- await asyncio.wait_for(
- asyncio.shield(tool_task), timeout=heartbeat_interval
- )
- except asyncio.TimeoutError:
- # Task still running, send heartbeat to keep connection alive
- logger.debug(f"Sending heartbeat for tool {tool_name} ({tool_call_id})")
- yield StreamHeartbeat(toolCallId=tool_call_id)
- except CancelledError:
- # Task was cancelled, clean up and propagate
- tool_task.cancel()
- logger.warning(f"Tool execution cancelled: {tool_name} ({tool_call_id})")
- raise
-
- # Get the result - handle any exceptions that occurred during execution
- try:
- tool_execution_response: StreamToolOutputAvailable = await tool_task
- except Exception as e:
- # Task raised an exception - ensure we send an error response to the frontend
- logger.error(
- f"Tool execution failed: {tool_name} ({tool_call_id}): {e}", exc_info=True
- )
- error_response = ErrorResponse(
- message=f"Tool execution failed: {e!s}",
- error=type(e).__name__,
- session_id=session.session_id,
- )
- tool_execution_response = StreamToolOutputAvailable(
- toolCallId=tool_call_id,
- toolName=tool_name,
- output=error_response.model_dump_json(),
- success=False,
- )
-
- yield tool_execution_response
-
-
-def _log_api_error(
- error: Exception,
- context: str,
- session_id: str | None = None,
- message_count: int | None = None,
- model: str | None = None,
- retry_count: int = 0,
-) -> None:
- """Log detailed API error information for debugging."""
- details = _extract_api_error_details(error)
- details["context"] = context
- details["session_id"] = session_id
- details["message_count"] = message_count
- details["model"] = model
- details["retry_count"] = retry_count
-
- if isinstance(error, RateLimitError):
- logger.warning(f"Rate limit error in {context}: {details}", exc_info=error)
- elif isinstance(error, APIConnectionError):
- logger.warning(f"API connection error in {context}: {details}", exc_info=error)
- elif isinstance(error, APIStatusError) and error.status_code >= 500:
- logger.error(f"API server error (5xx) in {context}: {details}", exc_info=error)
- else:
- logger.error(f"API error in {context}: {details}", exc_info=error)
-
-
-def _extract_api_error_details(error: Exception) -> dict[str, Any]:
- """Extract detailed information from OpenAI/OpenRouter API errors."""
- error_msg = str(error)
- details: dict[str, Any] = {
- "error_type": type(error).__name__,
- "error_message": error_msg[:500] + "..." if len(error_msg) > 500 else error_msg,
- }
-
- if hasattr(error, "code"):
- details["code"] = getattr(error, "code", None)
- if hasattr(error, "param"):
- details["param"] = getattr(error, "param", None)
-
- if isinstance(error, APIStatusError):
- details["status_code"] = error.status_code
- details["request_id"] = getattr(error, "request_id", None)
-
- if hasattr(error, "body") and error.body:
- details["response_body"] = _sanitize_error_body(error.body)
-
- if hasattr(error, "response") and error.response:
- headers = error.response.headers
- details["openrouter_provider"] = headers.get("x-openrouter-provider")
- details["openrouter_model"] = headers.get("x-openrouter-model")
- details["retry_after"] = headers.get("retry-after")
- details["rate_limit_remaining"] = headers.get("x-ratelimit-remaining")
-
- return details
-
-
-def _sanitize_error_body(
- body: Any, max_length: int = 2000
-) -> dict[str, Any] | str | None:
- """Extract only safe fields from error response body to avoid logging sensitive data."""
- if not isinstance(body, dict):
- # Non-dict bodies (e.g., HTML error pages) - return truncated string
- if body is not None:
- body_str = str(body)
- if len(body_str) > max_length:
- return body_str[:max_length] + "...[truncated]"
- return body_str
- return None
-
- safe_fields = ("message", "type", "code", "param", "error")
- sanitized: dict[str, Any] = {}
-
- for field in safe_fields:
- if field in body:
- value = body[field]
- if field == "error" and isinstance(value, dict):
- sanitized[field] = _sanitize_error_body(value, max_length)
- elif isinstance(value, str) and len(value) > max_length:
- sanitized[field] = value[:max_length] + "...[truncated]"
- else:
- sanitized[field] = value
-
- return sanitized if sanitized else None
diff --git a/autogpt_platform/backend/backend/copilot/service_test.py b/autogpt_platform/backend/backend/copilot/service_test.py
index ed6db06535..10d29f3fbf 100644
--- a/autogpt_platform/backend/backend/copilot/service_test.py
+++ b/autogpt_platform/backend/backend/copilot/service_test.py
@@ -4,75 +4,14 @@ from os import getenv
import pytest
-from . import service as chat_service
from .model import create_chat_session, get_chat_session, upsert_chat_session
-from .response_model import StreamError, StreamTextDelta, StreamToolOutputAvailable
+from .response_model import StreamError, StreamTextDelta
from .sdk import service as sdk_service
from .sdk.transcript import download_transcript
logger = logging.getLogger(__name__)
[email protected](loop_scope="session")
-async def test_stream_chat_completion(setup_test_user, test_user_id):
- """
- Test the stream_chat_completion function.
- """
- api_key: str | None = getenv("OPEN_ROUTER_API_KEY")
- if not api_key:
- return pytest.skip("OPEN_ROUTER_API_KEY is not set, skipping test")
-
- session = await create_chat_session(test_user_id)
-
- has_errors = False
- assistant_message = ""
- async for chunk in chat_service.stream_chat_completion(
- session.session_id, "Hello, how are you?", user_id=session.user_id
- ):
- logger.info(chunk)
- if isinstance(chunk, StreamError):
- has_errors = True
- if isinstance(chunk, StreamTextDelta):
- assistant_message += chunk.delta
-
- # StreamFinish is published by mark_session_completed (processor layer),
- # not by the service. The generator completing means the stream ended.
- assert not has_errors, "Error occurred while streaming chat completion"
- assert assistant_message, "Assistant message is empty"
-
-
[email protected](loop_scope="session")
-async def test_stream_chat_completion_with_tool_calls(setup_test_user, test_user_id):
- """
- Test the stream_chat_completion function.
- """
- api_key: str | None = getenv("OPEN_ROUTER_API_KEY")
- if not api_key:
- return pytest.skip("OPEN_ROUTER_API_KEY is not set, skipping test")
-
- session = await create_chat_session(test_user_id)
- session = await upsert_chat_session(session)
-
- has_errors = False
- had_tool_calls = False
- async for chunk in chat_service.stream_chat_completion(
- session.session_id,
- "Please find me an agent that can help me with my business. Use the query 'moneny printing agent'",
- user_id=session.user_id,
- ):
- logger.info(chunk)
- if isinstance(chunk, StreamError):
- has_errors = True
- if isinstance(chunk, StreamToolOutputAvailable):
- had_tool_calls = True
-
- assert not has_errors, "Error occurred while streaming chat completion"
- assert had_tool_calls, "Tool calls did not occur"
- session = await get_chat_session(session.session_id)
- assert session, "Session not found"
- assert session.usage, "Usage is empty"
-
-
@pytest.mark.asyncio(loop_scope="session")
async def test_sdk_resume_multi_turn(setup_test_user, test_user_id):
"""Test that the SDK --resume path captures and uses transcripts across turns.