Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/frontend): error handling, stream reconnection, and chat switching (#12205)

Zamil Majdy29ca034e4020
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.py
index 6255573099..dff315f4f6 100644
--- a/autogpt_platform/backend/backend/api/features/chat/routes.py
+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py
@@ -621,11 +621,10 @@ async def resume_session_stream(
if not active_session:
return Response(status_code=204)
- # Subscribe from the beginning ("0-0") to replay all chunks for this turn.
- # This is necessary because hydrated messages filter out incomplete tool calls
- # to avoid "No tool invocation found" errors. The resume stream delivers
- # those tool calls fresh with proper SDK state.
- # The AI SDK's deduplication will handle any duplicate chunks.
+ # Always replay from the beginning ("0-0") on resume.
+ # We can't use last_message_id because it's the latest ID in the backend
+ # stream, not the latest the frontend received — the gap causes lost
+ # messages. The frontend deduplicates replayed content.
subscriber_queue = await stream_registry.subscribe_to_session(
session_id=session_id,
user_id=user_id,
diff --git a/autogpt_platform/backend/backend/copilot/config.py b/autogpt_platform/backend/backend/copilot/config.py
index 8d05d894d6..ce6d37830f 100644
--- a/autogpt_platform/backend/backend/copilot/config.py
+++ b/autogpt_platform/backend/backend/copilot/config.py
@@ -85,7 +85,7 @@ class ChatConfig(BaseSettings):
)
claude_agent_max_subtasks: int = Field(
default=10,
- description="Max number of sub-agent Tasks the SDK can spawn per session.",
+ description="Max number of concurrent sub-agent Tasks the SDK can run per session.",
)
claude_agent_use_resume: bool = Field(
default=True,
diff --git a/autogpt_platform/backend/backend/copilot/executor/manager.py b/autogpt_platform/backend/backend/copilot/executor/manager.py
index f77344115c..da113ccc50 100644
--- a/autogpt_platform/backend/backend/copilot/executor/manager.py
+++ b/autogpt_platform/backend/backend/copilot/executor/manager.py
@@ -4,6 +4,7 @@ This module contains the CoPilotExecutor class that consumes chat tasks from
RabbitMQ and processes them using a thread pool, following the graph executor pattern.
"""
+import asyncio
import logging
import os
import threading
@@ -409,14 +410,19 @@ class CoPilotExecutor(AppProcess):
def on_run_done(f: Future):
logger.info(f"Run completed for {session_id}")
+ error_msg = None
try:
if exec_error := f.exception():
- logger.error(f"Execution for {session_id} failed: {exec_error}")
+ error_msg = str(exec_error) or type(exec_error).__name__
+ logger.error(f"Execution for {session_id} failed: {error_msg}")
ack_message(reject=True, requeue=False)
else:
ack_message(reject=False, requeue=False)
+ except asyncio.CancelledError:
+ logger.info(f"Run completion callback cancelled for {session_id}")
except BaseException as e:
- logger.exception(f"Error in run completion callback: {e}")
+ error_msg = str(e) or type(e).__name__
+ logger.exception(f"Error in run completion callback: {error_msg}")
finally:
# Release the cluster lock
if session_id in self._task_locks:
diff --git a/autogpt_platform/backend/backend/copilot/executor/processor.py b/autogpt_platform/backend/backend/copilot/executor/processor.py
index 48ea4cd931..38a1b4a851 100644
--- a/autogpt_platform/backend/backend/copilot/executor/processor.py
+++ b/autogpt_platform/backend/backend/copilot/executor/processor.py
@@ -125,7 +125,10 @@ class CoPilotProcessor:
)
future.result(timeout=5)
except Exception as e:
- logger.warning(f"[CoPilotExecutor] Worker {self.tid} cleanup error: {e}")
+ error_msg = str(e) or type(e).__name__
+ logger.warning(
+ f"[CoPilotExecutor] Worker {self.tid} cleanup error: {error_msg}"
+ )
# Stop the event loop
self.execution_loop.call_soon_threadsafe(self.execution_loop.stop)
@@ -157,47 +160,30 @@ class CoPilotProcessor:
start_time = time.monotonic()
- try:
- # Run the async execution in our event loop
- future = asyncio.run_coroutine_threadsafe(
- self._execute_async(entry, cancel, cluster_lock, log),
- self.execution_loop,
- )
-
- # Wait for completion, checking cancel periodically
- while not future.done():
- try:
- future.result(timeout=1.0)
- except asyncio.TimeoutError:
- if cancel.is_set():
- log.info("Cancellation requested")
- future.cancel()
- break
- # Refresh cluster lock to maintain ownership
- cluster_lock.refresh()
+ # Run the async execution in our event loop
+ future = asyncio.run_coroutine_threadsafe(
+ self._execute_async(entry, cancel, cluster_lock, log),
+ self.execution_loop,
+ )
- if not future.cancelled():
- # Get result to propagate any exceptions
- future.result()
+ # Wait for completion, checking cancel periodically
+ while not future.done():
+ try:
+ future.result(timeout=1.0)
+ except asyncio.TimeoutError:
+ if cancel.is_set():
+ log.info("Cancellation requested")
+ future.cancel()
+ break
+ # Refresh cluster lock to maintain ownership
+ cluster_lock.refresh()
- elapsed = time.monotonic() - start_time
- log.info(f"Execution completed in {elapsed:.2f}s")
+ if not future.cancelled():
+ # Get result to propagate any exceptions
+ future.result()
- except BaseException as e:
- elapsed = time.monotonic() - start_time
- log.error(f"Execution failed after {elapsed:.2f}s: {e}")
- # Safety net: if _execute_async's error handler failed to mark
- # the session (e.g. RuntimeError from SDK cleanup), do it here.
- try:
- asyncio.run_coroutine_threadsafe(
- stream_registry.mark_session_completed(
- entry.session_id, error_message=str(e) or "Unknown error"
- ),
- self.execution_loop,
- ).result(timeout=5.0)
- except Exception as cleanup_err:
- log.error(f"Safety net mark_session_completed failed: {cleanup_err}")
- raise
+ elapsed = time.monotonic() - start_time
+ log.info(f"Execution completed in {elapsed:.2f}s")
async def _execute_async(
self,
@@ -219,6 +205,7 @@ class CoPilotProcessor:
"""
last_refresh = time.monotonic()
refresh_interval = 30.0 # Refresh lock every 30 seconds
+ error_msg = None
try:
# Choose service based on LaunchDarkly flag
@@ -264,17 +251,26 @@ class CoPilotProcessor:
exc_info=True,
)
- error_message = "Operation cancelled" if cancel.is_set() else None
- await stream_registry.mark_session_completed(
- entry.session_id, error_message=error_message
- )
+ # Stream loop completed
+ if cancel.is_set():
+ log.info("Stream cancelled by user")
except BaseException as e:
- log.error(f"Turn failed: {e}")
+ # Handle all exceptions (including CancelledError) with appropriate logging
+ if isinstance(e, asyncio.CancelledError):
+ log.info("Turn cancelled")
+ error_msg = "Operation cancelled"
+ else:
+ error_msg = str(e) or type(e).__name__
+ log.error(f"Turn failed: {error_msg}")
+ raise
+ finally:
+ # If no exception but user cancelled, still mark as cancelled
+ if not error_msg and cancel.is_set():
+ error_msg = "Operation cancelled"
try:
await stream_registry.mark_session_completed(
- entry.session_id, error_message=str(e) or "Unknown error"
+ entry.session_id, error_message=error_msg
)
except Exception as mark_err:
- log.error(f"mark_session_completed also failed: {mark_err}")
- raise
+ log.error(f"Failed to mark session completed: {mark_err}")
diff --git a/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py b/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py
index 925cd99930..c2e7788d18 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py
@@ -160,7 +160,7 @@ def create_security_hooks(
Args:
user_id: Current user ID for isolation validation
sdk_cwd: SDK working directory for workspace-scoped tool validation
- max_subtasks: Maximum Task (sub-agent) spawns allowed per session
+ max_subtasks: Maximum concurrent Task (sub-agent) spawns allowed per session
on_stop: Callback ``(transcript_path, sdk_session_id)`` invoked when
the SDK finishes processing — used to read the JSONL transcript
before the CLI process exits.
@@ -172,8 +172,9 @@ def create_security_hooks(
from claude_agent_sdk import HookMatcher
from claude_agent_sdk.types import HookContext, HookInput, SyncHookJSONOutput
- # Per-session counter for Task sub-agent spawns
- task_spawn_count = 0
+ # Per-session tracking for Task sub-agent concurrency.
+ # Set of tool_use_ids that consumed a slot — len() is the active count.
+ task_tool_use_ids: set[str] = set()
async def pre_tool_use_hook(
input_data: HookInput,
@@ -181,7 +182,6 @@ def create_security_hooks(
context: HookContext,
) -> SyncHookJSONOutput:
"""Combined pre-tool-use validation hook."""
- nonlocal task_spawn_count
_ = context # unused but required by signature
tool_name = cast(str, input_data.get("tool_name", ""))
tool_input = cast(dict[str, Any], input_data.get("tool_input", {}))
@@ -200,18 +200,18 @@ def create_security_hooks(
"(remove the run_in_background parameter)."
),
)
- if task_spawn_count >= max_subtasks:
+ if len(task_tool_use_ids) >= max_subtasks:
logger.warning(
f"[SDK] Task limit reached ({max_subtasks}), user={user_id}"
)
return cast(
SyncHookJSONOutput,
_deny(
- f"Maximum {max_subtasks} sub-tasks per session. "
- "Please continue in the main conversation."
+ f"Maximum {max_subtasks} concurrent sub-tasks. "
+ "Wait for running sub-tasks to finish, "
+ "or continue in the main conversation."
),
)
- task_spawn_count += 1
# Strip MCP prefix for consistent validation
is_copilot_tool = tool_name.startswith(MCP_TOOL_PREFIX)
@@ -229,9 +229,24 @@ def create_security_hooks(
if result:
return cast(SyncHookJSONOutput, result)
+ # Reserve the Task slot only after all validations pass
+ if tool_name == "Task" and tool_use_id is not None:
+ task_tool_use_ids.add(tool_use_id)
+
logger.debug(f"[SDK] Tool start: {tool_name}, user={user_id}")
return cast(SyncHookJSONOutput, {})
+ def _release_task_slot(tool_name: str, tool_use_id: str | None) -> None:
+ """Release a Task concurrency slot if one was reserved."""
+ if tool_name == "Task" and tool_use_id in task_tool_use_ids:
+ task_tool_use_ids.discard(tool_use_id)
+ logger.info(
+ "[SDK] Task slot released, active=%d/%d, user=%s",
+ len(task_tool_use_ids),
+ max_subtasks,
+ user_id,
+ )
+
async def post_tool_use_hook(
input_data: HookInput,
tool_use_id: str | None,
@@ -246,6 +261,8 @@ def create_security_hooks(
"""
_ = context
tool_name = cast(str, input_data.get("tool_name", ""))
+
+ _release_task_slot(tool_name, tool_use_id)
is_builtin = not tool_name.startswith(MCP_TOOL_PREFIX)
logger.info(
"[SDK] PostToolUse: %s (builtin=%s, tool_use_id=%s)",
@@ -289,6 +306,9 @@ def create_security_hooks(
f"[SDK] Tool failed: {tool_name}, error={error}, "
f"user={user_id}, tool_use_id={tool_use_id}"
)
+
+ _release_task_slot(tool_name, tool_use_id)
+
return cast(SyncHookJSONOutput, {})
async def pre_compact_hook(
diff --git a/autogpt_platform/backend/backend/copilot/sdk/security_hooks_test.py b/autogpt_platform/backend/backend/copilot/sdk/security_hooks_test.py
index 01802f6410..bc49bbb1b9 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/security_hooks_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/security_hooks_test.py
@@ -208,19 +208,22 @@ def test_bash_builtin_blocked_message_clarity():
@pytest.fixture()
def _hooks():
- """Create security hooks and return the PreToolUse handler."""
+ """Create security hooks and return (pre, post, post_failure) handlers."""
from .security_hooks import create_security_hooks
hooks = create_security_hooks(user_id="u1", sdk_cwd=SDK_CWD, max_subtasks=2)
pre = hooks["PreToolUse"][0].hooks[0]
- return pre
+ post = hooks["PostToolUse"][0].hooks[0]
+ post_failure = hooks["PostToolUseFailure"][0].hooks[0]
+ return pre, post, post_failure
@pytest.mark.skipif(not _sdk_available(), reason="claude_agent_sdk not installed")
@pytest.mark.asyncio
async def test_task_background_blocked(_hooks):
"""Task with run_in_background=true must be denied."""
- result = await _hooks(
+ pre, _, _ = _hooks
+ result = await pre(
{"tool_name": "Task", "tool_input": {"run_in_background": True, "prompt": "x"}},
tool_use_id=None,
context={},
@@ -233,9 +236,10 @@ async def test_task_background_blocked(_hooks):
@pytest.mark.asyncio
async def test_task_foreground_allowed(_hooks):
"""Task without run_in_background should be allowed."""
- result = await _hooks(
+ pre, _, _ = _hooks
+ result = await pre(
{"tool_name": "Task", "tool_input": {"prompt": "do stuff"}},
- tool_use_id=None,
+ tool_use_id="tu-1",
context={},
)
assert not _is_denied(result)
@@ -245,25 +249,102 @@ async def test_task_foreground_allowed(_hooks):
@pytest.mark.asyncio
async def test_task_limit_enforced(_hooks):
"""Task spawns beyond max_subtasks should be denied."""
+ pre, _, _ = _hooks
# First two should pass
- for _ in range(2):
- result = await _hooks(
+ for i in range(2):
+ result = await pre(
{"tool_name": "Task", "tool_input": {"prompt": "ok"}},
- tool_use_id=None,
+ tool_use_id=f"tu-limit-{i}",
context={},
)
assert not _is_denied(result)
# Third should be denied (limit=2)
- result = await _hooks(
+ result = await pre(
{"tool_name": "Task", "tool_input": {"prompt": "over limit"}},
- tool_use_id=None,
+ tool_use_id="tu-limit-2",
context={},
)
assert _is_denied(result)
assert "Maximum" in _reason(result)
[email protected](not _sdk_available(), reason="claude_agent_sdk not installed")
+async def test_task_slot_released_on_completion(_hooks):
+ """Completing a Task should free a slot so new Tasks can be spawned."""
+ pre, post, _ = _hooks
+ # Fill both slots
+ for i in range(2):
+ result = await pre(
+ {"tool_name": "Task", "tool_input": {"prompt": "ok"}},
+ tool_use_id=f"tu-comp-{i}",
+ context={},
+ )
+ assert not _is_denied(result)
+
+ # Third should be denied — at capacity
+ result = await pre(
+ {"tool_name": "Task", "tool_input": {"prompt": "over"}},
+ tool_use_id="tu-comp-2",
+ context={},
+ )
+ assert _is_denied(result)
+
+ # Complete first task — frees a slot
+ await post(
+ {"tool_name": "Task", "tool_input": {}},
+ tool_use_id="tu-comp-0",
+ context={},
+ )
+
+ # Now a new Task should be allowed
+ result = await pre(
+ {"tool_name": "Task", "tool_input": {"prompt": "after release"}},
+ tool_use_id="tu-comp-3",
+ context={},
+ )
+ assert not _is_denied(result)
+
+
[email protected](not _sdk_available(), reason="claude_agent_sdk not installed")
+async def test_task_slot_released_on_failure(_hooks):
+ """A failed Task should also free its concurrency slot."""
+ pre, _, post_failure = _hooks
+ # Fill both slots
+ for i in range(2):
+ result = await pre(
+ {"tool_name": "Task", "tool_input": {"prompt": "ok"}},
+ tool_use_id=f"tu-fail-{i}",
+ context={},
+ )
+ assert not _is_denied(result)
+
+ # At capacity
+ result = await pre(
+ {"tool_name": "Task", "tool_input": {"prompt": "over"}},
+ tool_use_id="tu-fail-2",
+ context={},
+ )
+ assert _is_denied(result)
+
+ # Fail first task — should free a slot
+ await post_failure(
+ {"tool_name": "Task", "tool_input": {}, "error": "something broke"},
+ tool_use_id="tu-fail-0",
+ context={},
+ )
+
+ # New Task should be allowed
+ result = await pre(
+ {"tool_name": "Task", "tool_input": {"prompt": "after failure"}},
+ tool_use_id="tu-fail-3",
+ context={},
+ )
+ assert not _is_denied(result)
+
+
# -- _is_tool_error_or_denial ------------------------------------------------
@@ -298,7 +379,9 @@ class TestIsToolErrorOrDenial:
def test_subtask_limit_denial(self):
assert (
_is_tool_error_or_denial(
- "Maximum 2 sub-tasks per session. Please continue in the main conversation."
+ "Maximum 2 concurrent sub-tasks. "
+ "Wait for running sub-tasks to finish, "
+ "or continue in the main conversation."
)
is True
)
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.py
index 887f8cc18f..65618c3102 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py
@@ -75,6 +75,10 @@ class CapturedTranscript:
_SDK_CWD_PREFIX = WORKSPACE_PREFIX
+# Special message prefixes for text-based markers (parsed by frontend)
+COPILOT_ERROR_PREFIX = "[COPILOT_ERROR]" # Renders as ErrorCard
+COPILOT_SYSTEM_PREFIX = "[COPILOT_SYSTEM]" # Renders as system info message
+
# Heartbeat interval — keep SSE alive through proxies/LBs during tool execution.
# IMPORTANT: Must be less than frontend timeout (12s in useCopilotPage.ts)
_HEARTBEAT_INTERVAL = 10.0 # seconds
@@ -413,6 +417,20 @@ async def stream_chat_completion_sdk(
# Type narrowing: session is guaranteed ChatSession after the check above
session = cast(ChatSession, session)
+ # Clean up stale error markers from previous turn before starting new turn
+ # If the last message contains an error marker, remove it (user is retrying)
+ if (
+ len(session.messages) > 0
+ and session.messages[-1].role == "assistant"
+ and session.messages[-1].content
+ and COPILOT_ERROR_PREFIX in session.messages[-1].content
+ ):
+ logger.info(
+ "[SDK] [%s] Removing stale error marker from previous turn",
+ session_id[:12],
+ )
+ session.messages.pop()
+
# Append the new message to the session if it's not already there
new_message_role = "user" if is_user_message else "assistant"
if message and (
@@ -725,6 +743,25 @@ async def stream_chat_completion_sdk(
- len(adapter.resolved_tool_calls),
)
+ # Log ResultMessage details for debugging
+ if isinstance(sdk_msg, ResultMessage):
+ logger.info(
+ "[SDK] [%s] Received: ResultMessage %s "
+ "(unresolved=%d, current=%d, resolved=%d)",
+ session_id[:12],
+ sdk_msg.subtype,
+ len(adapter.current_tool_calls)
+ - len(adapter.resolved_tool_calls),
+ len(adapter.current_tool_calls),
+ len(adapter.resolved_tool_calls),
+ )
+ if sdk_msg.subtype in ("error", "error_during_execution"):
+ logger.error(
+ "[SDK] [%s] SDK execution failed with error: %s",
+ session_id[:12],
+ sdk_msg.result or "(no error message provided)",
+ )
+
for response in adapter.convert_message(sdk_msg):
if isinstance(response, StreamStart):
continue
@@ -749,6 +786,15 @@ async def stream_chat_completion_sdk(
extra,
)
+ # Log errors being sent to frontend
+ if isinstance(response, StreamError):
+ logger.error(
+ "[SDK] [%s] Sending error to frontend: %s (code=%s)",
+ session_id[:12],
+ response.errorText,
+ response.code,
+ )
+
yield response
if isinstance(response, StreamTextDelta):
@@ -855,13 +901,13 @@ async def stream_chat_completion_sdk(
yield response
# If the stream ended without a ResultMessage, the SDK
- # CLI exited unexpectedly. Close any open text/step so
- # the chunks are well-formed. StreamFinish is published
- # by mark_session_completed in the processor.
+ # CLI exited unexpectedly or the user stopped execution.
+ # Close any open text/step so chunks are well-formed, and
+ # append a cancellation message so users see feedback.
+ # StreamFinish is published by mark_session_completed in the processor.
if not stream_completed:
- logger.warning(
- "[SDK] [%s] Stream ended without ResultMessage "
- "(StopAsyncIteration)",
+ logger.info(
+ "[SDK] [%s] Stream ended without ResultMessage (stopped by user)",
session_id[:12],
)
closing_responses: list[StreamBaseResponse] = []
@@ -869,6 +915,15 @@ async def stream_chat_completion_sdk(
for r in closing_responses:
yield r
+ # Add "Stopped by user" message so it persists after refresh
+ # Use COPILOT_SYSTEM_PREFIX so frontend renders it as system message, not assistant
+ session.messages.append(
+ ChatMessage(
+ role="assistant",
+ content=f"{COPILOT_SYSTEM_PREFIX} Execution stopped by user",
+ )
+ )
+
if (
assistant_response.content or assistant_response.tool_calls
) and not has_appended_assistant:
@@ -922,43 +977,76 @@ async def stream_chat_completion_sdk(
"to use the OpenAI-compatible fallback."
)
- session = cast(ChatSession, await asyncio.shield(upsert_chat_session(session)))
logger.info(
- "[SDK] [%s] Session saved with %d messages",
+ "[SDK] [%s] Stream completed successfully with %d messages",
session_id[:12],
len(session.messages),
)
- except asyncio.CancelledError:
- # Client disconnect / server shutdown — save session before re-raising
- # so accumulated messages aren't lost.
- logger.warning("[SDK] [%s] Session cancelled (CancelledError)", session_id[:12])
+ except BaseException as e:
+ # Catch BaseException to handle both Exception and CancelledError
+ # (CancelledError inherits from BaseException in Python 3.8+)
+ if isinstance(e, asyncio.CancelledError):
+ logger.warning("[SDK] [%s] Session cancelled", session_id[:12])
+ error_msg = "Operation cancelled"
+ else:
+ error_msg = str(e) or type(e).__name__
+ # SDK cleanup RuntimeError is expected during cancellation, log as warning
+ if isinstance(e, RuntimeError) and "cancel scope" in str(e):
+ logger.warning(
+ "[SDK] [%s] SDK cleanup error: %s", session_id[:12], error_msg
+ )
+ else:
+ logger.error(
+ f"[SDK] [%s] Error: {error_msg}", session_id[:12], exc_info=True
+ )
+
+ # Append error marker to session (non-invasive text parsing approach)
+ # The finally block will persist the session with this error marker
if session:
+ session.messages.append(
+ ChatMessage(
+ role="assistant", content=f"{COPILOT_ERROR_PREFIX} {error_msg}"
+ )
+ )
+ logger.debug(
+ "[SDK] [%s] Appended error marker, will be persisted in finally",
+ session_id[:12],
+ )
+
+ # Yield StreamError for immediate feedback (only for non-cancellation errors)
+ # Skip for CancelledError and RuntimeError cleanup issues (both are cancellations)
+ is_cancellation = isinstance(e, asyncio.CancelledError) or (
+ isinstance(e, RuntimeError) and "cancel scope" in str(e)
+ )
+ if not is_cancellation:
+ yield StreamError(
+ errorText=error_msg,
+ code="sdk_error",
+ )
+
+ raise
+ finally:
+ # --- Persist session messages ---
+ # This MUST run in finally to persist messages even when the generator
+ # is stopped early (e.g., user clicks stop, processor breaks stream loop).
+ # Without this, messages disappear after refresh because they were never
+ # saved to the database.
+ if session is not None:
try:
await asyncio.shield(upsert_chat_session(session))
logger.info(
- "[SDK] [%s] Session saved on cancel (%d messages)",
+ "[SDK] [%s] Session persisted in finally with %d messages",
session_id[:12],
len(session.messages),
)
- except Exception as save_err:
+ except Exception as persist_err:
logger.error(
- "[SDK] [%s] Failed to save session on cancel: %s",
+ "[SDK] [%s] Failed to persist session in finally: %s",
session_id[:12],
- save_err,
+ persist_err,
+ exc_info=True,
)
- raise
- except Exception as e:
- logger.error(f"[SDK] Error: {e}", exc_info=True)
- if session:
- try:
- await asyncio.shield(upsert_chat_session(session))
- except Exception as save_err:
- logger.error(f"[SDK] Failed to save session on error: {save_err}")
- yield StreamError(
- errorText="An error occurred. Please try again.",
- code="sdk_error",
- )
- finally:
+
# --- Upload transcript for next-turn --resume ---
# This MUST run in finally so the transcript is uploaded even when
# the streaming loop raises an exception. The CLI uses
diff --git a/autogpt_platform/backend/backend/copilot/stream_registry.py b/autogpt_platform/backend/backend/copilot/stream_registry.py
index 90b871b238..80ddaa8321 100644
--- a/autogpt_platform/backend/backend/copilot/stream_registry.py
+++ b/autogpt_platform/backend/backend/copilot/stream_registry.py
@@ -707,7 +707,6 @@ async def mark_session_completed(
True if session was newly marked completed, False if already completed/failed
"""
status: Literal["completed", "failed"] = "failed" if error_message else "completed"
-
redis = await get_redis_async()
meta_key = _get_session_meta_key(session_id)
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/ChatMessagesContainer.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/ChatMessagesContainer.tsx
index c0a9e5da10..0dc8e60c69 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/ChatMessagesContainer.tsx
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/components/ChatMessagesContainer/ChatMessagesContainer.tsx
@@ -10,6 +10,7 @@ import {
MessageResponse,
} from "@/components/ai-elements/message";
import { LoadingSpinner } from "@/components/atoms/LoadingSpinner/LoadingSpinner";
+import { ErrorCard } from "@/components/molecules/ErrorCard/ErrorCard";
import { ToolUIPart, UIDataTypes, UIMessage, UITools } from "ai";
import { useEffect, useState } from "react";
import { CreateAgentTool } from "../../tools/CreateAgent/CreateAgent";
@@ -27,9 +28,56 @@ import { GenericTool } from "../../tools/GenericTool/GenericTool";
import { ViewAgentOutputTool } from "../../tools/ViewAgentOutput/ViewAgentOutput";
// ---------------------------------------------------------------------------
-// Workspace media support
+// Special text parsing (error markers, workspace URLs, etc.)
// ---------------------------------------------------------------------------
+// Special message prefixes for text-based markers (set by backend)
+const COPILOT_ERROR_PREFIX = "[COPILOT_ERROR]";
+const COPILOT_SYSTEM_PREFIX = "[COPILOT_SYSTEM]";
+
+type MarkerType = "error" | "system" | null;
+
+/**
+ * Parse special markers from message content (error, system).
+ *
+ * Detects markers added by the backend for special rendering:
+ * - `[COPILOT_ERROR] message` → ErrorCard
+ * - `[COPILOT_SYSTEM] message` → System info message
+ *
+ * Returns marker type, marker text, and cleaned text.
+ */
+function parseSpecialMarkers(text: string): {
+ markerType: MarkerType;
+ markerText: string;
+ cleanText: string;
+} {
+ // Check for error marker
+ const errorMatch = text.match(
+ new RegExp(`\\${COPILOT_ERROR_PREFIX}\\s*(.+?)$`, "s"),
+ );
+ if (errorMatch) {
+ return {
+ markerType: "error",
+ markerText: errorMatch[1].trim(),
+ cleanText: text.replace(errorMatch[0], "").trim(),
+ };
+ }
+
+ // Check for system marker
+ const systemMatch = text.match(
+ new RegExp(`\\${COPILOT_SYSTEM_PREFIX}\\s*(.+?)$`, "s"),
+ );
+ if (systemMatch) {
+ return {
+ markerType: "system",
+ markerText: systemMatch[1].trim(),
+ cleanText: text.replace(systemMatch[0], "").trim(),
+ };
+ }
+
+ return { markerType: null, markerText: "", cleanText: text };
+}
+
/**
* Resolve workspace:// URLs in markdown text to proxy download URLs.
*
@@ -147,24 +195,42 @@ export const ChatMessagesContainer = ({
}: ChatMessagesContainerProps) => {
const [thinkingPhrase, setThinkingPhrase] = useState(getRandomPhrase);
- useEffect(() => {
- if (status === "submitted") {
- setThinkingPhrase(getRandomPhrase());
- }
- }, [status]);
-
const lastMessage = messages[messages.length - 1];
- const lastAssistantHasVisibleContent =
- lastMessage?.role === "assistant" &&
- lastMessage.parts.some(
- (p) =>
- (p.type === "text" && p.text.trim().length > 0) ||
- p.type.startsWith("tool-"),
- );
+
+ // Determine if something is visibly "in-flight" in the last assistant message:
+ // - Text is actively streaming (last part is non-empty text)
+ // - A tool call is pending (state is input-streaming or input-available)
+ const hasInflight = (() => {
+ if (lastMessage?.role !== "assistant") return false;
+ const parts = lastMessage.parts;
+ if (parts.length === 0) return false;
+
+ const lastPart = parts[parts.length - 1];
+
+ // Text is actively being written
+ if (lastPart.type === "text" && lastPart.text.trim().length > 0)
+ return true;
+
+ // A tool call is still pending (no output yet)
+ if (
+ lastPart.type.startsWith("tool-") &&
+ "state" in lastPart &&
+ (lastPart.state === "input-streaming" ||
+ lastPart.state === "input-available")
+ )
+ return true;
+
+ return false;
+ })();
const showThinking =
- status === "submitted" ||
- (status === "streaming" && !lastAssistantHasVisibleContent);
+ status === "submitted" || (status === "streaming" && !hasInflight);
+
+ useEffect(() => {
+ if (showThinking) {
+ setThinkingPhrase(getRandomPhrase());
+ }
+ }, [showThinking]);
return (
<Conversation className="min-h-0 flex-1">
@@ -182,11 +248,6 @@ export const ChatMessagesContainer = ({
const isLastAssistant =
messageIndex === messages.length - 1 &&
message.role === "assistant";
- const messageHasVisibleContent = message.parts.some(
- (p) =>
- (p.type === "text" && p.text.trim().length > 0) ||
- p.type.startsWith("tool-"),
- );
return (
<Message from={message.role} key={message.id}>
@@ -199,15 +260,41 @@ export const ChatMessagesContainer = ({
>
{message.parts.map((part, i) => {
switch (part.type) {
- case "text":
+ case "text": {
+ // Check for special markers (error, system)
+ const { markerType, markerText, cleanText } =
+ parseSpecialMarkers(part.text);
+
+ if (markerType === "error") {
+ return (
+ <ErrorCard
+ key={`${message.id}-${i}`}
+ responseError={{ message: markerText }}
+ context="execution"
+ />
+ );
+ }
+
+ if (markerType === "system") {
+ return (
+ <div
+ key={`${message.id}-${i}`}
+ className="my-2 rounded-lg bg-neutral-100 px-3 py-2 text-sm italic text-neutral-600"
+ >
+ {markerText}
+ </div>
+ );
+ }
+
return (
<MessageResponse
key={`${message.id}-${i}`}
components={STREAMDOWN_COMPONENTS}
>
- {resolveWorkspaceUrls(part.text)}
+ {resolveWorkspaceUrls(cleanText)}
</MessageResponse>
);
+ }
case "tool-find_block":
return (
<FindBlocksTool
@@ -295,13 +382,11 @@ export const ChatMessagesContainer = ({
return null;
}
})}
- {isLastAssistant &&
- !messageHasVisibleContent &&
- showThinking && (
- <span className="inline-block animate-shimmer bg-gradient-to-r from-neutral-400 via-neutral-600 to-neutral-400 bg-[length:200%_100%] bg-clip-text text-transparent">
- {thinkingPhrase}
- </span>
- )}
+ {isLastAssistant && showThinking && (
+ <span className="inline-block animate-shimmer bg-gradient-to-r from-neutral-400 via-neutral-600 to-neutral-400 bg-[length:200%_100%] bg-clip-text text-transparent">
+ {thinkingPhrase}
+ </span>
+ )}
</MessageContent>
</Message>
);
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/useChatSession.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/useChatSession.ts
index b9a3702552..337eb37797 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/useChatSession.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/useChatSession.ts
@@ -18,7 +18,7 @@ export function useChatSession() {
const sessionQuery = useGetV2GetSession(sessionId ?? "", {
query: {
enabled: !!sessionId,
- staleTime: Infinity,
+ staleTime: Infinity, // Manual invalidation on session switch
refetchOnWindowFocus: false,
refetchOnReconnect: true,
refetchOnMount: true,
@@ -47,7 +47,7 @@ export function useChatSession() {
const hasActiveStream = useMemo(() => {
if (sessionQuery.data?.status !== 200) return false;
return !!sessionQuery.data.data.active_stream;
- }, [sessionQuery.data]);
+ }, [sessionQuery.data, sessionId]);
// Memoize so the effect in useCopilotPage doesn't infinite-loop on a new
// array reference every render. Re-derives only when query data changes.
@@ -119,5 +119,6 @@ export function useChatSession() {
isSessionError: sessionQuery.isError,
createSession,
isCreatingSession,
+ refetchSession: sessionQuery.refetch,
};
}
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/useCopilotPage.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/useCopilotPage.ts
index 4794a3da0a..20e1e6a4cd 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/useCopilotPage.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/useCopilotPage.ts
@@ -15,7 +15,9 @@ import type { UIMessage } from "ai";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { useChatSession } from "./useChatSession";
-const STREAM_START_TIMEOUT_MS = 12_000;
+const RECONNECT_BASE_DELAY_MS = 1_000;
+const RECONNECT_MAX_DELAY_MS = 30_000;
+const RECONNECT_MAX_ATTEMPTS = 5;
/** Mark any in-progress tool parts as completed/errored so spinners stop. */
function resolveInProgressTools(
@@ -35,42 +37,12 @@ function resolveInProgressTools(
}));
}
-/** Build a fingerprint from a message's role + text/tool content for cross-boundary dedup. */
-function messageFingerprint(msg: UIMessage): string {
- const fragments = msg.parts.map((p) => {
- if ("text" in p && typeof p.text === "string") return p.text;
- if ("toolCallId" in p && typeof p.toolCallId === "string")
- return `tool:${p.toolCallId}`;
- return "";
- });
- return `${msg.role}::${fragments.join("\n")}`;
-}
-
-/**
- * Deduplicate messages by ID *and* by content fingerprint.
- * ID-based dedup catches duplicates within the same source (e.g. two
- * identical stream events). Fingerprint-based dedup catches duplicates
- * across the hydration/stream boundary where IDs differ (synthetic
- * `${sessionId}-${index}` vs AI SDK nanoid).
- *
- * NOTE: Fingerprint dedup only applies to assistant messages, not user messages.
- * Users should be able to send the same message multiple times.
- */
+/** Simple ID-based deduplication - trust backend for correctness */
function deduplicateMessages(messages: UIMessage[]): UIMessage[] {
const seenIds = new Set<string>();
- const seenFingerprints = new Set<string>();
return messages.filter((msg) => {
if (seenIds.has(msg.id)) return false;
seenIds.add(msg.id);
-
- // Only apply fingerprint deduplication to assistant messages
- // User messages should allow duplicates (same text sent multiple times)
- if (msg.role === "assistant") {
- const fp = messageFingerprint(msg);
- if (fp !== "::" && seenFingerprints.has(fp)) return false;
- seenFingerprints.add(fp);
- }
-
return true;
});
}
@@ -94,6 +66,7 @@ export function useCopilotPage() {
isSessionError,
createSession,
isCreatingSession,
+ refetchSession,
} = useChatSession();
const { mutate: deleteSessionMutation, isPending: isDeleting } =
@@ -153,6 +126,48 @@ export function useCopilotPage() {
[sessionId],
);
+ // Reconnect state
+ const [reconnectAttempts, setReconnectAttempts] = useState(0);
+ const [isReconnectScheduled, setIsReconnectScheduled] = useState(false);
+ const reconnectTimerRef = useRef<ReturnType<typeof setTimeout>>();
+ const hasShownDisconnectToast = useRef(false);
+
+ // Consolidated reconnect logic
+ function handleReconnect(sid: string) {
+ if (isReconnectScheduled || !sid) return;
+
+ const nextAttempt = reconnectAttempts + 1;
+ if (nextAttempt > RECONNECT_MAX_ATTEMPTS) {
+ toast({
+ title: "Connection lost",
+ description: "Unable to reconnect. Please refresh the page.",
+ variant: "destructive",
+ });
+ return;
+ }
+
+ setIsReconnectScheduled(true);
+ setReconnectAttempts(nextAttempt);
+
+ if (!hasShownDisconnectToast.current) {
+ hasShownDisconnectToast.current = true;
+ toast({
+ title: "Connection lost",
+ description: "Reconnecting...",
+ });
+ }
+
+ const delay = Math.min(
+ RECONNECT_BASE_DELAY_MS * 2 ** reconnectAttempts,
+ RECONNECT_MAX_DELAY_MS,
+ );
+
+ reconnectTimerRef.current = setTimeout(() => {
+ setIsReconnectScheduled(false);
+ resumeStream();
+ }, delay);
+ }
+
const {
messages: rawMessages,
sendMessage,
@@ -164,9 +179,32 @@ export function useCopilotPage() {
} = useChat({
id: sessionId ?? undefined,
transport: transport ?? undefined,
- // Don't use resume: true — it fires before hydration completes, causing
- // the hydrated messages to overwrite the resumed stream. Instead we
- // call resumeStream() manually after hydration + active_stream detection.
+ onFinish: async ({ isDisconnect, isAbort }) => {
+ if (isAbort || !sessionId) return;
+
+ if (isDisconnect) {
+ handleReconnect(sessionId);
+ return;
+ }
+
+ // Check if backend executor is still running after clean close
+ const result = await refetchSession();
+ const backendActive =
+ result.data?.status === 200 && !!result.data.data.active_stream;
+
+ if (backendActive) {
+ handleReconnect(sessionId);
+ }
+ },
+ onError: (error) => {
+ if (!sessionId) return;
+ // Only reconnect on network errors (not HTTP errors)
+ const isNetworkError =
+ error.name === "TypeError" || error.name === "AbortError";
+ if (isNetworkError) {
+ handleReconnect(sessionId);
+ }
+ },
});
// Deduplicate messages continuously to prevent duplicates when resuming streams
@@ -205,44 +243,31 @@ export function useCopilotPage() {
}
}
- // Abort the stream if the backend doesn't start sending data within 12s.
- const stopRef = useRef(stop);
- stopRef.current = stop;
- useEffect(() => {
- if (status !== "submitted") return;
-
- const timer = setTimeout(() => {
- stopRef.current();
- toast({
- title: "Stream timed out",
- description: "The server took too long to respond. Please try again.",
- variant: "destructive",
- });
- }, STREAM_START_TIMEOUT_MS);
-
- return () => clearTimeout(timer);
- }, [status]);
-
- // Hydrate messages from the REST session endpoint.
- // Skip hydration while streaming to avoid overwriting the live stream.
+ // Hydrate messages from REST API when not actively streaming
useEffect(() => {
if (!hydratedMessages || hydratedMessages.length === 0) return;
if (status === "streaming" || status === "submitted") return;
+ if (isReconnectScheduled) return;
setMessages((prev) => {
if (prev.length >= hydratedMessages.length) return prev;
- // Deduplicate to handle rare cases where duplicate streams might occur
return deduplicateMessages(hydratedMessages);
});
- }, [hydratedMessages, setMessages, status]);
+ }, [hydratedMessages, setMessages, status, isReconnectScheduled]);
- // Ref: tracks whether we've already resumed for a given session.
- // Format: Map<sessionId, hasResumed>
+ // Track resume state per session
const hasResumedRef = useRef<Map<string, boolean>>(new Map());
- // When the stream ends (or drops), invalidate the session cache so the
- // next hydration fetches fresh messages from the backend. Without this,
- // staleTime: Infinity means the cache keeps the pre-stream data forever,
- // and any messages added during streaming are lost on remount/navigation.
+ // Clean up reconnect state on session switch
+ useEffect(() => {
+ clearTimeout(reconnectTimerRef.current);
+ reconnectTimerRef.current = undefined;
+ setReconnectAttempts(0);
+ setIsReconnectScheduled(false);
+ hasShownDisconnectToast.current = false;
+ prevStatusRef.current = status; // Reset to avoid cross-session state bleeding
+ }, [sessionId, status]);
+
+ // Invalidate session cache when stream completes
const prevStatusRef = useRef(status);
useEffect(() => {
const prev = prevStatusRef.current;
@@ -250,12 +275,17 @@ export function useCopilotPage() {
const wasActive = prev === "streaming" || prev === "submitted";
const isIdle = status === "ready" || status === "error";
- if (wasActive && isIdle && sessionId) {
+
+ if (wasActive && isIdle && sessionId && !isReconnectScheduled) {
queryClient.invalidateQueries({
queryKey: getGetV2GetSessionQueryKey(sessionId),
});
+ if (status === "ready") {
+ setReconnectAttempts(0);
+ hasShownDisconnectToast.current = false;
+ }
}
- }, [status, sessionId, queryClient]);
+ }, [status, sessionId, queryClient, isReconnectScheduled]);
// Resume an active stream AFTER hydration completes.
// IMPORTANT: Only runs when page loads with existing active stream (reconnection).
@@ -352,16 +382,16 @@ export function useCopilotPage() {
}
}, [isDeleting]);
- // True while we know the backend has an active stream but haven't
- // reconnected yet. Used to disable the send button and show stop UI.
+ // True while reconnecting or backend has active stream but we haven't connected yet
const isReconnecting =
- hasActiveStream && status !== "streaming" && status !== "submitted";
+ isReconnectScheduled ||
+ (hasActiveStream && status !== "streaming" && status !== "submitted");
return {
sessionId,
messages,
status,
- error,
+ error: isReconnecting ? undefined : error,
stop,
isReconnecting,
isLoadingSession,
diff --git a/autogpt_platform/frontend/src/app/api/chat/sessions/[sessionId]/stream/route.ts b/autogpt_platform/frontend/src/app/api/chat/sessions/[sessionId]/stream/route.ts
index bd27c77963..9b70ff2dca 100644
--- a/autogpt_platform/frontend/src/app/api/chat/sessions/[sessionId]/stream/route.ts
+++ b/autogpt_platform/frontend/src/app/api/chat/sessions/[sessionId]/stream/route.ts
@@ -3,6 +3,20 @@ import { getServerAuthToken } from "@/lib/autogpt-server-api/helpers";
import { NextRequest } from "next/server";
import { normalizeSSEStream, SSE_HEADERS } from "../../../sse-helpers";
+export const maxDuration = 800;
+
+const DEBUG_SSE_TIMEOUT_MS = process.env.NEXT_PUBLIC_SSE_TIMEOUT_MS
+ ? Number(process.env.NEXT_PUBLIC_SSE_TIMEOUT_MS)
+ : undefined;
+
+function debugSignal(): AbortSignal | undefined {
+ if (!DEBUG_SSE_TIMEOUT_MS) return undefined;
+ console.warn(
+ `[SSE_DEBUG] Simulating proxy timeout in ${DEBUG_SSE_TIMEOUT_MS}ms`,
+ );
+ return AbortSignal.timeout(DEBUG_SSE_TIMEOUT_MS);
+}
+
export async function POST(
request: NextRequest,
{ params }: { params: Promise<{ sessionId: string }> },
@@ -47,6 +61,7 @@ export async function POST(
is_user_message: is_user_message ?? true,
context: context || null,
}),
+ signal: debugSignal(),
});
if (!response.ok) {
@@ -110,6 +125,7 @@ export async function GET(
const response = await fetch(streamUrl.toString(), {
method: "GET",
headers,
+ signal: debugSignal(),
});
if (response.status === 204) {