Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): fix sub-agent session hang and orphan on E2B API stall (#12774)

Zamil Majdy92575ae76b43
diff --git a/autogpt_platform/backend/backend/blocks/autopilot.py b/autogpt_platform/backend/backend/blocks/autopilot.py
index 81d57e2372..af783b0757 100644
--- a/autogpt_platform/backend/backend/blocks/autopilot.py
+++ b/autogpt_platform/backend/backend/blocks/autopilot.py
@@ -4,6 +4,7 @@ import asyncio
import contextvars
import json
import logging
+import uuid
from typing import TYPE_CHECKING, Any
from typing_extensions import TypedDict # Needed for Python <3.12 compatibility
@@ -32,6 +33,10 @@ logger = logging.getLogger(__name__)
AUTOPILOT_BLOCK_ID = "c069dc6b-c3ed-4c12-b6e5-d47361e64ce6"
+class SubAgentRecursionError(RuntimeError):
+ """Raised when the sub-agent nesting depth limit is exceeded."""
+
+
class ToolCallEntry(TypedDict):
"""A single tool invocation record from an autopilot execution."""
@@ -410,8 +415,41 @@ class AutoPilotBlock(Block):
yield "session_id", sid
yield "error", "AutoPilot execution was cancelled."
raise
+ except SubAgentRecursionError as exc:
+ # Deliberate block — re-enqueueing would immediately hit the limit
+ # again, so skip recovery and just surface the error.
+ yield "session_id", sid
+ yield "error", str(exc)
except Exception as exc:
yield "session_id", sid
+ # Recovery enqueue must happen BEFORE yielding "error": the block
+ # framework (_base.execute) raises BlockExecutionError immediately
+ # when it sees ("error", ...) and stops consuming the generator,
+ # so any code after that yield is dead code in production.
+ effective_prompt = input_data.prompt
+ if input_data.system_context:
+ effective_prompt = (
+ f"[System Context: {input_data.system_context}]\n\n"
+ f"{input_data.prompt}"
+ )
+ try:
+ await _enqueue_for_recovery(
+ sid,
+ execution_context.user_id,
+ effective_prompt,
+ input_data.dry_run or execution_context.dry_run,
+ )
+ except asyncio.CancelledError:
+ # Task cancelled during recovery — still yield the error
+ # so the session_id + error pair is visible before re-raising.
+ yield "error", str(exc)
+ raise
+ except Exception:
+ logger.warning(
+ "AutoPilot session %s: recovery enqueue raised unexpectedly",
+ sid[:12],
+ exc_info=True,
+ )
yield "error", str(exc)
@@ -439,13 +477,13 @@ def _check_recursion(
when the caller exits to restore the previous depth.
Raises:
- RuntimeError: If the current depth already meets or exceeds the limit.
+ SubAgentRecursionError: If the current depth already meets or exceeds the limit.
"""
current = _autopilot_recursion_depth.get()
inherited = _autopilot_recursion_limit.get()
limit = max_depth if inherited is None else min(inherited, max_depth)
if current >= limit:
- raise RuntimeError(
+ raise SubAgentRecursionError(
f"AutoPilot recursion depth limit reached ({limit}). "
"The autopilot has called itself too many times."
)
@@ -536,3 +574,51 @@ def _merge_inherited_permissions(
# Return the token so the caller can restore the previous value in finally.
token = _inherited_permissions.set(merged)
return merged, token
+
+
+# ---------------------------------------------------------------------------
+# Recovery helpers
+# ---------------------------------------------------------------------------
+
+
+async def _enqueue_for_recovery(
+ session_id: str,
+ user_id: str,
+ message: str,
+ dry_run: bool,
+) -> None:
+ """Re-enqueue an orphaned sub-agent session so a fresh executor picks it up.
+
+ When ``execute_copilot`` raises an unexpected exception the sub-agent
+ session is left with ``last_role=user`` and no active consumer — identical
+ to the state that caused Toran's reports of silent sub-agents. Publishing
+ the original prompt back to the copilot queue lets the executor service
+ resume the session without manual intervention.
+
+ Skipped for dry-run sessions (no real consumers listen to the queue for
+ simulated sessions). Any failure to publish is logged and swallowed so
+ it never masks the original exception.
+ """
+ if dry_run:
+ return
+ try:
+ from backend.copilot.executor.utils import ( # avoid circular import
+ enqueue_copilot_turn,
+ )
+
+ await asyncio.wait_for(
+ enqueue_copilot_turn(
+ session_id=session_id,
+ user_id=user_id,
+ message=message,
+ turn_id=str(uuid.uuid4()),
+ ),
+ timeout=10,
+ )
+ logger.info("AutoPilot session %s enqueued for recovery", session_id[:12])
+ except Exception:
+ logger.warning(
+ "AutoPilot session %s: failed to enqueue for recovery",
+ session_id[:12],
+ exc_info=True,
+ )
diff --git a/autogpt_platform/backend/backend/blocks/test/test_autopilot.py b/autogpt_platform/backend/backend/blocks/test/test_autopilot.py
index a2b44ff38e..5fb468fb03 100644
--- a/autogpt_platform/backend/backend/blocks/test/test_autopilot.py
+++ b/autogpt_platform/backend/backend/blocks/test/test_autopilot.py
@@ -1,13 +1,14 @@
"""Tests for AutoPilotBlock: recursion guard, streaming, validation, and error paths."""
import asyncio
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, patch
import pytest
from backend.blocks.autopilot import (
AUTOPILOT_BLOCK_ID,
AutoPilotBlock,
+ SubAgentRecursionError,
_autopilot_recursion_depth,
_autopilot_recursion_limit,
_check_recursion,
@@ -57,7 +58,7 @@ class TestCheckRecursion:
try:
t2 = _check_recursion(2)
try:
- with pytest.raises(RuntimeError, match="recursion depth limit"):
+ with pytest.raises(SubAgentRecursionError):
_check_recursion(2)
finally:
_reset_recursion(t2)
@@ -71,7 +72,7 @@ class TestCheckRecursion:
t2 = _check_recursion(10) # inner wants 10, but inherited is 2
try:
# depth is now 2, limit is min(10, 2) = 2 → should raise
- with pytest.raises(RuntimeError, match="recursion depth limit"):
+ with pytest.raises(SubAgentRecursionError):
_check_recursion(10)
finally:
_reset_recursion(t2)
@@ -81,7 +82,7 @@ class TestCheckRecursion:
def test_limit_of_one_blocks_immediately_on_second_call(self):
t1 = _check_recursion(1)
try:
- with pytest.raises(RuntimeError):
+ with pytest.raises(SubAgentRecursionError):
_check_recursion(1)
finally:
_reset_recursion(t1)
@@ -244,3 +245,171 @@ class TestBlockRegistration:
# The field should exist (inherited) but there should be no explicit
# redefinition. We verify by checking the class __annotations__ directly.
assert "error" not in AutoPilotBlock.Output.__annotations__
+
+
+# ---------------------------------------------------------------------------
+# Recovery enqueue integration tests
+# ---------------------------------------------------------------------------
+
+
+class TestRecoveryEnqueue:
+ """Tests that run() enqueues orphaned sessions for recovery on failure."""
+
+ @pytest.fixture
+ def block(self):
+ return AutoPilotBlock()
+
+ @pytest.mark.asyncio
+ async def test_recovery_enqueued_on_transient_exception(self, block):
+ """A generic exception should trigger _enqueue_for_recovery."""
+ block.execute_copilot = AsyncMock(side_effect=RuntimeError("network error"))
+ block.create_session = AsyncMock(return_value="sess-recover")
+
+ input_data = block.Input(prompt="do work", max_recursion_depth=3)
+ ctx = _make_context()
+
+ with patch("backend.blocks.autopilot._enqueue_for_recovery") as mock_enqueue:
+ mock_enqueue.return_value = None
+ outputs = {}
+ async for name, value in block.run(input_data, execution_context=ctx):
+ outputs[name] = value
+
+ assert "network error" in outputs.get("error", "")
+ mock_enqueue.assert_awaited_once_with(
+ "sess-recover",
+ ctx.user_id,
+ "do work",
+ False,
+ )
+
+ @pytest.mark.asyncio
+ async def test_recovery_not_enqueued_for_recursion_limit(self, block):
+ """Recursion limit errors are deliberate — no recovery enqueue."""
+ block.execute_copilot = AsyncMock(
+ side_effect=SubAgentRecursionError(
+ "AutoPilot recursion depth limit reached (3). "
+ "The autopilot has called itself too many times."
+ )
+ )
+ block.create_session = AsyncMock(return_value="sess-rec-limit")
+
+ input_data = block.Input(prompt="recurse", max_recursion_depth=3)
+ ctx = _make_context()
+
+ with patch("backend.blocks.autopilot._enqueue_for_recovery") as mock_enqueue:
+ async for _ in block.run(input_data, execution_context=ctx):
+ pass
+
+ mock_enqueue.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_recovery_not_enqueued_for_dry_run(self, block):
+ """dry_run=True sessions must not be enqueued (no real consumers)."""
+ block.execute_copilot = AsyncMock(side_effect=RuntimeError("transient"))
+ block.create_session = AsyncMock(return_value="sess-dry-fail")
+
+ input_data = block.Input(prompt="test", max_recursion_depth=3, dry_run=True)
+ ctx = _make_context()
+
+ with patch("backend.blocks.autopilot._enqueue_for_recovery") as mock_enqueue:
+ mock_enqueue.return_value = None
+ async for _ in block.run(input_data, execution_context=ctx):
+ pass
+
+ # _enqueue_for_recovery is called with dry_run=True,
+ # so the inner guard returns early without publishing to the queue.
+ mock_enqueue.assert_awaited_once()
+ positional = mock_enqueue.call_args_list[0][0]
+ assert positional[3] is True # dry_run=True
+
+ @pytest.mark.asyncio
+ async def test_recovery_enqueue_failure_does_not_mask_original_error(self, block):
+ """If _enqueue_for_recovery itself raises, the original error is still yielded."""
+ block.execute_copilot = AsyncMock(side_effect=ValueError("original"))
+ block.create_session = AsyncMock(return_value="sess-enq-fail")
+
+ input_data = block.Input(prompt="hello", max_recursion_depth=3)
+ ctx = _make_context()
+
+ async def _failing_enqueue(*args, **kwargs):
+ raise OSError("rabbitmq down")
+
+ with patch(
+ "backend.blocks.autopilot._enqueue_for_recovery",
+ side_effect=_failing_enqueue,
+ ):
+ outputs = {}
+ async for name, value in block.run(input_data, execution_context=ctx):
+ outputs[name] = value
+
+ # Original error must still be surfaced despite the enqueue failure
+ assert outputs.get("error") == "original"
+ assert outputs.get("session_id") == "sess-enq-fail"
+
+ @pytest.mark.asyncio
+ async def test_recovery_uses_dry_run_from_context(self, block):
+ """execution_context.dry_run=True is OR-ed into the dry_run arg."""
+ block.execute_copilot = AsyncMock(side_effect=RuntimeError("fail"))
+ block.create_session = AsyncMock(return_value="sess-ctx-dry")
+
+ input_data = block.Input(prompt="test", max_recursion_depth=3, dry_run=False)
+ ctx = _make_context()
+ ctx.dry_run = True # outer execution is dry_run
+
+ with patch("backend.blocks.autopilot._enqueue_for_recovery") as mock_enqueue:
+ mock_enqueue.return_value = None
+ async for _ in block.run(input_data, execution_context=ctx):
+ pass
+
+ mock_enqueue.assert_awaited_once()
+ positional = mock_enqueue.call_args_list[0][0]
+ assert positional[3] is True # dry_run=True
+
+ @pytest.mark.asyncio
+ async def test_recovery_uses_effective_prompt_with_system_context(self, block):
+ """When system_context is set, _enqueue_for_recovery receives the
+ effective_prompt (system_context prepended) so the dedup check in
+ maybe_append_user_message passes on replay."""
+ block.execute_copilot = AsyncMock(side_effect=RuntimeError("e2b timeout"))
+ block.create_session = AsyncMock(return_value="sess-sys-ctx")
+
+ input_data = block.Input(
+ prompt="do work",
+ system_context="Be concise.",
+ max_recursion_depth=3,
+ )
+ ctx = _make_context()
+
+ with patch("backend.blocks.autopilot._enqueue_for_recovery") as mock_enqueue:
+ mock_enqueue.return_value = None
+ async for _ in block.run(input_data, execution_context=ctx):
+ pass
+
+ mock_enqueue.assert_awaited_once()
+ positional = mock_enqueue.call_args_list[0][0]
+ assert positional[2] == "[System Context: Be concise.]\n\ndo work"
+
+ @pytest.mark.asyncio
+ async def test_recovery_cancelled_error_still_yields_error(self, block):
+ """CancelledError during _enqueue_for_recovery still yields the error output."""
+ block.execute_copilot = AsyncMock(side_effect=RuntimeError("e2b stall"))
+ block.create_session = AsyncMock(return_value="sess-cancel")
+
+ async def _cancelled_enqueue(*args, **kwargs):
+ raise asyncio.CancelledError
+
+ outputs = {}
+ with patch(
+ "backend.blocks.autopilot._enqueue_for_recovery",
+ side_effect=_cancelled_enqueue,
+ ):
+ with pytest.raises(asyncio.CancelledError):
+ async for name, value in block.run(
+ block.Input(prompt="do work", max_recursion_depth=3),
+ execution_context=_make_context(),
+ ):
+ outputs[name] = value
+
+ # error must be yielded even when recovery raises CancelledError
+ assert outputs.get("error") == "e2b stall"
+ assert outputs.get("session_id") == "sess-cancel"
diff --git a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py
index 8da9749a65..038158fc41 100644
--- a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py
+++ b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py
@@ -31,14 +31,22 @@ The sandbox_id is stored in Redis. The same key doubles as a creation lock:
a ``"creating"`` sentinel value is written with a short TTL while a new sandbox
is being provisioned, preventing duplicate creation under concurrent requests.
-E2B project-level "paused sandbox lifetime" should be set to match
-``_SANDBOX_ID_TTL`` (48 h) so orphaned paused sandboxes are auto-killed before
-the Redis key expires.
+Sandbox lifetime
+----------------
+E2B assigns each sandbox an absolute ``end_at`` timestamp at create time:
+``end_at = now + timeout``. Pausing does NOT extend ``end_at``; only
+``connect()`` extends it (by ``timeout`` seconds from the moment of reconnect).
+Active sessions therefore stay alive as long as turns arrive within the timeout
+window. Orphaned sandboxes (e.g. leaked by a failed create retry) are paused
+(not killed) at ``end_at`` under the default ``on_timeout="pause"`` lifecycle;
+they persist until explicitly killed or until E2B's platform-level cleanup
+applies (30-day limit during beta).
"""
import asyncio
import contextlib
import logging
+import math
from typing import Any, Awaitable, Callable, Literal
from e2b import AsyncSandbox, SandboxLifecycle
@@ -50,11 +58,29 @@ logger = logging.getLogger(__name__)
_SANDBOX_KEY_PREFIX = "copilot:e2b:sandbox:"
_CREATING_SENTINEL = "creating"
+# Per-attempt timeout for AsyncSandbox.create(). E2B normally provisions a
+# sandbox in 5-15 s; 30 s gives generous headroom while ensuring a slow/hung
+# E2B API call fails fast rather than blocking an executor goroutine for hours.
+_SANDBOX_CREATE_TIMEOUT_SECONDS = 30
+
+# Number of creation attempts before giving up. Three attempts with 1 s / 2 s
+# backoff means the worst-case wait is ~93 s (30+1+30+2+30) — far better than
+# the indefinite hang that caused the original incident.
+_SANDBOX_CREATE_MAX_RETRIES = 3
+
# Short TTL for the "creating" sentinel — if the process dies mid-creation the
# lock auto-expires so other callers are not blocked forever.
-_CREATION_LOCK_TTL = 60 # seconds
+# Must be ≥ worst-case retry time: _SANDBOX_CREATE_MAX_RETRIES ×
+# _SANDBOX_CREATE_TIMEOUT_SECONDS + inter-retry backoff ≈ 93 s → 120 s.
+_CREATION_LOCK_TTL = 120 # seconds
+
+# Wait interval for followers polling the "creating" sentinel.
+_WAIT_INTERVAL_SECONDS = 0.5
-_MAX_WAIT_ATTEMPTS = 20 # 20 × 0.5 s = 10 s max wait
+# Derive follower budget from the lock TTL so it automatically tracks future
+# TTL changes. Add a 20% safety margin to handle slight clock drift / late
+# sentinel expiry. Result: ceil(120 / 0.5 * 1.2) = 288 iterations ≈ 144 s.
+_MAX_WAIT_ATTEMPTS = math.ceil(_CREATION_LOCK_TTL / _WAIT_INTERVAL_SECONDS * 1.2)
# Timeout for E2B API calls (pause/kill) — short because these are control-plane
# operations; if the sandbox is unreachable, fail fast and retry on the next turn.
@@ -145,7 +171,7 @@ async def get_or_create_sandbox(
if value == _CREATING_SENTINEL:
# Another coroutine is creating — wait for it to finish.
- await asyncio.sleep(0.5)
+ await asyncio.sleep(_WAIT_INTERVAL_SECONDS)
continue
# No sandbox and no active creation — atomically claim the creation slot.
@@ -157,25 +183,79 @@ async def get_or_create_sandbox(
await asyncio.sleep(0.1)
continue
- # We hold the slot — create the sandbox.
+ # We hold the slot — create the sandbox with per-attempt timeout and
+ # retry. The sentinel remains held throughout so concurrent callers
+ # for the same session wait rather than racing to create duplicates.
+ sandbox: AsyncSandbox | None = None
try:
lifecycle = SandboxLifecycle(
on_timeout=on_timeout,
auto_resume=on_timeout == "pause",
)
- sandbox = await AsyncSandbox.create(
- template=template,
- api_key=api_key,
- timeout=timeout,
- lifecycle=lifecycle,
- )
+ # Note: asyncio.wait_for() only cancels the client-side wait;
+ # E2B may complete provisioning server-side after a timeout.
+ # Since AsyncSandbox.create() returns no sandbox_id before
+ # completion, recovery via connect() is not possible and each
+ # timed-out attempt may leak a sandbox. Under the default
+ # on_timeout="pause" lifecycle, leaked orphans are paused (not
+ # killed) at end_at and persist until explicitly cleaned up.
+ # At most _SANDBOX_CREATE_MAX_RETRIES − 1 = 2 sandboxes can
+ # leak per incident.
+ last_exc: Exception | None = None
+ for attempt in range(1, _SANDBOX_CREATE_MAX_RETRIES + 1):
+ try:
+ sandbox = await asyncio.wait_for(
+ AsyncSandbox.create(
+ template=template,
+ api_key=api_key,
+ timeout=timeout,
+ lifecycle=lifecycle,
+ ),
+ timeout=_SANDBOX_CREATE_TIMEOUT_SECONDS,
+ )
+ last_exc = None
+ break
+ except Exception as exc:
+ last_exc = exc
+ logger.warning(
+ "[E2B] Sandbox creation attempt %d/%d failed for session %.12s: %s",
+ attempt,
+ _SANDBOX_CREATE_MAX_RETRIES,
+ session_id,
+ exc,
+ )
+ if attempt < _SANDBOX_CREATE_MAX_RETRIES:
+ await asyncio.sleep(2 ** (attempt - 1)) # 1 s, 2 s
+
+ if last_exc is not None:
+ raise last_exc
+
+ assert sandbox is not None # guaranteed: last_exc is None iff break was hit
try:
await _set_stored_sandbox_id(session_id, sandbox.sandbox_id)
except Exception:
# Redis save failed — kill the sandbox to avoid leaking it.
with contextlib.suppress(Exception):
- await sandbox.kill()
+ await asyncio.wait_for(
+ sandbox.kill(), timeout=_E2B_API_TIMEOUT_SECONDS
+ )
raise
+ except asyncio.CancelledError:
+ # Task cancelled during creation — release the slot so followers
+ # are not blocked for the full TTL (120 s). CancelledError inherits
+ # from BaseException, not Exception, so it is not caught above.
+ # Kill the sandbox if it was already created to avoid leaking it
+ # (can happen when cancellation fires during _set_stored_sandbox_id).
+ # Suppress BaseException (including a second CancelledError) so a
+ # re-entrant cancellation during cleanup cannot skip the redis.delete.
+ with contextlib.suppress(Exception, asyncio.CancelledError):
+ await redis.delete(key)
+ if sandbox is not None:
+ with contextlib.suppress(Exception, asyncio.CancelledError):
+ await asyncio.wait_for(
+ sandbox.kill(), timeout=_E2B_API_TIMEOUT_SECONDS
+ )
+ raise
except Exception:
# Release the creation slot so other callers can proceed.
await redis.delete(key)
diff --git a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py
index a4b72c079c..7eb8b78ec6 100644
--- a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py
@@ -18,6 +18,7 @@ import pytest
from .e2b_sandbox import (
_CREATING_SENTINEL,
+ _SANDBOX_CREATE_MAX_RETRIES,
_try_reconnect,
get_or_create_sandbox,
kill_sandbox,
@@ -259,6 +260,142 @@ class TestGetOrCreateSandbox:
assert result is sb
+ def test_create_retries_on_timeout_then_succeeds(self):
+ """On first-attempt timeout, retries and succeeds on second attempt."""
+ new_sb = _mock_sandbox("sb-retry")
+ redis = _mock_redis(set_nx_result=True, stored_sandbox_id=None)
+
+ call_count = 0
+
+ async def _create_side_effect(**kwargs):
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ raise asyncio.TimeoutError
+ return new_sb
+
+ with (
+ patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_redis(redis),
+ patch(
+ "backend.copilot.tools.e2b_sandbox.asyncio.sleep",
+ new_callable=AsyncMock,
+ ),
+ ):
+ mock_cls.create = AsyncMock(side_effect=_create_side_effect)
+ result = asyncio.run(
+ get_or_create_sandbox(_SESSION_ID, _API_KEY, timeout=_TIMEOUT)
+ )
+
+ assert result is new_sb
+ assert call_count == 2
+
+ def test_create_exhausts_all_retries_then_raises(self):
+ """When all retry attempts fail, the last exception is re-raised."""
+ redis = _mock_redis(set_nx_result=True, stored_sandbox_id=None)
+
+ with (
+ patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_redis(redis),
+ patch(
+ "backend.copilot.tools.e2b_sandbox.asyncio.sleep",
+ new_callable=AsyncMock,
+ ),
+ ):
+ mock_cls.create = AsyncMock(side_effect=asyncio.TimeoutError)
+ with pytest.raises(asyncio.TimeoutError):
+ asyncio.run(
+ get_or_create_sandbox(_SESSION_ID, _API_KEY, timeout=_TIMEOUT)
+ )
+
+ assert mock_cls.create.await_count == _SANDBOX_CREATE_MAX_RETRIES
+ # Creation slot must be released even after full retry exhaustion
+ redis.delete.assert_awaited_once()
+
+ def test_create_non_timeout_exception_also_retried(self):
+ """Non-timeout exceptions (e.g., network errors) are also retried."""
+ new_sb = _mock_sandbox("sb-net-retry")
+ redis = _mock_redis(set_nx_result=True, stored_sandbox_id=None)
+
+ call_count = 0
+
+ async def _create_side_effect(**kwargs):
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ raise ConnectionError("temporary network blip")
+ return new_sb
+
+ with (
+ patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_redis(redis),
+ patch(
+ "backend.copilot.tools.e2b_sandbox.asyncio.sleep",
+ new_callable=AsyncMock,
+ ),
+ ):
+ mock_cls.create = AsyncMock(side_effect=_create_side_effect)
+ result = asyncio.run(
+ get_or_create_sandbox(_SESSION_ID, _API_KEY, timeout=_TIMEOUT)
+ )
+
+ assert result is new_sb
+ assert call_count == 2
+
+ def test_create_cancellation_releases_creation_slot(self):
+ """CancelledError during creation must release the Redis sentinel."""
+ redis = _mock_redis(set_nx_result=True, stored_sandbox_id=None)
+
+ async def _create_side_effect(**kwargs):
+ raise asyncio.CancelledError
+
+ with (
+ patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ _patch_redis(redis),
+ patch(
+ "backend.copilot.tools.e2b_sandbox.asyncio.sleep",
+ new_callable=AsyncMock,
+ ),
+ ):
+ mock_cls.create = AsyncMock(side_effect=_create_side_effect)
+ with pytest.raises(asyncio.CancelledError):
+ asyncio.run(
+ get_or_create_sandbox(_SESSION_ID, _API_KEY, timeout=_TIMEOUT)
+ )
+
+ # Sentinel must be released even on task cancellation
+ redis.delete.assert_awaited_once()
+
+ def test_post_create_cancellation_kills_sandbox(self):
+ """CancelledError during _set_stored_sandbox_id must kill the already-created sandbox."""
+ redis = _mock_redis(set_nx_result=True, stored_sandbox_id=None)
+ created_sb = _mock_sandbox()
+
+ async def _set_side_effect(*_args, **_kwargs):
+ raise asyncio.CancelledError
+
+ with (
+ patch("backend.copilot.tools.e2b_sandbox.AsyncSandbox") as mock_cls,
+ patch(
+ "backend.copilot.tools.e2b_sandbox._set_stored_sandbox_id",
+ side_effect=_set_side_effect,
+ ),
+ _patch_redis(redis),
+ patch(
+ "backend.copilot.tools.e2b_sandbox.asyncio.sleep",
+ new_callable=AsyncMock,
+ ),
+ ):
+ mock_cls.create = AsyncMock(return_value=created_sb)
+ with pytest.raises(asyncio.CancelledError):
+ asyncio.run(
+ get_or_create_sandbox(_SESSION_ID, _API_KEY, timeout=_TIMEOUT)
+ )
+
+ # Sandbox must be killed and Redis sentinel cleared on post-create cancellation
+ created_sb.kill.assert_awaited_once()
+ redis.delete.assert_awaited_once()
+
def test_stale_reconnect_clears_and_creates(self):
"""When stored sandbox is stale (not running), clear it and create a new one."""
stale_sb = _mock_sandbox("sb-stale", running=False)