Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/copilot): bash_exec no longer reports a false timeout when the sandbox drops a running command (#15130)

Bently11ccebf4c5fa
diff --git a/autogpt_platform/backend/backend/copilot/constants.py b/autogpt_platform/backend/backend/copilot/constants.py
index 2978dc6f91..4d20792e17 100644
--- a/autogpt_platform/backend/backend/copilot/constants.py
+++ b/autogpt_platform/backend/backend/copilot/constants.py
@@ -76,6 +76,10 @@ MAX_TOOL_WAIT_SECONDS = 5 * 60 # 5 minutes
# "no tool blocks >= idle_timeout" holds by construction.
STREAM_IDLE_TIMEOUT_SECONDS = MAX_TOOL_WAIT_SECONDS * 2 # 10 minutes
+# The SDK turn gives up on a tool that is still pending after this long, aborts
+# the stream and pauses the E2B box.
+HUNG_TOOL_CAP_SECONDS = 2 * 60 * 60
+
# Redis key prefix for the SDK-stream-level lock that ensures only one
# active SDK stream per session. Released by the SDK turn's finally block
# and force-released by mark_session_completed on cancel/error.
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.py
index 14b851af5b..ee0b840be0 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py
@@ -90,6 +90,7 @@ from ..constants import (
COPILOT_ERROR_PREFIX,
COPILOT_RETRYABLE_ERROR_PREFIX,
FRIENDLY_TRANSIENT_MSG,
+ HUNG_TOOL_CAP_SECONDS,
STREAM_ERROR_MARKER,
STREAM_INCOMPLETE_MARKER,
STREAM_LOCK_PREFIX,
@@ -283,7 +284,7 @@ _CIRCUIT_BREAKER_ERROR_MSG = (
# Two regimes: no tool pending → 30 min (SDK genuinely idle); tool pending →
# 2 h hard cap (lets long sub-AutoPilots run, still backstops a hung tool).
_IDLE_TIMEOUT_SECONDS = 30 * 60
-_HUNG_TOOL_CAP_SECONDS = 2 * 60 * 60
+_HUNG_TOOL_CAP_SECONDS = HUNG_TOOL_CAP_SECONDS
# Floor on the per-query SDK budget — too small and the CLI refuses to
# start a turn at all. Caller (``_resolve_dynamic_max_budget_usd``) clamps
diff --git a/autogpt_platform/backend/backend/copilot/tools/bash_exec.py b/autogpt_platform/backend/backend/copilot/tools/bash_exec.py
index fda009a138..242fd9b61a 100644
--- a/autogpt_platform/backend/backend/copilot/tools/bash_exec.py
+++ b/autogpt_platform/backend/backend/copilot/tools/bash_exec.py
@@ -20,8 +20,10 @@ import shlex
from typing import Any
from e2b import AsyncSandbox, CommandExitException
-from e2b.exceptions import TimeoutException
+from e2b.exceptions import NotFoundException, TimeoutException
+from pydantic import BaseModel
+from backend.copilot.constants import HUNG_TOOL_CAP_SECONDS
from backend.copilot.context import (
E2B_WORKDIR,
get_current_sandbox,
@@ -38,6 +40,8 @@ from backend.copilot.integration_creds import (
get_integration_env_vars,
)
from backend.copilot.model import ChatSession
+from backend.copilot.sdk.env import config as chat_config
+from backend.util.e2b_network import reattach_command
from backend.util.sandbox_login import (
changed_login_files,
judged_text,
@@ -47,11 +51,71 @@ from backend.util.sandbox_login import (
from .base import BaseTool
from .connect_integration import requested_scopes
+from .e2b_sandbox import SandboxOwner, keep_sandbox_running
from .models import BashExecResponse, ErrorResponse, ToolResponseBase
from .sandbox import get_workspace_dir, has_full_sandbox, run_sandboxed
logger = logging.getLogger(__name__)
+# Headroom on the box's running-time limit past the command's own timeout, so
+# the box is still up to report the result (or the timeout) and to be killed.
+_SANDBOX_LIMIT_MARGIN_SECONDS = 60
+# A stream that breaks this close to the command's deadline is its timeout.
+# The SDK's own deadline starts after ours and is rounded to the millisecond,
+# so it never fires earlier than this; any earlier break is the box's doing.
+_DEADLINE_SLACK_SECONDS = 0.05
+# Reattaches to a running command after its stream to the box drops.
+_MAX_RECONNECTS = 3
+_KILL_TIMEOUT_SECONDS = 10
+# The turn's idle watchdog fires HUNG_TOOL_CAP_SECONDS after the last SDK
+# message, which is before the gate, keep_sandbox_running (up to ~40s) and
+# stream setup. Capping this far below it lets the tool's own timeout land first.
+_HUNG_CAP_MARGIN_SECONDS = 120
+
+# envd's kill reaches only the shell it started, so the children of a compound
+# command (`npm install && npm run build`) would keep running. This stops the
+# shell first, so it cannot start the next step once a child dies, then stops
+# and kills every process under it. The shell itself is left for envd's kill.
+# Exits non-zero if any of them is still alive afterwards (e.g. one run as root).
+_KILL_TREE_SCRIPT = r"""
+p=__PID__
+kill -STOP "$p" 2>/dev/null || exit 0
+under() {
+ awk -v root="$p" '
+ { pid = $1; s = $0; sub(/.*\) /, "", s); split(s, f, " ")
+ parent[pid] = f[2]; state[pid] = f[1] }
+ END { q[1] = root; n = 1
+ while (n) { r = q[n--]
+ for (k in parent) if (parent[k] == r) {
+ q[++n] = k; if (state[k] != "Z") print k } } }
+ ' /proc/[0-9]*/stat 2>/dev/null
+}
+kids=$(under); [ -n "$kids" ] && kill -STOP $kids 2>/dev/null
+kids=$(under); [ -n "$kids" ] && kill -KILL $kids 2>/dev/null
+sleep 0.2
+[ -z "$(under)" ]
+"""
+
+
+class _E2BRun(BaseModel):
+ """How a command on the box ended, with all the output we received."""
+
+ stdout: str
+ stderr: str
+ elapsed: float
+ exit_code: int | None = None
+ timed_out: bool = False
+ # Kill sent after a timeout: True done, False not found, None unknown.
+ killed: bool | None = None
+ # The processes the command started were killed along with it.
+ children_killed: bool = False
+ # Why we stopped following a command that had not ended: the stream
+ # error's type name.
+ lost: str | None = None
+ # The command ended while we were not attached: no exit code.
+ ended_unseen: bool = False
+ reconnects: int = 0
+
def _build_completion_response(
stdout: str | None,
@@ -199,6 +263,11 @@ class BashExecTool(BaseTool):
session_id,
user_id,
required_scopes=requested_scopes(session),
+ owner=(
+ SandboxOwner.for_session(session.session_id, session.expert_id)
+ if session
+ else None
+ ),
)
# Bubblewrap fallback: local isolated execution.
@@ -238,6 +307,7 @@ class BashExecTool(BaseTool):
session_id: str | None,
user_id: str | None = None,
required_scopes: dict[str, frozenset[str]] | None = None,
+ owner: SandboxOwner | None = None,
) -> ToolResponseBase:
"""Execute *command* on the E2B sandbox via commands.run().
@@ -245,6 +315,9 @@ class BashExecTool(BaseTool):
for any user with connected accounts. E2B has full internet access, so
CLI tools like ``gh`` work without manual authentication.
"""
+ # The turn stops following a tool after the hung-tool cap and pauses
+ # the box, so a longer timeout only keeps an unwatched box billing.
+ timeout = min(max(timeout, 1), HUNG_TOOL_CAP_SECONDS - _HUNG_CAP_MARGIN_SECONDS)
envs: dict[str, str] = {
"PATH": "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin",
}
@@ -266,31 +339,26 @@ class BashExecTool(BaseTool):
if git_identity:
envs.update(git_identity)
+ # The box's running-time limit was armed when this turn connected;
+ # E2B pauses the box at it, breaking the command's stream mid-run.
+ await keep_sandbox_running(
+ sandbox,
+ max(
+ chat_config.e2b_sandbox_timeout,
+ timeout + _SANDBOX_LIMIT_MARGIN_SECONDS,
+ ),
+ owner,
+ )
try:
- result = await sandbox.commands.run(
- f"bash -c {shlex.quote(command)}",
- cwd=E2B_WORKDIR,
- timeout=timeout,
- envs=envs,
- )
- return _build_completion_response(
- result.stdout,
- result.stderr,
- result.exit_code,
- secret_values,
- session_id,
- )
- except CommandExitException as exc:
- return _build_completion_response(
- exc.stdout, exc.stderr, exc.exit_code, secret_values, session_id
- )
- except TimeoutException:
- return BashExecResponse(
- message="Execution timed out",
- stdout="",
- stderr=f"Timed out after {timeout}s",
- exit_code=-1,
- timed_out=True,
+ run = await _follow_command(sandbox, command, timeout, envs)
+ except TimeoutException as exc:
+ return ErrorResponse(
+ message=(
+ "The sandbox did not confirm the command started within "
+ "the request timeout, so it may or may not have run "
+ f"({exc}). This is not the command's {timeout}s timeout."
+ ),
+ error="e2b_start_timeout",
session_id=session_id,
).from_outside()
except Exception as exc:
@@ -300,6 +368,174 @@ class BashExecTool(BaseTool):
error="e2b_execution_error",
session_id=session_id,
).from_outside(str(exc))
+ return _run_response(run, timeout, secret_values, session_id)
+
+
+async def _follow_command(
+ sandbox: AsyncSandbox, command: str, timeout: int, envs: dict[str, str]
+) -> _E2BRun:
+ """Run *command* on the box and follow it to its end or its *timeout*.
+
+ The SDK raises ``TimeoutException`` for more than the command's own
+ deadline: the box pausing or dying, or a proxy cancelling the stream,
+ surface the same way at any elapsed time. Only a break at the deadline
+ is the timeout (the command is then killed); before it, the command is
+ still running on the box, so the stream is reattached instead of the
+ command reported as failed. Output printed while detached is not
+ replayed by E2B. Raises only when the command could not be started.
+ """
+ loop = asyncio.get_running_loop()
+ stdout: list[str] = []
+ stderr: list[str] = []
+ started = loop.time()
+ deadline = started + timeout
+ handle = await sandbox.commands.run(
+ f"bash -c {shlex.quote(command)}",
+ background=True,
+ cwd=E2B_WORKDIR,
+ timeout=timeout,
+ envs=envs,
+ on_stdout=stdout.append,
+ on_stderr=stderr.append,
+ )
+ pid = handle.pid
+ reconnects = 0
+
+ def outcome(**kwargs: Any) -> _E2BRun:
+ return _E2BRun(
+ stdout="".join(stdout),
+ stderr="".join(stderr),
+ elapsed=loop.time() - started,
+ reconnects=reconnects,
+ **kwargs,
+ )
+
+ while True:
+ try:
+ result = await handle.wait()
+ return outcome(exit_code=result.exit_code)
+ except CommandExitException as exc:
+ return outcome(exit_code=exc.exit_code)
+ except Exception as exc:
+ remaining = deadline - loop.time()
+ if remaining <= _DEADLINE_SLACK_SECONDS:
+ return outcome(timed_out=True, **await _kill(sandbox, pid))
+ logger.warning(
+ "[E2B] bash_exec lost its stream to pid %s after %.1fs of %ds "
+ "(%s: %s)",
+ pid,
+ loop.time() - started,
+ timeout,
+ type(exc).__name__,
+ exc,
+ )
+ if reconnects >= _MAX_RECONNECTS:
+ return outcome(lost=type(exc).__name__)
+ reconnects += 1
+ try:
+ handle = await reattach_command(
+ sandbox,
+ pid,
+ timeout=remaining,
+ on_stdout=stdout.append,
+ on_stderr=stderr.append,
+ )
+ except NotFoundException:
+ return outcome(ended_unseen=True)
+ except Exception as reconnect_exc:
+ if deadline - loop.time() <= _DEADLINE_SLACK_SECONDS:
+ return outcome(timed_out=True, **await _kill(sandbox, pid))
+ logger.warning(
+ "[E2B] bash_exec could not reattach to pid %s: %s",
+ pid,
+ reconnect_exc,
+ )
+ return outcome(lost=type(exc).__name__)
+
+
+async def _kill(sandbox: AsyncSandbox, pid: int) -> dict[str, Any]:
+ children_killed = await _kill_children(sandbox, pid)
+ try:
+ killed = await asyncio.wait_for(
+ sandbox.commands.kill(pid), timeout=_KILL_TIMEOUT_SECONDS
+ )
+ except Exception as exc:
+ logger.warning("[E2B] Could not kill timed-out pid %s: %s", pid, exc)
+ killed = None
+ return {"killed": killed, "children_killed": children_killed}
+
+
+async def _kill_children(sandbox: AsyncSandbox, pid: int) -> bool:
+ try:
+ result = await run_internal(
+ sandbox,
+ _KILL_TREE_SCRIPT.replace("__PID__", str(pid)),
+ timeout=_KILL_TIMEOUT_SECONDS,
+ )
+ except Exception as exc:
+ logger.warning("[E2B] Could not kill the children of pid %s: %s", pid, exc)
+ return False
+ return result.exit_code == 0
+
+
+def _run_response(
+ run: _E2BRun, timeout: int, secret_values: list[str], session_id: str | None
+) -> BashExecResponse:
+ if run.exit_code is not None:
+ response = _build_completion_response(
+ run.stdout, run.stderr, run.exit_code, secret_values, session_id
+ )
+ if run.reconnects:
+ response.message += (
+ f" (the connection to the sandbox dropped {run.reconnects} "
+ "time(s) and was resumed; output printed meanwhile may be "
+ "missing)"
+ )
+ return response
+
+ elapsed = f"{run.elapsed:.1f}s"
+ if run.timed_out:
+ if run.killed is True and run.children_killed:
+ fate = "it was killed, with every process it started."
+ elif run.killed is True:
+ fate = (
+ "the shell was killed, but processes it started may still be "
+ "running."
+ )
+ elif run.killed is False:
+ fate = "it had already ended."
+ else:
+ fate = "it could not be killed and may still be running."
+ note = f"Timed out after {timeout}s"
+ message = f"{note}; {fate}"
+ elif run.ended_unseen:
+ note = (
+ f"Lost the connection to the sandbox after {elapsed} (timeout "
+ f"{timeout}s); the command ended meanwhile, exit code unknown"
+ )
+ message = (
+ f"{note}. Any output it printed after the drop is lost; it was "
+ "not killed and did not time out."
+ )
+ else:
+ note = (
+ f"Lost the connection to the sandbox after {elapsed} (timeout "
+ f"{timeout}s); the command may still be running"
+ )
+ message = (
+ f"{note}. It did not time out and was not killed (the stream "
+ f"to it broke: {run.lost}). "
+ "Check whether it is still running (e.g. with ps) before running "
+ "it again."
+ )
+ response = _build_completion_response(
+ run.stdout, run.stderr, -1, secret_values, session_id
+ )
+ # First line of stderr, so it is what the chat shows for this call.
+ response.stderr = f"{note}\n{response.stderr}".rstrip("\n")
+ response.message = message
+ response.timed_out = run.timed_out
+ return response
async def _lands_on(path: str) -> bool:
diff --git a/autogpt_platform/backend/backend/copilot/tools/bash_exec_test.py b/autogpt_platform/backend/backend/copilot/tools/bash_exec_test.py
index 00e51b266f..a9194542c7 100644
--- a/autogpt_platform/backend/backend/copilot/tools/bash_exec_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/bash_exec_test.py
@@ -1,9 +1,22 @@
"""Tests for BashExecTool — E2B path with token injection."""
+import asyncio
+import itertools
+import re
+import subprocess
+import sys
+import time
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
-from e2b import CommandExitException
+from connectrpc.code import Code
+from connectrpc.errors import ConnectError
+from e2b import CommandExitException, CommandResult
+from e2b.envd.rpc import handle_rpc_exception
+from e2b.exceptions import NotFoundException, TimeoutException
from ._test_data import make_session
from .bash_exec import BashExecTool
@@ -23,18 +36,66 @@ def picked_credentials():
yield
+class _FakeRedisLock:
+ """redis-py's ``Lock``, held in-process: one holder per name at a time."""
+
+ _held: dict[str, asyncio.Lock] = {}
+
+ def __init__(self, name: str, **kwargs):
+ self._lock = self._held.setdefault(name, asyncio.Lock())
+
+ async def acquire(self) -> bool:
+ await self._lock.acquire()
+ return True
+
+ async def release(self) -> None:
+ self._lock.release()
+
+
[email protected](autouse=True)
+def stream_password_store():
+ """Where the screen's stream password expiry lives, Redis in production."""
+ _FakeRedisLock._held.clear()
+ redis = MagicMock()
+ redis.expire = AsyncMock()
+ redis.lock = MagicMock(side_effect=_FakeRedisLock)
+ with patch(
+ "backend.copilot.tools.e2b_sandbox.get_redis_async",
+ new=AsyncMock(return_value=redis),
+ ):
+ yield redis
+
+
def _make_tool() -> BashExecTool:
return BashExecTool()
def _make_sandbox(exit_code: int = 0, stdout: str = "", stderr: str = "") -> MagicMock:
- result = MagicMock()
- result.exit_code = exit_code
- result.stdout = stdout
- result.stderr = stderr
+ """A box whose command prints *stdout*/*stderr* and exits *exit_code*."""
+
+ async def run(*args, on_stdout=None, on_stderr=None, **kwargs):
+ if stdout and on_stdout:
+ on_stdout(stdout)
+ if stderr and on_stderr:
+ on_stderr(stderr)
+ handle = MagicMock(pid=1)
+ if exit_code:
+ handle.wait = AsyncMock(
+ side_effect=CommandExitException(
+ stdout=stdout, stderr=stderr, exit_code=exit_code, error=None
+ )
+ )
+ else:
+ handle.wait = AsyncMock(
+ return_value=CommandResult(
+ stdout=stdout, stderr=stderr, exit_code=0, error=None
+ )
+ )
+ return handle
sandbox = MagicMock()
- sandbox.commands.run = AsyncMock(return_value=result)
+ sandbox.set_timeout = AsyncMock()
+ sandbox.commands.run = AsyncMock(side_effect=run)
return sandbox
@@ -147,14 +208,8 @@ class TestBashExecE2BTokenInjection:
tool = _make_tool()
session = make_session(user_id=_USER)
- sandbox = MagicMock()
- sandbox.commands.run = AsyncMock(
- side_effect=CommandExitException(
- stdout="not logged in gh-secret",
- stderr="oops gh-secret",
- exit_code=1,
- error=None,
- )
+ sandbox = _make_sandbox(
+ exit_code=1, stdout="not logged in gh-secret", stderr="oops gh-secret"
)
with (
@@ -328,3 +383,562 @@ class TestBashExecSdkToolResultRedirect:
)
assert isinstance(result, BashExecResponse)
sandbox.commands.run.assert_called_once()
+
+
+# ---------------------------------------------------------------------------
+# Timeouts against a box that behaves like E2B's
+# ---------------------------------------------------------------------------
+
+_TICK = 0.02
+
+
+def _sdk_error(code: Code, message: str) -> Exception:
+ """The exception the E2B SDK raises for an envd stream ending with *code*."""
+ return handle_rpc_exception(ConnectError(code, message))
+
+
+class _FakeProcess:
+ def __init__(self, output: list[tuple[float, str, str]], runtime: float, code: int):
+ self.output = output # (seconds of run time, "stdout"/"stderr", text)
+ self.runtime = runtime
+ self.exit_code = code
+ self.ran = 0.0
+ self.printed = 0
+ self.ended = False
+ self.killed = False
+
+
+class _FakeHandle:
+ """``AsyncCommandHandle``: streams one process until it ends, the stream's
+ own deadline (the command ``timeout``) passes, or the box stops answering."""
+
+ def __init__(self, box: "_FakeBox", pid: int, timeout: float, on_stdout, on_stderr):
+ self.pid = pid
+ self._box = box
+ self._proc = box.processes[pid]
+ self._timeout = timeout
+ self._on = {"stdout": on_stdout, "stderr": on_stderr}
+ self._seen: dict[str, list[str]] = {"stdout": [], "stderr": []}
+ self._task = asyncio.create_task(self._stream())
+
+ async def _stream(self) -> CommandResult:
+ loop = asyncio.get_running_loop()
+ opened = last = loop.time()
+ proc = self._proc
+ while True:
+ await asyncio.sleep(_TICK)
+ if self._box.reached_limit():
+ raise self._box.limit_error()
+ if proc.killed:
+ raise _sdk_error(Code.UNKNOWN, "process killed")
+ # Runs on the same clock as the deadlines, however late the sleep.
+ now = loop.time()
+ proc.ran += now - last
+ last = now
+ while (
+ proc.printed < len(proc.output)
+ and proc.output[proc.printed][0] <= proc.ran
+ ):
+ _, stream, text = proc.output[proc.printed]
+ proc.printed += 1
+ self._seen[stream].append(text)
+ if self._on[stream]:
+ self._on[stream](text)
+ if proc.ran >= proc.runtime:
+ proc.ended = True
+ return CommandResult(
+ stdout="".join(self._seen["stdout"]),
+ stderr="".join(self._seen["stderr"]),
+ exit_code=proc.exit_code,
+ error=None,
+ )
+ if self._timeout and loop.time() - opened >= self._timeout:
+ # A connection deadline only: E2B leaves the process running.
+ raise _sdk_error(Code.DEADLINE_EXCEEDED, "deadline exceeded")
+
+ async def wait(self) -> CommandResult:
+ result = await self._task
+ if result.exit_code:
+ raise CommandExitException(
+ stdout=result.stdout,
+ stderr=result.stderr,
+ exit_code=result.exit_code,
+ error=None,
+ )
+ return result
+
+
+class _FakeCommands:
+ def __init__(self, box: "_FakeBox"):
+ self._box = box
+ self._pids = itertools.count(100)
+
+ async def run(
+ self,
+ cmd,
+ background=None,
+ envs=None,
+ cwd=None,
+ on_stdout=None,
+ on_stderr=None,
+ stdin=None,
+ timeout=60,
+ request_timeout=None,
+ ):
+ self._box.answer()
+ pid = next(self._pids)
+ self._box.processes[pid] = self._box.next_process
+ handle = _FakeHandle(self._box, pid, timeout, on_stdout, on_stderr)
+ return handle if background else await handle.wait()
+
+ async def connect(
+ self, pid, timeout=60, request_timeout=None, on_stdout=None, on_stderr=None
+ ):
+ self._box.answer()
+ proc = self._box.processes.get(pid)
+ if proc is None or proc.ended or proc.killed:
+ raise NotFoundException(f"process with pid {pid} not found")
+ return _FakeHandle(self._box, pid, timeout, on_stdout, on_stderr)
+
+ async def kill(self, pid, request_timeout=None) -> bool:
+ self._box.answer()
+ proc = self._box.processes.get(pid)
+ if proc is None or proc.ended:
+ return False
+ proc.killed = True
+ return True
+
+
+class _FakeBox:
+ """``AsyncSandbox`` with E2B's running-time limit: when it passes, the box
+ is paused (``on_timeout="pause"``) with its processes frozen, and any open
+ command stream breaks with the SDK's ``TimeoutException``. Like
+ ``auto_resume``, the next request resumes it for another *limit*
+ seconds; with *gone_at_limit* it was killed instead and never answers."""
+
+ def __init__(
+ self, limit: float, process: _FakeProcess, gone_at_limit: bool = False
+ ):
+ self._loop = asyncio.get_running_loop()
+ self._limit = limit
+ self.end_at = self._loop.time() + limit
+ self.gone_at_limit = gone_at_limit
+ self.paused = False
+ self.next_process = process
+ self.processes: dict[int, _FakeProcess] = {}
+ self.sandbox_id = "box-under-test"
+ self.set_timeout = AsyncMock(side_effect=self._set_timeout)
+ self.get_info = AsyncMock(side_effect=self._get_info)
+ self.commands = _FakeCommands(self)
+
+ async def _set_timeout(self, seconds: int) -> None:
+ self.end_at = self._loop.time() + seconds
+
+ async def _get_info(self) -> SimpleNamespace:
+ left = self.end_at - self._loop.time()
+ return SimpleNamespace(
+ end_at=datetime.now(timezone.utc) + timedelta(seconds=left)
+ )
+
+ def seconds_left(self) -> float:
+ return self.end_at - self._loop.time()
+
+ def reached_limit(self) -> bool:
+ if self._loop.time() >= self.end_at:
+ self.paused = True
+ return self.paused
+
+ def limit_error(self) -> Exception:
+ if self.gone_at_limit:
+ return _sdk_error(Code.UNAVAILABLE, "sandbox not found")
+ return _sdk_error(Code.CANCELED, "stream cancelled")
+
+ def answer(self) -> None:
+ if not self.reached_limit():
+ return
+ if self.gone_at_limit:
+ raise self.limit_error()
+ self.paused = False
+ self.end_at = self._loop.time() + self._limit
+
+
+async def _run(box: _FakeBox, timeout: int) -> BashExecResponse | ErrorResponse:
+ result = await _make_tool()._execute_on_e2b(
+ sandbox=box, # type: ignore[arg-type]
+ command="./long-job.sh",
+ timeout=timeout,
+ session_id="session-under-test",
+ user_id=None,
+ )
+ assert isinstance(result, (BashExecResponse, ErrorResponse))
+ return result
+
+
+def _counting(runtime: float, every: float = 0.1) -> _FakeProcess:
+ """A process that prints one numbered line every *every* seconds."""
+ steps = int(runtime / every)
+ output = [
+ (every * (i + 1) - _TICK / 2, "stdout", f"{i + 1}\n") for i in range(steps)
+ ]
+ return _FakeProcess(output, runtime, 0)
+
+
+class TestBashExecE2BTimeouts:
+ """The SDK raises ``TimeoutException`` for the command's own deadline and
+ also for the box pausing or dying under the command, at whatever moment
+ that happens. Only the first may be reported as the command timing out."""
+
+ @pytest.fixture(autouse=True)
+ def kill_tree(self):
+ """The script that kills a timed-out command's children, run on the box."""
+ with patch(
+ "backend.copilot.tools.bash_exec.run_internal",
+ new=AsyncMock(return_value=SimpleNamespace(exit_code=0)),
+ ) as run:
+ yield run
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_command_outlives_the_box_running_time_limit(self):
+ # The turn's connect armed the limit long ago: 0.3s of it is left,
+ # and the command needs 0.8s of its 5s timeout.
+ box = _FakeBox(limit=0.3, process=_counting(0.8))
+
+ result = await _run(box, timeout=5)
+
+ assert isinstance(result, BashExecResponse)
+ assert result.timed_out is False
+ assert result.exit_code == 0
+ assert result.stdout == "".join(f"{i}\n" for i in range(1, 9))
+ assert "Timed out" not in result.stderr
+ # Re-armed to the configured safety net (420s), which covers the
+ # whole 5s timeout.
+ box.set_timeout.assert_awaited_once_with(420)
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_long_timeout_extends_the_limit_past_it(self):
+ box = _FakeBox(limit=30, process=_counting(0.1))
+
+ await _run(box, timeout=3600)
+
+ box.set_timeout.assert_awaited_once_with(3660)
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_timeout_past_the_hung_tool_cap_is_capped_below_it(self):
+ # The turn gives up on a pending tool after two hours and pauses the
+ # box, so a longer timeout would only bill a box nobody is following.
+ # The cap sits below the watchdog so the tool's own timeout lands first.
+ from backend.copilot.constants import HUNG_TOOL_CAP_SECONDS
+
+ from .bash_exec import _HUNG_CAP_MARGIN_SECONDS
+
+ box = _FakeBox(limit=30, process=_counting(0.1))
+
+ await _run(box, timeout=7 * 24 * 60 * 60)
+
+ box.set_timeout.assert_awaited_once_with(
+ HUNG_TOOL_CAP_SECONDS - _HUNG_CAP_MARGIN_SECONDS + 60
+ )
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_a_short_command_leaves_a_longer_limit_alone(self):
+ # Another command on the box (a parallel call, or another session of
+ # the same expert) already pushed the limit out to an hour.
+ box = _FakeBox(limit=3660, process=_counting(0.1))
+
+ result = await _run(box, timeout=30)
+
+ assert isinstance(result, BashExecResponse)
+ box.set_timeout.assert_not_awaited()
+ assert box.seconds_left() > 3600
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_commands_started_together_cannot_shorten_the_limit(self):
+ from .e2b_sandbox import keep_sandbox_running
+
+ box = _FakeBox(limit=30, process=_counting(0.1))
+ set_limit = box._set_timeout
+
+ async def slow_read():
+ await asyncio.sleep(0.01)
+ return await box._get_info()
+
+ async def short_lands_last(seconds: int) -> None:
+ # Both read the old limit; the short call's set reaches E2B last.
+ await asyncio.sleep(0.05 if seconds < 1000 else 0)
+ await set_limit(seconds)
+
+ box.get_info = AsyncMock(side_effect=slow_read)
+ box.set_timeout = AsyncMock(side_effect=short_lands_last)
+
+ assert await asyncio.gather(
+ keep_sandbox_running(box, 3660), # type: ignore[arg-type]
+ keep_sandbox_running(box, 420), # type: ignore[arg-type]
+ ) == [True, True]
+
+ assert box.seconds_left() > 3600
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_an_unreadable_limit_is_still_extended(self):
+ from .e2b_sandbox import keep_sandbox_running
+
+ box = _FakeBox(limit=30, process=_counting(0.1))
+ box.get_info = AsyncMock(side_effect=RuntimeError("E2B API down"))
+
+ assert await keep_sandbox_running(box, 900) is True # type: ignore[arg-type]
+
+ box.set_timeout.assert_awaited_once_with(900)
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_box_paused_mid_command_is_reattached(self):
+ # E2B refuses the new limit, so the box still pauses under the
+ # command; the command is frozen, not gone.
+ box = _FakeBox(limit=0.3, process=_counting(0.8))
+ box.set_timeout = AsyncMock(side_effect=RuntimeError("E2B API down"))
+
+ result = await _run(box, timeout=5)
+
+ assert isinstance(result, BashExecResponse)
+ assert result.timed_out is False
+ assert result.exit_code == 0
+ assert result.stdout == "".join(f"{i}\n" for i in range(1, 9))
+ assert "and was resumed" in result.message
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_box_gone_mid_command_is_not_reported_as_a_timeout(self):
+ box = _FakeBox(limit=0.3, process=_counting(0.8), gone_at_limit=True)
+ box.set_timeout = AsyncMock(side_effect=RuntimeError("E2B API down"))
+
+ result = await _run(box, timeout=120)
+
+ assert isinstance(result, BashExecResponse)
+ assert result.timed_out is False
+ assert result.exit_code == -1
+ assert "Timed out after 120s" not in result.stderr
+ assert "Timed out after 120s" not in result.message
+ # What it printed before the box went away is kept.
+ assert result.stdout.startswith("1\n2\n")
+ assert "8\n" not in result.stdout
+ assert re.match(
+ r"Lost the connection to the sandbox after 0\.\d+s \(timeout 120s\)",
+ result.stderr,
+ )
+ assert "may still be running" in result.message
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_box_pausing_past_the_reconnect_cap_gives_up_following(self):
+ # Every reattach works, but the box pauses again 0.2s later, so the
+ # 2s command is still running when the reattaches run out.
+ from .bash_exec import _MAX_RECONNECTS
+
+ proc = _counting(2.0)
+ box = _FakeBox(limit=0.2, process=proc)
+ box.set_timeout = AsyncMock(side_effect=RuntimeError("E2B API down"))
+ box.commands.connect = AsyncMock(wraps=box.commands.connect)
+
+ result = await _run(box, timeout=30)
+
+ assert isinstance(result, BashExecResponse)
+ assert box.commands.connect.await_count == _MAX_RECONNECTS
+ assert result.timed_out is False
+ assert result.exit_code == -1
+ assert result.stdout.startswith("1\n2\n")
+ assert "Timed out" not in result.stderr
+ assert "may still be running" in result.message
+ assert proc.killed is False
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_box_pausing_in_the_last_second_is_not_the_timeout(self):
+ # The box pauses 1.2s into a 2s timeout; the command needs 1.6s.
+ box = _FakeBox(limit=1.2, process=_counting(1.6))
+ box.set_timeout = AsyncMock(side_effect=RuntimeError("E2B API down"))
+
+ result = await _run(box, timeout=2)
+
+ assert isinstance(result, BashExecResponse)
+ assert result.timed_out is False
+ assert result.exit_code == 0
+ assert result.stdout == "".join(f"{i}\n" for i in range(1, 17))
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_reattach_failing_past_the_deadline_is_the_timeout(self):
+ # The box pauses just before the deadline and the reattach only
+ # fails once the deadline has passed.
+ proc = _counting(60)
+ box = _FakeBox(limit=1.8, process=proc)
+ box.set_timeout = AsyncMock(side_effect=RuntimeError("E2B API down"))
+
+ async def slow_failing_connect(*args, **kwargs):
+ await asyncio.sleep(0.4)
+ raise TimeoutException("deadline exceeded while reconnecting")
+
+ box.commands.connect = AsyncMock(side_effect=slow_failing_connect)
+
+ result = await _run(box, timeout=2)
+
+ assert isinstance(result, BashExecResponse)
+ assert result.timed_out is True
+ assert (
+ result.message
+ == "Timed out after 2s; it was killed, with every process it started."
+ )
+ assert proc.killed is True
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_real_timeout_keeps_partial_output_and_kills_the_command(self):
+ proc = _FakeProcess(
+ [(0.1, "stdout", "partial\n"), (0.2, "stderr", "warning: slow\n")],
+ runtime=60,
+ code=0,
+ )
+ box = _FakeBox(limit=30, process=proc)
+
+ result = await _run(box, timeout=1)
+
+ assert isinstance(result, BashExecResponse)
+ assert result.timed_out is True
+ assert result.exit_code == -1
+ assert result.stdout == "partial\n"
+ assert result.stderr == "Timed out after 1s\nwarning: slow"
+ assert (
+ result.message
+ == "Timed out after 1s; it was killed, with every process it started."
+ )
+ # E2B's deadline only closes the stream; the tool kills the command.
+ assert proc.killed is True
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_timeout_kills_the_children_before_the_shell(self, kill_tree):
+ # envd's kill reaches only the shell, so its children are killed first,
+ # while the shell is still there to find them under.
+ proc = _FakeProcess([], runtime=60, code=0)
+ box = _FakeBox(limit=30, process=proc)
+
+ def children_first(sandbox, script, **kwargs):
+ assert proc.killed is False
+ return SimpleNamespace(exit_code=0)
+
+ kill_tree.side_effect = children_first
+
+ await _run(box, timeout=1)
+
+ kill_tree.assert_awaited_once()
+ script = kill_tree.await_args.args[1]
+ assert "p=100\n" in script
+ assert proc.killed is True
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_children_that_could_not_be_killed_are_reported(self, kill_tree):
+ proc = _FakeProcess([], runtime=60, code=0)
+ box = _FakeBox(limit=30, process=proc)
+ kill_tree.side_effect = RuntimeError("login files changed")
+
+ result = await _run(box, timeout=1)
+
+ assert isinstance(result, BashExecResponse)
+ assert result.timed_out is True
+ assert result.message == (
+ "Timed out after 1s; the shell was killed, but processes it "
+ "started may still be running."
+ )
+ assert proc.killed is True
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_command_ending_while_detached_is_reported_unseen(self):
+ # Paused under the command, and the box's own process table no
+ # longer has it when we come back.
+ box = _FakeBox(limit=0.3, process=_counting(0.8))
+ box.set_timeout = AsyncMock(side_effect=RuntimeError("E2B API down"))
+ box.commands.connect = AsyncMock(
+ side_effect=NotFoundException("process with pid 100 not found")
+ )
+
+ result = await _run(box, timeout=120)
+
+ assert isinstance(result, BashExecResponse)
+ assert result.timed_out is False
+ assert result.exit_code == -1
+ assert "exit code unknown" in result.stderr
+ assert "Timed out" not in result.stderr
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_stream_not_opening_is_not_the_command_timeout(self):
+ box = _FakeBox(limit=30, process=_counting(0.1))
+ box.commands.run = AsyncMock(
+ side_effect=TimeoutException(
+ "Request timed out: the stream didn't open within "
+ "'request_timeout' (60 seconds)."
+ )
+ )
+
+ result = await _run(box, timeout=120)
+
+ assert isinstance(result, ErrorResponse)
+ assert result.error == "e2b_start_timeout"
+ assert "Timed out after 120s" not in result.message
+ assert "may or may not have run" in result.message
+
+ @pytest.mark.asyncio(loop_scope="session")
+ async def test_limit_extension_pushes_out_the_stream_password(
+ self, stream_password_store
+ ):
+ from .e2b_sandbox import SandboxOwner, keep_sandbox_running
+
+ box = _FakeBox(limit=30, process=_counting(0.1))
+ owner = SandboxOwner(kind="session", id="session-under-test")
+
+ assert await keep_sandbox_running(box, 900, owner) is True # type: ignore[arg-type]
+
+ box.set_timeout.assert_awaited_once_with(900)
+ stream_password_store.expire.assert_awaited_once_with(
+ owner.stream_key(), 900, gt=True
+ )
+
+
+def _children(pid: int) -> list[int]:
+ found = []
+ for stat in Path("/proc").glob("[0-9]*/stat"):
+ try:
+ fields = stat.read_text().rsplit(") ", 1)[1].split()
+ except (OSError, IndexError):
+ continue
+ if int(fields[1]) == pid:
+ found.append(int(stat.parent.name))
+ return found
+
+
+def _alive(pid: int) -> bool:
+ """Running or stopped; a killed child its stopped parent hasn't reaped is not."""
+ try:
+ state = Path(f"/proc/{pid}/stat").read_text().rsplit(") ", 1)[1].split()[0]
+ except (OSError, IndexError):
+ return False
+ return state not in ("Z", "X")
+
+
[email protected](sys.platform != "linux", reason="reads /proc")
+def test_kill_tree_script_kills_a_compound_commands_children():
+ # The reviewer's repro: killing the shell of `sleep 30 && echo done` left
+ # sleep running under a new parent.
+ from .bash_exec import _KILL_TREE_SCRIPT
+
+ shell = subprocess.Popen(["bash", "-c", "sleep 30 && echo done"])
+ try:
+ for _ in range(50):
+ if _children(shell.pid):
+ break
+ time.sleep(0.05)
+ (sleeper,) = _children(shell.pid)
+
+ subprocess.run(
+ ["bash", "-c", _KILL_TREE_SCRIPT.replace("__PID__", str(shell.pid))],
+ check=True,
+ timeout=10,
+ )
+
+ assert not _alive(sleeper)
+ # The shell is stopped, not killed: envd's kill does that, and the
+ # shell must not start `echo done` in between.
+ assert _alive(shell.pid)
+ finally:
+ shell.kill()
+ shell.wait(timeout=10)
diff --git a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py
index 3a757bca5a..8b82b10d8f 100644
--- a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py
+++ b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py
@@ -73,6 +73,7 @@ import asyncio
import contextlib
import logging
import math
+import time
from collections.abc import Awaitable, Callable, Mapping
from typing import Any, Literal
@@ -146,6 +147,14 @@ _MAX_WAIT_ATTEMPTS = math.ceil(_CREATION_LOCK_TTL / _WAIT_INTERVAL_SECONDS * 1.2
# on the next turn.
_E2B_API_TIMEOUT_SECONDS = 10
+# Serialises the read-compare-set of one box's running-time limit, so two
+# commands started together cannot each read the old limit and the shorter
+# one land last. The holder makes at most two E2B calls; a waiter that gives
+# up extends without the lock rather than not at all.
+_LIMIT_LOCK_PREFIX = "copilot:e2b:limit_lock:"
+_LIMIT_LOCK_TTL = 3 * _E2B_API_TIMEOUT_SECONDS
+_LIMIT_LOCK_WAIT_SECONDS = 2 * _E2B_API_TIMEOUT_SECONDS
+
# Bound on stopping the screen's stream before a pause: a box that does not
# answer must not hold the pause up for long.
_STOP_STREAM_TIMEOUT_SECONDS = 5
@@ -1066,6 +1075,72 @@ async def pause_sandbox_direct(
return False
+async def keep_sandbox_running(
+ sandbox: "AsyncSandbox", seconds: int, owner: SandboxOwner | None = None
+) -> bool:
+ """Make the box's running-time limit at least *seconds* from now.
+
+ Only a connect re-arms the limit, once per turn, and E2B pauses the box
+ at it whatever is running: a command started late in a long turn is cut
+ off mid-run. ``set_timeout`` can shorten the limit as well, and a box
+ runs commands side by side (parallel calls in a turn, every session of
+ an expert), so a limit already further out is left alone. The screen's
+ stream password lives as long as the box could run (``_settle_stream``),
+ so it is pushed out with it, never shortened.
+ Best effort: returns ``False`` when E2B did not take the new limit.
+ """
+ lock = None
+ with contextlib.suppress(Exception):
+ redis = await get_redis_async()
+ candidate = redis.lock(
+ f"{_LIMIT_LOCK_PREFIX}{sandbox.sandbox_id}",
+ timeout=_LIMIT_LOCK_TTL,
+ blocking_timeout=_LIMIT_LOCK_WAIT_SECONDS,
+ )
+ if await candidate.acquire():
+ lock = candidate
+ try:
+ if not await _extend_running_limit(sandbox, seconds):
+ return False
+ finally:
+ if lock is not None:
+ with contextlib.suppress(Exception):
+ await lock.release()
+ if owner is not None:
+ with contextlib.suppress(Exception):
+ redis = await get_redis_async()
+ await redis.expire(owner.stream_key(), seconds, gt=True)
+ return True
+
+
+async def _extend_running_limit(sandbox: "AsyncSandbox", seconds: int) -> bool:
+ """Set the box's limit to *seconds* from now unless it already ends later."""
+ wanted_end = time.time() + seconds
+ try:
+ info = await asyncio.wait_for(
+ sandbox.get_info(), timeout=_E2B_API_TIMEOUT_SECONDS
+ )
+ if info.end_at.timestamp() >= wanted_end:
+ return True
+ except Exception as exc:
+ logger.debug(
+ "[E2B] Could not read sandbox %.12s limit: %s", sandbox.sandbox_id, exc
+ )
+ try:
+ await asyncio.wait_for(
+ sandbox.set_timeout(seconds), timeout=_E2B_API_TIMEOUT_SECONDS
+ )
+ except Exception as exc:
+ logger.warning(
+ "[E2B] Could not extend sandbox %.12s to %ds: %s",
+ sandbox.sandbox_id,
+ seconds,
+ exc,
+ )
+ return False
+ return True
+
+
async def kill_sandbox(session_id: str, api_key: str) -> bool:
"""Kill a session's box: the chat is gone, so is its scratch computer.
diff --git a/autogpt_platform/backend/backend/copilot/tools/gate_subject_test.py b/autogpt_platform/backend/backend/copilot/tools/gate_subject_test.py
index 493d62ff81..3b07a1147e 100644
--- a/autogpt_platform/backend/backend/copilot/tools/gate_subject_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/gate_subject_test.py
@@ -822,6 +822,9 @@ class _LocalSandbox:
cmd: str,
envs: dict[str, str] | None = None,
cwd: str = "/home/user",
+ background: bool = False,
+ on_stdout: Any = None,
+ on_stderr: Any = None,
**_: Any,
) -> SimpleNamespace:
user_home = (envs or {}).get("HOME", "/home/user")
@@ -833,11 +836,18 @@ class _LocalSandbox:
text=True,
timeout=10,
)
- return SimpleNamespace(
+ result = SimpleNamespace(
exit_code=done.returncode,
stdout=done.stdout.replace(self.home, "/home/user"),
stderr=done.stderr,
)
+ if not background:
+ return result
+ # bash_exec follows its command through E2B's background handle.
+ for emit, text in ((on_stdout, result.stdout), (on_stderr, result.stderr)):
+ if emit and text:
+ emit(text)
+ return SimpleNamespace(pid=1, wait=AsyncMock(return_value=result))
@pytest.fixture
diff --git a/autogpt_platform/backend/backend/util/e2b_network.py b/autogpt_platform/backend/backend/util/e2b_network.py
index 61b3c4dd72..20d3adb3b2 100644
--- a/autogpt_platform/backend/backend/util/e2b_network.py
+++ b/autogpt_platform/backend/backend/util/e2b_network.py
@@ -1,8 +1,9 @@
"""One door for every E2B box: where its egress is pinned.
Every ``AsyncSandbox.create`` and ``.connect`` in the backend goes through
-``create_sandbox`` / ``connect_sandbox`` here; ``e2b_network_test`` fails on
-any that does not. That is the chokepoint at which a box's network is set:
+``create_sandbox`` / ``connect_sandbox`` here, and a reattach to a running
+command through ``reattach_command``; ``e2b_network_test`` fails on any that
+does not. That is the chokepoint at which a box's network is set:
the credential swap proxy (SECRT-2651) that every box will egress through,
so that the model never holds a real credential and nothing dials out
around the proxy.
@@ -40,7 +41,7 @@ import logging
import secrets
from typing import Any, Literal, Optional, TypeVar, cast
-from e2b import AsyncSandbox
+from e2b import AsyncCommandHandle, AsyncSandbox
from e2b.sandbox.sandbox_api import (
SandboxEgressProxyOpts,
SandboxNetworkOpts,
@@ -215,6 +216,21 @@ async def connect_sandbox(
return sandbox
+async def reattach_command(
+ sandbox: AsyncSandbox, pid: int, **kwargs: Any
+) -> AsyncCommandHandle:
+ """Reattach to a process still running on a box the caller already holds.
+
+ Not a box connect: it opens one more stream to the box's own process
+ service, like ``commands.run`` does, on a box that ``create_sandbox`` or
+ ``connect_sandbox`` has already pinned. It creates nothing and changes no
+ network: the box keeps the egress that create or connect pinned, and
+ there is no credential to rotate. Every other keyword goes to
+ ``commands.connect``.
+ """
+ return await sandbox.commands.connect(pid, **kwargs)
+
+
async def kill_sandbox(sandbox: AsyncSandbox) -> None:
"""Kill a box and revoke its proxy credential: the kill for any caller
that holds the handle. A kill that raises leaves the credential alone,
diff --git a/autogpt_platform/backend/backend/util/e2b_network_test.py b/autogpt_platform/backend/backend/util/e2b_network_test.py
index a071be901a..717d0b07d5 100644
--- a/autogpt_platform/backend/backend/util/e2b_network_test.py
+++ b/autogpt_platform/backend/backend/util/e2b_network_test.py
@@ -19,6 +19,7 @@ from backend.util.e2b_network import (
credential_record,
forget_sandbox,
kill_sandbox,
+ reattach_command,
secret_digest,
)
@@ -117,6 +118,22 @@ class TestOff:
class TestPinned:
+ @pytest.mark.asyncio
+ async def test_a_command_reattach_leaves_the_pinned_network_alone(self):
+ """Reattaching to a running command is not a box connect: no new
+ credential, no network update, the box keeps what it was pinned to."""
+ box, redis = _box("sb-1"), _redis({"e2b:egress:box:sb-1": "box-a1"})
+ handle = MagicMock()
+ box.commands.connect = AsyncMock(return_value=handle)
+ with (
+ _configured(_PROXY),
+ patch(f"{_M}.get_redis_async", AsyncMock(return_value=redis)),
+ ):
+ assert await reattach_command(box, 42, timeout=30) is handle
+ box.commands.connect.assert_awaited_once_with(42, timeout=30)
+ box.update_network.assert_not_awaited()
+ assert redis.store == {"e2b:egress:box:sb-1": "box-a1"}
+
@pytest.mark.asyncio
async def test_create_pins_the_box_under_a_credential_the_proxy_can_resolve(self):
box, redis = _box("sb-1"), _redis()