AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/copilot): stop the desktop stream when the box pauses (#14699)
Krzysztof Czerwinski39369be13b9c
diff --git a/autogpt_platform/backend/backend/blocks/desktop/_api.py b/autogpt_platform/backend/backend/blocks/desktop/_api.pyindex 73ed085d98..1eac84a264 100644--- a/autogpt_platform/backend/backend/blocks/desktop/_api.py+++ b/autogpt_platform/backend/backend/blocks/desktop/_api.py@@ -263,6 +263,14 @@ class DesktopSession:async def kill(self) -> None:await self.sandbox.kill()+ async def stop_stream(self) -> None:+ """Stop serving the screen; the display itself stays up.++ Whatever password the stream ran under stops working with it, and the+ next ``start_stream`` brings the stack back under a fresh one.+ """+ await self._vnc_command(_STOP_STREAM)+async def ensure_display(self, width: int, height: int) -> None:if await self._check("pgrep -x xfwm4"):returndiff --git a/autogpt_platform/backend/backend/blocks/desktop/desktop_stream_test.py b/autogpt_platform/backend/backend/blocks/desktop/desktop_stream_test.pyindex 138fc2e2e0..924beffb69 100644--- a/autogpt_platform/backend/backend/blocks/desktop/desktop_stream_test.py+++ b/autogpt_platform/backend/backend/blocks/desktop/desktop_stream_test.py@@ -99,6 +99,19 @@ async def test_a_remembered_password_is_dropped_once_the_proxy_is_gone():assert any(cmd.startswith("x11vnc") for cmd, _ in _commands(run))+async def test_stopping_the_stream_kills_only_the_vnc_stack_and_as_root():+ session, run = _session(listening=True)++ await session.stop_stream()++ ((command, user),) = _commands(run)+ assert "pkill -x x11vnc" in command and "pkill -f '[n]ovnc_proxy'" in command+ # The display keeps running: a later open serves the same screen again.+ assert "Xvfb" not in command and "xfce" not in command+ assert user == VNC_USER++@pytest.mark.asyncioasync def test_a_failed_x11vnc_says_why_in_its_own_words():"""Its stderr goes to a file in the box, so the bare exception is empty."""diff --git a/autogpt_platform/backend/backend/copilot/computer.py b/autogpt_platform/backend/backend/copilot/computer.pyindex a92c2859c4..cbf6164122 100644--- a/autogpt_platform/backend/backend/copilot/computer.py+++ b/autogpt_platform/backend/backend/copilot/computer.py@@ -7,8 +7,15 @@ that box* and hands back the live stream, whether the ask comes from the``start_desktop`` tool inside a turn or from the Computer tab and sidepanel. The box pauses at turn end like any other and comes back with thescreen exactly as it was, but under a fresh stream password: the password-is kept here, off the box, and forgotten whenever the box pauses, so a-stream URL that may have leaked is good for one running stretch only.+is kept here, off the box, and when the turn-end pause comes the stream is+stopped and the password forgotten, so a stream URL that may have leaked+stops working there. "Screen on" therefore means the display was started+in this box; the next open is what serves it again.++One pause is out of our hands: a box left idle is paused by E2B's own+timeout with its stream still up, and any request to the stream URL resumes+it. That stream is stopped at our next connect to the box+(``e2b_sandbox._settle_stream``), not before.``describe_computer`` only lists: it never connects, so a paused box stayspaused (connecting is what E2B's auto-resume reacts to). Whether the screen@@ -151,7 +158,7 @@ async def open_desktop(if not user_id:raise ValueError("A desktop needs an authenticated user to issue its link to")redis = await get_redis_async()- lock_key = f"{owner.display_key()}:lock"+ lock_key = owner.display_lock_key()token = uuid.uuid4().hexwaited = 0.0while not await redis.set(lock_key, token, nx=True, ex=_DESKTOP_LOCK_TTL_SECONDS):@@ -223,9 +230,11 @@ async def _stream_password(owner: SandboxOwner) -> Optional[str]:async def _remember_screen(owner: SandboxOwner, sandbox_id: str, password: str) -> None:"""Record the screen as on in this box, and the password its stream uses.- The password outlives a pause only as long as the box could have kept- running: its expiry is the box's running-time limit, and the turn-end- pause drops it outright (``e2b_sandbox._forget_stream``).+ The password is remembered only as long as the box could have kept+ running: its expiry is the box's running-time limit, pushed out again+ whenever a connect re-arms that limit (``e2b_sandbox._settle_stream``),+ and the turn-end pause stops the stream and drops it outright+ (``e2b_sandbox._revoke_stream``)."""redis = await get_redis_async()await redis.set(owner.display_key(), sandbox_id, ex=owner.ttl)diff --git a/autogpt_platform/backend/backend/copilot/computer_test.py b/autogpt_platform/backend/backend/copilot/computer_test.pyindex 5384c95568..5159bb0df6 100644--- a/autogpt_platform/backend/backend/copilot/computer_test.py+++ b/autogpt_platform/backend/backend/copilot/computer_test.py@@ -262,6 +262,18 @@ class TestOpenDesktop:await open_desktop(owner, {}, "k", user_id=_USER)desktop.start_stream.assert_awaited_once_with("issued-before")+ @pytest.mark.asyncio+ async def test_a_stream_stopped_at_the_pause_reopens_under_a_new_password(self):+ """The pause leaves an empty marker where the password was: to an open+ that is no password, so the stack restarts under a fresh one."""+ owner = SandboxOwner(kind="session", id=_SESSION)+ redis = _redis("sb-1", stream="")+ sandbox, desktop = _sandbox("sb-1"), _desktop("sb-1")+ redis_p, get_p, cls_p, cfg_p = self._patches(redis, sandbox, desktop)+ with redis_p, get_p, cls_p, cfg_p:+ await open_desktop(owner, {}, "k", user_id=_USER)+ desktop.start_stream.assert_awaited_once_with(None)+@pytest.mark.asyncioasync def test_a_password_left_over_from_a_replaced_box_is_not_reused(self):"""The screen flag names another box: whatever password Redis stilldiff --git a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.pyindex f7da254ac5..8b0a6c4221 100644--- a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py+++ b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py@@ -87,7 +87,7 @@ from e2b import (from e2b.exceptions import NotFoundExceptionfrom pydantic import BaseModel, ConfigDict-from backend.blocks.desktop._api import resolve_volume+from backend.blocks.desktop._api import DesktopSession, resolve_volumefrom backend.data.redis_client import get_redis_asyncfrom backend.util.e2b_template import ensure_template, forget_templatefrom backend.util.sandbox_metadata import MountState, SandboxMetadata@@ -134,6 +134,15 @@ _MAX_WAIT_ATTEMPTS = math.ceil(_CREATION_LOCK_TTL / _WAIT_INTERVAL_SECONDS * 1.2# on the next turn._E2B_API_TIMEOUT_SECONDS = 10+# Bound on stopping the screen's stream before a pause: a box that does not+# answer must not hold the pause up for long.+_STOP_STREAM_TIMEOUT_SECONDS = 5++# Held in place of a stream password once the stream has been stopped in the+# box: nothing is serving, so a reconnect has nothing to stop. Empty on+# purpose, so every reader that wants a password sees none.+_STREAM_STOPPED = ""+# Redis TTL for a session sandbox key. Must be ≥ the E2B project "paused# sandbox lifetime" setting (recommended: set both to 48 h)._SANDBOX_ID_TTL = 48 * 3600 # 48 hours@@ -224,11 +233,17 @@ class SandboxOwner(BaseModel):The password never rests on the box (see ``DesktopSession.start_stream``);this is what lets a re-open hand back the URL the user already holds.- It is dropped whenever the box pauses, so a URL that may have leaked- is good for one running stretch only.+ When we pause the box the stream is stopped and the password dropped+ (``_revoke_stream``), so a URL that may have leaked stops working at+ the turn-end pause. A pause E2B makes on its own timeout is only+ caught up with at our next connect (``_settle_stream``)."""return f"{self.key()}:stream"+ def display_lock_key(self) -> str:+ """Redis key held by whoever is turning the screen on right now."""+ return f"{self.display_key()}:lock"+def legacy_desktop_key(self) -> str:"""Where the pre-one-box desktop's id was cached; swept on kill."""if self.is_expert:@@ -290,12 +305,20 @@ async def connect_owned(is that limit for the owner's box (a resumed box would otherwise get theSDK's default)."""+ await _owned_info(sandbox_id, owner, api_key)+ return await AsyncSandbox.connect(sandbox_id, api_key=api_key, timeout=timeout)+++async def _owned_info(+ sandbox_id: str, owner: SandboxOwner, api_key: str+) -> SandboxInfo:+ """What E2B says about *sandbox_id*, refused unless it is *owner*'s box."""info = await AsyncSandbox.get_info(sandbox_id, api_key=api_key)expected = owner.metadata()stamped = info.metadata or {}if any(stamped.get(key) != value for key, value in expected.items()):raise SandboxNotOwnedError(f"Sandbox {sandbox_id[:12]} is not {owner}'s box")- return await AsyncSandbox.connect(sandbox_id, api_key=api_key, timeout=timeout)+ return infodef _as_owner(owner: "SandboxOwner | str") -> SandboxOwner:@@ -408,7 +431,13 @@ async def _try_reconnect("""owner = _as_owner(owner)try:- sandbox = await connect_owned(sandbox_id, owner, api_key, timeout=timeout)+ # Same order as ``connect_owned``: the stamp is read before the connect+ # wakes anything. The state read with it says whether this connect is+ # what resumes the box.+ info = await _owned_info(sandbox_id, owner, api_key)+ sandbox = await AsyncSandbox.connect(+ sandbox_id, api_key=api_key, timeout=timeout+ )except SandboxNotOwnedError as exc:logger.warning("[E2B] Refusing reconnect: %s", exc)except NotFoundException as exc:@@ -417,6 +446,12 @@ async def _try_reconnect(if await sandbox.is_running():# Refresh TTL so an active owner cannot lose its sandbox_id at expiry.await _set_stored_sandbox_id(owner, sandbox_id)+ await _settle_stream(+ owner,+ sandbox,+ resumed=info.state == SandboxState.PAUSED,+ timeout=timeout,+ )return sandboxlogger.warning("[E2B] Box %.12s came back not running", sandbox_id)@@ -761,6 +796,7 @@ async def _act_on_sandbox(*,sandbox_id: str | None = None,clear_stored_id: bool = False,+ timeout: float = _E2B_API_TIMEOUT_SECONDS,) -> bool:"""Connect to the owner's sandbox and run *fn* on it.@@ -779,7 +815,7 @@ async def _act_on_sandbox(await fn(await connect_owned(sandbox_id, owner, api_key))try:- await asyncio.wait_for(_run(), timeout=_E2B_API_TIMEOUT_SECONDS)+ await asyncio.wait_for(_run(), timeout=timeout)if clear_stored_id:await _clear_stored_sandbox_id(owner)logger.info(@@ -822,8 +858,23 @@ async def pause_sandbox(owner = SandboxOwner.for_session(session_id, expert_id)if not await _release_turn(owner):return False- paused = await _act_on_sandbox(owner, api_key, "pause", lambda sb: sb.pause())- if paused:+ revoked = False++ async def _pause(sandbox: AsyncSandbox) -> None:+ nonlocal revoked+ revoked = await _revoke_stream(owner, sandbox)+ await sandbox.pause()++ # The stop gets its own share of the budget, so a box slow to answer it+ # still leaves the connect and the pause the time they always had.+ paused = await _act_on_sandbox(+ owner,+ api_key,+ "pause",+ _pause,+ timeout=_E2B_API_TIMEOUT_SECONDS + _STOP_STREAM_TIMEOUT_SECONDS,+ )+ if paused and not revoked:await _forget_stream(owner)return paused@@ -844,10 +895,12 @@ async def pause_sandbox_direct(owner = SandboxOwner.for_session(session_id, expert_id)if not await _release_turn(owner):return False+ revoked = await _revoke_stream(owner, sandbox)try:await asyncio.wait_for(sandbox.pause(), timeout=_E2B_API_TIMEOUT_SECONDS)logger.info("[E2B] Paused sandbox %.12s for %s", sandbox.sandbox_id, owner)- await _forget_stream(owner)+ if not revoked:+ await _forget_stream(owner)return Trueexcept Exception as exc:logger.warning(@@ -979,11 +1032,99 @@ async def _forget_owner_state(owner: SandboxOwner) -> None:)+async def _revoke_stream(owner: SandboxOwner, sandbox: AsyncSandbox) -> bool:+ """Stop the screen's stream in *sandbox* so its password stops working.++ Forgetting the password is not enough on its own: a pause keeps the box's+ processes, so the stream would come back with the box and still answer to+ it. Only the stream goes; the display stays up and the next open serves+ it again under a fresh password. A box whose screen was never turned on+ costs one Redis read and no command. Never raises, because a box that+ cannot be told to stop must still pause. Returns whether the stream is+ known to be stopped.+ """+ try:+ redis = await get_redis_async()+ if not await _screen_started_in(owner, sandbox.sandbox_id):+ return False+ await asyncio.wait_for(+ DesktopSession(sandbox).stop_stream(),+ timeout=_STOP_STREAM_TIMEOUT_SECONDS,+ )+ await redis.set(owner.stream_key(), _STREAM_STOPPED, ex=owner.ttl)+ except Exception as exc:+ logger.warning(+ "[E2B] Could not stop the screen stream in %.12s for %s: %s",+ sandbox.sandbox_id,+ owner,+ exc,+ )+ return False+ logger.info("[E2B] Stopped the screen stream in %.12s", sandbox.sandbox_id)+ return True+++async def _settle_stream(+ owner: SandboxOwner,+ sandbox: AsyncSandbox,+ *,+ resumed: bool,+ timeout: int | None,+) -> None:+ """On reconnect, stop a stream that outlived its running stretch.++ *resumed* is exact: E2B reported the box paused just before this connect+ woke it. Unless the stream is already known to be stopped, it came back+ with the box (E2B paused it on its own timeout, or our stop before the+ pause failed) and is stopped now, whatever Redis still remembers.++ A box that was already running is left alone while its password is+ remembered, and the password's expiry is pushed out to the running-time+ limit this connect just re-armed, so a stream someone is watching on a+ box that never pauses is not mistaken for a leftover. Only a running box+ with no stream key at all is stopped: the password outlived the box's+ limit, so the box did pause and something other than us resumed it. An+ open in progress is left alone there; it restarts the stream itself.++ What this cannot reach: a box E2B paused on its timeout is resumed by any+ request to the stream URL, with no call through here, and serves under+ the old password until our next connect. Closing that takes a stop the+ box runs itself, or a sweep, neither of which exists yet.+ """+ try:+ redis = await get_redis_async()+ if not await _screen_started_in(owner, sandbox.sandbox_id):+ return+ raw = await redis.get(owner.stream_key())+ password = raw.decode() if isinstance(raw, bytes) else raw+ if password == _STREAM_STOPPED:+ return+ if not resumed:+ if password is not None:+ if timeout:+ await redis.expire(owner.stream_key(), timeout)+ return+ if await redis.get(owner.display_lock_key()) is not None:+ return+ except Exception as exc:+ logger.warning("[E2B] Could not read %s's screen state: %s", owner, exc)+ return+ await _revoke_stream(owner, sandbox)+++async def _screen_started_in(owner: SandboxOwner, sandbox_id: str) -> bool:+ redis = await get_redis_async()+ raw = await redis.get(owner.display_key())+ value = raw.decode() if isinstance(raw, bytes) else raw+ return value == sandbox_id++async def _forget_stream(owner: SandboxOwner) -> None:"""Drop the stream password: the screen's next open issues a fresh one.- Called whenever the box pauses, so the window in which a stream URL- works is one running stretch of the box (``SandboxOwner.stream_key``).+ Called when we pause the box without its stream known to be stopped; the+ next connect finds the box paused and stops the stream then+ (``_settle_stream``)."""with contextlib.suppress(Exception):redis = await get_redis_async()diff --git a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.pyindex 543090b470..95e2c907e5 100644--- a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py@@ -21,7 +21,7 @@ import pytestfrom e2b import SandboxStatefrom e2b.exceptions import SandboxNotFoundException-from backend.blocks.desktop._api import SHARED_PATH, WORKSPACE_PATH+from backend.blocks.desktop._api import _STOP_STREAM, SHARED_PATH, WORKSPACE_PATHfrom backend.blocks.desktop._common import (expert_volume_name,user_volume_name,@@ -69,6 +69,7 @@ def _mock_sandbox(sb.is_running = AsyncMock(return_value=running)sb.pause = AsyncMock()sb.kill = AsyncMock()+ sb.commands.run = AsyncMock()stamped = (owner or SandboxOwner(kind="session", id=_SESSION_ID)).metadata()sb.get_info = AsyncMock(return_value=MagicMock(metadata=stamped))_STAMPS[sandbox_id] = sb.get_info.return_value@@ -251,6 +252,8 @@ class TestGetOrCreateSandbox:)assert result is sb+ # A box whose screen was never on pays no command on the way back.+ sb.commands.run.assert_not_awaited()mock_cls.create.assert_not_called()# redis.set called once to refresh TTL, not to claim a creation slotredis.set.assert_awaited_once()@@ -786,7 +789,9 @@ class TestPauseSandboxDirect:assert result is Truesb.pause.assert_awaited_once()- redis.get.assert_not_awaited()+ # The screen was never on: one flag read, and no command in the box.+ redis.get.assert_awaited_once_with(f"copilot:e2b:sandbox:{_SESSION_ID}:display")+ sb.commands.run.assert_not_awaited()# The turn-end pause is where the stream password is dropped.redis.delete.assert_awaited_once_with(f"copilot:e2b:sandbox:{_SESSION_ID}:stream"@@ -819,6 +824,285 @@ class TestPauseSandboxDirect:assert result is False+# ---------------------------------------------------------------------------+# A pause ends the stream: the password is forgotten *and* nothing serves it+# ---------------------------------------------------------------------------++_SHELL_KEY = f"copilot:e2b:sandbox:{_SESSION_ID}"+_DISPLAY_KEY = f"{_SHELL_KEY}:display"+_STREAM_KEY = f"{_SHELL_KEY}:stream"+_DISPLAY_LOCK_KEY = f"{_DISPLAY_KEY}:lock"+++def _screen_redis(**keys: str) -> AsyncMock:+ """Redis holding the box id plus the given screen keys, as bytes.++ Unlike ``_keyed_redis`` an empty value is a value: the stopped-stream+ marker is an empty string, and a missing key is ``None``.+ """+ values = {_SHELL_KEY: _SANDBOX_ID, **keys}+ r = AsyncMock()+ r.get = AsyncMock(+ side_effect=lambda key: values[key].encode() if key in values else None+ )+ r.set = AsyncMock(return_value=True)+ r.delete = AsyncMock()+ return r+++def _stream_key_writes(redis: AsyncMock) -> list:+ return [c for c in redis.set.await_args_list if c.args[0] == _STREAM_KEY]+++def _assert_stream_stopped_as_root(sb: MagicMock) -> None:+ sb.commands.run.assert_awaited_once()+ call = sb.commands.run.await_args+ assert call.args[0] == _STOP_STREAM+ assert call.kwargs["user"] == "root"+++class TestPauseStopsTheStream:+ def test_pause_stops_the_stream_before_the_box_pauses(self):+ sb = _mock_sandbox()+ order: list[str] = []+ sb.commands.run = AsyncMock(side_effect=lambda *_, **__: order.append("stop"))+ sb.pause = AsyncMock(side_effect=lambda: order.append("pause"))+ redis = _screen_redis(**{_DISPLAY_KEY: _SANDBOX_ID, _STREAM_KEY: "issued"})+ with _patch_sdk() as mock_cls, _patch_redis(redis):+ mock_cls.connect = AsyncMock(return_value=sb)+ assert asyncio.run(pause_sandbox(_SESSION_ID, _API_KEY)) is True++ assert order == ["stop", "pause"]+ _assert_stream_stopped_as_root(sb)+ # The password is replaced by the stopped marker, not just deleted,+ # so the next reconnect knows there is nothing left to stop.+ [write] = _stream_key_writes(redis)+ assert write.args[1] == ""+ redis.delete.assert_not_awaited()++ def test_direct_pause_stops_the_stream_before_the_box_pauses(self):+ sb = _mock_sandbox()+ order: list[str] = []+ sb.commands.run = AsyncMock(side_effect=lambda *_, **__: order.append("stop"))+ sb.pause = AsyncMock(side_effect=lambda: order.append("pause"))+ redis = _screen_redis(**{_DISPLAY_KEY: _SANDBOX_ID, _STREAM_KEY: "issued"})+ with _patch_redis(redis):+ assert asyncio.run(pause_sandbox_direct(sb, _SESSION_ID)) is True++ assert order == ["stop", "pause"]+ _assert_stream_stopped_as_root(sb)+ [write] = _stream_key_writes(redis)+ assert write.args[1] == ""+ redis.delete.assert_not_awaited()++ def test_a_screen_flag_for_a_replaced_box_stops_nothing(self):+ sb = _mock_sandbox()+ redis = _screen_redis(**{_DISPLAY_KEY: "sb-old", _STREAM_KEY: "issued"})+ with _patch_redis(redis):+ assert asyncio.run(pause_sandbox_direct(sb, _SESSION_ID)) is True++ sb.commands.run.assert_not_awaited()+ redis.delete.assert_awaited_once_with(_STREAM_KEY)++ @pytest.mark.parametrize("direct", [True, False])+ def test_a_stop_that_fails_does_not_block_the_pause(self, direct: bool):+ sb = _mock_sandbox()+ sb.commands.run = AsyncMock(side_effect=RuntimeError("box not answering"))+ redis = _screen_redis(**{_DISPLAY_KEY: _SANDBOX_ID, _STREAM_KEY: "issued"})+ with _patch_sdk() as mock_cls, _patch_redis(redis):+ mock_cls.connect = AsyncMock(return_value=sb)+ paused = asyncio.run(+ pause_sandbox_direct(sb, _SESSION_ID)+ if direct+ else pause_sandbox(_SESSION_ID, _API_KEY)+ )++ assert paused is True+ sb.commands.run.assert_awaited_once()+ sb.pause.assert_awaited_once()+ # Not known to be stopped: the password is dropped with no marker, so+ # the box's next reconnect tries the stop again.+ assert _stream_key_writes(redis) == []+ redis.delete.assert_awaited_once_with(_STREAM_KEY)++ def test_a_stop_that_hangs_is_cut_off_and_the_box_still_pauses(self):+ async def _hang(*_, **__):+ await asyncio.sleep(30)++ sb = _mock_sandbox()+ sb.commands.run = AsyncMock(side_effect=_hang)+ redis = _screen_redis(**{_DISPLAY_KEY: _SANDBOX_ID, _STREAM_KEY: "issued"})+ with (+ _patch_redis(redis),+ patch(+ "backend.copilot.tools.e2b_sandbox._STOP_STREAM_TIMEOUT_SECONDS", 0.01+ ),+ ):+ assert asyncio.run(pause_sandbox_direct(sb, _SESSION_ID)) is True++ sb.commands.run.assert_awaited_once()+ sb.pause.assert_awaited_once()+ redis.delete.assert_awaited_once_with(_STREAM_KEY)++ def test_the_stop_does_not_eat_into_the_time_the_pause_has(self):+ """The lookup pause runs connect, stop and pause under one deadline:+ the stop's own limit is added to it rather than taken out of it."""+ sb = _mock_sandbox()+ redis = _screen_redis(**{_DISPLAY_KEY: _SANDBOX_ID, _STREAM_KEY: "issued"})+ real_wait_for = asyncio.wait_for+ budgets: list[float] = []++ async def _recording_wait_for(awaitable, timeout):+ budgets.append(timeout)+ return await real_wait_for(awaitable, timeout)++ with (+ _patch_sdk() as mock_cls,+ _patch_redis(redis),+ patch(+ "backend.copilot.tools.e2b_sandbox.asyncio.wait_for",+ _recording_wait_for,+ ),+ ):+ mock_cls.connect = AsyncMock(return_value=sb)+ assert asyncio.run(pause_sandbox(_SESSION_ID, _API_KEY)) is True++ assert budgets == [15, 5]++ def test_unreadable_screen_state_does_not_block_the_pause(self):+ sb = _mock_sandbox()+ redis = _screen_redis()+ redis.get = AsyncMock(side_effect=ConnectionError("redis down"))+ with _patch_redis(redis):+ assert asyncio.run(pause_sandbox_direct(sb, _SESSION_ID)) is True++ redis.get.assert_awaited_once_with(_DISPLAY_KEY)+ sb.pause.assert_awaited_once()+++class TestReconnectSettlesTheStream:+ """A stream that outlived its running stretch is stopped on reconnect."""++ def _reconnect(self, redis: AsyncMock, sb: MagicMock, *, paused: bool):+ """Reconnect to *sb*, which E2B reports as *paused* (or running) just+ before the connect wakes it."""+ _STAMPS[sb.sandbox_id].state = (+ SandboxState.PAUSED if paused else SandboxState.RUNNING+ )+ with _patch_sdk() as mock_cls, _patch_redis(redis):+ mock_cls.connect = AsyncMock(return_value=sb)+ return asyncio.run(+ get_or_create_sandbox(_SESSION_ID, _API_KEY, timeout=_TIMEOUT)+ )++ @pytest.mark.parametrize(+ "keys",+ [+ pytest.param({_DISPLAY_KEY: _SANDBOX_ID}, id="password-gone"),+ # E2B's own timeout pause can land before the password expires.+ pytest.param(+ {_DISPLAY_KEY: _SANDBOX_ID, _STREAM_KEY: "issued"},+ id="password-still-remembered",+ ),+ # The opener's own connect is what resumed the box: nobody can be+ # part-way through starting a stream in a box that was paused.+ pytest.param(+ {_DISPLAY_KEY: _SANDBOX_ID, _DISPLAY_LOCK_KEY: "token"},+ id="open-in-progress",+ ),+ ],+ )+ def test_a_box_that_was_paused_comes_back_with_its_stream_stopped(self, keys: dict):+ sb = _mock_sandbox()+ redis = _screen_redis(**keys)++ assert self._reconnect(redis, sb, paused=True) is sb++ _assert_stream_stopped_as_root(sb)+ [write] = _stream_key_writes(redis)+ assert write.args[1] == ""++ def test_a_running_box_with_no_stream_key_was_resumed_behind_our_back(self):+ """The password outlived the box's running limit, so the box did pause+ and something other than us woke it: its stream is a leftover."""+ sb = _mock_sandbox()+ redis = _screen_redis(**{_DISPLAY_KEY: _SANDBOX_ID})++ assert self._reconnect(redis, sb, paused=False) is sb++ _assert_stream_stopped_as_root(sb)+ [write] = _stream_key_writes(redis)+ assert write.args[1] == ""++ def test_a_watched_stream_on_a_box_that_never_paused_is_kept_alive(self):+ """An expert's overlapping turns keep the box running past the+ password's first expiry: each connect pushes the expiry out with the+ box's re-armed limit instead of letting the stream read as a leftover."""+ sb = _mock_sandbox()+ redis = _screen_redis(**{_DISPLAY_KEY: _SANDBOX_ID, _STREAM_KEY: "issued"})++ assert self._reconnect(redis, sb, paused=False) is sb++ sb.commands.run.assert_not_awaited()+ redis.expire.assert_awaited_once_with(_STREAM_KEY, _TIMEOUT)+ assert _stream_key_writes(redis) == []++ def test_a_stop_that_fails_still_hands_the_box_back(self):+ sb = _mock_sandbox()+ sb.commands.run = AsyncMock(side_effect=RuntimeError("box not answering"))+ redis = _screen_redis(**{_DISPLAY_KEY: _SANDBOX_ID})++ assert self._reconnect(redis, sb, paused=True) is sb++ sb.commands.run.assert_awaited_once()+ assert _stream_key_writes(redis) == []++ def test_unreadable_screen_state_still_hands_the_box_back(self):+ sb = _mock_sandbox()+ redis = _screen_redis()+ shell_get = redis.get.side_effect++ def _get(key: str):+ if key == _DISPLAY_KEY:+ raise ConnectionError("redis down")+ return shell_get(key)++ redis.get = AsyncMock(side_effect=_get)++ assert self._reconnect(redis, sb, paused=True) is sb++ sb.commands.run.assert_not_awaited()++ @pytest.mark.parametrize(+ "paused, keys",+ [+ pytest.param(+ True, {_DISPLAY_KEY: _SANDBOX_ID, _STREAM_KEY: ""}, id="already-stopped"+ ),+ pytest.param(+ False,+ {_DISPLAY_KEY: _SANDBOX_ID, _STREAM_KEY: ""},+ id="already-stopped-and-running",+ ),+ pytest.param(+ False,+ {_DISPLAY_KEY: _SANDBOX_ID, _DISPLAY_LOCK_KEY: "token"},+ id="open-in-progress-on-a-running-box",+ ),+ pytest.param(True, {_DISPLAY_KEY: "sb-old"}, id="flag-for-a-replaced-box"),+ pytest.param(True, {}, id="screen-never-on"),+ ],+ )+ def test_a_stream_that_is_accounted_for_is_left_alone(+ self, paused: bool, keys: dict+ ):+ sb = _mock_sandbox()++ assert self._reconnect(_screen_redis(**keys), sb, paused=paused) is sb++ sb.commands.run.assert_not_awaited()++# ---------------------------------------------------------------------------# Expert boxes: one persistent sandbox per hired expert# ---------------------------------------------------------------------------