AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): put gh, Node and python back into the AutoPilot sandbox image (#15058)
Reinier van der Leer90b19f8cc5d3
diff --git a/autogpt_platform/backend/backend/copilot/computer_test.py b/autogpt_platform/backend/backend/copilot/computer_test.pyindex 5159bb0df6..4296435320 100644--- a/autogpt_platform/backend/backend/copilot/computer_test.py+++ b/autogpt_platform/backend/backend/copilot/computer_test.py@@ -31,7 +31,7 @@ def _info(sandbox_id: str, state: SandboxState, mounts: str = "attached"):started_at=datetime(2026, 9, 5, 12, 0, tzinfo=timezone.utc),cpu_count=1,memory_mb=2048,- template_id="agpt-desktop-1x2",+ template_id="agpt-desktop-1x2-004d6e73",metadata={"autogpt_kind": "shell", "autogpt_mounts": mounts},)@@ -189,7 +189,7 @@ class TestOpenDesktop:redis_p, get_p, cls_p, cfg_p = self._patches(redis, sandbox, desktop)with redis_p, get_p as get_mock, cls_p as desktop_cls, cfg_p as cfg:cfg.e2b_sandbox_timeout = 420- cfg.e2b_sandbox_template = "agpt-desktop-1x2"+ cfg.e2b_sandbox_template = "agpt-desktop-1x2-004d6e73"cfg.e2b_sandbox_on_timeout = "pause"stream, first_time, shared = await open_desktop(owner, mounts, "k", user_id=_USER, session_id=_SESSION@@ -202,7 +202,7 @@ class TestOpenDesktop:owner,"k",timeout=420,- template="agpt-desktop-1x2",+ template="agpt-desktop-1x2-004d6e73",on_timeout="pause",volume_mounts=mounts,user_id=_USER,diff --git a/autogpt_platform/backend/backend/copilot/config.py b/autogpt_platform/backend/backend/copilot/config.pyindex 02d23e4a10..f1a6080c10 100644--- a/autogpt_platform/backend/backend/copilot/config.py+++ b/autogpt_platform/backend/backend/copilot/config.py@@ -746,7 +746,7 @@ class ChatConfig(BaseSettings):description="E2B API key. Falls back to E2B_API_KEY environment variable.",)e2b_sandbox_template: str = Field(- default="agpt-desktop-1x2",+ default="agpt-desktop-1x2-004d6e73",description="E2B sandbox template for copilot sessions. The default is our ""own image (E2B's desktop image at 1 vCPU / 2 GiB, ~$0.08/h running, ""no display started), built on the team automatically the first time "diff --git a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.pyindex 2e4ca01511..ea658298a9 100644--- a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py+++ b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox.py@@ -95,7 +95,11 @@ from backend.util.e2b_network import (create_sandbox,forget_sandbox,)-from backend.util.e2b_template import ensure_template, forget_template+from backend.util.e2b_template import (+ SUPERSEDED_TEMPLATES,+ ensure_template,+ forget_template,+)from backend.util.sandbox_metadata import MountState, SandboxMetadata, owned_by_userlogger = logging.getLogger(__name__)@@ -110,6 +114,7 @@ METADATA_KIND = "autogpt_kind"# "attached" when the workspace volumes were mounted, "none" when creation had# to fall back to a volume-less box — visible in the E2B dashboard and API.METADATA_MOUNTS = "autogpt_mounts"+METADATA_TEMPLATE = "autogpt_template"# Per-attempt timeout for AsyncSandbox.create(). E2B normally provisions a# sandbox in 5-15 s; 30 s gives generous headroom while ensuring a slow/hung@@ -477,12 +482,14 @@ async def _try_reconnect(*,timeout: int | None = None,user_id: str | None = None,+ template: str | None = None,) -> "AsyncSandbox | None":"""Reconnect to the owner's box, or ``None`` if it is gone.- Gone means E2B no longer has it, it is stamped for someone else, or it- came back not running: the cached id is dropped so a replacement can be- created. Anything else (a 5xx, a network blip) is raised, not swallowed.+ Gone means E2B no longer has it, it is stamped for someone else, it came+ back not running, or it runs a superseded image and was retired in favour+ of *template*: the cached id is dropped so a replacement can be created.+ Anything else (a 5xx, a network blip) is raised, not swallowed.The box may be perfectly fine, and replacing it on a guess would forkeverything on it that is not in a volume: the screen, running processes,installed tools. *timeout* re-arms the box's running-time limit.@@ -493,6 +500,11 @@ async def _try_reconnect(# wakes anything. The state read with it says whether this connect is# what resumes the box.info = await _owned_info(sandbox_id, owner, api_key)+ if template and await _retire_superseded_box(+ sandbox_id, info, owner, template, api_key+ ):+ await _clear_stored_sandbox_id(owner)+ return Nonesandbox = await _connect_pinned(sandbox_id,info,@@ -524,6 +536,64 @@ async def _try_reconnect(return None+async def _retire_superseded_box(+ sandbox_id: str,+ info: SandboxInfo,+ owner: SandboxOwner,+ template: str,+ api_key: str,+) -> bool:+ """Kill the owner's box, unconnected, if it runs a superseded image.++ A new image reaches an owner only through a new box. Its ``~/workspace``+ and ``~/shared`` volumes carry over and the rest of its filesystem does+ not, so a box without them, or with another turn on it, is kept.+ """+ stamped = info.metadata or {}+ built_from = stamped.get(METADATA_TEMPLATE)+ if (+ built_from not in SUPERSEDED_TEMPLATES+ or built_from == template+ or stamped.get(METADATA_MOUNTS) != "attached"+ or await _has_active_turns(owner)+ ):+ return False+ try:+ await asyncio.wait_for(+ AsyncSandbox.kill(sandbox_id, api_key=api_key),+ timeout=_E2B_API_TIMEOUT_SECONDS,+ )+ except Exception as exc:+ logger.warning(+ "[E2B] Could not retire %s's box %.12s (%s); keeping it",+ owner,+ sandbox_id,+ exc,+ )+ return False+ await forget_sandbox(sandbox_id)+ await _forget_owner_state(owner)+ logger.info(+ "[E2B] Retired %s's box %.12s: built from %s, replacing it with %s",+ owner,+ sandbox_id,+ built_from,+ template,+ )+ return True+++async def _has_active_turns(owner: SandboxOwner) -> bool:+ """Whether another turn is on the owner's box; unknown counts as yes."""+ if not owner.is_expert:+ return False+ try:+ redis = await get_redis_async()+ return int(await redis.get(_active_turns_key(owner)) or 0) > 0+ except Exception:+ return True++async def _resolve_volume_mounts(volume_mounts: Mapping[str, str] | None, api_key: str) -> dict[str, "AsyncVolume | str"] | None:@@ -665,7 +735,12 @@ async def get_or_create_owner_sandbox(# Existing sandbox ID — try to reconnect (auto-resumes if paused).try:sandbox = await _try_reconnect(- value, owner, api_key, timeout=timeout, user_id=user_id+ value,+ owner,+ api_key,+ timeout=timeout,+ user_id=user_id,+ template=template,)except Exception as exc:if value in retried_ids:diff --git a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.pyindex a4c26458ed..7a29f11e53 100644--- a/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/e2b_sandbox_test.py@@ -27,6 +27,7 @@ from backend.blocks.desktop._common import (user_volume_name,workspace_volume_mounts,)+from backend.util.e2b_template import DESKTOP_IMAGEfrom backend.util.sandbox_metadata import deployment_envfrom .e2b_sandbox import (@@ -321,13 +322,20 @@ class TestGetOrCreateSandbox:mock_cls.create = AsyncMock(side_effect=fake_create)asyncio.run(get_or_create_sandbox(- _SESSION_ID, _API_KEY, timeout=_TIMEOUT, template="agpt-desktop-1x2"+ _SESSION_ID,+ _API_KEY,+ timeout=_TIMEOUT,+ template="agpt-desktop-1x2-004d6e73",))# The build can take longer than the creation slot's TTL, so it must# finish before the slot is claimed.- assert order == [f"ensure:agpt-desktop-1x2:{_API_KEY}", "claim", "create"]+ assert order == [+ f"ensure:agpt-desktop-1x2-004d6e73:{_API_KEY}",+ "claim",+ "create",+ ]def test_create_with_on_timeout_kill(self):"""on_timeout='kill' disables auto_resume automatically."""@@ -1551,6 +1559,76 @@ class TestExpertShellBox:assert _turn_acquires(redis) == []+class TestSupersededImage:+ """A box built from a superseded image is swapped for one on the current one."""++ _OLD = "sb-old"+ _STAMP = {"autogpt_template": "agpt-desktop-1x2", "autogpt_mounts": "attached"}++ def _run(+ self,+ stamp: dict[str, str],+ redis_values: dict[str, str | None],+ template: str = DESKTOP_IMAGE.alias,+ ):+ owner = SandboxOwner(kind="expert", id=_EXPERT_ID)+ old = _mock_sandbox(self._OLD, owner=owner)+ _STAMPS[self._OLD].metadata = {**owner.metadata(), **stamp}+ new = _mock_sandbox("sb-new", owner=owner)+ values = {_EXPERT_SHELL_KEY: self._OLD, **redis_values}+ redis = _keyed_redis(values)+ redis.delete = AsyncMock(+ side_effect=lambda *keys: [values.pop(k, 0) for k in keys]+ )+ with (+ _patch_sdk() as mock_cls,+ _patch_redis(redis),+ patch("backend.copilot.tools.e2b_sandbox.ensure_template", AsyncMock()),+ ):+ mock_cls.connect = AsyncMock(return_value=old)+ mock_cls.create = AsyncMock(return_value=new)+ result = asyncio.run(+ get_or_create_sandbox(+ _SESSION_ID,+ _API_KEY,+ timeout=_TIMEOUT,+ template=template,+ expert_id=_EXPERT_ID,+ )+ )+ return result, old, new, mock_cls++ def test_an_idle_box_on_a_superseded_image_is_replaced(self):+ result, _, new, mock_cls = self._run(self._STAMP, {})++ assert result is new+ # Killed by id without connecting: a paused box is not resumed to die.+ mock_cls.connect.assert_not_awaited()+ mock_cls.kill.assert_awaited_once_with(self._OLD, api_key=_API_KEY)+ assert mock_cls.create.await_args.kwargs["template"] == DESKTOP_IMAGE.alias++ @pytest.mark.parametrize(+ "stamp, redis_values, template",+ [+ # Its ~/workspace is not a volume: replacing it would lose the files.+ ({**_STAMP, "autogpt_mounts": "none"}, {}, DESKTOP_IMAGE.alias),+ # Another turn is running commands on it.+ (_STAMP, {_EXPERT_ACTIVE_KEY: "1"}, DESKTOP_IMAGE.alias),+ # Still the configured image, e.g. pinned by an env override.+ (_STAMP, {}, "agpt-desktop-1x2"),+ ],+ ids=["no-volumes", "turn-in-flight", "configured-image"],+ )+ def test_a_box_that_cannot_be_replaced_safely_is_kept(+ self, stamp, redis_values, template+ ):+ result, old, _, mock_cls = self._run(stamp, redis_values, template)++ assert result is old+ mock_cls.kill.assert_not_awaited()+ mock_cls.create.assert_not_awaited()++class TestExpertPause:def test_last_turn_pauses_the_box(self):sb = _mock_sandbox()diff --git a/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.py b/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.pyindex 0abe5bf80b..3a72830bcf 100644--- a/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/start_desktop_test.py@@ -84,7 +84,7 @@ async def _run(tool, box: _Box, *, user_id, session):with redis_p, get_p, cls_p, computer_cfg as ccfg, tool_cfg as tcfg:tcfg.active_e2b_api_key = "e2b_test_key"ccfg.e2b_sandbox_timeout = 420- ccfg.e2b_sandbox_template = "agpt-desktop-1x2"+ ccfg.e2b_sandbox_template = "agpt-desktop-1x2-004d6e73"ccfg.e2b_sandbox_on_timeout = "pause"return await tool._execute(user_id=user_id, session=session)@@ -126,7 +126,7 @@ class TestStartDesktop:f"copilot:e2b:sandbox:{session.session_id}")assert kwargs["volume_mounts"] == {WORKSPACE_PATH: user_volume_name(_USER)}- assert kwargs["template"] == "agpt-desktop-1x2"+ assert kwargs["template"] == "agpt-desktop-1x2-004d6e73"assert kwargs["count_turn"] is Falseassert kwargs["user_id"] == _USERassert kwargs["session_id"] == session.session_iddiff --git a/autogpt_platform/backend/backend/util/e2b_template.py b/autogpt_platform/backend/backend/util/e2b_template.pyindex c8509dd1fc..b7ccdec5f5 100644--- a/autogpt_platform/backend/backend/util/e2b_template.py+++ b/autogpt_platform/backend/backend/util/e2b_template.py@@ -6,13 +6,16 @@ re-snapshot it at 1 vCPU / 2 GiB under our own alias, which is cheaper torun than E2B's ``base`` (2 vCPU / 512 MiB) with four times the RAM, and italready carries XFCE, Chrome, Firefox and VS Code, so a box can later turn ascreen on without changing image. Nothing graphical starts at boot: a shell-box on this image idles at about 90 MiB.+box on this image idles at about 90 MiB. On top of it we install the+developer tools ``base`` gave agents and ``desktop`` lacks (``desktop_image``).Template aliases live per E2B team, so the first sandbox on a new team (orkey) has to build it. ``ensure_template`` checks the alias and builds it-from ``desktop`` when missing (12-25 s, once per team), serialised through+from ``desktop`` when missing (about 50 s, once per team), serialised throughRedis so parallel first turns don't each start a build. Templates we don't-manage are left alone.+manage are left alone. A team keeps a ready alias forever, so the alias ends+in a digest of the image's build steps: changing them renames the image and+every team builds the new one on first use."Exists" is not "ready": E2B registers an alias the moment a build isrequested, before the build has run, and a failed build leaves the alias in@@ -39,6 +42,7 @@ from e2b.api.client.models import ()from e2b.api.client_async import get_api_clientfrom e2b.connection_config import ConnectionConfig+from e2b.template.main import TemplateBuilderfrom e2b.template.types import BuildInfofrom pydantic import BaseModel, ConfigDict@@ -70,14 +74,18 @@ class TemplateSpec(BaseModel):DESKTOP_IMAGE = TemplateSpec(- alias="agpt-desktop-1x2", source="desktop", cpu_count=1, memory_mb=2048+ alias="agpt-desktop-1x2-004d6e73", source="desktop", cpu_count=1, memory_mb=2048)MANAGED_TEMPLATES: dict[str, TemplateSpec] = {DESKTOP_IMAGE.alias: DESKTOP_IMAGE}--# A build takes 12-25 s. The build is cut off before the lock can expire, so-# the lock is only ever released by its owner (or by the TTL after a crash).-# Followers wait as long as the lock can live, so they never give up on a-# build that is still allowed to finish.+# Images whose boxes are replaced on their next reconnect because they lack+# tools agents rely on. Not every new image is worth that: a replacement+# keeps only the box's volumes.+SUPERSEDED_TEMPLATES = frozenset({"agpt-desktop-1x2"})++# A build takes under a minute. The build is cut off before the lock can+# expire, so the lock is only ever released by its owner (or by the TTL after+# a crash). Followers wait as long as the lock can live, so they never give+# up on a build that is still allowed to finish._BUILD_LOCK_TTL_SECONDS = 300_BUILD_TIMEOUT_SECONDS = 240_BUILD_WAIT_SECONDS = _BUILD_LOCK_TTL_SECONDS@@ -193,7 +201,7 @@ async def build_template(spec: TemplateSpec, api_key: str) -> BuildInfo:spec.source,)info = await AsyncTemplate.build(- Template().from_template(spec.source),+ desktop_image(spec.source),spec.alias,tags=spec.tags,cpu_count=spec.cpu_count,@@ -204,6 +212,69 @@ async def build_template(spec: TemplateSpec, api_key: str) -> BuildInfo:return info+def desktop_image(source: str) -> TemplateBuilder:+ """*source* plus the tools E2B's ``base`` image gave agents and it lacks.++ Changing these steps changes ``DESKTOP_IMAGE``'s alias (its test says to).+ """+ return (+ Template()+ .from_template(source)+ .run_cmd(+ [+ "install -d -m 755 /etc/apt/keyrings",+ f"curl -fsSL -o {_GH_KEYRING} {_GH_KEY}",+ f"curl -fsSL {_NODE_KEY} | gpg --dearmor -o {_NODE_KEYRING}",+ f"chmod go+r {_GH_KEYRING} {_NODE_KEYRING}",+ f"echo '{_GH_REPO}' > /etc/apt/sources.list.d/github-cli.list",+ f"echo '{_NODE_REPO}' > /etc/apt/sources.list.d/nodesource.list",+ ],+ user="root",+ )+ .apt_install(+ [+ "gh",+ "nodejs",+ "python-is-python3",+ "python3-venv",+ "file",+ "pkg-config",+ "imagemagick",+ ],+ no_install_recommends=True,+ )+ .npm_install("yarn@1", g=True)+ # apt-get update only warns about an unreachable repository, and gh and+ # nodejs then resolve from Ubuntu's archive: fail the build instead.+ .run_cmd(_VERIFY_IMAGE, user="user")+ )+++# Ubuntu 22.04's own gh is 2.4 (2022) and it ships Node 12, so both come+# from their vendors' apt repositories.+_GH_KEY = "https://cli.github.com/packages/githubcli-archive-keyring.gpg"+_GH_KEYRING = "/etc/apt/keyrings/githubcli.gpg"+_GH_REPO = f"deb [signed-by={_GH_KEYRING}] https://cli.github.com/packages stable main"+_NODE_KEY = "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key"+_NODE_KEYRING = "/etc/apt/keyrings/nodesource.gpg"+_NODE_REPO = f"deb [signed-by={_NODE_KEYRING}] https://deb.nodesource.com/node_24.x nodistro main"+_VERIFY_IMAGE = [+ "dpkg --compare-versions \"$(dpkg-query -W -f='${Version}' gh)\" ge 2.100",+ "dpkg --compare-versions \"$(dpkg-query -W -f='${Version}' nodejs)\" ge 24",+ "gh --version",+ "node --version",+ "npm --version",+ "npx --version",+ "yarn --version",+ "python --version",+ "python3 -m venv /tmp/venv-check",+ "rm -rf /tmp/venv-check",+ "file --version",+ "pkg-config --version",+ "convert --version",+]++async def get_template_state(spec: TemplateSpec, api_key: str) -> TemplateState:"""What the team has behind *spec*'s alias right now."""async with get_api_client(ConnectionConfig(api_key=api_key)) as client:diff --git a/autogpt_platform/backend/backend/util/e2b_template_test.py b/autogpt_platform/backend/backend/util/e2b_template_test.pyindex e60dcfff47..172aeccec5 100644--- a/autogpt_platform/backend/backend/util/e2b_template_test.py+++ b/autogpt_platform/backend/backend/util/e2b_template_test.py@@ -1,7 +1,9 @@import asyncio+import hashlibfrom unittest.mock import AsyncMock, MagicMock, patchimport pytest+from e2b import Templatefrom e2b.api.client.models import TemplateAliasResponse, TemplateBuildStatusfrom backend.copilot.config import ChatConfig@@ -10,6 +12,7 @@ from backend.util.e2b_template import (DESKTOP_IMAGE,TemplateSpec,TemplateState,+ desktop_image,ensure_template,forget_ready_templates,get_template_state,@@ -44,6 +47,15 @@ class TestSpec:assert ChatConfig().e2b_sandbox_template == DESKTOP_IMAGE.aliasassert DESKTOP_IMAGE.cpu_count == 1 and DESKTOP_IMAGE.memory_mb == 2048+ def test_the_alias_names_the_image_it_builds(self):+ # Teams keep a ready alias forever, so a changed image needs a new alias.+ steps = Template.to_json(desktop_image(DESKTOP_IMAGE.source))+ digest = hashlib.sha256(steps.encode()).hexdigest()[:8]+ assert DESKTOP_IMAGE.alias == f"agpt-desktop-1x2-{digest}", (+ "desktop_image() changed: rename DESKTOP_IMAGE and the "+ "ChatConfig.e2b_sandbox_template default to this alias"+ )+def test_tags_are_docker_safe_and_resolve_the_bare_alias(self):tags = TemplateSpec(alias="x", source="desktop", cpu_count=2, memory_mb=4096diff --git a/autogpt_platform/backend/scripts/build_desktop_template.py b/autogpt_platform/backend/scripts/build_desktop_template.pyindex fafc1cd52c..f06dbea809 100644--- a/autogpt_platform/backend/scripts/build_desktop_template.py+++ b/autogpt_platform/backend/scripts/build_desktop_template.py@@ -1,6 +1,6 @@"""Build the platform's sandbox image on an E2B team ahead of first use.-The backend builds ``agpt-desktop-1x2`` itself the first time a team needs+The backend builds ``DESKTOP_IMAGE`` itself the first time a team needsit (see ``backend.util.e2b_template``). Run this to do it up front, or tobuild an experimental size under another alias: