Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
"""Sandbox-level behaviours pinned against the E2B goldens (sbx_* conformance
cases): the 400/404 texts of E2B's API for malformed arguments, the not-found
and paused states, listing and metrics rules. In-memory backend."""
from __future__ import annotations
import asyncio
import dataclasses
import json
from datetime import datetime, timedelta, timezone
import e2b
import pytest
from e2b import AsyncSandbox, InvalidArgumentException, SandboxException
from e2b._backend import CuaMachine, set_backend
from fake_backend import FakeBackend
VALIDATION = (
"400: validation error: error in openapi3filter.RequestError: request body has an error: "
"doesn't match schema #/components/schemas/"
)
@pytest.fixture
def backend():
b = FakeBackend()
set_backend(b)
yield b
set_backend(None)
async def _sb(**kw):
return await AsyncSandbox.create(template="w", **kw)
async def _state(sb) -> str:
try:
return str((await AsyncSandbox.get_info(sb.sandbox_id)).state)
except e2b.SandboxNotFoundException:
return "gone"
def _text(e: pytest.ExceptionInfo) -> str:
return str(e.value)
# ── create: the request body is checked as E2B's API checks it ────────────────
@pytest.mark.parametrize(
"kw, pointer, what",
[
({"timeout": -1}, "/timeout", "number must be at least 1"),
({"timeout": 0}, "/timeout", "number must be at least 1"),
({"timeout": 30.5}, "/timeout", "value must be an integer"),
({"timeout": "30"}, "/timeout", "value must be an integer"),
({"metadata": {"k": 1}}, "/metadata/k", "value must be a string"),
({"envs": {"A": 1}}, "/envVars/A", "value must be a string"),
({"allow_internet_access": "no"}, "/allow_internet_access", "value must be a boolean"),
({"network": {"deny_out": [1]}}, "/network/denyOut/0", "value must be a string"),
({"mcp": "github"}, "/mcp", "value must be an object"),
(
{"lifecycle": {"on_timeout": "pause", "auto_resume": "yes"}},
"/autoResume/enabled",
"value must be a boolean",
),
],
)
async def test_create_refuses_what_the_api_schema_refuses(backend, kw, pointer, what):
with pytest.raises(SandboxException) as e:
await _sb(**kw)
assert type(e.value) is SandboxException
assert _text(e) == f'{VALIDATION}NewSandbox: Error at "{pointer}": {what}'
assert not backend.machines
async def test_create_accepts_what_the_api_accepts(backend):
# an empty metadata key, a long value and an odd env name are the API's to allow
await _sb(metadata={"": "v"}, envs={"A B": "1"})
await _sb(metadata={"k": "v" * 5000}, timeout=1)
async def test_allowed_domains_need_a_deny_of_all_traffic(backend):
text = (
"400: When specifying allowed domains in allow out, you must include "
"'ALL_TRAFFIC' in deny out to block all other traffic."
)
for allow in ("1.1.1.1", ["not an ip!!"], ["1.1.1.1/33"], [""]):
with pytest.raises(SandboxException) as e:
await _sb(network={"allow_out": allow})
assert _text(e) == text
await _sb(network={"allow_out": ["1.1.1.1"], "deny_out": [e2b.ALL_TRAFFIC]})
async def test_template_errors_read_like_the_api(backend):
from fake_backend import NotFound
backend.fail_create = NotFound("image not found")
with pytest.raises(SandboxException) as e:
await AsyncSandbox.create(template="no-such")
assert type(e.value) is SandboxException
assert _text(e) == "404: template 'no-such' not found"
backend.fail_create = None
with pytest.raises(SandboxException) as e:
await AsyncSandbox.create(template="Not A Template!!")
assert _text(e) == "400: Invalid template reference: invalid template ID: Not A Template!!"
async def test_a_failure_that_is_not_a_missing_template_keeps_its_text(backend):
backend.fail_create = SandboxException("qemu failed: Address already in use")
with pytest.raises(SandboxException, match="Address already in use"):
await _sb()
async def test_allow_internet_access_is_none_until_chosen(backend):
assert (await (await _sb()).get_info()).allow_internet_access is None
assert (await (await _sb(allow_internet_access=True)).get_info()).allow_internet_access is True
assert (
await (await _sb(allow_internet_access=False)).get_info()
).allow_internet_access is False
async def test_sandbox_has_no_envd_token(backend):
sb = await _sb()
assert sb._envd_access_token is None and sb.traffic_access_token is None
# ── set_timeout ───────────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"value, what",
[
(-5, "number must be at least 0"),
(1.5, "value must be an integer"),
("10", "value must be an integer"),
(True, "value must be an integer"),
(None, "Value is not nullable"),
],
)
async def test_set_timeout_edges(backend, value, what):
sb = await _sb()
for call in (sb.set_timeout(value), AsyncSandbox.set_timeout(sb.sandbox_id, value)):
with pytest.raises(SandboxException) as e:
await call
assert _text(e) == f'{VALIDATION}SandboxTimeoutRequest: Error at "/timeout": {what}'
assert await _state(sb) == "running"
async def test_set_timeout_zero_ends_the_sandbox_and_a_huge_one_is_capped(backend):
sb = await _sb()
await sb.set_timeout(10**9)
assert (await sb.get_info()).end_at - datetime.now(timezone.utc) < timedelta(hours=25)
await sb.set_timeout(0)
assert await _state(sb) == "gone"
with pytest.raises(e2b.SandboxNotFoundException, match="^Sandbox e2b-0 not found$"):
await sb.get_info()
# ── unknown, malformed and killed ids ─────────────────────────────────────────
async def test_unknown_ids_read_like_the_api(backend):
gone = "e2b-ffffffffff"
for call, text in (
(AsyncSandbox.connect(gone), f"Paused sandbox {gone} not found"),
(AsyncSandbox.get_info(gone), f"Sandbox {gone} not found"),
(AsyncSandbox.pause(gone), f"Sandbox {gone} not found"),
(AsyncSandbox.set_timeout(gone, 60), f"Sandbox {gone} not found"),
):
with pytest.raises(e2b.SandboxNotFoundException) as e:
await call
assert _text(e) == text
assert await AsyncSandbox.kill(gone) is False
assert await AsyncSandbox.kill("") is False
assert await AsyncSandbox.get_metrics(gone) == []
async def test_malformed_ids_are_a_400(backend):
for call in (AsyncSandbox.connect("not a valid id!"), AsyncSandbox.kill("not a valid id!")):
with pytest.raises(SandboxException) as e:
await call
assert type(e.value) is SandboxException
assert _text(e) == "400: Invalid sandbox ID"
async def test_calls_after_kill(backend):
sb = await _sb()
sid = sb.sandbox_id
assert await sb.kill() is True
assert await sb.kill() is False
assert await AsyncSandbox.kill(sid) is False
assert await sb.is_running() is False
assert await sb.get_metrics() == []
for call in (sb.get_info(), sb.set_timeout(60), sb.pause()):
with pytest.raises(e2b.SandboxNotFoundException, match=f"^Sandbox {sid} not found$"):
await call
with pytest.raises(e2b.SandboxNotRunningException):
await sb.commands.run("echo x")
# ── paused sandboxes ──────────────────────────────────────────────────────────
async def test_paused_sandbox_behaviour(backend):
sb = await _sb()
sid = sb.sandbox_id
assert await AsyncSandbox.pause(sid) is True
# another handle pausing again says so, as does the first
assert await AsyncSandbox.pause(sid) is False
assert await sb.pause() is False
assert (await sb.get_info()).state == e2b.SandboxState.PAUSED
assert str((await AsyncSandbox.get_info(sid)).state) == "paused"
assert await sb.is_running() is False
assert await AsyncSandbox.get_metrics(sid) == []
# the paused sandbox cannot run commands or have its lifetime set
with pytest.raises(e2b.SandboxNotRunningException):
await sb.commands.run("echo x")
with pytest.raises(e2b.SandboxNotFoundException, match=f"^Sandbox {sid} not found$"):
await AsyncSandbox.set_timeout(sid, 300)
with pytest.raises(e2b.SandboxNotFoundException):
await sb.set_timeout(300)
listed = await AsyncSandbox.list(
query=e2b.SandboxQuery(state=[e2b.SandboxState.PAUSED])
).next_items()
assert [i.sandbox_id for i in listed] == [sid]
# connect resumes it, and the first handle sees the sandbox running again
again = await AsyncSandbox.connect(sid)
assert (await again.commands.run("echo back")).stdout == "back\n"
assert await _state(sb) == "running"
assert (await sb.commands.run("echo mine")).stdout == "mine\n"
assert await AsyncSandbox.kill(sid) is True
# ── metrics, fork ─────────────────────────────────────────────────────────────
async def test_metrics_windows(backend):
sb = await _sb()
now = datetime.now(timezone.utc)
for kw in (
{"start": now + timedelta(hours=1)},
{"start": now, "end": now - timedelta(minutes=10)},
):
for call in (sb.get_metrics(**kw), AsyncSandbox.get_metrics(sb.sandbox_id, **kw)):
with pytest.raises(SandboxException) as e:
await call
assert _text(e) == "400: Invalid time range for metrics"
assert len(await sb.get_metrics(start=now - timedelta(minutes=10), end=now)) == 1
@pytest.mark.parametrize("count", [0, -1, 21, True, "2", 1.5])
async def test_fork_count_is_validated_before_anything_forks(backend, count):
sb = await _sb()
for call in (sb.fork(count=count), AsyncSandbox.fork(sb.sandbox_id, count=count)):
with pytest.raises(InvalidArgumentException) as e:
await call
assert _text(e) == "count must be an integer between 1 and 20"
assert len(backend.machines) == 1
# ── listing ───────────────────────────────────────────────────────────────────
async def test_listing_limits_order_and_tokens(backend):
made = [await _sb(metadata={"n": str(i)}) for i in range(3)]
for i, sb in enumerate(made):
backend.records[sb.sandbox_id]["started_at"] = datetime(
2026, 1, 1 + i, tzinfo=timezone.utc
).isoformat()
# newest first unless the order is given
default = [i.sandbox_id for i in await AsyncSandbox.list().next_items()]
assert default == [s.sandbox_id for s in reversed(made)]
asc = [i.sandbox_id for i in await AsyncSandbox.list(order="asc").next_items()]
assert asc == [s.sandbox_id for s in made]
assert len(await AsyncSandbox.list(limit=0).next_items()) == 3
for limit, what in ((-1, "number must be at least 1"), (100000, "number must be at most 100")):
with pytest.raises(SandboxException) as e:
await AsyncSandbox.list(limit=limit).next_items()
assert _text(e) == (
f'400: validation error: error in openapi3filter.RequestError: parameter "limit" in query has an error: {what}'
)
with pytest.raises(SandboxException) as e:
await AsyncSandbox.list(next_token="garbage-token").next_items()
assert _text(e) == "400: Invalid next token"
# an invalid order is the SDK's own error, raised when the page is fetched
paginator = AsyncSandbox.list(order="sideways")
with pytest.raises(InvalidArgumentException, match="Invalid order 'sideways'"):
await paginator.next_items()
async def test_a_listed_sandbox_reports_what_get_info_does(backend):
sb = await _sb(timeout=600, metadata={"a": "b"}, network={"deny_out": ["10.0.0.0/8"]})
backend.records[sb.sandbox_id]["started_at"] = "2026-01-01T00:00:00+00:00"
backend.records[sb.sandbox_id]["end_at"] = "2026-01-01T00:10:00+00:00"
listed = (await AsyncSandbox.list().next_items())[0]
info = await AsyncSandbox.get_info(sb.sandbox_id)
assert dataclasses.replace(listed, started_at=info.started_at) == info
assert listed.end_at == datetime(2026, 1, 1, 0, 10, tzinfo=timezone.utc) # the recorded end
assert listed.network == {"deny_out": ["10.0.0.0/8"], "allow_public_traffic": True}
assert listed.metadata == {"a": "b"}
async def test_snapshot_ids_are_checked_like_template_ids(backend):
with pytest.raises(SandboxException) as e:
await AsyncSandbox.delete_snapshot("!! bad")
assert _text(e) == "400: Invalid template ID: invalid template ID: !! bad"
assert await AsyncSandbox.delete_snapshot("conf/no-such-snapshot:v0") is False
# ── network updates ───────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"update, text",
[
({"deny_out": ["example.com"]}, "400: invalid denied CIDR example.com"),
({"deny_out": ["999.1.1.1"]}, "400: invalid denied CIDR 999.1.1.1"),
({"deny_out": ["1.1.1.1/33"]}, "400: invalid denied CIDR 1.1.1.1/33"),
({"deny_out": [""]}, "400: invalid denied CIDR "),
(
{"deny_out": [1]},
f'{VALIDATION}SandboxNetworkUpdateConfig: Error at "/denyOut/0": value must be a string',
),
],
)
async def test_network_updates_are_checked_like_e2bs_api(backend, update, text):
sb = await _sb(network={"allow_out": ["1.1.1.1"], "deny_out": [e2b.ALL_TRAFFIC]})
before = dict(backend.machines[sb.sandbox_id].egress)
for call in (sb.update_network(update), AsyncSandbox.update_network(sb.sandbox_id, update)):
with pytest.raises(SandboxException) as e:
await call
assert type(e.value) is SandboxException and _text(e) == text
assert backend.machines[sb.sandbox_id].egress == before # the sandbox keeps what it had
async def test_network_update_accepts_unknown_keys_and_refuses_none(backend):
sb = await _sb()
await sb.update_network({"bogus": []})
with pytest.raises(AttributeError):
await sb.update_network(None)
async def test_info_reports_network_as_e2b_does(backend):
sb = await _sb(network={"allow_out": ["1.1.1.1"], "deny_out": [e2b.ALL_TRAFFIC]})
assert (await sb.get_info()).network == {
"allow_out": ["1.1.1.1"],
"deny_out": ["0.0.0.0/0"],
"allow_public_traffic": True,
}
await sb.update_network({"deny_out": ["1.0.0.0/24"]})
assert (await sb.get_info()).network == {
"deny_out": ["1.0.0.0/24"],
"allow_public_traffic": True,
}
again = await AsyncSandbox.connect(sb.sandbox_id)
assert (await again.get_info()).network == (await sb.get_info()).network
# ── signed URLs ───────────────────────────────────────────────────────────────
async def test_signed_url_of_a_killed_sandbox_is_not_found(backend):
sb = await _sb()
sb.upload_url("/tmp/x")
backend.machines[sb.sandbox_id].deleted = True
await sb.kill()
with pytest.raises(e2b.SandboxNotFoundException):
sb.upload_url("/tmp/x")
class _Env:
def __init__(self, url_path: str):
self.url_path = url_path
self.requests = []
async def call_json(self, method, body):
self.requests.append(json.loads(body))
return json.dumps({"urlPath": self.url_path})
def endpoint(self):
return "http://10.0.0.5:3211"
async def _machine(env) -> CuaMachine:
m = CuaMachine(None, "e2b-x", env)
m._driver_user = "cua"
return m
async def test_signed_url_for_another_user_needs_spacesd_run_as(backend):
old = _Env("/files?path=%2Ftmp%2Fa&method=GET&exp=1&sig=s")
m = await _machine(old)
with pytest.raises(InvalidArgumentException, match="cannot sign file URLs for user='root'"):
await m.signed_url("/tmp/a", "GET", 60, "root")
assert old.requests[0]["runAs"] == "root"
new = _Env("/files?path=%2Ftmp%2Fa&method=GET&exp=1&ru=root&sig=s")
m = await _machine(new)
assert (await m.signed_url("/tmp/a", "GET", 60, "root")).endswith("&ru=root&sig=s")
# the default user needs no run_as, and an upload may create the parent directories
m = await _machine(_Env("/files?path=%2Ftmp%2Fa&method=PUT&exp=1&pp=1&sig=s"))
await m.signed_url("/tmp/a", "PUT", 60, "cua")
assert m._env.requests[0].get("runAs") is None and m._env.requests[0]["createParents"] is True
async def test_expiry_does_not_outlive_the_wait(backend):
# the lifetime timer of an embedded runtime still ends a sandbox on time
sb = await _sb()
await sb._set_lifetime(0.02)
await asyncio.sleep(0.08)
assert await _state(sb) == "gone"