Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(dify-agent): upgrade E2B to 2.55.1 with safe replay and full snapshots (#43930)

Ingram Za550230cfa2b
diff --git a/api/controllers/common/errors.py b/api/controllers/common/errors.py
index 2e3fb830cb8..05a3fd22389 100644
--- a/api/controllers/common/errors.py
+++ b/api/controllers/common/errors.py
@@ -87,12 +87,6 @@ class NoFileUploadedError(BaseHTTPException):
code = 400
-class ForbiddenError(BaseHTTPException):
- error_code = "forbidden"
- code = 403
- description = "Forbidden"
-
-
class NotFoundError(BaseHTTPException):
error_code = "not_found"
code = 404
diff --git a/api/uv.lock b/api/uv.lock
index 32b72147fea..4e5ba6a0b48 100644
--- a/api/uv.lock
+++ b/api/uv.lock
@@ -1351,7 +1351,7 @@ dependencies = [
[package.metadata]
requires-dist = [
- { name = "e2b", marker = "extra == 'server'", specifier = ">=2.38.0,<3.0.0" },
+ { name = "e2b", marker = "extra == 'server'", specifier = ">=2.55.1,<3.0.0" },
{ name = "fastapi", marker = "extra == 'server'", specifier = "==0.136.0" },
{ name = "graphon", marker = "extra == 'server'", specifier = "==0.5.2" },
{ name = "grpcio", marker = "extra == 'server'", specifier = ">=1.60.0,<2.0.0" },
diff --git a/dify-agent/docs/dify-agent/guide/index.md b/dify-agent/docs/dify-agent/guide/index.md
index f7bf9b8fc61..144fc2e65a1 100644
--- a/dify-agent/docs/dify-agent/guide/index.md
+++ b/dify-agent/docs/dify-agent/guide/index.md
@@ -283,6 +283,29 @@ provider `RuntimeError` observed first becomes a tool observation. In contrast,
run-deadline cancellation propagates through the Shell boundary; only the Dify
Agent run deadline owns the terminal `agent_run_limit_exceeded` failure.
+The E2B backend uses SDK 2.55.1 or later with three control-plane retries.
+The SDK retries `502` only for replayable operations, including when the response
+has `Retry-After`; a gateway can return `502` after a resource was already created.
+Sandbox and snapshot creation therefore surface `502` without replaying the
+request. The SDK still retries `503`, `429` with `Retry-After`, and safely
+replayable network failures. Backend adapters invoke each SDK operation once;
+exhausted retries and timeouts do not start a second SDK request budget.
+Resource creation and snapshot creation are also not replayed after an ambiguous
+post-write network failure. See the [SDK replay policy](https://github.com/e2b-dev/E2B/pull/1924).
+
+This policy covers E2B lifecycle API calls. Shell commands and normal file
+operations use the separate shellctl client, and usage collection uses its own
+HTTP client. Neither inherits the E2B SDK retry or connection-pool settings.
+
+The backend explicitly requests `mode="full"` for timeout pauses, explicit
+pauses, and Home Snapshots, preserving both filesystem and memory state. Binding
+acquisition uses `on_resume="restore"`; `connect` does not accept a snapshot
+mode. These settings preserve the existing full-memory restore intent while
+replacing the deprecated `keep_memory` option. They do not override platform
+fallbacks: E2B may save only the filesystem if a timeout pause is continuously
+refused because a previous snapshot is still being finalized. See the
+[E2B persistence contract](https://docs.e2b.dev/sandbox/persistence).
+
## OpenShell backend
See the [OpenShell Runtime Backend guide](openshell.md) for its configuration
diff --git a/dify-agent/pyproject.toml b/dify-agent/pyproject.toml
index 1c29370e03c..11b6594604b 100644
--- a/dify-agent/pyproject.toml
+++ b/dify-agent/pyproject.toml
@@ -19,7 +19,7 @@ dify-agent-stub-server = "dify_agent.agent_stub.server.cli:main"
[project.optional-dependencies]
server = [
- "e2b>=2.38.0,<3.0.0",
+ "e2b>=2.55.1,<3.0.0",
"fastapi==0.136.0",
"graphon==0.5.2",
# grpcio is a runtime requirement of the openshell SDK that its wheel
diff --git a/dify-agent/src/dify_agent/runtime_backend/e2b.py b/dify-agent/src/dify_agent/runtime_backend/e2b.py
index 398dd0e2f1e..27be29c68ac 100644
--- a/dify-agent/src/dify_agent/runtime_backend/e2b.py
+++ b/dify-agent/src/dify_agent/runtime_backend/e2b.py
@@ -9,12 +9,10 @@ operation-local and are never serialized into module JSON state.
from __future__ import annotations
import asyncio
-from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
import logging
-from typing import TYPE_CHECKING, Literal, Protocol, TypeVar, cast
+from typing import TYPE_CHECKING, Literal, Protocol, cast
-import httpx as e2b_httpx
import httpx2 as httpx
from shellctl.client import ShellctlClientError
@@ -45,19 +43,8 @@ if TYPE_CHECKING:
# One RuntimeLease spans the complete Agent run, not one Shell tool call.
E2B_MAX_ACTIVE_TIMEOUT_SECONDS = 60 * 60
-_E2B_CONTROL_PLANE_MAX_ATTEMPTS = 2
-_E2B_CONTROL_PLANE_RETRY_INTERVAL_SECONDS = 0.25
_SHELLCTL_READY_MAX_ATTEMPTS = 3
_SHELLCTL_READY_RETRY_INTERVAL_SECONDS = 0.5
-_RETRYABLE_E2B_TRANSPORT_ERRORS = (
- e2b_httpx.ConnectError,
- e2b_httpx.ReadTimeout,
- e2b_httpx.ReadError,
- e2b_httpx.WriteError,
- e2b_httpx.RemoteProtocolError,
-)
-
-_ResultT = TypeVar("_ResultT")
logger = logging.getLogger(__name__)
@@ -95,11 +82,11 @@ class _E2BSandbox(Protocol):
def get_host(self, port: int) -> str: ...
- async def pause(self, keep_memory: bool = True) -> bool: ...
+ async def pause(self, *, mode: Literal["full"]) -> bool: ...
async def kill(self) -> bool: ...
- async def create_snapshot(self, name: str | None = None) -> _E2BSnapshotInfo: ...
+ async def create_snapshot(self, name: str | None = None, *, mode: Literal["full"]) -> _E2BSnapshotInfo: ...
class E2BControlPlane(Protocol):
@@ -126,12 +113,17 @@ class E2BSDKControlPlane:
SDK Sandbox objects are returned only to operation-local backend adapters.
Native not-found exceptions are normalized so adapters can distinguish
confirmed resource loss from transient acquisition and cleanup failures.
+
+ The SDK owns control-plane retries and replay safety. Backend adapters call
+ each SDK operation once so an exhausted retry or timeout cannot start a new
+ request budget. A 502 is replayed only for safe operations; Retry-After does
+ not override replay safety. See https://github.com/e2b-dev/E2B/pull/1924.
"""
api_key: str
def _options(self) -> ApiParams:
- return {"api_key": self.api_key}
+ return {"api_key": self.api_key, "retries": 3}
async def create(
self,
@@ -144,20 +136,18 @@ class E2BSDKControlPlane:
from e2b import AsyncSandbox, NotFoundException, RateLimitException, SandboxNotFoundException
try:
- return cast(
- _E2BSandbox,
- cast(
- object,
- await AsyncSandbox.create(
- template,
- timeout=timeout,
- metadata=metadata,
- network={"allow_public_traffic": False},
- lifecycle={"on_timeout": on_timeout, "auto_resume": False},
- **self._options(),
- ),
- ),
+ sandbox: object = await AsyncSandbox.create(
+ template,
+ timeout=timeout,
+ metadata=metadata,
+ network={"allow_public_traffic": False},
+ lifecycle={
+ "on_timeout": {"action": "pause", "mode": "full"} if on_timeout == "pause" else "kill",
+ "auto_resume": False,
+ },
+ **self._options(),
)
+ return cast(_E2BSandbox, sandbox)
except (SandboxNotFoundException, NotFoundException) as exc:
raise _E2BControlPlaneNotFoundError(str(exc)) from exc
except RateLimitException as exc:
@@ -167,10 +157,10 @@ class E2BSDKControlPlane:
from e2b import AsyncSandbox, NotFoundException, SandboxNotFoundException
try:
- return cast(
- _E2BSandbox,
- cast(object, await AsyncSandbox.connect(handle, timeout=timeout, **self._options())),
+ sandbox: object = await AsyncSandbox.connect(
+ handle, timeout=timeout, on_resume="restore", **self._options()
)
+ return cast(_E2BSandbox, sandbox)
except (SandboxNotFoundException, NotFoundException) as exc:
raise _E2BControlPlaneNotFoundError(str(exc)) from exc
@@ -208,7 +198,7 @@ class E2BHomeSnapshotBackend:
if not isinstance(source, E2BRuntimeLease):
raise HomeSnapshotCreateError("E2B Home Snapshot requires an E2B RuntimeLease")
try:
- snapshot = await source.sandbox.create_snapshot()
+ snapshot = await source.sandbox.create_snapshot(mode="full")
return snapshot.snapshot_id
except BaseException as exc:
if isinstance(exc, Exception):
@@ -217,12 +207,7 @@ class E2BHomeSnapshotBackend:
async def delete(self, snapshot_ref: str) -> None:
try:
- _ = await _run_e2b_idempotent_operation(
- operation="delete_snapshot",
- sandbox_id=snapshot_ref,
- cleanup_stage="home_snapshot_delete",
- action=lambda: self.control_plane.delete_snapshot(snapshot_ref),
- )
+ _ = await self.control_plane.delete_snapshot(snapshot_ref)
except _E2BControlPlaneNotFoundError:
return
except Exception as exc:
@@ -270,30 +255,20 @@ class E2BExecutionBindingBackend:
for entry in await sandbox.files.list(self.layout.workspace_dir):
await sandbox.files.remove(entry.path)
sandbox_id = sandbox.sandbox_id
- _ = await _run_e2b_idempotent_operation(
- operation="pause",
- sandbox_id=sandbox_id,
- cleanup_stage="binding_create",
- action=lambda: sandbox.pause(keep_memory=True),
- )
+ _ = await sandbox.pause(mode="full")
return ExecutionBindingAllocation(binding_ref=sandbox_id, workspace_ref=sandbox_id)
except _E2BControlPlaneCapacityExhaustedError as exc:
raise BindingCapacityExhaustedError(str(exc)) from exc
except BaseException as exc:
if sandbox is not None:
try:
- _ = await _run_e2b_idempotent_operation(
- operation="kill",
- sandbox_id=sandbox.sandbox_id,
- cleanup_stage="binding_create_compensation",
- action=sandbox.kill,
- )
+ _ = await sandbox.kill()
except Exception as cleanup_exc:
_log_e2b_cleanup_warning(
"failed to remove partial E2B Binding",
operation="kill",
sandbox_id=sandbox.sandbox_id,
- attempt=_e2b_failure_attempt(cleanup_exc),
+ attempt=1,
cleanup_stage="binding_create_compensation",
exception=cleanup_exc,
)
@@ -308,12 +283,7 @@ class E2BExecutionBindingBackend:
sandbox: _E2BSandbox | None = None
lease: E2BRuntimeLease | None = None
try:
- sandbox = await _run_e2b_idempotent_operation(
- operation="connect",
- sandbox_id=binding_ref,
- cleanup_stage="binding_acquire",
- action=lambda: self.control_plane.connect(binding_ref, timeout=self.active_timeout_seconds),
- )
+ sandbox = await self.control_plane.connect(binding_ref, timeout=self.active_timeout_seconds)
if not await sandbox.files.exists(self.layout.workspace_dir):
raise BindingLostError(f"E2B Binding {binding_ref!r} no longer contains its Workspace")
lease = await self._lease(sandbox)
@@ -351,18 +321,13 @@ class E2BExecutionBindingBackend:
close_base_error = exc
pause_base_error: BaseException | None = None
try:
- _ = await _run_e2b_idempotent_operation(
- operation="pause",
- sandbox_id=lease.sandbox.sandbox_id,
- cleanup_stage="binding_release",
- action=lambda: lease.sandbox.pause(keep_memory=True),
- )
+ _ = await lease.sandbox.pause(mode="full")
except Exception as exc:
_log_e2b_cleanup_warning(
"failed to pause E2B RuntimeLease",
operation="pause",
sandbox_id=lease.sandbox.sandbox_id,
- attempt=_e2b_failure_attempt(exc),
+ attempt=1,
cleanup_stage="binding_release",
exception=exc,
)
@@ -384,12 +349,7 @@ class E2BExecutionBindingBackend:
if spec.workspace_ref != spec.binding_ref:
raise BindingDestroyError("E2B Workspace ref must equal its Binding ref")
try:
- _ = await _run_e2b_idempotent_operation(
- operation="kill",
- sandbox_id=spec.binding_ref,
- cleanup_stage="binding_destroy",
- action=lambda: self.control_plane.kill(spec.binding_ref),
- )
+ _ = await self.control_plane.kill(spec.binding_ref)
except _E2BControlPlaneNotFoundError:
return
except Exception as exc:
@@ -465,37 +425,6 @@ async def _wait_for_shellctl_ready(client: ShellctlClientProtocol) -> None:
await asyncio.sleep(_SHELLCTL_READY_RETRY_INTERVAL_SECONDS)
-async def _run_e2b_idempotent_operation(
- *,
- operation: str,
- sandbox_id: str,
- cleanup_stage: str,
- action: Callable[[], Awaitable[_ResultT]],
-) -> _ResultT:
- """Retry one idempotent E2B lifecycle operation after a transport failure."""
- for attempt in range(1, _E2B_CONTROL_PLANE_MAX_ATTEMPTS + 1):
- try:
- return await action()
- except _RETRYABLE_E2B_TRANSPORT_ERRORS as exc:
- exhausted = attempt == _E2B_CONTROL_PLANE_MAX_ATTEMPTS
- logger.warning(
- "E2B lifecycle operation exhausted retries" if exhausted else "retrying E2B lifecycle operation",
- exc_info=True,
- extra=_e2b_log_extra(
- operation=operation,
- sandbox_id=sandbox_id,
- attempt=attempt,
- cleanup_stage=cleanup_stage,
- exception=exc,
- outcome="retry_exhausted" if exhausted else "retrying",
- ),
- )
- if exhausted:
- raise
- await asyncio.sleep(_E2B_CONTROL_PLANE_RETRY_INTERVAL_SECONDS)
- raise AssertionError("unreachable")
-
-
def _e2b_log_extra(
*,
operation: str,
@@ -524,6 +453,7 @@ def _log_e2b_cleanup_warning(
cleanup_stage: str,
exception: BaseException,
) -> None:
+ # attempt counts adapter calls; the SDK's internal HTTP retries are opaque.
logger.warning(
message,
exc_info=True,
@@ -539,12 +469,6 @@ def _log_e2b_cleanup_warning(
)
-def _e2b_failure_attempt(exception: BaseException) -> int:
- if isinstance(exception, _RETRYABLE_E2B_TRANSPORT_ERRORS):
- return _E2B_CONTROL_PLANE_MAX_ATTEMPTS
- return 1
-
-
async def _best_effort_close_data_plane(lease: E2BRuntimeLease | None) -> None:
if lease is None:
return
@@ -565,18 +489,13 @@ async def _best_effort_pause(sandbox: _E2BSandbox | None) -> None:
if sandbox is None:
return
try:
- _ = await _run_e2b_idempotent_operation(
- operation="pause",
- sandbox_id=sandbox.sandbox_id,
- cleanup_stage="binding_acquire_compensation",
- action=lambda: sandbox.pause(keep_memory=True),
- )
+ _ = await sandbox.pause(mode="full")
except BaseException as exc:
_log_e2b_cleanup_warning(
"failed to pause E2B Binding after acquire failure",
operation="pause",
sandbox_id=sandbox.sandbox_id,
- attempt=_e2b_failure_attempt(exc),
+ attempt=1,
cleanup_stage="binding_acquire_compensation",
exception=exc,
)
diff --git a/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py b/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py
index 445e862a402..3bc46d033e7 100644
--- a/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py
+++ b/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py
@@ -5,7 +5,7 @@ import posixpath
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
import logging
-from typing import cast
+from typing import Literal, cast
import httpx as e2b_httpx
import httpx2 as httpx
@@ -16,6 +16,7 @@ from dify_agent.runtime_backend import (
BindingAcquireError,
BindingCapacityExhaustedError,
BindingCreateError,
+ BindingDestroyError,
BindingLostError,
ExecutionBindingCreateSpec,
ExecutionBindingDestroySpec,
@@ -91,19 +92,20 @@ class _Sandbox:
sandbox_id: str
files: _Files = field(default_factory=_Files)
traffic_access_token: str | None = "traffic-token"
- pauses: list[bool] = field(default_factory=list)
+ pauses: list[str] = field(default_factory=list)
killed: int = 0
snapshots: int = 0
pause_error: Exception | None = None
pause_errors: list[BaseException] = field(default_factory=list)
kill_errors: list[BaseException] = field(default_factory=list)
snapshot_errors: list[BaseException] = field(default_factory=list)
+ snapshot_modes: list[str] = field(default_factory=list)
def get_host(self, port: int) -> str:
return f"{self.sandbox_id}-{port}.example.test"
- async def pause(self, keep_memory: bool = True) -> bool:
- self.pauses.append(keep_memory)
+ async def pause(self, *, mode: Literal["full"]) -> bool:
+ self.pauses.append(mode)
if self.pause_errors:
raise self.pause_errors.pop(0)
if self.pause_error is not None:
@@ -116,9 +118,10 @@ class _Sandbox:
raise self.kill_errors.pop(0)
return True
- async def create_snapshot(self, name: str | None = None) -> _Snapshot:
+ async def create_snapshot(self, name: str | None = None, *, mode: Literal["full"]) -> _Snapshot:
del name
self.snapshots += 1
+ self.snapshot_modes.append(mode)
if self.snapshot_errors:
raise self.snapshot_errors.pop(0)
return _Snapshot(snapshot_id=f"snapshot-{self.sandbox_id}-{self.snapshots}")
@@ -257,7 +260,15 @@ def sleep_delays(monkeypatch: pytest.MonkeyPatch) -> list[float]:
@pytest.mark.anyio
-async def test_e2b_sdk_create_disables_public_traffic(monkeypatch: pytest.MonkeyPatch) -> None:
+ ("on_timeout", "expected_timeout_action"),
+ [("pause", {"action": "pause", "mode": "full"}), ("kill", "kill")],
+)
+async def test_e2b_sdk_create_disables_public_traffic_and_preserves_lifecycle(
+ monkeypatch: pytest.MonkeyPatch,
+ on_timeout: Literal["kill", "pause"],
+ expected_timeout_action: dict[str, str] | str,
+) -> None:
from e2b import AsyncSandbox
sandbox = _Sandbox(sandbox_id="sandbox-1")
@@ -279,11 +290,12 @@ async def test_e2b_sdk_create_disables_public_traffic(monkeypatch: pytest.Monkey
"prepared-template",
timeout=120,
metadata={"dify.resource": "runtime-sandbox"},
- on_timeout="pause",
+ on_timeout=on_timeout,
)
assert created is sandbox
assert create_options["network"] == {"allow_public_traffic": False}
+ assert create_options["lifecycle"] == {"on_timeout": expected_timeout_action, "auto_resume": False}
@pytest.mark.anyio
@@ -320,31 +332,19 @@ async def test_e2b_sdk_create_maps_provider_capacity_exhaustion(monkeypatch: pyt
"transport_error_type",
[e2b_httpx.ConnectError, e2b_httpx.ReadTimeout],
)
-async def test_e2b_connect_retries_one_transport_failure(
- monkeypatch: pytest.MonkeyPatch,
- caplog: pytest.LogCaptureFixture,
+async def test_e2b_connect_does_not_repeat_failed_sdk_call(
sleep_delays: list[float],
transport_error_type: type[e2b_httpx.RequestError],
) -> None:
- control = _ControlPlane(connect_errors=[_transport_error(transport_error_type)])
- sandbox = _Sandbox(sandbox_id="sandbox-1")
- sandbox.files.paths.add("/workspace")
- control.sandboxes[sandbox.sandbox_id] = sandbox
+ failure = _transport_error(transport_error_type)
+ control = _ControlPlane(connect_errors=[failure])
backend = _binding_backend(control)
- clients = _mock_http(monkeypatch, lambda _request: httpx.Response(200, json={"status": "ok"}))
- with caplog.at_level(logging.WARNING, logger="dify_agent.runtime_backend.e2b"):
- lease = await backend.acquire(sandbox.sandbox_id)
-
- assert control.connect_attempts == [sandbox.sandbox_id, sandbox.sandbox_id]
- assert sleep_delays == [0.25]
- retry_record = next(record for record in caplog.records if record.__dict__.get("outcome") == "retrying")
- assert retry_record.__dict__["e2b_operation"] == "connect"
- assert retry_record.__dict__["sandbox_id"] == sandbox.sandbox_id
- assert retry_record.__dict__["attempt"] == 1
- assert retry_record.__dict__["cleanup_stage"] == "binding_acquire"
- assert retry_record.__dict__["exception_type"] == transport_error_type.__name__
- await cast(E2BRuntimeLease, lease).data_plane.close()
- assert clients[0].is_closed
+ with pytest.raises(BindingAcquireError, match="stale E2B connection") as exc_info:
+ _ = await backend.acquire("sandbox-1")
+
+ assert exc_info.value.__cause__ is failure
+ assert control.connect_attempts == ["sandbox-1"]
+ assert sleep_delays == []
@pytest.mark.anyio
@@ -352,26 +352,30 @@ async def test_e2b_connect_retries_one_transport_failure(
"transport_error_type",
[e2b_httpx.ReadError, e2b_httpx.ReadTimeout],
)
-async def test_e2b_initial_pause_retries_one_transport_failure(
+async def test_e2b_initial_pause_failure_compensates_without_repeating_sdk_call(
sleep_delays: list[float],
transport_error_type: type[e2b_httpx.RequestError],
) -> None:
- control = _ControlPlane(sandbox_pause_errors=[_transport_error(transport_error_type)])
+ failure = _transport_error(transport_error_type)
+ control = _ControlPlane(sandbox_pause_errors=[failure])
backend = _binding_backend(control)
- allocation = await backend.create_binding(
- ExecutionBindingCreateSpec(
- tenant_id="tenant-1",
- agent_id="agent-1",
- binding_id="binding-1",
- workspace_id="workspace-1",
- existing_workspace_ref=None,
+ with pytest.raises(BindingCreateError, match="stale E2B connection") as exc_info:
+ _ = await backend.create_binding(
+ ExecutionBindingCreateSpec(
+ tenant_id="tenant-1",
+ agent_id="agent-1",
+ binding_id="binding-1",
+ workspace_id="workspace-1",
+ existing_workspace_ref=None,
+ )
)
- )
- sandbox = control.sandboxes[allocation.binding_ref]
+ sandbox = next(iter(control.sandboxes.values()))
+ assert exc_info.value.__cause__ is failure
assert control.created == [("prepared-template", "pause")]
- assert sandbox.pauses == [True, True]
- assert sleep_delays == [0.25]
+ assert sandbox.pauses == ["full"]
+ assert sandbox.killed == 1
+ assert sleep_delays == []
@pytest.mark.anyio
@@ -379,22 +383,25 @@ async def test_e2b_initial_pause_retries_one_transport_failure(
"transport_error_type",
[e2b_httpx.WriteError, e2b_httpx.ReadTimeout],
)
-async def test_e2b_destroy_retries_one_transport_failure(
+async def test_e2b_destroy_does_not_repeat_failed_sdk_call(
sleep_delays: list[float],
transport_error_type: type[e2b_httpx.RequestError],
) -> None:
- control = _ControlPlane(kill_errors=[_transport_error(transport_error_type)])
+ failure = _transport_error(transport_error_type)
+ control = _ControlPlane(kill_errors=[failure])
backend = _binding_backend(control)
- await backend.destroy_binding(
- ExecutionBindingDestroySpec(
- binding_ref="sandbox-1",
- workspace_ref="sandbox-1",
- destroy_workspace=True,
+ with pytest.raises(BindingDestroyError, match="stale E2B connection") as exc_info:
+ await backend.destroy_binding(
+ ExecutionBindingDestroySpec(
+ binding_ref="sandbox-1",
+ workspace_ref="sandbox-1",
+ destroy_workspace=True,
+ )
)
- )
- assert control.killed == ["sandbox-1", "sandbox-1"]
- assert sleep_delays == [0.25]
+ assert exc_info.value.__cause__ is failure
+ assert control.killed == ["sandbox-1"]
+ assert sleep_delays == []
@pytest.mark.anyio
@@ -402,37 +409,39 @@ async def test_e2b_destroy_retries_one_transport_failure(
"transport_error_type",
[e2b_httpx.RemoteProtocolError, e2b_httpx.ReadTimeout],
)
-async def test_e2b_snapshot_delete_retries_one_transport_failure(
+async def test_e2b_snapshot_delete_does_not_repeat_failed_sdk_call(
sleep_delays: list[float],
transport_error_type: type[e2b_httpx.RequestError],
) -> None:
- control = _ControlPlane(delete_snapshot_errors=[_transport_error(transport_error_type)])
+ failure = _transport_error(transport_error_type)
+ control = _ControlPlane(delete_snapshot_errors=[failure])
backend = E2BHomeSnapshotBackend(
control_plane=control, # pyright: ignore[reportArgumentType]
)
- await backend.delete("snapshot-1")
+ with pytest.raises(BindingDestroyError, match="stale E2B connection") as exc_info:
+ await backend.delete("snapshot-1")
- assert control.deleted_snapshots == ["snapshot-1", "snapshot-1"]
- assert sleep_delays == [0.25]
+ assert exc_info.value.__cause__ is failure
+ assert control.deleted_snapshots == ["snapshot-1"]
+ assert sleep_delays == []
@pytest.mark.anyio
-async def test_e2b_connect_raises_after_two_read_timeouts(
+async def test_e2b_connect_does_not_reset_sdk_timeout_budget(
sleep_delays: list[float],
) -> None:
- control = _ControlPlane(
- connect_errors=[
- _transport_error(e2b_httpx.ReadTimeout),
- _transport_error(e2b_httpx.ReadTimeout),
- ]
- )
+ first_timeout = _transport_error(e2b_httpx.ReadTimeout)
+ second_timeout = _transport_error(e2b_httpx.ReadTimeout)
+ control = _ControlPlane(connect_errors=[first_timeout, second_timeout])
backend = _binding_backend(control)
- with pytest.raises(BindingAcquireError, match="stale E2B connection"):
+ with pytest.raises(BindingAcquireError, match="stale E2B connection") as exc_info:
_ = await backend.acquire("sandbox-1")
- assert control.connect_attempts == ["sandbox-1", "sandbox-1"]
- assert sleep_delays == [0.25]
+ assert exc_info.value.__cause__ is first_timeout
+ assert control.connect_attempts == ["sandbox-1"]
+ assert control.connect_errors == [second_timeout]
+ assert sleep_delays == []
@pytest.mark.anyio
@@ -514,6 +523,7 @@ async def test_e2b_create_and_snapshot_create_are_not_retried(
assert control.created == [("prepared-template", "pause")]
assert snapshot_sandbox.snapshots == 1
+ assert snapshot_sandbox.snapshot_modes == ["full"]
@pytest.mark.anyio
@@ -615,10 +625,9 @@ async def test_e2b_binding_uses_default_template_or_exact_snapshot_and_couples_r
assert control.created == [("prepared-template", "pause"), ("snapshot-1", "pause")]
assert default_allocation.binding_ref == default_allocation.workspace_ref
assert snapshot_allocation.binding_ref == snapshot_allocation.workspace_ref
- assert control.sandboxes[default_allocation.binding_ref].pauses == [True]
-
for allocation in (default_allocation, snapshot_allocation):
runtime = control.sandboxes[allocation.binding_ref]
+ assert runtime.pauses == ["full"]
assert runtime.files.paths == {"/workspace"}
assert "/workspace" not in runtime.files.removed
@@ -685,10 +694,7 @@ async def test_e2b_binding_create_preserves_primary_error_when_compensation_kill
) -> None:
control = _ControlPlane(
pause_error=RuntimeError("primary pause failure"),
- sandbox_kill_errors=[
- _transport_error(e2b_httpx.WriteError),
- _transport_error(e2b_httpx.WriteError),
- ],
+ sandbox_kill_errors=[_transport_error(e2b_httpx.WriteError)],
)
backend = _binding_backend(control)
with caplog.at_level(logging.WARNING, logger="dify_agent.runtime_backend.e2b"):
@@ -704,30 +710,27 @@ async def test_e2b_binding_create_preserves_primary_error_when_compensation_kill
)
sandbox = next(iter(control.sandboxes.values()))
- assert sandbox.killed == 2
- assert sleep_delays == [0.25]
+ assert sandbox.killed == 1
+ assert sleep_delays == []
cleanup_record = next(
record for record in caplog.records if record.__dict__.get("outcome") == "ignored_cleanup_failure"
)
assert cleanup_record.funcName == "create_binding"
assert cleanup_record.__dict__["e2b_operation"] == "kill"
assert cleanup_record.__dict__["sandbox_id"] == sandbox.sandbox_id
+ assert cleanup_record.__dict__["attempt"] == 1
assert cleanup_record.__dict__["cleanup_stage"] == "binding_create_compensation"
assert cleanup_record.__dict__["exception_type"] == "WriteError"
@pytest.mark.anyio
-async def test_e2b_binding_create_fails_closed_when_initial_pause_retries_are_exhausted(
+async def test_e2b_binding_create_fails_closed_after_sdk_pause_timeout(
sleep_delays: list[float],
) -> None:
- control = _ControlPlane(
- sandbox_pause_errors=[
- _transport_error(e2b_httpx.ReadError),
- _transport_error(e2b_httpx.ReadError),
- ]
- )
+ failure = _transport_error(e2b_httpx.ReadTimeout)
+ control = _ControlPlane(sandbox_pause_errors=[failure])
backend = _binding_backend(control)
- with pytest.raises(BindingCreateError, match="stale E2B connection"):
+ with pytest.raises(BindingCreateError, match="stale E2B connection") as exc_info:
_ = await backend.create_binding(
ExecutionBindingCreateSpec(
tenant_id="tenant-1",
@@ -739,9 +742,10 @@ async def test_e2b_binding_create_fails_closed_when_initial_pause_retries_are_ex
)
sandbox = next(iter(control.sandboxes.values()))
- assert sandbox.pauses == [True, True]
+ assert exc_info.value.__cause__ is failure
+ assert sandbox.pauses == ["full"]
assert sandbox.killed == 1
- assert sleep_delays == [0.25]
+ assert sleep_delays == []
@pytest.mark.anyio
@@ -789,6 +793,7 @@ async def test_e2b_checkpoint_uses_exact_source_runtime() -> None:
assert snapshot_ref == "snapshot-source-1"
assert source_sandbox.snapshots == 1
+ assert source_sandbox.snapshot_modes == ["full"]
@pytest.mark.anyio
@@ -821,7 +826,7 @@ async def test_e2b_release_warns_without_raising_for_cleanup_failures(
await backend.release(lease)
assert data_plane.close_calls == 1
- assert sandbox.pauses == [True]
+ assert sandbox.pauses == ["full"]
records = [record for record in caplog.records if record.name == "dify_agent.runtime_backend.e2b"]
assert len(records) == expected_warning_count
for record in records:
@@ -829,12 +834,15 @@ async def test_e2b_release_warns_without_raising_for_cleanup_failures(
assert record.__dict__["cleanup_stage"] == "binding_release"
assert record.__dict__["outcome"] == "ignored_cleanup_failure"
assert isinstance(record.__dict__["e2b_operation"], str)
- assert isinstance(record.__dict__["attempt"], int)
+ assert record.__dict__["attempt"] == 1
assert isinstance(record.__dict__["exception_type"], str)
@pytest.mark.anyio
-async def test_e2b_release_retries_pause_transport_failure(sleep_delays: list[float]) -> None:
+async def test_e2b_release_closes_and_logs_failed_sdk_pause_once(
+ caplog: pytest.LogCaptureFixture,
+ sleep_delays: list[float],
+) -> None:
data_plane = _ReleaseDataPlane()
sandbox = _Sandbox(
sandbox_id="sandbox-release",
@@ -845,15 +853,24 @@ async def test_e2b_release_retries_pause_transport_failure(sleep_delays: list[fl
data_plane=cast(ShellctlRuntimeLease, cast(object, data_plane)),
)
backend, _ = _connected_backend()
- await backend.release(lease)
+ with caplog.at_level(logging.WARNING, logger="dify_agent.runtime_backend.e2b"):
+ await backend.release(lease)
assert data_plane.close_calls == 1
- assert sandbox.pauses == [True, True]
- assert sleep_delays == [0.25]
+ assert sandbox.pauses == ["full"]
+ assert sleep_delays == []
+ cleanup_record = next(
+ record for record in caplog.records if record.__dict__.get("outcome") == "ignored_cleanup_failure"
+ )
+ assert cleanup_record.__dict__["e2b_operation"] == "pause"
+ assert cleanup_record.__dict__["sandbox_id"] == sandbox.sandbox_id
+ assert cleanup_record.__dict__["attempt"] == 1
+ assert cleanup_record.__dict__["cleanup_stage"] == "binding_release"
+ assert cleanup_record.__dict__["exception_type"] == "RemoteProtocolError"
@pytest.mark.anyio
-async def test_e2b_release_ignores_exhausted_pause_read_timeouts(
+async def test_e2b_release_does_not_reset_sdk_pause_timeout_budget(
caplog: pytest.LogCaptureFixture,
sleep_delays: list[float],
) -> None:
@@ -875,14 +892,15 @@ async def test_e2b_release_ignores_exhausted_pause_read_timeouts(
await backend.release(lease)
assert data_plane.close_calls == 1
- assert sandbox.pauses == [True, True]
- assert sleep_delays == [0.25]
+ assert sandbox.pauses == ["full"]
+ assert len(sandbox.pause_errors) == 1
+ assert sleep_delays == []
cleanup_record = next(
record for record in caplog.records if record.__dict__.get("outcome") == "ignored_cleanup_failure"
)
assert cleanup_record.__dict__["e2b_operation"] == "pause"
assert cleanup_record.__dict__["sandbox_id"] == sandbox.sandbox_id
- assert cleanup_record.__dict__["attempt"] == 2
+ assert cleanup_record.__dict__["attempt"] == 1
assert cleanup_record.__dict__["cleanup_stage"] == "binding_release"
assert cleanup_record.__dict__["exception_type"] == "ReadTimeout"
@@ -907,7 +925,7 @@ async def test_e2b_release_does_not_swallow_base_exceptions(failure_stage: str)
await backend.release(lease)
assert data_plane.close_calls == 1
- assert sandbox.pauses == [True]
+ assert sandbox.pauses == ["full"]
@pytest.mark.anyio
@@ -928,7 +946,7 @@ async def test_e2b_release_preserves_close_base_exception_when_pause_also_raises
assert exc_info.value is close_error
assert exc_info.value.__cause__ is pause_error
assert data_plane.close_calls == 1
- assert sandbox.pauses == [True]
+ assert sandbox.pauses == ["full"]
@pytest.mark.anyio
@@ -982,7 +1000,7 @@ async def test_e2b_acquire_fails_closed_without_traffic_token(
_ = await backend.acquire(sandbox.sandbox_id)
assert clients == []
- assert sandbox.pauses == [True]
+ assert sandbox.pauses == ["full"]
@pytest.mark.anyio
@@ -1006,7 +1024,7 @@ async def test_e2b_acquire_closes_transport_and_pauses_after_readiness_retries_e
assert attempts == 3
assert sleep_delays == [0.5, 0.5]
assert clients[0].is_closed
- assert sandbox.pauses == [True]
+ assert sandbox.pauses == ["full"]
@pytest.mark.anyio
@@ -1033,7 +1051,7 @@ async def test_e2b_acquire_does_not_retry_shellctl_4xx_and_preserves_error_when_
assert attempts == 1
assert clients[0].is_closed
- assert sandbox.pauses == [True]
+ assert sandbox.pauses == ["full"]
@pytest.mark.anyio
@@ -1081,7 +1099,7 @@ async def test_e2b_acquire_preserves_health_failure_when_close_and_pause_fail(
assert client.calls == 3
assert data_plane.close_calls == 1
- assert sandbox.pauses == [True]
+ assert sandbox.pauses == ["full"]
@pytest.mark.anyio
@@ -1138,34 +1156,47 @@ async def test_acquire_task_cancellation_during_health_closes_transport_and_paus
with pytest.raises(asyncio.CancelledError, match="cancel acquisition"):
await task
assert clients[0].is_closed
- assert sandbox.pauses == [True]
+ assert sandbox.pauses == ["full"]
@pytest.mark.anyio
-async def test_release_task_cancellation_keeps_pause_retries_and_original_cancel(
+async def test_release_task_cancellation_attempts_pause_once_and_preserves_original_cancel(
sleep_delays: list[float],
) -> None:
entered = asyncio.Event()
+ @dataclass(slots=True)
class BlockingCloseDataPlane:
+ close_calls: int = 0
+ close_error: asyncio.CancelledError | None = None
+
async def close(self) -> None:
+ self.close_calls += 1
entered.set()
- await asyncio.Event().wait()
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError as exc:
+ self.close_error = exc
+ raise
backend, sandbox = _connected_backend()
sandbox.pause_errors = [_transport_error(e2b_httpx.ReadTimeout), _transport_error(e2b_httpx.ReadTimeout)]
+ data_plane = BlockingCloseDataPlane()
lease = E2BRuntimeLease(
sandbox=sandbox, # pyright: ignore[reportArgumentType]
- data_plane=cast(ShellctlRuntimeLease, cast(object, BlockingCloseDataPlane())),
+ data_plane=cast(ShellctlRuntimeLease, cast(object, data_plane)),
)
task = asyncio.create_task(backend.release(lease))
async with asyncio.timeout(2):
await entered.wait()
task.cancel("cancel release")
- with pytest.raises(asyncio.CancelledError, match="cancel release"):
+ with pytest.raises(asyncio.CancelledError, match="cancel release") as exc_info:
await task
- assert sandbox.pauses == [True, True]
- assert sleep_delays == [0.25]
+ assert exc_info.value is data_plane.close_error
+ assert data_plane.close_calls == 1
+ assert sandbox.pauses == ["full"]
+ assert len(sandbox.pause_errors) == 1
+ assert sleep_delays == []
@pytest.mark.anyio
@@ -1176,8 +1207,8 @@ async def test_acquire_primary_health_error_survives_task_cancel_during_compensa
# cleanup must not replace the primary acquisition error.
entered = asyncio.Event()
- async def blocked_pause(sandbox: _Sandbox, keep_memory: bool = True) -> bool:
- sandbox.pauses.append(keep_memory)
+ async def blocked_pause(sandbox: _Sandbox, *, mode: Literal["full"]) -> bool:
+ sandbox.pauses.append(mode)
entered.set()
await asyncio.Event().wait()
return True
@@ -1195,4 +1226,4 @@ async def test_acquire_primary_health_error_survives_task_cancel_during_compensa
with pytest.raises(BindingAcquireError, match="bad token"):
await task
assert clients[0].is_closed
- assert sandbox.pauses == [True]
+ assert sandbox.pauses == ["full"]
diff --git a/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b_sdk.py b/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b_sdk.py
new file mode 100644
index 00000000000..556f6f09b26
--- /dev/null
+++ b/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b_sdk.py
@@ -0,0 +1,398 @@
+"""Exercise Dify's real E2B SDK boundary with an in-memory HTTP transport.
+
+These tests retain SDK serialization, response parsing, and retry middleware.
+They do not exercise E2B services, envd RPC, or pyqwest's connection pools.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import json
+from typing import cast
+
+from e2b.api import client_async
+from e2b.exceptions import SandboxException
+from e2b.retry import AsyncRetryableTransport
+import httpx
+import pytest
+
+from dify_agent.runtime_backend import (
+ BindingCreateError,
+ BindingDestroyError,
+ ExecutionBindingCreateSpec,
+ ExecutionBindingDestroySpec,
+ HomeSnapshotCreateError,
+ HomeSnapshotCreateSpec,
+)
+from dify_agent.runtime_backend.e2b import (
+ E2BExecutionBindingBackend,
+ E2BHomeSnapshotBackend,
+ E2BRuntimeLease,
+ E2BSDKControlPlane,
+ _E2BControlPlaneCapacityExhaustedError,
+ _E2BControlPlaneNotFoundError,
+)
+from dify_agent.runtime_backend.shellctl import ShellctlRuntimeLease
+
+_API_KEY = "e2b_" + "0" * 40
+_SANDBOX_ID = "sandbox-sdk-contract"
+
+
+@dataclass
+class _ApiTransport:
+ responses: list[httpx.Response | httpx.RequestError] = field(default_factory=list)
+ requests: list[httpx.Request] = field(default_factory=list)
+ delays: list[float] = field(default_factory=list)
+ retry_limits: list[int] = field(default_factory=list)
+ now: float = 0.0
+
+ def handle(self, request: httpx.Request) -> httpx.Response:
+ self.requests.append(request)
+ assert request.url.host == "api.e2b.test"
+ assert self.responses, f"Unexpected SDK request: {request.method} {request.url.path}"
+ response = self.responses.pop(0)
+ if isinstance(response, httpx.RequestError):
+ raise response
+ return response
+
+ async def sleep(self, delay: float) -> None:
+ self.delays.append(delay)
+ self.now += delay
+
+
+def api_transport(monkeypatch: pytest.MonkeyPatch) -> _ApiTransport:
+ state = _ApiTransport()
+ transport = httpx.MockTransport(state.handle)
+
+ def retry_transport(inner: httpx.AsyncBaseTransport, retries: int) -> AsyncRetryableTransport:
+ state.retry_limits.append(retries)
+ return AsyncRetryableTransport(
+ inner,
+ retries,
+ sleep=state.sleep,
+ monotonic=lambda: state.now,
+ random_=lambda: 0.0,
+ )
+
+ # Inject below the SDK's retry middleware, preserving get_api_client and
+ # every public lifecycle method. Injectable clocks make its budget deterministic.
+ # Source: https://github.com/e2b-dev/E2B/blob/%40e2b/python-sdk%402.55.1/packages/python-sdk/e2b/api/client_async/__init__.py
+ monkeypatch.setattr(client_async, "get_transport", lambda *_args, **_kwargs: transport)
+ monkeypatch.setattr(client_async, "AsyncRetryableTransport", retry_transport)
+ monkeypatch.setenv("E2B_API_URL", "https://api.e2b.test")
+ monkeypatch.setenv("E2B_DEBUG", "false")
+ return state
+
+
+def _sandbox_response() -> httpx.Response:
+ return httpx.Response(
+ 201,
+ json={
+ "templateID": "prepared-template",
+ "sandboxID": _SANDBOX_ID,
+ "clientID": "contract-client",
+ "envdVersion": "0.5.0",
+ "envdAccessToken": "envd-contract-token",
+ "trafficAccessToken": "traffic-contract-token",
+ "domain": "e2b.test",
+ },
+ )
+
+
+def _control_plane() -> E2BSDKControlPlane:
+ return E2BSDKControlPlane(api_key=_API_KEY)
+
+
+def _binding_backend() -> E2BExecutionBindingBackend:
+ return E2BExecutionBindingBackend(
+ control_plane=_control_plane(),
+ template="prepared-template",
+ active_timeout_seconds=120,
+ )
+
+
+async def _create_sandbox():
+ return await _control_plane().create(
+ "prepared-template",
+ timeout=120,
+ metadata={"dify.resource": "runtime-sandbox"},
+ on_timeout="pause",
+ )
+
+
+def _destroy_spec() -> ExecutionBindingDestroySpec:
+ return ExecutionBindingDestroySpec(
+ binding_ref=_SANDBOX_ID,
+ workspace_ref=_SANDBOX_ID,
+ destroy_workspace=True,
+ )
+
+
+async def test_sdk_lifecycle_preserves_private_traffic_and_memory_resume(api_transport: _ApiTransport) -> None:
+ api_transport.responses = [
+ _sandbox_response(),
+ _sandbox_response(),
+ httpx.Response(204),
+ httpx.Response(201, json={"snapshotID": "team/home:default", "names": []}),
+ httpx.Response(204),
+ httpx.Response(204),
+ ]
+ sandbox = await _create_sandbox()
+ assert sandbox.sandbox_id == _SANDBOX_ID
+ assert sandbox.traffic_access_token == "traffic-contract-token"
+ assert sandbox.get_host(5004) == f"5004-{_SANDBOX_ID}.e2b.test"
+
+ connected = await _control_plane().connect(_SANDBOX_ID, timeout=120)
+ assert connected.traffic_access_token == "traffic-contract-token"
+ assert await connected.pause(mode="full")
+ snapshot = await connected.create_snapshot(mode="full")
+ assert snapshot.snapshot_id == "team/home:default"
+ assert await _control_plane().kill(_SANDBOX_ID)
+ assert await _control_plane().delete_snapshot(snapshot.snapshot_id)
+
+ assert [(request.method, request.url.raw_path) for request in api_transport.requests] == [
+ ("POST", b"/v2/sandboxes"),
+ ("POST", f"/v2/sandboxes/{_SANDBOX_ID}/connect".encode()),
+ ("POST", f"/sandboxes/{_SANDBOX_ID}/pause".encode()),
+ ("POST", f"/sandboxes/{_SANDBOX_ID}/snapshots".encode()),
+ ("DELETE", f"/sandboxes/{_SANDBOX_ID}".encode()),
+ ("DELETE", b"/templates/team%2Fhome%3Adefault"),
+ ]
+ create_body = json.loads(api_transport.requests[0].content)
+ assert create_body["templateID"] == "prepared-template"
+ assert create_body["timeout"] == 120
+ assert create_body["metadata"] == {"dify.resource": "runtime-sandbox"}
+ assert create_body["network"] == {"allowPublicTraffic": False}
+ assert create_body["autoPause"] is True
+ assert create_body["autoPauseMemory"] is True
+ assert create_body["autoResume"] == {"enabled": False}
+ # SDK "restore" leaves memory unset, preserving the connect API's memory-restore default.
+ assert json.loads(api_transport.requests[1].content) == {"timeout": 120}
+ assert json.loads(api_transport.requests[2].content) == {"memory": True}
+ assert json.loads(api_transport.requests[3].content) == {"memory": True}
+ assert all(request.headers["X-API-Key"] == _API_KEY for request in api_transport.requests)
+ assert api_transport.retry_limits == [3] * 6
+ assert api_transport.delays == []
+
+
+async def test_sdk_kill_lifecycle_omits_pause_memory_mode(api_transport: _ApiTransport) -> None:
+ api_transport.responses = [_sandbox_response()]
+ sandbox = await _control_plane().create(
+ "prepared-template",
+ timeout=120,
+ metadata={"dify.resource": "runtime-sandbox"},
+ on_timeout="kill",
+ )
+ assert sandbox.sandbox_id == _SANDBOX_ID
+ assert len(api_transport.requests) == 1
+ create_body = json.loads(api_transport.requests[0].content)
+ assert create_body["autoPause"] is False
+ assert "autoPauseMemory" not in create_body
+ assert "mode" not in create_body
+ assert create_body["autoResume"] == {"enabled": False}
+ assert create_body["network"] == {"allowPublicTraffic": False}
+
+
+async def test_sdk_preserves_already_paused_and_absent_resource_results(api_transport: _ApiTransport) -> None:
+ api_transport.responses = [
+ _sandbox_response(),
+ httpx.Response(409, json={"code": 409, "message": "already paused"}),
+ *[httpx.Response(404, json={"code": 404, "message": "not found"}) for _ in range(3)],
+ ]
+ sandbox = await _create_sandbox()
+ assert await sandbox.pause(mode="full") is False
+ assert await _control_plane().kill(_SANDBOX_ID) is False
+ assert await _control_plane().delete_snapshot("team/home:default") is False
+ with pytest.raises(_E2BControlPlaneNotFoundError, match="not found"):
+ await _control_plane().connect(_SANDBOX_ID, timeout=120)
+ assert len(api_transport.requests) == 5
+ assert api_transport.delays == []
+
+
[email protected](("status", "headers"), [(503, {}), (429, {"Retry-After": "2"})])
+async def test_sdk_recovers_transient_create_responses(
+ api_transport: _ApiTransport, status: int, headers: dict[str, str]
+) -> None:
+ api_transport.responses = [
+ httpx.Response(status, headers=headers, json={"code": status, "message": "busy"}),
+ _sandbox_response(),
+ ]
+ assert (await _create_sandbox()).sandbox_id == _SANDBOX_ID
+ assert len(api_transport.requests) == 2
+ assert api_transport.requests[0].content == api_transport.requests[1].content
+ assert api_transport.delays == ([2.0] if status == 429 else [0.05])
+
+
[email protected]("operation", ["create", "snapshot"])
[email protected]("retry_after", [None, "0", "2"])
+async def test_resource_creates_are_not_replayed_after_bad_gateway(
+ api_transport: _ApiTransport, operation: str, retry_after: str | None
+) -> None:
+ headers = {} if retry_after is None else {"Retry-After": retry_after}
+ error_response = httpx.Response(502, headers=headers, json={"code": 502, "message": "bad gateway"})
+ if operation == "create":
+ api_transport.responses = [error_response]
+ with pytest.raises(BindingCreateError, match="502") as raised:
+ await _binding_backend().create_binding(
+ ExecutionBindingCreateSpec(
+ tenant_id="tenant-contract",
+ agent_id="agent-contract",
+ binding_id="binding-contract",
+ workspace_id="workspace-contract",
+ existing_workspace_ref=None,
+ )
+ )
+ expected_path = "/v2/sandboxes"
+ else:
+ api_transport.responses = [_sandbox_response()]
+ sandbox = await _create_sandbox()
+ api_transport.requests.clear()
+ api_transport.retry_limits.clear()
+ api_transport.responses = [error_response]
+ source = E2BRuntimeLease(sandbox=sandbox, data_plane=cast(ShellctlRuntimeLease, object()))
+ with pytest.raises(HomeSnapshotCreateError, match="502") as raised:
+ await E2BHomeSnapshotBackend(control_plane=_control_plane()).create_from_runtime(
+ spec=HomeSnapshotCreateSpec(
+ tenant_id="tenant-contract", agent_id="agent-contract", home_snapshot_id="home-contract"
+ ),
+ source=source,
+ )
+ expected_path = f"/sandboxes/{_SANDBOX_ID}/snapshots"
+
+ assert isinstance(raised.value.__cause__, SandboxException)
+ assert raised.value.__cause__.status_code == 502
+ assert [(request.method, request.url.path) for request in api_transport.requests] == [("POST", expected_path)]
+ if operation == "snapshot":
+ assert json.loads(api_transport.requests[0].content) == {"memory": True}
+ else:
+ assert json.loads(api_transport.requests[0].content)["autoPauseMemory"] is True
+ assert api_transport.retry_limits == [3]
+ assert api_transport.delays == []
+
+
[email protected]("operation", ["connect", "pause"])
+async def test_sdk_replays_idempotent_lifecycle_after_bad_gateway(api_transport: _ApiTransport, operation: str) -> None:
+ error_response = httpx.Response(502, json={"code": 502, "message": "bad gateway"})
+ if operation == "connect":
+ api_transport.responses = [error_response, _sandbox_response()]
+ assert (await _control_plane().connect(_SANDBOX_ID, timeout=120)).sandbox_id == _SANDBOX_ID
+ expected_path = f"/v2/sandboxes/{_SANDBOX_ID}/connect"
+ else:
+ api_transport.responses = [_sandbox_response()]
+ sandbox = await _create_sandbox()
+ api_transport.requests.clear()
+ api_transport.retry_limits.clear()
+ api_transport.responses = [error_response, httpx.Response(204)]
+ assert await sandbox.pause(mode="full")
+ expected_path = f"/sandboxes/{_SANDBOX_ID}/pause"
+
+ assert [(request.method, request.url.path) for request in api_transport.requests] == [("POST", expected_path)] * 2
+ assert api_transport.requests[0].content == api_transport.requests[1].content
+ expected_body = {"timeout": 120} if operation == "connect" else {"memory": True}
+ assert json.loads(api_transport.requests[0].content) == expected_body
+ assert api_transport.retry_limits == [3]
+ assert api_transport.delays == [0.05]
+
+
[email protected]("status", [502, 503])
+async def test_destroy_exhausts_one_sdk_retry_budget(api_transport: _ApiTransport, status: int) -> None:
+ api_transport.responses = [httpx.Response(status, json={"code": status, "message": "busy"}) for _ in range(4)]
+ with pytest.raises(BindingDestroyError, match=str(status)):
+ await _binding_backend().destroy_binding(_destroy_spec())
+ assert len(api_transport.requests) == 4
+ assert api_transport.delays == [0.05, 0.1, 0.2]
+ assert api_transport.retry_limits == [3]
+ timeouts = [request.extensions["timeout"]["read"] for request in api_transport.requests]
+ assert timeouts[-1] < timeouts[0]
+
+
[email protected]("error_type", [httpx.ReadError, httpx.WriteError, httpx.RemoteProtocolError])
+async def test_destroy_does_not_repeat_sdk_exhausted_network_retries(
+ api_transport: _ApiTransport, error_type: type[httpx.RequestError]
+) -> None:
+ api_transport.responses = [error_type("request interrupted") for _ in range(4)]
+ with pytest.raises(BindingDestroyError, match="request interrupted"):
+ await _binding_backend().destroy_binding(_destroy_spec())
+ assert len(api_transport.requests) == 4
+ assert api_transport.delays == [0.05, 0.1, 0.2]
+ assert api_transport.retry_limits == [3]
+
+
[email protected](("status", "headers"), [(500, {}), (504, {}), (429, {}), (429, {"Retry-After": "61"})])
+async def test_sdk_does_not_retry_terminal_or_over_budget_responses(
+ api_transport: _ApiTransport, status: int, headers: dict[str, str]
+) -> None:
+ api_transport.responses = [httpx.Response(status, headers=headers, json={"code": status, "message": "unavailable"})]
+ expected_error = _E2BControlPlaneCapacityExhaustedError if status == 429 else SandboxException
+ with pytest.raises(expected_error, match=str(status)):
+ await _create_sandbox()
+ assert len(api_transport.requests) == 1
+ assert api_transport.delays == []
+
+
[email protected]("operation", ["create", "snapshot"])
+ "error_type", [httpx.ReadError, httpx.WriteError, httpx.RemoteProtocolError, httpx.ReadTimeout]
+)
+async def test_resource_creates_are_not_replayed_after_request_interruption(
+ api_transport: _ApiTransport, operation: str, error_type: type[httpx.RequestError]
+) -> None:
+ error = error_type("request interrupted after sending")
+ if operation == "create":
+ api_transport.responses = [error]
+ with pytest.raises(BindingCreateError, match="request interrupted"):
+ await _binding_backend().create_binding(
+ ExecutionBindingCreateSpec(
+ tenant_id="tenant-contract",
+ agent_id="agent-contract",
+ binding_id="binding-contract",
+ workspace_id="workspace-contract",
+ existing_workspace_ref=None,
+ )
+ )
+ else:
+ api_transport.responses = [_sandbox_response()]
+ sandbox = await _create_sandbox()
+ api_transport.requests.clear()
+ api_transport.responses = [error]
+ source = E2BRuntimeLease(sandbox=sandbox, data_plane=cast(ShellctlRuntimeLease, object()))
+ with pytest.raises(HomeSnapshotCreateError, match="request interrupted"):
+ await E2BHomeSnapshotBackend(control_plane=_control_plane()).create_from_runtime(
+ spec=HomeSnapshotCreateSpec(
+ tenant_id="tenant-contract", agent_id="agent-contract", home_snapshot_id="home-contract"
+ ),
+ source=source,
+ )
+ assert len(api_transport.requests) == 1
+ assert api_transport.delays == []
+
+
[email protected]("error_type", [httpx.ReadError, httpx.WriteError, httpx.RemoteProtocolError])
+async def test_sdk_replays_idempotent_connect_after_network_error(
+ api_transport: _ApiTransport, error_type: type[httpx.RequestError]
+) -> None:
+ api_transport.responses = [error_type("request interrupted") for _ in range(3)] + [_sandbox_response()]
+ assert (await _control_plane().connect(_SANDBOX_ID, timeout=120)).sandbox_id == _SANDBOX_ID
+ assert len(api_transport.requests) == 4
+ assert api_transport.delays == [0.05, 0.1, 0.2]
+
+
+async def test_destroy_does_not_restart_timeout_budget(api_transport: _ApiTransport) -> None:
+ api_transport.responses = [httpx.ReadTimeout("request timed out")]
+ with pytest.raises(BindingDestroyError, match="request timed out"):
+ await _binding_backend().destroy_binding(_destroy_spec())
+ assert len(api_transport.requests) == 1
+ assert api_transport.delays == []
diff --git a/dify-agent/tests/local/test_packaging.py b/dify-agent/tests/local/test_packaging.py
index ab2d0f96cac..a430fdff9ff 100644
--- a/dify-agent/tests/local/test_packaging.py
+++ b/dify-agent/tests/local/test_packaging.py
@@ -16,7 +16,7 @@ CLIENT_SHARED_DTO_DEPENDENCIES = {
}
SERVER_RUNTIME_DEPENDENCIES = {
- "e2b>=2.38.0,<3.0.0",
+ "e2b>=2.55.1,<3.0.0",
"fastapi==0.136.0",
"graphon==0.5.2",
"grpcio>=1.60.0,<2.0.0",
diff --git a/dify-agent/uv.lock b/dify-agent/uv.lock
index 060ab961211..5df5ee28263 100644
--- a/dify-agent/uv.lock
+++ b/dify-agent/uv.lock
@@ -578,7 +578,7 @@ docs = [
[package.metadata]
requires-dist = [
- { name = "e2b", marker = "extra == 'server'", specifier = ">=2.38.0,<3.0.0" },
+ { name = "e2b", marker = "extra == 'server'", specifier = ">=2.55.1,<3.0.0" },
{ name = "fastapi", marker = "extra == 'server'", specifier = "==0.136.0" },
{ name = "graphon", marker = "extra == 'server'", specifier = "==0.5.2" },
{ name = "grpcio", marker = "extra == 'server'", specifier = ">=1.60.0,<2.0.0" },
@@ -624,15 +624,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" },
]
-[[package]]
-name = "dockerfile-parse"
-version = "2.0.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/92/df/929ee0b5d2c8bd8d713c45e71b94ab57c7e11e322130724d54f469b2cd48/dockerfile-parse-2.0.1.tar.gz", hash = "sha256:3184ccdc513221983e503ac00e1aa504a2aa8f84e5de673c46b0b6eee99ec7bc", size = 24556, upload-time = "2023-07-18T13:36:07.897Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/7a/6c/79cd5bc1b880d8c1a9a5550aa8dacd57353fa3bb2457227e1fb47383eb49/dockerfile_parse-2.0.1-py2.py3-none-any.whl", hash = "sha256:bdffd126d2eb26acf1066acb54cb2e336682e1d72b974a40894fac76a4df17f6", size = 14845, upload-time = "2023-07-18T13:36:06.052Z" },
-]
-
[[package]]
name = "docstring-parser"
version = "0.18.0"
@@ -644,12 +635,12 @@ wheels = [
[[package]]
name = "e2b"
-version = "2.46.4"
+version = "2.55.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "attrs" },
{ name = "connectrpc" },
- { name = "dockerfile-parse" },
+ { name = "e2b-dockerfile-utils" },
{ name = "h2" },
{ name = "httpx" },
{ name = "packaging" },
@@ -660,9 +651,18 @@ dependencies = [
{ name = "typing-extensions" },
{ name = "wcmatch" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/7d/04/3e7d3ca8abaeffcfc41d36b9d6fe3aa055cecf3b56234e051991de848b2d/e2b-2.46.4.tar.gz", hash = "sha256:777d35beb00194b401a6c4ac2498aa836628839649f17aae86e85920382c8f6c", size = 228801, upload-time = "2026-09-02T20:48:36.725Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/40/ad/144bf647f526e65829c78d9b9f681d0d4af13cacaa80c326e1f05ba4cc42/e2b-2.55.1.tar.gz", hash = "sha256:5255f9a01dc484c343f750d13d3a9c857ef3511222a240572ddce2563dd301d9", size = 241589, upload-time = "2026-10-09T14:54:52.76Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a6/0c/7fca57469c04bdf30373dc1b49336fa29da3863ab095ba82e81f246517cb/e2b-2.55.1-py3-none-any.whl", hash = "sha256:30aec382a7cac5401e0aa0efb156422459f586101809c3e9f34a6fcf797097db", size = 402847, upload-time = "2026-10-09T14:54:54.319Z" },
+]
+
+[[package]]
+name = "e2b-dockerfile-utils"
+version = "0.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/17/82/204eaae693bf5c55c85879e055fbfe86941bccbe41e3d6d502319bb0a17d/e2b_dockerfile_utils-0.1.0.tar.gz", hash = "sha256:f840a3e30b2e4140ef4ebb67013a399b568ce4bf62de9f9c1d069d3e47ac9992", size = 16843, upload-time = "2026-10-09T10:00:42.208Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/a0/b5/585a30223d9ee95bc94f32f8d5358b147bf93832e37b809545a3ab7892a0/e2b-2.46.4-py3-none-any.whl", hash = "sha256:5d77e9fee8571a0d24a179f14244984127d18bd93a99a5e90226c876087035b5", size = 403067, upload-time = "2026-09-02T20:48:35.082Z" },
+ { url = "https://files.pythonhosted.org/packages/4b/a3/2851fb65169f6724ac6befeb8f593e465ad4479d2a630d3b281ec36f73aa/e2b_dockerfile_utils-0.1.0-py3-none-any.whl", hash = "sha256:fedc7d036d8a23cb962c01b90882ebec154520ec1bed064e1fe92796a7971553", size = 19901, upload-time = "2026-10-09T10:00:41.384Z" },
]
[[package]]