Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(dify-agent): wait for E2B shellctl readiness (#39996)

盐粒 Yanli1369a2cdfc12
diff --git a/dify-agent/src/dify_agent/adapters/shell/shellctl.py b/dify-agent/src/dify_agent/adapters/shell/shellctl.py
index 24cc9a576f8..d4cf4dfd663 100644
--- a/dify-agent/src/dify_agent/adapters/shell/shellctl.py
+++ b/dify-agent/src/dify_agent/adapters/shell/shellctl.py
@@ -24,6 +24,7 @@ from typing import Protocol, TypeVar, cast
import httpx2 as httpx
from shellctl.client import ShellctlClientError
+from shellctl.shared import HealthResponse
from dify_agent.adapters.shell.protocols import (
ShellCommandProtocol,
@@ -214,6 +215,8 @@ class ShellctlJobStatus(Protocol):
class ShellctlClientProtocol(Protocol):
+ async def health(self) -> HealthResponse: ...
+
async def run(
self,
script: str,
diff --git a/dify-agent/src/dify_agent/runtime_backend/e2b.py b/dify-agent/src/dify_agent/runtime_backend/e2b.py
index 4cd18e08d02..fdcaaa0c1fe 100644
--- a/dify-agent/src/dify_agent/runtime_backend/e2b.py
+++ b/dify-agent/src/dify_agent/runtime_backend/e2b.py
@@ -8,10 +8,12 @@ operation-local and are never serialized into Agenton state.
from __future__ import annotations
+import asyncio
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Literal, Protocol, cast
import httpx2 as httpx
+from shellctl.client import ShellctlClientError
from dify_agent.adapters.shell.protocols import ShellCommandProtocol
from dify_agent.adapters.shell.shellctl import ShellctlClientProtocol
@@ -39,6 +41,8 @@ if TYPE_CHECKING:
from e2b.connection_config import ApiParams
E2B_MAX_ACTIVE_TIMEOUT_SECONDS = 60 * 60
+_SHELLCTL_READY_MAX_ATTEMPTS = 3
+_SHELLCTL_READY_RETRY_INTERVAL_SECONDS = 0.5
class _E2BControlPlaneNotFoundError(RuntimeError):
@@ -250,17 +254,21 @@ class E2BExecutionBindingBackend:
async def acquire(self, binding_ref: str) -> RuntimeLease:
"""Acquire operation-scoped shellctl access for an opaque Binding ref."""
sandbox: _E2BSandbox | None = None
+ lease: E2BRuntimeLease | None = None
try:
sandbox = await self.control_plane.connect(binding_ref, timeout=self.active_timeout_seconds)
if not await sandbox.files.exists(self.layout.workspace_dir):
raise BindingLostError(f"E2B Binding {binding_ref!r} no longer contains its Workspace")
- return await self._lease(sandbox)
+ lease = await self._lease(sandbox)
+ await _wait_for_shellctl_ready(lease.data_plane.client)
+ return lease
except _E2BControlPlaneNotFoundError as exc:
raise BindingLostError(f"E2B Binding {binding_ref!r} no longer exists") from exc
except BindingLostError:
await _best_effort_pause(sandbox)
raise
except BaseException as exc:
+ await _best_effort_close_data_plane(lease)
await _best_effort_pause(sandbox)
if isinstance(exc, Exception):
raise BindingAcquireError(str(exc)) from exc
@@ -354,6 +362,29 @@ class E2BRuntimeLease:
return self.data_plane.files
+async def _wait_for_shellctl_ready(client: ShellctlClientProtocol) -> None:
+ for attempt in range(_SHELLCTL_READY_MAX_ATTEMPTS):
+ try:
+ _ = await client.health()
+ return
+ except (httpx.TimeoutException, httpx.RequestError):
+ if attempt == _SHELLCTL_READY_MAX_ATTEMPTS - 1:
+ raise
+ except ShellctlClientError as exc:
+ if not 500 <= exc.status_code < 600 or attempt == _SHELLCTL_READY_MAX_ATTEMPTS - 1:
+ raise
+ await asyncio.sleep(_SHELLCTL_READY_RETRY_INTERVAL_SECONDS)
+
+
+async def _best_effort_close_data_plane(lease: E2BRuntimeLease | None) -> None:
+ if lease is None:
+ return
+ try:
+ await lease.data_plane.close()
+ except BaseException:
+ pass
+
+
async def _best_effort_pause(sandbox: _E2BSandbox | None) -> None:
if sandbox is None:
return
diff --git a/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py b/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py
index bd3ec3fba11..085462b5146 100644
--- a/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py
+++ b/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py
@@ -1,11 +1,15 @@
from __future__ import annotations
+from collections.abc import Callable
from dataclasses import dataclass, field
from typing import cast
+import httpx2 as httpx
import pytest
+from shellctl.client import ShellctlClientError
from dify_agent.runtime_backend import (
+ BindingAcquireError,
BindingCreateError,
ExecutionBindingCreateSpec,
ExecutionBindingDestroySpec,
@@ -13,6 +17,7 @@ from dify_agent.runtime_backend import (
SharedWorkspaceUnsupportedError,
WorkspacePreservationUnsupportedError,
)
+from dify_agent.runtime_backend import e2b as e2b_module
from dify_agent.runtime_backend.e2b import (
E2BExecutionBindingBackend,
E2BHomeSnapshotBackend,
@@ -101,6 +106,39 @@ class _ControlPlane:
return True
+def _mock_http(
+ monkeypatch: pytest.MonkeyPatch,
+ handler: Callable[[httpx.Request], httpx.Response],
+) -> list[httpx.AsyncClient]:
+ original_async_client = httpx.AsyncClient
+ transport = httpx.MockTransport(handler)
+ clients: list[httpx.AsyncClient] = []
+
+ def create_client(*args: object, **kwargs: object) -> httpx.AsyncClient:
+ _ = kwargs.setdefault("transport", transport)
+ client = original_async_client(*args, **kwargs)
+ clients.append(client)
+ return client
+
+ monkeypatch.setattr(httpx, "AsyncClient", create_client)
+ return clients
+
+
+def _connected_backend(*, pause_error: Exception | None = None) -> tuple[E2BExecutionBindingBackend, _Sandbox]:
+ control = _ControlPlane()
+ sandbox = _Sandbox(sandbox_id="sandbox-1", pause_error=pause_error)
+ sandbox.files.paths.add("/home/dify/workspace")
+ control.sandboxes[sandbox.sandbox_id] = sandbox
+ return (
+ E2BExecutionBindingBackend(
+ control_plane=control, # pyright: ignore[reportArgumentType]
+ template="prepared-template",
+ active_timeout_seconds=3600,
+ ),
+ sandbox,
+ )
+
+
@pytest.mark.anyio
async def test_e2b_binding_uses_default_template_or_exact_snapshot_and_couples_refs() -> None:
control = _ControlPlane()
@@ -254,3 +292,140 @@ async def test_e2b_checkpoint_uses_exact_source_runtime() -> None:
assert snapshot_ref == "snapshot-source-1"
assert source_sandbox.snapshots == 1
+
+
+async def test_e2b_acquire_retries_transient_shellctl_failures_until_ready(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ attempts = 0
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ nonlocal attempts
+ attempts += 1
+ if attempts == 1:
+ raise httpx.ReadTimeout("shellctl starting", request=request)
+ if attempts == 2:
+ raise httpx.ConnectError("shellctl starting", request=request)
+ return httpx.Response(200, json={"status": "ok"})
+
+ clients = _mock_http(monkeypatch, handler)
+ sleeps: list[float] = []
+
+ async def record_sleep(delay: float) -> None:
+ sleeps.append(delay)
+
+ monkeypatch.setattr(e2b_module.asyncio, "sleep", record_sleep)
+ backend, sandbox = _connected_backend()
+
+ lease = await backend.acquire(sandbox.sandbox_id)
+
+ assert attempts == 3
+ assert sleeps == [0.5, 0.5]
+ assert not clients[0].is_closed
+ await backend.release(lease)
+ assert clients[0].is_closed
+
+
+async def test_e2b_acquire_closes_transport_and_pauses_after_readiness_retries_exhausted(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ attempts = 0
+
+ def handler(_request: httpx.Request) -> httpx.Response:
+ nonlocal attempts
+ attempts += 1
+ return httpx.Response(503, json={"error": {"code": "starting", "message": "not ready"}})
+
+ clients = _mock_http(monkeypatch, handler)
+ sleeps: list[float] = []
+
+ async def record_sleep(delay: float) -> None:
+ sleeps.append(delay)
+
+ monkeypatch.setattr(e2b_module.asyncio, "sleep", record_sleep)
+ backend, sandbox = _connected_backend()
+
+ with pytest.raises(BindingAcquireError, match="not ready"):
+ _ = await backend.acquire(sandbox.sandbox_id)
+
+ assert attempts == 3
+ assert sleeps == [0.5, 0.5]
+ assert clients[0].is_closed
+ assert sandbox.pauses == [True]
+
+
+async def test_e2b_acquire_does_not_retry_shellctl_4xx_and_preserves_error_when_pause_fails(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ attempts = 0
+
+ def handler(_request: httpx.Request) -> httpx.Response:
+ nonlocal attempts
+ attempts += 1
+ return httpx.Response(401, json={"error": {"code": "unauthorized", "message": "bad token"}})
+
+ clients = _mock_http(monkeypatch, handler)
+
+ async def fail_sleep(_delay: float) -> None:
+ raise AssertionError("non-retryable health failures must not sleep")
+
+ monkeypatch.setattr(e2b_module.asyncio, "sleep", fail_sleep)
+ backend, sandbox = _connected_backend(pause_error=RuntimeError("pause failed"))
+
+ with pytest.raises(BindingAcquireError, match="bad token"):
+ _ = await backend.acquire(sandbox.sandbox_id)
+
+ assert attempts == 1
+ assert clients[0].is_closed
+ assert sandbox.pauses == [True]
+
+
+async def test_e2b_acquire_preserves_health_failure_when_close_and_pause_fail(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ @dataclass(slots=True)
+ class _UnavailableHealthClient:
+ calls: int = 0
+
+ async def health(self) -> object:
+ self.calls += 1
+ raise ShellctlClientError(503, "starting", "primary health failure")
+
+ @dataclass(slots=True)
+ class _FailingCloseDataPlane:
+ client: _UnavailableHealthClient
+ close_calls: int = 0
+
+ async def close(self) -> None:
+ self.close_calls += 1
+ raise RuntimeError("close failed")
+
+ client = _UnavailableHealthClient()
+ data_plane = _FailingCloseDataPlane(client=client)
+
+ async def create_lease(
+ _self: E2BExecutionBindingBackend,
+ sandbox: _Sandbox,
+ ) -> E2BRuntimeLease:
+ return E2BRuntimeLease(
+ sandbox=sandbox, # pyright: ignore[reportArgumentType]
+ data_plane=cast(ShellctlRuntimeLease, cast(object, data_plane)),
+ )
+
+ async def skip_sleep(_delay: float) -> None:
+ return None
+
+ monkeypatch.setattr(E2BExecutionBindingBackend, "_lease", create_lease)
+ monkeypatch.setattr(e2b_module.asyncio, "sleep", skip_sleep)
+ backend, sandbox = _connected_backend(pause_error=RuntimeError("pause failed"))
+
+ with pytest.raises(BindingAcquireError, match="primary health failure"):
+ _ = await backend.acquire(sandbox.sandbox_id)
+
+ assert client.calls == 3
+ assert data_plane.close_calls == 1
+ assert sandbox.pauses == [True]