Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(dify-agent): wait for E2B shellctl readiness (#39996)
盐粒 Yanli1369a2cdfc12
diff --git a/dify-agent/src/dify_agent/adapters/shell/shellctl.py b/dify-agent/src/dify_agent/adapters/shell/shellctl.pyindex 24cc9a576f8..d4cf4dfd663 100644--- a/dify-agent/src/dify_agent/adapters/shell/shellctl.py+++ b/dify-agent/src/dify_agent/adapters/shell/shellctl.py@@ -24,6 +24,7 @@ from typing import Protocol, TypeVar, castimport httpx2 as httpxfrom shellctl.client import ShellctlClientError+from shellctl.shared import HealthResponsefrom dify_agent.adapters.shell.protocols import (ShellCommandProtocol,@@ -214,6 +215,8 @@ class ShellctlJobStatus(Protocol):class ShellctlClientProtocol(Protocol):+ async def health(self) -> HealthResponse: ...+async def run(self,script: str,diff --git a/dify-agent/src/dify_agent/runtime_backend/e2b.py b/dify-agent/src/dify_agent/runtime_backend/e2b.pyindex 4cd18e08d02..fdcaaa0c1fe 100644--- a/dify-agent/src/dify_agent/runtime_backend/e2b.py+++ b/dify-agent/src/dify_agent/runtime_backend/e2b.py@@ -8,10 +8,12 @@ operation-local and are never serialized into Agenton state.from __future__ import annotations+import asynciofrom dataclasses import dataclass, fieldfrom typing import TYPE_CHECKING, Literal, Protocol, castimport httpx2 as httpx+from shellctl.client import ShellctlClientErrorfrom dify_agent.adapters.shell.protocols import ShellCommandProtocolfrom dify_agent.adapters.shell.shellctl import ShellctlClientProtocol@@ -39,6 +41,8 @@ if TYPE_CHECKING:from e2b.connection_config import ApiParamsE2B_MAX_ACTIVE_TIMEOUT_SECONDS = 60 * 60+_SHELLCTL_READY_MAX_ATTEMPTS = 3+_SHELLCTL_READY_RETRY_INTERVAL_SECONDS = 0.5class _E2BControlPlaneNotFoundError(RuntimeError):@@ -250,17 +254,21 @@ class E2BExecutionBindingBackend:async def acquire(self, binding_ref: str) -> RuntimeLease:"""Acquire operation-scoped shellctl access for an opaque Binding ref."""sandbox: _E2BSandbox | None = None+ lease: E2BRuntimeLease | None = Nonetry:sandbox = await self.control_plane.connect(binding_ref, timeout=self.active_timeout_seconds)if not await sandbox.files.exists(self.layout.workspace_dir):raise BindingLostError(f"E2B Binding {binding_ref!r} no longer contains its Workspace")- return await self._lease(sandbox)+ lease = await self._lease(sandbox)+ await _wait_for_shellctl_ready(lease.data_plane.client)+ return leaseexcept _E2BControlPlaneNotFoundError as exc:raise BindingLostError(f"E2B Binding {binding_ref!r} no longer exists") from excexcept BindingLostError:await _best_effort_pause(sandbox)raiseexcept BaseException as exc:+ await _best_effort_close_data_plane(lease)await _best_effort_pause(sandbox)if isinstance(exc, Exception):raise BindingAcquireError(str(exc)) from exc@@ -354,6 +362,29 @@ class E2BRuntimeLease:return self.data_plane.files+async def _wait_for_shellctl_ready(client: ShellctlClientProtocol) -> None:+ for attempt in range(_SHELLCTL_READY_MAX_ATTEMPTS):+ try:+ _ = await client.health()+ return+ except (httpx.TimeoutException, httpx.RequestError):+ if attempt == _SHELLCTL_READY_MAX_ATTEMPTS - 1:+ raise+ except ShellctlClientError as exc:+ if not 500 <= exc.status_code < 600 or attempt == _SHELLCTL_READY_MAX_ATTEMPTS - 1:+ raise+ await asyncio.sleep(_SHELLCTL_READY_RETRY_INTERVAL_SECONDS)+++async def _best_effort_close_data_plane(lease: E2BRuntimeLease | None) -> None:+ if lease is None:+ return+ try:+ await lease.data_plane.close()+ except BaseException:+ pass++async def _best_effort_pause(sandbox: _E2BSandbox | None) -> None:if sandbox is None:returndiff --git a/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py b/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.pyindex bd3ec3fba11..085462b5146 100644--- a/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py+++ b/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py@@ -1,11 +1,15 @@from __future__ import annotations+from collections.abc import Callablefrom dataclasses import dataclass, fieldfrom typing import cast+import httpx2 as httpximport pytest+from shellctl.client import ShellctlClientErrorfrom dify_agent.runtime_backend import (+ BindingAcquireError,BindingCreateError,ExecutionBindingCreateSpec,ExecutionBindingDestroySpec,@@ -13,6 +17,7 @@ from dify_agent.runtime_backend import (SharedWorkspaceUnsupportedError,WorkspacePreservationUnsupportedError,)+from dify_agent.runtime_backend import e2b as e2b_modulefrom dify_agent.runtime_backend.e2b import (E2BExecutionBindingBackend,E2BHomeSnapshotBackend,@@ -101,6 +106,39 @@ class _ControlPlane:return True+def _mock_http(+ monkeypatch: pytest.MonkeyPatch,+ handler: Callable[[httpx.Request], httpx.Response],+) -> list[httpx.AsyncClient]:+ original_async_client = httpx.AsyncClient+ transport = httpx.MockTransport(handler)+ clients: list[httpx.AsyncClient] = []++ def create_client(*args: object, **kwargs: object) -> httpx.AsyncClient:+ _ = kwargs.setdefault("transport", transport)+ client = original_async_client(*args, **kwargs)+ clients.append(client)+ return client++ monkeypatch.setattr(httpx, "AsyncClient", create_client)+ return clients+++def _connected_backend(*, pause_error: Exception | None = None) -> tuple[E2BExecutionBindingBackend, _Sandbox]:+ control = _ControlPlane()+ sandbox = _Sandbox(sandbox_id="sandbox-1", pause_error=pause_error)+ sandbox.files.paths.add("/home/dify/workspace")+ control.sandboxes[sandbox.sandbox_id] = sandbox+ return (+ E2BExecutionBindingBackend(+ control_plane=control, # pyright: ignore[reportArgumentType]+ template="prepared-template",+ active_timeout_seconds=3600,+ ),+ sandbox,+ )++@pytest.mark.anyioasync def test_e2b_binding_uses_default_template_or_exact_snapshot_and_couples_refs() -> None:control = _ControlPlane()@@ -254,3 +292,140 @@ async def test_e2b_checkpoint_uses_exact_source_runtime() -> None:assert snapshot_ref == "snapshot-source-1"assert source_sandbox.snapshots == 1+++async def test_e2b_acquire_retries_transient_shellctl_failures_until_ready(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ attempts = 0++ def handler(request: httpx.Request) -> httpx.Response:+ nonlocal attempts+ attempts += 1+ if attempts == 1:+ raise httpx.ReadTimeout("shellctl starting", request=request)+ if attempts == 2:+ raise httpx.ConnectError("shellctl starting", request=request)+ return httpx.Response(200, json={"status": "ok"})++ clients = _mock_http(monkeypatch, handler)+ sleeps: list[float] = []++ async def record_sleep(delay: float) -> None:+ sleeps.append(delay)++ monkeypatch.setattr(e2b_module.asyncio, "sleep", record_sleep)+ backend, sandbox = _connected_backend()++ lease = await backend.acquire(sandbox.sandbox_id)++ assert attempts == 3+ assert sleeps == [0.5, 0.5]+ assert not clients[0].is_closed+ await backend.release(lease)+ assert clients[0].is_closed+++async def test_e2b_acquire_closes_transport_and_pauses_after_readiness_retries_exhausted(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ attempts = 0++ def handler(_request: httpx.Request) -> httpx.Response:+ nonlocal attempts+ attempts += 1+ return httpx.Response(503, json={"error": {"code": "starting", "message": "not ready"}})++ clients = _mock_http(monkeypatch, handler)+ sleeps: list[float] = []++ async def record_sleep(delay: float) -> None:+ sleeps.append(delay)++ monkeypatch.setattr(e2b_module.asyncio, "sleep", record_sleep)+ backend, sandbox = _connected_backend()++ with pytest.raises(BindingAcquireError, match="not ready"):+ _ = await backend.acquire(sandbox.sandbox_id)++ assert attempts == 3+ assert sleeps == [0.5, 0.5]+ assert clients[0].is_closed+ assert sandbox.pauses == [True]+++async def test_e2b_acquire_does_not_retry_shellctl_4xx_and_preserves_error_when_pause_fails(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ attempts = 0++ def handler(_request: httpx.Request) -> httpx.Response:+ nonlocal attempts+ attempts += 1+ return httpx.Response(401, json={"error": {"code": "unauthorized", "message": "bad token"}})++ clients = _mock_http(monkeypatch, handler)++ async def fail_sleep(_delay: float) -> None:+ raise AssertionError("non-retryable health failures must not sleep")++ monkeypatch.setattr(e2b_module.asyncio, "sleep", fail_sleep)+ backend, sandbox = _connected_backend(pause_error=RuntimeError("pause failed"))++ with pytest.raises(BindingAcquireError, match="bad token"):+ _ = await backend.acquire(sandbox.sandbox_id)++ assert attempts == 1+ assert clients[0].is_closed+ assert sandbox.pauses == [True]+++async def test_e2b_acquire_preserves_health_failure_when_close_and_pause_fail(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ @dataclass(slots=True)+ class _UnavailableHealthClient:+ calls: int = 0++ async def health(self) -> object:+ self.calls += 1+ raise ShellctlClientError(503, "starting", "primary health failure")++ @dataclass(slots=True)+ class _FailingCloseDataPlane:+ client: _UnavailableHealthClient+ close_calls: int = 0++ async def close(self) -> None:+ self.close_calls += 1+ raise RuntimeError("close failed")++ client = _UnavailableHealthClient()+ data_plane = _FailingCloseDataPlane(client=client)++ async def create_lease(+ _self: E2BExecutionBindingBackend,+ sandbox: _Sandbox,+ ) -> E2BRuntimeLease:+ return E2BRuntimeLease(+ sandbox=sandbox, # pyright: ignore[reportArgumentType]+ data_plane=cast(ShellctlRuntimeLease, cast(object, data_plane)),+ )++ async def skip_sleep(_delay: float) -> None:+ return None++ monkeypatch.setattr(E2BExecutionBindingBackend, "_lease", create_lease)+ monkeypatch.setattr(e2b_module.asyncio, "sleep", skip_sleep)+ backend, sandbox = _connected_backend(pause_error=RuntimeError("pause failed"))++ with pytest.raises(BindingAcquireError, match="primary health failure"):+ _ = await backend.acquire(sandbox.sandbox_id)++ assert client.calls == 3+ assert data_plane.close_calls == 1+ assert sandbox.pauses == [True]