Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(dify-agent): harden E2B binding lifecycle (#41623)

Ingram Za1a10f17e32e
diff --git a/api/uv.lock b/api/uv.lock
index d7539edbf7a..3146632b97d 100644
--- a/api/uv.lock
+++ b/api/uv.lock
@@ -1309,7 +1309,7 @@ dependencies = [
[package.metadata]
requires-dist = [
- { name = "e2b", marker = "extra == 'server'", specifier = ">=2.34.0,<3.0.0" },
+ { name = "e2b", marker = "extra == 'server'", specifier = ">=2.38.0,<3.0.0" },
{ name = "fastapi", marker = "extra == 'server'", specifier = "==0.136.0" },
{ name = "graphon", marker = "extra == 'server'", specifier = "==0.5.2" },
{ name = "httpx", specifier = "==0.28.1" },
diff --git a/dify-agent/pyproject.toml b/dify-agent/pyproject.toml
index 07d0dec20ad..58dcc648151 100644
--- a/dify-agent/pyproject.toml
+++ b/dify-agent/pyproject.toml
@@ -19,7 +19,7 @@ dify-agent-stub-server = "dify_agent.agent_stub.server.cli:main"
[project.optional-dependencies]
server = [
- "e2b>=2.34.0,<3.0.0",
+ "e2b>=2.38.0,<3.0.0",
"fastapi==0.136.0",
"graphon==0.5.2",
"jsonschema>=4.23.0,<5.0.0",
diff --git a/dify-agent/src/dify_agent/runtime_backend/__init__.py b/dify-agent/src/dify_agent/runtime_backend/__init__.py
index c274a93fb53..607fddd3242 100644
--- a/dify-agent/src/dify_agent/runtime_backend/__init__.py
+++ b/dify-agent/src/dify_agent/runtime_backend/__init__.py
@@ -2,6 +2,7 @@
from .errors import (
BindingAcquireError,
+ BindingCapacityExhaustedError,
BindingCreateError,
BindingDestroyError,
BindingLostError,
@@ -27,6 +28,7 @@ from .protocols import (
__all__ = [
"BindingAcquireError",
+ "BindingCapacityExhaustedError",
"BindingCreateError",
"BindingDestroyError",
"BindingLostError",
diff --git a/dify-agent/src/dify_agent/runtime_backend/e2b.py b/dify-agent/src/dify_agent/runtime_backend/e2b.py
index 4dad6ff13bc..7f09cb5a9f3 100644
--- a/dify-agent/src/dify_agent/runtime_backend/e2b.py
+++ b/dify-agent/src/dify_agent/runtime_backend/e2b.py
@@ -9,9 +9,12 @@ operation-local and are never serialized into Agenton state.
from __future__ import annotations
import asyncio
+from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Literal, Protocol, cast
+import logging
+from typing import TYPE_CHECKING, Literal, Protocol, TypeVar, cast
+import httpx as e2b_httpx
import httpx2 as httpx
from shellctl.client import ShellctlClientError
@@ -19,6 +22,7 @@ from dify_agent.adapters.shell.protocols import ShellCommandProtocol
from dify_agent.adapters.shell.shellctl import ShellctlClientProtocol
from dify_agent.runtime_backend.errors import (
BindingAcquireError,
+ BindingCapacityExhaustedError,
BindingCreateError,
BindingDestroyError,
BindingLostError,
@@ -41,14 +45,30 @@ if TYPE_CHECKING:
# One RuntimeLease spans the complete Agent run, not one Shell tool call.
E2B_MAX_ACTIVE_TIMEOUT_SECONDS = 60 * 60
+_E2B_CONTROL_PLANE_MAX_ATTEMPTS = 2
+_E2B_CONTROL_PLANE_RETRY_INTERVAL_SECONDS = 0.25
_SHELLCTL_READY_MAX_ATTEMPTS = 3
_SHELLCTL_READY_RETRY_INTERVAL_SECONDS = 0.5
+_RETRYABLE_E2B_TRANSPORT_ERRORS = (
+ e2b_httpx.ConnectError,
+ e2b_httpx.ReadTimeout,
+ e2b_httpx.ReadError,
+ e2b_httpx.WriteError,
+ e2b_httpx.RemoteProtocolError,
+)
+
+_ResultT = TypeVar("_ResultT")
+logger = logging.getLogger(__name__)
class _E2BControlPlaneNotFoundError(RuntimeError):
"""Typed boundary error for SDK resources that no longer exist."""
+class _E2BControlPlaneCapacityExhaustedError(RuntimeError):
+ """Typed boundary error for provider-side Sandbox capacity exhaustion."""
+
+
class _E2BFileEntry(Protocol):
path: str
@@ -121,7 +141,7 @@ class E2BSDKControlPlane:
metadata: dict[str, str],
on_timeout: Literal["kill", "pause"],
) -> _E2BSandbox:
- from e2b import AsyncSandbox, NotFoundException, SandboxNotFoundException
+ from e2b import AsyncSandbox, NotFoundException, RateLimitException, SandboxNotFoundException
try:
return cast(
@@ -140,6 +160,8 @@ class E2BSDKControlPlane:
)
except (SandboxNotFoundException, NotFoundException) as exc:
raise _E2BControlPlaneNotFoundError(str(exc)) from exc
+ except RateLimitException as exc:
+ raise _E2BControlPlaneCapacityExhaustedError(str(exc)) from exc
async def connect(self, handle: str, *, timeout: int) -> _E2BSandbox:
from e2b import AsyncSandbox, NotFoundException, SandboxNotFoundException
@@ -195,7 +217,12 @@ class E2BHomeSnapshotBackend:
async def delete(self, snapshot_ref: str) -> None:
try:
- _ = await self.control_plane.delete_snapshot(snapshot_ref)
+ _ = await _run_e2b_idempotent_operation(
+ operation="delete_snapshot",
+ sandbox_id=snapshot_ref,
+ cleanup_stage="home_snapshot_delete",
+ action=lambda: self.control_plane.delete_snapshot(snapshot_ref),
+ )
except _E2BControlPlaneNotFoundError:
return
except Exception as exc:
@@ -243,14 +270,33 @@ class E2BExecutionBindingBackend:
for entry in await sandbox.files.list(self.layout.workspace_dir):
await sandbox.files.remove(entry.path)
sandbox_id = sandbox.sandbox_id
- _ = await sandbox.pause(keep_memory=True)
+ _ = await _run_e2b_idempotent_operation(
+ operation="pause",
+ sandbox_id=sandbox_id,
+ cleanup_stage="binding_create",
+ action=lambda: sandbox.pause(keep_memory=True),
+ )
return ExecutionBindingAllocation(binding_ref=sandbox_id, workspace_ref=sandbox_id)
+ except _E2BControlPlaneCapacityExhaustedError as exc:
+ raise BindingCapacityExhaustedError(str(exc)) from exc
except BaseException as exc:
if sandbox is not None:
try:
- _ = await sandbox.kill()
- except BaseException:
- pass
+ _ = await _run_e2b_idempotent_operation(
+ operation="kill",
+ sandbox_id=sandbox.sandbox_id,
+ cleanup_stage="binding_create_compensation",
+ action=sandbox.kill,
+ )
+ except Exception as cleanup_exc:
+ _log_e2b_cleanup_warning(
+ "failed to remove partial E2B Binding",
+ operation="kill",
+ sandbox_id=sandbox.sandbox_id,
+ attempt=_e2b_failure_attempt(cleanup_exc),
+ cleanup_stage="binding_create_compensation",
+ exception=cleanup_exc,
+ )
if isinstance(exc, Exception):
if isinstance(exc, BindingCreateError):
raise
@@ -262,7 +308,12 @@ class E2BExecutionBindingBackend:
sandbox: _E2BSandbox | None = None
lease: E2BRuntimeLease | None = None
try:
- sandbox = await self.control_plane.connect(binding_ref, timeout=self.active_timeout_seconds)
+ sandbox = await _run_e2b_idempotent_operation(
+ operation="connect",
+ sandbox_id=binding_ref,
+ cleanup_stage="binding_acquire",
+ action=lambda: self.control_plane.connect(binding_ref, timeout=self.active_timeout_seconds),
+ )
if not await sandbox.files.exists(self.layout.workspace_dir):
raise BindingLostError(f"E2B Binding {binding_ref!r} no longer contains its Workspace")
lease = await self._lease(sandbox)
@@ -281,20 +332,48 @@ class E2BExecutionBindingBackend:
raise
async def release(self, lease: RuntimeLease) -> None:
- """Close operation-local transports and pause the physical E2B resource."""
+ """Close operation-local transports and best-effort pause the E2B resource."""
if not isinstance(lease, E2BRuntimeLease):
raise TypeError("E2BExecutionBindingBackend can only release its own RuntimeLease")
- close_error: Exception | None = None
+ close_base_error: BaseException | None = None
try:
await lease.data_plane.close()
except Exception as exc:
- close_error = exc
+ _log_e2b_cleanup_warning(
+ "failed to close E2B RuntimeLease data plane",
+ operation="close_data_plane",
+ sandbox_id=lease.sandbox.sandbox_id,
+ attempt=1,
+ cleanup_stage="binding_release",
+ exception=exc,
+ )
+ except BaseException as exc:
+ close_base_error = exc
+ pause_base_error: BaseException | None = None
try:
- _ = await lease.sandbox.pause(keep_memory=True)
+ _ = await _run_e2b_idempotent_operation(
+ operation="pause",
+ sandbox_id=lease.sandbox.sandbox_id,
+ cleanup_stage="binding_release",
+ action=lambda: lease.sandbox.pause(keep_memory=True),
+ )
except Exception as exc:
- raise BindingAcquireError(str(exc)) from exc
- if close_error is not None:
- raise BindingAcquireError(str(close_error)) from close_error
+ _log_e2b_cleanup_warning(
+ "failed to pause E2B RuntimeLease",
+ operation="pause",
+ sandbox_id=lease.sandbox.sandbox_id,
+ attempt=_e2b_failure_attempt(exc),
+ cleanup_stage="binding_release",
+ exception=exc,
+ )
+ except BaseException as exc:
+ pause_base_error = exc
+ if close_base_error is not None:
+ if pause_base_error is not None:
+ raise close_base_error from pause_base_error
+ raise close_base_error
+ if pause_base_error is not None:
+ raise pause_base_error
async def destroy_binding(self, spec: ExecutionBindingDestroySpec) -> None:
"""Destroy the coupled physical Binding and Workspace idempotently."""
@@ -305,7 +384,12 @@ class E2BExecutionBindingBackend:
if spec.workspace_ref != spec.binding_ref:
raise BindingDestroyError("E2B Workspace ref must equal its Binding ref")
try:
- _ = await self.control_plane.kill(spec.binding_ref)
+ _ = await _run_e2b_idempotent_operation(
+ operation="kill",
+ sandbox_id=spec.binding_ref,
+ cleanup_stage="binding_destroy",
+ action=lambda: self.control_plane.kill(spec.binding_ref),
+ )
except _E2BControlPlaneNotFoundError:
return
except Exception as exc:
@@ -381,22 +465,121 @@ async def _wait_for_shellctl_ready(client: ShellctlClientProtocol) -> None:
await asyncio.sleep(_SHELLCTL_READY_RETRY_INTERVAL_SECONDS)
+async def _run_e2b_idempotent_operation(
+ *,
+ operation: str,
+ sandbox_id: str,
+ cleanup_stage: str,
+ action: Callable[[], Awaitable[_ResultT]],
+) -> _ResultT:
+ """Retry one idempotent E2B lifecycle operation after a transport failure."""
+ for attempt in range(1, _E2B_CONTROL_PLANE_MAX_ATTEMPTS + 1):
+ try:
+ return await action()
+ except _RETRYABLE_E2B_TRANSPORT_ERRORS as exc:
+ exhausted = attempt == _E2B_CONTROL_PLANE_MAX_ATTEMPTS
+ logger.warning(
+ "E2B lifecycle operation exhausted retries" if exhausted else "retrying E2B lifecycle operation",
+ exc_info=True,
+ extra=_e2b_log_extra(
+ operation=operation,
+ sandbox_id=sandbox_id,
+ attempt=attempt,
+ cleanup_stage=cleanup_stage,
+ exception=exc,
+ outcome="retry_exhausted" if exhausted else "retrying",
+ ),
+ )
+ if exhausted:
+ raise
+ await asyncio.sleep(_E2B_CONTROL_PLANE_RETRY_INTERVAL_SECONDS)
+ raise AssertionError("unreachable")
+
+
+def _e2b_log_extra(
+ *,
+ operation: str,
+ sandbox_id: str,
+ attempt: int,
+ cleanup_stage: str,
+ exception: BaseException,
+ outcome: str,
+) -> dict[str, object]:
+ return {
+ "e2b_operation": operation,
+ "sandbox_id": sandbox_id,
+ "attempt": attempt,
+ "cleanup_stage": cleanup_stage,
+ "exception_type": type(exception).__name__,
+ "outcome": outcome,
+ }
+
+
+def _log_e2b_cleanup_warning(
+ message: str,
+ *,
+ operation: str,
+ sandbox_id: str,
+ attempt: int,
+ cleanup_stage: str,
+ exception: BaseException,
+) -> None:
+ logger.warning(
+ message,
+ exc_info=True,
+ stacklevel=2,
+ extra=_e2b_log_extra(
+ operation=operation,
+ sandbox_id=sandbox_id,
+ attempt=attempt,
+ cleanup_stage=cleanup_stage,
+ exception=exception,
+ outcome="ignored_cleanup_failure",
+ ),
+ )
+
+
+def _e2b_failure_attempt(exception: BaseException) -> int:
+ if isinstance(exception, _RETRYABLE_E2B_TRANSPORT_ERRORS):
+ return _E2B_CONTROL_PLANE_MAX_ATTEMPTS
+ return 1
+
+
async def _best_effort_close_data_plane(lease: E2BRuntimeLease | None) -> None:
if lease is None:
return
try:
await lease.data_plane.close()
- except BaseException:
- pass
+ except BaseException as exc:
+ _log_e2b_cleanup_warning(
+ "failed to close E2B RuntimeLease data plane after acquire failure",
+ operation="close_data_plane",
+ sandbox_id=lease.sandbox.sandbox_id,
+ attempt=1,
+ cleanup_stage="binding_acquire_compensation",
+ exception=exc,
+ )
async def _best_effort_pause(sandbox: _E2BSandbox | None) -> None:
if sandbox is None:
return
try:
- _ = await sandbox.pause(keep_memory=True)
- except BaseException:
- pass
+ _ = await _run_e2b_idempotent_operation(
+ operation="pause",
+ sandbox_id=sandbox.sandbox_id,
+ cleanup_stage="binding_acquire_compensation",
+ action=lambda: sandbox.pause(keep_memory=True),
+ )
+ except BaseException as exc:
+ _log_e2b_cleanup_warning(
+ "failed to pause E2B Binding after acquire failure",
+ operation="pause",
+ sandbox_id=sandbox.sandbox_id,
+ attempt=_e2b_failure_attempt(exc),
+ cleanup_stage="binding_acquire_compensation",
+ exception=exc,
+ )
__all__ = [
diff --git a/dify-agent/src/dify_agent/runtime_backend/errors.py b/dify-agent/src/dify_agent/runtime_backend/errors.py
index 627b4fb8851..c08b118b80e 100644
--- a/dify-agent/src/dify_agent/runtime_backend/errors.py
+++ b/dify-agent/src/dify_agent/runtime_backend/errors.py
@@ -21,6 +21,10 @@ class BindingCreateError(RuntimeBackendError):
pass
+class BindingCapacityExhaustedError(BindingCreateError):
+ """The selected runtime backend cannot allocate another Binding."""
+
+
class BindingAcquireError(RuntimeBackendError):
pass
@@ -47,6 +51,7 @@ class WorkspaceUnavailableError(RuntimeBackendError):
__all__ = [
"BindingAcquireError",
+ "BindingCapacityExhaustedError",
"BindingCreateError",
"BindingDestroyError",
"BindingLostError",
diff --git a/dify-agent/src/dify_agent/server/execution_bindings.py b/dify-agent/src/dify_agent/server/execution_bindings.py
index fbecb129daa..06e6ff39003 100644
--- a/dify-agent/src/dify_agent/server/execution_bindings.py
+++ b/dify-agent/src/dify_agent/server/execution_bindings.py
@@ -8,6 +8,7 @@ from dify_agent.protocol import (
DestroyExecutionBindingRequest,
)
from dify_agent.runtime_backend import (
+ BindingCapacityExhaustedError,
BindingCreateError,
BindingDestroyError,
ExecutionBindingBackend,
@@ -48,6 +49,8 @@ class ExecutionBindingService:
)
except SharedWorkspaceUnsupportedError as exc:
raise ExecutionBindingServiceError("shared_workspace_unsupported", str(exc), status_code=409) from exc
+ except BindingCapacityExhaustedError as exc:
+ raise ExecutionBindingServiceError("binding_capacity_exhausted", str(exc), status_code=429) from exc
except BindingCreateError as exc:
raise ExecutionBindingServiceError("binding_create_failed", str(exc), status_code=502) from exc
return CreateExecutionBindingResponse(
diff --git a/dify-agent/tests/integration/dify_agent/server/test_e2b_transport_soak.py b/dify-agent/tests/integration/dify_agent/server/test_e2b_transport_soak.py
new file mode 100644
index 00000000000..ff15dea6483
--- /dev/null
+++ b/dify-agent/tests/integration/dify_agent/server/test_e2b_transport_soak.py
@@ -0,0 +1,1490 @@
+"""Opt-in DIFY-2966 transport soak against a deployed E2B Agent Backend.
+
+The formal profile intentionally keeps its 32-way concurrency gate fixed. The
+``DIFY_AGENT_TEST_SOAK_*`` numeric overrides exist only for harness development;
+the JSONL ledger labels every non-default run as ``formal_gate=false``.
+
+E2B 2.38's public async API supports metadata-filtered pagination and a static
+kill-by-id operation, which lets the final reconciliation stay scoped to the
+run marker created by this test:
+https://github.com/e2b-dev/E2B/blob/%40e2b/python-sdk%402.38.0/packages/python-sdk/e2b/sandbox_async/sandbox_api.py
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import math
+import os
+import re
+import time
+from collections.abc import Awaitable, Callable, Mapping, Sequence
+from dataclasses import dataclass, replace
+from datetime import UTC, datetime
+from pathlib import Path
+from types import TracebackType
+from typing import IO, Self, cast
+
+import pytest
+
+from dify_agent.client import Client, DifyAgentHTTPError
+from dify_agent.protocol import CreateExecutionBindingRequest, DestroyExecutionBindingRequest
+
+pytestmark = pytest.mark.integration
+
+_FORMAL_CONCURRENCY = 32
+_FORMAL_BURST_ROUNDS = 10
+_FORMAL_IDLE_BINDINGS = 32
+_FORMAL_IDLE_INTERVAL_SECONDS = 600.0
+_FORMAL_IDLE_ROUNDS = 12
+_DEFAULT_REQUEST_TIMEOUT_SECONDS = 180.0
+_DEFAULT_RECONCILE_TIMEOUT_SECONDS = 60.0
+_RECONCILE_POLL_SECONDS = 2.0
+_RECONCILE_FINAL_PROOF_RESERVE_SECONDS = 10.0
+_RECONCILE_STABLE_EMPTY_SECONDS = 3.0
+_RECONCILE_FINAL_SCAN_INTERVAL_SECONDS = 0.5
+_RECONCILE_RETRY_BACKOFF_SECONDS = 0.1
+_TENANT_ID = "dify-2966-transport-soak"
+# Binding file reads intentionally reject symlinks via O_NOFOLLOW. `/etc/os-release`
+# is a symlink in the E2B image, while `/etc/hostname` is a small regular file.
+_READ_PROBE_PATH = "/etc/hostname"
+_RUN_ID_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,47}\Z")
+_REDACTED = "[REDACTED]"
+_SECRET_KEY_PARTS = ("api_key", "authorization", "password", "secret", "token")
+
+
+@dataclass(frozen=True, slots=True)
+class _SoakSettings:
+ service_url: str
+ service_token: str
+ e2b_api_key: str
+ run_id: str
+ artifact: Path
+ concurrency: int
+ burst_rounds: int
+ idle_bindings: int
+ idle_interval_seconds: float
+ idle_rounds: int
+ request_timeout_seconds: float
+ reconcile_timeout_seconds: float
+
+ @property
+ def binding_prefix(self) -> str:
+ return f"dify-2966-{self.run_id}-"
+
+ @property
+ def formal_gate(self) -> bool:
+ return (
+ self.concurrency == _FORMAL_CONCURRENCY
+ and self.burst_rounds == _FORMAL_BURST_ROUNDS
+ and self.idle_bindings == _FORMAL_IDLE_BINDINGS
+ and self.idle_interval_seconds == _FORMAL_IDLE_INTERVAL_SECONDS
+ and self.idle_rounds == _FORMAL_IDLE_ROUNDS
+ and self.request_timeout_seconds == _DEFAULT_REQUEST_TIMEOUT_SECONDS
+ and self.reconcile_timeout_seconds == _DEFAULT_RECONCILE_TIMEOUT_SECONDS
+ )
+
+
+@dataclass(frozen=True, slots=True)
+class _Binding:
+ binding_id: str
+ binding_ref: str
+ workspace_ref: str
+
+
+@dataclass(frozen=True, slots=True)
+class _OwnedSandbox:
+ sandbox_id: str
+ binding_id: str
+ workspace_id: str
+ state: str
+
+
+_ListOwnedSandboxes = Callable[[_SoakSettings], Awaitable[list[_OwnedSandbox]]]
+_KillOwnedSandbox = Callable[[_SoakSettings, str], Awaitable[bool]]
+_Sleep = Callable[[float], Awaitable[None]]
+_Monotonic = Callable[[], float]
+
+
+class _JsonlLedger:
+ """Flush one sanitized JSON object per line to an exclusive artifact."""
+
+ _handle: IO[str]
+ _secrets: tuple[str, ...]
+
+ def __init__(self, path: Path, *, secrets: Sequence[str]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ self._handle = path.open("x", encoding="utf-8")
+ self._secrets = tuple(secret for secret in secrets if secret)
+
+ def __enter__(self) -> Self:
+ return self
+
+ def __exit__(
+ self,
+ exc_type: type[BaseException] | None,
+ exc_value: BaseException | None,
+ traceback: TracebackType | None,
+ ) -> None:
+ del exc_type, exc_value, traceback
+ self._handle.close()
+
+ def record(self, event: str, **fields: object) -> None:
+ payload = {
+ "timestamp": datetime.now(UTC).isoformat(),
+ "event": event,
+ **fields,
+ }
+ sanitized = _sanitize_for_ledger(payload, secrets=self._secrets)
+ _ = self._handle.write(json.dumps(sanitized, separators=(",", ":"), sort_keys=True) + "\n")
+ self._handle.flush()
+
+
+def _sanitize_for_ledger(value: object, *, secrets: Sequence[str], key: str | None = None) -> object:
+ if key is not None and any(part in key.lower() for part in _SECRET_KEY_PARTS):
+ return _REDACTED
+ if isinstance(value, Mapping):
+ mapping = cast(Mapping[object, object], value)
+ return {
+ str(item_key): _sanitize_for_ledger(item, secrets=secrets, key=str(item_key))
+ for item_key, item in mapping.items()
+ }
+ if isinstance(value, list | tuple):
+ sequence = cast(Sequence[object], value)
+ return [_sanitize_for_ledger(item, secrets=secrets) for item in sequence]
+ if isinstance(value, str):
+ sanitized = value
+ for secret in secrets:
+ if sanitized == secret:
+ return _REDACTED
+ sanitized = sanitized.replace(secret, _REDACTED)
+ return sanitized
+ if value is None or isinstance(value, bool | int | float):
+ return value
+ return str(value)
+
+
+def _load_settings(env: Mapping[str, str]) -> _SoakSettings:
+ if env.get("DIFY_AGENT_TEST_SOAK_ENABLED", "").strip() != "1":
+ pytest.skip("set DIFY_AGENT_TEST_SOAK_ENABLED=1 to run the DIFY-2966 staging soak")
+
+ required_names = (
+ "DIFY_AGENT_TEST_SERVICE_URL",
+ "DIFY_AGENT_TEST_SERVICE_API_TOKEN",
+ "DIFY_AGENT_TEST_E2B_API_KEY",
+ "DIFY_AGENT_TEST_SOAK_RUN_ID",
+ "DIFY_AGENT_TEST_SOAK_ARTIFACT",
+ )
+ missing = [name for name in required_names if not env.get(name, "").strip()]
+ if missing:
+ pytest.skip(f"set {', '.join(missing)} to run the DIFY-2966 staging soak")
+
+ run_id = env["DIFY_AGENT_TEST_SOAK_RUN_ID"].strip()
+ if _RUN_ID_PATTERN.fullmatch(run_id) is None:
+ raise ValueError("DIFY_AGENT_TEST_SOAK_RUN_ID must be 1-48 ASCII letters, digits, dots, underscores, or dashes")
+
+ return _SoakSettings(
+ service_url=env["DIFY_AGENT_TEST_SERVICE_URL"].strip(),
+ service_token=env["DIFY_AGENT_TEST_SERVICE_API_TOKEN"].strip(),
+ e2b_api_key=env["DIFY_AGENT_TEST_E2B_API_KEY"].strip(),
+ run_id=run_id,
+ artifact=Path(env["DIFY_AGENT_TEST_SOAK_ARTIFACT"].strip()).expanduser(),
+ concurrency=_positive_int(env, "DIFY_AGENT_TEST_SOAK_CONCURRENCY", _FORMAL_CONCURRENCY),
+ burst_rounds=_positive_int(env, "DIFY_AGENT_TEST_SOAK_BURST_ROUNDS", _FORMAL_BURST_ROUNDS),
+ idle_bindings=_positive_int(env, "DIFY_AGENT_TEST_SOAK_IDLE_BINDINGS", _FORMAL_IDLE_BINDINGS),
+ idle_interval_seconds=_nonnegative_float(
+ env,
+ "DIFY_AGENT_TEST_SOAK_IDLE_INTERVAL_SECONDS",
+ _FORMAL_IDLE_INTERVAL_SECONDS,
+ ),
+ idle_rounds=_positive_int(env, "DIFY_AGENT_TEST_SOAK_IDLE_ROUNDS", _FORMAL_IDLE_ROUNDS),
+ request_timeout_seconds=_positive_float(
+ env,
+ "DIFY_AGENT_TEST_SOAK_REQUEST_TIMEOUT_SECONDS",
+ _DEFAULT_REQUEST_TIMEOUT_SECONDS,
+ ),
+ reconcile_timeout_seconds=_positive_float(
+ env,
+ "DIFY_AGENT_TEST_SOAK_RECONCILE_TIMEOUT_SECONDS",
+ _DEFAULT_RECONCILE_TIMEOUT_SECONDS,
+ ),
+ )
+
+
+def _positive_int(env: Mapping[str, str], name: str, default: int) -> int:
+ raw = env.get(name, "").strip()
+ value = int(raw) if raw else default
+ if value <= 0:
+ raise ValueError(f"{name} must be greater than zero")
+ return value
+
+
+def _positive_float(env: Mapping[str, str], name: str, default: float) -> float:
+ value = _nonnegative_float(env, name, default)
+ if value == 0:
+ raise ValueError(f"{name} must be greater than zero")
+ return value
+
+
+def _nonnegative_float(env: Mapping[str, str], name: str, default: float) -> float:
+ raw = env.get(name, "").strip()
+ value = float(raw) if raw else default
+ if not math.isfinite(value) or value < 0:
+ raise ValueError(f"{name} must be finite and nonnegative")
+ return value
+
+
+def _client(settings: _SoakSettings) -> Client:
+ # One Client per concurrent lifecycle produces independent HTTP connections
+ # so a multi-worker deployment is not accidentally exercised through only
+ # one persistent downstream connection.
+ return Client(
+ base_url=settings.service_url,
+ timeout=settings.request_timeout_seconds,
+ headers={"Authorization": f"Bearer {settings.service_token}"},
+ )
+
+
+def _http_status(exc: BaseException) -> int | None:
+ return exc.status_code if isinstance(exc, DifyAgentHTTPError) else None
+
+
+def _record_operation_failure(
+ ledger: _JsonlLedger,
+ *,
+ operation: str,
+ started_at: float,
+ exc: BaseException,
+ binding: _Binding | None = None,
+ binding_id: str | None = None,
+ cleanup_result: str | None = None,
+) -> None:
+ ledger.record(
+ "operation_end",
+ operation=operation,
+ outcome="failed",
+ binding_id=binding.binding_id if binding is not None else binding_id,
+ binding_ref=binding.binding_ref if binding is not None else None,
+ workspace_ref=binding.workspace_ref if binding is not None else None,
+ http_status=_http_status(exc),
+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),
+ exception_type=type(exc).__name__,
+ cleanup_result=cleanup_result,
+ )
+
+
+async def _create_binding(settings: _SoakSettings, ledger: _JsonlLedger, binding_id: str) -> _Binding:
+ started_at = time.perf_counter()
+ ledger.record("operation_start", operation="create", binding_id=binding_id)
+ try:
+ async with _client(settings) as client:
+ allocation = await client.create_execution_binding(
+ CreateExecutionBindingRequest(
+ tenant_id=_TENANT_ID,
+ agent_id=settings.binding_prefix,
+ binding_id=binding_id,
+ workspace_id=binding_id,
+ existing_workspace_ref=None,
+ home_snapshot_ref=None,
+ )
+ )
+ except BaseException as exc:
+ _record_operation_failure(ledger, operation="create", started_at=started_at, exc=exc, binding_id=binding_id)
+ raise
+
+ binding = _Binding(
+ binding_id=binding_id,
+ binding_ref=allocation.binding_ref,
+ workspace_ref=allocation.workspace_ref,
+ )
+ ledger.record(
+ "operation_end",
+ operation="create",
+ outcome="succeeded",
+ binding_id=binding.binding_id,
+ binding_ref=binding.binding_ref,
+ workspace_ref=binding.workspace_ref,
+ http_status=201,
+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),
+ exception_type=None,
+ cleanup_result=None,
+ )
+ return binding
+
+
+async def _probe_binding(settings: _SoakSettings, ledger: _JsonlLedger, binding: _Binding) -> None:
+ async with _client(settings) as client:
+ started_at = time.perf_counter()
+ ledger.record(
+ "operation_start",
+ operation="list",
+ binding_id=binding.binding_id,
+ binding_ref=binding.binding_ref,
+ workspace_ref=binding.workspace_ref,
+ )
+ try:
+ listing = await client.list_binding_files(binding.binding_ref, ".")
+ assert listing.path == "."
+ except BaseException as exc:
+ _record_operation_failure(ledger, operation="list", started_at=started_at, exc=exc, binding=binding)
+ raise
+ ledger.record(
+ "operation_end",
+ operation="list",
+ outcome="succeeded",
+ binding_id=binding.binding_id,
+ binding_ref=binding.binding_ref,
+ workspace_ref=binding.workspace_ref,
+ http_status=200,
+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),
+ exception_type=None,
+ cleanup_result=None,
+ )
+
+ started_at = time.perf_counter()
+ ledger.record(
+ "operation_start",
+ operation="read",
+ binding_id=binding.binding_id,
+ binding_ref=binding.binding_ref,
+ workspace_ref=binding.workspace_ref,
+ )
+ try:
+ preview = await client.read_binding_file(binding.binding_ref, _READ_PROBE_PATH)
+ assert preview.path == _READ_PROBE_PATH
+ assert not preview.binary
+ assert preview.text
+ except BaseException as exc:
+ _record_operation_failure(ledger, operation="read", started_at=started_at, exc=exc, binding=binding)
+ raise
+ ledger.record(
+ "operation_end",
+ operation="read",
+ outcome="succeeded",
+ binding_id=binding.binding_id,
+ binding_ref=binding.binding_ref,
+ workspace_ref=binding.workspace_ref,
+ http_status=200,
+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),
+ exception_type=None,
+ cleanup_result=None,
+ )
+
+
+async def _destroy_binding(
+ settings: _SoakSettings,
+ ledger: _JsonlLedger,
+ binding: _Binding,
+ *,
+ cleanup: bool,
+) -> None:
+ operation = "cleanup_destroy" if cleanup else "destroy"
+ started_at = time.perf_counter()
+ ledger.record(
+ "operation_start",
+ operation=operation,
+ binding_id=binding.binding_id,
+ binding_ref=binding.binding_ref,
+ workspace_ref=binding.workspace_ref,
+ )
+ try:
+ async with _client(settings) as client:
+ await client.destroy_execution_binding(
+ DestroyExecutionBindingRequest(
+ binding_ref=binding.binding_ref,
+ workspace_ref=binding.workspace_ref,
+ destroy_workspace=True,
+ )
+ )
+ except BaseException as exc:
+ _record_operation_failure(
+ ledger,
+ operation=operation,
+ started_at=started_at,
+ exc=exc,
+ binding=binding,
+ cleanup_result="failed" if cleanup else None,
+ )
+ raise
+ ledger.record(
+ "operation_end",
+ operation=operation,
+ outcome="succeeded",
+ binding_id=binding.binding_id,
+ binding_ref=binding.binding_ref,
+ workspace_ref=binding.workspace_ref,
+ http_status=204,
+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),
+ exception_type=None,
+ cleanup_result="destroyed" if cleanup else None,
+ )
+
+
+def _raise_task_failures(results: Sequence[object], *, message: str) -> None:
+ failures: list[Exception] = []
+ for result in results:
+ if not isinstance(result, BaseException):
+ continue
+ if not isinstance(result, Exception):
+ raise result
+ failures.append(result)
+ if failures:
+ raise ExceptionGroup(message, failures)
+
+
+async def _gather_checked(awaitables: Sequence[Awaitable[None]], *, message: str) -> None:
+ results = await asyncio.gather(*awaitables, return_exceptions=True)
+ _raise_task_failures(results, message=message)
+
+
+async def _run_burst(
+ settings: _SoakSettings,
+ ledger: _JsonlLedger,
+ known: dict[str, _Binding],
+ destroyed: set[str],
+) -> None:
+ for round_number in range(settings.burst_rounds):
+ ledger.record("stage_start", stage="burst_round", round=round_number, count=settings.concurrency)
+
+ async def run_one(sequence: int) -> None:
+ binding_id = f"{settings.binding_prefix}burst-{round_number:02d}-{sequence:04d}"
+ binding = await _create_binding(settings, ledger, binding_id)
+ known[binding.binding_ref] = binding
+ await _probe_binding(settings, ledger, binding)
+ await _destroy_binding(settings, ledger, binding, cleanup=False)
+ destroyed.add(binding.binding_ref)
+
+ try:
+ await _gather_checked(
+ [run_one(sequence) for sequence in range(settings.concurrency)],
+ message=f"burst round {round_number} failed",
+ )
+ except BaseException as exc:
+ ledger.record(
+ "stage_end",
+ stage="burst_round",
+ round=round_number,
+ count=settings.concurrency,
+ outcome="failed",
+ exception_type=type(exc).__name__,
+ )
+ raise
+ ledger.record(
+ "stage_end",
+ stage="burst_round",
+ round=round_number,
+ count=settings.concurrency,
+ outcome="succeeded",
+ exception_type=None,
+ )
+
+
+async def _run_idle(
+ settings: _SoakSettings,
+ ledger: _JsonlLedger,
+ known: dict[str, _Binding],
+ destroyed: set[str],
+) -> None:
+ bindings_by_sequence: dict[int, _Binding] = {}
+ ledger.record("stage_start", stage="idle_create", count=settings.idle_bindings)
+
+ async def create_one(sequence: int) -> None:
+ binding_id = f"{settings.binding_prefix}idle-{sequence:04d}"
+ binding = await _create_binding(settings, ledger, binding_id)
+ bindings_by_sequence[sequence] = binding
+ known[binding.binding_ref] = binding
+
+ bindings: list[_Binding] = []
+ try:
+ await _gather_checked(
+ [create_one(sequence) for sequence in range(settings.idle_bindings)],
+ message="idle binding creation failed; do not reduce the configured capacity gate",
+ )
+ bindings = [bindings_by_sequence[index] for index in range(settings.idle_bindings)]
+ await _require_exact_idle_inventory(settings, ledger, bindings, stage="idle_create_state")
+ except BaseException as exc:
+ ledger.record(
+ "stage_end",
+ stage="idle_create",
+ count=settings.idle_bindings,
+ outcome="failed",
+ exception_type=type(exc).__name__,
+ )
+ raise
+ ledger.record(
+ "stage_end",
+ stage="idle_create",
+ count=settings.idle_bindings,
+ outcome="succeeded",
+ exception_type=None,
+ )
+
+ for round_number in range(settings.idle_rounds):
+ ledger.record(
+ "stage_start",
+ stage="idle_wait",
+ round=round_number,
+ duration_seconds=settings.idle_interval_seconds,
+ )
+ await asyncio.sleep(settings.idle_interval_seconds)
+ ledger.record(
+ "stage_end",
+ stage="idle_wait",
+ round=round_number,
+ duration_seconds=settings.idle_interval_seconds,
+ outcome="succeeded",
+ )
+
+ ledger.record("stage_start", stage="idle_probe", round=round_number, count=len(bindings))
+ try:
+ await _gather_checked(
+ [_probe_binding(settings, ledger, binding) for binding in bindings],
+ message=f"idle probe round {round_number} failed",
+ )
+ await _require_exact_idle_inventory(
+ settings,
+ ledger,
+ bindings,
+ stage=f"idle_probe_{round_number:02d}_state",
+ )
+ except BaseException as exc:
+ ledger.record(
+ "stage_end",
+ stage="idle_probe",
+ round=round_number,
+ count=len(bindings),
+ outcome="failed",
+ exception_type=type(exc).__name__,
+ )
+ raise
+ ledger.record(
+ "stage_end",
+ stage="idle_probe",
+ round=round_number,
+ count=len(bindings),
+ outcome="succeeded",
+ exception_type=None,
+ )
+
+ ledger.record("stage_start", stage="idle_destroy", count=len(bindings))
+
+ async def destroy_one(binding: _Binding) -> None:
+ await _destroy_binding(settings, ledger, binding, cleanup=False)
+ destroyed.add(binding.binding_ref)
+
+ try:
+ await _gather_checked(
+ [destroy_one(binding) for binding in bindings],
+ message="idle binding destruction failed",
+ )
+ except BaseException as exc:
+ ledger.record(
+ "stage_end",
+ stage="idle_destroy",
+ count=len(bindings),
+ outcome="failed",
+ exception_type=type(exc).__name__,
+ )
+ raise
+ ledger.record(
+ "stage_end",
+ stage="idle_destroy",
+ count=len(bindings),
+ outcome="succeeded",
+ exception_type=None,
+ )
+
+
+async def _cleanup_known_bindings(
+ settings: _SoakSettings,
+ ledger: _JsonlLedger,
+ known: Mapping[str, _Binding],
+ destroyed: set[str],
+) -> list[Exception]:
+ pending = [binding for binding in known.values() if binding.binding_ref not in destroyed]
+ if not pending:
+ return []
+ results = await asyncio.gather(
+ *[_destroy_binding(settings, ledger, binding, cleanup=True) for binding in pending],
+ return_exceptions=True,
+ )
+ errors: list[Exception] = []
+ for binding, result in zip(pending, results, strict=True):
+ if isinstance(result, BaseException):
+ if not isinstance(result, Exception):
+ raise result
+ errors.append(result)
+ else:
+ destroyed.add(binding.binding_ref)
+ return errors
+
+
+async def _list_owned_e2b_sandboxes(settings: _SoakSettings) -> list[_OwnedSandbox]:
+ from e2b import AsyncSandbox, SandboxInfo, SandboxQuery
+
+ paginator = AsyncSandbox.list(
+ query=SandboxQuery(
+ metadata={
+ "dify.agent_id": settings.binding_prefix,
+ "dify.tenant_id": _TENANT_ID,
+ }
+ ),
+ limit=100,
+ api_key=settings.e2b_api_key,
+ )
+ candidates: list[SandboxInfo] = []
+ while paginator.has_next:
+ candidates.extend(await paginator.next_items())
+
+ owned: list[_OwnedSandbox] = []
+ for candidate in candidates:
+ metadata = candidate.metadata
+ binding_id = metadata.get("dify.binding_id", "")
+ workspace_id = metadata.get("dify.workspace_id", "")
+ if not binding_id.startswith(settings.binding_prefix) or not workspace_id.startswith(settings.binding_prefix):
+ continue
+ state = candidate.state.value
+ owned.append(
+ _OwnedSandbox(
+ sandbox_id=candidate.sandbox_id,
+ binding_id=binding_id,
+ workspace_id=workspace_id,
+ state=str(state),
+ )
+ )
+ return sorted(owned, key=lambda sandbox: sandbox.sandbox_id)
+
+
+def _inventory_identity(sandboxes: Sequence[_OwnedSandbox]) -> list[tuple[str, str, str, str]]:
+ return sorted(
+ (sandbox.sandbox_id, sandbox.binding_id, sandbox.workspace_id, sandbox.state) for sandbox in sandboxes
+ )
+
+
+def _expected_idle_identity(bindings: Sequence[_Binding]) -> list[tuple[str, str, str, str]]:
+ return sorted((binding.binding_ref, binding.binding_id, binding.binding_id, "paused") for binding in bindings)
+
+
+def _record_inventory(
+ ledger: _JsonlLedger,
+ *,
+ stage: str,
+ sandboxes: Sequence[_OwnedSandbox],
+ attempt: int | None = None,
+) -> None:
+ ledger.record("e2b_inventory", stage=stage, attempt=attempt, count=len(sandboxes))
+ for sandbox in sandboxes:
+ ledger.record(
+ "e2b_inventory_item",
+ stage=stage,
+ attempt=attempt,
+ sandbox_id=sandbox.sandbox_id,
+ binding_id=sandbox.binding_id,
+ workspace_id=sandbox.workspace_id,
+ state=sandbox.state,
+ )
+
+
+async def _list_owned_inventory_attempt(
+ settings: _SoakSettings,
+ ledger: _JsonlLedger,
+ *,
+ list_owned: _ListOwnedSandboxes,
+ stage: str,
+ attempt: int,
+ timeout_seconds: float,
+) -> tuple[list[_OwnedSandbox] | None, Exception | None]:
+ started_at = time.perf_counter()
+ ledger.record(
+ "operation_start",
+ operation="e2b_inventory_list",
+ stage=stage,
+ attempt=attempt,
+ )
+ try:
+ inventory = await asyncio.wait_for(list_owned(settings), timeout=timeout_seconds)
+ except Exception as exc:
+ ledger.record(
+ "operation_end",
+ operation="e2b_inventory_list",
+ stage=stage,
+ attempt=attempt,
+ outcome="failed",
+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),
+ exception_type=type(exc).__name__,
+ cleanup_result=None,
+ )
+ return None, exc
+ ledger.record(
+ "operation_end",
+ operation="e2b_inventory_list",
+ stage=stage,
+ attempt=attempt,
+ outcome="succeeded",
+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),
+ exception_type=None,
+ cleanup_result=None,
+ )
+ return inventory, None
+
+
+async def _require_exact_idle_inventory(
+ settings: _SoakSettings,
+ ledger: _JsonlLedger,
+ bindings: Sequence[_Binding],
+ *,
+ stage: str = "idle_identity",
+ list_owned: _ListOwnedSandboxes = _list_owned_e2b_sandboxes,
+ sleep: _Sleep = asyncio.sleep,
+ monotonic: _Monotonic = time.monotonic,
+) -> None:
+ """Require the exact run-owned Bindings to be visibly paused before continuing."""
+ if any(binding.binding_ref != binding.workspace_ref for binding in bindings):
+ ledger.record(
+ "e2b_identity_check",
+ stage=stage,
+ outcome="failed",
+ expected_count=len(bindings),
+ actual_count=None,
+ exception_type="BackendIdentityMismatch",
+ )
+ raise AssertionError("E2B idle bindings must return identical Binding and Workspace refs")
+
+ expected = _expected_idle_identity(bindings)
+ deadline = monotonic() + settings.reconcile_timeout_seconds
+ attempt = 0
+ last_list_error: Exception | None = None
+ last_observed_count: int | None = None
+ while True:
+ remaining_seconds = deadline - monotonic()
+ if remaining_seconds <= 0:
+ if last_list_error is not None:
+ raise AssertionError(
+ "E2B idle inventory could not be proven before the reconcile deadline"
+ ) from last_list_error
+ ledger.record(
+ "e2b_identity_check",
+ stage=stage,
+ attempt=attempt,
+ outcome="failed",
+ expected_count=len(expected),
+ actual_count=last_observed_count,
+ exception_type="BackendIdentityMismatch",
+ )
+ raise AssertionError(
+ f"E2B idle inventory mismatch: expected {len(expected)} exact, observed {last_observed_count}"
+ )
+ attempt += 1
+ actual_sandboxes, list_error = await _list_owned_inventory_attempt(
+ settings,
+ ledger,
+ list_owned=list_owned,
+ stage=stage,
+ attempt=attempt,
+ timeout_seconds=remaining_seconds,
+ )
+ if list_error is not None:
+ last_list_error = list_error
+ remaining_seconds = deadline - monotonic()
+ if remaining_seconds <= 0:
+ raise AssertionError(
+ "E2B idle inventory could not be proven before the reconcile deadline"
+ ) from last_list_error
+ await sleep(min(_RECONCILE_RETRY_BACKOFF_SECONDS, remaining_seconds))
+ continue
+ assert actual_sandboxes is not None
+ last_list_error = None
+ actual = _inventory_identity(actual_sandboxes)
+ last_observed_count = len(actual)
+ _record_inventory(ledger, stage=stage, sandboxes=actual_sandboxes, attempt=attempt)
+ if actual == expected:
+ ledger.record(
+ "e2b_identity_check",
+ stage=stage,
+ attempt=attempt,
+ outcome="succeeded",
+ expected_count=len(expected),
+ actual_count=len(actual),
+ exception_type=None,
+ )
+ return
+ if monotonic() >= deadline:
+ ledger.record(
+ "e2b_identity_check",
+ stage=stage,
+ attempt=attempt,
+ outcome="failed",
+ expected_count=len(expected),
+ actual_count=len(actual),
+ exception_type="BackendIdentityMismatch",
+ )
+ raise AssertionError(
+ f"E2B idle inventory mismatch: expected {len(expected)} exact allocations, observed {len(actual)}"
+ )
+ await sleep(min(_RECONCILE_POLL_SECONDS, max(0.0, deadline - monotonic())))
+
+
+async def _kill_owned_e2b_sandbox(settings: _SoakSettings, sandbox_id: str) -> bool:
+ from e2b import AsyncSandbox
+
+ return await AsyncSandbox.kill(sandbox_id, api_key=settings.e2b_api_key)
+
+
+async def _reconcile_owned_e2b_sandboxes(
+ settings: _SoakSettings,
+ ledger: _JsonlLedger,
+ *,
+ list_owned: _ListOwnedSandboxes = _list_owned_e2b_sandboxes,
+ kill_owned: _KillOwnedSandbox = _kill_owned_e2b_sandbox,
+ sleep: _Sleep = asyncio.sleep,
+ monotonic: _Monotonic = time.monotonic,
+) -> None:
+ deadline = monotonic() + settings.reconcile_timeout_seconds
+ final_proof_reserve = min(
+ _RECONCILE_FINAL_PROOF_RESERVE_SECONDS,
+ settings.reconcile_timeout_seconds / 2,
+ )
+ scan_deadline = deadline - final_proof_reserve
+ attempt = 0
+ last_list_error: Exception | None = None
+ last_inventory: list[_OwnedSandbox] | None = None
+
+ async def kill_one(sandbox: _OwnedSandbox, *, attempt: int) -> None:
+ started_at = time.perf_counter()
+ ledger.record(
+ "operation_start",
+ operation="e2b_direct_kill",
+ attempt=attempt,
+ sandbox_id=sandbox.sandbox_id,
+ binding_id=sandbox.binding_id,
+ workspace_id=sandbox.workspace_id,
+ )
+ try:
+ killed = await kill_owned(settings, sandbox.sandbox_id)
+ except Exception as exc:
+ ledger.record(
+ "operation_end",
+ operation="e2b_direct_kill",
+ attempt=attempt,
+ outcome="failed",
+ sandbox_id=sandbox.sandbox_id,
+ binding_id=sandbox.binding_id,
+ workspace_id=sandbox.workspace_id,
+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),
+ exception_type=type(exc).__name__,
+ cleanup_result="failed",
+ )
+ return
+ except asyncio.CancelledError:
+ ledger.record(
+ "operation_end",
+ operation="e2b_direct_kill",
+ attempt=attempt,
+ outcome="failed",
+ sandbox_id=sandbox.sandbox_id,
+ binding_id=sandbox.binding_id,
+ workspace_id=sandbox.workspace_id,
+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),
+ exception_type="CancelledError",
+ cleanup_result="cancelled",
+ )
+ raise
+ ledger.record(
+ "operation_end",
+ operation="e2b_direct_kill",
+ attempt=attempt,
+ outcome="succeeded",
+ sandbox_id=sandbox.sandbox_id,
+ binding_id=sandbox.binding_id,
+ workspace_id=sandbox.workspace_id,
+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),
+ exception_type=None,
+ cleanup_result="killed" if killed else "already_absent",
+ )
+
+ async def kill_batch(
+ sandboxes: Sequence[_OwnedSandbox],
+ *,
+ attempt: int,
+ timeout_seconds: float,
+ ) -> None:
+ started_at = time.perf_counter()
+ ledger.record(
+ "operation_start",
+ operation="e2b_direct_kill_batch",
+ attempt=attempt,
+ count=len(sandboxes),
+ )
+ try:
+ _ = await asyncio.wait_for(
+ asyncio.gather(*[kill_one(sandbox, attempt=attempt) for sandbox in sandboxes]),
+ timeout=timeout_seconds,
+ )
+ except TimeoutError:
+ ledger.record(
+ "operation_end",
+ operation="e2b_direct_kill_batch",
+ attempt=attempt,
+ outcome="failed",
+ count=len(sandboxes),
+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),
+ exception_type="TimeoutError",
+ cleanup_result="deadline_exhausted",
+ )
+ return
+ ledger.record(
+ "operation_end",
+ operation="e2b_direct_kill_batch",
+ attempt=attempt,
+ outcome="completed",
+ count=len(sandboxes),
+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),
+ exception_type=None,
+ cleanup_result="attempted",
+ )
+
+ while monotonic() < scan_deadline:
+ remaining_seconds = scan_deadline - monotonic()
+ if remaining_seconds <= 0:
+ break
+ attempt += 1
+ current, list_error = await _list_owned_inventory_attempt(
+ settings,
+ ledger,
+ list_owned=list_owned,
+ stage="direct_cleanup_scan",
+ attempt=attempt,
+ timeout_seconds=remaining_seconds,
+ )
+ if list_error is not None:
+ last_list_error = list_error
+ else:
+ assert current is not None
+ last_list_error = None
+ last_inventory = current
+ _record_inventory(ledger, stage="direct_cleanup_scan", sandboxes=current, attempt=attempt)
+ kill_remaining = scan_deadline - monotonic()
+ if current and kill_remaining > 0:
+ await kill_batch(current, attempt=attempt, timeout_seconds=kill_remaining)
+
+ remaining_seconds = scan_deadline - monotonic()
+ if remaining_seconds <= 0:
+ break
+ await sleep(min(_RECONCILE_POLL_SECONDS, remaining_seconds))
+
+ final_inventory: list[_OwnedSandbox] | None = None
+ stable_empty_since: float | None = None
+ while monotonic() < deadline:
+ remaining_seconds = deadline - monotonic()
+ if remaining_seconds <= 0:
+ break
+ attempt += 1
+ current, list_error = await _list_owned_inventory_attempt(
+ settings,
+ ledger,
+ list_owned=list_owned,
+ stage="direct_cleanup_final_scan",
+ attempt=attempt,
+ timeout_seconds=min(_RECONCILE_POLL_SECONDS, remaining_seconds),
+ )
+ if list_error is not None:
+ last_list_error = list_error
+ stable_empty_since = None
+ else:
+ assert current is not None
+ last_list_error = None
+ last_inventory = current
+ _record_inventory(ledger, stage="direct_cleanup_final_scan", sandboxes=current, attempt=attempt)
+ if current:
+ stable_empty_since = None
+ kill_remaining = deadline - monotonic()
+ if kill_remaining > 0:
+ await kill_batch(
+ current,
+ attempt=attempt,
+ timeout_seconds=min(_RECONCILE_POLL_SECONDS, kill_remaining),
+ )
+ else:
+ observed_at = monotonic()
+ if stable_empty_since is None:
+ stable_empty_since = observed_at
+ stable_empty_seconds = observed_at - stable_empty_since
+ if stable_empty_seconds >= _RECONCILE_STABLE_EMPTY_SECONDS:
+ ledger.record(
+ "e2b_stable_empty",
+ stage="direct_cleanup_final_scan",
+ attempt=attempt,
+ outcome="succeeded",
+ duration_seconds=stable_empty_seconds,
+ )
+ final_inventory = current
+ break
+
+ remaining_seconds = deadline - monotonic()
+ if remaining_seconds <= 0:
+ break
+ await sleep(min(_RECONCILE_FINAL_SCAN_INTERVAL_SECONDS, remaining_seconds))
+
+ if final_inventory is None:
+ if last_list_error is not None:
+ raise AssertionError("final E2B inventory proof failed before the reconcile deadline") from last_list_error
+ remaining_count = len(last_inventory) if last_inventory is not None else 0
+ if remaining_count:
+ raise AssertionError(
+ f"final E2B inventory proof was not empty before the reconcile deadline ({remaining_count} observed)"
+ )
+ raise AssertionError(
+ f"final E2B inventory did not remain empty for {_RECONCILE_STABLE_EMPTY_SECONDS:g}s before the deadline"
+ )
+
+ _record_inventory(
+ ledger,
+ stage="direct_cleanup_deadline",
+ sandboxes=final_inventory,
+ attempt=attempt,
+ )
+ _record_inventory(ledger, stage="after_direct_cleanup", sandboxes=final_inventory, attempt=attempt)
+ assert not final_inventory, f"{len(final_inventory)} run-owned E2B sandboxes remain after cleanup"
+
+
+def _raise_collected_errors(primary: BaseException | None, cleanup_errors: Sequence[Exception]) -> None:
+ if primary is not None and not isinstance(primary, Exception):
+ raise primary
+ errors = ([primary] if isinstance(primary, Exception) else []) + list(cleanup_errors)
+ if len(errors) == 1:
+ raise errors[0]
+ if errors:
+ raise ExceptionGroup("DIFY-2966 soak and cleanup failures", errors)
+
+
+async def test_e2b_transport_soak() -> None:
+ settings = _load_settings(os.environ)
+ primary_error: BaseException | None = None
+ cleanup_errors: list[Exception] = []
+ known: dict[str, _Binding] = {}
+ destroyed: set[str] = set()
+ cleanup_authorized = False
+
+ with _JsonlLedger(
+ settings.artifact,
+ secrets=(settings.service_token, settings.e2b_api_key),
+ ) as ledger:
+ ledger.record(
+ "run_start",
+ run_id=settings.run_id,
+ binding_prefix=settings.binding_prefix,
+ formal_gate=settings.formal_gate,
+ concurrency=settings.concurrency,
+ burst_rounds=settings.burst_rounds,
+ idle_bindings=settings.idle_bindings,
+ idle_interval_seconds=settings.idle_interval_seconds,
+ idle_rounds=settings.idle_rounds,
+ )
+
+ initial = await _list_owned_e2b_sandboxes(settings)
+ _record_inventory(ledger, stage="preflight", sandboxes=initial)
+ assert not initial, "DIFY_AGENT_TEST_SOAK_RUN_ID is already in use; choose a new run id"
+ cleanup_authorized = True
+
+ try:
+ await _run_burst(settings, ledger, known, destroyed)
+ await _run_idle(settings, ledger, known, destroyed)
+ except BaseException as exc:
+ primary_error = exc
+ finally:
+ if cleanup_authorized:
+ try:
+ cleanup_errors.extend(await _cleanup_known_bindings(settings, ledger, known, destroyed))
+ except BaseException as exc:
+ if not isinstance(exc, Exception):
+ raise
+ cleanup_errors.append(exc)
+ try:
+ await _reconcile_owned_e2b_sandboxes(settings, ledger)
+ except BaseException as exc:
+ if not isinstance(exc, Exception):
+ raise
+ cleanup_errors.append(exc)
+
+ outcome = "succeeded" if primary_error is None and not cleanup_errors else "failed"
+ ledger.record(
+ "run_end",
+ run_id=settings.run_id,
+ formal_gate=settings.formal_gate,
+ outcome=outcome,
+ exception_type=type(primary_error).__name__ if primary_error is not None else None,
+ cleanup_error_types=[type(error).__name__ for error in cleanup_errors],
+ )
+
+ _raise_collected_errors(primary_error, cleanup_errors)
+
+
+def test_soak_requires_explicit_enable(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delenv("DIFY_AGENT_TEST_SOAK_ENABLED", raising=False)
+ with pytest.raises(pytest.skip.Exception, match="DIFY_AGENT_TEST_SOAK_ENABLED=1"):
+ _ = _load_settings(os.environ)
+
+
+def test_soak_missing_credentials_skip_without_echoing_values(monkeypatch: pytest.MonkeyPatch) -> None:
+ for name in (
+ "DIFY_AGENT_TEST_SERVICE_URL",
+ "DIFY_AGENT_TEST_E2B_API_KEY",
+ "DIFY_AGENT_TEST_SOAK_RUN_ID",
+ "DIFY_AGENT_TEST_SOAK_ARTIFACT",
+ ):
+ monkeypatch.delenv(name, raising=False)
+ monkeypatch.setenv("DIFY_AGENT_TEST_SOAK_ENABLED", "1")
+ monkeypatch.setenv("DIFY_AGENT_TEST_SERVICE_API_TOKEN", "service-secret-value")
+ with pytest.raises(pytest.skip.Exception) as raised:
+ _ = _load_settings(os.environ)
+ assert "service-secret-value" not in str(raised.value)
+
+
+def test_ledger_redacts_secret_fields_and_values(tmp_path: Path) -> None:
+ service_token = "service-secret-value"
+ e2b_key = "e2b-secret-value"
+ artifact = tmp_path / "ledger.jsonl"
+
+ with _JsonlLedger(artifact, secrets=(service_token, e2b_key)) as ledger:
+ ledger.record(
+ "probe",
+ authorization=service_token,
+ nested={"api_key": e2b_key, "message": f"failed near {service_token}"},
+ )
+
+ raw = artifact.read_text(encoding="utf-8")
+ payload = cast(dict[str, object], json.loads(raw))
+ assert service_token not in raw
+ assert e2b_key not in raw
+ assert payload["authorization"] == _REDACTED
+ nested = cast(dict[str, object], payload["nested"])
+ assert nested["api_key"] == _REDACTED
+ assert nested["message"] == f"failed near {_REDACTED}"
+
+
+def test_formal_gate_includes_request_and_reconcile_timeouts(tmp_path: Path) -> None:
+ settings = _SoakSettings(
+ service_url="https://agent.invalid",
+ service_token="service-secret-value",
+ e2b_api_key="e2b-secret-value",
+ run_id="formal",
+ artifact=tmp_path / "formal.jsonl",
+ concurrency=_FORMAL_CONCURRENCY,
+ burst_rounds=_FORMAL_BURST_ROUNDS,
+ idle_bindings=_FORMAL_IDLE_BINDINGS,
+ idle_interval_seconds=_FORMAL_IDLE_INTERVAL_SECONDS,
+ idle_rounds=_FORMAL_IDLE_ROUNDS,
+ request_timeout_seconds=_DEFAULT_REQUEST_TIMEOUT_SECONDS,
+ reconcile_timeout_seconds=_DEFAULT_RECONCILE_TIMEOUT_SECONDS,
+ )
+
+ assert settings.formal_gate
+ assert not replace(settings, request_timeout_seconds=1).formal_gate
+ assert not replace(settings, reconcile_timeout_seconds=1).formal_gate
+
+
+def test_reconcile_timeout_override_must_be_positive(tmp_path: Path) -> None:
+ env = {
+ "DIFY_AGENT_TEST_SOAK_ENABLED": "1",
+ "DIFY_AGENT_TEST_SERVICE_URL": "https://agent.invalid",
+ "DIFY_AGENT_TEST_SERVICE_API_TOKEN": "service-secret-value",
+ "DIFY_AGENT_TEST_E2B_API_KEY": "e2b-secret-value",
+ "DIFY_AGENT_TEST_SOAK_RUN_ID": "positive-timeout",
+ "DIFY_AGENT_TEST_SOAK_ARTIFACT": str(tmp_path / "positive-timeout.jsonl"),
+ "DIFY_AGENT_TEST_SOAK_RECONCILE_TIMEOUT_SECONDS": "0",
+ }
+
+ with pytest.raises(ValueError, match="must be greater than zero"):
+ _ = _load_settings(env)
+
+
+def _test_settings(tmp_path: Path, *, name: str, reconcile_timeout_seconds: float = 60.0) -> _SoakSettings:
+ return _SoakSettings(
+ service_url="https://agent.invalid",
+ service_token="service-secret-value",
+ e2b_api_key="e2b-secret-value",
+ run_id=name,
+ artifact=tmp_path / f"{name}.jsonl",
+ concurrency=1,
+ burst_rounds=1,
+ idle_bindings=1,
+ idle_interval_seconds=0,
+ idle_rounds=1,
+ request_timeout_seconds=1,
+ reconcile_timeout_seconds=reconcile_timeout_seconds,
+ )
+
+
+class _FakeClock:
+ now: float
+
+ def __init__(self) -> None:
+ self.now = 0.0
+
+ def monotonic(self) -> float:
+ return self.now
+
+ async def sleep(self, seconds: float) -> None:
+ self.now += seconds
+
+
[email protected]("mismatch", ["workspace", "state"])
+async def test_idle_inventory_requires_exact_paused_identity(tmp_path: Path, mismatch: str) -> None:
+ settings = _test_settings(tmp_path, name="identity", reconcile_timeout_seconds=0.1)
+ clock = _FakeClock()
+ binding = _Binding(
+ binding_id=f"{settings.binding_prefix}idle-0000",
+ binding_ref="sandbox-1",
+ workspace_ref="sandbox-1",
+ )
+ mismatched = _OwnedSandbox(
+ sandbox_id=binding.binding_ref,
+ binding_id=binding.binding_id,
+ workspace_id=(f"{settings.binding_prefix}wrong-workspace" if mismatch == "workspace" else binding.binding_id),
+ state="running" if mismatch == "state" else "paused",
+ )
+
+ async def list_owned(_settings: _SoakSettings) -> list[_OwnedSandbox]:
+ return [mismatched]
+
+ with _JsonlLedger(settings.artifact, secrets=(settings.service_token, settings.e2b_api_key)) as ledger:
+ with pytest.raises(AssertionError, match="inventory mismatch"):
+ await _require_exact_idle_inventory(
+ settings,
+ ledger,
+ [binding],
+ list_owned=list_owned,
+ sleep=clock.sleep,
+ monotonic=clock.monotonic,
+ )
+
+
+async def test_reconcile_catches_sandbox_with_delayed_visibility(tmp_path: Path) -> None:
+ settings = _test_settings(tmp_path, name="delayed", reconcile_timeout_seconds=8)
+ clock = _FakeClock()
+ sandbox = _OwnedSandbox(
+ sandbox_id="sandbox-delayed",
+ binding_id=f"{settings.binding_prefix}idle-0000",
+ workspace_id=f"{settings.binding_prefix}idle-0000",
+ state="paused",
+ )
+ inventories: list[list[_OwnedSandbox]] = [[], [sandbox], [], []]
+ list_attempt = 0
+ killed: list[str] = []
+
+ async def list_owned(_settings: _SoakSettings) -> list[_OwnedSandbox]:
+ nonlocal list_attempt
+ inventory = inventories[list_attempt] if list_attempt < len(inventories) else []
+ list_attempt += 1
+ return inventory
+
+ async def kill_owned(_settings: _SoakSettings, sandbox_id: str) -> bool:
+ killed.append(sandbox_id)
+ return True
+
+ with _JsonlLedger(settings.artifact, secrets=(settings.service_token, settings.e2b_api_key)) as ledger:
+ await _reconcile_owned_e2b_sandboxes(
+ settings,
+ ledger,
+ list_owned=list_owned,
+ kill_owned=kill_owned,
+ sleep=clock.sleep,
+ monotonic=clock.monotonic,
+ )
+
+ assert killed == [sandbox.sandbox_id]
+ assert clock.monotonic() >= 7
+ assert clock.monotonic() < settings.reconcile_timeout_seconds
+
+
+async def test_reconcile_retries_first_list_failure_then_cleans_visible_sandbox(tmp_path: Path) -> None:
+ settings = _test_settings(tmp_path, name="retry-list", reconcile_timeout_seconds=8)
+ clock = _FakeClock()
+ sandbox = _OwnedSandbox(
+ sandbox_id="sandbox-after-list-error",
+ binding_id=f"{settings.binding_prefix}idle-0000",
+ workspace_id=f"{settings.binding_prefix}idle-0000",
+ state="paused",
+ )
+ list_attempts = 0
+ killed: list[str] = []
+
+ async def list_owned(_settings: _SoakSettings) -> list[_OwnedSandbox]:
+ nonlocal list_attempts
+ list_attempts += 1
+ if list_attempts == 1:
+ raise RuntimeError(f"temporary list failure near {settings.e2b_api_key}")
+ if list_attempts == 2:
+ return [sandbox]
+ return []
+
+ async def kill_owned(_settings: _SoakSettings, sandbox_id: str) -> bool:
+ killed.append(sandbox_id)
+ return True
+
+ with _JsonlLedger(settings.artifact, secrets=(settings.service_token, settings.e2b_api_key)) as ledger:
+ await _reconcile_owned_e2b_sandboxes(
+ settings,
+ ledger,
+ list_owned=list_owned,
+ kill_owned=kill_owned,
+ sleep=clock.sleep,
+ monotonic=clock.monotonic,
+ )
+
+ raw = settings.artifact.read_text(encoding="utf-8")
+ events = [cast(dict[str, object], json.loads(line)) for line in raw.splitlines()]
+ list_outcomes = [
+ event["outcome"]
+ for event in events
+ if event.get("event") == "operation_end" and event.get("operation") == "e2b_inventory_list"
+ ]
+ assert killed == [sandbox.sandbox_id]
+ assert list_outcomes[0] == "failed"
+ assert "succeeded" in list_outcomes[1:]
+ assert settings.e2b_api_key not in raw
+
+
+async def test_reconcile_retries_first_kill_failure_and_keeps_ledger_secret_safe(tmp_path: Path) -> None:
+ settings = _test_settings(tmp_path, name="retry-kill", reconcile_timeout_seconds=10)
+ clock = _FakeClock()
+ sandbox = _OwnedSandbox(
+ sandbox_id="sandbox-retry",
+ binding_id=f"{settings.binding_prefix}idle-0000",
+ workspace_id=f"{settings.binding_prefix}idle-0000",
+ state="paused",
+ )
+ present = True
+ kill_attempts = 0
+
+ async def list_owned(_settings: _SoakSettings) -> list[_OwnedSandbox]:
+ return [sandbox] if present else []
+
+ async def kill_owned(_settings: _SoakSettings, sandbox_id: str) -> bool:
+ nonlocal kill_attempts, present
+ assert sandbox_id == sandbox.sandbox_id
+ kill_attempts += 1
+ if kill_attempts == 1:
+ raise RuntimeError(f"temporary failure near {settings.e2b_api_key}")
+ present = False
+ return True
+
+ with _JsonlLedger(settings.artifact, secrets=(settings.service_token, settings.e2b_api_key)) as ledger:
+ await _reconcile_owned_e2b_sandboxes(
+ settings,
+ ledger,
+ list_owned=list_owned,
+ kill_owned=kill_owned,
+ sleep=clock.sleep,
+ monotonic=clock.monotonic,
+ )
+
+ raw = settings.artifact.read_text(encoding="utf-8")
+ events = [cast(dict[str, object], json.loads(line)) for line in raw.splitlines()]
+ kill_outcomes = [
+ event["outcome"]
+ for event in events
+ if event.get("event") == "operation_end" and event.get("operation") == "e2b_direct_kill"
+ ]
+ assert kill_attempts == 2
+ assert kill_outcomes == ["failed", "succeeded"]
+ assert clock.monotonic() >= 8
+ assert clock.monotonic() < settings.reconcile_timeout_seconds
+ assert settings.service_token not in raw
+ assert settings.e2b_api_key not in raw
+
+
+async def test_reconcile_recovers_when_first_final_list_is_slow_and_times_out(tmp_path: Path) -> None:
+ settings = _test_settings(tmp_path, name="slow-final-list", reconcile_timeout_seconds=20)
+ clock = _FakeClock()
+ slow_final_injected = False
+
+ async def list_owned(_settings: _SoakSettings) -> list[_OwnedSandbox]:
+ nonlocal slow_final_injected
+ if clock.monotonic() >= 10 and not slow_final_injected:
+ slow_final_injected = True
+ await clock.sleep(_RECONCILE_POLL_SECONDS)
+ raise TimeoutError("simulated slow final inventory list")
+ return []
+
+ async def kill_owned(_settings: _SoakSettings, _sandbox_id: str) -> bool:
+ raise AssertionError("kill must not run for stable empty inventory")
+
+ with _JsonlLedger(settings.artifact, secrets=(settings.service_token, settings.e2b_api_key)) as ledger:
+ await _reconcile_owned_e2b_sandboxes(
+ settings,
+ ledger,
+ list_owned=list_owned,
+ kill_owned=kill_owned,
+ sleep=clock.sleep,
+ monotonic=clock.monotonic,
+ )
+
+ raw = settings.artifact.read_text(encoding="utf-8")
+ events = [cast(dict[str, object], json.loads(line)) for line in raw.splitlines()]
+ final_list_outcomes = [
+ event["outcome"]
+ for event in events
+ if event.get("event") == "operation_end"
+ and event.get("operation") == "e2b_inventory_list"
+ and event.get("stage") == "direct_cleanup_final_scan"
+ ]
+ assert slow_final_injected
+ assert final_list_outcomes[0] == "failed"
+ assert "succeeded" in final_list_outcomes[1:]
+ assert clock.monotonic() >= 15
+ assert clock.monotonic() < settings.reconcile_timeout_seconds
+
+
+async def test_reconcile_bounds_never_returning_inventory_list(tmp_path: Path) -> None:
+ settings = _test_settings(tmp_path, name="list-deadline", reconcile_timeout_seconds=0.04)
+
+ async def list_owned(_settings: _SoakSettings) -> list[_OwnedSandbox]:
+ _ = await asyncio.Event().wait()
+ raise AssertionError("unreachable")
+
+ async def kill_owned(_settings: _SoakSettings, _sandbox_id: str) -> bool:
+ raise AssertionError("kill must not run without inventory ownership")
+
+ with _JsonlLedger(settings.artifact, secrets=(settings.service_token, settings.e2b_api_key)) as ledger:
+ with pytest.raises(AssertionError, match="final E2B inventory proof failed"):
+ await _reconcile_owned_e2b_sandboxes(
+ settings,
+ ledger,
+ list_owned=list_owned,
+ kill_owned=kill_owned,
+ )
+
+
+async def test_reconcile_bounds_never_returning_kill_batch(tmp_path: Path) -> None:
+ settings = _test_settings(tmp_path, name="kill-deadline", reconcile_timeout_seconds=0.04)
+ sandbox = _OwnedSandbox(
+ sandbox_id="sandbox-never-killed",
+ binding_id=f"{settings.binding_prefix}idle-0000",
+ workspace_id=f"{settings.binding_prefix}idle-0000",
+ state="paused",
+ )
+
+ async def list_owned(_settings: _SoakSettings) -> list[_OwnedSandbox]:
+ return [sandbox]
+
+ async def kill_owned(_settings: _SoakSettings, _sandbox_id: str) -> bool:
+ _ = await asyncio.Event().wait()
+ raise AssertionError("unreachable")
+
+ with _JsonlLedger(settings.artifact, secrets=(settings.service_token, settings.e2b_api_key)) as ledger:
+ with pytest.raises(AssertionError, match="was not empty"):
+ await _reconcile_owned_e2b_sandboxes(
+ settings,
+ ledger,
+ list_owned=list_owned,
+ kill_owned=kill_owned,
+ )
diff --git a/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py b/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py
index 62963e7066a..932b66bb34d 100644
--- a/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py
+++ b/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py
@@ -1,19 +1,25 @@
from __future__ import annotations
+import asyncio
import posixpath
from collections.abc import Callable
from dataclasses import dataclass, field
+import logging
from typing import cast
+import httpx as e2b_httpx
import httpx2 as httpx
import pytest
from shellctl.client import ShellctlClientError
from dify_agent.runtime_backend import (
BindingAcquireError,
+ BindingCapacityExhaustedError,
BindingCreateError,
+ BindingLostError,
ExecutionBindingCreateSpec,
ExecutionBindingDestroySpec,
+ HomeSnapshotCreateError,
HomeSnapshotCreateSpec,
SharedWorkspaceUnsupportedError,
WorkspacePreservationUnsupportedError,
@@ -38,8 +44,17 @@ class _FileEntry:
class _Files:
paths: set[str] = field(default_factory=set)
removed: list[str] = field(default_factory=list)
+ make_dir_errors: list[BaseException] = field(default_factory=list)
+ list_errors: list[BaseException] = field(default_factory=list)
+ remove_errors: list[BaseException] = field(default_factory=list)
+ make_dir_calls: int = 0
+ list_calls: int = 0
+ remove_calls: int = 0
async def make_dir(self, path: str) -> bool:
+ self.make_dir_calls += 1
+ if self.make_dir_errors:
+ raise self.make_dir_errors.pop(0)
self.paths.add(path)
return True
@@ -47,6 +62,9 @@ class _Files:
return path in self.paths
async def list(self, path: str) -> list[_FileEntry]:
+ self.list_calls += 1
+ if self.list_errors:
+ raise self.list_errors.pop(0)
prefix = f"{path.rstrip('/')}/"
return [
_FileEntry(path=entry)
@@ -55,6 +73,9 @@ class _Files:
]
async def remove(self, path: str) -> None:
+ self.remove_calls += 1
+ if self.remove_errors:
+ raise self.remove_errors.pop(0)
self.removed.append(path)
self.paths = {entry for entry in self.paths if entry != path and posixpath.commonpath((entry, path)) != path}
@@ -74,23 +95,32 @@ class _Sandbox:
killed: int = 0
snapshots: int = 0
pause_error: Exception | None = None
+ pause_errors: list[BaseException] = field(default_factory=list)
+ kill_errors: list[BaseException] = field(default_factory=list)
+ snapshot_errors: list[BaseException] = field(default_factory=list)
def get_host(self, port: int) -> str:
return f"{self.sandbox_id}-{port}.example.test"
async def pause(self, keep_memory: bool = True) -> bool:
self.pauses.append(keep_memory)
+ if self.pause_errors:
+ raise self.pause_errors.pop(0)
if self.pause_error is not None:
raise self.pause_error
return True
async def kill(self) -> bool:
self.killed += 1
+ if self.kill_errors:
+ raise self.kill_errors.pop(0)
return True
async def create_snapshot(self, name: str | None = None) -> _Snapshot:
del name
self.snapshots += 1
+ if self.snapshot_errors:
+ raise self.snapshot_errors.pop(0)
return _Snapshot(snapshot_id=f"snapshot-{self.sandbox_id}-{self.snapshots}")
@@ -101,11 +131,32 @@ class _ControlPlane:
killed: list[str] = field(default_factory=list)
deleted_snapshots: list[str] = field(default_factory=list)
pause_error: Exception | None = None
+ sandbox_pause_errors: list[BaseException] = field(default_factory=list)
+ sandbox_kill_errors: list[BaseException] = field(default_factory=list)
+ create_errors: list[BaseException] = field(default_factory=list)
+ connect_errors: list[BaseException] = field(default_factory=list)
+ kill_errors: list[BaseException] = field(default_factory=list)
+ delete_snapshot_errors: list[BaseException] = field(default_factory=list)
+ connect_attempts: list[str] = field(default_factory=list)
+ file_make_dir_errors: list[BaseException] = field(default_factory=list)
+ file_list_errors: list[BaseException] = field(default_factory=list)
+ file_remove_errors: list[BaseException] = field(default_factory=list)
async def create(self, template: str, *, timeout: int, metadata: dict[str, str], on_timeout: str) -> _Sandbox:
del timeout
+ self.created.append((template, on_timeout))
+ if self.create_errors:
+ raise self.create_errors.pop(0)
sandbox_id = f"sandbox-{len(self.sandboxes) + 1}"
- sandbox = _Sandbox(sandbox_id=sandbox_id, pause_error=self.pause_error)
+ sandbox = _Sandbox(
+ sandbox_id=sandbox_id,
+ pause_error=self.pause_error,
+ pause_errors=list(self.sandbox_pause_errors),
+ kill_errors=list(self.sandbox_kill_errors),
+ )
+ sandbox.files.make_dir_errors = list(self.file_make_dir_errors)
+ sandbox.files.list_errors = list(self.file_list_errors)
+ sandbox.files.remove_errors = list(self.file_remove_errors)
sandbox.files.paths.update(
{
"/workspace",
@@ -115,20 +166,26 @@ class _ControlPlane:
}
)
self.sandboxes[sandbox_id] = sandbox
- self.created.append((template, on_timeout))
assert metadata["dify.resource"] == "runtime-sandbox"
return sandbox
async def connect(self, handle: str, *, timeout: int) -> _Sandbox:
del timeout
+ self.connect_attempts.append(handle)
+ if self.connect_errors:
+ raise self.connect_errors.pop(0)
return self.sandboxes[handle]
async def kill(self, handle: str) -> bool:
self.killed.append(handle)
+ if self.kill_errors:
+ raise self.kill_errors.pop(0)
return True
async def delete_snapshot(self, snapshot_ref: str) -> bool:
self.deleted_snapshots.append(snapshot_ref)
+ if self.delete_snapshot_errors:
+ raise self.delete_snapshot_errors.pop(0)
return True
@@ -150,19 +207,53 @@ def _mock_http(
return clients
+def _binding_backend(control: _ControlPlane) -> E2BExecutionBindingBackend:
+ return E2BExecutionBindingBackend(
+ control_plane=control, # pyright: ignore[reportArgumentType]
+ template="prepared-template",
+ active_timeout_seconds=E2B_MAX_ACTIVE_TIMEOUT_SECONDS,
+ )
+
+
def _connected_backend(*, pause_error: Exception | None = None) -> tuple[E2BExecutionBindingBackend, _Sandbox]:
control = _ControlPlane()
sandbox = _Sandbox(sandbox_id="sandbox-1", pause_error=pause_error)
sandbox.files.paths.add("/workspace")
control.sandboxes[sandbox.sandbox_id] = sandbox
- return (
- E2BExecutionBindingBackend(
- control_plane=control, # pyright: ignore[reportArgumentType]
- template="prepared-template",
- active_timeout_seconds=E2B_MAX_ACTIVE_TIMEOUT_SECONDS,
- ),
- sandbox,
- )
+ return _binding_backend(control), sandbox
+
+
+@dataclass(slots=True)
+class _ReleaseDataPlane:
+ close_error: BaseException | None = None
+ close_calls: int = 0
+
+ async def close(self) -> None:
+ self.close_calls += 1
+ if self.close_error is not None:
+ raise self.close_error
+
+
+def _transport_error(error_type: type[e2b_httpx.RequestError]) -> e2b_httpx.RequestError:
+ request = e2b_httpx.Request("POST", "https://api.e2b.test/sandboxes")
+ return error_type("stale E2B connection", request=request)
+
+
+def _http_status_error() -> e2b_httpx.HTTPStatusError:
+ request = e2b_httpx.Request("POST", "https://api.e2b.test/sandboxes")
+ response = e2b_httpx.Response(503, request=request)
+ return e2b_httpx.HTTPStatusError("E2B unavailable", request=request, response=response)
+
+
+def sleep_delays(monkeypatch: pytest.MonkeyPatch) -> list[float]:
+ delays: list[float] = []
+
+ async def record_sleep(delay: float) -> None:
+ delays.append(delay)
+
+ monkeypatch.setattr(e2b_module.asyncio, "sleep", record_sleep)
+ return delays
@pytest.mark.anyio
@@ -195,17 +286,310 @@ async def test_e2b_sdk_create_disables_public_traffic(monkeypatch: pytest.Monkey
assert create_options["network"] == {"allow_public_traffic": False}
+async def test_e2b_sdk_create_maps_provider_capacity_exhaustion(monkeypatch: pytest.MonkeyPatch) -> None:
+ from e2b import AsyncSandbox, RateLimitException
+
+ calls = 0
+
+ async def create(
+ _cls: type[AsyncSandbox],
+ _template: str,
+ **_options: object,
+ ) -> _Sandbox:
+ nonlocal calls
+ calls += 1
+ raise RateLimitException("maximum concurrent sandboxes reached")
+
+ monkeypatch.setattr(AsyncSandbox, "create", classmethod(create))
+ control_plane = E2BSDKControlPlane(api_key="e2b-secret")
+
+ with pytest.raises(e2b_module._E2BControlPlaneCapacityExhaustedError, match="maximum concurrent"):
+ _ = await control_plane.create(
+ "prepared-template",
+ timeout=120,
+ metadata={"dify.resource": "runtime-sandbox"},
+ on_timeout="pause",
+ )
+
+ assert calls == 1
+
+
+ "transport_error_type",
+ [e2b_httpx.ConnectError, e2b_httpx.ReadTimeout],
+)
+async def test_e2b_connect_retries_one_transport_failure(
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+ sleep_delays: list[float],
+ transport_error_type: type[e2b_httpx.RequestError],
+) -> None:
+ control = _ControlPlane(connect_errors=[_transport_error(transport_error_type)])
+ sandbox = _Sandbox(sandbox_id="sandbox-1")
+ sandbox.files.paths.add("/workspace")
+ control.sandboxes[sandbox.sandbox_id] = sandbox
+ backend = _binding_backend(control)
+ clients = _mock_http(monkeypatch, lambda _request: httpx.Response(200, json={"status": "ok"}))
+ with caplog.at_level(logging.WARNING, logger="dify_agent.runtime_backend.e2b"):
+ lease = await backend.acquire(sandbox.sandbox_id)
+
+ assert control.connect_attempts == [sandbox.sandbox_id, sandbox.sandbox_id]
+ assert sleep_delays == [0.25]
+ retry_record = next(record for record in caplog.records if record.__dict__.get("outcome") == "retrying")
+ assert retry_record.__dict__["e2b_operation"] == "connect"
+ assert retry_record.__dict__["sandbox_id"] == sandbox.sandbox_id
+ assert retry_record.__dict__["attempt"] == 1
+ assert retry_record.__dict__["cleanup_stage"] == "binding_acquire"
+ assert retry_record.__dict__["exception_type"] == transport_error_type.__name__
+ await cast(E2BRuntimeLease, lease).data_plane.close()
+ assert clients[0].is_closed
+
+
+ "transport_error_type",
+ [e2b_httpx.ReadError, e2b_httpx.ReadTimeout],
+)
+async def test_e2b_initial_pause_retries_one_transport_failure(
+ sleep_delays: list[float],
+ transport_error_type: type[e2b_httpx.RequestError],
+) -> None:
+ control = _ControlPlane(sandbox_pause_errors=[_transport_error(transport_error_type)])
+ backend = _binding_backend(control)
+ allocation = await backend.create_binding(
+ ExecutionBindingCreateSpec(
+ tenant_id="tenant-1",
+ agent_id="agent-1",
+ binding_id="binding-1",
+ workspace_id="workspace-1",
+ existing_workspace_ref=None,
+ )
+ )
+
+ sandbox = control.sandboxes[allocation.binding_ref]
+ assert control.created == [("prepared-template", "pause")]
+ assert sandbox.pauses == [True, True]
+ assert sleep_delays == [0.25]
+
+
+ "transport_error_type",
+ [e2b_httpx.WriteError, e2b_httpx.ReadTimeout],
+)
+async def test_e2b_destroy_retries_one_transport_failure(
+ sleep_delays: list[float],
+ transport_error_type: type[e2b_httpx.RequestError],
+) -> None:
+ control = _ControlPlane(kill_errors=[_transport_error(transport_error_type)])
+ backend = _binding_backend(control)
+ await backend.destroy_binding(
+ ExecutionBindingDestroySpec(
+ binding_ref="sandbox-1",
+ workspace_ref="sandbox-1",
+ destroy_workspace=True,
+ )
+ )
+
+ assert control.killed == ["sandbox-1", "sandbox-1"]
+ assert sleep_delays == [0.25]
+
+
+ "transport_error_type",
+ [e2b_httpx.RemoteProtocolError, e2b_httpx.ReadTimeout],
+)
+async def test_e2b_snapshot_delete_retries_one_transport_failure(
+ sleep_delays: list[float],
+ transport_error_type: type[e2b_httpx.RequestError],
+) -> None:
+ control = _ControlPlane(delete_snapshot_errors=[_transport_error(transport_error_type)])
+ backend = E2BHomeSnapshotBackend(
+ control_plane=control, # pyright: ignore[reportArgumentType]
+ )
+ await backend.delete("snapshot-1")
+
+ assert control.deleted_snapshots == ["snapshot-1", "snapshot-1"]
+ assert sleep_delays == [0.25]
+
+
+async def test_e2b_connect_raises_after_two_read_timeouts(
+ sleep_delays: list[float],
+) -> None:
+ control = _ControlPlane(
+ connect_errors=[
+ _transport_error(e2b_httpx.ReadTimeout),
+ _transport_error(e2b_httpx.ReadTimeout),
+ ]
+ )
+ backend = _binding_backend(control)
+
+ with pytest.raises(BindingAcquireError, match="stale E2B connection"):
+ _ = await backend.acquire("sandbox-1")
+
+ assert control.connect_attempts == ["sandbox-1", "sandbox-1"]
+ assert sleep_delays == [0.25]
+
+
+ ("failure_kind", "expected_error"),
+ [
+ ("not_found", BindingLostError),
+ ("http_status", BindingAcquireError),
+ ("validation", BindingAcquireError),
+ ("cancelled", asyncio.CancelledError),
+ ],
+)
+async def test_e2b_connect_does_not_retry_non_transport_failures(
+ monkeypatch: pytest.MonkeyPatch,
+ failure_kind: str,
+ expected_error: type[BaseException],
+) -> None:
+ if failure_kind == "not_found":
+ failure: BaseException = e2b_module._E2BControlPlaneNotFoundError("missing")
+ elif failure_kind == "http_status":
+ failure = _http_status_error()
+ elif failure_kind == "validation":
+ failure = ValueError("invalid E2B response")
+ else:
+ failure = asyncio.CancelledError()
+ control = _ControlPlane(connect_errors=[failure])
+ backend = _binding_backend(control)
+
+ async def fail_sleep(_delay: float) -> None:
+ raise AssertionError("non-retryable E2B failures must not sleep")
+
+ monkeypatch.setattr(e2b_module.asyncio, "sleep", fail_sleep)
+ with pytest.raises(expected_error):
+ _ = await backend.acquire("sandbox-1")
+
+ assert control.connect_attempts == ["sandbox-1"]
+
+
+ "transport_error_type",
+ [e2b_httpx.ConnectError, e2b_httpx.ReadTimeout],
+)
+async def test_e2b_create_and_snapshot_create_are_not_retried(
+ monkeypatch: pytest.MonkeyPatch,
+ transport_error_type: type[e2b_httpx.RequestError],
+) -> None:
+ control = _ControlPlane(create_errors=[_transport_error(transport_error_type)])
+ bindings = _binding_backend(control)
+ snapshot_sandbox = _Sandbox(
+ sandbox_id="sandbox-snapshot",
+ snapshot_errors=[_transport_error(transport_error_type)],
+ )
+ source = E2BRuntimeLease(
+ sandbox=snapshot_sandbox, # pyright: ignore[reportArgumentType]
+ data_plane=cast(ShellctlRuntimeLease, object()),
+ )
+ snapshots = E2BHomeSnapshotBackend(control_plane=control) # pyright: ignore[reportArgumentType]
+
+ async def fail_sleep(_delay: float) -> None:
+ raise AssertionError("non-idempotent E2B creates must not sleep")
+
+ monkeypatch.setattr(e2b_module.asyncio, "sleep", fail_sleep)
+ with pytest.raises(BindingCreateError, match="stale E2B connection"):
+ _ = await bindings.create_binding(
+ ExecutionBindingCreateSpec(
+ tenant_id="tenant-1",
+ agent_id="agent-1",
+ binding_id="binding-1",
+ workspace_id="workspace-1",
+ existing_workspace_ref=None,
+ )
+ )
+ with pytest.raises(HomeSnapshotCreateError, match="stale E2B connection"):
+ _ = await snapshots.create_from_runtime(
+ spec=HomeSnapshotCreateSpec(tenant_id="tenant-1", agent_id="agent-1", home_snapshot_id="home-1"),
+ source=source,
+ )
+
+ assert control.created == [("prepared-template", "pause")]
+ assert snapshot_sandbox.snapshots == 1
+
+
+async def test_e2b_create_maps_capacity_exhaustion_without_retry(monkeypatch: pytest.MonkeyPatch) -> None:
+ control = _ControlPlane(
+ create_errors=[e2b_module._E2BControlPlaneCapacityExhaustedError("maximum concurrent sandboxes reached")]
+ )
+ backend = _binding_backend(control)
+
+ async def fail_sleep(_delay: float) -> None:
+ raise AssertionError("capacity exhaustion must not retry")
+
+ monkeypatch.setattr(e2b_module.asyncio, "sleep", fail_sleep)
+ with pytest.raises(BindingCapacityExhaustedError, match="maximum concurrent sandboxes reached"):
+ _ = await backend.create_binding(
+ ExecutionBindingCreateSpec(
+ tenant_id="tenant-1",
+ agent_id="agent-1",
+ binding_id="binding-1",
+ workspace_id="workspace-1",
+ existing_workspace_ref=None,
+ )
+ )
+
+ assert control.created == [("prepared-template", "pause")]
+
+
+ ("failure_stage", "expected_calls"),
+ [
+ ("make_dir", (1, 0, 0)),
+ ("list", (1, 1, 0)),
+ ("remove", (1, 1, 1)),
+ ],
+)
+async def test_e2b_file_initialization_is_one_shot_and_compensates_on_failure(
+ monkeypatch: pytest.MonkeyPatch,
+ failure_stage: str,
+ expected_calls: tuple[int, int, int],
+) -> None:
+ error = _transport_error(e2b_httpx.ReadError)
+ control = _ControlPlane(
+ file_make_dir_errors=[error] if failure_stage == "make_dir" else [],
+ file_list_errors=[error] if failure_stage == "list" else [],
+ file_remove_errors=[error] if failure_stage == "remove" else [],
+ )
+ backend = _binding_backend(control)
+
+ async def fail_sleep(_delay: float) -> None:
+ raise AssertionError("file initialization must not retry")
+
+ monkeypatch.setattr(e2b_module.asyncio, "sleep", fail_sleep)
+ with pytest.raises(BindingCreateError, match="stale E2B connection"):
+ _ = await backend.create_binding(
+ ExecutionBindingCreateSpec(
+ tenant_id="tenant-1",
+ agent_id="agent-1",
+ binding_id="binding-1",
+ workspace_id="workspace-1",
+ existing_workspace_ref=None,
+ )
+ )
+
+ sandbox = next(iter(control.sandboxes.values()))
+ assert (sandbox.files.make_dir_calls, sandbox.files.list_calls, sandbox.files.remove_calls) == expected_calls
+ assert sandbox.killed == 1
+
+
@pytest.mark.anyio
async def test_e2b_binding_uses_default_template_or_exact_snapshot_and_couples_refs() -> None:
control = _ControlPlane()
snapshots = E2BHomeSnapshotBackend(
control_plane=control, # pyright: ignore[reportArgumentType]
)
- bindings = E2BExecutionBindingBackend(
- control_plane=control, # pyright: ignore[reportArgumentType]
- template="prepared-template",
- active_timeout_seconds=E2B_MAX_ACTIVE_TIMEOUT_SECONDS,
- )
+ bindings = _binding_backend(control)
default_allocation = await bindings.create_binding(
ExecutionBindingCreateSpec(
@@ -255,11 +639,7 @@ async def test_e2b_binding_uses_default_template_or_exact_snapshot_and_couples_r
@pytest.mark.anyio
async def test_e2b_rejects_shared_workspace_and_binding_only_destroy() -> None:
control = _ControlPlane()
- backend = E2BExecutionBindingBackend(
- control_plane=control, # pyright: ignore[reportArgumentType]
- template="prepared-template",
- active_timeout_seconds=E2B_MAX_ACTIVE_TIMEOUT_SECONDS,
- )
+ backend = _binding_backend(control)
spec = ExecutionBindingCreateSpec(
tenant_id="tenant-1",
agent_id="agent-2",
@@ -280,11 +660,7 @@ async def test_e2b_rejects_shared_workspace_and_binding_only_destroy() -> None:
@pytest.mark.anyio
async def test_e2b_binding_create_kills_sandbox_when_initialization_fails() -> None:
control = _ControlPlane(pause_error=RuntimeError("pause failed"))
- backend = E2BExecutionBindingBackend(
- control_plane=control, # pyright: ignore[reportArgumentType]
- template="prepared-template",
- active_timeout_seconds=E2B_MAX_ACTIVE_TIMEOUT_SECONDS,
- )
+ backend = _binding_backend(control)
with pytest.raises(BindingCreateError, match="pause failed"):
await backend.create_binding(
@@ -302,6 +678,72 @@ async def test_e2b_binding_create_kills_sandbox_when_initialization_fails() -> N
assert sandbox.killed == 1
+async def test_e2b_binding_create_preserves_primary_error_when_compensation_kill_fails(
+ caplog: pytest.LogCaptureFixture,
+ sleep_delays: list[float],
+) -> None:
+ control = _ControlPlane(
+ pause_error=RuntimeError("primary pause failure"),
+ sandbox_kill_errors=[
+ _transport_error(e2b_httpx.WriteError),
+ _transport_error(e2b_httpx.WriteError),
+ ],
+ )
+ backend = _binding_backend(control)
+ with caplog.at_level(logging.WARNING, logger="dify_agent.runtime_backend.e2b"):
+ with pytest.raises(BindingCreateError, match="primary pause failure"):
+ _ = await backend.create_binding(
+ ExecutionBindingCreateSpec(
+ tenant_id="tenant-1",
+ agent_id="agent-1",
+ binding_id="binding-1",
+ workspace_id="workspace-1",
+ existing_workspace_ref=None,
+ )
+ )
+
+ sandbox = next(iter(control.sandboxes.values()))
+ assert sandbox.killed == 2
+ assert sleep_delays == [0.25]
+ cleanup_record = next(
+ record for record in caplog.records if record.__dict__.get("outcome") == "ignored_cleanup_failure"
+ )
+ assert cleanup_record.funcName == "create_binding"
+ assert cleanup_record.__dict__["e2b_operation"] == "kill"
+ assert cleanup_record.__dict__["sandbox_id"] == sandbox.sandbox_id
+ assert cleanup_record.__dict__["cleanup_stage"] == "binding_create_compensation"
+ assert cleanup_record.__dict__["exception_type"] == "WriteError"
+
+
+async def test_e2b_binding_create_fails_closed_when_initial_pause_retries_are_exhausted(
+ sleep_delays: list[float],
+) -> None:
+ control = _ControlPlane(
+ sandbox_pause_errors=[
+ _transport_error(e2b_httpx.ReadError),
+ _transport_error(e2b_httpx.ReadError),
+ ]
+ )
+ backend = _binding_backend(control)
+ with pytest.raises(BindingCreateError, match="stale E2B connection"):
+ _ = await backend.create_binding(
+ ExecutionBindingCreateSpec(
+ tenant_id="tenant-1",
+ agent_id="agent-1",
+ binding_id="binding-1",
+ workspace_id="workspace-1",
+ existing_workspace_ref=None,
+ )
+ )
+
+ sandbox = next(iter(control.sandboxes.values()))
+ assert sandbox.pauses == [True, True]
+ assert sandbox.killed == 1
+ assert sleep_delays == [0.25]
+
+
@pytest.mark.anyio
async def test_e2b_missing_explicit_snapshot_does_not_fall_back_to_template() -> None:
class _FailingControlPlane(_ControlPlane):
@@ -311,11 +753,7 @@ async def test_e2b_missing_explicit_snapshot_does_not_fall_back_to_template() ->
raise RuntimeError("snapshot unavailable")
control = _FailingControlPlane()
- backend = E2BExecutionBindingBackend(
- control_plane=control, # pyright: ignore[reportArgumentType]
- template="prepared-template",
- active_timeout_seconds=E2B_MAX_ACTIVE_TIMEOUT_SECONDS,
- )
+ backend = _binding_backend(control)
with pytest.raises(BindingCreateError, match="snapshot unavailable"):
await backend.create_binding(
@@ -353,9 +791,150 @@ async def test_e2b_checkpoint_uses_exact_source_runtime() -> None:
assert source_sandbox.snapshots == 1
+ ("close_fails", "pause_fails", "expected_warning_count"),
+ [
+ (True, False, 1),
+ (False, True, 1),
+ (True, True, 2),
+ ],
+)
+async def test_e2b_release_warns_without_raising_for_cleanup_failures(
+ caplog: pytest.LogCaptureFixture,
+ close_fails: bool,
+ pause_fails: bool,
+ expected_warning_count: int,
+) -> None:
+ data_plane = _ReleaseDataPlane(close_error=RuntimeError("close failed") if close_fails else None)
+ sandbox = _Sandbox(
+ sandbox_id="sandbox-release",
+ pause_error=RuntimeError("pause failed") if pause_fails else None,
+ )
+ lease = E2BRuntimeLease(
+ sandbox=sandbox, # pyright: ignore[reportArgumentType]
+ data_plane=cast(ShellctlRuntimeLease, cast(object, data_plane)),
+ )
+ backend, _ = _connected_backend()
+
+ with caplog.at_level(logging.WARNING, logger="dify_agent.runtime_backend.e2b"):
+ await backend.release(lease)
+
+ assert data_plane.close_calls == 1
+ assert sandbox.pauses == [True]
+ records = [record for record in caplog.records if record.name == "dify_agent.runtime_backend.e2b"]
+ assert len(records) == expected_warning_count
+ for record in records:
+ assert record.__dict__["sandbox_id"] == sandbox.sandbox_id
+ assert record.__dict__["cleanup_stage"] == "binding_release"
+ assert record.__dict__["outcome"] == "ignored_cleanup_failure"
+ assert isinstance(record.__dict__["e2b_operation"], str)
+ assert isinstance(record.__dict__["attempt"], int)
+ assert isinstance(record.__dict__["exception_type"], str)
+
+
+async def test_e2b_release_retries_pause_transport_failure(sleep_delays: list[float]) -> None:
+ data_plane = _ReleaseDataPlane()
+ sandbox = _Sandbox(
+ sandbox_id="sandbox-release",
+ pause_errors=[_transport_error(e2b_httpx.RemoteProtocolError)],
+ )
+ lease = E2BRuntimeLease(
+ sandbox=sandbox, # pyright: ignore[reportArgumentType]
+ data_plane=cast(ShellctlRuntimeLease, cast(object, data_plane)),
+ )
+ backend, _ = _connected_backend()
+ await backend.release(lease)
+
+ assert data_plane.close_calls == 1
+ assert sandbox.pauses == [True, True]
+ assert sleep_delays == [0.25]
+
+
+async def test_e2b_release_ignores_exhausted_pause_read_timeouts(
+ caplog: pytest.LogCaptureFixture,
+ sleep_delays: list[float],
+) -> None:
+ data_plane = _ReleaseDataPlane()
+ sandbox = _Sandbox(
+ sandbox_id="sandbox-release",
+ pause_errors=[
+ _transport_error(e2b_httpx.ReadTimeout),
+ _transport_error(e2b_httpx.ReadTimeout),
+ ],
+ )
+ lease = E2BRuntimeLease(
+ sandbox=sandbox, # pyright: ignore[reportArgumentType]
+ data_plane=cast(ShellctlRuntimeLease, cast(object, data_plane)),
+ )
+ backend, _ = _connected_backend()
+
+ with caplog.at_level(logging.WARNING, logger="dify_agent.runtime_backend.e2b"):
+ await backend.release(lease)
+
+ assert data_plane.close_calls == 1
+ assert sandbox.pauses == [True, True]
+ assert sleep_delays == [0.25]
+ cleanup_record = next(
+ record for record in caplog.records if record.__dict__.get("outcome") == "ignored_cleanup_failure"
+ )
+ assert cleanup_record.__dict__["e2b_operation"] == "pause"
+ assert cleanup_record.__dict__["sandbox_id"] == sandbox.sandbox_id
+ assert cleanup_record.__dict__["attempt"] == 2
+ assert cleanup_record.__dict__["cleanup_stage"] == "binding_release"
+ assert cleanup_record.__dict__["exception_type"] == "ReadTimeout"
+
+
[email protected]("failure_stage", ["close", "pause"])
+async def test_e2b_release_does_not_swallow_base_exceptions(failure_stage: str) -> None:
+ data_plane = _ReleaseDataPlane(
+ close_error=asyncio.CancelledError() if failure_stage == "close" else None,
+ )
+ sandbox = _Sandbox(
+ sandbox_id="sandbox-release",
+ pause_errors=[asyncio.CancelledError()] if failure_stage == "pause" else [],
+ )
+ lease = E2BRuntimeLease(
+ sandbox=sandbox, # pyright: ignore[reportArgumentType]
+ data_plane=cast(ShellctlRuntimeLease, cast(object, data_plane)),
+ )
+ backend, _ = _connected_backend()
+
+ with pytest.raises(asyncio.CancelledError):
+ await backend.release(lease)
+
+ assert data_plane.close_calls == 1
+ assert sandbox.pauses == [True]
+
+
+async def test_e2b_release_preserves_close_base_exception_when_pause_also_raises() -> None:
+ close_error = asyncio.CancelledError("close cancelled")
+ pause_error = asyncio.CancelledError("pause cancelled")
+ data_plane = _ReleaseDataPlane(close_error=close_error)
+ sandbox = _Sandbox(sandbox_id="sandbox-release", pause_errors=[pause_error])
+ lease = E2BRuntimeLease(
+ sandbox=sandbox, # pyright: ignore[reportArgumentType]
+ data_plane=cast(ShellctlRuntimeLease, cast(object, data_plane)),
+ )
+ backend, _ = _connected_backend()
+
+ with pytest.raises(asyncio.CancelledError) as exc_info:
+ await backend.release(lease)
+
+ assert exc_info.value is close_error
+ assert exc_info.value.__cause__ is pause_error
+ assert data_plane.close_calls == 1
+ assert sandbox.pauses == [True]
+
+
@pytest.mark.anyio
async def test_e2b_acquire_retries_transient_shellctl_failures_until_ready(
monkeypatch: pytest.MonkeyPatch,
+ sleep_delays: list[float],
) -> None:
monkeypatch.setenv("SHELLCTL_AUTH_TOKEN", "ambient-shellctl-token")
attempts = 0
@@ -373,18 +952,12 @@ async def test_e2b_acquire_retries_transient_shellctl_failures_until_ready(
return httpx.Response(200, json={"status": "ok"})
clients = _mock_http(monkeypatch, handler)
- sleeps: list[float] = []
-
- async def record_sleep(delay: float) -> None:
- sleeps.append(delay)
-
- monkeypatch.setattr(e2b_module.asyncio, "sleep", record_sleep)
backend, sandbox = _connected_backend()
lease = await backend.acquire(sandbox.sandbox_id)
assert attempts == 3
- assert sleeps == [0.5, 0.5]
+ assert sleep_delays == [0.5, 0.5]
assert lease.layout.home_dir == "/home/dify"
assert lease.layout.workspace_dir == "/workspace"
assert not clients[0].is_closed
@@ -415,6 +988,7 @@ async def test_e2b_acquire_fails_closed_without_traffic_token(
@pytest.mark.anyio
async def test_e2b_acquire_closes_transport_and_pauses_after_readiness_retries_exhausted(
monkeypatch: pytest.MonkeyPatch,
+ sleep_delays: list[float],
) -> None:
attempts = 0
@@ -424,19 +998,13 @@ async def test_e2b_acquire_closes_transport_and_pauses_after_readiness_retries_e
return httpx.Response(503, json={"error": {"code": "starting", "message": "not ready"}})
clients = _mock_http(monkeypatch, handler)
- sleeps: list[float] = []
-
- async def record_sleep(delay: float) -> None:
- sleeps.append(delay)
-
- monkeypatch.setattr(e2b_module.asyncio, "sleep", record_sleep)
backend, sandbox = _connected_backend()
with pytest.raises(BindingAcquireError, match="not ready"):
_ = await backend.acquire(sandbox.sandbox_id)
assert attempts == 3
- assert sleeps == [0.5, 0.5]
+ assert sleep_delays == [0.5, 0.5]
assert clients[0].is_closed
assert sandbox.pauses == [True]
diff --git a/dify-agent/tests/local/dify_agent/server/test_execution_bindings.py b/dify-agent/tests/local/dify_agent/server/test_execution_bindings.py
index 4ff354be6c9..82d3d69415d 100644
--- a/dify-agent/tests/local/dify_agent/server/test_execution_bindings.py
+++ b/dify-agent/tests/local/dify_agent/server/test_execution_bindings.py
@@ -1,23 +1,30 @@
from dataclasses import dataclass, field
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
import pytest
from dify_agent.protocol import CreateExecutionBindingRequest, DestroyExecutionBindingRequest
from dify_agent.runtime_backend import (
+ BindingCapacityExhaustedError,
ExecutionBindingAllocation,
ExecutionBindingCreateSpec,
ExecutionBindingDestroySpec,
)
from dify_agent.server.execution_bindings import ExecutionBindingService
+from dify_agent.server.routes.execution_bindings import create_execution_bindings_router
@dataclass(slots=True)
class _Backend:
created: list[ExecutionBindingCreateSpec] = field(default_factory=list)
destroyed: list[ExecutionBindingDestroySpec] = field(default_factory=list)
+ create_error: Exception | None = None
async def create_binding(self, spec: ExecutionBindingCreateSpec) -> ExecutionBindingAllocation:
self.created.append(spec)
+ if self.create_error is not None:
+ raise self.create_error
return ExecutionBindingAllocation(binding_ref="opaque-binding", workspace_ref="opaque-workspace")
async def destroy_binding(self, spec: ExecutionBindingDestroySpec) -> None:
@@ -64,3 +71,32 @@ async def test_execution_binding_service_forwards_final_contract() -> None:
destroy_workspace=True,
)
]
+
+
+def test_execution_binding_route_reports_capacity_exhaustion_as_429() -> None:
+ backend = _Backend(create_error=BindingCapacityExhaustedError("maximum concurrent sandboxes reached"))
+ service = ExecutionBindingService(backend=backend) # pyright: ignore[reportArgumentType]
+ app = FastAPI()
+ app.include_router(create_execution_bindings_router(lambda: service))
+
+ with TestClient(app) as client:
+ response = client.post(
+ "/execution-bindings",
+ json={
+ "tenant_id": "tenant-1",
+ "agent_id": "agent-1",
+ "binding_id": "binding-1",
+ "workspace_id": "workspace-1",
+ "existing_workspace_ref": None,
+ "home_snapshot_ref": None,
+ },
+ )
+
+ assert response.status_code == 429
+ assert response.json() == {
+ "detail": {
+ "code": "binding_capacity_exhausted",
+ "message": "maximum concurrent sandboxes reached",
+ }
+ }
+ assert len(backend.created) == 1
diff --git a/dify-agent/tests/local/test_packaging.py b/dify-agent/tests/local/test_packaging.py
index d382da2d405..d5f767dcdfd 100644
--- a/dify-agent/tests/local/test_packaging.py
+++ b/dify-agent/tests/local/test_packaging.py
@@ -16,7 +16,7 @@ CLIENT_SHARED_DTO_DEPENDENCIES = {
}
SERVER_RUNTIME_DEPENDENCIES = {
- "e2b>=2.34.0,<3.0.0",
+ "e2b>=2.38.0,<3.0.0",
"fastapi==0.136.0",
"graphon==0.5.2",
"jsonschema>=4.23.0,<5.0.0",
diff --git a/dify-agent/uv.lock b/dify-agent/uv.lock
index 9ae245a835b..4283e4be4e9 100644
--- a/dify-agent/uv.lock
+++ b/dify-agent/uv.lock
@@ -394,6 +394,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/8d/e4/d66708bdf0d92fb4d49b22cdff4b10cec38aca5dcd7e81d909bb55c65cd7/confection-1.3.3-py3-none-any.whl", hash = "sha256:b9fef9ee84b237ef4611ec3eb5797b70e13063e6310ad9f15536373f5e313c82", size = 35902, upload-time = "2026-03-24T18:45:22.664Z" },
]
+[[package]]
+name = "connectrpc"
+version = "0.11.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "protobuf-py" },
+ { name = "pyqwest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8b/b5/63e14ab9d4d4cc58818562db4120a35fa7454e3035633da41bdc6b712abd/connectrpc-0.11.1.tar.gz", hash = "sha256:18277f7838847b4271ca38d40c7d2387b5a2ea6a29f240689c19e1ec84aaff66", size = 46222, upload-time = "2026-07-15T06:33:31.601Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5c/ac/aa647675812cd075f2cb9acb00d62f4f2cbcbc6736049a62342b7371ce5b/connectrpc-0.11.1-py3-none-any.whl", hash = "sha256:8a52e2e92a485fa9681c1101a79a5ebeb31807e3ea3d5aabd41f484a6398bc7b", size = 64991, upload-time = "2026-07-15T06:33:29.396Z" },
+]
+
[[package]]
name = "coverage"
version = "7.14.0"
@@ -630,7 +643,7 @@ docs = [
[package.metadata]
requires-dist = [
- { name = "e2b", marker = "extra == 'server'", specifier = ">=2.34.0,<3.0.0" },
+ { name = "e2b", marker = "extra == 'server'", specifier = ">=2.38.0,<3.0.0" },
{ name = "fastapi", marker = "extra == 'server'", specifier = "==0.136.0" },
{ name = "graphon", marker = "extra == 'server'", specifier = "==0.5.2" },
{ name = "httpx", specifier = "==0.28.1" },
@@ -694,24 +707,25 @@ wheels = [
[[package]]
name = "e2b"
-version = "2.34.0"
+version = "2.38.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "attrs" },
+ { name = "connectrpc" },
{ name = "dockerfile-parse" },
{ name = "h2" },
- { name = "httpcore" },
{ name = "httpx" },
{ name = "packaging" },
- { name = "protobuf" },
+ { name = "protobuf-py" },
+ { name = "pyqwest" },
{ name = "python-dateutil" },
{ name = "rich" },
{ name = "typing-extensions" },
{ name = "wcmatch" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/5b/22/5bcf34304b62e6984df9089eb996a4b2aa63618f901befe1f7cc73a24437/e2b-2.34.0.tar.gz", hash = "sha256:0dcc2694b3ead62e87b3d1070a05f66842dd2fa320fc1d7bd84c8fcfadeaf1b1", size = 189367, upload-time = "2026-07-17T09:59:23.844Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/e2/e2/1acb7bd6e16811a3affc89521304420410f97040a0a386497b101f898de3/e2b-2.38.0.tar.gz", hash = "sha256:b245976cc946d144baf77341b19fa537bfd7cbd9028dc8ef59bd6947f5e329c1", size = 205432, upload-time = "2026-08-10T17:57:07.643Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/6d/ec/b2f9297afc25de7c27ab5a85db4060783680e2b132285e2498fdbad484cf/e2b-2.34.0-py3-none-any.whl", hash = "sha256:873323571d18bf633be45e59fc6271410b30dfbc81e8df85e711f4f184c03fea", size = 343447, upload-time = "2026-07-17T09:59:22.553Z" },
+ { url = "https://files.pythonhosted.org/packages/03/fa/e6d50d307a9dcde63038589fce7c1cb32ce3efde3d70c20a6adb11eee3c6/e2b-2.38.0-py3-none-any.whl", hash = "sha256:9f43803aa23e9a33ed48c1c778d588f83c0390db21edf3098255137981541b67", size = 365646, upload-time = "2026-08-10T17:57:08.947Z" },
]
[[package]]
@@ -2309,6 +2323,53 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" },
]
+[[package]]
+name = "protobuf-py"
+version = "0.1.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "protobuf-py-ext", marker = "(platform_machine == 'arm64' and platform_python_implementation == 'CPython' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_python_implementation == 'CPython' and sys_platform == 'linux') or (platform_machine == 'AMD64' and platform_python_implementation == 'CPython' and sys_platform == 'win32') or (platform_machine == 'ARM64' and platform_python_implementation == 'CPython' and sys_platform == 'win32')" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/72/ed/02fd902d9c51b7ff53dfc9a745eb11490722edfd30073af889e171f07b8e/protobuf_py-0.1.1.tar.gz", hash = "sha256:6bd08ac4d8f1661965bbe2685429d79043704cdd1ee720a7a89617331742240b", size = 133525, upload-time = "2026-06-24T19:02:15.271Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/fd/00/1b3775aca1c70e3007e06ef5996f6bb9b3a32341eb0cce3ffb6effad8dec/protobuf_py-0.1.1-py3-none-any.whl", hash = "sha256:efc4f50f275ed6dae10a1f30bb81ad1a75368557b3ff22a532b7a472050368f1", size = 181656, upload-time = "2026-06-24T19:01:29.556Z" },
+]
+
+[[package]]
+name = "protobuf-py-ext"
+version = "0.1.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/2f/05/6dc9ccff1e8159eb9a144e6d3c4acfd2211cd4fcd20c34fe9155d17d6a7f/protobuf_py_ext-0.1.1.tar.gz", hash = "sha256:e85bfdfdb3ed50634db8ccc7429dd9286520109489c735463971a418707b4fef", size = 31912, upload-time = "2026-06-24T19:02:16.321Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/19/70/2b5d62a60a2e0d88ecde1ae98db3132bcd2672fb39c8d581b82b34ac0bd8/protobuf_py_ext-0.1.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:310039e03cb15181781a0b78017419f6d4ee302e988c3c70b87f1facdf05532d", size = 306008, upload-time = "2026-06-24T19:01:31.399Z" },
+ { url = "https://files.pythonhosted.org/packages/2d/d6/73c06bb4cac2e04c0adc154965fe8b6520224bd737fd7e73bba405056321/protobuf_py_ext-0.1.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89ec8348d1ba045f79b2fedd14e40cca36f2a41b52f2c4fdf55a60c58add2353", size = 314769, upload-time = "2026-06-24T19:01:32.89Z" },
+ { url = "https://files.pythonhosted.org/packages/18/c5/e4e6bc6096b66d1c82639a1b501147f16ee65d32567304b987d002e2c666/protobuf_py_ext-0.1.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:182798e4861aba72d05855bd06febe4926aa7265e6f444a1b8af5252beee4f7e", size = 328122, upload-time = "2026-06-24T19:01:34.394Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/4b/1d792a40d0f0a0f914f1dfa8bb5e9573ca0ecd5fe5cecf80d77193212abd/protobuf_py_ext-0.1.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9eb25c3a329c0551cc86b209a5e5d8ecb8d834b9924a3aa019377853a703b6d3", size = 492338, upload-time = "2026-06-24T19:01:36.103Z" },
+ { url = "https://files.pythonhosted.org/packages/52/51/5ad329223c905e5c530cae38ae24cde3584d8ab7e09457f2a01afce80e60/protobuf_py_ext-0.1.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:aaecbde82bef10c7c40578cbb61b7a19896bf7fa450972050a3bb302acb7d5d6", size = 541578, upload-time = "2026-06-24T19:01:37.481Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/c7/01bd8a8bfe2df4b07aafac12ccfb7b7ebe2303f65296a9e02fcafa28ff3e/protobuf_py_ext-0.1.1-cp310-abi3-win_amd64.whl", hash = "sha256:6b0c615c48e95acc53cf33e9310eeaff8b30d2d7555bf93e7bca8fb4f40e9a5c", size = 251319, upload-time = "2026-06-24T19:01:38.946Z" },
+ { url = "https://files.pythonhosted.org/packages/24/dc/914065538b5db54b6a920b5af38c1ef142252ea1eea711820435fa259fac/protobuf_py_ext-0.1.1-cp310-abi3-win_arm64.whl", hash = "sha256:72956cd0af5dee24b41c6f5ba5e42622d17e6d555002b5efc1634e27a1446de2", size = 241635, upload-time = "2026-06-24T19:01:40.301Z" },
+ { url = "https://files.pythonhosted.org/packages/13/a5/0b5d73cab815fd50615cb87a02e04e8332f9e27380c890aba1459cf7e919/protobuf_py_ext-0.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c6f0cd58620f415a3534d195358338f4999a774e12510f91c592b38d13d37388", size = 304903, upload-time = "2026-06-24T19:01:41.796Z" },
+ { url = "https://files.pythonhosted.org/packages/37/a9/14c120b9ac36a0e11bb05e482fa6ae322de583a8e1068e4859035c289947/protobuf_py_ext-0.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21572764f625d829604fc4d83635f533f35775f11c6da142345b3f3c8d64bd09", size = 316148, upload-time = "2026-06-24T19:01:43.247Z" },
+ { url = "https://files.pythonhosted.org/packages/35/54/7c00a1a9783c3ba74f6b2f5d2f049f09037bbd489c465c09a34f32fb611b/protobuf_py_ext-0.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7f14f00c2678bfbe7ad46f057b9f9938c1677bcf39e405e163e272c6f9814b8", size = 326458, upload-time = "2026-06-24T19:01:44.655Z" },
+ { url = "https://files.pythonhosted.org/packages/0d/0e/81fa50c0d6c4d664e5be6d0a3c4f2c7edfef87ab12d0bac764abca1f2955/protobuf_py_ext-0.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1610865622e2e27568277ca63d8d2d23dcc55eaa767398865c21539ddf4ce24d", size = 493596, upload-time = "2026-06-24T19:01:46.344Z" },
+ { url = "https://files.pythonhosted.org/packages/56/19/3183cf6e4de62846c20549654a1d573dae51ca06aaf7fed06accdfab74f6/protobuf_py_ext-0.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8995efba9476e1ea18ef9873306871dba697308994bc2766558fec3387acc3de", size = 539656, upload-time = "2026-06-24T19:01:48.21Z" },
+ { url = "https://files.pythonhosted.org/packages/a4/6c/09841f3dbe7c3d4b3f2779f8f2832ac23fb96ac8ab71674e91af732cf9ff/protobuf_py_ext-0.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ba61d49dace8f874361583030a7c48139b42eb37c9ffbb1e7e8a227a51576f44", size = 305017, upload-time = "2026-06-24T19:01:49.84Z" },
+ { url = "https://files.pythonhosted.org/packages/52/43/e450b5e202f6715274acc9acd9f9769908cbdeab861a53c798fcbd467d35/protobuf_py_ext-0.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a9c2f026096ca4c595c89a297067ef371e241d3ff8e1f6d7c779aa8419cdca7", size = 316215, upload-time = "2026-06-24T19:01:51.249Z" },
+ { url = "https://files.pythonhosted.org/packages/97/5c/23e79b35f1b5755b9bdc0c0c9b8cd8abababaef1a1639608d8a96bb61a9d/protobuf_py_ext-0.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf78707a040b9294e5e1ec4a1875f0046acfe52e92150cea27de4e9fc9db39bb", size = 326419, upload-time = "2026-06-24T19:01:52.93Z" },
+ { url = "https://files.pythonhosted.org/packages/e8/de/5493de0ec12920a3b1dd72608ce0c1e8cdb7e0eb9e87accaecc5216df29e/protobuf_py_ext-0.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ae4373845cbb85bdade3ef5368cc2f4b5f80bf173383afc1ab063f95644e5599", size = 493734, upload-time = "2026-06-24T19:01:54.301Z" },
+ { url = "https://files.pythonhosted.org/packages/98/89/31da55b414e6332aad11d858e9a86bd36fbb324f52fa3fc6d8f1c57840a9/protobuf_py_ext-0.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2a4cc478eef7a2acc1daaebcde479a3ac2396d47e6bdc7e776ca4c4147ba8b4c", size = 539703, upload-time = "2026-06-24T19:01:55.707Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/71/39f231838ef06476d46ac40dd814894277a732a3688c0a0c994850b3b62f/protobuf_py_ext-0.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:359dccdc1c3eafed2a913c570bb082b8df848a1d27d548ce8385f246a7d68be2", size = 302145, upload-time = "2026-06-24T19:01:57.116Z" },
+ { url = "https://files.pythonhosted.org/packages/87/f1/01c81ff5f420600366a0e6a613ce2af20834534d2b3d7523f4f3b4ddb54f/protobuf_py_ext-0.1.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91c38b4a10a306366443273ee03ca554537a0965bf05b7ada8e7dbdee04cc93e", size = 314541, upload-time = "2026-06-24T19:01:58.745Z" },
+ { url = "https://files.pythonhosted.org/packages/86/d8/1cbf5a0298ceddc0cdbb28bf1ecaf8bd3cff54ed4ced6a1392d5226172fc/protobuf_py_ext-0.1.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79eee3bcbb289d6ea114eb8fe3a1469c5b59bf53e207238469f567d9c53ba56f", size = 325092, upload-time = "2026-06-24T19:02:00.382Z" },
+ { url = "https://files.pythonhosted.org/packages/af/23/8b1694616044cbfec2d18d4c6fffb03e05089c8bdb5970c6bafd60cc7fd5/protobuf_py_ext-0.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:10992141a8282a71ac3e3530d1a489efb27618d84000b9a47918cf70e5816d9b", size = 491684, upload-time = "2026-06-24T19:02:01.862Z" },
+ { url = "https://files.pythonhosted.org/packages/49/d8/99997a7a6cf6989c944d9183c5deb6a045c27460add0316c7b34763891b3/protobuf_py_ext-0.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4411ffe0e06a774b83c5c71c546ce097640a25f596c45f95f53d3e3148e3f22d", size = 538048, upload-time = "2026-06-24T19:02:03.362Z" },
+ { url = "https://files.pythonhosted.org/packages/15/3c/9e99ecbb68e1d5b46016d821eb1cbba9a91e6b50e71e75faf0c1e6189f0d/protobuf_py_ext-0.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:55a17d80ea419501ff221a6627523f38a431bb33e6aa3de81ae3a7f271c49c75", size = 296337, upload-time = "2026-06-24T19:02:04.787Z" },
+ { url = "https://files.pythonhosted.org/packages/af/f2/305338a28225fb54b28d6c3f5948109b9088b329a2b6f77ca610c2bdcd34/protobuf_py_ext-0.1.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dbb518b5638403ceaa08ac4fc7dac626f45ed9b856b3517de3906cf3de4d632", size = 308961, upload-time = "2026-06-24T19:02:06.185Z" },
+ { url = "https://files.pythonhosted.org/packages/60/f9/416103c93677ff2ea407704ea64fa6de9e700dc030c48360a182d91ce373/protobuf_py_ext-0.1.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a4adc65ab6a5e4885c67fc808bcacb83d755d05b566d312a0e10c2a873f2ad4", size = 321895, upload-time = "2026-06-24T19:02:07.714Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/83/eb3e81bb2f83834b7deff4cee2d56eeb1adcbc847492a56b7f910ab257db/protobuf_py_ext-0.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e97f45c49676efacfb8e95bfcb1a002bc337e618a6780be1e55df2ba5ebd2f1e", size = 486091, upload-time = "2026-06-24T19:02:09.243Z" },
+ { url = "https://files.pythonhosted.org/packages/99/96/bd88fca38556b3105e4dd41d6a176e31dcc583fe979e830aeedd2f8c20ee/protobuf_py_ext-0.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:53a6b3590f6aa7f97b8ed60f62fef9b096babfeae82f7283fd3a4c405827d4f8", size = 534582, upload-time = "2026-06-24T19:02:11.227Z" },
+]
+
[[package]]
name = "psutil"
version = "7.2.2"
@@ -2624,6 +2685,52 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/bc/67/4759522f5bca0ac4cda9f42c7f3f818aa826568793bd8b4532d2d2ffa515/pypdfium2-5.7.1-py3-none-win_arm64.whl", hash = "sha256:622821698fcc30fc560bd4eead6df9e6b846de9876b82861bed0091c09a4c27b", size = 3540903, upload-time = "2026-04-20T15:01:00.994Z" },
]
+[[package]]
+name = "pyqwest"
+version = "0.9.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "opentelemetry-api" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8d/59/97531fd9d0a06d54e84f5493775739b0c1d0d13ec704974d04fa423a3332/pyqwest-0.9.0.tar.gz", hash = "sha256:514dd0b37d7a1bcb978b5d6423d15f89cf7dcf8058ac2a37aa42cd30090de89e", size = 477462, upload-time = "2026-08-10T01:55:36.219Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/47/34/d556fa07d9bff21fc6c9c16738614c6336164b65d9ced1ce9f1c1044aaeb/pyqwest-0.9.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:517d2eb295d56ef1530cebe0cf290a66fa199c57955bd1c5f1dc592240fbb7d6", size = 5224594, upload-time = "2026-08-10T01:54:18.646Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/3f/1d7d1a08ebf8838c0c85e99ab5db5c1aae457f76ee03b60c8f4fc4948e01/pyqwest-0.9.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:408dc692835363b6824bee61ee9e4c9a5bba9c08d75b903f82f9a802e41d1b30", size = 5100753, upload-time = "2026-08-10T01:54:20.669Z" },
+ { url = "https://files.pythonhosted.org/packages/66/fe/22c4056b526baa4c2c953f4e99925ce0bb14e7747d66e6ac3448a34f76e5/pyqwest-0.9.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1abf1ab3a0c1b545651ab37a643acc7909c4e7644a63a2e4af36e3e230a0db5", size = 5619882, upload-time = "2026-08-10T01:54:22.468Z" },
+ { url = "https://files.pythonhosted.org/packages/de/2c/8f1159a30bbc905b2fcbc386740a260e76ae64372feb263103438ef5201c/pyqwest-0.9.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b02e5c9ce57e079eb3716c464d3818ad2573d7743483c39526ad77ee95af806", size = 5564744, upload-time = "2026-08-10T01:54:24.814Z" },
+ { url = "https://files.pythonhosted.org/packages/ac/fd/ff15034d7874fd208d606bd80c924fb0b1091db8159c892672168b656de1/pyqwest-0.9.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:98bdaa30a3a8d8c71506a33c084e485a243fbf51e77130bb39e0f249dd116142", size = 5779373, upload-time = "2026-08-10T01:54:26.753Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/b5/eeafdeb65ad8fe9c39044aedcfc97610f4e5c7b5620ca9fe2d504f669108/pyqwest-0.9.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:10eba84f9d8512d23bd16dd60b1c77dab956470361559214981dda065df4ad1a", size = 5973306, upload-time = "2026-08-10T01:54:28.729Z" },
+ { url = "https://files.pythonhosted.org/packages/34/db/a8979dab5c591529f858d1505d09b3c8b6dec337921f15bed4858c55716e/pyqwest-0.9.0-cp310-abi3-win_amd64.whl", hash = "sha256:af4e859800b02cd1fcdd898af26a881c4a8ce9520f71f6cb0a5403523d2cbbdf", size = 4834706, upload-time = "2026-08-10T01:54:30.335Z" },
+ { url = "https://files.pythonhosted.org/packages/1d/9e/d070ca1ce3c202952770436172889a4b0556a564a490baf726f7bb895eba/pyqwest-0.9.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:abe322cc1b63947b493bc06028615e0d5de655928159227aab64f60a0eb0e27d", size = 5244745, upload-time = "2026-08-10T01:54:32.257Z" },
+ { url = "https://files.pythonhosted.org/packages/41/97/ebded0bb35a79f51bb2d1519e34922d79c8825f9e71b696bb20f387a08c2/pyqwest-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c6b2dc6a9c583859d031941e1041b8e711afe0ad4c130628207731cef2b1a486", size = 5101044, upload-time = "2026-08-10T01:54:34.088Z" },
+ { url = "https://files.pythonhosted.org/packages/a9/2c/a7f117e793b4e27642807bf7230b9fa297ff1e1ac489fc260fc3d4350ae4/pyqwest-0.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:551e47714da72a72939958029eb37a754cecd974344471c4a2038583a66a5396", size = 5630841, upload-time = "2026-08-10T01:54:35.885Z" },
+ { url = "https://files.pythonhosted.org/packages/b4/b4/be47fc141529941352a848cd77da581803137b733993a51bf464f2c48915/pyqwest-0.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3be3d38ccdab3077bf1cfe90c346ce927f9b49c56dcdf20e060f5f2502d01021", size = 5572909, upload-time = "2026-08-10T01:54:37.653Z" },
+ { url = "https://files.pythonhosted.org/packages/cb/fd/703635e12710ce7ef5e961f5e1b22c609aecc8f07eedd79477b48d1edb74/pyqwest-0.9.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:45d9d424da878b22604799d1ce2a0a5df054924487ee5a851ad08025e2076e5c", size = 5794049, upload-time = "2026-08-10T01:54:39.363Z" },
+ { url = "https://files.pythonhosted.org/packages/3c/15/941aad56b108c743df9d41e124be82ecb6be0eba1007e945b6a39f5dd1a0/pyqwest-0.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e4420adcd2b756da706b31a04f095a521f6e440dab284b7a1b46c76e048c1f78", size = 5974542, upload-time = "2026-08-10T01:54:41.202Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/86/8445fe05b47322047347464a909ae95dba023bf8409b9967f66bda2905ad/pyqwest-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:25734bd9798bbfaa53c83d395649dd9ea9a791a64b9848734955cb78ad52c832", size = 4847034, upload-time = "2026-08-10T01:54:42.899Z" },
+ { url = "https://files.pythonhosted.org/packages/a7/92/013fc3c94f0dc4eeca556431797da477d76ad43d69134837ba149468dcc3/pyqwest-0.9.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:6e4c2afe6c7293d9e42cfdb0489b0b5f64002159c2dc9c45d9322c1b4ffb42cd", size = 5244793, upload-time = "2026-08-10T01:54:44.891Z" },
+ { url = "https://files.pythonhosted.org/packages/48/61/c685e0c595f3f1b7842a776c48aaf2b3f47270a61672967d9d38f7c5e7f6/pyqwest-0.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:67f153b94dfbb9aeabc38f20a1bada1acfd9361bec12db86268bd951be35a154", size = 5100715, upload-time = "2026-08-10T01:54:46.777Z" },
+ { url = "https://files.pythonhosted.org/packages/f9/a1/a28b2cd704d7dd37043f1e6b398798576c12293f3efae84f00e7f6a5625c/pyqwest-0.9.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9942ffa243c8c2243e3ce04e3e3ec9b4c119346c963fd0051cd989f0defecf6", size = 5628787, upload-time = "2026-08-10T01:54:48.488Z" },
+ { url = "https://files.pythonhosted.org/packages/36/31/6255c7f17cd00e39f1c4df3003cea2d2d087e483fb4c49a6535307cd21e0/pyqwest-0.9.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f8f9c880c19826fead1838e12b042865b15df3ec844928cc74188a8b740f761", size = 5571946, upload-time = "2026-08-10T01:54:50.181Z" },
+ { url = "https://files.pythonhosted.org/packages/37/71/344738bdb38bddbc07cb29e5b1287c08db9e53d6b322581492571bdef748/pyqwest-0.9.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:56040761f6ca7abb036c7d288863bf143a9b166d824d98a604908ef20c6906e0", size = 5792872, upload-time = "2026-08-10T01:54:52.045Z" },
+ { url = "https://files.pythonhosted.org/packages/fc/ee/c5c74e3afb6dd0a25bca23bb4757d1a7135fab09a0a054036e837d49dd11/pyqwest-0.9.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:02a51df744521e45eb5379bc325b91de4512ec64a9f4aba3248f9166ec4e5b88", size = 5974192, upload-time = "2026-08-10T01:54:53.818Z" },
+ { url = "https://files.pythonhosted.org/packages/6e/08/302960836e3e43ca3051313ee3b767534d1610bd3b8033c7bbad41b06e1b/pyqwest-0.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:e9a0e3123aee446d5f6e57c43cae87b4dab4b1af3fa970d114d74d5d4b9d075c", size = 4847050, upload-time = "2026-08-10T01:54:56.155Z" },
+ { url = "https://files.pythonhosted.org/packages/fc/3e/a36940844325fbc59adf92116d3bcd1155cc2c72d1a390880886ca492bb4/pyqwest-0.9.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:fdab21bbedff4a21209135423f5f54f287a01dd6143cbd7aeab63d5d8c889654", size = 5246741, upload-time = "2026-08-10T01:54:57.697Z" },
+ { url = "https://files.pythonhosted.org/packages/41/8d/efb9741449a81f2489f6929a7691a2145662c3650e06bcb139c64f543d0a/pyqwest-0.9.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:21a04ddf4497ef3c7fe36cdc473c094999b7710005d38bd8a0dfd4a3feace12f", size = 5102423, upload-time = "2026-08-10T01:54:59.253Z" },
+ { url = "https://files.pythonhosted.org/packages/8f/a8/0b667cf9c07e62861cfe24c09ad30357a9c2a0fc3863cfc5610af5b66e3e/pyqwest-0.9.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8c51c1a27ea529fffab90c127a045f0dfe1abc54d5b9ad5f4165e954439c9b2", size = 5627663, upload-time = "2026-08-10T01:55:01.036Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/7c/1f88be3d35afe92b79e361aa40faf3904e3a7852e8638a9d3d5c00891ab3/pyqwest-0.9.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10f283ffc40a2774e0dc8fbb47e4a8f9f8a698a99feae3bf4ad75b340c98031b", size = 5572750, upload-time = "2026-08-10T01:55:03.163Z" },
+ { url = "https://files.pythonhosted.org/packages/21/3e/98c1e151772e77dcf00cb8b1c070f2c8edcb15fe12fdee30c1022006ca8e/pyqwest-0.9.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1f73d4a8e75b5860741fbd0623a0ce792a53336dbfec5f34736004acd5446843", size = 5790615, upload-time = "2026-08-10T01:55:04.758Z" },
+ { url = "https://files.pythonhosted.org/packages/7b/f1/12bb4b119cffb7fe5301b2d997ec9ffc1719637f6f4faf2ff2315e13b201/pyqwest-0.9.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e34d55a53492b44b82b690c2b11957fc35e99ebab4db2e9bcb2a1e2621d71b62", size = 5976516, upload-time = "2026-08-10T01:55:06.453Z" },
+ { url = "https://files.pythonhosted.org/packages/53/55/046dbf68ccedd816770fd212c6c2612372a6f0d7ecf2d8273480db460ddf/pyqwest-0.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:48c367150783f6866d0af43c209cc405f8287743d7b2472c7d757fda2478fd57", size = 4841470, upload-time = "2026-08-10T01:55:08.035Z" },
+ { url = "https://files.pythonhosted.org/packages/55/cd/38ff77d145385afd71b2dbd6eb116251ce3ffddbed0edde16c64394fbec2/pyqwest-0.9.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:10fae206270355c993b49c49bf9778995167f5efe591807d50a8e682921d71ea", size = 5222689, upload-time = "2026-08-10T01:55:09.639Z" },
+ { url = "https://files.pythonhosted.org/packages/82/95/d8c6804eeb72b7b89842b58f1f05d5cfda5be410c4a22442d2574d510d39/pyqwest-0.9.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5f475129fb792593222a9ce0fefbbc10bda1544a7411f3946d771c9a61c50e61", size = 5085846, upload-time = "2026-08-10T01:55:11.569Z" },
+ { url = "https://files.pythonhosted.org/packages/f2/e7/04fe8962ec416be0199093ebdd326653a5a7b8f53ddfdb29af81756f7c08/pyqwest-0.9.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78f6f676d349535cb094e8e3bcf33863238a142fb7dcc7afb9e61749e2351047", size = 5613296, upload-time = "2026-08-10T01:55:13.315Z" },
+ { url = "https://files.pythonhosted.org/packages/03/0c/7a11728155a34838d2e7c78fb24c937f2cd844ec791ef416030c52f4bb5e/pyqwest-0.9.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f460f5fcdae8cc46e6a679128836f6cd47fbf4bf73e5c9d6ea868e4a90b2e7ee", size = 5557492, upload-time = "2026-08-10T01:55:15.103Z" },
+ { url = "https://files.pythonhosted.org/packages/4a/d5/8c6e68f0e978b5257b309809fec00d946c1ed20dacef97f25c23de5b91a9/pyqwest-0.9.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:cba6804151e973d27c99b9f02b3f0bcab7270c3f99725b508ada9c147365cabd", size = 5774204, upload-time = "2026-08-10T01:55:16.932Z" },
+ { url = "https://files.pythonhosted.org/packages/17/20/fed89b62d50b4efad2ebe78c3867e0e402f824726652034728a5dd0a342e/pyqwest-0.9.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:bc9dfeca326918d70a5368114de0fc627d9f64fafa36563ab484ae58ffe42c0d", size = 5964506, upload-time = "2026-08-10T01:55:19.031Z" },
+ { url = "https://files.pythonhosted.org/packages/11/de/30575019efebae84502269d097e706e89a30631e3719218b053e4e0dc179/pyqwest-0.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:384693adc0908f16324e5b508a3c25fb9a92b88f9bdf7ccb6820507db4a95888", size = 4809013, upload-time = "2026-08-10T01:55:21.042Z" },
+]
+
[[package]]
name = "pytest"
version = "9.0.3"