Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(dify-agent): harden E2B binding lifecycle (#41623)
Ingram Za1a10f17e32e
diff --git a/api/uv.lock b/api/uv.lockindex d7539edbf7a..3146632b97d 100644--- a/api/uv.lock+++ b/api/uv.lock@@ -1309,7 +1309,7 @@ dependencies = [[package.metadata]requires-dist = [- { name = "e2b", marker = "extra == 'server'", specifier = ">=2.34.0,<3.0.0" },+ { name = "e2b", marker = "extra == 'server'", specifier = ">=2.38.0,<3.0.0" },{ name = "fastapi", marker = "extra == 'server'", specifier = "==0.136.0" },{ name = "graphon", marker = "extra == 'server'", specifier = "==0.5.2" },{ name = "httpx", specifier = "==0.28.1" },diff --git a/dify-agent/pyproject.toml b/dify-agent/pyproject.tomlindex 07d0dec20ad..58dcc648151 100644--- a/dify-agent/pyproject.toml+++ b/dify-agent/pyproject.toml@@ -19,7 +19,7 @@ dify-agent-stub-server = "dify_agent.agent_stub.server.cli:main"[project.optional-dependencies]server = [- "e2b>=2.34.0,<3.0.0",+ "e2b>=2.38.0,<3.0.0","fastapi==0.136.0","graphon==0.5.2","jsonschema>=4.23.0,<5.0.0",diff --git a/dify-agent/src/dify_agent/runtime_backend/__init__.py b/dify-agent/src/dify_agent/runtime_backend/__init__.pyindex c274a93fb53..607fddd3242 100644--- a/dify-agent/src/dify_agent/runtime_backend/__init__.py+++ b/dify-agent/src/dify_agent/runtime_backend/__init__.py@@ -2,6 +2,7 @@from .errors import (BindingAcquireError,+ BindingCapacityExhaustedError,BindingCreateError,BindingDestroyError,BindingLostError,@@ -27,6 +28,7 @@ from .protocols import (__all__ = ["BindingAcquireError",+ "BindingCapacityExhaustedError","BindingCreateError","BindingDestroyError","BindingLostError",diff --git a/dify-agent/src/dify_agent/runtime_backend/e2b.py b/dify-agent/src/dify_agent/runtime_backend/e2b.pyindex 4dad6ff13bc..7f09cb5a9f3 100644--- a/dify-agent/src/dify_agent/runtime_backend/e2b.py+++ b/dify-agent/src/dify_agent/runtime_backend/e2b.py@@ -9,9 +9,12 @@ operation-local and are never serialized into Agenton state.from __future__ import annotationsimport asyncio+from collections.abc import Awaitable, Callablefrom dataclasses import dataclass, field-from typing import TYPE_CHECKING, Literal, Protocol, cast+import logging+from typing import TYPE_CHECKING, Literal, Protocol, TypeVar, cast+import httpx as e2b_httpximport httpx2 as httpxfrom shellctl.client import ShellctlClientError@@ -19,6 +22,7 @@ from dify_agent.adapters.shell.protocols import ShellCommandProtocolfrom dify_agent.adapters.shell.shellctl import ShellctlClientProtocolfrom dify_agent.runtime_backend.errors import (BindingAcquireError,+ BindingCapacityExhaustedError,BindingCreateError,BindingDestroyError,BindingLostError,@@ -41,14 +45,30 @@ if TYPE_CHECKING:# One RuntimeLease spans the complete Agent run, not one Shell tool call.E2B_MAX_ACTIVE_TIMEOUT_SECONDS = 60 * 60+_E2B_CONTROL_PLANE_MAX_ATTEMPTS = 2+_E2B_CONTROL_PLANE_RETRY_INTERVAL_SECONDS = 0.25_SHELLCTL_READY_MAX_ATTEMPTS = 3_SHELLCTL_READY_RETRY_INTERVAL_SECONDS = 0.5+_RETRYABLE_E2B_TRANSPORT_ERRORS = (+ e2b_httpx.ConnectError,+ e2b_httpx.ReadTimeout,+ e2b_httpx.ReadError,+ e2b_httpx.WriteError,+ e2b_httpx.RemoteProtocolError,+)++_ResultT = TypeVar("_ResultT")+logger = logging.getLogger(__name__)class _E2BControlPlaneNotFoundError(RuntimeError):"""Typed boundary error for SDK resources that no longer exist."""+class _E2BControlPlaneCapacityExhaustedError(RuntimeError):+ """Typed boundary error for provider-side Sandbox capacity exhaustion."""++class _E2BFileEntry(Protocol):path: str@@ -121,7 +141,7 @@ class E2BSDKControlPlane:metadata: dict[str, str],on_timeout: Literal["kill", "pause"],) -> _E2BSandbox:- from e2b import AsyncSandbox, NotFoundException, SandboxNotFoundException+ from e2b import AsyncSandbox, NotFoundException, RateLimitException, SandboxNotFoundExceptiontry:return cast(@@ -140,6 +160,8 @@ class E2BSDKControlPlane:)except (SandboxNotFoundException, NotFoundException) as exc:raise _E2BControlPlaneNotFoundError(str(exc)) from exc+ except RateLimitException as exc:+ raise _E2BControlPlaneCapacityExhaustedError(str(exc)) from excasync def connect(self, handle: str, *, timeout: int) -> _E2BSandbox:from e2b import AsyncSandbox, NotFoundException, SandboxNotFoundException@@ -195,7 +217,12 @@ class E2BHomeSnapshotBackend:async def delete(self, snapshot_ref: str) -> None:try:- _ = await self.control_plane.delete_snapshot(snapshot_ref)+ _ = await _run_e2b_idempotent_operation(+ operation="delete_snapshot",+ sandbox_id=snapshot_ref,+ cleanup_stage="home_snapshot_delete",+ action=lambda: self.control_plane.delete_snapshot(snapshot_ref),+ )except _E2BControlPlaneNotFoundError:returnexcept Exception as exc:@@ -243,14 +270,33 @@ class E2BExecutionBindingBackend:for entry in await sandbox.files.list(self.layout.workspace_dir):await sandbox.files.remove(entry.path)sandbox_id = sandbox.sandbox_id- _ = await sandbox.pause(keep_memory=True)+ _ = await _run_e2b_idempotent_operation(+ operation="pause",+ sandbox_id=sandbox_id,+ cleanup_stage="binding_create",+ action=lambda: sandbox.pause(keep_memory=True),+ )return ExecutionBindingAllocation(binding_ref=sandbox_id, workspace_ref=sandbox_id)+ except _E2BControlPlaneCapacityExhaustedError as exc:+ raise BindingCapacityExhaustedError(str(exc)) from excexcept BaseException as exc:if sandbox is not None:try:- _ = await sandbox.kill()- except BaseException:- pass+ _ = await _run_e2b_idempotent_operation(+ operation="kill",+ sandbox_id=sandbox.sandbox_id,+ cleanup_stage="binding_create_compensation",+ action=sandbox.kill,+ )+ except Exception as cleanup_exc:+ _log_e2b_cleanup_warning(+ "failed to remove partial E2B Binding",+ operation="kill",+ sandbox_id=sandbox.sandbox_id,+ attempt=_e2b_failure_attempt(cleanup_exc),+ cleanup_stage="binding_create_compensation",+ exception=cleanup_exc,+ )if isinstance(exc, Exception):if isinstance(exc, BindingCreateError):raise@@ -262,7 +308,12 @@ class E2BExecutionBindingBackend:sandbox: _E2BSandbox | None = Nonelease: E2BRuntimeLease | None = Nonetry:- sandbox = await self.control_plane.connect(binding_ref, timeout=self.active_timeout_seconds)+ sandbox = await _run_e2b_idempotent_operation(+ operation="connect",+ sandbox_id=binding_ref,+ cleanup_stage="binding_acquire",+ action=lambda: self.control_plane.connect(binding_ref, timeout=self.active_timeout_seconds),+ )if not await sandbox.files.exists(self.layout.workspace_dir):raise BindingLostError(f"E2B Binding {binding_ref!r} no longer contains its Workspace")lease = await self._lease(sandbox)@@ -281,20 +332,48 @@ class E2BExecutionBindingBackend:raiseasync def release(self, lease: RuntimeLease) -> None:- """Close operation-local transports and pause the physical E2B resource."""+ """Close operation-local transports and best-effort pause the E2B resource."""if not isinstance(lease, E2BRuntimeLease):raise TypeError("E2BExecutionBindingBackend can only release its own RuntimeLease")- close_error: Exception | None = None+ close_base_error: BaseException | None = Nonetry:await lease.data_plane.close()except Exception as exc:- close_error = exc+ _log_e2b_cleanup_warning(+ "failed to close E2B RuntimeLease data plane",+ operation="close_data_plane",+ sandbox_id=lease.sandbox.sandbox_id,+ attempt=1,+ cleanup_stage="binding_release",+ exception=exc,+ )+ except BaseException as exc:+ close_base_error = exc+ pause_base_error: BaseException | None = Nonetry:- _ = await lease.sandbox.pause(keep_memory=True)+ _ = await _run_e2b_idempotent_operation(+ operation="pause",+ sandbox_id=lease.sandbox.sandbox_id,+ cleanup_stage="binding_release",+ action=lambda: lease.sandbox.pause(keep_memory=True),+ )except Exception as exc:- raise BindingAcquireError(str(exc)) from exc- if close_error is not None:- raise BindingAcquireError(str(close_error)) from close_error+ _log_e2b_cleanup_warning(+ "failed to pause E2B RuntimeLease",+ operation="pause",+ sandbox_id=lease.sandbox.sandbox_id,+ attempt=_e2b_failure_attempt(exc),+ cleanup_stage="binding_release",+ exception=exc,+ )+ except BaseException as exc:+ pause_base_error = exc+ if close_base_error is not None:+ if pause_base_error is not None:+ raise close_base_error from pause_base_error+ raise close_base_error+ if pause_base_error is not None:+ raise pause_base_errorasync def destroy_binding(self, spec: ExecutionBindingDestroySpec) -> None:"""Destroy the coupled physical Binding and Workspace idempotently."""@@ -305,7 +384,12 @@ class E2BExecutionBindingBackend:if spec.workspace_ref != spec.binding_ref:raise BindingDestroyError("E2B Workspace ref must equal its Binding ref")try:- _ = await self.control_plane.kill(spec.binding_ref)+ _ = await _run_e2b_idempotent_operation(+ operation="kill",+ sandbox_id=spec.binding_ref,+ cleanup_stage="binding_destroy",+ action=lambda: self.control_plane.kill(spec.binding_ref),+ )except _E2BControlPlaneNotFoundError:returnexcept Exception as exc:@@ -381,22 +465,121 @@ async def _wait_for_shellctl_ready(client: ShellctlClientProtocol) -> None:await asyncio.sleep(_SHELLCTL_READY_RETRY_INTERVAL_SECONDS)+async def _run_e2b_idempotent_operation(+ *,+ operation: str,+ sandbox_id: str,+ cleanup_stage: str,+ action: Callable[[], Awaitable[_ResultT]],+) -> _ResultT:+ """Retry one idempotent E2B lifecycle operation after a transport failure."""+ for attempt in range(1, _E2B_CONTROL_PLANE_MAX_ATTEMPTS + 1):+ try:+ return await action()+ except _RETRYABLE_E2B_TRANSPORT_ERRORS as exc:+ exhausted = attempt == _E2B_CONTROL_PLANE_MAX_ATTEMPTS+ logger.warning(+ "E2B lifecycle operation exhausted retries" if exhausted else "retrying E2B lifecycle operation",+ exc_info=True,+ extra=_e2b_log_extra(+ operation=operation,+ sandbox_id=sandbox_id,+ attempt=attempt,+ cleanup_stage=cleanup_stage,+ exception=exc,+ outcome="retry_exhausted" if exhausted else "retrying",+ ),+ )+ if exhausted:+ raise+ await asyncio.sleep(_E2B_CONTROL_PLANE_RETRY_INTERVAL_SECONDS)+ raise AssertionError("unreachable")+++def _e2b_log_extra(+ *,+ operation: str,+ sandbox_id: str,+ attempt: int,+ cleanup_stage: str,+ exception: BaseException,+ outcome: str,+) -> dict[str, object]:+ return {+ "e2b_operation": operation,+ "sandbox_id": sandbox_id,+ "attempt": attempt,+ "cleanup_stage": cleanup_stage,+ "exception_type": type(exception).__name__,+ "outcome": outcome,+ }+++def _log_e2b_cleanup_warning(+ message: str,+ *,+ operation: str,+ sandbox_id: str,+ attempt: int,+ cleanup_stage: str,+ exception: BaseException,+) -> None:+ logger.warning(+ message,+ exc_info=True,+ stacklevel=2,+ extra=_e2b_log_extra(+ operation=operation,+ sandbox_id=sandbox_id,+ attempt=attempt,+ cleanup_stage=cleanup_stage,+ exception=exception,+ outcome="ignored_cleanup_failure",+ ),+ )+++def _e2b_failure_attempt(exception: BaseException) -> int:+ if isinstance(exception, _RETRYABLE_E2B_TRANSPORT_ERRORS):+ return _E2B_CONTROL_PLANE_MAX_ATTEMPTS+ return 1++async def _best_effort_close_data_plane(lease: E2BRuntimeLease | None) -> None:if lease is None:returntry:await lease.data_plane.close()- except BaseException:- pass+ except BaseException as exc:+ _log_e2b_cleanup_warning(+ "failed to close E2B RuntimeLease data plane after acquire failure",+ operation="close_data_plane",+ sandbox_id=lease.sandbox.sandbox_id,+ attempt=1,+ cleanup_stage="binding_acquire_compensation",+ exception=exc,+ )async def _best_effort_pause(sandbox: _E2BSandbox | None) -> None:if sandbox is None:returntry:- _ = await sandbox.pause(keep_memory=True)- except BaseException:- pass+ _ = await _run_e2b_idempotent_operation(+ operation="pause",+ sandbox_id=sandbox.sandbox_id,+ cleanup_stage="binding_acquire_compensation",+ action=lambda: sandbox.pause(keep_memory=True),+ )+ except BaseException as exc:+ _log_e2b_cleanup_warning(+ "failed to pause E2B Binding after acquire failure",+ operation="pause",+ sandbox_id=sandbox.sandbox_id,+ attempt=_e2b_failure_attempt(exc),+ cleanup_stage="binding_acquire_compensation",+ exception=exc,+ )__all__ = [diff --git a/dify-agent/src/dify_agent/runtime_backend/errors.py b/dify-agent/src/dify_agent/runtime_backend/errors.pyindex 627b4fb8851..c08b118b80e 100644--- a/dify-agent/src/dify_agent/runtime_backend/errors.py+++ b/dify-agent/src/dify_agent/runtime_backend/errors.py@@ -21,6 +21,10 @@ class BindingCreateError(RuntimeBackendError):pass+class BindingCapacityExhaustedError(BindingCreateError):+ """The selected runtime backend cannot allocate another Binding."""++class BindingAcquireError(RuntimeBackendError):pass@@ -47,6 +51,7 @@ class WorkspaceUnavailableError(RuntimeBackendError):__all__ = ["BindingAcquireError",+ "BindingCapacityExhaustedError","BindingCreateError","BindingDestroyError","BindingLostError",diff --git a/dify-agent/src/dify_agent/server/execution_bindings.py b/dify-agent/src/dify_agent/server/execution_bindings.pyindex fbecb129daa..06e6ff39003 100644--- a/dify-agent/src/dify_agent/server/execution_bindings.py+++ b/dify-agent/src/dify_agent/server/execution_bindings.py@@ -8,6 +8,7 @@ from dify_agent.protocol import (DestroyExecutionBindingRequest,)from dify_agent.runtime_backend import (+ BindingCapacityExhaustedError,BindingCreateError,BindingDestroyError,ExecutionBindingBackend,@@ -48,6 +49,8 @@ class ExecutionBindingService:)except SharedWorkspaceUnsupportedError as exc:raise ExecutionBindingServiceError("shared_workspace_unsupported", str(exc), status_code=409) from exc+ except BindingCapacityExhaustedError as exc:+ raise ExecutionBindingServiceError("binding_capacity_exhausted", str(exc), status_code=429) from excexcept BindingCreateError as exc:raise ExecutionBindingServiceError("binding_create_failed", str(exc), status_code=502) from excreturn CreateExecutionBindingResponse(diff --git a/dify-agent/tests/integration/dify_agent/server/test_e2b_transport_soak.py b/dify-agent/tests/integration/dify_agent/server/test_e2b_transport_soak.pynew file mode 100644index 00000000000..ff15dea6483--- /dev/null+++ b/dify-agent/tests/integration/dify_agent/server/test_e2b_transport_soak.py@@ -0,0 +1,1490 @@+"""Opt-in DIFY-2966 transport soak against a deployed E2B Agent Backend.++The formal profile intentionally keeps its 32-way concurrency gate fixed. The+``DIFY_AGENT_TEST_SOAK_*`` numeric overrides exist only for harness development;+the JSONL ledger labels every non-default run as ``formal_gate=false``.++E2B 2.38's public async API supports metadata-filtered pagination and a static+kill-by-id operation, which lets the final reconciliation stay scoped to the+run marker created by this test:+https://github.com/e2b-dev/E2B/blob/%40e2b/python-sdk%402.38.0/packages/python-sdk/e2b/sandbox_async/sandbox_api.py+"""++from __future__ import annotations++import asyncio+import json+import math+import os+import re+import time+from collections.abc import Awaitable, Callable, Mapping, Sequence+from dataclasses import dataclass, replace+from datetime import UTC, datetime+from pathlib import Path+from types import TracebackType+from typing import IO, Self, cast++import pytest++from dify_agent.client import Client, DifyAgentHTTPError+from dify_agent.protocol import CreateExecutionBindingRequest, DestroyExecutionBindingRequest++pytestmark = pytest.mark.integration++_FORMAL_CONCURRENCY = 32+_FORMAL_BURST_ROUNDS = 10+_FORMAL_IDLE_BINDINGS = 32+_FORMAL_IDLE_INTERVAL_SECONDS = 600.0+_FORMAL_IDLE_ROUNDS = 12+_DEFAULT_REQUEST_TIMEOUT_SECONDS = 180.0+_DEFAULT_RECONCILE_TIMEOUT_SECONDS = 60.0+_RECONCILE_POLL_SECONDS = 2.0+_RECONCILE_FINAL_PROOF_RESERVE_SECONDS = 10.0+_RECONCILE_STABLE_EMPTY_SECONDS = 3.0+_RECONCILE_FINAL_SCAN_INTERVAL_SECONDS = 0.5+_RECONCILE_RETRY_BACKOFF_SECONDS = 0.1+_TENANT_ID = "dify-2966-transport-soak"+# Binding file reads intentionally reject symlinks via O_NOFOLLOW. `/etc/os-release`+# is a symlink in the E2B image, while `/etc/hostname` is a small regular file.+_READ_PROBE_PATH = "/etc/hostname"+_RUN_ID_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,47}\Z")+_REDACTED = "[REDACTED]"+_SECRET_KEY_PARTS = ("api_key", "authorization", "password", "secret", "token")+++@dataclass(frozen=True, slots=True)+class _SoakSettings:+ service_url: str+ service_token: str+ e2b_api_key: str+ run_id: str+ artifact: Path+ concurrency: int+ burst_rounds: int+ idle_bindings: int+ idle_interval_seconds: float+ idle_rounds: int+ request_timeout_seconds: float+ reconcile_timeout_seconds: float++ @property+ def binding_prefix(self) -> str:+ return f"dify-2966-{self.run_id}-"++ @property+ def formal_gate(self) -> bool:+ return (+ self.concurrency == _FORMAL_CONCURRENCY+ and self.burst_rounds == _FORMAL_BURST_ROUNDS+ and self.idle_bindings == _FORMAL_IDLE_BINDINGS+ and self.idle_interval_seconds == _FORMAL_IDLE_INTERVAL_SECONDS+ and self.idle_rounds == _FORMAL_IDLE_ROUNDS+ and self.request_timeout_seconds == _DEFAULT_REQUEST_TIMEOUT_SECONDS+ and self.reconcile_timeout_seconds == _DEFAULT_RECONCILE_TIMEOUT_SECONDS+ )+++@dataclass(frozen=True, slots=True)+class _Binding:+ binding_id: str+ binding_ref: str+ workspace_ref: str+++@dataclass(frozen=True, slots=True)+class _OwnedSandbox:+ sandbox_id: str+ binding_id: str+ workspace_id: str+ state: str+++_ListOwnedSandboxes = Callable[[_SoakSettings], Awaitable[list[_OwnedSandbox]]]+_KillOwnedSandbox = Callable[[_SoakSettings, str], Awaitable[bool]]+_Sleep = Callable[[float], Awaitable[None]]+_Monotonic = Callable[[], float]+++class _JsonlLedger:+ """Flush one sanitized JSON object per line to an exclusive artifact."""++ _handle: IO[str]+ _secrets: tuple[str, ...]++ def __init__(self, path: Path, *, secrets: Sequence[str]) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ self._handle = path.open("x", encoding="utf-8")+ self._secrets = tuple(secret for secret in secrets if secret)++ def __enter__(self) -> Self:+ return self++ def __exit__(+ self,+ exc_type: type[BaseException] | None,+ exc_value: BaseException | None,+ traceback: TracebackType | None,+ ) -> None:+ del exc_type, exc_value, traceback+ self._handle.close()++ def record(self, event: str, **fields: object) -> None:+ payload = {+ "timestamp": datetime.now(UTC).isoformat(),+ "event": event,+ **fields,+ }+ sanitized = _sanitize_for_ledger(payload, secrets=self._secrets)+ _ = self._handle.write(json.dumps(sanitized, separators=(",", ":"), sort_keys=True) + "\n")+ self._handle.flush()+++def _sanitize_for_ledger(value: object, *, secrets: Sequence[str], key: str | None = None) -> object:+ if key is not None and any(part in key.lower() for part in _SECRET_KEY_PARTS):+ return _REDACTED+ if isinstance(value, Mapping):+ mapping = cast(Mapping[object, object], value)+ return {+ str(item_key): _sanitize_for_ledger(item, secrets=secrets, key=str(item_key))+ for item_key, item in mapping.items()+ }+ if isinstance(value, list | tuple):+ sequence = cast(Sequence[object], value)+ return [_sanitize_for_ledger(item, secrets=secrets) for item in sequence]+ if isinstance(value, str):+ sanitized = value+ for secret in secrets:+ if sanitized == secret:+ return _REDACTED+ sanitized = sanitized.replace(secret, _REDACTED)+ return sanitized+ if value is None or isinstance(value, bool | int | float):+ return value+ return str(value)+++def _load_settings(env: Mapping[str, str]) -> _SoakSettings:+ if env.get("DIFY_AGENT_TEST_SOAK_ENABLED", "").strip() != "1":+ pytest.skip("set DIFY_AGENT_TEST_SOAK_ENABLED=1 to run the DIFY-2966 staging soak")++ required_names = (+ "DIFY_AGENT_TEST_SERVICE_URL",+ "DIFY_AGENT_TEST_SERVICE_API_TOKEN",+ "DIFY_AGENT_TEST_E2B_API_KEY",+ "DIFY_AGENT_TEST_SOAK_RUN_ID",+ "DIFY_AGENT_TEST_SOAK_ARTIFACT",+ )+ missing = [name for name in required_names if not env.get(name, "").strip()]+ if missing:+ pytest.skip(f"set {', '.join(missing)} to run the DIFY-2966 staging soak")++ run_id = env["DIFY_AGENT_TEST_SOAK_RUN_ID"].strip()+ if _RUN_ID_PATTERN.fullmatch(run_id) is None:+ raise ValueError("DIFY_AGENT_TEST_SOAK_RUN_ID must be 1-48 ASCII letters, digits, dots, underscores, or dashes")++ return _SoakSettings(+ service_url=env["DIFY_AGENT_TEST_SERVICE_URL"].strip(),+ service_token=env["DIFY_AGENT_TEST_SERVICE_API_TOKEN"].strip(),+ e2b_api_key=env["DIFY_AGENT_TEST_E2B_API_KEY"].strip(),+ run_id=run_id,+ artifact=Path(env["DIFY_AGENT_TEST_SOAK_ARTIFACT"].strip()).expanduser(),+ concurrency=_positive_int(env, "DIFY_AGENT_TEST_SOAK_CONCURRENCY", _FORMAL_CONCURRENCY),+ burst_rounds=_positive_int(env, "DIFY_AGENT_TEST_SOAK_BURST_ROUNDS", _FORMAL_BURST_ROUNDS),+ idle_bindings=_positive_int(env, "DIFY_AGENT_TEST_SOAK_IDLE_BINDINGS", _FORMAL_IDLE_BINDINGS),+ idle_interval_seconds=_nonnegative_float(+ env,+ "DIFY_AGENT_TEST_SOAK_IDLE_INTERVAL_SECONDS",+ _FORMAL_IDLE_INTERVAL_SECONDS,+ ),+ idle_rounds=_positive_int(env, "DIFY_AGENT_TEST_SOAK_IDLE_ROUNDS", _FORMAL_IDLE_ROUNDS),+ request_timeout_seconds=_positive_float(+ env,+ "DIFY_AGENT_TEST_SOAK_REQUEST_TIMEOUT_SECONDS",+ _DEFAULT_REQUEST_TIMEOUT_SECONDS,+ ),+ reconcile_timeout_seconds=_positive_float(+ env,+ "DIFY_AGENT_TEST_SOAK_RECONCILE_TIMEOUT_SECONDS",+ _DEFAULT_RECONCILE_TIMEOUT_SECONDS,+ ),+ )+++def _positive_int(env: Mapping[str, str], name: str, default: int) -> int:+ raw = env.get(name, "").strip()+ value = int(raw) if raw else default+ if value <= 0:+ raise ValueError(f"{name} must be greater than zero")+ return value+++def _positive_float(env: Mapping[str, str], name: str, default: float) -> float:+ value = _nonnegative_float(env, name, default)+ if value == 0:+ raise ValueError(f"{name} must be greater than zero")+ return value+++def _nonnegative_float(env: Mapping[str, str], name: str, default: float) -> float:+ raw = env.get(name, "").strip()+ value = float(raw) if raw else default+ if not math.isfinite(value) or value < 0:+ raise ValueError(f"{name} must be finite and nonnegative")+ return value+++def _client(settings: _SoakSettings) -> Client:+ # One Client per concurrent lifecycle produces independent HTTP connections+ # so a multi-worker deployment is not accidentally exercised through only+ # one persistent downstream connection.+ return Client(+ base_url=settings.service_url,+ timeout=settings.request_timeout_seconds,+ headers={"Authorization": f"Bearer {settings.service_token}"},+ )+++def _http_status(exc: BaseException) -> int | None:+ return exc.status_code if isinstance(exc, DifyAgentHTTPError) else None+++def _record_operation_failure(+ ledger: _JsonlLedger,+ *,+ operation: str,+ started_at: float,+ exc: BaseException,+ binding: _Binding | None = None,+ binding_id: str | None = None,+ cleanup_result: str | None = None,+) -> None:+ ledger.record(+ "operation_end",+ operation=operation,+ outcome="failed",+ binding_id=binding.binding_id if binding is not None else binding_id,+ binding_ref=binding.binding_ref if binding is not None else None,+ workspace_ref=binding.workspace_ref if binding is not None else None,+ http_status=_http_status(exc),+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),+ exception_type=type(exc).__name__,+ cleanup_result=cleanup_result,+ )+++async def _create_binding(settings: _SoakSettings, ledger: _JsonlLedger, binding_id: str) -> _Binding:+ started_at = time.perf_counter()+ ledger.record("operation_start", operation="create", binding_id=binding_id)+ try:+ async with _client(settings) as client:+ allocation = await client.create_execution_binding(+ CreateExecutionBindingRequest(+ tenant_id=_TENANT_ID,+ agent_id=settings.binding_prefix,+ binding_id=binding_id,+ workspace_id=binding_id,+ existing_workspace_ref=None,+ home_snapshot_ref=None,+ )+ )+ except BaseException as exc:+ _record_operation_failure(ledger, operation="create", started_at=started_at, exc=exc, binding_id=binding_id)+ raise++ binding = _Binding(+ binding_id=binding_id,+ binding_ref=allocation.binding_ref,+ workspace_ref=allocation.workspace_ref,+ )+ ledger.record(+ "operation_end",+ operation="create",+ outcome="succeeded",+ binding_id=binding.binding_id,+ binding_ref=binding.binding_ref,+ workspace_ref=binding.workspace_ref,+ http_status=201,+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),+ exception_type=None,+ cleanup_result=None,+ )+ return binding+++async def _probe_binding(settings: _SoakSettings, ledger: _JsonlLedger, binding: _Binding) -> None:+ async with _client(settings) as client:+ started_at = time.perf_counter()+ ledger.record(+ "operation_start",+ operation="list",+ binding_id=binding.binding_id,+ binding_ref=binding.binding_ref,+ workspace_ref=binding.workspace_ref,+ )+ try:+ listing = await client.list_binding_files(binding.binding_ref, ".")+ assert listing.path == "."+ except BaseException as exc:+ _record_operation_failure(ledger, operation="list", started_at=started_at, exc=exc, binding=binding)+ raise+ ledger.record(+ "operation_end",+ operation="list",+ outcome="succeeded",+ binding_id=binding.binding_id,+ binding_ref=binding.binding_ref,+ workspace_ref=binding.workspace_ref,+ http_status=200,+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),+ exception_type=None,+ cleanup_result=None,+ )++ started_at = time.perf_counter()+ ledger.record(+ "operation_start",+ operation="read",+ binding_id=binding.binding_id,+ binding_ref=binding.binding_ref,+ workspace_ref=binding.workspace_ref,+ )+ try:+ preview = await client.read_binding_file(binding.binding_ref, _READ_PROBE_PATH)+ assert preview.path == _READ_PROBE_PATH+ assert not preview.binary+ assert preview.text+ except BaseException as exc:+ _record_operation_failure(ledger, operation="read", started_at=started_at, exc=exc, binding=binding)+ raise+ ledger.record(+ "operation_end",+ operation="read",+ outcome="succeeded",+ binding_id=binding.binding_id,+ binding_ref=binding.binding_ref,+ workspace_ref=binding.workspace_ref,+ http_status=200,+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),+ exception_type=None,+ cleanup_result=None,+ )+++async def _destroy_binding(+ settings: _SoakSettings,+ ledger: _JsonlLedger,+ binding: _Binding,+ *,+ cleanup: bool,+) -> None:+ operation = "cleanup_destroy" if cleanup else "destroy"+ started_at = time.perf_counter()+ ledger.record(+ "operation_start",+ operation=operation,+ binding_id=binding.binding_id,+ binding_ref=binding.binding_ref,+ workspace_ref=binding.workspace_ref,+ )+ try:+ async with _client(settings) as client:+ await client.destroy_execution_binding(+ DestroyExecutionBindingRequest(+ binding_ref=binding.binding_ref,+ workspace_ref=binding.workspace_ref,+ destroy_workspace=True,+ )+ )+ except BaseException as exc:+ _record_operation_failure(+ ledger,+ operation=operation,+ started_at=started_at,+ exc=exc,+ binding=binding,+ cleanup_result="failed" if cleanup else None,+ )+ raise+ ledger.record(+ "operation_end",+ operation=operation,+ outcome="succeeded",+ binding_id=binding.binding_id,+ binding_ref=binding.binding_ref,+ workspace_ref=binding.workspace_ref,+ http_status=204,+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),+ exception_type=None,+ cleanup_result="destroyed" if cleanup else None,+ )+++def _raise_task_failures(results: Sequence[object], *, message: str) -> None:+ failures: list[Exception] = []+ for result in results:+ if not isinstance(result, BaseException):+ continue+ if not isinstance(result, Exception):+ raise result+ failures.append(result)+ if failures:+ raise ExceptionGroup(message, failures)+++async def _gather_checked(awaitables: Sequence[Awaitable[None]], *, message: str) -> None:+ results = await asyncio.gather(*awaitables, return_exceptions=True)+ _raise_task_failures(results, message=message)+++async def _run_burst(+ settings: _SoakSettings,+ ledger: _JsonlLedger,+ known: dict[str, _Binding],+ destroyed: set[str],+) -> None:+ for round_number in range(settings.burst_rounds):+ ledger.record("stage_start", stage="burst_round", round=round_number, count=settings.concurrency)++ async def run_one(sequence: int) -> None:+ binding_id = f"{settings.binding_prefix}burst-{round_number:02d}-{sequence:04d}"+ binding = await _create_binding(settings, ledger, binding_id)+ known[binding.binding_ref] = binding+ await _probe_binding(settings, ledger, binding)+ await _destroy_binding(settings, ledger, binding, cleanup=False)+ destroyed.add(binding.binding_ref)++ try:+ await _gather_checked(+ [run_one(sequence) for sequence in range(settings.concurrency)],+ message=f"burst round {round_number} failed",+ )+ except BaseException as exc:+ ledger.record(+ "stage_end",+ stage="burst_round",+ round=round_number,+ count=settings.concurrency,+ outcome="failed",+ exception_type=type(exc).__name__,+ )+ raise+ ledger.record(+ "stage_end",+ stage="burst_round",+ round=round_number,+ count=settings.concurrency,+ outcome="succeeded",+ exception_type=None,+ )+++async def _run_idle(+ settings: _SoakSettings,+ ledger: _JsonlLedger,+ known: dict[str, _Binding],+ destroyed: set[str],+) -> None:+ bindings_by_sequence: dict[int, _Binding] = {}+ ledger.record("stage_start", stage="idle_create", count=settings.idle_bindings)++ async def create_one(sequence: int) -> None:+ binding_id = f"{settings.binding_prefix}idle-{sequence:04d}"+ binding = await _create_binding(settings, ledger, binding_id)+ bindings_by_sequence[sequence] = binding+ known[binding.binding_ref] = binding++ bindings: list[_Binding] = []+ try:+ await _gather_checked(+ [create_one(sequence) for sequence in range(settings.idle_bindings)],+ message="idle binding creation failed; do not reduce the configured capacity gate",+ )+ bindings = [bindings_by_sequence[index] for index in range(settings.idle_bindings)]+ await _require_exact_idle_inventory(settings, ledger, bindings, stage="idle_create_state")+ except BaseException as exc:+ ledger.record(+ "stage_end",+ stage="idle_create",+ count=settings.idle_bindings,+ outcome="failed",+ exception_type=type(exc).__name__,+ )+ raise+ ledger.record(+ "stage_end",+ stage="idle_create",+ count=settings.idle_bindings,+ outcome="succeeded",+ exception_type=None,+ )++ for round_number in range(settings.idle_rounds):+ ledger.record(+ "stage_start",+ stage="idle_wait",+ round=round_number,+ duration_seconds=settings.idle_interval_seconds,+ )+ await asyncio.sleep(settings.idle_interval_seconds)+ ledger.record(+ "stage_end",+ stage="idle_wait",+ round=round_number,+ duration_seconds=settings.idle_interval_seconds,+ outcome="succeeded",+ )++ ledger.record("stage_start", stage="idle_probe", round=round_number, count=len(bindings))+ try:+ await _gather_checked(+ [_probe_binding(settings, ledger, binding) for binding in bindings],+ message=f"idle probe round {round_number} failed",+ )+ await _require_exact_idle_inventory(+ settings,+ ledger,+ bindings,+ stage=f"idle_probe_{round_number:02d}_state",+ )+ except BaseException as exc:+ ledger.record(+ "stage_end",+ stage="idle_probe",+ round=round_number,+ count=len(bindings),+ outcome="failed",+ exception_type=type(exc).__name__,+ )+ raise+ ledger.record(+ "stage_end",+ stage="idle_probe",+ round=round_number,+ count=len(bindings),+ outcome="succeeded",+ exception_type=None,+ )++ ledger.record("stage_start", stage="idle_destroy", count=len(bindings))++ async def destroy_one(binding: _Binding) -> None:+ await _destroy_binding(settings, ledger, binding, cleanup=False)+ destroyed.add(binding.binding_ref)++ try:+ await _gather_checked(+ [destroy_one(binding) for binding in bindings],+ message="idle binding destruction failed",+ )+ except BaseException as exc:+ ledger.record(+ "stage_end",+ stage="idle_destroy",+ count=len(bindings),+ outcome="failed",+ exception_type=type(exc).__name__,+ )+ raise+ ledger.record(+ "stage_end",+ stage="idle_destroy",+ count=len(bindings),+ outcome="succeeded",+ exception_type=None,+ )+++async def _cleanup_known_bindings(+ settings: _SoakSettings,+ ledger: _JsonlLedger,+ known: Mapping[str, _Binding],+ destroyed: set[str],+) -> list[Exception]:+ pending = [binding for binding in known.values() if binding.binding_ref not in destroyed]+ if not pending:+ return []+ results = await asyncio.gather(+ *[_destroy_binding(settings, ledger, binding, cleanup=True) for binding in pending],+ return_exceptions=True,+ )+ errors: list[Exception] = []+ for binding, result in zip(pending, results, strict=True):+ if isinstance(result, BaseException):+ if not isinstance(result, Exception):+ raise result+ errors.append(result)+ else:+ destroyed.add(binding.binding_ref)+ return errors+++async def _list_owned_e2b_sandboxes(settings: _SoakSettings) -> list[_OwnedSandbox]:+ from e2b import AsyncSandbox, SandboxInfo, SandboxQuery++ paginator = AsyncSandbox.list(+ query=SandboxQuery(+ metadata={+ "dify.agent_id": settings.binding_prefix,+ "dify.tenant_id": _TENANT_ID,+ }+ ),+ limit=100,+ api_key=settings.e2b_api_key,+ )+ candidates: list[SandboxInfo] = []+ while paginator.has_next:+ candidates.extend(await paginator.next_items())++ owned: list[_OwnedSandbox] = []+ for candidate in candidates:+ metadata = candidate.metadata+ binding_id = metadata.get("dify.binding_id", "")+ workspace_id = metadata.get("dify.workspace_id", "")+ if not binding_id.startswith(settings.binding_prefix) or not workspace_id.startswith(settings.binding_prefix):+ continue+ state = candidate.state.value+ owned.append(+ _OwnedSandbox(+ sandbox_id=candidate.sandbox_id,+ binding_id=binding_id,+ workspace_id=workspace_id,+ state=str(state),+ )+ )+ return sorted(owned, key=lambda sandbox: sandbox.sandbox_id)+++def _inventory_identity(sandboxes: Sequence[_OwnedSandbox]) -> list[tuple[str, str, str, str]]:+ return sorted(+ (sandbox.sandbox_id, sandbox.binding_id, sandbox.workspace_id, sandbox.state) for sandbox in sandboxes+ )+++def _expected_idle_identity(bindings: Sequence[_Binding]) -> list[tuple[str, str, str, str]]:+ return sorted((binding.binding_ref, binding.binding_id, binding.binding_id, "paused") for binding in bindings)+++def _record_inventory(+ ledger: _JsonlLedger,+ *,+ stage: str,+ sandboxes: Sequence[_OwnedSandbox],+ attempt: int | None = None,+) -> None:+ ledger.record("e2b_inventory", stage=stage, attempt=attempt, count=len(sandboxes))+ for sandbox in sandboxes:+ ledger.record(+ "e2b_inventory_item",+ stage=stage,+ attempt=attempt,+ sandbox_id=sandbox.sandbox_id,+ binding_id=sandbox.binding_id,+ workspace_id=sandbox.workspace_id,+ state=sandbox.state,+ )+++async def _list_owned_inventory_attempt(+ settings: _SoakSettings,+ ledger: _JsonlLedger,+ *,+ list_owned: _ListOwnedSandboxes,+ stage: str,+ attempt: int,+ timeout_seconds: float,+) -> tuple[list[_OwnedSandbox] | None, Exception | None]:+ started_at = time.perf_counter()+ ledger.record(+ "operation_start",+ operation="e2b_inventory_list",+ stage=stage,+ attempt=attempt,+ )+ try:+ inventory = await asyncio.wait_for(list_owned(settings), timeout=timeout_seconds)+ except Exception as exc:+ ledger.record(+ "operation_end",+ operation="e2b_inventory_list",+ stage=stage,+ attempt=attempt,+ outcome="failed",+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),+ exception_type=type(exc).__name__,+ cleanup_result=None,+ )+ return None, exc+ ledger.record(+ "operation_end",+ operation="e2b_inventory_list",+ stage=stage,+ attempt=attempt,+ outcome="succeeded",+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),+ exception_type=None,+ cleanup_result=None,+ )+ return inventory, None+++async def _require_exact_idle_inventory(+ settings: _SoakSettings,+ ledger: _JsonlLedger,+ bindings: Sequence[_Binding],+ *,+ stage: str = "idle_identity",+ list_owned: _ListOwnedSandboxes = _list_owned_e2b_sandboxes,+ sleep: _Sleep = asyncio.sleep,+ monotonic: _Monotonic = time.monotonic,+) -> None:+ """Require the exact run-owned Bindings to be visibly paused before continuing."""+ if any(binding.binding_ref != binding.workspace_ref for binding in bindings):+ ledger.record(+ "e2b_identity_check",+ stage=stage,+ outcome="failed",+ expected_count=len(bindings),+ actual_count=None,+ exception_type="BackendIdentityMismatch",+ )+ raise AssertionError("E2B idle bindings must return identical Binding and Workspace refs")++ expected = _expected_idle_identity(bindings)+ deadline = monotonic() + settings.reconcile_timeout_seconds+ attempt = 0+ last_list_error: Exception | None = None+ last_observed_count: int | None = None+ while True:+ remaining_seconds = deadline - monotonic()+ if remaining_seconds <= 0:+ if last_list_error is not None:+ raise AssertionError(+ "E2B idle inventory could not be proven before the reconcile deadline"+ ) from last_list_error+ ledger.record(+ "e2b_identity_check",+ stage=stage,+ attempt=attempt,+ outcome="failed",+ expected_count=len(expected),+ actual_count=last_observed_count,+ exception_type="BackendIdentityMismatch",+ )+ raise AssertionError(+ f"E2B idle inventory mismatch: expected {len(expected)} exact, observed {last_observed_count}"+ )+ attempt += 1+ actual_sandboxes, list_error = await _list_owned_inventory_attempt(+ settings,+ ledger,+ list_owned=list_owned,+ stage=stage,+ attempt=attempt,+ timeout_seconds=remaining_seconds,+ )+ if list_error is not None:+ last_list_error = list_error+ remaining_seconds = deadline - monotonic()+ if remaining_seconds <= 0:+ raise AssertionError(+ "E2B idle inventory could not be proven before the reconcile deadline"+ ) from last_list_error+ await sleep(min(_RECONCILE_RETRY_BACKOFF_SECONDS, remaining_seconds))+ continue+ assert actual_sandboxes is not None+ last_list_error = None+ actual = _inventory_identity(actual_sandboxes)+ last_observed_count = len(actual)+ _record_inventory(ledger, stage=stage, sandboxes=actual_sandboxes, attempt=attempt)+ if actual == expected:+ ledger.record(+ "e2b_identity_check",+ stage=stage,+ attempt=attempt,+ outcome="succeeded",+ expected_count=len(expected),+ actual_count=len(actual),+ exception_type=None,+ )+ return+ if monotonic() >= deadline:+ ledger.record(+ "e2b_identity_check",+ stage=stage,+ attempt=attempt,+ outcome="failed",+ expected_count=len(expected),+ actual_count=len(actual),+ exception_type="BackendIdentityMismatch",+ )+ raise AssertionError(+ f"E2B idle inventory mismatch: expected {len(expected)} exact allocations, observed {len(actual)}"+ )+ await sleep(min(_RECONCILE_POLL_SECONDS, max(0.0, deadline - monotonic())))+++async def _kill_owned_e2b_sandbox(settings: _SoakSettings, sandbox_id: str) -> bool:+ from e2b import AsyncSandbox++ return await AsyncSandbox.kill(sandbox_id, api_key=settings.e2b_api_key)+++async def _reconcile_owned_e2b_sandboxes(+ settings: _SoakSettings,+ ledger: _JsonlLedger,+ *,+ list_owned: _ListOwnedSandboxes = _list_owned_e2b_sandboxes,+ kill_owned: _KillOwnedSandbox = _kill_owned_e2b_sandbox,+ sleep: _Sleep = asyncio.sleep,+ monotonic: _Monotonic = time.monotonic,+) -> None:+ deadline = monotonic() + settings.reconcile_timeout_seconds+ final_proof_reserve = min(+ _RECONCILE_FINAL_PROOF_RESERVE_SECONDS,+ settings.reconcile_timeout_seconds / 2,+ )+ scan_deadline = deadline - final_proof_reserve+ attempt = 0+ last_list_error: Exception | None = None+ last_inventory: list[_OwnedSandbox] | None = None++ async def kill_one(sandbox: _OwnedSandbox, *, attempt: int) -> None:+ started_at = time.perf_counter()+ ledger.record(+ "operation_start",+ operation="e2b_direct_kill",+ attempt=attempt,+ sandbox_id=sandbox.sandbox_id,+ binding_id=sandbox.binding_id,+ workspace_id=sandbox.workspace_id,+ )+ try:+ killed = await kill_owned(settings, sandbox.sandbox_id)+ except Exception as exc:+ ledger.record(+ "operation_end",+ operation="e2b_direct_kill",+ attempt=attempt,+ outcome="failed",+ sandbox_id=sandbox.sandbox_id,+ binding_id=sandbox.binding_id,+ workspace_id=sandbox.workspace_id,+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),+ exception_type=type(exc).__name__,+ cleanup_result="failed",+ )+ return+ except asyncio.CancelledError:+ ledger.record(+ "operation_end",+ operation="e2b_direct_kill",+ attempt=attempt,+ outcome="failed",+ sandbox_id=sandbox.sandbox_id,+ binding_id=sandbox.binding_id,+ workspace_id=sandbox.workspace_id,+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),+ exception_type="CancelledError",+ cleanup_result="cancelled",+ )+ raise+ ledger.record(+ "operation_end",+ operation="e2b_direct_kill",+ attempt=attempt,+ outcome="succeeded",+ sandbox_id=sandbox.sandbox_id,+ binding_id=sandbox.binding_id,+ workspace_id=sandbox.workspace_id,+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),+ exception_type=None,+ cleanup_result="killed" if killed else "already_absent",+ )++ async def kill_batch(+ sandboxes: Sequence[_OwnedSandbox],+ *,+ attempt: int,+ timeout_seconds: float,+ ) -> None:+ started_at = time.perf_counter()+ ledger.record(+ "operation_start",+ operation="e2b_direct_kill_batch",+ attempt=attempt,+ count=len(sandboxes),+ )+ try:+ _ = await asyncio.wait_for(+ asyncio.gather(*[kill_one(sandbox, attempt=attempt) for sandbox in sandboxes]),+ timeout=timeout_seconds,+ )+ except TimeoutError:+ ledger.record(+ "operation_end",+ operation="e2b_direct_kill_batch",+ attempt=attempt,+ outcome="failed",+ count=len(sandboxes),+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),+ exception_type="TimeoutError",+ cleanup_result="deadline_exhausted",+ )+ return+ ledger.record(+ "operation_end",+ operation="e2b_direct_kill_batch",+ attempt=attempt,+ outcome="completed",+ count=len(sandboxes),+ latency_ms=round((time.perf_counter() - started_at) * 1000, 3),+ exception_type=None,+ cleanup_result="attempted",+ )++ while monotonic() < scan_deadline:+ remaining_seconds = scan_deadline - monotonic()+ if remaining_seconds <= 0:+ break+ attempt += 1+ current, list_error = await _list_owned_inventory_attempt(+ settings,+ ledger,+ list_owned=list_owned,+ stage="direct_cleanup_scan",+ attempt=attempt,+ timeout_seconds=remaining_seconds,+ )+ if list_error is not None:+ last_list_error = list_error+ else:+ assert current is not None+ last_list_error = None+ last_inventory = current+ _record_inventory(ledger, stage="direct_cleanup_scan", sandboxes=current, attempt=attempt)+ kill_remaining = scan_deadline - monotonic()+ if current and kill_remaining > 0:+ await kill_batch(current, attempt=attempt, timeout_seconds=kill_remaining)++ remaining_seconds = scan_deadline - monotonic()+ if remaining_seconds <= 0:+ break+ await sleep(min(_RECONCILE_POLL_SECONDS, remaining_seconds))++ final_inventory: list[_OwnedSandbox] | None = None+ stable_empty_since: float | None = None+ while monotonic() < deadline:+ remaining_seconds = deadline - monotonic()+ if remaining_seconds <= 0:+ break+ attempt += 1+ current, list_error = await _list_owned_inventory_attempt(+ settings,+ ledger,+ list_owned=list_owned,+ stage="direct_cleanup_final_scan",+ attempt=attempt,+ timeout_seconds=min(_RECONCILE_POLL_SECONDS, remaining_seconds),+ )+ if list_error is not None:+ last_list_error = list_error+ stable_empty_since = None+ else:+ assert current is not None+ last_list_error = None+ last_inventory = current+ _record_inventory(ledger, stage="direct_cleanup_final_scan", sandboxes=current, attempt=attempt)+ if current:+ stable_empty_since = None+ kill_remaining = deadline - monotonic()+ if kill_remaining > 0:+ await kill_batch(+ current,+ attempt=attempt,+ timeout_seconds=min(_RECONCILE_POLL_SECONDS, kill_remaining),+ )+ else:+ observed_at = monotonic()+ if stable_empty_since is None:+ stable_empty_since = observed_at+ stable_empty_seconds = observed_at - stable_empty_since+ if stable_empty_seconds >= _RECONCILE_STABLE_EMPTY_SECONDS:+ ledger.record(+ "e2b_stable_empty",+ stage="direct_cleanup_final_scan",+ attempt=attempt,+ outcome="succeeded",+ duration_seconds=stable_empty_seconds,+ )+ final_inventory = current+ break++ remaining_seconds = deadline - monotonic()+ if remaining_seconds <= 0:+ break+ await sleep(min(_RECONCILE_FINAL_SCAN_INTERVAL_SECONDS, remaining_seconds))++ if final_inventory is None:+ if last_list_error is not None:+ raise AssertionError("final E2B inventory proof failed before the reconcile deadline") from last_list_error+ remaining_count = len(last_inventory) if last_inventory is not None else 0+ if remaining_count:+ raise AssertionError(+ f"final E2B inventory proof was not empty before the reconcile deadline ({remaining_count} observed)"+ )+ raise AssertionError(+ f"final E2B inventory did not remain empty for {_RECONCILE_STABLE_EMPTY_SECONDS:g}s before the deadline"+ )++ _record_inventory(+ ledger,+ stage="direct_cleanup_deadline",+ sandboxes=final_inventory,+ attempt=attempt,+ )+ _record_inventory(ledger, stage="after_direct_cleanup", sandboxes=final_inventory, attempt=attempt)+ assert not final_inventory, f"{len(final_inventory)} run-owned E2B sandboxes remain after cleanup"+++def _raise_collected_errors(primary: BaseException | None, cleanup_errors: Sequence[Exception]) -> None:+ if primary is not None and not isinstance(primary, Exception):+ raise primary+ errors = ([primary] if isinstance(primary, Exception) else []) + list(cleanup_errors)+ if len(errors) == 1:+ raise errors[0]+ if errors:+ raise ExceptionGroup("DIFY-2966 soak and cleanup failures", errors)+++async def test_e2b_transport_soak() -> None:+ settings = _load_settings(os.environ)+ primary_error: BaseException | None = None+ cleanup_errors: list[Exception] = []+ known: dict[str, _Binding] = {}+ destroyed: set[str] = set()+ cleanup_authorized = False++ with _JsonlLedger(+ settings.artifact,+ secrets=(settings.service_token, settings.e2b_api_key),+ ) as ledger:+ ledger.record(+ "run_start",+ run_id=settings.run_id,+ binding_prefix=settings.binding_prefix,+ formal_gate=settings.formal_gate,+ concurrency=settings.concurrency,+ burst_rounds=settings.burst_rounds,+ idle_bindings=settings.idle_bindings,+ idle_interval_seconds=settings.idle_interval_seconds,+ idle_rounds=settings.idle_rounds,+ )++ initial = await _list_owned_e2b_sandboxes(settings)+ _record_inventory(ledger, stage="preflight", sandboxes=initial)+ assert not initial, "DIFY_AGENT_TEST_SOAK_RUN_ID is already in use; choose a new run id"+ cleanup_authorized = True++ try:+ await _run_burst(settings, ledger, known, destroyed)+ await _run_idle(settings, ledger, known, destroyed)+ except BaseException as exc:+ primary_error = exc+ finally:+ if cleanup_authorized:+ try:+ cleanup_errors.extend(await _cleanup_known_bindings(settings, ledger, known, destroyed))+ except BaseException as exc:+ if not isinstance(exc, Exception):+ raise+ cleanup_errors.append(exc)+ try:+ await _reconcile_owned_e2b_sandboxes(settings, ledger)+ except BaseException as exc:+ if not isinstance(exc, Exception):+ raise+ cleanup_errors.append(exc)++ outcome = "succeeded" if primary_error is None and not cleanup_errors else "failed"+ ledger.record(+ "run_end",+ run_id=settings.run_id,+ formal_gate=settings.formal_gate,+ outcome=outcome,+ exception_type=type(primary_error).__name__ if primary_error is not None else None,+ cleanup_error_types=[type(error).__name__ for error in cleanup_errors],+ )++ _raise_collected_errors(primary_error, cleanup_errors)+++def test_soak_requires_explicit_enable(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delenv("DIFY_AGENT_TEST_SOAK_ENABLED", raising=False)+ with pytest.raises(pytest.skip.Exception, match="DIFY_AGENT_TEST_SOAK_ENABLED=1"):+ _ = _load_settings(os.environ)+++def test_soak_missing_credentials_skip_without_echoing_values(monkeypatch: pytest.MonkeyPatch) -> None:+ for name in (+ "DIFY_AGENT_TEST_SERVICE_URL",+ "DIFY_AGENT_TEST_E2B_API_KEY",+ "DIFY_AGENT_TEST_SOAK_RUN_ID",+ "DIFY_AGENT_TEST_SOAK_ARTIFACT",+ ):+ monkeypatch.delenv(name, raising=False)+ monkeypatch.setenv("DIFY_AGENT_TEST_SOAK_ENABLED", "1")+ monkeypatch.setenv("DIFY_AGENT_TEST_SERVICE_API_TOKEN", "service-secret-value")+ with pytest.raises(pytest.skip.Exception) as raised:+ _ = _load_settings(os.environ)+ assert "service-secret-value" not in str(raised.value)+++def test_ledger_redacts_secret_fields_and_values(tmp_path: Path) -> None:+ service_token = "service-secret-value"+ e2b_key = "e2b-secret-value"+ artifact = tmp_path / "ledger.jsonl"++ with _JsonlLedger(artifact, secrets=(service_token, e2b_key)) as ledger:+ ledger.record(+ "probe",+ authorization=service_token,+ nested={"api_key": e2b_key, "message": f"failed near {service_token}"},+ )++ raw = artifact.read_text(encoding="utf-8")+ payload = cast(dict[str, object], json.loads(raw))+ assert service_token not in raw+ assert e2b_key not in raw+ assert payload["authorization"] == _REDACTED+ nested = cast(dict[str, object], payload["nested"])+ assert nested["api_key"] == _REDACTED+ assert nested["message"] == f"failed near {_REDACTED}"+++def test_formal_gate_includes_request_and_reconcile_timeouts(tmp_path: Path) -> None:+ settings = _SoakSettings(+ service_url="https://agent.invalid",+ service_token="service-secret-value",+ e2b_api_key="e2b-secret-value",+ run_id="formal",+ artifact=tmp_path / "formal.jsonl",+ concurrency=_FORMAL_CONCURRENCY,+ burst_rounds=_FORMAL_BURST_ROUNDS,+ idle_bindings=_FORMAL_IDLE_BINDINGS,+ idle_interval_seconds=_FORMAL_IDLE_INTERVAL_SECONDS,+ idle_rounds=_FORMAL_IDLE_ROUNDS,+ request_timeout_seconds=_DEFAULT_REQUEST_TIMEOUT_SECONDS,+ reconcile_timeout_seconds=_DEFAULT_RECONCILE_TIMEOUT_SECONDS,+ )++ assert settings.formal_gate+ assert not replace(settings, request_timeout_seconds=1).formal_gate+ assert not replace(settings, reconcile_timeout_seconds=1).formal_gate+++def test_reconcile_timeout_override_must_be_positive(tmp_path: Path) -> None:+ env = {+ "DIFY_AGENT_TEST_SOAK_ENABLED": "1",+ "DIFY_AGENT_TEST_SERVICE_URL": "https://agent.invalid",+ "DIFY_AGENT_TEST_SERVICE_API_TOKEN": "service-secret-value",+ "DIFY_AGENT_TEST_E2B_API_KEY": "e2b-secret-value",+ "DIFY_AGENT_TEST_SOAK_RUN_ID": "positive-timeout",+ "DIFY_AGENT_TEST_SOAK_ARTIFACT": str(tmp_path / "positive-timeout.jsonl"),+ "DIFY_AGENT_TEST_SOAK_RECONCILE_TIMEOUT_SECONDS": "0",+ }++ with pytest.raises(ValueError, match="must be greater than zero"):+ _ = _load_settings(env)+++def _test_settings(tmp_path: Path, *, name: str, reconcile_timeout_seconds: float = 60.0) -> _SoakSettings:+ return _SoakSettings(+ service_url="https://agent.invalid",+ service_token="service-secret-value",+ e2b_api_key="e2b-secret-value",+ run_id=name,+ artifact=tmp_path / f"{name}.jsonl",+ concurrency=1,+ burst_rounds=1,+ idle_bindings=1,+ idle_interval_seconds=0,+ idle_rounds=1,+ request_timeout_seconds=1,+ reconcile_timeout_seconds=reconcile_timeout_seconds,+ )+++class _FakeClock:+ now: float++ def __init__(self) -> None:+ self.now = 0.0++ def monotonic(self) -> float:+ return self.now++ async def sleep(self, seconds: float) -> None:+ self.now += seconds++[email protected]("mismatch", ["workspace", "state"])+async def test_idle_inventory_requires_exact_paused_identity(tmp_path: Path, mismatch: str) -> None:+ settings = _test_settings(tmp_path, name="identity", reconcile_timeout_seconds=0.1)+ clock = _FakeClock()+ binding = _Binding(+ binding_id=f"{settings.binding_prefix}idle-0000",+ binding_ref="sandbox-1",+ workspace_ref="sandbox-1",+ )+ mismatched = _OwnedSandbox(+ sandbox_id=binding.binding_ref,+ binding_id=binding.binding_id,+ workspace_id=(f"{settings.binding_prefix}wrong-workspace" if mismatch == "workspace" else binding.binding_id),+ state="running" if mismatch == "state" else "paused",+ )++ async def list_owned(_settings: _SoakSettings) -> list[_OwnedSandbox]:+ return [mismatched]++ with _JsonlLedger(settings.artifact, secrets=(settings.service_token, settings.e2b_api_key)) as ledger:+ with pytest.raises(AssertionError, match="inventory mismatch"):+ await _require_exact_idle_inventory(+ settings,+ ledger,+ [binding],+ list_owned=list_owned,+ sleep=clock.sleep,+ monotonic=clock.monotonic,+ )+++async def test_reconcile_catches_sandbox_with_delayed_visibility(tmp_path: Path) -> None:+ settings = _test_settings(tmp_path, name="delayed", reconcile_timeout_seconds=8)+ clock = _FakeClock()+ sandbox = _OwnedSandbox(+ sandbox_id="sandbox-delayed",+ binding_id=f"{settings.binding_prefix}idle-0000",+ workspace_id=f"{settings.binding_prefix}idle-0000",+ state="paused",+ )+ inventories: list[list[_OwnedSandbox]] = [[], [sandbox], [], []]+ list_attempt = 0+ killed: list[str] = []++ async def list_owned(_settings: _SoakSettings) -> list[_OwnedSandbox]:+ nonlocal list_attempt+ inventory = inventories[list_attempt] if list_attempt < len(inventories) else []+ list_attempt += 1+ return inventory++ async def kill_owned(_settings: _SoakSettings, sandbox_id: str) -> bool:+ killed.append(sandbox_id)+ return True++ with _JsonlLedger(settings.artifact, secrets=(settings.service_token, settings.e2b_api_key)) as ledger:+ await _reconcile_owned_e2b_sandboxes(+ settings,+ ledger,+ list_owned=list_owned,+ kill_owned=kill_owned,+ sleep=clock.sleep,+ monotonic=clock.monotonic,+ )++ assert killed == [sandbox.sandbox_id]+ assert clock.monotonic() >= 7+ assert clock.monotonic() < settings.reconcile_timeout_seconds+++async def test_reconcile_retries_first_list_failure_then_cleans_visible_sandbox(tmp_path: Path) -> None:+ settings = _test_settings(tmp_path, name="retry-list", reconcile_timeout_seconds=8)+ clock = _FakeClock()+ sandbox = _OwnedSandbox(+ sandbox_id="sandbox-after-list-error",+ binding_id=f"{settings.binding_prefix}idle-0000",+ workspace_id=f"{settings.binding_prefix}idle-0000",+ state="paused",+ )+ list_attempts = 0+ killed: list[str] = []++ async def list_owned(_settings: _SoakSettings) -> list[_OwnedSandbox]:+ nonlocal list_attempts+ list_attempts += 1+ if list_attempts == 1:+ raise RuntimeError(f"temporary list failure near {settings.e2b_api_key}")+ if list_attempts == 2:+ return [sandbox]+ return []++ async def kill_owned(_settings: _SoakSettings, sandbox_id: str) -> bool:+ killed.append(sandbox_id)+ return True++ with _JsonlLedger(settings.artifact, secrets=(settings.service_token, settings.e2b_api_key)) as ledger:+ await _reconcile_owned_e2b_sandboxes(+ settings,+ ledger,+ list_owned=list_owned,+ kill_owned=kill_owned,+ sleep=clock.sleep,+ monotonic=clock.monotonic,+ )++ raw = settings.artifact.read_text(encoding="utf-8")+ events = [cast(dict[str, object], json.loads(line)) for line in raw.splitlines()]+ list_outcomes = [+ event["outcome"]+ for event in events+ if event.get("event") == "operation_end" and event.get("operation") == "e2b_inventory_list"+ ]+ assert killed == [sandbox.sandbox_id]+ assert list_outcomes[0] == "failed"+ assert "succeeded" in list_outcomes[1:]+ assert settings.e2b_api_key not in raw+++async def test_reconcile_retries_first_kill_failure_and_keeps_ledger_secret_safe(tmp_path: Path) -> None:+ settings = _test_settings(tmp_path, name="retry-kill", reconcile_timeout_seconds=10)+ clock = _FakeClock()+ sandbox = _OwnedSandbox(+ sandbox_id="sandbox-retry",+ binding_id=f"{settings.binding_prefix}idle-0000",+ workspace_id=f"{settings.binding_prefix}idle-0000",+ state="paused",+ )+ present = True+ kill_attempts = 0++ async def list_owned(_settings: _SoakSettings) -> list[_OwnedSandbox]:+ return [sandbox] if present else []++ async def kill_owned(_settings: _SoakSettings, sandbox_id: str) -> bool:+ nonlocal kill_attempts, present+ assert sandbox_id == sandbox.sandbox_id+ kill_attempts += 1+ if kill_attempts == 1:+ raise RuntimeError(f"temporary failure near {settings.e2b_api_key}")+ present = False+ return True++ with _JsonlLedger(settings.artifact, secrets=(settings.service_token, settings.e2b_api_key)) as ledger:+ await _reconcile_owned_e2b_sandboxes(+ settings,+ ledger,+ list_owned=list_owned,+ kill_owned=kill_owned,+ sleep=clock.sleep,+ monotonic=clock.monotonic,+ )++ raw = settings.artifact.read_text(encoding="utf-8")+ events = [cast(dict[str, object], json.loads(line)) for line in raw.splitlines()]+ kill_outcomes = [+ event["outcome"]+ for event in events+ if event.get("event") == "operation_end" and event.get("operation") == "e2b_direct_kill"+ ]+ assert kill_attempts == 2+ assert kill_outcomes == ["failed", "succeeded"]+ assert clock.monotonic() >= 8+ assert clock.monotonic() < settings.reconcile_timeout_seconds+ assert settings.service_token not in raw+ assert settings.e2b_api_key not in raw+++async def test_reconcile_recovers_when_first_final_list_is_slow_and_times_out(tmp_path: Path) -> None:+ settings = _test_settings(tmp_path, name="slow-final-list", reconcile_timeout_seconds=20)+ clock = _FakeClock()+ slow_final_injected = False++ async def list_owned(_settings: _SoakSettings) -> list[_OwnedSandbox]:+ nonlocal slow_final_injected+ if clock.monotonic() >= 10 and not slow_final_injected:+ slow_final_injected = True+ await clock.sleep(_RECONCILE_POLL_SECONDS)+ raise TimeoutError("simulated slow final inventory list")+ return []++ async def kill_owned(_settings: _SoakSettings, _sandbox_id: str) -> bool:+ raise AssertionError("kill must not run for stable empty inventory")++ with _JsonlLedger(settings.artifact, secrets=(settings.service_token, settings.e2b_api_key)) as ledger:+ await _reconcile_owned_e2b_sandboxes(+ settings,+ ledger,+ list_owned=list_owned,+ kill_owned=kill_owned,+ sleep=clock.sleep,+ monotonic=clock.monotonic,+ )++ raw = settings.artifact.read_text(encoding="utf-8")+ events = [cast(dict[str, object], json.loads(line)) for line in raw.splitlines()]+ final_list_outcomes = [+ event["outcome"]+ for event in events+ if event.get("event") == "operation_end"+ and event.get("operation") == "e2b_inventory_list"+ and event.get("stage") == "direct_cleanup_final_scan"+ ]+ assert slow_final_injected+ assert final_list_outcomes[0] == "failed"+ assert "succeeded" in final_list_outcomes[1:]+ assert clock.monotonic() >= 15+ assert clock.monotonic() < settings.reconcile_timeout_seconds+++async def test_reconcile_bounds_never_returning_inventory_list(tmp_path: Path) -> None:+ settings = _test_settings(tmp_path, name="list-deadline", reconcile_timeout_seconds=0.04)++ async def list_owned(_settings: _SoakSettings) -> list[_OwnedSandbox]:+ _ = await asyncio.Event().wait()+ raise AssertionError("unreachable")++ async def kill_owned(_settings: _SoakSettings, _sandbox_id: str) -> bool:+ raise AssertionError("kill must not run without inventory ownership")++ with _JsonlLedger(settings.artifact, secrets=(settings.service_token, settings.e2b_api_key)) as ledger:+ with pytest.raises(AssertionError, match="final E2B inventory proof failed"):+ await _reconcile_owned_e2b_sandboxes(+ settings,+ ledger,+ list_owned=list_owned,+ kill_owned=kill_owned,+ )+++async def test_reconcile_bounds_never_returning_kill_batch(tmp_path: Path) -> None:+ settings = _test_settings(tmp_path, name="kill-deadline", reconcile_timeout_seconds=0.04)+ sandbox = _OwnedSandbox(+ sandbox_id="sandbox-never-killed",+ binding_id=f"{settings.binding_prefix}idle-0000",+ workspace_id=f"{settings.binding_prefix}idle-0000",+ state="paused",+ )++ async def list_owned(_settings: _SoakSettings) -> list[_OwnedSandbox]:+ return [sandbox]++ async def kill_owned(_settings: _SoakSettings, _sandbox_id: str) -> bool:+ _ = await asyncio.Event().wait()+ raise AssertionError("unreachable")++ with _JsonlLedger(settings.artifact, secrets=(settings.service_token, settings.e2b_api_key)) as ledger:+ with pytest.raises(AssertionError, match="was not empty"):+ await _reconcile_owned_e2b_sandboxes(+ settings,+ ledger,+ list_owned=list_owned,+ kill_owned=kill_owned,+ )diff --git a/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py b/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.pyindex 62963e7066a..932b66bb34d 100644--- a/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py+++ b/dify-agent/tests/local/dify_agent/runtime_backend/test_e2b.py@@ -1,19 +1,25 @@from __future__ import annotations+import asyncioimport posixpathfrom collections.abc import Callablefrom dataclasses import dataclass, field+import loggingfrom typing import cast+import httpx as e2b_httpximport httpx2 as httpximport pytestfrom shellctl.client import ShellctlClientErrorfrom dify_agent.runtime_backend import (BindingAcquireError,+ BindingCapacityExhaustedError,BindingCreateError,+ BindingLostError,ExecutionBindingCreateSpec,ExecutionBindingDestroySpec,+ HomeSnapshotCreateError,HomeSnapshotCreateSpec,SharedWorkspaceUnsupportedError,WorkspacePreservationUnsupportedError,@@ -38,8 +44,17 @@ class _FileEntry:class _Files:paths: set[str] = field(default_factory=set)removed: list[str] = field(default_factory=list)+ make_dir_errors: list[BaseException] = field(default_factory=list)+ list_errors: list[BaseException] = field(default_factory=list)+ remove_errors: list[BaseException] = field(default_factory=list)+ make_dir_calls: int = 0+ list_calls: int = 0+ remove_calls: int = 0async def make_dir(self, path: str) -> bool:+ self.make_dir_calls += 1+ if self.make_dir_errors:+ raise self.make_dir_errors.pop(0)self.paths.add(path)return True@@ -47,6 +62,9 @@ class _Files:return path in self.pathsasync def list(self, path: str) -> list[_FileEntry]:+ self.list_calls += 1+ if self.list_errors:+ raise self.list_errors.pop(0)prefix = f"{path.rstrip('/')}/"return [_FileEntry(path=entry)@@ -55,6 +73,9 @@ class _Files:]async def remove(self, path: str) -> None:+ self.remove_calls += 1+ if self.remove_errors:+ raise self.remove_errors.pop(0)self.removed.append(path)self.paths = {entry for entry in self.paths if entry != path and posixpath.commonpath((entry, path)) != path}@@ -74,23 +95,32 @@ class _Sandbox:killed: int = 0snapshots: int = 0pause_error: Exception | None = None+ pause_errors: list[BaseException] = field(default_factory=list)+ kill_errors: list[BaseException] = field(default_factory=list)+ snapshot_errors: list[BaseException] = field(default_factory=list)def get_host(self, port: int) -> str:return f"{self.sandbox_id}-{port}.example.test"async def pause(self, keep_memory: bool = True) -> bool:self.pauses.append(keep_memory)+ if self.pause_errors:+ raise self.pause_errors.pop(0)if self.pause_error is not None:raise self.pause_errorreturn Trueasync def kill(self) -> bool:self.killed += 1+ if self.kill_errors:+ raise self.kill_errors.pop(0)return Trueasync def create_snapshot(self, name: str | None = None) -> _Snapshot:del nameself.snapshots += 1+ if self.snapshot_errors:+ raise self.snapshot_errors.pop(0)return _Snapshot(snapshot_id=f"snapshot-{self.sandbox_id}-{self.snapshots}")@@ -101,11 +131,32 @@ class _ControlPlane:killed: list[str] = field(default_factory=list)deleted_snapshots: list[str] = field(default_factory=list)pause_error: Exception | None = None+ sandbox_pause_errors: list[BaseException] = field(default_factory=list)+ sandbox_kill_errors: list[BaseException] = field(default_factory=list)+ create_errors: list[BaseException] = field(default_factory=list)+ connect_errors: list[BaseException] = field(default_factory=list)+ kill_errors: list[BaseException] = field(default_factory=list)+ delete_snapshot_errors: list[BaseException] = field(default_factory=list)+ connect_attempts: list[str] = field(default_factory=list)+ file_make_dir_errors: list[BaseException] = field(default_factory=list)+ file_list_errors: list[BaseException] = field(default_factory=list)+ file_remove_errors: list[BaseException] = field(default_factory=list)async def create(self, template: str, *, timeout: int, metadata: dict[str, str], on_timeout: str) -> _Sandbox:del timeout+ self.created.append((template, on_timeout))+ if self.create_errors:+ raise self.create_errors.pop(0)sandbox_id = f"sandbox-{len(self.sandboxes) + 1}"- sandbox = _Sandbox(sandbox_id=sandbox_id, pause_error=self.pause_error)+ sandbox = _Sandbox(+ sandbox_id=sandbox_id,+ pause_error=self.pause_error,+ pause_errors=list(self.sandbox_pause_errors),+ kill_errors=list(self.sandbox_kill_errors),+ )+ sandbox.files.make_dir_errors = list(self.file_make_dir_errors)+ sandbox.files.list_errors = list(self.file_list_errors)+ sandbox.files.remove_errors = list(self.file_remove_errors)sandbox.files.paths.update({"/workspace",@@ -115,20 +166,26 @@ class _ControlPlane:})self.sandboxes[sandbox_id] = sandbox- self.created.append((template, on_timeout))assert metadata["dify.resource"] == "runtime-sandbox"return sandboxasync def connect(self, handle: str, *, timeout: int) -> _Sandbox:del timeout+ self.connect_attempts.append(handle)+ if self.connect_errors:+ raise self.connect_errors.pop(0)return self.sandboxes[handle]async def kill(self, handle: str) -> bool:self.killed.append(handle)+ if self.kill_errors:+ raise self.kill_errors.pop(0)return Trueasync def delete_snapshot(self, snapshot_ref: str) -> bool:self.deleted_snapshots.append(snapshot_ref)+ if self.delete_snapshot_errors:+ raise self.delete_snapshot_errors.pop(0)return True@@ -150,19 +207,53 @@ def _mock_http(return clients+def _binding_backend(control: _ControlPlane) -> E2BExecutionBindingBackend:+ return E2BExecutionBindingBackend(+ control_plane=control, # pyright: ignore[reportArgumentType]+ template="prepared-template",+ active_timeout_seconds=E2B_MAX_ACTIVE_TIMEOUT_SECONDS,+ )++def _connected_backend(*, pause_error: Exception | None = None) -> tuple[E2BExecutionBindingBackend, _Sandbox]:control = _ControlPlane()sandbox = _Sandbox(sandbox_id="sandbox-1", pause_error=pause_error)sandbox.files.paths.add("/workspace")control.sandboxes[sandbox.sandbox_id] = sandbox- return (- E2BExecutionBindingBackend(- control_plane=control, # pyright: ignore[reportArgumentType]- template="prepared-template",- active_timeout_seconds=E2B_MAX_ACTIVE_TIMEOUT_SECONDS,- ),- sandbox,- )+ return _binding_backend(control), sandbox+++@dataclass(slots=True)+class _ReleaseDataPlane:+ close_error: BaseException | None = None+ close_calls: int = 0++ async def close(self) -> None:+ self.close_calls += 1+ if self.close_error is not None:+ raise self.close_error+++def _transport_error(error_type: type[e2b_httpx.RequestError]) -> e2b_httpx.RequestError:+ request = e2b_httpx.Request("POST", "https://api.e2b.test/sandboxes")+ return error_type("stale E2B connection", request=request)+++def _http_status_error() -> e2b_httpx.HTTPStatusError:+ request = e2b_httpx.Request("POST", "https://api.e2b.test/sandboxes")+ response = e2b_httpx.Response(503, request=request)+ return e2b_httpx.HTTPStatusError("E2B unavailable", request=request, response=response)+++def sleep_delays(monkeypatch: pytest.MonkeyPatch) -> list[float]:+ delays: list[float] = []++ async def record_sleep(delay: float) -> None:+ delays.append(delay)++ monkeypatch.setattr(e2b_module.asyncio, "sleep", record_sleep)+ return delays@pytest.mark.anyio@@ -195,17 +286,310 @@ async def test_e2b_sdk_create_disables_public_traffic(monkeypatch: pytest.Monkeyassert create_options["network"] == {"allow_public_traffic": False}+async def test_e2b_sdk_create_maps_provider_capacity_exhaustion(monkeypatch: pytest.MonkeyPatch) -> None:+ from e2b import AsyncSandbox, RateLimitException++ calls = 0++ async def create(+ _cls: type[AsyncSandbox],+ _template: str,+ **_options: object,+ ) -> _Sandbox:+ nonlocal calls+ calls += 1+ raise RateLimitException("maximum concurrent sandboxes reached")++ monkeypatch.setattr(AsyncSandbox, "create", classmethod(create))+ control_plane = E2BSDKControlPlane(api_key="e2b-secret")++ with pytest.raises(e2b_module._E2BControlPlaneCapacityExhaustedError, match="maximum concurrent"):+ _ = await control_plane.create(+ "prepared-template",+ timeout=120,+ metadata={"dify.resource": "runtime-sandbox"},+ on_timeout="pause",+ )++ assert calls == 1+++ "transport_error_type",+ [e2b_httpx.ConnectError, e2b_httpx.ReadTimeout],+)+async def test_e2b_connect_retries_one_transport_failure(+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+ sleep_delays: list[float],+ transport_error_type: type[e2b_httpx.RequestError],+) -> None:+ control = _ControlPlane(connect_errors=[_transport_error(transport_error_type)])+ sandbox = _Sandbox(sandbox_id="sandbox-1")+ sandbox.files.paths.add("/workspace")+ control.sandboxes[sandbox.sandbox_id] = sandbox+ backend = _binding_backend(control)+ clients = _mock_http(monkeypatch, lambda _request: httpx.Response(200, json={"status": "ok"}))+ with caplog.at_level(logging.WARNING, logger="dify_agent.runtime_backend.e2b"):+ lease = await backend.acquire(sandbox.sandbox_id)++ assert control.connect_attempts == [sandbox.sandbox_id, sandbox.sandbox_id]+ assert sleep_delays == [0.25]+ retry_record = next(record for record in caplog.records if record.__dict__.get("outcome") == "retrying")+ assert retry_record.__dict__["e2b_operation"] == "connect"+ assert retry_record.__dict__["sandbox_id"] == sandbox.sandbox_id+ assert retry_record.__dict__["attempt"] == 1+ assert retry_record.__dict__["cleanup_stage"] == "binding_acquire"+ assert retry_record.__dict__["exception_type"] == transport_error_type.__name__+ await cast(E2BRuntimeLease, lease).data_plane.close()+ assert clients[0].is_closed+++ "transport_error_type",+ [e2b_httpx.ReadError, e2b_httpx.ReadTimeout],+)+async def test_e2b_initial_pause_retries_one_transport_failure(+ sleep_delays: list[float],+ transport_error_type: type[e2b_httpx.RequestError],+) -> None:+ control = _ControlPlane(sandbox_pause_errors=[_transport_error(transport_error_type)])+ backend = _binding_backend(control)+ allocation = await backend.create_binding(+ ExecutionBindingCreateSpec(+ tenant_id="tenant-1",+ agent_id="agent-1",+ binding_id="binding-1",+ workspace_id="workspace-1",+ existing_workspace_ref=None,+ )+ )++ sandbox = control.sandboxes[allocation.binding_ref]+ assert control.created == [("prepared-template", "pause")]+ assert sandbox.pauses == [True, True]+ assert sleep_delays == [0.25]+++ "transport_error_type",+ [e2b_httpx.WriteError, e2b_httpx.ReadTimeout],+)+async def test_e2b_destroy_retries_one_transport_failure(+ sleep_delays: list[float],+ transport_error_type: type[e2b_httpx.RequestError],+) -> None:+ control = _ControlPlane(kill_errors=[_transport_error(transport_error_type)])+ backend = _binding_backend(control)+ await backend.destroy_binding(+ ExecutionBindingDestroySpec(+ binding_ref="sandbox-1",+ workspace_ref="sandbox-1",+ destroy_workspace=True,+ )+ )++ assert control.killed == ["sandbox-1", "sandbox-1"]+ assert sleep_delays == [0.25]+++ "transport_error_type",+ [e2b_httpx.RemoteProtocolError, e2b_httpx.ReadTimeout],+)+async def test_e2b_snapshot_delete_retries_one_transport_failure(+ sleep_delays: list[float],+ transport_error_type: type[e2b_httpx.RequestError],+) -> None:+ control = _ControlPlane(delete_snapshot_errors=[_transport_error(transport_error_type)])+ backend = E2BHomeSnapshotBackend(+ control_plane=control, # pyright: ignore[reportArgumentType]+ )+ await backend.delete("snapshot-1")++ assert control.deleted_snapshots == ["snapshot-1", "snapshot-1"]+ assert sleep_delays == [0.25]+++async def test_e2b_connect_raises_after_two_read_timeouts(+ sleep_delays: list[float],+) -> None:+ control = _ControlPlane(+ connect_errors=[+ _transport_error(e2b_httpx.ReadTimeout),+ _transport_error(e2b_httpx.ReadTimeout),+ ]+ )+ backend = _binding_backend(control)++ with pytest.raises(BindingAcquireError, match="stale E2B connection"):+ _ = await backend.acquire("sandbox-1")++ assert control.connect_attempts == ["sandbox-1", "sandbox-1"]+ assert sleep_delays == [0.25]+++ ("failure_kind", "expected_error"),+ [+ ("not_found", BindingLostError),+ ("http_status", BindingAcquireError),+ ("validation", BindingAcquireError),+ ("cancelled", asyncio.CancelledError),+ ],+)+async def test_e2b_connect_does_not_retry_non_transport_failures(+ monkeypatch: pytest.MonkeyPatch,+ failure_kind: str,+ expected_error: type[BaseException],+) -> None:+ if failure_kind == "not_found":+ failure: BaseException = e2b_module._E2BControlPlaneNotFoundError("missing")+ elif failure_kind == "http_status":+ failure = _http_status_error()+ elif failure_kind == "validation":+ failure = ValueError("invalid E2B response")+ else:+ failure = asyncio.CancelledError()+ control = _ControlPlane(connect_errors=[failure])+ backend = _binding_backend(control)++ async def fail_sleep(_delay: float) -> None:+ raise AssertionError("non-retryable E2B failures must not sleep")++ monkeypatch.setattr(e2b_module.asyncio, "sleep", fail_sleep)+ with pytest.raises(expected_error):+ _ = await backend.acquire("sandbox-1")++ assert control.connect_attempts == ["sandbox-1"]+++ "transport_error_type",+ [e2b_httpx.ConnectError, e2b_httpx.ReadTimeout],+)+async def test_e2b_create_and_snapshot_create_are_not_retried(+ monkeypatch: pytest.MonkeyPatch,+ transport_error_type: type[e2b_httpx.RequestError],+) -> None:+ control = _ControlPlane(create_errors=[_transport_error(transport_error_type)])+ bindings = _binding_backend(control)+ snapshot_sandbox = _Sandbox(+ sandbox_id="sandbox-snapshot",+ snapshot_errors=[_transport_error(transport_error_type)],+ )+ source = E2BRuntimeLease(+ sandbox=snapshot_sandbox, # pyright: ignore[reportArgumentType]+ data_plane=cast(ShellctlRuntimeLease, object()),+ )+ snapshots = E2BHomeSnapshotBackend(control_plane=control) # pyright: ignore[reportArgumentType]++ async def fail_sleep(_delay: float) -> None:+ raise AssertionError("non-idempotent E2B creates must not sleep")++ monkeypatch.setattr(e2b_module.asyncio, "sleep", fail_sleep)+ with pytest.raises(BindingCreateError, match="stale E2B connection"):+ _ = await bindings.create_binding(+ ExecutionBindingCreateSpec(+ tenant_id="tenant-1",+ agent_id="agent-1",+ binding_id="binding-1",+ workspace_id="workspace-1",+ existing_workspace_ref=None,+ )+ )+ with pytest.raises(HomeSnapshotCreateError, match="stale E2B connection"):+ _ = await snapshots.create_from_runtime(+ spec=HomeSnapshotCreateSpec(tenant_id="tenant-1", agent_id="agent-1", home_snapshot_id="home-1"),+ source=source,+ )++ assert control.created == [("prepared-template", "pause")]+ assert snapshot_sandbox.snapshots == 1+++async def test_e2b_create_maps_capacity_exhaustion_without_retry(monkeypatch: pytest.MonkeyPatch) -> None:+ control = _ControlPlane(+ create_errors=[e2b_module._E2BControlPlaneCapacityExhaustedError("maximum concurrent sandboxes reached")]+ )+ backend = _binding_backend(control)++ async def fail_sleep(_delay: float) -> None:+ raise AssertionError("capacity exhaustion must not retry")++ monkeypatch.setattr(e2b_module.asyncio, "sleep", fail_sleep)+ with pytest.raises(BindingCapacityExhaustedError, match="maximum concurrent sandboxes reached"):+ _ = await backend.create_binding(+ ExecutionBindingCreateSpec(+ tenant_id="tenant-1",+ agent_id="agent-1",+ binding_id="binding-1",+ workspace_id="workspace-1",+ existing_workspace_ref=None,+ )+ )++ assert control.created == [("prepared-template", "pause")]+++ ("failure_stage", "expected_calls"),+ [+ ("make_dir", (1, 0, 0)),+ ("list", (1, 1, 0)),+ ("remove", (1, 1, 1)),+ ],+)+async def test_e2b_file_initialization_is_one_shot_and_compensates_on_failure(+ monkeypatch: pytest.MonkeyPatch,+ failure_stage: str,+ expected_calls: tuple[int, int, int],+) -> None:+ error = _transport_error(e2b_httpx.ReadError)+ control = _ControlPlane(+ file_make_dir_errors=[error] if failure_stage == "make_dir" else [],+ file_list_errors=[error] if failure_stage == "list" else [],+ file_remove_errors=[error] if failure_stage == "remove" else [],+ )+ backend = _binding_backend(control)++ async def fail_sleep(_delay: float) -> None:+ raise AssertionError("file initialization must not retry")++ monkeypatch.setattr(e2b_module.asyncio, "sleep", fail_sleep)+ with pytest.raises(BindingCreateError, match="stale E2B connection"):+ _ = await backend.create_binding(+ ExecutionBindingCreateSpec(+ tenant_id="tenant-1",+ agent_id="agent-1",+ binding_id="binding-1",+ workspace_id="workspace-1",+ existing_workspace_ref=None,+ )+ )++ sandbox = next(iter(control.sandboxes.values()))+ assert (sandbox.files.make_dir_calls, sandbox.files.list_calls, sandbox.files.remove_calls) == expected_calls+ assert sandbox.killed == 1++@pytest.mark.anyioasync def test_e2b_binding_uses_default_template_or_exact_snapshot_and_couples_refs() -> None:control = _ControlPlane()snapshots = E2BHomeSnapshotBackend(control_plane=control, # pyright: ignore[reportArgumentType])- bindings = E2BExecutionBindingBackend(- control_plane=control, # pyright: ignore[reportArgumentType]- template="prepared-template",- active_timeout_seconds=E2B_MAX_ACTIVE_TIMEOUT_SECONDS,- )+ bindings = _binding_backend(control)default_allocation = await bindings.create_binding(ExecutionBindingCreateSpec(@@ -255,11 +639,7 @@ async def test_e2b_binding_uses_default_template_or_exact_snapshot_and_couples_r@pytest.mark.anyioasync def test_e2b_rejects_shared_workspace_and_binding_only_destroy() -> None:control = _ControlPlane()- backend = E2BExecutionBindingBackend(- control_plane=control, # pyright: ignore[reportArgumentType]- template="prepared-template",- active_timeout_seconds=E2B_MAX_ACTIVE_TIMEOUT_SECONDS,- )+ backend = _binding_backend(control)spec = ExecutionBindingCreateSpec(tenant_id="tenant-1",agent_id="agent-2",@@ -280,11 +660,7 @@ async def test_e2b_rejects_shared_workspace_and_binding_only_destroy() -> None:@pytest.mark.anyioasync def test_e2b_binding_create_kills_sandbox_when_initialization_fails() -> None:control = _ControlPlane(pause_error=RuntimeError("pause failed"))- backend = E2BExecutionBindingBackend(- control_plane=control, # pyright: ignore[reportArgumentType]- template="prepared-template",- active_timeout_seconds=E2B_MAX_ACTIVE_TIMEOUT_SECONDS,- )+ backend = _binding_backend(control)with pytest.raises(BindingCreateError, match="pause failed"):await backend.create_binding(@@ -302,6 +678,72 @@ async def test_e2b_binding_create_kills_sandbox_when_initialization_fails() -> Nassert sandbox.killed == 1+async def test_e2b_binding_create_preserves_primary_error_when_compensation_kill_fails(+ caplog: pytest.LogCaptureFixture,+ sleep_delays: list[float],+) -> None:+ control = _ControlPlane(+ pause_error=RuntimeError("primary pause failure"),+ sandbox_kill_errors=[+ _transport_error(e2b_httpx.WriteError),+ _transport_error(e2b_httpx.WriteError),+ ],+ )+ backend = _binding_backend(control)+ with caplog.at_level(logging.WARNING, logger="dify_agent.runtime_backend.e2b"):+ with pytest.raises(BindingCreateError, match="primary pause failure"):+ _ = await backend.create_binding(+ ExecutionBindingCreateSpec(+ tenant_id="tenant-1",+ agent_id="agent-1",+ binding_id="binding-1",+ workspace_id="workspace-1",+ existing_workspace_ref=None,+ )+ )++ sandbox = next(iter(control.sandboxes.values()))+ assert sandbox.killed == 2+ assert sleep_delays == [0.25]+ cleanup_record = next(+ record for record in caplog.records if record.__dict__.get("outcome") == "ignored_cleanup_failure"+ )+ assert cleanup_record.funcName == "create_binding"+ assert cleanup_record.__dict__["e2b_operation"] == "kill"+ assert cleanup_record.__dict__["sandbox_id"] == sandbox.sandbox_id+ assert cleanup_record.__dict__["cleanup_stage"] == "binding_create_compensation"+ assert cleanup_record.__dict__["exception_type"] == "WriteError"+++async def test_e2b_binding_create_fails_closed_when_initial_pause_retries_are_exhausted(+ sleep_delays: list[float],+) -> None:+ control = _ControlPlane(+ sandbox_pause_errors=[+ _transport_error(e2b_httpx.ReadError),+ _transport_error(e2b_httpx.ReadError),+ ]+ )+ backend = _binding_backend(control)+ with pytest.raises(BindingCreateError, match="stale E2B connection"):+ _ = await backend.create_binding(+ ExecutionBindingCreateSpec(+ tenant_id="tenant-1",+ agent_id="agent-1",+ binding_id="binding-1",+ workspace_id="workspace-1",+ existing_workspace_ref=None,+ )+ )++ sandbox = next(iter(control.sandboxes.values()))+ assert sandbox.pauses == [True, True]+ assert sandbox.killed == 1+ assert sleep_delays == [0.25]++@pytest.mark.anyioasync def test_e2b_missing_explicit_snapshot_does_not_fall_back_to_template() -> None:class _FailingControlPlane(_ControlPlane):@@ -311,11 +753,7 @@ async def test_e2b_missing_explicit_snapshot_does_not_fall_back_to_template() ->raise RuntimeError("snapshot unavailable")control = _FailingControlPlane()- backend = E2BExecutionBindingBackend(- control_plane=control, # pyright: ignore[reportArgumentType]- template="prepared-template",- active_timeout_seconds=E2B_MAX_ACTIVE_TIMEOUT_SECONDS,- )+ backend = _binding_backend(control)with pytest.raises(BindingCreateError, match="snapshot unavailable"):await backend.create_binding(@@ -353,9 +791,150 @@ async def test_e2b_checkpoint_uses_exact_source_runtime() -> None:assert source_sandbox.snapshots == 1+ ("close_fails", "pause_fails", "expected_warning_count"),+ [+ (True, False, 1),+ (False, True, 1),+ (True, True, 2),+ ],+)+async def test_e2b_release_warns_without_raising_for_cleanup_failures(+ caplog: pytest.LogCaptureFixture,+ close_fails: bool,+ pause_fails: bool,+ expected_warning_count: int,+) -> None:+ data_plane = _ReleaseDataPlane(close_error=RuntimeError("close failed") if close_fails else None)+ sandbox = _Sandbox(+ sandbox_id="sandbox-release",+ pause_error=RuntimeError("pause failed") if pause_fails else None,+ )+ lease = E2BRuntimeLease(+ sandbox=sandbox, # pyright: ignore[reportArgumentType]+ data_plane=cast(ShellctlRuntimeLease, cast(object, data_plane)),+ )+ backend, _ = _connected_backend()++ with caplog.at_level(logging.WARNING, logger="dify_agent.runtime_backend.e2b"):+ await backend.release(lease)++ assert data_plane.close_calls == 1+ assert sandbox.pauses == [True]+ records = [record for record in caplog.records if record.name == "dify_agent.runtime_backend.e2b"]+ assert len(records) == expected_warning_count+ for record in records:+ assert record.__dict__["sandbox_id"] == sandbox.sandbox_id+ assert record.__dict__["cleanup_stage"] == "binding_release"+ assert record.__dict__["outcome"] == "ignored_cleanup_failure"+ assert isinstance(record.__dict__["e2b_operation"], str)+ assert isinstance(record.__dict__["attempt"], int)+ assert isinstance(record.__dict__["exception_type"], str)+++async def test_e2b_release_retries_pause_transport_failure(sleep_delays: list[float]) -> None:+ data_plane = _ReleaseDataPlane()+ sandbox = _Sandbox(+ sandbox_id="sandbox-release",+ pause_errors=[_transport_error(e2b_httpx.RemoteProtocolError)],+ )+ lease = E2BRuntimeLease(+ sandbox=sandbox, # pyright: ignore[reportArgumentType]+ data_plane=cast(ShellctlRuntimeLease, cast(object, data_plane)),+ )+ backend, _ = _connected_backend()+ await backend.release(lease)++ assert data_plane.close_calls == 1+ assert sandbox.pauses == [True, True]+ assert sleep_delays == [0.25]+++async def test_e2b_release_ignores_exhausted_pause_read_timeouts(+ caplog: pytest.LogCaptureFixture,+ sleep_delays: list[float],+) -> None:+ data_plane = _ReleaseDataPlane()+ sandbox = _Sandbox(+ sandbox_id="sandbox-release",+ pause_errors=[+ _transport_error(e2b_httpx.ReadTimeout),+ _transport_error(e2b_httpx.ReadTimeout),+ ],+ )+ lease = E2BRuntimeLease(+ sandbox=sandbox, # pyright: ignore[reportArgumentType]+ data_plane=cast(ShellctlRuntimeLease, cast(object, data_plane)),+ )+ backend, _ = _connected_backend()++ with caplog.at_level(logging.WARNING, logger="dify_agent.runtime_backend.e2b"):+ await backend.release(lease)++ assert data_plane.close_calls == 1+ assert sandbox.pauses == [True, True]+ assert sleep_delays == [0.25]+ cleanup_record = next(+ record for record in caplog.records if record.__dict__.get("outcome") == "ignored_cleanup_failure"+ )+ assert cleanup_record.__dict__["e2b_operation"] == "pause"+ assert cleanup_record.__dict__["sandbox_id"] == sandbox.sandbox_id+ assert cleanup_record.__dict__["attempt"] == 2+ assert cleanup_record.__dict__["cleanup_stage"] == "binding_release"+ assert cleanup_record.__dict__["exception_type"] == "ReadTimeout"++[email protected]("failure_stage", ["close", "pause"])+async def test_e2b_release_does_not_swallow_base_exceptions(failure_stage: str) -> None:+ data_plane = _ReleaseDataPlane(+ close_error=asyncio.CancelledError() if failure_stage == "close" else None,+ )+ sandbox = _Sandbox(+ sandbox_id="sandbox-release",+ pause_errors=[asyncio.CancelledError()] if failure_stage == "pause" else [],+ )+ lease = E2BRuntimeLease(+ sandbox=sandbox, # pyright: ignore[reportArgumentType]+ data_plane=cast(ShellctlRuntimeLease, cast(object, data_plane)),+ )+ backend, _ = _connected_backend()++ with pytest.raises(asyncio.CancelledError):+ await backend.release(lease)++ assert data_plane.close_calls == 1+ assert sandbox.pauses == [True]+++async def test_e2b_release_preserves_close_base_exception_when_pause_also_raises() -> None:+ close_error = asyncio.CancelledError("close cancelled")+ pause_error = asyncio.CancelledError("pause cancelled")+ data_plane = _ReleaseDataPlane(close_error=close_error)+ sandbox = _Sandbox(sandbox_id="sandbox-release", pause_errors=[pause_error])+ lease = E2BRuntimeLease(+ sandbox=sandbox, # pyright: ignore[reportArgumentType]+ data_plane=cast(ShellctlRuntimeLease, cast(object, data_plane)),+ )+ backend, _ = _connected_backend()++ with pytest.raises(asyncio.CancelledError) as exc_info:+ await backend.release(lease)++ assert exc_info.value is close_error+ assert exc_info.value.__cause__ is pause_error+ assert data_plane.close_calls == 1+ assert sandbox.pauses == [True]++@pytest.mark.anyioasync def test_e2b_acquire_retries_transient_shellctl_failures_until_ready(monkeypatch: pytest.MonkeyPatch,+ sleep_delays: list[float],) -> None:monkeypatch.setenv("SHELLCTL_AUTH_TOKEN", "ambient-shellctl-token")attempts = 0@@ -373,18 +952,12 @@ async def test_e2b_acquire_retries_transient_shellctl_failures_until_ready(return httpx.Response(200, json={"status": "ok"})clients = _mock_http(monkeypatch, handler)- sleeps: list[float] = []-- async def record_sleep(delay: float) -> None:- sleeps.append(delay)-- monkeypatch.setattr(e2b_module.asyncio, "sleep", record_sleep)backend, sandbox = _connected_backend()lease = await backend.acquire(sandbox.sandbox_id)assert attempts == 3- assert sleeps == [0.5, 0.5]+ assert sleep_delays == [0.5, 0.5]assert lease.layout.home_dir == "/home/dify"assert lease.layout.workspace_dir == "/workspace"assert not clients[0].is_closed@@ -415,6 +988,7 @@ async def test_e2b_acquire_fails_closed_without_traffic_token(@pytest.mark.anyioasync def test_e2b_acquire_closes_transport_and_pauses_after_readiness_retries_exhausted(monkeypatch: pytest.MonkeyPatch,+ sleep_delays: list[float],) -> None:attempts = 0@@ -424,19 +998,13 @@ async def test_e2b_acquire_closes_transport_and_pauses_after_readiness_retries_ereturn httpx.Response(503, json={"error": {"code": "starting", "message": "not ready"}})clients = _mock_http(monkeypatch, handler)- sleeps: list[float] = []-- async def record_sleep(delay: float) -> None:- sleeps.append(delay)-- monkeypatch.setattr(e2b_module.asyncio, "sleep", record_sleep)backend, sandbox = _connected_backend()with pytest.raises(BindingAcquireError, match="not ready"):_ = await backend.acquire(sandbox.sandbox_id)assert attempts == 3- assert sleeps == [0.5, 0.5]+ assert sleep_delays == [0.5, 0.5]assert clients[0].is_closedassert sandbox.pauses == [True]diff --git a/dify-agent/tests/local/dify_agent/server/test_execution_bindings.py b/dify-agent/tests/local/dify_agent/server/test_execution_bindings.pyindex 4ff354be6c9..82d3d69415d 100644--- a/dify-agent/tests/local/dify_agent/server/test_execution_bindings.py+++ b/dify-agent/tests/local/dify_agent/server/test_execution_bindings.py@@ -1,23 +1,30 @@from dataclasses import dataclass, field+from fastapi import FastAPI+from fastapi.testclient import TestClientimport pytestfrom dify_agent.protocol import CreateExecutionBindingRequest, DestroyExecutionBindingRequestfrom dify_agent.runtime_backend import (+ BindingCapacityExhaustedError,ExecutionBindingAllocation,ExecutionBindingCreateSpec,ExecutionBindingDestroySpec,)from dify_agent.server.execution_bindings import ExecutionBindingService+from dify_agent.server.routes.execution_bindings import create_execution_bindings_router@dataclass(slots=True)class _Backend:created: list[ExecutionBindingCreateSpec] = field(default_factory=list)destroyed: list[ExecutionBindingDestroySpec] = field(default_factory=list)+ create_error: Exception | None = Noneasync def create_binding(self, spec: ExecutionBindingCreateSpec) -> ExecutionBindingAllocation:self.created.append(spec)+ if self.create_error is not None:+ raise self.create_errorreturn ExecutionBindingAllocation(binding_ref="opaque-binding", workspace_ref="opaque-workspace")async def destroy_binding(self, spec: ExecutionBindingDestroySpec) -> None:@@ -64,3 +71,32 @@ async def test_execution_binding_service_forwards_final_contract() -> None:destroy_workspace=True,)]+++def test_execution_binding_route_reports_capacity_exhaustion_as_429() -> None:+ backend = _Backend(create_error=BindingCapacityExhaustedError("maximum concurrent sandboxes reached"))+ service = ExecutionBindingService(backend=backend) # pyright: ignore[reportArgumentType]+ app = FastAPI()+ app.include_router(create_execution_bindings_router(lambda: service))++ with TestClient(app) as client:+ response = client.post(+ "/execution-bindings",+ json={+ "tenant_id": "tenant-1",+ "agent_id": "agent-1",+ "binding_id": "binding-1",+ "workspace_id": "workspace-1",+ "existing_workspace_ref": None,+ "home_snapshot_ref": None,+ },+ )++ assert response.status_code == 429+ assert response.json() == {+ "detail": {+ "code": "binding_capacity_exhausted",+ "message": "maximum concurrent sandboxes reached",+ }+ }+ assert len(backend.created) == 1diff --git a/dify-agent/tests/local/test_packaging.py b/dify-agent/tests/local/test_packaging.pyindex d382da2d405..d5f767dcdfd 100644--- a/dify-agent/tests/local/test_packaging.py+++ b/dify-agent/tests/local/test_packaging.py@@ -16,7 +16,7 @@ CLIENT_SHARED_DTO_DEPENDENCIES = {}SERVER_RUNTIME_DEPENDENCIES = {- "e2b>=2.34.0,<3.0.0",+ "e2b>=2.38.0,<3.0.0","fastapi==0.136.0","graphon==0.5.2","jsonschema>=4.23.0,<5.0.0",diff --git a/dify-agent/uv.lock b/dify-agent/uv.lockindex 9ae245a835b..4283e4be4e9 100644--- a/dify-agent/uv.lock+++ b/dify-agent/uv.lock@@ -394,6 +394,19 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/8d/e4/d66708bdf0d92fb4d49b22cdff4b10cec38aca5dcd7e81d909bb55c65cd7/confection-1.3.3-py3-none-any.whl", hash = "sha256:b9fef9ee84b237ef4611ec3eb5797b70e13063e6310ad9f15536373f5e313c82", size = 35902, upload-time = "2026-03-24T18:45:22.664Z" },]+[[package]]+name = "connectrpc"+version = "0.11.1"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "protobuf-py" },+ { name = "pyqwest" },+]+sdist = { url = "https://files.pythonhosted.org/packages/8b/b5/63e14ab9d4d4cc58818562db4120a35fa7454e3035633da41bdc6b712abd/connectrpc-0.11.1.tar.gz", hash = "sha256:18277f7838847b4271ca38d40c7d2387b5a2ea6a29f240689c19e1ec84aaff66", size = 46222, upload-time = "2026-07-15T06:33:31.601Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/5c/ac/aa647675812cd075f2cb9acb00d62f4f2cbcbc6736049a62342b7371ce5b/connectrpc-0.11.1-py3-none-any.whl", hash = "sha256:8a52e2e92a485fa9681c1101a79a5ebeb31807e3ea3d5aabd41f484a6398bc7b", size = 64991, upload-time = "2026-07-15T06:33:29.396Z" },+]+[[package]]name = "coverage"version = "7.14.0"@@ -630,7 +643,7 @@ docs = [[package.metadata]requires-dist = [- { name = "e2b", marker = "extra == 'server'", specifier = ">=2.34.0,<3.0.0" },+ { name = "e2b", marker = "extra == 'server'", specifier = ">=2.38.0,<3.0.0" },{ name = "fastapi", marker = "extra == 'server'", specifier = "==0.136.0" },{ name = "graphon", marker = "extra == 'server'", specifier = "==0.5.2" },{ name = "httpx", specifier = "==0.28.1" },@@ -694,24 +707,25 @@ wheels = [[[package]]name = "e2b"-version = "2.34.0"+version = "2.38.0"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "attrs" },+ { name = "connectrpc" },{ name = "dockerfile-parse" },{ name = "h2" },- { name = "httpcore" },{ name = "httpx" },{ name = "packaging" },- { name = "protobuf" },+ { name = "protobuf-py" },+ { name = "pyqwest" },{ name = "python-dateutil" },{ name = "rich" },{ name = "typing-extensions" },{ name = "wcmatch" },]-sdist = { url = "https://files.pythonhosted.org/packages/5b/22/5bcf34304b62e6984df9089eb996a4b2aa63618f901befe1f7cc73a24437/e2b-2.34.0.tar.gz", hash = "sha256:0dcc2694b3ead62e87b3d1070a05f66842dd2fa320fc1d7bd84c8fcfadeaf1b1", size = 189367, upload-time = "2026-07-17T09:59:23.844Z" }+sdist = { url = "https://files.pythonhosted.org/packages/e2/e2/1acb7bd6e16811a3affc89521304420410f97040a0a386497b101f898de3/e2b-2.38.0.tar.gz", hash = "sha256:b245976cc946d144baf77341b19fa537bfd7cbd9028dc8ef59bd6947f5e329c1", size = 205432, upload-time = "2026-08-10T17:57:07.643Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/6d/ec/b2f9297afc25de7c27ab5a85db4060783680e2b132285e2498fdbad484cf/e2b-2.34.0-py3-none-any.whl", hash = "sha256:873323571d18bf633be45e59fc6271410b30dfbc81e8df85e711f4f184c03fea", size = 343447, upload-time = "2026-07-17T09:59:22.553Z" },+ { url = "https://files.pythonhosted.org/packages/03/fa/e6d50d307a9dcde63038589fce7c1cb32ce3efde3d70c20a6adb11eee3c6/e2b-2.38.0-py3-none-any.whl", hash = "sha256:9f43803aa23e9a33ed48c1c778d588f83c0390db21edf3098255137981541b67", size = 365646, upload-time = "2026-08-10T17:57:08.947Z" },][[package]]@@ -2309,6 +2323,53 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" },]+[[package]]+name = "protobuf-py"+version = "0.1.1"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "protobuf-py-ext", marker = "(platform_machine == 'arm64' and platform_python_implementation == 'CPython' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_python_implementation == 'CPython' and sys_platform == 'linux') or (platform_machine == 'AMD64' and platform_python_implementation == 'CPython' and sys_platform == 'win32') or (platform_machine == 'ARM64' and platform_python_implementation == 'CPython' and sys_platform == 'win32')" },+]+sdist = { url = "https://files.pythonhosted.org/packages/72/ed/02fd902d9c51b7ff53dfc9a745eb11490722edfd30073af889e171f07b8e/protobuf_py-0.1.1.tar.gz", hash = "sha256:6bd08ac4d8f1661965bbe2685429d79043704cdd1ee720a7a89617331742240b", size = 133525, upload-time = "2026-06-24T19:02:15.271Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/fd/00/1b3775aca1c70e3007e06ef5996f6bb9b3a32341eb0cce3ffb6effad8dec/protobuf_py-0.1.1-py3-none-any.whl", hash = "sha256:efc4f50f275ed6dae10a1f30bb81ad1a75368557b3ff22a532b7a472050368f1", size = 181656, upload-time = "2026-06-24T19:01:29.556Z" },+]++[[package]]+name = "protobuf-py-ext"+version = "0.1.1"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/2f/05/6dc9ccff1e8159eb9a144e6d3c4acfd2211cd4fcd20c34fe9155d17d6a7f/protobuf_py_ext-0.1.1.tar.gz", hash = "sha256:e85bfdfdb3ed50634db8ccc7429dd9286520109489c735463971a418707b4fef", size = 31912, upload-time = "2026-06-24T19:02:16.321Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/19/70/2b5d62a60a2e0d88ecde1ae98db3132bcd2672fb39c8d581b82b34ac0bd8/protobuf_py_ext-0.1.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:310039e03cb15181781a0b78017419f6d4ee302e988c3c70b87f1facdf05532d", size = 306008, upload-time = "2026-06-24T19:01:31.399Z" },+ { url = "https://files.pythonhosted.org/packages/2d/d6/73c06bb4cac2e04c0adc154965fe8b6520224bd737fd7e73bba405056321/protobuf_py_ext-0.1.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89ec8348d1ba045f79b2fedd14e40cca36f2a41b52f2c4fdf55a60c58add2353", size = 314769, upload-time = "2026-06-24T19:01:32.89Z" },+ { url = "https://files.pythonhosted.org/packages/18/c5/e4e6bc6096b66d1c82639a1b501147f16ee65d32567304b987d002e2c666/protobuf_py_ext-0.1.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:182798e4861aba72d05855bd06febe4926aa7265e6f444a1b8af5252beee4f7e", size = 328122, upload-time = "2026-06-24T19:01:34.394Z" },+ { url = "https://files.pythonhosted.org/packages/b0/4b/1d792a40d0f0a0f914f1dfa8bb5e9573ca0ecd5fe5cecf80d77193212abd/protobuf_py_ext-0.1.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9eb25c3a329c0551cc86b209a5e5d8ecb8d834b9924a3aa019377853a703b6d3", size = 492338, upload-time = "2026-06-24T19:01:36.103Z" },+ { url = "https://files.pythonhosted.org/packages/52/51/5ad329223c905e5c530cae38ae24cde3584d8ab7e09457f2a01afce80e60/protobuf_py_ext-0.1.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:aaecbde82bef10c7c40578cbb61b7a19896bf7fa450972050a3bb302acb7d5d6", size = 541578, upload-time = "2026-06-24T19:01:37.481Z" },+ { url = "https://files.pythonhosted.org/packages/ed/c7/01bd8a8bfe2df4b07aafac12ccfb7b7ebe2303f65296a9e02fcafa28ff3e/protobuf_py_ext-0.1.1-cp310-abi3-win_amd64.whl", hash = "sha256:6b0c615c48e95acc53cf33e9310eeaff8b30d2d7555bf93e7bca8fb4f40e9a5c", size = 251319, upload-time = "2026-06-24T19:01:38.946Z" },+ { url = "https://files.pythonhosted.org/packages/24/dc/914065538b5db54b6a920b5af38c1ef142252ea1eea711820435fa259fac/protobuf_py_ext-0.1.1-cp310-abi3-win_arm64.whl", hash = "sha256:72956cd0af5dee24b41c6f5ba5e42622d17e6d555002b5efc1634e27a1446de2", size = 241635, upload-time = "2026-06-24T19:01:40.301Z" },+ { url = "https://files.pythonhosted.org/packages/13/a5/0b5d73cab815fd50615cb87a02e04e8332f9e27380c890aba1459cf7e919/protobuf_py_ext-0.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c6f0cd58620f415a3534d195358338f4999a774e12510f91c592b38d13d37388", size = 304903, upload-time = "2026-06-24T19:01:41.796Z" },+ { url = "https://files.pythonhosted.org/packages/37/a9/14c120b9ac36a0e11bb05e482fa6ae322de583a8e1068e4859035c289947/protobuf_py_ext-0.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21572764f625d829604fc4d83635f533f35775f11c6da142345b3f3c8d64bd09", size = 316148, upload-time = "2026-06-24T19:01:43.247Z" },+ { url = "https://files.pythonhosted.org/packages/35/54/7c00a1a9783c3ba74f6b2f5d2f049f09037bbd489c465c09a34f32fb611b/protobuf_py_ext-0.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7f14f00c2678bfbe7ad46f057b9f9938c1677bcf39e405e163e272c6f9814b8", size = 326458, upload-time = "2026-06-24T19:01:44.655Z" },+ { url = "https://files.pythonhosted.org/packages/0d/0e/81fa50c0d6c4d664e5be6d0a3c4f2c7edfef87ab12d0bac764abca1f2955/protobuf_py_ext-0.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1610865622e2e27568277ca63d8d2d23dcc55eaa767398865c21539ddf4ce24d", size = 493596, upload-time = "2026-06-24T19:01:46.344Z" },+ { url = "https://files.pythonhosted.org/packages/56/19/3183cf6e4de62846c20549654a1d573dae51ca06aaf7fed06accdfab74f6/protobuf_py_ext-0.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8995efba9476e1ea18ef9873306871dba697308994bc2766558fec3387acc3de", size = 539656, upload-time = "2026-06-24T19:01:48.21Z" },+ { url = "https://files.pythonhosted.org/packages/a4/6c/09841f3dbe7c3d4b3f2779f8f2832ac23fb96ac8ab71674e91af732cf9ff/protobuf_py_ext-0.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ba61d49dace8f874361583030a7c48139b42eb37c9ffbb1e7e8a227a51576f44", size = 305017, upload-time = "2026-06-24T19:01:49.84Z" },+ { url = "https://files.pythonhosted.org/packages/52/43/e450b5e202f6715274acc9acd9f9769908cbdeab861a53c798fcbd467d35/protobuf_py_ext-0.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a9c2f026096ca4c595c89a297067ef371e241d3ff8e1f6d7c779aa8419cdca7", size = 316215, upload-time = "2026-06-24T19:01:51.249Z" },+ { url = "https://files.pythonhosted.org/packages/97/5c/23e79b35f1b5755b9bdc0c0c9b8cd8abababaef1a1639608d8a96bb61a9d/protobuf_py_ext-0.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf78707a040b9294e5e1ec4a1875f0046acfe52e92150cea27de4e9fc9db39bb", size = 326419, upload-time = "2026-06-24T19:01:52.93Z" },+ { url = "https://files.pythonhosted.org/packages/e8/de/5493de0ec12920a3b1dd72608ce0c1e8cdb7e0eb9e87accaecc5216df29e/protobuf_py_ext-0.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ae4373845cbb85bdade3ef5368cc2f4b5f80bf173383afc1ab063f95644e5599", size = 493734, upload-time = "2026-06-24T19:01:54.301Z" },+ { url = "https://files.pythonhosted.org/packages/98/89/31da55b414e6332aad11d858e9a86bd36fbb324f52fa3fc6d8f1c57840a9/protobuf_py_ext-0.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2a4cc478eef7a2acc1daaebcde479a3ac2396d47e6bdc7e776ca4c4147ba8b4c", size = 539703, upload-time = "2026-06-24T19:01:55.707Z" },+ { url = "https://files.pythonhosted.org/packages/1c/71/39f231838ef06476d46ac40dd814894277a732a3688c0a0c994850b3b62f/protobuf_py_ext-0.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:359dccdc1c3eafed2a913c570bb082b8df848a1d27d548ce8385f246a7d68be2", size = 302145, upload-time = "2026-06-24T19:01:57.116Z" },+ { url = "https://files.pythonhosted.org/packages/87/f1/01c81ff5f420600366a0e6a613ce2af20834534d2b3d7523f4f3b4ddb54f/protobuf_py_ext-0.1.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91c38b4a10a306366443273ee03ca554537a0965bf05b7ada8e7dbdee04cc93e", size = 314541, upload-time = "2026-06-24T19:01:58.745Z" },+ { url = "https://files.pythonhosted.org/packages/86/d8/1cbf5a0298ceddc0cdbb28bf1ecaf8bd3cff54ed4ced6a1392d5226172fc/protobuf_py_ext-0.1.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79eee3bcbb289d6ea114eb8fe3a1469c5b59bf53e207238469f567d9c53ba56f", size = 325092, upload-time = "2026-06-24T19:02:00.382Z" },+ { url = "https://files.pythonhosted.org/packages/af/23/8b1694616044cbfec2d18d4c6fffb03e05089c8bdb5970c6bafd60cc7fd5/protobuf_py_ext-0.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:10992141a8282a71ac3e3530d1a489efb27618d84000b9a47918cf70e5816d9b", size = 491684, upload-time = "2026-06-24T19:02:01.862Z" },+ { url = "https://files.pythonhosted.org/packages/49/d8/99997a7a6cf6989c944d9183c5deb6a045c27460add0316c7b34763891b3/protobuf_py_ext-0.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4411ffe0e06a774b83c5c71c546ce097640a25f596c45f95f53d3e3148e3f22d", size = 538048, upload-time = "2026-06-24T19:02:03.362Z" },+ { url = "https://files.pythonhosted.org/packages/15/3c/9e99ecbb68e1d5b46016d821eb1cbba9a91e6b50e71e75faf0c1e6189f0d/protobuf_py_ext-0.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:55a17d80ea419501ff221a6627523f38a431bb33e6aa3de81ae3a7f271c49c75", size = 296337, upload-time = "2026-06-24T19:02:04.787Z" },+ { url = "https://files.pythonhosted.org/packages/af/f2/305338a28225fb54b28d6c3f5948109b9088b329a2b6f77ca610c2bdcd34/protobuf_py_ext-0.1.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dbb518b5638403ceaa08ac4fc7dac626f45ed9b856b3517de3906cf3de4d632", size = 308961, upload-time = "2026-06-24T19:02:06.185Z" },+ { url = "https://files.pythonhosted.org/packages/60/f9/416103c93677ff2ea407704ea64fa6de9e700dc030c48360a182d91ce373/protobuf_py_ext-0.1.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a4adc65ab6a5e4885c67fc808bcacb83d755d05b566d312a0e10c2a873f2ad4", size = 321895, upload-time = "2026-06-24T19:02:07.714Z" },+ { url = "https://files.pythonhosted.org/packages/5a/83/eb3e81bb2f83834b7deff4cee2d56eeb1adcbc847492a56b7f910ab257db/protobuf_py_ext-0.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e97f45c49676efacfb8e95bfcb1a002bc337e618a6780be1e55df2ba5ebd2f1e", size = 486091, upload-time = "2026-06-24T19:02:09.243Z" },+ { url = "https://files.pythonhosted.org/packages/99/96/bd88fca38556b3105e4dd41d6a176e31dcc583fe979e830aeedd2f8c20ee/protobuf_py_ext-0.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:53a6b3590f6aa7f97b8ed60f62fef9b096babfeae82f7283fd3a4c405827d4f8", size = 534582, upload-time = "2026-06-24T19:02:11.227Z" },+]+[[package]]name = "psutil"version = "7.2.2"@@ -2624,6 +2685,52 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/bc/67/4759522f5bca0ac4cda9f42c7f3f818aa826568793bd8b4532d2d2ffa515/pypdfium2-5.7.1-py3-none-win_arm64.whl", hash = "sha256:622821698fcc30fc560bd4eead6df9e6b846de9876b82861bed0091c09a4c27b", size = 3540903, upload-time = "2026-04-20T15:01:00.994Z" },]+[[package]]+name = "pyqwest"+version = "0.9.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "opentelemetry-api" },+]+sdist = { url = "https://files.pythonhosted.org/packages/8d/59/97531fd9d0a06d54e84f5493775739b0c1d0d13ec704974d04fa423a3332/pyqwest-0.9.0.tar.gz", hash = "sha256:514dd0b37d7a1bcb978b5d6423d15f89cf7dcf8058ac2a37aa42cd30090de89e", size = 477462, upload-time = "2026-08-10T01:55:36.219Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/47/34/d556fa07d9bff21fc6c9c16738614c6336164b65d9ced1ce9f1c1044aaeb/pyqwest-0.9.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:517d2eb295d56ef1530cebe0cf290a66fa199c57955bd1c5f1dc592240fbb7d6", size = 5224594, upload-time = "2026-08-10T01:54:18.646Z" },+ { url = "https://files.pythonhosted.org/packages/c9/3f/1d7d1a08ebf8838c0c85e99ab5db5c1aae457f76ee03b60c8f4fc4948e01/pyqwest-0.9.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:408dc692835363b6824bee61ee9e4c9a5bba9c08d75b903f82f9a802e41d1b30", size = 5100753, upload-time = "2026-08-10T01:54:20.669Z" },+ { url = "https://files.pythonhosted.org/packages/66/fe/22c4056b526baa4c2c953f4e99925ce0bb14e7747d66e6ac3448a34f76e5/pyqwest-0.9.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1abf1ab3a0c1b545651ab37a643acc7909c4e7644a63a2e4af36e3e230a0db5", size = 5619882, upload-time = "2026-08-10T01:54:22.468Z" },+ { url = "https://files.pythonhosted.org/packages/de/2c/8f1159a30bbc905b2fcbc386740a260e76ae64372feb263103438ef5201c/pyqwest-0.9.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b02e5c9ce57e079eb3716c464d3818ad2573d7743483c39526ad77ee95af806", size = 5564744, upload-time = "2026-08-10T01:54:24.814Z" },+ { url = "https://files.pythonhosted.org/packages/ac/fd/ff15034d7874fd208d606bd80c924fb0b1091db8159c892672168b656de1/pyqwest-0.9.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:98bdaa30a3a8d8c71506a33c084e485a243fbf51e77130bb39e0f249dd116142", size = 5779373, upload-time = "2026-08-10T01:54:26.753Z" },+ { url = "https://files.pythonhosted.org/packages/ed/b5/eeafdeb65ad8fe9c39044aedcfc97610f4e5c7b5620ca9fe2d504f669108/pyqwest-0.9.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:10eba84f9d8512d23bd16dd60b1c77dab956470361559214981dda065df4ad1a", size = 5973306, upload-time = "2026-08-10T01:54:28.729Z" },+ { url = "https://files.pythonhosted.org/packages/34/db/a8979dab5c591529f858d1505d09b3c8b6dec337921f15bed4858c55716e/pyqwest-0.9.0-cp310-abi3-win_amd64.whl", hash = "sha256:af4e859800b02cd1fcdd898af26a881c4a8ce9520f71f6cb0a5403523d2cbbdf", size = 4834706, upload-time = "2026-08-10T01:54:30.335Z" },+ { url = "https://files.pythonhosted.org/packages/1d/9e/d070ca1ce3c202952770436172889a4b0556a564a490baf726f7bb895eba/pyqwest-0.9.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:abe322cc1b63947b493bc06028615e0d5de655928159227aab64f60a0eb0e27d", size = 5244745, upload-time = "2026-08-10T01:54:32.257Z" },+ { url = "https://files.pythonhosted.org/packages/41/97/ebded0bb35a79f51bb2d1519e34922d79c8825f9e71b696bb20f387a08c2/pyqwest-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c6b2dc6a9c583859d031941e1041b8e711afe0ad4c130628207731cef2b1a486", size = 5101044, upload-time = "2026-08-10T01:54:34.088Z" },+ { url = "https://files.pythonhosted.org/packages/a9/2c/a7f117e793b4e27642807bf7230b9fa297ff1e1ac489fc260fc3d4350ae4/pyqwest-0.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:551e47714da72a72939958029eb37a754cecd974344471c4a2038583a66a5396", size = 5630841, upload-time = "2026-08-10T01:54:35.885Z" },+ { url = "https://files.pythonhosted.org/packages/b4/b4/be47fc141529941352a848cd77da581803137b733993a51bf464f2c48915/pyqwest-0.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3be3d38ccdab3077bf1cfe90c346ce927f9b49c56dcdf20e060f5f2502d01021", size = 5572909, upload-time = "2026-08-10T01:54:37.653Z" },+ { url = "https://files.pythonhosted.org/packages/cb/fd/703635e12710ce7ef5e961f5e1b22c609aecc8f07eedd79477b48d1edb74/pyqwest-0.9.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:45d9d424da878b22604799d1ce2a0a5df054924487ee5a851ad08025e2076e5c", size = 5794049, upload-time = "2026-08-10T01:54:39.363Z" },+ { url = "https://files.pythonhosted.org/packages/3c/15/941aad56b108c743df9d41e124be82ecb6be0eba1007e945b6a39f5dd1a0/pyqwest-0.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e4420adcd2b756da706b31a04f095a521f6e440dab284b7a1b46c76e048c1f78", size = 5974542, upload-time = "2026-08-10T01:54:41.202Z" },+ { url = "https://files.pythonhosted.org/packages/b7/86/8445fe05b47322047347464a909ae95dba023bf8409b9967f66bda2905ad/pyqwest-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:25734bd9798bbfaa53c83d395649dd9ea9a791a64b9848734955cb78ad52c832", size = 4847034, upload-time = "2026-08-10T01:54:42.899Z" },+ { url = "https://files.pythonhosted.org/packages/a7/92/013fc3c94f0dc4eeca556431797da477d76ad43d69134837ba149468dcc3/pyqwest-0.9.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:6e4c2afe6c7293d9e42cfdb0489b0b5f64002159c2dc9c45d9322c1b4ffb42cd", size = 5244793, upload-time = "2026-08-10T01:54:44.891Z" },+ { url = "https://files.pythonhosted.org/packages/48/61/c685e0c595f3f1b7842a776c48aaf2b3f47270a61672967d9d38f7c5e7f6/pyqwest-0.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:67f153b94dfbb9aeabc38f20a1bada1acfd9361bec12db86268bd951be35a154", size = 5100715, upload-time = "2026-08-10T01:54:46.777Z" },+ { url = "https://files.pythonhosted.org/packages/f9/a1/a28b2cd704d7dd37043f1e6b398798576c12293f3efae84f00e7f6a5625c/pyqwest-0.9.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9942ffa243c8c2243e3ce04e3e3ec9b4c119346c963fd0051cd989f0defecf6", size = 5628787, upload-time = "2026-08-10T01:54:48.488Z" },+ { url = "https://files.pythonhosted.org/packages/36/31/6255c7f17cd00e39f1c4df3003cea2d2d087e483fb4c49a6535307cd21e0/pyqwest-0.9.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f8f9c880c19826fead1838e12b042865b15df3ec844928cc74188a8b740f761", size = 5571946, upload-time = "2026-08-10T01:54:50.181Z" },+ { url = "https://files.pythonhosted.org/packages/37/71/344738bdb38bddbc07cb29e5b1287c08db9e53d6b322581492571bdef748/pyqwest-0.9.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:56040761f6ca7abb036c7d288863bf143a9b166d824d98a604908ef20c6906e0", size = 5792872, upload-time = "2026-08-10T01:54:52.045Z" },+ { url = "https://files.pythonhosted.org/packages/fc/ee/c5c74e3afb6dd0a25bca23bb4757d1a7135fab09a0a054036e837d49dd11/pyqwest-0.9.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:02a51df744521e45eb5379bc325b91de4512ec64a9f4aba3248f9166ec4e5b88", size = 5974192, upload-time = "2026-08-10T01:54:53.818Z" },+ { url = "https://files.pythonhosted.org/packages/6e/08/302960836e3e43ca3051313ee3b767534d1610bd3b8033c7bbad41b06e1b/pyqwest-0.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:e9a0e3123aee446d5f6e57c43cae87b4dab4b1af3fa970d114d74d5d4b9d075c", size = 4847050, upload-time = "2026-08-10T01:54:56.155Z" },+ { url = "https://files.pythonhosted.org/packages/fc/3e/a36940844325fbc59adf92116d3bcd1155cc2c72d1a390880886ca492bb4/pyqwest-0.9.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:fdab21bbedff4a21209135423f5f54f287a01dd6143cbd7aeab63d5d8c889654", size = 5246741, upload-time = "2026-08-10T01:54:57.697Z" },+ { url = "https://files.pythonhosted.org/packages/41/8d/efb9741449a81f2489f6929a7691a2145662c3650e06bcb139c64f543d0a/pyqwest-0.9.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:21a04ddf4497ef3c7fe36cdc473c094999b7710005d38bd8a0dfd4a3feace12f", size = 5102423, upload-time = "2026-08-10T01:54:59.253Z" },+ { url = "https://files.pythonhosted.org/packages/8f/a8/0b667cf9c07e62861cfe24c09ad30357a9c2a0fc3863cfc5610af5b66e3e/pyqwest-0.9.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8c51c1a27ea529fffab90c127a045f0dfe1abc54d5b9ad5f4165e954439c9b2", size = 5627663, upload-time = "2026-08-10T01:55:01.036Z" },+ { url = "https://files.pythonhosted.org/packages/5f/7c/1f88be3d35afe92b79e361aa40faf3904e3a7852e8638a9d3d5c00891ab3/pyqwest-0.9.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10f283ffc40a2774e0dc8fbb47e4a8f9f8a698a99feae3bf4ad75b340c98031b", size = 5572750, upload-time = "2026-08-10T01:55:03.163Z" },+ { url = "https://files.pythonhosted.org/packages/21/3e/98c1e151772e77dcf00cb8b1c070f2c8edcb15fe12fdee30c1022006ca8e/pyqwest-0.9.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1f73d4a8e75b5860741fbd0623a0ce792a53336dbfec5f34736004acd5446843", size = 5790615, upload-time = "2026-08-10T01:55:04.758Z" },+ { url = "https://files.pythonhosted.org/packages/7b/f1/12bb4b119cffb7fe5301b2d997ec9ffc1719637f6f4faf2ff2315e13b201/pyqwest-0.9.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e34d55a53492b44b82b690c2b11957fc35e99ebab4db2e9bcb2a1e2621d71b62", size = 5976516, upload-time = "2026-08-10T01:55:06.453Z" },+ { url = "https://files.pythonhosted.org/packages/53/55/046dbf68ccedd816770fd212c6c2612372a6f0d7ecf2d8273480db460ddf/pyqwest-0.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:48c367150783f6866d0af43c209cc405f8287743d7b2472c7d757fda2478fd57", size = 4841470, upload-time = "2026-08-10T01:55:08.035Z" },+ { url = "https://files.pythonhosted.org/packages/55/cd/38ff77d145385afd71b2dbd6eb116251ce3ffddbed0edde16c64394fbec2/pyqwest-0.9.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:10fae206270355c993b49c49bf9778995167f5efe591807d50a8e682921d71ea", size = 5222689, upload-time = "2026-08-10T01:55:09.639Z" },+ { url = "https://files.pythonhosted.org/packages/82/95/d8c6804eeb72b7b89842b58f1f05d5cfda5be410c4a22442d2574d510d39/pyqwest-0.9.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5f475129fb792593222a9ce0fefbbc10bda1544a7411f3946d771c9a61c50e61", size = 5085846, upload-time = "2026-08-10T01:55:11.569Z" },+ { url = "https://files.pythonhosted.org/packages/f2/e7/04fe8962ec416be0199093ebdd326653a5a7b8f53ddfdb29af81756f7c08/pyqwest-0.9.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78f6f676d349535cb094e8e3bcf33863238a142fb7dcc7afb9e61749e2351047", size = 5613296, upload-time = "2026-08-10T01:55:13.315Z" },+ { url = "https://files.pythonhosted.org/packages/03/0c/7a11728155a34838d2e7c78fb24c937f2cd844ec791ef416030c52f4bb5e/pyqwest-0.9.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f460f5fcdae8cc46e6a679128836f6cd47fbf4bf73e5c9d6ea868e4a90b2e7ee", size = 5557492, upload-time = "2026-08-10T01:55:15.103Z" },+ { url = "https://files.pythonhosted.org/packages/4a/d5/8c6e68f0e978b5257b309809fec00d946c1ed20dacef97f25c23de5b91a9/pyqwest-0.9.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:cba6804151e973d27c99b9f02b3f0bcab7270c3f99725b508ada9c147365cabd", size = 5774204, upload-time = "2026-08-10T01:55:16.932Z" },+ { url = "https://files.pythonhosted.org/packages/17/20/fed89b62d50b4efad2ebe78c3867e0e402f824726652034728a5dd0a342e/pyqwest-0.9.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:bc9dfeca326918d70a5368114de0fc627d9f64fafa36563ab484ae58ffe42c0d", size = 5964506, upload-time = "2026-08-10T01:55:19.031Z" },+ { url = "https://files.pythonhosted.org/packages/11/de/30575019efebae84502269d097e706e89a30631e3719218b053e4e0dc179/pyqwest-0.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:384693adc0908f16324e5b508a3c25fb9a92b88f9bdf7ccb6820507db4a95888", size = 4809013, upload-time = "2026-08-10T01:55:21.042Z" },+]+[[package]]name = "pytest"version = "9.0.3"