Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(sdk-python): retry gateway wake timeouts on streaming requests

Maohaoaf9910f8da1f
diff --git a/sdks/code-interpreter/python/src/code_interpreter/adapters/code_adapter.py b/sdks/code-interpreter/python/src/code_interpreter/adapters/code_adapter.py
index 64e0d2c86..8970a7398 100644
--- a/sdks/code-interpreter/python/src/code_interpreter/adapters/code_adapter.py
+++ b/sdks/code-interpreter/python/src/code_interpreter/adapters/code_adapter.py
@@ -41,7 +41,7 @@ from opensandbox.config import ConnectionConfig
from opensandbox.exceptions import InvalidArgumentException, SandboxApiException
from opensandbox.models.execd import Execution, ExecutionHandlers
from opensandbox.models.sandboxes import SandboxEndpoint
-from opensandbox.transport import unwrap_retry_transport
+from opensandbox.transport import streaming_transport
from code_interpreter.adapters.converter.code_execution_converter import (
CodeExecutionConverter,
@@ -128,9 +128,8 @@ class CodesAdapter(Codes):
"Accept": "text/event-stream",
"Cache-Control": "no-cache",
}
- # SSE bootstraps bypass the retry wrapper: request bodies are
- # not replayable and a non-idempotent status opt-in would cause
- # duplicate execution on a resent SSE POST.
+ # SSE bootstraps keep only the wake-timeout retry: a non-idempotent
+ # status opt-in would cause duplicate execution on a resent SSE POST.
sse_transport = (
self.connection_config.new_owned_transport()
if self._owns_clients
@@ -144,7 +143,7 @@ class CodesAdapter(Codes):
write=timeout_seconds,
pool=None,
),
- transport=unwrap_retry_transport(sse_transport),
+ transport=streaming_transport(sse_transport),
)
async def _get_client(self):
diff --git a/sdks/code-interpreter/python/src/code_interpreter/sync/adapters/code_adapter.py b/sdks/code-interpreter/python/src/code_interpreter/sync/adapters/code_adapter.py
index 217b7d5d5..4f9d7926c 100644
--- a/sdks/code-interpreter/python/src/code_interpreter/sync/adapters/code_adapter.py
+++ b/sdks/code-interpreter/python/src/code_interpreter/sync/adapters/code_adapter.py
@@ -39,7 +39,7 @@ from opensandbox.models.sandboxes import SandboxEndpoint
from opensandbox.sync.adapters.converter.execution_event_dispatcher import (
ExecutionEventDispatcherSync,
)
-from opensandbox.transport import unwrap_retry_transport
+from opensandbox.transport import streaming_transport
from code_interpreter.models.code_sync import CodeContextSync, SupportedLanguageSync
from code_interpreter.sync.services.code import CodesSync
@@ -123,9 +123,8 @@ class CodesAdapterSync(CodesSync):
"Accept": "text/event-stream",
"Cache-Control": "no-cache",
}
- # SSE bootstraps bypass the retry wrapper: request bodies are
- # not replayable and a non-idempotent status opt-in would cause
- # duplicate execution on a resent SSE POST.
+ # SSE bootstraps keep only the wake-timeout retry: a non-idempotent
+ # status opt-in would cause duplicate execution on a resent SSE POST.
sse_transport = (
self.connection_config.new_owned_transport()
if self._owns_clients
@@ -136,7 +135,7 @@ class CodesAdapterSync(CodesSync):
timeout=httpx.Timeout(
connect=timeout_seconds, read=None, write=timeout_seconds, pool=None
),
- transport=unwrap_retry_transport(sse_transport),
+ transport=streaming_transport(sse_transport),
)
def _get_execd_url(self, path: str) -> str:
diff --git a/sdks/sandbox/python/src/opensandbox/adapters/command_adapter.py b/sdks/sandbox/python/src/opensandbox/adapters/command_adapter.py
index c5aa78038..d1fb9dbf1 100644
--- a/sdks/sandbox/python/src/opensandbox/adapters/command_adapter.py
+++ b/sdks/sandbox/python/src/opensandbox/adapters/command_adapter.py
@@ -69,7 +69,7 @@ from opensandbox.models.execd import (
)
from opensandbox.models.sandboxes import SandboxEndpoint
from opensandbox.services.command import Commands
-from opensandbox.transport import unwrap_retry_transport
+from opensandbox.transport import streaming_transport
logger = logging.getLogger(__name__)
@@ -135,9 +135,8 @@ class CommandsAdapter(Commands):
"Accept": "text/event-stream",
"Cache-Control": "no-cache",
}
- # SSE bootstraps bypass the retry wrapper: request bodies are
- # not replayable and a non-idempotent status opt-in would cause
- # duplicate execution on a resent SSE POST.
+ # SSE bootstraps keep only the wake-timeout retry: a non-idempotent
+ # status opt-in would cause duplicate execution on a resent SSE POST.
self._sse_client = httpx.AsyncClient(
headers=sse_headers,
timeout=httpx.Timeout(
@@ -146,7 +145,7 @@ class CommandsAdapter(Commands):
write=timeout_seconds,
pool=None,
),
- transport=unwrap_retry_transport(self.connection_config.transport),
+ transport=streaming_transport(self.connection_config.transport),
**build_async_redirect_client_options(self.connection_config, base_url),
)
diff --git a/sdks/sandbox/python/src/opensandbox/adapters/isolated_adapter.py b/sdks/sandbox/python/src/opensandbox/adapters/isolated_adapter.py
index 55f37c5a6..78b5968fa 100644
--- a/sdks/sandbox/python/src/opensandbox/adapters/isolated_adapter.py
+++ b/sdks/sandbox/python/src/opensandbox/adapters/isolated_adapter.py
@@ -56,7 +56,7 @@ from opensandbox.services.isolated import (
IsolationServiceMixin,
IsolationSession,
)
-from opensandbox.transport import unwrap_retry_transport
+from opensandbox.transport import streaming_transport
logger = logging.getLogger(__name__)
@@ -258,9 +258,8 @@ class IsolatedSessionsAdapter(IsolationServiceMixin, IsolationService):
"Accept": "text/event-stream",
"Cache-Control": "no-cache",
}
- # SSE bootstraps bypass the retry wrapper: request bodies are
- # not replayable and a non-idempotent status opt-in would cause
- # duplicate execution on a resent SSE POST.
+ # SSE bootstraps keep only the wake-timeout retry: a non-idempotent
+ # status opt-in would cause duplicate execution on a resent SSE POST.
self._sse_client = httpx.AsyncClient(
headers=sse_headers,
timeout=httpx.Timeout(
@@ -269,7 +268,7 @@ class IsolatedSessionsAdapter(IsolationServiceMixin, IsolationService):
write=timeout_seconds,
pool=None,
),
- transport=unwrap_retry_transport(self.connection_config.transport),
+ transport=streaming_transport(self.connection_config.transport),
**build_async_redirect_client_options(self.connection_config, base_url),
)
diff --git a/sdks/sandbox/python/src/opensandbox/sync/adapters/command_adapter.py b/sdks/sandbox/python/src/opensandbox/sync/adapters/command_adapter.py
index ebb772fe2..acd0494ce 100644
--- a/sdks/sandbox/python/src/opensandbox/sync/adapters/command_adapter.py
+++ b/sdks/sandbox/python/src/opensandbox/sync/adapters/command_adapter.py
@@ -64,7 +64,7 @@ from opensandbox.sync.adapters.converter.execution_event_dispatcher import (
ExecutionEventDispatcherSync,
)
from opensandbox.sync.services.command import CommandsSync
-from opensandbox.transport import unwrap_retry_transport
+from opensandbox.transport import streaming_transport
logger = logging.getLogger(__name__)
@@ -123,9 +123,8 @@ class CommandsAdapterSync(CommandsSync):
"Accept": "text/event-stream",
"Cache-Control": "no-cache",
}
- # SSE bootstraps bypass the retry wrapper: request bodies are
- # not replayable and a non-idempotent status opt-in would cause
- # duplicate execution on a resent SSE POST.
+ # SSE bootstraps keep only the wake-timeout retry: a non-idempotent
+ # status opt-in would cause duplicate execution on a resent SSE POST.
self._sse_client = httpx.Client(
headers=sse_headers,
timeout=httpx.Timeout(
@@ -134,7 +133,7 @@ class CommandsAdapterSync(CommandsSync):
write=timeout_seconds,
pool=None,
),
- transport=unwrap_retry_transport(self.connection_config.transport),
+ transport=streaming_transport(self.connection_config.transport),
**build_redirect_client_options(self.connection_config, base_url),
)
diff --git a/sdks/sandbox/python/src/opensandbox/sync/adapters/isolated_adapter.py b/sdks/sandbox/python/src/opensandbox/sync/adapters/isolated_adapter.py
index 73e6ccf13..8aba6e863 100644
--- a/sdks/sandbox/python/src/opensandbox/sync/adapters/isolated_adapter.py
+++ b/sdks/sandbox/python/src/opensandbox/sync/adapters/isolated_adapter.py
@@ -58,7 +58,7 @@ from opensandbox.sync.services.isolated import (
IsolationServiceSyncMixin,
IsolationSessionSync,
)
-from opensandbox.transport import unwrap_retry_transport
+from opensandbox.transport import streaming_transport
logger = logging.getLogger(__name__)
@@ -199,9 +199,8 @@ class IsolatedSessionsAdapterSync(IsolationServiceSyncMixin, IsolationServiceSyn
"Accept": "text/event-stream",
"Cache-Control": "no-cache",
}
- # SSE bootstraps bypass the retry wrapper: request bodies are
- # not replayable and a non-idempotent status opt-in would cause
- # duplicate execution on a resent SSE POST.
+ # SSE bootstraps keep only the wake-timeout retry: a non-idempotent
+ # status opt-in would cause duplicate execution on a resent SSE POST.
self._sse_client = httpx.Client(
headers=sse_headers,
timeout=httpx.Timeout(
@@ -210,7 +209,7 @@ class IsolatedSessionsAdapterSync(IsolationServiceSyncMixin, IsolationServiceSyn
write=timeout_seconds,
pool=None,
),
- transport=unwrap_retry_transport(self.connection_config.transport),
+ transport=streaming_transport(self.connection_config.transport),
**build_redirect_client_options(self.connection_config, base_url),
)
diff --git a/sdks/sandbox/python/src/opensandbox/transport/__init__.py b/sdks/sandbox/python/src/opensandbox/transport/__init__.py
index 5cc5859ad..f67d02374 100644
--- a/sdks/sandbox/python/src/opensandbox/transport/__init__.py
+++ b/sdks/sandbox/python/src/opensandbox/transport/__init__.py
@@ -48,6 +48,21 @@ def unwrap_retry_transport(transport: _T) -> _T:
return transport
+def streaming_transport(transport: _T) -> _T:
+ """
+ Return the transport for a streaming (SSE) client.
+
+ SSE requests run commands, so general retries must not resend them: a
+ caller ``retryable_status_codes_non_idempotent`` opt-in would cause
+ duplicate execution. A retry wrapper is replaced by one that keeps only
+ the OSEP-0024 wake-timeout retry, which is safe because the gateway
+ never forwarded the request. Other transports are returned unchanged.
+ """
+ if isinstance(transport, (RetryAsyncTransport, RetrySyncTransport)):
+ return transport.for_streaming() # type: ignore[return-value]
+ return transport
+
+
__all__ = [
"JitterMode",
"RetryAsyncTransport",
@@ -55,5 +70,6 @@ __all__ = [
"RetryEvent",
"RetryPolicy",
"RetrySyncTransport",
+ "streaming_transport",
"unwrap_retry_transport",
]
diff --git a/sdks/sandbox/python/src/opensandbox/transport/_async_retry.py b/sdks/sandbox/python/src/opensandbox/transport/_async_retry.py
index 9ddfb890e..125a453a2 100644
--- a/sdks/sandbox/python/src/opensandbox/transport/_async_retry.py
+++ b/sdks/sandbox/python/src/opensandbox/transport/_async_retry.py
@@ -122,9 +122,20 @@ class RetryAsyncTransport(httpx.AsyncBaseTransport):
@property
def inner(self) -> httpx.AsyncBaseTransport:
- """The wrapped raw transport, safe to hand to SSE clients."""
+ """The wrapped raw transport."""
return self._inner
+ def for_streaming(self) -> httpx.AsyncBaseTransport:
+ """Transport for streaming (SSE) clients; see :meth:`RetryPolicy.wake_only`."""
+ if not self._policy.wake_retry_enabled:
+ return self._inner
+ return RetryAsyncTransport(
+ self._inner,
+ self._policy.wake_only(),
+ owns_inner=self._owns_inner,
+ rng=self._rng,
+ )
+
async def handle_async_request(
self, request: httpx.Request
) -> httpx.Response:
diff --git a/sdks/sandbox/python/src/opensandbox/transport/_sync_retry.py b/sdks/sandbox/python/src/opensandbox/transport/_sync_retry.py
index 8ab0dbb2a..6425cd0e3 100644
--- a/sdks/sandbox/python/src/opensandbox/transport/_sync_retry.py
+++ b/sdks/sandbox/python/src/opensandbox/transport/_sync_retry.py
@@ -72,9 +72,20 @@ class RetrySyncTransport(httpx.BaseTransport):
@property
def inner(self) -> httpx.BaseTransport:
- """The wrapped raw transport, safe to hand to SSE clients."""
+ """The wrapped raw transport."""
return self._inner
+ def for_streaming(self) -> httpx.BaseTransport:
+ """Transport for streaming (SSE) clients; see :meth:`RetryPolicy.wake_only`."""
+ if not self._policy.wake_retry_enabled:
+ return self._inner
+ return RetrySyncTransport(
+ self._inner,
+ self._policy.wake_only(),
+ owns_inner=self._owns_inner,
+ rng=self._rng,
+ )
+
def handle_request(self, request: httpx.Request) -> httpx.Response:
policy = self._policy
method = request.method.upper()
diff --git a/sdks/sandbox/python/src/opensandbox/transport/retry.py b/sdks/sandbox/python/src/opensandbox/transport/retry.py
index 63e5b0d5a..9b67bd736 100644
--- a/sdks/sandbox/python/src/opensandbox/transport/retry.py
+++ b/sdks/sandbox/python/src/opensandbox/transport/retry.py
@@ -18,7 +18,7 @@
from __future__ import annotations
from collections.abc import Callable
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
from datetime import timedelta
from enum import Enum
from http import HTTPStatus
@@ -184,6 +184,18 @@ class RetryPolicy:
"""Never retry. Also suppresses fresh-connection recovery and wake retry."""
return cls(max_retries=0, wake_retry_enabled=False)
+ def wake_only(self) -> RetryPolicy:
+ """
+ This policy with only the wake-timeout retry left.
+
+ Used for streaming (SSE) requests. They run commands, so they must
+ not be resent after a status code or a post-send failure, but a
+ marked wake timeout proves the gateway never forwarded the request.
+ """
+ return replace(
+ self, max_retries=0, per_attempt_timeout=None, overall_deadline=None
+ )
+
def wraps_transport(self) -> bool:
"""
Whether this policy requires the retry transport to be installed.
diff --git a/sdks/sandbox/python/tests/test_command_service_sse_client_config.py b/sdks/sandbox/python/tests/test_command_service_sse_client_config.py
index f77037630..b1a95fc03 100644
--- a/sdks/sandbox/python/tests/test_command_service_sse_client_config.py
+++ b/sdks/sandbox/python/tests/test_command_service_sse_client_config.py
@@ -14,10 +14,38 @@
# limitations under the License.
#
+import httpx
+import pytest
+
from opensandbox.adapters.command_adapter import CommandsAdapter
from opensandbox.config import ConnectionConfig
+from opensandbox.config.connection_sync import ConnectionConfigSync
from opensandbox.models.sandboxes import SandboxEndpoint
-from opensandbox.transport import RetryAsyncTransport
+from opensandbox.sync.adapters.command_adapter import CommandsAdapterSync
+from opensandbox.transport import RetryAsyncTransport, RetryPolicy, RetrySyncTransport
+
+_SSE_BODY = (
+ b'data: {"type":"init","text":"exec-1","timestamp":1}\n\n'
+ b'data: {"type":"execution_complete","timestamp":2,"execution_time":1}\n\n'
+)
+
+
+def _wake_then_stream_handler(calls: list[str]):
+ """First /command answer is a gateway wake timeout, then the SSE stream."""
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ calls.append(request.url.path)
+ if len(calls) == 1:
+ return httpx.Response(
+ 503,
+ headers={"OpenSandbox-Wake": "expired", "Retry-After": "0"},
+ json={"code": "sandbox_wake_timeout", "message": "restore in progress"},
+ )
+ return httpx.Response(
+ 200, headers={"Content-Type": "text/event-stream"}, content=_SSE_BODY
+ )
+
+ return handler
def test_sse_client_has_event_stream_headers_and_no_read_timeout() -> None:
@@ -31,21 +59,49 @@ def test_sse_client_has_event_stream_headers_and_no_read_timeout() -> None:
assert sse_client.timeout.read is None
-def test_sse_client_bypasses_retry_wrapper() -> None:
+def test_sse_client_keeps_only_wake_retry() -> None:
"""
- SSE bootstraps must not go through the retry wrapper: request bodies
- are not replayable, and a caller that opts into non-idempotent
- status retries would otherwise trigger duplicate execution.
+ SSE bootstraps must not go through the general retries: a caller that
+ opts into non-idempotent status retries would otherwise trigger
+ duplicate execution. The wake-timeout retry stays, since the gateway
+ never forwarded a request it answered with a wake timeout.
"""
cfg = ConnectionConfig(protocol="http").with_transport_if_missing()
endpoint = SandboxEndpoint(endpoint="localhost:44772", port=44772)
adapter = CommandsAdapter(cfg, endpoint)
- # Sanity: the normal client goes through the retry wrapper.
- assert isinstance(adapter._httpx_client._transport, RetryAsyncTransport)
- # SSE client's transport is the raw inner of the wrapper.
- assert not isinstance(adapter._sse_client._transport, RetryAsyncTransport)
- assert (
- adapter._sse_client._transport
- is adapter._httpx_client._transport.inner # type: ignore[union-attr]
+ client_transport = adapter._httpx_client._transport
+ sse_transport = adapter._sse_client._transport
+ assert isinstance(client_transport, RetryAsyncTransport)
+ assert isinstance(sse_transport, RetryAsyncTransport)
+ assert sse_transport is not client_transport
+ assert sse_transport.inner is client_transport.inner
+
+
+async def test_run_retries_gateway_wake_timeout() -> None:
+ calls: list[str] = []
+ transport = RetryAsyncTransport(
+ httpx.MockTransport(_wake_then_stream_handler(calls)), RetryPolicy()
)
+ cfg = ConnectionConfig(protocol="http", transport=transport)
+ adapter = CommandsAdapter(cfg, SandboxEndpoint(endpoint="localhost:44772"))
+
+ execution = await adapter.run("echo hi")
+
+ assert execution.id == "exec-1"
+ assert calls == ["/command", "/command"]
+
+
+def test_sync_run_retries_gateway_wake_timeout() -> None:
+ calls: list[str] = []
+ transport = RetrySyncTransport(
+ httpx.MockTransport(_wake_then_stream_handler(calls)), RetryPolicy()
+ )
+ cfg = ConnectionConfigSync(protocol="http", transport=transport)
+ adapter = CommandsAdapterSync(cfg, SandboxEndpoint(endpoint="localhost:44772"))
+
+ execution = adapter.run("echo hi")
+
+ assert execution.id == "exec-1"
+ assert calls == ["/command", "/command"]
diff --git a/sdks/sandbox/python/tests/test_retry_transport.py b/sdks/sandbox/python/tests/test_retry_transport.py
index 17583a754..acf7ab9bb 100644
--- a/sdks/sandbox/python/tests/test_retry_transport.py
+++ b/sdks/sandbox/python/tests/test_retry_transport.py
@@ -673,3 +673,73 @@ class TestUnwrapRetryTransport:
from opensandbox.transport import unwrap_retry_transport
assert unwrap_retry_transport(None) is None
+
+
+class TestStreamingTransport:
+ """SSE clients keep the wake-timeout retry and nothing else."""
+
+ @staticmethod
+ def _wake_503() -> httpx.Response:
+ return httpx.Response(
+ 503, headers={"OpenSandbox-Wake": "expired", "Retry-After": "0"}
+ )
+
+ @pytest.mark.asyncio
+ async def test_async_keeps_wake_retry(self) -> None:
+ from opensandbox.transport import streaming_transport
+
+ inner = _CountingAsyncTransport([self._wake_503(), httpx.Response(200)])
+ transport = streaming_transport(RetryAsyncTransport(inner, _fast_policy()))
+ async with httpx.AsyncClient(
+ transport=transport, base_url="http://x"
+ ) as client:
+ response = await client.post("/command", json={"command": "true"})
+ assert response.status_code == 200
+ assert inner.calls == 2
+
+ @pytest.mark.asyncio
+ async def test_async_drops_general_retries(self) -> None:
+ from opensandbox.transport import streaming_transport
+
+ inner = _CountingAsyncTransport([httpx.Response(503)])
+ policy = _fast_policy(
+ retryable_status_codes_non_idempotent=frozenset(
+ {HTTPStatus.SERVICE_UNAVAILABLE}
+ )
+ )
+ transport = streaming_transport(RetryAsyncTransport(inner, policy))
+ async with httpx.AsyncClient(
+ transport=transport, base_url="http://x"
+ ) as client:
+ response = await client.post("/command", json={"command": "true"})
+ assert response.status_code == 503
+ assert inner.calls == 1
+
+ def test_sync_keeps_wake_retry_and_drops_general_retries(self) -> None:
+ from opensandbox.transport import streaming_transport
+
+ inner = _CountingSyncTransport([self._wake_503(), httpx.Response(503)])
+ policy = _fast_policy(
+ retryable_status_codes_non_idempotent=frozenset(
+ {HTTPStatus.SERVICE_UNAVAILABLE}
+ )
+ )
+ transport = streaming_transport(RetrySyncTransport(inner, policy))
+ with httpx.Client(transport=transport, base_url="http://x") as client:
+ response = client.post("/command", json={"command": "true"})
+ assert response.status_code == 503
+ assert inner.calls == 2
+
+ def test_returns_inner_when_wake_retry_disabled(self) -> None:
+ from opensandbox.transport import streaming_transport
+
+ inner = httpx.AsyncHTTPTransport()
+ wrapper = RetryAsyncTransport(inner, RetryPolicy(wake_retry_enabled=False))
+ assert streaming_transport(wrapper) is inner
+
+ def test_passes_through_other_transports(self) -> None:
+ from opensandbox.transport import streaming_transport
+
+ raw = httpx.AsyncHTTPTransport()
+ assert streaming_transport(raw) is raw
+ assert streaming_transport(None) is None