Secure, Fast, and Extensible Sandbox runtime for AI agents.
fix(sdk-python): retry gateway wake timeouts on streaming requests
Maohaoaf9910f8da1f
diff --git a/sdks/code-interpreter/python/src/code_interpreter/adapters/code_adapter.py b/sdks/code-interpreter/python/src/code_interpreter/adapters/code_adapter.pyindex 64e0d2c86..8970a7398 100644--- a/sdks/code-interpreter/python/src/code_interpreter/adapters/code_adapter.py+++ b/sdks/code-interpreter/python/src/code_interpreter/adapters/code_adapter.py@@ -41,7 +41,7 @@ from opensandbox.config import ConnectionConfigfrom opensandbox.exceptions import InvalidArgumentException, SandboxApiExceptionfrom opensandbox.models.execd import Execution, ExecutionHandlersfrom opensandbox.models.sandboxes import SandboxEndpoint-from opensandbox.transport import unwrap_retry_transport+from opensandbox.transport import streaming_transportfrom code_interpreter.adapters.converter.code_execution_converter import (CodeExecutionConverter,@@ -128,9 +128,8 @@ class CodesAdapter(Codes):"Accept": "text/event-stream","Cache-Control": "no-cache",}- # SSE bootstraps bypass the retry wrapper: request bodies are- # not replayable and a non-idempotent status opt-in would cause- # duplicate execution on a resent SSE POST.+ # SSE bootstraps keep only the wake-timeout retry: a non-idempotent+ # status opt-in would cause duplicate execution on a resent SSE POST.sse_transport = (self.connection_config.new_owned_transport()if self._owns_clients@@ -144,7 +143,7 @@ class CodesAdapter(Codes):write=timeout_seconds,pool=None,),- transport=unwrap_retry_transport(sse_transport),+ transport=streaming_transport(sse_transport),)async def _get_client(self):diff --git a/sdks/code-interpreter/python/src/code_interpreter/sync/adapters/code_adapter.py b/sdks/code-interpreter/python/src/code_interpreter/sync/adapters/code_adapter.pyindex 217b7d5d5..4f9d7926c 100644--- a/sdks/code-interpreter/python/src/code_interpreter/sync/adapters/code_adapter.py+++ b/sdks/code-interpreter/python/src/code_interpreter/sync/adapters/code_adapter.py@@ -39,7 +39,7 @@ from opensandbox.models.sandboxes import SandboxEndpointfrom opensandbox.sync.adapters.converter.execution_event_dispatcher import (ExecutionEventDispatcherSync,)-from opensandbox.transport import unwrap_retry_transport+from opensandbox.transport import streaming_transportfrom code_interpreter.models.code_sync import CodeContextSync, SupportedLanguageSyncfrom code_interpreter.sync.services.code import CodesSync@@ -123,9 +123,8 @@ class CodesAdapterSync(CodesSync):"Accept": "text/event-stream","Cache-Control": "no-cache",}- # SSE bootstraps bypass the retry wrapper: request bodies are- # not replayable and a non-idempotent status opt-in would cause- # duplicate execution on a resent SSE POST.+ # SSE bootstraps keep only the wake-timeout retry: a non-idempotent+ # status opt-in would cause duplicate execution on a resent SSE POST.sse_transport = (self.connection_config.new_owned_transport()if self._owns_clients@@ -136,7 +135,7 @@ class CodesAdapterSync(CodesSync):timeout=httpx.Timeout(connect=timeout_seconds, read=None, write=timeout_seconds, pool=None),- transport=unwrap_retry_transport(sse_transport),+ transport=streaming_transport(sse_transport),)def _get_execd_url(self, path: str) -> str:diff --git a/sdks/sandbox/python/src/opensandbox/adapters/command_adapter.py b/sdks/sandbox/python/src/opensandbox/adapters/command_adapter.pyindex c5aa78038..d1fb9dbf1 100644--- a/sdks/sandbox/python/src/opensandbox/adapters/command_adapter.py+++ b/sdks/sandbox/python/src/opensandbox/adapters/command_adapter.py@@ -69,7 +69,7 @@ from opensandbox.models.execd import ()from opensandbox.models.sandboxes import SandboxEndpointfrom opensandbox.services.command import Commands-from opensandbox.transport import unwrap_retry_transport+from opensandbox.transport import streaming_transportlogger = logging.getLogger(__name__)@@ -135,9 +135,8 @@ class CommandsAdapter(Commands):"Accept": "text/event-stream","Cache-Control": "no-cache",}- # SSE bootstraps bypass the retry wrapper: request bodies are- # not replayable and a non-idempotent status opt-in would cause- # duplicate execution on a resent SSE POST.+ # SSE bootstraps keep only the wake-timeout retry: a non-idempotent+ # status opt-in would cause duplicate execution on a resent SSE POST.self._sse_client = httpx.AsyncClient(headers=sse_headers,timeout=httpx.Timeout(@@ -146,7 +145,7 @@ class CommandsAdapter(Commands):write=timeout_seconds,pool=None,),- transport=unwrap_retry_transport(self.connection_config.transport),+ transport=streaming_transport(self.connection_config.transport),**build_async_redirect_client_options(self.connection_config, base_url),)diff --git a/sdks/sandbox/python/src/opensandbox/adapters/isolated_adapter.py b/sdks/sandbox/python/src/opensandbox/adapters/isolated_adapter.pyindex 55f37c5a6..78b5968fa 100644--- a/sdks/sandbox/python/src/opensandbox/adapters/isolated_adapter.py+++ b/sdks/sandbox/python/src/opensandbox/adapters/isolated_adapter.py@@ -56,7 +56,7 @@ from opensandbox.services.isolated import (IsolationServiceMixin,IsolationSession,)-from opensandbox.transport import unwrap_retry_transport+from opensandbox.transport import streaming_transportlogger = logging.getLogger(__name__)@@ -258,9 +258,8 @@ class IsolatedSessionsAdapter(IsolationServiceMixin, IsolationService):"Accept": "text/event-stream","Cache-Control": "no-cache",}- # SSE bootstraps bypass the retry wrapper: request bodies are- # not replayable and a non-idempotent status opt-in would cause- # duplicate execution on a resent SSE POST.+ # SSE bootstraps keep only the wake-timeout retry: a non-idempotent+ # status opt-in would cause duplicate execution on a resent SSE POST.self._sse_client = httpx.AsyncClient(headers=sse_headers,timeout=httpx.Timeout(@@ -269,7 +268,7 @@ class IsolatedSessionsAdapter(IsolationServiceMixin, IsolationService):write=timeout_seconds,pool=None,),- transport=unwrap_retry_transport(self.connection_config.transport),+ transport=streaming_transport(self.connection_config.transport),**build_async_redirect_client_options(self.connection_config, base_url),)diff --git a/sdks/sandbox/python/src/opensandbox/sync/adapters/command_adapter.py b/sdks/sandbox/python/src/opensandbox/sync/adapters/command_adapter.pyindex ebb772fe2..acd0494ce 100644--- a/sdks/sandbox/python/src/opensandbox/sync/adapters/command_adapter.py+++ b/sdks/sandbox/python/src/opensandbox/sync/adapters/command_adapter.py@@ -64,7 +64,7 @@ from opensandbox.sync.adapters.converter.execution_event_dispatcher import (ExecutionEventDispatcherSync,)from opensandbox.sync.services.command import CommandsSync-from opensandbox.transport import unwrap_retry_transport+from opensandbox.transport import streaming_transportlogger = logging.getLogger(__name__)@@ -123,9 +123,8 @@ class CommandsAdapterSync(CommandsSync):"Accept": "text/event-stream","Cache-Control": "no-cache",}- # SSE bootstraps bypass the retry wrapper: request bodies are- # not replayable and a non-idempotent status opt-in would cause- # duplicate execution on a resent SSE POST.+ # SSE bootstraps keep only the wake-timeout retry: a non-idempotent+ # status opt-in would cause duplicate execution on a resent SSE POST.self._sse_client = httpx.Client(headers=sse_headers,timeout=httpx.Timeout(@@ -134,7 +133,7 @@ class CommandsAdapterSync(CommandsSync):write=timeout_seconds,pool=None,),- transport=unwrap_retry_transport(self.connection_config.transport),+ transport=streaming_transport(self.connection_config.transport),**build_redirect_client_options(self.connection_config, base_url),)diff --git a/sdks/sandbox/python/src/opensandbox/sync/adapters/isolated_adapter.py b/sdks/sandbox/python/src/opensandbox/sync/adapters/isolated_adapter.pyindex 73e6ccf13..8aba6e863 100644--- a/sdks/sandbox/python/src/opensandbox/sync/adapters/isolated_adapter.py+++ b/sdks/sandbox/python/src/opensandbox/sync/adapters/isolated_adapter.py@@ -58,7 +58,7 @@ from opensandbox.sync.services.isolated import (IsolationServiceSyncMixin,IsolationSessionSync,)-from opensandbox.transport import unwrap_retry_transport+from opensandbox.transport import streaming_transportlogger = logging.getLogger(__name__)@@ -199,9 +199,8 @@ class IsolatedSessionsAdapterSync(IsolationServiceSyncMixin, IsolationServiceSyn"Accept": "text/event-stream","Cache-Control": "no-cache",}- # SSE bootstraps bypass the retry wrapper: request bodies are- # not replayable and a non-idempotent status opt-in would cause- # duplicate execution on a resent SSE POST.+ # SSE bootstraps keep only the wake-timeout retry: a non-idempotent+ # status opt-in would cause duplicate execution on a resent SSE POST.self._sse_client = httpx.Client(headers=sse_headers,timeout=httpx.Timeout(@@ -210,7 +209,7 @@ class IsolatedSessionsAdapterSync(IsolationServiceSyncMixin, IsolationServiceSynwrite=timeout_seconds,pool=None,),- transport=unwrap_retry_transport(self.connection_config.transport),+ transport=streaming_transport(self.connection_config.transport),**build_redirect_client_options(self.connection_config, base_url),)diff --git a/sdks/sandbox/python/src/opensandbox/transport/__init__.py b/sdks/sandbox/python/src/opensandbox/transport/__init__.pyindex 5cc5859ad..f67d02374 100644--- a/sdks/sandbox/python/src/opensandbox/transport/__init__.py+++ b/sdks/sandbox/python/src/opensandbox/transport/__init__.py@@ -48,6 +48,21 @@ def unwrap_retry_transport(transport: _T) -> _T:return transport+def streaming_transport(transport: _T) -> _T:+ """+ Return the transport for a streaming (SSE) client.++ SSE requests run commands, so general retries must not resend them: a+ caller ``retryable_status_codes_non_idempotent`` opt-in would cause+ duplicate execution. A retry wrapper is replaced by one that keeps only+ the OSEP-0024 wake-timeout retry, which is safe because the gateway+ never forwarded the request. Other transports are returned unchanged.+ """+ if isinstance(transport, (RetryAsyncTransport, RetrySyncTransport)):+ return transport.for_streaming() # type: ignore[return-value]+ return transport++__all__ = ["JitterMode","RetryAsyncTransport",@@ -55,5 +70,6 @@ __all__ = ["RetryEvent","RetryPolicy","RetrySyncTransport",+ "streaming_transport","unwrap_retry_transport",]diff --git a/sdks/sandbox/python/src/opensandbox/transport/_async_retry.py b/sdks/sandbox/python/src/opensandbox/transport/_async_retry.pyindex 9ddfb890e..125a453a2 100644--- a/sdks/sandbox/python/src/opensandbox/transport/_async_retry.py+++ b/sdks/sandbox/python/src/opensandbox/transport/_async_retry.py@@ -122,9 +122,20 @@ class RetryAsyncTransport(httpx.AsyncBaseTransport):@propertydef inner(self) -> httpx.AsyncBaseTransport:- """The wrapped raw transport, safe to hand to SSE clients."""+ """The wrapped raw transport."""return self._inner+ def for_streaming(self) -> httpx.AsyncBaseTransport:+ """Transport for streaming (SSE) clients; see :meth:`RetryPolicy.wake_only`."""+ if not self._policy.wake_retry_enabled:+ return self._inner+ return RetryAsyncTransport(+ self._inner,+ self._policy.wake_only(),+ owns_inner=self._owns_inner,+ rng=self._rng,+ )+async def handle_async_request(self, request: httpx.Request) -> httpx.Response:diff --git a/sdks/sandbox/python/src/opensandbox/transport/_sync_retry.py b/sdks/sandbox/python/src/opensandbox/transport/_sync_retry.pyindex 8ab0dbb2a..6425cd0e3 100644--- a/sdks/sandbox/python/src/opensandbox/transport/_sync_retry.py+++ b/sdks/sandbox/python/src/opensandbox/transport/_sync_retry.py@@ -72,9 +72,20 @@ class RetrySyncTransport(httpx.BaseTransport):@propertydef inner(self) -> httpx.BaseTransport:- """The wrapped raw transport, safe to hand to SSE clients."""+ """The wrapped raw transport."""return self._inner+ def for_streaming(self) -> httpx.BaseTransport:+ """Transport for streaming (SSE) clients; see :meth:`RetryPolicy.wake_only`."""+ if not self._policy.wake_retry_enabled:+ return self._inner+ return RetrySyncTransport(+ self._inner,+ self._policy.wake_only(),+ owns_inner=self._owns_inner,+ rng=self._rng,+ )+def handle_request(self, request: httpx.Request) -> httpx.Response:policy = self._policymethod = request.method.upper()diff --git a/sdks/sandbox/python/src/opensandbox/transport/retry.py b/sdks/sandbox/python/src/opensandbox/transport/retry.pyindex 63e5b0d5a..9b67bd736 100644--- a/sdks/sandbox/python/src/opensandbox/transport/retry.py+++ b/sdks/sandbox/python/src/opensandbox/transport/retry.py@@ -18,7 +18,7 @@from __future__ import annotationsfrom collections.abc import Callable-from dataclasses import dataclass, field+from dataclasses import dataclass, field, replacefrom datetime import timedeltafrom enum import Enumfrom http import HTTPStatus@@ -184,6 +184,18 @@ class RetryPolicy:"""Never retry. Also suppresses fresh-connection recovery and wake retry."""return cls(max_retries=0, wake_retry_enabled=False)+ def wake_only(self) -> RetryPolicy:+ """+ This policy with only the wake-timeout retry left.++ Used for streaming (SSE) requests. They run commands, so they must+ not be resent after a status code or a post-send failure, but a+ marked wake timeout proves the gateway never forwarded the request.+ """+ return replace(+ self, max_retries=0, per_attempt_timeout=None, overall_deadline=None+ )+def wraps_transport(self) -> bool:"""Whether this policy requires the retry transport to be installed.diff --git a/sdks/sandbox/python/tests/test_command_service_sse_client_config.py b/sdks/sandbox/python/tests/test_command_service_sse_client_config.pyindex f77037630..b1a95fc03 100644--- a/sdks/sandbox/python/tests/test_command_service_sse_client_config.py+++ b/sdks/sandbox/python/tests/test_command_service_sse_client_config.py@@ -14,10 +14,38 @@# limitations under the License.#+import httpx+import pytest+from opensandbox.adapters.command_adapter import CommandsAdapterfrom opensandbox.config import ConnectionConfig+from opensandbox.config.connection_sync import ConnectionConfigSyncfrom opensandbox.models.sandboxes import SandboxEndpoint-from opensandbox.transport import RetryAsyncTransport+from opensandbox.sync.adapters.command_adapter import CommandsAdapterSync+from opensandbox.transport import RetryAsyncTransport, RetryPolicy, RetrySyncTransport++_SSE_BODY = (+ b'data: {"type":"init","text":"exec-1","timestamp":1}\n\n'+ b'data: {"type":"execution_complete","timestamp":2,"execution_time":1}\n\n'+)+++def _wake_then_stream_handler(calls: list[str]):+ """First /command answer is a gateway wake timeout, then the SSE stream."""++ def handler(request: httpx.Request) -> httpx.Response:+ calls.append(request.url.path)+ if len(calls) == 1:+ return httpx.Response(+ 503,+ headers={"OpenSandbox-Wake": "expired", "Retry-After": "0"},+ json={"code": "sandbox_wake_timeout", "message": "restore in progress"},+ )+ return httpx.Response(+ 200, headers={"Content-Type": "text/event-stream"}, content=_SSE_BODY+ )++ return handlerdef test_sse_client_has_event_stream_headers_and_no_read_timeout() -> None:@@ -31,21 +59,49 @@ def test_sse_client_has_event_stream_headers_and_no_read_timeout() -> None:assert sse_client.timeout.read is None-def test_sse_client_bypasses_retry_wrapper() -> None:+def test_sse_client_keeps_only_wake_retry() -> None:"""- SSE bootstraps must not go through the retry wrapper: request bodies- are not replayable, and a caller that opts into non-idempotent- status retries would otherwise trigger duplicate execution.+ SSE bootstraps must not go through the general retries: a caller that+ opts into non-idempotent status retries would otherwise trigger+ duplicate execution. The wake-timeout retry stays, since the gateway+ never forwarded a request it answered with a wake timeout."""cfg = ConnectionConfig(protocol="http").with_transport_if_missing()endpoint = SandboxEndpoint(endpoint="localhost:44772", port=44772)adapter = CommandsAdapter(cfg, endpoint)- # Sanity: the normal client goes through the retry wrapper.- assert isinstance(adapter._httpx_client._transport, RetryAsyncTransport)- # SSE client's transport is the raw inner of the wrapper.- assert not isinstance(adapter._sse_client._transport, RetryAsyncTransport)- assert (- adapter._sse_client._transport- is adapter._httpx_client._transport.inner # type: ignore[union-attr]+ client_transport = adapter._httpx_client._transport+ sse_transport = adapter._sse_client._transport+ assert isinstance(client_transport, RetryAsyncTransport)+ assert isinstance(sse_transport, RetryAsyncTransport)+ assert sse_transport is not client_transport+ assert sse_transport.inner is client_transport.inner+++async def test_run_retries_gateway_wake_timeout() -> None:+ calls: list[str] = []+ transport = RetryAsyncTransport(+ httpx.MockTransport(_wake_then_stream_handler(calls)), RetryPolicy())+ cfg = ConnectionConfig(protocol="http", transport=transport)+ adapter = CommandsAdapter(cfg, SandboxEndpoint(endpoint="localhost:44772"))++ execution = await adapter.run("echo hi")++ assert execution.id == "exec-1"+ assert calls == ["/command", "/command"]+++def test_sync_run_retries_gateway_wake_timeout() -> None:+ calls: list[str] = []+ transport = RetrySyncTransport(+ httpx.MockTransport(_wake_then_stream_handler(calls)), RetryPolicy()+ )+ cfg = ConnectionConfigSync(protocol="http", transport=transport)+ adapter = CommandsAdapterSync(cfg, SandboxEndpoint(endpoint="localhost:44772"))++ execution = adapter.run("echo hi")++ assert execution.id == "exec-1"+ assert calls == ["/command", "/command"]diff --git a/sdks/sandbox/python/tests/test_retry_transport.py b/sdks/sandbox/python/tests/test_retry_transport.pyindex 17583a754..acf7ab9bb 100644--- a/sdks/sandbox/python/tests/test_retry_transport.py+++ b/sdks/sandbox/python/tests/test_retry_transport.py@@ -673,3 +673,73 @@ class TestUnwrapRetryTransport:from opensandbox.transport import unwrap_retry_transportassert unwrap_retry_transport(None) is None+++class TestStreamingTransport:+ """SSE clients keep the wake-timeout retry and nothing else."""++ @staticmethod+ def _wake_503() -> httpx.Response:+ return httpx.Response(+ 503, headers={"OpenSandbox-Wake": "expired", "Retry-After": "0"}+ )++ @pytest.mark.asyncio+ async def test_async_keeps_wake_retry(self) -> None:+ from opensandbox.transport import streaming_transport++ inner = _CountingAsyncTransport([self._wake_503(), httpx.Response(200)])+ transport = streaming_transport(RetryAsyncTransport(inner, _fast_policy()))+ async with httpx.AsyncClient(+ transport=transport, base_url="http://x"+ ) as client:+ response = await client.post("/command", json={"command": "true"})+ assert response.status_code == 200+ assert inner.calls == 2++ @pytest.mark.asyncio+ async def test_async_drops_general_retries(self) -> None:+ from opensandbox.transport import streaming_transport++ inner = _CountingAsyncTransport([httpx.Response(503)])+ policy = _fast_policy(+ retryable_status_codes_non_idempotent=frozenset(+ {HTTPStatus.SERVICE_UNAVAILABLE}+ )+ )+ transport = streaming_transport(RetryAsyncTransport(inner, policy))+ async with httpx.AsyncClient(+ transport=transport, base_url="http://x"+ ) as client:+ response = await client.post("/command", json={"command": "true"})+ assert response.status_code == 503+ assert inner.calls == 1++ def test_sync_keeps_wake_retry_and_drops_general_retries(self) -> None:+ from opensandbox.transport import streaming_transport++ inner = _CountingSyncTransport([self._wake_503(), httpx.Response(503)])+ policy = _fast_policy(+ retryable_status_codes_non_idempotent=frozenset(+ {HTTPStatus.SERVICE_UNAVAILABLE}+ )+ )+ transport = streaming_transport(RetrySyncTransport(inner, policy))+ with httpx.Client(transport=transport, base_url="http://x") as client:+ response = client.post("/command", json={"command": "true"})+ assert response.status_code == 503+ assert inner.calls == 2++ def test_returns_inner_when_wake_retry_disabled(self) -> None:+ from opensandbox.transport import streaming_transport++ inner = httpx.AsyncHTTPTransport()+ wrapper = RetryAsyncTransport(inner, RetryPolicy(wake_retry_enabled=False))+ assert streaming_transport(wrapper) is inner++ def test_passes_through_other_transports(self) -> None:+ from opensandbox.transport import streaming_transport++ raw = httpx.AsyncHTTPTransport()+ assert streaming_transport(raw) is raw+ assert streaming_transport(None) is None