Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(sandbox): retry transient 5xx on POST /cmd (#1361)

r33drichardsed8e1e89a183
diff --git a/libs/python/cua-sandbox/cua_sandbox/transport/http.py b/libs/python/cua-sandbox/cua_sandbox/transport/http.py
index 121c591b05..2a73ecc4dc 100644
--- a/libs/python/cua-sandbox/cua_sandbox/transport/http.py
+++ b/libs/python/cua-sandbox/cua_sandbox/transport/http.py
@@ -6,13 +6,27 @@ and returns an SSE stream with a single ``data: {...}`` frame containing the res
from __future__ import annotations
+import asyncio
import base64
import json
+import logging
from typing import Any, Dict, Optional
import httpx
from cua_sandbox.transport.base import Transport
+logger = logging.getLogger(__name__)
+
+# Retry transient 5xx responses on /cmd. The computer-server can briefly
+# return 5xx (e.g. when Traefik temporarily drops the pod from its
+# endpoint list, or when the emulator's gRPC subsystem hangs during
+# fork/exec). 4xx errors are not retried (client error, won't change).
+# Read/transport timeouts are not retried either — the command may
+# already be running on the server, and most /cmd actions aren't
+# idempotent.
+_CMD_MAX_RETRIES = 3
+_CMD_RETRY_BACKOFF_S = 0.5 # doubled each retry: 0.5s, 1.0s, 2.0s
+
class HTTPTransport(Transport):
"""Transport that communicates with computer-server over HTTP POST /cmd (SSE)."""
@@ -57,7 +71,13 @@ class HTTPTransport(Transport):
self._client = None
async def _cmd(self, command: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
- """Send a command to POST /cmd and parse the SSE response."""
+ """Send a command to POST /cmd and parse the SSE response.
+
+ Retries on transient 5xx responses (server briefly unavailable,
+ grpc fork hiccups, Traefik backend-unready). Does not retry on
+ httpx exceptions — the request may have reached the server and
+ started running a non-idempotent command.
+ """
assert self._client is not None, "Transport not connected"
body = {"command": command}
if params:
@@ -72,7 +92,19 @@ class HTTPTransport(Transport):
req_timeout = httpx.Timeout(self._timeout, read=float(server_timeout) + 10)
else:
req_timeout = None # use client default
- resp = await self._client.post("/cmd", json=body, timeout=req_timeout)
+
+ resp: httpx.Response
+ for attempt in range(_CMD_MAX_RETRIES):
+ resp = await self._client.post("/cmd", json=body, timeout=req_timeout)
+ if resp.status_code < 500 or attempt == _CMD_MAX_RETRIES - 1:
+ break
+ backoff = _CMD_RETRY_BACKOFF_S * (2 ** attempt)
+ logger.debug(
+ "[http] /cmd %s returned %d, retrying in %.1fs (attempt %d/%d)",
+ command, resp.status_code, backoff, attempt + 1, _CMD_MAX_RETRIES,
+ )
+ await asyncio.sleep(backoff)
+
resp.raise_for_status()
return self._parse_sse(resp.text)
diff --git a/libs/python/cua-sandbox/tests/test_transport_http_retry.py b/libs/python/cua-sandbox/tests/test_transport_http_retry.py
new file mode 100644
index 0000000000..39526111e7
--- /dev/null
+++ b/libs/python/cua-sandbox/tests/test_transport_http_retry.py
@@ -0,0 +1,129 @@
+"""Unit tests for HTTPTransport._cmd retry-on-5xx behavior.
+
+The computer-server can briefly return 5xx (Traefik dropping the pod
+from its endpoint list, grpc fork hiccups). The transport retries
+up to 3 times with exponential backoff for 5xx responses. 4xx
+errors are raised immediately. httpx transport exceptions (read
+timeout, connection reset) are not retried because the command may
+have already started on the server.
+"""
+
+from __future__ import annotations
+
+import json
+from itertools import cycle
+
+import httpx
+import pytest
+
+from cua_sandbox.transport.http import HTTPTransport
+
+pytestmark = pytest.mark.asyncio
+
+
+def _sse_body(payload: dict) -> str:
+ return f"data: {json.dumps(payload)}\n\n"
+
+
+async def _make_transport(handler) -> HTTPTransport:
+ t = HTTPTransport("http://server:8000", timeout=5.0)
+ # Patch the async client after connect so the mock transport is in use.
+ await t.connect()
+ assert t._client is not None
+ await t._client.aclose()
+ t._client = httpx.AsyncClient(
+ base_url="http://server:8000",
+ transport=httpx.MockTransport(handler),
+ timeout=5.0,
+ )
+ return t
+
+
+async def test_cmd_success_no_retry():
+ calls = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ calls.append(request.url.path)
+ return httpx.Response(200, text=_sse_body({"width": 1, "height": 2}))
+
+ t = await _make_transport(handler)
+ result = await t._cmd("get_screen_size")
+ assert result == {"width": 1, "height": 2}
+ assert len(calls) == 1, "success should not retry"
+
+
+async def test_cmd_retries_5xx_then_succeeds(monkeypatch):
+ # Skip sleeps so the test runs instantly.
+ import cua_sandbox.transport.http as http_mod
+
+ async def _no_sleep(_seconds):
+ pass
+
+ monkeypatch.setattr(http_mod.asyncio, "sleep", _no_sleep)
+
+ responses = iter([
+ httpx.Response(503, text="service unavailable"),
+ httpx.Response(502, text="bad gateway"),
+ httpx.Response(200, text=_sse_body({"ok": True})),
+ ])
+ calls = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ calls.append(request.url.path)
+ return next(responses)
+
+ t = await _make_transport(handler)
+ result = await t._cmd("screenshot")
+ assert result == {"ok": True}
+ assert len(calls) == 3, "should retry twice before succeeding"
+
+
+async def test_cmd_gives_up_after_max_retries(monkeypatch):
+ import cua_sandbox.transport.http as http_mod
+
+ async def _no_sleep(_seconds):
+ pass
+
+ monkeypatch.setattr(http_mod.asyncio, "sleep", _no_sleep)
+
+ calls = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ calls.append(request.url.path)
+ return httpx.Response(503, text="still unavailable")
+
+ t = await _make_transport(handler)
+ with pytest.raises(httpx.HTTPStatusError) as exc_info:
+ await t._cmd("screenshot")
+ assert exc_info.value.response.status_code == 503
+ # _CMD_MAX_RETRIES (3) attempts total, then raise.
+ assert len(calls) == 3
+
+
+async def test_cmd_does_not_retry_4xx():
+ calls = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ calls.append(request.url.path)
+ return httpx.Response(401, text="unauthorized")
+
+ t = await _make_transport(handler)
+ with pytest.raises(httpx.HTTPStatusError) as exc_info:
+ await t._cmd("screenshot")
+ assert exc_info.value.response.status_code == 401
+ assert len(calls) == 1, "4xx should not retry"
+
+
+async def test_cmd_does_not_retry_read_timeout():
+ calls = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ calls.append(request.url.path)
+ raise httpx.ReadTimeout("server too slow", request=request)
+
+ t = await _make_transport(handler)
+ with pytest.raises(httpx.ReadTimeout):
+ await t._cmd("run_command", params={"command": "slow"})
+ # ReadTimeout means the request reached the server — don't retry
+ # or we risk double-executing a non-idempotent command.
+ assert len(calls) == 1