Fara1.5 – A family of frontier computer use agent models
graceful_client: guarantee create() terminates; fix infinite retry hang
corby19224e97dab1
diff --git a/webeval/src/webeval/oai_clients/graceful_client.py b/webeval/src/webeval/oai_clients/graceful_client.pyindex 01d5ede..73d3d63 100644--- a/webeval/src/webeval/oai_clients/graceful_client.py+++ b/webeval/src/webeval/oai_clients/graceful_client.py@@ -48,6 +48,7 @@ class GracefulRetryClient(ChatCompletionClient):max_retries: int = 8,max_tokens: int = 115000,timeout: Optional[float] = None,+ max_total_attempts: Optional[int] = None,):super().__init__(max_tokens=max_tokens)if not clients:@@ -57,6 +58,13 @@ class GracefulRetryClient(ChatCompletionClient):self.max_retries = max_retriesself.max_tokens = max_tokensself.timeout = timeout+ # Hard ceiling on total loop iterations per create() call so the+ # loop always terminates even when branches don't consume `tries`.+ self.max_total_attempts = (+ max_total_attempts+ if max_total_attempts is not None+ else max_retries + 2 * len(clients)+ )self.support_json = support_jsonself.blocklist: set = set()self._client_idx = random.randint(0, len(clients) - 1)@@ -177,12 +185,14 @@ class GracefulRetryClient(ChatCompletionClient):extra_create_args: Mapping[str, Any] = {},) -> CreateResult:tries = self.max_retries+ total_attempts = 0last_error: Optional[Exception] = Noneclient = self.next_client(no_increment=True)# Mutable copy so we can mutate per-request without affecting callers.extra = dict(extra_create_args)- while tries > 0:+ while tries > 0 and total_attempts < self.max_total_attempts:+ total_attempts += 1request_tokens = client.count_tokens(messages=messages)self._remove_reasoning_effort_if_needed(client, extra)self.logger.info(@@ -222,6 +232,10 @@ class GracefulRetryClient(ChatCompletionClient):continueexcept openai.BadRequestError as e:if "check-access-response-enc" in str(e):+ # Consume the retry budget so a persistent access-token+ # failure on a small pool can't loop forever.+ tries -= 1+ last_error = eself.logger.error(f"GracefulRetryClient.create() AccessTokenError: {client.description}, refreshing credentials\n{e}")@@ -298,12 +312,17 @@ class GracefulRetryClient(ChatCompletionClient):await asyncio.sleep(1)continueexcept openai.AuthenticationError as e:+ # Consume the retry budget so a persistent auth failure on a+ # single-endpoint pool can't loop forever.+ tries -= 1+ last_error = eself.logger.error(f"GracefulRetryClient.create() AuthenticationError: {client.description}: {e}")if hasattr(client, "refresh_credentials"):client.refresh_credentials()client = self.next_client()+ await asyncio.sleep(1)continueexcept openai.APIStatusError as e:if "Prompt is too large" in str(e):@@ -346,7 +365,8 @@ class GracefulRetryClient(ChatCompletionClient):raise last_errorvalid_clients = [c for c in self._clients if c.endpoint not in self.blocklist]raise Exception(- f"GracefulRetryClient.create(): all clients exhausted after {self.max_retries} retries; "+ f"GracefulRetryClient.create(): gave up after {total_attempts} attempts "+ f"(max_retries={self.max_retries}, max_total_attempts={self.max_total_attempts}); "f"{len(valid_clients)}/{len(self._clients)} clients reachable. Blocklist size: {len(self.blocklist)}")diff --git a/webeval/tests/test_oai_clients.py b/webeval/tests/test_oai_clients.pyindex 4954cbc..c57b225 100644--- a/webeval/tests/test_oai_clients.py+++ b/webeval/tests/test_oai_clients.py@@ -14,10 +14,13 @@ from __future__ import annotationsimport asyncioimport ioimport json+import loggingimport sysfrom pathlib import Pathfrom types import SimpleNamespace+import httpx+import openaiimport pytestfrom PIL import Image as PILImage@@ -130,6 +133,106 @@ def test_should_include_model_exact_match():assert not GracefulRetryClient._should_include_model("gpt-4o-mini", "gpt-4o")+class _AlwaysFailsClient:+ """Fake ChatCompletionClient whose create() always raises ``exc``.++ Exposes the minimal surface GracefulRetryClient.create() touches:+ ``endpoint``/``description``/``count_tokens``/``create`` (+ an optional+ ``refresh_credentials`` no-op). Records how many times create() ran so the+ test can assert the loop is bounded.+ """++ def __init__(self, exc: Exception, endpoint: str = "https://fake"):+ self._exc = exc+ self.endpoint = endpoint+ self.description = f"fake-gpt-4o@{endpoint}"+ self.calls = 0++ def count_tokens(self, messages=()): # noqa: ARG002+ return 0++ def refresh_credentials(self):+ pass++ async def create(self, *args, **kwargs): # noqa: ARG002+ self.calls += 1+ raise self._exc++ async def close(self):+ pass++[email protected]("n_endpoints", [1, 3])+def test_graceful_retry_terminates_on_persistent_auth_error(n_endpoints):+ """Regression: a pool that only ever raises AuthenticationError must make+ create() *terminate* (raising the last error), not spin forever.++ Pre-fix, the AuthenticationError branch neither blocklisted nor consumed+ the retry budget, so a single bad endpoint wedged create() indefinitely+ (the 19h hang). The per-error budget + global ``max_total_attempts`` cap+ now guarantee a bounded number of underlying create() calls.+ """+ from webeval.oai_clients import GracefulRetryClient++ auth_err = openai.AuthenticationError(+ "bad creds",+ response=httpx.Response(401, request=httpx.Request("POST", "https://fake")),+ body=None,+ )+ clients = [+ _AlwaysFailsClient(auth_err, endpoint=f"https://fake-{i}")+ for i in range(n_endpoints)+ ]+ g = GracefulRetryClient(+ clients=clients,+ logger=logging.getLogger(__name__),+ max_retries=3,+ )++ async def _run():+ # Hard timeout so a regression to the old behaviour fails loudly+ # instead of hanging the whole suite.+ return await asyncio.wait_for(g.create(messages=[]), timeout=30)++ with pytest.raises(openai.AuthenticationError):+ asyncio.run(_run())++ total_calls = sum(c.calls for c in clients)+ assert total_calls <= g.max_total_attempts+ # Budget-bounded: max_retries decrements drive termination here.+ assert total_calls <= g.max_retries + 1+++def test_graceful_retry_terminates_when_all_endpoints_blocklisted():+ """A pool that only raises NotFoundError must terminate once every+ endpoint is blocklisted (next_client() raises), never looping forever."""+ from webeval.oai_clients import GracefulRetryClient++ not_found = openai.NotFoundError(+ "no such deployment",+ response=httpx.Response(404, request=httpx.Request("POST", "https://fake")),+ body=None,+ )+ clients = [+ _AlwaysFailsClient(not_found, endpoint=f"https://fake-{i}") for i in range(3)+ ]+ g = GracefulRetryClient(+ clients=clients,+ logger=logging.getLogger(__name__),+ max_retries=8,+ )++ async def _run():+ return await asyncio.wait_for(g.create(messages=[]), timeout=30)++ # All endpoints get blocklisted → next_client() raises RuntimeError.+ with pytest.raises(RuntimeError):+ asyncio.run(_run())+ # One create() per endpoint at most before the pool is exhausted.+ assert sum(c.calls for c in clients) <= len(clients)+ assert len(g.blocklist) == len(clients)++def test_openai_wrapper_create_round_trip(monkeypatch):"""Stub the OpenAI client and verify .create() returns a CreateResultwith text content and updated usage."""