Skip to content

armgit / fara

Public Imported from GitHub
0

Fara1.5 – A family of frontier computer use agent models

graceful_client: guarantee create() terminates; fix infinite retry hang

corby19224e97dab1
diff --git a/webeval/src/webeval/oai_clients/graceful_client.py b/webeval/src/webeval/oai_clients/graceful_client.py
index 01d5ede..73d3d63 100644
--- a/webeval/src/webeval/oai_clients/graceful_client.py
+++ b/webeval/src/webeval/oai_clients/graceful_client.py
@@ -48,6 +48,7 @@ class GracefulRetryClient(ChatCompletionClient):
max_retries: int = 8,
max_tokens: int = 115000,
timeout: Optional[float] = None,
+ max_total_attempts: Optional[int] = None,
):
super().__init__(max_tokens=max_tokens)
if not clients:
@@ -57,6 +58,13 @@ class GracefulRetryClient(ChatCompletionClient):
self.max_retries = max_retries
self.max_tokens = max_tokens
self.timeout = timeout
+ # Hard ceiling on total loop iterations per create() call so the
+ # loop always terminates even when branches don't consume `tries`.
+ self.max_total_attempts = (
+ max_total_attempts
+ if max_total_attempts is not None
+ else max_retries + 2 * len(clients)
+ )
self.support_json = support_json
self.blocklist: set = set()
self._client_idx = random.randint(0, len(clients) - 1)
@@ -177,12 +185,14 @@ class GracefulRetryClient(ChatCompletionClient):
extra_create_args: Mapping[str, Any] = {},
) -> CreateResult:
tries = self.max_retries
+ total_attempts = 0
last_error: Optional[Exception] = None
client = self.next_client(no_increment=True)
# Mutable copy so we can mutate per-request without affecting callers.
extra = dict(extra_create_args)
- while tries > 0:
+ while tries > 0 and total_attempts < self.max_total_attempts:
+ total_attempts += 1
request_tokens = client.count_tokens(messages=messages)
self._remove_reasoning_effort_if_needed(client, extra)
self.logger.info(
@@ -222,6 +232,10 @@ class GracefulRetryClient(ChatCompletionClient):
continue
except openai.BadRequestError as e:
if "check-access-response-enc" in str(e):
+ # Consume the retry budget so a persistent access-token
+ # failure on a small pool can't loop forever.
+ tries -= 1
+ last_error = e
self.logger.error(
f"GracefulRetryClient.create() AccessTokenError: {client.description}, refreshing credentials\n{e}"
)
@@ -298,12 +312,17 @@ class GracefulRetryClient(ChatCompletionClient):
await asyncio.sleep(1)
continue
except openai.AuthenticationError as e:
+ # Consume the retry budget so a persistent auth failure on a
+ # single-endpoint pool can't loop forever.
+ tries -= 1
+ last_error = e
self.logger.error(
f"GracefulRetryClient.create() AuthenticationError: {client.description}: {e}"
)
if hasattr(client, "refresh_credentials"):
client.refresh_credentials()
client = self.next_client()
+ await asyncio.sleep(1)
continue
except openai.APIStatusError as e:
if "Prompt is too large" in str(e):
@@ -346,7 +365,8 @@ class GracefulRetryClient(ChatCompletionClient):
raise last_error
valid_clients = [c for c in self._clients if c.endpoint not in self.blocklist]
raise Exception(
- f"GracefulRetryClient.create(): all clients exhausted after {self.max_retries} retries; "
+ f"GracefulRetryClient.create(): gave up after {total_attempts} attempts "
+ f"(max_retries={self.max_retries}, max_total_attempts={self.max_total_attempts}); "
f"{len(valid_clients)}/{len(self._clients)} clients reachable. Blocklist size: {len(self.blocklist)}"
)
diff --git a/webeval/tests/test_oai_clients.py b/webeval/tests/test_oai_clients.py
index 4954cbc..c57b225 100644
--- a/webeval/tests/test_oai_clients.py
+++ b/webeval/tests/test_oai_clients.py
@@ -14,10 +14,13 @@ from __future__ import annotations
import asyncio
import io
import json
+import logging
import sys
from pathlib import Path
from types import SimpleNamespace
+import httpx
+import openai
import pytest
from PIL import Image as PILImage
@@ -130,6 +133,106 @@ def test_should_include_model_exact_match():
assert not GracefulRetryClient._should_include_model("gpt-4o-mini", "gpt-4o")
+class _AlwaysFailsClient:
+ """Fake ChatCompletionClient whose create() always raises ``exc``.
+
+ Exposes the minimal surface GracefulRetryClient.create() touches:
+ ``endpoint``/``description``/``count_tokens``/``create`` (+ an optional
+ ``refresh_credentials`` no-op). Records how many times create() ran so the
+ test can assert the loop is bounded.
+ """
+
+ def __init__(self, exc: Exception, endpoint: str = "https://fake"):
+ self._exc = exc
+ self.endpoint = endpoint
+ self.description = f"fake-gpt-4o@{endpoint}"
+ self.calls = 0
+
+ def count_tokens(self, messages=()): # noqa: ARG002
+ return 0
+
+ def refresh_credentials(self):
+ pass
+
+ async def create(self, *args, **kwargs): # noqa: ARG002
+ self.calls += 1
+ raise self._exc
+
+ async def close(self):
+ pass
+
+
[email protected]("n_endpoints", [1, 3])
+def test_graceful_retry_terminates_on_persistent_auth_error(n_endpoints):
+ """Regression: a pool that only ever raises AuthenticationError must make
+ create() *terminate* (raising the last error), not spin forever.
+
+ Pre-fix, the AuthenticationError branch neither blocklisted nor consumed
+ the retry budget, so a single bad endpoint wedged create() indefinitely
+ (the 19h hang). The per-error budget + global ``max_total_attempts`` cap
+ now guarantee a bounded number of underlying create() calls.
+ """
+ from webeval.oai_clients import GracefulRetryClient
+
+ auth_err = openai.AuthenticationError(
+ "bad creds",
+ response=httpx.Response(401, request=httpx.Request("POST", "https://fake")),
+ body=None,
+ )
+ clients = [
+ _AlwaysFailsClient(auth_err, endpoint=f"https://fake-{i}")
+ for i in range(n_endpoints)
+ ]
+ g = GracefulRetryClient(
+ clients=clients,
+ logger=logging.getLogger(__name__),
+ max_retries=3,
+ )
+
+ async def _run():
+ # Hard timeout so a regression to the old behaviour fails loudly
+ # instead of hanging the whole suite.
+ return await asyncio.wait_for(g.create(messages=[]), timeout=30)
+
+ with pytest.raises(openai.AuthenticationError):
+ asyncio.run(_run())
+
+ total_calls = sum(c.calls for c in clients)
+ assert total_calls <= g.max_total_attempts
+ # Budget-bounded: max_retries decrements drive termination here.
+ assert total_calls <= g.max_retries + 1
+
+
+def test_graceful_retry_terminates_when_all_endpoints_blocklisted():
+ """A pool that only raises NotFoundError must terminate once every
+ endpoint is blocklisted (next_client() raises), never looping forever."""
+ from webeval.oai_clients import GracefulRetryClient
+
+ not_found = openai.NotFoundError(
+ "no such deployment",
+ response=httpx.Response(404, request=httpx.Request("POST", "https://fake")),
+ body=None,
+ )
+ clients = [
+ _AlwaysFailsClient(not_found, endpoint=f"https://fake-{i}") for i in range(3)
+ ]
+ g = GracefulRetryClient(
+ clients=clients,
+ logger=logging.getLogger(__name__),
+ max_retries=8,
+ )
+
+ async def _run():
+ return await asyncio.wait_for(g.create(messages=[]), timeout=30)
+
+ # All endpoints get blocklisted → next_client() raises RuntimeError.
+ with pytest.raises(RuntimeError):
+ asyncio.run(_run())
+ # One create() per endpoint at most before the pool is exhausted.
+ assert sum(c.calls for c in clients) <= len(clients)
+ assert len(g.blocklist) == len(clients)
+
+
def test_openai_wrapper_create_round_trip(monkeypatch):
"""Stub the OpenAI client and verify .create() returns a CreateResult
with text content and updated usage."""