The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(bedrock): a hung Bedrock call fails after one read timeout, not five
Tekniume286e63e0ec1
diff --git a/agent/bedrock_adapter.py b/agent/bedrock_adapter.pyindex 0339d66d01e..273aba07f6a 100644--- a/agent/bedrock_adapter.py+++ b/agent/bedrock_adapter.py@@ -129,12 +129,15 @@ def _require_boto3():def _new_client(factory: Any, service: str, region: str, read_timeout: Optional[float]):- """``factory.client(...)``; no ``Config`` at all when no timeout is configured, so the unset- case builds exactly the client botocore always built (60s read / 60s connect)."""- if read_timeout is None:- return factory.client(service, region_name=region)+ """``factory.client(...)`` with botocore's own timeouts (60s read / 60s connect) unless a+ request timeout is configured. The model-call client makes one attempt per call: Hermes's+ retry loop owns retries and backoff (throttles classify as rate_limit there), and botocore's+ default of 5 attempts re-sends a read timeout, so a hung call took ~5x the timeout to fail.+ ``total_max_attempts`` counts the first request; ``max_attempts`` would count retries only."""from botocore.config import Config- config = Config(read_timeout=read_timeout, connect_timeout=_BOTO_CONNECT_TIMEOUT_SECONDS)+ config = Config(retries={"total_max_attempts": 1}) if service == "bedrock-runtime" else Config()+ if read_timeout is not None:+ config = config.merge(Config(read_timeout=read_timeout, connect_timeout=_BOTO_CONNECT_TIMEOUT_SECONDS))return factory.client(service, region_name=region, config=config)diff --git a/tests/agent/test_bedrock_no_sdk_retry.py b/tests/agent/test_bedrock_no_sdk_retry.pynew file mode 100644index 00000000000..213bee667e5--- /dev/null+++ b/tests/agent/test_bedrock_no_sdk_retry.py@@ -0,0 +1,98 @@+"""A Bedrock model call makes ONE HTTP attempt; Hermes's retry loop owns retries and backoff.++botocore's default retry policy (5 attempts) re-sent a Converse read timeout, so a hung call took+~5x the configured request timeout to surface, and re-sent throttles under Hermes's own rate-limit+backoff. These tests drive the real boto3 client the adapter builds against a local fake endpoint+and count the requests that reach it.+"""++import json+import threading+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer++import pytest++pytest.importorskip("boto3")+botocore_exceptions = pytest.importorskip("botocore.exceptions")++from agent import bedrock_adapter+from agent.bedrock_adapter import _get_bedrock_runtime_client+from agent.error_classifier import FailoverReason, classify_api_error++REGION = "us-east-1"+MODEL = "us.amazon.nova-pro-v1:0"+MESSAGES = [{"role": "user", "content": [{"text": "hi"}]}]+++def fake_bedrock(monkeypatch):+ """Local bedrock-runtime endpoint; ``mode`` is ``hang`` (never answers until teardown) or+ ``throttle`` (400 ThrottlingException). Yields the list of request paths it received."""+ state = {"mode": "hang", "hits": []}+ release = threading.Event()++ class Handler(BaseHTTPRequestHandler):+ def do_POST(self):+ state["hits"].append(self.path)+ self.rfile.read(int(self.headers.get("content-length", 0)))+ if state["mode"] == "hang":+ release.wait(30)+ return+ body = json.dumps({"message": "Too many requests, please wait before trying again."}).encode()+ self.send_response(400)+ self.send_header("content-type", "application/json")+ self.send_header("x-amzn-ErrorType", "ThrottlingException")+ self.send_header("content-length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)++ def log_message(self, format, *args):+ pass++ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ server.daemon_threads = True+ threading.Thread(target=server.serve_forever, daemon=True).start()+ monkeypatch.setenv("AWS_ENDPOINT_URL_BEDROCK_RUNTIME", f"http://127.0.0.1:{server.server_port}")+ monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIA-FAKE")+ monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fake-secret")+ monkeypatch.setenv("AWS_EC2_METADATA_DISABLED", "true")+ for var in ("AWS_PROFILE", "AWS_SESSION_TOKEN", "AWS_BEARER_TOKEN_BEDROCK", "AWS_MAX_ATTEMPTS",+ "AWS_RETRY_MODE", "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS"):+ monkeypatch.delenv(var, raising=False)+ bedrock_adapter.reset_client_cache()+ try:+ yield state+ finally:+ release.set()+ server.shutdown()+ server.server_close()+ bedrock_adapter.reset_client_cache()+++def _use_config(monkeypatch, cfg):+ monkeypatch.setattr("hermes_cli.config.load_config_readonly", lambda: cfg)+++def test_hung_converse_fails_after_one_read_timeout_not_five(fake_bedrock, monkeypatch):+ _use_config(monkeypatch, {"providers": {"bedrock": {"request_timeout_seconds": 0.5}}})+ client = _get_bedrock_runtime_client(REGION, MODEL)++ with pytest.raises(botocore_exceptions.ReadTimeoutError) as excinfo:+ client.converse(modelId=MODEL, messages=MESSAGES)++ assert len(fake_bedrock["hits"]) == 1, "botocore re-sent the timed-out call"+ assert classify_api_error(excinfo.value, provider="bedrock", model=MODEL).retryable+++def test_throttle_reaches_hermes_retry_loop_on_first_refusal_with_default_timeouts(fake_bedrock, monkeypatch):+ _use_config(monkeypatch, {})+ fake_bedrock["mode"] = "throttle"+ client = _get_bedrock_runtime_client(REGION, MODEL)++ with pytest.raises(botocore_exceptions.ClientError) as excinfo:+ client.converse(modelId=MODEL, messages=MESSAGES)++ assert len(fake_bedrock["hits"]) == 1, "botocore retried the throttle under Hermes's own backoff"+ assert client.meta.config.read_timeout == 60, "unset config keeps botocore's own timeouts"+ classified = classify_api_error(excinfo.value, provider="bedrock", model=MODEL)+ assert (classified.reason, classified.retryable) == (FailoverReason.rate_limit, True)diff --git a/tests/agent/test_bedrock_request_timeout.py b/tests/agent/test_bedrock_request_timeout.pyindex 20134602c17..517b552ef3f 100644--- a/tests/agent/test_bedrock_request_timeout.py+++ b/tests/agent/test_bedrock_request_timeout.py@@ -82,8 +82,8 @@ def test_unset_config_builds_exactly_the_botocore_default_client(config):built = _get_bedrock_runtime_client(REGION, CLAUDE).meta.configassert (built.read_timeout, built.connect_timeout) == (BOTOCORE_DEFAULT_TIMEOUT, BOTOCORE_DEFAULT_TIMEOUT)- assert (built.read_timeout, built.connect_timeout, built.retries, built.max_pool_connections) == (- baseline.read_timeout, baseline.connect_timeout, baseline.retries, baseline.max_pool_connections)+ assert (built.read_timeout, built.connect_timeout, built.max_pool_connections) == (+ baseline.read_timeout, baseline.connect_timeout, baseline.max_pool_connections)@pytest.mark.parametrize("model, expected", [(NOVA, 600), (CLAUDE, 300)], ids=["provider", "model-wins"])@@ -94,7 +94,6 @@ def test_configured_timeout_is_the_client_read_timeout_and_connect_stays_boundedassert client_config.read_timeout == expectedassert client_config.connect_timeout == BOTOCORE_DEFAULT_TIMEOUT, "a long read budget must not stretch connect"- assert client_config.retries == boto3.client("bedrock-runtime", region_name=REGION).meta.config.retries@pytest.mark.parametrize("provider_value, model_value", [("soon", -5), (0, "later"), ("", None)])