The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(compression): report a summary call_llm recovered on the main model as a fallback
Teknium3a5407ac73e4
diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.pyindex 5b541da88bb..37616d40eb5 100644--- a/agent/auxiliary_client.py+++ b/agent/auxiliary_client.py@@ -7835,12 +7835,9 @@ def _ladder_provider_fallback(first_err: Exception, route: _LadderRoute):regardless of user intent. Auth errors from an explicit provider may only use the task'sown configured fallback_chain; they never imply an unconfigured provider hop."""task, tag, resolved_provider = route.task, route.tag, route.resolved_provider- # Respect explicit provider choice for transient errors (auth, request validation, etc.) but allow- # fallback when the provider clearly cannot serve the request due to capacity: payment/quota exhaustion- # and connection failures are capacity problems, not request constraints. See #26803: daily token quota- # (429 + "too many tokens per day") must fall back just like a 402 credit error.- # Rate limits are included: after retries are exhausted, a 429 means the provider is at capacity. See- # #52228. See #26803: daily token quota must fall back like a 402 credit error.+ # Respect explicit provider choice for transient errors (auth, request validation, etc.) but fall back when+ # the provider cannot serve due to capacity: payment/quota exhaustion (a daily-token 429 like a 402, #26803),+ # connection failures, and a 429 after retries are exhausted (#52228) are not request constraints.is_auto = resolved_provider in {"auto", "", None}reason = next((label for predicate, label in _FALLBACK_REASONS if predicate(first_err)), None)is_capacity_error = any(@@ -7909,6 +7906,8 @@ def _ladder_provider_fallback(first_err: Exception, route: _LadderRoute):_record_route_info(route.route_info, _fallback_provider_from_label(fb_label), fb_model)fb_resp = yield _LadderStep("fallback", (fb_client, fb_model, fb_label))if fb_resp is not None:+ if route.route_info is not None: # tell the caller its own route failed, not only where it landed+ route.route_info.update(failed_model=route.final_model or "", failure=f"{reason}: {first_err}")return fb_respfb_client, fb_model, fb_label = _next_fallback_after_quarantine(task, resolved_provider, is_auto, route, _chain_failed_model, _chain_failure_scope,diff --git a/agent/context_compressor.py b/agent/context_compressor.pyindex b01a7edd87a..fa26f20e2db 100644--- a/agent/context_compressor.py+++ b/agent/context_compressor.py@@ -3822,7 +3822,6 @@ Summary generation was unavailable, so this is a best-effort deterministic fallbself.summary_model or self.model, f"{len(prompt):,}", _latency_info["prompt_build_ms"],)try:- # Compression is atomic: shield the summary call from gateway interrupts. Re-entrant.with aux_interrupt_protection():response = call_llm(**call_kwargs)finally:@@ -3843,6 +3842,7 @@ Summary generation was unavailable, so this is a best-effort deterministic fallb)if self._compression_cancelled():raise AuxiliaryExplicitCancellation()+ self._note_aux_route_fallback(_aux_route)# Reasoning-field fallback (DeepSeek/Qwen/Kimi put the summary in reasoning_content); capped.content = extract_content_or_reasoning(response, max_reasoning_chars=8000)where = f"(provider={self.provider or 'auto'} model={self.summary_model or self.model})"diff --git a/agent/context_compressor_telemetry.py b/agent/context_compressor_telemetry.pyindex d8b9c4d92fe..bee7df84ff0 100644--- a/agent/context_compressor_telemetry.py+++ b/agent/context_compressor_telemetry.py@@ -100,6 +100,22 @@ class CompressionTelemetryMixin:accumulate = key in {"queue_wait_ms", "summary_generation_ms"} and value is not Nonetelemetry[key] = (telemetry.get(key) or 0) + value if accumulate else value+ def _note_aux_route_fallback(self, route: dict[str, str]) -> None:+ """``call_llm`` answered the summary from a fallback route after the configured one failed (dead+ endpoint, quota): record it like the compressor's own main-model fallback, so telemetry says+ ``fallback_used`` and, when the main model answered, the user is told their compression model is broken."""+ failed_model = route.get("failed_model")+ if not failed_model:+ return+ telemetry = getattr(self, "_active_compression_telemetry", None)+ if isinstance(telemetry, dict):+ telemetry["fallback_used"] = True+ telemetry["failure_class"] = telemetry.get("failure_class") or "aux_model_fallback"+ if route.get("model") == self.model: # the user warning says "summarised with your main model"+ self._last_aux_model_failure_model = failed_model+ failure = route.get("failure") or "unknown error"+ self._last_aux_model_failure_error = failure if len(failure) <= 220 else failure[:217].rstrip() + "..."+def _compression_method(self) -> str:"""How the summary that replaced the middle window was produced."""if getattr(self, "_last_summary_fallback_used", False):diff --git a/tests/agent/test_compression_call_llm_fallback_reported.py b/tests/agent/test_compression_call_llm_fallback_reported.pynew file mode 100644index 00000000000..2f1b56d74a6--- /dev/null+++ b/tests/agent/test_compression_call_llm_fallback_reported.py@@ -0,0 +1,135 @@+"""A compression summary that ``call_llm`` recovered on a fallback route is reported as a fallback.++When ``auxiliary.compression`` points at a dead endpoint, ``call_llm``'s recovery ladder answers the summary+with the main model. The compressor saw a plain success: no user warning, and telemetry+``method=llm_summary fallback_used=false``. Real HTTP path (closed port + loopback main model, temp ``HERMES_HOME``).+"""++import json+import socket+import threading+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer++import pytest++import agent.auxiliary_client as aux+from agent.context_compressor import ContextCompressor++SUMMARY = "## Goal\nAnswer the numbered questions.\n## Progress\nQuestions answered.\n## Next Steps\nContinue.\n"+++def _chunk(text, finish=None):+ payload = {"id": "c", "object": "chat.completion.chunk", "created": 1, "model": "m",+ "choices": [{"index": 0, "delta": {"content": text} if text else {}, "finish_reason": finish}]}+ return f"data: {json.dumps(payload)}\n\n".encode()+++def _serve(handler):+ server = ThreadingHTTPServer(("127.0.0.1", 0), handler)+ thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True)+ thread.start()+ return server, thread+++def _stop(server, thread):+ aux.shutdown_cached_clients()+ server.shutdown()+ server.server_close()+ thread.join(timeout=5)+++def _write_config(home, config):+ (home / "config.yaml").write_text(json.dumps(config), encoding="utf-8")+++def main_model_endpoint():+ """The main model: answers every chat completion (streamed or not) with a summary."""+ seen = []++ class Handler(BaseHTTPRequestHandler):+ protocol_version = "HTTP/1.1"++ def log_message(self, *_args):+ pass++ def do_POST(self):+ body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)) or b"{}")+ seen.append(body.get("model"))+ self.send_response(200)+ self.send_header("Content-Type", "text/event-stream")+ self.send_header("Connection", "close")+ self.end_headers()+ self.wfile.write(_chunk(SUMMARY) + _chunk("", "stop") + b"data: [DONE]\n\n")+ self.wfile.flush()+ self.close_connection = True++ aux.shutdown_cached_clients()+ aux._reset_aux_unhealthy_cache()+ server, thread = _serve(Handler)+ try:+ yield f"http://127.0.0.1:{server.server_port}/v1", seen+ finally:+ _stop(server, thread)+++def _closed_port_url():+ with socket.socket() as sock:+ sock.bind(("127.0.0.1", 0))+ port = sock.getsockname()[1]+ return f"http://127.0.0.1:{port}/v1"+++def _history():+ messages = []+ for i in range(30):+ messages.append({"role": "user", "content": f"Question {i}: explain topic {i}. " + "lorem ipsum " * 200})+ messages.append({"role": "assistant", "content": f"Answer {i}: " + "dolor sit amet " * 200})+ return messages+++def test_summary_recovered_on_the_main_model_by_call_llm_is_reported(tmp_path, monkeypatch, main_model_endpoint):+ main_url, seen = main_model_endpoint+ monkeypatch.setenv("HERMES_HOME", str(tmp_path))+ monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost")+ _write_config(tmp_path, {+ "model": {"provider": "custom", "base_url": main_url, "api_key": "dummy", "default": "main-fake",+ "context_length": 64000},+ "auxiliary": {"compression": {"provider": "custom", "base_url": _closed_port_url(), "api_key": "dummy",+ "model": "dead-summarizer", "timeout": 20}},+ })+ compressor = ContextCompressor(model="main-fake", provider="custom", base_url=main_url, api_key="dummy",+ config_context_length=64000, quiet_mode=True)++ compressed = compressor.compress(_history(), force=True)++ assert len(compressed) < 60 and seen == ["main-fake"] # the main model wrote the summary+ telemetry = compressor._last_compression_telemetry+ assert telemetry["fallback_used"] is True+ assert telemetry["method"] == "aux_fallback_main"+ # The user warning (``_warn_summary_or_aux_fallback``) keys on these.+ assert compressor._last_aux_model_failure_model == "dead-summarizer"+ assert "connection error" in compressor._last_aux_model_failure_error++[email protected]("route, warns", [+ ({"provider": "custom", "model": "main-fake", "failed_model": "dead", "failure": "connection error: x"}, True),+ ({"provider": "openrouter", "model": "chain-model", "failed_model": "dead", "failure": "rate limit: x"}, False),+])+def test_only_a_main_model_recovery_names_the_broken_compression_model(route, warns):+ compressor = ContextCompressor(model="main-fake", config_context_length=64000, quiet_mode=True)+ compressor._active_compression_telemetry = telemetry = {}++ compressor._note_aux_route_fallback(route)++ assert telemetry["fallback_used"] is True+ assert (compressor._last_aux_model_failure_model == "dead") is warns+++def test_a_route_that_answered_itself_is_not_a_fallback():+ compressor = ContextCompressor(model="main-fake", config_context_length=64000, quiet_mode=True)+ compressor._active_compression_telemetry = telemetry = {}++ compressor._note_aux_route_fallback({"provider": "custom", "model": "summarizer"})++ assert telemetry == {} and compressor._last_aux_model_failure_model is None