The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(agent): keep status/all-failed-batch failures counted under top-level judging
Teknium28734be0e5a6
diff --git a/agent/display.py b/agent/display.pyindex 191470cac0a..83e8d9a49e1 100644--- a/agent/display.py+++ b/agent/display.py@@ -18,7 +18,8 @@ from urllib.parse import urlsplitfrom utils import safe_json_loadsfrom agent.i18n import tfrom agent.redact import redact_sensitive_text-from agent.tool_result_classification import file_mutation_result_landed, has_failure_marker, is_guardrail_refusal+from agent.tool_result_classification import (+ shape_failure_error, file_mutation_result_landed, has_failure_marker, is_guardrail_refusal)logger = logging.getLogger(__name__)@@ -1048,6 +1049,8 @@ def _detect_tool_failure(tool_name: str, result: Any) -> tuple[bool, str]:err = data.get("error") or data.get("message")if err and (failed or "error" in data):return True, f" [{_trim_error(str(err))}]"+ if shape_err := shape_failure_error(data):+ return True, f" [{_trim_error(shape_err)}]"# A JSON object is judged by its top-level shape only. An ``error`` key nested# inside a success payload is data the tool read (a page's own validation# message inside browser_cdp's ``result``), not a failure of the call.diff --git a/agent/tool_guardrails.py b/agent/tool_guardrails.pyindex 0328cb84dd7..116cd9e95b5 100644--- a/agent/tool_guardrails.py+++ b/agent/tool_guardrails.py@@ -14,7 +14,8 @@ from dataclasses import asdict, dataclass, field, fieldsfrom typing import Any, Mappingfrom utils import safe_json_loads-from agent.tool_result_classification import file_mutation_result_landed, has_failure_marker, is_guardrail_refusal+from agent.tool_result_classification import (+ shape_failure_error, file_mutation_result_landed, has_failure_marker, is_guardrail_refusal)IDEMPOTENT_TOOL_NAMES = frozenset({@@ -241,7 +242,7 @@ def classify_tool_failure(tool_name: str, result: str | None) -> tuple[bool, str# Same rule as _detect_tool_failure: a JSON object is judged by its top-level# shape, never by a substring scan of its nested content.err = data.get("error") or data.get("message")- return (True, " [error]") if err and (failed or "error" in data) else (False, "")+ return (True, " [error]") if (err and (failed or "error" in data)) or shape_failure_error(data) else (False, "")return (True, " [error]") if has_failure_marker(result) else (False, "")diff --git a/agent/tool_result_classification.py b/agent/tool_result_classification.pyindex 5a5738d626a..77aac93c8bc 100644--- a/agent/tool_result_classification.py+++ b/agent/tool_result_classification.py@@ -57,6 +57,28 @@ def has_failure_marker(result: str) -> bool:return any(m.start() < 500 for m in _FAILURE_MARKER_RE.finditer(head)) or result.startswith("Error")+# ``timeout`` is deliberately absent: process(wait) returns it for a still-running job.+_FAILED_STATUSES = frozenset({"failed", "error"})+++def shape_failure_error(data: dict) -> str | None:+ """Error text when a JSON-object result failed by SHAPE rather than by its top-level+ ``error``: a ``status`` of failed/error, or a ``results`` batch where EVERY item carries an+ error (delegate_task with all children interrupted, a connector batch with every call+ refused). None for a clean or partial result. Top-level JSON objects skip the substring+ scan, so this is what keeps those failures counted."""+ if str(data.get("status", "")).lower() in _FAILED_STATUSES:+ return str(data.get("status"))+ items = data.get("results")+ if not isinstance(items, list) or not items or not all(isinstance(i, dict) for i in items):+ return None+ errors = [i.get("error") for i in items]+ if not all(errors):+ return None+ first = errors[0]+ return str(first.get("message") or first) if isinstance(first, dict) else str(first)++def file_mutation_result_landed(tool_name: str, result: Any) -> bool:"""Return True when a file mutation result proves the write landed."""if tool_name not in FILE_MUTATING_TOOL_NAMES or not isinstance(result, str):diff --git a/tests/agent/test_display_tool_failure.py b/tests/agent/test_display_tool_failure.pyindex b2fbd57dd5d..83579468509 100644--- a/tests/agent/test_display_tool_failure.py+++ b/tests/agent/test_display_tool_failure.py@@ -154,33 +154,28 @@ class TestDetectToolFailureStructured:- def test_nested_error_inside_a_success_object_is_not_a_failure(self):- # browser_cdp wraps a raw CDP result as {"success": True, "method", "result"}.- # A Runtime.evaluate that reads a page's own validation object puts that- # object's "error" key inside "result": data the tool read, not a failure.- result = json.dumps({- "success": True,- "method": "Runtime.evaluate",- "result": {"result": {"type": "object",- "value": {"error": ["Vul een geldig telefoonnummer in."]}}},- })- assert _detect_tool_failure("browser_cdp", result) == (False, "")-- def test_success_object_with_error_word_in_a_nested_string_is_not_a_failure(self):- result = json.dumps({"success": True, "rows": [{"ok": False, "error": "matched 0 elements"}]})- assert _detect_tool_failure("browser_fill_form", result) == (False, "")-- def test_top_level_error_without_success_flag_is_still_a_failure(self):- result = json.dumps({"error": "CDP call timed out after 30.0s", "method": "Runtime.evaluate"})- is_failure, suffix = _detect_tool_failure("browser_cdp", result)- assert is_failure is True- assert "timed out" in suffix-- def test_plain_text_failures_still_trip_the_substring_scan(self):- # The substring scan is for results that are not a JSON object.- assert _detect_tool_failure("web_search", "Error executing tool 'web_search': boom") == (True, " [error]")- assert _detect_tool_failure("web_search", 'request "failed" upstream') == (True, " [error]")- assert _detect_tool_failure("web_search", "all good") == (False, "")+ def test_json_object_is_judged_by_top_level_shape(self):+ """An "error" nested inside a success payload is data the tool read (a page's own+ validation message in browser_cdp's result), never a failure; a top-level error, an+ all-failed batch, and plain-text failures still flag, in both classifiers."""+ ok = [+ json.dumps({"success": True, "method": "Runtime.evaluate",+ "result": {"result": {"type": "object", "value": {"error": ["Vul een geldig telefoonnummer in."]}}}}),+ json.dumps({"success": True, "rows": [{"ok": False, "error": "matched 0 elements"}]}),+ json.dumps({"results": [{"task_index": 0, "error": "interrupted"}, {"task_index": 1, "summary": "done", "error": None}]}),+ ]+ bad = [+ json.dumps({"error": "CDP call timed out after 30.0s", "method": "Runtime.evaluate"}),+ json.dumps({"results": [{"task_index": 0, "status": "interrupted", "error": "child did not finish"},+ {"task_index": 1, "status": "interrupted", "error": "child did not finish"}]}),+ "Error executing tool 'web_search': boom",+ ]+ for result in ok:+ assert _detect_tool_failure("browser_cdp", result) == (False, ""), result+ assert classify_tool_failure("browser_cdp", result) == (False, ""), result+ for result in bad:+ assert _detect_tool_failure("delegate_task", result)[0] is True, result+ assert classify_tool_failure("delegate_task", result)[0] is True, resultclass TestGetCuteToolMessageFailureSuffix:diff --git a/tests/agent/test_tool_guardrails.py b/tests/agent/test_tool_guardrails.pyindex f6153c777ce..430fdb5e935 100644--- a/tests/agent/test_tool_guardrails.py+++ b/tests/agent/test_tool_guardrails.py@@ -478,26 +478,3 @@ def test_execute_code_replay_streak_notice_fires_on_warn_only_desktop_config():assert notices[:2] == [None, None]assert all(n is not None and "consecutive identical call to execute_code" in n for n in notices[2:]), noticesassert controller.halt_decision is None, "warn-only surfaces must not halt"---def test_classify_tool_failure_judges_a_json_object_by_its_top_level_shape():- # Mirrors agent.display._detect_tool_failure: an "error" key nested inside a- # success payload (browser_cdp's wrapped Runtime.evaluate result) is not a failure.- nested = json.dumps({- "success": True,- "method": "Runtime.evaluate",- "result": {"result": {"type": "object", "value": {"error": ["Vul een geldig telefoonnummer in."]}}},- })- assert classify_tool_failure("browser_cdp", nested) == (False, "")- assert classify_tool_failure("browser_cdp", json.dumps({"error": "CDP call timed out"})) == (True, " [error]")- assert classify_tool_failure("read_file", json.dumps({"success": False, "error": "File not found"})) == (True, " [error]")- assert classify_tool_failure("memory", json.dumps({"success": False, "error": "would exceed the limit"})) == (True, " [full]")- assert classify_tool_failure("web_search", json.dumps({"success": True, "data": "hello"})) == (False, "")---def test_classify_tool_failure_keeps_the_substring_scan_for_plain_text():- assert classify_tool_failure("web_search", "Error executing tool 'web_search': boom") == (True, " [error]")- assert classify_tool_failure("web_search", 'request "failed" upstream') == (True, " [error]")- assert classify_tool_failure("web_search", "all good") == (False, "")- assert classify_tool_failure("terminal", json.dumps({"output": "x", "exit_code": 2})) == (True, " [exit 2]")- assert classify_tool_failure("terminal", json.dumps({"output": "x", "exit_code": 0})) == (False, "")