Skip to content

The agent that grows with you

fix(agent): keep status/all-failed-batch failures counted under top-level judging

Teknium28734be0e5a6
diff --git a/agent/display.py b/agent/display.py
index 191470cac0a..83e8d9a49e1 100644
--- a/agent/display.py
+++ b/agent/display.py
@@ -18,7 +18,8 @@ from urllib.parse import urlsplit
from utils import safe_json_loads
from agent.i18n import t
from agent.redact import redact_sensitive_text
-from agent.tool_result_classification import file_mutation_result_landed, has_failure_marker, is_guardrail_refusal
+from agent.tool_result_classification import (
+ shape_failure_error, file_mutation_result_landed, has_failure_marker, is_guardrail_refusal)
logger = logging.getLogger(__name__)
@@ -1048,6 +1049,8 @@ def _detect_tool_failure(tool_name: str, result: Any) -> tuple[bool, str]:
err = data.get("error") or data.get("message")
if err and (failed or "error" in data):
return True, f" [{_trim_error(str(err))}]"
+ if shape_err := shape_failure_error(data):
+ return True, f" [{_trim_error(shape_err)}]"
# A JSON object is judged by its top-level shape only. An ``error`` key nested
# inside a success payload is data the tool read (a page's own validation
# message inside browser_cdp's ``result``), not a failure of the call.
diff --git a/agent/tool_guardrails.py b/agent/tool_guardrails.py
index 0328cb84dd7..116cd9e95b5 100644
--- a/agent/tool_guardrails.py
+++ b/agent/tool_guardrails.py
@@ -14,7 +14,8 @@ from dataclasses import asdict, dataclass, field, fields
from typing import Any, Mapping
from utils import safe_json_loads
-from agent.tool_result_classification import file_mutation_result_landed, has_failure_marker, is_guardrail_refusal
+from agent.tool_result_classification import (
+ shape_failure_error, file_mutation_result_landed, has_failure_marker, is_guardrail_refusal)
IDEMPOTENT_TOOL_NAMES = frozenset({
@@ -241,7 +242,7 @@ def classify_tool_failure(tool_name: str, result: str | None) -> tuple[bool, str
# Same rule as _detect_tool_failure: a JSON object is judged by its top-level
# shape, never by a substring scan of its nested content.
err = data.get("error") or data.get("message")
- return (True, " [error]") if err and (failed or "error" in data) else (False, "")
+ return (True, " [error]") if (err and (failed or "error" in data)) or shape_failure_error(data) else (False, "")
return (True, " [error]") if has_failure_marker(result) else (False, "")
diff --git a/agent/tool_result_classification.py b/agent/tool_result_classification.py
index 5a5738d626a..77aac93c8bc 100644
--- a/agent/tool_result_classification.py
+++ b/agent/tool_result_classification.py
@@ -57,6 +57,28 @@ def has_failure_marker(result: str) -> bool:
return any(m.start() < 500 for m in _FAILURE_MARKER_RE.finditer(head)) or result.startswith("Error")
+# ``timeout`` is deliberately absent: process(wait) returns it for a still-running job.
+_FAILED_STATUSES = frozenset({"failed", "error"})
+
+
+def shape_failure_error(data: dict) -> str | None:
+ """Error text when a JSON-object result failed by SHAPE rather than by its top-level
+ ``error``: a ``status`` of failed/error, or a ``results`` batch where EVERY item carries an
+ error (delegate_task with all children interrupted, a connector batch with every call
+ refused). None for a clean or partial result. Top-level JSON objects skip the substring
+ scan, so this is what keeps those failures counted."""
+ if str(data.get("status", "")).lower() in _FAILED_STATUSES:
+ return str(data.get("status"))
+ items = data.get("results")
+ if not isinstance(items, list) or not items or not all(isinstance(i, dict) for i in items):
+ return None
+ errors = [i.get("error") for i in items]
+ if not all(errors):
+ return None
+ first = errors[0]
+ return str(first.get("message") or first) if isinstance(first, dict) else str(first)
+
+
def file_mutation_result_landed(tool_name: str, result: Any) -> bool:
"""Return True when a file mutation result proves the write landed."""
if tool_name not in FILE_MUTATING_TOOL_NAMES or not isinstance(result, str):
diff --git a/tests/agent/test_display_tool_failure.py b/tests/agent/test_display_tool_failure.py
index b2fbd57dd5d..83579468509 100644
--- a/tests/agent/test_display_tool_failure.py
+++ b/tests/agent/test_display_tool_failure.py
@@ -154,33 +154,28 @@ class TestDetectToolFailureStructured:
- def test_nested_error_inside_a_success_object_is_not_a_failure(self):
- # browser_cdp wraps a raw CDP result as {"success": True, "method", "result"}.
- # A Runtime.evaluate that reads a page's own validation object puts that
- # object's "error" key inside "result": data the tool read, not a failure.
- result = json.dumps({
- "success": True,
- "method": "Runtime.evaluate",
- "result": {"result": {"type": "object",
- "value": {"error": ["Vul een geldig telefoonnummer in."]}}},
- })
- assert _detect_tool_failure("browser_cdp", result) == (False, "")
-
- def test_success_object_with_error_word_in_a_nested_string_is_not_a_failure(self):
- result = json.dumps({"success": True, "rows": [{"ok": False, "error": "matched 0 elements"}]})
- assert _detect_tool_failure("browser_fill_form", result) == (False, "")
-
- def test_top_level_error_without_success_flag_is_still_a_failure(self):
- result = json.dumps({"error": "CDP call timed out after 30.0s", "method": "Runtime.evaluate"})
- is_failure, suffix = _detect_tool_failure("browser_cdp", result)
- assert is_failure is True
- assert "timed out" in suffix
-
- def test_plain_text_failures_still_trip_the_substring_scan(self):
- # The substring scan is for results that are not a JSON object.
- assert _detect_tool_failure("web_search", "Error executing tool 'web_search': boom") == (True, " [error]")
- assert _detect_tool_failure("web_search", 'request "failed" upstream') == (True, " [error]")
- assert _detect_tool_failure("web_search", "all good") == (False, "")
+ def test_json_object_is_judged_by_top_level_shape(self):
+ """An "error" nested inside a success payload is data the tool read (a page's own
+ validation message in browser_cdp's result), never a failure; a top-level error, an
+ all-failed batch, and plain-text failures still flag, in both classifiers."""
+ ok = [
+ json.dumps({"success": True, "method": "Runtime.evaluate",
+ "result": {"result": {"type": "object", "value": {"error": ["Vul een geldig telefoonnummer in."]}}}}),
+ json.dumps({"success": True, "rows": [{"ok": False, "error": "matched 0 elements"}]}),
+ json.dumps({"results": [{"task_index": 0, "error": "interrupted"}, {"task_index": 1, "summary": "done", "error": None}]}),
+ ]
+ bad = [
+ json.dumps({"error": "CDP call timed out after 30.0s", "method": "Runtime.evaluate"}),
+ json.dumps({"results": [{"task_index": 0, "status": "interrupted", "error": "child did not finish"},
+ {"task_index": 1, "status": "interrupted", "error": "child did not finish"}]}),
+ "Error executing tool 'web_search': boom",
+ ]
+ for result in ok:
+ assert _detect_tool_failure("browser_cdp", result) == (False, ""), result
+ assert classify_tool_failure("browser_cdp", result) == (False, ""), result
+ for result in bad:
+ assert _detect_tool_failure("delegate_task", result)[0] is True, result
+ assert classify_tool_failure("delegate_task", result)[0] is True, result
class TestGetCuteToolMessageFailureSuffix:
diff --git a/tests/agent/test_tool_guardrails.py b/tests/agent/test_tool_guardrails.py
index f6153c777ce..430fdb5e935 100644
--- a/tests/agent/test_tool_guardrails.py
+++ b/tests/agent/test_tool_guardrails.py
@@ -478,26 +478,3 @@ def test_execute_code_replay_streak_notice_fires_on_warn_only_desktop_config():
assert notices[:2] == [None, None]
assert all(n is not None and "consecutive identical call to execute_code" in n for n in notices[2:]), notices
assert controller.halt_decision is None, "warn-only surfaces must not halt"
-
-
-def test_classify_tool_failure_judges_a_json_object_by_its_top_level_shape():
- # Mirrors agent.display._detect_tool_failure: an "error" key nested inside a
- # success payload (browser_cdp's wrapped Runtime.evaluate result) is not a failure.
- nested = json.dumps({
- "success": True,
- "method": "Runtime.evaluate",
- "result": {"result": {"type": "object", "value": {"error": ["Vul een geldig telefoonnummer in."]}}},
- })
- assert classify_tool_failure("browser_cdp", nested) == (False, "")
- assert classify_tool_failure("browser_cdp", json.dumps({"error": "CDP call timed out"})) == (True, " [error]")
- assert classify_tool_failure("read_file", json.dumps({"success": False, "error": "File not found"})) == (True, " [error]")
- assert classify_tool_failure("memory", json.dumps({"success": False, "error": "would exceed the limit"})) == (True, " [full]")
- assert classify_tool_failure("web_search", json.dumps({"success": True, "data": "hello"})) == (False, "")
-
-
-def test_classify_tool_failure_keeps_the_substring_scan_for_plain_text():
- assert classify_tool_failure("web_search", "Error executing tool 'web_search': boom") == (True, " [error]")
- assert classify_tool_failure("web_search", 'request "failed" upstream') == (True, " [error]")
- assert classify_tool_failure("web_search", "all good") == (False, "")
- assert classify_tool_failure("terminal", json.dumps({"output": "x", "exit_code": 2})) == (True, " [exit 2]")
- assert classify_tool_failure("terminal", json.dumps({"output": "x", "exit_code": 0})) == (False, "")