Python
fix(observability): arguments that do not fit a tool are bad_arguments, not a tool failure
Devika Vermadaaf32d908e9
diff --git a/coworker/engine.py b/coworker/engine.pyindex be5d508a..4085d1d9 100644--- a/coworker/engine.py+++ b/coworker/engine.py@@ -340,6 +340,21 @@ def _settings_changed(verdict: Any) -> Any:return replace(verdict, verdict="unsure", reason=msg, **extra)+def _args_fit(spec: Any, arguments: Any) -> bool:+ """Whether the arguments fit the tool's own parameters: no name it lacks, no required+ one missing. True when that cannot be told (no function, or one taking anything)."""+ func = getattr(spec, "func", None)+ if func is None or not isinstance(arguments, dict):+ return True+ try:+ inspect.signature(func).bind(**arguments)+ except TypeError:+ return False+ except ValueError: # no signature to read+ return True+ return True++class TurnEngine:def __init__(self,@@ -2974,7 +2989,9 @@ class TurnEngine:if status != "error":out["error_kind"] = Noneelif ran:- out["error_kind"] = "raised"+ # Arguments the tool does not take, or a required one missing, fail at the call+ # itself: the model's mistake, not the tool's (found by the labelling judge).+ out["error_kind"] = "raised" if _args_fit(spec, arguments) else "bad_arguments"elif mangled:out["error_kind"] = "bad_arguments"elif spec is None or reason.startswith("unknown tool"):diff --git a/coworker/observability/signals.py b/coworker/observability/signals.pyindex 8c499ef6..481ebf6c 100644--- a/coworker/observability/signals.py+++ b/coworker/observability/signals.py@@ -327,7 +327,7 @@ def c_made_up_tool(turn, top, all_steps, p):k = s["attrs"].get("error_kind") if s["kind"] == "tool" else Noneif s["kind"] == "tool" and s["status"] == "error" and k in ("unknown_tool", "bad_arguments"):text = (f"The model asked for a tool that does not exist: {s['name']}." if k == "unknown_tool"- else f"The model sent arguments for {s['name']} that could not be read.")+ else f"The model sent arguments that do not fit {s['name']}.")found.append(issue("made_up_tool", s, text, kind=k))return ("fired" if found else "fine"), found, "No unknown tools or invalid arguments"diff --git a/coworker/observability/signals.yaml b/coworker/observability/signals.yamlindex 901ef175..f070ebe5 100644--- a/coworker/observability/signals.yaml+++ b/coworker/observability/signals.yaml@@ -37,8 +37,10 @@# golden the scenarios that test it (phase-2/scenarios.md)# source where the check was asked for-version: "0.4"+version: "0.5"draft: 2026-11-03+# 0.5: bad_arguments also covers arguments that do not fit the tool (a name it lacks, a required one+# missing), recorded as such by the engine; its words say so (labelling judge, T11).# 0.4: waited_unwatched also counts the wait's overlap with nobody_watching gaps (OB-14).# 0.3: ran_without_sandbox runs only where a sandbox is expected (found in the first live run).# 0.2: reviewer_allowed_send (TD2-20), sandbox_failed, the mock-review wording (M2-R1 to M2-R9),@@ -315,7 +317,7 @@ checks:params: {}says:unknown_tool: "The model asked for a tool that does not exist: {name}."- bad_arguments: "The model sent arguments for {name} that could not be read."+ bad_arguments: "The model sent arguments that do not fit {name}."fine: "No unknown tools or invalid arguments"needs: [tool.error_kind]golden: [S2-13]diff --git a/tests/corpora/observability/golden-phase2/README.md b/tests/corpora/observability/golden-phase2/README.mdindex 02bf4c44..1e04648c 100644--- a/tests/corpora/observability/golden-phase2/README.md+++ b/tests/corpora/observability/golden-phase2/README.md@@ -1,6 +1,6 @@# Golden test, Phase 2-*Written 2026-11-02 with [recording-format.md](../../recording-format.md) 1.2 and [signals.yaml](../../signals.yaml) 0.3; rebuilt 2026-11-03 for signals 0.4 (S2-17d added). Made by `phase-2/lab/make_golden2.py`, checked by `phase-2/lab/check_golden2.py`.*+*Written 2026-11-02 with [recording-format.md](../../recording-format.md) 1.2 and [signals.yaml](../../signals.yaml) 0.3; rebuilt 2026-11-03 for signals 0.4 (S2-17d added) and 0.5. Made by `phase-2/lab/make_golden2.py`, checked by `phase-2/lab/check_golden2.py`.*## What it isdiff --git a/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-01/expected-issues.jsonindex 359a9f69..4359b744 100644--- a/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"repeated_step": [diff --git a/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.jsonindex bc994a3a..ed90f2f0 100644--- a/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02/expected-issues.jsonindex 579901cb..0eb4fb45 100644--- a/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"off_plan": [diff --git a/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.jsonindex 739d59d6..df8b065b 100644--- a/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"off_plan": [diff --git a/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.jsonindex 5061dd2d..f8a6411d 100644--- a/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.jsonindex bc994a3a..ed90f2f0 100644--- a/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-03/expected-issues.jsonindex 53a46fa6..b5c944ac 100644--- a/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"refused": [diff --git a/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.jsonindex bc994a3a..ed90f2f0 100644--- a/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-04/expected-issues.jsonindex fdb534a5..10871ddf 100644--- a/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {"overridden": [diff --git a/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.jsonindex bc994a3a..ed90f2f0 100644--- a/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-05/expected-issues.jsonindex da408400..3ec0303c 100644--- a/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"reviewer_could_not_run": [diff --git a/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.jsonindex 4042fe25..83a99105 100644--- a/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-06/expected-issues.jsonindex e525f777..f7ec744d 100644--- a/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"reviewer_paused": [diff --git a/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-07/expected-issues.jsonindex 2fe645b2..34bd1342 100644--- a/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {"shadow_disagreed": [diff --git a/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08/expected-issues.jsonindex 513b11c2..f89b8cec 100644--- a/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"tool_failed": [diff --git a/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.jsonindex 513b11c2..f89b8cec 100644--- a/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"tool_failed": [diff --git a/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.jsonindex 2598d4db..f47be3d7 100644--- a/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"made_up_tool": [diff --git a/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-09/expected-issues.jsonindex 7e9020eb..800c6e71 100644--- a/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"model_failed": [diff --git a/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.jsonindex bc994a3a..ed90f2f0 100644--- a/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-10/expected-issues.jsonindex 2a7940af..e2770015 100644--- a/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"retried": [diff --git a/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.jsonindex bc994a3a..ed90f2f0 100644--- a/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-11/expected-issues.jsonindex 0ed0fd33..619158fa 100644--- a/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"cut_off": [diff --git a/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-12/expected-issues.jsonindex cf9ac100..fc3306e1 100644--- a/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"step_limit": [diff --git a/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-13/expected-issues.jsonindex 934ff556..c0ad169b 100644--- a/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 2,"fired": {"made_up_tool": [diff --git a/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14/expected-issues.jsonindex dde419fa..ebfdc8fb 100644--- a/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"context_full": [diff --git a/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.jsonindex bc994a3a..ed90f2f0 100644--- a/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.jsonindex 8fcfa96b..3f8e4d2c 100644--- a/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"context_full": [diff --git a/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-15/expected-issues.jsonindex 140ad354..bc43f204 100644--- a/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"blocked_by_sandbox": [diff --git a/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.jsonindex bf937979..5c4325bc 100644--- a/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-16/expected-issues.jsonindex 03d5f06d..218625a7 100644--- a/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"ran_without_sandbox": [diff --git a/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.jsonindex 9665d26d..b529b521 100644--- a/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17/expected-issues.jsonindex e41d83a0..ddc23868 100644--- a/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"waited_unwatched": [diff --git a/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.jsonindex bc994a3a..ed90f2f0 100644--- a/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.jsonindex bc994a3a..ed90f2f0 100644--- a/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.jsonindex e41d83a0..ddc23868 100644--- a/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"waited_unwatched": [diff --git a/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-18/expected-issues.jsonindex afb49b23..63b33744 100644--- a/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 1,"fired": {"sandbox_failed": [diff --git a/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.jsonindex 6c2d41c1..1cbc3355 100644--- a/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-19/expected-issues.jsonindex ef073414..1d12177d 100644--- a/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {"reviewer_allowed_send": [diff --git a/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.jsonindex 4042fe25..83a99105 100644--- a/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json+++ b/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json@@ -1,5 +1,5 @@{- "signals_version": "0.4",+ "signals_version": "0.5","warn": 0,"fired": {},"fine": [diff --git a/tests/corpora/observability/golden-phase2/index.json b/tests/corpora/observability/golden-phase2/index.jsonindex 56e98666..e0ed24d9 100644--- a/tests/corpora/observability/golden-phase2/index.json+++ b/tests/corpora/observability/golden-phase2/index.json@@ -1,6 +1,6 @@{"format": "1.2",- "signals_version": "0.4",+ "signals_version": "0.5","built": true,"scenarios": [{diff --git a/tests/test_observability_issues.py b/tests/test_observability_issues.pyindex a144cf5d..c4d583b1 100644--- a/tests/test_observability_issues.py+++ b/tests/test_observability_issues.py@@ -153,6 +153,44 @@ def test_a_recorded_turn_gives_its_issues(tmp_path):assert any(f["check"] == "step_limit" and "of 12" in f["text"] for f in res["fine"])+def test_arguments_that_do_not_fit_the_tool_are_bad_arguments(tmp_path):+ """Labelling (T11): a model gave replace_in_file a name it does not take. The call fails at+ the call itself: the model's mistake, so made_up_tool, not "A tool failed"."""+ d = _run(tmp_path, [+ _call("c1", "read_file", {"pth": "a"}), # a name the tool does not take+ _call("c2", "read_file", {}), # the required one missing+ _call("c3", "read_file", {"path": "a"}), # fits+ AssistantTurn(text="done"),+ ])+ tools = {s["call_id"]: s for s in d["steps"] if s["kind"] == "tool"}+ assert tools["c1"]["status"] == "error" and tools["c1"]["attrs"]["error_kind"] == "bad_arguments"+ assert tools["c2"]["attrs"]["error_kind"] == "bad_arguments"+ assert tools["c3"]["attrs"]["error_kind"] is None+ res = signals.check_turn(d["turns"][0], d["steps"])+ found = [i for i in res["issues"] if i["check"] == "made_up_tool"]+ assert len(found) == 2 and found[0]["text"] == "The model sent arguments that do not fit read_file."+ assert "tool_failed" not in {i["check"] for i in res["issues"]}+++def test_a_tool_that_fails_by_itself_is_still_raised(tmp_path):+ def broken(path: str) -> dict:+ """Break.++ Args:+ path: anything+ """+ raise RuntimeError("disk on fire")++ from coworker.engine import _args_fit+ from coworker.tools import ToolRegistry as _R++ reg = _R()+ reg.register(broken, metadata=_Meta())+ assert _args_fit(reg.get("broken"), {"path": "x"}) is True+ assert _args_fit(reg.get("broken"), {"path": "x", "extra": 1}) is False+ assert _args_fit(None, {"anything": 1}) is True # nothing to check against++# -- E8c: stored with the turn and served ------------------------------------------------------def test_issues_are_stored_at_turn_end_and_served(tmp_path):