Skip to content
Python

fix(observability): arguments that do not fit a tool are bad_arguments, not a tool failure

Devika Vermadaaf32d908e9
diff --git a/coworker/engine.py b/coworker/engine.py
index be5d508a..4085d1d9 100644
--- a/coworker/engine.py
+++ b/coworker/engine.py
@@ -340,6 +340,21 @@ def _settings_changed(verdict: Any) -> Any:
return replace(verdict, verdict="unsure", reason=msg, **extra)
+def _args_fit(spec: Any, arguments: Any) -> bool:
+ """Whether the arguments fit the tool's own parameters: no name it lacks, no required
+ one missing. True when that cannot be told (no function, or one taking anything)."""
+ func = getattr(spec, "func", None)
+ if func is None or not isinstance(arguments, dict):
+ return True
+ try:
+ inspect.signature(func).bind(**arguments)
+ except TypeError:
+ return False
+ except ValueError: # no signature to read
+ return True
+ return True
+
+
class TurnEngine:
def __init__(
self,
@@ -2974,7 +2989,9 @@ class TurnEngine:
if status != "error":
out["error_kind"] = None
elif ran:
- out["error_kind"] = "raised"
+ # Arguments the tool does not take, or a required one missing, fail at the call
+ # itself: the model's mistake, not the tool's (found by the labelling judge).
+ out["error_kind"] = "raised" if _args_fit(spec, arguments) else "bad_arguments"
elif mangled:
out["error_kind"] = "bad_arguments"
elif spec is None or reason.startswith("unknown tool"):
diff --git a/coworker/observability/signals.py b/coworker/observability/signals.py
index 8c499ef6..481ebf6c 100644
--- a/coworker/observability/signals.py
+++ b/coworker/observability/signals.py
@@ -327,7 +327,7 @@ def c_made_up_tool(turn, top, all_steps, p):
k = s["attrs"].get("error_kind") if s["kind"] == "tool" else None
if s["kind"] == "tool" and s["status"] == "error" and k in ("unknown_tool", "bad_arguments"):
text = (f"The model asked for a tool that does not exist: {s['name']}." if k == "unknown_tool"
- else f"The model sent arguments for {s['name']} that could not be read.")
+ else f"The model sent arguments that do not fit {s['name']}.")
found.append(issue("made_up_tool", s, text, kind=k))
return ("fired" if found else "fine"), found, "No unknown tools or invalid arguments"
diff --git a/coworker/observability/signals.yaml b/coworker/observability/signals.yaml
index 901ef175..f070ebe5 100644
--- a/coworker/observability/signals.yaml
+++ b/coworker/observability/signals.yaml
@@ -37,8 +37,10 @@
# golden the scenarios that test it (phase-2/scenarios.md)
# source where the check was asked for
-version: "0.4"
+version: "0.5"
draft: 2026-11-03
+# 0.5: bad_arguments also covers arguments that do not fit the tool (a name it lacks, a required one
+# missing), recorded as such by the engine; its words say so (labelling judge, T11).
# 0.4: waited_unwatched also counts the wait's overlap with nobody_watching gaps (OB-14).
# 0.3: ran_without_sandbox runs only where a sandbox is expected (found in the first live run).
# 0.2: reviewer_allowed_send (TD2-20), sandbox_failed, the mock-review wording (M2-R1 to M2-R9),
@@ -315,7 +317,7 @@ checks:
params: {}
says:
unknown_tool: "The model asked for a tool that does not exist: {name}."
- bad_arguments: "The model sent arguments for {name} that could not be read."
+ bad_arguments: "The model sent arguments that do not fit {name}."
fine: "No unknown tools or invalid arguments"
needs: [tool.error_kind]
golden: [S2-13]
diff --git a/tests/corpora/observability/golden-phase2/README.md b/tests/corpora/observability/golden-phase2/README.md
index 02bf4c44..1e04648c 100644
--- a/tests/corpora/observability/golden-phase2/README.md
+++ b/tests/corpora/observability/golden-phase2/README.md
@@ -1,6 +1,6 @@
# Golden test, Phase 2
-*Written 2026-11-02 with [recording-format.md](../../recording-format.md) 1.2 and [signals.yaml](../../signals.yaml) 0.3; rebuilt 2026-11-03 for signals 0.4 (S2-17d added). Made by `phase-2/lab/make_golden2.py`, checked by `phase-2/lab/check_golden2.py`.*
+*Written 2026-11-02 with [recording-format.md](../../recording-format.md) 1.2 and [signals.yaml](../../signals.yaml) 0.3; rebuilt 2026-11-03 for signals 0.4 (S2-17d added) and 0.5. Made by `phase-2/lab/make_golden2.py`, checked by `phase-2/lab/check_golden2.py`.*
## What it is
diff --git a/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json
index 359a9f69..4359b744 100644
--- a/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-01/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"repeated_step": [
diff --git a/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json
index bc994a3a..ed90f2f0 100644
--- a/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-01c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json
index 579901cb..0eb4fb45 100644
--- a/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-02/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"off_plan": [
diff --git a/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json
index 739d59d6..df8b065b 100644
--- a/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-02b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"off_plan": [
diff --git a/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json
index 5061dd2d..f8a6411d 100644
--- a/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-02c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json
index bc994a3a..ed90f2f0 100644
--- a/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-02n/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json
index 53a46fa6..b5c944ac 100644
--- a/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-03/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"refused": [
diff --git a/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json
index bc994a3a..ed90f2f0 100644
--- a/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-03b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json
index fdb534a5..10871ddf 100644
--- a/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-04/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {
"overridden": [
diff --git a/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json
index bc994a3a..ed90f2f0 100644
--- a/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-04n/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json
index da408400..3ec0303c 100644
--- a/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-05/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"reviewer_could_not_run": [
diff --git a/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json
index 4042fe25..83a99105 100644
--- a/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-05c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json
index e525f777..f7ec744d 100644
--- a/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-06/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"reviewer_paused": [
diff --git a/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json
index 2fe645b2..34bd1342 100644
--- a/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-07/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {
"shadow_disagreed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json
index 513b11c2..f89b8cec 100644
--- a/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-08/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"tool_failed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json
index 513b11c2..f89b8cec 100644
--- a/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-08b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"tool_failed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json
index 2598d4db..f47be3d7 100644
--- a/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-08c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"made_up_tool": [
diff --git a/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json
index 7e9020eb..800c6e71 100644
--- a/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-09/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"model_failed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json
index bc994a3a..ed90f2f0 100644
--- a/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-09c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json
index 2a7940af..e2770015 100644
--- a/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-10/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"retried": [
diff --git a/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json
index bc994a3a..ed90f2f0 100644
--- a/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-10b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json
index 0ed0fd33..619158fa 100644
--- a/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-11/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"cut_off": [
diff --git a/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json
index cf9ac100..fc3306e1 100644
--- a/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-12/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"step_limit": [
diff --git a/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json
index 934ff556..c0ad169b 100644
--- a/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-13/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 2,
"fired": {
"made_up_tool": [
diff --git a/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json
index dde419fa..ebfdc8fb 100644
--- a/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-14/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"context_full": [
diff --git a/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json
index bc994a3a..ed90f2f0 100644
--- a/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-14b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json
index 8fcfa96b..3f8e4d2c 100644
--- a/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-14d/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"context_full": [
diff --git a/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json
index 140ad354..bc43f204 100644
--- a/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-15/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"blocked_by_sandbox": [
diff --git a/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json
index bf937979..5c4325bc 100644
--- a/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-15c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json
index 03d5f06d..218625a7 100644
--- a/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-16/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"ran_without_sandbox": [
diff --git a/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json
index 9665d26d..b529b521 100644
--- a/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-16c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json
index e41d83a0..ddc23868 100644
--- a/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-17/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"waited_unwatched": [
diff --git a/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json
index bc994a3a..ed90f2f0 100644
--- a/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-17b/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json
index bc994a3a..ed90f2f0 100644
--- a/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-17c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json
index e41d83a0..ddc23868 100644
--- a/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-17d/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"waited_unwatched": [
diff --git a/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json
index afb49b23..63b33744 100644
--- a/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-18/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 1,
"fired": {
"sandbox_failed": [
diff --git a/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json
index 6c2d41c1..1cbc3355 100644
--- a/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-18c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json
index ef073414..1d12177d 100644
--- a/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-19/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {
"reviewer_allowed_send": [
diff --git a/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json b/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json
index 4042fe25..83a99105 100644
--- a/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json
+++ b/tests/corpora/observability/golden-phase2/S2-19c/expected-issues.json
@@ -1,5 +1,5 @@
{
- "signals_version": "0.4",
+ "signals_version": "0.5",
"warn": 0,
"fired": {},
"fine": [
diff --git a/tests/corpora/observability/golden-phase2/index.json b/tests/corpora/observability/golden-phase2/index.json
index 56e98666..e0ed24d9 100644
--- a/tests/corpora/observability/golden-phase2/index.json
+++ b/tests/corpora/observability/golden-phase2/index.json
@@ -1,6 +1,6 @@
{
"format": "1.2",
- "signals_version": "0.4",
+ "signals_version": "0.5",
"built": true,
"scenarios": [
{
diff --git a/tests/test_observability_issues.py b/tests/test_observability_issues.py
index a144cf5d..c4d583b1 100644
--- a/tests/test_observability_issues.py
+++ b/tests/test_observability_issues.py
@@ -153,6 +153,44 @@ def test_a_recorded_turn_gives_its_issues(tmp_path):
assert any(f["check"] == "step_limit" and "of 12" in f["text"] for f in res["fine"])
+def test_arguments_that_do_not_fit_the_tool_are_bad_arguments(tmp_path):
+ """Labelling (T11): a model gave replace_in_file a name it does not take. The call fails at
+ the call itself: the model's mistake, so made_up_tool, not "A tool failed"."""
+ d = _run(tmp_path, [
+ _call("c1", "read_file", {"pth": "a"}), # a name the tool does not take
+ _call("c2", "read_file", {}), # the required one missing
+ _call("c3", "read_file", {"path": "a"}), # fits
+ AssistantTurn(text="done"),
+ ])
+ tools = {s["call_id"]: s for s in d["steps"] if s["kind"] == "tool"}
+ assert tools["c1"]["status"] == "error" and tools["c1"]["attrs"]["error_kind"] == "bad_arguments"
+ assert tools["c2"]["attrs"]["error_kind"] == "bad_arguments"
+ assert tools["c3"]["attrs"]["error_kind"] is None
+ res = signals.check_turn(d["turns"][0], d["steps"])
+ found = [i for i in res["issues"] if i["check"] == "made_up_tool"]
+ assert len(found) == 2 and found[0]["text"] == "The model sent arguments that do not fit read_file."
+ assert "tool_failed" not in {i["check"] for i in res["issues"]}
+
+
+def test_a_tool_that_fails_by_itself_is_still_raised(tmp_path):
+ def broken(path: str) -> dict:
+ """Break.
+
+ Args:
+ path: anything
+ """
+ raise RuntimeError("disk on fire")
+
+ from coworker.engine import _args_fit
+ from coworker.tools import ToolRegistry as _R
+
+ reg = _R()
+ reg.register(broken, metadata=_Meta())
+ assert _args_fit(reg.get("broken"), {"path": "x"}) is True
+ assert _args_fit(reg.get("broken"), {"path": "x", "extra": 1}) is False
+ assert _args_fit(None, {"anything": 1}) is True # nothing to check against
+
+
# -- E8c: stored with the turn and served ------------------------------------------------------
def test_issues_are_stored_at_turn_end_and_served(tmp_path):