Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Catch a repeat the model rewords to dodge the guard (#1706)

Rach Pradhan44d4b6fdaf8c
diff --git a/evals/harness_behavior.jsonl b/evals/harness_behavior.jsonl
index f4634067..4a6cd036 100644
--- a/evals/harness_behavior.jsonl
+++ b/evals/harness_behavior.jsonl
@@ -139,3 +139,5 @@
{"id": "repeat-identical-call-stops-a-workflow-synthesis-child", "why": "#1706: the guard sits in the shared turn loop, so a workflow child gets it too. The sweep phase reports, then the synthesis child repeats one call with one result; its 8th batch ends its turn and the parent reads the stop text as the synthesis result instead of the child running on.", "args": [], "prompt": "how does configuration discovery work", "script": [{"tool": "workflow", "arguments": {"phases": [{"title": "sweep config sources", "tasks": [{"description": "sweep-config", "prompt": "search for configuration discovery behavior", "isolation": "shared_cwd"}]}, {"title": "synthesize findings", "tasks": [{"description": "summary", "prompt": "summarize the findings from {{prev}}", "isolation": "shared_cwd"}]}]}}, {"text": "SWEEP-OK"}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"text": "the parent read the synthesis stop"}], "assert": [{"request_lacks": {"index": 5, "text": "times in a row"}}, {"request_contains": {"index": 6, "text": "times in a row"}}, {"request_count": 11}, {"request_contains": {"index": -1, "text": "Stopped: the model repeated the same tool call"}}, {"final_text_contains": "the parent read the synthesis stop"}], "timeout_s": 120}
{"id": "repeat-guard-survives-a-worker-retry", "why": "#1706: a worker's transient-failure re-ask resumes the same history through a fresh runTurn. The identical-batch count must carry over, or each re-ask buys the loop up to seven more batches. Five batches, a failed ask, then the 8th batch overall ends the worker.", "args": [], "prompt": "how does the loader decide which config source wins", "script": [{"tool": "workflow", "arguments": {"phases": [{"title": "sweep config sources", "tasks": [{"description": "sweep-config", "prompt": "search for how the loader picks a config source", "isolation": "shared_cwd"}]}]}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"http_status": 418, "error": "scripted flake"}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"text": "the parent read the worker stop"}], "assert": [{"request_count": 11}, {"request_contains": {"index": -1, "text": "Stopped: the model repeated the same tool call"}}, {"final_text_contains": "the parent read the worker stop"}], "timeout_s": 120}
{"id": "repeat-guard-survives-a-stream-retry", "why": "#1706: a request-layer retry after a mid-stream failure stays inside the turn and must not reset the identical-batch count. Five batches, one failed stream, then the 8th batch overall ends the turn.", "prompt": "Check the box.", "script": [{"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"text": "partial", "stream_error": {"type": "api_error", "message": "Internal error during token parsing"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"text": "never reached"}], "assert": [{"request_count": 9}, {"final_text_contains": "Stopped: the model repeated the same tool call"}]}
+{"id": "reworded-call-same-result-stops-the-turn", "why": "#1706: a live model dodged the identical-call guard by rewording one poll (`cat ./status.txt`, `cat 'status.txt'`, `cat -- status.txt`...) while the result never changed. The same tool returning the same result is the signal: the 6th such batch carries the reworded notice and the 12th ends the turn.", "prompt": "Wait for the deploy in status.txt.", "workspace_seed": {"status.txt": "PENDING\n"}, "script": [{"tool": "bash", "arguments": {"command": "cat status.txt"}}, {"tool": "bash", "arguments": {"command": "cat ./status.txt"}}, {"tool": "bash", "arguments": {"command": "cat status.txt"}}, {"tool": "bash", "arguments": {"command": "cat 'status.txt'"}}, {"tool": "bash", "arguments": {"command": "cat \"status.txt\""}}, {"tool": "bash", "arguments": {"command": "cat -- status.txt"}}, {"tool": "bash", "arguments": {"command": "cat -- ./status.txt"}}, {"tool": "bash", "arguments": {"command": "cat .//status.txt"}}, {"tool": "bash", "arguments": {"command": "cat ././status.txt"}}, {"tool": "bash", "arguments": {"command": "cat -- 'status.txt'"}}, {"tool": "bash", "arguments": {"command": "cat './status.txt'"}}, {"tool": "bash", "arguments": {"command": "cat ./././status.txt"}}, {"text": "never reached"}], "assert": [{"request_lacks": {"index": 5, "text": "only the wording of the call changed"}}, {"request_contains": {"index": 6, "text": "only the wording of the call changed"}}, {"request_lacks": {"index": -1, "text": "times in a row. Repeating it"}}, {"request_count": 12}, {"final_text_contains": "Stopped: the model got the same tool result 12 times in a row"}]}
+{"id": "quiet-commands-do-not-trip-the-repeat-guard", "why": "#1706 counterfactual: commands that print nothing return the same `(no output)` text, but a run of them is ordinary work. Fourteen different quiet commands all run and the turn ends on the model's own reply.", "prompt": "Make the directories.", "script": [{"tool": "bash", "arguments": {"command": "mkdir -p d0"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d1"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d2"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d3"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d4"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d5"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d6"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d7"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d8"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d9"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d10"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d11"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d12"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d13"}}, {"text": "all directories made"}], "assert": [{"request_count": 15}, {"request_lacks": {"index": -1, "text": "in a row"}}, {"final_text_contains": "all directories made"}]}
diff --git a/src/agent_turn.zig b/src/agent_turn.zig
index 2fa59caf..772356be 100644
--- a/src/agent_turn.zig
+++ b/src/agent_turn.zig
@@ -91,12 +91,11 @@ pub fn run(self: *Agent, root_turn_prepared: *std.atomic.Value(bool)) anyerror![
const reply_start = self.messages.items.len;
defer @import("agent_length_cutoff.zig").discardEmptyReply(self, root, reply_start);
const done = try length.step(self, root);
- if (done == null and self.tool_repeat.stopped()) { // #1706: every wire's tool batch lands here
- const stop = @import("tool_repeat.zig").stop_text;
+ if (done == null) if (self.tool_repeat.stopText()) |stop| { // #1706: every wire's tool batch lands here
try self.say("{s}\n", .{stop});
_ = @import("tool_call_repair.zig").endsTurn(self, stop);
return review_deadline.finish(stop);
- }
+ };
if (done) |final_text| {
if (self.yielded != null) return review_deadline.finish(final_text);
if (@import("tool_call_repair.zig").endsTurn(self, final_text)) return review_deadline.finish(final_text); // final: no retry/nudge
diff --git a/src/tool_call_repair.zig b/src/tool_call_repair.zig
index c7e1c8de..dc28c237 100644
--- a/src/tool_call_repair.zig
+++ b/src/tool_call_repair.zig
@@ -25,7 +25,8 @@ pub const loop_stop_text = "Stopped: the model sent tool calls with unusable arg
/// The turn loop ends on this text outright: no retry, nudge, or open-work pass.
pub fn isLoopStop(text: []const u8) bool {
- return text.ptr == loop_stop_text.ptr or text.ptr == @import("tool_repeat.zig").stop_text.ptr;
+ const repeat = @import("tool_repeat.zig");
+ return text.ptr == loop_stop_text.ptr or text.ptr == repeat.stop_text.ptr or text.ptr == repeat.reworded_stop_text.ptr;
}
/// runTurn's check for that stop; a child also closes its feedback inbox.
diff --git a/src/tool_repeat.zig b/src/tool_repeat.zig
index 50199ee1..6e1f4a8a 100644
--- a/src/tool_repeat.zig
+++ b/src/tool_repeat.zig
@@ -4,29 +4,65 @@
//! keep going until a wall clock kills the run. Nothing else ends that turn:
//! `tool_call_repair.brokenCallLoop` only covers unusable arguments, and the
//! dedupe gate in `agent_tools.rejectToolCall` is opt-in. Each executed batch
-//! folds into a fingerprint of its normalized calls and their results; the
-//! `nudge_at`-th identical batch in a row tells the model so, and the
-//! `stop_at`-th ends the turn. A batch still waiting on work is never counted:
-//! a pending or cancelled result, or a shell job poll that reports the job is
-//! still running (`[job N: running…]`), which is how a model waits on CI or a
-//! build. Reset at `runTurn`, except when a worker's transient-failure re-ask
-//! resumes the same history (`carryOver`): a fresh count there would let a
-//! looping worker repeat up to `stop_at - 1` more batches per ask.
+//! folds into two runs:
+//!
+//! - the call run: the same normalized calls with the same results. The
+//! `nudge_at`-th in a row tells the model so, the `stop_at`-th ends the turn.
+//! - the result run: the same tools with the same results, whatever the input.
+//! A model that rewords the call (`cat ./x`, `cat 'x'`, `cat -- x`) to get
+//! the same answer again is still not making progress, so the
+//! `reworded_nudge_at`-th tells it and the `reworded_stop_at`-th ends the
+//! turn. A batch whose results carry no content (`(no output)`, a bare exit
+//! code) breaks this run: quiet commands in a row are ordinary work.
+//!
+//! A batch still waiting on work breaks both runs: a pending or cancelled
+//! result, or a shell job poll that reports the job is still running
+//! (`[job N: running…]`), which is how a model waits on CI or a build. Reset
+//! at `runTurn`, except when a worker's transient-failure re-ask resumes the
+//! same history (`carryOver`): a fresh count there would let a looping worker
+//! repeat up to `stop_at - 1` more batches per ask.
const std = @import("std");
const tools = @import("tools.zig");
pub const nudge_at: u8 = 4;
pub const stop_at: u8 = 8;
+pub const reworded_nudge_at: u8 = 6;
+pub const reworded_stop_at: u8 = 12;
pub const nudge_text = std.fmt.comptimePrint("[graff] This call ran with this input and returned this result {d} times in a row. Repeating it will not make progress: take a different step, or finish with what you have.", .{nudge_at});
pub const stop_text = std.fmt.comptimePrint("Stopped: the model repeated the same tool call with the same result {d} times in a row, so the turn ended instead of running it again. Send a new message to continue, or switch models.", .{stop_at});
-/// Turn-scoped run of identical batches. No heap.
-pub const Tracker = struct {
+pub const reworded_nudge_text = std.fmt.comptimePrint("[graff] This tool returned this same result {d} times in a row; only the wording of the call changed. Rewording it will not make progress: take a different step, or finish with what you have.", .{reworded_nudge_at});
+
+pub const reworded_stop_text = std.fmt.comptimePrint("Stopped: the model got the same tool result {d} times in a row while rewording the call, so the turn ended instead of running it again. Send a new message to continue, or switch models.", .{reworded_stop_at});
+
+/// One run of equal fingerprints; null (no countable batch) breaks it.
+const Run = struct {
last: u64 = 0,
- run: u8 = 0,
+ len: u8 = 0,
+
+ fn note(self: *Run, fp: ?u64) u8 {
+ const f = fp orelse {
+ self.* = .{};
+ return 0;
+ };
+ if (self.len > 0 and f == self.last) {
+ self.len +|= 1;
+ } else {
+ self.* = .{ .last = f, .len = 1 };
+ }
+ return self.len;
+ }
+};
+
+pub const Notice = enum { none, repeated, reworded };
+
+/// Turn-scoped runs of repeating batches. No heap.
+pub const Tracker = struct {
+ calls: Run = .{},
+ results: Run = .{},
carry: bool = false,
pub fn reset(self: *Tracker) void {
@@ -37,29 +73,26 @@ pub const Tracker = struct {
self.* = .{};
}
- /// The next `reset` keeps the run: the coming `runTurn` resumes this
+ /// The next `reset` keeps the runs: the coming `runTurn` resumes this
/// turn's history rather than starting a new one.
pub fn carryOver(self: *Tracker) void {
self.carry = true;
}
- /// Folds one batch; a null fingerprint (work still pending) breaks the run.
- pub fn note(self: *Tracker, fp: ?u64) u8 {
- const f = fp orelse {
- self.* = .{};
- return 0;
- };
- if (self.run > 0 and f == self.last) {
- self.run +|= 1;
- } else {
- self.last = f;
- self.run = 1;
- }
- return self.run;
+ /// Folds one batch into both runs; says which notice, if any, it earned.
+ pub fn note(self: *Tracker, calls_fp: ?u64, results_fp: ?u64) Notice {
+ const same_call = self.calls.note(calls_fp);
+ const same_result = self.results.note(results_fp);
+ if (same_call == nudge_at) return .repeated;
+ if (same_result == reworded_nudge_at and same_call < nudge_at) return .reworded;
+ return .none;
}
- pub fn stopped(self: *const Tracker) bool {
- return self.run >= stop_at;
+ /// The text that ends the turn, once a run reached its stop.
+ pub fn stopText(self: *const Tracker) ?[]const u8 {
+ if (self.calls.len >= stop_at) return stop_text;
+ if (self.results.len >= reworded_stop_at) return reworded_stop_text;
+ return null;
}
};
@@ -70,6 +103,60 @@ fn jobRunning(text: []const u8) bool {
return std.mem.indexOf(u8, head, ": running") != null;
}
+/// Inputs that only bound or label a call. A model that varies them, or the
+/// spacing inside a command, to dodge the guard is still repeating itself:
+/// one live run polled `cat status.txt` as `cat status.txt`, then with
+/// `timeout` 1000, 1001, 1002...
+const knobs = [_][]const u8{ "timeout", "timeout_ms", "wait_ms", "description" };
+
+/// The call's identity for the fingerprint: its name and input, minus the
+/// top-level knobs, with whitespace runs in strings collapsed.
+pub fn callKey(arena: std.mem.Allocator, call: tools.ToolCall) ![]const u8 {
+ var aw: std.Io.Writer.Allocating = .init(arena);
+ try aw.writer.writeAll(call.name);
+ try aw.writer.writeByte('\n');
+ try writeInput(&aw.writer, call.input, true);
+ return aw.writer.buffered();
+}
+
+fn writeInput(w: *std.Io.Writer, v: std.json.Value, top: bool) !void {
+ switch (v) {
+ .string => |text| {
+ try w.writeByte('"');
+ var it = std.mem.tokenizeAny(u8, text, " \t\r\n");
+ var first = true;
+ while (it.next()) |word| : (first = false) {
+ if (!first) try w.writeByte(' ');
+ try w.writeAll(word);
+ }
+ try w.writeByte(0);
+ },
+ .object => |o| {
+ try w.writeByte('{');
+ var it = o.iterator();
+ entries: while (it.next()) |kv| {
+ if (top) for (knobs) |k| if (std.mem.eql(u8, kv.key_ptr.*, k)) continue :entries;
+ try w.print("{s}:", .{kv.key_ptr.*});
+ try writeInput(w, kv.value_ptr.*, false);
+ try w.writeByte(',');
+ }
+ try w.writeByte('}');
+ },
+ .array => |a| {
+ try w.writeByte('[');
+ for (a.items) |item| {
+ try writeInput(w, item, false);
+ try w.writeByte(',');
+ }
+ try w.writeByte(']');
+ },
+ else => {
+ var j: std.json.Stringify = .{ .writer = w };
+ try j.write(v);
+ },
+ }
+}
+
/// Null when any result is still waiting: waiting is not repeating.
pub fn fingerprint(keys: []const []const u8, results: []const tools.ExecResult) ?u64 {
var h = std.hash.Wyhash.init(0x1706);
@@ -83,15 +170,41 @@ pub fn fingerprint(keys: []const []const u8, results: []const tools.ExecResult)
return h.final();
}
+/// No output and no error text: `(no output)`, or nothing but `[exit code N]`.
+fn contentFree(text: []const u8) bool {
+ var rest = std.mem.trim(u8, text, " \t\r\n");
+ if (std.mem.endsWith(u8, rest, "]")) {
+ if (std.mem.lastIndexOf(u8, rest, "[exit code ")) |i| rest = std.mem.trim(u8, rest[0..i], " \t\r\n");
+ }
+ return rest.len == 0 or std.mem.eql(u8, rest, "(no output)");
+}
+
+/// The batch's tools and results, input ignored. Null when every result is
+/// content-free, or when any is still waiting.
+pub fn resultFingerprint(names: []const []const u8, results: []const tools.ExecResult) ?u64 {
+ for (results) |r| {
+ if (!contentFree(r.text)) break;
+ } else return null;
+ return fingerprint(names, results);
+}
+
/// After a batch ran: count it, and on the nudge append the notice to the
/// last result so the model reads it with what it just got back.
pub fn observe(self: anytype, calls: []const tools.ToolCall, results: []tools.ExecResult) !void {
if (calls.len == 0) return;
const keys = try self.arena.alloc([]const u8, calls.len);
- for (calls, keys) |call, *key| key.* = try self.toolDedupeKey(call);
- if (self.tool_repeat.note(fingerprint(keys, results)) != nudge_at) return;
+ const names = try self.arena.alloc([]const u8, calls.len);
+ for (calls, keys, names) |call, *key, *name| {
+ key.* = try callKey(self.arena, call);
+ name.* = call.name;
+ }
+ const notice = switch (self.tool_repeat.note(fingerprint(keys, results), resultFingerprint(names, results))) {
+ .none => return,
+ .repeated => nudge_text,
+ .reworded => reworded_nudge_text,
+ };
const last = &results[results.len - 1];
- last.text = try std.fmt.allocPrint(self.arena, "{s}\n\n{s}", .{ last.text, nudge_text });
+ last.text = try std.fmt.allocPrint(self.arena, "{s}\n\n{s}", .{ last.text, notice });
}
test "identical batches build a run that stops at stop_at" {
@@ -100,13 +213,13 @@ test "identical batches build a run that stops at stop_at" {
const ok = [_]tools.ExecResult{.{ .text = "ok\n", .is_error = false }};
var i: u8 = 1;
while (i < stop_at) : (i += 1) {
- try std.testing.expectEqual(i, t.note(fingerprint(&keys, &ok)));
- try std.testing.expect(!t.stopped());
+ try std.testing.expectEqual(i, t.calls.note(fingerprint(&keys, &ok)));
+ try std.testing.expect(t.stopText() == null);
}
- try std.testing.expectEqual(stop_at, t.note(fingerprint(&keys, &ok)));
- try std.testing.expect(t.stopped());
+ try std.testing.expectEqual(stop_at, t.calls.note(fingerprint(&keys, &ok)));
+ try std.testing.expectEqual(@as(?[]const u8, stop_text), t.stopText());
t.reset();
- try std.testing.expect(!t.stopped());
+ try std.testing.expect(t.stopText() == null);
}
test "a carried run survives one reset, then resets clear it again" {
@@ -114,14 +227,14 @@ test "a carried run survives one reset, then resets clear it again" {
const keys = [_][]const u8{"shell\n{\"command\":\"echo ok\"}"};
const ok = [_]tools.ExecResult{.{ .text = "ok\n", .is_error = false }};
var i: u8 = 1;
- while (i < stop_at) : (i += 1) _ = t.note(fingerprint(&keys, &ok));
+ while (i < stop_at) : (i += 1) _ = t.calls.note(fingerprint(&keys, &ok));
t.carryOver();
t.reset();
- try std.testing.expectEqual(stop_at, t.note(fingerprint(&keys, &ok)));
- try std.testing.expect(t.stopped());
+ try std.testing.expectEqual(stop_at, t.calls.note(fingerprint(&keys, &ok)));
+ try std.testing.expect(t.stopText() != null);
t.reset();
- try std.testing.expect(!t.stopped());
- try std.testing.expectEqual(@as(u8, 1), t.note(fingerprint(&keys, &ok)));
+ try std.testing.expect(t.stopText() == null);
+ try std.testing.expectEqual(@as(u8, 1), t.calls.note(fingerprint(&keys, &ok)));
}
test "a different input, result, or error flag starts a new run" {
@@ -131,11 +244,11 @@ test "a different input, result, or error flag starts a new run" {
const ok = [_]tools.ExecResult{.{ .text = "ok\n", .is_error = false }};
const ok_err = [_]tools.ExecResult{.{ .text = "ok\n", .is_error = true }};
const other = [_]tools.ExecResult{.{ .text = "ok 2\n", .is_error = false }};
- _ = t.note(fingerprint(&a, &ok));
- try std.testing.expectEqual(@as(u8, 2), t.note(fingerprint(&a, &ok)));
- try std.testing.expectEqual(@as(u8, 1), t.note(fingerprint(&b, &ok)));
- try std.testing.expectEqual(@as(u8, 1), t.note(fingerprint(&b, &other)));
- try std.testing.expectEqual(@as(u8, 1), t.note(fingerprint(&b, &ok_err)));
+ _ = t.calls.note(fingerprint(&a, &ok));
+ try std.testing.expectEqual(@as(u8, 2), t.calls.note(fingerprint(&a, &ok)));
+ try std.testing.expectEqual(@as(u8, 1), t.calls.note(fingerprint(&b, &ok)));
+ try std.testing.expectEqual(@as(u8, 1), t.calls.note(fingerprint(&b, &other)));
+ try std.testing.expectEqual(@as(u8, 1), t.calls.note(fingerprint(&b, &ok_err)));
}
test "a pending or cancelled result never counts and breaks the run" {
@@ -144,11 +257,11 @@ test "a pending or cancelled result never counts and breaks the run" {
const done = [_]tools.ExecResult{.{ .text = "still running", .is_error = false }};
const waiting = [_]tools.ExecResult{.{ .text = "still running", .is_error = false, .pending = true }};
const cancelled = [_]tools.ExecResult{.{ .text = "still running", .is_error = false, .cancelled = true }};
- _ = t.note(fingerprint(&keys, &done));
- _ = t.note(fingerprint(&keys, &done));
- try std.testing.expectEqual(@as(u8, 0), t.note(fingerprint(&keys, &waiting)));
- try std.testing.expectEqual(@as(u8, 1), t.note(fingerprint(&keys, &done)));
- try std.testing.expectEqual(@as(u8, 0), t.note(fingerprint(&keys, &cancelled)));
+ _ = t.calls.note(fingerprint(&keys, &done));
+ _ = t.calls.note(fingerprint(&keys, &done));
+ try std.testing.expectEqual(@as(u8, 0), t.calls.note(fingerprint(&keys, &waiting)));
+ try std.testing.expectEqual(@as(u8, 1), t.calls.note(fingerprint(&keys, &done)));
+ try std.testing.expectEqual(@as(u8, 0), t.calls.note(fingerprint(&keys, &cancelled)));
}
test "polling a job that is still running never counts, a finished job's output does" {
@@ -158,12 +271,12 @@ test "polling a job that is still running never counts, a finished job's output
const parked = [_]tools.ExecResult{.{ .text = "[job 9: running — persistent server; parked in the background. You are notified on exit; do not poll]", .is_error = false }};
const exited = [_]tools.ExecResult{.{ .text = "[job 34: exited with code 0]\nok", .is_error = false }};
var i: u8 = 0;
- while (i < 12) : (i += 1) try std.testing.expectEqual(@as(u8, 0), t.note(fingerprint(&keys, &running)));
- while (i < 24) : (i += 1) try std.testing.expectEqual(@as(u8, 0), t.note(fingerprint(&keys, &parked)));
- try std.testing.expect(!t.stopped());
+ while (i < 12) : (i += 1) try std.testing.expectEqual(@as(u8, 0), t.calls.note(fingerprint(&keys, &running)));
+ while (i < 24) : (i += 1) try std.testing.expectEqual(@as(u8, 0), t.calls.note(fingerprint(&keys, &parked)));
+ try std.testing.expect(t.stopText() == null);
i = 1;
- while (i <= stop_at) : (i += 1) _ = t.note(fingerprint(&keys, &exited));
- try std.testing.expect(t.stopped());
+ while (i <= stop_at) : (i += 1) _ = t.calls.note(fingerprint(&keys, &exited));
+ try std.testing.expect(t.stopText() != null);
}
test "an empty-argument error loop still stops" {
@@ -171,8 +284,8 @@ test "an empty-argument error loop still stops" {
const keys = [_][]const u8{"read_file\n{}"};
const err = [_]tools.ExecResult{.{ .text = "[error] tool call arguments were truncated", .is_error = true }};
var i: u8 = 0;
- while (i < stop_at) : (i += 1) _ = t.note(fingerprint(&keys, &err));
- try std.testing.expect(t.stopped());
+ while (i < stop_at) : (i += 1) _ = t.calls.note(fingerprint(&keys, &err));
+ try std.testing.expect(t.stopText() != null);
}
test "observe appends the notice on the nudge batch only" {
@@ -181,9 +294,6 @@ test "observe appends the notice on the nudge batch only" {
const Fake = struct {
arena: std.mem.Allocator,
tool_repeat: Tracker = .{},
- fn toolDedupeKey(_: *@This(), call: tools.ToolCall) ![]const u8 {
- return call.name;
- }
};
var fake: Fake = .{ .arena = arena_state.allocator() };
const calls = [_]tools.ToolCall{.{ .id = "c", .name = "shell", .input = .null }};
@@ -195,3 +305,101 @@ test "observe appends the notice on the nudge batch only" {
try std.testing.expectEqual(i == nudge_at, noticed);
}
}
+
+test "the same result under reworded calls nudges at 6 and stops at 12" {
+ var t: Tracker = .{};
+ const names = [_][]const u8{"shell"};
+ const pending = [_]tools.ExecResult{.{ .text = "PENDING\n", .is_error = false }};
+ const spellings = [_][]const u8{ "cat status.txt", "cat ./status.txt", "cat 'status.txt'", "cat -- status.txt", "cat \"status.txt\"", "cat ././status.txt" };
+ var i: u8 = 1;
+ while (i <= reworded_stop_at) : (i += 1) {
+ const keys = [_][]const u8{spellings[(i - 1) % spellings.len]};
+ const notice = t.note(fingerprint(&keys, &pending), resultFingerprint(&names, &pending));
+ try std.testing.expectEqual(if (i == reworded_nudge_at) Notice.reworded else Notice.none, notice);
+ try std.testing.expectEqual(i == reworded_stop_at, t.stopText() != null);
+ }
+ try std.testing.expectEqual(@as(?[]const u8, reworded_stop_text), t.stopText());
+}
+
+test "identical calls take the call notice and stop, never the reworded one" {
+ var t: Tracker = .{};
+ const keys = [_][]const u8{"shell\n{\"command\":\"echo ok\"}"};
+ const names = [_][]const u8{"shell"};
+ const ok = [_]tools.ExecResult{.{ .text = "ok\n", .is_error = false }};
+ var i: u8 = 1;
+ while (i <= stop_at) : (i += 1) {
+ const notice = t.note(fingerprint(&keys, &ok), resultFingerprint(&names, &ok));
+ try std.testing.expectEqual(if (i == nudge_at) Notice.repeated else Notice.none, notice);
+ }
+ try std.testing.expectEqual(@as(?[]const u8, stop_text), t.stopText());
+}
+
+test "content-free results under different calls are ordinary work" {
+ var t: Tracker = .{};
+ const names = [_][]const u8{"shell"};
+ const quiet = [_][]const tools.ExecResult{
+ &.{.{ .text = "(no output)", .is_error = false }},
+ &.{.{ .text = "\n[exit code 1]", .is_error = true }},
+ &.{.{ .text = "", .is_error = false }},
+ };
+ var i: u8 = 0;
+ while (i < 3 * reworded_stop_at) : (i += 1) {
+ const r = quiet[i % quiet.len];
+ try std.testing.expectEqual(@as(?u64, null), resultFingerprint(&names, r));
+ const keys = [_][]const u8{if (i % 2 == 0) "grep a" else "grep b"};
+ try std.testing.expectEqual(Notice.none, t.note(fingerprint(&keys, r), resultFingerprint(&names, r)));
+ }
+ try std.testing.expect(t.stopText() == null);
+ const noisy = [_]tools.ExecResult{.{ .text = "boom\n[exit code 1]", .is_error = true }};
+ try std.testing.expect(resultFingerprint(&names, &noisy) != null);
+}
+
+test "observe appends the reworded notice when only the wording changes" {
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+ const Fake = struct {
+ arena: std.mem.Allocator,
+ tool_repeat: Tracker = .{},
+ };
+ var fake: Fake = .{ .arena = arena };
+ const dots = "./././././././.";
+ var i: u8 = 1;
+ while (i <= reworded_nudge_at + 1) : (i += 1) {
+ const input = try std.json.parseFromSliceLeaky(std.json.Value, arena, try std.fmt.allocPrint(arena, "{{\"command\":\"cat {s}status.txt\"}}", .{dots[0 .. 2 * @as(usize, i)]}), .{});
+ const calls = [_]tools.ToolCall{.{ .id = "c", .name = "shell", .input = input }};
+ var results = [_]tools.ExecResult{.{ .text = "PENDING\n", .is_error = false }};
+ try observe(&fake, &calls, &results);
+ try std.testing.expectEqual(i == reworded_nudge_at, std.mem.indexOf(u8, results[0].text, reworded_nudge_text) != null);
+ try std.testing.expect(std.mem.indexOf(u8, results[0].text, nudge_text) == null);
+ }
+}
+
+test "the key ignores knobs and spacing, not what the call does" {
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+ const inputs = [_][]const u8{
+ "{\"command\":\"cat status.txt\"}",
+ "{\"command\":\"cat status.txt\"}",
+ "{\"command\":\"cat status.txt\",\"timeout\":1000}",
+ "{\"command\":\" cat status.txt\\n\",\"timeout\":1001,\"description\":\"poll\"}",
+ };
+ var keys: [inputs.len][]const u8 = undefined;
+ for (inputs, &keys) |text, *key| {
+ const v = try std.json.parseFromSliceLeaky(std.json.Value, arena, text, .{});
+ key.* = try callKey(arena, .{ .id = "c", .name = "shell", .input = v });
+ }
+ for (keys[1..]) |key| try std.testing.expectEqualStrings(keys[0], key);
+ const differ = [_][]const u8{
+ "{\"command\":\"cat other.txt\"}",
+ "{\"command\":\"cat status.txt\",\"run_in_background\":true}",
+ "{\"command\":\"cat status.txt\",\"opts\":{\"timeout\":1}}",
+ };
+ for (differ) |text| {
+ const v = try std.json.parseFromSliceLeaky(std.json.Value, arena, text, .{});
+ try std.testing.expect(!std.mem.eql(u8, keys[0], try callKey(arena, .{ .id = "c", .name = "shell", .input = v })));
+ }
+ const read = try std.json.parseFromSliceLeaky(std.json.Value, arena, inputs[0], .{});
+ try std.testing.expect(!std.mem.eql(u8, keys[0], try callKey(arena, .{ .id = "c", .name = "read_file", .input = read })));
+}