graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Catch a repeat the model rewords to dodge the guard (#1706)
Rach Pradhan44d4b6fdaf8c
diff --git a/evals/harness_behavior.jsonl b/evals/harness_behavior.jsonlindex f4634067..4a6cd036 100644--- a/evals/harness_behavior.jsonl+++ b/evals/harness_behavior.jsonl@@ -139,3 +139,5 @@{"id": "repeat-identical-call-stops-a-workflow-synthesis-child", "why": "#1706: the guard sits in the shared turn loop, so a workflow child gets it too. The sweep phase reports, then the synthesis child repeats one call with one result; its 8th batch ends its turn and the parent reads the stop text as the synthesis result instead of the child running on.", "args": [], "prompt": "how does configuration discovery work", "script": [{"tool": "workflow", "arguments": {"phases": [{"title": "sweep config sources", "tasks": [{"description": "sweep-config", "prompt": "search for configuration discovery behavior", "isolation": "shared_cwd"}]}, {"title": "synthesize findings", "tasks": [{"description": "summary", "prompt": "summarize the findings from {{prev}}", "isolation": "shared_cwd"}]}]}}, {"text": "SWEEP-OK"}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"text": "the parent read the synthesis stop"}], "assert": [{"request_lacks": {"index": 5, "text": "times in a row"}}, {"request_contains": {"index": 6, "text": "times in a row"}}, {"request_count": 11}, {"request_contains": {"index": -1, "text": "Stopped: the model repeated the same tool call"}}, {"final_text_contains": "the parent read the synthesis stop"}], "timeout_s": 120}{"id": "repeat-guard-survives-a-worker-retry", "why": "#1706: a worker's transient-failure re-ask resumes the same history through a fresh runTurn. The identical-batch count must carry over, or each re-ask buys the loop up to seven more batches. Five batches, a failed ask, then the 8th batch overall ends the worker.", "args": [], "prompt": "how does the loader decide which config source wins", "script": [{"tool": "workflow", "arguments": {"phases": [{"title": "sweep config sources", "tasks": [{"description": "sweep-config", "prompt": "search for how the loader picks a config source", "isolation": "shared_cwd"}]}]}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"http_status": 418, "error": "scripted flake"}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"text": "the parent read the worker stop"}], "assert": [{"request_count": 11}, {"request_contains": {"index": -1, "text": "Stopped: the model repeated the same tool call"}}, {"final_text_contains": "the parent read the worker stop"}], "timeout_s": 120}{"id": "repeat-guard-survives-a-stream-retry", "why": "#1706: a request-layer retry after a mid-stream failure stays inside the turn and must not reset the identical-batch count. Five batches, one failed stream, then the 8th batch overall ends the turn.", "prompt": "Check the box.", "script": [{"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"text": "partial", "stream_error": {"type": "api_error", "message": "Internal error during token parsing"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"tool": "bash", "arguments": {"command": "echo ok"}}, {"text": "never reached"}], "assert": [{"request_count": 9}, {"final_text_contains": "Stopped: the model repeated the same tool call"}]}+{"id": "reworded-call-same-result-stops-the-turn", "why": "#1706: a live model dodged the identical-call guard by rewording one poll (`cat ./status.txt`, `cat 'status.txt'`, `cat -- status.txt`...) while the result never changed. The same tool returning the same result is the signal: the 6th such batch carries the reworded notice and the 12th ends the turn.", "prompt": "Wait for the deploy in status.txt.", "workspace_seed": {"status.txt": "PENDING\n"}, "script": [{"tool": "bash", "arguments": {"command": "cat status.txt"}}, {"tool": "bash", "arguments": {"command": "cat ./status.txt"}}, {"tool": "bash", "arguments": {"command": "cat status.txt"}}, {"tool": "bash", "arguments": {"command": "cat 'status.txt'"}}, {"tool": "bash", "arguments": {"command": "cat \"status.txt\""}}, {"tool": "bash", "arguments": {"command": "cat -- status.txt"}}, {"tool": "bash", "arguments": {"command": "cat -- ./status.txt"}}, {"tool": "bash", "arguments": {"command": "cat .//status.txt"}}, {"tool": "bash", "arguments": {"command": "cat ././status.txt"}}, {"tool": "bash", "arguments": {"command": "cat -- 'status.txt'"}}, {"tool": "bash", "arguments": {"command": "cat './status.txt'"}}, {"tool": "bash", "arguments": {"command": "cat ./././status.txt"}}, {"text": "never reached"}], "assert": [{"request_lacks": {"index": 5, "text": "only the wording of the call changed"}}, {"request_contains": {"index": 6, "text": "only the wording of the call changed"}}, {"request_lacks": {"index": -1, "text": "times in a row. Repeating it"}}, {"request_count": 12}, {"final_text_contains": "Stopped: the model got the same tool result 12 times in a row"}]}+{"id": "quiet-commands-do-not-trip-the-repeat-guard", "why": "#1706 counterfactual: commands that print nothing return the same `(no output)` text, but a run of them is ordinary work. Fourteen different quiet commands all run and the turn ends on the model's own reply.", "prompt": "Make the directories.", "script": [{"tool": "bash", "arguments": {"command": "mkdir -p d0"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d1"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d2"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d3"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d4"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d5"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d6"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d7"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d8"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d9"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d10"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d11"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d12"}}, {"tool": "bash", "arguments": {"command": "mkdir -p d13"}}, {"text": "all directories made"}], "assert": [{"request_count": 15}, {"request_lacks": {"index": -1, "text": "in a row"}}, {"final_text_contains": "all directories made"}]}diff --git a/src/agent_turn.zig b/src/agent_turn.zigindex 2fa59caf..772356be 100644--- a/src/agent_turn.zig+++ b/src/agent_turn.zig@@ -91,12 +91,11 @@ pub fn run(self: *Agent, root_turn_prepared: *std.atomic.Value(bool)) anyerror![const reply_start = self.messages.items.len;defer @import("agent_length_cutoff.zig").discardEmptyReply(self, root, reply_start);const done = try length.step(self, root);- if (done == null and self.tool_repeat.stopped()) { // #1706: every wire's tool batch lands here- const stop = @import("tool_repeat.zig").stop_text;+ if (done == null) if (self.tool_repeat.stopText()) |stop| { // #1706: every wire's tool batch lands heretry self.say("{s}\n", .{stop});_ = @import("tool_call_repair.zig").endsTurn(self, stop);return review_deadline.finish(stop);- }+ };if (done) |final_text| {if (self.yielded != null) return review_deadline.finish(final_text);if (@import("tool_call_repair.zig").endsTurn(self, final_text)) return review_deadline.finish(final_text); // final: no retry/nudgediff --git a/src/tool_call_repair.zig b/src/tool_call_repair.zigindex c7e1c8de..dc28c237 100644--- a/src/tool_call_repair.zig+++ b/src/tool_call_repair.zig@@ -25,7 +25,8 @@ pub const loop_stop_text = "Stopped: the model sent tool calls with unusable arg/// The turn loop ends on this text outright: no retry, nudge, or open-work pass.pub fn isLoopStop(text: []const u8) bool {- return text.ptr == loop_stop_text.ptr or text.ptr == @import("tool_repeat.zig").stop_text.ptr;+ const repeat = @import("tool_repeat.zig");+ return text.ptr == loop_stop_text.ptr or text.ptr == repeat.stop_text.ptr or text.ptr == repeat.reworded_stop_text.ptr;}/// runTurn's check for that stop; a child also closes its feedback inbox.diff --git a/src/tool_repeat.zig b/src/tool_repeat.zigindex 50199ee1..6e1f4a8a 100644--- a/src/tool_repeat.zig+++ b/src/tool_repeat.zig@@ -4,29 +4,65 @@//! keep going until a wall clock kills the run. Nothing else ends that turn://! `tool_call_repair.brokenCallLoop` only covers unusable arguments, and the//! dedupe gate in `agent_tools.rejectToolCall` is opt-in. Each executed batch-//! folds into a fingerprint of its normalized calls and their results; the-//! `nudge_at`-th identical batch in a row tells the model so, and the-//! `stop_at`-th ends the turn. A batch still waiting on work is never counted:-//! a pending or cancelled result, or a shell job poll that reports the job is-//! still running (`[job N: running…]`), which is how a model waits on CI or a-//! build. Reset at `runTurn`, except when a worker's transient-failure re-ask-//! resumes the same history (`carryOver`): a fresh count there would let a-//! looping worker repeat up to `stop_at - 1` more batches per ask.+//! folds into two runs:+//!+//! - the call run: the same normalized calls with the same results. The+//! `nudge_at`-th in a row tells the model so, the `stop_at`-th ends the turn.+//! - the result run: the same tools with the same results, whatever the input.+//! A model that rewords the call (`cat ./x`, `cat 'x'`, `cat -- x`) to get+//! the same answer again is still not making progress, so the+//! `reworded_nudge_at`-th tells it and the `reworded_stop_at`-th ends the+//! turn. A batch whose results carry no content (`(no output)`, a bare exit+//! code) breaks this run: quiet commands in a row are ordinary work.+//!+//! A batch still waiting on work breaks both runs: a pending or cancelled+//! result, or a shell job poll that reports the job is still running+//! (`[job N: running…]`), which is how a model waits on CI or a build. Reset+//! at `runTurn`, except when a worker's transient-failure re-ask resumes the+//! same history (`carryOver`): a fresh count there would let a looping worker+//! repeat up to `stop_at - 1` more batches per ask.const std = @import("std");const tools = @import("tools.zig");pub const nudge_at: u8 = 4;pub const stop_at: u8 = 8;+pub const reworded_nudge_at: u8 = 6;+pub const reworded_stop_at: u8 = 12;pub const nudge_text = std.fmt.comptimePrint("[graff] This call ran with this input and returned this result {d} times in a row. Repeating it will not make progress: take a different step, or finish with what you have.", .{nudge_at});pub const stop_text = std.fmt.comptimePrint("Stopped: the model repeated the same tool call with the same result {d} times in a row, so the turn ended instead of running it again. Send a new message to continue, or switch models.", .{stop_at});-/// Turn-scoped run of identical batches. No heap.-pub const Tracker = struct {+pub const reworded_nudge_text = std.fmt.comptimePrint("[graff] This tool returned this same result {d} times in a row; only the wording of the call changed. Rewording it will not make progress: take a different step, or finish with what you have.", .{reworded_nudge_at});++pub const reworded_stop_text = std.fmt.comptimePrint("Stopped: the model got the same tool result {d} times in a row while rewording the call, so the turn ended instead of running it again. Send a new message to continue, or switch models.", .{reworded_stop_at});++/// One run of equal fingerprints; null (no countable batch) breaks it.+const Run = struct {last: u64 = 0,- run: u8 = 0,+ len: u8 = 0,++ fn note(self: *Run, fp: ?u64) u8 {+ const f = fp orelse {+ self.* = .{};+ return 0;+ };+ if (self.len > 0 and f == self.last) {+ self.len +|= 1;+ } else {+ self.* = .{ .last = f, .len = 1 };+ }+ return self.len;+ }+};++pub const Notice = enum { none, repeated, reworded };++/// Turn-scoped runs of repeating batches. No heap.+pub const Tracker = struct {+ calls: Run = .{},+ results: Run = .{},carry: bool = false,pub fn reset(self: *Tracker) void {@@ -37,29 +73,26 @@ pub const Tracker = struct {self.* = .{};}- /// The next `reset` keeps the run: the coming `runTurn` resumes this+ /// The next `reset` keeps the runs: the coming `runTurn` resumes this/// turn's history rather than starting a new one.pub fn carryOver(self: *Tracker) void {self.carry = true;}- /// Folds one batch; a null fingerprint (work still pending) breaks the run.- pub fn note(self: *Tracker, fp: ?u64) u8 {- const f = fp orelse {- self.* = .{};- return 0;- };- if (self.run > 0 and f == self.last) {- self.run +|= 1;- } else {- self.last = f;- self.run = 1;- }- return self.run;+ /// Folds one batch into both runs; says which notice, if any, it earned.+ pub fn note(self: *Tracker, calls_fp: ?u64, results_fp: ?u64) Notice {+ const same_call = self.calls.note(calls_fp);+ const same_result = self.results.note(results_fp);+ if (same_call == nudge_at) return .repeated;+ if (same_result == reworded_nudge_at and same_call < nudge_at) return .reworded;+ return .none;}- pub fn stopped(self: *const Tracker) bool {- return self.run >= stop_at;+ /// The text that ends the turn, once a run reached its stop.+ pub fn stopText(self: *const Tracker) ?[]const u8 {+ if (self.calls.len >= stop_at) return stop_text;+ if (self.results.len >= reworded_stop_at) return reworded_stop_text;+ return null;}};@@ -70,6 +103,60 @@ fn jobRunning(text: []const u8) bool {return std.mem.indexOf(u8, head, ": running") != null;}+/// Inputs that only bound or label a call. A model that varies them, or the+/// spacing inside a command, to dodge the guard is still repeating itself:+/// one live run polled `cat status.txt` as `cat status.txt`, then with+/// `timeout` 1000, 1001, 1002...+const knobs = [_][]const u8{ "timeout", "timeout_ms", "wait_ms", "description" };++/// The call's identity for the fingerprint: its name and input, minus the+/// top-level knobs, with whitespace runs in strings collapsed.+pub fn callKey(arena: std.mem.Allocator, call: tools.ToolCall) ![]const u8 {+ var aw: std.Io.Writer.Allocating = .init(arena);+ try aw.writer.writeAll(call.name);+ try aw.writer.writeByte('\n');+ try writeInput(&aw.writer, call.input, true);+ return aw.writer.buffered();+}++fn writeInput(w: *std.Io.Writer, v: std.json.Value, top: bool) !void {+ switch (v) {+ .string => |text| {+ try w.writeByte('"');+ var it = std.mem.tokenizeAny(u8, text, " \t\r\n");+ var first = true;+ while (it.next()) |word| : (first = false) {+ if (!first) try w.writeByte(' ');+ try w.writeAll(word);+ }+ try w.writeByte(0);+ },+ .object => |o| {+ try w.writeByte('{');+ var it = o.iterator();+ entries: while (it.next()) |kv| {+ if (top) for (knobs) |k| if (std.mem.eql(u8, kv.key_ptr.*, k)) continue :entries;+ try w.print("{s}:", .{kv.key_ptr.*});+ try writeInput(w, kv.value_ptr.*, false);+ try w.writeByte(',');+ }+ try w.writeByte('}');+ },+ .array => |a| {+ try w.writeByte('[');+ for (a.items) |item| {+ try writeInput(w, item, false);+ try w.writeByte(',');+ }+ try w.writeByte(']');+ },+ else => {+ var j: std.json.Stringify = .{ .writer = w };+ try j.write(v);+ },+ }+}+/// Null when any result is still waiting: waiting is not repeating.pub fn fingerprint(keys: []const []const u8, results: []const tools.ExecResult) ?u64 {var h = std.hash.Wyhash.init(0x1706);@@ -83,15 +170,41 @@ pub fn fingerprint(keys: []const []const u8, results: []const tools.ExecResult)return h.final();}+/// No output and no error text: `(no output)`, or nothing but `[exit code N]`.+fn contentFree(text: []const u8) bool {+ var rest = std.mem.trim(u8, text, " \t\r\n");+ if (std.mem.endsWith(u8, rest, "]")) {+ if (std.mem.lastIndexOf(u8, rest, "[exit code ")) |i| rest = std.mem.trim(u8, rest[0..i], " \t\r\n");+ }+ return rest.len == 0 or std.mem.eql(u8, rest, "(no output)");+}++/// The batch's tools and results, input ignored. Null when every result is+/// content-free, or when any is still waiting.+pub fn resultFingerprint(names: []const []const u8, results: []const tools.ExecResult) ?u64 {+ for (results) |r| {+ if (!contentFree(r.text)) break;+ } else return null;+ return fingerprint(names, results);+}+/// After a batch ran: count it, and on the nudge append the notice to the/// last result so the model reads it with what it just got back.pub fn observe(self: anytype, calls: []const tools.ToolCall, results: []tools.ExecResult) !void {if (calls.len == 0) return;const keys = try self.arena.alloc([]const u8, calls.len);- for (calls, keys) |call, *key| key.* = try self.toolDedupeKey(call);- if (self.tool_repeat.note(fingerprint(keys, results)) != nudge_at) return;+ const names = try self.arena.alloc([]const u8, calls.len);+ for (calls, keys, names) |call, *key, *name| {+ key.* = try callKey(self.arena, call);+ name.* = call.name;+ }+ const notice = switch (self.tool_repeat.note(fingerprint(keys, results), resultFingerprint(names, results))) {+ .none => return,+ .repeated => nudge_text,+ .reworded => reworded_nudge_text,+ };const last = &results[results.len - 1];- last.text = try std.fmt.allocPrint(self.arena, "{s}\n\n{s}", .{ last.text, nudge_text });+ last.text = try std.fmt.allocPrint(self.arena, "{s}\n\n{s}", .{ last.text, notice });}test "identical batches build a run that stops at stop_at" {@@ -100,13 +213,13 @@ test "identical batches build a run that stops at stop_at" {const ok = [_]tools.ExecResult{.{ .text = "ok\n", .is_error = false }};var i: u8 = 1;while (i < stop_at) : (i += 1) {- try std.testing.expectEqual(i, t.note(fingerprint(&keys, &ok)));- try std.testing.expect(!t.stopped());+ try std.testing.expectEqual(i, t.calls.note(fingerprint(&keys, &ok)));+ try std.testing.expect(t.stopText() == null);}- try std.testing.expectEqual(stop_at, t.note(fingerprint(&keys, &ok)));- try std.testing.expect(t.stopped());+ try std.testing.expectEqual(stop_at, t.calls.note(fingerprint(&keys, &ok)));+ try std.testing.expectEqual(@as(?[]const u8, stop_text), t.stopText());t.reset();- try std.testing.expect(!t.stopped());+ try std.testing.expect(t.stopText() == null);}test "a carried run survives one reset, then resets clear it again" {@@ -114,14 +227,14 @@ test "a carried run survives one reset, then resets clear it again" {const keys = [_][]const u8{"shell\n{\"command\":\"echo ok\"}"};const ok = [_]tools.ExecResult{.{ .text = "ok\n", .is_error = false }};var i: u8 = 1;- while (i < stop_at) : (i += 1) _ = t.note(fingerprint(&keys, &ok));+ while (i < stop_at) : (i += 1) _ = t.calls.note(fingerprint(&keys, &ok));t.carryOver();t.reset();- try std.testing.expectEqual(stop_at, t.note(fingerprint(&keys, &ok)));- try std.testing.expect(t.stopped());+ try std.testing.expectEqual(stop_at, t.calls.note(fingerprint(&keys, &ok)));+ try std.testing.expect(t.stopText() != null);t.reset();- try std.testing.expect(!t.stopped());- try std.testing.expectEqual(@as(u8, 1), t.note(fingerprint(&keys, &ok)));+ try std.testing.expect(t.stopText() == null);+ try std.testing.expectEqual(@as(u8, 1), t.calls.note(fingerprint(&keys, &ok)));}test "a different input, result, or error flag starts a new run" {@@ -131,11 +244,11 @@ test "a different input, result, or error flag starts a new run" {const ok = [_]tools.ExecResult{.{ .text = "ok\n", .is_error = false }};const ok_err = [_]tools.ExecResult{.{ .text = "ok\n", .is_error = true }};const other = [_]tools.ExecResult{.{ .text = "ok 2\n", .is_error = false }};- _ = t.note(fingerprint(&a, &ok));- try std.testing.expectEqual(@as(u8, 2), t.note(fingerprint(&a, &ok)));- try std.testing.expectEqual(@as(u8, 1), t.note(fingerprint(&b, &ok)));- try std.testing.expectEqual(@as(u8, 1), t.note(fingerprint(&b, &other)));- try std.testing.expectEqual(@as(u8, 1), t.note(fingerprint(&b, &ok_err)));+ _ = t.calls.note(fingerprint(&a, &ok));+ try std.testing.expectEqual(@as(u8, 2), t.calls.note(fingerprint(&a, &ok)));+ try std.testing.expectEqual(@as(u8, 1), t.calls.note(fingerprint(&b, &ok)));+ try std.testing.expectEqual(@as(u8, 1), t.calls.note(fingerprint(&b, &other)));+ try std.testing.expectEqual(@as(u8, 1), t.calls.note(fingerprint(&b, &ok_err)));}test "a pending or cancelled result never counts and breaks the run" {@@ -144,11 +257,11 @@ test "a pending or cancelled result never counts and breaks the run" {const done = [_]tools.ExecResult{.{ .text = "still running", .is_error = false }};const waiting = [_]tools.ExecResult{.{ .text = "still running", .is_error = false, .pending = true }};const cancelled = [_]tools.ExecResult{.{ .text = "still running", .is_error = false, .cancelled = true }};- _ = t.note(fingerprint(&keys, &done));- _ = t.note(fingerprint(&keys, &done));- try std.testing.expectEqual(@as(u8, 0), t.note(fingerprint(&keys, &waiting)));- try std.testing.expectEqual(@as(u8, 1), t.note(fingerprint(&keys, &done)));- try std.testing.expectEqual(@as(u8, 0), t.note(fingerprint(&keys, &cancelled)));+ _ = t.calls.note(fingerprint(&keys, &done));+ _ = t.calls.note(fingerprint(&keys, &done));+ try std.testing.expectEqual(@as(u8, 0), t.calls.note(fingerprint(&keys, &waiting)));+ try std.testing.expectEqual(@as(u8, 1), t.calls.note(fingerprint(&keys, &done)));+ try std.testing.expectEqual(@as(u8, 0), t.calls.note(fingerprint(&keys, &cancelled)));}test "polling a job that is still running never counts, a finished job's output does" {@@ -158,12 +271,12 @@ test "polling a job that is still running never counts, a finished job's outputconst parked = [_]tools.ExecResult{.{ .text = "[job 9: running — persistent server; parked in the background. You are notified on exit; do not poll]", .is_error = false }};const exited = [_]tools.ExecResult{.{ .text = "[job 34: exited with code 0]\nok", .is_error = false }};var i: u8 = 0;- while (i < 12) : (i += 1) try std.testing.expectEqual(@as(u8, 0), t.note(fingerprint(&keys, &running)));- while (i < 24) : (i += 1) try std.testing.expectEqual(@as(u8, 0), t.note(fingerprint(&keys, &parked)));- try std.testing.expect(!t.stopped());+ while (i < 12) : (i += 1) try std.testing.expectEqual(@as(u8, 0), t.calls.note(fingerprint(&keys, &running)));+ while (i < 24) : (i += 1) try std.testing.expectEqual(@as(u8, 0), t.calls.note(fingerprint(&keys, &parked)));+ try std.testing.expect(t.stopText() == null);i = 1;- while (i <= stop_at) : (i += 1) _ = t.note(fingerprint(&keys, &exited));- try std.testing.expect(t.stopped());+ while (i <= stop_at) : (i += 1) _ = t.calls.note(fingerprint(&keys, &exited));+ try std.testing.expect(t.stopText() != null);}test "an empty-argument error loop still stops" {@@ -171,8 +284,8 @@ test "an empty-argument error loop still stops" {const keys = [_][]const u8{"read_file\n{}"};const err = [_]tools.ExecResult{.{ .text = "[error] tool call arguments were truncated", .is_error = true }};var i: u8 = 0;- while (i < stop_at) : (i += 1) _ = t.note(fingerprint(&keys, &err));- try std.testing.expect(t.stopped());+ while (i < stop_at) : (i += 1) _ = t.calls.note(fingerprint(&keys, &err));+ try std.testing.expect(t.stopText() != null);}test "observe appends the notice on the nudge batch only" {@@ -181,9 +294,6 @@ test "observe appends the notice on the nudge batch only" {const Fake = struct {arena: std.mem.Allocator,tool_repeat: Tracker = .{},- fn toolDedupeKey(_: *@This(), call: tools.ToolCall) ![]const u8 {- return call.name;- }};var fake: Fake = .{ .arena = arena_state.allocator() };const calls = [_]tools.ToolCall{.{ .id = "c", .name = "shell", .input = .null }};@@ -195,3 +305,101 @@ test "observe appends the notice on the nudge batch only" {try std.testing.expectEqual(i == nudge_at, noticed);}}++test "the same result under reworded calls nudges at 6 and stops at 12" {+ var t: Tracker = .{};+ const names = [_][]const u8{"shell"};+ const pending = [_]tools.ExecResult{.{ .text = "PENDING\n", .is_error = false }};+ const spellings = [_][]const u8{ "cat status.txt", "cat ./status.txt", "cat 'status.txt'", "cat -- status.txt", "cat \"status.txt\"", "cat ././status.txt" };+ var i: u8 = 1;+ while (i <= reworded_stop_at) : (i += 1) {+ const keys = [_][]const u8{spellings[(i - 1) % spellings.len]};+ const notice = t.note(fingerprint(&keys, &pending), resultFingerprint(&names, &pending));+ try std.testing.expectEqual(if (i == reworded_nudge_at) Notice.reworded else Notice.none, notice);+ try std.testing.expectEqual(i == reworded_stop_at, t.stopText() != null);+ }+ try std.testing.expectEqual(@as(?[]const u8, reworded_stop_text), t.stopText());+}++test "identical calls take the call notice and stop, never the reworded one" {+ var t: Tracker = .{};+ const keys = [_][]const u8{"shell\n{\"command\":\"echo ok\"}"};+ const names = [_][]const u8{"shell"};+ const ok = [_]tools.ExecResult{.{ .text = "ok\n", .is_error = false }};+ var i: u8 = 1;+ while (i <= stop_at) : (i += 1) {+ const notice = t.note(fingerprint(&keys, &ok), resultFingerprint(&names, &ok));+ try std.testing.expectEqual(if (i == nudge_at) Notice.repeated else Notice.none, notice);+ }+ try std.testing.expectEqual(@as(?[]const u8, stop_text), t.stopText());+}++test "content-free results under different calls are ordinary work" {+ var t: Tracker = .{};+ const names = [_][]const u8{"shell"};+ const quiet = [_][]const tools.ExecResult{+ &.{.{ .text = "(no output)", .is_error = false }},+ &.{.{ .text = "\n[exit code 1]", .is_error = true }},+ &.{.{ .text = "", .is_error = false }},+ };+ var i: u8 = 0;+ while (i < 3 * reworded_stop_at) : (i += 1) {+ const r = quiet[i % quiet.len];+ try std.testing.expectEqual(@as(?u64, null), resultFingerprint(&names, r));+ const keys = [_][]const u8{if (i % 2 == 0) "grep a" else "grep b"};+ try std.testing.expectEqual(Notice.none, t.note(fingerprint(&keys, r), resultFingerprint(&names, r)));+ }+ try std.testing.expect(t.stopText() == null);+ const noisy = [_]tools.ExecResult{.{ .text = "boom\n[exit code 1]", .is_error = true }};+ try std.testing.expect(resultFingerprint(&names, &noisy) != null);+}++test "observe appends the reworded notice when only the wording changes" {+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena_state.deinit();+ const arena = arena_state.allocator();+ const Fake = struct {+ arena: std.mem.Allocator,+ tool_repeat: Tracker = .{},+ };+ var fake: Fake = .{ .arena = arena };+ const dots = "./././././././.";+ var i: u8 = 1;+ while (i <= reworded_nudge_at + 1) : (i += 1) {+ const input = try std.json.parseFromSliceLeaky(std.json.Value, arena, try std.fmt.allocPrint(arena, "{{\"command\":\"cat {s}status.txt\"}}", .{dots[0 .. 2 * @as(usize, i)]}), .{});+ const calls = [_]tools.ToolCall{.{ .id = "c", .name = "shell", .input = input }};+ var results = [_]tools.ExecResult{.{ .text = "PENDING\n", .is_error = false }};+ try observe(&fake, &calls, &results);+ try std.testing.expectEqual(i == reworded_nudge_at, std.mem.indexOf(u8, results[0].text, reworded_nudge_text) != null);+ try std.testing.expect(std.mem.indexOf(u8, results[0].text, nudge_text) == null);+ }+}++test "the key ignores knobs and spacing, not what the call does" {+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena_state.deinit();+ const arena = arena_state.allocator();+ const inputs = [_][]const u8{+ "{\"command\":\"cat status.txt\"}",+ "{\"command\":\"cat status.txt\"}",+ "{\"command\":\"cat status.txt\",\"timeout\":1000}",+ "{\"command\":\" cat status.txt\\n\",\"timeout\":1001,\"description\":\"poll\"}",+ };+ var keys: [inputs.len][]const u8 = undefined;+ for (inputs, &keys) |text, *key| {+ const v = try std.json.parseFromSliceLeaky(std.json.Value, arena, text, .{});+ key.* = try callKey(arena, .{ .id = "c", .name = "shell", .input = v });+ }+ for (keys[1..]) |key| try std.testing.expectEqualStrings(keys[0], key);+ const differ = [_][]const u8{+ "{\"command\":\"cat other.txt\"}",+ "{\"command\":\"cat status.txt\",\"run_in_background\":true}",+ "{\"command\":\"cat status.txt\",\"opts\":{\"timeout\":1}}",+ };+ for (differ) |text| {+ const v = try std.json.parseFromSliceLeaky(std.json.Value, arena, text, .{});+ try std.testing.expect(!std.mem.eql(u8, keys[0], try callKey(arena, .{ .id = "c", .name = "shell", .input = v })));+ }+ const read = try std.json.parseFromSliceLeaky(std.json.Value, arena, inputs[0], .{});+ try std.testing.expect(!std.mem.eql(u8, keys[0], try callKey(arena, .{ .id = "c", .name = "read_file", .input = read })));+}