graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Trust the provider's count for the context meter
Rach Pradhan01620eb6f902
diff --git a/evals/harness_behavior.jsonl b/evals/harness_behavior.jsonlindex 73188640..9b1d2263 100644--- a/evals/harness_behavior.jsonl+++ b/evals/harness_behavior.jsonl@@ -8,7 +8,7 @@{"id": "ambiguous-asks-instead-of-guessing", "why": "#106: an ambiguous request has to reach the human as an ask_user event and come back carrying THEIR words. The regression is a harness that buries the question in an ordinary tool_call the client cannot answer, or that answers it itself and lets the run continue on the guess.", "args": [], "prompt": "add caching", "controls_after_turn": [{"type": "answer", "text": "an external store, and cap it at 500 entries"}], "script": [{"tool": "ask_user", "arguments": {"question": "Should the cache live in process memory or in an external store?", "options": ["process memory", "external store"]}}, {"text": "using the external store, capped as you asked"}], "assert": [{"event": {"type": "ask_user", "question": "Should the cache live in process memory or in an external store?"}}, {"events_at_most": {"match": {"type": "tool_call", "name": "ask_user"}, "count": 0}}, {"event": {"type": "tool_result", "name": "ask_user", "is_error": false, "text": "an external store, and cap it at 500 entries"}}, {"request_contains": {"index": -1, "text": "cap it at 500 entries"}}, {"no_event": {"type": "tool_call", "name": "write_file"}}, {"final_text_contains": "using the external store"}]}{"id": "two-file-edit-lands-both", "why": "#106: a two-file change has to land on disk, not merely be reported. The second edit_file only matches if the first write really happened, and the closing bash check prints its marker only when both files hold the new text - a reported-but-absent edit fails the case instead of passing it.", "args": [], "env": {"GRAFF_NO_CODEDB_GUARD": "1"}, "prompt": "raise the timeout to 60 and mark the import in the client", "script": [{"tool": "write_file", "arguments": {"path": "settings.py", "content": "TIMEOUT = 30\n"}}, {"tool": "write_file", "arguments": {"path": "client.py", "content": "from settings import TIMEOUT\n"}}, {"tool": "edit_file", "arguments": {"path": "settings.py", "old_string": "TIMEOUT = 30", "new_string": "TIMEOUT = 60"}}, {"tool": "edit_file", "arguments": {"path": "client.py", "old_string": "from settings import TIMEOUT", "new_string": "from settings import TIMEOUT # noqa"}}, {"tool": "bash", "arguments": {"command": "grep -q 'TIMEOUT = 60' settings.py && grep -q noqa client.py && echo both-edits-on-disk"}}, {"text": "both files are updated"}], "assert": [{"events_at_least": {"match": {"type": "tool_call", "name": "edit_file"}, "count": 2}}, {"events_at_most": {"match": {"type": "tool_call_finished", "is_error": true}, "count": 0}}, {"event": {"type": "tool_result", "name": "bash", "text": "both-edits-on-disk\n"}}, {"request_contains": {"index": -1, "text": "settings.py"}}, {"request_contains": {"index": -1, "text": "client.py"}}, {"final_text_contains": "both files are updated"}]}{"id": "failed-bash-is-not-reported-as-success", "why": "#106: a command that fails must come back marked is_error with its exit status and stderr intact, so the next turn cannot mistake it for success. The regression is a swallowed failure - a non-zero exit finishing as an ok tool_call, or output trimmed away before the model sees why it broke.", "args": [], "prompt": "run the config check", "script": [{"tool": "bash", "arguments": {"command": "ls missing_config.yaml; exit 7"}}, {"text": "the config check failed, so I am not claiming it passed"}], "assert": [{"event": {"type": "tool_call_finished", "name": "bash", "is_error": true}}, {"events_at_most": {"match": {"type": "tool_call_finished", "name": "bash", "is_error": false}, "count": 0}}, {"request_contains": {"index": -1, "text": "[exit code 7]"}}, {"request_contains": {"index": -1, "text": "No such file or directory"}}, {"final_text_contains": "the config check failed"}]}-{"id": "subagent-compaction-pins-task", "why": "G2/H2: compact() must call pinChildTask before it summarizes a subagent's history, or a compacted child loses its own mandate. No unit test can prove the wiring (compact() hits the network right after the prelude), so this drives a real child through auto-compaction against the scripted model and checks the request sent right after: it must still carry the child's task, restated verbatim, not just the model's summary.", "args": [], "env": {"GRAFF_COMPACT_PCT": "5"}, "prompt": "spawn a subagent to survey the payments module", "script": [{"tool": "subagent", "arguments": {"description": "survey payments", "prompt": "SENTINEL-TASK-9f2c: read every file under payments/ and list the exported functions", "isolation": "shared_cwd"}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c0'*2000)\""}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c1'*2000)\""}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c2'*2000)\""}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c3'*2000)\""}}, {"text": "the payments module has been surveyed so far"}, {"text": "functions listed: chargeCard, refund, listInvoices"}, {"text": "the subagent found chargeCard, refund, and listInvoices"}], "assert": [{"event": {"type": "tool_call", "name": "subagent"}}, {"event": {"type": "tool_result", "name": "subagent", "is_error": false}}, {"request_contains": {"index": 6, "text": "SENTINEL-TASK-9f2c"}}, {"request_contains": {"index": 6, "text": "restated verbatim"}}, {"final_text_contains": "the subagent found chargeCard, refund, and listInvoices"}]}+{"id": "subagent-compaction-pins-task", "why": "G2/H2: compact() must call pinChildTask before it summarizes a subagent's history, or a compacted child loses its own mandate. No unit test can prove the wiring (compact() hits the network right after the prelude), so this drives a real child through auto-compaction against the scripted model and checks the request sent right after: it must still carry the child's task, restated verbatim, not just the model's summary. The tool outputs are sized so the child's real context (bytes/4, with the provider's count trusted) crosses the 5% line at the same request the scripted summary expects.", "args": [], "env": {"GRAFF_COMPACT_PCT": "5"}, "prompt": "spawn a subagent to survey the payments module", "script": [{"tool": "subagent", "arguments": {"description": "survey payments", "prompt": "SENTINEL-TASK-9f2c: read every file under payments/ and list the exported functions", "isolation": "shared_cwd"}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c0'*3500)\""}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c1'*3500)\""}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c2'*3500)\""}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c3'*3500)\""}}, {"text": "the payments module has been surveyed so far"}, {"text": "functions listed: chargeCard, refund, listInvoices"}, {"text": "the subagent found chargeCard, refund, and listInvoices"}], "assert": [{"event": {"type": "tool_call", "name": "subagent"}}, {"event": {"type": "tool_result", "name": "subagent", "is_error": false}}, {"request_contains": {"index": 6, "text": "SENTINEL-TASK-9f2c"}}, {"request_contains": {"index": 6, "text": "restated verbatim"}}, {"final_text_contains": "the subagent found chargeCard, refund, and listInvoices"}]}{"id": "escalation-r0-declines-then-r0d-revision", "why": "escalation R0d: a 1-file bugfix must be DECLINED solo (the 0.0.237 study's 30-calls-no-edit regression), and the SECOND ask of the same class must get the R0d revision prescription — evidence-carrying sequential revision before any fleet. The regression is a fleet on first ask, or a flat second decline that never prescribes the revision.", "args": [], "prompt": "ultracode fix the bug in stats.py", "script": [{"tool": "workflow", "arguments": {"phases": [{"title": "find the defect", "tasks": [{"description": "find", "prompt": "read stats.py and locate the defect"}]}, {"title": "implement the fix", "tasks": [{"description": "fix", "prompt": "fix the defect in stats.py"}]}]}}, {"text": "declined by the ladder — I made the edit solo but the tests still fail"}, {"tool": "workflow", "arguments": {"phases": [{"title": "find the defect", "tasks": [{"description": "find", "prompt": "read stats.py and locate the defect"}]}, {"title": "implement the fix", "tasks": [{"description": "fix", "prompt": "fix the defect in stats.py"}]}]}}, {"text": "revising with deeper analysis as prescribed"}], "second_prompt": "the fix did not take — pytest still fails in stats.py", "assert": [{"event": {"type": "tool_result", "name": "workflow", "is_error": false}}, {"request_contains": {"index": 1, "text": "(escalation R0):"}}, {"request_contains": {"index": 1, "text": "Do it yourself"}}, {"request_lacks": {"index": 1, "text": "(escalation R0d):"}}, {"request_contains": {"index": -1, "text": "(escalation R0d):"}}, {"request_contains": {"index": -1, "text": "revision case"}}, {"request_contains": {"index": -1, "text": "second failure earns the fleet"}}, {"final_text_contains": "revising with deeper analysis"}]}{"id": "escalation-no-verifier-no-revision-loop", "why": "Weng's revision gate (failure_evidence.verifierFor): an `other`-class ask has no verifier — no diff contract, no eval, no citable anchors — so a repeat failure must NOT be prescribed a solo revision; R0d stays shut and with the fleet unaffordable the decline is the plain R0 advisory. The regression is a revision loop spun without external feedback.", "args": ["--max-model-calls", "8"], "prompt": "polish the onboarding copy in welcome.txt", "script": [{"tool": "workflow", "arguments": {"phases": [{"title": "find problems", "tasks": [{"description": "find", "prompt": "read welcome.txt and list the problems"}]}, {"title": "verify each finding", "tasks": [{"description": "verify", "prompt": "verify the problems in welcome.txt"}]}]}}, {"text": "polished it solo as told"}, {"tool": "workflow", "arguments": {"phases": [{"title": "find problems", "tasks": [{"description": "find", "prompt": "read welcome.txt and list the problems"}]}, {"title": "verify each finding", "tasks": [{"description": "verify", "prompt": "verify the problems in welcome.txt"}]}]}}, {"text": "continuing the polish myself"}], "second_prompt": "the onboarding copy in welcome.txt still reads badly — go again", "assert": [{"request_contains": {"index": 1, "text": "(escalation R0):"}}, {"request_contains": {"index": -1, "text": "(escalation R0):"}}, {"request_lacks": {"index": -1, "text": "(escalation R0d):"}}, {"request_lacks": {"index": -1, "text": "revision case"}}, {"final_text_contains": "continuing the polish myself"}]}{"id": "review-repeat-failure-gets-anchor-revision", "why": "report_anchors gives review/research a real verifier (cited paths must resolve against the tree), so a review-class repeat failure now earns the R0d revision prescription instead of jumping to the judge fleet — the anchors are the external feedback Weng's gate demands.", "args": [], "prompt": "review the findings in report.py", "script": [{"tool": "workflow", "arguments": {"phases": [{"title": "find problems", "tasks": [{"description": "find", "prompt": "read report.py and list the problems"}]}, {"title": "verify each finding", "tasks": [{"description": "verify", "prompt": "verify the problems in report.py"}]}]}}, {"text": "reviewed solo as told"}, {"tool": "workflow", "arguments": {"phases": [{"title": "find problems", "tasks": [{"description": "find", "prompt": "read report.py and list the problems"}]}, {"title": "verify each finding", "tasks": [{"description": "verify", "prompt": "verify the problems in report.py"}]}]}}, {"text": "re-reviewing against the cited code as prescribed"}], "second_prompt": "that review missed real problems in report.py — go again", "assert": [{"request_contains": {"index": 1, "text": "(escalation R0):"}}, {"request_lacks": {"index": 1, "text": "(escalation R0d):"}}, {"request_contains": {"index": -1, "text": "(escalation R0d):"}}, {"request_contains": {"index": -1, "text": "revision case"}}, {"final_text_contains": "re-reviewing against the cited code"}]}diff --git a/src/agent_context.zig b/src/agent_context.zigindex 10403081..6b2638b5 100644--- a/src/agent_context.zig+++ b/src/agent_context.zig@@ -54,7 +54,7 @@ test "recordUsage (#202): floors the meter from the local estimate when usage isvar response: std.json.ObjectMap = .empty;try response.put(a, "usage", .{ .object = usage });recordUsage(&agent, response, 400);- try std.testing.expectEqual(@max(fullRequestEstimateTokens(&agent), @as(u64, 15)), agent.last_context_tokens);+ try std.testing.expectEqual(@max(measuredRequestTokens(&agent, context_tokens.serializedLen(Value{ .array = agent.messages })), @as(u64, 15)), agent.last_context_tokens);// A malformed smaller total must not override stronger component evidence.try usage.put(a, "prompt_tokens", .{ .integer = 220_000 });@@ -172,13 +172,13 @@ pub fn floorContextTokens(self: *Agent, est: u64) void {self.context_local_tokens = estimate.local;}-/// A full-history request's nonzero provider usage is authoritative enough to-/// correct a stale high meter after an explicit trim/provider mutation. Still-/// retain the complete local request estimate as a structural lower bound.+/// A full-history request's provider usage replaces the local estimate, whose 8k+/// pre-usage baseline read ~3k high on Anthropic. Plain bytes/4 under-counts code+/// on every provider measured, so a report below it is not credible and floors there.pub fn replaceContextTokens(self: *Agent, reported: u64) void {- const local = self.fullRequestEstimateTokens();- self.last_context_tokens = @max(local, reported);- self.context_local_tokens = local;+ const input_bytes = context_tokens.serializedLen(Value{ .array = self.messages });+ self.last_context_tokens = @max(reported, measuredRequestTokens(self, input_bytes));+ self.context_local_tokens = requestEstimateFromInputBytes(self, input_bytes);}/// An integer usage field, or 0 if absent / wrong type.@@ -208,17 +208,19 @@ pub fn fullInputEstimateTokens(self: *Agent) u64 {return context_tokens.estimatedTokens(Value{ .array = self.messages });}-/// Estimate the complete fixed+history input sent by a normal model request.-/// fullInputEstimateTokens intentionally covers messages only; the pre-send gate-/// must additionally count the mutable system prompt and active tool schema. Keep-/// the old 8k conservative baseline when those serialize smaller, but never let it-/// hide an unbounded set_system_prompt/set_agent payload.+/// The complete fixed+history input of a normal request (fullInputEstimateTokens is+/// messages only): system prompt and tool schema too, with the 8k conservative+/// baseline when those serialize smaller, never hiding a large set_system_prompt.fn requestEstimateFromInputBytes(self: *Agent, input_bytes: usize) u64 {const input_tokens = context_tokens.estimatedTokensFromLen(Value{ .array = self.messages }, input_bytes);- const fixed_bytes = context_tokens.serializedLen(self.systemPrompt()) +| self.toolsJson().len;- const measured = input_tokens +| @as(u64, @intCast(fixed_bytes / 4));const baseline = @min(@as(u64, 8000), self.provider.context / 8);- return @max(measured, input_tokens +| baseline);+ return @max(measuredRequestTokens(self, input_bytes), input_tokens +| baseline);+}++/// bytes/4 of history + system prompt + tool schema, with no baseline.+pub fn measuredRequestTokens(self: *Agent, input_bytes: usize) u64 {+ const fixed_bytes = context_tokens.serializedLen(self.systemPrompt()) +| self.toolsJson().len;+ return context_tokens.estimatedTokensFromLen(Value{ .array = self.messages }, input_bytes) +| @as(u64, @intCast(fixed_bytes / 4));}pub fn fullRequestEstimateTokens(self: *Agent) u64 {@@ -245,10 +247,8 @@ pub fn contextEstimate(self: *Agent) ContextEstimate {/// Reusing the observed byte count avoids a redundant full JSON walk.pub fn contextEstimateFromInputBytes(self: *Agent, input_bytes: usize) ContextEstimate {const local = requestEstimateFromInputBytes(self, input_bytes);- return .{- .local = local,- .effective = local +| (self.last_context_tokens -| self.context_local_tokens),- };+ const last = self.last_context_tokens; // a provider sample may sit below the local estimate+ return .{ .local = local, .effective = if (last == 0 or last >= self.context_local_tokens) local +| (last - @min(last, self.context_local_tokens)) else last +| (local -| self.context_local_tokens) };}/// Current context occupancy: today's locally measurable request plus the@@ -264,9 +264,7 @@ pub fn effectiveContextTokens(self: *Agent) u64 {/// advancing only the local anchor prevents counting them twice.pub fn pairContextMeterWithCurrentLocal(self: *Agent) void {if (!self.last_usage_includes_output) return;- const local = self.fullRequestEstimateTokens();- self.last_context_tokens = @max(self.last_context_tokens, local);- self.context_local_tokens = local;+ self.context_local_tokens = self.fullRequestEstimateTokens();self.last_usage_includes_output = false;}@@ -279,6 +277,7 @@ pub fn rebaseContextMeter(self: *Agent) void {}test "rebaseContextMeter preserves hidden context while replacing local input" {+ _ = @import("agent_context_meter_tests.zig"); // the provider-count meter tests ride alongvar arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);defer arena_state.deinit();const a = arena_state.allocator();@@ -533,9 +532,9 @@ test "recordUsageResponses: usage fallbacks never lower an authoritative meter"try smaller_usage.put(a, "total_tokens", .{ .integer = 100 });var smaller_response: std.json.ObjectMap = .empty;try smaller_response.put(a, "usage", .{ .object = smaller_usage });- // A full re-anchor trusts its provider reading (with the local floor).+ // A full re-anchor is its provider reading; the pre-usage floor no longer lifts it.recordUsageResponses(&agent, smaller_response, 400);- try std.testing.expectEqual(@max(fullRequestEstimateTokens(&agent), @as(u64, 100)), agent.last_context_tokens);+ try std.testing.expectEqual(@as(u64, 100), agent.last_context_tokens);try std.testing.expect(agent.last_usage_includes_output);// The aggregate is advisory when larger input/output components arediff --git a/src/agent_context_meter_tests.zig b/src/agent_context_meter_tests.zignew file mode 100644index 00000000..3116070e--- /dev/null+++ b/src/agent_context_meter_tests.zig@@ -0,0 +1,104 @@+//! The context meter trusts a provider's count of a full request (agent_context.zig).++const std = @import("std");+const util = @import("util.zig");+const Agent = @import("agent.zig").Agent;+const messages_mod = @import("messages.zig");+const agent_context = @import("agent_context.zig");++/// What a real provider reports: more than bytes/4, since code and JSON+/// tokenize at fewer than four bytes a token.+fn providerCount(agent: *Agent) u64 {+ return agent_context.measuredRequestTokens(agent, @import("context_tokens.zig").serializedLen(std.json.Value{ .array = agent.messages })) + 1_000;+}++fn anthropicAgent(a: std.mem.Allocator) Agent {+ var agent: Agent = undefined;+ agent.io = std.testing.io;+ agent.arena = a;+ agent.messages = std.json.Array.init(a);+ agent.last_context_tokens = 0;+ agent.context_local_tokens = 0;+ agent.last_usage_includes_output = false;+ agent.provider = .{ .id = "anthropic", .kind = .anthropic, .auth = .x_api_key, .url = "", .api_key = "", .model = "claude", .context = 200_000 };+ agent.sub = false;+ agent.strict = false;+ agent.sys_normal = &util.repeatBytes("s", 6_000);+ agent.sys_strict = "";+ agent.tools_anthropic = &util.repeatBytes("t", 7_000);+ return agent;+}++test "before any usage the meter keeps the conservative local estimate" {+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena_state.deinit();+ var agent = anthropicAgent(arena_state.allocator());+ try agent.messages.append(try messages_mod.textMessage(arena_state.allocator(), "user", "hi"));+ // 13 KB of system + tools is ~3.3k by bytes/4; the pre-usage baseline holds it at 8k.+ try std.testing.expect(agent.fullRequestEstimateTokens() >= 8_000);+ try std.testing.expectEqual(agent.fullRequestEstimateTokens(), agent.effectiveContextTokens());+}++test "a provider count below the local estimate is the meter" {+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena_state.deinit();+ const a = arena_state.allocator();+ var agent = anthropicAgent(a);+ try agent.messages.append(try messages_mod.textMessage(a, "user", "hi"));+ const local = agent.fullRequestEstimateTokens();+ const reported = providerCount(&agent);+ agent_context.replaceContextTokens(&agent, reported);+ try std.testing.expect(local > reported);+ try std.testing.expectEqual(reported, agent.effectiveContextTokens());+}++test "a report below plain bytes/4 is not credible and floors there" {+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena_state.deinit();+ const a = arena_state.allocator();+ var agent = anthropicAgent(a);+ try agent.messages.append(try messages_mod.textMessage(a, "user", "hi"));+ const floor = providerCount(&agent) - 1_000;+ agent_context.replaceContextTokens(&agent, 12); // a server that under-reports+ try std.testing.expectEqual(floor, agent.effectiveContextTokens());+}++test "growth after the sample adds to the provider count" {+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena_state.deinit();+ const a = arena_state.allocator();+ var agent = anthropicAgent(a);+ try agent.messages.append(try messages_mod.textMessage(a, "user", "hi"));+ const reported = providerCount(&agent);+ agent_context.replaceContextTokens(&agent, reported);+ const anchor = agent.context_local_tokens;+ try agent.messages.append(try messages_mod.textMessage(a, "user", &util.repeatBytes("x", 4_000)));+ const grown = agent.fullRequestEstimateTokens() - anchor;+ try std.testing.expect(grown >= 900);+ try std.testing.expectEqual(reported + grown, agent.effectiveContextTokens());+}++test "pairing provider-covered output does not re-floor at the local estimate" {+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena_state.deinit();+ const a = arena_state.allocator();+ var agent = anthropicAgent(a);+ try agent.messages.append(try messages_mod.textMessage(a, "user", "hi"));+ const reported = providerCount(&agent) + 200; // input + the reply's output tokens+ agent_context.replaceContextTokens(&agent, reported);+ agent.last_usage_includes_output = true;+ try agent.messages.append(try messages_mod.textMessage(a, "assistant", &util.repeatBytes("y", 600)));+ agent.pairContextMeterWithCurrentLocal();+ try std.testing.expectEqual(reported, agent.effectiveContextTokens());+}++test "a cleared sample falls back to the local estimate even with a stale anchor" {+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena_state.deinit();+ const a = arena_state.allocator();+ var agent = anthropicAgent(a);+ try agent.messages.append(try messages_mod.textMessage(a, "user", "hi"));+ agent_context.replaceContextTokens(&agent, providerCount(&agent));+ agent.last_context_tokens = 0; // what compaction does+ try std.testing.expectEqual(agent.fullRequestEstimateTokens(), agent.effectiveContextTokens());+}