Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Trust the provider's count for the context meter

Rach Pradhan01620eb6f902
diff --git a/evals/harness_behavior.jsonl b/evals/harness_behavior.jsonl
index 73188640..9b1d2263 100644
--- a/evals/harness_behavior.jsonl
+++ b/evals/harness_behavior.jsonl
@@ -8,7 +8,7 @@
{"id": "ambiguous-asks-instead-of-guessing", "why": "#106: an ambiguous request has to reach the human as an ask_user event and come back carrying THEIR words. The regression is a harness that buries the question in an ordinary tool_call the client cannot answer, or that answers it itself and lets the run continue on the guess.", "args": [], "prompt": "add caching", "controls_after_turn": [{"type": "answer", "text": "an external store, and cap it at 500 entries"}], "script": [{"tool": "ask_user", "arguments": {"question": "Should the cache live in process memory or in an external store?", "options": ["process memory", "external store"]}}, {"text": "using the external store, capped as you asked"}], "assert": [{"event": {"type": "ask_user", "question": "Should the cache live in process memory or in an external store?"}}, {"events_at_most": {"match": {"type": "tool_call", "name": "ask_user"}, "count": 0}}, {"event": {"type": "tool_result", "name": "ask_user", "is_error": false, "text": "an external store, and cap it at 500 entries"}}, {"request_contains": {"index": -1, "text": "cap it at 500 entries"}}, {"no_event": {"type": "tool_call", "name": "write_file"}}, {"final_text_contains": "using the external store"}]}
{"id": "two-file-edit-lands-both", "why": "#106: a two-file change has to land on disk, not merely be reported. The second edit_file only matches if the first write really happened, and the closing bash check prints its marker only when both files hold the new text - a reported-but-absent edit fails the case instead of passing it.", "args": [], "env": {"GRAFF_NO_CODEDB_GUARD": "1"}, "prompt": "raise the timeout to 60 and mark the import in the client", "script": [{"tool": "write_file", "arguments": {"path": "settings.py", "content": "TIMEOUT = 30\n"}}, {"tool": "write_file", "arguments": {"path": "client.py", "content": "from settings import TIMEOUT\n"}}, {"tool": "edit_file", "arguments": {"path": "settings.py", "old_string": "TIMEOUT = 30", "new_string": "TIMEOUT = 60"}}, {"tool": "edit_file", "arguments": {"path": "client.py", "old_string": "from settings import TIMEOUT", "new_string": "from settings import TIMEOUT # noqa"}}, {"tool": "bash", "arguments": {"command": "grep -q 'TIMEOUT = 60' settings.py && grep -q noqa client.py && echo both-edits-on-disk"}}, {"text": "both files are updated"}], "assert": [{"events_at_least": {"match": {"type": "tool_call", "name": "edit_file"}, "count": 2}}, {"events_at_most": {"match": {"type": "tool_call_finished", "is_error": true}, "count": 0}}, {"event": {"type": "tool_result", "name": "bash", "text": "both-edits-on-disk\n"}}, {"request_contains": {"index": -1, "text": "settings.py"}}, {"request_contains": {"index": -1, "text": "client.py"}}, {"final_text_contains": "both files are updated"}]}
{"id": "failed-bash-is-not-reported-as-success", "why": "#106: a command that fails must come back marked is_error with its exit status and stderr intact, so the next turn cannot mistake it for success. The regression is a swallowed failure - a non-zero exit finishing as an ok tool_call, or output trimmed away before the model sees why it broke.", "args": [], "prompt": "run the config check", "script": [{"tool": "bash", "arguments": {"command": "ls missing_config.yaml; exit 7"}}, {"text": "the config check failed, so I am not claiming it passed"}], "assert": [{"event": {"type": "tool_call_finished", "name": "bash", "is_error": true}}, {"events_at_most": {"match": {"type": "tool_call_finished", "name": "bash", "is_error": false}, "count": 0}}, {"request_contains": {"index": -1, "text": "[exit code 7]"}}, {"request_contains": {"index": -1, "text": "No such file or directory"}}, {"final_text_contains": "the config check failed"}]}
-{"id": "subagent-compaction-pins-task", "why": "G2/H2: compact() must call pinChildTask before it summarizes a subagent's history, or a compacted child loses its own mandate. No unit test can prove the wiring (compact() hits the network right after the prelude), so this drives a real child through auto-compaction against the scripted model and checks the request sent right after: it must still carry the child's task, restated verbatim, not just the model's summary.", "args": [], "env": {"GRAFF_COMPACT_PCT": "5"}, "prompt": "spawn a subagent to survey the payments module", "script": [{"tool": "subagent", "arguments": {"description": "survey payments", "prompt": "SENTINEL-TASK-9f2c: read every file under payments/ and list the exported functions", "isolation": "shared_cwd"}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c0'*2000)\""}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c1'*2000)\""}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c2'*2000)\""}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c3'*2000)\""}}, {"text": "the payments module has been surveyed so far"}, {"text": "functions listed: chargeCard, refund, listInvoices"}, {"text": "the subagent found chargeCard, refund, and listInvoices"}], "assert": [{"event": {"type": "tool_call", "name": "subagent"}}, {"event": {"type": "tool_result", "name": "subagent", "is_error": false}}, {"request_contains": {"index": 6, "text": "SENTINEL-TASK-9f2c"}}, {"request_contains": {"index": 6, "text": "restated verbatim"}}, {"final_text_contains": "the subagent found chargeCard, refund, and listInvoices"}]}
+{"id": "subagent-compaction-pins-task", "why": "G2/H2: compact() must call pinChildTask before it summarizes a subagent's history, or a compacted child loses its own mandate. No unit test can prove the wiring (compact() hits the network right after the prelude), so this drives a real child through auto-compaction against the scripted model and checks the request sent right after: it must still carry the child's task, restated verbatim, not just the model's summary. The tool outputs are sized so the child's real context (bytes/4, with the provider's count trusted) crosses the 5% line at the same request the scripted summary expects.", "args": [], "env": {"GRAFF_COMPACT_PCT": "5"}, "prompt": "spawn a subagent to survey the payments module", "script": [{"tool": "subagent", "arguments": {"description": "survey payments", "prompt": "SENTINEL-TASK-9f2c: read every file under payments/ and list the exported functions", "isolation": "shared_cwd"}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c0'*3500)\""}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c1'*3500)\""}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c2'*3500)\""}}, {"tool": "bash", "arguments": {"command": "python3 -c \"print('c3'*3500)\""}}, {"text": "the payments module has been surveyed so far"}, {"text": "functions listed: chargeCard, refund, listInvoices"}, {"text": "the subagent found chargeCard, refund, and listInvoices"}], "assert": [{"event": {"type": "tool_call", "name": "subagent"}}, {"event": {"type": "tool_result", "name": "subagent", "is_error": false}}, {"request_contains": {"index": 6, "text": "SENTINEL-TASK-9f2c"}}, {"request_contains": {"index": 6, "text": "restated verbatim"}}, {"final_text_contains": "the subagent found chargeCard, refund, and listInvoices"}]}
{"id": "escalation-r0-declines-then-r0d-revision", "why": "escalation R0d: a 1-file bugfix must be DECLINED solo (the 0.0.237 study's 30-calls-no-edit regression), and the SECOND ask of the same class must get the R0d revision prescription — evidence-carrying sequential revision before any fleet. The regression is a fleet on first ask, or a flat second decline that never prescribes the revision.", "args": [], "prompt": "ultracode fix the bug in stats.py", "script": [{"tool": "workflow", "arguments": {"phases": [{"title": "find the defect", "tasks": [{"description": "find", "prompt": "read stats.py and locate the defect"}]}, {"title": "implement the fix", "tasks": [{"description": "fix", "prompt": "fix the defect in stats.py"}]}]}}, {"text": "declined by the ladder — I made the edit solo but the tests still fail"}, {"tool": "workflow", "arguments": {"phases": [{"title": "find the defect", "tasks": [{"description": "find", "prompt": "read stats.py and locate the defect"}]}, {"title": "implement the fix", "tasks": [{"description": "fix", "prompt": "fix the defect in stats.py"}]}]}}, {"text": "revising with deeper analysis as prescribed"}], "second_prompt": "the fix did not take — pytest still fails in stats.py", "assert": [{"event": {"type": "tool_result", "name": "workflow", "is_error": false}}, {"request_contains": {"index": 1, "text": "(escalation R0):"}}, {"request_contains": {"index": 1, "text": "Do it yourself"}}, {"request_lacks": {"index": 1, "text": "(escalation R0d):"}}, {"request_contains": {"index": -1, "text": "(escalation R0d):"}}, {"request_contains": {"index": -1, "text": "revision case"}}, {"request_contains": {"index": -1, "text": "second failure earns the fleet"}}, {"final_text_contains": "revising with deeper analysis"}]}
{"id": "escalation-no-verifier-no-revision-loop", "why": "Weng's revision gate (failure_evidence.verifierFor): an `other`-class ask has no verifier — no diff contract, no eval, no citable anchors — so a repeat failure must NOT be prescribed a solo revision; R0d stays shut and with the fleet unaffordable the decline is the plain R0 advisory. The regression is a revision loop spun without external feedback.", "args": ["--max-model-calls", "8"], "prompt": "polish the onboarding copy in welcome.txt", "script": [{"tool": "workflow", "arguments": {"phases": [{"title": "find problems", "tasks": [{"description": "find", "prompt": "read welcome.txt and list the problems"}]}, {"title": "verify each finding", "tasks": [{"description": "verify", "prompt": "verify the problems in welcome.txt"}]}]}}, {"text": "polished it solo as told"}, {"tool": "workflow", "arguments": {"phases": [{"title": "find problems", "tasks": [{"description": "find", "prompt": "read welcome.txt and list the problems"}]}, {"title": "verify each finding", "tasks": [{"description": "verify", "prompt": "verify the problems in welcome.txt"}]}]}}, {"text": "continuing the polish myself"}], "second_prompt": "the onboarding copy in welcome.txt still reads badly — go again", "assert": [{"request_contains": {"index": 1, "text": "(escalation R0):"}}, {"request_contains": {"index": -1, "text": "(escalation R0):"}}, {"request_lacks": {"index": -1, "text": "(escalation R0d):"}}, {"request_lacks": {"index": -1, "text": "revision case"}}, {"final_text_contains": "continuing the polish myself"}]}
{"id": "review-repeat-failure-gets-anchor-revision", "why": "report_anchors gives review/research a real verifier (cited paths must resolve against the tree), so a review-class repeat failure now earns the R0d revision prescription instead of jumping to the judge fleet — the anchors are the external feedback Weng's gate demands.", "args": [], "prompt": "review the findings in report.py", "script": [{"tool": "workflow", "arguments": {"phases": [{"title": "find problems", "tasks": [{"description": "find", "prompt": "read report.py and list the problems"}]}, {"title": "verify each finding", "tasks": [{"description": "verify", "prompt": "verify the problems in report.py"}]}]}}, {"text": "reviewed solo as told"}, {"tool": "workflow", "arguments": {"phases": [{"title": "find problems", "tasks": [{"description": "find", "prompt": "read report.py and list the problems"}]}, {"title": "verify each finding", "tasks": [{"description": "verify", "prompt": "verify the problems in report.py"}]}]}}, {"text": "re-reviewing against the cited code as prescribed"}], "second_prompt": "that review missed real problems in report.py — go again", "assert": [{"request_contains": {"index": 1, "text": "(escalation R0):"}}, {"request_lacks": {"index": 1, "text": "(escalation R0d):"}}, {"request_contains": {"index": -1, "text": "(escalation R0d):"}}, {"request_contains": {"index": -1, "text": "revision case"}}, {"final_text_contains": "re-reviewing against the cited code"}]}
diff --git a/src/agent_context.zig b/src/agent_context.zig
index 10403081..6b2638b5 100644
--- a/src/agent_context.zig
+++ b/src/agent_context.zig
@@ -54,7 +54,7 @@ test "recordUsage (#202): floors the meter from the local estimate when usage is
var response: std.json.ObjectMap = .empty;
try response.put(a, "usage", .{ .object = usage });
recordUsage(&agent, response, 400);
- try std.testing.expectEqual(@max(fullRequestEstimateTokens(&agent), @as(u64, 15)), agent.last_context_tokens);
+ try std.testing.expectEqual(@max(measuredRequestTokens(&agent, context_tokens.serializedLen(Value{ .array = agent.messages })), @as(u64, 15)), agent.last_context_tokens);
// A malformed smaller total must not override stronger component evidence.
try usage.put(a, "prompt_tokens", .{ .integer = 220_000 });
@@ -172,13 +172,13 @@ pub fn floorContextTokens(self: *Agent, est: u64) void {
self.context_local_tokens = estimate.local;
}
-/// A full-history request's nonzero provider usage is authoritative enough to
-/// correct a stale high meter after an explicit trim/provider mutation. Still
-/// retain the complete local request estimate as a structural lower bound.
+/// A full-history request's provider usage replaces the local estimate, whose 8k
+/// pre-usage baseline read ~3k high on Anthropic. Plain bytes/4 under-counts code
+/// on every provider measured, so a report below it is not credible and floors there.
pub fn replaceContextTokens(self: *Agent, reported: u64) void {
- const local = self.fullRequestEstimateTokens();
- self.last_context_tokens = @max(local, reported);
- self.context_local_tokens = local;
+ const input_bytes = context_tokens.serializedLen(Value{ .array = self.messages });
+ self.last_context_tokens = @max(reported, measuredRequestTokens(self, input_bytes));
+ self.context_local_tokens = requestEstimateFromInputBytes(self, input_bytes);
}
/// An integer usage field, or 0 if absent / wrong type.
@@ -208,17 +208,19 @@ pub fn fullInputEstimateTokens(self: *Agent) u64 {
return context_tokens.estimatedTokens(Value{ .array = self.messages });
}
-/// Estimate the complete fixed+history input sent by a normal model request.
-/// fullInputEstimateTokens intentionally covers messages only; the pre-send gate
-/// must additionally count the mutable system prompt and active tool schema. Keep
-/// the old 8k conservative baseline when those serialize smaller, but never let it
-/// hide an unbounded set_system_prompt/set_agent payload.
+/// The complete fixed+history input of a normal request (fullInputEstimateTokens is
+/// messages only): system prompt and tool schema too, with the 8k conservative
+/// baseline when those serialize smaller, never hiding a large set_system_prompt.
fn requestEstimateFromInputBytes(self: *Agent, input_bytes: usize) u64 {
const input_tokens = context_tokens.estimatedTokensFromLen(Value{ .array = self.messages }, input_bytes);
- const fixed_bytes = context_tokens.serializedLen(self.systemPrompt()) +| self.toolsJson().len;
- const measured = input_tokens +| @as(u64, @intCast(fixed_bytes / 4));
const baseline = @min(@as(u64, 8000), self.provider.context / 8);
- return @max(measured, input_tokens +| baseline);
+ return @max(measuredRequestTokens(self, input_bytes), input_tokens +| baseline);
+}
+
+/// bytes/4 of history + system prompt + tool schema, with no baseline.
+pub fn measuredRequestTokens(self: *Agent, input_bytes: usize) u64 {
+ const fixed_bytes = context_tokens.serializedLen(self.systemPrompt()) +| self.toolsJson().len;
+ return context_tokens.estimatedTokensFromLen(Value{ .array = self.messages }, input_bytes) +| @as(u64, @intCast(fixed_bytes / 4));
}
pub fn fullRequestEstimateTokens(self: *Agent) u64 {
@@ -245,10 +247,8 @@ pub fn contextEstimate(self: *Agent) ContextEstimate {
/// Reusing the observed byte count avoids a redundant full JSON walk.
pub fn contextEstimateFromInputBytes(self: *Agent, input_bytes: usize) ContextEstimate {
const local = requestEstimateFromInputBytes(self, input_bytes);
- return .{
- .local = local,
- .effective = local +| (self.last_context_tokens -| self.context_local_tokens),
- };
+ const last = self.last_context_tokens; // a provider sample may sit below the local estimate
+ return .{ .local = local, .effective = if (last == 0 or last >= self.context_local_tokens) local +| (last - @min(last, self.context_local_tokens)) else last +| (local -| self.context_local_tokens) };
}
/// Current context occupancy: today's locally measurable request plus the
@@ -264,9 +264,7 @@ pub fn effectiveContextTokens(self: *Agent) u64 {
/// advancing only the local anchor prevents counting them twice.
pub fn pairContextMeterWithCurrentLocal(self: *Agent) void {
if (!self.last_usage_includes_output) return;
- const local = self.fullRequestEstimateTokens();
- self.last_context_tokens = @max(self.last_context_tokens, local);
- self.context_local_tokens = local;
+ self.context_local_tokens = self.fullRequestEstimateTokens();
self.last_usage_includes_output = false;
}
@@ -279,6 +277,7 @@ pub fn rebaseContextMeter(self: *Agent) void {
}
test "rebaseContextMeter preserves hidden context while replacing local input" {
+ _ = @import("agent_context_meter_tests.zig"); // the provider-count meter tests ride along
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const a = arena_state.allocator();
@@ -533,9 +532,9 @@ test "recordUsageResponses: usage fallbacks never lower an authoritative meter"
try smaller_usage.put(a, "total_tokens", .{ .integer = 100 });
var smaller_response: std.json.ObjectMap = .empty;
try smaller_response.put(a, "usage", .{ .object = smaller_usage });
- // A full re-anchor trusts its provider reading (with the local floor).
+ // A full re-anchor is its provider reading; the pre-usage floor no longer lifts it.
recordUsageResponses(&agent, smaller_response, 400);
- try std.testing.expectEqual(@max(fullRequestEstimateTokens(&agent), @as(u64, 100)), agent.last_context_tokens);
+ try std.testing.expectEqual(@as(u64, 100), agent.last_context_tokens);
try std.testing.expect(agent.last_usage_includes_output);
// The aggregate is advisory when larger input/output components are
diff --git a/src/agent_context_meter_tests.zig b/src/agent_context_meter_tests.zig
new file mode 100644
index 00000000..3116070e
--- /dev/null
+++ b/src/agent_context_meter_tests.zig
@@ -0,0 +1,104 @@
+//! The context meter trusts a provider's count of a full request (agent_context.zig).
+
+const std = @import("std");
+const util = @import("util.zig");
+const Agent = @import("agent.zig").Agent;
+const messages_mod = @import("messages.zig");
+const agent_context = @import("agent_context.zig");
+
+/// What a real provider reports: more than bytes/4, since code and JSON
+/// tokenize at fewer than four bytes a token.
+fn providerCount(agent: *Agent) u64 {
+ return agent_context.measuredRequestTokens(agent, @import("context_tokens.zig").serializedLen(std.json.Value{ .array = agent.messages })) + 1_000;
+}
+
+fn anthropicAgent(a: std.mem.Allocator) Agent {
+ var agent: Agent = undefined;
+ agent.io = std.testing.io;
+ agent.arena = a;
+ agent.messages = std.json.Array.init(a);
+ agent.last_context_tokens = 0;
+ agent.context_local_tokens = 0;
+ agent.last_usage_includes_output = false;
+ agent.provider = .{ .id = "anthropic", .kind = .anthropic, .auth = .x_api_key, .url = "", .api_key = "", .model = "claude", .context = 200_000 };
+ agent.sub = false;
+ agent.strict = false;
+ agent.sys_normal = &util.repeatBytes("s", 6_000);
+ agent.sys_strict = "";
+ agent.tools_anthropic = &util.repeatBytes("t", 7_000);
+ return agent;
+}
+
+test "before any usage the meter keeps the conservative local estimate" {
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ var agent = anthropicAgent(arena_state.allocator());
+ try agent.messages.append(try messages_mod.textMessage(arena_state.allocator(), "user", "hi"));
+ // 13 KB of system + tools is ~3.3k by bytes/4; the pre-usage baseline holds it at 8k.
+ try std.testing.expect(agent.fullRequestEstimateTokens() >= 8_000);
+ try std.testing.expectEqual(agent.fullRequestEstimateTokens(), agent.effectiveContextTokens());
+}
+
+test "a provider count below the local estimate is the meter" {
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ const a = arena_state.allocator();
+ var agent = anthropicAgent(a);
+ try agent.messages.append(try messages_mod.textMessage(a, "user", "hi"));
+ const local = agent.fullRequestEstimateTokens();
+ const reported = providerCount(&agent);
+ agent_context.replaceContextTokens(&agent, reported);
+ try std.testing.expect(local > reported);
+ try std.testing.expectEqual(reported, agent.effectiveContextTokens());
+}
+
+test "a report below plain bytes/4 is not credible and floors there" {
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ const a = arena_state.allocator();
+ var agent = anthropicAgent(a);
+ try agent.messages.append(try messages_mod.textMessage(a, "user", "hi"));
+ const floor = providerCount(&agent) - 1_000;
+ agent_context.replaceContextTokens(&agent, 12); // a server that under-reports
+ try std.testing.expectEqual(floor, agent.effectiveContextTokens());
+}
+
+test "growth after the sample adds to the provider count" {
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ const a = arena_state.allocator();
+ var agent = anthropicAgent(a);
+ try agent.messages.append(try messages_mod.textMessage(a, "user", "hi"));
+ const reported = providerCount(&agent);
+ agent_context.replaceContextTokens(&agent, reported);
+ const anchor = agent.context_local_tokens;
+ try agent.messages.append(try messages_mod.textMessage(a, "user", &util.repeatBytes("x", 4_000)));
+ const grown = agent.fullRequestEstimateTokens() - anchor;
+ try std.testing.expect(grown >= 900);
+ try std.testing.expectEqual(reported + grown, agent.effectiveContextTokens());
+}
+
+test "pairing provider-covered output does not re-floor at the local estimate" {
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ const a = arena_state.allocator();
+ var agent = anthropicAgent(a);
+ try agent.messages.append(try messages_mod.textMessage(a, "user", "hi"));
+ const reported = providerCount(&agent) + 200; // input + the reply's output tokens
+ agent_context.replaceContextTokens(&agent, reported);
+ agent.last_usage_includes_output = true;
+ try agent.messages.append(try messages_mod.textMessage(a, "assistant", &util.repeatBytes("y", 600)));
+ agent.pairContextMeterWithCurrentLocal();
+ try std.testing.expectEqual(reported, agent.effectiveContextTokens());
+}
+
+test "a cleared sample falls back to the local estimate even with a stale anchor" {
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ const a = arena_state.allocator();
+ var agent = anthropicAgent(a);
+ try agent.messages.append(try messages_mod.textMessage(a, "user", "hi"));
+ agent_context.replaceContextTokens(&agent, providerCount(&agent));
+ agent.last_context_tokens = 0; // what compaction does
+ try std.testing.expectEqual(agent.fullRequestEstimateTokens(), agent.effectiveContextTokens());
+}