Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Trace the server's token usage per response

justrach3cbc7928d8b3
diff --git a/build.zig b/build.zig
index 2decf784..12fa968b 100644
--- a/build.zig
+++ b/build.zig
@@ -150,6 +150,9 @@ pub fn build(b: *std.Build) void {
const acp_harness_room_test = b.addSystemCommand(&.{ "python3", "scripts/test-acp-harness-room.py" });
acp_harness_room_test.addArtifactArg(exe);
test_step.dependOn(&acp_harness_room_test.step);
+ const usage_trace_test = b.addSystemCommand(&.{ "python3", "scripts/test-usage-trace.py" });
+ usage_trace_test.addArtifactArg(exe);
+ test_step.dependOn(&usage_trace_test.step);
const acp_view_meta_test = b.addSystemCommand(&.{ "python3", "scripts/test-acp-view-meta.py" });
acp_view_meta_test.addArtifactArg(exe);
test_step.dependOn(&acp_view_meta_test.step);
diff --git a/docs/architecture.md b/docs/architecture.md
index 28fee398..2837dcc2 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -337,6 +337,15 @@ cache-read count from whichever field the provider uses
`prompt_tokens_details.cached_tokens`) into `Agent.last_cache_read`, which
`request()` writes to the `cache_read_tokens` field of each `api` trace line.
+The `api` line's `context_tokens` is the context meter, not the server's
+input count: it is floored at graff's estimate of a full resend, which on a
+chained Codex request still counts every retained encrypted reasoning item
+the server has dropped. Each response therefore also writes a `usage` line
+with the server's own split: `input_tokens` (cached and written tokens
+included), `cache_read_tokens`, `cache_write_tokens`, `output_tokens`, and
+`chained` (sent as `previous_response_id` plus new items). A cache hit rate is
+`cache_read_tokens / input_tokens` from `usage`, never from `context_tokens`.
+
## Session persistence
`/save` serializes `{provider, model, strict, messages}` with
diff --git a/scripts/eval/mock_model.py b/scripts/eval/mock_model.py
index cd9a8913..fd305d35 100644
--- a/scripts/eval/mock_model.py
+++ b/scripts/eval/mock_model.py
@@ -125,7 +125,7 @@ class ScriptedModel:
self._send(json.dumps({
"id": "mock", "object": "chat.completion", "model": "mock",
"choices": [{"index": 0, "message": message, "finish_reason": finish}],
- "usage": {"prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12},
+ "usage": reply.get("usage", {"prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12}),
}).encode(), "application/json")
def _stream(self, reply: dict[str, Any]) -> None:
@@ -168,7 +168,7 @@ class ScriptedModel:
return
self._chunk({
"choices": [{"index": 0, "delta": {}, "finish_reason": finish}],
- "usage": {"prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12},
+ "usage": reply.get("usage", {"prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12}),
})
self.wfile.write(b"data: [DONE]\n\n")
self.wfile.flush()
diff --git a/scripts/test-usage-trace.py b/scripts/test-usage-trace.py
new file mode 100644
index 00000000..1ef1a0d1
--- /dev/null
+++ b/scripts/test-usage-trace.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""Each model response writes a `usage` trace event with the server's counts.
+
+- `input_tokens` is what the server billed as input (cached and written
+ tokens included), `cache_read_tokens` / `cache_write_tokens` / `output_tokens`
+ are its split, and `chained` says whether the request was a held-socket delta
+- the `api` event's `context_tokens` is the context meter and may differ; the
+ cache hit rate is cache_read_tokens / input_tokens from `usage`
+"""
+import json, os, shutil, subprocess, sys, tempfile
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent / 'eval'))
+from mock_model import ScriptedModel
+
+USAGE = {'prompt_tokens': 20000, 'completion_tokens': 17, 'total_tokens': 20017,
+ 'prompt_tokens_details': {'cached_tokens': 18432}}
+
+
+def main():
+ binary = str(Path(sys.argv[1] if len(sys.argv) > 1 else 'zig-out/bin/graff').resolve())
+ temp = Path(tempfile.mkdtemp(prefix='graff-usage-trace-'))
+ model = ScriptedModel([
+ {'tool': 'todo_read', 'arguments': {}, 'usage': USAGE},
+ {'text': 'done', 'usage': {'prompt_tokens': 21000, 'completion_tokens': 3, 'total_tokens': 21003}},
+ ])
+ try:
+ work = temp / 'proj'
+ work.mkdir()
+ env = {k: v for k, v in os.environ.items() if not k.endswith('_API_KEY')}
+ env.update(HOME=str(temp), USERPROFILE=str(temp), AI_GATEWAY_API_KEY='local', GRAFF_NO_TELEMETRY='1',
+ GRAFF_FLEET='off', GRAFF_NO_SMOLIFY='1', GRAFF_NO_CODEDB_GUARD='1')
+ port = model.start(0)
+ env['GRAFF_VERCEL_URL'] = f'http://127.0.0.1:{port}/v1/chat/completions'
+ run = subprocess.run([binary, '--model', 'vercel', '--old', '--no-lean', '--yolo', '-p', 'go'], cwd=work, env=env,
+ capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=120)
+ assert run.returncode == 0, run.stderr[-1500:]
+ rows = []
+ for path in sorted((work / '.graff' / 'traces').glob('*.jsonl')):
+ rows += [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
+ usage = [r for r in rows if r.get('ev') == 'usage' and not r.get('from_sub')]
+ api = [r for r in rows if r.get('ev') == 'api' and not r.get('is_error')]
+ assert len(usage) == 2 and len(api) == 2, (usage, api)
+ first = usage[0]
+ assert first['input_tokens'] == 20000, first
+ assert first['cache_read_tokens'] == 18432 and first['cache_write_tokens'] == 0, first
+ assert first['output_tokens'] == 17 and first['chained'] is False, first
+ assert first['provider'] == 'vercel' and first['model'], first
+ assert usage[1]['input_tokens'] == 21000 and usage[1]['cache_read_tokens'] == 0, usage[1]
+ print('PASS each response writes a usage event with the server input, cache read/write and output tokens', flush=True)
+ finally:
+ model.stop()
+ shutil.rmtree(temp, ignore_errors=True)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/src/agent_context.zig b/src/agent_context.zig
index 2b82359a..dbed42f4 100644
--- a/src/agent_context.zig
+++ b/src/agent_context.zig
@@ -187,13 +187,13 @@ pub fn usageInt(obj: std.json.ObjectMap, name: []const u8) i64 {
return 0;
}
-/// Record one request's usage into the session-wide tally (g_cost):
-/// token counts always; USD only for metered seats with a price_table row.
+/// Record one request's usage into the session-wide tally (g_cost) and trace: token counts always; USD only for metered seats with a price_table row.
/// #471: the seat is classified from the provider AND how its credential was
/// obtained — a flat-rate login (codex/kimi/xai) tallies as sub_calls and adds
/// $0, while an env key on that same provider is metered like any other.
pub fn recordCost(self: *Agent, ordinary_in: i64, cache_in: i64, cache_write_in: i64, out: i64) void {
g_cost.addForProvider(self.io, billing.forProvider(self.provider), self.provider.id, self.provider.model, ordinary_in, cache_in, cache_write_in, out);
+ @import("usage_trace.zig").note(self, ordinary_in, cache_in, cache_write_in, out);
if (!self.sub) @import("prompt_cache_hud.zig").noteUsage(@intCast(@max(cache_in, 0)), @intCast(@max(cache_write_in, 0)));
}
diff --git a/src/main.zig b/src/main.zig
index fab25bbc..56c6e40f 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -559,7 +559,7 @@ const workflow = @import("workflow_test.zig"); // the engine's tests live here (
const exec = @import("exec.zig");
test { // ── Unit tests (`zig build test`): pull in tests from imported modules (mcp.zig)
_ = .{ @import("beta_feed.zig"), @import("jev_tool.zig"), @import("jev_model_scope.zig"), @import("jev_provider_switch_tests.zig"), @import("jev_effort_state.zig") };
- _ = .{ @import("mcp_apps.zig"), @import("html_view.zig"), @import("providers_confirmation_tests.zig"), @import("repl_model_confirmation_tests.zig"), @import("repl_model_pick_tests.zig"), @import("engine_sink_citation_tests.zig"), @import("cite_markup_stream_tests.zig"), @import("oneshot_citation_tests.zig"), @import("tui_acp.zig"), @import("acp_permission.zig"), @import("acp_usage.zig") };
+ _ = .{ @import("mcp_apps.zig"), @import("html_view.zig"), @import("usage_trace.zig"), @import("providers_confirmation_tests.zig"), @import("repl_model_confirmation_tests.zig"), @import("repl_model_pick_tests.zig"), @import("engine_sink_citation_tests.zig"), @import("cite_markup_stream_tests.zig"), @import("oneshot_citation_tests.zig"), @import("tui_acp.zig"), @import("acp_permission.zig"), @import("acp_usage.zig") };
_ = .{ @import("mcp_server_tests.zig"), @import("main_test.zig"), @import("artifact_claim_command.zig"), @import("cli_path_hint.zig"), @import("run_budget.zig"), @import("pr_review_input.zig"), @import("pr_claim_review.zig"), @import("tui_paste_transfer.zig") };
_ = .{ @import("session_catalog.zig"), @import("session_prompt.zig"), @import("session_discovery.zig") };
_ = .{ @import("prompt_astra.zig"), @import("prompt_guidance.zig") }; // per-model guidance (Astra, GPT-5.6)
diff --git a/src/trace.zig b/src/trace.zig
index 3a407a84..2115c213 100644
--- a/src/trace.zig
+++ b/src/trace.zig
@@ -127,6 +127,7 @@ pub const Tracer = struct {
if (is_error) {
// Failed calls did not report usage. Omit it rather than claiming
// an observed zero; request/response byte counts remain known.
+ @import("usage_trace.zig").drop();
self.write(.{
.t = self.elapsedMs(),
.ev = "api",
@@ -153,6 +154,7 @@ pub const Tracer = struct {
.cache_read_tokens = cache_read,
.is_error = is_error,
});
+ @import("usage_trace.zig").emit(self, label, from_subagent, model); // the server's own split, same request
}
pub fn tool(self: *Tracer, name: []const u8, call_id: u64, ms: i64, is_error: bool, result_bytes: usize, from_sub: bool) void {
diff --git a/src/usage_trace.zig b/src/usage_trace.zig
new file mode 100644
index 00000000..13c1012f
--- /dev/null
+++ b/src/usage_trace.zig
@@ -0,0 +1,93 @@
+//! One `usage` trace event per model response: the server's own token counts.
+//!
+//! The `api` event carries `context_tokens`, the context meter, which is the
+//! larger of the server total and graff's local estimate of a full resend
+//! (agent_context.zig). On a chained Codex request the server drops earlier
+//! turns' reasoning while the local estimate still counts every retained
+//! encrypted reasoning item, so `cache_read_tokens / context_tokens` is not a
+//! cache hit rate. This event records what the server billed, split the same
+//! way recordCost receives it, so trajectory analysis can compute
+//! `cache_read_tokens / input_tokens` per request.
+//!
+//! recordCost only stashes the numbers (tests build partial agents whose
+//! tracer is undefined); Tracer.api writes them right after its `api` line.
+//! A request and its trace line run on the same thread, so the slot is
+//! thread-local.
+
+const std = @import("std");
+const Agent = @import("agent.zig").Agent;
+
+const Usage = struct {
+ provider: []const u8,
+ input: i64,
+ cache_read: i64,
+ cache_write: i64,
+ output: i64,
+ chained: bool,
+};
+
+threadlocal var pending: ?Usage = null;
+
+/// From recordCost: remember this response's usage for the next `api` line.
+pub fn note(self: *const Agent, ordinary: i64, cache_read: i64, cache_write: i64, out: i64) void {
+ pending = .{
+ .provider = self.provider.id,
+ .input = ordinary + cache_read + cache_write,
+ .cache_read = cache_read,
+ .cache_write = cache_write,
+ .output = out,
+ // Sent as previous_response_id + the new items on a held socket.
+ .chained = self.codex_prev_id != null,
+ };
+}
+
+/// From Tracer.api on a successful call: write the pending usage, if any.
+pub fn emit(tr: anytype, label: []const u8, from_sub: bool, model: []const u8) void {
+ const u = pending orelse return;
+ pending = null;
+ tr.write(.{
+ .t = tr.elapsedMs(),
+ .ev = "usage",
+ .agent = label,
+ .from_sub = from_sub,
+ .provider = u.provider,
+ .model = model,
+ .input_tokens = u.input,
+ .cache_read_tokens = u.cache_read,
+ .cache_write_tokens = u.cache_write,
+ .output_tokens = u.output,
+ .chained = u.chained,
+ });
+}
+
+/// From Tracer.api on a failed call: nothing was billed for it.
+pub fn drop() void {
+ pending = null;
+}
+
+test "usage waits for its api line and is written once" {
+ const Sink = struct {
+ lines: u32 = 0,
+ last_input: i64 = 0,
+ fn write(self: *@This(), event: anytype) void {
+ self.lines += 1;
+ self.last_input = event.input_tokens;
+ }
+ fn elapsedMs(_: *@This()) i64 {
+ return 0;
+ }
+ };
+ var agent: Agent = undefined; // only provider and codex_prev_id are read
+ agent.provider.id = "codex";
+ agent.codex_prev_id = null;
+ var sink: Sink = .{};
+ note(&agent, 100, 900, 0, 7);
+ emit(&sink, "main", false, "gpt-6-sol");
+ emit(&sink, "main", false, "gpt-6-sol"); // taken: no second line
+ try std.testing.expectEqual(@as(u32, 1), sink.lines);
+ try std.testing.expectEqual(@as(i64, 1000), sink.last_input);
+ note(&agent, 1, 2, 3, 4);
+ drop(); // the call failed
+ emit(&sink, "main", false, "gpt-6-sol");
+ try std.testing.expectEqual(@as(u32, 1), sink.lines);
+}