graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Name why a request read nothing from the cache; keep slow Anthropic calls warm (#1320)
justrach1d1a7c7a07ce
diff --git a/docs/architecture.md b/docs/architecture.mdindex 2837dcc2..d998f2a4 100644--- a/docs/architecture.md+++ b/docs/architecture.md@@ -346,6 +346,16 @@ included), `cache_read_tokens`, `cache_write_tokens`, `output_tokens`, and`chained` (sent as `previous_response_id` plus new items). A cache hit rate is`cache_read_tokens / input_tokens` from `usage`, never from `context_tokens`.+`usage` also says why a read dropped (#1320). It records `gap_ms` (time since+this agent's previous request began), `cache_ttl` (`"1h"` when requested),+`prefix_changed` and `prefix_bust` (the root's system+tools prefix moved, and+the named reason: tools, mcp, mode, compact, …), and, on a request with real+input that read nothing, `cache_miss`: `prefix_changed`, `first_request`,+`idle_gap` (5+ minutes since the previous request, past a default entry's+life), or `unknown`. On the Anthropic API a request whose predecessor began+3+ minutes earlier asks for the 1-hour TTL, so a slow call no longer returns+to an expired prefix.+## Session persistence`/save` serializes `{provider, model, strict, messages}` withdiff --git a/src/agent_request.zig b/src/agent_request.zigindex c10343e5..f40b397c 100644--- a/src/agent_request.zig+++ b/src/agent_request.zig@@ -202,6 +202,7 @@ pub fn request(self: *Agent, tools_in: ?[]const u8) !std.json.ObjectMap {var openai404_retries: usize = 0; // #opencode-parity: bounded retries for OpenAI's spurious model 404sconst max_openai404_retries: usize = 2;var gw_retry = policy.GatewayRetryState{}; // #gateway-artifact state (agent_gateway_retry.zig)+ @import("cache_ttl.zig").begin(self); // #1320: before request_started movesrebuild: while (true) {if (@import("agent_async_tools.zig").started(self)) return error.AsyncToolStreamFailed;const live = self.usesLiveTransport();diff --git a/src/agent_request_body.zig b/src/agent_request_body.zigindex 9b35199c..d3e1399b 100644--- a/src/agent_request_body.zig+++ b/src/agent_request_body.zig@@ -76,7 +76,7 @@ pub fn buildBody(self: *Agent, tools_in: ?[]const u8, force_tool: bool, stream:// other anthropic-format providers (minimax) get a plain string.try s.objectField("system");const sys = try @import("agent_request_body_responses.zig").schemaAwarePrompt(self);- const cc = "{\"type\":\"ephemeral\"}";+ const cc = @import("cache_ttl.zig").control(self); // #1320: 1h when calls outlive 5mif (std.mem.eql(u8, self.provider.id, "anthropic") or is_kimi) {try s.beginArray();try s.beginObject();@@ -106,7 +106,7 @@ pub fn buildBody(self: *Agent, tools_in: ?[]const u8, force_tool: bool, stream:// Anthropic API and Kimi's declared Anthropic transport. Kimi also// normalizes all string content into Anthropic text blocks.const cache_msgs = std.mem.eql(u8, self.provider.id, "anthropic") or is_kimi;- try writeAnthropicMessages(&s, self.messages, cache_msgs, is_kimi);+ try writeAnthropicMessages(&s, self.messages, if (cache_msgs) cc else null, is_kimi);},.openai => {// graff's MakeOpenAiCompat: OpenAI deprecated max_tokens indiff --git a/src/cache_ttl.zig b/src/cache_ttl.zignew file mode 100644index 00000000..30968eae--- /dev/null+++ b/src/cache_ttl.zig@@ -0,0 +1,105 @@+//! #1320: an Anthropic cache entry lives 5 minutes from its last use. A model+//! call that runs longer (slow reasoning at a large context) comes back to an+//! expired prefix, and the next request reads nothing. When this agent's+//! previous request began 3+ minutes ago, calls are that slow: its successor+//! asks for the 1-hour TTL (a 2x write instead of 1.25x, on the tokens it+//! writes) so the entry outlives the next slow call. The gap is taken once+//! per request, before any retry, and usage_trace.zig reports it with the+//! reason a request read nothing.++const std = @import("std");+const Agent = @import("agent.zig").Agent;++pub const ephemeral = "{\"type\":\"ephemeral\"}";+pub const ephemeral_1h = "{\"type\":\"ephemeral\",\"ttl\":\"1h\"}";+/// A previous request this long ago means calls outlive a 5-minute entry.+pub const slow_gap_ms: i64 = 3 * 60 * 1000;+/// The provider's default entry lifetime (Anthropic's; others are similar).+pub const default_ttl_ms: i64 = 5 * 60 * 1000;++/// Requests and their usage lines run on one thread (see usage_trace.zig).+threadlocal var gap_ms: ?i64 = null;++/// Before a request's retry loop: time since this agent's previous request began.+pub fn begin(self: *const Agent) void {+ gap_ms = if (self.request_started) |t| t.untilNow(self.io, .awake).toMilliseconds() else null;+}++pub fn gap() ?i64 {+ return gap_ms;+}++/// Only the Anthropic API gets a TTL; Anthropic-format providers keep the default.+pub fn longTtl(provider_id: []const u8, since_previous: ?i64) bool {+ const g = since_previous orelse return false;+ return std.mem.eql(u8, provider_id, "anthropic") and g >= slow_gap_ms;+}++/// The cache_control object for this request's breakpoints.+pub fn control(self: *const Agent) []const u8 {+ return if (longTtl(self.provider.id, gap_ms)) ephemeral_1h else ephemeral;+}++/// Why a request with real input read nothing from the cache. Content-free.+pub fn missReason(cache_read: i64, input: i64, prefix_changed: bool, since_previous: ?i64) ?[]const u8 {+ if (cache_read > 0 or input < 1024) return null;+ if (prefix_changed) return "prefix_changed";+ const g = since_previous orelse return "first_request";+ return if (g >= default_ttl_ms) "idle_gap" else "unknown";+}++test "#1320: slow calls get the 1-hour TTL on the Anthropic API only" {+ try std.testing.expect(!longTtl("anthropic", null));+ try std.testing.expect(!longTtl("anthropic", 90_000));+ try std.testing.expect(longTtl("anthropic", 307_000));+ try std.testing.expect(!longTtl("kimi", 307_000));+ try std.testing.expect(!longTtl("codex", 307_000));+}++test "#1320: an Anthropic body after a slow call marks every breakpoint 1h; Kimi keeps the default" {+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);+ defer arena_state.deinit();+ const arena = arena_state.allocator();+ var messages = std.json.Array.init(arena);+ var user: std.json.ObjectMap = .empty;+ try user.put(arena, "role", .{ .string = "user" });+ try user.put(arena, "content", .{ .string = "hello" });+ try messages.append(.{ .object = user });+ var agent: Agent = .{+ .gpa = std.testing.allocator,+ .arena = arena,+ .io = std.testing.io,+ .client = undefined,+ .provider = .{ .id = "anthropic", .kind = .anthropic, .auth = .x_api_key, .url = "", .api_key = "k", .model = "claude-fixture", .context = 200_000 },+ .messages = messages,+ .sub = false,+ .label = "",+ .out = null,+ .sys_normal = "system",+ };+ defer gap_ms = null;+ const long = "\"cache_control\":" ++ ephemeral_1h;+ gap_ms = 307_000;+ const slow = try agent.buildBody(null, false, true, true);+ defer std.testing.allocator.free(slow);+ try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, slow, long)); // system + last message+ try std.testing.expect(std.mem.indexOf(u8, slow, "\"cache_control\":" ++ ephemeral ++ "") == null);+ gap_ms = 20_000;+ const fast = try agent.buildBody(null, false, true, true);+ defer std.testing.allocator.free(fast);+ try std.testing.expect(std.mem.indexOf(u8, fast, "\"ttl\"") == null);+ agent.provider.id = "kimi";+ gap_ms = 307_000;+ const kimi = try agent.buildBody(null, false, true, true);+ defer std.testing.allocator.free(kimi);+ try std.testing.expect(std.mem.indexOf(u8, kimi, "\"ttl\"") == null);+}++test "#1320: a zero-read request names its likely cause" {+ try std.testing.expect(missReason(900, 1000, true, 1) == null);+ try std.testing.expect(missReason(0, 200, false, null) == null); // below any cacheable size+ try std.testing.expectEqualStrings("prefix_changed", missReason(0, 90_000, true, 400_000).?);+ try std.testing.expectEqualStrings("first_request", missReason(0, 90_000, false, null).?);+ try std.testing.expectEqualStrings("idle_gap", missReason(0, 90_000, false, 307_000).?);+ try std.testing.expectEqualStrings("unknown", missReason(0, 90_000, false, 20_000).?);+}diff --git a/src/serde.zig b/src/serde.zigindex 4f174f9f..00134b5d 100644--- a/src/serde.zig+++ b/src/serde.zig@@ -91,7 +91,7 @@ fn anthropicCacheableBlock(value: Value) bool {return false;}-fn writeObjectWithCache(s: *std.json.Stringify, obj: std.json.ObjectMap) !void {+fn writeObjectWithCache(s: *std.json.Stringify, obj: std.json.ObjectMap, cache_control: []const u8) !void {try s.beginObject();var it = obj.iterator();while (it.next()) |kv| {@@ -100,16 +100,16 @@ fn writeObjectWithCache(s: *std.json.Stringify, obj: std.json.ObjectMap) !void {}if (obj.get("cache_control") == null) {try s.objectField("cache_control");- try s.print("{s}", .{"{\"type\":\"ephemeral\"}"});+ try s.print("{s}", .{cache_control});}try s.endObject();}/// Serialize Anthropic messages. `normalize_blocks` matches the official Kimi/// adapter by turning every plain string into a `{type:text,text}` content-/// array. `cache` marks the final cacheable block (including tool_result), not-/// just the plain-string happy path.-pub fn writeAnthropicMessages(s: *std.json.Stringify, messages: std.json.Array, cache: bool, normalize_blocks: bool) !void {+/// array. `cache` (the cache_control object, #1320) marks the final cacheable+/// block (including tool_result), not just the plain-string happy path.+pub fn writeAnthropicMessages(s: *std.json.Stringify, messages: std.json.Array, cache: ?[]const u8, normalize_blocks: bool) !void {const items = messages.items;try s.beginArray();for (items, 0..) |m, i| {@@ -118,7 +118,7 @@ pub fn writeAnthropicMessages(s: *std.json.Stringify, messages: std.json.Array,continue;}const content = m.object.get("content").?;- const cache_this = cache and i + 1 == items.len;+ const cache_this = cache != null and i + 1 == items.len;const string_content = content == .string;const array_cache = cache_this and content == .array and content.array.items.len > 0 and anthropicCacheableBlock(content.array.items[content.array.items.len - 1]);if (!normalize_blocks and !cache_this) {@@ -147,7 +147,7 @@ pub fn writeAnthropicMessages(s: *std.json.Stringify, messages: std.json.Array,try s.write(content.string);if (cache_this) {try s.objectField("cache_control");- try s.print("{s}", .{"{\"type\":\"ephemeral\"}"});+ try s.print("{s}", .{cache.?});}try s.endObject();try s.endArray();@@ -155,7 +155,7 @@ pub fn writeAnthropicMessages(s: *std.json.Stringify, messages: std.json.Array,try s.beginArray();for (content.array.items, 0..) |block, block_i| {if (block_i + 1 == content.array.items.len and anthropicCacheableBlock(block))- try writeObjectWithCache(s, block.object)+ try writeObjectWithCache(s, block.object, cache.?)elsetry s.write(block);}@@ -203,7 +203,7 @@ pub fn writeAnthropicTools(s: *std.json.Stringify, arena: Allocator, raw: []constry s.beginArray();for (value.array.items, 0..) |tool, i| {if (cache and i + 1 == value.array.items.len and tool == .object)- try writeObjectWithCache(s, tool.object)+ try writeObjectWithCache(s, tool.object, @import("cache_ttl.zig").ephemeral)elsetry s.write(tool);}diff --git a/src/session_wake.zig b/src/session_wake.zigindex 22c4fcb2..36964405 100644--- a/src/session_wake.zig+++ b/src/session_wake.zig@@ -142,7 +142,7 @@ test "notification metadata stays off chat, responses and anthropic wires includif (mode == 0) try writeWireArray(&s, msgs.items) else if (mode == 1)try serde.writeOpenAIMessageNormalized(&s, m)else- try serde.writeAnthropicMessages(&s, msgs, mode & 1 == 1, mode & 2 == 2);+ try serde.writeAnthropicMessages(&s, msgs, if (mode & 1 == 1) @import("cache_ttl.zig").ephemeral else null, mode & 2 == 2);try std.testing.expect(std.mem.indexOf(u8, aw.written(), origin_key) == null);try std.testing.expect(std.mem.indexOf(u8, aw.written(), "wake up") != null);try std.testing.expect(std.mem.indexOf(u8, aw.written(), "\"role\":\"user\"") != null);diff --git a/src/usage_trace.zig b/src/usage_trace.zigindex 13c1012f..9fe20fd1 100644--- a/src/usage_trace.zig+++ b/src/usage_trace.zig@@ -16,6 +16,8 @@const std = @import("std");const Agent = @import("agent.zig").Agent;+const cache_ttl = @import("cache_ttl.zig");+const hud = @import("prompt_cache_hud.zig");const Usage = struct {provider: []const u8,@@ -24,20 +26,38 @@ const Usage = struct {cache_write: i64,output: i64,chained: bool,+ /// #1320: why a read may have dropped. Time since this agent's previous+ /// request began, the TTL asked for (null: the provider default), whether+ /// the system+tools prefix changed and the named reason, and on a+ /// zero-read request with real input the likely cause.+ gap_ms: ?i64 = null,+ cache_ttl: ?[]const u8 = null,+ prefix_changed: bool = false,+ prefix_bust: ?[]const u8 = null,+ cache_miss: ?[]const u8 = null,};threadlocal var pending: ?Usage = null;/// From recordCost: remember this response's usage for the next `api` line.pub fn note(self: *const Agent, ordinary: i64, cache_read: i64, cache_write: i64, out: i64) void {+ const gap = cache_ttl.gap();+ // The prefix tracker follows the root session only.+ const snap = if (self.sub) hud.Snap{} else hud.snapshot();+ const input = ordinary + cache_read + cache_write;pending = .{.provider = self.provider.id,- .input = ordinary + cache_read + cache_write,+ .input = input,.cache_read = cache_read,.cache_write = cache_write,.output = out,// Sent as previous_response_id + the new items on a held socket..chained = self.codex_prev_id != null,+ .gap_ms = gap,+ .cache_ttl = if (cache_ttl.longTtl(self.provider.id, gap)) "1h" else null,+ .prefix_changed = !snap.same,+ .prefix_bust = if (snap.same) null else hud.bustLabel(snap.last_bust),+ .cache_miss = cache_ttl.missReason(cache_read, input, !snap.same, gap),};}@@ -57,6 +77,11 @@ pub fn emit(tr: anytype, label: []const u8, from_sub: bool, model: []const u8) v.cache_write_tokens = u.cache_write,.output_tokens = u.output,.chained = u.chained,+ .gap_ms = u.gap_ms,+ .cache_ttl = u.cache_ttl,+ .prefix_changed = u.prefix_changed,+ .prefix_bust = u.prefix_bust,+ .cache_miss = u.cache_miss,});}@@ -69,25 +94,39 @@ test "usage waits for its api line and is written once" {const Sink = struct {lines: u32 = 0,last_input: i64 = 0,+ last_miss: ?[]const u8 = null,fn write(self: *@This(), event: anytype) void {self.lines += 1;self.last_input = event.input_tokens;+ self.last_miss = event.cache_miss;}fn elapsedMs(_: *@This()) i64 {return 0;}};- var agent: Agent = undefined; // only provider and codex_prev_id are read+ var agent: Agent = undefined; // only provider, sub and codex_prev_id are readagent.provider.id = "codex";agent.codex_prev_id = null;+ agent.sub = true;+ agent.request_started = null;+ cache_ttl.begin(&agent); // this thread's slot: no previous requestvar sink: Sink = .{};note(&agent, 100, 900, 0, 7);emit(&sink, "main", false, "gpt-6-sol");emit(&sink, "main", false, "gpt-6-sol"); // taken: no second linetry std.testing.expectEqual(@as(u32, 1), sink.lines);try std.testing.expectEqual(@as(i64, 1000), sink.last_input);+ try std.testing.expect(sink.last_miss == null);note(&agent, 1, 2, 3, 4);drop(); // the call failedemit(&sink, "main", false, "gpt-6-sol");try std.testing.expectEqual(@as(u32, 1), sink.lines);+ // #1320: a zero-read request says why.+ note(&agent, 50_000, 0, 0, 9);+ emit(&sink, "main", false, "gpt-6-sol");+ try std.testing.expectEqualStrings("first_request", sink.last_miss.?);+}++test {+ _ = cache_ttl;}