Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Name why a request read nothing from the cache; keep slow Anthropic calls warm (#1320)

justrach1d1a7c7a07ce
diff --git a/docs/architecture.md b/docs/architecture.md
index 2837dcc2..d998f2a4 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -346,6 +346,16 @@ included), `cache_read_tokens`, `cache_write_tokens`, `output_tokens`, and
`chained` (sent as `previous_response_id` plus new items). A cache hit rate is
`cache_read_tokens / input_tokens` from `usage`, never from `context_tokens`.
+`usage` also says why a read dropped (#1320). It records `gap_ms` (time since
+this agent's previous request began), `cache_ttl` (`"1h"` when requested),
+`prefix_changed` and `prefix_bust` (the root's system+tools prefix moved, and
+the named reason: tools, mcp, mode, compact, …), and, on a request with real
+input that read nothing, `cache_miss`: `prefix_changed`, `first_request`,
+`idle_gap` (5+ minutes since the previous request, past a default entry's
+life), or `unknown`. On the Anthropic API a request whose predecessor began
+3+ minutes earlier asks for the 1-hour TTL, so a slow call no longer returns
+to an expired prefix.
+
## Session persistence
`/save` serializes `{provider, model, strict, messages}` with
diff --git a/src/agent_request.zig b/src/agent_request.zig
index c10343e5..f40b397c 100644
--- a/src/agent_request.zig
+++ b/src/agent_request.zig
@@ -202,6 +202,7 @@ pub fn request(self: *Agent, tools_in: ?[]const u8) !std.json.ObjectMap {
var openai404_retries: usize = 0; // #opencode-parity: bounded retries for OpenAI's spurious model 404s
const max_openai404_retries: usize = 2;
var gw_retry = policy.GatewayRetryState{}; // #gateway-artifact state (agent_gateway_retry.zig)
+ @import("cache_ttl.zig").begin(self); // #1320: before request_started moves
rebuild: while (true) {
if (@import("agent_async_tools.zig").started(self)) return error.AsyncToolStreamFailed;
const live = self.usesLiveTransport();
diff --git a/src/agent_request_body.zig b/src/agent_request_body.zig
index 9b35199c..d3e1399b 100644
--- a/src/agent_request_body.zig
+++ b/src/agent_request_body.zig
@@ -76,7 +76,7 @@ pub fn buildBody(self: *Agent, tools_in: ?[]const u8, force_tool: bool, stream:
// other anthropic-format providers (minimax) get a plain string.
try s.objectField("system");
const sys = try @import("agent_request_body_responses.zig").schemaAwarePrompt(self);
- const cc = "{\"type\":\"ephemeral\"}";
+ const cc = @import("cache_ttl.zig").control(self); // #1320: 1h when calls outlive 5m
if (std.mem.eql(u8, self.provider.id, "anthropic") or is_kimi) {
try s.beginArray();
try s.beginObject();
@@ -106,7 +106,7 @@ pub fn buildBody(self: *Agent, tools_in: ?[]const u8, force_tool: bool, stream:
// Anthropic API and Kimi's declared Anthropic transport. Kimi also
// normalizes all string content into Anthropic text blocks.
const cache_msgs = std.mem.eql(u8, self.provider.id, "anthropic") or is_kimi;
- try writeAnthropicMessages(&s, self.messages, cache_msgs, is_kimi);
+ try writeAnthropicMessages(&s, self.messages, if (cache_msgs) cc else null, is_kimi);
},
.openai => {
// graff's MakeOpenAiCompat: OpenAI deprecated max_tokens in
diff --git a/src/cache_ttl.zig b/src/cache_ttl.zig
new file mode 100644
index 00000000..30968eae
--- /dev/null
+++ b/src/cache_ttl.zig
@@ -0,0 +1,105 @@
+//! #1320: an Anthropic cache entry lives 5 minutes from its last use. A model
+//! call that runs longer (slow reasoning at a large context) comes back to an
+//! expired prefix, and the next request reads nothing. When this agent's
+//! previous request began 3+ minutes ago, calls are that slow: its successor
+//! asks for the 1-hour TTL (a 2x write instead of 1.25x, on the tokens it
+//! writes) so the entry outlives the next slow call. The gap is taken once
+//! per request, before any retry, and usage_trace.zig reports it with the
+//! reason a request read nothing.
+
+const std = @import("std");
+const Agent = @import("agent.zig").Agent;
+
+pub const ephemeral = "{\"type\":\"ephemeral\"}";
+pub const ephemeral_1h = "{\"type\":\"ephemeral\",\"ttl\":\"1h\"}";
+/// A previous request this long ago means calls outlive a 5-minute entry.
+pub const slow_gap_ms: i64 = 3 * 60 * 1000;
+/// The provider's default entry lifetime (Anthropic's; others are similar).
+pub const default_ttl_ms: i64 = 5 * 60 * 1000;
+
+/// Requests and their usage lines run on one thread (see usage_trace.zig).
+threadlocal var gap_ms: ?i64 = null;
+
+/// Before a request's retry loop: time since this agent's previous request began.
+pub fn begin(self: *const Agent) void {
+ gap_ms = if (self.request_started) |t| t.untilNow(self.io, .awake).toMilliseconds() else null;
+}
+
+pub fn gap() ?i64 {
+ return gap_ms;
+}
+
+/// Only the Anthropic API gets a TTL; Anthropic-format providers keep the default.
+pub fn longTtl(provider_id: []const u8, since_previous: ?i64) bool {
+ const g = since_previous orelse return false;
+ return std.mem.eql(u8, provider_id, "anthropic") and g >= slow_gap_ms;
+}
+
+/// The cache_control object for this request's breakpoints.
+pub fn control(self: *const Agent) []const u8 {
+ return if (longTtl(self.provider.id, gap_ms)) ephemeral_1h else ephemeral;
+}
+
+/// Why a request with real input read nothing from the cache. Content-free.
+pub fn missReason(cache_read: i64, input: i64, prefix_changed: bool, since_previous: ?i64) ?[]const u8 {
+ if (cache_read > 0 or input < 1024) return null;
+ if (prefix_changed) return "prefix_changed";
+ const g = since_previous orelse return "first_request";
+ return if (g >= default_ttl_ms) "idle_gap" else "unknown";
+}
+
+test "#1320: slow calls get the 1-hour TTL on the Anthropic API only" {
+ try std.testing.expect(!longTtl("anthropic", null));
+ try std.testing.expect(!longTtl("anthropic", 90_000));
+ try std.testing.expect(longTtl("anthropic", 307_000));
+ try std.testing.expect(!longTtl("kimi", 307_000));
+ try std.testing.expect(!longTtl("codex", 307_000));
+}
+
+test "#1320: an Anthropic body after a slow call marks every breakpoint 1h; Kimi keeps the default" {
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+ var messages = std.json.Array.init(arena);
+ var user: std.json.ObjectMap = .empty;
+ try user.put(arena, "role", .{ .string = "user" });
+ try user.put(arena, "content", .{ .string = "hello" });
+ try messages.append(.{ .object = user });
+ var agent: Agent = .{
+ .gpa = std.testing.allocator,
+ .arena = arena,
+ .io = std.testing.io,
+ .client = undefined,
+ .provider = .{ .id = "anthropic", .kind = .anthropic, .auth = .x_api_key, .url = "", .api_key = "k", .model = "claude-fixture", .context = 200_000 },
+ .messages = messages,
+ .sub = false,
+ .label = "",
+ .out = null,
+ .sys_normal = "system",
+ };
+ defer gap_ms = null;
+ const long = "\"cache_control\":" ++ ephemeral_1h;
+ gap_ms = 307_000;
+ const slow = try agent.buildBody(null, false, true, true);
+ defer std.testing.allocator.free(slow);
+ try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, slow, long)); // system + last message
+ try std.testing.expect(std.mem.indexOf(u8, slow, "\"cache_control\":" ++ ephemeral ++ "") == null);
+ gap_ms = 20_000;
+ const fast = try agent.buildBody(null, false, true, true);
+ defer std.testing.allocator.free(fast);
+ try std.testing.expect(std.mem.indexOf(u8, fast, "\"ttl\"") == null);
+ agent.provider.id = "kimi";
+ gap_ms = 307_000;
+ const kimi = try agent.buildBody(null, false, true, true);
+ defer std.testing.allocator.free(kimi);
+ try std.testing.expect(std.mem.indexOf(u8, kimi, "\"ttl\"") == null);
+}
+
+test "#1320: a zero-read request names its likely cause" {
+ try std.testing.expect(missReason(900, 1000, true, 1) == null);
+ try std.testing.expect(missReason(0, 200, false, null) == null); // below any cacheable size
+ try std.testing.expectEqualStrings("prefix_changed", missReason(0, 90_000, true, 400_000).?);
+ try std.testing.expectEqualStrings("first_request", missReason(0, 90_000, false, null).?);
+ try std.testing.expectEqualStrings("idle_gap", missReason(0, 90_000, false, 307_000).?);
+ try std.testing.expectEqualStrings("unknown", missReason(0, 90_000, false, 20_000).?);
+}
diff --git a/src/serde.zig b/src/serde.zig
index 4f174f9f..00134b5d 100644
--- a/src/serde.zig
+++ b/src/serde.zig
@@ -91,7 +91,7 @@ fn anthropicCacheableBlock(value: Value) bool {
return false;
}
-fn writeObjectWithCache(s: *std.json.Stringify, obj: std.json.ObjectMap) !void {
+fn writeObjectWithCache(s: *std.json.Stringify, obj: std.json.ObjectMap, cache_control: []const u8) !void {
try s.beginObject();
var it = obj.iterator();
while (it.next()) |kv| {
@@ -100,16 +100,16 @@ fn writeObjectWithCache(s: *std.json.Stringify, obj: std.json.ObjectMap) !void {
}
if (obj.get("cache_control") == null) {
try s.objectField("cache_control");
- try s.print("{s}", .{"{\"type\":\"ephemeral\"}"});
+ try s.print("{s}", .{cache_control});
}
try s.endObject();
}
/// Serialize Anthropic messages. `normalize_blocks` matches the official Kimi
/// adapter by turning every plain string into a `{type:text,text}` content
-/// array. `cache` marks the final cacheable block (including tool_result), not
-/// just the plain-string happy path.
-pub fn writeAnthropicMessages(s: *std.json.Stringify, messages: std.json.Array, cache: bool, normalize_blocks: bool) !void {
+/// array. `cache` (the cache_control object, #1320) marks the final cacheable
+/// block (including tool_result), not just the plain-string happy path.
+pub fn writeAnthropicMessages(s: *std.json.Stringify, messages: std.json.Array, cache: ?[]const u8, normalize_blocks: bool) !void {
const items = messages.items;
try s.beginArray();
for (items, 0..) |m, i| {
@@ -118,7 +118,7 @@ pub fn writeAnthropicMessages(s: *std.json.Stringify, messages: std.json.Array,
continue;
}
const content = m.object.get("content").?;
- const cache_this = cache and i + 1 == items.len;
+ const cache_this = cache != null and i + 1 == items.len;
const string_content = content == .string;
const array_cache = cache_this and content == .array and content.array.items.len > 0 and anthropicCacheableBlock(content.array.items[content.array.items.len - 1]);
if (!normalize_blocks and !cache_this) {
@@ -147,7 +147,7 @@ pub fn writeAnthropicMessages(s: *std.json.Stringify, messages: std.json.Array,
try s.write(content.string);
if (cache_this) {
try s.objectField("cache_control");
- try s.print("{s}", .{"{\"type\":\"ephemeral\"}"});
+ try s.print("{s}", .{cache.?});
}
try s.endObject();
try s.endArray();
@@ -155,7 +155,7 @@ pub fn writeAnthropicMessages(s: *std.json.Stringify, messages: std.json.Array,
try s.beginArray();
for (content.array.items, 0..) |block, block_i| {
if (block_i + 1 == content.array.items.len and anthropicCacheableBlock(block))
- try writeObjectWithCache(s, block.object)
+ try writeObjectWithCache(s, block.object, cache.?)
else
try s.write(block);
}
@@ -203,7 +203,7 @@ pub fn writeAnthropicTools(s: *std.json.Stringify, arena: Allocator, raw: []cons
try s.beginArray();
for (value.array.items, 0..) |tool, i| {
if (cache and i + 1 == value.array.items.len and tool == .object)
- try writeObjectWithCache(s, tool.object)
+ try writeObjectWithCache(s, tool.object, @import("cache_ttl.zig").ephemeral)
else
try s.write(tool);
}
diff --git a/src/session_wake.zig b/src/session_wake.zig
index 22c4fcb2..36964405 100644
--- a/src/session_wake.zig
+++ b/src/session_wake.zig
@@ -142,7 +142,7 @@ test "notification metadata stays off chat, responses and anthropic wires includ
if (mode == 0) try writeWireArray(&s, msgs.items) else if (mode == 1)
try serde.writeOpenAIMessageNormalized(&s, m)
else
- try serde.writeAnthropicMessages(&s, msgs, mode & 1 == 1, mode & 2 == 2);
+ try serde.writeAnthropicMessages(&s, msgs, if (mode & 1 == 1) @import("cache_ttl.zig").ephemeral else null, mode & 2 == 2);
try std.testing.expect(std.mem.indexOf(u8, aw.written(), origin_key) == null);
try std.testing.expect(std.mem.indexOf(u8, aw.written(), "wake up") != null);
try std.testing.expect(std.mem.indexOf(u8, aw.written(), "\"role\":\"user\"") != null);
diff --git a/src/usage_trace.zig b/src/usage_trace.zig
index 13c1012f..9fe20fd1 100644
--- a/src/usage_trace.zig
+++ b/src/usage_trace.zig
@@ -16,6 +16,8 @@
const std = @import("std");
const Agent = @import("agent.zig").Agent;
+const cache_ttl = @import("cache_ttl.zig");
+const hud = @import("prompt_cache_hud.zig");
const Usage = struct {
provider: []const u8,
@@ -24,20 +26,38 @@ const Usage = struct {
cache_write: i64,
output: i64,
chained: bool,
+ /// #1320: why a read may have dropped. Time since this agent's previous
+ /// request began, the TTL asked for (null: the provider default), whether
+ /// the system+tools prefix changed and the named reason, and on a
+ /// zero-read request with real input the likely cause.
+ gap_ms: ?i64 = null,
+ cache_ttl: ?[]const u8 = null,
+ prefix_changed: bool = false,
+ prefix_bust: ?[]const u8 = null,
+ cache_miss: ?[]const u8 = null,
};
threadlocal var pending: ?Usage = null;
/// From recordCost: remember this response's usage for the next `api` line.
pub fn note(self: *const Agent, ordinary: i64, cache_read: i64, cache_write: i64, out: i64) void {
+ const gap = cache_ttl.gap();
+ // The prefix tracker follows the root session only.
+ const snap = if (self.sub) hud.Snap{} else hud.snapshot();
+ const input = ordinary + cache_read + cache_write;
pending = .{
.provider = self.provider.id,
- .input = ordinary + cache_read + cache_write,
+ .input = input,
.cache_read = cache_read,
.cache_write = cache_write,
.output = out,
// Sent as previous_response_id + the new items on a held socket.
.chained = self.codex_prev_id != null,
+ .gap_ms = gap,
+ .cache_ttl = if (cache_ttl.longTtl(self.provider.id, gap)) "1h" else null,
+ .prefix_changed = !snap.same,
+ .prefix_bust = if (snap.same) null else hud.bustLabel(snap.last_bust),
+ .cache_miss = cache_ttl.missReason(cache_read, input, !snap.same, gap),
};
}
@@ -57,6 +77,11 @@ pub fn emit(tr: anytype, label: []const u8, from_sub: bool, model: []const u8) v
.cache_write_tokens = u.cache_write,
.output_tokens = u.output,
.chained = u.chained,
+ .gap_ms = u.gap_ms,
+ .cache_ttl = u.cache_ttl,
+ .prefix_changed = u.prefix_changed,
+ .prefix_bust = u.prefix_bust,
+ .cache_miss = u.cache_miss,
});
}
@@ -69,25 +94,39 @@ test "usage waits for its api line and is written once" {
const Sink = struct {
lines: u32 = 0,
last_input: i64 = 0,
+ last_miss: ?[]const u8 = null,
fn write(self: *@This(), event: anytype) void {
self.lines += 1;
self.last_input = event.input_tokens;
+ self.last_miss = event.cache_miss;
}
fn elapsedMs(_: *@This()) i64 {
return 0;
}
};
- var agent: Agent = undefined; // only provider and codex_prev_id are read
+ var agent: Agent = undefined; // only provider, sub and codex_prev_id are read
agent.provider.id = "codex";
agent.codex_prev_id = null;
+ agent.sub = true;
+ agent.request_started = null;
+ cache_ttl.begin(&agent); // this thread's slot: no previous request
var sink: Sink = .{};
note(&agent, 100, 900, 0, 7);
emit(&sink, "main", false, "gpt-6-sol");
emit(&sink, "main", false, "gpt-6-sol"); // taken: no second line
try std.testing.expectEqual(@as(u32, 1), sink.lines);
try std.testing.expectEqual(@as(i64, 1000), sink.last_input);
+ try std.testing.expect(sink.last_miss == null);
note(&agent, 1, 2, 3, 4);
drop(); // the call failed
emit(&sink, "main", false, "gpt-6-sol");
try std.testing.expectEqual(@as(u32, 1), sink.lines);
+ // #1320: a zero-read request says why.
+ note(&agent, 50_000, 0, 0, 9);
+ emit(&sink, "main", false, "gpt-6-sol");
+ try std.testing.expectEqualStrings("first_request", sink.last_miss.?);
+}
+
+test {
+ _ = cache_ttl;
}