Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Cache prompts on OpenRouter: sticky session and Anthropic breakpoint (#1284)

justrach216ef6cd9b3d
diff --git a/src/agent_request_body.zig b/src/agent_request_body.zig
index d3e1399b..d49f816a 100644
--- a/src/agent_request_body.zig
+++ b/src/agent_request_body.zig
@@ -170,6 +170,8 @@ pub fn buildBody(self: *Agent, tools_in: ?[]const u8, force_tool: bool, stream:
// official maximize-hits). Root is the project id; children
// share a role lane so sibling scouts reuse system+tools.
try http_headers.writeRequestCacheKey(&s, self.io, self.label, self, self.provider.id);
+ var session_key: [96]u8 = undefined; // #1284: OpenRouter's sticky session + Anthropic breakpoint
+ try @import("openrouter_cache.zig").write(&s, self.provider.id, self.provider.model, http_headers.requestCacheKey(self.io, self.label, self, self.provider.id, &session_key), @import("cache_ttl.zig").control(self));
// Provider-specific thinking controls and reasoning effort.
try @import("zai_wire.zig").writeChatExtras(&s, self.provider.id, self.provider.model, self.sendReasoningEffort(), @import("effort_route.zig").wireEffort(self.provider.id, self.provider.model, @tagName(self.reasoning)));
// --output-schema: structured outputs (xAI docs' response_format).
diff --git a/src/cache_ttl.zig b/src/cache_ttl.zig
index 30968eae..d49d6d0e 100644
--- a/src/cache_ttl.zig
+++ b/src/cache_ttl.zig
@@ -29,15 +29,21 @@ pub fn gap() ?i64 {
return gap_ms;
}
-/// Only the Anthropic API gets a TTL; Anthropic-format providers keep the default.
-pub fn longTtl(provider_id: []const u8, since_previous: ?i64) bool {
+/// The Anthropic API, and Anthropic models through OpenRouter (#1284), take a
+/// TTL; other Anthropic-format providers keep the default.
+pub fn takesTtl(provider_id: []const u8, model: []const u8) bool {
+ return std.mem.eql(u8, provider_id, "anthropic") or
+ (std.mem.eql(u8, provider_id, "openrouter") and std.mem.startsWith(u8, model, "anthropic/"));
+}
+
+pub fn longTtl(provider_id: []const u8, model: []const u8, since_previous: ?i64) bool {
const g = since_previous orelse return false;
- return std.mem.eql(u8, provider_id, "anthropic") and g >= slow_gap_ms;
+ return takesTtl(provider_id, model) and g >= slow_gap_ms;
}
/// The cache_control object for this request's breakpoints.
pub fn control(self: *const Agent) []const u8 {
- return if (longTtl(self.provider.id, gap_ms)) ephemeral_1h else ephemeral;
+ return if (longTtl(self.provider.id, self.provider.model, gap_ms)) ephemeral_1h else ephemeral;
}
/// Why a request with real input read nothing from the cache. Content-free.
@@ -49,11 +55,14 @@ pub fn missReason(cache_read: i64, input: i64, prefix_changed: bool, since_previ
}
test "#1320: slow calls get the 1-hour TTL on the Anthropic API only" {
- try std.testing.expect(!longTtl("anthropic", null));
- try std.testing.expect(!longTtl("anthropic", 90_000));
- try std.testing.expect(longTtl("anthropic", 307_000));
- try std.testing.expect(!longTtl("kimi", 307_000));
- try std.testing.expect(!longTtl("codex", 307_000));
+ try std.testing.expect(!longTtl("anthropic", "claude-opus-5", null));
+ try std.testing.expect(!longTtl("anthropic", "claude-opus-5", 90_000));
+ try std.testing.expect(longTtl("anthropic", "claude-opus-5", 307_000));
+ try std.testing.expect(!longTtl("kimi", "k3", 307_000));
+ try std.testing.expect(!longTtl("codex", "gpt-6-sol", 307_000));
+ // #1284: Anthropic models through OpenRouter take the TTL too.
+ try std.testing.expect(longTtl("openrouter", "anthropic/claude-sonnet-5", 307_000));
+ try std.testing.expect(!longTtl("openrouter", "deepseek/deepseek-v4-pro", 307_000));
}
test "#1320: an Anthropic body after a slow call marks every breakpoint 1h; Kimi keeps the default" {
@@ -103,3 +112,7 @@ test "#1320: a zero-read request names its likely cause" {
try std.testing.expectEqualStrings("idle_gap", missReason(0, 90_000, false, 307_000).?);
try std.testing.expectEqualStrings("unknown", missReason(0, 90_000, false, 20_000).?);
}
+
+test {
+ _ = @import("openrouter_cache.zig");
+}
diff --git a/src/openrouter_cache.zig b/src/openrouter_cache.zig
new file mode 100644
index 00000000..68125c8b
--- /dev/null
+++ b/src/openrouter_cache.zig
@@ -0,0 +1,58 @@
+//! #1284: OpenRouter chat completions carried no cache hints, so Anthropic
+//! models routed there paid full input price every turn. OpenRouter documents
+//! two fields for it: a top-level `cache_control` that puts the breakpoint on
+//! the last cacheable block (Anthropic models; the others cache on their own),
+//! and `session_id`, the sticky-routing key that sends later requests to the
+//! provider endpoint holding the prefix. Its usage already reports
+//! `prompt_tokens_details.cached_tokens` and `cache_write_tokens`, which
+//! agent_context reads, so hits show up in cost and the `usage` trace.
+
+const std = @import("std");
+const Agent = @import("agent.zig").Agent;
+
+pub fn write(s: anytype, provider_id: []const u8, model: []const u8, session_key: []const u8, cache_control: []const u8) !void {
+ if (!std.mem.eql(u8, provider_id, "openrouter")) return;
+ try s.objectField("session_id");
+ try s.write(session_key);
+ if (!std.mem.startsWith(u8, model, "anthropic/")) return;
+ try s.objectField("cache_control");
+ try s.print("{s}", .{cache_control});
+}
+
+fn body(arena: std.mem.Allocator, provider_id: []const u8, model: []const u8) ![]u8 {
+ var messages = std.json.Array.init(arena);
+ var user: std.json.ObjectMap = .empty;
+ try user.put(arena, "role", .{ .string = "user" });
+ try user.put(arena, "content", .{ .string = "hello" });
+ try messages.append(.{ .object = user });
+ var agent: Agent = .{
+ .gpa = std.testing.allocator,
+ .arena = arena,
+ .io = std.testing.io,
+ .client = undefined,
+ .provider = .{ .id = provider_id, .kind = .openai, .auth = .bearer, .url = "", .api_key = "k", .model = model, .context = 200_000 },
+ .messages = messages,
+ .sub = false,
+ .label = "main",
+ .out = null,
+ .sys_normal = "system",
+ };
+ const built = try agent.buildBody(null, false, true, true);
+ defer std.testing.allocator.free(built);
+ return arena.dupe(u8, built);
+}
+
+test "#1284: OpenRouter requests pin a session; Anthropic models there get a cache breakpoint" {
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ const arena = arena_state.allocator();
+ const claude = try body(arena, "openrouter", "anthropic/claude-sonnet-5");
+ try std.testing.expect(std.mem.indexOf(u8, claude, "\"session_id\":\"") != null);
+ try std.testing.expect(std.mem.indexOf(u8, claude, "\"cache_control\":{\"type\":\"ephemeral\"}") != null);
+ const other = try body(arena, "openrouter", "deepseek/deepseek-v4-pro");
+ try std.testing.expect(std.mem.indexOf(u8, other, "\"session_id\":\"") != null);
+ try std.testing.expect(std.mem.indexOf(u8, other, "cache_control") == null); // caches on its own
+ const direct = try body(arena, "openai", "gpt-5.6");
+ try std.testing.expect(std.mem.indexOf(u8, direct, "session_id") == null);
+ try std.testing.expect(std.mem.indexOf(u8, direct, "cache_control") == null);
+}
diff --git a/src/usage_trace.zig b/src/usage_trace.zig
index 9fe20fd1..6597f9ce 100644
--- a/src/usage_trace.zig
+++ b/src/usage_trace.zig
@@ -54,7 +54,7 @@ pub fn note(self: *const Agent, ordinary: i64, cache_read: i64, cache_write: i64
// Sent as previous_response_id + the new items on a held socket.
.chained = self.codex_prev_id != null,
.gap_ms = gap,
- .cache_ttl = if (cache_ttl.longTtl(self.provider.id, gap)) "1h" else null,
+ .cache_ttl = if (cache_ttl.longTtl(self.provider.id, self.provider.model, gap)) "1h" else null,
.prefix_changed = !snap.same,
.prefix_bust = if (snap.same) null else hud.bustLabel(snap.last_bust),
.cache_miss = cache_ttl.missReason(cache_read, input, !snap.same, gap),