Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

chat: fixed opening exchange so a new session's first call hits the prompt cache (#1657)

Rach409170671f1a
diff --git a/src/agent_length_cutoff.zig b/src/agent_length_cutoff.zig
index b8030415..8acd6fbc 100644
--- a/src/agent_length_cutoff.zig
+++ b/src/agent_length_cutoff.zig
@@ -30,8 +30,8 @@ pub const note = "Your previous reply ran long without a tool call and was cut o
/// that end in a tool call stay well under it.
pub const runaway_bytes: usize = 24 * 1024;
-/// Rewinds per turn. Argon can run away again after a rewind (a second
-/// runaway in one turn streamed 180 KB for 145 s to the output cap).
+/// Rewinds per turn. A model can run away again after a rewind, and a second
+/// runaway left alone streams all the way to the output cap.
pub const max_rewinds: u8 = 2;
/// The provider stopped because it ran out of output budget, in any of the
diff --git a/src/agent_request_body.zig b/src/agent_request_body.zig
index 7f0735f1..8f9f4409 100644
--- a/src/agent_request_body.zig
+++ b/src/agent_request_body.zig
@@ -168,6 +168,7 @@ pub fn buildBody(self: *Agent, tools_in: ?[]const u8, force_tool: bool, stream:
try s.objectField("content");
try s.write(repo.instructions);
try s.endObject();
+ if (repo_context.statefulChat(self)) try repo_context.writeOpening(&s);
if (repo.context.len != 0) try repo_context.writeChatMessage(&s, self.scratchAlloc(), repo.context);
for (self.messages.items) |m| try writeOpenAIMessageNormalized(&s, m);
try s.endArray();
diff --git a/src/repo_context.zig b/src/repo_context.zig
index d9dc2a1a..3782fc26 100644
--- a/src/repo_context.zig
+++ b/src/repo_context.zig
@@ -104,6 +104,27 @@ pub fn writeItem(s: *std.json.Stringify, text: []const u8) !void {
/// framed so the model reads it as standing context rather than a request.
pub const chat_frame = "Repository context for this session (standing instructions and layout, not a request):\n\n";
+/// Gemini's implicit cache reuses a request's prefix only up to the start of
+/// a later turn: the system prompt and tools alone never count, and the repo
+/// context plus the first prompt is one user turn, so every new session's
+/// first call read nothing from the cache. One fixed exchange ahead of the
+/// context gives every session, in any repo, the same turn boundary after the
+/// system prompt and tools, so a first call in a new repo reads them from the
+/// cache.
+pub const opening_user = "A new coding session is starting.";
+pub const opening_model = "Ready.";
+
+pub fn writeOpening(s: *std.json.Stringify) !void {
+ for ([_][2][]const u8{ .{ "user", opening_user }, .{ "assistant", opening_model } }) |m| {
+ try s.beginObject();
+ try s.objectField("role");
+ try s.write(m[0]);
+ try s.objectField("content");
+ try s.write(m[1]);
+ try s.endObject();
+ }
+}
+
pub fn writeChatMessage(s: *std.json.Stringify, arena: std.mem.Allocator, text: []const u8) !void {
try s.beginObject();
try s.objectField("role");
diff --git a/src/repo_context_tests.zig b/src/repo_context_tests.zig
index 7e9197b5..eacfcbd0 100644
--- a/src/repo_context_tests.zig
+++ b/src/repo_context_tests.zig
@@ -107,11 +107,16 @@ test "ADR 0258: Gemini on the Codegraff chat wire sends the per-repo context onc
const system = messages[0].object.get("content").?.string;
try std.testing.expect(std.mem.startsWith(u8, system, "BASE\n\n# Skills\nstatic tail"));
try std.testing.expect(std.mem.indexOf(u8, system, "# Project") == null);
+ // A fixed opening exchange gives every session the same cacheable turn boundary.
try std.testing.expectEqualStrings("user", messages[1].object.get("role").?.string);
- const context = messages[1].object.get("content").?.string;
+ try std.testing.expectEqualStrings(repo_context.opening_user, messages[1].object.get("content").?.string);
+ try std.testing.expectEqualStrings("assistant", messages[2].object.get("role").?.string);
+ try std.testing.expectEqualStrings(repo_context.opening_model, messages[2].object.get("content").?.string);
+ try std.testing.expectEqualStrings("user", messages[3].object.get("role").?.string);
+ const context = messages[3].object.get("content").?.string;
try std.testing.expect(std.mem.startsWith(u8, context, repo_context.chat_frame ++ "# Project instructions (from AGENTS.md)\nUse tabs."));
try std.testing.expect(std.mem.indexOf(u8, context, "logs/auth.log") != null);
- try std.testing.expectEqualStrings("hello", messages[2].object.get("content").?.string);
+ try std.testing.expectEqualStrings("hello", messages[4].object.get("content").?.string);
// A second repo sends the same system prompt.
var other = try chatRoot(a, "codegraff", "google/gemini-3.7-flash", layout_b);