Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

chatgpt-new: loaded tools ride additional_tools items, tools stays frozen

justrachdd76d45f8265
diff --git a/docs/adr/0221-chatgpt-new-sign-in.md b/docs/adr/0221-chatgpt-new-sign-in.md
index a94355c6..2ebc25b7 100644
--- a/docs/adr/0221-chatgpt-new-sign-in.md
+++ b/docs/adr/0221-chatgpt-new-sign-in.md
@@ -133,6 +133,18 @@ process rather than restarting anything.
reads its SSE text. The tool catalog goes flat, as it does to the metered
API; if the route starts enforcing namespaces, the fix is to wrap it in
one.
+- **Deferred tools keep `tools` frozen.** The stable catalog appends each
+ loaded tool to the end of `tools`, but OpenAI renders tools ahead of every
+ message and checks cache breakpoints only at message ends, so each load
+ made the next request miss the cache for the whole conversation. Codex
+ hides that tail behind hosted tool search; this route refuses it. So here
+ the tail never reaches `tools`: before each request, every loaded tool that
+ the history does not carry yet is announced in one developer-role
+ `additional_tools` input item, the mechanism OpenAI's prompt-caching guide
+ recommends (`additional_tools.zig`). An item that compaction prunes is
+ announced again; on any other route the items are removed and the tail
+ carries the tools as before. Before this, the MCP suite hit the cache on
+ 12 to 29% of its input on this route against 60 to 65% on Codex.
- **Transport**: live turns use the Responses WebSocket like codex, chaining
`previous_response_id` on the held socket, steering GPT-6 turns, and
prewarming when `GRAFF_WS_PREWARM` is set. One-shot and quiet turns stay on
diff --git a/src/additional_tools.zig b/src/additional_tools.zig
new file mode 100644
index 00000000..8599edc7
--- /dev/null
+++ b/src/additional_tools.zig
@@ -0,0 +1,118 @@
+//! Deferred tools on the ChatGPT plan's new sign-in (ADR 0221), loaded the way
+//! OpenAI's prompt-caching guide asks: without touching `tools`.
+//!
+//! The stable catalog (ADR 0011) appends each loaded tool to the END of the
+//! `tools` array. OpenAI renders tools ahead of every message and checks cache
+//! breakpoints only at message ends, so any change to `tools` makes the whole
+//! conversation miss the cache on the next request. Codex hides that tail
+//! behind hosted tool search (`defer_loading`), which this route refuses.
+//! OpenAI's documented alternative is a developer-role `additional_tools`
+//! input item: it adds the loaded definitions where the load happened, after
+//! the prefix it would otherwise invalidate.
+//!
+//! So on this route the tail never reaches `tools`, and before each request
+//! `sync` announces every loaded tool that no `additional_tools` item in the
+//! history carries yet: once per tool, and again only if compaction pruned
+//! the item that carried it.
+
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const Value = std.json.Value;
+const Io = std.Io;
+
+const Agent = @import("agent.zig").Agent;
+const Provider = @import("provider.zig").Provider;
+const mcp = @import("mcp.zig");
+const mcp_schema_gate = @import("mcp_schema_gate.zig");
+const native_fold = @import("native_fold.zig");
+
+pub const item_type = "additional_tools";
+
+/// Routes that load deferred tools through `additional_tools` items.
+pub fn active(provider_id: []const u8, kind: Provider.Kind) bool {
+ return kind == .responses and std.mem.eql(u8, provider_id, "chatgpt-new") and mcp_schema_gate.g_stable_catalog;
+}
+
+/// The loaded tail as Responses tool entries: exactly what the stable catalog
+/// would append to `tools`.
+fn tailEntries(self: *Agent, arena: Allocator) ![]const Value {
+ const connected: []const mcp.Tool = if (self.registry) |reg| try reg.snapshotTools(arena) else &.{};
+ var aw: Io.Writer.Allocating = .init(arena);
+ var s: std.json.Stringify = .{ .writer = &aw.writer };
+ try s.beginArray();
+ try native_fold.renderLoadedTail(&s, .responses, arena, connected);
+ try s.endArray();
+ const v = try std.json.parseFromSliceLeaky(Value, arena, aw.writer.buffered(), .{ .allocate = .alloc_always });
+ return if (v == .array) v.array.items else &.{};
+}
+
+fn toolName(tool: Value) []const u8 {
+ if (tool != .object) return "";
+ const n = tool.object.get("name") orelse return "";
+ return if (n == .string) n.string else "";
+}
+
+pub fn isItem(m: Value) bool {
+ if (m != .object) return false;
+ const t = m.object.get("type") orelse return false;
+ return t == .string and std.mem.eql(u8, t.string, item_type);
+}
+
+fn announced(messages: []const Value, name: []const u8) bool {
+ for (messages) |m| if (isItem(m)) if (m.object.get("tools")) |list| if (list == .array) {
+ for (list.array.items) |t| if (std.mem.eql(u8, toolName(t), name)) return true;
+ };
+ return false;
+}
+
+fn inTail(tail: []const Value, name: []const u8) bool {
+ for (tail) |t| if (std.mem.eql(u8, toolName(t), name)) return true;
+ return false;
+}
+
+/// Before a request: announce each loaded tool the history does not carry yet.
+/// On any other route, remove the items instead. Other wires do not know them,
+/// and there the catalog tail carries the loaded tools.
+pub fn sync(self: *Agent) void {
+ if (self.sub) return;
+ if (!active(self.provider.id, self.provider.kind)) return dropItems(&self.messages);
+ const arena = self.messageMutationAlloc();
+ const tail = tailEntries(self, arena) catch return;
+ var fresh = std.json.Array.init(arena);
+ for (tail) |t| if (!announced(self.messages.items, toolName(t))) fresh.append(t) catch return;
+ if (fresh.items.len == 0) return;
+ var item: std.json.ObjectMap = .empty;
+ item.put(arena, "type", .{ .string = item_type }) catch return;
+ item.put(arena, "role", .{ .string = "developer" }) catch return;
+ item.put(arena, "tools", .{ .array = fresh }) catch return;
+ self.messages.append(.{ .object = item }) catch {};
+}
+
+fn dropItems(messages: *std.json.Array) void {
+ var i: usize = 0;
+ while (i < messages.items.len) {
+ if (isItem(messages.items[i])) _ = messages.orderedRemove(i) else i += 1;
+ }
+}
+
+/// The request's `tools` without the loaded tail. Always re-serialized on this
+/// route, even with nothing loaded, so the bytes stay identical from the first
+/// request to the last.
+pub fn stripTail(self: *Agent, arena: Allocator, tools_json: []const u8) []const u8 {
+ const tail = tailEntries(self, arena) catch return tools_json;
+ var v = std.json.parseFromSliceLeaky(Value, arena, tools_json, .{ .allocate = .alloc_always }) catch return tools_json;
+ if (v != .array) return tools_json;
+ var i: usize = 0;
+ while (i < v.array.items.len) {
+ const name = toolName(v.array.items[i]);
+ if (name.len > 0 and inTail(tail, name)) _ = v.array.orderedRemove(i) else i += 1;
+ }
+ var aw: Io.Writer.Allocating = .init(arena);
+ var s: std.json.Stringify = .{ .writer = &aw.writer };
+ s.write(v) catch return tools_json;
+ return aw.writer.buffered();
+}
+
+test {
+ _ = @import("additional_tools_tests.zig");
+}
diff --git a/src/additional_tools_tests.zig b/src/additional_tools_tests.zig
new file mode 100644
index 00000000..5c941b73
--- /dev/null
+++ b/src/additional_tools_tests.zig
@@ -0,0 +1,109 @@
+//! Tests for additional_tools.zig (ADR 0221): a load on the ChatGPT plan
+//! route must leave `tools` byte-identical and announce the tool in the input.
+
+const std = @import("std");
+const additional_tools = @import("additional_tools.zig");
+const native_fold = @import("native_fold.zig");
+const mcp_schema_gate = @import("mcp_schema_gate.zig");
+const testAgentFor = @import("agent_request_body_responses.zig").testAgentFor;
+
+/// The body's top-level `tools` array, verbatim: the key right after `input`
+/// closes, not an `additional_tools` item's own `tools` inside the input.
+fn toolsOf(body: []const u8) []const u8 {
+ const start = (std.mem.indexOf(u8, body, "],\"tools\":") orelse return "") + "],\"tools\":".len;
+ const end = std.mem.indexOfPos(u8, body, start, ",\"tool_choice\"") orelse return "";
+ return body[start..end];
+}
+
+fn items(messages: []const std.json.Value) usize {
+ var n: usize = 0;
+ for (messages) |m| if (additional_tools.isItem(m)) {
+ n += 1;
+ };
+ return n;
+}
+
+test "ChatGPT plan: a tool load leaves tools byte-identical and adds one additional_tools item" {
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ const a = arena_state.allocator();
+ const saved_fold = native_fold.enabled;
+ const saved_stable = mcp_schema_gate.g_stable_catalog;
+ defer {
+ native_fold.enabled = saved_fold;
+ mcp_schema_gate.g_stable_catalog = saved_stable;
+ native_fold.clearLoadedSession();
+ }
+ native_fold.enabled = true;
+ mcp_schema_gate.g_stable_catalog = true;
+ native_fold.clearLoadedSession();
+
+ var agent = try testAgentFor(a, "chatgpt-new", .responses, "gpt-6.1-sol");
+ agent.invalidateRootTools(); // the test agent starts on the subagent catalog
+ try agent.ensureRootTools(.responses);
+ additional_tools.sync(&agent);
+ try std.testing.expectEqual(@as(usize, 0), items(agent.messages.items)); // nothing loaded yet
+ const before = try agent.buildBody(agent.tools_responses, false, true, true);
+ defer std.testing.allocator.free(before);
+
+ // The model loads a folded native; the catalog tail now carries it.
+ native_fold.markLoaded("workflow");
+ agent.invalidateRootTools();
+ try agent.ensureRootTools(.responses);
+ try std.testing.expect(std.mem.indexOf(u8, agent.tools_responses, "\"name\":\"workflow\"") != null);
+
+ additional_tools.sync(&agent);
+ try std.testing.expectEqual(@as(usize, 1), items(agent.messages.items));
+ const after = try agent.buildBody(agent.tools_responses, false, true, true);
+ defer std.testing.allocator.free(after);
+
+ // `tools` is the same bytes before and after the load, without the tail...
+ try std.testing.expect(toolsOf(before).len > 2);
+ try std.testing.expectEqualStrings(toolsOf(before), toolsOf(after));
+ try std.testing.expect(std.mem.indexOf(u8, toolsOf(after), "\"name\":\"workflow\"") == null);
+ // ...and the definition arrives as an input item after the prefix.
+ try std.testing.expect(std.mem.indexOf(u8, before, "\"type\":\"additional_tools\"") == null);
+ const at = std.mem.indexOf(u8, after, "\"type\":\"additional_tools\"") orelse return error.TestExpectedItem;
+ try std.testing.expect(at < (std.mem.indexOf(u8, after, "],\"tools\":") orelse return error.TestExpectedTools));
+ try std.testing.expect(std.mem.indexOf(u8, after[at..], "\"name\":\"workflow\"") != null);
+
+ // A second sync announces nothing new.
+ additional_tools.sync(&agent);
+ try std.testing.expectEqual(@as(usize, 1), items(agent.messages.items));
+}
+
+test "ChatGPT plan: a pruned item is announced again; other routes drop the items" {
+ var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena_state.deinit();
+ const a = arena_state.allocator();
+ const saved_fold = native_fold.enabled;
+ const saved_stable = mcp_schema_gate.g_stable_catalog;
+ defer {
+ native_fold.enabled = saved_fold;
+ mcp_schema_gate.g_stable_catalog = saved_stable;
+ native_fold.clearLoadedSession();
+ }
+ native_fold.enabled = true;
+ mcp_schema_gate.g_stable_catalog = true;
+ native_fold.clearLoadedSession();
+ native_fold.markLoaded("workflow");
+
+ var agent = try testAgentFor(a, "chatgpt-new", .responses, "gpt-6.1-sol");
+ additional_tools.sync(&agent);
+ try std.testing.expectEqual(@as(usize, 1), items(agent.messages.items));
+
+ // Compaction pruned everything before its blob, the item included.
+ _ = agent.messages.pop();
+ additional_tools.sync(&agent);
+ try std.testing.expectEqual(@as(usize, 1), items(agent.messages.items));
+
+ // Switching to Codex removes the items: there the catalog tail carries the tool.
+ agent.provider.id = "codex";
+ additional_tools.sync(&agent);
+ try std.testing.expectEqual(@as(usize, 0), items(agent.messages.items));
+ agent.invalidateRootTools();
+ try agent.ensureRootTools(.responses);
+ try std.testing.expect(std.mem.indexOf(u8, agent.tools_responses, "\"name\":\"workflow\"") != null);
+ try std.testing.expect(!additional_tools.active("codex", .responses));
+ try std.testing.expect(!additional_tools.active("chatgpt-new", .openai));
+}
diff --git a/src/agent_request.zig b/src/agent_request.zig
index 69dd9349..74a851e5 100644
--- a/src/agent_request.zig
+++ b/src/agent_request.zig
@@ -157,6 +157,8 @@ pub fn request(self: *Agent, tools_in: ?[]const u8) !std.json.ObjectMap {
// #95: scrub any malformed function_call_output before it hits the wire.
const message_arena = self.messageMutationAlloc();
@import("history_wire.zig").prepare(message_arena, self.provider.kind, &self.messages);
+ // ADR 0221: on the ChatGPT plan route loaded tools ride additional_tools items, so `tools` never changes.
+ if (!self.compaction_request and !self.server_compaction_request) @import("additional_tools.zig").sync(self);
// #193 follow-up: bound any single oversized tool output (an uncapped MCP
// result, a huge fetch on a small-window model) before send. The responses
// path already hard-caps output above (normalizeResponsesHistory); this is the
diff --git a/src/agent_request_body_responses.zig b/src/agent_request_body_responses.zig
index 4dcb04a8..cac34f99 100644
--- a/src/agent_request_body_responses.zig
+++ b/src/agent_request_body_responses.zig
@@ -12,6 +12,7 @@ const codex_chain = @import("codex_chain.zig");
const server_compact = @import("agent_server_compact.zig");
const xai_hosted = @import("xai_hosted.zig");
const codex_tool_search = @import("codex_tool_search.zig");
+const additional_tools = @import("additional_tools.zig");
pub fn write(self: *Agent, s: *std.json.Stringify, tools: ?[]const u8, force_tool: bool) !void {
// Responses API (codex / ChatGPT, xAI, and native Codegraff aliases).
@@ -69,6 +70,8 @@ pub fn write(self: *Agent, s: *std.json.Stringify, tools: ?[]const u8, force_too
next = codex_tool_search.splice(self.scratchAlloc(), next) catch next;
next = codex_tool_search.spliceWebSearch(self.scratchAlloc(), next) catch next;
}
+ // ADR 0221: no hosted tool search here; the loaded tail rides additional_tools items instead.
+ if (additional_tools.active(self.provider.id, self.provider.kind)) next = additional_tools.stripTail(self, self.scratchAlloc(), next);
break :blk next;
};
const async_payload = try @import("async_tool_policy.zig").decorate(self.scratchAlloc(), self.provider, payload, self.async_tools_armed and !self.sub and !self.compaction_request and !self.server_compaction_request);
diff --git a/src/native_fold.zig b/src/native_fold.zig
index c6b8b528..34ce4c4a 100644
--- a/src/native_fold.zig
+++ b/src/native_fold.zig
@@ -274,9 +274,11 @@ pub fn loadedNames() []const []const u8 {
/// GRAFF_STABLE_CATALOG tail (schema.zig renderRootTools, #476): every tool
/// whose schema has loaded this session, appended in LOAD ORDER after the
/// stable catalog head — natives and MCP tools share one admission sequence.
-/// Append-only is the point: a load changes only the tools array's tail
-/// bytes, and the provider's prefix cache (tools serialize before messages)
-/// survives what a mid-array re-insertion would bust.
+/// Append-only keeps the head's bytes, but the tail still changes the tools
+/// array, and tools serialize before every message: on OpenAI's Responses
+/// caching, whose breakpoints sit at message ends, the next request misses.
+/// Codex hides the tail behind hosted tool search (defer_loading); the ChatGPT
+/// plan route moves it into additional_tools input items (additional_tools.zig).
pub fn renderLoadedTail(s: *std.json.Stringify, kind: @import("provider.zig").Provider.Kind, out: Allocator, mcp_tools: []const @import("mcp.zig").Tool) !void {
const schema_mod = @import("schema.zig");
const mcp = @import("mcp.zig");
diff --git a/src/schema.zig b/src/schema.zig
index 22fae4b4..459efd29 100644
--- a/src/schema.zig
+++ b/src/schema.zig
@@ -258,7 +258,8 @@ pub fn renderRootTools(
for (mcp_tools) |m| if (!mcp_schema_gate.omitMcp(m.qualified_name) and !mcp_schema_gate.isDeferred(mcp_tools, m) and !(mcp_schema_gate.g_stable_catalog and mcp_schema_gate.policyDeferred(mcp_tools, m)))
try writeToolEntry(&s, kind, m.qualified_name, m.description, .{ .value = m.input_schema });
// GRAFF_STABLE_CATALOG (#476): loaded tools append in LOAD ORDER after the
- // stable head — loads change only tail bytes, the prefix cache survives.
+ // stable head, so loads change only tail bytes (see renderLoadedTail for
+ // why that alone does not keep an OpenAI cache hit).
if (mcp_schema_gate.g_stable_catalog) try native_fold.renderLoadedTail(&s, kind, out, mcp_tools);
try s.endArray();
return aw.toOwnedSlice();
diff --git a/src/test_hooks.zig b/src/test_hooks.zig
index 26919cfb..043e7659 100644
--- a/src/test_hooks.zig
+++ b/src/test_hooks.zig
@@ -396,4 +396,5 @@ test {
_ = @import("destructive_git.zig"); // #1269: destructive-git tokenizer tables
_ = @import("output_elide.zig"); // #1271: over-cap output keeps head and tail
_ = @import("graff_dir.zig"); // #1273: .graff/.gitignore keeps run state out of git
+ _ = @import("additional_tools.zig"); // ADR 0221: loaded tools as additional_tools items on the ChatGPT plan route
}