graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
//! #421 + #410: prompt-snapshot tests. Absent capabilities contribute zero
//! instruction text (exact length + dropped-segment bytes unfindable). The
//! full-capability prompt is the inline golden below so wording drift is a
//! reviewed choice, not a hash nobody reads.
const std = @import("std");
const prompts = @import("prompts.zig");
const skills = @import("skills.zig");
const skill_docs = @import("skill_docs.zig");
const imagegen = @import("imagegen.zig");
const tool_gates = @import("tool_gates.zig");
const no_local_tools = @import("no_local_tools.zig");
const repl_glue = @import("repl_glue.zig");
const session_index = @import("session_index.zig");
const agent_mod = @import("agent.zig");
const golden_full_prompt = @import("prompt_golden.zig").full_prompt;
/// Every capability configuration the gates can actually produce, plus the
/// all-off floor. `.{}` is full capability (the Caps defaults).
const matrix = [_]struct { name: []const u8, caps: prompts.Caps }{
.{ .name = "full", .caps = .{} },
.{ .name = "no-local-tools (#330 embedder)", .caps = .{ .local_tools = false } },
.{ .name = "no-subagents", .caps = .{ .subagents = false } },
.{ .name = "no-todos", .caps = .{ .todos = false } },
.{ .name = "no-constraints", .caps = .{ .constraints = false } },
.{ .name = "no-git-repo (scratch cwd)", .caps = .{ .git_repo = false } },
.{ .name = "floor", .caps = .{ .local_tools = false, .subagents = false, .todos = false, .constraints = false, .git_repo = false } },
};
test "#421 golden: the full-capability root prompt is exactly this, byte for byte" {
var a_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer a_state.deinit();
const a = a_state.allocator();
try std.testing.expectEqualStrings(golden_full_prompt, prompts.main_system_prompt);
// The segment table composes to the same bytes the comptime constant does,
// so a reordered or forgotten segment cannot hide behind the fast path.
try std.testing.expectEqualStrings(golden_full_prompt, try prompts.composeSegments(a, .{}));
// ...and the fast path really is a fast path: full capability allocates nothing.
var b_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer b_state.deinit();
try std.testing.expectEqualStrings(golden_full_prompt, try prompts.composeBase(b_state.allocator(), .{}));
try std.testing.expectEqual(@as(usize, 0), b_state.queryCapacity());
// The strict/ultra variants still compose ONTO this base rather than
// replacing it (#326), which the segmentation must not have disturbed.
try std.testing.expect(std.mem.startsWith(u8, prompts.main_system_prompt_strict, golden_full_prompt));
}
test "#421: an absent capability contributes ZERO instruction text, in every configuration" {
for (matrix) |row| {
var a_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer a_state.deinit();
const out = try prompts.composeSegments(a_state.allocator(), row.caps);
var expected: usize = 0;
for (prompts.segments) |seg| {
if (row.caps.has(seg.gate)) expected += seg.text.len;
}
std.testing.expectEqual(expected, out.len) catch |err| {
std.debug.print("config '{s}': composed {d} bytes, expected {d}\n", .{ row.name, out.len, expected });
return err;
};
// Exact length is necessary but not sufficient: prove each dropped
// segment's own bytes are unfindable, and each kept one is still there.
for (prompts.segments) |seg| {
const present = std.mem.indexOf(u8, out, seg.text) != null;
std.testing.expectEqual(row.caps.has(seg.gate), present) catch |err| {
std.debug.print("config '{s}': segment '{s}' present={} expected={}\n", .{ row.name, seg.name, present, row.caps.has(seg.gate) });
return err;
};
}
}
}
test "#421: a gated-off capability's tool names disappear from the prompt entirely" {
var a_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer a_state.deinit();
const a = a_state.allocator();
// #330 embedder mode: bash, read_file, edit_file, write_file and codedb are
// hard-removed from every catalog, so no sentence may still name them.
const embedder = try prompts.composeSegments(a, .{ .local_tools = false });
// "Treat that path as the result" is #440's handle contract: a session with
// no way to OPEN a path is never told to slice one.
for ([_][]const u8{ "read_file", "edit_file", "write_file", "codedb", "bash grep", "/trace", "gh issue create", ".graff/traces", "Treat that path as the result" }) |dead|
try std.testing.expect(std.mem.indexOf(u8, embedder, dead) == null);
const no_subs = try prompts.composeSegments(a, .{ .subagents = false });
for ([_][]const u8{ "subagent tool", "workflow tool", "{{prev}}" }) |dead|
try std.testing.expect(std.mem.indexOf(u8, no_subs, dead) == null);
const no_todos = try prompts.composeSegments(a, .{ .todos = false });
try std.testing.expect(std.mem.indexOf(u8, no_todos, "todo_write") == null);
const no_constraints = try prompts.composeSegments(a, .{ .constraints = false });
try std.testing.expect(std.mem.indexOf(u8, no_constraints, "note_constraint") == null);
// Outside a git repo, the AUTHORING guidance goes — commit identity and
// PR discipline have nothing to act on — while the safety rail stays.
const no_git = try prompts.composeSegments(a, .{ .git_repo = false });
for ([_][]const u8{ "## What changed", "Co-Authored-By: Codegraff", "GIT_AUTHOR_" }) |dead|
try std.testing.expect(std.mem.indexOf(u8, no_git, dead) == null);
// What survives EVERY gate: identity, the two prompt-doctrine lines, the
// do-not-discard-work rail, and the closing style.
for (matrix) |row| {
const out = try prompts.composeSegments(a, row.caps);
for ([_][]const u8{
"You are a coding agent",
"never invent", // #421 doctrine 1: no wrapper APIs, no assumed capabilities
"its OWN environment", // #421 doctrine 2: verify where the project lives
"Never run git commands that discard work",
"Parallelize tool calls",
"Be direct and concise",
"A comment alone does not complete a recurrence report.", // #1701: MCP can report without local tools
}) |keep| {
std.testing.expect(std.mem.indexOf(u8, out, keep) != null) catch |err| {
std.debug.print("config '{s}' lost '{s}'\n", .{ row.name, keep });
return err;
};
}
}
}
test "#421: detectCaps reads the same gates dispatch refuses a call with" {
const saved = no_local_tools.enabled;
defer no_local_tools.enabled = saved;
no_local_tools.enabled = false;
try std.testing.expect(prompts.detectCaps().full());
no_local_tools.enabled = true;
const caps = prompts.detectCaps();
try std.testing.expect(!caps.local_tools);
// #330 keeps the orchestration and meta tools: the CHILD inherits the gate,
// so the fan-out guidance is still guidance for a tool that exists.
try std.testing.expect(caps.subagents);
try std.testing.expect(caps.todos);
try std.testing.expect(caps.constraints);
var a_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer a_state.deinit();
const gated = try prompts.baseForSession(a_state.allocator());
try std.testing.expect(gated.len < prompts.main_system_prompt.len);
try std.testing.expect(std.mem.indexOf(u8, gated, "read_file") == null);
}
test "#410: the transcript line appears exactly when a durable session exists" {
var a_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer a_state.deinit();
const a = a_state.allocator();
// No durable session (a subagent, a stub, a future --no-session): no line.
try std.testing.expectEqualStrings("", prompts.sessionTranscriptNote(a, ""));
const note = prompts.sessionTranscriptNote(a, "session-1750000000000");
// The path is the one session_index actually writes, not a hand-written twin.
try std.testing.expect(std.mem.indexOf(u8, note, try session_index.sessionPath(a, "session-1750000000000")) != null);
try std.testing.expect(std.mem.indexOf(u8, note, "lags") != null); // the caveat #410 asks for
try std.testing.expect(std.mem.indexOf(u8, note, "JSON object") != null);
// NOT JSONL: that is the .graff/traces event stream, a different file. A
// wrong format claim costs the model a turn discovering it.
try std.testing.expect(std.mem.indexOf(u8, note, "JSONL") == null);
try std.testing.expect(std.mem.indexOf(u8, prompts.main_system_prompt, "durable transcript") == null); // session-scoped, never comptime
}
test "#410: the transcript line rides the funnel, so a persona swap cannot drop it" {
var a_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer a_state.deinit();
const a = a_state.allocator();
var agent: agent_mod.Agent = undefined;
defer prompts.armSessionTranscript(a, "", .{}, false); // leave the global as the rest of the suite expects it
// Armed as #445 arms it in production: after the first compaction.
prompts.armSessionTranscript(a, "session-42", .{}, true);
try prompts.setSystemPrompts(&agent, "BASE", a);
for ([_][]const u8{ agent.sys_normal, agent.sys_strict, agent.sys_ultra, agent.sys_ultra_strict }) |v| {
try std.testing.expect(std.mem.indexOf(u8, v, "BASE") != null);
try std.testing.expect(std.mem.indexOf(u8, v, "session-42.session.json") != null);
}
// A later /agent persona or set_system_prompt goes through the same funnel
// and keeps it — the #326 staleness class, closed by construction.
try prompts.setSystemPrompts(&agent, "PERSONA", a);
try std.testing.expect(std.mem.indexOf(u8, agent.sys_normal, "session-42.session.json") != null);
// sys_base stays the pure base: the next refresh composes from it, once.
try std.testing.expectEqualStrings("PERSONA", agent.sys_base);
try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, agent.sys_normal, "session-42.session.json"));
// Unreadable session file (#330 removed read_file/bash): no line at all,
// compacted or not — the capability half of the gate is independent.
prompts.armSessionTranscript(a, "session-42", .{ .local_tools = false }, true);
try prompts.setSystemPrompts(&agent, "BASE", a);
try std.testing.expectEqualStrings("BASE", agent.sys_normal);
}
// #445: the measured cost of the #421/#410 prompt additions was ~240 input
// tokens on EVERY call, and the transcript line's share of it buys nothing
// until the live window stops being a superset of the file it names. So the
// line is deferred to the first compaction, and this pins both ends of that:
// absent on a fresh session, present once the boundary has been crossed.
test "#445: the transcript line is absent until the first compaction, then rides every variant" {
var a_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer a_state.deinit();
const a = a_state.allocator();
const saved = prompts.g_session_compacted;
defer prompts.g_session_compacted = saved;
defer prompts.armSessionTranscript(a, "", .{}, false); // leave the global as the rest of the suite expects it
defer prompts.armCompactNotes(""); // #391's store is a process global: never leak an armed session into another test
var agent: agent_mod.Agent = undefined;
agent.sub = false;
agent.session_name = "session-445";
prompts.g_session_compacted = false;
// A fresh session: setRootSystemPrompts arms with the flag as it stands,
// so every variant is exactly the base and the session pays nothing.
prompts.armSessionTranscript(a, agent.session_name, .{}, prompts.g_session_compacted);
try prompts.setSystemPrompts(&agent, "BASE", a);
try std.testing.expectEqualStrings("BASE", agent.sys_normal);
for ([_][]const u8{ agent.sys_normal, agent.sys_strict, agent.sys_ultra, agent.sys_ultra_strict }) |v|
try std.testing.expect(std.mem.indexOf(u8, v, "session-445.session.json") == null);
// The compaction boundary re-derives all four, in place, from sys_base.
prompts.noteSessionCompacted(&agent, a);
try std.testing.expect(prompts.g_session_compacted);
for ([_][]const u8{ agent.sys_normal, agent.sys_strict, agent.sys_ultra, agent.sys_ultra_strict }) |v| {
try std.testing.expect(std.mem.indexOf(u8, v, "BASE") != null);
try std.testing.expect(std.mem.indexOf(u8, v, "session-445.session.json") != null);
}
try std.testing.expectEqualStrings("BASE", agent.sys_base); // still the pure base
// Idempotent: a session that compacts ten times still carries ONE line.
prompts.noteSessionCompacted(&agent, a);
prompts.noteSessionCompacted(&agent, a);
try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, agent.sys_normal, "session-445.session.json"));
// And a later persona swap keeps it — the #326 staleness class stays closed.
try prompts.setSystemPrompts(&agent, "PERSONA", a);
try std.testing.expect(std.mem.indexOf(u8, agent.sys_normal, "session-445.session.json") != null);
// A compacting SUBAGENT has no durable session of its own, and must not
// spend the root's tokens on a line about a file it never wrote.
prompts.g_session_compacted = false;
prompts.armSessionTranscript(a, "", .{}, false);
var child: agent_mod.Agent = undefined;
child.sub = true;
child.session_name = "session-child";
child.sys_base = "CHILD-BASE";
prompts.noteSessionCompacted(&child, a);
try std.testing.expect(!prompts.g_session_compacted);
try prompts.setSystemPrompts(&child, "CHILD-BASE", a);
try std.testing.expectEqualStrings("CHILD-BASE", child.sys_normal);
}
// #445: one PROCESS can host several root conversations. /new mints a fresh
// session_name and /clear empties the history and saves over the file, so both
// put the durable file back to holding no more than the live window. A flag
// that only ever went true made one compaction arm the line for the rest of
// the process — the exact per-call waste this issue exists to remove, walked
// back in through the commonest flow there is.
test "#445: /new and /clear disarm the transcript line again, and a later compaction re-arms it" {
var a_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer a_state.deinit();
const a = a_state.allocator();
const saved = prompts.g_session_compacted;
defer prompts.g_session_compacted = saved;
defer prompts.armSessionTranscript(a, "", .{}, false); // leave the global as the rest of the suite expects it
defer prompts.armCompactNotes(""); // #391's store is a process global: never leak an armed session into another test
var agent: agent_mod.Agent = undefined;
agent.sub = false;
agent.session_name = "session-long";
prompts.g_session_compacted = false;
prompts.armSessionTranscript(a, agent.session_name, .{}, false);
try prompts.setSystemPrompts(&agent, "BASE", a);
// A long session compacts: the line arms, as #445's forward boundary says.
prompts.noteSessionCompacted(&agent, a);
try std.testing.expect(std.mem.indexOf(u8, agent.sys_normal, "session-long.session.json") != null);
// /new: a brand-new conversation under a brand-new name, in the SAME
// process. commands_session calls the reset AFTER the rename, so the
// re-arm reads the session that exists now — and finds nothing to say.
agent.session_name = "session-new";
prompts.resetSessionCompacted(&agent, a);
try std.testing.expect(!prompts.g_session_compacted);
try std.testing.expectEqualStrings("BASE", agent.sys_normal);
for ([_][]const u8{ agent.sys_normal, agent.sys_strict, agent.sys_ultra, agent.sys_ultra_strict }) |v| {
try std.testing.expect(std.mem.indexOf(u8, v, "session-long.session.json") == null); // no stale name either
try std.testing.expect(std.mem.indexOf(u8, v, "session-new.session.json") == null);
}
// ...and the new conversation earns the line back on its own first
// compaction, naming ITS file. The flag is a boundary, not a one-way latch.
prompts.noteSessionCompacted(&agent, a);
try std.testing.expect(std.mem.indexOf(u8, agent.sys_normal, "session-new.session.json") != null);
try std.testing.expect(std.mem.indexOf(u8, agent.sys_normal, "session-long.session.json") == null);
// /clear: same name, but the immediate saveSession rewrites the durable
// file down to an empty history, so the line again describes nothing.
prompts.resetSessionCompacted(&agent, a);
try std.testing.expect(!prompts.g_session_compacted);
try std.testing.expectEqualStrings("BASE", agent.sys_normal);
// Already false: a second /clear must not touch the prompt at all.
agent.sys_normal = "SENTINEL";
prompts.resetSessionCompacted(&agent, a);
try std.testing.expectEqualStrings("SENTINEL", agent.sys_normal);
// A subagent's own /clear-equivalent can never disarm the ROOT's line.
prompts.g_session_compacted = true;
var child: agent_mod.Agent = undefined;
child.sub = true;
child.session_name = "session-child";
child.sys_base = "CHILD-BASE";
child.sys_normal = "CHILD-SENTINEL";
prompts.resetSessionCompacted(&child, a);
try std.testing.expect(prompts.g_session_compacted);
try std.testing.expectEqualStrings("CHILD-SENTINEL", child.sys_normal);
}
// #445: /resume is the third door onto the same state. loadSession replaces
// the live history with a COPY of the file it just read, so the transcript
// line is redundant for exactly the reason it is redundant on a fresh session
// — and worse than redundant if left armed, since it would still be naming the
// conversation the user resumed AWAY from.
test "#445: /resume disarms the transcript line and leaves no stale session path" {
var a_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer a_state.deinit();
const a = a_state.allocator();
const saved = prompts.g_session_compacted;
defer prompts.g_session_compacted = saved;
defer prompts.armSessionTranscript(a, "", .{}, false); // leave the global as the rest of the suite expects it
defer prompts.armCompactNotes(""); // #391's store is a process global: never leak an armed session into another test
var agent: agent_mod.Agent = undefined;
agent.sub = false;
agent.session_name = "session-before";
prompts.g_session_compacted = false;
prompts.armSessionTranscript(a, agent.session_name, .{}, false);
try prompts.setSystemPrompts(&agent, "BASE", a);
// The conversation the user is about to leave compacted, so it is armed.
prompts.noteSessionCompacted(&agent, a);
try std.testing.expect(std.mem.indexOf(u8, agent.sys_normal, "session-before.session.json") != null);
// /resume: commands_misc assigns the new name, THEN resets — the ordering
// the reset's doc comment requires, so the re-arm sees the resumed session.
agent.session_name = "session-resumed";
prompts.resetSessionCompacted(&agent, a);
try std.testing.expect(!prompts.g_session_compacted);
try std.testing.expectEqualStrings("BASE", agent.sys_normal);
for ([_][]const u8{ agent.sys_normal, agent.sys_strict, agent.sys_ultra, agent.sys_ultra_strict }) |v| {
try std.testing.expect(std.mem.indexOf(u8, v, "session-before.session.json") == null); // the stale path is gone
try std.testing.expect(std.mem.indexOf(u8, v, "session-resumed.session.json") == null); // and the new one is not owed one yet
}
// The resumed conversation earns the line back on its OWN first compaction.
prompts.noteSessionCompacted(&agent, a);
try std.testing.expect(std.mem.indexOf(u8, agent.sys_normal, "session-resumed.session.json") != null);
try std.testing.expect(std.mem.indexOf(u8, agent.sys_normal, "session-before.session.json") == null);
}
test "#421: MCP, skill and optional-tool guidance all cost zero when absent" {
var a_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer a_state.deinit();
const a = a_state.allocator();
// MCP: startup.buildSystemPrompt injects a server's note only when that
// server actually connected, and nothing is baked into the base prompt.
for (skills.mcp_notes) |mn| {
try std.testing.expect(!skills.mcpServerConnected(&.{}, mn.server));
try std.testing.expect(std.mem.indexOf(u8, prompts.main_system_prompt, mn.note) == null);
}
// Markdown skills: an empty catalog is the empty string, not a header.
try std.testing.expectEqualStrings("", skill_docs.promptCatalog(a, &.{}));
// #352 optional tools: no availability, no advertisement, and the base
// prompt never mentions imagegen either way (its guidance is the skill).
const saved = imagegen.available;
defer imagegen.available = saved;
imagegen.available = false;
try std.testing.expect(!tool_gates.advertised(imagegen.tool_name));
try std.testing.expect(std.mem.indexOf(u8, prompts.main_system_prompt, "imagegen") == null);
}
test "#421: goal steering is empty outside a goal run" {
var a_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer a_state.deinit();
const a = a_state.allocator();
try std.testing.expectEqualStrings("", try repl_glue.goalSteeringNote(a, null));
// ...and a goal that is no longer active steers nobody either (#223).
try std.testing.expectEqualStrings("", try repl_glue.goalSteeringNote(a, .{ .objective = "ship it", .status = .complete }));
try std.testing.expect((try repl_glue.goalSteeringNote(a, .{ .objective = "ship it", .status = .active })).len > 0);
}
// #391 + #445 regression. The two features collided at integration in a way
// neither suite could see: #391's note store is a PROCESS global whose
// composition reads the filesystem through `agent.io`, and #445's tests drive
// the funnel with stub Agents whose `io` is `undefined`. Once anything armed
// the store, every later composition in the binary segfaulted inside
// compact_note.pathFor — three tests away from the cause.
//
// The fix is that the funnel's IMPLICIT arming is gated on `!builtin.is_test`,
// so a test can only arm the store by asking for it. This pins that: remove the
// gate and this goes red instead of the segfault coming back.
test "#391/#445: the prompt funnel never arms the note store in a test build" {
var a_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer a_state.deinit();
const a = a_state.allocator();
const saved = prompts.g_session_compacted;
defer prompts.g_session_compacted = saved;
defer prompts.armCompactNotes("");
defer prompts.armSessionTranscript(a, "", .{}, false);
prompts.armCompactNotes(""); // start disarmed, as a fresh process would
try std.testing.expect(!prompts.compactNotesArmed());
var agent: agent_mod.Agent = undefined;
agent.sub = false;
agent.session_name = "session-purity";
agent.io = std.testing.io; // a real io: this test is about arming, not about io
try prompts.setRootSystemPrompts(&agent, "BASE", a);
// Production arms here. A test build must not, or the funnel stops being
// the pure string function the rest of this suite depends on.
try std.testing.expect(!prompts.compactNotesArmed());
}