Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
11.9 KB
Raw Blame
//! Tests for agent_eval.zig's eval-driven-loop behavioral producer (issue
//! #256): the first production caller of turn_committed/model_mispredicted.
//! Reached through the `test { _ = @import(...) }` hook at the bottom of
//! agent_eval.zig, mirroring behavior_trace.zig's own test-file pattern.

const std = @import("std");
const builtin = @import("builtin");
const Io = std.Io;
const Value = std.json.Value;

const agent_mod = @import("agent.zig");
const Agent = agent_mod.Agent;
const trace = @import("trace.zig");
const Tracer = trace.Tracer;
const btrace = @import("behavior_trace.zig");
const BehaviorTrace = btrace.BehaviorTrace;
const process_runner = @import("process_runner.zig");
const verify_fingerprint = @import("verify_fingerprint.zig");

test "runEval: commits before the command runs, mispredicts on a missed target, and leaves a met target commitment-only (#256)" {
    // The chdir below is POSIX-only, matching session_start.zig's own
    // --worktree isolation (Windows has no libc chdir link here).
    if (builtin.os.tag == .windows) return;
    const gpa = std.testing.allocator;
    const io = std.testing.io;

    // runEval's appendEvalLog() writes .graff/eval-log.tsv against the real
    // process cwd (it takes no Dir param), so this test moves the process
    // into a scratch directory for its duration instead of touching the repo
    // checkout. fchdir on real directory handles is used instead of
    // path-based chdir/realPath: realPath on Dir.cwd()'s AT_FDCWD handle is
    // unreliable on this platform (see session_start.zig's own PWD
    // fallback), and this way sidesteps it entirely in both directions.
    var tmp = std.testing.tmpDir(.{});
    defer tmp.cleanup();
    var orig_dir = try Io.Dir.cwd().openDir(io, ".", .{});
    defer orig_dir.close(io);
    defer _ = std.posix.system.fchdir(orig_dir.handle);
    if (std.posix.system.fchdir(tmp.dir.handle) != 0) return error.ChdirFailed;

    var aw: Io.Writer.Allocating = .init(gpa);
    defer aw.deinit();
    var behavior: BehaviorTrace = .{
        .io = io,
        .gpa = gpa,
        .out = &aw.writer,
        .run_id = "runeval-test",
    };
    behavior.start("test", 1);
    const turn = behavior.beginTurn(0);
    var tracer: Tracer = .{
        .io = io,
        .gpa = gpa,
        .out = null,
        .start = Io.Timestamp.now(io, .awake),
        .behavior = &behavior,
    };

    var arena_state = std.heap.ArenaAllocator.init(gpa);
    defer arena_state.deinit();
    var agent: Agent = .{
        .gpa = gpa,
        .arena = arena_state.allocator(),
        .io = io,
        .client = undefined,
        .provider = undefined,
        .messages = undefined,
        .sub = false,
        .label = "test",
        .out = null,
        .tracer = &tracer,
        .eval_cmd = "printf 'score: 10\\n'; exit 1",
        .eval_target = 90,
    };

    const failure = try agent.runEval("");
    try std.testing.expect(!failure.is_error);
    try std.testing.expectEqual(@as(u32, 1), agent.eval_iter);
    try std.testing.expect(!agent.eval_verified);
    try std.testing.expect(agent.eval_repair_pending);
    try std.testing.expect(agent.completed == null);

    agent.eval_cmd = "printf 'score: 100\\n'";
    const success = try agent.runEval("");
    try std.testing.expect(!success.is_error);
    try std.testing.expectEqual(@as(u32, 2), agent.eval_iter);
    try std.testing.expect(agent.eval_verified);
    try std.testing.expect(!agent.eval_repair_pending);
    try std.testing.expect(agent.completed == null);
    const notes = try Io.Dir.cwd().readFileAlloc(io, ".graff/eval-notes/session.md", gpa, .limited(16 * 1024));
    defer gpa.free(notes);
    try std.testing.expect(std.mem.indexOf(u8, notes, "eval #1") != null);
    try std.testing.expect(std.mem.indexOf(u8, notes, "met=no") != null);
    try std.testing.expect(std.mem.indexOf(u8, notes, "eval #2") != null);
    try std.testing.expect(std.mem.indexOf(u8, notes, "met=yes") != null);

    const out = aw.writer.buffered();
    var lines = std.mem.splitScalar(u8, std.mem.trimEnd(u8, out, "\n"), '\n');
    _ = lines.next(); // run_started
    _ = lines.next(); // turn_started

    var committed1 = try std.json.parseFromSlice(Value, gpa, lines.next().?, .{});
    defer committed1.deinit();
    try std.testing.expectEqualStrings("turn_committed", committed1.value.object.get("kind").?.string);
    const commitment_id_1 = committed1.value.object.get("commitment_id").?.string;
    try std.testing.expectEqual(@as(i64, @intCast(turn)), committed1.value.object.get("turn").?.integer);
    try std.testing.expectEqualStrings("eval", committed1.value.object.get("action").?.object.get("kind").?.string);
    try std.testing.expect(committed1.value.object.get("expect").?.object.get("pass").?.bool);

    var mispredicted = try std.json.parseFromSlice(Value, gpa, lines.next().?, .{});
    defer mispredicted.deinit();
    try std.testing.expectEqualStrings("model_mispredicted", mispredicted.value.object.get("kind").?.string);
    try std.testing.expectEqualStrings(commitment_id_1, mispredicted.value.object.get("commitment_id").?.string);
    try std.testing.expectEqual(@as(i64, @intCast(turn)), mispredicted.value.object.get("turn").?.integer);
    try std.testing.expectEqual(@as(i64, 1), mispredicted.value.object.get("actual").?.object.get("exit").?.integer);
    try std.testing.expect(!mispredicted.value.object.get("actual").?.object.get("pass").?.bool);

    // Privacy contract: the command text and its stdout/stderr are content
    // and must never reach the local behavioral stream for this producer
    // (docs/behavioral-trajectories.md).
    try std.testing.expect(std.mem.indexOf(u8, out, "printf") == null);
    try std.testing.expect(std.mem.indexOf(u8, out, "exit 1") == null);
    try std.testing.expect(std.mem.indexOf(u8, out, "score: 10") == null);

    var committed2 = try std.json.parseFromSlice(Value, gpa, lines.next().?, .{});
    defer committed2.deinit();
    try std.testing.expectEqualStrings("turn_committed", committed2.value.object.get("kind").?.string);
    const commitment_id_2 = committed2.value.object.get("commitment_id").?.string;
    try std.testing.expect(!std.mem.eql(u8, commitment_id_1, commitment_id_2));

    // The met-target commitment has no paired misprediction: absence is the
    // success signal (docs/behavioral-trajectories.md).
    try std.testing.expect(lines.next() == null);
}

/// Run a command in the process cwd, failing the test if it did not succeed.
/// The #412 test needs a REAL git repo: reading git is the whole feature, and
/// a mocked probe would prove nothing about the porcelain/diff/untracked split.
fn fixtureCmd(gpa: std.mem.Allocator, io: Io, argv: []const []const u8) !void {
    const r = process_runner.runCapped(gpa, io, argv, 1 << 16, 1 << 16, 60_000) catch
        return error.SkipZigTest; // no git on this machine: skip rather than fail on the environment
    defer gpa.free(r.stdout);
    defer gpa.free(r.stderr);
    if (!process_runner.ranOk(r)) return error.FixtureCommandFailed;
}

/// How many times the --eval command was actually SPAWNED. The counter lives
/// under .graff/, which the fixture repo gitignores, so counting can never
/// itself move the fingerprint the guard is reading.
fn verifierCalls(gpa: std.mem.Allocator, io: Io) usize {
    const body = Io.Dir.cwd().readFileAlloc(io, ".graff/verifier-calls", gpa, .limited(4096)) catch return 0;
    defer gpa.free(body);
    return body.len;
}

test "#412: an unchanged worktree is not re-verified, and any change re-arms the verifier" {
    // POSIX-only for the same reason as the test above: the fchdir isolation.
    if (builtin.os.tag == .windows) return;
    const gpa = std.testing.allocator;
    const io = std.testing.io;

    var tmp = std.testing.tmpDir(.{});
    defer tmp.cleanup();
    var orig_dir = try Io.Dir.cwd().openDir(io, ".", .{});
    defer orig_dir.close(io);
    defer _ = std.posix.system.fchdir(orig_dir.handle);
    if (std.posix.system.fchdir(tmp.dir.handle) != 0) return error.ChdirFailed;

    // A real single-commit repo. `.graff/` is ignored so the harness's own eval
    // log, its notes and the call counter can never read as work the model did
    // - without that the tree would move on every eval and never skip.
    try Io.Dir.cwd().writeFile(io, .{ .sub_path = ".gitignore", .data = ".graff/\n" });
    try Io.Dir.cwd().writeFile(io, .{ .sub_path = "work.txt", .data = "one\n" });
    try fixtureCmd(gpa, io, &.{ "git", "init", "-q" });
    try fixtureCmd(gpa, io, &.{ "git", "add", "-A" });
    try fixtureCmd(gpa, io, &.{ "git", "-c", "[email protected]", "-c", "user.name=t", "-c", "commit.gpgsign=false", "commit", "-q", "--no-verify", "-m", "fixture" });

    var arena_state = std.heap.ArenaAllocator.init(gpa);
    defer arena_state.deinit();
    var agent: Agent = .{
        .gpa = gpa,
        .arena = arena_state.allocator(),
        .io = io,
        .client = undefined,
        .provider = undefined,
        .messages = undefined,
        .sub = false,
        .label = "test",
        .out = null,
        .tracer = null,
        // Always RED, and it leaves one byte behind per real invocation.
        .eval_cmd = "mkdir -p .graff && printf x >> .graff/verifier-calls; exit 1",
        .eval_target = 90,
    };

    // Attempt 1: nothing to compare against, so the verifier runs. Its RED
    // verdict arms the guard over the tree it failed on.
    const first = try agent.runEval("");
    try std.testing.expect(!first.is_error); // a RED is a verdict, not a tool error
    try std.testing.expectEqual(@as(usize, 1), verifierCalls(gpa, io));
    try std.testing.expect(agent.eval_repair_pending);
    try std.testing.expect(agent.eval_fp != null);

    // Attempt 2, workspace untouched. This is the whole feature: before it, a
    // /goal continuation re-ran the entire scoring command (plus a --judge
    // model call) to re-derive a verdict that could not have changed.
    const second = try agent.runEval("");
    try std.testing.expectEqual(@as(usize, 1), verifierCalls(gpa, io)); // NOT spawned
    try std.testing.expectEqual(@as(u32, 2), agent.eval_iter); // but the attempt still counts
    try std.testing.expect(second.is_error);
    try std.testing.expect(std.mem.indexOf(u8, second.text, verify_fingerprint.no_progress_steer) != null);
    try std.testing.expect(std.mem.indexOf(u8, second.text, "eval output (tail)") == null); // no verdict was manufactured
    // A skipped run verifies nothing, so completion stays blocked.
    try std.testing.expect(agent.eval_repair_pending and !agent.eval_verified);

    // A TRACKED edit is progress: the verifier runs again...
    try Io.Dir.cwd().writeFile(io, .{ .sub_path = "work.txt", .data = "two\n" });
    _ = try agent.runEval("");
    try std.testing.expectEqual(@as(usize, 2), verifierCalls(gpa, io));
    // ...and its RED arms the skip over the NEW tree, not the old one.
    _ = try agent.runEval("");
    try std.testing.expectEqual(@as(usize, 2), verifierCalls(gpa, io));

    // An UNTRACKED file is progress too, and its CONTENTS are what prove it:
    // `git status --porcelain` says `?? scratch.txt` either way and
    // `git diff HEAD` stays empty, so a fingerprint built from those two alone
    // would skip both of these attempts over real work.
    try Io.Dir.cwd().writeFile(io, .{ .sub_path = "scratch.txt", .data = "a" });
    _ = try agent.runEval("");
    try std.testing.expectEqual(@as(usize, 3), verifierCalls(gpa, io));
    try Io.Dir.cwd().writeFile(io, .{ .sub_path = "scratch.txt", .data = "b" });
    _ = try agent.runEval("");
    try std.testing.expectEqual(@as(usize, 4), verifierCalls(gpa, io));

    // Six attempts, four verifier runs: the two no-progress ones were free.
    try std.testing.expectEqual(@as(u32, 6), agent.eval_iter);

    // A green eval disarms the guard entirely - the next RED must be measured
    // against its own tree, never against one this run already left behind.
    agent.eval_cmd = "printf 'score: 100\\n'";
    _ = try agent.runEval("");
    try std.testing.expect(agent.eval_verified and agent.eval_fp == null);
}