Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
24.2 KB
Raw Blame
//! Eval-driven scoring loop and its optional LLM-as-judge scorer.
//! Tests live in agent_eval_tests.zig.

const std = @import("std");
const Io = std.Io;

const Agent = @import("agent.zig").Agent;
const ExecResult = @import("tools.zig").ExecResult;
const ToolCtx = @import("tools.zig").ToolCtx;
const ToolOutput = @import("tools.zig").ToolOutput;
const parseEvalScore = @import("repl_glue.zig").parseEvalScore;
const utf8Prefix = @import("util.zig").utf8Prefix;

const scoring = @import("scoring.zig");
const promptFingerprint = scoring.promptFingerprint;
const providerClass = scoring.providerClass;
const signScore = scoring.signScore;

const telemetry = @import("telemetry.zig");
const runCappedWithOptions = @import("jobs.zig").runCappedWithOptions;
const judgeTask = @import("subagent.zig").judgeTask;
const eval_memory = @import("eval_memory.zig");
const trace = @import("trace.zig");
const fleet = @import("fleet.zig");
const main_mod = @import("main.zig");
const route_policy = @import("route_policy.zig"); // #372 (shape, role, tier) coordinates on the local capture rows
const playbook_reflect = @import("playbook_reflect.zig"); // #383 Reflector: one bounded distillation per process, on a target-met eval
const orch_rows = @import("orchestration_rows.zig"); // the orchestration outcome rides the same score funnel
const shapes = @import("shapes.zig");
const failure_evidence = @import("failure_evidence.zig"); // a RED verdict is parked as escalation evidence
const verify_fingerprint = @import("verify_fingerprint.zig"); // #412 no-progress guard: an unmoved worktree is not re-verified

test {
    _ = @import("agent_eval_tests.zig");
}

/// Run the configured --eval scoring command, append the result to the
/// scores log (.graff/eval-log.tsv), and return a verdict for the model:
/// score (0-100), best so far, target, and whether the target is met. The
/// harness runs the command, so the model cannot fake the number. (eval tool)
pub fn runEval(self: *Agent, note: []const u8) !ExecResult {
    const cmd = self.eval_cmd orelse return .{
        .text = "no eval command configured - relaunch graff with --eval <scoring cmd> and --until <N>, or ask the user to set one",
        .is_error = true,
    };

    // #412 no-progress guard. The tree the verifier is about to run against is
    // fingerprinted BEFORE the command runs (the command itself may write), and
    // when the last verdict was RED over a byte-identical tree the verifier is
    // not run at all: nothing it could report can have changed, so the whole
    // scoring command, its judge model call and its output tail are saved and
    // the model is steered at the real blocker instead. The attempt still
    // counts - a model that keeps calling eval without editing has to converge
    // on the iteration cap, not spin for free. Unknown (no repo, git error,
    // truncated probe) never matches, so the guard fails open.
    const tree_fp = verify_fingerprint.capture(self.gpa, self.io, cmd);
    if (verify_fingerprint.skipReverify(self.eval_repair_pending, self.eval_fp, tree_fp)) {
        self.eval_iter += 1;
        return .{ .text = try verify_fingerprint.noProgressText(self.arena, self.eval_iter), .is_error = true };
    }

    // Behavioral commitment (issue #256): the eval-driven loop is the first
    // production caller of turn_committed/model_mispredicted. The commitment
    // asserts the loop's own belief - the command will meet the target -
    // before the command runs, so a later contradiction is provable rather
    // than reconstructed after the fact. commitment_id is opaque and
    // per-invocation; the command text and its stdout/stderr are content and
    // must never reach either typed field (docs/behavioral-trajectories.md).
    const behavior = if (self.tracer) |tracer| tracer.behavior else null;
    const eval_turn = if (behavior) |bt| bt.currentTurn() else 0;
    var commitment_buf: [48]u8 = undefined;
    const commitment_id = std.fmt.bufPrint(&commitment_buf, "eval-{d}-{d}", .{ eval_turn, self.eval_iter + 1 }) catch "";
    if (behavior) |bt| bt.recordExpectedAction(eval_turn, commitment_id, .{ .kind = "eval" }, .{ .pass = true }, "eval-driven loop verifier");

    const run = runCappedWithOptions(self.gpa, self.io, &.{ "/bin/sh", "-c", cmd }, 64 * 1024, 16 * 1024, 0, .{ .environ_map = @import("tool_env.zig").get() }) catch |e| { // #1267
        // The command never ran, so the commitment can never be verified by
        // the normal exit-code/score path below; resolve it here instead of
        // leaving a dangling turn_committed that a scorer would misread as
        // an unresolved success.
        if (behavior) |bt| bt.recordMisprediction(eval_turn, commitment_id, .{ .pass = true }, .{ .pass = false, .exit = @as(i32, -1) }, "eval command could not run");
        self.eval_verified = false;
        self.eval_repair_pending = true;
        // #412: a command that never ran is not a verdict about the workspace,
        // and "edit source files" is not the fix for it - disarm the guard so
        // the next call really does try the command again (and so a fingerprint
        // left by an earlier RED cannot suppress it).
        self.eval_fp = null;
        eval_memory.record(self, note, null, -1, false);
        return .{ .text = try std.fmt.allocPrint(self.arena, "eval command could not run: {t}", .{e}), .is_error = true };
    };
    defer self.gpa.free(run.stdout);
    defer self.gpa.free(run.stderr);
    self.eval_iter += 1;
    const exit_code: i32 = switch (run.term) {
        .exited => |c| @intCast(c),
        else => -1,
    };
    // #367: a command that prints no parseable score falls back to exit-code
    // semantics — 0 is a pass, anything else a fail — so pointing --eval at a
    // plain test runner (`pytest -q`, `zig build test`, `false`) works out of
    // the box instead of scraping nonsense off its summary line. An explicit
    // `score: N` line always wins; judges (runJudge) keep parse-only.
    const parsed = parseEvalScore(run.stdout) orelse parseEvalScore(run.stderr);
    const exit_derived = parsed == null;
    const det: ?f64 = parsed orelse if (exit_code == 0) @as(f64, 100) else @as(f64, 0);

    // LLM-as-judge (--judge): an independent subagent inspects the actual
    // artifacts against the rubric and returns its own 0-100 score. Both
    // scores must clear the target, so the binding value is their min().
    // Skip the judge when the deterministic command itself produced no
    // score - fix that first rather than burn a judge run.
    const judge: ?f64 = if (self.eval_judge != null and det != null) self.runJudge(self.eval_judge.?, run.stdout, note) else null;
    const combined: ?f64 = if (self.eval_judge == null)
        det
    else if (det != null and judge != null)
        @min(det.?, judge.?)
    else
        null;

    const target_f: f64 = @floatFromInt(self.eval_target);
    const improved = if (combined) |s| (self.eval_best < 0 or s > self.eval_best) else false;
    if (combined) |s| {
        if (self.eval_best < 0 or s > self.eval_best) self.eval_best = s;
    }
    const met = if (combined) |s| s >= target_f else false;
    self.eval_repair_pending = exit_code != 0 or !met;
    self.eval_verified = !self.eval_repair_pending;
    // #412: a RED remembers the tree it failed on, so the next eval over that
    // same tree is skipped; a green forgets it, because the next RED must be
    // measured against its own tree and never against a stale one.
    self.eval_fp = if (self.eval_repair_pending) tree_fp else null;
    // A RED verdict is harness ground truth about a FAILED attempt: park a
    // capped excerpt so the next escalation decision carries the evidence
    // (the R0d revision advisory, or an R3 fleet's briefs).
    if (exit_code != 0 or !met) {
        var ev_buf: [256]u8 = undefined;
        const ev = std.fmt.bufPrint(&ev_buf, "eval RED: score {d:.1}/100 (target {d}, exit {d}) — {s}", .{ combined orelse -1.0, self.eval_target, exit_code, utf8Prefix(note, 128) }) catch "eval RED: scoring command failed";
        failure_evidence.note(shapes.classOf(shapes.rawAsk()), ev);
    }
    // A green eval refunds the RED-continuation budget (see grantRepairTurn):
    // red→repair→green→red progress cycles keep running; only sustained
    // failure exhausts it.
    if (self.eval_verified) self.eval_repair_grants = 0;
    // Resolve the commitment made above: a nonzero exit or an unmet/unparsed
    // target contradicts "this command will meet the target", so it is a
    // misprediction. Meeting the target leaves the commitment unresolved by
    // design - docs/behavioral-trajectories.md: a commitment with no paired
    // misprediction is itself the success signal.
    if (behavior) |bt| if (exit_code != 0 or !met)
        bt.recordMisprediction(eval_turn, commitment_id, .{ .pass = true }, .{ .pass = false, .exit = exit_code }, "target not met");
    self.appendEvalLog(note, det, judge, combined, exit_code, met) catch {};
    eval_memory.record(self, note, combined, exit_code, met);
    // #383 Reflector: a target-met eval is the moment a run is verifiably
    // finished, so distil it into candidate playbook bullets — ONCE per
    // process, one tool-less model call on the worker seat. Gated inside on
    // trace.g_traj, the same null-sink guard the local DGM capture below
    // uses, which is what keeps agent_eval_tests' stub Agent (undefined
    // provider/client, and a second eval that DOES meet its target) out of a
    // live model call.
    if (met) playbook_reflect.afterEval(self, note, combined orelse 0, run.stdout);

    // The orchestration outcome (§5). This is the same funnel the DGM score
    // rows use, and for the same reason: an eval verdict is the one place the
    // harness holds ground truth about whether a run WORKED. If this turn made
    // an escalation decision, that decision now gets its measured cost and
    // result — score, calls actually spent, and whether the edit-contract
    // probe ever saw a diff.
    //
    // Gated on `met`, and that gate is load-bearing. An `--until` loop
    // evaluates repeatedly: red, repair, green. Filing on the FIRST verdict
    // recorded score 0 for a run that went on to score 100 — observed, on the
    // very first acceptance run of this feature — which would teach the policy
    // that the rung it chose had failed when it had succeeded. A run that
    // never goes green files nothing here and stays uncelled, which is the
    // same doctrine foldArms applies to every incomplete row: an uncelled
    // observation is honest, a mis-celled one poisons the cell. The one
    // failure that DOES get recorded is the one that matters most, and it is
    // recorded elsewhere — run_budget.exhaustedFatal files score 0,
    // exhausted:true before the process dies.
    if (met) if (orch_rows.pending() != null) if (combined) |s| {
        if (s >= 0 and s <= 100)
            orch_rows.flushPending(s / 100.0, if (self.run_budget) |b| b.used() else 0, 0, false, &promptFingerprint(cmd));
    };

    // Feed the eval-driven score into the fleet (docs/hyperagents.md §9.B):
    // on a NEW BEST, submit the genome (this agent's persona) with its achieved
    // score on the pinned eval set (the eval command's fingerprint). Without this
    // the score only ever reached .graff/eval-log.tsv and the DGM/fleet never saw
    // real eval-driven work — only darwincode/JSON-proto runs ever submitted.
    if (combined) |s| {
        if (improved and s > 0) { // s>0: skip the initial-state / total-failure 0 (don't pollute the cell mean)
            if (s > 100) {
                // Review F8: parseEvalScore does NOT bound its result (a stray
                // "score: 9000" line parses as 9000) — an out-of-[0,100] score
                // must never be signed or submitted. Skip the score AND its
                // paired propose/submit, mirroring mainloop /score's explicit
                // rejection, so the submit counter stays in sync with stored
                // scores. The local eval verdict below still shows the number.
                if (self.tracer) |tr| tr.note("fleet", "score skipped: eval score outside [0,100]");
            } else if (trace.g_traj != null or telemetry.g_telem != null) {
                // Skipped entirely when neither sink exists (also keeps stub
                // agents in tests from touching provider/systemPrompt).
                const sys = self.systemPrompt();
                const genome_fp = promptFingerprint(sys);
                const esh_fp = promptFingerprint(cmd);
                const genome: []const u8 = &genome_fp;
                const esh: []const u8 = &esh_fp;
                const pclass = providerClass(self.provider.model);
                // --niche tags this score's cell. Without it the score lands in the
                // anonymous "" niche, which pullElites can never match to a builtin —
                // so an eval session that wants to grow a champion must name its role.
                // Truncated to 64 chars and sanitized (tab/newline/CR → ' ', review
                // F7) BEFORE signing (fleetEvent's own niche cap, same as mainloop
                // /score) so signed bytes equal ingested bytes.
                var niche_buf: [64]u8 = undefined;
                const niche = scoring.sanitizeMetaField(&niche_buf, utf8Prefix(self.eval_niche, 64));
                // SCORE SCALE CONTRACT (issue #168 Gap 4): local UX stays 0-100
                // (the /100 verdicts below, eval_best, eval_target), but every
                // score that leaves this function — the local archive row and
                // the signed envelope alike — is [0,1].
                const s01 = s / 100.0;
                // Local DGM capture, deliberately OUTSIDE the telemetry gate:
                // the same genome + score the fleet events carry also lands in
                // .graff/trajectories, so `/agents promote` can grow a champion
                // from eval-driven runs with zero egress. Until this block the
                // eval loop fed only the backend — a default-local-privacy
                // session scored work the local promote could never see. The
                // kind:"eval" row carries the niche tag: promoteAgents reads
                // niche off non-prompt/non-score records only. Same niche gate
                // as the backend path — a "" cell is unpromotable either way.
                // #372: the same two rows also carry the (shape, role, tier,
                // model) coordinates the learned tier policy partitions on.
                // An eval loop instantiates no catalog shape, so its cell is
                // (adhoc, <the slot --niche names>); the rung is read back off
                // whatever model this session actually ran.
                const rrole = route_policy.roleOf("", niche);
                const rtier = route_policy.tierLabelFor(self.provider.id, self.provider.model);
                if (niche.len > 0) if (trace.g_traj) |traj| {
                    traj.capturePrompt(genome_fp, sys);
                    traj.node(.{ .kind = "eval", .prompt_sha = genome, .niche = niche, .eval_set_hash = esh, .provider_class = pclass, .shape = "adhoc", .role = rrole, .tier = rtier, .model = self.provider.model, .t = traj.elapsedMs() });
                    traj.node(.{ .kind = "score", .prompt_sha = genome, .score = s01, .niche = niche, .eval_set_hash = esh, .provider_class = pclass, .shape = "adhoc", .role = rrole, .tier = rtier, .model = self.provider.model, .t = traj.elapsedMs() });
                };
                // Auto-learn: evolution is a default part of the harness, not
                // a command. A target-met NEW BEST is the learning moment —
                // the work just verified green — so recompute this project's
                // champions from the archive right now and hot-reload them,
                // making the very next spawn run the evolved genome with no
                // restart and no /agents promote. Auto mode never overwrites
                // a hand-written persona (fleet.promoteAgents); the manual
                // command remains for --personal tiers and forced overwrites.
                if (met and niche.len > 0 and main_mod.g_fleet and !self.sub) {
                    var pout: Io.Writer.Allocating = .init(self.arena);
                    const n = fleet.promoteAgents(self.io, self.gpa, &pout.writer, fleet.g_home, false, true);
                    if (n > 0) {
                        fleet.g_agent_types = fleet.loadAgentTypes(self.io, self.arena, fleet.g_home);
                        self.say("  auto-promoted {d} champion persona(s) → {s} (live now)\n", .{ n, fleet.agents_dir }) catch {};
                        if (self.tracer) |tr| tr.note("fleet", "auto-promoted champions after green eval");
                    }
                }
                if (telemetry.g_telem) |t| {
                    const run_id: []const u8 = &scoring.g_run_id;
                    // Genome-send (docs/proposals/graff-dgm.md §B): the eval genome is this agent's own
                    // persona, never spawned via runSub, so its prompt_text never reached
                    // the worker. A cell only promotes when harness_scores joins to a
                    // harness_genomes row, so ride the genome text over on a `propose`
                    // (deduped by prompt_sha) before the score — else a winning eval cell
                    // has nothing to serve. Gated on a niche: a "" cell is unpromotable.
                    // Oversized genomes skip the propose (review F6): the server verifies
                    // the fingerprint over the carried text, so a truncated genome would
                    // be dropped there anyway.
                    if (niche.len > 0) {
                        if (sys.len <= telemetry.Telemetry.max_propose_text)
                            t.fleetEvent("propose", niche, genome, "", pclass, "", 0, sys)
                        else if (self.tracer) |tr| tr.note("fleet", "propose skipped: genome > 64KB");
                    }
                    const sig = signScore(genome, "", s01, run_id, "", "", esh, niche, pclass);
                    const sig_s: []const u8 = if (scoring.g_score_key != null) &sig else "";
                    var provbuf: [512]u8 = undefined;
                    // #376: the 7th component is the routing STRATUM (index 5
                    // is the phase slot an eval loop never has). Same layout
                    // as scoreVariants' tuple, so one collector reads both.
                    const prov = std.fmt.bufPrint(&provbuf, "{s}\t{s}\t{s}\t{s}\t{s}\t{s}\t{s}", .{ "", "", esh, pclass, niche, "", route_policy.stratumOf(self.provider.model) }) catch "";
                    t.scoreEvent(genome, "", s01, run_id, sig_s, prov);
                    t.fleetEvent("submit", niche, genome, "", pclass, esh, 0, "");
                }
            }
        }
    }

    var aw: Io.Writer.Allocating = .init(self.arena);
    const w = &aw.writer;
    if (combined) |s| {
        if (self.eval_judge != null) {
            try w.print("eval #{d}: deterministic {d:.1} + judge {d:.1} -> {d:.1}/100 (best {d:.1}, target {d}). ", .{ self.eval_iter, det.?, judge.?, s, self.eval_best, self.eval_target });
        } else {
            try w.print("eval #{d}: score {d:.1}/100 (best {d:.1}, target {d}). ", .{ self.eval_iter, s, self.eval_best, self.eval_target });
        }
        if (exit_derived) try w.writeAll("(Score derived from the command's exit code — its output had no score line; make it print `score: N` for finer grading.) ");
        if (met)
            try w.writeAll("TARGET MET - verifier gate is green; finish only if no workspace-changing tool runs after this eval.")
        else if (improved)
            try w.writeAll("Improved, but still red - the prior plan is dropped. Make one focused repair, then run eval again; completion is blocked.")
        else
            try w.writeAll("No gain - the prior plan is dropped. Repair from current evidence and re-run eval; completion is blocked.");
    } else if (self.eval_judge != null and det != null and judge == null) {
        try w.print("eval #{d}: deterministic score {d:.1}/100, but the judge returned no parseable score (it may have errored). Re-run after checking the rubric. ", .{ self.eval_iter, det.? });
    } else {
        try w.print("eval #{d}: command ran but no score parsed - print a bare number, or JSON with a score field (0-100 or 0-1), on the last line. ", .{self.eval_iter});
    }
    const tail = if (run.stdout.len > 1500) run.stdout[run.stdout.len - 1500 ..] else run.stdout;
    try w.print("\n[exit {d}] eval output (tail):\n{s}", .{ exit_code, tail });
    return .{ .text = try self.arena.dupe(u8, aw.writer.buffered()), .is_error = false };
}

/// Append one tab-separated row to the scores log (.graff/eval-log.tsv).
/// Best-effort - a failed write never breaks the loop.
pub fn appendEvalLog(self: *Agent, note: []const u8, det: ?f64, judge: ?f64, score: ?f64, exit_code: i32, met: bool) !void {
    Io.Dir.cwd().createDir(self.io, ".graff", .default_dir) catch {};
    const path = ".graff/eval-log.tsv";
    const existing = Io.Dir.cwd().readFileAlloc(self.io, path, self.arena, .limited(2 * 1024 * 1024)) catch "";
    var aw: Io.Writer.Allocating = .init(self.arena);
    const w = &aw.writer;
    try w.writeAll(existing);
    if (existing.len > 0 and existing[existing.len - 1] != '\n') try w.writeByte('\n');
    try w.print("iter={d}\tscore=", .{self.eval_iter});
    if (score) |s| try w.print("{d:.2}", .{s}) else try w.writeAll("NA");
    try w.writeAll("\tdet=");
    if (det) |d| try w.print("{d:.2}", .{d}) else try w.writeAll("NA");
    try w.writeAll("\tjudge=");
    if (judge) |j| try w.print("{d:.2}", .{j}) else try w.writeAll("NA");
    try w.print("\tbest={d:.2}\ttarget={d}\tmet={s}\texit={d}\tnote=", .{ self.eval_best, self.eval_target, if (met) "yes" else "no", exit_code });
    for (note) |ch| try w.writeByte(if (ch < 0x20) ' ' else ch);
    try w.writeByte('\n');
    Io.Dir.cwd().writeFile(self.io, .{ .sub_path = path, .data = aw.writer.buffered() }) catch {};
}

/// Spawn an independent LLM judge (a read-only subagent) to score the
/// current work against the --judge rubric on a 0-100 scale. The judge
/// inspects the real artifacts with its own tools and ends its report with
/// a `score:` line, parsed the same way as a deterministic eval. Runs on a
/// pool thread via judgeTask (mirrors workflowTask) so the eval handler can
/// await it. Returns null if the judge could not run or gave no score.
pub fn runJudge(self: *Agent, rubric: []const u8, eval_output: []const u8, note: []const u8) ?f64 {
    const ctx: ToolCtx = .{
        .gpa = self.gpa,
        .io = self.io,
        .client = self.client,
        .provider = self.provider,
        .subagent_provider = self.subagent_provider,
        .subagent_cross_provider = self.subagent_cross_provider,
        .registry = if (self.sub) null else self.registry,
        .from_sub = self.sub,
        .has_eval = self.eval_cmd != null,
        .approvals = self.approvals,
        .tracer = self.tracer,
        .run_budget = self.run_budget,
        .depth = self.depth,
        .snapshots = self.snapshots,
        .tools_used = &self.tools_used,
    };
    const evidence = if (eval_output.len > 1200) eval_output[eval_output.len - 1200 ..] else eval_output;
    const what = if (note.len > 0) note else "(no note given)";
    const judge_prompt = std.fmt.allocPrint(self.arena,
        \\Score the current state of the work in this directory against the rubric below, on a 0-100 scale.
        \\
        \\RUBRIC:
        \\{s}
        \\
        \\The author's note on the latest change: {s}
        \\
        \\An automated check was also run; its output (tail) is below as evidence. Form your OWN independent judgement of how well the artifacts satisfy the rubric - do not simply echo the check:
        \\---
        \\{s}
        \\---
        \\
        \\Inspect the actual files and artifacts the work produced (read them with your tools; do not modify anything), then score how fully they satisfy the rubric. End your reply with a single final line `score: <N>` where N is an integer from 0 to 100.
    , .{ rubric, what, evidence }) catch return null;
    var fut: Io.Future(ToolOutput) = self.io.async(judgeTask, .{ ctx, judge_prompt });
    const out = fut.await(self.io);
    defer self.gpa.free(out.text);
    if (out.is_error) return null;
    return parseEvalScore(out.text);
}