Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
16.7 KB
Raw Blame
//! Tests for #376 increment 2 — one learned seat per workflow phase.
//! Split out of route_phase.zig for the 600-line cap; reached from its
//! `test {}` block and from test_hooks.zig.

const std = @import("std");
const Io = std.Io;

const phase = @import("route_phase.zig");
const policy = @import("route_policy.zig");
const route_trace = @import("route_trace.zig");
const bench = @import("bench_priors.zig");
const selection = @import("subagent_selection.zig");
const pin_mod = @import("subagent_pin.zig");
const tier_ladder = @import("subagent_tier_ladder.zig");
const Provider = @import("provider.zig").Provider;

fn codex(model: []const u8) Provider {
    return .{ .id = "codex", .kind = .responses, .auth = .bearer, .url = "", .api_key = "sk-test", .model = model, .context = 272_000 };
}

/// The same synthetic sheet #372's demonstration uses: on the SHEET ALONE
/// sol outscores luna and is not dominated by it, so anything that moves
/// below is provably the lived evidence talking, not the priors.
const demo_sheet = [_]bench.Entry{
    .{ .model = "gpt-6-sol", .effort = "medium", .score = 0.60, .cost = 0.9 }, // effective $0.90
    .{ .model = "gpt-6-luna", .effort = "max", .score = 0.40, .cost = 0.61 }, // effective $0.61
};

/// Lived scores from a `research` shape's `sweep` phase: the ladder's mid rung
/// (sol) did badly there, the cheap rung (luna) did well. Exactly the rows
/// route_trace.captureScore writes — no recipe rows, so bench_priors' own
/// #374 sheet feedback is deliberately NOT what is being exercised here.
const demo_archive =
    \\{"kind":"score","prompt_sha":"aa","score":0.2,"shape":"research","role":"sweep","tier":"mid","model":"gpt-6-sol"}
    \\{"kind":"score","prompt_sha":"aa","score":0.2,"shape":"research","role":"sweep","tier":"mid","model":"gpt-6-sol"}
    \\{"kind":"score","prompt_sha":"aa","score":0.2,"shape":"research","role":"sweep","tier":"mid","model":"gpt-6-sol"}
    \\{"kind":"score","prompt_sha":"bb","score":0.95,"shape":"research","role":"sweep","tier":"small","model":"gpt-6-luna"}
    \\{"kind":"score","prompt_sha":"bb","score":0.95,"shape":"research","role":"sweep","tier":"small","model":"gpt-6-luna"}
    \\{"kind":"score","prompt_sha":"bb","score":0.95,"shape":"research","role":"sweep","tier":"small","model":"gpt-6-luna"}
;

/// Install `demo_archive` + `demo_sheet` as the session's live policy, with
/// the session's worker route coming from the #291 ladder descent (the one
/// opening #376 gives the policy). Returns a restorer for `defer`.
const Env = struct {
    cells: []const policy.CellObs,
    entries: []const bench.Entry,
    ladders: []const tier_ladder.TierLadder,
    from_ladder: bool,

    fn install(arena: std.mem.Allocator, archive: []const u8, from_ladder: bool) Env {
        const saved: Env = .{
            .cells = policy.g_cells,
            .entries = bench.g_entries,
            .ladders = bench.g_ladders,
            .from_ladder = selection.g_default_from_ladder,
        };
        bench.g_ladders = &.{}; // compiled ladder only, so the baseline is pinned
        bench.g_entries = &demo_sheet;
        policy.g_cells = policy.foldCells(arena, archive);
        selection.g_default_from_ladder = from_ladder;
        return saved;
    }

    fn restore(self: Env) void {
        policy.g_cells = self.cells;
        bench.g_entries = self.entries;
        bench.g_ladders = self.ladders;
        selection.g_default_from_ladder = self.from_ladder;
    }
};

/// The two workers of a `sweep` phase, as workflow.zig hands them over: the
/// niche is the persona name (or the phase title for an inline variant).
const sweep_niches = [_][]const u8{ "researcher", "skeptic" };

test "role prior: search roles ride the small rung; judged roles keep the session rung" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();
    // NO lived evidence: an empty archive, so any move is the prior alone.
    const saved = Env.install(a, "", true);
    defer saved.restore();

    const find = phase.forPhase(codex("gpt-6-sol"), .review, "find the defects", &.{}, true);
    try std.testing.expectEqualStrings("gpt-6-luna", find.provider.model);
    try std.testing.expect(find.pin != null);
    try std.testing.expect(find.role_prior);
    // The source stays the ladder's — a hand rule must not claim evidence.
    try std.testing.expectEqual(policy.Source.ladder, find.sourceFor(false));

    const sweep = phase.forPhase(codex("gpt-6-sol"), .research, "sweep the sources", &.{}, true);
    try std.testing.expectEqualStrings("gpt-6-luna", sweep.provider.model);

    const verify = phase.forPhase(codex("gpt-6-sol"), .review, "verify each finding", &.{}, true);
    try std.testing.expect(verify.pin == null);
    try std.testing.expectEqualStrings("gpt-6-sol", verify.provider.model);

    // A user-chosen seat is never moved by the prior, same as #372's rule.
    const pinned = Env.install(a, "", false);
    defer pinned.restore();
    const kept = phase.forPhase(codex("gpt-6-sol"), .review, "find the defects", &.{}, true);
    try std.testing.expect(kept.pin == null);
    try std.testing.expectEqualStrings("gpt-6-sol", kept.provider.model);
}

test "#376 uniformRole: a phase routes only when all of its workers agree on one role" {
    // The phase title's own canonical slot is the role every worker reports
    // (policy.roleOf reads the title first), whatever their niches are.
    try std.testing.expectEqualStrings("sweep", phase.uniformRole("sweep the repo", &sweep_niches));
    try std.testing.expectEqualStrings("sweep", phase.uniformRole("sweep the repo", &.{}));
    // An off-vocabulary title falls back to the niches — and only when they
    // agree, since a phase whose workers file under different cells has no
    // single cell to route on.
    try std.testing.expectEqualStrings("review", phase.uniformRole("do the thing", &.{ "review the diff", "review again" }));
    try std.testing.expectEqualStrings("", phase.uniformRole("do the thing", &.{ "review the diff", "sweep the repo" }));
    try std.testing.expectEqualStrings("", phase.uniformRole("do the thing", &.{"some-persona"}));
    try std.testing.expectEqualStrings("", phase.uniformRole("do the thing", &.{}));
    // Whatever it answers, it agrees with what every worker's trace reports.
    for (sweep_niches) |n| {
        try std.testing.expectEqualStrings(phase.uniformRole("sweep the repo", &sweep_niches), policy.roleOf("sweep the repo", n));
    }
}

test "#376 DEMONSTRATION: an observed phase re-seats EVERY worker; an unobserved one keeps today's route" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const env = Env.install(arena_state.allocator(), demo_archive, true);
    defer env.restore();

    // The session descended the #291 ladder to sol; the (research, sweep)
    // cell has measured luna as strictly better value there.
    const base = codex("gpt-6-sol");
    const seat = phase.forPhase(base, .research, "sweep the repo", &sweep_niches, true);
    try std.testing.expect(seat.pin != null);
    try std.testing.expectEqualStrings("gpt-6-luna", seat.provider.model);
    try std.testing.expectEqualStrings("gpt-6-luna", seat.pin.?.model);

    // PHASE-UNIFORM: one seat, so every worker of the phase resolves to the
    // same model and files under the same cell — with or without a
    // root-authored genome. That is the property #290 needs preserved.
    for (sweep_niches) |niche| {
        try std.testing.expectEqualStrings("gpt-6-luna", seat.provider.model);
        const cell = seat.cellOf(niche);
        try std.testing.expectEqual(policy.Shape.research, cell.shape);
        try std.testing.expectEqualStrings("sweep", cell.role);
        try std.testing.expectEqual(policy.Source.learned_policy, seat.sourceFor(false));
        try std.testing.expectEqual(policy.Source.learned_policy, seat.sourceFor(true));
    }

    // An UNOBSERVED phase in the same session is untouched: same session, same
    // ladder, no evidence for its cell — so it keeps exactly today's behavior,
    // including today's `source` attribution.
    const quiet = phase.forPhase(base, .feature, "implement the fix", &.{"implementer"}, true);
    try std.testing.expect(quiet.pin == null);
    try std.testing.expectEqualStrings("gpt-6-sol", quiet.provider.model);
    try std.testing.expectEqual(policy.Source.ladder, quiet.sourceFor(false));
    try std.testing.expectEqual(policy.Source.workflow_override, quiet.sourceFor(true));
}

test "#376: the re-seated phase's trace says learned-policy on every worker" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();
    const env = Env.install(a, demo_archive, true);
    defer env.restore();

    var sink: Io.Writer.Allocating = .init(a);
    route_trace.g_test_sink = &sink;
    defer route_trace.g_test_sink = null;

    // Drive the exact emission workflow.zig makes, once per worker.
    const seat = phase.forPhase(codex("gpt-6-sol"), .research, "sweep the repo", &sweep_niches, true);
    for (sweep_niches) |niche| {
        route_trace.emitSpawnProvider(std.testing.io, null, niche, seat.provider, seat.cellOf(niche), seat.sourceFor(false), null, niche);
    }

    var lines = std.mem.splitScalar(u8, std.mem.trimEnd(u8, sink.writer.buffered(), "\n"), '\n');
    var n: usize = 0;
    while (lines.next()) |ln| : (n += 1) {
        const ev = try std.json.parseFromSliceLeaky(std.json.Value, a, ln, .{});
        try std.testing.expectEqualStrings("agent_route", ev.object.get("type").?.string);
        try std.testing.expectEqualStrings("gpt-6-luna", ev.object.get("resolved_model").?.string);
        try std.testing.expectEqualStrings("learned-policy", ev.object.get("source").?.string);
        try std.testing.expectEqualStrings("research", ev.object.get("shape").?.string);
        try std.testing.expectEqualStrings("sweep", ev.object.get("role").?.string);
        // The rung is read back off the seated model, and the id names the
        // exact cell that moved the phase.
        try std.testing.expectEqualStrings("small", ev.object.get("tier").?.string);
        try std.testing.expectEqualStrings("research/sweep@small", ev.object.get("policy_or_genome_id").?.string);
        try std.testing.expectEqualStrings("codex", ev.object.get("provider").?.string);
    }
    try std.testing.expectEqual(sweep_niches.len, n);
}

test "#376: an explicitly chosen worker model outranks the phase policy" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const base = codex("gpt-6-sol");

    // --subagent-model: the session named the worker model, so the route is
    // `session-default`, not the ladder's, and evidence never overrides a
    // human's stated choice.
    {
        const env = Env.install(arena_state.allocator(), demo_archive, false);
        defer env.restore();
        const seat = phase.forPhase(base, .research, "sweep the repo", &sweep_niches, true);
        try std.testing.expect(seat.pin == null);
        try std.testing.expectEqualStrings("gpt-6-sol", seat.provider.model);
        try std.testing.expectEqual(policy.Source.session_default, seat.sourceFor(false));
    }
    // A plain inherit-the-root (no worker provider at all) is the same story:
    // `session-default` is a choice, not a default the policy may improve on.
    {
        const env = Env.install(arena_state.allocator(), demo_archive, true);
        defer env.restore();
        const seat = phase.forPhase(base, .research, "sweep the repo", &sweep_niches, false);
        try std.testing.expect(seat.pin == null);
        try std.testing.expectEqual(policy.Source.session_default, seat.sourceFor(false));
    }
}

test "#376: the phase seat can only trade DOWN on cost, and never leaves the provider" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();

    // The mirror image of the demonstration: sol is the loved model but it
    // is also the DEARER one, so a phase already seated on luna can never be
    // moved up to it however well sol scored in the cell.
    const upward =
        \\{"kind":"score","prompt_sha":"aa","score":1,"shape":"research","role":"sweep","tier":"mid","model":"gpt-6-sol"}
        \\{"kind":"score","prompt_sha":"aa","score":1,"shape":"research","role":"sweep","tier":"mid","model":"gpt-6-sol"}
        \\{"kind":"score","prompt_sha":"bb","score":0,"shape":"research","role":"sweep","tier":"small","model":"gpt-6-luna"}
        \\{"kind":"score","prompt_sha":"bb","score":0,"shape":"research","role":"sweep","tier":"small","model":"gpt-6-luna"}
    ;
    const env = Env.install(a, upward, true);
    defer env.restore();
    const seat = phase.forPhase(codex("gpt-6-luna"), .research, "sweep the repo", &sweep_niches, true);
    try std.testing.expect(seat.pin == null);
    try std.testing.expectEqualStrings("gpt-6-luna", seat.provider.model);

    // And whatever it decides, the seat is provider-local by construction: the
    // id, the credential and the account all ride across unchanged, so no
    // phase can move prompts or billing to a second vendor. Cross-provider
    // workers keep needing --subagent-provider + the consent flag, and the
    // sub-first flat-rate routing in subagent_pin is reached only by an
    // explicit `tier` ask, which a phase task never makes.
    const env2 = Env.install(a, demo_archive, true);
    defer env2.restore();
    const base = codex("gpt-6-sol");
    const moved = phase.forPhase(base, .research, "sweep the repo", &sweep_niches, true);
    try std.testing.expect(moved.pin != null);
    try std.testing.expectEqualStrings(base.id, moved.provider.id);
    try std.testing.expectEqualStrings(base.api_key, moved.provider.api_key);
    try std.testing.expect(pin_mod.rungAffordable(base.model, moved.provider.model));
    try std.testing.expect(std.mem.indexOf(u8, @embedFile("route_phase.zig"), "subscriptionRung") == null);
}

test "#376: an uncelled phase, and a bootstrap install, keep today's behavior exactly" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();
    const base = codex("gpt-6-sol");
    {
        const env = Env.install(a, demo_archive, true);
        defer env.restore();
        // Off-vocabulary title, niches that disagree → no single role, so the
        // evidence that WOULD have moved a `sweep` phase moves nothing.
        const seat = phase.forPhase(base, .research, "ponder", &.{ "researcher", "review the diff" }, true);
        try std.testing.expect(seat.pin == null);
        try std.testing.expectEqualStrings("gpt-6-sol", seat.provider.model);
    }
    {
        // An install with no archive at all: zero cells — the learned layer
        // says nothing, so the search-role PRIOR seats sweep at the small
        // rung. Still reported as the ladder's answer: a hand rule, not
        // evidence, and never a claim of `learned-policy`.
        const env = Env.install(a, "", true);
        defer env.restore();
        const seat = phase.forPhase(base, .research, "sweep the repo", &sweep_niches, true);
        try std.testing.expect(seat.pin != null);
        try std.testing.expectEqualStrings("gpt-6-luna", seat.provider.model);
        try std.testing.expect(seat.role_prior);
        try std.testing.expectEqual(policy.Source.ladder, seat.sourceFor(false));
    }
}

test "#376 firewall: the seat is resolved ONCE per phase, outside the task loop" {
    // The property is about the CALL SITE — that no task can influence its own
    // model — so it is pinned as source text, the same way #372 pinned the
    // no-per-task-pin invariant it replaces.
    const src = @embedFile("workflow.zig");
    // #380 wraps the same one-shot resolution in vision_ask.phaseSeat, which
    // may swap the phase's model but still yields ONE seat for the whole
    // phase — the property this test exists to protect.
    const seat_at = std.mem.indexOf(u8, src, "const seat = vision_ask.phaseSeat(route_phase.forPhase(").?;
    // (The loop indexes from 0 rather than 1 since the §2b collapse gate needs
    // the task INDEX to look up its survivor, not a 1-based display number.)
    const loop_at = std.mem.indexOf(u8, src, "for (labels, prompts, overrides, niches, isolations, isolation_fallbacks, futures, 0..)").?;
    try std.testing.expect(seat_at < loop_at); // resolved before the fan-out
    // Every spawn in the phase — first attempt and retry alike — receives that
    // one value verbatim; nothing per task is ever passed as a model pin.
    try std.testing.expect(std.mem.indexOf(u8, src, "isolation_fallback, seat.pin }") != null);
    try std.testing.expect(std.mem.indexOf(u8, src, "isolation_fallbacks[i], seat.pin }") != null);
}