Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
24.5 KB
Raw Blame
//! Tests for the escalation ladder and the edit contract, split out of
//! escalation.zig (600-line cap; the `<mod>_tests.zig` pattern, wired into the
//! test root by test_hooks.zig).
//!
//! The ladder is deliberately PURE over cheap observables, which is what makes
//! this file possible: the whole threshold matrix — scope x budget x
//! audit-language x prior-failure — is driven here with no allocator, no
//! network, no process and no trace file.

const std = @import("std");
const Value = std.json.Value;

const escalation = @import("escalation.zig");
const orch = @import("orchestration_policy.zig");
const pb = @import("phase_budget.zig");
const shapes = @import("shapes.zig");

const Rung = orch.Rung;

/// The observables the study's cap-30 runs actually presented, as a base to
/// vary one axis at a time from. 27 remaining is what is left after the root
/// has read the task and authored a workflow — the real moment `admit` runs.
fn base() escalation.Observables {
    return .{
        .shape = .review,
        .task_class = .bugfix,
        .files = 1,
        .widest = 3,
        .tasks = 5,
        .plan_estimate = 17,
        .audit = false,
        .prior_failure = false,
        .remaining = 27,
        .cap = 30,
    };
}

fn phasesFrom(a: std.mem.Allocator, src: []const u8) []const Value {
    const v = std.json.parseFromSliceLeaky(Value, a, src, .{}) catch unreachable;
    return v.array.items;
}

test "ladder: the study's cap-30 bugfix is R0 — one file is not a fleet" {
    // THE regression. 01-bugfix-B: two buggy functions in stats.py, given five
    // workers and three judges, 30 calls spent, zero edits. Scope decides, and
    // the scope here is one file.
    var o = base();
    try std.testing.expectEqual(Rung.R0, escalation.ladderRung(o));
    // Note WHY it is R0 and not a budget refusal: a review fleet (17) plus the
    // landing reserve (7) does fit the 27 that remain. The budget said yes.
    try std.testing.expect(pb.Ledger.init(30).fits(27, pb.fleetFloor(.review)));
    // Two files is still a pair, not a fan-out.
    o.files = 2;
    try std.testing.expectEqual(Rung.R0, escalation.ladderRung(o));
}

test "ladder: scope alone opens R2, and only at 3+ files with a plan that fans out" {
    var o = base();
    o.files = 3;
    try std.testing.expectEqual(Rung.R2, escalation.ladderRung(o));
    o.files = 8;
    try std.testing.expectEqual(Rung.R2, escalation.ladderRung(o));
    // A plan that authored ONE task per phase is not a fleet however many
    // files the ask names — there is nothing to run in parallel.
    o.widest = 1;
    try std.testing.expectEqual(Rung.R0, escalation.ladderRung(o));
}

test "ladder: budget vetoes R2 even at full scope" {
    var o = base();
    o.files = 6;
    // 12 left, 7 reserved, a review fleet needs 17: the fleet cannot finish,
    // so the answer is the rung that can.
    o.remaining = 12;
    try std.testing.expectEqual(Rung.R0, escalation.ladderRung(o));
    // Raise the ceiling and the same ask fans out.
    o.remaining = 30;
    try std.testing.expectEqual(Rung.R2, escalation.ladderRung(o));
}

test "ladder: audit-class language buys R3, and a prior failed attempt buys it for free" {
    var o = base();
    // Still one file — but the USER asked for breadth, so breadth is what they
    // are paying for. This is the deliberate escape hatch from R0.
    o.audit = true;
    try std.testing.expectEqual(Rung.R3, escalation.ladderRung(o));
    // …unless it cannot finish. R3 is not exempt from arithmetic.
    o.remaining = 10;
    try std.testing.expectEqual(Rung.R0, escalation.ladderRung(o));
    // First-attempt failure is the free escalation signal: same ask, same
    // scope, but the cheap answer has already been tried and did not stick.
    o = base();
    o.prior_failure = true;
    try std.testing.expectEqual(Rung.R3, escalation.ladderRung(o));
}

test "ladder: an open-ended research ask is R1, one scout — never a sweep fleet" {
    var o = base();
    o.shape = .research;
    o.task_class = .research;
    o.files = 4; // enough files for R2 on any other shape
    // 05-research-B: three sweepers over three small files, briefs measuring
    // 1.000 similar to each other, 24 calls — for an answer one scout reached.
    try std.testing.expectEqual(Rung.R1, escalation.ladderRung(o));
    // Audit language still overrides upward: "exhaustively map every module"
    // is a real fleet ask even though it is research-shaped.
    o.audit = true;
    try std.testing.expectEqual(Rung.R3, escalation.ladderRung(o));
}

test "verdictFor: R0 declines non-error, R1 caps at one scout, R2 downsizes to fit" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();
    const phases = phasesFrom(a,
        \\[{"title":"find bugs","tasks":[1,2,3,4,5,6]},
        \\ {"title":"synthesize","tasks":[1]}]
    );

    var o = base();
    // R0's advisory is a RESULT, not an error: a declined fleet is not a
    // failed tool call, and is_error here would burn a retry and read to the
    // model as a broken harness.
    switch (escalation.verdictFor(.R0, o, phases)) {
        .solo => |advice| {
            try std.testing.expect(std.mem.indexOf(u8, advice, "R0") != null);
            try std.testing.expect(std.mem.indexOf(u8, advice, "Do it yourself") != null);
            // It also says what would earn a fleet, so the model has a path.
            try std.testing.expect(std.mem.indexOf(u8, advice, "verification fails") != null);
        },
        else => return error.ExpectedSolo,
    }
    try std.testing.expectEqual(@as(usize, 1), (escalation.verdictFor(.R1, o, phases)).downsize);

    // 6 finders (24) + 1 synthesis (2) = 26, against 21 spendable → trim.
    o.plan_estimate = 26;
    const w = (escalation.verdictFor(.R2, o, phases)).downsize;
    try std.testing.expect(w >= 1 and w < 6);
    // A plan that already fits is not touched.
    o.plan_estimate = 14;
    try std.testing.expect(escalation.verdictFor(.R2, o, phases) == .proceed);
}

test "downsizeWidth: trims redundant finders and never the landing phase" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();
    // A-fix at width: 4 finders (16) + 1 implementer (6) + 1 verify (3) = 25,
    // against 20 spendable at cap 30 / 27 remaining.
    const phases = phasesFrom(a,
        \\[{"title":"find the defect","tasks":[1,2,3,4]},
        \\ {"title":"implement the fix","tasks":[1]},
        \\ {"title":"verify the diff","tasks":[1]}]
    );
    const w = escalation.downsizeWidth(phases, 30, 27);
    // 2 finders (8) + 6 + 3 = 17 fits; 3 (12+6+3=21) is one call past the
    // reserve, because the reserve also holds the narration turn (#390).
    try std.testing.expectEqual(@as(usize, 2), w);
    // Whatever the width, the implement phase carries one task, so the edit
    // contract survives every trim — which is §4-P1's inviolable rule.
    try std.testing.expect(w >= 1);
    // Squeeze until only the skinniest plan fits: 1 finder + 1 implementer +
    // 1 verify = 13, on top of the 7-call reserve, so 20 left is the floor.
    try std.testing.expectEqual(@as(usize, 1), escalation.downsizeWidth(phases, 30, 20));
    // And below that, nothing fits — the caller must fall back to solo. Note
    // WHAT does not fit: the landing phase's own implementer, which is the
    // point at which a fleet stops being able to produce a patch at all.
    try std.testing.expectEqual(@as(usize, 0), escalation.downsizeWidth(phases, 30, 19));
    // An unlimited pool never trims.
    try std.testing.expectEqual(@as(usize, 4), escalation.downsizeWidth(phases, 0, std.math.maxInt(u64)));
}

test "decide: an explicit user shape choice is honoured and never overridden (#290)" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();
    const phases = phasesFrom(a, "[{\"title\":\"find bugs\",\"tasks\":[1,2,3]}]");
    const key: orch.Key = .{ .task_class = .bugfix, .budget_band = .b40, .stratum = "m" };

    // A learned cell that would normally trade this run DOWN to R0.
    var arms = [_]orch.ArmObs{
        armWith(key, .R2, 3, 0.2, 30),
        armWith(key, .R0, 4, 0.9, 8),
    };

    // Without an explicit choice, the ladder's R0 stands and the source says
    // which layer decided.
    const implicit = escalation.decide(base(), phases, false, &arms, key);
    try std.testing.expectEqual(Rung.R0, implicit.rung);
    try std.testing.expectEqual(orch.Source.bootstrap, implicit.source);

    // With one, the user's intent wins outright: R2 or above, marked explicit,
    // and no learned arm may move it in either direction.
    const explicit = escalation.decide(base(), phases, true, &arms, key);
    try std.testing.expect(explicit.rung.level() >= Rung.R2.level());
    try std.testing.expectEqual(orch.Source.explicit, explicit.source);
    try std.testing.expect(explicit.verdict != .solo);
}

test "decide: a learned cell overrides the hand ladder and says so" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();
    const phases = phasesFrom(a, "[{\"title\":\"find bugs\",\"tasks\":[1,2,3]}]");
    const key: orch.Key = .{ .task_class = .bugfix, .budget_band = .b40, .stratum = "m" };
    var o = base();
    o.files = 4; // the hand ladder would say R2 here

    var arms = [_]orch.ArmObs{
        armWith(key, .R2, 4, 0.4, 28),
        armWith(key, .R0, 5, 0.95, 9),
    };
    const d = escalation.decide(o, phases, false, &arms, key);
    // Trading DOWN needs only "not worse", and this is much better.
    try std.testing.expectEqual(Rung.R0, d.rung);
    try std.testing.expectEqual(orch.Source.learned, d.source);
    // The advisory names the learned policy, not the ladder's scope arithmetic.
    try std.testing.expect(std.mem.indexOf(u8, d.verdict.solo, "(escalation R0, learned):") != null);
}

fn armWith(key: orch.Key, arm: Rung, n: u32, score: f64, calls: u32) orch.ArmObs {
    var a: orch.ArmObs = .{ .key = key, .arm = arm };
    var i: u32 = 0;
    while (i < n) : (i += 1) {
        a.sum_score += score;
        a.n += 1;
        a.landed_n += 1;
        a.calls[a.n_calls] = calls;
        a.n_calls += 1;
    }
    return a;
}

test "userChoseShape: names the fan-out, not merely the codeword" {
    // The codeword alone is NOT an explicit shape choice — that is the whole
    // premise of the redesign: `ultracode` opts into the ladder, not into a
    // fleet. Only naming the structure counts.
    try std.testing.expect(!escalation.userChoseShape("ultracode fix the bug in stats.py"));
    try std.testing.expect(escalation.userChoseShape("ultracode use shape A on this diff"));
    try std.testing.expect(escalation.userChoseShape("fan out three reviewers"));
    try std.testing.expect(escalation.userChoseShape("spawn one agent per module"));
    try std.testing.expect(escalation.userChoseShape("review these IN PARALLEL"));
    try std.testing.expect(!escalation.userChoseShape("summarize the changelog"));
}

test "countDistinctFiles: paths, not version numbers or sentence periods" {
    // The study's five prompts, as scope evidence.
    try std.testing.expectEqual(@as(usize, 2), escalation.countDistinctFiles(
        "This directory contains stats.py and test_stats.py. Two of the three functions in stats.py have bugs.",
    ));
    try std.testing.expectEqual(@as(usize, 2), escalation.countDistinctFiles(
        "Create a command-line tool wordfreq.py in this directory. There is a sample.txt to test against.",
    ));
    try std.testing.expectEqual(@as(usize, 4), escalation.countDistinctFiles(
        "This directory is a small Python project: engine.py, report.py, main.py and settings.json.",
    ));
    // A trailing period, a version number and a bare sentence are not files.
    try std.testing.expectEqual(@as(usize, 0), escalation.countDistinctFiles(
        "The minimum supported Python version is 3.11. Prefer the most recent source.",
    ));
    // Directory-qualified paths count once each, case-insensitively.
    try std.testing.expectEqual(@as(usize, 2), escalation.countDistinctFiles("src/main.zig and SRC/MAIN.ZIG and src/util.zig"));
    // Bounded: a brief naming hundreds of distinct paths still fits the fixed
    // scratch, and the count saturates rather than growing without limit.
    var many: [9 * 300]u8 = @splat(' ');
    for (0..300) |i| _ = std.fmt.bufPrint(many[i * 9 ..][0..8], "f{d:0>4}.py", .{i}) catch unreachable;
    const n = escalation.countDistinctFiles(&many);
    try std.testing.expectEqual(@as(usize, escalation.max_files_counted), n);
    // Which is still far above the only threshold that reads it.
    try std.testing.expect(n >= escalation.fleet_scope_min);
}

test "ladder: a first small-scope failure with a verifier is R0d — revision before fleet" {
    var o = base();
    o.prior_failure = true;
    o.prior_failure_count = 1;
    o.verifier = .diff;
    try std.testing.expectEqual(Rung.R0d, escalation.ladderRung(o));
    // R0d spends no spawns, so it stays admissible where a fleet is not.
    o.remaining = 3;
    try std.testing.expectEqual(Rung.R0d, escalation.ladderRung(o));
    // A SECOND failure means the revision was already spent: fleet next.
    o = base();
    o.prior_failure = true;
    o.prior_failure_count = 2;
    o.verifier = .diff;
    try std.testing.expectEqual(Rung.R3, escalation.ladderRung(o));
    // No verifier, no revision loop (Weng's gate): the R3 judges ARE the
    // external feedback a solo retry would lack.
    o.prior_failure_count = 1;
    o.verifier = .none;
    try std.testing.expectEqual(Rung.R3, escalation.ladderRung(o));
    // A coverage-shaped failure (3+ files) wants breadth, not revision.
    o.verifier = .diff;
    o.files = 4;
    try std.testing.expectEqual(Rung.R3, escalation.ladderRung(o));
    // Audit language keeps its precedence over the revision rung.
    o = base();
    o.prior_failure = true;
    o.prior_failure_count = 1;
    o.verifier = .diff;
    o.audit = true;
    try std.testing.expectEqual(Rung.R3, escalation.ladderRung(o));
}

test "R0d: the advisory prescribes a revision and carries the parked evidence" {
    escalation.resetSession();
    defer escalation.resetSession();
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();
    // Bare advisory: a revision prescription with a path to the fleet.
    const bare = escalation.deepRetryAdvice(a, .bugfix);
    try std.testing.expect(std.mem.indexOf(u8, bare, "R0d") != null);
    try std.testing.expect(std.mem.indexOf(u8, bare, "second failure earns the fleet") != null);
    // With parked evidence the advisory embeds it verbatim — the retry acts
    // on the observed signal, not on the reasoning that already missed.
    escalation.failure_evidence.note(.bugfix, "eval RED: score 40.0/100 (target 90, exit 1)");
    const with_ev = escalation.deepRetryAdvice(a, .bugfix);
    try std.testing.expect(std.mem.indexOf(u8, with_ev, "PRIOR ATTEMPT EVIDENCE") != null);
    try std.testing.expect(std.mem.indexOf(u8, with_ev, "eval RED: score 40.0/100") != null);
    // Another class stays bare — evidence never leaks across classes.
    try std.testing.expect(std.mem.indexOf(u8, escalation.deepRetryAdvice(a, .research), "PRIOR ATTEMPT EVIDENCE") == null);
    // resetSession clears the ledger with the rest of the session state.
    escalation.resetSession();
    try std.testing.expectEqualStrings("", escalation.failure_evidence.evidence(.bugfix));
}

test "observe: decline count and the class verifier feed the revision gate" {
    escalation.resetSession();
    defer escalation.resetSession();
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();
    const phases = phasesFrom(a, "[{\"title\":\"implement the fix\",\"tasks\":[1]}]");
    escalation.noteDeclined(.bugfix);
    const o = escalation.observe(phases, "fix the bug in stats.py", .review, 27, 30);
    try std.testing.expect(o.prior_failure);
    try std.testing.expectEqual(@as(u8, 1), o.prior_failure_count);
    try std.testing.expectEqual(escalation.failure_evidence.Verifier.diff, o.verifier);
    try std.testing.expectEqual(Rung.R0d, escalation.ladderRung(o));
    // verdictFor(R0d) declines non-error with the revision prescription.
    switch (escalation.verdictFor(.R0d, o, phases)) {
        .solo => |advice| try std.testing.expect(std.mem.indexOf(u8, advice, "revision case") != null),
        else => return error.ExpectedSolo,
    }
}

test "edit contract: a clean tree is is_error, a tree that moved is ok" {
    // The PURE half of §3b, which is the half with the interesting logic — the
    // impure half is one `git status --porcelain` and a string compare.
    //
    // An uncontracted phase is never judged on the tree at all.
    try std.testing.expect(!escalation.contractUnmet(false, "", ""));
    try std.testing.expect(!escalation.contractUnmet(false, " M stats.py\n", " M stats.py\n"));

    // Contracted + clean tree: nothing landed, period. This is the 30-calls-
    // no-edit run, caught by a probe that costs zero model calls.
    try std.testing.expect(escalation.contractUnmet(true, "", ""));
    try std.testing.expect(escalation.contractUnmet(true, "", "   \n \t\n"));

    // Contracted + a tree that moved: the work landed.
    try std.testing.expect(!escalation.contractUnmet(true, "", " M stats.py\n"));
    try std.testing.expect(!escalation.contractUnmet(true, " M stats.py\n", " M stats.py\n M util.py\n"));

    // Contracted + an ALREADY-dirty tree this phase did not touch: also unmet.
    // Without this, any phase run in a workspace with uncommitted work would
    // pass the contract for free, which is most real workspaces.
    try std.testing.expect(escalation.contractUnmet(true, " M stats.py\n", " M stats.py\n"));
    try std.testing.expect(escalation.contractUnmet(true, " M stats.py", " M stats.py\n"));
}

test "the contract's texts say what to DO, not merely that something failed" {
    // The failure text is fed straight back to a worker that gets one retry,
    // so it has to be an instruction.
    try std.testing.expect(std.mem.indexOf(u8, escalation.contract_unmet, "no files changed") != null);
    try std.testing.expect(std.mem.indexOf(u8, escalation.contract_unmet, "changed path") != null);
    // And it must NOT carry the not-retry-safe marker, or the retry the
    // contract depends on would be skipped.
    try std.testing.expect(std.mem.indexOf(u8, escalation.contract_unmet, "retry is NOT safe") == null);
    // The brief boilerplate is codex's changed-path listing requirement.
    try std.testing.expect(std.mem.indexOf(u8, escalation.contract_brief_note, "changed:") != null);
}

test "isContracted / isVariantsPhase read the closed slot vocabulary" {
    try std.testing.expect(escalation.isContracted(shapes.canonicalSlot("implement the fix")));
    try std.testing.expect(escalation.isContracted(shapes.canonicalSlot("build from the winner")));
    try std.testing.expect(!escalation.isContracted(shapes.canonicalSlot("find security bugs")));
    try std.testing.expect(!escalation.isContracted(shapes.canonicalSlot("verify each finding")));

    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();
    const phases = phasesFrom(a,
        \\[{"title":"variants of the landing page","tasks":[1,2]},
        \\ {"title":"find bugs","tasks":[1]}]
    );
    try std.testing.expect(escalation.isVariantsPhase(phases[0]));
    try std.testing.expect(!escalation.isVariantsPhase(phases[1]));
}

test "prior-failure tracking: the second ask of a class carries the escalation signal" {
    escalation.resetSession();
    defer escalation.resetSession();
    try std.testing.expect(!escalation.priorAttemptFailed(.bugfix));
    escalation.noteDeclined(.bugfix);
    try std.testing.expect(escalation.priorAttemptFailed(.bugfix));
    // Per class: declining a bugfix says nothing about a research ask.
    try std.testing.expect(!escalation.priorAttemptFailed(.research));
    // A pipeline that could not fit has no class, so it counts for all.
    escalation.notePipelineDeclined();
    try std.testing.expect(escalation.priorAttemptFailed(.research));
}

test "observe: reads scope off the ask AND the plan, and classifies the ask" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();
    const phases = phasesFrom(a,
        \\[{"title":"find bugs","tasks":[
        \\   {"prompt":"read report.py and settings.json"},
        \\   {"prompt":"read main.py"}]}]
    );
    const o = escalation.observe(phases, "ultracode fix the bug in stats.py", .review, 27, 30);
    // stats.py from the ASK plus three from the briefs: the ask is scope
    // evidence even when the plan paraphrases it.
    try std.testing.expectEqual(@as(usize, 4), o.files);
    try std.testing.expectEqual(@as(usize, 2), o.tasks);
    try std.testing.expectEqual(@as(usize, 2), o.widest);
    try std.testing.expectEqual(shapes.TaskClass.bugfix, o.task_class);
    try std.testing.expect(!o.audit);
    // 2 finders at 4 calls each.
    try std.testing.expectEqual(@as(u64, 8), o.plan_estimate);
}

test "shapes.classOf / isAuditClass: the study's five prompts land where the ladder needs them" {
    try std.testing.expectEqual(shapes.TaskClass.bugfix, shapes.classOf(
        "Two of the three functions in stats.py have bugs. Fix stats.py so that the whole pytest suite passes.",
    ));
    try std.testing.expectEqual(shapes.TaskClass.refactor, shapes.classOf(
        "Refactor it: 1. Rename the class TaxEngine to LevyEngine.",
    ));
    try std.testing.expectEqual(shapes.TaskClass.research, shapes.classOf(
        "Write summary.md answering these four questions, and cite the file each answer comes from.",
    ));
    try std.testing.expectEqual(shapes.TaskClass.feature, shapes.classOf(
        "Create a command-line tool wordfreq.py in this directory.",
    ));
    // A repair ask that MENTIONS review is still a repair — the precedence
    // that stops shape A from being handed a bugfix.
    try std.testing.expectEqual(shapes.TaskClass.bugfix, shapes.classOf("fix the bug the review found"));
    // …and an ordinary feature spec that happens to say "broken" is NOT a
    // bugfix. The bare needle mis-classified the study's own wordfreq task,
    // caught on the first acceptance run: the full prompt says "ties broken by
    // ascending ASCII order", which is a sort rule, not a defect report.
    const wordfreq_full =
        "Create a command-line tool wordfreq.py in this directory. Output: one line per token, " ++
        "sorted by count descending, ties broken by ascending ASCII order of the token.";
    try std.testing.expectEqual(shapes.TaskClass.feature, shapes.classOf(wordfreq_full));
    // The qualified forms still read as defects.
    try std.testing.expectEqual(shapes.TaskClass.bugfix, shapes.classOf("the parser is broken on empty input"));
    try std.testing.expectEqual(shapes.TaskClass.bugfix, shapes.classOf("two broken tests in the suite"));
    // Breadth language outranks the noun it audits FOR: "for bugs" must not
    // file an audit fleet's outcome under the bugfix learning cell (the
    // mis-cell the branch's first real-model smoke caught).
    try std.testing.expectEqual(shapes.TaskClass.review, shapes.classOf(
        "thoroughly audit every file in this repo for bugs and robustness problems",
    ));
    // Audit language is narrow on purpose: a fleet takes an explicit word.
    try std.testing.expect(shapes.isAuditClass("thoroughly audit the auth layer"));
    try std.testing.expect(shapes.isAuditClass("check every file for this pattern"));
    try std.testing.expect(!shapes.isAuditClass("fix the bug in stats.py"));
    try std.testing.expect(!shapes.isAuditClass("add a --top flag"));
}

test "shapes.rawAsk: captured by value, so a freed per-turn arena cannot dangle" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    const a = arena_state.allocator();
    const typed = try std.fmt.allocPrint(a, "ultracode fix {s}", .{"stats.py"});
    _ = try shapes.applyUltracodeSteering(a, typed, typed, false);
    try std.testing.expectEqualStrings("ultracode fix stats.py", shapes.rawAsk());
    // Free the arena the string came from; the capture must survive it.
    arena_state.deinit();
    try std.testing.expectEqualStrings("ultracode fix stats.py", shapes.rawAsk());
    try std.testing.expectEqual(shapes.TaskClass.bugfix, shapes.classOf(shapes.rawAsk()));
}