Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
17.3 KB
Raw Blame
//! Tests for the workflow engine (phases + pipeline). Split out of
//! src/workflow.zig, which crossed the 600-line cap once the audit fixes for
//! the {{prev}} phase budget, the all-failed abort, the shared context slot
//! and the pipeline isolation guard rail all landed in it. Reached through the
//! `test { _ = ... }` hook in main.zig - without that line these silently
//! compile to nothing and the suite still reports green.

const std = @import("std");
const Io = std.Io;
const Value = std.json.Value;
const workflow = @import("workflow.zig");
const subagent = @import("subagent.zig");
const subagent_run = @import("subagent_run.zig");
const json_args = @import("json_args.zig");
const tools = @import("tools.zig");
const util = @import("util.zig");
const ToolCtx = tools.ToolCtx;
const cappedPrevBody = workflow.cappedPrevBody;
const phaseTaskCap = workflow.phaseTaskCap;
const admissionNotice = workflow.admissionNotice;
const withContext = workflow.withContext;
const gateAllows = workflow.gateAllows;
const buildAbortText = workflow.buildAbortText;
const failExcerpt = workflow.failExcerpt;
const fail_excerpt_cap = workflow.fail_excerpt_cap;
const buildManifest = workflow.buildManifest;
const pipelinePrompt = workflow.pipelinePrompt;
const pipelineIsolationError = workflow.pipelineIsolationError;
const phase_prev_budget = workflow.phase_prev_budget;
const PhaseTally = workflow.PhaseTally;
const min_task_prev_cap = workflow.min_task_prev_cap;
const max_workflow_tasks = workflow.max_workflow_tasks;
const execWorkflow = workflow.execWorkflow;

test "cappedPrevBody bounds a wide phase output, keeps head + inspect tail at several caps (#4/U3)" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();

    // A short result passes through untouched — most outputs never hit the
    // cap, at any cap size.
    try std.testing.expectEqualStrings("hi", cappedPrevBody(a, "hi", phase_prev_budget));
    try std.testing.expectEqualStrings("hi", cappedPrevBody(a, "hi", min_task_prev_cap));

    // A huge result is capped at several different cap sizes (the divided
    // per-task cap varies with phase width): head kept, the trailing
    // inspect: pointer never truncated away, a truncation marker added, and
    // the full text never lands verbatim.
    const big = util.repeatBytes("X", 9000) ++ "\n[subagent sa-007-abcd · inspect: .graff/subagents/sa-007-abcd.md]";
    for ([_]usize{ phase_prev_budget, 3000, 1500, min_task_prev_cap }) |cap| {
        const capped = cappedPrevBody(a, big, cap);
        try std.testing.expect(capped.len < big.len);
        try std.testing.expect(capped.len <= cap + 200); // head + tail + marker overhead
        try std.testing.expect(std.mem.indexOf(u8, capped, "truncated") != null);
        try std.testing.expect(std.mem.indexOf(u8, capped, "inspect: .graff/subagents/sa-007-abcd.md") != null);
        try std.testing.expect(std.mem.indexOf(u8, capped, big) == null);
    }
}

test "phaseTaskCap divides the phase budget across tasks, floored at min_task_prev_cap (#U3)" {
    // Narrow phases: budget/n_tasks comfortably clears the floor.
    try std.testing.expectEqual(@as(usize, phase_prev_budget), phaseTaskCap(1));
    try std.testing.expectEqual(@as(usize, phase_prev_budget / 2), phaseTaskCap(2));

    // Wide phases (up to max_workflow_tasks, and beyond): the divided cap
    // would fall below min_task_prev_cap, so the floor wins — never an
    // unreadably tiny per-task slice.
    try std.testing.expect(phase_prev_budget / max_workflow_tasks < min_task_prev_cap);
    try std.testing.expectEqual(@as(usize, min_task_prev_cap), phaseTaskCap(max_workflow_tasks));
    try std.testing.expectEqual(@as(usize, min_task_prev_cap), phaseTaskCap(50));
}

test "admissionNotice exposes omitted workflow tasks to synthesis and the final manifest (#863)" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();

    try std.testing.expectEqualStrings(
        "workflow warning [phase 1/2 sweep]: requested=4, executed=1, omitted=3",
        admissionNotice(a, 1, 2, "sweep", 4, 1),
    );
    try std.testing.expectEqualStrings("", admissionNotice(a, 1, 2, "sweep", 4, 4));
    try std.testing.expectEqualStrings("", admissionNotice(a, 2, 2, "synthesize", 1, 1));
}

test "withContext: set prepends context + blank line; absent/empty is byte-identical (U5)" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();

    // Context set: context, blank line, raw task — nothing else.
    const with = try withContext(a, "Repo: zig 0.16, 600-line file cap.", "Implement the thing.");
    try std.testing.expectEqualStrings("Repo: zig 0.16, 600-line file cap.\n\nImplement the thing.", with);

    // Context absent → byte-identical to the raw task (no stray blank line).
    const absent = try withContext(a, "", "Implement the thing.");
    try std.testing.expectEqualStrings("Implement the thing.", absent);

    // Context present but empty string behaves the same as absent.
    const empty = try withContext(a, "", "Do the other thing.");
    try std.testing.expectEqualStrings("Do the other thing.", empty);
}

test "gateAllows: empty when always runs, else case-insensitive substring (#5)" {
    try std.testing.expect(gateAllows("anything", "")); // no gate → always run
    try std.testing.expect(gateAllows("found 3 ISSUES here", "issues")); // case-insensitive hit
    try std.testing.expect(!gateAllows("all clean, no findings", "ISSUE")); // absent → skip
    try std.testing.expect(!gateAllows("", "ready")); // empty prev → skip
}

test "buildAbortText names the phase index and title, and surfaces each failed task's error (U2/#248)" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();

    const labels = [_][]const u8{ "scan A", "scan B" };
    // Parallel per-task excerpts: one real API error, one task that left
    // nothing to report.
    // Mirrors what subagentFailure actually emits today, attempt count and all
    // — a fixture that has drifted from production cannot catch the excerpt
    // regressing, which is the only thing this test is for.
    const details = [_][]const u8{ "subagent sa-001 failed before producing a report: 429 rate_limit_exceeded [quota/rate-limit failure, 3 attempts]. retry is likely safe", "" };
    const text = try buildAbortText(a, &labels, &details, 2, 3, "Recon");

    // Every failed task's header survives, so a human can see what was tried.
    try std.testing.expect(std.mem.indexOf(u8, text, "### scan A (no result — task failed)") != null);
    try std.testing.expect(std.mem.indexOf(u8, text, "### scan B (no result — task failed)") != null);
    // #248 — the underlying API error travels WITH the header instead of being
    // consumed by the retry gate and dropped. This is the whole ask on the
    // issue: a caller that only reads "task failed" cannot adapt.
    try std.testing.expect(std.mem.indexOf(u8, text, "429 rate_limit_exceeded") != null);
    try std.testing.expect(std.mem.indexOf(u8, text, "### scan A (no result — task failed)\nsubagent sa-001 failed") != null);
    // The abort line names the phase index/total and title — the whole point
    // of this text existing instead of a silent empty {{prev}}.
    try std.testing.expect(std.mem.indexOf(u8, text, "workflow aborted: every task in phase 2/3 (Recon) failed") != null);
}

test "failExcerpt keeps the API error on one capped line, and stays empty when there is none (#248)" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();

    // A real subagentFailure text: the cause survives, flattened, so it can sit
    // under a "### label" header without breaking that layout.
    const raw = "subagent sa-001 failed before producing a report:\n{\"error\":\"rate_limit_exceeded\"}\t[quota/rate-limit failure, 3 attempts]. retry is likely safe";
    const one = failExcerpt(a, raw);
    try std.testing.expect(std.mem.indexOf(u8, one, "rate_limit_exceeded") != null);
    try std.testing.expect(std.mem.indexOfAny(u8, one, "\n\r\t") == null);

    // A huge error body is capped, so one failure can never blow up the next
    // phase's prompt — the reason the excerpt is bounded at all.
    const huge = util.repeatBytes("E", 5000);
    const capped = failExcerpt(a, &huge);
    try std.testing.expect(capped.len < huge.len);
    try std.testing.expect(capped.len <= fail_excerpt_cap + 8); // + the "…" marker
    try std.testing.expect(std.mem.endsWith(u8, capped, "…"));

    // Nothing to report → "" so the render sites keep the bare header rather
    // than printing an empty detail line.
    try std.testing.expectEqualStrings("", failExcerpt(a, "  \n\t "));
}

test "buildManifest reports ok/retried per phase and SKIPPED for a gated phase (U2)" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();

    const tallies = [_]PhaseTally{
        .{ .phase_no = 1, .total_phases = 3, .title = "Gather", .ok = 3, .total = 3, .retried = 1, .diversity = admissionNotice(a, 1, 3, "Gather", 4, 3) },
        .{ .phase_no = 2, .total_phases = 3, .title = "Verify", .ok = 0, .total = 2, .retried = 0, .skipped_when = "no findings" },
        .{ .phase_no = 3, .total_phases = 3, .title = "Report", .ok = 1, .total = 2, .retried = 0 },
    };
    const block = try buildManifest(a, &tallies);

    try std.testing.expect(std.mem.startsWith(u8, block, "## workflow"));
    try std.testing.expect(std.mem.indexOf(u8, block, "phase 1/3 Gather: 3/3 ok, 1 retried") != null);
    try std.testing.expect(std.mem.indexOf(u8, block, "workflow warning [phase 1/3 Gather]: requested=4, executed=3, omitted=1") != null);
    try std.testing.expect(std.mem.indexOf(u8, block, "phase 2/3 Verify: SKIPPED (when=\"no findings\")") != null);
    try std.testing.expect(std.mem.indexOf(u8, block, "phase 3/3 Report: 1/2 ok, 0 retried") != null);
    // A skipped phase must never claim an ok/retried count it never earned.
    try std.testing.expect(std.mem.indexOf(u8, block, "phase 2/3 Verify: 0/2") == null);
}

test "#382: a phase's diversity note rides the manifest, under its own phase line" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();

    // The manifest is where the ROOT reads a run's shape, so it is where a
    // note addressed to the orchestrator belongs — a phase's own results are
    // summarised into {{prev}} and go to the next phase's WORKERS instead.
    const note = "diversity warning: 3 variant briefs are ~62% similar — proceed if intentional.";
    const tallies = [_]PhaseTally{
        .{ .phase_no = 1, .total_phases = 2, .title = "variants", .ok = 3, .total = 3, .retried = 0, .diversity = note },
        .{ .phase_no = 2, .total_phases = 2, .title = "build", .ok = 1, .total = 1, .retried = 0 },
    };
    const block = try buildManifest(a, &tallies);

    const phase1 = std.mem.indexOf(u8, block, "phase 1/2 variants: 3/3 ok, 0 retried").?;
    const warned = std.mem.indexOf(u8, block, note).?;
    const phase2 = std.mem.indexOf(u8, block, "phase 2/2 build").?;
    try std.testing.expect(phase1 < warned and warned < phase2);

    // A quiet phase adds nothing at all — no "diversity ok" line to train the
    // root to skim past.
    const quiet = [_]PhaseTally{.{ .phase_no = 1, .total_phases = 1, .title = "variants", .ok = 3, .total = 3, .retried = 0 }};
    try std.testing.expectEqualStrings("## workflow\nphase 1/1 variants: 3/3 ok, 0 retried", try buildManifest(a, &quiet));
}

test "pipelinePrompt substitutes {{item}}/{{prev}} and appends when omitted (#3)" {
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const a = arena_state.allocator();

    // {{item}} substituted on stage 1; no previous-stage section.
    const s1 = try pipelinePrompt(a, "Audit {{item}} for bugs.", "src/x.zig", "", 1);
    try std.testing.expect(std.mem.indexOf(u8, s1, "Audit src/x.zig for bugs.") != null);
    try std.testing.expect(std.mem.indexOf(u8, s1, "previous stage") == null);

    // {{item}} and {{prev}} both substituted on stage 2.
    const s2 = try pipelinePrompt(a, "Given {{prev}}, fix {{item}}.", "src/x.zig", "BUG: off-by-one", 2);
    try std.testing.expect(std.mem.indexOf(u8, s2, "Given BUG: off-by-one, fix src/x.zig.") != null);

    // Omitted placeholders are appended (item on stage 1, prev on stage 2+).
    const s3 = try pipelinePrompt(a, "Summarize the result.", "ticket-7", "DONE", 2);
    try std.testing.expect(std.mem.indexOf(u8, s3, "Item: ticket-7") != null);
    try std.testing.expect(std.mem.indexOf(u8, s3, "Result from the previous stage:") != null);
    try std.testing.expect(std.mem.indexOf(u8, s3, "DONE") != null);
}

test "execWorkflow rejects a non-object argument tree instead of dereferencing it" {
    var ctx: ToolCtx = undefined;
    ctx.gpa = std.testing.allocator;
    ctx.from_sub = false;

    const out1 = try execWorkflow(ctx, .{ .string = "do it" });
    try std.testing.expect(out1.is_error);
    try std.testing.expect(std.mem.indexOf(u8, out1.text, "object") != null);
    std.testing.allocator.free(out1.text);

    const parsed = try std.json.parseFromSlice(Value, std.testing.allocator, "{\"phases\":\"nope\"}", .{});
    defer parsed.deinit();
    const out2 = try execWorkflow(ctx, parsed.value);
    try std.testing.expect(out2.is_error);
    try std.testing.expect(std.mem.indexOf(u8, out2.text, "array") != null);
    std.testing.allocator.free(out2.text);
}

test "execSubagent rejects a non-object argument tree instead of dereferencing it" {
    var ctx: ToolCtx = undefined;
    ctx.gpa = std.testing.allocator;
    ctx.from_sub = false;

    const parsed = try std.json.parseFromSlice(Value, std.testing.allocator, "[1,2]", .{});
    defer parsed.deinit();
    const out = try subagent.execSubagent(ctx, parsed.value);
    try std.testing.expect(out.is_error);
    std.testing.allocator.free(out.text);
}

test "pipelineIsolationError: stage 0 may isolate, stage 1+ worktree is refused naming the stage, non-worktree is always allowed (#295 guard rail)" {
    // Stage 0 (the first stage) may isolate with its own worktree.
    try std.testing.expect(pipelineIsolationError(0, .worktree) == null);

    // Stage 1+ requesting worktree isolation is refused — a pipeline is a
    // dependent chain over one item, so a later stage in its own worktree
    // cannot see what the prior stage did.
    const msg1 = pipelineIsolationError(1, .worktree) orelse return error.TestExpectedNonNull;
    try std.testing.expect(std.mem.indexOf(u8, msg1, "stage 1") != null);
    try std.testing.expect(std.mem.indexOf(u8, msg1, "pipeline") != null);

    const msg3 = pipelineIsolationError(3, .worktree) orelse return error.TestExpectedNonNull;
    try std.testing.expect(std.mem.indexOf(u8, msg3, "stage 3") != null);

    // Non-worktree isolation (shared_cwd) is allowed at any stage index.
    try std.testing.expect(pipelineIsolationError(0, .shared_cwd) == null);
    try std.testing.expect(pipelineIsolationError(1, .shared_cwd) == null);
    try std.testing.expect(pipelineIsolationError(4, .shared_cwd) == null);
}

test "a 5xx outage is transient, so a fan-out keeps retrying it (U1 regression guard)" {
    // The classifier was advisory prose until failureAllowsRetry made it control
    // flow. Its .model arm matched the bare word "unavailable", so "503 Service
    // Unavailable" - the single most common transient failure - classified as a
    // dead model and stopped the whole phase retrying.
    for ([_][]const u8{ "503 Service Unavailable", "502 Bad Gateway", "500 internal server error", "upstream temporarily unavailable" }) |detail|
        try std.testing.expectEqual(subagent_run.FailKind.transport, subagent_run.classifyFailure(error.Unexpected, detail));
    // A genuinely dead model still classifies as .model and still stops retrying.
    for ([_][]const u8{ "model unavailable", "no such model", "model_not_found" }) |detail|
        try std.testing.expectEqual(subagent_run.FailKind.model, subagent_run.classifyFailure(error.Unexpected, detail));
}

test "the ROOT permission gate refuses a malformed bash call instead of dereferencing it" {
    // gateTool is root-only (subagents take the early return at its top), so
    // the json_args guards added across the workflow/subagent paths in v0.0.223
    // never covered it. A model that sends bash arguments as a bare string or
    // array reaches this gate on the ordinary single-agent path.
    for ([_]std.json.Value{ .{ .string = "rm -rf /" }, .null, .{ .integer = 7 } }) |bad|
        try std.testing.expect(json_args.object(bad) == null);
    // The shape the gate actually wants still resolves, so the guard is not
    // simply refusing everything.
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const good = try std.json.parseFromSliceLeaky(Value, arena_state.allocator(), "{\"command\":\"ls\"}", .{});
    const ok = json_args.object(good) orelse return error.TestExpectedNonNull;
    try std.testing.expectEqualStrings("ls", ok.get("command").?.string);
}