Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
18.8 KB
Raw Blame
const std = @import("std");
const rlm = @import("rlm.zig");
const spec = @import("rlm_spec.zig");
const tools = @import("tools.zig");
const gpa = std.testing.allocator;
const io = std.testing.io;

pub const Fixture = struct { // shared with rlm_literal_tests.zig
    tmp: std.testing.TmpDir,
    client: std.http.Client,
    cwd: []u8,
    pub fn init() !Fixture {
        var tmp = std.testing.tmpDir(.{});
        errdefer tmp.cleanup();
        try tmp.dir.writeFile(io, .{ .sub_path = "target.txt", .data = "old" });
        var buf: [std.fs.max_path_bytes]u8 = undefined;
        const n = try tmp.dir.realPath(io, &buf);
        return .{ .tmp = tmp, .client = .{ .allocator = gpa, .io = io }, .cwd = try gpa.dupe(u8, buf[0..n]) };
    }
    pub fn ctx(self: *Fixture) tools.ToolCtx {
        return .{ .gpa = gpa, .io = io, .client = &self.client, .provider = undefined, .registry = null, .from_sub = false, .approvals = null, .tracer = null, .agent_cwd = self.cwd };
    }
    pub fn deinit(self: *Fixture) void {
        @import("jobs.zig").jobsReap(gpa, io);
        @import("job_notify.zig").resetForTest(io);
        rlm.resetLive(gpa, io);
        self.client.deinit();
        gpa.free(self.cwd);
        self.tmp.cleanup();
    }
    pub fn run(self: *Fixture, code: []const u8) !tools.ToolOutput {
        return rlm.runScript(self.ctx(), code);
    }
};

test "rlm order applies native edit before dependent verification" {
    var f = try Fixture.init();
    defer f.deinit();
    const code = if (@import("builtin").os.tag == .windows)
        "e = edit_file(path=\"target.txt\", old_string=\"old\", new_string=\"new\")\nv = bash(\"type target.txt\")\nprint(v)"
    else
        "e = edit_file(path=\"target.txt\", old_string=\"old\", new_string=\"new\")\nv = bash(\"test $(cat target.txt) = new && printf VERIFIED\")\nprint(v)";
    const out = try f.run(code);
    defer gpa.free(out.text);
    try std.testing.expect(!out.is_error and !out.pending);
    try std.testing.expectEqualStrings(if (@import("builtin").os.tag == .windows) "new" else "VERIFIED", std.mem.trim(u8, out.text, "\r\n"));
}

test "rlm order failed edit prevents following verification side effects" {
    var f = try Fixture.init();
    defer f.deinit();
    const out = try f.run("e = edit_file(path=\"target.txt\", old_string=\"absent\", new_string=\"new\")\nv = bash(\"printf ran > marker\")\nprint(v)");
    defer gpa.free(out.text);
    try std.testing.expect(out.is_error);
    try std.testing.expectError(error.FileNotFound, f.tmp.dir.openFile(io, "marker", .{}));
}

test "rlm order repeated mutations execute twice and post mutation read is fresh" {
    var f = try Fixture.init();
    defer f.deinit();
    const out = try f.run("before = read_file(\"target.txt\")\na = bash(\"printf x >> target.txt\")\nb = bash(\"printf x >> target.txt\")\nafter = read_file(\"target.txt\")\nprint(before, after)");
    defer gpa.free(out.text);
    try std.testing.expect(!out.is_error);
    try std.testing.expectEqualStrings("old\noldxx", out.text);
}

test "rlm order nested failed host prevents later mutation" {
    var f = try Fixture.init();
    defer f.deinit();
    const code = if (@import("builtin").os.tag == .windows)
        "print(bash(\"echo FAILED & exit /b 7\"), write_file(path=\"marker\", content=\"ran\"))"
    else
        "print(bash(\"printf FAILED; exit 7\"), write_file(path=\"marker\", content=\"ran\"))";
    const out = try f.run(code);
    defer gpa.free(out.text);
    try std.testing.expect(out.is_error);
    try std.testing.expect(std.mem.indexOf(u8, out.text, "FAILED") != null);
    try std.testing.expectError(error.FileNotFound, f.tmp.dir.openFile(io, "marker", .{}));
}

test "rlm order pending direct and nested hosts stop and preserve status" {
    var f = try Fixture.init();
    defer f.deinit();
    for ([_][]const u8{
        "v = bash(command=\"sleep 0.1\", run_in_background=true)\nx = write_file(path=\"marker\", content=\"ran\")\nprint(v)",
        "print(bash(command=\"sleep 0.1\", run_in_background=true), write_file(path=\"marker\", content=\"ran\"))",
    }) |code| {
        const out = try f.run(code);
        defer gpa.free(out.text);
        try std.testing.expect(out.pending and !out.is_error);
        try std.testing.expectError(error.FileNotFound, f.tmp.dir.openFile(io, "marker", .{}));
    }
}

const ReadHost = struct {
    var started: std.atomic.Value(usize) = .init(0);
    var overlap: std.atomic.Value(bool) = .init(false);
    fn run(ctx: tools.ToolCtx, _: @import("spec_ptc.zig").Call) tools.ToolOutput {
        _ = started.fetchAdd(1, .seq_cst);
        for (0..100) |_| {
            if (started.load(.seq_cst) >= 2) {
                overlap.store(true, .seq_cst);
                break;
            }
            ctx.io.sleep(.fromMilliseconds(1), .awake) catch {};
        }
        return .{ .text = ctx.gpa.dupe(u8, "read") catch unreachable };
    }
};

test "rlm order streamed prefix reads overlap and barrier survives partial chunks" {
    var f = try Fixture.init();
    defer f.deinit();
    const saved = spec.run_host;
    defer spec.run_host = saved;
    spec.run_host = ReadHost.run;
    ReadHost.started.store(0, .seq_cst);
    ReadHost.overlap.store(false, .seq_cst);
    spec.feedLive(f.ctx(), "a = read_file(\"a\")\nb = read_file(\"b\")\ne = edit_file(path=\"target.txt\", ");
    spec.feedLive(f.ctx(), "old_string=\"old\", new_string=\"new\")\nc = read_file(\"target.txt\")\n");
    var arena = std.heap.ArenaAllocator.init(gpa);
    defer arena.deinit();
    var claimed = std.StringHashMap(tools.ToolOutput).init(gpa);
    defer {
        var it = claimed.valueIterator();
        while (it.next()) |out| gpa.free(out.text);
        claimed.deinit();
    }
    spec.takeLive(f.ctx(), &claimed, arena.allocator());
    try std.testing.expectEqual(@as(usize, 2), ReadHost.started.load(.seq_cst));
    try std.testing.expect(ReadHost.overlap.load(.seq_cst));
}

test "rlm order cancelled shell preserves cancellation and prevents tail mutation" {
    var f = try Fixture.init();
    defer f.deinit();
    const cancel = @import("cancel_source.zig");
    defer cancel.clear();
    cancel.cancel(.json_cancel);
    const out = try f.run("print(bash(\"sleep 0.1\"), write_file(path=\"marker\", content=\"ran\"))");
    defer gpa.free(out.text);
    try std.testing.expect(out.cancelled and out.is_error);
    try std.testing.expectError(error.FileNotFound, f.tmp.dir.openFile(io, "marker", .{}));
}

const StoppedHost = struct {
    var count: usize = 0;
    var pending: bool = false;
    fn run(ctx: tools.ToolCtx, _: @import("spec_ptc.zig").Call) tools.ToolOutput {
        count += 1;
        return .{ .text = ctx.gpa.dupe(u8, "stopped") catch unreachable, .pending = pending, .cancelled = !pending };
    }
};

test "rlm order each preserves pending and cancellation without binding partial results" {
    var f = try Fixture.init();
    defer f.deinit();
    var arena = std.heap.ArenaAllocator.init(gpa);
    defer arena.deinit();
    for ([_]bool{ true, false }) |pending| {
        StoppedHost.count = 0;
        StoppedHost.pending = pending;
        const binds = [_]spec.Binding{.{ .name = "items", .text = "[{\"id\":1},{\"id\":2}]" }};
        var outputs: std.ArrayList(spec.Binding) = .empty;
        const hit = try @import("rlm_mcp.zig").evalEach(f.ctx(), arena.allocator(), "out = each(items, \"read_file\", \"id\")", &binds, &outputs, StoppedHost.run);
        try std.testing.expect(hit == .fail);
        defer gpa.free(hit.fail.text);
        try std.testing.expectEqual(pending, hit.fail.pending);
        try std.testing.expectEqual(!pending, hit.fail.cancelled);
        try std.testing.expectEqual(@as(usize, 1), StoppedHost.count);
        try std.testing.expectEqual(@as(usize, 0), outputs.items.len);
    }
}

test "rlm order unresolved print argument prevents speculative tail reads" {
    var arena = std.heap.ArenaAllocator.init(gpa);
    defer arena.deinit();
    const order = @import("rlm_order.zig");
    try std.testing.expect(try order.leading(arena.allocator(), "print(bash(command=cmd), read_file(\"x\"))") == null);
    try std.testing.expect(try order.leading(arena.allocator(), "print(read_file(\"a\"), read_file(\"b\"))") != null);
}

test "rlm unsupported indexed print refuses before executing its nested command or tail" {
    var f = try Fixture.init();
    defer f.deinit();
    const out = try f.run("print(bash(\"printf ran > nested-marker\")[\"stdout\"])\nx = write_file(path=\"marker\", content=\"ran\")");
    defer gpa.free(out.text);
    try std.testing.expect(out.is_error);
    try std.testing.expect(std.mem.indexOf(u8, out.text, "host results are text") != null);
    try std.testing.expectError(error.FileNotFound, f.tmp.dir.openFile(io, "nested-marker", .{}));
    try std.testing.expectError(error.FileNotFound, f.tmp.dir.openFile(io, "marker", .{}));
}

test "rlm unsupported bound field access reports text contract without rerunning host" {
    var f = try Fixture.init();
    defer f.deinit();
    for ([_][]const u8{ "r[\"stdout\"]", "r.get(\"exit_code\")" }) |expr| {
        const code = try std.fmt.allocPrint(gpa, "r = bash(\"printf x >> target.txt\")\nprint({s})", .{expr});
        defer gpa.free(code);
        const out = try f.run(code);
        defer gpa.free(out.text);
        try std.testing.expect(out.is_error);
        try std.testing.expect(std.mem.indexOf(u8, out.text, "assign then print(name)") != null);
    }
    const out = try f.run("print(read_file(\"target.txt\"))");
    defer gpa.free(out.text);
    try std.testing.expectEqualStrings("oldxx", out.text);
}

test "rlm print keeps literals known and missing binds and supported reducers" {
    var f = try Fixture.init();
    defer f.deinit();
    try f.tmp.dir.writeFile(io, .{ .sub_path = "items.json", .data = "[{\"id\":1},{\"id\":2}]" });
    const out = try f.run("items = read_file(\"items.json\")\ncount = len(items)\nids = project(items, \"id\")\nprint(\"literal [data]\", 'quoted', count, ids, len(items), project(items, \"id\"), missing_bind)");
    defer gpa.free(out.text);
    try std.testing.expect(!out.is_error);
    try std.testing.expectEqualStrings("literal [data]\nquoted\n2\n[1,2]\n2\n[1,2]\nmissing_bind", out.text);
}

const Gate = struct {
    seen: usize = 0,
    call_id: []const u8 = "",
    fn check(context: *anyopaque, call: tools.ToolCall) ?tools.ToolOutput {
        const self: *Gate = @ptrCast(@alignCast(context));
        self.seen += 1;
        self.call_id = call.id;
        if (!@import("shell_tool.zig").runsCommand(call.name)) return null;
        return .{ .text = gpa.dupe(u8, "gate denied") catch &.{}, .is_error = true };
    }
};

test "rlm host calls pass the agent gate and a denial stops the script (#1292)" {
    var f = try Fixture.init();
    defer f.deinit();
    var gate: Gate = .{};
    var ctx = f.ctx();
    ctx.host_gate = .{ .context = &gate, .call_id = "call-rlm", .check = Gate.check };
    const out = try rlm.runScript(ctx, "r = read_file(\"target.txt\")\nv = bash(\"printf ran > marker\")\nw = write_file(path=\"after\", content=\"x\")\nprint(v)");
    defer gpa.free(out.text);
    try std.testing.expect(out.is_error);
    try std.testing.expectEqualStrings("gate denied", out.text);
    try std.testing.expectEqual(@as(usize, 2), gate.seen); // read_file, then the denied bash
    try std.testing.expectEqualStrings("call-rlm", gate.call_id);
    try std.testing.expectError(error.FileNotFound, f.tmp.dir.openFile(io, "marker", .{}));
    try std.testing.expectError(error.FileNotFound, f.tmp.dir.openFile(io, "after", .{}));
}

const Agent = @import("agent.zig").Agent;
const Approvals = @import("approvals.zig").Approvals;
const permission = @import("engine_permission.zig");

const Deny = struct {
    calls: usize = 0,
    fn ask(ptr: *anyopaque, _: std.Io, req: permission.Request) permission.Decision {
        const self: *@This() = @ptrCast(@alignCast(ptr));
        self.calls += 1;
        std.debug.assert(std.mem.eql(u8, req.call_id, "call-rlm"));
        return .deny;
    }
};

fn rlmCall(arena: std.mem.Allocator, code: []const u8) !tools.ToolCall {
    var args: std.json.ObjectMap = .empty;
    try args.put(arena, "code", .{ .string = code });
    return .{ .id = "call-rlm", .name = rlm.tool_name, .input = .{ .object = args } };
}

test "rlm bash and write_file at the root ask the permission handler like catalog calls (#1292)" {
    var f = try Fixture.init();
    defer f.deinit();
    var arena = std.heap.ArenaAllocator.init(gpa);
    defer arena.deinit();
    for ([_][]const u8{
        "print(bash(\"printf ran > marker\"))",
        "w = write_file(path=\"marker\", content=\"ran\")\nprint(w)",
    }) |code| {
        var deny: Deny = .{};
        var approvals: Approvals = .{};
        var agent: Agent = .{ .gpa = gpa, .arena = arena.allocator(), .io = io, .client = &f.client, .provider = .{ .id = "fixture", .kind = .openai, .auth = .bearer, .url = "", .api_key = "", .model = "fixture", .context = 1000 }, .messages = .init(arena.allocator()), .sub = false, .label = "root", .out = null, .approvals = &approvals, .permission = .{ .ctx = &deny, .request = Deny.ask }, .agent_cwd = f.cwd };
        defer agent.tools_used.deinit(gpa);
        const results = try agent.runTools(&.{try rlmCall(arena.allocator(), code)});
        try std.testing.expectEqual(@as(usize, 1), deny.calls);
        try std.testing.expect(results[0].is_error);
        try std.testing.expect(std.mem.indexOf(u8, results[0].text, "declined") != null);
        try std.testing.expectError(error.FileNotFound, f.tmp.dir.openFile(io, "marker", .{}));
    }
}

test "rlm loaded peer_message reports structured-only support and stops the script (#1413)" {
    var f = try Fixture.init();
    defer f.deinit();
    var arena = std.heap.ArenaAllocator.init(gpa);
    defer arena.deinit();
    const fold = @import("native_fold.zig");
    const saved_enabled = fold.enabled;
    defer {
        fold.enabled = saved_enabled;
        fold.resetSession();
    }
    fold.enabled = true;
    fold.resetSession();
    var agent: Agent = .{ .gpa = gpa, .arena = arena.allocator(), .io = io, .client = &f.client, .provider = .{ .id = "fixture", .kind = .openai, .auth = .bearer, .url = "", .api_key = "", .model = "fixture", .context = 1000 }, .messages = .init(arena.allocator()), .sub = false, .label = "root", .out = null, .agent_cwd = f.cwd };
    defer agent.tools_used.deinit(gpa);
    const load = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), "{\"tools\":[\"peer_message\"]}", .{});
    const loaded = try agent.runTools(&.{.{ .id = "load", .name = "load_tool_schemas", .input = load }});
    try std.testing.expect(!loaded[0].is_error);
    try std.testing.expect(fold.isLoaded("peer_message"));
    const results = try agent.runTools(&.{try rlmCall(arena.allocator(), "p = peer_message(action=\"inbox\")\nw = write_file(path=\"marker\", content=\"ran\")\nprint(p)")});
    try std.testing.expect(results[0].is_error);
    try std.testing.expectEqualStrings("rlm: peer_message is agent-managed and is not supported inside rlm; call peer_message directly as a structured tool.", results[0].text);
    try std.testing.expectError(error.FileNotFound, f.tmp.dir.openFile(io, "marker", .{}));
    // Loading still enables the normal inline handler, not a host function.
    const input = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), "{\"action\":\"invalid-fixture-action\"}", .{});
    const direct = try agent.runTools(&.{.{ .id = "direct", .name = "peer_message", .input = input }});
    try std.testing.expect(direct[0].is_error);
    try std.testing.expect(std.mem.startsWith(u8, direct[0].text, "peer_message action must be "));
}

test "rlm agent-managed hosts refuse direct nested and each calls without executing the tail (#1413)" {
    var f = try Fixture.init();
    defer f.deinit();
    for (@import("schema.zig").meta_names) |name| {
        const code = try std.fmt.allocPrint(gpa, "x = {s}()\nw = write_file(path=\"marker\", content=\"ran\")", .{name});
        defer gpa.free(code);
        const out = try f.run(code);
        defer gpa.free(out.text);
        try std.testing.expect(out.is_error);
        try std.testing.expect(std.mem.indexOf(u8, out.text, "is not supported inside rlm") != null);
        try std.testing.expect(std.mem.indexOf(u8, out.text, name) != null);
        try std.testing.expectError(error.FileNotFound, f.tmp.dir.openFile(io, "marker", .{}));
    }
    for ([_][]const u8{
        "print(peer_message(action=\"inbox\"), write_file(path=\"marker\", content=\"ran\"))",
        "x = each([\"inbox\", \"list\"], \"peer_message\", \"action\")\nw = write_file(path=\"marker\", content=\"ran\")",
    }) |code| {
        const out = try f.run(code);
        defer gpa.free(out.text);
        try std.testing.expect(out.is_error);
        try std.testing.expect(std.mem.indexOf(u8, out.text, "call peer_message directly as a structured tool") != null);
        try std.testing.expectError(error.FileNotFound, f.tmp.dir.openFile(io, "marker", .{}));
    }
}

test "rlm destructive git in a yolo subagent hits the subagent block (#1292)" {
    var f = try Fixture.init();
    defer f.deinit();
    var arena = std.heap.ArenaAllocator.init(gpa);
    defer arena.deinit();
    var approvals: Approvals = .{ .yolo = true };
    var agent: Agent = .{ .gpa = gpa, .arena = arena.allocator(), .io = io, .client = &f.client, .provider = .{ .id = "fixture", .kind = .openai, .auth = .bearer, .url = "", .api_key = "", .model = "fixture", .context = 1000 }, .messages = .init(arena.allocator()), .sub = true, .label = "sub", .out = null, .approvals = &approvals, .agent_cwd = f.cwd };
    defer agent.tools_used.deinit(gpa);
    // A missing branch: harmless even if the gate ever regresses.
    const code = "v = bash(\"git branch -D graff-1294-no-such-branch\")\nw = write_file(path=\"marker\", content=\"ran\")\nprint(v)";
    const results = try agent.runTools(&.{try rlmCall(arena.allocator(), code)});
    try std.testing.expect(results[0].is_error);
    try std.testing.expect(std.mem.indexOf(u8, results[0].text, "destructive git is blocked for subagents") != null);
    try std.testing.expectError(error.FileNotFound, f.tmp.dir.openFile(io, "marker", .{}));
}

test "rlm projection assignments take comments, keywords and sparse fields, then print (#1536)" {
    var f = try Fixture.init();
    defer f.deinit();
    try f.tmp.dir.writeFile(io, .{ .sub_path = "items.json", .data = "[{\"id\":1,\"name\":\"a\"},{\"id\":2}]" });
    const out = try f.run("items = read_file(\"items.json\")\nids = project(items, \"id\")  # plain ids\nnames = project(x=items, field=\"name\")\nn = len(items) # count\nprint(ids, names, n)");
    defer gpa.free(out.text);
    try std.testing.expect(!out.is_error);
    try std.testing.expectEqualStrings("[1,2]\n[\"a\",null]\n2", out.text);

    const bad = try f.run("t = read_file(\"target.txt\")\nv = project(t, \"id\")\nprint(v)");
    defer gpa.free(bad.text);
    try std.testing.expect(bad.is_error);
    try std.testing.expect(std.mem.indexOf(u8, bad.text, "unsupported statement") == null);
    try std.testing.expect(std.mem.indexOf(u8, bad.text, "not a JSON array") != null);
}