graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
//! Tests split out of rlm.zig so the REPL stays under the 600-line cap.
const std = @import("std");
const Io = std.Io;
const rlm = @import("rlm.zig");
const rlm_spec = @import("rlm_spec.zig");
const no_local_tools = @import("no_local_tools.zig");
const tools = @import("tools.zig");
const ToolCtx = tools.ToolCtx;
const ToolOutput = tools.ToolOutput;
fn testCtx(gpa: std.mem.Allocator, io: Io, client: *std.http.Client) ToolCtx {
return .{
.gpa = gpa,
.io = io,
.client = client,
.provider = undefined,
.registry = null,
.from_sub = false,
.approvals = null,
.tracer = null,
};
}
test "maybeAppend is a no-op when --old, and refuses to double-append" {
const gpa = std.testing.allocator;
var arena_state = std.heap.ArenaAllocator.init(gpa);
defer arena_state.deinit();
const arena = arena_state.allocator();
const Spec = struct { name: []const u8, desc: []const u8 = "", schema: []const u8 = "{}" };
const base = [_]Spec{.{ .name = "read_file" }};
const saved = rlm.available;
const saved_local = no_local_tools.enabled;
defer {
rlm.available = saved;
no_local_tools.enabled = saved_local;
}
rlm.available = false;
no_local_tools.enabled = false;
try std.testing.expectEqual(@as(usize, 1), (try rlm.maybeAppend(Spec, arena, &base)).len);
rlm.available = true;
const one = try rlm.maybeAppend(Spec, arena, &base);
try std.testing.expectEqual(@as(usize, 2), one.len);
try std.testing.expectEqualStrings(rlm.tool_name, one[1].name);
const two = try rlm.maybeAppend(Spec, arena, one);
try std.testing.expectEqual(@as(usize, 2), two.len);
rlm.available = true;
no_local_tools.enabled = true;
try std.testing.expectEqual(@as(usize, 1), (try rlm.maybeAppend(Spec, arena, &base)).len);
}
test "runScript speculates three sleeps so wall time is ~one sleep, not three" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var dummy_client: std.http.Client = .{ .allocator = gpa, .io = io };
defer dummy_client.deinit();
const saved = rlm.available;
defer {
rlm.available = saved;
rlm.resetLive(gpa, io);
}
rlm.available = true;
const ctx = testCtx(gpa, io, &dummy_client);
const t0: Io.Timestamp = .now(io, .awake);
const out = try rlm.runScript(ctx, "a = sleep_ms(40)\nb = sleep_ms(41)\nc = sleep_ms(42)\nprint(a)");
defer gpa.free(out.text);
const dt = t0.untilNow(io, .awake).toMilliseconds();
try std.testing.expect(!out.is_error);
try std.testing.expectEqualStrings("slept 40ms", out.text);
try std.testing.expect(dt < 100);
}
test "runScript reads a fixture via speculated read_file and prints it" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try tmp.dir.writeFile(io, .{ .sub_path = "note.txt", .data = "hello-rlm\n" });
var path_buf: [std.fs.max_path_bytes]u8 = undefined;
const n = try tmp.dir.realPath(io, &path_buf);
const dir = path_buf[0..n];
var dummy_client: std.http.Client = .{ .allocator = gpa, .io = io };
defer dummy_client.deinit();
const saved = rlm.available;
defer {
rlm.available = saved;
rlm.resetLive(gpa, io);
}
rlm.available = true;
var ctx = testCtx(gpa, io, &dummy_client);
ctx.agent_cwd = dir;
const out = try rlm.runScript(ctx, "x = read_file(\"note.txt\")\nprint(x)");
defer gpa.free(out.text);
try std.testing.expect(!out.is_error);
try std.testing.expect(std.mem.indexOf(u8, out.text, "hello-rlm") != null);
const nested = try rlm.runScript(ctx, "print(read_file(\"note.txt\"))");
defer gpa.free(nested.text);
try std.testing.expect(!nested.is_error);
try std.testing.expect(std.mem.indexOf(u8, nested.text, "hello-rlm") != null);
}
test "runScript saves a bound result with write_file(path, name) for the shell" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try tmp.dir.writeFile(io, .{ .sub_path = "note.txt", .data = "hello-rlm\n" });
var path_buf: [std.fs.max_path_bytes]u8 = undefined;
const n = try tmp.dir.realPath(io, &path_buf);
var dummy_client: std.http.Client = .{ .allocator = gpa, .io = io };
defer dummy_client.deinit();
const saved = rlm.available;
defer {
rlm.available = saved;
rlm.resetLive(gpa, io);
}
rlm.available = true;
var ctx = testCtx(gpa, io, &dummy_client);
ctx.agent_cwd = path_buf[0..n];
const out = try rlm.runScript(ctx, "x = read_file(\"note.txt\")\nwrite_file(\"copy.txt\", x)");
defer gpa.free(out.text);
try std.testing.expect(!out.is_error);
var buf: [64]u8 = undefined;
const copied = try tmp.dir.readFile(io, "copy.txt", &buf);
try std.testing.expect(std.mem.indexOf(u8, copied, "hello-rlm") != null);
}
test "runScript splits a semicolon one-liner and prints both binds" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var dummy_client: std.http.Client = .{ .allocator = gpa, .io = io };
defer dummy_client.deinit();
const saved = rlm.available;
defer {
rlm.available = saved;
rlm.resetLive(gpa, io);
}
rlm.available = true;
const ctx = testCtx(gpa, io, &dummy_client);
const t0: Io.Timestamp = .now(io, .awake);
const out = try rlm.runScript(ctx, "a = sleep_ms(40); b = sleep_ms(41); print(a, b)");
defer gpa.free(out.text);
const dt = t0.untilNow(io, .awake).toMilliseconds();
try std.testing.expect(!out.is_error);
try std.testing.expectEqualStrings("slept 40ms\nslept 41ms", out.text);
try std.testing.expect(dt < 90);
}
test "feedLive launches sleeps before runScript, so claim is a hit not a re-run" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var dummy_client: std.http.Client = .{ .allocator = gpa, .io = io };
defer dummy_client.deinit();
const saved = rlm.available;
defer {
rlm.available = saved;
rlm.resetLive(gpa, io);
}
rlm.available = true;
const ctx = testCtx(gpa, io, &dummy_client);
const t0: Io.Timestamp = .now(io, .awake);
rlm.feedLive(ctx, "a = sleep_ms(40)\n");
rlm.feedLive(ctx, "b = sleep_ms(41)\n");
rlm.feedLive(ctx, "c = sleep_ms(42)\n");
const out = try rlm.runScript(ctx, "a = sleep_ms(40)\nb = sleep_ms(41)\nc = sleep_ms(42)\nprint(a)");
defer gpa.free(out.text);
const dt = t0.untilNow(io, .awake).toMilliseconds();
try std.testing.expect(!out.is_error);
try std.testing.expectEqualStrings("slept 40ms", out.text);
try std.testing.expect(dt < 100);
}
test "feedLive splits a semicolon line before runScript claims" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var dummy_client: std.http.Client = .{ .allocator = gpa, .io = io };
defer dummy_client.deinit();
const saved = rlm.available;
defer {
rlm.available = saved;
rlm.resetLive(gpa, io);
}
rlm.available = true;
const ctx = testCtx(gpa, io, &dummy_client);
const t0: Io.Timestamp = .now(io, .awake);
rlm.feedLive(ctx, "a = sleep_ms(40); b = sleep_ms(41)\n");
const out = try rlm.runScript(ctx, "a = sleep_ms(40); b = sleep_ms(41); print(a, b)");
defer gpa.free(out.text);
const dt = t0.untilNow(io, .awake).toMilliseconds();
try std.testing.expect(!out.is_error);
try std.testing.expectEqualStrings("slept 40ms\nslept 41ms", out.text);
try std.testing.expect(dt < 90);
}
test "runScript reuses last-script binds; resetLive drops them" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var dummy_client: std.http.Client = .{ .allocator = gpa, .io = io };
defer dummy_client.deinit();
const saved = rlm.available;
defer {
rlm.available = saved;
rlm.resetLive(gpa, io);
}
rlm.available = true;
const ctx = testCtx(gpa, io, &dummy_client);
const first = try rlm.runScript(ctx, "prev = sleep_ms(1)");
defer gpa.free(first.text);
try std.testing.expect(!first.is_error);
const reuse = try rlm.runScript(ctx, "print(prev)");
defer gpa.free(reuse.text);
try std.testing.expect(!reuse.is_error);
try std.testing.expectEqualStrings("slept 1ms", reuse.text);
const overwrite = try rlm.runScript(ctx, "prev = sleep_ms(2); print(prev)");
defer gpa.free(overwrite.text);
try std.testing.expectEqualStrings("slept 2ms", overwrite.text);
const kept = try rlm.runScript(ctx, "print(prev)");
defer gpa.free(kept.text);
try std.testing.expectEqualStrings("slept 2ms", kept.text);
rlm.resetLive(gpa, io);
const gone = try rlm.runScript(ctx, "print(prev)");
defer gpa.free(gone.text);
try std.testing.expectEqualStrings("prev", gone.text);
}
test "runScript last assignment wins when a name is rebound in one script" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var dummy_client: std.http.Client = .{ .allocator = gpa, .io = io };
defer dummy_client.deinit();
const saved = rlm.available;
defer {
rlm.available = saved;
rlm.resetLive(gpa, io);
}
rlm.available = true;
const ctx = testCtx(gpa, io, &dummy_client);
const out = try rlm.runScript(ctx, "a = sleep_ms(1); a = sleep_ms(2); print(a)");
defer gpa.free(out.text);
try std.testing.expect(!out.is_error);
try std.testing.expectEqualStrings("slept 2ms", out.text);
}
fn catalogHas(specs: anytype, name: []const u8) bool {
for (specs) |s| if (std.mem.eql(u8, s.name, name)) return true;
return false;
}
test "maybeAppend keeps subagent and workflow; rlm is never the only catalog tool" {
const gpa = std.testing.allocator;
var arena_state = std.heap.ArenaAllocator.init(gpa);
defer arena_state.deinit();
const arena = arena_state.allocator();
const Spec = struct { name: []const u8, desc: []const u8 = "", schema: []const u8 = "{}" };
const base = [_]Spec{
.{ .name = "read_file" },
.{ .name = "subagent" },
.{ .name = "workflow" },
};
const saved = rlm.available;
const saved_local = no_local_tools.enabled;
defer {
rlm.available = saved;
no_local_tools.enabled = saved_local;
rlm.sync();
}
rlm.available = true;
no_local_tools.enabled = false;
const on = try rlm.maybeAppend(Spec, arena, &base);
try std.testing.expectEqual(@as(usize, 4), on.len);
try std.testing.expect(catalogHas(on, "subagent"));
try std.testing.expect(catalogHas(on, "workflow"));
try std.testing.expect(catalogHas(on, rlm.tool_name));
try std.testing.expect(on.len > 1);
rlm.available = false;
const off = try rlm.maybeAppend(Spec, arena, &base);
try std.testing.expectEqual(@as(usize, 3), off.len);
try std.testing.expect(catalogHas(off, "subagent"));
try std.testing.expect(!catalogHas(off, rlm.tool_name));
}
test "effectiveRootSpecs: default rlm keeps subagent; --old and --lean do not drop it" {
const schema = @import("schema.zig");
const root = @import("main.zig");
const learn_store = @import("learn_store.zig");
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
const saved = rlm.available;
const saved_lean = no_local_tools.lean;
const saved_local = no_local_tools.enabled;
const saved_clock = root.g_clock_sleep;
const saved_learn = learn_store.active_agent_loaded;
defer {
rlm.available = saved;
no_local_tools.lean = saved_lean;
no_local_tools.enabled = saved_local;
root.g_clock_sleep = saved_clock;
learn_store.active_agent_loaded = saved_learn;
rlm.sync();
}
no_local_tools.enabled = false;
root.g_clock_sleep = false;
learn_store.active_agent_loaded = true;
rlm.available = true;
no_local_tools.lean = false;
rlm.sync();
const def = try schema.effectiveRootSpecs(arena);
try std.testing.expect(catalogHas(def, "subagent"));
try std.testing.expect(catalogHas(def, "workflow"));
try std.testing.expect(catalogHas(def, rlm.tool_name));
try std.testing.expect(def.len > 2);
rlm.available = false;
rlm.sync();
const old = try schema.effectiveRootSpecs(arena);
try std.testing.expect(catalogHas(old, "subagent"));
try std.testing.expect(catalogHas(old, "workflow"));
try std.testing.expect(!catalogHas(old, rlm.tool_name));
rlm.available = true;
no_local_tools.lean = true;
rlm.sync();
const lean_on = try schema.effectiveRootSpecs(arena);
try std.testing.expect(catalogHas(lean_on, "subagent"));
try std.testing.expect(catalogHas(lean_on, rlm.tool_name));
try std.testing.expect(!catalogHas(lean_on, "workflow"));
try std.testing.expect(catalogHas(lean_on, "shell"));
try std.testing.expect(no_local_tools.leanKeeps("subagent"));
try std.testing.expect(!no_local_tools.leanKeeps(rlm.tool_name));
}
test "runScript empty subagent() reaches execSubagent; persist binds do not break that" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var dummy_client: std.http.Client = .{ .allocator = gpa, .io = io };
defer dummy_client.deinit();
const saved = rlm.available;
defer {
rlm.available = saved;
rlm.resetLive(gpa, io);
}
rlm.available = true;
const ctx = testCtx(gpa, io, &dummy_client);
const first = try rlm.runScript(ctx, "prev = sleep_ms(1)");
defer gpa.free(first.text);
try std.testing.expect(!first.is_error);
const miss = try rlm.runScript(ctx, "h = subagent()");
defer gpa.free(miss.text);
try std.testing.expect(miss.is_error);
try std.testing.expect(std.mem.indexOf(u8, miss.text, "missing required") != null);
try std.testing.expect(std.mem.indexOf(u8, miss.text, "prompt") != null);
const reuse = try rlm.runScript(ctx, "print(prev)");
defer gpa.free(reuse.text);
try std.testing.expect(!reuse.is_error);
try std.testing.expectEqualStrings("slept 1ms", reuse.text);
}
test "feedLive leaves side-effecting subagents for lexical execution" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var dummy_client: std.http.Client = .{ .allocator = gpa, .io = io };
defer dummy_client.deinit();
const saved = rlm.available;
defer {
rlm.available = saved;
rlm.resetLive(gpa, io);
}
rlm.available = true;
const ctx = testCtx(gpa, io, &dummy_client);
rlm.feedLive(ctx, "a = subagent()\n");
rlm.feedLive(ctx, "b = subagent(\"\")\n");
var claimed = std.StringHashMap(ToolOutput).init(gpa);
defer {
var it = claimed.iterator();
while (it.next()) |e| gpa.free(e.value_ptr.text);
claimed.deinit();
}
var arena_state = std.heap.ArenaAllocator.init(gpa);
defer arena_state.deinit();
rlm_spec.takeLive(ctx, &claimed, arena_state.allocator());
try std.testing.expectEqual(@as(usize, 0), claimed.count());
}
test "rlm prompt: subagent() is advertised as sidecar-only, not a critical-path handoff" {
try std.testing.expect(std.mem.indexOf(u8, rlm.tool_desc, "sidecar-only") != null);
try std.testing.expect(std.mem.indexOf(u8, rlm.tool_desc, "critical-path") != null);
try std.testing.expect(std.mem.indexOf(u8, rlm_spec.system_note, "sidecar-only") != null);
try std.testing.expect(std.mem.indexOf(u8, rlm_spec.system_note, "critical-path") != null);
try std.testing.expect(rlm.tool_desc.len < 600);
try std.testing.expect(std.mem.indexOf(u8, rlm.lean_tool_desc, "load_tool_schemas") != null);
try std.testing.expect(std.mem.indexOf(u8, rlm.lean_tool_desc, "MCP") != null);
}
test "runScript refuses an unloaded MCP name and does not treat it as a host tool" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var dummy_client: std.http.Client = .{ .allocator = gpa, .io = io };
defer dummy_client.deinit();
const saved = rlm.available;
const saved_mcp = @import("rlm_mcp.zig").host_enabled;
defer {
rlm.available = saved;
@import("rlm_mcp.zig").host_enabled = saved_mcp;
rlm.resetLive(gpa, io);
}
rlm.available = true;
@import("rlm_mcp.zig").host_enabled = true;
const ctx = testCtx(gpa, io, &dummy_client);
const out = try rlm.runScript(ctx, "x = mcp__linear__list_issues()\nprint(x)");
defer gpa.free(out.text);
try std.testing.expect(out.is_error);
try std.testing.expect(std.mem.indexOf(u8, out.text, "MCP not available") != null or std.mem.indexOf(u8, out.text, "load_tool_schemas") != null);
}
test "#1016: literal read_file stays native when an MCP short name collides" {
const gpa = std.testing.allocator;
const io = std.testing.io;
const mcp = @import("mcp.zig");
const mcp_schema_gate = @import("mcp_schema_gate.zig");
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try tmp.dir.writeFile(io, .{ .sub_path = "note.txt", .data = "hello-native\n" });
var path_buf: [std.fs.max_path_bytes]u8 = undefined;
const n = try tmp.dir.realPath(io, &path_buf);
const dir = path_buf[0..n];
var arena_state = std.heap.ArenaAllocator.init(gpa);
defer arena_state.deinit();
const arena = arena_state.allocator();
mcp_schema_gate.reset();
defer mcp_schema_gate.reset();
mcp_schema_gate.g_policy = .{ .enabled = true, .budget = 0, .eager = &.{} };
var inner: std.json.ObjectMap = .empty;
try inner.put(arena, "type", .{ .string = "string" });
var props: std.json.ObjectMap = .empty;
try props.put(arena, "path", .{ .object = inner });
var schema: std.json.ObjectMap = .empty;
try schema.put(arena, "type", .{ .string = "object" });
try schema.put(arena, "properties", .{ .object = props });
var colliding = [_]mcp.Tool{.{
.server_index = 0,
.original_name = "read_file",
.qualified_name = "mcp__wiki__read_file",
.description = "wiki",
.input_schema = .{ .object = schema },
}};
var registry = mcp.Registry.empty(gpa, io);
defer registry.deinit();
registry.tools = colliding[0..];
var dummy_client: std.http.Client = .{ .allocator = gpa, .io = io };
defer dummy_client.deinit();
const saved = rlm.available;
defer {
rlm.available = saved;
rlm.resetLive(gpa, io);
}
rlm.available = true;
var ctx = testCtx(gpa, io, &dummy_client);
ctx.registry = ®istry;
ctx.agent_cwd = dir;
const out = try rlm.runScript(ctx, "print(read_file(\"note.txt\"))");
defer gpa.free(out.text);
try std.testing.expect(!out.is_error);
try std.testing.expect(std.mem.indexOf(u8, out.text, "hello-native") != null);
try std.testing.expect(std.mem.indexOf(u8, out.text, "MCP tool schema is not loaded") == null);
}
test "a printed slim view names itself and the full size (ADR 0238)" {
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const full: [56005]u8 = @splat('x');
const note = @import("rlm.zig").slimNote(arena_state.allocator(), "[{\"id\":\"ISS-1\"}]", &full);
try std.testing.expect(std.mem.startsWith(u8, note, "[{\"id\":\"ISS-1\"}]\n[slim view of a 56005-byte value"));
try std.testing.expect(std.mem.indexOf(u8, note, "write_file") != null);
}
test "a printed slim view names the fields the whole value has (ADR 0240)" {
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const note = @import("rlm.zig").slimNote(arena_state.allocator(), "[{\"id\":\"ISS-1\"}]", "[{\"id\":\"ISS-1\",\"priority\":2,\"estimate\":3}]");
try std.testing.expect(std.mem.indexOf(u8, note, "-byte value; fields: id, priority, estimate; the value keeps every field") != null);
}
test "an unbound call and a bare each() print their results (ADR 0240)" {
const gpa = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try tmp.dir.writeFile(io, .{ .sub_path = "note.txt", .data = "hello-rlm\n" });
var path_buf: [std.fs.max_path_bytes]u8 = undefined;
const n = try tmp.dir.realPath(io, &path_buf);
var dummy_client: std.http.Client = .{ .allocator = gpa, .io = io };
defer dummy_client.deinit();
const saved = rlm.available;
defer {
rlm.available = saved;
rlm.resetLive(gpa, io);
}
rlm.available = true;
var ctx = testCtx(gpa, io, &dummy_client);
ctx.agent_cwd = path_buf[0..n];
const bare = try rlm.runScript(ctx, "read_file(\"note.txt\")");
defer gpa.free(bare.text);
try std.testing.expect(!bare.is_error);
try std.testing.expect(std.mem.indexOf(u8, bare.text, "hello-rlm") != null);
// each() hands a native tool `arg`; sleep_ms takes it (rlm_order_tests does the same).
const mapped = try rlm.runScript(ctx, "each([1, 2], \"sleep_ms\")");
defer gpa.free(mapped.text);
try std.testing.expect(!mapped.is_error);
try std.testing.expect(std.mem.indexOf(u8, mapped.text, "slept") != null);
}