graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
//! Resume re-attaches durable background jobs (ADR 0218). A graff that died
//! mid-task left its finite background jobs running with their output in
//! `.graff/job-output/<id>/` (job_capture.zig). Loading that session puts
//! each one back in the job pool under its old id: a job still running is
//! watched to its end, one that ended while graff was down is reported, and
//! either way `bash_output` and `bash_kill` reach it again. Captures of other
//! sessions are left for their own resume, and swept once stale and over.
const std = @import("std");
const builtin = @import("builtin");
const Io = std.Io;
const Allocator = std.mem.Allocator;
const jobs = @import("jobs.zig");
const Job = jobs.Job;
const g_jobs = &jobs.g_jobs;
const job_capture = @import("job_capture.zig");
const Meta = job_capture.Meta;
const job_registry = @import("job_registry.zig");
const proc_identity = @import("proc_identity.zig");
const util = @import("util.zig");
/// Another session's capture is swept once this old and over.
const stale_ms: i64 = 7 * std.time.ms_per_day;
pub const Seen = struct {
meta: ?Meta,
reported: bool = false,
owner_alive: bool = false,
over: bool = false, // an exit receipt, or its leader is gone
age_ms: i64 = 0,
};
pub const Verdict = enum { adopt, remove, skip };
/// What resume does with one capture directory.
pub fn verdict(seen: Seen, session: []const u8) Verdict {
const meta = seen.meta orelse return if (seen.age_ms > stale_ms) .remove else .skip;
if (seen.owner_alive) return .skip; // a live graff still watches it
if (seen.reported) return .remove; // its exit already reached the model
if (std.mem.eql(u8, meta.session, session)) return .adopt;
return if (seen.over and seen.age_ms > stale_ms) .remove else .skip;
}
/// session.loadSession: recover from the workspace's capture root.
pub fn onLoad(gpa: Allocator, io: Io, session: []const u8) void {
if (comptime !job_capture.enabled) return;
if (builtin.is_test or session.len == 0) return;
var buf: [std.fs.max_path_bytes]u8 = undefined;
const root = job_capture.rootPath(io, &buf) orelse return;
_ = recoverIn(gpa, io, root, session);
}
/// Returns how many jobs went back into the pool.
pub fn recoverIn(gpa: Allocator, io: Io, root: []const u8, session: []const u8) usize {
if (comptime !job_capture.enabled) return 0;
var arena_state = std.heap.ArenaAllocator.init(gpa);
defer arena_state.deinit();
const arena = arena_state.allocator();
var names: std.ArrayList([]const u8) = .empty;
{
var dir = Io.Dir.cwd().openDir(io, root, .{ .iterate = true }) catch return 0;
defer dir.close(io);
var it = dir.iterate();
while (it.next(io) catch null) |entry| {
if (entry.kind != .directory) continue;
_ = std.fmt.parseInt(u64, entry.name, 10) catch continue;
names.append(arena, arena.dupe(u8, entry.name) catch continue) catch continue;
}
}
const now = util.unixMs(io);
var adopted: usize = 0;
for (names.items) |name| { // acted on after the listing: no deletes mid-iteration
const path = std.fmt.allocPrint(arena, "{s}/{s}", .{ root, name }) catch continue;
const seen = look(io, arena, path, now);
switch (verdict(seen, session)) {
.skip => {},
.remove => Io.Dir.cwd().deleteTree(io, path) catch {},
.adopt => if (adopt(gpa, io, path, seen.meta.?)) {
adopted += 1;
} else |_| {},
}
}
return adopted;
}
fn look(io: Io, arena: Allocator, path: []const u8, now: i64) Seen {
const meta = job_capture.readMeta(io, arena, path) orelse {
const stat = Io.Dir.cwd().statFile(io, path, .{}) catch return .{ .meta = null };
const mtime_ms: i64 = @intCast(@divTrunc(stat.mtime.nanoseconds, std.time.ns_per_ms));
return .{ .meta = null, .age_ms = now - mtime_ms };
};
const owner: proc_identity.Probe = if (meta.owner_pid > 0) proc_identity.probe(io, meta.owner_pid) else .gone;
const leader = proc_identity.probe(io, meta.pid);
return .{
.meta = meta,
.reported = job_capture.reported(io, path),
.owner_alive = job_registry.stateOf(meta.owner_start_id, owner) != .gone,
.over = job_capture.exitStatus(io, path) != null or job_registry.stateOf(meta.start_id, leader) == .gone,
.age_ms = now - meta.started_ms,
};
}
/// Back into the pool under its old id, watched by the durable pump.
fn adopt(gpa: Allocator, io: Io, path: []const u8, meta: Meta) !void {
{
g_jobs.mutex.lockUncancelable(io);
defer g_jobs.mutex.unlock(io);
if (g_jobs.find(meta.id) != null) return error.AlreadyTracked;
}
const capture = try gpa.create(job_capture.Capture);
errdefer gpa.destroy(capture);
capture.* = .{ .dir = try gpa.dupe(u8, path), .adopted = true, .start_id = meta.start_id };
errdefer gpa.free(capture.dir);
const cmd = try gpa.dupe(u8, meta.cmd);
errdefer gpa.free(cmd);
const cwd: ?[]u8 = if (meta.cwd.len > 0) try gpa.dupe(u8, meta.cwd) else null;
errdefer if (cwd) |c| gpa.free(c);
const job = try gpa.create(Job);
errdefer gpa.destroy(job);
job.* = .{
.id = meta.id,
.cmd = cmd,
// Not graff's child: nothing may wait on or signal it through `child`.
.child = .{ .id = null, .thread_handle = {}, .stdin = null, .stdout = null, .stderr = null, .request_resource_usage_statistics = false },
.cwd = cwd,
.started_ms = meta.started_ms,
.last_active_ms = jobs.nowMs(io),
.group_pid = meta.pid,
.capture = capture,
};
// This graff watches it now: another resume of the session leaves it be.
// Both records land before the pump, which forgets the registry one.
var owned = meta;
owned.owner_pid = proc_identity.selfPid();
owned.owner_start_id = proc_identity.selfStartId(io);
job_capture.writeMeta(io, capture, owned);
if (meta.pid > 0) job_registry.write(io, job_registry.home, @import("jobs_admin.zig").recordOf(io, job));
errdefer if (meta.pid > 0) job_registry.forget(io, job_registry.home, meta.pid);
{
g_jobs.mutex.lockUncancelable(io);
defer g_jobs.mutex.unlock(io);
try g_jobs.list.append(gpa, job);
}
job.future = io.concurrent(@import("job_durable.zig").pump, .{ job, gpa, io }) catch |e| {
g_jobs.mutex.lockUncancelable(io);
defer g_jobs.mutex.unlock(io);
for (g_jobs.list.items, 0..) |j, i| {
if (j == job) {
_ = g_jobs.list.swapRemove(i);
break;
}
}
return e;
};
}
test "resume adopts its own session's jobs and leaves live owners and other sessions alone" {
const own: Meta = .{ .id = 1, .session = "mine", .cmd = "make" };
const other: Meta = .{ .id = 2, .session = "theirs", .cmd = "make" };
try std.testing.expectEqual(Verdict.adopt, verdict(.{ .meta = own }, "mine"));
try std.testing.expectEqual(Verdict.skip, verdict(.{ .meta = own, .owner_alive = true }, "mine"));
try std.testing.expectEqual(Verdict.remove, verdict(.{ .meta = own, .reported = true }, "mine"));
try std.testing.expectEqual(Verdict.skip, verdict(.{ .meta = own, .reported = true, .owner_alive = true }, "mine"));
try std.testing.expectEqual(Verdict.skip, verdict(.{ .meta = other, .over = true }, "mine"));
try std.testing.expectEqual(Verdict.skip, verdict(.{ .meta = other, .age_ms = stale_ms + 1 }, "mine"));
try std.testing.expectEqual(Verdict.remove, verdict(.{ .meta = other, .over = true, .age_ms = stale_ms + 1 }, "mine"));
try std.testing.expectEqual(Verdict.skip, verdict(.{ .meta = null }, "mine"));
try std.testing.expectEqual(Verdict.remove, verdict(.{ .meta = null, .age_ms = stale_ms + 1 }, "mine"));
}
/// A capture as a graff that then died would leave it: the job spawned with
/// its receipt, its record naming an owner that no longer exists.
fn orphan(io: Io, root: []const u8, id: u64, cmd: []const u8, session: []const u8) !std.process.Child {
const gpa = std.testing.allocator;
var setup = try job_capture.create(io, gpa, root, id, cmd);
const child = std.process.spawn(io, .{ .argv = &setup.argv, .stdin = .ignore, .stdout = .{ .file = setup.out }, .stderr = .{ .file = setup.err }, .pgid = 0 }) catch |e| {
setup.abandon(gpa, io);
return e;
};
const pid = child.id.?;
const capture = setup.capture;
setup.spawned(gpa, io);
job_capture.writeMeta(io, capture, .{
.id = id,
.session = session,
.cmd = cmd,
.pid = pid,
.start_id = switch (proc_identity.probe(io, pid)) {
.id => |v| v,
else => 0,
},
.owner_pid = 0,
.started_ms = util.unixMs(io),
});
job_capture.release(gpa, io, capture, false);
return child;
}
test "a job that outlived graff is re-attached on resume and reports its real exit once" {
if (!job_capture.enabled) return error.SkipZigTest;
const gpa = std.testing.allocator;
const io = std.testing.io;
const job_notify = @import("job_notify.zig");
jobs.g_jobs = .{};
job_notify.resetForTest(io);
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var rbuf: [std.fs.max_path_bytes]u8 = undefined;
const root = rbuf[0..try tmp.dir.realPath(io, &rbuf)];
// One finished while graff was down, one is still running at resume, and
// one belongs to another session.
var done = try orphan(io, root, 7001, "printf early; exit 3", "s-resume");
_ = try done.wait(io);
var late = try orphan(io, root, 7002, "sleep 0.3; printf late", "s-resume");
defer if (late.wait(io)) |_| {} else |_| {};
var other = try orphan(io, root, 7003, "exit 0", "s-other");
_ = try other.wait(io);
try std.testing.expectEqual(@as(usize, 2), recoverIn(gpa, io, root, "s-resume"));
defer jobs.jobsReap(gpa, io);
const early = try jobs.jobOutput(gpa, io, 7001, 5000);
defer gpa.free(early.text);
try std.testing.expect(std.mem.indexOf(u8, early.text, "exited with code 3") != null);
try std.testing.expect(std.mem.indexOf(u8, early.text, "early") != null);
const running = try jobs.jobOutput(gpa, io, 7002, 5000);
defer gpa.free(running.text);
try std.testing.expect(std.mem.indexOf(u8, running.text, "exited with code 0") != null);
try std.testing.expect(std.mem.indexOf(u8, running.text, "late") != null);
// Both exits reached the model; the other session's capture is untouched.
var pbuf: [std.fs.max_path_bytes]u8 = undefined;
try std.testing.expect(job_capture.reported(io, try std.fmt.bufPrint(&pbuf, "{s}/7001", .{root})));
try std.testing.expect(job_capture.reported(io, try std.fmt.bufPrint(&pbuf, "{s}/7002", .{root})));
try std.testing.expect(!job_capture.reported(io, try std.fmt.bufPrint(&pbuf, "{s}/7003", .{root})));
// A second resume while this process owns them adopts nothing.
try std.testing.expectEqual(@as(usize, 0), recoverIn(gpa, io, root, "s-resume"));
}
test "a resumed job's completion notice marks it reported" {
if (!job_capture.enabled) return error.SkipZigTest;
const gpa = std.testing.allocator;
const io = std.testing.io;
const job_notify = @import("job_notify.zig");
jobs.g_jobs = .{};
job_notify.resetForTest(io);
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var rbuf: [std.fs.max_path_bytes]u8 = undefined;
const root = rbuf[0..try tmp.dir.realPath(io, &rbuf)];
var done = try orphan(io, root, 7101, "printf gone; exit 5", "s-notice");
_ = try done.wait(io);
try std.testing.expectEqual(@as(usize, 1), recoverIn(gpa, io, root, "s-notice"));
defer jobs.jobsReap(gpa, io);
var buf: [4096]u8 = undefined;
var text: []const u8 = "";
for (0..200) |_| {
text = job_notify.takeWake(io, &buf) orelse {
try io.sleep(.fromMilliseconds(10), .awake);
continue;
};
break;
}
try std.testing.expect(std.mem.indexOf(u8, text, "[job 7101 exited 5: printf gone; exit 5]") != null);
try std.testing.expect(std.mem.indexOf(u8, text, "gone") != null);
var pbuf: [std.fs.max_path_bytes]u8 = undefined;
try std.testing.expect(job_capture.reported(io, try std.fmt.bufPrint(&pbuf, "{s}/7101", .{root})));
}