Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
8.3 KB
Raw Blame
//! Is a proposed harness tree one graff can run? Every problem comes back as
//! one line of actionable prose, because the list is fed verbatim to the
//! proposer as retry feedback: say what is wrong and what to do instead.
//!
//! Beyond shape (paths, sizes, frontmatter, hook JSON), two rules keep a
//! refine loop honest. References to helper scripts must resolve inside the
//! tree, so a probe never fails on a file the proposal forgot. And text the
//! proposal adds may not talk about grading: the loop improves the work from
//! how runs behaved, and a harness that cites a rubric or a grader is tuning
//! to a measurement it should never see.

const std = @import("std");
const Allocator = std.mem.Allocator;
const Value = std.json.Value;
const tree_mod = @import("harness_tree.zig");
const Tree = tree_mod.Tree;

pub const grader_words = [_][]const u8{ "rubric", "grader", "grading criteria", "hidden test", "judge's", "the judge" };
const hook_events = [_][]const u8{ "pre_tool", "post_tool", "turn_end" };

/// Every problem with `tree`, one line each; empty means it can run. `base`
/// is the tree the proposal started from: grading language already in it is
/// the user's and is not held against the proposal.
pub fn problems(arena: Allocator, tree: Tree, base: ?Tree) ![]const []const u8 {
    var out: std.ArrayList([]const u8) = .empty;
    if (tree.files.len > tree_mod.max_files)
        try say(arena, &out, "the harness has {d} files; keep it to {d} or fewer by merging or dropping the least useful ones", .{ tree.files.len, tree_mod.max_files });
    var n_instructions: usize = 0;
    for (tree.files, 0..) |f, i| {
        if (i > 0 and !std.mem.lessThan(u8, tree.files[i - 1].path, f.path))
            try say(arena, &out, "{s} appears twice or out of order; list each path once", .{f.path});
        const part = tree_mod.classify(f.path) orelse {
            try say(arena, &out, "{s} is not part of the harness; use the instruction file, {s}/<name>/SKILL.md, {s}/<name>.md, {s}/<file>, or the hooks in {s} (names are lowercase letters, digits, `_` or `-`)", .{ f.path, tree_mod.skills_dir, tree_mod.agents_dir, tree_mod.scripts_dir, tree_mod.hooks_path });
            continue;
        };
        if (f.data.len > tree_mod.max_file_bytes)
            try say(arena, &out, "{s} is {d} bytes; keep each file under {d}", .{ f.path, f.data.len, tree_mod.max_file_bytes });
        switch (part) {
            .instructions => {
                n_instructions += 1;
                const cap = @import("context_limits.zig").agents_md_bytes;
                if (f.data.len > cap)
                    try say(arena, &out, "{s} is {d} bytes but graff reads only the first {d}; move detail into a skill and keep the instruction file short", .{ f.path, f.data.len, cap });
            },
            .skill => try checkSkill(arena, &out, f),
            .skill_file => {
                const name = tree_mod.skillName(f.path).?;
                const doc = try std.fmt.allocPrint(arena, "{s}/{s}/SKILL.md", .{ tree_mod.skills_dir, name });
                if (tree.get(doc) == null)
                    try say(arena, &out, "{s} belongs to skill `{s}`, which has no {s}; add one or drop the file", .{ f.path, name, doc });
            },
            .subagent => {
                const base_name = std.fs.path.basename(f.path);
                if (@import("agent_file.zig").parse(arena, base_name, f.data) == null)
                    try say(arena, &out, "{s} has no prompt body; a subagent file needs instructions after its frontmatter", .{f.path});
            },
            .script => {},
            .hooks => try checkHooks(arena, &out, f.data),
        }
        try checkScriptRefs(arena, &out, tree, f);
        try checkGraderWords(arena, &out, f, if (base) |b| b.get(f.path) else null);
    }
    if (n_instructions > 1)
        try say(arena, &out, "the harness has {d} instruction files but graff reads only the first non-empty one; keep a single instruction file", .{n_instructions});
    return out.items;
}

/// The problems as one message, or null when there are none.
pub fn report(arena: Allocator, tree: Tree, base: ?Tree) !?[]const u8 {
    const list = try problems(arena, tree, base);
    if (list.len == 0) return null;
    var buf: std.Io.Writer.Allocating = .init(arena);
    for (list) |p| try buf.writer.print("- {s}\n", .{p});
    return buf.writer.buffered();
}

fn say(arena: Allocator, out: *std.ArrayList([]const u8), comptime fmt: []const u8, args: anytype) !void {
    try out.append(arena, try std.fmt.allocPrint(arena, fmt, args));
}

fn checkSkill(arena: Allocator, out: *std.ArrayList([]const u8), f: tree_mod.File) !void {
    const dir_name = tree_mod.skillName(f.path).?;
    const doc = @import("skill_docs.zig").parseDoc(dir_name, f.data);
    if (!std.mem.startsWith(u8, f.data, "---"))
        return say(arena, out, "{s} needs frontmatter: start it with ---, then `name: {s}` and a one-line `description:`, then ---", .{ f.path, dir_name });
    if (doc.desc.len == 0)
        try say(arena, out, "{s} has no `description:`; graff shows only the description until the skill is loaded, so say when to use it", .{f.path});
    if (!std.mem.eql(u8, doc.name, dir_name))
        try say(arena, out, "{s} is named `{s}` in its frontmatter; use `name: {s}` to match its directory", .{ f.path, doc.name, dir_name });
    if (doc.body.len == 0) try say(arena, out, "{s} has no body after its frontmatter", .{f.path});
}

fn checkHooks(arena: Allocator, out: *std.ArrayList([]const u8), data: []const u8) !void {
    const v = std.json.parseFromSliceLeaky(Value, arena, data, .{}) catch
        return say(arena, out, "the hooks are not valid JSON; write an object like {{\"pre_tool\": [{{\"match\": \"shell\", \"command\": \"python3 .harness/scripts/guard.py\"}}]}}", .{});
    if (v != .object) return say(arena, out, "the hooks must be a JSON object keyed by pre_tool, post_tool or turn_end", .{});
    var it = v.object.iterator();
    while (it.next()) |kv| {
        const event = kv.key_ptr.*;
        if (!known(event)) {
            try say(arena, out, "hook event `{s}` does not exist; use pre_tool, post_tool or turn_end", .{event});
            continue;
        }
        if (kv.value_ptr.* != .array) {
            try say(arena, out, "hooks.{s} must be a list of {{\"command\": ...}} objects", .{event});
            continue;
        }
        for (kv.value_ptr.array.items, 0..) |h, i| {
            const cmd = if (h == .object) h.object.get("command") else null;
            if (cmd == null or cmd.? != .string or std.mem.trim(u8, cmd.?.string, " \t").len == 0)
                try say(arena, out, "hooks.{s}[{d}] needs a non-empty string `command`", .{ event, i });
        }
    }
}

fn known(event: []const u8) bool {
    for (hook_events) |e| if (std.mem.eql(u8, e, event)) return true;
    return false;
}

/// Every `.harness/scripts/<file>` a harness file mentions must be in the tree.
fn checkScriptRefs(arena: Allocator, out: *std.ArrayList([]const u8), tree: Tree, f: tree_mod.File) !void {
    const prefix = tree_mod.scripts_dir ++ "/";
    var at: usize = 0;
    while (std.mem.indexOfPos(u8, f.data, at, prefix)) |start| {
        var end = start + prefix.len;
        while (end < f.data.len and isPathByte(f.data[end])) end += 1;
        while (end > start + prefix.len and f.data[end - 1] == '.') end -= 1; // sentence period
        at = end;
        if (end == start + prefix.len) continue;
        const ref = f.data[start..end];
        if (tree.get(ref) == null)
            try say(arena, out, "{s} mentions {s}, which is not in the harness; add the script or drop the mention", .{ f.path, ref });
    }
}

fn isPathByte(c: u8) bool {
    return std.ascii.isAlphanumeric(c) or c == '_' or c == '-' or c == '.' or c == '/';
}

fn checkGraderWords(arena: Allocator, out: *std.ArrayList([]const u8), f: tree_mod.File, before: ?[]const u8) !void {
    for (grader_words) |word| {
        if (countFold(f.data, word) > if (before) |b| countFold(b, word) else 0)
            try say(arena, out, "{s} adds text about `{s}`; describe the work itself, not how it is graded", .{ f.path, word });
    }
}

fn countFold(hay: []const u8, needle: []const u8) usize {
    if (needle.len == 0 or hay.len < needle.len) return 0;
    var n: usize = 0;
    var i: usize = 0;
    while (i + needle.len <= hay.len) : (i += 1) {
        if (std.ascii.eqlIgnoreCase(hay[i .. i + needle.len], needle)) n += 1;
    }
    return n;
}