Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
16.5 KB
Raw Blame
//! Optional, pinned model-check baseline for native prompt-policy learning.
//! The result concerns shared engine source, not the candidate prompt's behavior.
const std = @import("std");
const Io = std.Io;
const Allocator = std.mem.Allocator;
const store_mod = @import("learn_store.zig");
const jobs = @import("jobs.zig");
const util = @import("util.zig");

pub const evidence_schema = "codegraff.learn.formal-evidence.v1";
const check_schema = "codegraff.dgm.formal-check.v1";
const stdout_cap = 4096;
const stderr_cap = 4096;

pub const Evidence = struct {
    schema: []const u8 = evidence_schema,
    phase: []const u8,
    config_id: []const u8,
    trial_id: []const u8,
    candidate_prompt_sha256: []const u8,
    formal_identity_sha256: []const u8,
    checker_output_sha256: []const u8,
    binary_sha256: []const u8,
    formal_pin_sha256: []const u8,
    checker_sha256: []const u8,
    helper_sha256: []const u8,
};

fn validHash(value: []const u8) bool {
    return store_mod.validId(value);
}

fn promptHash(prompt: []const u8) [64]u8 {
    var digest: [32]u8 = undefined;
    std.crypto.hash.sha2.Sha256.hash(prompt, &digest, .{});
    return std.fmt.bytesToHex(digest, .lower);
}

fn checkedString(obj: std.json.ObjectMap, key: []const u8) ![]const u8 {
    const value = util.strFieldObj(obj, key) orelse return error.InvalidFormalCheckResult;
    if (!validHash(value)) return error.InvalidFormalCheckResult;
    return value;
}

fn parseResult(arena: Allocator, raw: []const u8, candidate_sha: []const u8, binary_sha: []const u8) !Evidence {
    const parsed = try std.json.parseFromSliceLeaky(std.json.Value, arena, raw, .{});
    if (parsed != .object or parsed.object.count() != 6 or
        !std.mem.eql(u8, util.strFieldObj(parsed.object, "schema") orelse "", check_schema))
        return error.InvalidFormalCheckResult;
    const ok = parsed.object.get("ok") orelse return error.InvalidFormalCheckResult;
    if (ok != .bool or !ok.bool) return error.InvalidFormalCheckResult;
    const got_candidate = try checkedString(parsed.object, "candidate_prompt_sha256");
    const got_binary = try checkedString(parsed.object, "binary_sha256");
    if (!std.mem.eql(u8, got_candidate, candidate_sha) or !std.mem.eql(u8, got_binary, binary_sha))
        return error.FormalCheckIdentityMismatch;
    return .{
        .phase = "",
        .config_id = "",
        .trial_id = "",
        .candidate_prompt_sha256 = got_candidate,
        .formal_identity_sha256 = try checkedString(parsed.object, "formal_identity_sha256"),
        .checker_output_sha256 = try checkedString(parsed.object, "checker_output_sha256"),
        .binary_sha256 = got_binary,
        .formal_pin_sha256 = "",
        .checker_sha256 = "",
        .helper_sha256 = "",
    };
}

fn binaryPin(arena: Allocator, io: Io, formal: store_mod.FormalCheck) !store_mod.PinnedFile {
    const bytes = try store_mod.readPinnedFileAlloc(io, arena, formal.pin, store_mod.max_config_bytes);
    const parsed = try std.json.parseFromSliceLeaky(std.json.Value, arena, bytes, .{});
    if (parsed != .object) return error.InvalidFormalPin;
    const binary = parsed.object.get("binary") orelse return error.InvalidFormalPin;
    if (binary != .object) return error.InvalidFormalPin;
    const sha = util.strFieldObj(binary.object, "sha256") orelse return error.InvalidFormalPin;
    const path = util.strFieldObj(binary.object, "path") orelse return error.InvalidFormalPin;
    if (!validHash(sha) or !std.fs.path.isAbsolute(path)) return error.InvalidFormalPin;
    return .{ .path = path, .sha256 = sha };
}

fn matchesAdapter(program: store_mod.Program, binary: store_mod.PinnedFile) bool {
    var pinned = false;
    var passed = false;
    for (program.inputs) |input| {
        if (std.mem.eql(u8, input.path, binary.path) and std.mem.eql(u8, input.sha256, binary.sha256))
            pinned = true;
    }
    for (program.args) |arg| {
        if (std.mem.eql(u8, arg, binary.path)) passed = true;
    }
    return pinned and passed;
}

pub fn verifyBinaryBinding(arena: Allocator, io: Io, config: store_mod.Config) !void {
    const formal = config.formal_check orelse return;
    const binary = try binaryPin(arena, io, formal);
    if (!matchesAdapter(config.mutator, binary) or !matchesAdapter(config.evaluator, binary))
        return error.FormalBinaryMismatch;
}

fn check(
    gpa: Allocator,
    arena: Allocator,
    io: Io,
    environ: *const std.process.Environ.Map,
    formal: store_mod.FormalCheck,
    prompt: []const u8,
) !Evidence {
    const key_path = environ.get("GRAFF_SCORE_KEY_FILE") orelse return error.FormalSigningKeyRequired;
    if (!std.fs.path.isAbsolute(key_path)) return error.FormalSigningKeyRequired;
    if (formal.checker.args.len != 1 or formal.checker.inputs.len != 1)
        return error.InvalidFormalChecker;
    try store_mod.verifyProgram(io, formal.checker);
    try store_mod.verifyPinnedFile(io, formal.pin);
    const expected_binary = (try binaryPin(arena, io, formal)).sha256;
    const candidate_sha = promptHash(prompt);
    var child_env = std.process.Environ.Map.init(gpa);
    defer child_env.deinit();
    try child_env.put("PATH", "/usr/bin:/bin");
    try child_env.put("LANG", "C");
    try child_env.put("LC_ALL", "C");
    try child_env.put("GRAFF_SCORE_KEY_FILE", key_path);
    try child_env.put("PYTHONDONTWRITEBYTECODE", "1");
    const argv = [_][]const u8{
        formal.checker.program,
        formal.checker.args[0],
        "--check",
        formal.pin.path,
        "--candidate-sha256",
        &candidate_sha,
    };
    const run = try jobs.runCappedWithOptions(gpa, io, &argv, stdout_cap, stderr_cap, formal.timeout_ms, .{
        .environ_map = &child_env,
        .kill_process_tree = true,
    });
    defer {
        gpa.free(run.stdout);
        gpa.free(run.stderr);
    }
    if (run.timed_out or run.cancelled) return error.FormalCheckTimedOut;
    if (run.stdout_truncated or run.stderr_truncated) return error.FormalCheckOutputTooLarge;
    if (run.term != .exited or run.term.exited != 0) return error.FormalCheckFailed;
    // The pin is checked after execution too. The external pin/helper is an
    // operator trust input; this detects drift, not adversarial transient swap.
    try store_mod.verifyProgram(io, formal.checker);
    try store_mod.verifyPinnedFile(io, formal.pin);
    var result = try parseResult(arena, run.stdout, &candidate_sha, expected_binary);
    result.formal_pin_sha256 = formal.pin.sha256;
    result.checker_sha256 = formal.checker.sha256;
    result.helper_sha256 = formal.checker.inputs[0].sha256;
    return result;
}

pub fn record(
    gpa: Allocator,
    arena: Allocator,
    io: Io,
    environ: *const std.process.Environ.Map,
    store: *store_mod.Store,
    formal: store_mod.FormalCheck,
    config_id: []const u8,
    trial_id: []const u8,
    phase: []const u8,
    prompt: []const u8,
) ![]const u8 {
    if (!std.mem.eql(u8, phase, "admission") and !std.mem.eql(u8, phase, "selection"))
        return error.InvalidFormalPhase;
    var result = try check(gpa, arena, io, environ, formal, prompt);
    result.phase = phase;
    result.config_id = config_id;
    result.trial_id = trial_id;
    const bytes = try store_mod.jsonBytes(gpa, result);
    defer gpa.free(bytes);
    const id = try store.writeEvidence(gpa, bytes);
    return try arena.dupe(u8, &id);
}

pub fn verify(
    gpa: Allocator,
    arena: Allocator,
    io: Io,
    environ: *const std.process.Environ.Map,
    store: *store_mod.Store,
    formal: store_mod.FormalCheck,
    evidence_id: []const u8,
    config_id: []const u8,
    trial_id: []const u8,
    phase: []const u8,
    prompt: []const u8,
) !void {
    if (formal.checker.args.len != 1 or formal.checker.inputs.len != 1)
        return error.InvalidFormalChecker;
    const bytes = try store.readEvidence(arena, evidence_id, store_mod.max_record_bytes);
    const saved = try std.json.parseFromSliceLeaky(Evidence, arena, bytes, .{});
    const expected_sha = promptHash(prompt);
    if (!std.mem.eql(u8, saved.schema, evidence_schema) or
        !std.mem.eql(u8, saved.phase, phase) or
        !std.mem.eql(u8, saved.config_id, config_id) or
        !std.mem.eql(u8, saved.trial_id, trial_id) or
        !std.mem.eql(u8, saved.candidate_prompt_sha256, &expected_sha) or
        !std.mem.eql(u8, saved.formal_pin_sha256, formal.pin.sha256) or
        !std.mem.eql(u8, saved.checker_sha256, formal.checker.sha256) or
        !std.mem.eql(u8, saved.helper_sha256, formal.checker.inputs[0].sha256) or
        !validHash(saved.formal_identity_sha256) or !validHash(saved.checker_output_sha256) or
        !validHash(saved.binary_sha256)) return error.FormalEvidenceMismatch;
    const fresh = try check(gpa, arena, io, environ, formal, prompt);
    // TLC output contains timestamps/seeds, so its output hash may differ on a
    // fresh run. The pinned identity, binary and candidate must match.
    if (!std.mem.eql(u8, saved.formal_identity_sha256, fresh.formal_identity_sha256) or
        !std.mem.eql(u8, saved.binary_sha256, fresh.binary_sha256))
        return error.FormalEvidenceMismatch;
}

test "formal helper result rejects false success and wrong candidate" {
    const a = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
    const b = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
    var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
    defer arena_state.deinit();
    const arena = arena_state.allocator();
    const good = try std.fmt.allocPrint(arena, "{{\"schema\":\"{s}\",\"ok\":true,\"candidate_prompt_sha256\":\"{s}\",\"formal_identity_sha256\":\"{s}\",\"checker_output_sha256\":\"{s}\",\"binary_sha256\":\"{s}\"}}", .{ check_schema, a, a, a, b });
    _ = try parseResult(arena, good, a, b);
    try std.testing.expectError(error.FormalCheckIdentityMismatch, parseResult(arena, good, b, b));
    const false_success = try std.fmt.allocPrint(arena, "{{\"schema\":\"{s}\",\"ok\":false,\"candidate_prompt_sha256\":\"{s}\",\"formal_identity_sha256\":\"{s}\",\"checker_output_sha256\":\"{s}\",\"binary_sha256\":\"{s}\"}}", .{ check_schema, a, a, a, b });
    try std.testing.expectError(error.InvalidFormalCheckResult, parseResult(arena, false_success, a, b));
}

test "formal binary must be pinned and passed to both adapters" {
    const a = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
    const b = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
    const binary: store_mod.PinnedFile = .{ .path = "/opt/graff", .sha256 = a };
    const correct: store_mod.Program = .{
        .program = "/opt/adapter",
        .sha256 = b,
        .args = &.{"/opt/graff"},
        .inputs = &.{binary},
    };
    try std.testing.expect(matchesAdapter(correct, binary));
    var wrong = correct;
    wrong.args = &.{"/opt/other"};
    try std.testing.expect(!matchesAdapter(wrong, binary));
    wrong = correct;
    wrong.inputs = &.{.{ .path = "/opt/graff", .sha256 = b }};
    try std.testing.expect(!matchesAdapter(wrong, binary));
}

test "default configuration skips the optional formal checker" {
    const hash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
    const config: store_mod.Config = .{
        .schema = store_mod.config_schema,
        .agent_name = "candidate",
        .mutation_instruction = "change one behavior",
        .mutator = .{ .program = "/missing/mutator", .sha256 = hash },
        .evaluator = .{ .program = "/missing/evaluator", .sha256 = hash },
        .evaluation_suite = .{ .path = "/missing/suite", .sha256 = hash },
        .cohort = .{ .provider = "test", .model = "test", .task_family = "test", .adapter_version = "v1", .verifier_version = "v1" },
    };
    try verifyBinaryBinding(std.testing.allocator, std.testing.io, config);
}

test "pinned helper evidence is rechecked for resume and promotion contexts" {
    if (@import("builtin").os.tag == .windows) return;
    const io = std.testing.io;
    const gpa = std.testing.allocator;
    var arena_state = std.heap.ArenaAllocator.init(gpa);
    defer arena_state.deinit();
    const arena = arena_state.allocator();
    var tmp = std.testing.tmpDir(.{});
    defer tmp.cleanup();
    var root_buf: [std.fs.max_path_bytes]u8 = undefined;
    const root_len = try tmp.dir.realPath(io, &root_buf);
    const root = root_buf[0..root_len];
    const helper_path = try std.fmt.allocPrint(arena, "{s}/helper.sh", .{root});
    const pin_path = try std.fmt.allocPrint(arena, "{s}/pin.json", .{root});
    const key_path = try std.fmt.allocPrint(arena, "{s}/key", .{root});
    var shell_buf: [std.fs.max_path_bytes]u8 = undefined;
    const shell_len = try Io.Dir.realPathFileAbsolute(io, "/bin/sh", &shell_buf);
    const shell_path = shell_buf[0..shell_len];
    const binary_hash = try store_mod.hashFileNoFollow(io, shell_path);
    const identity = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
    const script = try std.fmt.allocPrint(arena, "printf '{{\"schema\":\"{s}\",\"ok\":true,\"candidate_prompt_sha256\":\"%s\",\"formal_identity_sha256\":\"{s}\",\"checker_output_sha256\":\"{s}\",\"binary_sha256\":\"{s}\"}}\\n' \"$4\"\n", .{ check_schema, identity, identity, &binary_hash });
    {
        const file = try tmp.dir.createFile(io, "helper.sh", .{ .permissions = .fromMode(0o600) });
        defer file.close(io);
        try file.writeStreamingAll(io, script);
    }
    const pin_bytes = try std.fmt.allocPrint(arena, "{{\"binary\":{{\"path\":\"{s}\",\"sha256\":\"{s}\"}}}}\n", .{ shell_path, &binary_hash });
    {
        const file = try tmp.dir.createFile(io, "pin.json", .{ .permissions = .fromMode(0o600) });
        defer file.close(io);
        try file.writeStreamingAll(io, pin_bytes);
    }
    {
        const file = try tmp.dir.createFile(io, "key", .{ .permissions = .fromMode(0o600) });
        defer file.close(io);
        try file.writeStreamingAll(io, "test key");
    }
    const helper_hash = store_mod.rawSha256(script);
    const pin_hash = store_mod.rawSha256(pin_bytes);
    const inputs = [_]store_mod.PinnedFile{.{ .path = helper_path, .sha256 = &helper_hash }};
    const formal: store_mod.FormalCheck = .{
        .checker = .{ .program = shell_path, .sha256 = &binary_hash, .args = &.{helper_path}, .inputs = &inputs },
        .pin = .{ .path = pin_path, .sha256 = &pin_hash },
        .timeout_ms = 2000,
    };
    var env = std.process.Environ.Map.init(gpa);
    defer env.deinit();
    try std.testing.expectError(error.FormalSigningKeyRequired, check(gpa, arena, io, &env, formal, "parent prompt"));
    try env.put("GRAFF_SCORE_KEY_FILE", key_path);
    var store = try store_mod.Store.initAt(io, tmp.dir);
    defer store.deinit();
    const config_id = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
    const trial_id = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc";
    const admission = try record(gpa, arena, io, &env, &store, formal, config_id, trial_id, "admission", "parent prompt");
    try verify(gpa, arena, io, &env, &store, formal, admission, config_id, trial_id, "admission", "parent prompt");
    try std.testing.expectError(error.FormalEvidenceMismatch, verify(gpa, arena, io, &env, &store, formal, admission, config_id, trial_id, "admission", "changed parent"));
    try std.testing.expectError(error.FormalEvidenceMismatch, verify(gpa, arena, io, &env, &store, formal, admission, config_id, trial_id, "selection", "parent prompt"));
    const selection = try record(gpa, arena, io, &env, &store, formal, config_id, trial_id, "selection", "selected prompt");
    try verify(gpa, arena, io, &env, &store, formal, selection, config_id, trial_id, "selection", "selected prompt");
    try std.testing.expectError(error.FormalEvidenceMismatch, verify(gpa, arena, io, &env, &store, formal, selection, config_id, trial_id, "selection", "different selection"));
    const missing = "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd";
    try std.testing.expectError(error.FileNotFound, verify(gpa, arena, io, &env, &store, formal, missing, config_id, trial_id, "admission", "parent prompt"));

    const slow_script = "sleep 2\n";
    {
        const file = try tmp.dir.createFile(io, "helper.sh", .{ .truncate = true, .permissions = .fromMode(0o600) });
        defer file.close(io);
        try file.writeStreamingAll(io, slow_script);
    }
    const slow_hash = store_mod.rawSha256(slow_script);
    const slow_inputs = [_]store_mod.PinnedFile{.{ .path = helper_path, .sha256 = &slow_hash }};
    var slow_formal = formal;
    slow_formal.checker.inputs = &slow_inputs;
    slow_formal.timeout_ms = 50;
    try std.testing.expectError(error.FormalCheckTimedOut, check(gpa, arena, io, &env, slow_formal, "parent prompt"));
}