Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
11.5 KB
Raw Blame
//! #352 anti-fabrication verification for the `imagegen` tool.
//!
//! Why this module exists at all: during the #352 investigation a model was
//! asked to generate a 64x64 PNG, could not (the hosted `image_gen` tool is
//! server-side and never fires outside the Codex app), and reported success
//! anyway — it globbed a two-week-old file out of ~/.codex/generated_images/,
//! resized it with `sips`, and wrote "Verified it is a 64x64 PNG". Every part
//! of that sentence was true except the part that mattered: the image was not
//! generated by that turn. An md5 comparison is what caught it.
//!
//! So a successful `imagegen` result is not a claim, it is a receipt. The
//! generator's exit code is necessary and nowhere near sufficient; the file it
//! points at has to independently prove it is a NEW artifact:
//!   * it exists;
//!   * its mtime is at or after the moment we spawned the generator (a stale
//!     glob fails here — this is the check that would have caught #352);
//!   * if it existed before the run, its size or mtime actually moved (a
//!     no-op rewrite of the same bytes is not a generation);
//!   * it is bigger than a floor no real image undercuts (a 0-byte or
//!     truncated download is not an image);
//!   * its leading bytes match the container the caller asked for (a JSON
//!     error body saved to `out.png` is not a PNG).
//! Any single failure is a tool error naming the failed check. There is no
//! path to a success result that skips one.
//!
//! Everything here is pure over metadata + the file's first bytes, so the
//! whole decision table is unit-testable with synthetic values and never
//! needs a network, an API key, or a real generator.

const std = @import("std");
const Io = std.Io;

/// Below this a "generated image" is a truncated download or an error page.
pub const min_bytes: u64 = 1024;

/// Filesystem mtime granularity varies (APFS is nanoseconds, some network and
/// FAT-derived filesystems round to 1-2 seconds), so the freshness comparison
/// gets a small slack rather than risking a false "stale" on a file that WAS
/// just written. The #352 artifact was two weeks old; the `unchanged` check
/// below covers anything that lands inside the slack window.
pub const mtime_slack_ns: i128 = 2 * std.time.ns_per_s;

pub const Format = enum {
    png,
    jpeg,
    webp,

    /// An explicit `output_format` argument (the CLI's spelling).
    pub fn fromName(name: []const u8) ?Format {
        if (std.ascii.eqlIgnoreCase(name, "png")) return .png;
        if (std.ascii.eqlIgnoreCase(name, "jpeg") or std.ascii.eqlIgnoreCase(name, "jpg")) return .jpeg;
        if (std.ascii.eqlIgnoreCase(name, "webp")) return .webp;
        return null;
    }

    /// The container implied by the output path's extension. PNG is the
    /// default because it is the CLI's default output format.
    pub fn fromPath(path: []const u8) Format {
        const dot = std.mem.lastIndexOfScalar(u8, path, '.') orelse return .png;
        return fromName(path[dot + 1 ..]) orelse .png;
    }

    /// Container signature check against the file's leading bytes.
    pub fn magicOk(self: Format, head: []const u8) bool {
        return switch (self) {
            .png => head.len >= 8 and std.mem.eql(u8, head[0..8], "\x89PNG\r\n\x1a\n"),
            .jpeg => head.len >= 3 and std.mem.eql(u8, head[0..3], "\xff\xd8\xff"),
            // RIFF....WEBP — the 4 bytes between the two tags are the length.
            .webp => head.len >= 12 and std.mem.eql(u8, head[0..4], "RIFF") and std.mem.eql(u8, head[8..12], "WEBP"),
        };
    }

    pub fn label(self: Format) []const u8 {
        return switch (self) {
            .png => "png",
            .jpeg => "jpeg",
            .webp => "webp",
        };
    }

    /// What the bytes actually ARE, as opposed to what a path claims. The
    /// codex engine does not let the caller choose a container, so the honest
    /// thing to report is whatever the hosted tool produced.
    pub fn detect(head: []const u8) ?Format {
        for ([_]Format{ .png, .jpeg, .webp }) |f| {
            if (f.magicOk(head)) return f;
        }
        return null;
    }
};

/// What we knew about the output path at one instant. `existed = false` is the
/// normal case (a fresh default path); the other fields are then meaningless.
pub const Snapshot = struct {
    existed: bool = false,
    size: u64 = 0,
    mtime_ns: i128 = 0,
};

/// Read the output path's current metadata. Absent is a valid answer, not an
/// error — it is the expected state before a generation.
pub fn snapshot(io: Io, path: []const u8) Snapshot {
    const st = Io.Dir.cwd().statFile(io, path, .{}) catch return .{};
    return .{ .existed = true, .size = st.size, .mtime_ns = st.mtime.nanoseconds };
}

pub const Verdict = enum {
    ok,
    /// The generator exited 0 and wrote nothing at the path it was given.
    missing,
    /// The file predates the run — the #352 stale-glob signature.
    stale,
    /// The pre-existing file is byte-for-byte where it was.
    unchanged,
    /// Too small to be an image (truncated download / error page).
    too_small,
    /// Leading bytes are not the requested container.
    bad_magic,

    pub fn failedCheck(self: Verdict) []const u8 {
        return switch (self) {
            .ok => "none",
            .missing => "output-file-exists",
            .stale => "output-file-is-fresh (mtime at or after spawn)",
            .unchanged => "output-file-changed (pre-existing file was not rewritten)",
            .too_small => "output-file-size-floor",
            .bad_magic => "output-file-magic-bytes",
        };
    }

    pub fn detail(self: Verdict) []const u8 {
        return switch (self) {
            .ok => "",
            .missing => "the generator exited 0 but no file exists at the requested path, so nothing was generated",
            .stale => "the file at that path is OLDER than this run — it is a pre-existing artifact, not a newly generated image (this is exactly the #352 failure: a stale file passed off as a fresh generation)",
            .unchanged => "the file at that path existed before this run and its size and mtime are unchanged, so this run did not write it",
            .too_small => "the file is far too small to be a real image — most likely a truncated download or an error body",
            .bad_magic => "the file's leading bytes are not the requested image container, so whatever was written is not that image format",
        };
    }
};

/// The whole decision, in order. `after == null` means the path does not exist
/// after the run. `head` is the file's first bytes (>= 12 is enough for every
/// signature we check).
pub fn evaluate(before: Snapshot, after: ?Snapshot, started_ns: i128, head: []const u8, fmt: Format) Verdict {
    const now = after orelse return .missing;
    if (now.mtime_ns + mtime_slack_ns < started_ns) return .stale;
    if (before.existed and now.size == before.size and now.mtime_ns == before.mtime_ns) return .unchanged;
    if (now.size < min_bytes) return .too_small;
    if (!fmt.magicOk(head)) return .bad_magic;
    return .ok;
}

const png_head = "\x89PNG\r\n\x1a\n" ++ "IHDR-ish padding";
const spawn_ns: i128 = 1_000_000_000_000;

test "#352: a fresh, big-enough, correctly-signed file is the only thing that passes" {
    const fresh: Snapshot = .{ .existed = true, .size = 8192, .mtime_ns = spawn_ns + 5_000_000 };
    try std.testing.expectEqual(Verdict.ok, evaluate(.{}, fresh, spawn_ns, png_head, .png));
}

test "#352: a stale glob is rejected even though it is a real, valid PNG" {
    // The actual #352 artifact: a genuine PNG, correct magic, plenty big —
    // and two weeks old. Exit code and file-type checks both pass; only the
    // freshness check catches it.
    const two_weeks_ns: i128 = 14 * 24 * 60 * 60 * std.time.ns_per_s;
    const stale: Snapshot = .{ .existed = true, .size = 8279, .mtime_ns = spawn_ns - two_weeks_ns };
    try std.testing.expectEqual(Verdict.stale, evaluate(.{}, stale, spawn_ns, png_head, .png));
    // ...and a file written a hair before the spawn still counts as fresh,
    // because filesystem mtime granularity is not ours to trust.
    const borderline: Snapshot = .{ .existed = true, .size = 8279, .mtime_ns = spawn_ns - std.time.ns_per_s };
    try std.testing.expectEqual(Verdict.ok, evaluate(.{}, borderline, spawn_ns, png_head, .png));
}

test "#352: a pre-existing file that did not move is not a generation" {
    const before: Snapshot = .{ .existed = true, .size = 4096, .mtime_ns = spawn_ns + 1 };
    // Same size, same mtime: the generator did not touch it. (mtime is inside
    // the slack window, so the freshness check alone would have let it pass.)
    try std.testing.expectEqual(Verdict.unchanged, evaluate(before, before, spawn_ns, png_head, .png));
    // One byte different is enough to prove a rewrite.
    const rewritten: Snapshot = .{ .existed = true, .size = 4097, .mtime_ns = spawn_ns + 1 };
    try std.testing.expectEqual(Verdict.ok, evaluate(before, rewritten, spawn_ns, png_head, .png));
}

test "#352: missing, undersized and wrong-container outputs each fail their own check" {
    try std.testing.expectEqual(Verdict.missing, evaluate(.{}, null, spawn_ns, "", .png));

    const tiny: Snapshot = .{ .existed = true, .size = 64, .mtime_ns = spawn_ns + 1 };
    try std.testing.expectEqual(Verdict.too_small, evaluate(.{}, tiny, spawn_ns, png_head, .png));

    const big: Snapshot = .{ .existed = true, .size = 9000, .mtime_ns = spawn_ns + 1 };
    // An OpenAI JSON error body saved to out.png: fresh, big, and not a PNG.
    try std.testing.expectEqual(Verdict.bad_magic, evaluate(.{}, big, spawn_ns, "{\"error\": {\"message\"", .png));
    // A real PNG when JPEG was asked for is equally wrong.
    try std.testing.expectEqual(Verdict.bad_magic, evaluate(.{}, big, spawn_ns, png_head, .jpeg));

    try std.testing.expect(std.mem.indexOf(u8, Verdict.stale.detail(), "#352") != null);
    try std.testing.expectEqualStrings("none", Verdict.ok.failedCheck());
}

test "#352: container signatures and the format each output path implies" {
    try std.testing.expect(Format.png.magicOk(png_head));
    try std.testing.expect(!Format.png.magicOk("\x89PNG")); // truncated signature
    try std.testing.expect(Format.jpeg.magicOk("\xff\xd8\xff\xe0JFIF"));
    try std.testing.expect(Format.webp.magicOk("RIFF\x24\x00\x00\x00WEBPVP8 "));
    try std.testing.expect(!Format.webp.magicOk("RIFF\x24\x00\x00\x00WAVEfmt ")); // RIFF, wrong payload

    try std.testing.expectEqual(Format.png, Format.fromPath("out/render.png"));
    try std.testing.expectEqual(Format.jpeg, Format.fromPath("a.JPG"));
    try std.testing.expectEqual(Format.webp, Format.fromPath("dir.png/thing.webp"));
    try std.testing.expectEqual(Format.png, Format.fromPath("no-extension")); // CLI default
    try std.testing.expectEqual(Format.jpeg, Format.fromName("jpeg").?);
    try std.testing.expect(Format.fromName("gif") == null);
}

test "#352: snapshot reports absence as absence and reads back a real file's size" {
    const io = std.testing.io;
    var tmp = std.testing.tmpDir(.{});
    defer tmp.cleanup();
    var real_buf: [std.fs.max_path_bytes]u8 = undefined;
    const dir_path = real_buf[0..try tmp.dir.realPath(io, &real_buf)];
    const path = try std.fmt.allocPrint(std.testing.allocator, "{s}/shot.png", .{dir_path});
    defer std.testing.allocator.free(path);

    try std.testing.expect(!snapshot(io, path).existed);
    try Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = png_head });
    const after = snapshot(io, path);
    try std.testing.expect(after.existed);
    try std.testing.expectEqual(@as(u64, png_head.len), after.size);
}