graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
//! #352 anti-fabrication verification for the `imagegen` tool.
//!
//! Why this module exists at all: during the #352 investigation a model was
//! asked to generate a 64x64 PNG, could not (the hosted `image_gen` tool is
//! server-side and never fires outside the Codex app), and reported success
//! anyway — it globbed a two-week-old file out of ~/.codex/generated_images/,
//! resized it with `sips`, and wrote "Verified it is a 64x64 PNG". Every part
//! of that sentence was true except the part that mattered: the image was not
//! generated by that turn. An md5 comparison is what caught it.
//!
//! So a successful `imagegen` result is not a claim, it is a receipt. The
//! generator's exit code is necessary and nowhere near sufficient; the file it
//! points at has to independently prove it is a NEW artifact:
//! * it exists;
//! * its mtime is at or after the moment we spawned the generator (a stale
//! glob fails here — this is the check that would have caught #352);
//! * if it existed before the run, its size or mtime actually moved (a
//! no-op rewrite of the same bytes is not a generation);
//! * it is bigger than a floor no real image undercuts (a 0-byte or
//! truncated download is not an image);
//! * its leading bytes match the container the caller asked for (a JSON
//! error body saved to `out.png` is not a PNG).
//! Any single failure is a tool error naming the failed check. There is no
//! path to a success result that skips one.
//!
//! Everything here is pure over metadata + the file's first bytes, so the
//! whole decision table is unit-testable with synthetic values and never
//! needs a network, an API key, or a real generator.
const std = @import("std");
const Io = std.Io;
/// Below this a "generated image" is a truncated download or an error page.
pub const min_bytes: u64 = 1024;
/// Filesystem mtime granularity varies (APFS is nanoseconds, some network and
/// FAT-derived filesystems round to 1-2 seconds), so the freshness comparison
/// gets a small slack rather than risking a false "stale" on a file that WAS
/// just written. The #352 artifact was two weeks old; the `unchanged` check
/// below covers anything that lands inside the slack window.
pub const mtime_slack_ns: i128 = 2 * std.time.ns_per_s;
pub const Format = enum {
png,
jpeg,
webp,
/// An explicit `output_format` argument (the CLI's spelling).
pub fn fromName(name: []const u8) ?Format {
if (std.ascii.eqlIgnoreCase(name, "png")) return .png;
if (std.ascii.eqlIgnoreCase(name, "jpeg") or std.ascii.eqlIgnoreCase(name, "jpg")) return .jpeg;
if (std.ascii.eqlIgnoreCase(name, "webp")) return .webp;
return null;
}
/// The container implied by the output path's extension. PNG is the
/// default because it is the CLI's default output format.
pub fn fromPath(path: []const u8) Format {
const dot = std.mem.lastIndexOfScalar(u8, path, '.') orelse return .png;
return fromName(path[dot + 1 ..]) orelse .png;
}
/// Container signature check against the file's leading bytes.
pub fn magicOk(self: Format, head: []const u8) bool {
return switch (self) {
.png => head.len >= 8 and std.mem.eql(u8, head[0..8], "\x89PNG\r\n\x1a\n"),
.jpeg => head.len >= 3 and std.mem.eql(u8, head[0..3], "\xff\xd8\xff"),
// RIFF....WEBP — the 4 bytes between the two tags are the length.
.webp => head.len >= 12 and std.mem.eql(u8, head[0..4], "RIFF") and std.mem.eql(u8, head[8..12], "WEBP"),
};
}
pub fn label(self: Format) []const u8 {
return switch (self) {
.png => "png",
.jpeg => "jpeg",
.webp => "webp",
};
}
/// What the bytes actually ARE, as opposed to what a path claims. The
/// codex engine does not let the caller choose a container, so the honest
/// thing to report is whatever the hosted tool produced.
pub fn detect(head: []const u8) ?Format {
for ([_]Format{ .png, .jpeg, .webp }) |f| {
if (f.magicOk(head)) return f;
}
return null;
}
};
/// What we knew about the output path at one instant. `existed = false` is the
/// normal case (a fresh default path); the other fields are then meaningless.
pub const Snapshot = struct {
existed: bool = false,
size: u64 = 0,
mtime_ns: i128 = 0,
};
/// Read the output path's current metadata. Absent is a valid answer, not an
/// error — it is the expected state before a generation.
pub fn snapshot(io: Io, path: []const u8) Snapshot {
const st = Io.Dir.cwd().statFile(io, path, .{}) catch return .{};
return .{ .existed = true, .size = st.size, .mtime_ns = st.mtime.nanoseconds };
}
pub const Verdict = enum {
ok,
/// The generator exited 0 and wrote nothing at the path it was given.
missing,
/// The file predates the run — the #352 stale-glob signature.
stale,
/// The pre-existing file is byte-for-byte where it was.
unchanged,
/// Too small to be an image (truncated download / error page).
too_small,
/// Leading bytes are not the requested container.
bad_magic,
pub fn failedCheck(self: Verdict) []const u8 {
return switch (self) {
.ok => "none",
.missing => "output-file-exists",
.stale => "output-file-is-fresh (mtime at or after spawn)",
.unchanged => "output-file-changed (pre-existing file was not rewritten)",
.too_small => "output-file-size-floor",
.bad_magic => "output-file-magic-bytes",
};
}
pub fn detail(self: Verdict) []const u8 {
return switch (self) {
.ok => "",
.missing => "the generator exited 0 but no file exists at the requested path, so nothing was generated",
.stale => "the file at that path is OLDER than this run — it is a pre-existing artifact, not a newly generated image (this is exactly the #352 failure: a stale file passed off as a fresh generation)",
.unchanged => "the file at that path existed before this run and its size and mtime are unchanged, so this run did not write it",
.too_small => "the file is far too small to be a real image — most likely a truncated download or an error body",
.bad_magic => "the file's leading bytes are not the requested image container, so whatever was written is not that image format",
};
}
};
/// The whole decision, in order. `after == null` means the path does not exist
/// after the run. `head` is the file's first bytes (>= 12 is enough for every
/// signature we check).
pub fn evaluate(before: Snapshot, after: ?Snapshot, started_ns: i128, head: []const u8, fmt: Format) Verdict {
const now = after orelse return .missing;
if (now.mtime_ns + mtime_slack_ns < started_ns) return .stale;
if (before.existed and now.size == before.size and now.mtime_ns == before.mtime_ns) return .unchanged;
if (now.size < min_bytes) return .too_small;
if (!fmt.magicOk(head)) return .bad_magic;
return .ok;
}
const png_head = "\x89PNG\r\n\x1a\n" ++ "IHDR-ish padding";
const spawn_ns: i128 = 1_000_000_000_000;
test "#352: a fresh, big-enough, correctly-signed file is the only thing that passes" {
const fresh: Snapshot = .{ .existed = true, .size = 8192, .mtime_ns = spawn_ns + 5_000_000 };
try std.testing.expectEqual(Verdict.ok, evaluate(.{}, fresh, spawn_ns, png_head, .png));
}
test "#352: a stale glob is rejected even though it is a real, valid PNG" {
// The actual #352 artifact: a genuine PNG, correct magic, plenty big —
// and two weeks old. Exit code and file-type checks both pass; only the
// freshness check catches it.
const two_weeks_ns: i128 = 14 * 24 * 60 * 60 * std.time.ns_per_s;
const stale: Snapshot = .{ .existed = true, .size = 8279, .mtime_ns = spawn_ns - two_weeks_ns };
try std.testing.expectEqual(Verdict.stale, evaluate(.{}, stale, spawn_ns, png_head, .png));
// ...and a file written a hair before the spawn still counts as fresh,
// because filesystem mtime granularity is not ours to trust.
const borderline: Snapshot = .{ .existed = true, .size = 8279, .mtime_ns = spawn_ns - std.time.ns_per_s };
try std.testing.expectEqual(Verdict.ok, evaluate(.{}, borderline, spawn_ns, png_head, .png));
}
test "#352: a pre-existing file that did not move is not a generation" {
const before: Snapshot = .{ .existed = true, .size = 4096, .mtime_ns = spawn_ns + 1 };
// Same size, same mtime: the generator did not touch it. (mtime is inside
// the slack window, so the freshness check alone would have let it pass.)
try std.testing.expectEqual(Verdict.unchanged, evaluate(before, before, spawn_ns, png_head, .png));
// One byte different is enough to prove a rewrite.
const rewritten: Snapshot = .{ .existed = true, .size = 4097, .mtime_ns = spawn_ns + 1 };
try std.testing.expectEqual(Verdict.ok, evaluate(before, rewritten, spawn_ns, png_head, .png));
}
test "#352: missing, undersized and wrong-container outputs each fail their own check" {
try std.testing.expectEqual(Verdict.missing, evaluate(.{}, null, spawn_ns, "", .png));
const tiny: Snapshot = .{ .existed = true, .size = 64, .mtime_ns = spawn_ns + 1 };
try std.testing.expectEqual(Verdict.too_small, evaluate(.{}, tiny, spawn_ns, png_head, .png));
const big: Snapshot = .{ .existed = true, .size = 9000, .mtime_ns = spawn_ns + 1 };
// An OpenAI JSON error body saved to out.png: fresh, big, and not a PNG.
try std.testing.expectEqual(Verdict.bad_magic, evaluate(.{}, big, spawn_ns, "{\"error\": {\"message\"", .png));
// A real PNG when JPEG was asked for is equally wrong.
try std.testing.expectEqual(Verdict.bad_magic, evaluate(.{}, big, spawn_ns, png_head, .jpeg));
try std.testing.expect(std.mem.indexOf(u8, Verdict.stale.detail(), "#352") != null);
try std.testing.expectEqualStrings("none", Verdict.ok.failedCheck());
}
test "#352: container signatures and the format each output path implies" {
try std.testing.expect(Format.png.magicOk(png_head));
try std.testing.expect(!Format.png.magicOk("\x89PNG")); // truncated signature
try std.testing.expect(Format.jpeg.magicOk("\xff\xd8\xff\xe0JFIF"));
try std.testing.expect(Format.webp.magicOk("RIFF\x24\x00\x00\x00WEBPVP8 "));
try std.testing.expect(!Format.webp.magicOk("RIFF\x24\x00\x00\x00WAVEfmt ")); // RIFF, wrong payload
try std.testing.expectEqual(Format.png, Format.fromPath("out/render.png"));
try std.testing.expectEqual(Format.jpeg, Format.fromPath("a.JPG"));
try std.testing.expectEqual(Format.webp, Format.fromPath("dir.png/thing.webp"));
try std.testing.expectEqual(Format.png, Format.fromPath("no-extension")); // CLI default
try std.testing.expectEqual(Format.jpeg, Format.fromName("jpeg").?);
try std.testing.expect(Format.fromName("gif") == null);
}
test "#352: snapshot reports absence as absence and reads back a real file's size" {
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var real_buf: [std.fs.max_path_bytes]u8 = undefined;
const dir_path = real_buf[0..try tmp.dir.realPath(io, &real_buf)];
const path = try std.fmt.allocPrint(std.testing.allocator, "{s}/shot.png", .{dir_path});
defer std.testing.allocator.free(path);
try std.testing.expect(!snapshot(io, path).existed);
try Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = png_head });
const after = snapshot(io, path);
try std.testing.expect(after.existed);
try std.testing.expectEqual(@as(u64, png_head.len), after.size);
}