Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Type staged images by their bytes, not their file extension

Rach Pradhan461e45af8b29
diff --git a/src/acp_images.zig b/src/acp_images.zig
index b7028a46..be8b2b54 100644
--- a/src/acp_images.zig
+++ b/src/acp_images.zig
@@ -88,6 +88,13 @@ test "stage: a codex model receives ACP images as the next request's input_image
try std.testing.expect(std.mem.indexOf(u8, wire, vision_queue.attached_note) != null); // image-only prompt
try std.testing.expectEqual(@as(u8, 0), codex.pending_image_len);
+ // #1762: WebP bytes labelled image/png go out as image/webp.
+ const mislabeled = try std.json.parseFromSliceLeaky(Value, a,
+ \\[{"type":"image","data":"UklGRiQAAABXRUJQVlA4IA==","mimeType":"image/png"}]
+ , .{});
+ try std.testing.expectEqual(@as(usize, 1), stage(&codex, mislabeled));
+ try std.testing.expectEqualStrings("image/webp", codex.pending_image.?.media_type);
+
var text_only = testAgent(a, "openrouter", "qwen3.8-27b");
try std.testing.expectEqual(@as(usize, 0), stage(&text_only, prompt));
try std.testing.expectEqual(@as(u8, 0), text_only.pending_image_len);
diff --git a/src/commands_model.zig b/src/commands_model.zig
index 1089a0e9..22e5cd08 100644
--- a/src/commands_model.zig
+++ b/src/commands_model.zig
@@ -472,7 +472,7 @@ pub fn tryHandle(root: *Agent, keys: *Keys, arena: Allocator, line: []const u8,
.no_vision => try out.print("⚠ {s} can't see images — switch to a vision model first, e.g. /model claude-opus-4-8 or /model gpt-6-sol\n", .{root.provider.model}),
.ok => |o| try out.print("📎 attached {s} ({s}) — sent with your next message\n", .{ path, vision.fmtBytes(mbuf[0..16], o.bytes) }),
// Sized, distinct lines instead of "missing, or larger than 5MB" (#349).
- .too_large, .not_found, .read_error => try out.print("{s}\n", .{vision.stageMessage(&mbuf, staged, path)}),
+ .too_large, .not_found, .read_error, .unsupported => try out.print("{s}\n", .{vision.stageMessage(&mbuf, staged, path)}),
}
try out.flush();
return true;
diff --git a/src/commands_paste.zig b/src/commands_paste.zig
index 27e01a6e..66b65a73 100644
--- a/src/commands_paste.zig
+++ b/src/commands_paste.zig
@@ -38,7 +38,7 @@ pub fn tryHandle(root: *Agent, line: []const u8, out: *Io.Writer) !bool {
switch (pasted) {
.ok => |o| try out.print("📎 clipboard image attached ({s}, via {s}) — sent with your next message\n", .{ vision.fmtBytes(pbuf[0..16], o.bytes), grab.flavor.name() }),
.no_vision => try out.print("⚠ {s} can't see images\n", .{root.provider.model}),
- .too_large, .not_found, .read_error => try out.print("{s}\n", .{vision.stageMessage(&pbuf, pasted, "the clipboard image")}),
+ .too_large, .not_found, .read_error, .unsupported => try out.print("{s}\n", .{vision.stageMessage(&pbuf, pasted, "the clipboard image")}),
}
},
}
diff --git a/src/image_sniff.zig b/src/image_sniff.zig
new file mode 100644
index 00000000..52a066f9
--- /dev/null
+++ b/src/image_sniff.zig
@@ -0,0 +1,77 @@
+//! An image's media type from its bytes, not its name (#1762).
+//!
+//! Providers check the declared media type against the pixels and reject a
+//! mismatch, and the rejected block stays in the transcript, so every later
+//! turn fails the same way. A WebP saved as `shot.png` used to go out as
+//! `image/png` because the type came from the file extension. Every path that
+//! holds the bytes asks here first; the four types below are the raster
+//! formats every vision provider accepts.
+
+const std = @import("std");
+
+/// The formats `sniff` recognizes, for error text.
+pub const supported_names = "PNG, JPEG, GIF, or WebP";
+
+/// The media type the bytes actually are, or null when they are none of the
+/// four supported raster formats (BMP, TIFF, HEIC, SVG, text, truncated...).
+pub fn sniff(bytes: []const u8) ?[]const u8 {
+ if (std.mem.startsWith(u8, bytes, "\x89PNG\r\n\x1a\n")) return "image/png";
+ if (std.mem.startsWith(u8, bytes, "\xff\xd8\xff")) return "image/jpeg";
+ if (std.mem.startsWith(u8, bytes, "GIF87a") or std.mem.startsWith(u8, bytes, "GIF89a")) return "image/gif";
+ if (bytes.len >= 12 and std.mem.eql(u8, bytes[0..4], "RIFF") and std.mem.eql(u8, bytes[8..12], "WEBP")) return "image/webp";
+ return null;
+}
+
+/// `sniff` on base64 text, decoding only the 12-byte head every signature
+/// fits in. Null for unrecognized bytes or base64 that does not decode.
+pub fn sniffBase64(b64: []const u8) ?[]const u8 {
+ const dec = std.base64.standard.Decoder;
+ var head: [12]u8 = undefined;
+ // 16 base64 chars are exactly 12 bytes with no padding; shorter input
+ // decodes whole (its own padding included).
+ const text = b64[0..@min(b64.len, 16)];
+ const n = dec.calcSizeForSlice(text) catch return null;
+ if (n > head.len) return null;
+ dec.decode(head[0..n], text) catch return null;
+ return sniff(head[0..n]);
+}
+
+/// The type to send for pixels that arrived with a declared type (an MCP or
+/// ACP `mimeType`, a request `media_type`): the bytes win when they are a
+/// supported format, and the declaration stands only when they are not.
+pub fn resolveDeclared(b64: []const u8, declared: []const u8) []const u8 {
+ return sniffBase64(b64) orelse declared;
+}
+
+const webp_head = "RIFF\x24\x00\x00\x00WEBPVP8 ";
+
+test "sniff: each supported format by its signature, whatever the file is called" {
+ try std.testing.expectEqualStrings("image/png", sniff("\x89PNG\r\n\x1a\n\x00\x00").?);
+ try std.testing.expectEqualStrings("image/jpeg", sniff("\xff\xd8\xff\xe0\x00\x10JFIF").?);
+ try std.testing.expectEqualStrings("image/gif", sniff("GIF89a\x01\x00").?);
+ try std.testing.expectEqualStrings("image/gif", sniff("GIF87a\x01\x00").?);
+ try std.testing.expectEqualStrings("image/webp", sniff(webp_head).?);
+}
+
+test "sniff: unsupported or truncated bytes are not guessed" {
+ try std.testing.expect(sniff("") == null);
+ try std.testing.expect(sniff("BM\x36\x00\x00\x00") == null); // BMP
+ try std.testing.expect(sniff("II*\x00\x08\x00\x00\x00") == null); // TIFF
+ try std.testing.expect(sniff("\x00\x00\x00\x18ftypheic") == null); // HEIC
+ try std.testing.expect(sniff("<svg xmlns=") == null);
+ try std.testing.expect(sniff("RIFF\x24\x00\x00\x00WAVE") == null); // RIFF, but audio
+ try std.testing.expect(sniff("RIFF\x24\x00") == null); // truncated WebP
+ try std.testing.expect(sniff("\x89PNG") == null); // truncated PNG
+}
+
+test "#1762: WebP bytes declared as image/png resolve to image/webp" {
+ var b64: [std.base64.standard.Encoder.calcSize(webp_head.len)]u8 = undefined;
+ _ = std.base64.standard.Encoder.encode(&b64, webp_head);
+ try std.testing.expectEqualStrings("image/webp", sniffBase64(&b64).?);
+ try std.testing.expectEqualStrings("image/webp", resolveDeclared(&b64, "image/png"));
+ // A real PNG keeps its type; bytes we cannot place keep the declaration.
+ try std.testing.expectEqualStrings("image/png", resolveDeclared("iVBORw0KGgo=", "image/jpeg"));
+ try std.testing.expectEqualStrings("image/png", resolveDeclared("aGVsbG8=", "image/png"));
+ try std.testing.expect(sniffBase64("!!!!") == null);
+ try std.testing.expect(sniffBase64("") == null);
+}
diff --git a/src/read_image.zig b/src/read_image.zig
index a1adff14..0bac27bb 100644
--- a/src/read_image.zig
+++ b/src/read_image.zig
@@ -15,6 +15,7 @@ const ToolCtx = tools.ToolCtx;
const ToolOutput = tools.ToolOutput;
const vision = @import("vision.zig");
const clip = @import("vision_clipboard.zig");
+const sniff = @import("image_sniff.zig");
/// Stage pixels after exec has confined the path. The tool-batch checkpoint
/// delivers them to the next model request as a native image content block.
@@ -29,21 +30,35 @@ pub fn pngDims(bytes: []const u8) ?[2]u32 {
return .{ std.mem.readInt(u32, bytes[16..20], .big), std.mem.readInt(u32, bytes[20..24], .big) };
}
-fn fileDims(io: Io, path: []const u8) ?[2]u32 {
+/// The first bytes of `path`: enough for a PNG's dimensions and any
+/// `image_sniff` signature. Null when the file cannot be read.
+fn fileHead(io: Io, path: []const u8, buf: *[24]u8) ?[]const u8 {
const file = Io.Dir.cwd().openFile(io, path, .{}) catch return null;
defer file.close(io);
- var head: [24]u8 = undefined;
- const got = file.readPositionalAll(io, &head, 0) catch return null;
- return pngDims(head[0..got]);
+ const got = file.readPositionalAll(io, buf, 0) catch return null;
+ return buf[0..got];
+}
+
+fn fileDims(io: Io, path: []const u8) ?[2]u32 {
+ var buf: [24]u8 = undefined;
+ return pngDims(fileHead(io, path, &buf) orelse return null);
+}
+
+fn notRaster(gpa: Allocator, path: []const u8) !?ToolOutput {
+ return .{ .text = try std.fmt.allocPrint(gpa, "Image could not be attached: {s} is not a {s} image (its bytes match none of them, whatever its extension says), so it was not sent. Convert it to one of those formats first.", .{ path, sniff.supported_names }), .is_error = true };
}
fn stageWith(gpa: Allocator, io: Io, ctx: ToolCtx, resolved: []const u8, path: []const u8, size: u64, resize: clip.Resizer) !?ToolOutput {
const reg = ctx.registry orelse return .{ .text = try gpa.dupe(u8, "Image could not be attached: the image queue is unavailable."), .is_error = true };
if (size == 0) return .{ .text = try gpa.dupe(u8, "Image could not be attached: the file is empty."), .is_error = true };
- const media_type = vision.imageMediaType(path);
+ // #1762: the type comes from the bytes. The extension is only a label for
+ // a model that will not see the pixels anyway.
+ var head_buf: [24]u8 = undefined;
+ const sniffed = sniff.sniff(fileHead(io, resolved, &head_buf) orelse return null);
if (!vision.visionCapable(ctx.provider)) return .{
- .text = try std.fmt.allocPrint(gpa, "[image: {s}, {d} bytes — the active model does not accept images, so it was not attached]", .{ media_type, size }),
+ .text = try std.fmt.allocPrint(gpa, "[image: {s}, {d} bytes — the active model does not accept images, so it was not attached]", .{ sniffed orelse vision.imageMediaType(path), size }),
};
+ const media_type = sniffed orelse return notRaster(gpa, path);
const limit = clip.max_staged_image_bytes;
const fit = switch (clip.planStage(io, gpa, resolved, limit, resize)) {
.not_found => return null,
@@ -58,9 +73,9 @@ fn stageWith(gpa: Allocator, io: Io, ctx: ToolCtx, resolved: []const u8, path: [
defer if (fit.temp) clip.discard(io, gpa, fit.path);
const data = Io.Dir.cwd().readFileAlloc(io, fit.path, gpa, .limited(@intCast(limit))) catch return null;
defer gpa.free(data);
- // A downscale step is always PNG (sipsResize forces it, fitToBudget checks
- // the magic), so the media type follows the file actually encoded.
- const sent_type = if (fit.temp) vision.imageMediaType(fit.path) else media_type;
+ // The type of the bytes actually encoded: a downscale step re-encodes
+ // (sipsResize forces PNG), so it is sniffed again rather than assumed.
+ const sent_type = sniff.sniff(data) orelse return notRaster(gpa, path);
const enc = std.base64.standard.Encoder;
const b64 = try gpa.alloc(u8, enc.calcSize(data.len));
defer gpa.free(b64);
@@ -188,3 +203,37 @@ test "#1272: an oversize image is downscaled or refused, never queued as-is" {
try std.testing.expectEqualStrings("image/png", pending.media_type);
try std.testing.expect(std.mem.indexOf(u8, scaled.text, "downscaled from 4000x3000 to 1x1") != null);
}
+
+test "#1762: read_image types a WebP named .png by its bytes, and refuses non-raster bytes" {
+ const a = std.testing.allocator;
+ const io = std.testing.io;
+ var reg = @import("mcp.zig").Registry.empty(a, io);
+ defer reg.deinit();
+ var tmp = std.testing.tmpDir(.{});
+ defer tmp.cleanup();
+ var ctx: ToolCtx = undefined;
+ ctx.registry = &reg;
+ ctx.provider = .{ .id = "test", .kind = .anthropic, .auth = .bearer, .url = "", .api_key = "", .model = "claude-x", .context = 100000 };
+ const webp = "RIFF\x24\x00\x00\x00WEBPVP8 \x18\x00\x00\x00";
+ try tmp.dir.writeFile(io, .{ .sub_path = "shot.png", .data = webp });
+ const webp_path = try std.fmt.allocPrint(a, ".zig-cache/tmp/{s}/shot.png", .{tmp.sub_path});
+ defer a.free(webp_path);
+ const out = (try stageWith(a, io, ctx, webp_path, "shot.png", webp.len, noResizer)).?;
+ defer a.free(out.text);
+ try std.testing.expect(!out.is_error);
+ try std.testing.expect(std.mem.indexOf(u8, out.text, "image/webp") != null);
+ const pending = reg.pending_image orelse return error.TestUnexpectedResult;
+ try std.testing.expectEqualStrings("image/webp", pending.media_type);
+ reg.pending_image = null;
+
+ // Text saved as .png: a tool error, and no block that would wedge the chat.
+ const text = "not an image at all";
+ try tmp.dir.writeFile(io, .{ .sub_path = "notes.png", .data = text });
+ const text_path = try std.fmt.allocPrint(a, ".zig-cache/tmp/{s}/notes.png", .{tmp.sub_path});
+ defer a.free(text_path);
+ const refused = (try stageWith(a, io, ctx, text_path, "notes.png", text.len, noResizer)).?;
+ defer a.free(refused.text);
+ try std.testing.expect(refused.is_error);
+ try std.testing.expect(std.mem.indexOf(u8, refused.text, sniff.supported_names) != null);
+ try std.testing.expect(reg.pending_image == null);
+}
diff --git a/src/readline_data_uri.zig b/src/readline_data_uri.zig
index bd9059a8..98106fce 100644
--- a/src/readline_data_uri.zig
+++ b/src/readline_data_uri.zig
@@ -29,11 +29,8 @@ pub fn feedback(r: Result) []const u8 {
}
fn matchesMagic(mime: []const u8, bytes: []const u8) bool {
- if (std.mem.eql(u8, mime, "image/png")) return std.mem.startsWith(u8, bytes, "\x89PNG\r\n\x1a\n");
- if (std.mem.eql(u8, mime, "image/jpeg")) return std.mem.startsWith(u8, bytes, "\xff\xd8\xff");
- if (std.mem.eql(u8, mime, "image/gif")) return std.mem.startsWith(u8, bytes, "GIF87a") or std.mem.startsWith(u8, bytes, "GIF89a");
- if (std.mem.eql(u8, mime, "image/webp")) return bytes.len >= 12 and std.mem.eql(u8, bytes[0..4], "RIFF") and std.mem.eql(u8, bytes[8..12], "WEBP");
- return false;
+ const actual = @import("image_sniff.zig").sniff(bytes) orelse return false;
+ return std.mem.eql(u8, mime, actual);
}
/// Called on the actual bracketed-paste path, before normal paste expansion.
diff --git a/src/remote_images.zig b/src/remote_images.zig
index 0d1d67dd..d13af346 100644
--- a/src/remote_images.zig
+++ b/src/remote_images.zig
@@ -66,14 +66,16 @@ pub fn stage(root: *Agent, request: std.json.ObjectMap) bool {
return fail(root, "image_base64 data is not valid base64");
if (bytes == 0 or bytes > vision.max_staged_image_bytes)
return fail(root, "image_base64 is empty or exceeds the staged-image limit");
- {
+ // #1762: the bytes, when recognized, decide the type sent.
+ const sent_type = blk: {
const scratch = root.gpa.alloc(u8, bytes) catch
return fail(root, "could not validate remote image data");
defer root.gpa.free(scratch);
std.base64.standard.Decoder.decode(scratch, data) catch
return fail(root, "image_base64 data is not valid base64");
- }
- specs[i] = .{ .base64 = .{ .media_type = media_type, .data = data } };
+ break :blk @import("image_sniff.zig").sniff(scratch) orelse media_type;
+ };
+ specs[i] = .{ .base64 = .{ .media_type = sent_type, .data = data } };
continue;
}
return fail(root, "request image type must be image_url or image_base64");
diff --git a/src/test_hooks.zig b/src/test_hooks.zig
index c89c1ba9..03755918 100644
--- a/src/test_hooks.zig
+++ b/src/test_hooks.zig
@@ -338,6 +338,7 @@ test {
_ = @import("worker_jev.zig"); // ADR 0232: a child asks Jev for its own effort mid-run
_ = @import("piped_paste.zig"); // ADR 0231
_ = @import("mcp_connect_notice.zig"); // ADR 0230
+ _ = @import("image_sniff.zig"); // #1762: image media type from the bytes
_ = @import("oauth_callback.zig");
_ = @import("cache_key_tests.zig"); // ADR 0223: moved off http_headers.zig (600-line cap)
_ = @import("router_headers_tests.zig"); // moved off http_headers.zig (600-line cap)
diff --git a/src/vision.zig b/src/vision.zig
index 06e370b9..250c3183 100644
--- a/src/vision.zig
+++ b/src/vision.zig
@@ -89,7 +89,8 @@ pub fn visionCapable(p: Provider) bool {
return visionModel(p.model) or std.mem.eql(u8, p.id, "openai") or std.mem.eql(u8, p.id, "codex");
}
-/// Guess an image media type from a file extension (default image/png).
+/// Guess an image media type from a file extension (default image/png). A
+/// label only: anything sent to a provider is typed by `image_sniff` (#1762).
pub fn imageMediaType(path: []const u8) []const u8 {
if (std.ascii.endsWithIgnoreCase(path, ".jpg") or std.ascii.endsWithIgnoreCase(path, ".jpeg")) return "image/jpeg";
if (std.ascii.endsWithIgnoreCase(path, ".gif")) return "image/gif";
@@ -167,6 +168,8 @@ pub const StageResult = union(enum) {
not_found,
/// It was there and the right size, but the read or the encode failed.
read_error,
+ /// The bytes are not PNG, JPEG, GIF or WebP, whatever the name says (#1762).
+ unsupported,
pub fn isOk(self: StageResult) bool {
return std.meta.activeTag(self) == .ok;
@@ -201,15 +204,13 @@ pub fn stageImagePath(root: *Agent, path: []const u8) StageResult {
const arena = root.arena;
const data = readStagedBytes(io, arena, fit.path) orelse return .read_error;
+ // Media type from the bytes we actually encode, never the name (#1762): a
+ // WebP saved as .png must go out as image/webp, and a downscale temp is
+ // whatever `sips` wrote. Labelled with the path the USER named.
+ const media = @import("image_sniff.zig").sniff(data) orelse return .unsupported;
const enc = std.base64.standard.Encoder;
const b64 = arena.alloc(u8, enc.calcSize(data.len)) catch return .read_error;
_ = enc.encode(b64, data);
- // Media type from the file we actually encoded, but labelled with the path
- // the USER named — a temp file's random name means nothing to them. A
- // downscale step is PNG because `sipsResize` forces `-s format png` AND
- // `fitToBudget` re-checks the magic bytes; without both, `sips -Z` would
- // keep the source encoding and we'd send JPEG bytes as `image/png`.
- const media = imageMediaType(fit.path);
@import("vision_queue.zig").stage(root, .{ .media_type = media, .b64 = b64, .label = arena.dupe(u8, path) catch path });
return .{ .ok = .{ .bytes = fit.bytes, .media_type = media } };
}
@@ -229,6 +230,7 @@ pub fn stageMessage(buf: []u8, r: StageResult, what: []const u8) []const u8 {
) catch "that image is too large to send",
.not_found => std.fmt.bufPrint(buf, "can't find {s}", .{what}) catch "can't find that image",
.read_error => std.fmt.bufPrint(buf, "couldn't read {s}", .{what}) catch "couldn't read that image",
+ .unsupported => std.fmt.bufPrint(buf, "{s} is not a PNG, JPEG, GIF or WebP image; convert it first", .{what}) catch "that file is not a supported image",
};
}
@@ -323,10 +325,13 @@ pub fn mcpImageBlock(block: Value) ?McpImageBlock {
// mimeType is required by the spec; default rather than drop an otherwise
// usable image, but refuse a mime that is not an image at all.
const mime = if (block.object.get("mimeType")) |m| (if (m == .string) m.string else "") else "";
- const media_type = if (mime.len == 0) "image/png" else mime;
- if (!std.mem.startsWith(u8, media_type, "image/")) return null;
+ const declared = if (mime.len == 0) "image/png" else mime;
+ if (!std.mem.startsWith(u8, declared, "image/")) return null;
const bytes = std.base64.standard.Decoder.calcSizeForSlice(data.string) catch return null;
if (bytes == 0 or bytes > max_mcp_image_bytes) return null;
+ // #1762: a server (or ACP client) that mislabels a WebP as image/png
+ // would otherwise wedge the chat; the bytes win when we recognize them.
+ const media_type = @import("image_sniff.zig").resolveDeclared(data.string, declared);
return .{ .media_type = media_type, .b64 = data.string, .bytes = bytes };
}
diff --git a/src/vision_queue.zig b/src/vision_queue.zig
index fc0c2fb6..ac175e0a 100644
--- a/src/vision_queue.zig
+++ b/src/vision_queue.zig
@@ -348,6 +348,13 @@ test "remote image validation rejects malformed data before staging (#615)" {
, .{});
try std.testing.expect(!@import("remote_images.zig").stage(&root, bad.object));
try std.testing.expectEqual(@as(u8, 0), root.pending_image_len);
+ // #1762: a recognized format overrides the declared media_type.
+ const webp = try std.json.parseFromSliceLeaky(Value, arena,
+ \\{"type":"user","text":"read it","images":[
+ \\ {"type":"image_base64","media_type":"image/png","data":"UklGRiQAAABXRUJQVlA4IA=="}]}
+ , .{});
+ try std.testing.expect(@import("remote_images.zig").stage(&root, webp.object));
+ try std.testing.expectEqualStrings("image/webp", root.pending_image.?.media_type);
}
test "placeholder-only ask_user reply gets an explicit fallback" {