Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Accept images from openai/codex models and ACP image blocks

Rach Pradhanae863aca9849
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e9ae5ce4..b7c12d7a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,12 @@ current is part of cutting a release.
- ACP clients can pass MCP servers in `session/new` and `session/load`, and `GRAFF_NO_PLUGINS` now applies before those servers connect.
- Opted-in ACP clients can preview child-agent activity live, and background workers stream through a Graff extension across turns. Session teardown waits for background agents, and deferred MCP startup joins run under the registry lock.
+### Image input
+
+- Every OpenAI `gpt-*` model is treated as image-capable (gpt-oss and gpt-3.5 excepted), and so is every model on the `openai` and `codex` providers, including o-series and `codex-*` ids. New releases no longer ship text-only by omission.
+- ACP clients can send standard `image` prompt blocks, and `promptCapabilities.image` is now advertised. Images join the next model request when the model accepts them and are skipped for text-only models. An image with no caption is sent with a short note, since some providers reject an empty text part.
+- Graff no longer crashes at startup when its presence socket path would exceed the platform limit (a long `$HOME`); that best-effort listener is skipped instead.
+
### Optional formal evidence for local learning
- Bounded lifecycle models cover effort changes, asynchronous tools, permission ownership, and connection leases. Deliberately weakened configurations provide counterexamples for the invariants those models are meant to guard.
diff --git a/build.zig b/build.zig
index 84e05193..7971aa49 100644
--- a/build.zig
+++ b/build.zig
@@ -155,6 +155,9 @@ pub fn build(b: *std.Build) void {
const workspace_lifecycle_test = b.addSystemCommand(&.{ "python3", "scripts/test-workspace-lifecycle.py" });
workspace_lifecycle_test.addArtifactArg(exe);
test_step.dependOn(&workspace_lifecycle_test.step);
+ const acp_images_test = b.addSystemCommand(&.{ "python3", "scripts/test-acp-images.py" });
+ acp_images_test.addArtifactArg(exe);
+ test_step.dependOn(&acp_images_test.step);
const run_tool_budget_test = b.addSystemCommand(&.{ "python3", "scripts/test-run-tool-budget.py" });
run_tool_budget_test.addArtifactArg(exe);
test_step.dependOn(&run_tool_budget_test.step);
diff --git a/docs/acp-compatibility.md b/docs/acp-compatibility.md
index 5136220f..6ffb1555 100644
--- a/docs/acp-compatibility.md
+++ b/docs/acp-compatibility.md
@@ -28,7 +28,7 @@ optional feature or draft extension is enabled.
| Permissions | Send the advertised choices to the active client and await its response. Clients must honor an explicit rejection or cancellation; automatic approval is a separate user setting. | `scripts/test-acp-permissions.py` |
| Effort configuration | Expose supported `thought_level` values in session configuration. Validate setters, return the full option list, and emit `config_option_update` when agent-originated changes alter that state. | `scripts/test-acp-effort-config.py`, `src/acp_config.zig` |
| Configuration timing | Client changes arriving during a prompt are queued for after that turn. An effort-tool result applies on the owning agent thread before the next model request; clients refresh the affected session's controls. | `src/acp_idle.zig`, `src/jev_effort_state.zig`, `apps/native/lib/acp-client-stream.test.ts` |
-| Content | Text, resource links and supported embedded text resources are accepted. Image and audio prompt capabilities are currently advertised as false. | `src/acp_protocol.zig` |
+| Content | Text, resource links and supported embedded text resources are accepted. `image` blocks (`data` + `mimeType`) are accepted and advertised (`promptCapabilities.image`); they reach the next model request when the model accepts images and are skipped otherwise, the same as `@[path]` attachments. Audio is advertised as false. | `src/acp_protocol.zig`, `src/acp_images.zig`, `scripts/test-acp-images.py` |
| Client filesystem and terminals | These are optional negotiated ACP facilities, not a requirement that local harness tools be replaced by client tools. The local tool implementation does not imply support for every client-side method. | Capability negotiation and the upstream v1 schema |
| Usage | Connection usage is a namespaced extension. It preserves unknown totals and is not a standard ACP v1 billing contract. | `scripts/test-acp-usage.py`, ADR 0182 |
| Subagents | The proposed child-session update stream is an opt-in live preview: client `subagents` capability plus `GRAFF_ACP_DRAFT_SUBAGENTS=1`. Explicit `run_in_background:false` foreground children stream separately; the default detached workers retain their parent tool row and `graff/agents` inspection. Child history is not yet replayed on `session/load`. | `scripts/test-acp-subagent-update.py`, ADR 0194 |
diff --git a/scripts/eval-tier1.sh b/scripts/eval-tier1.sh
index c21b7eb9..5a614a4a 100755
--- a/scripts/eval-tier1.sh
+++ b/scripts/eval-tier1.sh
@@ -188,7 +188,8 @@ if wanted shell; then
python3 scripts/eval/process_guard.py --timeout 90 python3 scripts/test-shell-restart.py zig-out/bin/graff &&
python3 scripts/eval/process_guard.py --timeout 90 python3 scripts/test-acp-released-fixes.py zig-out/bin/graff &&
python3 scripts/eval/process_guard.py --timeout 120 python3 scripts/test-acp-permissions.py zig-out/bin/graff &&
- python3 scripts/eval/process_guard.py --timeout 90 python3 scripts/test-acp-session-load.py zig-out/bin/graff
+ python3 scripts/eval/process_guard.py --timeout 90 python3 scripts/test-acp-session-load.py zig-out/bin/graff &&
+ python3 scripts/eval/process_guard.py --timeout 90 python3 scripts/test-acp-images.py zig-out/bin/graff
); then :; else
record_fail shell
fi
diff --git a/scripts/eval/tier1-manifest.json b/scripts/eval/tier1-manifest.json
index 857b45e2..510a5812 100644
--- a/scripts/eval/tier1-manifest.json
+++ b/scripts/eval/tier1-manifest.json
@@ -15,7 +15,7 @@
"src/repl.zig",
"TUI/root.zig"
],
- "test_count_baseline": 2813,
+ "test_count_baseline": 2818,
"test_count_slack": 25,
"required_invariants": [
{
diff --git a/scripts/test-acp-images.py b/scripts/test-acp-images.py
new file mode 100755
index 00000000..bfdea840
--- /dev/null
+++ b/scripts/test-acp-images.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""Offline ACP image input: `promptCapabilities.image` is advertised, and an
+`image` block never reaches a text-only model as a provider 400. Delivery to
+vision models is covered by src/acp_images.zig (the offline gateway catalog has
+no vision model to route here)."""
+import importlib.util
+import json
+import sys
+import tempfile
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(REPO / "scripts/eval"))
+from mock_model import ScriptedModel
+
+_spec = importlib.util.spec_from_file_location("acp_session_load_fixture", REPO / "scripts/test-acp-session-load.py")
+_fixture = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_fixture)
+Acp = _fixture.Acp
+
+PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+
+
+def run(binary):
+ with tempfile.TemporaryDirectory(prefix="graff-acp-images-") as temporary:
+ base = Path(temporary).resolve()
+ cwd, home = base / "work", base / "home"
+ cwd.mkdir()
+ home.mkdir()
+ model = ScriptedModel([{"text": "no pixels here"}])
+ port = model.start(0)
+ a = Acp(binary, cwd, home, port)
+ try:
+ init = a.request("initialize", {"protocolVersion": 1})
+ caps = init["result"]["agentCapabilities"]["promptCapabilities"]
+ assert caps["image"] is True and caps["audio"] is False, caps
+ sid = a.request("session/new", {"cwd": str(cwd), "mcpServers": []})["result"]["sessionId"]
+ turn = a.request("session/prompt", {"sessionId": sid, "prompt": [
+ {"type": "text", "text": "what is in this picture?"},
+ {"type": "image", "data": PNG, "mimeType": "image/png"},
+ ]}, 35)
+ assert turn["result"]["stopReason"] == "end_turn", turn
+ sent = json.dumps(model.requests[0]["messages"][-1])
+ assert "what is in this picture?" in sent, sent
+ assert PNG not in sent and "image_url" not in sent, "a text-only model must not receive image parts"
+ finally:
+ a.close()
+ model.stop()
+ print("ACP image input: capability advertised; text-only model receives text only: ok")
+
+
+if __name__ == "__main__":
+ run(Path(sys.argv[1] if len(sys.argv) > 1 else REPO / "zig-out/bin/graff").resolve())
diff --git a/src/acp.zig b/src/acp.zig
index 973955b6..09a457a3 100644
--- a/src/acp.zig
+++ b/src/acp.zig
@@ -83,9 +83,11 @@ fn liveSlash(ctx: *anyopaque, arena: Allocator, text: []const u8) anyerror!?[]co
return try engine.stripSgr(arena, aw.writer.buffered());
}
-fn liveAfter(ctx: *anyopaque, arena: Allocator, text: []const u8) void {
+fn liveAfter(ctx: *anyopaque, arena: Allocator, text: []const u8, prompt: ?std.json.Value) void {
const live: *LiveTurn = @ptrCast(@alignCast(ctx));
_ = playbook_glue.applyUserOverride(live.root, arena, text);
+ // ACP image blocks join the next model request (promptCapabilities.image).
+ _ = @import("acp_images.zig").stage(live.root, prompt);
}
fn liveBind(ctx: *anyopaque, session_id: []const u8) void {
diff --git a/src/acp_engine.zig b/src/acp_engine.zig
index d68c7309..434c623d 100644
--- a/src/acp_engine.zig
+++ b/src/acp_engine.zig
@@ -30,7 +30,7 @@ pub var extra_cancelled: ?*const fn () bool = null;
pub const TurnFn = *const fn (ctx: *anyopaque, arena: Allocator, text: []const u8) anyerror![]const u8;
pub const SlashFn = *const fn (ctx: *anyopaque, arena: Allocator, text: []const u8) anyerror!?[]const u8;
-pub const AfterUserFn = *const fn (ctx: *anyopaque, arena: Allocator, text: []const u8) void;
+pub const AfterUserFn = *const fn (ctx: *anyopaque, arena: Allocator, text: []const u8, prompt: ?std.json.Value) void;
pub const BindSessionFn = *const fn (ctx: *anyopaque, session_id: []const u8) void;
pub const LoadSessionFn = *const fn (ctx: *anyopaque, arena: Allocator, w: *Io.Writer, req: proto.Request) anyerror!void;
/// Optional per-turn context meter: used and window tokens for the
@@ -187,7 +187,7 @@ fn promptTurn(d: *Dispatch, arena: Allocator, w: *Io.Writer, req: proto.Request)
return respond(w, req, .{ .stopReason = "end_turn" });
}
}
- if (d.after_user) |after| after(d.ctx, arena, text);
+ if (d.after_user) |after| after(d.ctx, arena, text, if (obj) |o| o.get("prompt") else null);
const final = d.turn(d.ctx, arena, text) catch |err| {
emitConfigChange(d, arena, w, sid, config_before) catch |config_err| return turnError(d, w, req, config_err);
return turnError(d, w, req, err);
@@ -268,7 +268,7 @@ fn respondInitialize(w: *Io.Writer, req: proto.Request, can_load: bool) !void {
.agentCapabilities = .{
.loadSession = can_load,
._meta = .{ .@"codegraff/usage" = can_load, .@"graff/backgroundSubagents" = true },
- .promptCapabilities = proto.PromptCapabilities{},
+ .promptCapabilities = proto.PromptCapabilities{ .image = can_load },
// session/new and session/load connect a client's MCP servers
// (stdio always; http advertised; sse not run).
.mcpCapabilities = .{ .http = true, .sse = false },
diff --git a/src/acp_images.zig b/src/acp_images.zig
new file mode 100644
index 00000000..b7028a46
--- /dev/null
+++ b/src/acp_images.zig
@@ -0,0 +1,94 @@
+//! ACP `image` prompt blocks (`{type:"image", data, mimeType}`) become vision
+//! input for the next model request, the same queue GUI `@[path]` attachments
+//! and MCP screenshots use. Parsing and the size/mime gate are
+//! `vision.mcpImageBlock` (the shape is identical). Models that cannot read
+//! images skip staging, matching `vision.stageGuiImageAttachment`.
+
+const std = @import("std");
+const Allocator = std.mem.Allocator;
+const Value = std.json.Value;
+
+const Agent = @import("agent.zig").Agent;
+const vision = @import("vision.zig");
+const vision_queue = @import("vision_queue.zig");
+
+/// Stage every image block in `prompt`. Returns how many were queued.
+pub fn stage(root: *Agent, prompt: ?Value) usize {
+ const p = prompt orelse return 0;
+ if (p != .array or !vision.visionCapable(root.provider)) return 0;
+ var staged: usize = 0;
+ for (p.array.items) |block| {
+ const img = vision.mcpImageBlock(block) orelse continue;
+ var label_buf: [32]u8 = undefined;
+ const label = std.fmt.bufPrint(&label_buf, "image {d}", .{staged + 1}) catch "image";
+ // The session arena outlives the request that carried the bytes.
+ const pending = img.stage(root.arena, label) catch continue;
+ vision_queue.stage(root, pending);
+ staged += 1;
+ }
+ return staged;
+}
+
+/// Pure half for tests: the image blocks `stage` would queue.
+pub fn count(prompt: ?Value) usize {
+ const p = prompt orelse return 0;
+ if (p != .array) return 0;
+ var n: usize = 0;
+ for (p.array.items) |block| {
+ if (vision.mcpImageBlock(block) != null) n += 1;
+ }
+ return n;
+}
+
+test "count: ACP image blocks are recognized; text, resource links and non-images are not" {
+ var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena.deinit();
+ const prompt = try std.json.parseFromSliceLeaky(Value, arena.allocator(),
+ \\[{"type":"text","text":"what is this?"},
+ \\ {"type":"image","data":"iVBORw0KGgo=","mimeType":"image/png"},
+ \\ {"type":"image","data":"/9j/4AAQ","mimeType":"image/jpeg"},
+ \\ {"type":"image","data":"AAAA","mimeType":"application/pdf"},
+ \\ {"type":"resource_link","uri":"file:///x.png","name":"x.png"}]
+ , .{});
+ try std.testing.expectEqual(@as(usize, 2), count(prompt));
+ try std.testing.expectEqual(@as(usize, 0), count(null));
+}
+
+fn testAgent(arena: Allocator, id: []const u8, model: []const u8) Agent {
+ return .{
+ .gpa = arena,
+ .arena = arena,
+ .io = undefined,
+ .client = undefined,
+ .provider = .{ .id = id, .kind = .responses, .auth = .bearer, .url = "", .api_key = "", .model = model, .context = 100_000 },
+ .messages = undefined,
+ .sub = false,
+ .label = "test",
+ .out = null,
+ };
+}
+
+test "stage: a codex model receives ACP images as the next request's input_image; text-only models skip them" {
+ var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena.deinit();
+ const a = arena.allocator();
+ const prompt = try std.json.parseFromSliceLeaky(Value, a,
+ \\[{"type":"image","data":"iVBORw0KGgo=","mimeType":"image/png"}]
+ , .{});
+ // A non-gpt id on the codex provider: only the provider rule makes it vision-capable.
+ var codex = testAgent(a, "codex", "o4-mini");
+ try std.testing.expectEqual(@as(usize, 1), stage(&codex, prompt));
+ const msg = try vision_queue.consumePromptImages(a, &codex, "");
+ var out: std.Io.Writer.Allocating = .init(a);
+ var s: std.json.Stringify = .{ .writer = &out.writer };
+ try s.write(msg);
+ const wire = out.written();
+ try std.testing.expect(std.mem.indexOf(u8, wire, "\"input_image\"") != null);
+ try std.testing.expect(std.mem.indexOf(u8, wire, "data:image/png;base64,iVBORw0KGgo=") != null);
+ try std.testing.expect(std.mem.indexOf(u8, wire, vision_queue.attached_note) != null); // image-only prompt
+ try std.testing.expectEqual(@as(u8, 0), codex.pending_image_len);
+
+ var text_only = testAgent(a, "openrouter", "qwen3.8-27b");
+ try std.testing.expectEqual(@as(usize, 0), stage(&text_only, prompt));
+ try std.testing.expectEqual(@as(u8, 0), text_only.pending_image_len);
+}
diff --git a/src/presence_accord.zig b/src/presence_accord.zig
index 349a0a69..4806eec6 100644
--- a/src/presence_accord.zig
+++ b/src/presence_accord.zig
@@ -186,6 +186,13 @@ fn sweepDead(io: Io, dir_path: []const u8) void {
}
/// Bind `{live_dir}/{pid}-{start}.accord.sock` and accept live posts.
+/// `sockaddr_un.sun_path` is 104 bytes on macOS/BSD and 108 on Linux,
+/// including the terminating NUL. std's listener indexes past it otherwise.
+pub fn fitsUnixSocketPath(path: []const u8) bool {
+ const cap: usize = if (builtin.os.tag == .linux) 108 else 104;
+ return path.len < cap;
+}
+
pub fn listen(io: Io, gpa: std.mem.Allocator, dir_path: ?[]const u8) void {
if (!enabled()) return;
if (builtin.os.tag == .windows) return;
@@ -197,6 +204,8 @@ pub fn listen(io: Io, gpa: std.mem.Allocator, dir_path: ?[]const u8) void {
const name = std.fmt.bufPrint(&name_buf, "{d}-{x}.accord.sock", .{ self.pid, self.start_id }) catch return;
var path_buf: [std.fs.max_path_bytes]u8 = undefined;
const path = std.fmt.bufPrint(&path_buf, "{s}/{s}", .{ dir, name }) catch return;
+ // Presence is best-effort: a long $HOME must not trip the sockaddr bound.
+ if (!fitsUnixSocketPath(path)) return;
const owned_dir = gpa.dupe(u8, dir) catch return;
const owned_name = gpa.dupe(u8, name) catch {
gpa.free(owned_dir);
@@ -577,3 +586,9 @@ test "liveClaim is a no-op when Accord is off" {
test_enabled = false;
liveClaim(std.testing.io, "[]");
}
+
+test "fitsUnixSocketPath: a socket under a long HOME is skipped, not a panic" {
+ try std.testing.expect(fitsUnixSocketPath("/Users/me/.graff/live/123-abc.accord.sock"));
+ const long = "/var/folders/65/qh0z1swn4g3ghw352z5w94w80000gn/T/graff-acp-images-abcdefgh/home/.graff/live/12345-1a2b3c4d5e6f.accord.sock";
+ try std.testing.expect(!fitsUnixSocketPath(long));
+}
diff --git a/src/test_hooks.zig b/src/test_hooks.zig
index b692093c..14871df8 100644
--- a/src/test_hooks.zig
+++ b/src/test_hooks.zig
@@ -350,6 +350,7 @@ test {
_ = @import("acp_config.zig");
_ = @import("session_start_plugins_test.zig");
_ = @import("acp_mcp_servers.zig");
+ _ = @import("acp_images.zig");
_ = @import("libgraff.zig");
_ = @import("tui_peer.zig");
_ = @import("acp_auth.zig");
diff --git a/src/vision.zig b/src/vision.zig
index 7c210ec3..79dcb517 100644
--- a/src/vision.zig
+++ b/src/vision.zig
@@ -83,7 +83,9 @@ fn openaiGptVision(m: []const u8) bool {
}
pub fn visionCapable(p: Provider) bool {
- return visionModel(p.model);
+ // OpenAI's API and ChatGPT/Codex login serve only multimodal models, so
+ // any id there (o-series, codex-*, future names) takes images.
+ return visionModel(p.model) or std.mem.eql(u8, p.id, "openai") or std.mem.eql(u8, p.id, "codex");
}
/// Guess an image media type from a file extension (default image/png).
@@ -388,6 +390,21 @@ test "visionCapable allowlist" {
try std.testing.expect(!visionCapable(mk("minimax-m3")));
}
+test "visionCapable: every model on the openai and codex providers takes images" {
+ const mk = struct {
+ fn p(id: []const u8, model: []const u8) Provider {
+ return .{ .id = id, .kind = .responses, .auth = .bearer, .url = "", .model = model, .context = 0, .api_key = "", .account = "" };
+ }
+ }.p;
+ for ([_][]const u8{ "openai", "codex" }) |id| {
+ for ([_][]const u8{ "gpt-6-astra", "gpt-6-sol", "gpt-6-luna", "gpt-5.6-terra", "gpt-5.3-codex-spark", "o4-mini", "codex-mini-latest" }) |model|
+ try std.testing.expect(visionCapable(mk(id, model)));
+ }
+ // The provider rule is not a name rule: an o-series id elsewhere stays text-only.
+ try std.testing.expect(!visionCapable(mk("openrouter", "o4-mini")));
+ try std.testing.expect(!visionCapable(mk("fireworks", "accounts/fireworks/models/gpt-oss-120b")));
+}
+
test "visionModel: every OpenAI gpt-* model sees images, gpt-oss and gpt-3.5 do not" {
// Catalog-driven so a new gpt-* entry can't ship text-only by omission.
for (@import("pricing.zig").model_table) |m| {
diff --git a/src/vision_queue.zig b/src/vision_queue.zig
index 8a65edb0..fc0c2fb6 100644
--- a/src/vision_queue.zig
+++ b/src/vision_queue.zig
@@ -99,7 +99,7 @@ pub fn consumePromptImages(arena: Allocator, root: *Agent, text: []const u8) !Va
vision.tracePaste(root, "sent", "submit", bytes, img.media_type);
}
if (imgs.len == 0) return messages.textMessage(arena, "user", text);
- return vision.imageMessages(arena, root.provider.kind, text, imgs);
+ return vision.imageMessages(arena, root.provider.kind, if (std.mem.trim(u8, text, " \t\r\n").len == 0) attached_note else text, imgs);
}
/// After an ask_user (or any tool that staged pixels) tool result, attach the