Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Claude Agent: keep a prompt's images when it follows a tool result (#1792)

justrach064cecf0e959
diff --git a/src/claude_agent.zig b/src/claude_agent.zig
index aef10a22..a60998ec 100644
--- a/src/claude_agent.zig
+++ b/src/claude_agent.zig
@@ -268,7 +268,8 @@ fn sendDelta(self: *Agent, a: std.mem.Allocator, s: *Session, items: []const Val
}
const last_extra = if (note.len == 0) d.extra_text else if (d.extra_text.len == 0) note else try std.mem.concat(a, u8, &.{ d.extra_text, "\n\n", note });
for (d.results, 0..) |r, i| {
- const result = try wire.toolResult(a, r.block, if (i + 1 == d.results.len) last_extra else "");
+ const last = i + 1 == d.results.len;
+ const result = try wire.toolResult(a, r.block, if (last) last_extra else "", if (last) d.extra_images else &.{});
if (pendingIndex(s, r.id)) |pi| {
const p = s.pending.items[pi];
try s.send(try wire.mcpReply(a, p.request_id, p.rpc_id, result));
diff --git a/src/claude_agent_history.zig b/src/claude_agent_history.zig
index a5bccc15..db218cc2 100644
--- a/src/claude_agent_history.zig
+++ b/src/claude_agent_history.zig
@@ -53,20 +53,23 @@ fn hasToolResult(m: Value) bool {
}
/// Content for the first message of a fresh session: the checkpoint of
-/// everything up to the last assistant reply, then the user messages after
-/// it as the prompt (graff may follow a prompt with notices of its own).
+/// everything up to the last assistant reply or tool result, then the user
+/// messages after it as the prompt (graff may follow a prompt with notices
+/// of its own). A turn that ended on a call (`attempt_completion`) leaves
+/// its result first in the trailing user run: the prompt after it keeps
+/// its blocks, images too, instead of joining the checkpoint as text (#1792).
pub fn freshContent(a: Allocator, items: []const Value) !Value {
var blocks: std.json.Array = .init(a);
var start = items.len;
while (start > 0 and std.mem.eql(u8, role(items[start - 1]), "user")) start -= 1;
- var prompt_run = start < items.len;
- for (items[start..]) |m| if (hasToolResult(m)) {
- prompt_run = false; // results of a call in the checkpoint: no new prompt
+ var cut = start;
+ for (items[start..], start..) |m, i| if (hasToolResult(m)) {
+ cut = i + 1; // results of a call belong to the checkpoint
};
- const head = if (prompt_run) items[0..start] else items;
+ const head = items[0..cut];
if (head.len > 0) try blocks.append(try textBlock(a, try checkpoint(a, head)));
- if (prompt_run) {
- for (items[start..]) |m| try appendPromptBlocks(a, &blocks, m.object.get("content") orelse Value{ .string = "" });
+ if (cut < items.len) {
+ for (items[cut..]) |m| try appendPromptBlocks(a, &blocks, m.object.get("content") orelse Value{ .string = "" });
} else if (head.len > 0) {
try blocks.append(try textBlock(a, "Continue from where the checkpoint ends."));
}
@@ -170,6 +173,9 @@ pub const Delta = struct {
results: []const ToolRes,
/// Non-result text that arrived with results (reminders, a steer).
extra_text: []const u8,
+ /// Images that arrived with results (a prompt after a call, a vision
+ /// attachment): sent with the last result as images, not text (#1792).
+ extra_images: []const Value = &.{},
/// A new prompt when there are no results.
user: ?Value,
};
@@ -200,13 +206,19 @@ pub fn delta(a: Allocator, items: []const Value) !?Delta {
}
}
if (results.items.len > 0) {
+ var images: std.ArrayList(Value) = .empty;
for (prompt.items) |b| {
- const t = wire.strField(b.object, "text") orelse "[image]";
+ const kind = wire.strField(b.object, "type") orelse "";
+ if (std.mem.eql(u8, kind, "image")) {
+ try images.append(a, b);
+ continue;
+ }
+ const t = wire.strField(b.object, "text") orelse try std.fmt.allocPrint(a, "[{s}]", .{kind});
if (t.len == 0) continue;
if (extra.items.len > 0) try extra.append(a, '\n');
try extra.appendSlice(a, t);
}
- return .{ .results = results.items, .extra_text = extra.items, .user = null };
+ return .{ .results = results.items, .extra_text = extra.items, .extra_images = images.items, .user = null };
}
if (prompt.items.len == 0) return null;
return .{ .results = &.{}, .extra_text = "", .user = .{ .array = prompt } };
diff --git a/src/claude_agent_tests.zig b/src/claude_agent_tests.zig
index 414820f9..c5e450c3 100644
--- a/src/claude_agent_tests.zig
+++ b/src/claude_agent_tests.zig
@@ -111,7 +111,7 @@ test "claude-agent MCP: tools/list renames input_schema; a tool_result becomes M
try std.testing.expectEqualStrings("shell", t0.get("name").?.string);
try std.testing.expectEqualStrings("object", t0.get("inputSchema").?.object.get("type").?.string);
const block = try parse(a, "{\"type\":\"tool_result\",\"tool_use_id\":\"toolu_1\",\"is_error\":true,\"content\":[{\"type\":\"text\",\"text\":\"boom\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"image/jpeg\",\"data\":\"AAAA\"}}]}");
- const result = try parse(a, try wire.toolResult(a, block.object, "steer: also check tests"));
+ const result = try parse(a, try wire.toolResult(a, block.object, "steer: also check tests", &.{}));
const content = result.object.get("content").?.array.items;
try std.testing.expectEqual(@as(usize, 3), content.len);
try std.testing.expectEqualStrings("boom", content[0].object.get("text").?.string);
@@ -237,3 +237,38 @@ test "claude-agent routing: once chosen it is the Claude plan; otherwise Claude
try std.testing.expectEqualStrings("anthropic", (try keys.providerFor("claude-opus-4-8")).id);
try std.testing.expectEqualStrings("anthropic", (try keys.providerById("anthropic", "claude-opus-5-5")).id);
}
+
+test "claude-agent: a prompt after a call's result keeps its image, fresh or live (#1792)" {
+ var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena.deinit();
+ const a = arena.allocator();
+ // The turn ended on attempt_completion; the user then sent a screenshot.
+ const msgs = try parse(a,
+ \\[{"role":"user","content":"build it"},
+ \\ {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"attempt_completion","input":{}}]},
+ \\ {"role":"user","content":[{"type":"tool_result","tool_use_id":"t1","content":"completion recorded"}]},
+ \\ {"role":"user","content":[{"type":"text","text":"what is that?"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"iVBORw0KGgo="}}]}]
+ );
+ const fresh = (try history.freshContent(a, msgs.array.items)).array.items;
+ try std.testing.expectEqual(@as(usize, 3), fresh.len);
+ const cp = fresh[0].object.get("text").?.string;
+ try std.testing.expect(std.mem.indexOf(u8, cp, "completion recorded") != null);
+ try std.testing.expect(std.mem.indexOf(u8, cp, "what is that?") == null);
+ try std.testing.expect(std.mem.indexOf(u8, cp, "[image]") == null);
+ try std.testing.expectEqualStrings("what is that?", fresh[1].object.get("text").?.string);
+ try std.testing.expectEqualStrings("image", fresh[2].object.get("type").?.string);
+
+ // A live session waiting on that call: the image rides with the result.
+ const d = (try history.delta(a, msgs.array.items[1..])).?;
+ try std.testing.expectEqual(@as(usize, 1), d.results.len);
+ try std.testing.expectEqualStrings("what is that?", d.extra_text);
+ try std.testing.expectEqual(@as(usize, 1), d.extra_images.len);
+ const wire_out = try wire.toolResult(a, d.results[0].block, d.extra_text, d.extra_images);
+ try std.testing.expect(std.mem.indexOf(u8, wire_out, "[image]") == null);
+ const result = try parse(a, wire_out);
+ const items = result.object.get("content").?.array.items;
+ try std.testing.expectEqual(@as(usize, 3), items.len);
+ try std.testing.expectEqualStrings("image", items[2].object.get("type").?.string);
+ try std.testing.expectEqualStrings("iVBORw0KGgo=", items[2].object.get("data").?.string);
+ try std.testing.expectEqualStrings("image/png", items[2].object.get("mimeType").?.string);
+}
diff --git a/src/claude_agent_wire.zig b/src/claude_agent_wire.zig
index f74d9bc2..f126ac58 100644
--- a/src/claude_agent_wire.zig
+++ b/src/claude_agent_wire.zig
@@ -257,9 +257,10 @@ pub fn toolsList(a: Allocator, tools: ?Value) ![]u8 {
/// An Anthropic `tool_result` block as an MCP `tools/call` result. Text that
/// graff appended to the same user message (reminders, a steer) rides along
-/// as one more text item: the model reads it with the result, as it would
-/// on the Messages wire.
-pub fn toolResult(a: Allocator, block: ObjectMap, extra_text: []const u8) ![]u8 {
+/// as one more text item, and Anthropic image blocks that came with it as
+/// MCP image items (#1792): the model reads them with the result, as it
+/// would on the Messages wire.
+pub fn toolResult(a: Allocator, block: ObjectMap, extra_text: []const u8, extra_images: []const Value) ![]u8 {
var aw: std.Io.Writer.Allocating = .init(a);
var s: std.json.Stringify = .{ .writer = &aw.writer };
try s.beginObject();
@@ -272,23 +273,12 @@ pub fn toolResult(a: Allocator, block: ObjectMap, extra_text: []const u8) ![]u8
const kind = strField(item.object, "type") orelse continue;
if (std.mem.eql(u8, kind, "text")) {
try textItem(&s, strField(item.object, "text") orelse "");
- } else if (std.mem.eql(u8, kind, "image")) {
- const src = item.object.get("source") orelse continue;
- if (src != .object) continue;
- const data = strField(src.object, "data") orelse continue;
- try s.beginObject();
- try s.objectField("type");
- try s.write("image");
- try s.objectField("data");
- try s.write(data);
- try s.objectField("mimeType");
- try s.write(strField(src.object, "media_type") orelse "image/png");
- try s.endObject();
- }
+ } else if (std.mem.eql(u8, kind, "image")) try imageItem(&s, item.object);
},
else => {},
};
if (extra_text.len > 0) try textItem(&s, extra_text);
+ for (extra_images) |img| if (img == .object) try imageItem(&s, img.object);
try s.endArray();
if (block.get("is_error")) |e| if (e == .bool and e.bool) {
try s.objectField("isError");
@@ -298,6 +288,21 @@ pub fn toolResult(a: Allocator, block: ObjectMap, extra_text: []const u8) ![]u8
return aw.toOwnedSlice();
}
+/// An Anthropic base64 image block as an MCP image item; others are skipped.
+fn imageItem(s: *std.json.Stringify, block: ObjectMap) !void {
+ const src = block.get("source") orelse return;
+ if (src != .object) return;
+ const data = strField(src.object, "data") orelse return;
+ try s.beginObject();
+ try s.objectField("type");
+ try s.write("image");
+ try s.objectField("data");
+ try s.write(data);
+ try s.objectField("mimeType");
+ try s.write(strField(src.object, "media_type") orelse "image/png");
+ try s.endObject();
+}
+
fn textItem(s: *std.json.Stringify, text: []const u8) !void {
try s.beginObject();
try s.objectField("type");