Skip to content

Lightpanda: the headless browser designed for AI and automation

agent: hand the model inline screenshots

Adrià Arrufate6203ff1fea4
diff --git a/build.zig.zon b/build.zig.zon
index 02db8bf57..8fb4eac97 100644
--- a/build.zig.zon
+++ b/build.zig.zon
@@ -36,8 +36,8 @@
.hash = "sqlite3-3.53.2-DMxLWuAOAAA_Px0arJOIOaP4AKEu5prbsQgPMA35W1zz",
},
.zenai = .{
- .url = "git+https://github.com/lightpanda-io/zenai.git#a745bb1b654e150955f7f4d84ddd1e35ed8beb9e",
- .hash = "zenai-0.0.0-iOY_VI6EBgAryRPO5dH1uS_knQguW5hqcL8EfpnBKBFK",
+ .url = "git+https://github.com/lightpanda-io/zenai.git#b76da59f1b7c36d367a0914adf60a72ca6d87dd4",
+ .hash = "zenai-0.0.0-iOY_VEm3BgDBXrcf-iedI2lNaTE5oF7vyG0AxB85b-2q",
},
.isocline = .{
.url = "git+https://github.com/arrufat/isocline?ref=lightpanda#832a9fe25f5f4458fcc47b5acc7c21db669c2f47",
diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig
index 4ecef3857..dd2d568a2 100644
--- a/src/agent/Agent.zig
+++ b/src/agent/Agent.zig
@@ -1922,16 +1922,28 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co
self.terminal.spinner.setTool(tool_name, args_str);
defer self.terminal.spinner.setThinking();
- const outcome: zenai.provider.Client.ToolHandler.Result = if (browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{})) |result|
- .{ .content = capToolOutput(allocator, tool_name, result.text), .is_error = result.is_error }
- else |err|
- .{ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{browser_tools.errorMessage(err)}) catch "Error: tool execution failed", .is_error = true };
+ const outcome: zenai.provider.Client.ToolHandler.Result = if (browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{ .inline_image = true })) |result| blk: {
+ const content = capToolOutput(allocator, tool_name, result.text);
+ break :blk .{
+ .content = content,
+ .is_error = result.is_error,
+ .parts = if (result.image) |image| imageParts(allocator, content, &image) catch null else null,
+ };
+ } else |err| .{ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{browser_tools.errorMessage(err)}) catch "Error: tool execution failed", .is_error = true };
self.terminal.agentToolDone(tool_name, args_str, !outcome.is_error);
if (self.terminal.verbosity == .high) self.terminal.printToolOutcome(tool_name, outcome.content, outcome.is_error);
return outcome;
}
+/// The text plus the rendered PNG, for backends that can show the model an image.
+fn imageParts(allocator: std.mem.Allocator, text: []const u8, image: *const lp.screenshot.Prepared) ![]const zenai.provider.ContentPart {
+ const parts = try allocator.alloc(zenai.provider.ContentPart, 2);
+ parts[0] = .{ .text = text };
+ parts[1] = .{ .image = .{ .data = try image.base64Alloc(allocator), .mime_type = "image/png" } };
+ return parts;
+}
+
/// One-shot for `--list-models`: resolve provider+key, fetch chat-capable model
/// IDs, print to stdout one per line.
pub fn listModels(allocator: std.mem.Allocator, opts: Config.Agent) !void {
diff --git a/src/browser/screenshot.zig b/src/browser/screenshot.zig
index bbf0cb224..9643c3950 100644
--- a/src/browser/screenshot.zig
+++ b/src/browser/screenshot.zig
@@ -130,12 +130,24 @@ pub const Prepared = struct {
pub fn jsonStringify(self: *const Prepared, jws: *std.json.Stringify) std.Io.Writer.Error!void {
try jws.beginWriteRaw();
try jws.writer.writeByte('"');
- var b64 = Base64Writer.init(jws.writer, .standard);
- _ = try self.write(&b64.writer);
- try b64.finish();
+ try self.writeBase64(jws.writer);
try jws.writer.writeByte('"');
jws.endWriteRaw();
}
+
+ /// The PNG as base64, for APIs that want it as one string.
+ pub fn base64Alloc(self: *const Prepared, allocator: Allocator) ![]const u8 {
+ var aw: std.Io.Writer.Allocating = .init(allocator);
+ errdefer aw.deinit();
+ try self.writeBase64(&aw.writer);
+ return aw.toOwnedSlice();
+ }
+
+ fn writeBase64(self: *const Prepared, writer: *std.Io.Writer) std.Io.Writer.Error!void {
+ var b64 = Base64Writer.init(writer, .standard);
+ _ = try self.write(&b64.writer);
+ try b64.finish();
+ }
};
const Sink = struct {
@@ -664,6 +676,23 @@ test "browser.screenshot: png signature and dimensions" {
try testing.expectEqual(true, height > 60 and height < 200);
}
+test "browser.screenshot: base64Alloc is the png, base64 encoded" {
+ defer testing.test_session.closeAllPages();
+ const frame = try testing.createFrame();
+ frame.url = "http://localhost/";
+ const div = try frame.window._document.createElement("div", null, frame);
+ try Frame.parse.htmlAsChildren(frame, div.asNode(), "<p>Hello</p>");
+
+ const prepared = try prepare(testing.arena_allocator, div.asNode(), .{ .width = 320 }, frame);
+ const b64 = try prepared.base64Alloc(testing.arena_allocator);
+ try testing.expect(std.mem.startsWith(u8, b64, "iVBORw0KGgo"));
+
+ const decoder = std.base64.standard.Decoder;
+ const bytes = try testing.arena_allocator.alloc(u8, try decoder.calcSizeForSlice(b64));
+ try decoder.decode(bytes, b64);
+ try testing.expectEqual("\x89PNG\r\n\x1a\n", bytes[0..8]);
+}
+
test "browser.screenshot: fixed height, clip and scale" {
defer testing.test_session.closeAllPages();
const frame = try testing.createFrame();