Lightpanda: the headless browser designed for AI and automation
agent: hand the model inline screenshots
Adrià Arrufate6203ff1fea4
diff --git a/build.zig.zon b/build.zig.zonindex 02db8bf57..8fb4eac97 100644--- a/build.zig.zon+++ b/build.zig.zon@@ -36,8 +36,8 @@.hash = "sqlite3-3.53.2-DMxLWuAOAAA_Px0arJOIOaP4AKEu5prbsQgPMA35W1zz",},.zenai = .{- .url = "git+https://github.com/lightpanda-io/zenai.git#a745bb1b654e150955f7f4d84ddd1e35ed8beb9e",- .hash = "zenai-0.0.0-iOY_VI6EBgAryRPO5dH1uS_knQguW5hqcL8EfpnBKBFK",+ .url = "git+https://github.com/lightpanda-io/zenai.git#b76da59f1b7c36d367a0914adf60a72ca6d87dd4",+ .hash = "zenai-0.0.0-iOY_VEm3BgDBXrcf-iedI2lNaTE5oF7vyG0AxB85b-2q",},.isocline = .{.url = "git+https://github.com/arrufat/isocline?ref=lightpanda#832a9fe25f5f4458fcc47b5acc7c21db669c2f47",diff --git a/src/agent/Agent.zig b/src/agent/Agent.zigindex 4ecef3857..dd2d568a2 100644--- a/src/agent/Agent.zig+++ b/src/agent/Agent.zig@@ -1922,16 +1922,28 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []coself.terminal.spinner.setTool(tool_name, args_str);defer self.terminal.spinner.setThinking();- const outcome: zenai.provider.Client.ToolHandler.Result = if (browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{})) |result|- .{ .content = capToolOutput(allocator, tool_name, result.text), .is_error = result.is_error }- else |err|- .{ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{browser_tools.errorMessage(err)}) catch "Error: tool execution failed", .is_error = true };+ const outcome: zenai.provider.Client.ToolHandler.Result = if (browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{ .inline_image = true })) |result| blk: {+ const content = capToolOutput(allocator, tool_name, result.text);+ break :blk .{+ .content = content,+ .is_error = result.is_error,+ .parts = if (result.image) |image| imageParts(allocator, content, &image) catch null else null,+ };+ } else |err| .{ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{browser_tools.errorMessage(err)}) catch "Error: tool execution failed", .is_error = true };self.terminal.agentToolDone(tool_name, args_str, !outcome.is_error);if (self.terminal.verbosity == .high) self.terminal.printToolOutcome(tool_name, outcome.content, outcome.is_error);return outcome;}+/// The text plus the rendered PNG, for backends that can show the model an image.+fn imageParts(allocator: std.mem.Allocator, text: []const u8, image: *const lp.screenshot.Prepared) ![]const zenai.provider.ContentPart {+ const parts = try allocator.alloc(zenai.provider.ContentPart, 2);+ parts[0] = .{ .text = text };+ parts[1] = .{ .image = .{ .data = try image.base64Alloc(allocator), .mime_type = "image/png" } };+ return parts;+}+/// One-shot for `--list-models`: resolve provider+key, fetch chat-capable model/// IDs, print to stdout one per line.pub fn listModels(allocator: std.mem.Allocator, opts: Config.Agent) !void {diff --git a/src/browser/screenshot.zig b/src/browser/screenshot.zigindex bbf0cb224..9643c3950 100644--- a/src/browser/screenshot.zig+++ b/src/browser/screenshot.zig@@ -130,12 +130,24 @@ pub const Prepared = struct {pub fn jsonStringify(self: *const Prepared, jws: *std.json.Stringify) std.Io.Writer.Error!void {try jws.beginWriteRaw();try jws.writer.writeByte('"');- var b64 = Base64Writer.init(jws.writer, .standard);- _ = try self.write(&b64.writer);- try b64.finish();+ try self.writeBase64(jws.writer);try jws.writer.writeByte('"');jws.endWriteRaw();}++ /// The PNG as base64, for APIs that want it as one string.+ pub fn base64Alloc(self: *const Prepared, allocator: Allocator) ![]const u8 {+ var aw: std.Io.Writer.Allocating = .init(allocator);+ errdefer aw.deinit();+ try self.writeBase64(&aw.writer);+ return aw.toOwnedSlice();+ }++ fn writeBase64(self: *const Prepared, writer: *std.Io.Writer) std.Io.Writer.Error!void {+ var b64 = Base64Writer.init(writer, .standard);+ _ = try self.write(&b64.writer);+ try b64.finish();+ }};const Sink = struct {@@ -664,6 +676,23 @@ test "browser.screenshot: png signature and dimensions" {try testing.expectEqual(true, height > 60 and height < 200);}+test "browser.screenshot: base64Alloc is the png, base64 encoded" {+ defer testing.test_session.closeAllPages();+ const frame = try testing.createFrame();+ frame.url = "http://localhost/";+ const div = try frame.window._document.createElement("div", null, frame);+ try Frame.parse.htmlAsChildren(frame, div.asNode(), "<p>Hello</p>");++ const prepared = try prepare(testing.arena_allocator, div.asNode(), .{ .width = 320 }, frame);+ const b64 = try prepared.base64Alloc(testing.arena_allocator);+ try testing.expect(std.mem.startsWith(u8, b64, "iVBORw0KGgo"));++ const decoder = std.base64.standard.Decoder;+ const bytes = try testing.arena_allocator.alloc(u8, try decoder.calcSizeForSlice(b64));+ try decoder.decode(bytes, b64);+ try testing.expectEqual("\x89PNG\r\n\x1a\n", bytes[0..8]);+}+test "browser.screenshot: fixed height, clip and scale" {defer testing.test_session.closeAllPages();const frame = try testing.createFrame();