Lightpanda: the headless browser designed for AI and automation
agent: let the model see a REPL /screenshot; fold the tool-result adapters
Adrià Arrufatcff32f6c047a
diff --git a/src/agent/Agent.zig b/src/agent/Agent.zigindex dd2d568a2..f033575a1 100644--- a/src/agent/Agent.zig+++ b/src/agent/Agent.zig@@ -1488,7 +1488,8 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool.tool_call => |t| t,else => return .{ .text = "internal: command has no tool mapping", .is_error = true },};- return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args, .{}) catch |err| .{+ // The terminal can't show an image, but the conversation can.+ return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args, .{ .inline_image = self.ai_client != null }) catch |err| .{.text = switch (err) {error.OutOfMemory => "out of memory",error.FrameNotLoaded => "no page loaded — run /goto <url> first",@@ -1620,6 +1621,7 @@ fn recordSlashToolCall(.id = try ma.dupe(u8, tool_calls[0].id),.name = try ma.dupe(u8, tool_calls[0].name),.content = content,+ .parts = if (result.image) |image| try imageParts(ma, content, &image) else null,.is_error = result.is_error,};@@ -1922,14 +1924,10 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []coself.terminal.spinner.setTool(tool_name, args_str);defer self.terminal.spinner.setThinking();- const outcome: zenai.provider.Client.ToolHandler.Result = if (browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{ .inline_image = true })) |result| blk: {- const content = capToolOutput(allocator, tool_name, result.text);- break :blk .{- .content = content,- .is_error = result.is_error,- .parts = if (result.image) |image| imageParts(allocator, content, &image) catch null else null,- };- } else |err| .{ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{browser_tools.errorMessage(err)}) catch "Error: tool execution failed", .is_error = true };+ const outcome = self.toolOutcome(allocator, tool_name, arguments) catch |err| zenai.provider.Client.ToolHandler.Result{+ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{browser_tools.errorMessage(err)}) catch "Error: tool execution failed",+ .is_error = true,+ };self.terminal.agentToolDone(tool_name, args_str, !outcome.is_error);if (self.terminal.verbosity == .high) self.terminal.printToolOutcome(tool_name, outcome.content, outcome.is_error);@@ -1937,11 +1935,19 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co}/// The text plus the rendered PNG, for backends that can show the model an image.-fn imageParts(allocator: std.mem.Allocator, text: []const u8, image: *const lp.screenshot.Prepared) ![]const zenai.provider.ContentPart {- const parts = try allocator.alloc(zenai.provider.ContentPart, 2);- parts[0] = .{ .text = text };- parts[1] = .{ .image = .{ .data = try image.base64Alloc(allocator), .mime_type = "image/png" } };- return parts;+fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value) browser_tools.ToolError!zenai.provider.Client.ToolHandler.Result {+ const result = try browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{ .inline_image = true });+ const content = capToolOutput(allocator, tool_name, result.text);+ return .{+ .content = content,+ .is_error = result.is_error,+ .parts = if (result.image) |image| try imageParts(allocator, content, &image) else null,+ };+}++fn imageParts(arena: std.mem.Allocator, text: []const u8, image: *const lp.screenshot.Prepared) browser_tools.ToolError![]const zenai.provider.ContentPart {+ const data = image.base64Alloc(arena) catch return error.InternalError;+ return try arena.dupe(zenai.provider.ContentPart, &.{ .{ .text = text }, .{ .image = .{ .data = data, .mime_type = "image/png" } } });}/// One-shot for `--list-models`: resolve provider+key, fetch chat-capable modeldiff --git a/src/browser/screenshot.zig b/src/browser/screenshot.zigindex 9643c3950..ee50b0522 100644--- a/src/browser/screenshot.zig+++ b/src/browser/screenshot.zig@@ -136,11 +136,10 @@ pub const Prepared = struct {}/// The PNG as base64, for APIs that want it as one string.- pub fn base64Alloc(self: *const Prepared, allocator: Allocator) ![]const u8 {- var aw: std.Io.Writer.Allocating = .init(allocator);- errdefer aw.deinit();+ pub fn base64Alloc(self: *const Prepared, arena: Allocator) ![]const u8 {+ var aw: std.Io.Writer.Allocating = .init(arena);try self.writeBase64(&aw.writer);- return aw.toOwnedSlice();+ return aw.written();}fn writeBase64(self: *const Prepared, writer: *std.Io.Writer) std.Io.Writer.Error!void {@@ -676,23 +675,6 @@ test "browser.screenshot: png signature and dimensions" {try testing.expectEqual(true, height > 60 and height < 200);}-test "browser.screenshot: base64Alloc is the png, base64 encoded" {- defer testing.test_session.closeAllPages();- const frame = try testing.createFrame();- frame.url = "http://localhost/";- const div = try frame.window._document.createElement("div", null, frame);- try Frame.parse.htmlAsChildren(frame, div.asNode(), "<p>Hello</p>");-- const prepared = try prepare(testing.arena_allocator, div.asNode(), .{ .width = 320 }, frame);- const b64 = try prepared.base64Alloc(testing.arena_allocator);- try testing.expect(std.mem.startsWith(u8, b64, "iVBORw0KGgo"));-- const decoder = std.base64.standard.Decoder;- const bytes = try testing.arena_allocator.alloc(u8, try decoder.calcSizeForSlice(b64));- try decoder.decode(bytes, b64);- try testing.expectEqual("\x89PNG\r\n\x1a\n", bytes[0..8]);-}-test "browser.screenshot: fixed height, clip and scale" {defer testing.test_session.closeAllPages();const frame = try testing.createFrame();@@ -810,6 +792,7 @@ test "browser.screenshot: json streams base64" {_ = enc.encode(expected[9 .. expected.len - 2], raw.written());@memcpy(expected[expected.len - 2 ..], "\"}");try testing.expectString(expected, json.written());+ try testing.expectString(expected[9 .. expected.len - 2], try prepared.base64Alloc(testing.arena_allocator));}test "browser.screenshot: block extraction" {diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex ee2832e73..d8a315ace 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -40,8 +40,7 @@ const Selector = @import("webapi/selector/Selector.zig");pub const driver_guidance =\\You are driving Lightpanda, a headless browser, through text tools:\\you reason over pages as a semantic tree, markdown or HTML. `screenshot`- \\renders that text layout as a PNG (no images, fonts or CSS) — use it- \\for spatial layout, not as a primary read.+ \\renders that text layout as a PNG: for spatial layout, not a primary read.\\\\Reading pages (cheap → expensive — prefer cheaper):\\- `tree` → semantic overview (role, name, value, backendNodeId per@@ -215,14 +214,6 @@ pub const save_script_rules =/// `..` segment. Operator-controlled symlinks already inside CWD are out/// of scope — the threat we close here is "client supplies an arbitrary/// path string".-pub const unsafe_path_message = "path must be relative and must not contain '..' segments";--/// The cwd is the server's, not one the user picked, so report where a file-/// really went.-pub fn absolutePath(arena: std.mem.Allocator, path: []const u8) []const u8 {- return std.Io.Dir.cwd().realPathFileAlloc(lp.io, path, arena) catch path;-}-pub fn isPathSafe(path: []const u8) bool {if (path.len == 0) return false;if (std.fs.path.isAbsolute(path)) return false;@@ -233,6 +224,14 @@ pub fn isPathSafe(path: []const u8) bool {return true;}+pub const unsafe_path_message = "path must be relative and must not contain '..' segments";++/// The cwd is the server's, not one the user picked, so report where a file+/// really went.+pub fn absolutePath(arena: std.mem.Allocator, path: []const u8) []const u8 {+ return std.Io.Dir.cwd().realPathFileAlloc(lp.io, path, arena) catch path;+}+/// Hand-written so per-tool semantics (record/heal/locator/data) and/// LLM-facing metadata (`definition`) live as exhaustive switches on the/// tag — adding a new tool is a compile error until each predicate AND@@ -320,13 +319,6 @@ pub const Tool = enum {};}- pub fn needsLocator(self: Tool) bool {- for (self.replayRequires()) |field| {- if (std.mem.eql(u8, field, "selector")) return true;- }- return false;- }-/// Result is data the caller probably wants on stdout (extracted JSON,/// markdown, evaluate return value) rather than a status line on stderr.pub fn producesData(self: Tool) bool {@@ -1322,9 +1314,8 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNo/// The node a read tool works on: the selector match, the registry node, or/// the whole document. All three live in the current frame.fn resolveScope(session: *lp.Session, registry: *CDPNode.Registry, page: *lp.Frame, selector: ?[]const u8, node_id: ?CDPNode.Id) ToolError!*DOMNode {- if (selector) |sel| return (try resolveBySelector(session, sel)).node;- if (node_id) |nid| return (try resolveNodeAndPage(session, registry, nid)).node;- return page.document.asNode();+ if (selector == null and node_id == null) return page.document.asNode();+ return (try resolveTarget(session, registry, selector, node_id)).node;}const HtmlParams = struct {@@ -1345,7 +1336,7 @@ fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Rif (args.selector == null and args.backendNodeId == null) {lp.dump.root(page.document, opts, &aw.writer, page) catch return ToolError.InternalError;} else {- const node = try resolveScope(session, registry, page, args.selector, args.backendNodeId);+ const node = (try resolveTarget(session, registry, args.selector, args.backendNodeId)).node;lp.dump.deep(node, opts, &aw.writer, page) catch return ToolError.InternalError;}return aw.written();@@ -1368,19 +1359,18 @@ fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDP}const page = try ensurePage(session, registry, args.url, args.timeout);const node = try resolveScope(session, registry, page, args.selector, args.backendNodeId);- const prepared = lp.screenshot.prepare(arena, node, .fromViewport(page._page.getViewport(), args.fullPage), page) catch- return ToolError.InternalError;- const width = prepared.opts.width;+ const opts: lp.screenshot.Opts = .fromViewport(page._page.getViewport(), args.fullPage);+ const prepared = lp.screenshot.prepare(arena, node, opts, page) catch return ToolError.InternalError;if (args.path) |path| {const height = writePng(&prepared, path) catch |err| return .{.text = std.fmt.allocPrint(arena, "could not write {s}: {s}", .{ path, @errorName(err) }) catch return ToolError.OutOfMemory,.is_error = true,};- return .{ .text = std.fmt.allocPrint(arena, "Saved {d}x{d} PNG to {s}", .{ width, height, absolutePath(arena, path) }) catch return ToolError.OutOfMemory };+ return .{ .text = std.fmt.allocPrint(arena, "Saved {d}x{d} PNG to {s}", .{ opts.width, height, absolutePath(arena, path) }) catch return ToolError.OutOfMemory };}return .{- .text = std.fmt.allocPrint(arena, "PNG, {d}px wide", .{width}) catch return ToolError.OutOfMemory,+ .text = std.fmt.allocPrint(arena, "PNG, {d}px wide", .{opts.width}) catch return ToolError.OutOfMemory,.image = prepared,};}diff --git a/src/mcp/protocol.zig b/src/mcp/protocol.zigindex 8b657bd4b..f4a6f4a46 100644--- a/src/mcp/protocol.zig+++ b/src/mcp/protocol.zig@@ -167,12 +167,6 @@ pub const CallParams = struct {arguments: ?std.json.Value = null,};-pub const ImageContent = struct {- type: []const u8 = "image",- data: @import("../browser/screenshot.zig").Prepared,- mimeType: []const u8 = "image/png",-};-pub fn TextContent(comptime T: type) type {return struct {type: []const u8 = "text",@@ -180,9 +174,19 @@ pub fn TextContent(comptime T: type) type {};}-pub fn CallToolResult(comptime T: type) type {+/// `T` serializes as the base64 payload.+pub fn ImageContent(comptime T: type) type {+ return struct {+ type: []const u8 = "image",+ data: T,+ mimeType: []const u8,+ };+}++/// `Content` is the content array: a slice or tuple of `TextContent`/`ImageContent`.+pub fn CallToolResult(comptime Content: type) type {return struct {- content: []const TextContent(T),+ content: Content,isError: bool = false,};}diff --git a/src/mcp/tools.zig b/src/mcp/tools.zigindex 259e1c497..66f86dbe3 100644--- a/src/mcp/tools.zig+++ b/src/mcp/tools.zig@@ -173,8 +173,9 @@ fn dispatchBrowserTool(};if (result.image) |image| {- return server.sendResult(id, .{- .content = .{ protocol.ImageContent{ .data = image }, protocol.TextContent([]const u8){ .text = result.text } },+ const Content = struct { protocol.ImageContent(lp.screenshot.Prepared), protocol.TextContent([]const u8) };+ return server.sendResult(id, protocol.CallToolResult(Content){+ .content = .{ .{ .data = image, .mimeType = "image/png" }, .{ .text = result.text } },.isError = result.is_error,});}@@ -289,7 +290,7 @@ fn writeScript(path: []const u8, content: []const u8) !void {fn sendToolResultText(server: *Server, id: std.json.Value, msg: []const u8, is_error: bool) !void {const content = [_]protocol.TextContent([]const u8){.{ .text = msg }};- try server.sendResult(id, protocol.CallToolResult([]const u8){ .content = &content, .isError = is_error });+ try server.sendResult(id, protocol.CallToolResult([]const protocol.TextContent([]const u8)){ .content = &content, .isError = is_error });}fn sendErrorContent(server: *Server, id: std.json.Value, msg: []const u8) !void {diff --git a/src/script/skill.zig b/src/script/skill.zigindex 3cbf47b7a..7ad4306c9 100644--- a/src/script/skill.zig+++ b/src/script/skill.zig@@ -180,7 +180,9 @@ fn positionalOptional(s: *const Schema, p: []const u8) bool {if (s.findField(p)) |f| {if (f.default_true) return true;}- if (std.mem.eql(u8, p, "selector") and s.tool.needsLocator()) return false;+ for (s.tool.replayRequires()) |r| {+ if (std.mem.eql(u8, r, p)) return false;+ }for (s.required) |r| {if (std.mem.eql(u8, r, p)) return false;}