Lightpanda: the headless browser designed for AI and automation
tools: add a screenshot tool
Adrià Arrufat9239e1eadddb
diff --git a/src/agent/Agent.zig b/src/agent/Agent.zigindex 2895737b9..83b2b4127 100644--- a/src/agent/Agent.zig+++ b/src/agent/Agent.zig@@ -1488,7 +1488,7 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool.tool_call => |t| t,else => return .{ .text = "internal: command has no tool mapping", .is_error = true },};- return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args) catch |err| .{+ const result = browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args) catch |err| return .{.text = switch (err) {error.OutOfMemory => "out of memory",error.FrameNotLoaded => "no page loaded — run /goto <url> first",@@ -1496,6 +1496,9 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool},.is_error = true,};+ // Tool results reach the model as text; an inline image has nowhere to go.+ if (result.image != null) return .{ .text = "screenshot needs `path` here; the inline image is MCP-only", .is_error = true };+ return result;}/// Data output (/extract, /evaluate, /markdown, /tree, …) → plain stdout ondiff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 774ab7cee..643bd0388 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -39,8 +39,9 @@ const Selector = @import("webapi/selector/Selector.zig");/// recordable as JavaScript agent scripts.pub const driver_guidance =\\You are driving Lightpanda, a headless browser, through text tools:- \\no screenshots, no images, no PDFs — you reason over pages as a- \\semantic tree, markdown or HTML.+ \\you reason over pages as a semantic tree, markdown or HTML. `screenshot`+ \\renders that text layout as a PNG (no images, fonts or CSS) — use it+ \\for spatial layout, not as a primary read.\\\\Reading pages (cheap → expensive — prefer cheaper):\\- `tree` → semantic overview (role, name, value, backendNodeId per@@ -234,6 +235,7 @@ pub const Tool = enum {search,markdown,html,+ screenshot,links,evaluate,extract,@@ -263,7 +265,7 @@ pub const Tool = enum {/// with noise.pub fn isRecorded(self: Tool) bool {return switch (self) {- .goto, .evaluate, .extract, .click, .fill, .scroll, .waitForSelector, .waitForScript, .waitForState, .hover, .press, .selectOption, .setChecked => true,+ .goto, .screenshot, .evaluate, .extract, .click, .fill, .scroll, .waitForSelector, .waitForScript, .waitForState, .hover, .press, .selectOption, .setChecked => true,.search, .markdown, .html, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,};}@@ -275,7 +277,7 @@ pub const Tool = enum {pub fn isAsync(self: Tool) bool {return switch (self) {.goto => true,- .evaluate, .extract, .click, .fill, .scroll, .waitForSelector, .waitForScript, .waitForState, .hover, .press, .selectOption, .setChecked, .search, .markdown, .html, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,+ .evaluate, .extract, .click, .fill, .scroll, .waitForSelector, .waitForScript, .waitForState, .hover, .press, .selectOption, .setChecked, .search, .markdown, .html, .screenshot, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,};}@@ -285,7 +287,7 @@ pub const Tool = enum {pub fn waitsForReadiness(self: Tool) bool {return switch (self) {.waitForSelector, .waitForScript, .waitForState => true,- .goto, .evaluate, .extract, .click, .fill, .scroll, .hover, .press, .selectOption, .setChecked, .search, .markdown, .html, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,+ .goto, .evaluate, .extract, .click, .fill, .scroll, .hover, .press, .selectOption, .setChecked, .search, .markdown, .html, .screenshot, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,};}@@ -295,7 +297,7 @@ pub const Tool = enum {/// `getCookies` (`url` filters, not navigates).pub fn navigatesToUrl(self: Tool) bool {return switch (self) {- .markdown, .html, .links, .tree, .interactiveElements, .structuredData, .detectForms => true,+ .markdown, .html, .screenshot, .links, .tree, .interactiveElements, .structuredData, .detectForms => true,.goto, .search, .evaluate, .extract, .nodeDetails, .click, .fill, .scroll, .waitForSelector, .waitForScript, .waitForState, .hover, .press, .selectOption, .setChecked, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,};}@@ -306,7 +308,7 @@ pub const Tool = enum {pub fn needsLocator(self: Tool) bool {return switch (self) {.click, .fill, .hover, .selectOption, .setChecked => true,- .goto, .search, .markdown, .html, .links, .evaluate, .extract, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .scroll, .waitForSelector, .waitForScript, .waitForState, .press, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,+ .goto, .search, .markdown, .html, .screenshot, .links, .evaluate, .extract, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .scroll, .waitForSelector, .waitForScript, .waitForState, .press, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,};}@@ -315,7 +317,7 @@ pub const Tool = enum {pub fn producesData(self: Tool) bool {return switch (self) {.search, .markdown, .html, .links, .evaluate, .extract, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => true,- .goto, .click, .fill, .scroll, .waitForSelector, .waitForScript, .waitForState, .hover, .press, .selectOption, .setChecked => false,+ .goto, .screenshot, .click, .fill, .scroll, .waitForSelector, .waitForScript, .waitForState, .hover, .press, .selectOption, .setChecked => false,};}@@ -398,6 +400,23 @@ pub const Tool = enum {\\}),},+ .screenshot => .{+ .description = "Render the page, or one node, as a PNG: the text layout Lightpanda computes, not a pixel-accurate browser rendering (no images, fonts or CSS colours). With `path`, writes the file and returns its location; without it, returns the image inline (MCP only). Use it to see spatial layout; read content with `markdown`/`tree`.",+ .summary = "Screenshot of the page or a node",+ .input_schema = minify(+ \\{+ \\ "type": "object",+ \\ "properties": {+ \\ "path": { "type": "string", "description": "Optional relative path (no '..' segments) to write the PNG to. Created or overwritten. Required outside MCP." },+ \\ "selector": { "type": "string", "description": "Optional CSS selector. When set, render only that element." },+ \\ "backendNodeId": { "type": "integer", "description": "Optional backend node ID. When set, render only that node. 0 is treated as omitted." },+ \\ "fullPage": { "type": "boolean", "description": "Render the whole content height instead of one viewport. Defaults to false." },+ \\ "url": { "type": "string", "description": "Optional URL to navigate to before rendering." },+ \\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." }+ \\ }+ \\}+ ),+ },.links => .{.description = "Extract the visible links in the opened page as JSON objects with `text` (anchor text, falling back to aria-label/title/image alt), `href` (resolved URL), and `backendNodeId` (pass to click/nodeDetails). One entry per href; hidden links are omitted. If a url is provided, it navigates to that url first.",.summary = "List all links on the page",@@ -703,7 +722,7 @@ pub const Tool = enum {};pub fn minify(comptime json: []const u8) []const u8 {- @setEvalBranchQuota(10000);+ @setEvalBranchQuota(100_000);return comptime blk: {var buf: [json.len]u8 = undefined;var len: usize = 0;@@ -790,6 +809,8 @@ pub fn errorMessage(err: ToolError) []const u8 {pub const ToolResult = struct {text: []const u8,is_error: bool = false,+ /// A rendered PNG the caller streams out itself (MCP image content).+ image: ?lp.screenshot.Prepared = null,};pub const GotoParams = struct {@@ -862,6 +883,7 @@ fn dispatch(.search => execSearch(arena, substituted),.markdown => .{ .text = try execMarkdown(arena, session, registry, substituted) },.html => .{ .text = try execHtml(arena, session, registry, substituted) },+ .screenshot => try execScreenshot(arena, session, registry, substituted),.links => .{ .text = try execLinks(arena, session, registry, substituted) },.tree => .{ .text = try execTree(arena, session, registry, substituted) },.nodeDetails => .{ .text = try execNodeDetails(arena, session, registry, substituted) },@@ -1309,6 +1331,64 @@ fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Rreturn aw.written();}+fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError!ToolResult {+ const Params = struct {+ path: ?[]const u8 = null,+ selector: ?[]const u8 = null,+ backendNodeId: ?CDPNode.Id = null,+ fullPage: bool = false,+ url: ?[:0]const u8 = null,+ timeout: ?u32 = null,+ };+ const args = try parseArgsOrDefault(Params, arena, arguments);+ if (args.path) |path| {+ if (!isPathSafe(path)) return .{ .text = "path must be relative and must not contain '..' segments", .is_error = true };+ }+ const page = try ensurePage(session, registry, args.url, args.timeout);++ var node = page.document.asNode();+ var frame = page;+ if (args.selector) |sel| {+ const resolved = try resolveBySelector(session, sel);+ node = resolved.node;+ frame = resolved.page;+ } else if (args.backendNodeId) |nid| {+ const resolved = try resolveNodeAndPage(session, registry, nid);+ node = resolved.node;+ frame = resolved.page;+ }++ const viewport = page._page.getViewport();+ const prepared = lp.screenshot.prepare(arena, node, .{+ .width = viewport.width,+ .height = if (args.fullPage) 0 else viewport.height,+ .scale = viewport.scale,+ }, frame) catch return ToolError.InternalError;++ const path = args.path orelse return .{+ .text = std.fmt.allocPrint(arena, "PNG, {d}px wide", .{viewport.width}) catch return ToolError.OutOfMemory,+ .image = prepared,+ };++ const height = writePng(&prepared, path) catch |err| return .{+ .text = std.fmt.allocPrint(arena, "could not write {s}: {s}", .{ path, @errorName(err) }) catch return ToolError.OutOfMemory,+ .is_error = true,+ };+ // Absolute path: the cwd is the server's, not one the user picked.+ const where = std.Io.Dir.cwd().realPathFileAlloc(lp.io, path, arena) catch path;+ return .{ .text = std.fmt.allocPrint(arena, "Saved {d}x{d} PNG to {s}", .{ viewport.width, height, where }) catch return ToolError.OutOfMemory };+}++fn writePng(prepared: *const lp.screenshot.Prepared, path: []const u8) !u32 {+ const file = try std.Io.Dir.cwd().createFile(lp.io, path, .{ .truncate = true });+ defer file.close(lp.io);+ var buf: [8 * 1024]u8 = undefined;+ var writer = file.writer(lp.io, &buf);+ const height = try prepared.write(&writer.interface);+ try writer.interface.flush();+ return height;+}+fn execLinks(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError![]const u8 {const Params = struct {limit: ?u32 = null,diff --git a/src/mcp/tools.zig b/src/mcp/tools.zigindex 0eb10ee0d..b8010379c 100644--- a/src/mcp/tools.zig+++ b/src/mcp/tools.zig@@ -38,6 +38,8 @@ fn annotations(tool: BrowserTool) protocol.ToolAnnotations {// Drains the buffer: a second call returns something else..consoleLogs => .{ .readOnlyHint = true, .destructiveHint = false, .openWorldHint = false },.goto, .search, .markdown, .html, .links, .tree, .interactiveElements, .structuredData, .detectForms => .{ .destructiveHint = false, .idempotentHint = true },+ // Overwrites `path` when given.+ .screenshot => .{ .idempotentHint = true },.waitForSelector, .waitForScript, .waitForState => .{ .destructiveHint = false, .idempotentHint = true, .openWorldHint = false },.evaluate, .click, .press => .{},.fill, .selectOption, .setChecked, .hover => .{ .destructiveHint = false, .idempotentHint = true, .openWorldHint = false },@@ -170,9 +172,42 @@ fn dispatchBrowserTool(return server.sendError(id, code, browser_tools.errorMessage(err));};+ if (result.image) |image| {+ return server.sendResult(id, ImageResult{ .image = image, .text = result.text });+ }try sendToolResultText(server, id, result.text, result.is_error);}+/// MCP image content: the PNG streams straight into the message as base64.+const ImageResult = struct {+ image: lp.screenshot.Prepared,+ text: []const u8,++ pub fn jsonStringify(self: @This(), jw: anytype) !void {+ try jw.beginObject();+ try jw.objectField("content");+ try jw.beginArray();+ try jw.beginObject();+ try jw.objectField("type");+ try jw.write("image");+ try jw.objectField("data");+ try jw.write(self.image);+ try jw.objectField("mimeType");+ try jw.write("image/png");+ try jw.endObject();+ try jw.beginObject();+ try jw.objectField("type");+ try jw.write("text");+ try jw.objectField("text");+ try jw.write(self.text);+ try jw.endObject();+ try jw.endArray();+ try jw.objectField("isError");+ try jw.write(false);+ try jw.endObject();+ }+};+fn surfacesErrorInBand(tool: BrowserTool) bool {return tool == .evaluate or tool == .extract;}@@ -1443,6 +1478,43 @@ test "MCP - html: maxBytes truncation and strip" {try testing.expect(std.mem.indexOf(u8, out.written(), "[truncated]") != null);}+test "MCP - screenshot: inline image, file, unsafe path" {+ var out: std.Io.Writer.Allocating = .init(testing.arena_allocator);+ const server = try testLoadPage("http://localhost:9582/src/browser/tests/mcp_actions.html", &out.writer);+ defer server.deinit();++ const inline_call =+ \\{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"screenshot"}}+ ;+ try router.handleMessage(server, testing.arena_allocator, inline_call);+ try testing.expect(std.mem.indexOf(u8, out.written(), "\"type\":\"image\"") != null);+ try testing.expect(std.mem.indexOf(u8, out.written(), "\"mimeType\":\"image/png\"") != null);+ // base64 of the PNG signature+ try testing.expect(std.mem.indexOf(u8, out.written(), "\"data\":\"iVBORw0KGgo") != null);+ try testing.expect(std.mem.indexOf(u8, out.written(), "\"isError\":false") != null);++ const path = "mcp-screenshot-test.png";+ std.Io.Dir.cwd().deleteFile(lp.io, path) catch {};+ defer std.Io.Dir.cwd().deleteFile(lp.io, path) catch {};++ out.clearRetainingCapacity();+ const to_file =+ \\{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"screenshot","arguments":{"path":"mcp-screenshot-test.png","selector":"#hoverTarget"}}}+ ;+ try router.handleMessage(server, testing.arena_allocator, to_file);+ try testing.expect(std.mem.indexOf(u8, out.written(), "Saved 1920x") != null);+ try testing.expect(std.mem.indexOf(u8, out.written(), "\"type\":\"image\"") == null);+ const png = try std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(1024 * 1024));+ try testing.expect(std.mem.startsWith(u8, png, "\x89PNG\r\n\x1a\n"));++ out.clearRetainingCapacity();+ const unsafe =+ \\{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"screenshot","arguments":{"path":"../escape.png"}}}+ ;+ try router.handleMessage(server, testing.arena_allocator, unsafe);+ try testing.expect(std.mem.indexOf(u8, out.written(), "\"isError\":true") != null);+}+test "MCP - links: dedup, hidden, text fallback, limit" {var out: std.Io.Writer.Allocating = .init(testing.arena_allocator);const server = try testLoadPage("http://localhost:9582/src/browser/tests/mcp_links.html", &out.writer);diff --git a/src/script/Runtime.zig b/src/script/Runtime.zigindex ee4ceee41..e8ddf247c 100644--- a/src/script/Runtime.zig+++ b/src/script/Runtime.zig@@ -721,6 +721,7 @@ fn callTool(};if (result.is_error) return .{ .fail = result.text };+ if (result.image != null) return .{ .fail = "screenshot needs `path` in a script" };return .{ .ok = result.text };}diff --git a/src/script/command.zig b/src/script/command.zigindex 5d7381cb1..0136e7d5d 100644--- a/src/script/command.zig+++ b/src/script/command.zig@@ -110,8 +110,10 @@ pub const Command = union(enum) {fn isRecorded(self: ToolCall) bool {if (!self.tool.isRecorded()) return false;const s = self.schema();- const args = self.args orelse return s.required.len == 0 and !self.tool.needsLocator();+ const args = self.args orelse return s.required.len == 0 and !self.tool.needsLocator() and self.tool != .screenshot;if (args != .object) return !self.tool.needsLocator();+ // An inline screenshot (no `path`) only exists as MCP image content.+ if (self.tool == .screenshot and !args.object.contains("path")) return false;const has_selector = args.object.contains("selector");if (!has_selector and (self.tool.needsLocator() or args.object.contains("backendNodeId"))) return false;@@ -498,6 +500,22 @@ test "isRecorded / producesData via tool flags" {try testing.expect(!login.isRecorded());}+test "isRecorded: screenshot needs a path to replay" {+ var arena: std.heap.ArenaAllocator = .init(testing.allocator);+ defer arena.deinit();+ const aa = arena.allocator();++ try testing.expect(!Command.fromToolCall(.screenshot, null).isRecorded());++ var without: std.json.ObjectMap = .empty;+ try without.put(aa, "selector", .{ .string = "#main" });+ try testing.expect(!Command.fromToolCall(.screenshot, .{ .object = without }).isRecorded());++ var with: std.json.ObjectMap = .empty;+ try with.put(aa, "path", .{ .string = "shot.png" });+ try testing.expect(Command.fromToolCall(.screenshot, .{ .object = with }).isRecorded());+}+test "isRecorded: args shape and locator semantics" {var arena: std.heap.ArenaAllocator = .init(testing.allocator);defer arena.deinit();diff --git a/src/script/skill.zig b/src/script/skill.zigindex 20c16de40..79749cb07 100644--- a/src/script/skill.zig+++ b/src/script/skill.zig@@ -198,6 +198,7 @@ fn note(tool: browser_tools.Tool) []const u8 {.waitForSelector => "`waitFor*` default timeout 5000 ms.",.waitForScript => "Re-evaluates page JS until truthy.",.waitForState => "",+ .screenshot => "Needs `path`; writes a PNG of the text layout.",.press => "Selector first! `page.press(\"Enter\")` binds \"Enter\" to `selector` and fails — use `page.press(null, \"Enter\")` or `page.press({ key: \"Enter\" })`.",.click, .fill, .scroll, .hover, .selectOption, .setChecked => "",.search, .markdown, .html, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => "",