Skip to content

Lightpanda: the headless browser designed for AI and automation

tools: add a screenshot tool

Adrià Arrufat9239e1eadddb
diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig
index 2895737b9..83b2b4127 100644
--- a/src/agent/Agent.zig
+++ b/src/agent/Agent.zig
@@ -1488,7 +1488,7 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool
.tool_call => |t| t,
else => return .{ .text = "internal: command has no tool mapping", .is_error = true },
};
- return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args) catch |err| .{
+ const result = browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args) catch |err| return .{
.text = switch (err) {
error.OutOfMemory => "out of memory",
error.FrameNotLoaded => "no page loaded — run /goto <url> first",
@@ -1496,6 +1496,9 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool
},
.is_error = true,
};
+ // Tool results reach the model as text; an inline image has nowhere to go.
+ if (result.image != null) return .{ .text = "screenshot needs `path` here; the inline image is MCP-only", .is_error = true };
+ return result;
}
/// Data output (/extract, /evaluate, /markdown, /tree, …) → plain stdout on
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index 774ab7cee..643bd0388 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -39,8 +39,9 @@ const Selector = @import("webapi/selector/Selector.zig");
/// recordable as JavaScript agent scripts.
pub const driver_guidance =
\\You are driving Lightpanda, a headless browser, through text tools:
- \\no screenshots, no images, no PDFs — you reason over pages as a
- \\semantic tree, markdown or HTML.
+ \\you reason over pages as a semantic tree, markdown or HTML. `screenshot`
+ \\renders that text layout as a PNG (no images, fonts or CSS) — use it
+ \\for spatial layout, not as a primary read.
\\
\\Reading pages (cheap → expensive — prefer cheaper):
\\- `tree` → semantic overview (role, name, value, backendNodeId per
@@ -234,6 +235,7 @@ pub const Tool = enum {
search,
markdown,
html,
+ screenshot,
links,
evaluate,
extract,
@@ -263,7 +265,7 @@ pub const Tool = enum {
/// with noise.
pub fn isRecorded(self: Tool) bool {
return switch (self) {
- .goto, .evaluate, .extract, .click, .fill, .scroll, .waitForSelector, .waitForScript, .waitForState, .hover, .press, .selectOption, .setChecked => true,
+ .goto, .screenshot, .evaluate, .extract, .click, .fill, .scroll, .waitForSelector, .waitForScript, .waitForState, .hover, .press, .selectOption, .setChecked => true,
.search, .markdown, .html, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,
};
}
@@ -275,7 +277,7 @@ pub const Tool = enum {
pub fn isAsync(self: Tool) bool {
return switch (self) {
.goto => true,
- .evaluate, .extract, .click, .fill, .scroll, .waitForSelector, .waitForScript, .waitForState, .hover, .press, .selectOption, .setChecked, .search, .markdown, .html, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,
+ .evaluate, .extract, .click, .fill, .scroll, .waitForSelector, .waitForScript, .waitForState, .hover, .press, .selectOption, .setChecked, .search, .markdown, .html, .screenshot, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,
};
}
@@ -285,7 +287,7 @@ pub const Tool = enum {
pub fn waitsForReadiness(self: Tool) bool {
return switch (self) {
.waitForSelector, .waitForScript, .waitForState => true,
- .goto, .evaluate, .extract, .click, .fill, .scroll, .hover, .press, .selectOption, .setChecked, .search, .markdown, .html, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,
+ .goto, .evaluate, .extract, .click, .fill, .scroll, .hover, .press, .selectOption, .setChecked, .search, .markdown, .html, .screenshot, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,
};
}
@@ -295,7 +297,7 @@ pub const Tool = enum {
/// `getCookies` (`url` filters, not navigates).
pub fn navigatesToUrl(self: Tool) bool {
return switch (self) {
- .markdown, .html, .links, .tree, .interactiveElements, .structuredData, .detectForms => true,
+ .markdown, .html, .screenshot, .links, .tree, .interactiveElements, .structuredData, .detectForms => true,
.goto, .search, .evaluate, .extract, .nodeDetails, .click, .fill, .scroll, .waitForSelector, .waitForScript, .waitForState, .hover, .press, .selectOption, .setChecked, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,
};
}
@@ -306,7 +308,7 @@ pub const Tool = enum {
pub fn needsLocator(self: Tool) bool {
return switch (self) {
.click, .fill, .hover, .selectOption, .setChecked => true,
- .goto, .search, .markdown, .html, .links, .evaluate, .extract, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .scroll, .waitForSelector, .waitForScript, .waitForState, .press, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,
+ .goto, .search, .markdown, .html, .screenshot, .links, .evaluate, .extract, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .scroll, .waitForSelector, .waitForScript, .waitForState, .press, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => false,
};
}
@@ -315,7 +317,7 @@ pub const Tool = enum {
pub fn producesData(self: Tool) bool {
return switch (self) {
.search, .markdown, .html, .links, .evaluate, .extract, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => true,
- .goto, .click, .fill, .scroll, .waitForSelector, .waitForScript, .waitForState, .hover, .press, .selectOption, .setChecked => false,
+ .goto, .screenshot, .click, .fill, .scroll, .waitForSelector, .waitForScript, .waitForState, .hover, .press, .selectOption, .setChecked => false,
};
}
@@ -398,6 +400,23 @@ pub const Tool = enum {
\\}
),
},
+ .screenshot => .{
+ .description = "Render the page, or one node, as a PNG: the text layout Lightpanda computes, not a pixel-accurate browser rendering (no images, fonts or CSS colours). With `path`, writes the file and returns its location; without it, returns the image inline (MCP only). Use it to see spatial layout; read content with `markdown`/`tree`.",
+ .summary = "Screenshot of the page or a node",
+ .input_schema = minify(
+ \\{
+ \\ "type": "object",
+ \\ "properties": {
+ \\ "path": { "type": "string", "description": "Optional relative path (no '..' segments) to write the PNG to. Created or overwritten. Required outside MCP." },
+ \\ "selector": { "type": "string", "description": "Optional CSS selector. When set, render only that element." },
+ \\ "backendNodeId": { "type": "integer", "description": "Optional backend node ID. When set, render only that node. 0 is treated as omitted." },
+ \\ "fullPage": { "type": "boolean", "description": "Render the whole content height instead of one viewport. Defaults to false." },
+ \\ "url": { "type": "string", "description": "Optional URL to navigate to before rendering." },
+ \\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." }
+ \\ }
+ \\}
+ ),
+ },
.links => .{
.description = "Extract the visible links in the opened page as JSON objects with `text` (anchor text, falling back to aria-label/title/image alt), `href` (resolved URL), and `backendNodeId` (pass to click/nodeDetails). One entry per href; hidden links are omitted. If a url is provided, it navigates to that url first.",
.summary = "List all links on the page",
@@ -703,7 +722,7 @@ pub const Tool = enum {
};
pub fn minify(comptime json: []const u8) []const u8 {
- @setEvalBranchQuota(10000);
+ @setEvalBranchQuota(100_000);
return comptime blk: {
var buf: [json.len]u8 = undefined;
var len: usize = 0;
@@ -790,6 +809,8 @@ pub fn errorMessage(err: ToolError) []const u8 {
pub const ToolResult = struct {
text: []const u8,
is_error: bool = false,
+ /// A rendered PNG the caller streams out itself (MCP image content).
+ image: ?lp.screenshot.Prepared = null,
};
pub const GotoParams = struct {
@@ -862,6 +883,7 @@ fn dispatch(
.search => execSearch(arena, substituted),
.markdown => .{ .text = try execMarkdown(arena, session, registry, substituted) },
.html => .{ .text = try execHtml(arena, session, registry, substituted) },
+ .screenshot => try execScreenshot(arena, session, registry, substituted),
.links => .{ .text = try execLinks(arena, session, registry, substituted) },
.tree => .{ .text = try execTree(arena, session, registry, substituted) },
.nodeDetails => .{ .text = try execNodeDetails(arena, session, registry, substituted) },
@@ -1309,6 +1331,64 @@ fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.R
return aw.written();
}
+fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError!ToolResult {
+ const Params = struct {
+ path: ?[]const u8 = null,
+ selector: ?[]const u8 = null,
+ backendNodeId: ?CDPNode.Id = null,
+ fullPage: bool = false,
+ url: ?[:0]const u8 = null,
+ timeout: ?u32 = null,
+ };
+ const args = try parseArgsOrDefault(Params, arena, arguments);
+ if (args.path) |path| {
+ if (!isPathSafe(path)) return .{ .text = "path must be relative and must not contain '..' segments", .is_error = true };
+ }
+ const page = try ensurePage(session, registry, args.url, args.timeout);
+
+ var node = page.document.asNode();
+ var frame = page;
+ if (args.selector) |sel| {
+ const resolved = try resolveBySelector(session, sel);
+ node = resolved.node;
+ frame = resolved.page;
+ } else if (args.backendNodeId) |nid| {
+ const resolved = try resolveNodeAndPage(session, registry, nid);
+ node = resolved.node;
+ frame = resolved.page;
+ }
+
+ const viewport = page._page.getViewport();
+ const prepared = lp.screenshot.prepare(arena, node, .{
+ .width = viewport.width,
+ .height = if (args.fullPage) 0 else viewport.height,
+ .scale = viewport.scale,
+ }, frame) catch return ToolError.InternalError;
+
+ const path = args.path orelse return .{
+ .text = std.fmt.allocPrint(arena, "PNG, {d}px wide", .{viewport.width}) catch return ToolError.OutOfMemory,
+ .image = prepared,
+ };
+
+ const height = writePng(&prepared, path) catch |err| return .{
+ .text = std.fmt.allocPrint(arena, "could not write {s}: {s}", .{ path, @errorName(err) }) catch return ToolError.OutOfMemory,
+ .is_error = true,
+ };
+ // Absolute path: the cwd is the server's, not one the user picked.
+ const where = std.Io.Dir.cwd().realPathFileAlloc(lp.io, path, arena) catch path;
+ return .{ .text = std.fmt.allocPrint(arena, "Saved {d}x{d} PNG to {s}", .{ viewport.width, height, where }) catch return ToolError.OutOfMemory };
+}
+
+fn writePng(prepared: *const lp.screenshot.Prepared, path: []const u8) !u32 {
+ const file = try std.Io.Dir.cwd().createFile(lp.io, path, .{ .truncate = true });
+ defer file.close(lp.io);
+ var buf: [8 * 1024]u8 = undefined;
+ var writer = file.writer(lp.io, &buf);
+ const height = try prepared.write(&writer.interface);
+ try writer.interface.flush();
+ return height;
+}
+
fn execLinks(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError![]const u8 {
const Params = struct {
limit: ?u32 = null,
diff --git a/src/mcp/tools.zig b/src/mcp/tools.zig
index 0eb10ee0d..b8010379c 100644
--- a/src/mcp/tools.zig
+++ b/src/mcp/tools.zig
@@ -38,6 +38,8 @@ fn annotations(tool: BrowserTool) protocol.ToolAnnotations {
// Drains the buffer: a second call returns something else.
.consoleLogs => .{ .readOnlyHint = true, .destructiveHint = false, .openWorldHint = false },
.goto, .search, .markdown, .html, .links, .tree, .interactiveElements, .structuredData, .detectForms => .{ .destructiveHint = false, .idempotentHint = true },
+ // Overwrites `path` when given.
+ .screenshot => .{ .idempotentHint = true },
.waitForSelector, .waitForScript, .waitForState => .{ .destructiveHint = false, .idempotentHint = true, .openWorldHint = false },
.evaluate, .click, .press => .{},
.fill, .selectOption, .setChecked, .hover => .{ .destructiveHint = false, .idempotentHint = true, .openWorldHint = false },
@@ -170,9 +172,42 @@ fn dispatchBrowserTool(
return server.sendError(id, code, browser_tools.errorMessage(err));
};
+ if (result.image) |image| {
+ return server.sendResult(id, ImageResult{ .image = image, .text = result.text });
+ }
try sendToolResultText(server, id, result.text, result.is_error);
}
+/// MCP image content: the PNG streams straight into the message as base64.
+const ImageResult = struct {
+ image: lp.screenshot.Prepared,
+ text: []const u8,
+
+ pub fn jsonStringify(self: @This(), jw: anytype) !void {
+ try jw.beginObject();
+ try jw.objectField("content");
+ try jw.beginArray();
+ try jw.beginObject();
+ try jw.objectField("type");
+ try jw.write("image");
+ try jw.objectField("data");
+ try jw.write(self.image);
+ try jw.objectField("mimeType");
+ try jw.write("image/png");
+ try jw.endObject();
+ try jw.beginObject();
+ try jw.objectField("type");
+ try jw.write("text");
+ try jw.objectField("text");
+ try jw.write(self.text);
+ try jw.endObject();
+ try jw.endArray();
+ try jw.objectField("isError");
+ try jw.write(false);
+ try jw.endObject();
+ }
+};
+
fn surfacesErrorInBand(tool: BrowserTool) bool {
return tool == .evaluate or tool == .extract;
}
@@ -1443,6 +1478,43 @@ test "MCP - html: maxBytes truncation and strip" {
try testing.expect(std.mem.indexOf(u8, out.written(), "[truncated]") != null);
}
+test "MCP - screenshot: inline image, file, unsafe path" {
+ var out: std.Io.Writer.Allocating = .init(testing.arena_allocator);
+ const server = try testLoadPage("http://localhost:9582/src/browser/tests/mcp_actions.html", &out.writer);
+ defer server.deinit();
+
+ const inline_call =
+ \\{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"screenshot"}}
+ ;
+ try router.handleMessage(server, testing.arena_allocator, inline_call);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "\"type\":\"image\"") != null);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "\"mimeType\":\"image/png\"") != null);
+ // base64 of the PNG signature
+ try testing.expect(std.mem.indexOf(u8, out.written(), "\"data\":\"iVBORw0KGgo") != null);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "\"isError\":false") != null);
+
+ const path = "mcp-screenshot-test.png";
+ std.Io.Dir.cwd().deleteFile(lp.io, path) catch {};
+ defer std.Io.Dir.cwd().deleteFile(lp.io, path) catch {};
+
+ out.clearRetainingCapacity();
+ const to_file =
+ \\{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"screenshot","arguments":{"path":"mcp-screenshot-test.png","selector":"#hoverTarget"}}}
+ ;
+ try router.handleMessage(server, testing.arena_allocator, to_file);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "Saved 1920x") != null);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "\"type\":\"image\"") == null);
+ const png = try std.Io.Dir.cwd().readFileAlloc(lp.io, path, testing.arena_allocator, .limited(1024 * 1024));
+ try testing.expect(std.mem.startsWith(u8, png, "\x89PNG\r\n\x1a\n"));
+
+ out.clearRetainingCapacity();
+ const unsafe =
+ \\{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"screenshot","arguments":{"path":"../escape.png"}}}
+ ;
+ try router.handleMessage(server, testing.arena_allocator, unsafe);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "\"isError\":true") != null);
+}
+
test "MCP - links: dedup, hidden, text fallback, limit" {
var out: std.Io.Writer.Allocating = .init(testing.arena_allocator);
const server = try testLoadPage("http://localhost:9582/src/browser/tests/mcp_links.html", &out.writer);
diff --git a/src/script/Runtime.zig b/src/script/Runtime.zig
index ee4ceee41..e8ddf247c 100644
--- a/src/script/Runtime.zig
+++ b/src/script/Runtime.zig
@@ -721,6 +721,7 @@ fn callTool(
};
if (result.is_error) return .{ .fail = result.text };
+ if (result.image != null) return .{ .fail = "screenshot needs `path` in a script" };
return .{ .ok = result.text };
}
diff --git a/src/script/command.zig b/src/script/command.zig
index 5d7381cb1..0136e7d5d 100644
--- a/src/script/command.zig
+++ b/src/script/command.zig
@@ -110,8 +110,10 @@ pub const Command = union(enum) {
fn isRecorded(self: ToolCall) bool {
if (!self.tool.isRecorded()) return false;
const s = self.schema();
- const args = self.args orelse return s.required.len == 0 and !self.tool.needsLocator();
+ const args = self.args orelse return s.required.len == 0 and !self.tool.needsLocator() and self.tool != .screenshot;
if (args != .object) return !self.tool.needsLocator();
+ // An inline screenshot (no `path`) only exists as MCP image content.
+ if (self.tool == .screenshot and !args.object.contains("path")) return false;
const has_selector = args.object.contains("selector");
if (!has_selector and (self.tool.needsLocator() or args.object.contains("backendNodeId"))) return false;
@@ -498,6 +500,22 @@ test "isRecorded / producesData via tool flags" {
try testing.expect(!login.isRecorded());
}
+test "isRecorded: screenshot needs a path to replay" {
+ var arena: std.heap.ArenaAllocator = .init(testing.allocator);
+ defer arena.deinit();
+ const aa = arena.allocator();
+
+ try testing.expect(!Command.fromToolCall(.screenshot, null).isRecorded());
+
+ var without: std.json.ObjectMap = .empty;
+ try without.put(aa, "selector", .{ .string = "#main" });
+ try testing.expect(!Command.fromToolCall(.screenshot, .{ .object = without }).isRecorded());
+
+ var with: std.json.ObjectMap = .empty;
+ try with.put(aa, "path", .{ .string = "shot.png" });
+ try testing.expect(Command.fromToolCall(.screenshot, .{ .object = with }).isRecorded());
+}
+
test "isRecorded: args shape and locator semantics" {
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
diff --git a/src/script/skill.zig b/src/script/skill.zig
index 20c16de40..79749cb07 100644
--- a/src/script/skill.zig
+++ b/src/script/skill.zig
@@ -198,6 +198,7 @@ fn note(tool: browser_tools.Tool) []const u8 {
.waitForSelector => "`waitFor*` default timeout 5000 ms.",
.waitForScript => "Re-evaluates page JS until truthy.",
.waitForState => "",
+ .screenshot => "Needs `path`; writes a PNG of the text layout.",
.press => "Selector first! `page.press(\"Enter\")` binds \"Enter\" to `selector` and fails — use `page.press(null, \"Enter\")` or `page.press({ key: \"Enter\" })`.",
.click, .fill, .scroll, .hover, .selectOption, .setChecked => "",
.search, .markdown, .html, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => "",