Lightpanda: the headless browser designed for AI and automation
agent: bound what a screenshot costs the model
Adrià Arrufatc69397946888
diff --git a/src/agent/Agent.zig b/src/agent/Agent.zigindex f033575a1..a44398c54 100644--- a/src/agent/Agent.zig+++ b/src/agent/Agent.zig@@ -572,6 +572,7 @@ fn runTurn(self: *Agent, input: TurnInput) bool {error.UnsupportedAttachment, error.AttachmentReadFailed => return false,error.UserCancelled => {self.terminal.printInfo("Interrupted.", .{});+ self.conversation.expireImages();self.conversation.prune();return false;},@@ -593,6 +594,7 @@ fn runTurn(self: *Agent, input: TurnInput) bool {elseself.terminal.printInfo("(no response from model)", .{});}+ self.conversation.expireImages();self.conversation.prune();return true;}@@ -1489,7 +1491,7 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_toolelse => return .{ .text = "internal: command has no tool mapping", .is_error = true },};// The terminal can't show an image, but the conversation can.- return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args, .{ .inline_image = self.ai_client != null }) catch |err| .{+ return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args, .{ .inline_image = if (self.ai_client != null) .{} else null }) catch |err| .{.text = switch (err) {error.OutOfMemory => "out of memory",error.FrameNotLoaded => "no page loaded — run /goto <url> first",@@ -1936,7 +1938,7 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co/// The text plus the rendered PNG, for backends that can show the model an image.fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value) browser_tools.ToolError!zenai.provider.Client.ToolHandler.Result {- const result = try browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{ .inline_image = true });+ const result = try browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{ .inline_image = .{} });const content = capToolOutput(allocator, tool_name, result.text);return .{.content = content,@@ -2116,3 +2118,7 @@ test "capToolOutput: extract is exempt from the default cap" {try std.testing.expectEqual(extract_output_max_bytes, toolOutputCap("extract"));try std.testing.expectEqual(tool_output_max_bytes, toolOutputCap("html"));}++test {+ std.testing.refAllDecls(Conversation);+}diff --git a/src/agent/Conversation.zig b/src/agent/Conversation.zigindex e1e9acdc6..22b93c0af 100644--- a/src/agent/Conversation.zig+++ b/src/agent/Conversation.zig@@ -33,6 +33,8 @@ const Message = zenai.provider.Message;// system prompt plus the most recent `prune_keep`.const prune_high = 30;const prune_keep = 20;+// Every image in history is re-sent on every request; keep only the newest.+const image_keep = 2;allocator: std.mem.Allocator,/// Seeded as `messages[0]` on the first turn. Lives outside `arena` (static or@@ -68,6 +70,29 @@ pub fn ensureSystemPrompt(self: *Conversation) !void {}}+/// Drop tool-result images older than the newest `image_keep`; their text+/// stays, with a note so the model knows to look again.+pub fn expireImages(self: *Conversation) void {+ var kept: usize = 0;+ var i = self.messages.items.len;+ while (i > 0) {+ i -= 1;+ for (self.messages.items[i].tool_results orelse continue) |*res| {+ if (!hasImage(res.parts orelse continue)) continue;+ kept += 1;+ if (kept <= image_keep) continue;+ const result = @constCast(res);+ result.parts = null;+ result.content = std.fmt.allocPrint(self.arena.allocator(), "{s} (image dropped from context; call the tool again to look)", .{res.content}) catch res.content;+ }+ }+}++fn hasImage(parts: []const zenai.provider.ContentPart) bool {+ for (parts) |part| if (part == .image) return true;+ return false;+}+/// Cap history growth: once it exceeds `prune_high`, keep the system prompt plus/// the most recent `prune_keep` messages, snapped to a safe boundary so a/// tool_call isn't split from its result.@@ -109,3 +134,25 @@ fn repackTail(self: *Conversation, tail: []const Message) void {self.arena.deinit();self.arena = new_arena;}++test "expireImages keeps the newest images and annotates the rest" {+ var conv: Conversation = .init(std.testing.allocator, "sys");+ defer conv.deinit();+ const a = conv.arena.allocator();++ const image = [_]zenai.provider.ContentPart{.{ .image = .{ .data = "AAAA", .mime_type = "image/png" } }};+ for (0..4) |n| {+ const results = try a.alloc(zenai.provider.ToolResult, 1);+ results[0] = .{ .id = "c", .name = "screenshot", .content = try std.fmt.allocPrint(a, "shot {d}", .{n}), .parts = &image };+ try conv.messages.append(std.testing.allocator, .{ .role = .tool, .tool_results = results });+ }++ conv.expireImages();++ try std.testing.expect(conv.messages.items[0].tool_results.?[0].parts == null);+ try std.testing.expectEqualStrings("shot 0 (image dropped from context; call the tool again to look)", conv.messages.items[0].tool_results.?[0].content);+ try std.testing.expect(conv.messages.items[1].tool_results.?[0].parts == null);+ try std.testing.expect(conv.messages.items[2].tool_results.?[0].parts != null);+ try std.testing.expect(conv.messages.items[3].tool_results.?[0].parts != null);+ try std.testing.expectEqualStrings("shot 3", conv.messages.items[3].tool_results.?[0].content);+}diff --git a/src/browser/screenshot.zig b/src/browser/screenshot.zigindex ee50b0522..a49a8147e 100644--- a/src/browser/screenshot.zig+++ b/src/browser/screenshot.zig@@ -64,8 +64,7 @@ pub fn png(arena: Allocator, node: *Node, opts: Opts, writer: *std.Io.Writer, fr// get the height of the PNG if we were to render it.pub fn contentHeight(arena: Allocator, node: *Node, width: u32, frame: *Frame) !u32 {const prepared = try prepare(arena, node, .{ .width = width }, frame);- var discard: std.Io.Writer.Discarding = .init(&.{});- return prepared.render(&discard.writer, RENDER_MEASURE_ONLY);+ return prepared.measure();}// The DOM walk, done up front so it can fail (allocation) before any output@@ -92,6 +91,12 @@ pub const Prepared = struct {return self.render(writer, 0);}+ /// The content height at `opts.width`, without rasterizing.+ pub fn measure(self: *const Prepared) std.Io.Writer.Error!u32 {+ var discard: std.Io.Writer.Discarding = .init(&.{});+ return self.render(&discard.writer, RENDER_MEASURE_ONLY);+ }+fn render(self: *const Prepared, writer: *std.Io.Writer, flags: u32) std.Io.Writer.Error!u32 {const opts = self.opts;const clip = opts.clip orelse Opts.Clip{ .x = 0, .y = 0, .width = 0, .height = 0 };diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex d8a315ace..5f09db31d 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -408,7 +408,7 @@ pub const Tool = enum {),},.screenshot => .{- .description = "Render the page, or one node, as a PNG: the text layout Lightpanda computes, not a pixel-accurate browser rendering (no images, fonts or CSS colours). With `path`, writes the file and returns its location; without it, returns the image inline where the client can display one. Use it to see spatial layout; read content with `markdown`/`tree`.",+ .description = "Render the page, or one node, as a PNG: the text layout Lightpanda computes, not a pixel-accurate browser rendering (no images, fonts or CSS colours). With `path`, writes the file at full size and returns its location; without it, returns the image inline where the client can display one, at most 1280px wide and 4096px tall. Use it to see spatial layout; read content with `markdown`/`tree`.",.summary = "Screenshot of the page or a node",.input_schema = minify(\\{@@ -848,7 +848,14 @@ const NodeAndPage = struct { node: *DOMNode, page: *lp.Frame, target: ActionTarg/// What the caller can do with a result beyond its text.pub const CallOpts = struct {- inline_image: bool = false,+ /// Set when the caller can hand an image to a model; the limits keep a+ /// screenshot within what models consume (and re-send every turn).+ inline_image: ?InlineImage = null,++ pub const InlineImage = struct {+ max_width: u32 = 1280,+ max_height: u32 = 4096,+ };};pub fn call(@@ -1342,7 +1349,7 @@ fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Rreturn aw.written();}-fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value, inline_image: bool) ToolError!ToolResult {+fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value, inline_image: ?CallOpts.InlineImage) ToolError!ToolResult {const Params = struct {path: ?[]const u8 = null,selector: ?[]const u8 = null,@@ -1354,23 +1361,31 @@ fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPconst args = try parseArgsOrDefault(Params, arena, arguments);if (args.path) |path| {if (!isPathSafe(path)) return .{ .text = unsafe_path_message, .is_error = true };- } else if (!inline_image) {+ } else if (inline_image == null) {return .{ .text = "pass `path`: this client cannot display an inline image", .is_error = true };}const page = try ensurePage(session, registry, args.url, args.timeout);const node = try resolveScope(session, registry, page, args.selector, args.backendNodeId);- const opts: lp.screenshot.Opts = .fromViewport(page._page.getViewport(), args.fullPage);- const prepared = lp.screenshot.prepare(arena, node, opts, page) catch return ToolError.InternalError;+ var prepared = lp.screenshot.prepare(arena, node, .fromViewport(page._page.getViewport(), args.fullPage), page) catch+ return ToolError.InternalError;if (args.path) |path| {const height = writePng(&prepared, path) catch |err| return .{.text = std.fmt.allocPrint(arena, "could not write {s}: {s}", .{ path, @errorName(err) }) catch return ToolError.OutOfMemory,.is_error = true,};- return .{ .text = std.fmt.allocPrint(arena, "Saved {d}x{d} PNG to {s}", .{ opts.width, height, absolutePath(arena, path) }) catch return ToolError.OutOfMemory };+ return .{ .text = std.fmt.allocPrint(arena, "Saved {d}x{d} PNG to {s}", .{ prepared.opts.width, height, absolutePath(arena, path) }) catch return ToolError.OutOfMemory };+ }++ // Layout reflows to the width, so measure after narrowing.+ const limits = inline_image.?;+ prepared.opts.width = @min(prepared.opts.width, limits.max_width);+ const content_height = prepared.measure() catch return ToolError.InternalError;+ if (prepared.opts.height == 0 or prepared.opts.height > limits.max_height) {+ prepared.opts.height = @min(content_height, limits.max_height);}return .{- .text = std.fmt.allocPrint(arena, "PNG, {d}px wide", .{opts.width}) catch return ToolError.OutOfMemory,+ .text = std.fmt.allocPrint(arena, "PNG, {d}x{d}", .{ prepared.opts.width, prepared.opts.height }) catch return ToolError.OutOfMemory,.image = prepared,};}diff --git a/src/mcp/Transport.zig b/src/mcp/Transport.zigindex 15a1d9882..b174373ad 100644--- a/src/mcp/Transport.zig+++ b/src/mcp/Transport.zig@@ -53,8 +53,17 @@ pub fn sendResponse(self: *Self, response: anytype) !void {try self.aw.writer.writeByte('\n');try self.writer.writeAll(self.aw.writer.buffered());try self.writer.flush();++ // A screenshot response can be megabytes; don't keep that much parked.+ if (self.aw.writer.buffer.len > large_response) {+ const allocator = self.aw.allocator;+ self.aw.deinit();+ self.aw = .init(allocator);+ }}+const large_response = 1024 * 1024;+pub fn sendResult(self: *Self, id: std.json.Value, result: anytype) !void {const GenericResponse = struct {jsonrpc: []const u8 = "2.0",diff --git a/src/mcp/tools.zig b/src/mcp/tools.zigindex 66f86dbe3..f0c34b899 100644--- a/src/mcp/tools.zig+++ b/src/mcp/tools.zig@@ -156,7 +156,7 @@ fn dispatchBrowserTool(};const active = server.active_session;- const result = browser_tools.call(arena, active.session, &active.node_registry, name, arguments, .{ .inline_image = true }) catch |err| {+ const result = browser_tools.call(arena, active.session, &active.node_registry, name, arguments, .{ .inline_image = .{} }) catch |err| {// evaluate/extract surface failures in-band so the LLM can self-correct;// other tools' operational failures are protocol-level.if (surfacesErrorInBand(tool)) {@@ -1465,6 +1465,13 @@ test "MCP - screenshot: inline image, file, unsafe path" {// base64 of the PNG signaturetry testing.expect(std.mem.indexOf(u8, out.written(), "\"data\":\"iVBORw0KGgo") != null);try testing.expect(std.mem.indexOf(u8, out.written(), "\"isError\":false") != null);+ // Inline images are narrowed to 1280px; IHDR width is big-endian at offset 16.+ try testing.expect(std.mem.indexOf(u8, out.written(), "PNG, 1280x") != null);+ const start = std.mem.indexOf(u8, out.written(), "\"data\":\"").? + "\"data\":\"".len;+ const b64 = out.written()[start .. start + 32];+ var header: [24]u8 = undefined;+ try std.base64.standard.Decoder.decode(&header, b64);+ try testing.expectEqual(1280, std.mem.readInt(u32, header[16..20], .big));const path = "mcp-screenshot-test.png";std.Io.Dir.cwd().deleteFile(lp.io, path) catch {};