Lightpanda: the headless browser designed for AI and automation
screenshot: fit() owns the inline sizing; prune() owns image expiry
Adrià Arrufat45744407afe1
diff --git a/src/agent/Agent.zig b/src/agent/Agent.zigindex a44398c54..c6366c6f1 100644--- a/src/agent/Agent.zig+++ b/src/agent/Agent.zig@@ -572,7 +572,6 @@ fn runTurn(self: *Agent, input: TurnInput) bool {error.UnsupportedAttachment, error.AttachmentReadFailed => return false,error.UserCancelled => {self.terminal.printInfo("Interrupted.", .{});- self.conversation.expireImages();self.conversation.prune();return false;},@@ -594,7 +593,6 @@ fn runTurn(self: *Agent, input: TurnInput) bool {elseself.terminal.printInfo("(no response from model)", .{});}- self.conversation.expireImages();self.conversation.prune();return true;}@@ -1491,7 +1489,7 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_toolelse => return .{ .text = "internal: command has no tool mapping", .is_error = true },};// The terminal can't show an image, but the conversation can.- return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args, .{ .inline_image = if (self.ai_client != null) .{} else null }) catch |err| .{+ return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args, .{ .inline_image = self.ai_client != null }) catch |err| .{.text = switch (err) {error.OutOfMemory => "out of memory",error.FrameNotLoaded => "no page loaded — run /goto <url> first",@@ -1938,7 +1936,7 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co/// The text plus the rendered PNG, for backends that can show the model an image.fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value) browser_tools.ToolError!zenai.provider.Client.ToolHandler.Result {- const result = try browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{ .inline_image = .{} });+ const result = try browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{ .inline_image = true });const content = capToolOutput(allocator, tool_name, result.text);return .{.content = content,@@ -2064,6 +2062,7 @@ test {_ = save;_ = settings;_ = picker;+ _ = Conversation;}test "savePrompt: save instructions followed by the rendered script skill" {@@ -2118,7 +2117,3 @@ test "capToolOutput: extract is exempt from the default cap" {try std.testing.expectEqual(extract_output_max_bytes, toolOutputCap("extract"));try std.testing.expectEqual(tool_output_max_bytes, toolOutputCap("html"));}--test {- std.testing.refAllDecls(Conversation);-}diff --git a/src/agent/Conversation.zig b/src/agent/Conversation.zigindex 22b93c0af..45c7b05de 100644--- a/src/agent/Conversation.zig+++ b/src/agent/Conversation.zig@@ -70,21 +70,32 @@ pub fn ensureSystemPrompt(self: *Conversation) !void {}}+const image_dropped_note = " (image dropped from context; call the tool again to look)";+/// Drop tool-result images older than the newest `image_keep`; their text-/// stays, with a note so the model knows to look again.-pub fn expireImages(self: *Conversation) void {- var kept: usize = 0;+/// stays, with a note so the model knows to look again. Tool results are+/// immutable, so a message that loses an image gets a re-homed copy.+fn expireImages(self: *Conversation) void {+ const arena = self.arena.allocator();+ var budget: usize = image_keep;var i = self.messages.items.len;while (i > 0) {i -= 1;- for (self.messages.items[i].tool_results orelse continue) |*res| {+ const msg = &self.messages.items[i];+ const results = msg.tool_results orelse continue;+ var stripped: ?[]zenai.provider.ToolResult = null;+ for (results, 0..) |res, n| {if (!hasImage(res.parts orelse continue)) continue;- kept += 1;- if (kept <= image_keep) continue;- const result = @constCast(res);- result.parts = null;- result.content = std.fmt.allocPrint(self.arena.allocator(), "{s} (image dropped from context; call the tool again to look)", .{res.content}) catch res.content;+ if (budget > 0) {+ budget -= 1;+ continue;+ }+ const copy = stripped orelse arena.dupe(zenai.provider.ToolResult, results) catch return;+ stripped = copy;+ copy[n].parts = null;+ copy[n].content = std.mem.concat(arena, u8, &.{ res.content, image_dropped_note }) catch res.content;}+ if (stripped) |s| msg.tool_results = s;}}@@ -93,10 +104,12 @@ fn hasImage(parts: []const zenai.provider.ContentPart) bool {return false;}-/// Cap history growth: once it exceeds `prune_high`, keep the system prompt plus-/// the most recent `prune_keep` messages, snapped to a safe boundary so a-/// tool_call isn't split from its result.+/// Cap history growth: expire stale images, then once history exceeds+/// `prune_high`, keep the system prompt plus the most recent `prune_keep`+/// messages, snapped to a safe boundary so a tool_call isn't split from its+/// result.pub fn prune(self: *Conversation) void {+ self.expireImages();const msgs = self.messages.items;if (msgs.len <= prune_high) return;const tail_start = zenai.provider.safeTruncationStart(msgs, msgs.len - prune_keep) orelse return;@@ -150,7 +163,7 @@ test "expireImages keeps the newest images and annotates the rest" {conv.expireImages();try std.testing.expect(conv.messages.items[0].tool_results.?[0].parts == null);- try std.testing.expectEqualStrings("shot 0 (image dropped from context; call the tool again to look)", conv.messages.items[0].tool_results.?[0].content);+ try std.testing.expectEqualStrings("shot 0" ++ image_dropped_note, conv.messages.items[0].tool_results.?[0].content);try std.testing.expect(conv.messages.items[1].tool_results.?[0].parts == null);try std.testing.expect(conv.messages.items[2].tool_results.?[0].parts != null);try std.testing.expect(conv.messages.items[3].tool_results.?[0].parts != null);diff --git a/src/browser/screenshot.zig b/src/browser/screenshot.zigindex a49a8147e..3098aa0ce 100644--- a/src/browser/screenshot.zig+++ b/src/browser/screenshot.zig@@ -61,7 +61,7 @@ pub fn png(arena: Allocator, node: *Node, opts: Opts, writer: *std.Io.Writer, frreturn prepared.write(writer);}-// get the height of the PNG if we were to render it.+/// The height a render at `width` would have.pub fn contentHeight(arena: Allocator, node: *Node, width: u32, frame: *Frame) !u32 {const prepared = try prepare(arena, node, .{ .width = width }, frame);return prepared.measure();@@ -97,6 +97,16 @@ pub const Prepared = struct {return self.render(&discard.writer, RENDER_MEASURE_ONLY);}+ /// Bound the render for a consumer with size limits. Layout reflows to+ /// the width, so the height is measured after narrowing, and only when it+ /// isn't already a fixed strip within the limit.+ pub fn fit(self: *Prepared, max_width: u32, max_height: u32) std.Io.Writer.Error!void {+ self.opts.width = @min(self.opts.width, max_width);+ if (self.opts.height == 0 or self.opts.height > max_height) {+ self.opts.height = @min(try self.measure(), max_height);+ }+ }+fn render(self: *const Prepared, writer: *std.Io.Writer, flags: u32) std.Io.Writer.Error!u32 {const opts = self.opts;const clip = opts.clip orelse Opts.Clip{ .x = 0, .y = 0, .width = 0, .height = 0 };diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 5f09db31d..f4856bccd 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -408,7 +408,7 @@ pub const Tool = enum {),},.screenshot => .{- .description = "Render the page, or one node, as a PNG: the text layout Lightpanda computes, not a pixel-accurate browser rendering (no images, fonts or CSS colours). With `path`, writes the file at full size and returns its location; without it, returns the image inline where the client can display one, at most 1280px wide and 4096px tall. Use it to see spatial layout; read content with `markdown`/`tree`.",+ .description = std.fmt.comptimePrint("Render the page, or one node, as a PNG: the text layout Lightpanda computes, not a pixel-accurate browser rendering (no images, fonts or CSS colours). With `path`, writes the file at full size and returns its location; without it, returns the image inline where the client can display one, at most {d}px wide and {d}px tall. Use it to see spatial layout; read content with `markdown`/`tree`.", .{ inline_image_max_width, inline_image_max_height }),.summary = "Screenshot of the page or a node",.input_schema = minify(\\{@@ -848,16 +848,15 @@ const NodeAndPage = struct { node: *DOMNode, page: *lp.Frame, target: ActionTarg/// What the caller can do with a result beyond its text.pub const CallOpts = struct {- /// Set when the caller can hand an image to a model; the limits keep a- /// screenshot within what models consume (and re-send every turn).- inline_image: ?InlineImage = null,-- pub const InlineImage = struct {- max_width: u32 = 1280,- max_height: u32 = 4096,- };+ /// The caller can hand an image to a model.+ inline_image: bool = false,};+// An inline screenshot is re-sent on every turn; keep it within what models+// consume. Files written to `path` are full size.+pub const inline_image_max_width = 1280;+pub const inline_image_max_height = 4096;+pub fn call(arena: std.mem.Allocator,session: *lp.Session,@@ -1349,7 +1348,7 @@ fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Rreturn aw.written();}-fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value, inline_image: ?CallOpts.InlineImage) ToolError!ToolResult {+fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value, inline_image: bool) ToolError!ToolResult {const Params = struct {path: ?[]const u8 = null,selector: ?[]const u8 = null,@@ -1361,7 +1360,7 @@ fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPconst args = try parseArgsOrDefault(Params, arena, arguments);if (args.path) |path| {if (!isPathSafe(path)) return .{ .text = unsafe_path_message, .is_error = true };- } else if (inline_image == null) {+ } else if (!inline_image) {return .{ .text = "pass `path`: this client cannot display an inline image", .is_error = true };}const page = try ensurePage(session, registry, args.url, args.timeout);@@ -1377,13 +1376,7 @@ fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPreturn .{ .text = std.fmt.allocPrint(arena, "Saved {d}x{d} PNG to {s}", .{ prepared.opts.width, height, absolutePath(arena, path) }) catch return ToolError.OutOfMemory };}- // Layout reflows to the width, so measure after narrowing.- const limits = inline_image.?;- prepared.opts.width = @min(prepared.opts.width, limits.max_width);- const content_height = prepared.measure() catch return ToolError.InternalError;- if (prepared.opts.height == 0 or prepared.opts.height > limits.max_height) {- prepared.opts.height = @min(content_height, limits.max_height);- }+ prepared.fit(inline_image_max_width, inline_image_max_height) catch return ToolError.InternalError;return .{.text = std.fmt.allocPrint(arena, "PNG, {d}x{d}", .{ prepared.opts.width, prepared.opts.height }) catch return ToolError.OutOfMemory,.image = prepared,diff --git a/src/mcp/Transport.zig b/src/mcp/Transport.zigindex b174373ad..e81bae88f 100644--- a/src/mcp/Transport.zig+++ b/src/mcp/Transport.zig@@ -25,6 +25,8 @@ const protocol = @import("protocol.zig");const Self = @This();+const large_response = 256 * 1024;+writer: *std.Io.Writer,mutex: std.Io.Mutex = .init,aw: std.Io.Writer.Allocating,@@ -54,7 +56,7 @@ pub fn sendResponse(self: *Self, response: anytype) !void {try self.writer.writeAll(self.aw.writer.buffered());try self.writer.flush();- // A screenshot response can be megabytes; don't keep that much parked.+ // A screenshot response is hundreds of KB; don't keep that parked.if (self.aw.writer.buffer.len > large_response) {const allocator = self.aw.allocator;self.aw.deinit();@@ -62,8 +64,6 @@ pub fn sendResponse(self: *Self, response: anytype) !void {}}-const large_response = 1024 * 1024;-pub fn sendResult(self: *Self, id: std.json.Value, result: anytype) !void {const GenericResponse = struct {jsonrpc: []const u8 = "2.0",diff --git a/src/mcp/tools.zig b/src/mcp/tools.zigindex f0c34b899..2f6602143 100644--- a/src/mcp/tools.zig+++ b/src/mcp/tools.zig@@ -156,7 +156,7 @@ fn dispatchBrowserTool(};const active = server.active_session;- const result = browser_tools.call(arena, active.session, &active.node_registry, name, arguments, .{ .inline_image = .{} }) catch |err| {+ const result = browser_tools.call(arena, active.session, &active.node_registry, name, arguments, .{ .inline_image = true }) catch |err| {// evaluate/extract surface failures in-band so the LLM can self-correct;// other tools' operational failures are protocol-level.if (surfacesErrorInBand(tool)) {@@ -1465,13 +1465,8 @@ test "MCP - screenshot: inline image, file, unsafe path" {// base64 of the PNG signaturetry testing.expect(std.mem.indexOf(u8, out.written(), "\"data\":\"iVBORw0KGgo") != null);try testing.expect(std.mem.indexOf(u8, out.written(), "\"isError\":false") != null);- // Inline images are narrowed to 1280px; IHDR width is big-endian at offset 16.+ // Inline images are narrowed to the model-facing limit.try testing.expect(std.mem.indexOf(u8, out.written(), "PNG, 1280x") != null);- const start = std.mem.indexOf(u8, out.written(), "\"data\":\"").? + "\"data\":\"".len;- const b64 = out.written()[start .. start + 32];- var header: [24]u8 = undefined;- try std.base64.standard.Decoder.decode(&header, b64);- try testing.expectEqual(1280, std.mem.readInt(u32, header[16..20], .big));const path = "mcp-screenshot-test.png";std.Io.Dir.cwd().deleteFile(lp.io, path) catch {};