Skip to content

Lightpanda: the headless browser designed for AI and automation

screenshot: fit() owns the inline sizing; prune() owns image expiry

Adrià Arrufat051a70cfdd38
diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig
index a44398c54..c6366c6f1 100644
--- a/src/agent/Agent.zig
+++ b/src/agent/Agent.zig
@@ -572,7 +572,6 @@ fn runTurn(self: *Agent, input: TurnInput) bool {
error.UnsupportedAttachment, error.AttachmentReadFailed => return false,
error.UserCancelled => {
self.terminal.printInfo("Interrupted.", .{});
- self.conversation.expireImages();
self.conversation.prune();
return false;
},
@@ -594,7 +593,6 @@ fn runTurn(self: *Agent, input: TurnInput) bool {
else
self.terminal.printInfo("(no response from model)", .{});
}
- self.conversation.expireImages();
self.conversation.prune();
return true;
}
@@ -1491,7 +1489,7 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool
else => return .{ .text = "internal: command has no tool mapping", .is_error = true },
};
// The terminal can't show an image, but the conversation can.
- return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args, .{ .inline_image = if (self.ai_client != null) .{} else null }) catch |err| .{
+ return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args, .{ .inline_image = self.ai_client != null }) catch |err| .{
.text = switch (err) {
error.OutOfMemory => "out of memory",
error.FrameNotLoaded => "no page loaded — run /goto <url> first",
@@ -1938,7 +1936,7 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co
/// The text plus the rendered PNG, for backends that can show the model an image.
fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value) browser_tools.ToolError!zenai.provider.Client.ToolHandler.Result {
- const result = try browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{ .inline_image = .{} });
+ const result = try browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{ .inline_image = true });
const content = capToolOutput(allocator, tool_name, result.text);
return .{
.content = content,
@@ -2064,6 +2062,7 @@ test {
_ = save;
_ = settings;
_ = picker;
+ _ = Conversation;
}
test "savePrompt: save instructions followed by the rendered script skill" {
@@ -2118,7 +2117,3 @@ test "capToolOutput: extract is exempt from the default cap" {
try std.testing.expectEqual(extract_output_max_bytes, toolOutputCap("extract"));
try std.testing.expectEqual(tool_output_max_bytes, toolOutputCap("html"));
}
-
-test {
- std.testing.refAllDecls(Conversation);
-}
diff --git a/src/agent/Conversation.zig b/src/agent/Conversation.zig
index 22b93c0af..45c7b05de 100644
--- a/src/agent/Conversation.zig
+++ b/src/agent/Conversation.zig
@@ -70,21 +70,32 @@ pub fn ensureSystemPrompt(self: *Conversation) !void {
}
}
+const image_dropped_note = " (image dropped from context; call the tool again to look)";
+
/// Drop tool-result images older than the newest `image_keep`; their text
-/// stays, with a note so the model knows to look again.
-pub fn expireImages(self: *Conversation) void {
- var kept: usize = 0;
+/// stays, with a note so the model knows to look again. Tool results are
+/// immutable, so a message that loses an image gets a re-homed copy.
+fn expireImages(self: *Conversation) void {
+ const arena = self.arena.allocator();
+ var budget: usize = image_keep;
var i = self.messages.items.len;
while (i > 0) {
i -= 1;
- for (self.messages.items[i].tool_results orelse continue) |*res| {
+ const msg = &self.messages.items[i];
+ const results = msg.tool_results orelse continue;
+ var stripped: ?[]zenai.provider.ToolResult = null;
+ for (results, 0..) |res, n| {
if (!hasImage(res.parts orelse continue)) continue;
- kept += 1;
- if (kept <= image_keep) continue;
- const result = @constCast(res);
- result.parts = null;
- result.content = std.fmt.allocPrint(self.arena.allocator(), "{s} (image dropped from context; call the tool again to look)", .{res.content}) catch res.content;
+ if (budget > 0) {
+ budget -= 1;
+ continue;
+ }
+ const copy = stripped orelse arena.dupe(zenai.provider.ToolResult, results) catch return;
+ stripped = copy;
+ copy[n].parts = null;
+ copy[n].content = std.mem.concat(arena, u8, &.{ res.content, image_dropped_note }) catch res.content;
}
+ if (stripped) |s| msg.tool_results = s;
}
}
@@ -93,10 +104,12 @@ fn hasImage(parts: []const zenai.provider.ContentPart) bool {
return false;
}
-/// Cap history growth: once it exceeds `prune_high`, keep the system prompt plus
-/// the most recent `prune_keep` messages, snapped to a safe boundary so a
-/// tool_call isn't split from its result.
+/// Cap history growth: expire stale images, then once history exceeds
+/// `prune_high`, keep the system prompt plus the most recent `prune_keep`
+/// messages, snapped to a safe boundary so a tool_call isn't split from its
+/// result.
pub fn prune(self: *Conversation) void {
+ self.expireImages();
const msgs = self.messages.items;
if (msgs.len <= prune_high) return;
const tail_start = zenai.provider.safeTruncationStart(msgs, msgs.len - prune_keep) orelse return;
@@ -150,7 +163,7 @@ test "expireImages keeps the newest images and annotates the rest" {
conv.expireImages();
try std.testing.expect(conv.messages.items[0].tool_results.?[0].parts == null);
- try std.testing.expectEqualStrings("shot 0 (image dropped from context; call the tool again to look)", conv.messages.items[0].tool_results.?[0].content);
+ try std.testing.expectEqualStrings("shot 0" ++ image_dropped_note, conv.messages.items[0].tool_results.?[0].content);
try std.testing.expect(conv.messages.items[1].tool_results.?[0].parts == null);
try std.testing.expect(conv.messages.items[2].tool_results.?[0].parts != null);
try std.testing.expect(conv.messages.items[3].tool_results.?[0].parts != null);
diff --git a/src/browser/screenshot.zig b/src/browser/screenshot.zig
index a49a8147e..3098aa0ce 100644
--- a/src/browser/screenshot.zig
+++ b/src/browser/screenshot.zig
@@ -61,7 +61,7 @@ pub fn png(arena: Allocator, node: *Node, opts: Opts, writer: *std.Io.Writer, fr
return prepared.write(writer);
}
-// get the height of the PNG if we were to render it.
+/// The height a render at `width` would have.
pub fn contentHeight(arena: Allocator, node: *Node, width: u32, frame: *Frame) !u32 {
const prepared = try prepare(arena, node, .{ .width = width }, frame);
return prepared.measure();
@@ -97,6 +97,16 @@ pub const Prepared = struct {
return self.render(&discard.writer, RENDER_MEASURE_ONLY);
}
+ /// Bound the render for a consumer with size limits. Layout reflows to
+ /// the width, so the height is measured after narrowing, and only when it
+ /// isn't already a fixed strip within the limit.
+ pub fn fit(self: *Prepared, max_width: u32, max_height: u32) std.Io.Writer.Error!void {
+ self.opts.width = @min(self.opts.width, max_width);
+ if (self.opts.height == 0 or self.opts.height > max_height) {
+ self.opts.height = @min(try self.measure(), max_height);
+ }
+ }
+
fn render(self: *const Prepared, writer: *std.Io.Writer, flags: u32) std.Io.Writer.Error!u32 {
const opts = self.opts;
const clip = opts.clip orelse Opts.Clip{ .x = 0, .y = 0, .width = 0, .height = 0 };
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index 5f09db31d..f4856bccd 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -408,7 +408,7 @@ pub const Tool = enum {
),
},
.screenshot => .{
- .description = "Render the page, or one node, as a PNG: the text layout Lightpanda computes, not a pixel-accurate browser rendering (no images, fonts or CSS colours). With `path`, writes the file at full size and returns its location; without it, returns the image inline where the client can display one, at most 1280px wide and 4096px tall. Use it to see spatial layout; read content with `markdown`/`tree`.",
+ .description = std.fmt.comptimePrint("Render the page, or one node, as a PNG: the text layout Lightpanda computes, not a pixel-accurate browser rendering (no images, fonts or CSS colours). With `path`, writes the file at full size and returns its location; without it, returns the image inline where the client can display one, at most {d}px wide and {d}px tall. Use it to see spatial layout; read content with `markdown`/`tree`.", .{ inline_image_max_width, inline_image_max_height }),
.summary = "Screenshot of the page or a node",
.input_schema = minify(
\\{
@@ -848,16 +848,15 @@ const NodeAndPage = struct { node: *DOMNode, page: *lp.Frame, target: ActionTarg
/// What the caller can do with a result beyond its text.
pub const CallOpts = struct {
- /// Set when the caller can hand an image to a model; the limits keep a
- /// screenshot within what models consume (and re-send every turn).
- inline_image: ?InlineImage = null,
-
- pub const InlineImage = struct {
- max_width: u32 = 1280,
- max_height: u32 = 4096,
- };
+ /// The caller can hand an image to a model.
+ inline_image: bool = false,
};
+// An inline screenshot is re-sent on every turn; keep it within what models
+// consume. Files written to `path` are full size.
+pub const inline_image_max_width = 1280;
+pub const inline_image_max_height = 4096;
+
pub fn call(
arena: std.mem.Allocator,
session: *lp.Session,
@@ -1349,7 +1348,7 @@ fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.R
return aw.written();
}
-fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value, inline_image: ?CallOpts.InlineImage) ToolError!ToolResult {
+fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value, inline_image: bool) ToolError!ToolResult {
const Params = struct {
path: ?[]const u8 = null,
selector: ?[]const u8 = null,
@@ -1361,7 +1360,7 @@ fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDP
const args = try parseArgsOrDefault(Params, arena, arguments);
if (args.path) |path| {
if (!isPathSafe(path)) return .{ .text = unsafe_path_message, .is_error = true };
- } else if (inline_image == null) {
+ } else if (!inline_image) {
return .{ .text = "pass `path`: this client cannot display an inline image", .is_error = true };
}
const page = try ensurePage(session, registry, args.url, args.timeout);
@@ -1377,13 +1376,7 @@ fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDP
return .{ .text = std.fmt.allocPrint(arena, "Saved {d}x{d} PNG to {s}", .{ prepared.opts.width, height, absolutePath(arena, path) }) catch return ToolError.OutOfMemory };
}
- // Layout reflows to the width, so measure after narrowing.
- const limits = inline_image.?;
- prepared.opts.width = @min(prepared.opts.width, limits.max_width);
- const content_height = prepared.measure() catch return ToolError.InternalError;
- if (prepared.opts.height == 0 or prepared.opts.height > limits.max_height) {
- prepared.opts.height = @min(content_height, limits.max_height);
- }
+ prepared.fit(inline_image_max_width, inline_image_max_height) catch return ToolError.InternalError;
return .{
.text = std.fmt.allocPrint(arena, "PNG, {d}x{d}", .{ prepared.opts.width, prepared.opts.height }) catch return ToolError.OutOfMemory,
.image = prepared,
diff --git a/src/mcp/Transport.zig b/src/mcp/Transport.zig
index b174373ad..e81bae88f 100644
--- a/src/mcp/Transport.zig
+++ b/src/mcp/Transport.zig
@@ -25,6 +25,8 @@ const protocol = @import("protocol.zig");
const Self = @This();
+const large_response = 256 * 1024;
+
writer: *std.Io.Writer,
mutex: std.Io.Mutex = .init,
aw: std.Io.Writer.Allocating,
@@ -54,7 +56,7 @@ pub fn sendResponse(self: *Self, response: anytype) !void {
try self.writer.writeAll(self.aw.writer.buffered());
try self.writer.flush();
- // A screenshot response can be megabytes; don't keep that much parked.
+ // A screenshot response is hundreds of KB; don't keep that parked.
if (self.aw.writer.buffer.len > large_response) {
const allocator = self.aw.allocator;
self.aw.deinit();
@@ -62,8 +64,6 @@ pub fn sendResponse(self: *Self, response: anytype) !void {
}
}
-const large_response = 1024 * 1024;
-
pub fn sendResult(self: *Self, id: std.json.Value, result: anytype) !void {
const GenericResponse = struct {
jsonrpc: []const u8 = "2.0",
diff --git a/src/mcp/tools.zig b/src/mcp/tools.zig
index f0c34b899..2f6602143 100644
--- a/src/mcp/tools.zig
+++ b/src/mcp/tools.zig
@@ -156,7 +156,7 @@ fn dispatchBrowserTool(
};
const active = server.active_session;
- const result = browser_tools.call(arena, active.session, &active.node_registry, name, arguments, .{ .inline_image = .{} }) catch |err| {
+ const result = browser_tools.call(arena, active.session, &active.node_registry, name, arguments, .{ .inline_image = true }) catch |err| {
// evaluate/extract surface failures in-band so the LLM can self-correct;
// other tools' operational failures are protocol-level.
if (surfacesErrorInBand(tool)) {
@@ -1465,13 +1465,8 @@ test "MCP - screenshot: inline image, file, unsafe path" {
// base64 of the PNG signature
try testing.expect(std.mem.indexOf(u8, out.written(), "\"data\":\"iVBORw0KGgo") != null);
try testing.expect(std.mem.indexOf(u8, out.written(), "\"isError\":false") != null);
- // Inline images are narrowed to 1280px; IHDR width is big-endian at offset 16.
+ // Inline images are narrowed to the model-facing limit.
try testing.expect(std.mem.indexOf(u8, out.written(), "PNG, 1280x") != null);
- const start = std.mem.indexOf(u8, out.written(), "\"data\":\"").? + "\"data\":\"".len;
- const b64 = out.written()[start .. start + 32];
- var header: [24]u8 = undefined;
- try std.base64.standard.Decoder.decode(&header, b64);
- try testing.expectEqual(1280, std.mem.readInt(u32, header[16..20], .big));
const path = "mcp-screenshot-test.png";
std.Io.Dir.cwd().deleteFile(lp.io, path) catch {};