Skip to content

Lightpanda: the headless browser designed for AI and automation

agent: bound what a screenshot costs the model

Adrià Arrufatc69397946888
diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig
index f033575a1..a44398c54 100644
--- a/src/agent/Agent.zig
+++ b/src/agent/Agent.zig
@@ -572,6 +572,7 @@ fn runTurn(self: *Agent, input: TurnInput) bool {
error.UnsupportedAttachment, error.AttachmentReadFailed => return false,
error.UserCancelled => {
self.terminal.printInfo("Interrupted.", .{});
+ self.conversation.expireImages();
self.conversation.prune();
return false;
},
@@ -593,6 +594,7 @@ fn runTurn(self: *Agent, input: TurnInput) bool {
else
self.terminal.printInfo("(no response from model)", .{});
}
+ self.conversation.expireImages();
self.conversation.prune();
return true;
}
@@ -1489,7 +1491,7 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool
else => return .{ .text = "internal: command has no tool mapping", .is_error = true },
};
// The terminal can't show an image, but the conversation can.
- return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args, .{ .inline_image = self.ai_client != null }) catch |err| .{
+ return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args, .{ .inline_image = if (self.ai_client != null) .{} else null }) catch |err| .{
.text = switch (err) {
error.OutOfMemory => "out of memory",
error.FrameNotLoaded => "no page loaded — run /goto <url> first",
@@ -1936,7 +1938,7 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co
/// The text plus the rendered PNG, for backends that can show the model an image.
fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value) browser_tools.ToolError!zenai.provider.Client.ToolHandler.Result {
- const result = try browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{ .inline_image = true });
+ const result = try browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{ .inline_image = .{} });
const content = capToolOutput(allocator, tool_name, result.text);
return .{
.content = content,
@@ -2116,3 +2118,7 @@ test "capToolOutput: extract is exempt from the default cap" {
try std.testing.expectEqual(extract_output_max_bytes, toolOutputCap("extract"));
try std.testing.expectEqual(tool_output_max_bytes, toolOutputCap("html"));
}
+
+test {
+ std.testing.refAllDecls(Conversation);
+}
diff --git a/src/agent/Conversation.zig b/src/agent/Conversation.zig
index e1e9acdc6..22b93c0af 100644
--- a/src/agent/Conversation.zig
+++ b/src/agent/Conversation.zig
@@ -33,6 +33,8 @@ const Message = zenai.provider.Message;
// system prompt plus the most recent `prune_keep`.
const prune_high = 30;
const prune_keep = 20;
+// Every image in history is re-sent on every request; keep only the newest.
+const image_keep = 2;
allocator: std.mem.Allocator,
/// Seeded as `messages[0]` on the first turn. Lives outside `arena` (static or
@@ -68,6 +70,29 @@ pub fn ensureSystemPrompt(self: *Conversation) !void {
}
}
+/// Drop tool-result images older than the newest `image_keep`; their text
+/// stays, with a note so the model knows to look again.
+pub fn expireImages(self: *Conversation) void {
+ var kept: usize = 0;
+ var i = self.messages.items.len;
+ while (i > 0) {
+ i -= 1;
+ for (self.messages.items[i].tool_results orelse continue) |*res| {
+ if (!hasImage(res.parts orelse continue)) continue;
+ kept += 1;
+ if (kept <= image_keep) continue;
+ const result = @constCast(res);
+ result.parts = null;
+ result.content = std.fmt.allocPrint(self.arena.allocator(), "{s} (image dropped from context; call the tool again to look)", .{res.content}) catch res.content;
+ }
+ }
+}
+
+fn hasImage(parts: []const zenai.provider.ContentPart) bool {
+ for (parts) |part| if (part == .image) return true;
+ return false;
+}
+
/// Cap history growth: once it exceeds `prune_high`, keep the system prompt plus
/// the most recent `prune_keep` messages, snapped to a safe boundary so a
/// tool_call isn't split from its result.
@@ -109,3 +134,25 @@ fn repackTail(self: *Conversation, tail: []const Message) void {
self.arena.deinit();
self.arena = new_arena;
}
+
+test "expireImages keeps the newest images and annotates the rest" {
+ var conv: Conversation = .init(std.testing.allocator, "sys");
+ defer conv.deinit();
+ const a = conv.arena.allocator();
+
+ const image = [_]zenai.provider.ContentPart{.{ .image = .{ .data = "AAAA", .mime_type = "image/png" } }};
+ for (0..4) |n| {
+ const results = try a.alloc(zenai.provider.ToolResult, 1);
+ results[0] = .{ .id = "c", .name = "screenshot", .content = try std.fmt.allocPrint(a, "shot {d}", .{n}), .parts = &image };
+ try conv.messages.append(std.testing.allocator, .{ .role = .tool, .tool_results = results });
+ }
+
+ conv.expireImages();
+
+ try std.testing.expect(conv.messages.items[0].tool_results.?[0].parts == null);
+ try std.testing.expectEqualStrings("shot 0 (image dropped from context; call the tool again to look)", conv.messages.items[0].tool_results.?[0].content);
+ try std.testing.expect(conv.messages.items[1].tool_results.?[0].parts == null);
+ try std.testing.expect(conv.messages.items[2].tool_results.?[0].parts != null);
+ try std.testing.expect(conv.messages.items[3].tool_results.?[0].parts != null);
+ try std.testing.expectEqualStrings("shot 3", conv.messages.items[3].tool_results.?[0].content);
+}
diff --git a/src/browser/screenshot.zig b/src/browser/screenshot.zig
index ee50b0522..a49a8147e 100644
--- a/src/browser/screenshot.zig
+++ b/src/browser/screenshot.zig
@@ -64,8 +64,7 @@ pub fn png(arena: Allocator, node: *Node, opts: Opts, writer: *std.Io.Writer, fr
// get the height of the PNG if we were to render it.
pub fn contentHeight(arena: Allocator, node: *Node, width: u32, frame: *Frame) !u32 {
const prepared = try prepare(arena, node, .{ .width = width }, frame);
- var discard: std.Io.Writer.Discarding = .init(&.{});
- return prepared.render(&discard.writer, RENDER_MEASURE_ONLY);
+ return prepared.measure();
}
// The DOM walk, done up front so it can fail (allocation) before any output
@@ -92,6 +91,12 @@ pub const Prepared = struct {
return self.render(writer, 0);
}
+ /// The content height at `opts.width`, without rasterizing.
+ pub fn measure(self: *const Prepared) std.Io.Writer.Error!u32 {
+ var discard: std.Io.Writer.Discarding = .init(&.{});
+ return self.render(&discard.writer, RENDER_MEASURE_ONLY);
+ }
+
fn render(self: *const Prepared, writer: *std.Io.Writer, flags: u32) std.Io.Writer.Error!u32 {
const opts = self.opts;
const clip = opts.clip orelse Opts.Clip{ .x = 0, .y = 0, .width = 0, .height = 0 };
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index d8a315ace..5f09db31d 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -408,7 +408,7 @@ pub const Tool = enum {
),
},
.screenshot => .{
- .description = "Render the page, or one node, as a PNG: the text layout Lightpanda computes, not a pixel-accurate browser rendering (no images, fonts or CSS colours). With `path`, writes the file and returns its location; without it, returns the image inline where the client can display one. Use it to see spatial layout; read content with `markdown`/`tree`.",
+ .description = "Render the page, or one node, as a PNG: the text layout Lightpanda computes, not a pixel-accurate browser rendering (no images, fonts or CSS colours). With `path`, writes the file at full size and returns its location; without it, returns the image inline where the client can display one, at most 1280px wide and 4096px tall. Use it to see spatial layout; read content with `markdown`/`tree`.",
.summary = "Screenshot of the page or a node",
.input_schema = minify(
\\{
@@ -848,7 +848,14 @@ const NodeAndPage = struct { node: *DOMNode, page: *lp.Frame, target: ActionTarg
/// What the caller can do with a result beyond its text.
pub const CallOpts = struct {
- inline_image: bool = false,
+ /// Set when the caller can hand an image to a model; the limits keep a
+ /// screenshot within what models consume (and re-send every turn).
+ inline_image: ?InlineImage = null,
+
+ pub const InlineImage = struct {
+ max_width: u32 = 1280,
+ max_height: u32 = 4096,
+ };
};
pub fn call(
@@ -1342,7 +1349,7 @@ fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.R
return aw.written();
}
-fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value, inline_image: bool) ToolError!ToolResult {
+fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value, inline_image: ?CallOpts.InlineImage) ToolError!ToolResult {
const Params = struct {
path: ?[]const u8 = null,
selector: ?[]const u8 = null,
@@ -1354,23 +1361,31 @@ fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDP
const args = try parseArgsOrDefault(Params, arena, arguments);
if (args.path) |path| {
if (!isPathSafe(path)) return .{ .text = unsafe_path_message, .is_error = true };
- } else if (!inline_image) {
+ } else if (inline_image == null) {
return .{ .text = "pass `path`: this client cannot display an inline image", .is_error = true };
}
const page = try ensurePage(session, registry, args.url, args.timeout);
const node = try resolveScope(session, registry, page, args.selector, args.backendNodeId);
- const opts: lp.screenshot.Opts = .fromViewport(page._page.getViewport(), args.fullPage);
- const prepared = lp.screenshot.prepare(arena, node, opts, page) catch return ToolError.InternalError;
+ var prepared = lp.screenshot.prepare(arena, node, .fromViewport(page._page.getViewport(), args.fullPage), page) catch
+ return ToolError.InternalError;
if (args.path) |path| {
const height = writePng(&prepared, path) catch |err| return .{
.text = std.fmt.allocPrint(arena, "could not write {s}: {s}", .{ path, @errorName(err) }) catch return ToolError.OutOfMemory,
.is_error = true,
};
- return .{ .text = std.fmt.allocPrint(arena, "Saved {d}x{d} PNG to {s}", .{ opts.width, height, absolutePath(arena, path) }) catch return ToolError.OutOfMemory };
+ return .{ .text = std.fmt.allocPrint(arena, "Saved {d}x{d} PNG to {s}", .{ prepared.opts.width, height, absolutePath(arena, path) }) catch return ToolError.OutOfMemory };
+ }
+
+ // Layout reflows to the width, so measure after narrowing.
+ const limits = inline_image.?;
+ prepared.opts.width = @min(prepared.opts.width, limits.max_width);
+ const content_height = prepared.measure() catch return ToolError.InternalError;
+ if (prepared.opts.height == 0 or prepared.opts.height > limits.max_height) {
+ prepared.opts.height = @min(content_height, limits.max_height);
}
return .{
- .text = std.fmt.allocPrint(arena, "PNG, {d}px wide", .{opts.width}) catch return ToolError.OutOfMemory,
+ .text = std.fmt.allocPrint(arena, "PNG, {d}x{d}", .{ prepared.opts.width, prepared.opts.height }) catch return ToolError.OutOfMemory,
.image = prepared,
};
}
diff --git a/src/mcp/Transport.zig b/src/mcp/Transport.zig
index 15a1d9882..b174373ad 100644
--- a/src/mcp/Transport.zig
+++ b/src/mcp/Transport.zig
@@ -53,8 +53,17 @@ pub fn sendResponse(self: *Self, response: anytype) !void {
try self.aw.writer.writeByte('\n');
try self.writer.writeAll(self.aw.writer.buffered());
try self.writer.flush();
+
+ // A screenshot response can be megabytes; don't keep that much parked.
+ if (self.aw.writer.buffer.len > large_response) {
+ const allocator = self.aw.allocator;
+ self.aw.deinit();
+ self.aw = .init(allocator);
+ }
}
+const large_response = 1024 * 1024;
+
pub fn sendResult(self: *Self, id: std.json.Value, result: anytype) !void {
const GenericResponse = struct {
jsonrpc: []const u8 = "2.0",
diff --git a/src/mcp/tools.zig b/src/mcp/tools.zig
index 66f86dbe3..f0c34b899 100644
--- a/src/mcp/tools.zig
+++ b/src/mcp/tools.zig
@@ -156,7 +156,7 @@ fn dispatchBrowserTool(
};
const active = server.active_session;
- const result = browser_tools.call(arena, active.session, &active.node_registry, name, arguments, .{ .inline_image = true }) catch |err| {
+ const result = browser_tools.call(arena, active.session, &active.node_registry, name, arguments, .{ .inline_image = .{} }) catch |err| {
// evaluate/extract surface failures in-band so the LLM can self-correct;
// other tools' operational failures are protocol-level.
if (surfacesErrorInBand(tool)) {
@@ -1465,6 +1465,13 @@ test "MCP - screenshot: inline image, file, unsafe path" {
// base64 of the PNG signature
try testing.expect(std.mem.indexOf(u8, out.written(), "\"data\":\"iVBORw0KGgo") != null);
try testing.expect(std.mem.indexOf(u8, out.written(), "\"isError\":false") != null);
+ // Inline images are narrowed to 1280px; IHDR width is big-endian at offset 16.
+ try testing.expect(std.mem.indexOf(u8, out.written(), "PNG, 1280x") != null);
+ const start = std.mem.indexOf(u8, out.written(), "\"data\":\"").? + "\"data\":\"".len;
+ const b64 = out.written()[start .. start + 32];
+ var header: [24]u8 = undefined;
+ try std.base64.standard.Decoder.decode(&header, b64);
+ try testing.expectEqual(1280, std.mem.readInt(u32, header[16..20], .big));
const path = "mcp-screenshot-test.png";
std.Io.Dir.cwd().deleteFile(lp.io, path) catch {};