Skip to content

Lightpanda: the headless browser designed for AI and automation

tools: add maxBytes and strip to the html tool

Adrià Arrufatf05789353b2d
diff --git a/src/LimitedWriter.zig b/src/LimitedWriter.zig
new file mode 100644
index 000000000..d94886430
--- /dev/null
+++ b/src/LimitedWriter.zig
@@ -0,0 +1,76 @@
+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+
+const truncateUtf8 = @import("string.zig").truncateUtf8;
+
+/// Caps what reaches `inner`. Past the cap, writes fail with `WriteFailed`
+/// and `truncated` is set so the caller can tell the cap from an I/O error.
+/// A null cap passes everything through.
+const LimitedWriter = @This();
+
+pub const truncation_marker = "\n\n[truncated]\n";
+
+inner: *std.Io.Writer,
+remaining: ?usize,
+truncated: bool = false,
+writer: std.Io.Writer,
+
+pub fn init(inner: *std.Io.Writer, max_bytes: ?u32) LimitedWriter {
+ return .{
+ .inner = inner,
+ .remaining = if (max_bytes) |m| @as(usize, m) else null,
+ .writer = .{
+ .vtable = &vtable,
+ .buffer = &.{},
+ },
+ };
+}
+
+const vtable = std.Io.Writer.VTable{ .drain = drain };
+
+fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize {
+ const self: *LimitedWriter = @alignCast(@fieldParentPtr("writer", w));
+ var total: usize = 0;
+ for (data[0 .. data.len - 1]) |slice| {
+ try self.consume(slice);
+ total += slice.len;
+ }
+ const pattern = data[data.len - 1];
+ for (0..splat) |_| {
+ try self.consume(pattern);
+ total += pattern.len;
+ }
+ return total;
+}
+
+fn consume(self: *LimitedWriter, bytes: []const u8) std.Io.Writer.Error!void {
+ const remaining = self.remaining orelse return self.inner.writeAll(bytes);
+ if (bytes.len <= remaining) {
+ try self.inner.writeAll(bytes);
+ self.remaining = remaining - bytes.len;
+ return;
+ }
+ if (remaining > 0) {
+ try self.inner.writeAll(truncateUtf8(bytes, remaining));
+ self.remaining = 0;
+ }
+ self.truncated = true;
+ return error.WriteFailed;
+}
diff --git a/src/browser/markdown.zig b/src/browser/markdown.zig
index f8934cd58..28dc7ccfa 100644
--- a/src/browser/markdown.zig
+++ b/src/browser/markdown.zig
@@ -27,62 +27,13 @@ const TreeWalker = @import("webapi/TreeWalker.zig");
const Slot = @import("webapi/element/html/Slot.zig");
const isAllWhitespace = @import("../string.zig").isAllWhitespace;
-const truncateUtf8 = @import("../string.zig").truncateUtf8;
+const LimitedWriter = @import("../LimitedWriter.zig");
pub const Opts = struct {
max_bytes: ?u32 = null,
};
-const truncation_marker = "\n\n[truncated]\n";
-
-const LimitedWriter = struct {
- inner: *std.Io.Writer,
- remaining: usize,
- truncated: bool = false,
- writer: std.Io.Writer,
-
- fn init(inner: *std.Io.Writer, max_bytes: u32) LimitedWriter {
- return .{
- .inner = inner,
- .remaining = max_bytes,
- .writer = .{
- .vtable = &vtable,
- .buffer = &.{},
- },
- };
- }
-
- const vtable = std.Io.Writer.VTable{ .drain = drain };
-
- fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize {
- const self: *LimitedWriter = @alignCast(@fieldParentPtr("writer", w));
- var total: usize = 0;
- for (data[0 .. data.len - 1]) |slice| {
- try self.consume(slice);
- total += slice.len;
- }
- const pattern = data[data.len - 1];
- for (0..splat) |_| {
- try self.consume(pattern);
- total += pattern.len;
- }
- return total;
- }
-
- fn consume(self: *LimitedWriter, bytes: []const u8) std.Io.Writer.Error!void {
- if (bytes.len <= self.remaining) {
- try self.inner.writeAll(bytes);
- self.remaining -= bytes.len;
- return;
- }
- if (self.remaining > 0) {
- try self.inner.writeAll(truncateUtf8(bytes, self.remaining));
- self.remaining = 0;
- }
- self.truncated = true;
- return error.WriteFailed;
- }
-};
+const truncation_marker = LimitedWriter.truncation_marker;
const State = struct {
const ListType = enum { ordered, unordered };
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index 5154c8fa7..3f3768f3a 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -28,6 +28,7 @@ const keenable = zenai.search.keenable;
const DOMNode = @import("webapi/Node.zig");
const CDPNode = @import("../cdp/Node.zig");
+const LimitedWriter = @import("../LimitedWriter.zig");
const Selector = @import("webapi/selector/Selector.zig");
/// Conventions any LLM driving Lightpanda should follow. The standalone
@@ -389,6 +390,8 @@ pub const Tool = enum {
\\ "properties": {
\\ "selector": { "type": "string", "description": "Optional CSS selector. When set, dump only that element's outerHTML." },
\\ "backendNodeId": { "type": "integer", "description": "Optional backend node ID. When set, dump only that node's outerHTML. 0 is treated as omitted." },
+ \\ "maxBytes": { "type": "integer", "description": "Optional soft cap on output size in bytes. Content is truncated at a UTF-8 boundary and a short '[truncated]' marker is appended past the cap." },
+ \\ "strip": { "type": "object", "description": "Optional. Omit element groups from the output: `js` (script, noscript, script preloads), `css` (style, stylesheet links), `ui` (css plus img, picture, video, audio, svg, canvas, iframe), `invisible` (elements an author rule or inline style sets to display:none). {\"js\":true,\"css\":true} keeps a page dump small.", "properties": { "js": { "type": "boolean" }, "css": { "type": "boolean" }, "ui": { "type": "boolean" }, "invisible": { "type": "boolean" } } },
\\ "url": { "type": "string", "description": "Optional URL to navigate to before dumping." },
\\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." }
\\ }
@@ -1277,27 +1280,45 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNo
return aw.written();
}
+const HtmlParams = struct {
+ selector: ?[]const u8 = null,
+ backendNodeId: ?CDPNode.Id = null,
+ maxBytes: ?u32 = null,
+ strip: lp.dump.Opts.Strip = .{},
+ url: ?[:0]const u8 = null,
+ timeout: ?u32 = null,
+};
+
fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError![]const u8 {
- const Params = struct {
- selector: ?[]const u8 = null,
- backendNodeId: ?CDPNode.Id = null,
- url: ?[:0]const u8 = null,
- timeout: ?u32 = null,
- };
- const args = try parseArgsOrDefault(Params, arena, arguments);
+ const args = try parseArgsOrDefault(HtmlParams, arena, arguments);
const page = try ensurePage(session, registry, args.url, args.timeout);
var aw: std.Io.Writer.Allocating = .init(arena);
+ var lw: LimitedWriter = .init(&aw.writer, args.maxBytes);
+ dumpHtml(session, registry, page, args, &lw.writer) catch |err| switch (err) {
+ error.WriteFailed => if (!lw.truncated) return ToolError.InternalError,
+ else => |e| return e,
+ };
+ if (lw.truncated) {
+ aw.writer.writeAll(LimitedWriter.truncation_marker) catch return ToolError.InternalError;
+ }
+ return aw.written();
+}
+
+fn dumpHtml(session: *lp.Session, registry: *CDPNode.Registry, page: *lp.Frame, args: HtmlParams, writer: *std.Io.Writer) (ToolError || error{WriteFailed})!void {
+ const opts: lp.dump.Opts = .{ .strip = args.strip };
if (args.selector) |sel| {
const resolved = try resolveBySelector(session, sel);
- lp.dump.deep(resolved.node, .{}, &aw.writer, resolved.page) catch return ToolError.InternalError;
- } else if (args.backendNodeId) |nid| {
+ return lp.dump.deep(resolved.node, opts, writer, resolved.page);
+ }
+ if (args.backendNodeId) |nid| {
const resolved = try resolveNodeAndPage(session, registry, nid);
- lp.dump.deep(resolved.node, .{}, &aw.writer, resolved.page) catch return ToolError.InternalError;
- } else {
- lp.dump.root(page.document, .{}, &aw.writer, page) catch return ToolError.InternalError;
+ return lp.dump.deep(resolved.node, opts, writer, resolved.page);
}
- return aw.written();
+ return lp.dump.root(page.document, opts, writer, page) catch |err| switch (err) {
+ error.WriteFailed => error.WriteFailed,
+ else => ToolError.InternalError,
+ };
}
fn execLinks(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError![]const u8 {
diff --git a/src/mcp/tools.zig b/src/mcp/tools.zig
index 47ff1c746..fd53b4a3b 100644
--- a/src/mcp/tools.zig
+++ b/src/mcp/tools.zig
@@ -1353,6 +1353,38 @@ test "MCP - html: full document, selector subtree, backendNodeId subtree" {
try testing.expect(std.mem.indexOf(u8, out.written(), "<form") == null);
}
+test "MCP - html: maxBytes truncation and strip" {
+ var out: std.Io.Writer.Allocating = .init(testing.arena_allocator);
+ const server = try testLoadPage("http://localhost:9582/src/browser/tests/dump.html", &out.writer);
+ defer server.deinit();
+
+ const full =
+ \\{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"html"}}
+ ;
+ try router.handleMessage(server, testing.arena_allocator, full);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "<script>") != null);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "<style>") != null);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "[truncated]") == null);
+
+ out.clearRetainingCapacity();
+ const stripped =
+ \\{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"html","arguments":{"strip":{"js":true,"css":true}}}}
+ ;
+ try router.handleMessage(server, testing.arena_allocator, stripped);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "<script>") == null);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "<style>") == null);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "<h1>Title</h1>") != null);
+
+ out.clearRetainingCapacity();
+ const capped =
+ \\{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"html","arguments":{"maxBytes":20}}}
+ ;
+ try router.handleMessage(server, testing.arena_allocator, capped);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "<!DOCTYPE html>") != null);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "<h1>") == null);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "[truncated]") != null);
+}
+
test "MCP - waitForScript: truthy returns, falsy times out" {
var out: std.Io.Writer.Allocating = .init(testing.arena_allocator);
const server = try testLoadPage("about:blank", &out.writer);