Lightpanda: the headless browser designed for AI and automation
tools: add maxBytes and strip to the html tool
Adrià Arrufatf05789353b2d
diff --git a/src/LimitedWriter.zig b/src/LimitedWriter.zignew file mode 100644index 000000000..d94886430--- /dev/null+++ b/src/LimitedWriter.zig@@ -0,0 +1,76 @@+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)+//+// Francis Bouvier <[email protected]>+// Pierre Tachoire <[email protected]>+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++const std = @import("std");++const truncateUtf8 = @import("string.zig").truncateUtf8;++/// Caps what reaches `inner`. Past the cap, writes fail with `WriteFailed`+/// and `truncated` is set so the caller can tell the cap from an I/O error.+/// A null cap passes everything through.+const LimitedWriter = @This();++pub const truncation_marker = "\n\n[truncated]\n";++inner: *std.Io.Writer,+remaining: ?usize,+truncated: bool = false,+writer: std.Io.Writer,++pub fn init(inner: *std.Io.Writer, max_bytes: ?u32) LimitedWriter {+ return .{+ .inner = inner,+ .remaining = if (max_bytes) |m| @as(usize, m) else null,+ .writer = .{+ .vtable = &vtable,+ .buffer = &.{},+ },+ };+}++const vtable = std.Io.Writer.VTable{ .drain = drain };++fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize {+ const self: *LimitedWriter = @alignCast(@fieldParentPtr("writer", w));+ var total: usize = 0;+ for (data[0 .. data.len - 1]) |slice| {+ try self.consume(slice);+ total += slice.len;+ }+ const pattern = data[data.len - 1];+ for (0..splat) |_| {+ try self.consume(pattern);+ total += pattern.len;+ }+ return total;+}++fn consume(self: *LimitedWriter, bytes: []const u8) std.Io.Writer.Error!void {+ const remaining = self.remaining orelse return self.inner.writeAll(bytes);+ if (bytes.len <= remaining) {+ try self.inner.writeAll(bytes);+ self.remaining = remaining - bytes.len;+ return;+ }+ if (remaining > 0) {+ try self.inner.writeAll(truncateUtf8(bytes, remaining));+ self.remaining = 0;+ }+ self.truncated = true;+ return error.WriteFailed;+}diff --git a/src/browser/markdown.zig b/src/browser/markdown.zigindex f8934cd58..28dc7ccfa 100644--- a/src/browser/markdown.zig+++ b/src/browser/markdown.zig@@ -27,62 +27,13 @@ const TreeWalker = @import("webapi/TreeWalker.zig");const Slot = @import("webapi/element/html/Slot.zig");const isAllWhitespace = @import("../string.zig").isAllWhitespace;-const truncateUtf8 = @import("../string.zig").truncateUtf8;+const LimitedWriter = @import("../LimitedWriter.zig");pub const Opts = struct {max_bytes: ?u32 = null,};-const truncation_marker = "\n\n[truncated]\n";--const LimitedWriter = struct {- inner: *std.Io.Writer,- remaining: usize,- truncated: bool = false,- writer: std.Io.Writer,-- fn init(inner: *std.Io.Writer, max_bytes: u32) LimitedWriter {- return .{- .inner = inner,- .remaining = max_bytes,- .writer = .{- .vtable = &vtable,- .buffer = &.{},- },- };- }-- const vtable = std.Io.Writer.VTable{ .drain = drain };-- fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize {- const self: *LimitedWriter = @alignCast(@fieldParentPtr("writer", w));- var total: usize = 0;- for (data[0 .. data.len - 1]) |slice| {- try self.consume(slice);- total += slice.len;- }- const pattern = data[data.len - 1];- for (0..splat) |_| {- try self.consume(pattern);- total += pattern.len;- }- return total;- }-- fn consume(self: *LimitedWriter, bytes: []const u8) std.Io.Writer.Error!void {- if (bytes.len <= self.remaining) {- try self.inner.writeAll(bytes);- self.remaining -= bytes.len;- return;- }- if (self.remaining > 0) {- try self.inner.writeAll(truncateUtf8(bytes, self.remaining));- self.remaining = 0;- }- self.truncated = true;- return error.WriteFailed;- }-};+const truncation_marker = LimitedWriter.truncation_marker;const State = struct {const ListType = enum { ordered, unordered };diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 5154c8fa7..3f3768f3a 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -28,6 +28,7 @@ const keenable = zenai.search.keenable;const DOMNode = @import("webapi/Node.zig");const CDPNode = @import("../cdp/Node.zig");+const LimitedWriter = @import("../LimitedWriter.zig");const Selector = @import("webapi/selector/Selector.zig");/// Conventions any LLM driving Lightpanda should follow. The standalone@@ -389,6 +390,8 @@ pub const Tool = enum {\\ "properties": {\\ "selector": { "type": "string", "description": "Optional CSS selector. When set, dump only that element's outerHTML." },\\ "backendNodeId": { "type": "integer", "description": "Optional backend node ID. When set, dump only that node's outerHTML. 0 is treated as omitted." },+ \\ "maxBytes": { "type": "integer", "description": "Optional soft cap on output size in bytes. Content is truncated at a UTF-8 boundary and a short '[truncated]' marker is appended past the cap." },+ \\ "strip": { "type": "object", "description": "Optional. Omit element groups from the output: `js` (script, noscript, script preloads), `css` (style, stylesheet links), `ui` (css plus img, picture, video, audio, svg, canvas, iframe), `invisible` (elements an author rule or inline style sets to display:none). {\"js\":true,\"css\":true} keeps a page dump small.", "properties": { "js": { "type": "boolean" }, "css": { "type": "boolean" }, "ui": { "type": "boolean" }, "invisible": { "type": "boolean" } } },\\ "url": { "type": "string", "description": "Optional URL to navigate to before dumping." },\\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." }\\ }@@ -1277,27 +1280,45 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNoreturn aw.written();}+const HtmlParams = struct {+ selector: ?[]const u8 = null,+ backendNodeId: ?CDPNode.Id = null,+ maxBytes: ?u32 = null,+ strip: lp.dump.Opts.Strip = .{},+ url: ?[:0]const u8 = null,+ timeout: ?u32 = null,+};+fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError![]const u8 {- const Params = struct {- selector: ?[]const u8 = null,- backendNodeId: ?CDPNode.Id = null,- url: ?[:0]const u8 = null,- timeout: ?u32 = null,- };- const args = try parseArgsOrDefault(Params, arena, arguments);+ const args = try parseArgsOrDefault(HtmlParams, arena, arguments);const page = try ensurePage(session, registry, args.url, args.timeout);var aw: std.Io.Writer.Allocating = .init(arena);+ var lw: LimitedWriter = .init(&aw.writer, args.maxBytes);+ dumpHtml(session, registry, page, args, &lw.writer) catch |err| switch (err) {+ error.WriteFailed => if (!lw.truncated) return ToolError.InternalError,+ else => |e| return e,+ };+ if (lw.truncated) {+ aw.writer.writeAll(LimitedWriter.truncation_marker) catch return ToolError.InternalError;+ }+ return aw.written();+}++fn dumpHtml(session: *lp.Session, registry: *CDPNode.Registry, page: *lp.Frame, args: HtmlParams, writer: *std.Io.Writer) (ToolError || error{WriteFailed})!void {+ const opts: lp.dump.Opts = .{ .strip = args.strip };if (args.selector) |sel| {const resolved = try resolveBySelector(session, sel);- lp.dump.deep(resolved.node, .{}, &aw.writer, resolved.page) catch return ToolError.InternalError;- } else if (args.backendNodeId) |nid| {+ return lp.dump.deep(resolved.node, opts, writer, resolved.page);+ }+ if (args.backendNodeId) |nid| {const resolved = try resolveNodeAndPage(session, registry, nid);- lp.dump.deep(resolved.node, .{}, &aw.writer, resolved.page) catch return ToolError.InternalError;- } else {- lp.dump.root(page.document, .{}, &aw.writer, page) catch return ToolError.InternalError;+ return lp.dump.deep(resolved.node, opts, writer, resolved.page);}- return aw.written();+ return lp.dump.root(page.document, opts, writer, page) catch |err| switch (err) {+ error.WriteFailed => error.WriteFailed,+ else => ToolError.InternalError,+ };}fn execLinks(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError![]const u8 {diff --git a/src/mcp/tools.zig b/src/mcp/tools.zigindex 47ff1c746..fd53b4a3b 100644--- a/src/mcp/tools.zig+++ b/src/mcp/tools.zig@@ -1353,6 +1353,38 @@ test "MCP - html: full document, selector subtree, backendNodeId subtree" {try testing.expect(std.mem.indexOf(u8, out.written(), "<form") == null);}+test "MCP - html: maxBytes truncation and strip" {+ var out: std.Io.Writer.Allocating = .init(testing.arena_allocator);+ const server = try testLoadPage("http://localhost:9582/src/browser/tests/dump.html", &out.writer);+ defer server.deinit();++ const full =+ \\{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"html"}}+ ;+ try router.handleMessage(server, testing.arena_allocator, full);+ try testing.expect(std.mem.indexOf(u8, out.written(), "<script>") != null);+ try testing.expect(std.mem.indexOf(u8, out.written(), "<style>") != null);+ try testing.expect(std.mem.indexOf(u8, out.written(), "[truncated]") == null);++ out.clearRetainingCapacity();+ const stripped =+ \\{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"html","arguments":{"strip":{"js":true,"css":true}}}}+ ;+ try router.handleMessage(server, testing.arena_allocator, stripped);+ try testing.expect(std.mem.indexOf(u8, out.written(), "<script>") == null);+ try testing.expect(std.mem.indexOf(u8, out.written(), "<style>") == null);+ try testing.expect(std.mem.indexOf(u8, out.written(), "<h1>Title</h1>") != null);++ out.clearRetainingCapacity();+ const capped =+ \\{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"html","arguments":{"maxBytes":20}}}+ ;+ try router.handleMessage(server, testing.arena_allocator, capped);+ try testing.expect(std.mem.indexOf(u8, out.written(), "<!DOCTYPE html>") != null);+ try testing.expect(std.mem.indexOf(u8, out.written(), "<h1>") == null);+ try testing.expect(std.mem.indexOf(u8, out.written(), "[truncated]") != null);+}+test "MCP - waitForScript: truthy returns, falsy times out" {var out: std.Io.Writer.Allocating = .init(testing.arena_allocator);const server = try testLoadPage("about:blank", &out.writer);