Lightpanda: the headless browser designed for AI and automation
dump: take max_bytes in Opts
Adrià Arrufat56241e26b84f
diff --git a/src/browser/dump.zig b/src/browser/dump.zigindex 89bd6fd64..b8cc01fe6 100644--- a/src/browser/dump.zig+++ b/src/browser/dump.zig@@ -19,6 +19,7 @@const std = @import("std");const lp = @import("lightpanda");const Frame = @import("Frame.zig");+const LimitedWriter = @import("../LimitedWriter.zig");const Node = @import("webapi/Node.zig");const Slot = @import("webapi/element/html/Slot.zig");const IFrame = @import("webapi/element/html/IFrame.zig");@@ -28,6 +29,9 @@ pub const Opts = struct {with_frames: bool = false,strip: Opts.Strip = .{},shadow: Opts.Shadow = .rendered,+ /// Soft cap: output is cut at a UTF-8 boundary and a truncation marker+ /// appended.+ max_bytes: ?u32 = null,pub const Strip = packed struct(u4) {js: bool = false,@@ -59,6 +63,16 @@ pub const Opts = struct {};pub fn root(doc: *Node.Document, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {+ if (opts.max_bytes == null) return rootUncapped(doc, opts, writer, frame);++ var lw: LimitedWriter = .init(writer, opts.max_bytes);+ rootUncapped(doc, opts, &lw.writer, frame) catch |err| {+ if (!lw.truncated) return err;+ try writer.writeAll(LimitedWriter.truncation_marker);+ };+}++fn rootUncapped(doc: *Node.Document, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {if (doc.is(Node.Document.HTMLDocument)) |html_doc| {blk: {// Ideally we just render the doctype which is part of the document@@ -80,11 +94,17 @@ pub fn root(doc: *Node.Document, opts: Opts, writer: *std.Io.Writer, frame: *Fra}}- return deep(doc.asNode(), opts, writer, frame);+ return _deep(doc.asNode(), opts, false, writer, frame);}pub fn deep(node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) error{WriteFailed}!void {- return _deep(node, opts, false, writer, frame);+ if (opts.max_bytes == null) return _deep(node, opts, false, writer, frame);++ var lw: LimitedWriter = .init(writer, opts.max_bytes);+ _deep(node, opts, false, &lw.writer, frame) catch |err| {+ if (!lw.truncated) return err;+ try writer.writeAll(LimitedWriter.truncation_marker);+ };}fn _deep(node: *Node, opts: Opts, comptime force_slot: bool, writer: *std.Io.Writer, frame: *Frame) error{WriteFailed}!void {@@ -499,6 +519,20 @@ test "dump: strip.ui removes css plus visual elements" {);}+test "dump: max_bytes truncates with a marker" {+ var page = try testing.pageTest("dump.html", .{});+ defer page.close();+ const frame = page.frame().?;++ var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);+ try root(frame.window._document, .{ .max_bytes = 24 }, &aw.writer, frame);+ try testing.expectString("<!DOCTYPE html>\n<html><h" ++ LimitedWriter.truncation_marker, aw.written());++ aw.clearRetainingCapacity();+ try deep(frame.window._document.asNode().lastChild().?, .{ .max_bytes = 6 }, &aw.writer, frame);+ try testing.expectString("<html>" ++ LimitedWriter.truncation_marker, aw.written());+}+test "dump: strip.invisible removes author display:none elements" {try expectDump(.{ .strip = .{ .invisible = true } },\\<!DOCTYPE html>diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 7862e7874..09306e082 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -28,7 +28,6 @@ const keenable = zenai.search.keenable;const DOMNode = @import("webapi/Node.zig");const CDPNode = @import("../cdp/Node.zig");-const LimitedWriter = @import("../LimitedWriter.zig");const Selector = @import("webapi/selector/Selector.zig");/// Conventions any LLM driving Lightpanda should follow. The standalone@@ -1287,32 +1286,18 @@ fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Rconst args = try parseArgsOrDefault(HtmlParams, arena, arguments);const page = try ensurePage(session, registry, args.url, args.timeout);+ const opts: lp.dump.Opts = .{ .strip = args.strip, .max_bytes = args.maxBytes };var aw: std.Io.Writer.Allocating = .init(arena);- var lw: LimitedWriter = .init(&aw.writer, args.maxBytes);- dumpHtml(session, registry, page, args, &lw.writer) catch |err| switch (err) {- error.WriteFailed => if (!lw.truncated) return ToolError.InternalError,- else => |e| return e,- };- if (lw.truncated) {- aw.writer.writeAll(LimitedWriter.truncation_marker) catch return ToolError.InternalError;- }- return aw.written();-}--fn dumpHtml(session: *lp.Session, registry: *CDPNode.Registry, page: *lp.Frame, args: HtmlParams, writer: *std.Io.Writer) (ToolError || error{WriteFailed})!void {- const opts: lp.dump.Opts = .{ .strip = args.strip };if (args.selector) |sel| {const resolved = try resolveBySelector(session, sel);- return lp.dump.deep(resolved.node, opts, writer, resolved.page);- }- if (args.backendNodeId) |nid| {+ lp.dump.deep(resolved.node, opts, &aw.writer, resolved.page) catch return ToolError.InternalError;+ } else if (args.backendNodeId) |nid| {const resolved = try resolveNodeAndPage(session, registry, nid);- return lp.dump.deep(resolved.node, opts, writer, resolved.page);+ lp.dump.deep(resolved.node, opts, &aw.writer, resolved.page) catch return ToolError.InternalError;+ } else {+ lp.dump.root(page.document, opts, &aw.writer, page) catch return ToolError.InternalError;}- return lp.dump.root(page.document, opts, writer, page) catch |err| switch (err) {- error.WriteFailed => error.WriteFailed,- else => ToolError.InternalError,- };+ return aw.written();}fn execLinks(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError![]const u8 {diff --git a/src/lightpanda.zig b/src/lightpanda.zigindex fbbaf9010..148bf7a38 100644--- a/src/lightpanda.zig+++ b/src/lightpanda.zig@@ -40,7 +40,6 @@ pub const dump = @import("browser/dump.zig");pub const markdown = @import("browser/markdown.zig");pub const screenshot = @import("browser/screenshot.zig");pub const Base64Writer = @import("Base64Writer.zig");-const LimitedWriter = @import("LimitedWriter.zig");const Selector = @import("browser/webapi/selector/Selector.zig");const Node = @import("browser/webapi/Node.zig");pub const SemanticTree = @import("SemanticTree.zig");@@ -191,8 +190,6 @@ pub const FetchOpts = struct {dump_mode: ?Config.DumpFormat = null,/// Dump only the first match instead of the document.selector: ?[:0]const u8 = null,- /// html and markdown only.- max_bytes: ?u32 = null,/// Any page with an HTTP status >= 400 fails the fetch with `error.HttpError`.fail_on_http_error: bool = false,writer: ?*std.Io.Writer = null,@@ -438,18 +435,11 @@ fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Frame, writer: *std.Io.Writer) !void {const root = try dumpRoot(frame, opts.selector);switch (mode) {- .html => {- var lw: LimitedWriter = .init(writer, opts.max_bytes);- const result = if (opts.selector == null)- dump.root(frame.window._document, opts.dump, &lw.writer, frame)- else- dump.deep(root, opts.dump, &lw.writer, frame);- result catch |err| {- if (!lw.truncated) return err;- try writer.writeAll(LimitedWriter.truncation_marker);- };- },- .markdown => try markdown.dump(root, .{ .max_bytes = opts.max_bytes, .strip = opts.dump.strip }, writer, frame),+ .html => if (opts.selector == null)+ try dump.root(frame.window._document, opts.dump, writer, frame)+ else+ try dump.deep(root, opts.dump, writer, frame),+ .markdown => try markdown.dump(root, .{ .max_bytes = opts.dump.max_bytes, .strip = opts.dump.strip }, writer, frame),.png => {var arena: std.heap.ArenaAllocator = .init(app.allocator);defer arena.deinit();diff --git a/src/main.zig b/src/main.zigindex 7b68b7e6a..80e40a802 100644--- a/src/main.zig+++ b/src/main.zig@@ -177,12 +177,12 @@ fn run(allocator: Allocator, main_arena: Allocator, proc_args: std.process.Args).wait_selector = opts.wait_selector,.dump_mode = opts.dump,.selector = opts.dump_selector,- .max_bytes = opts.dump_max_bytes,.fail_on_http_error = opts.fail_on_http_error,.dump = .{.strip = opts.strip_mode,.with_base = opts.with_base,.with_frames = opts.with_frames,+ .max_bytes = opts.dump_max_bytes,},.json = opts.json,};