Skip to content

Lightpanda: the headless browser designed for AI and automation

dump: take max_bytes in Opts

Adrià Arrufat56241e26b84f
diff --git a/src/browser/dump.zig b/src/browser/dump.zig
index 89bd6fd64..b8cc01fe6 100644
--- a/src/browser/dump.zig
+++ b/src/browser/dump.zig
@@ -19,6 +19,7 @@
const std = @import("std");
const lp = @import("lightpanda");
const Frame = @import("Frame.zig");
+const LimitedWriter = @import("../LimitedWriter.zig");
const Node = @import("webapi/Node.zig");
const Slot = @import("webapi/element/html/Slot.zig");
const IFrame = @import("webapi/element/html/IFrame.zig");
@@ -28,6 +29,9 @@ pub const Opts = struct {
with_frames: bool = false,
strip: Opts.Strip = .{},
shadow: Opts.Shadow = .rendered,
+ /// Soft cap: output is cut at a UTF-8 boundary and a truncation marker
+ /// appended.
+ max_bytes: ?u32 = null,
pub const Strip = packed struct(u4) {
js: bool = false,
@@ -59,6 +63,16 @@ pub const Opts = struct {
};
pub fn root(doc: *Node.Document, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {
+ if (opts.max_bytes == null) return rootUncapped(doc, opts, writer, frame);
+
+ var lw: LimitedWriter = .init(writer, opts.max_bytes);
+ rootUncapped(doc, opts, &lw.writer, frame) catch |err| {
+ if (!lw.truncated) return err;
+ try writer.writeAll(LimitedWriter.truncation_marker);
+ };
+}
+
+fn rootUncapped(doc: *Node.Document, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {
if (doc.is(Node.Document.HTMLDocument)) |html_doc| {
blk: {
// Ideally we just render the doctype which is part of the document
@@ -80,11 +94,17 @@ pub fn root(doc: *Node.Document, opts: Opts, writer: *std.Io.Writer, frame: *Fra
}
}
- return deep(doc.asNode(), opts, writer, frame);
+ return _deep(doc.asNode(), opts, false, writer, frame);
}
pub fn deep(node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) error{WriteFailed}!void {
- return _deep(node, opts, false, writer, frame);
+ if (opts.max_bytes == null) return _deep(node, opts, false, writer, frame);
+
+ var lw: LimitedWriter = .init(writer, opts.max_bytes);
+ _deep(node, opts, false, &lw.writer, frame) catch |err| {
+ if (!lw.truncated) return err;
+ try writer.writeAll(LimitedWriter.truncation_marker);
+ };
}
fn _deep(node: *Node, opts: Opts, comptime force_slot: bool, writer: *std.Io.Writer, frame: *Frame) error{WriteFailed}!void {
@@ -499,6 +519,20 @@ test "dump: strip.ui removes css plus visual elements" {
);
}
+test "dump: max_bytes truncates with a marker" {
+ var page = try testing.pageTest("dump.html", .{});
+ defer page.close();
+ const frame = page.frame().?;
+
+ var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);
+ try root(frame.window._document, .{ .max_bytes = 24 }, &aw.writer, frame);
+ try testing.expectString("<!DOCTYPE html>\n<html><h" ++ LimitedWriter.truncation_marker, aw.written());
+
+ aw.clearRetainingCapacity();
+ try deep(frame.window._document.asNode().lastChild().?, .{ .max_bytes = 6 }, &aw.writer, frame);
+ try testing.expectString("<html>" ++ LimitedWriter.truncation_marker, aw.written());
+}
+
test "dump: strip.invisible removes author display:none elements" {
try expectDump(.{ .strip = .{ .invisible = true } },
\\<!DOCTYPE html>
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index 7862e7874..09306e082 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -28,7 +28,6 @@ const keenable = zenai.search.keenable;
const DOMNode = @import("webapi/Node.zig");
const CDPNode = @import("../cdp/Node.zig");
-const LimitedWriter = @import("../LimitedWriter.zig");
const Selector = @import("webapi/selector/Selector.zig");
/// Conventions any LLM driving Lightpanda should follow. The standalone
@@ -1287,32 +1286,18 @@ fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.R
const args = try parseArgsOrDefault(HtmlParams, arena, arguments);
const page = try ensurePage(session, registry, args.url, args.timeout);
+ const opts: lp.dump.Opts = .{ .strip = args.strip, .max_bytes = args.maxBytes };
var aw: std.Io.Writer.Allocating = .init(arena);
- var lw: LimitedWriter = .init(&aw.writer, args.maxBytes);
- dumpHtml(session, registry, page, args, &lw.writer) catch |err| switch (err) {
- error.WriteFailed => if (!lw.truncated) return ToolError.InternalError,
- else => |e| return e,
- };
- if (lw.truncated) {
- aw.writer.writeAll(LimitedWriter.truncation_marker) catch return ToolError.InternalError;
- }
- return aw.written();
-}
-
-fn dumpHtml(session: *lp.Session, registry: *CDPNode.Registry, page: *lp.Frame, args: HtmlParams, writer: *std.Io.Writer) (ToolError || error{WriteFailed})!void {
- const opts: lp.dump.Opts = .{ .strip = args.strip };
if (args.selector) |sel| {
const resolved = try resolveBySelector(session, sel);
- return lp.dump.deep(resolved.node, opts, writer, resolved.page);
- }
- if (args.backendNodeId) |nid| {
+ lp.dump.deep(resolved.node, opts, &aw.writer, resolved.page) catch return ToolError.InternalError;
+ } else if (args.backendNodeId) |nid| {
const resolved = try resolveNodeAndPage(session, registry, nid);
- return lp.dump.deep(resolved.node, opts, writer, resolved.page);
+ lp.dump.deep(resolved.node, opts, &aw.writer, resolved.page) catch return ToolError.InternalError;
+ } else {
+ lp.dump.root(page.document, opts, &aw.writer, page) catch return ToolError.InternalError;
}
- return lp.dump.root(page.document, opts, writer, page) catch |err| switch (err) {
- error.WriteFailed => error.WriteFailed,
- else => ToolError.InternalError,
- };
+ return aw.written();
}
fn execLinks(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError![]const u8 {
diff --git a/src/lightpanda.zig b/src/lightpanda.zig
index fbbaf9010..148bf7a38 100644
--- a/src/lightpanda.zig
+++ b/src/lightpanda.zig
@@ -40,7 +40,6 @@ pub const dump = @import("browser/dump.zig");
pub const markdown = @import("browser/markdown.zig");
pub const screenshot = @import("browser/screenshot.zig");
pub const Base64Writer = @import("Base64Writer.zig");
-const LimitedWriter = @import("LimitedWriter.zig");
const Selector = @import("browser/webapi/selector/Selector.zig");
const Node = @import("browser/webapi/Node.zig");
pub const SemanticTree = @import("SemanticTree.zig");
@@ -191,8 +190,6 @@ pub const FetchOpts = struct {
dump_mode: ?Config.DumpFormat = null,
/// Dump only the first match instead of the document.
selector: ?[:0]const u8 = null,
- /// html and markdown only.
- max_bytes: ?u32 = null,
/// Any page with an HTTP status >= 400 fails the fetch with `error.HttpError`.
fail_on_http_error: bool = false,
writer: ?*std.Io.Writer = null,
@@ -438,18 +435,11 @@ fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {
fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Frame, writer: *std.Io.Writer) !void {
const root = try dumpRoot(frame, opts.selector);
switch (mode) {
- .html => {
- var lw: LimitedWriter = .init(writer, opts.max_bytes);
- const result = if (opts.selector == null)
- dump.root(frame.window._document, opts.dump, &lw.writer, frame)
- else
- dump.deep(root, opts.dump, &lw.writer, frame);
- result catch |err| {
- if (!lw.truncated) return err;
- try writer.writeAll(LimitedWriter.truncation_marker);
- };
- },
- .markdown => try markdown.dump(root, .{ .max_bytes = opts.max_bytes, .strip = opts.dump.strip }, writer, frame),
+ .html => if (opts.selector == null)
+ try dump.root(frame.window._document, opts.dump, writer, frame)
+ else
+ try dump.deep(root, opts.dump, writer, frame),
+ .markdown => try markdown.dump(root, .{ .max_bytes = opts.dump.max_bytes, .strip = opts.dump.strip }, writer, frame),
.png => {
var arena: std.heap.ArenaAllocator = .init(app.allocator);
defer arena.deinit();
diff --git a/src/main.zig b/src/main.zig
index 7b68b7e6a..80e40a802 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -177,12 +177,12 @@ fn run(allocator: Allocator, main_arena: Allocator, proc_args: std.process.Args)
.wait_selector = opts.wait_selector,
.dump_mode = opts.dump,
.selector = opts.dump_selector,
- .max_bytes = opts.dump_max_bytes,
.fail_on_http_error = opts.fail_on_http_error,
.dump = .{
.strip = opts.strip_mode,
.with_base = opts.with_base,
.with_frames = opts.with_frames,
+ .max_bytes = opts.dump_max_bytes,
},
.json = opts.json,
};