Lightpanda: the headless browser designed for AI and automation
fetch: add --selector and --max-bytes, honor --strip-mode for markdown
Adrià Arrufatd3bad960c36a
diff --git a/src/Config.zig b/src/Config.zigindex 676995c7c..2a26867af 100644--- a/src/Config.zig+++ b/src/Config.zig@@ -374,6 +374,8 @@ const Commands = cli.Builder(.{.{ .name = "with_base", .type = bool },.{ .name = "with_frames", .type = bool },.{ .name = "strip_mode", .type = dump.Opts.Strip, .default = dump.Opts.Strip{} },+ .{ .name = "selector", .type = ?[:0]const u8 },+ .{ .name = "max_bytes", .type = ?u32 },.{ .name = "wait_ms", .type = u32, .default = 5_000 },.{ .name = "wait_until", .type = ?WaitUntil },.{diff --git a/src/browser/markdown.zig b/src/browser/markdown.zigindex 50dc77b8b..ca9cecc81 100644--- a/src/browser/markdown.zig+++ b/src/browser/markdown.zig@@ -28,9 +28,13 @@ const Slot = @import("webapi/element/html/Slot.zig");const isAllWhitespace = @import("../string.zig").isAllWhitespace;const LimitedWriter = @import("../LimitedWriter.zig");+const Strip = @import("dump.zig").Opts.Strip;pub const Opts = struct {max_bytes: ?u32 = null,+ // Only `ui` (images) applies: scripts, styles and hidden elements are+ // never rendered.+ strip: Strip = .{},};const truncation_marker = LimitedWriter.truncation_marker;@@ -148,6 +152,7 @@ const Context = struct {writer: *std.Io.Writer,frame: *Frame,root: *Node,+ strip: Strip,// When there's a slot-attribute, we skip rendering, unless this flag has// bet set to true.@@ -318,6 +323,7 @@ const Context = struct {return;},.img => {+ if (self.strip.ui) return;try self.writer.writeAll("![");if (el.getAttributeSafe(comptime .wrap("alt"))) |alt| {try self.escape(alt);@@ -524,6 +530,7 @@ pub fn dump(node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !voi.writer = &lw.writer,.frame = frame,.root = node,+ .strip = opts.strip,};ctx.render(node) catch |err| switch (err) {error.WriteFailed => {@@ -543,6 +550,7 @@ pub fn dump(node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !voi.writer = writer,.frame = frame,.root = node,+ .strip = opts.strip,};try ctx.render(node);if (!ctx.state.last_char_was_newline) {@@ -857,6 +865,22 @@ test "browser.markdown: scoped dump of a hidden subtree still renders it" {try testing.expectString("\ndialog text\n", aw.written());}+test "browser.markdown: strip.ui drops images" {+ const frame = try testing.createFrame();+ defer testing.test_session.closeAllPages();+ frame.url = "http://localhost/";++ const doc = frame.window._document;+ const div = try doc.createElement("div", null, frame);+ try Frame.parse.htmlAsChildren(frame, div.asNode(), "<p>Text <img src=\"a.png\" alt=\"A\"> more</p>");++ var aw: std.Io.Writer.Allocating = .init(testing.allocator);+ defer aw.deinit();+ try dump(div.asNode(), .{ .strip = .{ .ui = true } }, &aw.writer, frame);++ try testing.expectString("\nText more\n", aw.written());+}+test "browser.markdown: max_bytes leaves output untouched when under cap" {const frame = try testing.createFrame();defer testing.test_session.closeAllPages();diff --git a/src/help.zon b/src/help.zonindex 225e8cccb..8df00c1e1 100644--- a/src/help.zon+++ b/src/help.zon@@ -84,12 +84,21 @@\\ prints one object; multiple URLs print {{"results": [ ... ]}} with\\ one object per URL. When used with --dump <MODE> the dumped\\ content is wrapped within each object.+ \\ --max-bytes <INT>+ \\ Soft cap on the dumped html or markdown, in bytes. Output is cut+ \\ at a UTF-8 boundary and a '[truncated]' marker is appended.+ \\ Defaults to no cap.\\ --metrics\\ Write metrics (Prometheus text format) to stdout on exit, after\\ any --dump output.\\ Defaults to false.+ \\ --selector <QUERY>+ \\ Dump only the first element matching the CSS selector instead+ \\ of the whole document. Fails if nothing matches.\\ --strip-mode <STRIP>\\ Tag group to remove from dump. Can be passed multiple times.+ \\ Markdown never contains scripts, styles or hidden elements, so+ \\ only 'ui' (images) applies to it.\\ Defaults to no-strip.\\ Allowed values:\\ js Script and link[as=script, rel=preload].diff --git a/src/lightpanda.zig b/src/lightpanda.zigindex 72ef4aa4c..7e5c2050d 100644--- a/src/lightpanda.zig+++ b/src/lightpanda.zig@@ -40,6 +40,9 @@ pub const dump = @import("browser/dump.zig");pub const markdown = @import("browser/markdown.zig");pub const screenshot = @import("browser/screenshot.zig");pub const Base64Writer = @import("Base64Writer.zig");+const LimitedWriter = @import("LimitedWriter.zig");+const Selector = @import("browser/webapi/selector/Selector.zig");+const Node = @import("browser/webapi/Node.zig");pub const SemanticTree = @import("SemanticTree.zig");pub const CDPNode = @import("cdp/Node.zig");pub const interactive = @import("browser/interactive.zig");@@ -186,6 +189,10 @@ pub const FetchOpts = struct {wait_selector: ?[:0]const u8 = null,dump: dump.Opts,dump_mode: ?Config.DumpFormat = null,+ /// Dump only the first match instead of the document.+ selector: ?[:0]const u8 = null,+ /// html and markdown only.+ max_bytes: ?u32 = null,writer: ?*std.Io.Writer = null,json: bool = false,};@@ -345,7 +352,7 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fetconst arena = try app.arena_pool.acquire(.large, "screenshot.dump");defer arena.release();- const shot = try screenshot.prepare(arena.allocator(), frame.?.window._document.asNode(), .{+ const shot = try screenshot.prepare(arena.allocator(), try dumpRoot(frame.?, opts.selector), .{.width = frame.?._page.getViewport().width,}, frame.?);try writeJsonEnvelope(writer, frame, opts.dump_mode, shot);@@ -356,7 +363,7 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fetdefer aw.deinit();if (opts.dump_mode) |mode| {- if (frame) |f| try dumpContent(app, mode, opts.dump, f, &aw.writer);+ if (frame) |f| try dumpContent(app, mode, opts, f, &aw.writer);}try writeJsonEnvelope(writer, frame, opts.dump_mode, aw.written());}@@ -372,20 +379,38 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fettry writer.writeAll("Frame closed. Please open a bug report including the URL\n");break :blk;};- try dumpContent(app, mode, opts.dump, frame, writer);+ try dumpContent(app, mode, opts, frame, writer);}}try writer.flush();}-fn dumpContent(app: *App, mode: Config.DumpFormat, dump_opts: dump.Opts, frame: *Frame, writer: *std.Io.Writer) !void {+fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {+ const document = frame.window._document.asNode();+ const sel = selector orelse return document;+ const element = try Selector.querySelector(document, sel, frame);+ return (element orelse return error.SelectorNotFound).asNode();+}++fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Frame, writer: *std.Io.Writer) !void {+ const root = try dumpRoot(frame, opts.selector);switch (mode) {- .html => try dump.root(frame.window._document, dump_opts, writer, frame),- .markdown => try markdown.dump(frame.window._document.asNode(), .{}, writer, frame),+ .html => {+ var lw: LimitedWriter = .init(writer, opts.max_bytes);+ const result = if (opts.selector == null)+ dump.root(frame.window._document, opts.dump, &lw.writer, frame)+ else+ dump.deep(root, opts.dump, &lw.writer, frame);+ result catch |err| {+ if (!lw.truncated) return err;+ try writer.writeAll(LimitedWriter.truncation_marker);+ };+ },+ .markdown => try markdown.dump(root, .{ .max_bytes = opts.max_bytes, .strip = opts.dump.strip }, writer, frame),.png => {var arena: std.heap.ArenaAllocator = .init(app.allocator);defer arena.deinit();- _ = try screenshot.png(arena.allocator(), frame.window._document.asNode(), .{+ _ = try screenshot.png(arena.allocator(), root, .{.width = frame._page.getViewport().width,}, writer, frame);},@@ -394,7 +419,7 @@ fn dumpContent(app: *App, mode: Config.DumpFormat, dump_opts: dump.Opts, frame:defer registry.deinit();const st: SemanticTree = .{- .dom_node = frame.window._document.asNode(),+ .dom_node = root,.registry = ®istry,.frame = frame,.arena = frame.call_arena,diff --git a/src/main.zig b/src/main.zigindex c1c19c984..ef0726458 100644--- a/src/main.zig+++ b/src/main.zig@@ -163,6 +163,11 @@ fn run(allocator: Allocator, main_arena: Allocator, proc_args: std.process.Args)});}+ if (opts.max_bytes != null and opts.dump != .html and opts.dump != .markdown) {+ log.fatal(.app, "--max-bytes needs a text dump", .{ .dump = opts.dump, .allowed = "html, markdown" });+ return error.InvalidArgument;+ }+var fetch_opts = lp.FetchOpts{.wait_ms = opts.wait_ms,.wait_until = opts.wait_until,@@ -170,6 +175,8 @@ fn run(allocator: Allocator, main_arena: Allocator, proc_args: std.process.Args).inject_script = opts.inject_script,.wait_selector = opts.wait_selector,.dump_mode = opts.dump,+ .selector = opts.selector,+ .max_bytes = opts.max_bytes,.dump = .{.strip = opts.strip_mode,.with_base = opts.with_base,