Skip to content

Lightpanda: the headless browser designed for AI and automation

fetch: add --selector and --max-bytes, honor --strip-mode for markdown

Adrià Arrufatd3bad960c36a
diff --git a/src/Config.zig b/src/Config.zig
index 676995c7c..2a26867af 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -374,6 +374,8 @@ const Commands = cli.Builder(.{
.{ .name = "with_base", .type = bool },
.{ .name = "with_frames", .type = bool },
.{ .name = "strip_mode", .type = dump.Opts.Strip, .default = dump.Opts.Strip{} },
+ .{ .name = "selector", .type = ?[:0]const u8 },
+ .{ .name = "max_bytes", .type = ?u32 },
.{ .name = "wait_ms", .type = u32, .default = 5_000 },
.{ .name = "wait_until", .type = ?WaitUntil },
.{
diff --git a/src/browser/markdown.zig b/src/browser/markdown.zig
index 50dc77b8b..ca9cecc81 100644
--- a/src/browser/markdown.zig
+++ b/src/browser/markdown.zig
@@ -28,9 +28,13 @@ const Slot = @import("webapi/element/html/Slot.zig");
const isAllWhitespace = @import("../string.zig").isAllWhitespace;
const LimitedWriter = @import("../LimitedWriter.zig");
+const Strip = @import("dump.zig").Opts.Strip;
pub const Opts = struct {
max_bytes: ?u32 = null,
+ // Only `ui` (images) applies: scripts, styles and hidden elements are
+ // never rendered.
+ strip: Strip = .{},
};
const truncation_marker = LimitedWriter.truncation_marker;
@@ -148,6 +152,7 @@ const Context = struct {
writer: *std.Io.Writer,
frame: *Frame,
root: *Node,
+ strip: Strip,
// When there's a slot-attribute, we skip rendering, unless this flag has
// bet set to true.
@@ -318,6 +323,7 @@ const Context = struct {
return;
},
.img => {
+ if (self.strip.ui) return;
try self.writer.writeAll("![");
if (el.getAttributeSafe(comptime .wrap("alt"))) |alt| {
try self.escape(alt);
@@ -524,6 +530,7 @@ pub fn dump(node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !voi
.writer = &lw.writer,
.frame = frame,
.root = node,
+ .strip = opts.strip,
};
ctx.render(node) catch |err| switch (err) {
error.WriteFailed => {
@@ -543,6 +550,7 @@ pub fn dump(node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !voi
.writer = writer,
.frame = frame,
.root = node,
+ .strip = opts.strip,
};
try ctx.render(node);
if (!ctx.state.last_char_was_newline) {
@@ -857,6 +865,22 @@ test "browser.markdown: scoped dump of a hidden subtree still renders it" {
try testing.expectString("\ndialog text\n", aw.written());
}
+test "browser.markdown: strip.ui drops images" {
+ const frame = try testing.createFrame();
+ defer testing.test_session.closeAllPages();
+ frame.url = "http://localhost/";
+
+ const doc = frame.window._document;
+ const div = try doc.createElement("div", null, frame);
+ try Frame.parse.htmlAsChildren(frame, div.asNode(), "<p>Text <img src=\"a.png\" alt=\"A\"> more</p>");
+
+ var aw: std.Io.Writer.Allocating = .init(testing.allocator);
+ defer aw.deinit();
+ try dump(div.asNode(), .{ .strip = .{ .ui = true } }, &aw.writer, frame);
+
+ try testing.expectString("\nText more\n", aw.written());
+}
+
test "browser.markdown: max_bytes leaves output untouched when under cap" {
const frame = try testing.createFrame();
defer testing.test_session.closeAllPages();
diff --git a/src/help.zon b/src/help.zon
index 225e8cccb..8df00c1e1 100644
--- a/src/help.zon
+++ b/src/help.zon
@@ -84,12 +84,21 @@
\\ prints one object; multiple URLs print {{"results": [ ... ]}} with
\\ one object per URL. When used with --dump <MODE> the dumped
\\ content is wrapped within each object.
+ \\ --max-bytes <INT>
+ \\ Soft cap on the dumped html or markdown, in bytes. Output is cut
+ \\ at a UTF-8 boundary and a '[truncated]' marker is appended.
+ \\ Defaults to no cap.
\\ --metrics
\\ Write metrics (Prometheus text format) to stdout on exit, after
\\ any --dump output.
\\ Defaults to false.
+ \\ --selector <QUERY>
+ \\ Dump only the first element matching the CSS selector instead
+ \\ of the whole document. Fails if nothing matches.
\\ --strip-mode <STRIP>
\\ Tag group to remove from dump. Can be passed multiple times.
+ \\ Markdown never contains scripts, styles or hidden elements, so
+ \\ only 'ui' (images) applies to it.
\\ Defaults to no-strip.
\\ Allowed values:
\\ js Script and link[as=script, rel=preload].
diff --git a/src/lightpanda.zig b/src/lightpanda.zig
index 72ef4aa4c..7e5c2050d 100644
--- a/src/lightpanda.zig
+++ b/src/lightpanda.zig
@@ -40,6 +40,9 @@ pub const dump = @import("browser/dump.zig");
pub const markdown = @import("browser/markdown.zig");
pub const screenshot = @import("browser/screenshot.zig");
pub const Base64Writer = @import("Base64Writer.zig");
+const LimitedWriter = @import("LimitedWriter.zig");
+const Selector = @import("browser/webapi/selector/Selector.zig");
+const Node = @import("browser/webapi/Node.zig");
pub const SemanticTree = @import("SemanticTree.zig");
pub const CDPNode = @import("cdp/Node.zig");
pub const interactive = @import("browser/interactive.zig");
@@ -186,6 +189,10 @@ pub const FetchOpts = struct {
wait_selector: ?[:0]const u8 = null,
dump: dump.Opts,
dump_mode: ?Config.DumpFormat = null,
+ /// Dump only the first match instead of the document.
+ selector: ?[:0]const u8 = null,
+ /// html and markdown only.
+ max_bytes: ?u32 = null,
writer: ?*std.Io.Writer = null,
json: bool = false,
};
@@ -345,7 +352,7 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet
const arena = try app.arena_pool.acquire(.large, "screenshot.dump");
defer arena.release();
- const shot = try screenshot.prepare(arena.allocator(), frame.?.window._document.asNode(), .{
+ const shot = try screenshot.prepare(arena.allocator(), try dumpRoot(frame.?, opts.selector), .{
.width = frame.?._page.getViewport().width,
}, frame.?);
try writeJsonEnvelope(writer, frame, opts.dump_mode, shot);
@@ -356,7 +363,7 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet
defer aw.deinit();
if (opts.dump_mode) |mode| {
- if (frame) |f| try dumpContent(app, mode, opts.dump, f, &aw.writer);
+ if (frame) |f| try dumpContent(app, mode, opts, f, &aw.writer);
}
try writeJsonEnvelope(writer, frame, opts.dump_mode, aw.written());
}
@@ -372,20 +379,38 @@ pub fn fetch(app: *App, browser: *Browser, urls: []const [:0]const u8, opts: Fet
try writer.writeAll("Frame closed. Please open a bug report including the URL\n");
break :blk;
};
- try dumpContent(app, mode, opts.dump, frame, writer);
+ try dumpContent(app, mode, opts, frame, writer);
}
}
try writer.flush();
}
-fn dumpContent(app: *App, mode: Config.DumpFormat, dump_opts: dump.Opts, frame: *Frame, writer: *std.Io.Writer) !void {
+fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {
+ const document = frame.window._document.asNode();
+ const sel = selector orelse return document;
+ const element = try Selector.querySelector(document, sel, frame);
+ return (element orelse return error.SelectorNotFound).asNode();
+}
+
+fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Frame, writer: *std.Io.Writer) !void {
+ const root = try dumpRoot(frame, opts.selector);
switch (mode) {
- .html => try dump.root(frame.window._document, dump_opts, writer, frame),
- .markdown => try markdown.dump(frame.window._document.asNode(), .{}, writer, frame),
+ .html => {
+ var lw: LimitedWriter = .init(writer, opts.max_bytes);
+ const result = if (opts.selector == null)
+ dump.root(frame.window._document, opts.dump, &lw.writer, frame)
+ else
+ dump.deep(root, opts.dump, &lw.writer, frame);
+ result catch |err| {
+ if (!lw.truncated) return err;
+ try writer.writeAll(LimitedWriter.truncation_marker);
+ };
+ },
+ .markdown => try markdown.dump(root, .{ .max_bytes = opts.max_bytes, .strip = opts.dump.strip }, writer, frame),
.png => {
var arena: std.heap.ArenaAllocator = .init(app.allocator);
defer arena.deinit();
- _ = try screenshot.png(arena.allocator(), frame.window._document.asNode(), .{
+ _ = try screenshot.png(arena.allocator(), root, .{
.width = frame._page.getViewport().width,
}, writer, frame);
},
@@ -394,7 +419,7 @@ fn dumpContent(app: *App, mode: Config.DumpFormat, dump_opts: dump.Opts, frame:
defer registry.deinit();
const st: SemanticTree = .{
- .dom_node = frame.window._document.asNode(),
+ .dom_node = root,
.registry = &registry,
.frame = frame,
.arena = frame.call_arena,
diff --git a/src/main.zig b/src/main.zig
index c1c19c984..ef0726458 100644
--- a/src/main.zig
+++ b/src/main.zig
@@ -163,6 +163,11 @@ fn run(allocator: Allocator, main_arena: Allocator, proc_args: std.process.Args)
});
}
+ if (opts.max_bytes != null and opts.dump != .html and opts.dump != .markdown) {
+ log.fatal(.app, "--max-bytes needs a text dump", .{ .dump = opts.dump, .allowed = "html, markdown" });
+ return error.InvalidArgument;
+ }
+
var fetch_opts = lp.FetchOpts{
.wait_ms = opts.wait_ms,
.wait_until = opts.wait_until,
@@ -170,6 +175,8 @@ fn run(allocator: Allocator, main_arena: Allocator, proc_args: std.process.Args)
.inject_script = opts.inject_script,
.wait_selector = opts.wait_selector,
.dump_mode = opts.dump,
+ .selector = opts.selector,
+ .max_bytes = opts.max_bytes,
.dump = .{
.strip = opts.strip_mode,
.with_base = opts.with_base,