Lightpanda: the headless browser designed for AI and automation
address feedback
Karl Seguine8f65914115e
diff --git a/src/browser/clutter.zig b/src/browser/clutter.zigindex 793159753..e7964767a 100644--- a/src/browser/clutter.zig+++ b/src/browser/clutter.zig@@ -126,7 +126,9 @@ const Stats = struct {cells: u32 = 0,th: u32 = 0,tables: u32 = 0,- table_head: bool = false,+ // A <caption>/<col>/<colgroup>/<thead>/<tfoot> descendant: markup only a+ // real data table carries.+ table_parts: bool = false,has_data_table: bool = false,// A direct child that keeps a <div> from reading as a paragraph.@@ -152,7 +154,7 @@ const Stats = struct {self.cells += other.cells;self.th += other.th;self.tables += other.tables;- self.table_head = self.table_head or other.table_head;+ self.table_parts = self.table_parts or other.table_parts;self.has_data_table = self.has_data_table or other.has_data_table;}@@ -242,7 +244,8 @@ const Pass = struct {const top = self.topCandidate() orelse return null;const candidate = self.refine(top);- log.debug(.browser, "clutter candidate", .{ .top = describe(top.asNode()), .score = self.score(top), .refined = describe(candidate.asNode()) });+ log.debug(.browser, "clutter candidate", .{ .top = describe(top.asNode()), .score = self.score(top) });+ log.debug(.browser, "clutter refined", .{ .el = describe(candidate.asNode()) });const tag = candidate.getTag();if (tag == .body or tag == .html) {// readability wraps the whole body here; that is no selection.@@ -293,6 +296,9 @@ const Pass = struct {.code, .pre => inner.in_code = true,.table => inner.in_table = true,.figure => inner.in_figure = true,+ // readability's _markDataTables descendants: a header row+ // built from <td> is still a header row.+ .caption, .col, .colgroup, .tfoot, .thead => st.table_parts = true,else => {},}@@ -448,16 +454,22 @@ const Pass = struct {const class = candidate.getClassName() orelse "";var kept: usize = 0;- var it = self.tree.children(parent.asNode(), false);- while (it.next()) |child| {- if (child.what != .element) {+ // Raw children, as in pruneBeside: a node RenderTree never yields has+ // to enter the set too, or only the HTML dump keeps it.+ var kid = parent.asNode().firstChild();+ while (kid) |k| : (kid = k.nextSibling()) {+ const sibling = k.is(Element) orelse {// Loose text between siblings is not part of any of them.- self.pruned.put(self.arena, child.node, {}) catch {};+ self.pruned.put(self.arena, k, {}) catch {};continue;- }- const sibling = child.node.subtype(Element);+ };if (sibling == candidate) continue;- const st = self.stats.get(sibling) orelse continue;+ // No stats: it never rendered, or the walk already pruned it as+ // unlikely. Neither is content.+ const st = self.stats.get(sibling) orelse {+ self.pruned.put(self.arena, k, {}) catch {};+ continue;+ };var keep = false;var bonus: f32 = 0;@@ -485,19 +497,30 @@ const Pass = struct {}/// Everything beside the path from the dump root down to `selected`:- /// each ancestor keeps only the child on the path. Shadow trees render- /// in place of their host, so a shadow root's parent is its host.+ /// each ancestor keeps only the child on the path.+ ///+ /// Raw siblings, not rendering ones: the HTML dump walks the DOM and+ /// leaves out only what this set holds, so a <script> or a [hidden]+ /// banner beside the path has to be in it to be left out of that dump+ /// too. The renderers that go through RenderTree never see those.fn pruneBeside(self: *Pass, selected: *Node) !void {var current = selected;while (current != self.root) {- const parent = current.parentNode() orelse blk: {- const shadow = current.is(Node.ShadowRoot) orelse return;- break :blk shadow.getHost().asNode();- };- var it = self.tree.children(parent, false);- while (it.next()) |child| {- if (child.node == current) continue;- try self.pruned.put(self.arena, child.node, {});+ if (current.is(Node.ShadowRoot)) |shadow| {+ // A shadow tree renders in place of its host, and the host's+ // light children are not beside it: they render through its+ // slots, inside the selection.+ current = shadow.getHost().asNode();+ continue;+ }+ const parent = current.parentNode() orelse return;+ var child = parent.firstChild();+ while (child) |c| : (child = c.nextSibling()) {+ if (c == current) continue;+ // <head> sits beside <body> but holds no content of its own,+ // and the HTML dump still needs its title and base.+ if (c.is(Element)) |el| if (el.getTag() == .head) continue;+ try self.pruned.put(self.arena, c, {});}current = parent;}@@ -716,7 +739,10 @@ const Pass = struct {var it = self.tree.children(node, false);while (it.next()) |child| {switch (child.what) {- .text => |text| n += measureText(text).len,+ .text => |text| {+ if (self.pruned.contains(child.node)) continue;+ n += measureText(text).len;+ },.element => {const el = child.node.subtype(Element);if (self.pruned.contains(child.node)) continue;@@ -778,7 +804,7 @@ fn isDataTable(el: *Element, st: *const Stats) bool {if (std.mem.eql(u8, v, "0")) return false;}if (el.getAttributeSafe(comptime .wrap("summary")) != null) return true;- if (st.th > 0 or st.table_head) return true;+ if (st.th > 0 or st.table_parts) return true;if (st.tables > 0) return false;const cols = if (st.rows == 0) st.cells else st.cells / st.rows;if (st.rows >= 10 or cols > 4) return true;@@ -861,6 +887,7 @@ fn hasRole(el: *Element, roles: []const []const u8) bool {const testing = @import("../testing.zig");const markdown = @import("markdown.zig");+const dump = @import("dump.zig");const prose = "Sourdough is a bread made by the fermentation of dough using wild lactobacillaceae and yeast, which give it a mildly sour taste. The lactic acid produced by the bacteria gives it a longer shelf life than breads made with baker's yeast. ";@@ -919,6 +946,30 @@ test "clutter: the prune set lives in the caller's allocator" {try testing.expectEqual(null, second.pruned);}+test "clutter: the HTML dump leaves out what the markdown dump does" {+ const frame = try testing.createFrame();+ defer testing.test_session.closeAllPages();++ const doc = frame.window._document;+ const div = try doc.createElement("div", null, frame);+ try Frame.parse.htmlAsChildren(frame, div.asNode(), "<script>window.__DATA__ = 1;</script>" +++ "<div hidden>Accept our cookies</div>" +++ "<div class=\"teasers\"><div><a href=\"/1\">Ten things to know</a></div><div><a href=\"/2\">Trending now</a></div></div>" +++ "<div class=\"article\"><p>" ++ prose ++ "</p><p>" ++ prose ++ "</p><p>" ++ prose ++ "</p></div>");++ const state = try RenderTree.resolve(testing.arena_allocator, div.asNode(), .{ .clutter = true }, frame);+ var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);+ try dump.render(state, .{}, &aw.writer, frame);+ const out = aw.written();++ try testing.expectEqual(true, std.mem.indexOf(u8, out, "wild lactobacillaceae") != null);+ // Neither node renders, so RenderTree never yields either one; the set+ // still has to hold them, because the HTML dump walks the DOM itself.+ try testing.expectEqual(null, std.mem.indexOf(u8, out, "__DATA__"));+ try testing.expectEqual(null, std.mem.indexOf(u8, out, "Accept our cookies"));+ try testing.expectEqual(null, std.mem.indexOf(u8, out, "Trending now"));+}+fn extract(html: []const u8) ![]const u8 {const frame = try testing.createFrame();defer testing.test_session.closeAllPages();diff --git a/src/browser/dump.zig b/src/browser/dump.zigindex b86fcf4b3..5f81a4533 100644--- a/src/browser/dump.zig+++ b/src/browser/dump.zig@@ -114,6 +114,14 @@ pub fn deep(node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) erro};}+pub fn render(state: RenderTree.State, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {+ var o = opts;+ o.strip = state.strip;+ o.pruned = state.pruned;+ if (state.root.is(Node.Document)) |doc| return root(doc, o, writer, frame);+ return deep(state.root, o, writer, frame);+}+fn _deep(node: *Node, opts: Opts, comptime force_slot: bool, writer: *std.Io.Writer, frame: *Frame) error{WriteFailed}!void {switch (node._type) {.cdata => {@@ -439,7 +447,8 @@ pub fn shouldStripElement(el: *Node.Element, strip: Opts.Strip, pruned: ?*const/// level: inside an article, section, main, nav or aside they belong to that/// content, which is also how the banner/contentinfo roles are assigned.pub fn isShellElement(el: *Node.Element) bool {- switch (el.getTag()) {+ const tag = el.getTag();+ switch (tag) {.nav, .aside, .dialog => return true,.header, .footer => return !hasSectioningAncestor(el),else => {},@@ -447,12 +456,24 @@ pub fn isShellElement(el: *Node.Element) bool {if (hasRole(el, &.{ "banner", "complementary", "contentinfo", "navigation", "search", "dialog", "alertdialog", "menu", "menubar" })) {return true;}- if (hasShellToken(el.getClassName()) or hasShellToken(el.getId())) {- return !hasSectioningAncestor(el);+ if (canHoldChrome(tag)) {+ if (hasShellToken(el.getClassName()) or hasShellToken(el.getId())) {+ return !hasSectioningAncestor(el);+ }}return false;}+// `td` stays in because table-based layout will do things like+// <td class=sidebar>, but `tr`, `thead` and `li` because "header" is often used+// to mean something other than the header of the site+fn canHoldChrome(tag: Node.Element.Tag) bool {+ return switch (tag) {+ .div, .section, .ul, .ol, .form, .table, .td, .p => true,+ else => false,+ };+}+// Words that name page chrome and nothing else. "menu" is left out: it also// names content.const shell_tokens = [_][]const u8{ "header", "footer", "nav", "navbar", "navigation", "sidebar", "masthead" };@@ -467,8 +488,8 @@ fn hasShellToken(value: ?[]const u8) bool {return false;}fn hasSectioningAncestor(el: *Node.Element) bool {- var node = el.asNode().parentNode();- while (node) |n| : (node = n.parentNode()) {+ var node = renderParent(el.asNode());+ while (node) |n| : (node = renderParent(n)) {if (n.is(Node.Element)) |ancestor| {switch (ancestor.getTag()) {.article, .aside, .main, .nav, .section => return true,@@ -482,6 +503,16 @@ fn hasSectioningAncestor(el: *Node.Element) bool {return false;}+// A shadow tree renders in place of its host, so the host continues the+// ancestor chain that a shadow root's null parent would otherwise end.+fn renderParent(node: *Node) ?*Node {+ if (node.parentNode()) |parent| {+ return parent;+ }+ const shadow = node.is(Node.ShadowRoot) orelse return null;+ return shadow.getHost().asNode();+}+// ARIA `role` is a space-separated fallback list; the first token wins.fn hasRole(el: *Node.Element, roles: []const []const u8) bool {const attr = el.getAttributeSafe(comptime .wrap("role")) orelse return false;@@ -689,6 +720,14 @@ test "dump: strip.shell honours landmark roles" {);}+test "dump: strip.shell's token rule only applies to chrome containers" {+ try expectShellDump(+ \\<table><thead><tr class="header"><td>Plan</td></tr></thead><tbody><tr><td>Basic</td><td class="sidebar">Legacy</td></tr></tbody></table><ul><li class="header">Item</li></ul>+ ,+ \\<div><table><thead><tr class="header"><td>Plan</td></tr></thead><tbody><tr><td>Basic</td></tr></tbody></table><ul><li class="header">Item</li></ul></div>+ );+}+fn expectShellDump(html: []const u8, expected: []const u8) !void {const frame = try testing.createFrame();defer testing.test_session.closeAllPages();diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 48567baff..ab2499905 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -385,6 +385,7 @@ pub const Tool = enum {\\ "selector": { "type": "string", "description": "Optional CSS selector. Render markdown for just that element's subtree." },\\ "backendNodeId": { "type": "integer", "description": "Optional backend node ID. Render markdown for just that node's subtree. 0 is treated as omitted." },\\ "maxBytes": { "type": "integer", "description": "Optional soft cap on output size in bytes. Content is truncated at a UTF-8 boundary and a short '[truncated]' marker is appended past the cap." },+ \\ "strip": { "type": "object", "description": "Optional. Omit element groups from the output; same groups as the html tool's strip. `shell` (page chrome by markup) and `clutter` (keep only the main content, in the manner of reader modes) are the ones that matter for reading; `ui` also drops images.", "properties": { "js": { "type": "boolean" }, "css": { "type": "boolean" }, "ui": { "type": "boolean" }, "invisible": { "type": "boolean" }, "shell": { "type": "boolean" }, "clutter": { "type": "boolean" } } },\\ "url": { "type": "string", "description": "Optional URL to navigate to before rendering." },\\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." }\\ }@@ -401,7 +402,7 @@ pub const Tool = enum {\\ "selector": { "type": "string", "description": "Optional CSS selector. When set, dump only that element's outerHTML." },\\ "backendNodeId": { "type": "integer", "description": "Optional backend node ID. When set, dump only that node's outerHTML. 0 is treated as omitted." },\\ "maxBytes": { "type": "integer", "description": "Optional soft cap on output size in bytes. Content is truncated at a UTF-8 boundary and a short '[truncated]' marker is appended past the cap." },- \\ "strip": { "type": "object", "description": "Optional. Omit element groups from the output: `js` (script, noscript, script preloads), `css` (style, stylesheet links), `ui` (css plus img, picture, video, audio, svg, canvas, iframe), `invisible` (elements an author rule or inline style sets to display:none), `shell` (nav, aside, dialog, page-level header/footer and the matching landmark roles; skipped when that would drop most of the text). {\"js\":true,\"css\":true} keeps a page dump small.", "properties": { "js": { "type": "boolean" }, "css": { "type": "boolean" }, "ui": { "type": "boolean" }, "invisible": { "type": "boolean" }, "shell": { "type": "boolean" } } },+ \\ "strip": { "type": "object", "description": "Optional. Omit element groups from the output: `js` (script, noscript, script preloads), `css` (style, stylesheet links), `ui` (css plus img, picture, video, audio, svg, canvas, iframe), `invisible` (elements an author rule or inline style sets to display:none), `shell` (nav, aside, dialog, page-level header/footer and the matching landmark roles; skipped when that would drop most of the text), `clutter` (keep only the main content, in the manner of reader modes; includes `shell` and `invisible`, and falls back to `shell` when it finds too little). {\"js\":true,\"css\":true} keeps a page dump small.", "properties": { "js": { "type": "boolean" }, "css": { "type": "boolean" }, "ui": { "type": "boolean" }, "invisible": { "type": "boolean" }, "shell": { "type": "boolean" }, "clutter": { "type": "boolean" } } },\\ "url": { "type": "string", "description": "Optional URL to navigate to before dumping." },\\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." }\\ }@@ -419,7 +420,7 @@ pub const Tool = enum {\\ "selector": { "type": "string", "description": "Optional CSS selector. When set, render only that element." },\\ "backendNodeId": { "type": "integer", "description": "Optional backend node ID. When set, render only that node. 0 is treated as omitted." },\\ "fullPage": { "type": "boolean", "description": "Render the whole content height instead of one viewport. Defaults to false." },- \\ "strip": { "type": "object", "description": "Optional. Omit element groups from the render; same groups as the html tool's strip (`js`, `css`, `ui`, `invisible`, `shell`).", "properties": { "js": { "type": "boolean" }, "css": { "type": "boolean" }, "ui": { "type": "boolean" }, "invisible": { "type": "boolean" }, "shell": { "type": "boolean" } } },+ \\ "strip": { "type": "object", "description": "Optional. Omit element groups from the render; same groups as the html tool's strip (`js`, `css`, `ui`, `invisible`, `shell`, `clutter`).", "properties": { "js": { "type": "boolean" }, "css": { "type": "boolean" }, "ui": { "type": "boolean" }, "invisible": { "type": "boolean" }, "shell": { "type": "boolean" }, "clutter": { "type": "boolean" } } },\\ "url": { "type": "string", "description": "Optional URL to navigate to before rendering." },\\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." }\\ }@@ -1306,6 +1307,7 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeRselector: ?[]const u8 = null,backendNodeId: ?NodeRegistry.Id = null,maxBytes: ?u32 = null,+ strip: lp.dump.Opts.Strip = .{},url: ?[:0]const u8 = null,timeout: ?u32 = null,};@@ -1313,9 +1315,10 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeRconst page = try ensurePage(session, registry, args.url, args.timeout);const node = try resolveScope(session, registry, page, args.selector, args.backendNodeId);+ const state = lp.RenderTree.resolve(arena, node, args.strip, page) catch return ToolError.OutOfMemory;var aw: std.Io.Writer.Allocating = .init(arena);- lp.markdown.dump(.{ .root = node }, .{ .max_bytes = args.maxBytes }, &aw.writer, page) catch return ToolError.InternalError;+ lp.markdown.dump(state, .{ .max_bytes = args.maxBytes }, &aw.writer, page) catch return ToolError.InternalError;return aw.written();}@@ -1339,20 +1342,10 @@ fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeRegisconst args = try parseArgsOrDefault(HtmlParams, arena, arguments);const page = try ensurePage(session, registry, args.url, args.timeout);- const whole = args.selector == null and args.backendNodeId == null;- const target = if (whole) page.document.asNode() else (try resolveTarget(session, registry, args.selector, args.backendNodeId)).node;+ const target = try resolveScope(session, registry, page, args.selector, args.backendNodeId);const state = lp.RenderTree.resolve(arena, target, args.strip, page) catch return ToolError.OutOfMemory;- const opts: lp.dump.Opts = .{- .strip = state.strip,- .pruned = state.pruned,- .max_bytes = args.maxBytes,- };var aw: std.Io.Writer.Allocating = .init(arena);- if (state.root.is(DOMNode.Document)) |document| {- lp.dump.root(document, opts, &aw.writer, page) catch return ToolError.InternalError;- } else {- lp.dump.deep(state.root, opts, &aw.writer, page) catch return ToolError.InternalError;- }+ lp.dump.render(state, .{ .max_bytes = args.maxBytes }, &aw.writer, page) catch return ToolError.InternalError;return aw.written();}diff --git a/src/cli.zig b/src/cli.zigindex b2ce4d5e5..e5e9abedc 100644--- a/src/cli.zig+++ b/src/cli.zig@@ -104,7 +104,7 @@ const string = @import("string.zig");/// - Enums — parsed via `std.meta.stringToEnum`. Returns/// `error.InvalidArgument` on a bad value. Requires `default` unless `?`./// - Packed structs of `bool` fields — parsed from a comma-separated list-/// (e.g. `--strip-mode js,css`). The literal `"full"` sets every field.+/// (e.g. `--strip-mode js,css`)./// Unknown names return `error.InvalidArgument`. Requires `default`./// `multiple` is not supported./// - Optional types default to `null` when `default` is omitted.@@ -661,32 +661,25 @@ pub fn Builder(comptime commands: anytype) type {const str = args.next() orelse return error.MissingArgument;- if (std.mem.eql(u8, str, "full")) {- // "full" sets all the fields of packed struct.- const Int = _struct.backing_integer orelse @compileError("packed struct must provide a backing integer");- target.* = @bitCast(@as(Int, std.math.maxInt(Int)));- } else {- // Parse given args.- var it = std.mem.tokenizeScalar(u8, str, ',');- outer: while (it.next()) |part| {- const trimmed = std.mem.trim(u8, part, &std.ascii.whitespace);-- inline for (_struct.fields) |f| {- lp.assert(f.type == bool, "all fields of packed struct must be boolean", .{- .option = option.name,- .field = f.name,- });-- if (std.mem.eql(u8, trimmed, @as([]const u8, f.name))) {- @field(target, f.name) = true;- continue :outer;- }- }+ var it = std.mem.tokenizeScalar(u8, str, ',');+ outer: while (it.next()) |part| {+ const trimmed = std.mem.trim(u8, part, &std.ascii.whitespace);++ inline for (_struct.fields) |f| {+ lp.assert(f.type == bool, "all fields of packed struct must be boolean", .{+ .option = option.name,+ .field = f.name,+ });- // Invalid option choice.- log.fatal(.app, "invalid option choice", .{ .arg = kebab_cased, .value = trimmed });- return error.InvalidArgument;+ if (std.mem.eql(u8, trimmed, @as([]const u8, f.name))) {+ @field(target, f.name) = true;+ continue :outer;+ }}++ // Invalid option choice.+ log.fatal(.app, "invalid option choice", .{ .arg = kebab_cased, .value = trimmed });+ return error.InvalidArgument;}},.@"enum" => {diff --git a/src/help.zon b/src/help.zonindex 6a6c9bf58..0ee486c95 100644--- a/src/help.zon+++ b/src/help.zon@@ -122,7 +122,7 @@\\ Allowed values:\\ clutter Heuristic to keep only the main content, in the\\ manner of reader modes. May fallback to `shell`- \\ (which my itself fallback to a full dump).+ \\ (which may itself fall back to a full dump).\\ css Includes style and link[rel=stylesheet].\\ js Script and link[as=script, rel=preload].\\ invisible Best-effort (e.g. display:none) hidden elementsdiff --git a/src/lightpanda.zig b/src/lightpanda.zigindex 48af3a51e..7cb7c9c83 100644--- a/src/lightpanda.zig+++ b/src/lightpanda.zig@@ -481,15 +481,9 @@ fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Framvar arena: std.heap.ArenaAllocator = .init(app.allocator);defer arena.deinit();const state = try RenderTree.resolve(arena.allocator(), try dumpRoot(frame, opts.selector), opts.dump.strip, frame);- var dump_opts = opts.dump;- dump_opts.strip = state.strip;- dump_opts.pruned = state.pruned;switch (mode) {- .html => if (opts.selector == null)- try dump.root(frame.window._document, dump_opts, writer, frame)- else- try dump.deep(state.root, dump_opts, writer, frame),- .markdown => try markdown.dump(state, .{ .max_bytes = dump_opts.max_bytes }, writer, frame),+ .html => try dump.render(state, opts.dump, writer, frame),+ .markdown => try markdown.dump(state, .{ .max_bytes = opts.dump.max_bytes }, writer, frame),.png => {_ = try screenshot.png(arena.allocator(), state, pngOpts(frame), writer, frame);},diff --git a/src/server/cdp/domains/lp.zig b/src/server/cdp/domains/lp.zigindex a7ddf4594..fcfb8fd33 100644--- a/src/server/cdp/domains/lp.zig+++ b/src/server/cdp/domains/lp.zig@@ -186,19 +186,15 @@ fn dump(cmd: anytype) !void {};const state = try lp.RenderTree.resolve(cmd.arena, target, params.strip, frame);- const root = state.root;switch (params.format) {.html, .markdown => {var aw: std.Io.Writer.Allocating = .init(cmd.arena);defer aw.deinit();- const opts: lp.dump.Opts = .{ .strip = state.strip, .pruned = state.pruned, .max_bytes = params.maxBytes };if (params.format == .markdown) {try markdown.dump(state, .{ .max_bytes = params.maxBytes }, &aw.writer, frame);- } else if (root.is(DOMNode.Document)) |doc| {- try lp.dump.root(doc, opts, &aw.writer, frame);} else {- try lp.dump.deep(root, opts, &aw.writer, frame);+ try lp.dump.render(state, .{ .max_bytes = params.maxBytes }, &aw.writer, frame);}return cmd.sendResult(.{ .format = params.format, .content = aw.written() }, .{});},