Lightpanda: the headless browser designed for AI and automation
dump: clutter/invisible strip out template
Karl Seguin345f865a4bbc
diff --git a/src/browser/RenderTree.zig b/src/browser/RenderTree.zigindex fe3ea26c9..e58d91430 100644--- a/src/browser/RenderTree.zig+++ b/src/browser/RenderTree.zig@@ -46,6 +46,12 @@ pub const State = struct {state: State,frame: *Frame,+skipped: ?Skipped = null,++pub const Skipped = struct {+ allocator: std.mem.Allocator,+ set: *PruneSet,+};pub const Child = struct {node: *Node,@@ -68,7 +74,10 @@ const Children = struct {pub fn next(self: *Children) ?Child {while (self.next_node) |node| {self.next_node = node.nextSibling();- var child = self.tree.classify(node, .{ .boxed = self.boxed }) orelse continue;+ var child = self.tree.classify(node, .{ .boxed = self.boxed }) orelse {+ self.tree.recordSkipped(node);+ continue;+ };child.separated = self.boxed and self.yielded;self.yielded = true;return child;@@ -91,12 +100,25 @@ pub const Slotted = struct {while (self.assigned.len > 0) {const node = self.assigned[0];self.assigned = self.assigned[1..];- return self.tree.classify(node, .{ .slotted = true }) orelse continue;+ return self.tree.classify(node, .{ .slotted = true }) orelse {+ self.tree.recordSkipped(node);+ continue;+ };}return self.fallback.next();}};+fn recordSkipped(self: *const RenderTree, node: *Node) void {+ const skipped = self.skipped orelse return;+ const el = node.is(Element) orelse return;+ if (el.getTag() == .head) {+ // <head> renders nothing, but we still need its title & base+ return;+ }+ skipped.set.put(skipped.allocator, node, {}) catch {};+}+pub fn children(self: *const RenderTree, parent: *Node, boxed: bool) Children {return .{ .tree = self, .next_node = parent.firstChild(), .boxed = boxed };}diff --git a/src/browser/clutter.zig b/src/browser/clutter.zigindex 5c64fb482..bf9db8868 100644--- a/src/browser/clutter.zig+++ b/src/browser/clutter.zig@@ -79,6 +79,7 @@ pub fn select(allocator: Allocator, root: *Node, strip_: RenderTree.Strip, frame.flags = flags,.tree = .{ .frame = frame, .state = .{ .root = root, .strip = strip } },};+ pass.tree.skipped = .{ .allocator = arena, .set = &pass.pruned };const selected = try pass.run() orelse continue;const chars = pass.selectedText(selected);log.debug(.browser, "strip clutter", .{ .attempt = i, .chars = chars, .total = pass.total, .root = describe(selected) });@@ -965,7 +966,8 @@ test "clutter: the HTML dump leaves out what the markdown dump does" {try Frame.parse.htmlAsChildren(frame, div.asNode(), "<script>window.__DATA__ = 1;</script>" ++"<div hidden>Accept our cookies</div>" ++"<div class=\"teasers\"><div><a href=\"/1\">Ten things to know</a></div><div><a href=\"/2\">Trending now</a></div></div>" ++- "<div class=\"article\"><p>" ++ prose ++ "</p><p>" ++ prose ++ "</p><p>" ++ prose ++ "</p></div>");+ "<div class=\"article\"><script>INNER_SCRIPT</script><div hidden>INNER_HIDDEN</div><span aria-hidden=\"true\">INNER_ARIA</span><template><p>INNER_TEMPLATE</p></template>" +++ "<p>" ++ prose ++ "</p><p>" ++ prose ++ "</p><p>" ++ prose ++ "</p></div>");const state = try RenderTree.resolve(testing.arena_allocator, div.asNode(), .{ .clutter = true }, frame);var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);@@ -978,6 +980,11 @@ test "clutter: the HTML dump leaves out what the markdown dump does" {try testing.expectEqual(null, std.mem.indexOf(u8, out, "__DATA__"));try testing.expectEqual(null, std.mem.indexOf(u8, out, "Accept our cookies"));try testing.expectEqual(null, std.mem.indexOf(u8, out, "Trending now"));+ // Inside the selection too: the walk records what it skips.+ try testing.expectEqual(null, std.mem.indexOf(u8, out, "INNER_SCRIPT"));+ try testing.expectEqual(null, std.mem.indexOf(u8, out, "INNER_HIDDEN"));+ try testing.expectEqual(null, std.mem.indexOf(u8, out, "INNER_ARIA"));+ try testing.expectEqual(null, std.mem.indexOf(u8, out, "INNER_TEMPLATE"));}fn extract(html: []const u8) ![]const u8 {diff --git a/src/browser/dump.zig b/src/browser/dump.zigindex 95d33a8a0..361bfa459 100644--- a/src/browser/dump.zig+++ b/src/browser/dump.zig@@ -510,6 +510,9 @@ pub fn shouldStripElement(el: *Node.Element, strip: Opts.Strip, pruned: ?*const}if (strip.invisible) {+ if (el.is(Template) != null) {+ return true;+ }if (el.ownerFrame(frame)) |owner| {if (owner._style_manager.hasAuthorDisplayNone(el)) {return true;@@ -787,6 +790,24 @@ test "dump: strip.invisible removes author display:none elements" {);}+test "dump: strip.invisible removes templates and their content" {+ const frame = try testing.createFrame();+ defer testing.test_session.closeAllPages();++ const doc = frame.window._document;+ const div = try doc.createElement("div", null, frame);+ try Frame.parse.htmlAsChildren(frame, div.asNode(), "<template><p>row</p></template><p>text</p>");++ var aw: std.Io.Writer.Allocating = .init(testing.allocator);+ defer aw.deinit();+ try deep(div.asNode(), .{}, &aw.writer, frame);+ try testing.expectString("<div><template><p>row</p></template><p>text</p></div>", aw.written());++ aw.clearRetainingCapacity();+ try deep(div.asNode(), .{ .strip = .{ .invisible = true } }, &aw.writer, frame);+ try testing.expectString("<div><p>text</p></div>", aw.written());+}+test "dump: strip.shell removes page chrome but keeps sectioned header/footer" {try expectShellDump(\\<header>H</header><nav>N</nav><main><header>MH</header><p>body</p><footer>MF</footer></main><article><footer>AF</footer></article><aside>A</aside><dialog>D</dialog><footer>F</footer>diff --git a/src/help.zon b/src/help.zonindex d00ecca73..c0d7e2d74 100644--- a/src/help.zon+++ b/src/help.zon@@ -134,7 +134,8 @@\\ (which may itself fall back to a full dump).\\ css Includes style and link[rel=stylesheet].\\ js Script and link[as=script, rel=preload].- \\ invisible Best-effort (e.g. display:none) hidden elements+ \\ invisible Best-effort hidden elements (e.g. display:none,+ \\ <template>).\\ shell Page chrome by markup: nav, aside, dialog,\\ page-level header and footer. Full dump when\\ `shell` would remove most of the text.