Skip to content

Lightpanda: the headless browser designed for AI and automation

dump: clutter/invisible strip out template

Karl Seguin345f865a4bbc
diff --git a/src/browser/RenderTree.zig b/src/browser/RenderTree.zig
index fe3ea26c9..e58d91430 100644
--- a/src/browser/RenderTree.zig
+++ b/src/browser/RenderTree.zig
@@ -46,6 +46,12 @@ pub const State = struct {
state: State,
frame: *Frame,
+skipped: ?Skipped = null,
+
+pub const Skipped = struct {
+ allocator: std.mem.Allocator,
+ set: *PruneSet,
+};
pub const Child = struct {
node: *Node,
@@ -68,7 +74,10 @@ const Children = struct {
pub fn next(self: *Children) ?Child {
while (self.next_node) |node| {
self.next_node = node.nextSibling();
- var child = self.tree.classify(node, .{ .boxed = self.boxed }) orelse continue;
+ var child = self.tree.classify(node, .{ .boxed = self.boxed }) orelse {
+ self.tree.recordSkipped(node);
+ continue;
+ };
child.separated = self.boxed and self.yielded;
self.yielded = true;
return child;
@@ -91,12 +100,25 @@ pub const Slotted = struct {
while (self.assigned.len > 0) {
const node = self.assigned[0];
self.assigned = self.assigned[1..];
- return self.tree.classify(node, .{ .slotted = true }) orelse continue;
+ return self.tree.classify(node, .{ .slotted = true }) orelse {
+ self.tree.recordSkipped(node);
+ continue;
+ };
}
return self.fallback.next();
}
};
+fn recordSkipped(self: *const RenderTree, node: *Node) void {
+ const skipped = self.skipped orelse return;
+ const el = node.is(Element) orelse return;
+ if (el.getTag() == .head) {
+ // <head> renders nothing, but we still need its title & base
+ return;
+ }
+ skipped.set.put(skipped.allocator, node, {}) catch {};
+}
+
pub fn children(self: *const RenderTree, parent: *Node, boxed: bool) Children {
return .{ .tree = self, .next_node = parent.firstChild(), .boxed = boxed };
}
diff --git a/src/browser/clutter.zig b/src/browser/clutter.zig
index 5c64fb482..bf9db8868 100644
--- a/src/browser/clutter.zig
+++ b/src/browser/clutter.zig
@@ -79,6 +79,7 @@ pub fn select(allocator: Allocator, root: *Node, strip_: RenderTree.Strip, frame
.flags = flags,
.tree = .{ .frame = frame, .state = .{ .root = root, .strip = strip } },
};
+ pass.tree.skipped = .{ .allocator = arena, .set = &pass.pruned };
const selected = try pass.run() orelse continue;
const chars = pass.selectedText(selected);
log.debug(.browser, "strip clutter", .{ .attempt = i, .chars = chars, .total = pass.total, .root = describe(selected) });
@@ -965,7 +966,8 @@ test "clutter: the HTML dump leaves out what the markdown dump does" {
try Frame.parse.htmlAsChildren(frame, div.asNode(), "<script>window.__DATA__ = 1;</script>" ++
"<div hidden>Accept our cookies</div>" ++
"<div class=\"teasers\"><div><a href=\"/1\">Ten things to know</a></div><div><a href=\"/2\">Trending now</a></div></div>" ++
- "<div class=\"article\"><p>" ++ prose ++ "</p><p>" ++ prose ++ "</p><p>" ++ prose ++ "</p></div>");
+ "<div class=\"article\"><script>INNER_SCRIPT</script><div hidden>INNER_HIDDEN</div><span aria-hidden=\"true\">INNER_ARIA</span><template><p>INNER_TEMPLATE</p></template>" ++
+ "<p>" ++ prose ++ "</p><p>" ++ prose ++ "</p><p>" ++ prose ++ "</p></div>");
const state = try RenderTree.resolve(testing.arena_allocator, div.asNode(), .{ .clutter = true }, frame);
var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);
@@ -978,6 +980,11 @@ test "clutter: the HTML dump leaves out what the markdown dump does" {
try testing.expectEqual(null, std.mem.indexOf(u8, out, "__DATA__"));
try testing.expectEqual(null, std.mem.indexOf(u8, out, "Accept our cookies"));
try testing.expectEqual(null, std.mem.indexOf(u8, out, "Trending now"));
+ // Inside the selection too: the walk records what it skips.
+ try testing.expectEqual(null, std.mem.indexOf(u8, out, "INNER_SCRIPT"));
+ try testing.expectEqual(null, std.mem.indexOf(u8, out, "INNER_HIDDEN"));
+ try testing.expectEqual(null, std.mem.indexOf(u8, out, "INNER_ARIA"));
+ try testing.expectEqual(null, std.mem.indexOf(u8, out, "INNER_TEMPLATE"));
}
fn extract(html: []const u8) ![]const u8 {
diff --git a/src/browser/dump.zig b/src/browser/dump.zig
index 95d33a8a0..361bfa459 100644
--- a/src/browser/dump.zig
+++ b/src/browser/dump.zig
@@ -510,6 +510,9 @@ pub fn shouldStripElement(el: *Node.Element, strip: Opts.Strip, pruned: ?*const
}
if (strip.invisible) {
+ if (el.is(Template) != null) {
+ return true;
+ }
if (el.ownerFrame(frame)) |owner| {
if (owner._style_manager.hasAuthorDisplayNone(el)) {
return true;
@@ -787,6 +790,24 @@ test "dump: strip.invisible removes author display:none elements" {
);
}
+test "dump: strip.invisible removes templates and their content" {
+ const frame = try testing.createFrame();
+ defer testing.test_session.closeAllPages();
+
+ const doc = frame.window._document;
+ const div = try doc.createElement("div", null, frame);
+ try Frame.parse.htmlAsChildren(frame, div.asNode(), "<template><p>row</p></template><p>text</p>");
+
+ var aw: std.Io.Writer.Allocating = .init(testing.allocator);
+ defer aw.deinit();
+ try deep(div.asNode(), .{}, &aw.writer, frame);
+ try testing.expectString("<div><template><p>row</p></template><p>text</p></div>", aw.written());
+
+ aw.clearRetainingCapacity();
+ try deep(div.asNode(), .{ .strip = .{ .invisible = true } }, &aw.writer, frame);
+ try testing.expectString("<div><p>text</p></div>", aw.written());
+}
+
test "dump: strip.shell removes page chrome but keeps sectioned header/footer" {
try expectShellDump(
\\<header>H</header><nav>N</nav><main><header>MH</header><p>body</p><footer>MF</footer></main><article><footer>AF</footer></article><aside>A</aside><dialog>D</dialog><footer>F</footer>
diff --git a/src/help.zon b/src/help.zon
index d00ecca73..c0d7e2d74 100644
--- a/src/help.zon
+++ b/src/help.zon
@@ -134,7 +134,8 @@
\\ (which may itself fall back to a full dump).
\\ css Includes style and link[rel=stylesheet].
\\ js Script and link[as=script, rel=preload].
- \\ invisible Best-effort (e.g. display:none) hidden elements
+ \\ invisible Best-effort hidden elements (e.g. display:none,
+ \\ <template>).
\\ shell Page chrome by markup: nav, aside, dialog,
\\ page-level header and footer. Full dump when
\\ `shell` would remove most of the text.