Lightpanda: the headless browser designed for AI and automation
markdown: skip hidden elements
Adrià Arrufate76cc816c061
diff --git a/src/browser/StyleManager.zig b/src/browser/StyleManager.zigindex c7dd36601..b633d2204 100644--- a/src/browser/StyleManager.zig+++ b/src/browser/StyleManager.zig@@ -644,6 +644,9 @@ fn matchesUaDisplayNoneRule(el: *Element) bool {}}+ // dialog:not([open]) { display: none }+ if (tag == .dialog and !el.hasAttributeSafe(comptime .wrap("open"))) return true;+// details:not([open]) > *:not(summary) { display: none }if (tag != .summary) {if (el.parentElement()) |parent| {diff --git a/src/browser/markdown.zig b/src/browser/markdown.zigindex f8934cd58..a4632311f 100644--- a/src/browser/markdown.zig+++ b/src/browser/markdown.zig@@ -114,7 +114,7 @@ fn isLayoutBlock(tag: Element.Tag) bool {};}-pub fn isStandaloneAnchor(el: *Element) bool {+pub fn isStandaloneAnchor(el: *Element, frame: *Frame) bool {const node = el.asNode();const parent = node.parentNode() orelse return false;const parent_el = parent.is(Element) orelse return false;@@ -125,7 +125,7 @@ pub fn isStandaloneAnchor(el: *Element) bool {while (prev) |p| : (prev = p.previousSibling()) {if (isSignificantText(p)) return false;if (p.is(Element)) |pe| {- if (isVisibleElement(pe)) break;+ if (isVisibleElement(pe, frame)) break;}}@@ -133,7 +133,7 @@ pub fn isStandaloneAnchor(el: *Element) bool {while (next) |n| : (next = n.nextSibling()) {if (isSignificantText(n)) return false;if (n.is(Element)) |ne| {- if (isVisibleElement(ne)) break;+ if (isVisibleElement(ne, frame)) break;}}@@ -145,9 +145,16 @@ fn isSignificantText(node: *Node) bool {return !isAllWhitespace(text.ownData());}-fn isVisibleElement(el: *Element) bool {+// Own state only; the dump root is exempt so a scoped dump of a hidden+// subtree still renders it.+fn isVisibleElement(el: *Element, frame: *Frame) bool {const tag = el.getTag();- return !tag.isMetadata() and tag != .svg;+ if (tag.isMetadata() or tag == .svg) return false;+ if (frame._style_manager.hasDisplayNone(el)) return false;+ if (el.getAttributeSafe(comptime .wrap("aria-hidden"))) |v| {+ if (std.ascii.eqlIgnoreCase(v, "true")) return false;+ }+ return true;}fn getAnchorLabel(el: *Element) ?[]const u8 {@@ -159,7 +166,7 @@ pub const ContentInfo = struct {has_block: bool,};-pub fn analyzeContent(root: *Node) ContentInfo {+pub fn analyzeContent(root: *Node, frame: *Frame) ContentInfo {var result: ContentInfo = .{ .has_visible = false, .has_block = false };var tw = TreeWalker.FullExcludeSelf.init(root, .{});while (tw.next()) |node| {@@ -167,7 +174,7 @@ pub fn analyzeContent(root: *Node) ContentInfo {result.has_visible = true;if (result.has_block) return result;} else if (node.is(Element)) |el| {- if (!isVisibleElement(el)) {+ if (!isVisibleElement(el, frame)) {tw.skipChildren();} else {const tag = el.getTag();@@ -189,6 +196,7 @@ const Context = struct {state: State,writer: *std.Io.Writer,frame: *Frame,+ root: *Node,// When there's a slot-attribute, we skip rendering, unless this flag has// bet set to true.@@ -252,7 +260,7 @@ const Context = struct {const tag = el.getTag();- if (!isVisibleElement(el)) return;+ if (el.asNode() != self.root and !isVisibleElement(el, self.frame)) return;if (!force_slot) {if (el.getAttributeSafe(comptime .wrap("slot")) != null) {@@ -375,7 +383,7 @@ const Context = struct {},.anchor => {const frame = self.frame;- const info = analyzeContent(el.asNode());+ const info = analyzeContent(el.asNode(), frame);const label = getAnchorLabel(el);const href_raw = el.getAttributeSafe(comptime .wrap("href"));@@ -397,7 +405,7 @@ const Context = struct {return;}- const standalone = isStandaloneAnchor(el);+ const standalone = isStandaloneAnchor(el, frame);if (standalone) {if (!self.state.last_char_was_newline) try self.writer.writeByte('\n');}@@ -564,6 +572,7 @@ pub fn dump(node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !voi.state = .{},.writer = &lw.writer,.frame = frame,+ .root = node,};ctx.render(node) catch |err| switch (err) {error.WriteFailed => {@@ -582,6 +591,7 @@ pub fn dump(node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !voi.state = .{},.writer = writer,.frame = frame,+ .root = node,};try ctx.render(node);if (!ctx.state.last_char_was_newline) {@@ -832,6 +842,70 @@ test "browser.markdown: anchor fallback label" {, "[](http://localhost/no-label)\n");}+test "browser.markdown: hidden elements are skipped" {+ try testMarkdownHTML(+ \\<p>before</p>+ \\<p style="display:none">inline</p>+ \\<div hidden><p>attribute</p></div>+ \\<p aria-hidden="true">aria</p>+ \\<span aria-hidden="TRUE">aria caps</span>+ \\<p aria-hidden="false">aria false</p>+ \\<details><summary>Summary</summary><p>collapsed</p></details><dialog><p>closed dialog</p></dialog><p>after</p>+ ,+ \\+ \\before+ \\+ \\aria false+ \\Summary+ \\+ \\after+ \\+ );+}++test "browser.markdown: stylesheet display:none is skipped" {+ var page = try testing.pageTest("dump.html", .{});+ defer page.close();+ const frame = page.frame().?;++ var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);+ try dump(frame.window._document.asNode(), .{}, &aw.writer, frame);++ try testing.expectString(+ \\+ \\# Title+ \\![]()+ \\+ \\visible & well+ \\+ , aw.written());+}++test "browser.markdown: anchor with only hidden content falls back to label" {+ try testMarkdownHTML(+ \\<a href="/x" aria-label="Label"><span hidden>secret</span></a>+ , "[Label](http://localhost/x)\n");+}++test "browser.markdown: scoped dump of a hidden subtree still renders it" {+ const frame = try testing.createFrame();+ defer testing.test_session.closeAllPages();+ frame.url = "http://localhost/";++ const doc = frame.window._document;+ const div = try doc.createElement("div", null, frame);+ try Frame.parse.htmlAsChildren(frame, div.asNode(),+ \\<div id="modal" style="display:none"><p>dialog text</p><p hidden>nested hidden</p></div>+ );+ const modal = div.asNode().firstChild().?;++ var aw: std.Io.Writer.Allocating = .init(testing.allocator);+ defer aw.deinit();+ try dump(modal, .{}, &aw.writer, frame);++ try testing.expectString("\ndialog text\n", aw.written());+}+test "browser.markdown: max_bytes leaves output untouched when under cap" {const frame = try testing.createFrame();defer testing.test_session.closeAllPages();diff --git a/src/browser/screenshot.zig b/src/browser/screenshot.zigindex 25aea5ccb..5beb3f5dc 100644--- a/src/browser/screenshot.zig+++ b/src/browser/screenshot.zig@@ -556,13 +556,13 @@ const Builder = struct {.anchor => {const href = el.getAttributeSafe(comptime .wrap("href"));const label = el.getAttributeSafe(comptime .wrap("aria-label")) orelse el.getAttributeSafe(comptime .wrap("title"));- const info = markdown.analyzeContent(el.asNode());+ const info = markdown.analyzeContent(el.asNode(), self.frame);if (!info.has_visible and label == null) return;// Same split as markdown: an anchor wrapping blocks, or one// sitting among element-only siblings (nav bars, post lists),// gets its own tight block instead of flowing inline.- const standalone = info.has_block or markdown.isStandaloneAnchor(el);+ const standalone = info.has_block or markdown.isStandaloneAnchor(el, self.frame);if (standalone) {try self.closeBlock();self.tight += 1;diff --git a/src/browser/tests/element/check_visibility.html b/src/browser/tests/element/check_visibility.htmlindex e4a421b69..9b5ad29c4 100644--- a/src/browser/tests/element/check_visibility.html+++ b/src/browser/tests/element/check_visibility.html@@ -271,6 +271,29 @@}</script>+<script id="ua_closed_dialog_is_hidden">+ {+ // dialog:not([open]) { display: none }+ const dialog = document.createElement('dialog');+ const child = document.createElement('p');+ dialog.appendChild(child);+ document.body.appendChild(dialog);++ testing.expectEqual(false, dialog.checkVisibility());+ testing.expectEqual(false, child.checkVisibility());+ testing.expectEqual('none', window.getComputedStyle(dialog).display);++ dialog.show();+ testing.expectEqual(true, dialog.checkVisibility());+ testing.expectEqual(true, child.checkVisibility());++ dialog.close();+ testing.expectEqual(false, dialog.checkVisibility());++ dialog.remove();+ }+</script>+<script id="hidden_attribute_propagates_through_check_visibility">{// [hidden] { display: none } applies to the element itself...