Lightpanda: the headless browser designed for AI and automation
tree: rank headings first and site navigation last
Adrià Arrufatbe73338fd681
diff --git a/src/SemanticTree.zig b/src/SemanticTree.zigindex b7db9bbcd..e481d9d9d 100644--- a/src/SemanticTree.zig+++ b/src/SemanticTree.zig@@ -133,7 +133,7 @@ const TextVisitor = struct {}};-const Priority = enum { critical, named, text };+const Priority = enum(u3) { critical, heading, named, text, chrome };const Line = struct {id: NodeRegistry.Id,@@ -150,6 +150,8 @@ const Line = struct {hidden: bool,// Inside an alert or status message, all of which is critical.critical_subtree: bool,+ // Inside site navigation, all of which ranks last unless critical.+ chrome_subtree: bool,subtree_len: u32 = 1,// Printed lines in the subtree, this one included.printed_len: u32,@@ -205,16 +207,18 @@ const LineCollector = struct {const inherited = if (self.current) |p| self.lines.items[p].critical_subtree else false;const role_scope = critical_roles.get(data.role);const critical = inherited or role_scope != null;+ const chrome = (if (self.current) |p| self.lines.items[p].chrome_subtree else false) or chrome_roles.has(data.role);try self.lines.append(self.arena, .{.id = data.id,.start = @intCast(start),.end = @intCast(self.text.written().len),.depth = self.depth(),.parent = self.current,- .priority = if (critical) .critical else own.priority,+ .priority = if (critical) .critical else if (chrome) .chrome else own.priority,.contentful = own.contentful,.hidden = own.hidden,.critical_subtree = inherited or role_scope == .subtree,+ .chrome_subtree = chrome,.printed_len = @intFromBool(!own.hidden),});self.current = @intCast(self.lines.items.len - 1);@@ -237,14 +241,24 @@ const critical_roles = std.StaticStringMap(enum { self, subtree }).initComptime(.{ "alertdialog", .self },});+// Menus and footers often come before the content, with more links than it.+// A banner is left out: it holds the search box and the cart.+const chrome_roles = std.StaticStringMap(void).initComptime(.{+ .{ "navigation", {} },+ .{ "complementary", {} },+ .{ "contentinfo", {} },+});+fn linePriority(node: *Node, data: *const NodeData, focused: ?*Node) Priority {if (focused == node) return .critical;if (node.is(Element)) |el| {if (std.mem.eql(u8, el.getAttributeSafe(comptime .wrap("aria-invalid")) orelse "", "true")) return .critical;}if (data.value != null or data.options != null or data.checked != null) return .named;- // Controls and headings share a bucket: a record's title is often a heading.- if (!std.mem.eql(u8, data.role, "StaticText") and data.name != null and data.name.?.len > 0) return .named;+ const named = data.name != null and data.name.?.len > 0;+ // Headings outline the page, and a record's title is often one.+ if (named and std.mem.eql(u8, data.role, "heading")) return .heading;+ if (named and !std.mem.eql(u8, data.role, "StaticText")) return .named;return .text;}@@ -354,8 +368,8 @@ const Budget = struct {rank[i] = @max(counter.*, rank[parent]);counter.* += 1;if (!line.contentful and line.priority != .critical) continue;- const r: u64 = @min(rank[i], std.math.maxInt(u30));- try keys.append(arena, @as(u64, @backingInt(line.priority)) << 62 | r << 32 | i);+ const r: u64 = @min(rank[i], std.math.maxInt(u29));+ try keys.append(arena, @as(u64, @backingInt(line.priority)) << 61 | r << 32 | i);}std.mem.sort(u64, keys.items, {}, std.sort.asc(u64));@@ -1184,3 +1198,28 @@ test "SemanticTree: maxBytes keeps a dialog, not its text" {try testing.expect(std.mem.find(u8, out, "dialog 'Your privacy'") != null);try testing.expect(std.mem.find(u8, out, "1,042 partners") == null);}++test "SemanticTree: maxBytes ranks headings first and navigation last" {+ var registry: NodeRegistry = .init(testing.allocator);+ defer registry.deinit();++ var page = try testing.pageTest("cdp/tree_budget_nav.html", .{});+ defer page.close();+ const frame = page.frame().?;++ const max_bytes = 300;+ const st: Self = try .init(testing.arena_allocator, frame.window._document.asNode(), ®istry, frame, .{ .max_bytes = max_bytes });++ var aw: std.Io.Writer.Allocating = .init(testing.allocator);+ defer aw.deinit();+ try st.textStringify(&aw.writer);+ const out = aw.written();++ try testing.expect(out.len <= max_bytes);+ try testing.expect(std.mem.find(u8, out, "heading 'Mini PC 128GB'") != null);+ try testing.expect(std.mem.find(u8, out, "heading 'Warranty'") != null);+ try testing.expect(std.mem.find(u8, out, "button 'Add to cart'") != null);+ try testing.expect(std.mem.find(u8, out, "'$3,879.00'") != null);+ try testing.expect(std.mem.find(u8, out, "Category 1'") == null);+ try testing.expect(std.mem.find(u8, out, "Footer link 1'") == null);+}diff --git a/src/browser/tests/cdp/tree_budget_nav.html b/src/browser/tests/cdp/tree_budget_nav.htmlnew file mode 100644index 000000000..8ee8b5518--- /dev/null+++ b/src/browser/tests/cdp/tree_budget_nav.html@@ -0,0 +1,58 @@+<!DOCTYPE html>+<html>+<body>+<nav aria-label="Menu">+ <ul>+ <li><a href="#c1">Category 1</a></li>+ <li><a href="#c2">Category 2</a></li>+ <li><a href="#c3">Category 3</a></li>+ <li><a href="#c4">Category 4</a></li>+ <li><a href="#c5">Category 5</a></li>+ <li><a href="#c6">Category 6</a></li>+ <li><a href="#c7">Category 7</a></li>+ <li><a href="#c8">Category 8</a></li>+ <li><a href="#c9">Category 9</a></li>+ <li><a href="#c10">Category 10</a></li>+ <li><a href="#c11">Category 11</a></li>+ <li><a href="#c12">Category 12</a></li>+ <li><a href="#c13">Category 13</a></li>+ <li><a href="#c14">Category 14</a></li>+ <li><a href="#c15">Category 15</a></li>+ <li><a href="#c16">Category 16</a></li>+ <li><a href="#c17">Category 17</a></li>+ <li><a href="#c18">Category 18</a></li>+ <li><a href="#c19">Category 19</a></li>+ <li><a href="#c20">Category 20</a></li>+ <li><a href="#c21">Category 21</a></li>+ <li><a href="#c22">Category 22</a></li>+ <li><a href="#c23">Category 23</a></li>+ <li><a href="#c24">Category 24</a></li>+ <li><a href="#c25">Category 25</a></li>+ <li><a href="#c26">Category 26</a></li>+ <li><a href="#c27">Category 27</a></li>+ <li><a href="#c28">Category 28</a></li>+ <li><a href="#c29">Category 29</a></li>+ <li><a href="#c30">Category 30</a></li>+ </ul>+</nav>+<div>+ <h1>Mini PC 128GB</h1>+ <span>$3,879.00</span>+ <button>Add to cart</button>+ <h2>Specifications</h2>+ <h2>Warranty</h2>+</div>+<footer>+ <a href="#f1">Footer link 1</a>+ <a href="#f2">Footer link 2</a>+ <a href="#f3">Footer link 3</a>+ <a href="#f4">Footer link 4</a>+ <a href="#f5">Footer link 5</a>+ <a href="#f6">Footer link 6</a>+ <a href="#f7">Footer link 7</a>+ <a href="#f8">Footer link 8</a>+ <a href="#f9">Footer link 9</a>+ <a href="#f10">Footer link 10</a>+</footer>+</body>+</html>