Skip to content

Lightpanda: the headless browser designed for AI and automation

tree: rank headings first and site navigation last

Adrià Arrufatbe73338fd681
diff --git a/src/SemanticTree.zig b/src/SemanticTree.zig
index b7db9bbcd..e481d9d9d 100644
--- a/src/SemanticTree.zig
+++ b/src/SemanticTree.zig
@@ -133,7 +133,7 @@ const TextVisitor = struct {
}
};
-const Priority = enum { critical, named, text };
+const Priority = enum(u3) { critical, heading, named, text, chrome };
const Line = struct {
id: NodeRegistry.Id,
@@ -150,6 +150,8 @@ const Line = struct {
hidden: bool,
// Inside an alert or status message, all of which is critical.
critical_subtree: bool,
+ // Inside site navigation, all of which ranks last unless critical.
+ chrome_subtree: bool,
subtree_len: u32 = 1,
// Printed lines in the subtree, this one included.
printed_len: u32,
@@ -205,16 +207,18 @@ const LineCollector = struct {
const inherited = if (self.current) |p| self.lines.items[p].critical_subtree else false;
const role_scope = critical_roles.get(data.role);
const critical = inherited or role_scope != null;
+ const chrome = (if (self.current) |p| self.lines.items[p].chrome_subtree else false) or chrome_roles.has(data.role);
try self.lines.append(self.arena, .{
.id = data.id,
.start = @intCast(start),
.end = @intCast(self.text.written().len),
.depth = self.depth(),
.parent = self.current,
- .priority = if (critical) .critical else own.priority,
+ .priority = if (critical) .critical else if (chrome) .chrome else own.priority,
.contentful = own.contentful,
.hidden = own.hidden,
.critical_subtree = inherited or role_scope == .subtree,
+ .chrome_subtree = chrome,
.printed_len = @intFromBool(!own.hidden),
});
self.current = @intCast(self.lines.items.len - 1);
@@ -237,14 +241,24 @@ const critical_roles = std.StaticStringMap(enum { self, subtree }).initComptime(
.{ "alertdialog", .self },
});
+// Menus and footers often come before the content, with more links than it.
+// A banner is left out: it holds the search box and the cart.
+const chrome_roles = std.StaticStringMap(void).initComptime(.{
+ .{ "navigation", {} },
+ .{ "complementary", {} },
+ .{ "contentinfo", {} },
+});
+
fn linePriority(node: *Node, data: *const NodeData, focused: ?*Node) Priority {
if (focused == node) return .critical;
if (node.is(Element)) |el| {
if (std.mem.eql(u8, el.getAttributeSafe(comptime .wrap("aria-invalid")) orelse "", "true")) return .critical;
}
if (data.value != null or data.options != null or data.checked != null) return .named;
- // Controls and headings share a bucket: a record's title is often a heading.
- if (!std.mem.eql(u8, data.role, "StaticText") and data.name != null and data.name.?.len > 0) return .named;
+ const named = data.name != null and data.name.?.len > 0;
+ // Headings outline the page, and a record's title is often one.
+ if (named and std.mem.eql(u8, data.role, "heading")) return .heading;
+ if (named and !std.mem.eql(u8, data.role, "StaticText")) return .named;
return .text;
}
@@ -354,8 +368,8 @@ const Budget = struct {
rank[i] = @max(counter.*, rank[parent]);
counter.* += 1;
if (!line.contentful and line.priority != .critical) continue;
- const r: u64 = @min(rank[i], std.math.maxInt(u30));
- try keys.append(arena, @as(u64, @backingInt(line.priority)) << 62 | r << 32 | i);
+ const r: u64 = @min(rank[i], std.math.maxInt(u29));
+ try keys.append(arena, @as(u64, @backingInt(line.priority)) << 61 | r << 32 | i);
}
std.mem.sort(u64, keys.items, {}, std.sort.asc(u64));
@@ -1184,3 +1198,28 @@ test "SemanticTree: maxBytes keeps a dialog, not its text" {
try testing.expect(std.mem.find(u8, out, "dialog 'Your privacy'") != null);
try testing.expect(std.mem.find(u8, out, "1,042 partners") == null);
}
+
+test "SemanticTree: maxBytes ranks headings first and navigation last" {
+ var registry: NodeRegistry = .init(testing.allocator);
+ defer registry.deinit();
+
+ var page = try testing.pageTest("cdp/tree_budget_nav.html", .{});
+ defer page.close();
+ const frame = page.frame().?;
+
+ const max_bytes = 300;
+ const st: Self = try .init(testing.arena_allocator, frame.window._document.asNode(), &registry, frame, .{ .max_bytes = max_bytes });
+
+ var aw: std.Io.Writer.Allocating = .init(testing.allocator);
+ defer aw.deinit();
+ try st.textStringify(&aw.writer);
+ const out = aw.written();
+
+ try testing.expect(out.len <= max_bytes);
+ try testing.expect(std.mem.find(u8, out, "heading 'Mini PC 128GB'") != null);
+ try testing.expect(std.mem.find(u8, out, "heading 'Warranty'") != null);
+ try testing.expect(std.mem.find(u8, out, "button 'Add to cart'") != null);
+ try testing.expect(std.mem.find(u8, out, "'$3,879.00'") != null);
+ try testing.expect(std.mem.find(u8, out, "Category 1'") == null);
+ try testing.expect(std.mem.find(u8, out, "Footer link 1'") == null);
+}
diff --git a/src/browser/tests/cdp/tree_budget_nav.html b/src/browser/tests/cdp/tree_budget_nav.html
new file mode 100644
index 000000000..8ee8b5518
--- /dev/null
+++ b/src/browser/tests/cdp/tree_budget_nav.html
@@ -0,0 +1,58 @@
+<!DOCTYPE html>
+<html>
+<body>
+<nav aria-label="Menu">
+ <ul>
+ <li><a href="#c1">Category 1</a></li>
+ <li><a href="#c2">Category 2</a></li>
+ <li><a href="#c3">Category 3</a></li>
+ <li><a href="#c4">Category 4</a></li>
+ <li><a href="#c5">Category 5</a></li>
+ <li><a href="#c6">Category 6</a></li>
+ <li><a href="#c7">Category 7</a></li>
+ <li><a href="#c8">Category 8</a></li>
+ <li><a href="#c9">Category 9</a></li>
+ <li><a href="#c10">Category 10</a></li>
+ <li><a href="#c11">Category 11</a></li>
+ <li><a href="#c12">Category 12</a></li>
+ <li><a href="#c13">Category 13</a></li>
+ <li><a href="#c14">Category 14</a></li>
+ <li><a href="#c15">Category 15</a></li>
+ <li><a href="#c16">Category 16</a></li>
+ <li><a href="#c17">Category 17</a></li>
+ <li><a href="#c18">Category 18</a></li>
+ <li><a href="#c19">Category 19</a></li>
+ <li><a href="#c20">Category 20</a></li>
+ <li><a href="#c21">Category 21</a></li>
+ <li><a href="#c22">Category 22</a></li>
+ <li><a href="#c23">Category 23</a></li>
+ <li><a href="#c24">Category 24</a></li>
+ <li><a href="#c25">Category 25</a></li>
+ <li><a href="#c26">Category 26</a></li>
+ <li><a href="#c27">Category 27</a></li>
+ <li><a href="#c28">Category 28</a></li>
+ <li><a href="#c29">Category 29</a></li>
+ <li><a href="#c30">Category 30</a></li>
+ </ul>
+</nav>
+<div>
+ <h1>Mini PC 128GB</h1>
+ <span>$3,879.00</span>
+ <button>Add to cart</button>
+ <h2>Specifications</h2>
+ <h2>Warranty</h2>
+</div>
+<footer>
+ <a href="#f1">Footer link 1</a>
+ <a href="#f2">Footer link 2</a>
+ <a href="#f3">Footer link 3</a>
+ <a href="#f4">Footer link 4</a>
+ <a href="#f5">Footer link 5</a>
+ <a href="#f6">Footer link 6</a>
+ <a href="#f7">Footer link 7</a>
+ <a href="#f8">Footer link 8</a>
+ <a href="#f9">Footer link 9</a>
+ <a href="#f10">Footer link 10</a>
+</footer>
+</body>
+</html>