Skip to content

Lightpanda: the headless browser designed for AI and automation

interactive: take roles and names from AXNode

Adrià Arrufat6bff402a2451
diff --git a/src/SemanticTree.zig b/src/SemanticTree.zig
index 20f6010ab..f41797619 100644
--- a/src/SemanticTree.zig
+++ b/src/SemanticTree.zig
@@ -25,6 +25,7 @@ const SelectorPath = @import("browser/SelectorPath.zig");
const CData = @import("browser/webapi/CData.zig");
const Element = @import("browser/webapi/Element.zig");
+const Label = @import("browser/webapi/element/html/Label.zig");
const Node = @import("browser/webapi/Node.zig");
const AXNode = @import("cdp/AXNode.zig");
const CDPNode = @import("cdp/Node.zig");
@@ -51,11 +52,13 @@ pub fn jsonStringify(self: @This(), jw: *std.json.Stringify) error{WriteFailed}!
};
var visibility_cache: Element.VisibilityCache = .empty;
var pointer_events_cache: Element.PointerEventsCache = .empty;
+ var label_index: Label.LabelByForIndex = .{};
var ctx: WalkContext = .{
.xpath_buffer = &xpath_buffer,
.listener_targets = listener_targets,
.visibility_cache = &visibility_cache,
.pointer_events_cache = &pointer_events_cache,
+ .label_index = &label_index,
};
self.walk(&ctx, self.dom_node, null, &visitor, 1, 0) catch |err| {
log.err(.app, "semantic tree json dump failed", .{ .err = err });
@@ -72,11 +75,13 @@ pub fn textStringify(self: @This(), writer: *std.Io.Writer) error{WriteFailed}!v
};
var visibility_cache: Element.VisibilityCache = .empty;
var pointer_events_cache: Element.PointerEventsCache = .empty;
+ var label_index: Label.LabelByForIndex = .{};
var ctx: WalkContext = .{
.xpath_buffer = &xpath_buffer,
.listener_targets = listener_targets,
.visibility_cache = &visibility_cache,
.pointer_events_cache = &pointer_events_cache,
+ .label_index = &label_index,
};
self.walk(&ctx, self.dom_node, null, &visitor, 1, 0) catch |err| {
log.err(.app, "semantic tree text dump failed", .{ .err = err });
@@ -109,6 +114,7 @@ const WalkContext = struct {
listener_targets: interactive.ListenerTargetMap,
visibility_cache: *Element.VisibilityCache,
pointer_events_cache: *Element.PointerEventsCache,
+ label_index: *Label.LabelByForIndex,
};
fn walk(
@@ -191,7 +197,7 @@ fn walk(
try appendXPathSegment(node, ctx.xpath_buffer, self.arena, index);
const xpath = ctx.xpath_buffer.items;
- var name = try axn.getName(self.frame, self.arena);
+ var name = try axn.getName(self.frame, self.arena, ctx.label_index);
const has_explicit_label = if (node.is(Element)) |el|
el.getAttributeSafe(comptime .wrap("aria-label")) != null or el.getAttributeSafe(comptime .wrap("title")) != null
@@ -655,7 +661,7 @@ pub fn getNodeDetails(
const cdp_node = try registry.register(node);
const axn = AXNode.fromNode(node);
const role = try axn.getRole();
- const name = try axn.getName(frame, arena);
+ const name = try axn.getName(frame, arena, null);
var is_interactive = false;
var is_disabled = false;
diff --git a/src/browser/interactive.zig b/src/browser/interactive.zig
index 9b008cb88..972337996 100644
--- a/src/browser/interactive.zig
+++ b/src/browser/interactive.zig
@@ -21,6 +21,8 @@ const std = @import("std");
const Frame = @import("Frame.zig");
const URL = @import("URL.zig");
const TreeWalker = @import("webapi/TreeWalker.zig");
+const Label = @import("webapi/element/html/Label.zig");
+const AXNode = @import("../cdp/AXNode.zig");
const Element = @import("webapi/Element.zig");
const Node = @import("webapi/Node.zig");
const EventTarget = @import("webapi/EventTarget.zig");
@@ -179,6 +181,7 @@ fn walkInteractive(
const listener_targets = try buildListenerTargetMap(frame, arena);
var css_cache: Element.PointerEventsCache = .empty;
+ var label_index: Label.LabelByForIndex = .{};
var results: std.ArrayList(InteractiveElement) = .empty;
@@ -212,15 +215,16 @@ fn walkInteractive(
const itype = classifyInteractivity(frame, el, html_el, listener_targets, &css_cache) orelse continue;
- const role = getRole(el);
+ const axn = AXNode.fromNode(node);
+ const role = axRole(axn);
if (filter.role) |rf| {
const r = role orelse continue;
if (!std.ascii.eqlIgnoreCase(r, rf)) continue;
}
- // Resolve accessible name only after the cheap role filter passes,
- // since getAccessibleName walks the element's text subtree.
- const name = try getAccessibleName(el, arena);
+ // Resolve the name only after the cheap role filter passes: it walks
+ // labels and the element's text subtree.
+ const name = (try axn.getName(frame, arena, &label_index)) orelse try getTextContent(node, arena);
if (filter.name) |nf| {
const n = name orelse continue;
if (std.ascii.indexOfIgnoreCase(n, nf) == null) continue;
@@ -391,65 +395,11 @@ pub fn explicitRole(el: *Element) ?[]const u8 {
return it.next();
}
-fn getRole(el: *Element) ?[]const u8 {
- if (explicitRole(el)) |role| return role;
-
- // Implicit role from tag
- return switch (el.getTag()) {
- .button, .summary => "button",
- .anchor, .area => if (el.getAttributeSafe(comptime .wrap("href")) != null) "link" else null,
- .input => blk: {
- if (el.is(Element.Html.Input)) |input| {
- break :blk switch (input._input_type) {
- .text, .tel, .url, .email => "textbox",
- .checkbox => "checkbox",
- .radio => "radio",
- .button, .submit, .reset, .image => "button",
- .range => "slider",
- .number => "spinbutton",
- .search => "searchbox",
- else => null,
- };
- }
- break :blk null;
- },
- .select => "combobox",
- .textarea => "textbox",
- .details => "group",
- else => null,
- };
-}
-
-fn getAccessibleName(el: *Element, arena: Allocator) !?[]const u8 {
- // aria-label
- if (el.getAttributeSafe(comptime .wrap("aria-label"))) |v| {
- if (v.len > 0) return v;
- }
-
- // alt (for img, input[type=image])
- if (el.getAttributeSafe(comptime .wrap("alt"))) |v| {
- if (v.len > 0) return v;
- }
-
- // title
- if (el.getAttributeSafe(comptime .wrap("title"))) |v| {
- if (v.len > 0) return v;
- }
-
- // placeholder
- if (el.getAttributeSafe(comptime .wrap("placeholder"))) |v| {
- if (v.len > 0) return v;
- }
-
- // value (for buttons)
- if (el.getTag() == .input) {
- if (el.getAttributeSafe(comptime .wrap("value"))) |v| {
- if (v.len > 0) return v;
- }
- }
-
- // Text content (first non-empty text node, trimmed)
- return try getTextContent(el.asNode(), arena);
+/// The accessibility role, or null where there is none: a plain element
+/// that is interactive only by its listener or tabindex.
+fn axRole(axn: AXNode) ?[]const u8 {
+ const role = axn.getRole() catch return null;
+ return if (std.mem.eql(u8, role, "none")) null else role;
}
pub fn getTextContent(node: *Node, arena: Allocator) !?[]const u8 {
@@ -529,6 +479,45 @@ fn testInteractive(html: []const u8) ![]InteractiveElement {
return collectInteractiveElements(div.asNode(), frame.call_arena, frame);
}
+/// Like `testInteractive`, but attached to the document so `<label for>` resolves.
+fn testInteractiveInBody(html: []const u8) ![]InteractiveElement {
+ const frame = try testing.createFrame();
+ errdefer testing.test_session.closeAllPages();
+
+ const doc = frame.window._document;
+ const div = try doc.createElement("div", null, frame);
+ _ = try doc.asNode().appendChild(div.asNode(), frame);
+ try Frame.parse.htmlAsChildren(frame, div.asNode(), html);
+
+ return collectInteractiveElements(div.asNode(), frame.call_arena, frame);
+}
+
+test "browser.interactive: names come from labels, like the tree" {
+ const elements = try testInteractiveInBody(
+ \\<label for="email">Email address</label><input id="email" type="text">
+ \\<label>Password <input type="password"></label>
+ \\<span id="q-label">Search</span><input type="search" aria-labelledby="q-label">
+ \\<input type="text" placeholder="City">
+ );
+ defer testing.test_session.closeAllPages();
+ try testing.expectEqual(4, elements.len);
+ try testing.expectEqual("textbox", elements[0].role.?);
+ try testing.expectEqual("Email address", elements[0].name.?);
+ try testing.expectEqual("textbox", elements[1].role.?);
+ try testing.expectEqual("Password", elements[1].name.?);
+ try testing.expectEqual("searchbox", elements[2].role.?);
+ try testing.expectEqual("Search", elements[2].name.?);
+ try testing.expectEqual("City", elements[3].name.?);
+}
+
+test "browser.interactive: an element with no role keeps its text as name" {
+ const elements = try testInteractive("<div tabindex=\"0\">Open menu</div>");
+ defer testing.test_session.closeAllPages();
+ try testing.expectEqual(1, elements.len);
+ try testing.expectEqual(null, elements[0].role);
+ try testing.expectEqual("Open menu", elements[0].name.?);
+}
+
test "browser.interactive: button" {
const elements = try testInteractive("<button>Click me</button>");
defer testing.test_session.closeAllPages();
diff --git a/src/browser/links.zig b/src/browser/links.zig
index 3769a0af3..41838b745 100644
--- a/src/browser/links.zig
+++ b/src/browser/links.zig
@@ -80,12 +80,12 @@ pub fn collectLinks(arena: Allocator, root: *Node, frame: *Frame) ![]Link {
const gop = try links.getOrPut(arena, href);
if (gop.found_existing) {
- if (gop.value_ptr.text == null) gop.value_ptr.text = try AXNode.fromNode(node).getName(frame, arena);
+ if (gop.value_ptr.text == null) gop.value_ptr.text = try AXNode.fromNode(node).getName(frame, arena, null);
continue;
}
gop.value_ptr.* = .{
.node = node,
- .text = try AXNode.fromNode(node).getName(frame, arena),
+ .text = try AXNode.fromNode(node).getName(frame, arena, null),
.href = href,
};
}
diff --git a/src/browser/webapi/WebDriver.zig b/src/browser/webapi/WebDriver.zig
index d50abd255..750123423 100644
--- a/src/browser/webapi/WebDriver.zig
+++ b/src/browser/webapi/WebDriver.zig
@@ -50,7 +50,7 @@ pub fn deleteAllCookies(_: *const WebDriver, page: *Page) void {
pub fn getComputedLabel(_: *const WebDriver, element: *Element, frame: *Frame) ![]const u8 {
const AXNode = @import("../../cdp/AXNode.zig");
const axnode = AXNode.fromNode(element.asNode());
- return (try axnode.getName(frame, frame.call_arena)) orelse "";
+ return (try axnode.getName(frame, frame.call_arena, null)) orelse "";
}
// Implements testdriver's `click`: a full trusted primary-button click
diff --git a/src/cdp/AXNode.zig b/src/cdp/AXNode.zig
index 60a9c6d54..704a1f08e 100644
--- a/src/cdp/AXNode.zig
+++ b/src/cdp/AXNode.zig
@@ -707,7 +707,7 @@ pub const Writer = struct {
if (!std.mem.eql(u8, needle, resolved.role)) return;
}
- const name = (try axn.getName(self.frame, self.temp_arena.allocator())) orelse "";
+ const name = (try axn.getName(self.frame, self.temp_arena.allocator(), self.label_index)) orelse "";
if (filter.accessible_name) |needle| {
if (!std.mem.eql(u8, needle, name)) return;
}
@@ -932,7 +932,9 @@ const AXSource = enum(u8) {
value, // input value
};
-pub fn getName(self: AXNode, frame: *Frame, allocator: std.mem.Allocator) !?[]const u8 {
+/// `label_index` makes repeated `<label for>` lookups one document scan
+/// instead of one per element.
+pub fn getName(self: AXNode, frame: *Frame, allocator: std.mem.Allocator, label_index: ?*Label.LabelByForIndex) !?[]const u8 {
var aw: std.Io.Writer.Allocating = .init(allocator);
defer aw.deinit();
@@ -959,7 +961,7 @@ pub fn getName(self: AXNode, frame: *Frame, allocator: std.mem.Allocator) !?[]co
const w: TextCaptureWriter = .{ .aw = &aw, .writer = &aw.writer };
- const source = try self.writeName(null, w, frame, null);
+ const source = try self.writeName(null, w, frame, label_index);
if (source != null) {
// Remove literal quotes inserted by writeString.
var raw_text = std.mem.trim(u8, aw.written(), "\"");
@@ -2036,7 +2038,7 @@ test "AXNode: getName name-from-content honors explicit role" {
for (cases) |c| {
const el = (try doc.querySelector(.wrap(c.selector), frame)).?;
const axn = AXNode.fromNode(el.asNode());
- const name = try axn.getName(frame, testing.allocator);
+ const name = try axn.getName(frame, testing.allocator, null);
defer if (name) |n| testing.allocator.free(n);
if (c.expected) |exp| {