Lightpanda: the headless browser designed for AI and automation
interactive: take roles and names from AXNode
Adrià Arrufat6bff402a2451
diff --git a/src/SemanticTree.zig b/src/SemanticTree.zigindex 20f6010ab..f41797619 100644--- a/src/SemanticTree.zig+++ b/src/SemanticTree.zig@@ -25,6 +25,7 @@ const SelectorPath = @import("browser/SelectorPath.zig");const CData = @import("browser/webapi/CData.zig");const Element = @import("browser/webapi/Element.zig");+const Label = @import("browser/webapi/element/html/Label.zig");const Node = @import("browser/webapi/Node.zig");const AXNode = @import("cdp/AXNode.zig");const CDPNode = @import("cdp/Node.zig");@@ -51,11 +52,13 @@ pub fn jsonStringify(self: @This(), jw: *std.json.Stringify) error{WriteFailed}!};var visibility_cache: Element.VisibilityCache = .empty;var pointer_events_cache: Element.PointerEventsCache = .empty;+ var label_index: Label.LabelByForIndex = .{};var ctx: WalkContext = .{.xpath_buffer = &xpath_buffer,.listener_targets = listener_targets,.visibility_cache = &visibility_cache,.pointer_events_cache = &pointer_events_cache,+ .label_index = &label_index,};self.walk(&ctx, self.dom_node, null, &visitor, 1, 0) catch |err| {log.err(.app, "semantic tree json dump failed", .{ .err = err });@@ -72,11 +75,13 @@ pub fn textStringify(self: @This(), writer: *std.Io.Writer) error{WriteFailed}!v};var visibility_cache: Element.VisibilityCache = .empty;var pointer_events_cache: Element.PointerEventsCache = .empty;+ var label_index: Label.LabelByForIndex = .{};var ctx: WalkContext = .{.xpath_buffer = &xpath_buffer,.listener_targets = listener_targets,.visibility_cache = &visibility_cache,.pointer_events_cache = &pointer_events_cache,+ .label_index = &label_index,};self.walk(&ctx, self.dom_node, null, &visitor, 1, 0) catch |err| {log.err(.app, "semantic tree text dump failed", .{ .err = err });@@ -109,6 +114,7 @@ const WalkContext = struct {listener_targets: interactive.ListenerTargetMap,visibility_cache: *Element.VisibilityCache,pointer_events_cache: *Element.PointerEventsCache,+ label_index: *Label.LabelByForIndex,};fn walk(@@ -191,7 +197,7 @@ fn walk(try appendXPathSegment(node, ctx.xpath_buffer, self.arena, index);const xpath = ctx.xpath_buffer.items;- var name = try axn.getName(self.frame, self.arena);+ var name = try axn.getName(self.frame, self.arena, ctx.label_index);const has_explicit_label = if (node.is(Element)) |el|el.getAttributeSafe(comptime .wrap("aria-label")) != null or el.getAttributeSafe(comptime .wrap("title")) != null@@ -655,7 +661,7 @@ pub fn getNodeDetails(const cdp_node = try registry.register(node);const axn = AXNode.fromNode(node);const role = try axn.getRole();- const name = try axn.getName(frame, arena);+ const name = try axn.getName(frame, arena, null);var is_interactive = false;var is_disabled = false;diff --git a/src/browser/interactive.zig b/src/browser/interactive.zigindex 9b008cb88..972337996 100644--- a/src/browser/interactive.zig+++ b/src/browser/interactive.zig@@ -21,6 +21,8 @@ const std = @import("std");const Frame = @import("Frame.zig");const URL = @import("URL.zig");const TreeWalker = @import("webapi/TreeWalker.zig");+const Label = @import("webapi/element/html/Label.zig");+const AXNode = @import("../cdp/AXNode.zig");const Element = @import("webapi/Element.zig");const Node = @import("webapi/Node.zig");const EventTarget = @import("webapi/EventTarget.zig");@@ -179,6 +181,7 @@ fn walkInteractive(const listener_targets = try buildListenerTargetMap(frame, arena);var css_cache: Element.PointerEventsCache = .empty;+ var label_index: Label.LabelByForIndex = .{};var results: std.ArrayList(InteractiveElement) = .empty;@@ -212,15 +215,16 @@ fn walkInteractive(const itype = classifyInteractivity(frame, el, html_el, listener_targets, &css_cache) orelse continue;- const role = getRole(el);+ const axn = AXNode.fromNode(node);+ const role = axRole(axn);if (filter.role) |rf| {const r = role orelse continue;if (!std.ascii.eqlIgnoreCase(r, rf)) continue;}- // Resolve accessible name only after the cheap role filter passes,- // since getAccessibleName walks the element's text subtree.- const name = try getAccessibleName(el, arena);+ // Resolve the name only after the cheap role filter passes: it walks+ // labels and the element's text subtree.+ const name = (try axn.getName(frame, arena, &label_index)) orelse try getTextContent(node, arena);if (filter.name) |nf| {const n = name orelse continue;if (std.ascii.indexOfIgnoreCase(n, nf) == null) continue;@@ -391,65 +395,11 @@ pub fn explicitRole(el: *Element) ?[]const u8 {return it.next();}-fn getRole(el: *Element) ?[]const u8 {- if (explicitRole(el)) |role| return role;-- // Implicit role from tag- return switch (el.getTag()) {- .button, .summary => "button",- .anchor, .area => if (el.getAttributeSafe(comptime .wrap("href")) != null) "link" else null,- .input => blk: {- if (el.is(Element.Html.Input)) |input| {- break :blk switch (input._input_type) {- .text, .tel, .url, .email => "textbox",- .checkbox => "checkbox",- .radio => "radio",- .button, .submit, .reset, .image => "button",- .range => "slider",- .number => "spinbutton",- .search => "searchbox",- else => null,- };- }- break :blk null;- },- .select => "combobox",- .textarea => "textbox",- .details => "group",- else => null,- };-}--fn getAccessibleName(el: *Element, arena: Allocator) !?[]const u8 {- // aria-label- if (el.getAttributeSafe(comptime .wrap("aria-label"))) |v| {- if (v.len > 0) return v;- }-- // alt (for img, input[type=image])- if (el.getAttributeSafe(comptime .wrap("alt"))) |v| {- if (v.len > 0) return v;- }-- // title- if (el.getAttributeSafe(comptime .wrap("title"))) |v| {- if (v.len > 0) return v;- }-- // placeholder- if (el.getAttributeSafe(comptime .wrap("placeholder"))) |v| {- if (v.len > 0) return v;- }-- // value (for buttons)- if (el.getTag() == .input) {- if (el.getAttributeSafe(comptime .wrap("value"))) |v| {- if (v.len > 0) return v;- }- }-- // Text content (first non-empty text node, trimmed)- return try getTextContent(el.asNode(), arena);+/// The accessibility role, or null where there is none: a plain element+/// that is interactive only by its listener or tabindex.+fn axRole(axn: AXNode) ?[]const u8 {+ const role = axn.getRole() catch return null;+ return if (std.mem.eql(u8, role, "none")) null else role;}pub fn getTextContent(node: *Node, arena: Allocator) !?[]const u8 {@@ -529,6 +479,45 @@ fn testInteractive(html: []const u8) ![]InteractiveElement {return collectInteractiveElements(div.asNode(), frame.call_arena, frame);}+/// Like `testInteractive`, but attached to the document so `<label for>` resolves.+fn testInteractiveInBody(html: []const u8) ![]InteractiveElement {+ const frame = try testing.createFrame();+ errdefer testing.test_session.closeAllPages();++ const doc = frame.window._document;+ const div = try doc.createElement("div", null, frame);+ _ = try doc.asNode().appendChild(div.asNode(), frame);+ try Frame.parse.htmlAsChildren(frame, div.asNode(), html);++ return collectInteractiveElements(div.asNode(), frame.call_arena, frame);+}++test "browser.interactive: names come from labels, like the tree" {+ const elements = try testInteractiveInBody(+ \\<label for="email">Email address</label><input id="email" type="text">+ \\<label>Password <input type="password"></label>+ \\<span id="q-label">Search</span><input type="search" aria-labelledby="q-label">+ \\<input type="text" placeholder="City">+ );+ defer testing.test_session.closeAllPages();+ try testing.expectEqual(4, elements.len);+ try testing.expectEqual("textbox", elements[0].role.?);+ try testing.expectEqual("Email address", elements[0].name.?);+ try testing.expectEqual("textbox", elements[1].role.?);+ try testing.expectEqual("Password", elements[1].name.?);+ try testing.expectEqual("searchbox", elements[2].role.?);+ try testing.expectEqual("Search", elements[2].name.?);+ try testing.expectEqual("City", elements[3].name.?);+}++test "browser.interactive: an element with no role keeps its text as name" {+ const elements = try testInteractive("<div tabindex=\"0\">Open menu</div>");+ defer testing.test_session.closeAllPages();+ try testing.expectEqual(1, elements.len);+ try testing.expectEqual(null, elements[0].role);+ try testing.expectEqual("Open menu", elements[0].name.?);+}+test "browser.interactive: button" {const elements = try testInteractive("<button>Click me</button>");defer testing.test_session.closeAllPages();diff --git a/src/browser/links.zig b/src/browser/links.zigindex 3769a0af3..41838b745 100644--- a/src/browser/links.zig+++ b/src/browser/links.zig@@ -80,12 +80,12 @@ pub fn collectLinks(arena: Allocator, root: *Node, frame: *Frame) ![]Link {const gop = try links.getOrPut(arena, href);if (gop.found_existing) {- if (gop.value_ptr.text == null) gop.value_ptr.text = try AXNode.fromNode(node).getName(frame, arena);+ if (gop.value_ptr.text == null) gop.value_ptr.text = try AXNode.fromNode(node).getName(frame, arena, null);continue;}gop.value_ptr.* = .{.node = node,- .text = try AXNode.fromNode(node).getName(frame, arena),+ .text = try AXNode.fromNode(node).getName(frame, arena, null),.href = href,};}diff --git a/src/browser/webapi/WebDriver.zig b/src/browser/webapi/WebDriver.zigindex d50abd255..750123423 100644--- a/src/browser/webapi/WebDriver.zig+++ b/src/browser/webapi/WebDriver.zig@@ -50,7 +50,7 @@ pub fn deleteAllCookies(_: *const WebDriver, page: *Page) void {pub fn getComputedLabel(_: *const WebDriver, element: *Element, frame: *Frame) ![]const u8 {const AXNode = @import("../../cdp/AXNode.zig");const axnode = AXNode.fromNode(element.asNode());- return (try axnode.getName(frame, frame.call_arena)) orelse "";+ return (try axnode.getName(frame, frame.call_arena, null)) orelse "";}// Implements testdriver's `click`: a full trusted primary-button clickdiff --git a/src/cdp/AXNode.zig b/src/cdp/AXNode.zigindex 60a9c6d54..704a1f08e 100644--- a/src/cdp/AXNode.zig+++ b/src/cdp/AXNode.zig@@ -707,7 +707,7 @@ pub const Writer = struct {if (!std.mem.eql(u8, needle, resolved.role)) return;}- const name = (try axn.getName(self.frame, self.temp_arena.allocator())) orelse "";+ const name = (try axn.getName(self.frame, self.temp_arena.allocator(), self.label_index)) orelse "";if (filter.accessible_name) |needle| {if (!std.mem.eql(u8, needle, name)) return;}@@ -932,7 +932,9 @@ const AXSource = enum(u8) {value, // input value};-pub fn getName(self: AXNode, frame: *Frame, allocator: std.mem.Allocator) !?[]const u8 {+/// `label_index` makes repeated `<label for>` lookups one document scan+/// instead of one per element.+pub fn getName(self: AXNode, frame: *Frame, allocator: std.mem.Allocator, label_index: ?*Label.LabelByForIndex) !?[]const u8 {var aw: std.Io.Writer.Allocating = .init(allocator);defer aw.deinit();@@ -959,7 +961,7 @@ pub fn getName(self: AXNode, frame: *Frame, allocator: std.mem.Allocator) !?[]coconst w: TextCaptureWriter = .{ .aw = &aw, .writer = &aw.writer };- const source = try self.writeName(null, w, frame, null);+ const source = try self.writeName(null, w, frame, label_index);if (source != null) {// Remove literal quotes inserted by writeString.var raw_text = std.mem.trim(u8, aw.written(), "\"");@@ -2036,7 +2038,7 @@ test "AXNode: getName name-from-content honors explicit role" {for (cases) |c| {const el = (try doc.querySelector(.wrap(c.selector), frame)).?;const axn = AXNode.fromNode(el.asNode());- const name = try axn.getName(frame, testing.allocator);+ const name = try axn.getName(frame, testing.allocator, null);defer if (name) |n| testing.allocator.free(n);if (c.expected) |exp| {