Lightpanda: the headless browser designed for AI and automation
`findElement`: match accessible names by regex
Adrià Arrufat158de60d6478
diff --git a/src/browser/interactive.zig b/src/browser/interactive.zigindex 655af76b6..0b1df93d4 100644--- a/src/browser/interactive.zig+++ b/src/browser/interactive.zig@@ -20,6 +20,7 @@ const std = @import("std");const Frame = @import("Frame.zig");const URL = @import("URL.zig");+const Regex = @import("../Regex.zig");const TreeWalker = @import("webapi/TreeWalker.zig");const Label = @import("webapi/element/html/Label.zig");const AXNode = @import("../server/cdp/AXNode.zig");@@ -154,6 +155,8 @@ const FindFilter = struct {role: ?[]const u8 = null,/// Accessible-name substring match (case-insensitive). When null, name is not filtered.name: ?[]const u8 = null,+ /// Compiled pattern searched in the accessible name.+ name_regex: ?Regex = null,/// Stop walking once this many matches accumulate. When null, walks the full subtree.max: ?usize = null,};@@ -229,6 +232,10 @@ fn walkInteractive(const n = name orelse continue;if (std.ascii.indexOfIgnoreCase(n, nf) == null) continue;}+ if (filter.name_regex) |re| {+ const n = name orelse continue;+ if (!re.matches(n)) continue;+ }const listener_types = getListenerTypes(el.asEventTarget(), listener_targets);@@ -490,6 +497,30 @@ fn testInteractiveInBody(html: []const u8) ![]InteractiveElement {return collectInteractiveElements(div.asNode(), frame.call_arena, frame);}+test "browser.interactive: a name regex filters the walk" {+ const frame = try testing.createFrame();+ defer testing.test_session.closeAllPages();+ const doc = frame.window._document;+ const div = try doc.createElement("div", null, frame);+ try Frame.parse.htmlAsChildren(frame, div.asNode(), "<button>Add to cart</button><button>Cart</button><a href=\"#\">Add item</a>");++ const context = testing.test_app.regex_context;+ const options: Regex.Options = .{ .case_insensitive = true, .unicode = true };++ const starts_add = try Regex.compile(context, "^add", options, null);+ defer starts_add.deinit();+ const found_add = try findInteractiveElements(div.asNode(), frame.call_arena, frame, .{ .name_regex = starts_add });+ try testing.expectEqual(2, found_add.len);+ try testing.expectEqual("Add to cart", found_add[0].name.?);+ try testing.expectEqual("Add item", found_add[1].name.?);++ const only_cart = try Regex.compile(context, "^cart$", options, null);+ defer only_cart.deinit();+ const found_cart = try findInteractiveElements(div.asNode(), frame.call_arena, frame, .{ .name_regex = only_cart });+ try testing.expectEqual(1, found_cart.len);+ try testing.expectEqual("Cart", found_cart[0].name.?);+}+test "browser.interactive: names come from labels, like the tree" {const elements = try testInteractiveInBody(\\<label for="email">Email address</label><input id="email" type="text">diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex ab2499905..777cf1525 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -50,8 +50,8 @@ pub const driver_guidance =\\ values are already in the tree — don't re-fetch via `nodeDetails`.\\- `nodeDetails(backendNodeId)` → a ready-to-use CSS `selector` that\\ resolves to one node, plus its id/class/attrs.- \\- `findElement(role, name)` → locate a candidate by role/name without- \\ parsing the whole tree.+ \\- `findElement(role, name)` → locate a candidate by role and name (a+ \\ substring, or `/regex/`) without parsing the whole tree.\\- `markdown(selector | backendNodeId)` → readable text for one\\ subtree. Use after `tree` has shown you where the interesting\\ region is.@@ -683,7 +683,7 @@ pub const Tool = enum {\\ "type": "object",\\ "properties": {\\ "role": { "type": "string", "description": "Optional ARIA role to match (e.g. 'button', 'link', 'textbox', 'checkbox')." },- \\ "name": { "type": "string", "description": "Optional accessible name substring to match (case-insensitive)." }+ \\ "name": { "type": "string", "description": "Optional accessible name to match, case-insensitive: a substring, or a JavaScript-syntax regex written as /.../ (unanchored: use ^...$ for the whole name; prefix (?-i) for case-sensitive)." }\\ }\\}),@@ -924,7 +924,7 @@ fn dispatch(.press => .{ .text = try execPress(arena, session, registry, substituted) },.selectOption => .{ .text = try execSelectOption(arena, session, registry, substituted) },.setChecked => .{ .text = try execSetChecked(arena, session, registry, substituted) },- .findElement => .{ .text = try execFindElement(arena, session, registry, substituted) },+ .findElement => execFindElement(arena, session, registry, substituted),.evaluate => execEvaluate(arena, session, registry, substituted),.extract => execExtract(arena, session, registry, substituted),.getEnv => .{ .text = try execGetEnv(arena, substituted) },@@ -2062,7 +2062,7 @@ fn execSetChecked(arena: std.mem.Allocator, session: *lp.Session, registry: *Nodreturn finalizeAction(arena, session, registry, scope, body);}-fn execFindElement(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeRegistry, arguments: ?std.json.Value) ToolError![]const u8 {+fn execFindElement(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeRegistry, arguments: ?std.json.Value) ToolError!ToolResult {const Params = struct {role: ?[]const u8 = null,name: ?[]const u8 = null,@@ -2073,14 +2073,36 @@ fn execFindElement(arena: std.mem.Allocator, session: *lp.Session, registry: *Noconst page = try requireFrame(session);+ const pattern: ?[]const u8 = if (args.name) |name| regexBody(name) else null;+ var diag: lp.Regex.Diagnostic = .{};+ const name_regex: ?lp.Regex = if (pattern) |p|+ lp.Regex.compile(session.browser.app.regex_context, p, .{ .case_insensitive = true, .unicode = true }, &diag) catch |err| switch (err) {+ error.OutOfMemory => return error.OutOfMemory,+ error.InvalidRegex => return .{+ .text = try std.fmt.allocPrint(arena, "findElement: invalid name regex '{s}': {s} at offset {d}", .{ p, diag.message(), diag.offset }),+ .is_error = true,+ },+ }+ else+ null;+ defer if (name_regex) |re| re.deinit();+const matched = lp.interactive.findInteractiveElements(page.document.asNode(), arena, page, .{.role = args.role,- .name = args.name,+ .name = if (pattern == null) args.name else null,+ .name_regex = name_regex,}) catch return ToolError.InternalError;lp.interactive.registerNodes(matched, registry) catchreturn ToolError.InternalError;- return renderJson(arena, matched);+ return .{ .text = try renderJson(arena, matched) };+}++/// The body of a `/.../` literal, the spelling adblock lists use for a regex+/// too. Unanchored, so a name that really is written that way still matches.+fn regexBody(text: []const u8) ?[]const u8 {+ if (text.len > 2 and text[0] == '/' and text[text.len - 1] == '/') return text[1 .. text.len - 1];+ return null;}fn execGetEnv(arena: std.mem.Allocator, arguments: ?std.json.Value) ToolError![]const u8 {diff --git a/src/mcp/tools.zig b/src/mcp/tools.zigindex 49561705a..0c3b59a03 100644--- a/src/mcp/tools.zig+++ b/src/mcp/tools.zig@@ -1411,6 +1411,26 @@ test "MCP - findElement" {try testing.expect(std.mem.indexOf(u8, out.written(), "error") != null);out.clearRetainingCapacity();}++ {+ const msg =+ \\{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"findElement","arguments":{"name":"/^prevent.*default$/"}}}+ ;+ try router.handleMessage(server, aa, msg);+ try testing.expect(std.mem.indexOf(u8, out.written(), "Prevent Default") != null);+ try testing.expect(std.mem.indexOf(u8, out.written(), "Click Me") == null);+ out.clearRetainingCapacity();+ }++ {+ const msg =+ \\{"jsonrpc":"2.0","id":6,"method":"tools/call","params":{"name":"findElement","arguments":{"name":"/(/"}}}+ ;+ try router.handleMessage(server, aa, msg);+ try testing.expect(std.mem.indexOf(u8, out.written(), "\"isError\":true") != null);+ try testing.expect(std.mem.indexOf(u8, out.written(), "missing closing parenthesis at offset 1") != null);+ out.clearRetainingCapacity();+ }}test "MCP - waitForSelector: existing element" {diff --git a/src/script/skill.zig b/src/script/skill.zigindex 7ad4306c9..0792a91f5 100644--- a/src/script/skill.zig+++ b/src/script/skill.zig@@ -203,7 +203,8 @@ fn note(tool: browser_tools.Tool) []const u8 {.screenshot => "`path` is required: writes a PNG of the text layout.",.press => "Selector first! `page.press(\"Enter\")` binds \"Enter\" to `selector` and fails — use `page.press(null, \"Enter\")` or `page.press({ key: \"Enter\" })`.",.click, .fill, .scroll, .hover, .selectOption, .setChecked => "",- .search, .markdown, .html, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => "",+ .findElement => "`name` is a case-insensitive substring, or a JS-syntax regex written as `/.../`.",+ .search, .markdown, .html, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .consoleLogs, .getUrl, .getCookies, .getEnv => "",};}