Lightpanda: the headless browser designed for AI and automation
links: dedup by href, skip hidden, fall back to alt/aria-label/title
Adrià Arrufat6dca2c89177b
diff --git a/src/browser/links.zig b/src/browser/links.zigindex 95624ce0f..771646969 100644--- a/src/browser/links.zig+++ b/src/browser/links.zig@@ -22,6 +22,7 @@ const Element = @import("webapi/Element.zig");const Node = @import("webapi/Node.zig");const Frame = @import("Frame.zig");const Selector = @import("webapi/selector/Selector.zig");+const TreeWalker = @import("webapi/TreeWalker.zig");const interactive = @import("interactive.zig");const log = @import("../lightpanda.zig").log;@@ -57,24 +58,36 @@ pub fn registerNodes(links: []Link, registry: anytype) !void {}}-/// Collect all links (anchor tags with an href) under `root`.+/// Collect the visible links (anchor tags with an href) under `root`, one+/// entry per resolved href.pub fn collectLinks(arena: Allocator, root: *Node, frame: *Frame) ![]Link {var links: std.ArrayList(Link) = .empty;+ var by_href: std.StringHashMapUnmanaged(usize) = .empty;+ var visibility_cache: Element.VisibilityCache = .empty;if (Selector.querySelectorAll(root, "a[href]", frame)) |list| {defer list.deinit(frame._page);for (list._nodes) |node| {+ const el = node.is(Element) orelse continue;const anchor = node.is(Element.Html.Anchor) orelse continue;+ if (!el.checkVisibilityCached(&visibility_cache, frame, .scan)) continue;+const href = anchor.getHref(frame) catch |err| {log.err(.app, "resolve href failed", .{ .err = err });continue;};if (href.len == 0) continue;+ const text = linkText(el, arena);+ if (by_href.get(href)) |i| {+ if (links.items[i].text == null) links.items[i].text = text;+ continue;+ }+ try by_href.put(arena, href, links.items.len);try links.append(arena, .{.node = node,- .text = interactive.getTextContent(node, arena) catch null,+ .text = text,.href = href,});}@@ -86,6 +99,22 @@ pub fn collectLinks(arena: Allocator, root: *Node, frame: *Frame) ![]Link {return links.items;}+// Same fallbacks markdown uses for an anchor without text.+fn linkText(el: *Element, arena: Allocator) ?[]const u8 {+ if (interactive.getTextContent(el.asNode(), arena) catch null) |text| return text;+ if (el.getAttributeSafe(comptime .wrap("aria-label"))) |label| return label;+ if (el.getAttributeSafe(comptime .wrap("title"))) |title| return title;++ var tw: TreeWalker.FullExcludeSelf = .init(el.asNode(), .{});+ while (tw.next()) |child| {+ const img = child.is(Element) orelse continue;+ if (img.getTag() != .img) continue;+ const alt = img.getAttributeSafe(comptime .wrap("alt")) orelse continue;+ if (alt.len > 0) return alt;+ }+ return null;+}+const testing = @import("../testing.zig");fn testLinks(html: []const u8) ![]Link {@@ -122,3 +151,47 @@ test "links: empty text" {try testing.expectEqual(1, links.len);try testing.expectEqual(null, links[0].text);}++test "links: text fallbacks for image and icon links" {+ defer testing.test_session.closeAllPages();+ const links = try testLinks(+ \\<a href="/logo"><img src="logo.png" alt="Acme"></a>+ \\<a href="/discord" aria-label="Discord"><svg></svg></a>+ \\<a href="/help" title="Help"><svg></svg></a>+ \\<a href="/both" aria-label="Label"><img alt="Alt"></a>+ );++ try testing.expectEqual(4, links.len);+ try testing.expectEqual("Acme", links[0].text.?);+ try testing.expectEqual("Discord", links[1].text.?);+ try testing.expectEqual("Help", links[2].text.?);+ try testing.expectEqual("Label", links[3].text.?);+}++test "links: one entry per href, keeping the first with text" {+ defer testing.test_session.closeAllPages();+ const links = try testLinks(+ \\<a href="/post/1"><img src="t.png"></a>+ \\<a href="/post/1">Read the post</a>+ \\<a href="/post/1">Comments</a>+ \\<a href="/post/2">Other</a>+ );++ try testing.expectEqual(2, links.len);+ try testing.expectEqual("Read the post", links[0].text.?);+ try testing.expectEqual("http://localhost/post/1", links[0].href);+ try testing.expectEqual("Other", links[1].text.?);+}++test "links: hidden links are skipped" {+ defer testing.test_session.closeAllPages();+ const links = try testLinks(+ \\<a href="/a" style="display:none">Inline</a>+ \\<a href="/b" hidden>Attribute</a>+ \\<div style="display:none"><a href="/c">Ancestor</a></div>+ \\<a href="/d">Visible</a>+ );++ try testing.expectEqual(1, links.len);+ try testing.expectEqual("Visible", links[0].text.?);+}diff --git a/src/browser/tests/mcp_links.html b/src/browser/tests/mcp_links.htmlnew file mode 100644index 000000000..571c99415--- /dev/null+++ b/src/browser/tests/mcp_links.html@@ -0,0 +1,9 @@+<!DOCTYPE html>+<html>+<body>+ <a href="/first">First</a>+ <a href="/first">First again</a>+ <a href="/second" hidden>Hidden</a>+ <a href="/third"><img src="t.png" alt="Third"></a>+</body>+</html>diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 7862e7874..3992bce47 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -400,9 +400,18 @@ pub const Tool = enum {),},.links => .{- .description = "Extract all links in the opened page as JSON objects with `text` (visible anchor text), `href` (resolved URL), and `backendNodeId` (pass to click/nodeDetails). If a url is provided, it navigates to that url first.",+ .description = "Extract the visible links in the opened page as JSON objects with `text` (anchor text, falling back to aria-label/title/image alt), `href` (resolved URL), and `backendNodeId` (pass to click/nodeDetails). One entry per href; hidden links are omitted. If a url is provided, it navigates to that url first.",.summary = "List all links on the page",- .input_schema = url_params_schema,+ .input_schema = minify(+ \\{+ \\ "type": "object",+ \\ "properties": {+ \\ "limit": { "type": "integer", "description": "Optional. Return at most this many links, in document order." },+ \\ "url": { "type": "string", "description": "Optional URL to navigate to before processing." },+ \\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." }+ \\ }+ \\}+ ),},.evaluate => .{.description = "Evaluate JavaScript in the current page context — an escape hatch for page-side logic the dedicated tools can't express; prefer `extract` for data and click/fill/etc. for actions. It runs in the page, so it cannot see the agent script's variables or builtins — interpolate any value into the `script` string. A bare trailing expression yields its value; top-level `await` and `return` are supported (the body then runs as an async function, so use `return` to produce a value). Objects and arrays return as JSON, so no `JSON.stringify` is needed. If a url is provided, it navigates there first. The `globalThis.lp` object exposes a Session-scoped bridge store: values written via `lp.foo = ...` auto-sync at end of evaluate, surviving navigation; values previously set via `/extract save=` or `/evaluate save=` appear as `lp.<name>`.",@@ -1316,11 +1325,19 @@ fn dumpHtml(session: *lp.Session, registry: *CDPNode.Registry, page: *lp.Frame,}fn execLinks(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError![]const u8 {- const args = try parseArgsOrDefault(UrlParams, arena, arguments);+ const Params = struct {+ limit: ?u32 = null,+ url: ?[:0]const u8 = null,+ timeout: ?u32 = null,+ };+ const args = try parseArgsOrDefault(Params, arena, arguments);const page = try ensurePage(session, registry, args.url, args.timeout);- const links_list = lp.links.collectLinks(arena, page.document.asNode(), page) catch+ var links_list = lp.links.collectLinks(arena, page.document.asNode(), page) catchreturn ToolError.InternalError;+ if (args.limit) |limit| {+ links_list = links_list[0..@min(limit, links_list.len)];+ }lp.links.registerNodes(links_list, registry) catchreturn ToolError.InternalError;return renderJson(arena, links_list);diff --git a/src/mcp/tools.zig b/src/mcp/tools.zigindex 771d212eb..0eb10ee0d 100644--- a/src/mcp/tools.zig+++ b/src/mcp/tools.zig@@ -1443,6 +1443,28 @@ test "MCP - html: maxBytes truncation and strip" {try testing.expect(std.mem.indexOf(u8, out.written(), "[truncated]") != null);}+test "MCP - links: dedup, hidden, text fallback, limit" {+ var out: std.Io.Writer.Allocating = .init(testing.arena_allocator);+ const server = try testLoadPage("http://localhost:9582/src/browser/tests/mcp_links.html", &out.writer);+ defer server.deinit();++ const all =+ \\{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"links"}}+ ;+ try router.handleMessage(server, testing.arena_allocator, all);+ try testing.expectEqual(2, std.mem.count(u8, out.written(), "href"));+ try testing.expect(std.mem.indexOf(u8, out.written(), "/second") == null);+ try testing.expect(std.mem.indexOf(u8, out.written(), "Third") != null);++ out.clearRetainingCapacity();+ const limited =+ \\{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"links","arguments":{"limit":1}}}+ ;+ try router.handleMessage(server, testing.arena_allocator, limited);+ try testing.expectEqual(1, std.mem.count(u8, out.written(), "href"));+ try testing.expect(std.mem.indexOf(u8, out.written(), "/first") != null);+}+test "MCP - waitForScript: truthy returns, falsy times out" {var out: std.Io.Writer.Allocating = .init(testing.arena_allocator);const server = try testLoadPage("about:blank", &out.writer);