Skip to content

Lightpanda: the headless browser designed for AI and automation

links: dedup by href, skip hidden, fall back to alt/aria-label/title

Adrià Arrufat6dca2c89177b
diff --git a/src/browser/links.zig b/src/browser/links.zig
index 95624ce0f..771646969 100644
--- a/src/browser/links.zig
+++ b/src/browser/links.zig
@@ -22,6 +22,7 @@ const Element = @import("webapi/Element.zig");
const Node = @import("webapi/Node.zig");
const Frame = @import("Frame.zig");
const Selector = @import("webapi/selector/Selector.zig");
+const TreeWalker = @import("webapi/TreeWalker.zig");
const interactive = @import("interactive.zig");
const log = @import("../lightpanda.zig").log;
@@ -57,24 +58,36 @@ pub fn registerNodes(links: []Link, registry: anytype) !void {
}
}
-/// Collect all links (anchor tags with an href) under `root`.
+/// Collect the visible links (anchor tags with an href) under `root`, one
+/// entry per resolved href.
pub fn collectLinks(arena: Allocator, root: *Node, frame: *Frame) ![]Link {
var links: std.ArrayList(Link) = .empty;
+ var by_href: std.StringHashMapUnmanaged(usize) = .empty;
+ var visibility_cache: Element.VisibilityCache = .empty;
if (Selector.querySelectorAll(root, "a[href]", frame)) |list| {
defer list.deinit(frame._page);
for (list._nodes) |node| {
+ const el = node.is(Element) orelse continue;
const anchor = node.is(Element.Html.Anchor) orelse continue;
+ if (!el.checkVisibilityCached(&visibility_cache, frame, .scan)) continue;
+
const href = anchor.getHref(frame) catch |err| {
log.err(.app, "resolve href failed", .{ .err = err });
continue;
};
if (href.len == 0) continue;
+ const text = linkText(el, arena);
+ if (by_href.get(href)) |i| {
+ if (links.items[i].text == null) links.items[i].text = text;
+ continue;
+ }
+ try by_href.put(arena, href, links.items.len);
try links.append(arena, .{
.node = node,
- .text = interactive.getTextContent(node, arena) catch null,
+ .text = text,
.href = href,
});
}
@@ -86,6 +99,22 @@ pub fn collectLinks(arena: Allocator, root: *Node, frame: *Frame) ![]Link {
return links.items;
}
+// Same fallbacks markdown uses for an anchor without text.
+fn linkText(el: *Element, arena: Allocator) ?[]const u8 {
+ if (interactive.getTextContent(el.asNode(), arena) catch null) |text| return text;
+ if (el.getAttributeSafe(comptime .wrap("aria-label"))) |label| return label;
+ if (el.getAttributeSafe(comptime .wrap("title"))) |title| return title;
+
+ var tw: TreeWalker.FullExcludeSelf = .init(el.asNode(), .{});
+ while (tw.next()) |child| {
+ const img = child.is(Element) orelse continue;
+ if (img.getTag() != .img) continue;
+ const alt = img.getAttributeSafe(comptime .wrap("alt")) orelse continue;
+ if (alt.len > 0) return alt;
+ }
+ return null;
+}
+
const testing = @import("../testing.zig");
fn testLinks(html: []const u8) ![]Link {
@@ -122,3 +151,47 @@ test "links: empty text" {
try testing.expectEqual(1, links.len);
try testing.expectEqual(null, links[0].text);
}
+
+test "links: text fallbacks for image and icon links" {
+ defer testing.test_session.closeAllPages();
+ const links = try testLinks(
+ \\<a href="/logo"><img src="logo.png" alt="Acme"></a>
+ \\<a href="/discord" aria-label="Discord"><svg></svg></a>
+ \\<a href="/help" title="Help"><svg></svg></a>
+ \\<a href="/both" aria-label="Label"><img alt="Alt"></a>
+ );
+
+ try testing.expectEqual(4, links.len);
+ try testing.expectEqual("Acme", links[0].text.?);
+ try testing.expectEqual("Discord", links[1].text.?);
+ try testing.expectEqual("Help", links[2].text.?);
+ try testing.expectEqual("Label", links[3].text.?);
+}
+
+test "links: one entry per href, keeping the first with text" {
+ defer testing.test_session.closeAllPages();
+ const links = try testLinks(
+ \\<a href="/post/1"><img src="t.png"></a>
+ \\<a href="/post/1">Read the post</a>
+ \\<a href="/post/1">Comments</a>
+ \\<a href="/post/2">Other</a>
+ );
+
+ try testing.expectEqual(2, links.len);
+ try testing.expectEqual("Read the post", links[0].text.?);
+ try testing.expectEqual("http://localhost/post/1", links[0].href);
+ try testing.expectEqual("Other", links[1].text.?);
+}
+
+test "links: hidden links are skipped" {
+ defer testing.test_session.closeAllPages();
+ const links = try testLinks(
+ \\<a href="/a" style="display:none">Inline</a>
+ \\<a href="/b" hidden>Attribute</a>
+ \\<div style="display:none"><a href="/c">Ancestor</a></div>
+ \\<a href="/d">Visible</a>
+ );
+
+ try testing.expectEqual(1, links.len);
+ try testing.expectEqual("Visible", links[0].text.?);
+}
diff --git a/src/browser/tests/mcp_links.html b/src/browser/tests/mcp_links.html
new file mode 100644
index 000000000..571c99415
--- /dev/null
+++ b/src/browser/tests/mcp_links.html
@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+<body>
+ <a href="/first">First</a>
+ <a href="/first">First again</a>
+ <a href="/second" hidden>Hidden</a>
+ <a href="/third"><img src="t.png" alt="Third"></a>
+</body>
+</html>
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index 7862e7874..3992bce47 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -400,9 +400,18 @@ pub const Tool = enum {
),
},
.links => .{
- .description = "Extract all links in the opened page as JSON objects with `text` (visible anchor text), `href` (resolved URL), and `backendNodeId` (pass to click/nodeDetails). If a url is provided, it navigates to that url first.",
+ .description = "Extract the visible links in the opened page as JSON objects with `text` (anchor text, falling back to aria-label/title/image alt), `href` (resolved URL), and `backendNodeId` (pass to click/nodeDetails). One entry per href; hidden links are omitted. If a url is provided, it navigates to that url first.",
.summary = "List all links on the page",
- .input_schema = url_params_schema,
+ .input_schema = minify(
+ \\{
+ \\ "type": "object",
+ \\ "properties": {
+ \\ "limit": { "type": "integer", "description": "Optional. Return at most this many links, in document order." },
+ \\ "url": { "type": "string", "description": "Optional URL to navigate to before processing." },
+ \\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." }
+ \\ }
+ \\}
+ ),
},
.evaluate => .{
.description = "Evaluate JavaScript in the current page context — an escape hatch for page-side logic the dedicated tools can't express; prefer `extract` for data and click/fill/etc. for actions. It runs in the page, so it cannot see the agent script's variables or builtins — interpolate any value into the `script` string. A bare trailing expression yields its value; top-level `await` and `return` are supported (the body then runs as an async function, so use `return` to produce a value). Objects and arrays return as JSON, so no `JSON.stringify` is needed. If a url is provided, it navigates there first. The `globalThis.lp` object exposes a Session-scoped bridge store: values written via `lp.foo = ...` auto-sync at end of evaluate, surviving navigation; values previously set via `/extract save=` or `/evaluate save=` appear as `lp.<name>`.",
@@ -1316,11 +1325,19 @@ fn dumpHtml(session: *lp.Session, registry: *CDPNode.Registry, page: *lp.Frame,
}
fn execLinks(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError![]const u8 {
- const args = try parseArgsOrDefault(UrlParams, arena, arguments);
+ const Params = struct {
+ limit: ?u32 = null,
+ url: ?[:0]const u8 = null,
+ timeout: ?u32 = null,
+ };
+ const args = try parseArgsOrDefault(Params, arena, arguments);
const page = try ensurePage(session, registry, args.url, args.timeout);
- const links_list = lp.links.collectLinks(arena, page.document.asNode(), page) catch
+ var links_list = lp.links.collectLinks(arena, page.document.asNode(), page) catch
return ToolError.InternalError;
+ if (args.limit) |limit| {
+ links_list = links_list[0..@min(limit, links_list.len)];
+ }
lp.links.registerNodes(links_list, registry) catch
return ToolError.InternalError;
return renderJson(arena, links_list);
diff --git a/src/mcp/tools.zig b/src/mcp/tools.zig
index 771d212eb..0eb10ee0d 100644
--- a/src/mcp/tools.zig
+++ b/src/mcp/tools.zig
@@ -1443,6 +1443,28 @@ test "MCP - html: maxBytes truncation and strip" {
try testing.expect(std.mem.indexOf(u8, out.written(), "[truncated]") != null);
}
+test "MCP - links: dedup, hidden, text fallback, limit" {
+ var out: std.Io.Writer.Allocating = .init(testing.arena_allocator);
+ const server = try testLoadPage("http://localhost:9582/src/browser/tests/mcp_links.html", &out.writer);
+ defer server.deinit();
+
+ const all =
+ \\{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"links"}}
+ ;
+ try router.handleMessage(server, testing.arena_allocator, all);
+ try testing.expectEqual(2, std.mem.count(u8, out.written(), "href"));
+ try testing.expect(std.mem.indexOf(u8, out.written(), "/second") == null);
+ try testing.expect(std.mem.indexOf(u8, out.written(), "Third") != null);
+
+ out.clearRetainingCapacity();
+ const limited =
+ \\{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"links","arguments":{"limit":1}}}
+ ;
+ try router.handleMessage(server, testing.arena_allocator, limited);
+ try testing.expectEqual(1, std.mem.count(u8, out.written(), "href"));
+ try testing.expect(std.mem.indexOf(u8, out.written(), "/first") != null);
+}
+
test "MCP - waitForScript: truthy returns, falsy times out" {
var out: std.Io.Writer.Allocating = .init(testing.arena_allocator);
const server = try testLoadPage("about:blank", &out.writer);