Lightpanda: the headless browser designed for AI and automation
search: drop results the browser would block
Adrià Arrufat4ba1f9e05dc5
diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 65016e236..cd4de2955 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -1381,7 +1381,7 @@ fn execSearch(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.js// Fall through on any failure so one outage doesn't kill// a whole benchmark run.var detail: Failure = .{};- if (session.runPumped(apiSearch, .{ engine, arena, api_key, timeout_ms, args.query, &detail })) |markdown_| {+ if (session.runPumped(apiSearch, .{ engine, arena, api_key, timeout_ms, args.query, &detail, &session.browser.http_client })) |markdown_| {return .{ .text = markdown_ };} else |err| {last_err = err;@@ -1411,7 +1411,7 @@ fn searchExplicit(arena: std.mem.Allocator, session: *lp.Session, comptime engin.is_error = true,};var detail: Failure = .{};- const markdown_ = session.runPumped(apiSearch, .{ engine, arena, api_key, timeout_ms, query, &detail }) catch |err|+ const markdown_ = session.runPumped(apiSearch, .{ engine, arena, api_key, timeout_ms, query, &detail, &session.browser.http_client }) catch |err|return searchFailed(arena, label, err, detail);return .{ .text = markdown_ };}@@ -1450,6 +1450,7 @@ fn apiSearch(timeout_ms: u32,query: []const u8,detail: *Failure,+ url_filter: anytype,) ![]const u8 {var init_options = engine.init_options;// The cascade (or the model) is the retry; honoring a Retry-After (60 s@@ -1479,7 +1480,20 @@ fn apiSearch(};defer response.deinit();- return renderResults(arena, try engine.collect(arena, response.value));+ return renderResults(arena, try dropBlocked(arena, try engine.collect(arena, response.value), url_filter));+}++/// Drops hits whose URL the browser's `--block-urls` patterns would refuse,+/// so a search can't hand the model a page it isn't allowed to open, or that+/// page's text in a snippet. A synthesized answer may draw on a dropped hit,+/// so it goes too. `url_filter` is anything with `blocksUrl(url) bool`.+fn dropBlocked(arena: std.mem.Allocator, results: SearchResults, url_filter: anytype) std.mem.Allocator.Error!SearchResults {+ var kept: std.ArrayList(Hit) = .empty;+ for (results.hits) |hit| {+ if (!url_filter.blocksUrl(hit.url)) try kept.append(arena, hit);+ }+ if (kept.items.len == results.hits.len) return results;+ return .{ .hits = kept.items };}pub const Hit = struct {@@ -3343,6 +3357,35 @@ test "brave titles and descriptions render on one line" {);}+test "dropBlocked: blocked hits and the synthesized answer go, the rest stay in order" {+ var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);+ defer arena.deinit();+ const aa = arena.allocator();++ const Filter = struct {+ fn blocksUrl(_: @This(), url: []const u8) bool {+ return std.mem.find(u8, url, "huggingface.co/datasets/") != null;+ }+ };+ const results: SearchResults = .{ .answer = "42", .hits = &.{+ .{ .title = "a", .url = "https://example.org/a", .snippet = "" },+ .{ .title = "gold", .url = "https://huggingface.co/datasets/x/y", .snippet = "answer: 42" },+ .{ .title = "b", .url = "https://example.org/b", .snippet = "" },+ } };++ const filtered = try dropBlocked(aa, results, Filter{});+ try std.testing.expectEqual(2, filtered.hits.len);+ try std.testing.expectEqualStrings("https://example.org/a", filtered.hits[0].url);+ try std.testing.expectEqualStrings("https://example.org/b", filtered.hits[1].url);+ try std.testing.expectEqualStrings("", filtered.answer);++ const untouched = try dropBlocked(aa, .{ .answer = "42", .hits = results.hits[0..1] }, Filter{});+ try std.testing.expectEqualStrings("42", untouched.answer);++ const all_blocked = try dropBlocked(aa, .{ .hits = results.hits[1..2] }, Filter{});+ try std.testing.expectEqualStrings("No results.", try renderResults(aa, all_blocked));+}+test "searchFailed: a rate limit says so, a bare failure stays short" {var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);defer arena.deinit();diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 24159230b..8ee687d17 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -424,6 +424,13 @@ fn clearUrlBlocklist(self: *Client) void {}}+/// Whether a `--block-urls` pattern (or CDP `Network.setBlockedURLs`)+/// matches `url`. The search tool uses it to drop results the browser would+/// refuse to open anyway.+pub fn blocksUrl(self: *const Client, url: []const u8) bool {+ return if (self.url_blocklist) |*blocklist| blocklist.isBlocked(url) else false;+}+/// Every reason a request is refused before it reaches the network:/// `--block-urls` patterns and the `--adblock-lists` filters both land here/// so that no call site can apply one without the other.