Skip to content

Lightpanda: the headless browser designed for AI and automation

search: drop results the browser would block

Adrià Arrufat4ba1f9e05dc5
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index 65016e236..cd4de2955 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -1381,7 +1381,7 @@ fn execSearch(arena: std.mem.Allocator, session: *lp.Session, arguments: ?std.js
// Fall through on any failure so one outage doesn't kill
// a whole benchmark run.
var detail: Failure = .{};
- if (session.runPumped(apiSearch, .{ engine, arena, api_key, timeout_ms, args.query, &detail })) |markdown_| {
+ if (session.runPumped(apiSearch, .{ engine, arena, api_key, timeout_ms, args.query, &detail, &session.browser.http_client })) |markdown_| {
return .{ .text = markdown_ };
} else |err| {
last_err = err;
@@ -1411,7 +1411,7 @@ fn searchExplicit(arena: std.mem.Allocator, session: *lp.Session, comptime engin
.is_error = true,
};
var detail: Failure = .{};
- const markdown_ = session.runPumped(apiSearch, .{ engine, arena, api_key, timeout_ms, query, &detail }) catch |err|
+ const markdown_ = session.runPumped(apiSearch, .{ engine, arena, api_key, timeout_ms, query, &detail, &session.browser.http_client }) catch |err|
return searchFailed(arena, label, err, detail);
return .{ .text = markdown_ };
}
@@ -1450,6 +1450,7 @@ fn apiSearch(
timeout_ms: u32,
query: []const u8,
detail: *Failure,
+ url_filter: anytype,
) ![]const u8 {
var init_options = engine.init_options;
// The cascade (or the model) is the retry; honoring a Retry-After (60 s
@@ -1479,7 +1480,20 @@ fn apiSearch(
};
defer response.deinit();
- return renderResults(arena, try engine.collect(arena, response.value));
+ return renderResults(arena, try dropBlocked(arena, try engine.collect(arena, response.value), url_filter));
+}
+
+/// Drops hits whose URL the browser's `--block-urls` patterns would refuse,
+/// so a search can't hand the model a page it isn't allowed to open, or that
+/// page's text in a snippet. A synthesized answer may draw on a dropped hit,
+/// so it goes too. `url_filter` is anything with `blocksUrl(url) bool`.
+fn dropBlocked(arena: std.mem.Allocator, results: SearchResults, url_filter: anytype) std.mem.Allocator.Error!SearchResults {
+ var kept: std.ArrayList(Hit) = .empty;
+ for (results.hits) |hit| {
+ if (!url_filter.blocksUrl(hit.url)) try kept.append(arena, hit);
+ }
+ if (kept.items.len == results.hits.len) return results;
+ return .{ .hits = kept.items };
}
pub const Hit = struct {
@@ -3343,6 +3357,35 @@ test "brave titles and descriptions render on one line" {
);
}
+test "dropBlocked: blocked hits and the synthesized answer go, the rest stay in order" {
+ var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);
+ defer arena.deinit();
+ const aa = arena.allocator();
+
+ const Filter = struct {
+ fn blocksUrl(_: @This(), url: []const u8) bool {
+ return std.mem.find(u8, url, "huggingface.co/datasets/") != null;
+ }
+ };
+ const results: SearchResults = .{ .answer = "42", .hits = &.{
+ .{ .title = "a", .url = "https://example.org/a", .snippet = "" },
+ .{ .title = "gold", .url = "https://huggingface.co/datasets/x/y", .snippet = "answer: 42" },
+ .{ .title = "b", .url = "https://example.org/b", .snippet = "" },
+ } };
+
+ const filtered = try dropBlocked(aa, results, Filter{});
+ try std.testing.expectEqual(2, filtered.hits.len);
+ try std.testing.expectEqualStrings("https://example.org/a", filtered.hits[0].url);
+ try std.testing.expectEqualStrings("https://example.org/b", filtered.hits[1].url);
+ try std.testing.expectEqualStrings("", filtered.answer);
+
+ const untouched = try dropBlocked(aa, .{ .answer = "42", .hits = results.hits[0..1] }, Filter{});
+ try std.testing.expectEqualStrings("42", untouched.answer);
+
+ const all_blocked = try dropBlocked(aa, .{ .hits = results.hits[1..2] }, Filter{});
+ try std.testing.expectEqualStrings("No results.", try renderResults(aa, all_blocked));
+}
+
test "searchFailed: a rate limit says so, a bare failure stays short" {
var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);
defer arena.deinit();
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 24159230b..8ee687d17 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -424,6 +424,13 @@ fn clearUrlBlocklist(self: *Client) void {
}
}
+/// Whether a `--block-urls` pattern (or CDP `Network.setBlockedURLs`)
+/// matches `url`. The search tool uses it to drop results the browser would
+/// refuse to open anyway.
+pub fn blocksUrl(self: *const Client, url: []const u8) bool {
+ return if (self.url_blocklist) |*blocklist| blocklist.isBlocked(url) else false;
+}
+
/// Every reason a request is refused before it reaches the network:
/// `--block-urls` patterns and the `--adblock-lists` filters both land here
/// so that no call site can apply one without the other.