Lightpanda: the headless browser designed for AI and automation
tools: collect search hits once, render them once
Adrià Arrufat1e081b5ac801
diff --git a/src/browser/tools.zig b/src/browser/tools.zigindex 9a7bfcf88..68b53c43f 100644--- a/src/browser/tools.zig+++ b/src/browser/tools.zig@@ -1092,7 +1092,7 @@ const api_engines = .{.init_options = brave.Client.InitOptions{},// text_decorations=false: no <strong> markup in model-read snippets..options = brave.types.SearchOptions{ .count = 10, .text_decorations = false },- .format = formatBraveMarkdown,+ .collect = collectBrave,},.{.tag = SearchEngine.tavily,@@ -1100,7 +1100,7 @@ const api_engines = .{.Client = tavily.Client,.init_options = tavily.Client.InitOptions{},.options = tavily.types.SearchOptions{ .max_results = 10 },- .format = formatTavilyMarkdown,+ .collect = collectTavily,},.{.tag = SearchEngine.exa,@@ -1110,7 +1110,7 @@ const api_engines = .{// highlights: Exa returns no snippet text unless contents is requested;// capped at 3 sentences since the default excerpts run long..options = exa.types.SearchOptions{ .numResults = 10, .contents = .{ .highlights = .{ .numSentences = 3 } } },- .format = formatExaMarkdown,+ .collect = collectExa,},.{.tag = SearchEngine.keenable,@@ -1120,7 +1120,7 @@ const api_engines = .{// snippet_max_length is a hint the API may round up to a word// boundary; 500 keeps ten results within a few KB of context..options = keenable.types.SearchOptions{ .max_results = 10, .snippet_max_length = 500 },- .format = formatKeenableMarkdown,+ .collect = collectKeenable,},};@@ -1261,54 +1261,66 @@ fn apiSearch(};defer response.deinit();- var aw: std.Io.Writer.Allocating = .init(arena);- try engine.format(&aw.writer, response.value);- return aw.written();+ return renderResults(arena, try engine.collect(arena, response.value));}-fn formatTavilyMarkdown(w: *std.Io.Writer, resp: tavily.types.SearchResponse) !void {- const answer = resp.answer orelse "";- if (answer.len == 0 and resp.results.len == 0) {- return w.writeAll("No results.");- }- if (answer.len > 0) {- try w.print("**Answer:** {s}\n\n", .{answer});- }- for (resp.results, 0..) |r, i| {- try writeResultItem(w, i, r.title, r.url, r.content);- }+pub const Hit = struct {+ title: []const u8,+ url: []const u8,+ snippet: []const u8,+};++pub const SearchResults = struct {+ /// Tavily's synthesized answer; empty for the engines that have none.+ answer: []const u8 = "",+ hits: []const Hit = &.{},+};++fn collectTavily(arena: std.mem.Allocator, resp: tavily.types.SearchResponse) !SearchResults {+ const hits = try arena.alloc(Hit, resp.results.len);+ for (resp.results, hits) |r, *hit| hit.* = .{ .title = r.title, .url = r.url, .snippet = r.content };+ return .{ .answer = resp.answer orelse "", .hits = hits };}-fn formatBraveMarkdown(w: *std.Io.Writer, resp: brave.types.SearchResponse) !void {+fn collectBrave(arena: std.mem.Allocator, resp: brave.types.SearchResponse) !SearchResults {const results: []const brave.types.Result = if (resp.web) |web| web.results else &.{};- if (results.len == 0) {- return w.writeAll("No results.");- }- for (results, 0..) |r, i| {- try writeResultItem(w, i, r.title, r.url, r.description);- }+ const hits = try arena.alloc(Hit, results.len);+ for (results, hits) |r, *hit| hit.* = .{ .title = r.title, .url = r.url, .snippet = r.description };+ return .{ .hits = hits };}-fn formatExaMarkdown(w: *std.Io.Writer, resp: exa.types.SearchResponse) !void {- if (resp.results.len == 0) {- return w.writeAll("No results.");- }- for (resp.results, 0..) |r, i| {+fn collectExa(arena: std.mem.Allocator, resp: exa.types.SearchResponse) !SearchResults {+ const hits = try arena.alloc(Hit, resp.results.len);+ for (resp.results, hits) |r, *hit| {const highlights = r.highlights orelse &[_][]const u8{};- const snippet = if (highlights.len > 0) highlights[0] else "";- try writeResultItem(w, i, r.title orelse "", r.url, snippet);+ hit.* = .{+ .title = r.title orelse "",+ .url = r.url,+ .snippet = if (highlights.len > 0) highlights[0] else "",+ };}+ return .{ .hits = hits };}-fn formatKeenableMarkdown(w: *std.Io.Writer, resp: keenable.types.SearchResponse) !void {- if (resp.results.len == 0) {- return w.writeAll("No results.");- }- for (resp.results, 0..) |r, i| {- // snippet carries the page text (the wire format's always-empty- // `description` is deliberately not even mapped by the client).- try writeResultItem(w, i, r.title, r.url, r.snippet);- }+fn collectKeenable(arena: std.mem.Allocator, resp: keenable.types.SearchResponse) !SearchResults {+ const hits = try arena.alloc(Hit, resp.results.len);+ // snippet carries the page text (the wire format's always-empty+ // `description` is deliberately not even mapped by the client).+ for (resp.results, hits) |r, *hit| hit.* = .{ .title = r.title, .url = r.url, .snippet = r.snippet };+ return .{ .hits = hits };+}++fn renderResults(arena: std.mem.Allocator, results: SearchResults) ToolError![]const u8 {+ if (results.answer.len == 0 and results.hits.len == 0) return "No results.";+ var aw: std.Io.Writer.Allocating = .init(arena);+ const w = &aw.writer;+ renderInto(w, results) catch return ToolError.OutOfMemory;+ return aw.written();+}++fn renderInto(w: *std.Io.Writer, results: SearchResults) !void {+ if (results.answer.len > 0) try w.print("**Answer:** {s}\n\n", .{results.answer});+ for (results.hits, 0..) |hit, i| try writeResultItem(w, i, hit.title, hit.url, hit.snippet);}/// An empty title (providers default it to "") would render as `****`.@@ -2782,7 +2794,7 @@ test "formatLpEnvNames reports empty when no names" {try std.testing.expectEqualStrings("No LP_* environment variables are set.", r);}-test "formatTavilyMarkdown renders answer and results" {+test "collectTavily renders answer and results" {var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);defer arena.deinit();const aa = arena.allocator();@@ -2796,9 +2808,7 @@ test "formatTavilyMarkdown renders answer and results" {},};- var aw: std.Io.Writer.Allocating = .init(aa);- try formatTavilyMarkdown(&aw.writer, resp);- const md = aw.written();+ const md = try renderResults(aa, try collectTavily(aa, resp));try std.testing.expect(std.mem.indexOf(u8, md, "**Answer:** Paris") != null);try std.testing.expect(std.mem.indexOf(u8, md, "1. **Paris - Wikipedia**") != null);try std.testing.expect(std.mem.indexOf(u8, md, "https://en.wikipedia.org/wiki/Paris") != null);@@ -2806,17 +2816,15 @@ test "formatTavilyMarkdown renders answer and results" {try std.testing.expect(std.mem.indexOf(u8, md, "2. **France**") != null);}-test "formatTavilyMarkdown handles empty results" {+test "collectTavily handles empty results" {var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);defer arena.deinit();const aa = arena.allocator();- var aw: std.Io.Writer.Allocating = .init(aa);- try formatTavilyMarkdown(&aw.writer, .{});- try std.testing.expectEqualStrings("No results.", aw.written());+ try std.testing.expectEqualStrings("No results.", try renderResults(aa, try collectTavily(aa, .{})));}-test "formatBraveMarkdown renders web results" {+test "collectBrave renders web results" {var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);defer arena.deinit();const aa = arena.allocator();@@ -2830,30 +2838,24 @@ test "formatBraveMarkdown renders web results" {},};- var aw: std.Io.Writer.Allocating = .init(aa);- try formatBraveMarkdown(&aw.writer, resp);- const md = aw.written();+ const md = try renderResults(aa, try collectBrave(aa, resp));try std.testing.expect(std.mem.indexOf(u8, md, "1. **Paris - Wikipedia**") != null);try std.testing.expect(std.mem.indexOf(u8, md, "https://en.wikipedia.org/wiki/Paris") != null);try std.testing.expect(std.mem.indexOf(u8, md, "Paris is the capital of France.") != null);try std.testing.expect(std.mem.indexOf(u8, md, "2. **France**") != null);}-test "formatBraveMarkdown handles empty results" {+test "collectBrave handles empty results" {var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);defer arena.deinit();const aa = arena.allocator();- var no_web: std.Io.Writer.Allocating = .init(aa);- try formatBraveMarkdown(&no_web.writer, .{});- try std.testing.expectEqualStrings("No results.", no_web.written());+ try std.testing.expectEqualStrings("No results.", try renderResults(aa, try collectBrave(aa, .{})));- var empty_web: std.Io.Writer.Allocating = .init(aa);- try formatBraveMarkdown(&empty_web.writer, .{ .web = .{} });- try std.testing.expectEqualStrings("No results.", empty_web.written());+ try std.testing.expectEqualStrings("No results.", try renderResults(aa, try collectBrave(aa, .{ .web = .{} })));}-test "formatKeenableMarkdown reads snippet" {+test "collectKeenable reads snippet" {var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);defer arena.deinit();const aa = arena.allocator();@@ -2866,9 +2868,7 @@ test "formatKeenableMarkdown reads snippet" {},};- var aw: std.Io.Writer.Allocating = .init(aa);- try formatKeenableMarkdown(&aw.writer, resp);- const md = aw.written();+ const md = try renderResults(aa, try collectKeenable(aa, resp));try std.testing.expect(std.mem.indexOf(u8, md, "1. **Zig (programming language)**") != null);try std.testing.expect(std.mem.indexOf(u8, md, "Zig is a system programming language.") != null);try std.testing.expect(std.mem.indexOf(u8, md, "2. **Zig guide**") != null);@@ -2883,15 +2883,14 @@ test "writeResultItem uses the URL as title when the title is empty" {try std.testing.expectEqualStrings("1. **https://example.org/x.pdf** — https://example.org/x.pdf\n snippet\n\n", aw.written());}-test "formatKeenableMarkdown handles empty results" {+test "collectKeenable handles empty results" {var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);defer arena.deinit();- var aw: std.Io.Writer.Allocating = .init(arena.allocator());- try formatKeenableMarkdown(&aw.writer, .{});- try std.testing.expectEqualStrings("No results.", aw.written());+ const aa = arena.allocator();+ try std.testing.expectEqualStrings("No results.", try renderResults(aa, try collectKeenable(aa, .{})));}-test "formatBraveMarkdown flattens newlines in titles and descriptions" {+test "collectBrave flattens newlines in titles and descriptions" {var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);defer arena.deinit();const aa = arena.allocator();@@ -2904,9 +2903,10 @@ test "formatBraveMarkdown flattens newlines in titles and descriptions" {},};- var aw: std.Io.Writer.Allocating = .init(aa);- try formatBraveMarkdown(&aw.writer, resp);- try std.testing.expectEqualStrings("1. **Multi line title** — https://example.org\n line one line two\n\n", aw.written());+ try std.testing.expectEqualStrings(+ "1. **Multi line title** — https://example.org\n line one line two\n\n",+ try renderResults(aa, try collectBrave(aa, resp)),+ );}test "isPathSafe: relative paths without traversal are accepted" {