Skip to content

Lightpanda: the headless browser designed for AI and automation

browser: add exa search engine support

Adrià Arrufatd76ead300af7
diff --git a/build.zig.zon b/build.zig.zon
index 9318956c0..d67bf5ddb 100644
--- a/build.zig.zon
+++ b/build.zig.zon
@@ -36,8 +36,8 @@
.hash = "sqlite3-3.53.2-DMxLWuAOAAA_Px0arJOIOaP4AKEu5prbsQgPMA35W1zz",
},
.zenai = .{
- .url = "git+https://github.com/lightpanda-io/zenai.git#f721fd4171afb95c65182923554d0f1de10d99bf",
- .hash = "zenai-0.0.0-iOY_VNgmBgBok23JrHb7N0U67VKSSqRb0BJWIpMMcnoy",
+ .url = "git+https://github.com/lightpanda-io/zenai.git#00bdd10117eefa247c815ebca47efbde9a91e3f0",
+ .hash = "zenai-0.0.0-iOY_VOE8BgCgb8oZtxgmYCfE9vlXdDaHYEsKMsUDCUyC",
},
.isocline = .{
.url = "git+https://github.com/arrufat/isocline?ref=lightpanda#832a9fe25f5f4458fcc47b5acc7c21db669c2f47",
diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig
index fb30f414e..21cf4775b 100644
--- a/src/agent/Agent.zig
+++ b/src/agent/Agent.zig
@@ -769,11 +769,7 @@ fn handleStream(self: *Agent, rest: []const u8) void {
fn handleSearchEngine(self: *Agent, rest: []const u8) void {
self.setEnumOption("searchEngine", &browser_tools.search_engine, rest);
const selected = std.meta.stringToEnum(browser_tools.SearchEngine, rest) orelse return;
- const env_var: [:0]const u8 = switch (selected) {
- .tavily => "TAVILY_API_KEY",
- .brave => "BRAVE_API_KEY",
- .auto, .duckduckgo => return,
- };
+ const env_var = browser_tools.searchEnvVar(selected) orelse return;
if (std.c.getenv(env_var) == null) {
self.terminal.printWarning("{s} is not set; the search tool will fail until you export it", .{env_var});
}
@@ -1458,7 +1454,7 @@ fn printSlashHelp(self: *Agent, arena: std.mem.Allocator, target: []const u8) vo
.{},
),
.searchEngine => self.terminal.printInfo(
- "/searchEngine " ++ Config.tagHint(browser_tools.SearchEngine) ++ " — set the web search engine behind the search tool (currently: {s}); saved to {s}. 'auto' tries Brave then Tavily (when their API keys are set) and falls back to the DuckDuckGo scrape; an explicit engine is used alone. Bare /searchEngine prints the engine.",
+ "/searchEngine " ++ Config.tagHint(browser_tools.SearchEngine) ++ " — set the web search engine behind the search tool (currently: {s}); saved to {s}. 'auto' tries Brave, Tavily, then Exa (when their API keys are set) and falls back to the DuckDuckGo scrape; an explicit engine is used alone. Bare /searchEngine prints the engine.",
.{ @tagName(browser_tools.search_engine), settings.remembered_path },
),
}
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index a3b989796..7baaf939a 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -23,6 +23,7 @@ const zenai = @import("zenai");
const log = lp.log;
const tavily = zenai.search.tavily;
const brave = zenai.search.brave;
+const exa = zenai.search.exa;
const DOMNode = @import("webapi/Node.zig");
const CDPNode = @import("../cdp/Node.zig");
@@ -349,7 +350,7 @@ pub const Tool = enum {
),
},
.search => .{
- .description = "Run a web search and return results as markdown. When BRAVE_API_KEY or TAVILY_API_KEY is set, queries that search API (Brave preferred) and returns a numbered list of {title, url, snippet}. Otherwise (or on API failure) falls back to scraping the DuckDuckGo HTML endpoint — degraded results, may rate-limit on bursty traffic. Prefer this over goto-ing google.com/search directly (Google blocks the browser on User-Agent/TLS). Browser state after this call is unspecified — to interact with a result, use `goto` with its URL; do not assume the browser DOM matches the results page.",
+ .description = "Run a web search and return results as markdown. When BRAVE_API_KEY, TAVILY_API_KEY or EXA_API_KEY is set, queries that search API (in that preference order) and returns a numbered list of {title, url, snippet}. Otherwise (or on API failure) falls back to scraping the DuckDuckGo HTML endpoint — degraded results, may rate-limit on bursty traffic. Prefer this over goto-ing google.com/search directly (Google blocks the browser on User-Agent/TLS). Browser state after this call is unspecified — to interact with a result, use `goto` with its URL; do not assume the browser DOM matches the results page.",
.summary = "Web search, results as markdown",
.input_schema = minify(
\\{
@@ -977,12 +978,55 @@ pub const SearchParams = struct {
timeout: ?u32 = null,
};
-/// Search backend for `execSearch`. `.auto` tries Brave then Tavily (each
-/// only when its API key is set), then the DuckDuckGo scrape; an explicit
-/// engine is used alone. Only the agent REPL's `/searchEngine` mutates this.
-pub const SearchEngine = enum { auto, tavily, brave, duckduckgo };
+/// Search backend for `execSearch`. `.auto` tries the `api_engines` in
+/// order (each only when its API key is set), then the DuckDuckGo scrape;
+/// an explicit engine is used alone. Only the agent REPL's `/searchEngine`
+/// mutates this.
+pub const SearchEngine = enum { auto, tavily, brave, exa, duckduckgo };
pub var search_engine: SearchEngine = .auto;
+// Ordered by `.auto` preference; the `search` tool description prose must
+// agree with this table.
+const api_engines = .{
+ .{
+ .tag = SearchEngine.brave,
+ .env_var = "BRAVE_API_KEY",
+ .Client = brave.Client,
+ // text_decorations=false: no <strong> markup in model-read snippets.
+ .options = brave.types.SearchOptions{ .count = 10, .text_decorations = false },
+ .format = formatBraveMarkdown,
+ },
+ .{
+ .tag = SearchEngine.tavily,
+ .env_var = "TAVILY_API_KEY",
+ .Client = tavily.Client,
+ .options = tavily.types.SearchOptions{ .max_results = 10 },
+ .format = formatTavilyMarkdown,
+ },
+ .{
+ .tag = SearchEngine.exa,
+ .env_var = "EXA_API_KEY",
+ .Client = exa.Client,
+ // highlights: Exa returns no snippet text unless contents is requested.
+ .options = exa.types.SearchOptions{ .numResults = 10, .contents = .{ .highlights = true } },
+ .format = formatExaMarkdown,
+ },
+};
+
+pub fn searchEnvVar(engine: SearchEngine) ?[:0]const u8 {
+ inline for (api_engines) |e| {
+ if (engine == e.tag) return e.env_var;
+ }
+ return null;
+}
+
+fn engineIndex(comptime tag: SearchEngine) usize {
+ inline for (api_engines, 0..) |e, i| {
+ if (e.tag == tag) return i;
+ }
+ @compileError("engine missing from api_engines: " ++ @tagName(tag));
+}
+
fn execSearch(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError!ToolResult {
const args = try parseArgs(SearchParams, arena, arguments);
if (args.query.len == 0) return ToolError.InvalidParams;
@@ -990,25 +1034,17 @@ fn execSearch(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode
switch (search_engine) {
// Any failure (network, non-2xx, parse) falls through to the next
// engine so a single outage doesn't kill a whole benchmark run.
- .auto => {
- if (std.c.getenv("BRAVE_API_KEY")) |api_key_z| {
- if (braveSearch(arena, std.mem.span(api_key_z), args.query)) |markdown_| {
- return .{ .text = markdown_ };
- } else |err| {
- log.warn(.browser, "brave fallback", .{ .err = err });
- }
- }
- if (std.c.getenv("TAVILY_API_KEY")) |api_key_z| {
- if (tavilySearch(arena, std.mem.span(api_key_z), args.query)) |markdown_| {
+ .auto => inline for (api_engines) |engine| {
+ if (std.c.getenv(engine.env_var)) |api_key_z| {
+ if (apiSearch(engine, arena, std.mem.span(api_key_z), args.query)) |markdown_| {
return .{ .text = markdown_ };
} else |err| {
- log.warn(.browser, "tavily fallback", .{ .err = err });
+ log.warn(.browser, @tagName(engine.tag) ++ " fallback", .{ .err = err });
}
}
},
- .tavily => return searchExplicit(arena, "tavily", "TAVILY_API_KEY", tavilySearch, args.query),
- .brave => return searchExplicit(arena, "brave", "BRAVE_API_KEY", braveSearch, args.query),
.duckduckgo => {},
+ inline else => |tag| return searchExplicit(arena, api_engines[comptime engineIndex(tag)], args.query),
}
const encoded = lp.URL.percentEncodeSegment(arena, args.query, .component) catch return ToolError.OutOfMemory;
@@ -1025,18 +1061,13 @@ fn execSearch(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode
/// Search with an explicitly selected engine: a missing key or a failed call
/// comes back as an error result — no DDG fallback.
-fn searchExplicit(
- arena: std.mem.Allocator,
- comptime label: []const u8,
- comptime env_var: [:0]const u8,
- searchFn: anytype,
- query: []const u8,
-) ToolError!ToolResult {
- const api_key_z = std.c.getenv(env_var) orelse return .{
- .text = "web search engine is set to " ++ label ++ " but " ++ env_var ++ " is not set in the environment",
+fn searchExplicit(arena: std.mem.Allocator, comptime engine: anytype, query: []const u8) ToolError!ToolResult {
+ const label = @tagName(engine.tag);
+ const api_key_z = std.c.getenv(engine.env_var) orelse return .{
+ .text = "web search engine is set to " ++ label ++ " but " ++ engine.env_var ++ " is not set in the environment",
.is_error = true,
};
- const markdown_ = searchFn(arena, std.mem.span(api_key_z), query) catch |err| {
+ const markdown_ = apiSearch(engine, arena, std.mem.span(api_key_z), query) catch |err| {
const msg = std.fmt.allocPrint(arena, label ++ " search failed: {s}", .{@errorName(err)}) catch
return ToolError.OutOfMemory;
return .{ .text = msg, .is_error = true };
@@ -1044,22 +1075,21 @@ fn searchExplicit(
return .{ .text = markdown_ };
}
-/// Thin wrapper over `zenai.search.tavily.Client` that handles client
-/// lifetime and renders the structured response as markdown for the agent.
-/// `arena` owns the returned slice. `api_key` is the value of TAVILY_API_KEY.
-fn tavilySearch(
+/// `arena` owns the returned slice.
+fn apiSearch(
+ comptime engine: anytype,
arena: std.mem.Allocator,
api_key: []const u8,
query: []const u8,
) ![]const u8 {
- var client: tavily.Client = .init(lp.io, arena, api_key, .{});
+ var client: engine.Client = .init(lp.io, arena, api_key, .{});
defer client.deinit();
- var response = client.search(query, .{ .max_results = 10 }) catch |err| {
- if (client.last_error_status) |status| {
- log.warn(.browser, "tavily non-2xx", .{
+ var response = client.search(query, engine.options) catch |err| {
+ if (client.last_error.status) |status| {
+ log.warn(.browser, @tagName(engine.tag) ++ " non-2xx", .{
.status = status,
- .body = client.last_error_body,
+ .body = client.last_error.body,
});
}
return err;
@@ -1067,7 +1097,7 @@ fn tavilySearch(
defer response.deinit();
var aw: std.Io.Writer.Allocating = .init(arena);
- try formatTavilyMarkdown(&aw.writer, response.value);
+ try engine.format(&aw.writer, response.value);
return aw.written();
}
@@ -1084,34 +1114,6 @@ fn formatTavilyMarkdown(w: *std.Io.Writer, resp: tavily.types.SearchResponse) !v
}
}
-/// Thin wrapper over `zenai.search.brave.Client` that handles client
-/// lifetime and renders the structured response as markdown for the agent.
-/// `arena` owns the returned slice. `api_key` is the value of BRAVE_API_KEY.
-fn braveSearch(
- arena: std.mem.Allocator,
- api_key: []const u8,
- query: []const u8,
-) ![]const u8 {
- var client: brave.Client = .init(lp.io, arena, api_key, .{});
- defer client.deinit();
-
- // text_decorations=false: no <strong> markup in model-read snippets.
- var response = client.search(query, .{ .count = 10, .text_decorations = false }) catch |err| {
- if (client.last_error_status) |status| {
- log.warn(.browser, "brave non-2xx", .{
- .status = status,
- .body = client.last_error_body,
- });
- }
- return err;
- };
- defer response.deinit();
-
- var aw: std.Io.Writer.Allocating = .init(arena);
- try formatBraveMarkdown(&aw.writer, response.value);
- return aw.written();
-}
-
fn formatBraveMarkdown(w: *std.Io.Writer, resp: brave.types.SearchResponse) !void {
const results: []const brave.types.Result = if (resp.web) |web| web.results else &.{};
if (results.len == 0) {
@@ -1122,6 +1124,17 @@ fn formatBraveMarkdown(w: *std.Io.Writer, resp: brave.types.SearchResponse) !voi
}
}
+fn formatExaMarkdown(w: *std.Io.Writer, resp: exa.types.SearchResponse) !void {
+ if (resp.results.len == 0) {
+ return w.writeAll("No results.");
+ }
+ for (resp.results, 0..) |r, i| {
+ const highlights = r.highlights orelse &[_][]const u8{};
+ const snippet = if (highlights.len > 0) highlights[0] else "";
+ try writeResultItem(w, i, r.title orelse r.url, r.url, snippet);
+ }
+}
+
fn writeResultItem(w: *std.Io.Writer, i: usize, title: []const u8, url: []const u8, snippet: []const u8) !void {
try w.print("{d}. **", .{i + 1});
try writeSingleLine(w, title);