Skip to content

Lightpanda: the headless browser designed for AI and automation

search: default to Keenable's keyless endpoint, drop the DuckDuckGo scrape

Adrià Arrufat42c3cbf20b27
diff --git a/build.zig.zon b/build.zig.zon
index c11965f32..9448ac039 100644
--- a/build.zig.zon
+++ b/build.zig.zon
@@ -36,8 +36,8 @@
.hash = "sqlite3-3.53.2-DMxLWuAOAAA_Px0arJOIOaP4AKEu5prbsQgPMA35W1zz",
},
.zenai = .{
- .url = "git+https://github.com/lightpanda-io/zenai.git#4aa2cccdaa765938fd96e4a9e74a0765362e9a17",
- .hash = "zenai-0.0.0-iOY_VApeBgDxgim4YNZ8UQd3XkAyUGOIrgAMx2pD6zxs",
+ .url = "git+https://github.com/lightpanda-io/zenai.git#a745bb1b654e150955f7f4d84ddd1e35ed8beb9e",
+ .hash = "zenai-0.0.0-iOY_VI6EBgAryRPO5dH1uS_knQguW5hqcL8EfpnBKBFK",
},
.isocline = .{
.url = "git+https://github.com/arrufat/isocline?ref=lightpanda#832a9fe25f5f4458fcc47b5acc7c21db669c2f47",
diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig
index 8b00360de..ad681520f 100644
--- a/src/agent/Agent.zig
+++ b/src/agent/Agent.zig
@@ -769,13 +769,11 @@ fn handleStream(self: *Agent, rest: []const u8) void {
fn handleSearchEngine(self: *Agent, rest: []const u8) void {
self.setEnumOption("searchEngine", &browser_tools.search_engine, rest);
const selected = std.meta.stringToEnum(browser_tools.SearchEngine, rest) orelse return;
- const env_var = browser_tools.searchEnvVar(selected) orelse return;
- if (std.c.getenv(env_var) == null) {
- if (browser_tools.searchEngineKeyless(selected)) {
- self.terminal.printInfo("{s} is not set; using the keyless endpoint (rate-limited per client IP)", .{env_var});
- } else {
- self.terminal.printWarning("{s} is not set; the search tool will fail until you export it", .{env_var});
- }
+ const key = browser_tools.searchKeyStatus(selected) orelse return;
+ switch (key.state) {
+ .set => {},
+ .keyless => self.terminal.printInfo("{s} is not set; using the keyless endpoint (rate-limited per client IP)", .{key.env_var}),
+ .missing => self.terminal.printWarning("{s} is not set; the search tool will fail until you export it", .{key.env_var}),
}
}
@@ -1458,7 +1456,7 @@ fn printSlashHelp(self: *Agent, arena: std.mem.Allocator, target: []const u8) vo
.{},
),
.searchEngine => self.terminal.printInfo(
- "/searchEngine " ++ Config.tagHint(browser_tools.SearchEngine) ++ " — set the web search engine behind the search tool (currently: {s}); saved to {s}. 'auto' tries Brave, Tavily, Exa, then Keenable (each when its API key is set), then Keenable's keyless endpoint, and falls back to the DuckDuckGo scrape; an explicit engine is used alone. Bare /searchEngine prints the engine.",
+ "/searchEngine " ++ Config.tagHint(browser_tools.SearchEngine) ++ " — set the web search engine behind the search tool (currently: {s}); saved to {s}. 'auto' tries " ++ browser_tools.search_engine_order ++ " (each when its API key is set; keenable's public endpoint otherwise); an explicit engine is used alone. Bare /searchEngine prints the engine.",
.{ @tagName(browser_tools.search_engine), settings.remembered_path },
),
}
diff --git a/src/agent/settings.zig b/src/agent/settings.zig
index f428751d5..bea91c4f7 100644
--- a/src/agent/settings.zig
+++ b/src/agent/settings.zig
@@ -461,7 +461,7 @@ test "parseRemembered: search_engine field round-trips" {
test "resolveSearchEngine: default auto, remembered wins" {
try testing.expect(resolveSearchEngine(null) == .auto);
try testing.expect(resolveSearchEngine(.{ .model = "m", .search_engine = null }) == .auto);
- try testing.expect(resolveSearchEngine(.{ .model = "m", .search_engine = .duckduckgo }) == .duckduckgo);
+ try testing.expect(resolveSearchEngine(.{ .model = "m", .search_engine = .keenable }) == .keenable);
}
test "resolveStream: default on, remembered wins" {
diff --git a/src/browser/tools.zig b/src/browser/tools.zig
index 5154c8fa7..f28f82958 100644
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@ -351,7 +351,7 @@ pub const Tool = enum {
),
},
.search => .{
- .description = "Run a web search and return results as markdown. When BRAVE_API_KEY, TAVILY_API_KEY, EXA_API_KEY or KEENABLE_API_KEY is set, queries that search API (in that preference order) and returns a numbered list of {title, url, snippet}. Otherwise (or on API failure) tries Keenable's keyless endpoint (no credential, rate-limited per client IP), then falls back to scraping the DuckDuckGo HTML endpoint — degraded results, may rate-limit on bursty traffic. Prefer this over goto-ing google.com/search directly (Google blocks the browser on User-Agent/TLS). Browser state after this call is unspecified — to interact with a result, use `goto` with its URL; do not assume the browser DOM matches the results page.",
+ .description = "Run a web search and return results as markdown: a numbered list of {title, url, snippet}. Tries " ++ search_engine_order ++ " in order, each when its API key (" ++ search_key_env_vars ++ ") is set; keenable also works without a key through its public endpoint (rate-limited per client IP). Prefer this over goto-ing google.com/search directly (Google blocks the browser on User-Agent/TLS). The browser does not navigate — to open a result, use `goto` with its URL.",
.summary = "Web search, results as markdown",
.input_schema = minify(
\\{
@@ -847,7 +847,7 @@ fn dispatch(
) ToolError!ToolResult {
return switch (tool) {
.goto => .{ .text = try execGoto(arena, session, registry, substituted) },
- .search => execSearch(arena, session, registry, substituted),
+ .search => execSearch(arena, substituted),
.markdown => .{ .text = try execMarkdown(arena, session, registry, substituted) },
.html => .{ .text = try execHtml(arena, session, registry, substituted) },
.links => .{ .text = try execLinks(arena, session, registry, substituted) },
@@ -979,23 +979,15 @@ pub const SearchParams = struct {
timeout: ?u32 = null,
};
-/// Search backend for `execSearch`. `.auto` tries the API engines in
-/// declaration order (each only when its API key is set), then the keyless
-/// public endpoint of `.keyless` engines (keenable), then the DuckDuckGo
-/// scrape; an explicit engine is used alone. Declaration order after `auto`
-/// is the `.auto` preference order (asserted against `api_engines` below);
-/// `duckduckgo` (the scrape fallback, no table entry) is last. Only the
-/// agent REPL's `/searchEngine` mutates this.
-pub const SearchEngine = enum { auto, brave, tavily, exa, keenable, duckduckgo };
+pub const SearchEngine = enum { auto, brave, tavily, exa, keenable };
+/// Only the agent REPL's `/searchEngine` mutates this.
pub var search_engine: SearchEngine = .auto;
-// One entry per API engine, in `SearchEngine` declaration order (asserted
-// below); the `search` tool description prose must agree with this table.
+// One entry per API engine, in `.auto` preference order.
const api_engines = .{
.{
.tag = SearchEngine.brave,
.env_var = "BRAVE_API_KEY",
- .keyless = false,
.Client = brave.Client,
.init_options = brave.Client.InitOptions{},
// text_decorations=false: no <strong> markup in model-read snippets.
@@ -1005,7 +997,6 @@ const api_engines = .{
.{
.tag = SearchEngine.tavily,
.env_var = "TAVILY_API_KEY",
- .keyless = false,
.Client = tavily.Client,
.init_options = tavily.Client.InitOptions{},
.options = tavily.types.SearchOptions{ .max_results = 10 },
@@ -1014,7 +1005,6 @@ const api_engines = .{
.{
.tag = SearchEngine.exa,
.env_var = "EXA_API_KEY",
- .keyless = false,
.Client = exa.Client,
.init_options = exa.Client.InitOptions{},
// highlights: Exa returns no snippet text unless contents is requested;
@@ -1025,14 +1015,8 @@ const api_engines = .{
.{
.tag = SearchEngine.keenable,
.env_var = "KEENABLE_API_KEY",
- // Also works with no API key: a `null` key routes the client to the
- // keyless public endpoint (rate-limited per client IP).
- .keyless = true,
.Client = keenable.Client,
- // Retries off: the fallback cascade is the retry mechanism, and
- // honoring the public endpoint's Retry-After (60s per sleep) would
- // stall the search tool instead of routing to the next rung.
- .init_options = keenable.Client.InitOptions{ .app_title = "lightpanda", .retry_policy = .disabled },
+ .init_options = keenable.Client.InitOptions{ .app_title = "lightpanda" },
// snippet_max_length is a hint the API may round up to a word
// boundary; 500 keeps ten results within a few KB of context.
.options = keenable.types.SearchOptions{ .max_results = 10, .snippet_max_length = 500 },
@@ -1040,125 +1024,125 @@ const api_engines = .{
},
};
-// `SearchEngine` declaration order is the single source of truth: tags must
-// appear in `api_engines` in declaration order, with `auto` first and
-// `duckduckgo` (scrape fallback, no table entry) last.
-comptime {
- std.debug.assert(@intFromEnum(SearchEngine.auto) == 0);
- std.debug.assert(api_engines.len == std.enums.values(SearchEngine).len - 2);
- std.debug.assert(@intFromEnum(SearchEngine.duckduckgo) == api_engines.len + 1);
- for (api_engines, 0..) |e, i| {
- if (@intFromEnum(e.tag) != i + 1)
- @compileError("api_engines order must match SearchEngine declaration order: " ++ @tagName(e.tag));
+/// `api_engines` rendered as prose for the tool description and REPL help,
+/// so neither has to be kept in step with the table by hand.
+fn joinEngines(comptime field: enum { tag, env_var }, comptime last_sep: []const u8) []const u8 {
+ comptime {
+ var out: []const u8 = "";
+ for (api_engines, 0..) |e, i| {
+ const sep = if (i == 0) "" else if (i + 1 == api_engines.len) last_sep else ", ";
+ out = out ++ sep ++ switch (field) {
+ .tag => @tagName(e.tag),
+ .env_var => e.env_var,
+ };
+ }
+ return out;
}
}
+pub const search_engine_order = joinEngines(.tag, ", then ");
+pub const search_key_env_vars = joinEngines(.env_var, " or ");
-/// Engines that also work with no API key (a `null` key routes the client
-/// to the provider's public endpoint). `searchExplicit` and the REPL's
-/// `/searchEngine` warning consult this.
-pub fn searchEngineKeyless(engine: SearchEngine) bool {
- inline for (api_engines) |e| {
- if (engine == e.tag) return e.keyless;
- }
- return false;
+/// A client whose `api_key` is optional also works with no API key: a
+/// `null` key routes it to the provider's public endpoint.
+fn isKeyless(comptime Client: type) bool {
+ return @typeInfo(@FieldType(Client, "api_key")) == .optional;
}
-pub fn searchEnvVar(engine: SearchEngine) ?[:0]const u8 {
+fn envValue(name: [:0]const u8) ?[]const u8 {
+ return std.mem.span(std.c.getenv(name) orelse return null);
+}
+
+pub const KeyStatus = struct {
+ env_var: [:0]const u8,
+ state: enum { set, keyless, missing },
+};
+
+/// How `engine` will authenticate right now; `null` for `.auto`, which has
+/// no key of its own.
+pub fn searchKeyStatus(engine: SearchEngine) ?KeyStatus {
inline for (api_engines) |e| {
- if (engine == e.tag) return e.env_var;
+ if (engine == e.tag) return .{
+ .env_var = e.env_var,
+ .state = if (envValue(e.env_var) != null) .set else if (comptime isKeyless(e.Client)) .keyless else .missing,
+ };
}
return null;
}
-/// Valid for any API engine tag; the comptime block above pins tag order.
-fn engineIndex(comptime tag: SearchEngine) usize {
- return @intFromEnum(tag) - 1;
-}
-
-fn execSearch(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError!ToolResult {
+fn execSearch(arena: std.mem.Allocator, arguments: ?std.json.Value) ToolError!ToolResult {
const args = try parseArgs(SearchParams, arena, arguments);
if (args.query.len == 0) return ToolError.InvalidParams;
+ const timeout_ms = args.timeout orelse default_nav_timeout_ms;
switch (search_engine) {
- // Any failure (network, non-2xx, parse) falls through to the next
- // engine so a single outage doesn't kill a whole benchmark run.
.auto => {
+ var last_err: anyerror = error.NoSearchEngine;
inline for (api_engines) |engine| {
- if (std.c.getenv(engine.env_var)) |api_key_z| {
- if (apiSearch(engine, arena, std.mem.span(api_key_z), args.query)) |markdown_| {
+ const api_key = envValue(engine.env_var);
+ if (api_key != null or comptime isKeyless(engine.Client)) {
+ // Any failure (network, non-2xx, parse) falls through to
+ // the next engine so a single outage doesn't kill a whole
+ // benchmark run; the cascade is the retry mechanism, so
+ // a Retry-After sleep would only stall the tool.
+ if (apiSearch(engine, arena, api_key, .disabled, timeout_ms, args.query)) |markdown_| {
return .{ .text = markdown_ };
} else |err| {
+ last_err = err;
log.warn(.browser, @tagName(engine.tag) ++ " fallback", .{ .err = err });
}
}
}
- // One rung above the DDG scrape: keyless public endpoints return
- // the same structured results under their own rate-limit regime
- // — worth trying even after a failed keyed attempt.
+ return searchFailed(arena, "web", last_err);
+ },
+ inline else => |tag| {
inline for (api_engines) |engine| {
- if (comptime engine.keyless) {
- if (apiSearch(engine, arena, null, args.query)) |markdown_| {
- return .{ .text = markdown_ };
- } else |err| {
- log.warn(.browser, @tagName(engine.tag) ++ " keyless fallback", .{ .err = err });
- }
- }
+ if (engine.tag == tag) return searchExplicit(arena, engine, timeout_ms, args.query);
}
+ @compileError("engine missing from api_engines: " ++ @tagName(tag));
},
- .duckduckgo => {},
- inline else => |tag| return searchExplicit(arena, api_engines[comptime engineIndex(tag)], args.query),
}
-
- const encoded = lp.URL.percentEncodeSegment(arena, args.query, .component) catch return ToolError.OutOfMemory;
- const ddg_url = std.fmt.allocPrintSentinel(
- arena,
- "https://html.duckduckgo.com/html/?q={s}",
- .{encoded},
- 0,
- ) catch return ToolError.OutOfMemory;
- _ = try performGoto(session, registry, ddg_url, .{ .timeout = args.timeout });
- const ddg_frame = try requireFrame(session);
- return .{ .text = try renderFrameMarkdown(arena, ddg_frame) };
}
-/// Search with an explicitly selected engine — no DDG fallback. A failed
-/// call comes back as an error result, as does a missing key unless the
-/// engine is keyless-capable (it then uses its public endpoint).
-fn searchExplicit(arena: std.mem.Allocator, comptime engine: anytype, query: []const u8) ToolError!ToolResult {
+/// Search with an explicitly selected engine — no fallback. A failed call
+/// comes back as an error result, as does a missing key unless the engine
+/// is keyless (it then uses its public endpoint).
+fn searchExplicit(arena: std.mem.Allocator, comptime engine: anytype, timeout_ms: u32, query: []const u8) ToolError!ToolResult {
const label = @tagName(engine.tag);
- const markdown_ = blk: {
- if (std.c.getenv(engine.env_var)) |z|
- break :blk apiSearch(engine, arena, std.mem.span(z), query);
- // Comptime-gated so the null-key call is only instantiated for
- // clients whose `init` takes an optional key.
- if (comptime engine.keyless)
- break :blk apiSearch(engine, arena, null, query);
- return .{
- .text = "web search engine is set to " ++ label ++ " but " ++ engine.env_var ++ " is not set in the environment",
- .is_error = true,
- };
- } catch |err| {
- const msg = std.fmt.allocPrint(arena, label ++ " search failed: {s}", .{@errorName(err)}) catch
- return ToolError.OutOfMemory;
- return .{ .text = msg, .is_error = true };
+ const api_key: ?[]const u8 = envValue(engine.env_var) orelse if (comptime isKeyless(engine.Client)) null else return .{
+ .text = "web search engine is set to " ++ label ++ " but " ++ engine.env_var ++ " is not set in the environment",
+ .is_error = true,
};
+ const markdown_ = apiSearch(engine, arena, api_key, engine.init_options.retry_policy, timeout_ms, query) catch |err|
+ return searchFailed(arena, label, err);
return .{ .text = markdown_ };
}
-/// `arena` owns the returned slice. `api_key` is the environment value, or
-/// `null` for a `.keyless` engine's public endpoint (its client's `init`
-/// takes a `?[]const u8` key; keyed clients get the unwrapped slice).
+fn searchFailed(arena: std.mem.Allocator, comptime label: []const u8, err: anyerror) ToolError!ToolResult {
+ const msg = std.fmt.allocPrint(arena, label ++ " search failed: {s}", .{@errorName(err)}) catch
+ return ToolError.OutOfMemory;
+ return .{ .text = msg, .is_error = true };
+}
+
+/// `arena` owns the returned slice. `api_key` is `null` only for a keyless
+/// engine's public endpoint. `timeout_ms` bounds each attempt once the
+/// connection is up (the search clients' `std.http.Client` has no connect
+/// timeout, unlike the browser's libcurl stack).
fn apiSearch(
comptime engine: anytype,
arena: std.mem.Allocator,
api_key: ?[]const u8,
+ retry_policy: zenai.retry.RetryPolicy,
+ timeout_ms: u32,
query: []const u8,
) ![]const u8 {
+ var init_options = engine.init_options;
+ init_options.retry_policy = retry_policy;
+ init_options.request_timeout_ms = timeout_ms;
var client: engine.Client = .init(
lp.io,
arena,
- if (comptime engine.keyless) api_key else api_key.?,
- engine.init_options,
+ if (comptime isKeyless(engine.Client)) api_key else api_key.?,
+ init_options,
);
defer client.deinit();
@@ -1208,7 +1192,7 @@ fn formatExaMarkdown(w: *std.Io.Writer, resp: exa.types.SearchResponse) !void {
for (resp.results, 0..) |r, i| {
const highlights = r.highlights orelse &[_][]const u8{};
const snippet = if (highlights.len > 0) highlights[0] else "";
- try writeResultItem(w, i, r.title orelse r.url, r.url, snippet);
+ try writeResultItem(w, i, r.title orelse "", r.url, snippet);
}
}
@@ -1223,9 +1207,11 @@ fn formatKeenableMarkdown(w: *std.Io.Writer, resp: keenable.types.SearchResponse
}
}
+/// Providers default a missing title to ""; the URL stands in rather than
+/// rendering an empty bold run.
fn writeResultItem(w: *std.Io.Writer, i: usize, title: []const u8, url: []const u8, snippet: []const u8) !void {
try w.print("{d}. **", .{i + 1});
- try writeSingleLine(w, title);
+ try writeSingleLine(w, if (title.len == 0) url else title);
try w.print("** — {s}\n ", .{url});
try writeSingleLine(w, snippet);
try w.writeAll("\n\n");
@@ -2595,6 +2581,19 @@ test "formatKeenableMarkdown reads snippet" {
try std.testing.expect(std.mem.indexOf(u8, md, "Compile-time execution.") != null);
}
+test "search engine prose is derived from api_engines" {
+ try std.testing.expectEqualStrings("brave, tavily, exa, then keenable", search_engine_order);
+ try std.testing.expectEqualStrings("BRAVE_API_KEY, TAVILY_API_KEY, EXA_API_KEY or KEENABLE_API_KEY", search_key_env_vars);
+}
+
+test "writeResultItem uses the URL as title when the title is empty" {
+ var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);
+ defer arena.deinit();
+ var aw: std.Io.Writer.Allocating = .init(arena.allocator());
+ try writeResultItem(&aw.writer, 0, "", "https://example.org/x.pdf", "snippet");
+ try std.testing.expectEqualStrings("1. **https://example.org/x.pdf** — https://example.org/x.pdf\n snippet\n\n", aw.written());
+}
+
test "formatKeenableMarkdown handles empty results" {
var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);
defer arena.deinit();