Skip to content

Lightpanda: the headless browser designed for AI and automation

Merge branch 'main' into brave-search

Adrià Arrufatbae9388f1ce3
diff --cc build.zig.zon
index c2e50cc14,725fbed4d..17648a3b2
--- a/build.zig.zon
+++ b/build.zig.zon
@@@ -31,12 -32,12 +32,12 @@@
.hash = "N-V-__8AAHM9RQHjzJo8Wn4dbGPqNOC21zZJJA9PcCj7FPF5",
},
.sqlite3 = .{
- .url = "https://github.com/allyourcodebase/sqlite3/archive/8f840560eae88ab66668c6827c64ffbd0d74ef37.tar.gz",
- .hash = "sqlite3-3.51.0-DMxLWssOAABZ8cAvU_LfBIbp0kZjm824PU8sSLXpEDdr",
+ .url = "https://github.com/allyourcodebase/sqlite3/archive/7a615f5af79009cd733e3480658586d9b0d28b35.tar.gz",
+ .hash = "sqlite3-3.53.2-DMxLWuAOAAA_Px0arJOIOaP4AKEu5prbsQgPMA35W1zz",
},
.zenai = .{
- .url = "git+https://github.com/lightpanda-io/zenai.git#31da07c1fe9805267ab4dbe4ce41c0cc2d16a36a",
- .hash = "zenai-0.0.0-iOY_VGnDBQDS-bBNsgXuHhwEMqTh9Q7bQuP0BKziOdrR",
- .url = "git+https://github.com/lightpanda-io/zenai.git#00c793fdd2797ae5fc9700c189d8617f167ae0dc",
- .hash = "zenai-0.0.0-iOY_VGOnBQDaWE_Gn8cwhh0XXE8RZ8ESE7QtcITF-P73",
++ .url = "git+https://github.com/lightpanda-io/zenai.git#cab4242b2c498d18e5637a97045f8a51a74e1c32",
++ .hash = "zenai-0.0.0-iOY_VBbNBQCo_e6zGGSUX5yyB2UYmx0dy3XuVpgO6h2n",
},
.isocline = .{
.url = "git+https://github.com/arrufat/isocline?ref=lightpanda#832a9fe25f5f4458fcc47b5acc7c21db669c2f47",
diff --cc src/agent/Agent.zig
index ab2a3868d,c2481a919..fcd62388a
--- a/src/agent/Agent.zig
+++ b/src/agent/Agent.zig
@@@ -354,7 -353,7 +354,7 @@@ pub fn init(allocator: std.mem.Allocato
try self.startSession();
- self.ai_client = if (llm) |l| try zenai.provider.Client.init(allocator, l, .{ .base_url = opts.base_url, .retry_policy = .long_running, .bill_to = hfBillTo(l.provider) }) else null;
- self.ai_client = if (llm) |l| try zenai.provider.Client.init(allocator, lp.io, l, .{ .base_url = opts.base_url, .retry_policy = .long_running, .bill_to = hfBillTo(l.provider), .environ = lp.environ() }) else null;
++ self.ai_client = if (llm) |l| try zenai.provider.Client.init(lp.io, allocator, l, .{ .base_url = opts.base_url, .retry_policy = .long_running, .bill_to = hfBillTo(l.provider), .environ = lp.environ() }) else null;
errdefer if (self.ai_client) |c| c.deinit(allocator);
if (self.ai_client) |c| c.setInterrupt(&self.http_interrupt);
@@@ -762,21 -760,6 +762,21 @@@ fn handleStream(self: *Agent, rest: []c
self.reportSaved("stream", if (on) "on" else "off");
}
+/// `/searchEngine`: bare prints the current engine; an argument sets and
+/// persists it, warning when the chosen engine's API key is unset.
+fn handleSearchEngine(self: *Agent, rest: []const u8) void {
+ self.setEnumOption("searchEngine", &browser_tools.search_engine, rest);
+ const selected = std.meta.stringToEnum(browser_tools.SearchEngine, rest) orelse return;
- const env_var: []const u8 = switch (selected) {
++ const env_var: [:0]const u8 = switch (selected) {
+ .tavily => "TAVILY_API_KEY",
+ .brave => "BRAVE_API_KEY",
+ .auto, .duckduckgo => return,
+ };
- if (std.posix.getenv(env_var) == null) {
++ if (std.c.getenv(env_var) == null) {
+ self.terminal.printWarning("{s} is not set; the search tool will fail until you export it", .{env_var});
+ }
+}
+
/// Print cumulative session token usage, broken down so the cache's effect is
/// visible — the REPL otherwise never surfaces the `$usage` line `--task`
/// prints. Reads `total_usage` (accumulated per turn by `processUserMessage`);
@@@ -984,7 -968,7 +985,7 @@@ fn hfBillTo(provider: Config.AiProvider
/// `owned_key` transfers ownership of an allocated `credentials.key` (Vertex
/// gcloud token) on success; on error the caller still owns it.
fn setProvider(self: *Agent, credentials: Credentials, owned_key: ?[:0]const u8) !void {
- const new_client = try zenai.provider.Client.init(self.allocator, credentials, .{ .base_url = self.model_base_url, .retry_policy = .long_running, .bill_to = hfBillTo(credentials.provider) });
- const new_client = try zenai.provider.Client.init(self.allocator, lp.io, credentials, .{ .base_url = self.model_base_url, .retry_policy = .long_running, .bill_to = hfBillTo(credentials.provider), .environ = lp.environ() });
++ const new_client = try zenai.provider.Client.init(lp.io, self.allocator, credentials, .{ .base_url = self.model_base_url, .retry_policy = .long_running, .bill_to = hfBillTo(credentials.provider), .environ = lp.environ() });
errdefer new_client.deinit(self.allocator);
// A same-provider re-select (vertex token refresh) must not reset the model.
@@@ -1864,7 -1845,7 +1866,7 @@@ pub fn listModels(allocator: std.mem.Al
var arena: std.heap.ArenaAllocator = .init(allocator);
defer arena.deinit();
- const ids = zenai.provider.listChatModelIds(allocator, arena.allocator(), llm.provider, llm.key, opts.base_url) catch |err| {
- const ids = zenai.provider.listChatModelIds(allocator, lp.io, arena.allocator(), llm.provider, llm.key, opts.base_url, lp.environ()) catch |err| {
++ const ids = zenai.provider.listChatModelIds(lp.io, allocator, arena.allocator(), llm.provider, llm.key, .{ .base_url = opts.base_url, .environ = lp.environ() }) catch |err| {
if (llm.provider == .vertex and !settings.vertexProjectMode()) {
std.debug.print("Vertex express mode cannot list models (the endpoint requires OAuth); set GOOGLE_CLOUD_PROJECT for project mode.\n", .{});
}
@@@ -1921,11 -1902,13 +1923,12 @@@ fn completionModels(context: *anyopaque
_ = self.model_completion_arena.reset(.retain_capacity);
const ids = zenai.provider.listChatModelIds(
- self.allocator,
+ lp.io,
+ self.allocator,
self.model_completion_arena.allocator(),
llm.provider,
llm.key,
-- self.model_base_url,
- lp.environ(),
++ .{ .base_url = self.model_base_url, .environ = lp.environ() },
) catch &.{};
self.model_completions = .{ .provider = llm.provider, .ids = ids };
return ids;
diff --cc src/agent/settings.zig
index 1908305df,f819758dd..d7d3bcae1
--- a/src/agent/settings.zig
+++ b/src/agent/settings.zig
@@@ -47,10 -47,10 +47,10 @@@ pub const ResolvedProvider = struct
/// placeholder, so the only honest availability signal is the server answering
/// `/v1/models` with a loaded model. Null means no server responded.
pub fn detectLocalProvider(allocator: std.mem.Allocator, tag: Config.AiProvider, base_url: ?[:0]const u8) ?Credentials {
- const key = zenai.provider.envApiKey(tag) orelse return null;
+ const key = zenai.provider.envApiKey(lp.environ(), tag) orelse return null;
var arena: std.heap.ArenaAllocator = .init(allocator);
defer arena.deinit();
- const ids = zenai.provider.listChatModelIds(allocator, arena.allocator(), tag, key, base_url) catch return null;
- const ids = zenai.provider.listChatModelIds(allocator, lp.io, arena.allocator(), tag, key, base_url, lp.environ()) catch return null;
++ const ids = zenai.provider.listChatModelIds(lp.io, allocator, arena.allocator(), tag, key, .{ .base_url = base_url, .environ = lp.environ() }) catch return null;
if (ids.len == 0) return null;
return .{ .provider = tag, .key = key };
}
@@@ -306,7 -302,7 +310,7 @@@ pub fn reconcileModel
) !ReconciledModel {
var arena: std.heap.ArenaAllocator = .init(allocator);
defer arena.deinit();
- const ids: []const []const u8 = zenai.provider.listChatModelIds(allocator, arena.allocator(), llm.provider, llm.key, base_url) catch &.{};
- const ids: []const []const u8 = zenai.provider.listChatModelIds(allocator, lp.io, arena.allocator(), llm.provider, llm.key, base_url, lp.environ()) catch &.{};
++ const ids: []const []const u8 = zenai.provider.listChatModelIds(lp.io, allocator, arena.allocator(), llm.provider, llm.key, .{ .base_url = base_url, .environ = lp.environ() }) catch &.{};
if (ids.len == 0 or string.isOneOf(desired, ids)) return .{ .use = try allocator.dupe(u8, desired) };
if (!explicit) {
diff --cc src/browser/tools.zig
index 1d53a66c5,6e295d8a1..c877685b9
--- a/src/browser/tools.zig
+++ b/src/browser/tools.zig
@@@ -957,28 -950,16 +957,28 @@@ fn execSearch(arena: std.mem.Allocator
const args = try parseArgs(SearchParams, arena, arguments);
if (args.query.len == 0) return ToolError.InvalidParams;
- // Tavily path: only when TAVILY_API_KEY is set in the process env. On any
- // failure (network, non-2xx, parse) fall through to the DuckDuckGo scrape
- // so a single Tavily outage doesn't kill a whole benchmark run.
- if (std.c.getenv("TAVILY_API_KEY")) |api_key_z| {
- const api_key = std.mem.span(api_key_z);
- if (tavilySearch(arena, api_key, args.query)) |markdown_| {
- return markdown_;
- } else |err| {
- log.warn(.browser, "tavily fallback", .{ .err = err });
- }
+ switch (search_engine) {
+ // Any failure (network, non-2xx, parse) falls through to the next
+ // engine so a single outage doesn't kill a whole benchmark run.
+ .auto => {
- if (std.posix.getenv("BRAVE_API_KEY")) |api_key| {
- if (braveSearch(arena, api_key, args.query)) |markdown_| {
++ if (std.c.getenv("BRAVE_API_KEY")) |api_key_z| {
++ if (braveSearch(arena, std.mem.span(api_key_z), args.query)) |markdown_| {
+ return .{ .text = markdown_ };
+ } else |err| {
+ log.warn(.browser, "brave fallback", .{ .err = err });
+ }
+ }
- if (std.posix.getenv("TAVILY_API_KEY")) |api_key| {
- if (tavilySearch(arena, api_key, args.query)) |markdown_| {
++ if (std.c.getenv("TAVILY_API_KEY")) |api_key_z| {
++ if (tavilySearch(arena, std.mem.span(api_key_z), args.query)) |markdown_| {
+ return .{ .text = markdown_ };
+ } else |err| {
+ log.warn(.browser, "tavily fallback", .{ .err = err });
+ }
+ }
+ },
+ .tavily => return searchExplicit(arena, "tavily", "TAVILY_API_KEY", tavilySearch, args.query),
+ .brave => return searchExplicit(arena, "brave", "BRAVE_API_KEY", braveSearch, args.query),
+ .duckduckgo => {},
}
const encoded = lp.URL.percentEncodeSegment(arena, args.query, .component) catch return ToolError.OutOfMemory;
@@@ -990,28 -971,7 +990,28 @@@
) catch return ToolError.OutOfMemory;
_ = try performGoto(session, registry, ddg_url, args.timeout);
const ddg_frame = try requireFrame(session);
- return renderFrameMarkdown(arena, ddg_frame);
+ return .{ .text = try renderFrameMarkdown(arena, ddg_frame) };
+}
+
+/// Search with an explicitly selected engine: a missing key or a failed call
+/// comes back as an error result — no DDG fallback.
+fn searchExplicit(
+ arena: std.mem.Allocator,
+ comptime label: []const u8,
- comptime env_var: []const u8,
++ comptime env_var: [:0]const u8,
+ searchFn: anytype,
+ query: []const u8,
+) ToolError!ToolResult {
- const api_key = std.posix.getenv(env_var) orelse return .{
++ const api_key_z = std.c.getenv(env_var) orelse return .{
+ .text = "web search engine is set to " ++ label ++ " but " ++ env_var ++ " is not set in the environment",
+ .is_error = true,
+ };
- const markdown_ = searchFn(arena, api_key, query) catch |err| {
++ const markdown_ = searchFn(arena, std.mem.span(api_key_z), query) catch |err| {
+ const msg = std.fmt.allocPrint(arena, label ++ " search failed: {s}", .{@errorName(err)}) catch
+ return ToolError.OutOfMemory;
+ return .{ .text = msg, .is_error = true };
+ };
+ return .{ .text = markdown_ };
}
/// Thin wrapper over `zenai.search.tavily.Client` that handles client
@@@ -1022,7 -982,7 +1022,7 @@@ fn tavilySearch
api_key: []const u8,
query: []const u8,
) ![]const u8 {
- var client: tavily.Client = .init(arena, api_key, .{});
- var client: tavily.Client = .init(arena, lp.io, api_key, .{});
++ var client: tavily.Client = .init(lp.io, arena, api_key, .{});
defer client.deinit();
var response = client.search(query, .{ .max_results = 10 }) catch |err| {
@@@ -1056,45 -1016,6 +1056,45 @@@ fn formatTavilyMarkdown(arena: std.mem.
return aw.written();
}
+/// Thin wrapper over `zenai.search.brave.Client` that handles client
+/// lifetime and renders the structured response as markdown for the agent.
+/// `arena` owns the returned slice. `api_key` is the value of BRAVE_API_KEY.
+fn braveSearch(
+ arena: std.mem.Allocator,
+ api_key: []const u8,
+ query: []const u8,
+) ![]const u8 {
- var client: brave.Client = .init(arena, api_key, .{});
++ var client: brave.Client = .init(lp.io, arena, api_key, .{});
+ defer client.deinit();
+
+ // text_decorations=false: no <strong> markup in model-read snippets.
+ var response = client.search(query, .{ .count = 10, .text_decorations = false }) catch |err| {
+ if (client.last_error_status) |status| {
+ log.warn(.browser, "brave non-2xx", .{
+ .status = status,
+ .body = client.last_error_body orelse "",
+ });
+ }
+ return err;
+ };
+ defer response.deinit();
+
+ return formatBraveMarkdown(arena, response.value);
+}
+
+fn formatBraveMarkdown(arena: std.mem.Allocator, resp: brave.types.SearchResponse) ![]const u8 {
+ var aw: std.Io.Writer.Allocating = .init(arena);
+ const w = &aw.writer;
+ const results: []const brave.types.Result = if (resp.web) |web| web.results else &.{};
+ for (results, 0..) |r, i| {
+ try w.print("{d}. **{s}** — {s}\n {s}\n\n", .{ i + 1, r.title, r.url, r.description });
+ }
+ if (results.len == 0) {
+ try w.writeAll("No results.");
+ }
+ return aw.written();
+}
+
fn renderFrameMarkdown(arena: std.mem.Allocator, frame: *lp.Frame) ToolError![]const u8 {
var aw: std.Io.Writer.Allocating = .init(arena);
lp.markdown.dump(frame.document.asNode(), .{}, &aw.writer, frame) catch
diff --cc src/network/Network.zig
index 3b5d27b90,3a9f9cf2f..868e82680
--- a/src/network/Network.zig
+++ b/src/network/Network.zig
@@@ -388,7 -393,12 +393,12 @@@ fn dropCdp(self: *Network, link: *CdpLi
self.cdp_links.remove(&link.node);
link.state = .removed;
self.cdp_dirty = true;
- if (notify) {
+
+ if (opts.shutdown_socket) {
- posix.shutdown(link.socket, .both) catch {};
++ sys_net.shutdown(link.socket, .both) catch {};
+ }
+
+ if (opts.notify) {
link.cdp.terminateFromNetwork();
// notify=true means the worker hasn't been told yet — push the