Lightpanda: the headless browser designed for AI and automation
chore: Network cleanup
Karl Seguindecedca6e7bf
diff --git a/src/Updater.zig b/src/Updater.zigindex 837c8e42d..2c753a6a0 100644--- a/src/Updater.zig+++ b/src/Updater.zig@@ -19,70 +19,31 @@const std = @import("std");const lp = @import("lightpanda");-const Network = @import("network/Network.zig");const http = @import("network/http.zig");+const Certificates = @import("network/Certificates.zig");const libcurl = @import("sys/libcurl.zig");-const crypto = @import("sys/libcrypto.zig");-const Config = @import("Config.zig");const Allocator = std.mem.Allocator;-/// Sole purpose of this client is to do updates; hence, its very minimal.-const Updater = @This();-x509_store: *crypto.X509_STORE,-config: *const Config,--/// Initializes the update client; meant to be used as singleton.-pub fn init(allocator: Allocator, config: *const Config) !Updater {- Network.globalInit(allocator);- errdefer Network.globalDeinit();- const x509_store = try Network.prepareX509Store(allocator, config);+/// Sends running Lightpanda version to remote to get update information.+/// Outputs directly to given `Writer`.+pub fn inform(allocator: Allocator, config: *const lp.Config, writer: *std.Io.Writer) !void {+ const certificates = try Certificates.init(allocator, config);+ defer certificates.deinit();- return .{- .x509_store = x509_store,- .config = config,+ libcurl.curl_global_init(.{ .ssl = true }, null) catch |err| {+ lp.assert(false, "curl global init", .{ .err = err });};-}+ defer libcurl.curl_global_cleanup();-pub fn deinit(self: *Updater) void {- Network.globalDeinit();- crypto.X509_STORE_free(self.x509_store);-}--/// Sends running Lightpanda version to remote to get update information.-/// Outputs directly to given `Writer`.-pub fn inform(self: *Updater, writer: *std.Io.Writer) !void {- var conn = try http.Connection.init(self.x509_store, self.config, null);+ var conn = try http.Connection.init(certificates, config, null);defer conn.deinit();- const url = std.fmt.comptimePrint("https://telemetry.lightpanda.io/v/{s}", .{lp.build_config.version});- // Prepare the request.- try conn.setURL(url);+ try conn.setURL("https://telemetry.lightpanda.io/v/" ++ lp.build_config.version);try conn.setGetMode();try conn.setFollowLocation(true);// Wraps everything needed to receive bytes.- const ReceiverContext = struct {- writer: *std.Io.Writer,- err: std.Io.Writer.Error!void = {},-- /// curl -> writer.- fn drain(- buffer: [*]const u8,- buf_count: usize,- buf_len: usize,- raw_ctx: *anyopaque,- ) callconv(.c) usize {- const ctx: *@This() = @ptrCast(@alignCast(raw_ctx));- const chunk = buffer[0 .. buf_count * buf_len];- ctx.writer.writeAll(chunk) catch |err| {- ctx.err = err;- return 0;- };-- return chunk.len;- }- };// Set receiver context.var ctx = ReceiverContext{ .writer = writer };@@ -97,3 +58,22 @@ pub fn inform(self: *Updater, writer: *std.Io.Writer) !void {_ = status_int;return writer.flush();}++const ReceiverContext = struct {+ writer: *std.Io.Writer,+ err: std.Io.Writer.Error!void = {},++ fn drain(buffer: [*]const u8, buf_count: usize, buf_len: usize, ctx: *anyopaque) callconv(.c) usize {+ // libcurl only ever sends 1 buffer+ std.debug.assert(buf_count == 1);++ const self: *ReceiverContext = @ptrCast(@alignCast(ctx));+ const chunk = buffer[0..buf_len];+ self.writer.writeAll(chunk) catch |err| {+ self.err = err;+ return 0;+ };++ return buf_len;+ }+};diff --git a/src/browser/Browser.zig b/src/browser/Browser.zigindex f4cab1c65..3302a48bd 100644--- a/src/browser/Browser.zig+++ b/src/browser/Browser.zig@@ -153,9 +153,6 @@ pub fn deinit(self: *Browser) void {// fire — only now is it safe to free the pool backing their parameters.self.fc_identity_pool.deinit(allocator);self.page_pool.deinit(allocator);- if (self.http_client.cache) |cache| {- cache.maintenance(lp.datetime.timestamp(.real));- }self.http_client.deinit();self.clearPermissions();self.permissions.deinit(allocator);diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zigindex 99bf6cb37..c5fd99de7 100644--- a/src/cdp/domains/network.zig+++ b/src/cdp/domains/network.zig@@ -216,14 +216,14 @@ fn clearBrowserCache(cmd: *CDP.Command) !void {// Chrome accepts that and clears the jar; reject only on truly malformed JSON.const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;const network = bc.cdp.browser.http_client.network;- if (network.cache) |*c| try c.clear();+ try network.cache.clear();return cmd.sendResult(null, .{});}fn canClearBrowserCache(cmd: *CDP.Command) !void {const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;const network = bc.cdp.browser.http_client.network;- return cmd.sendResult(.{ .result = network.cache != null }, .{});+ return cmd.sendResult(.{ .result = network.cache.active() != null }, .{});}fn clearBrowserCookies(cmd: *CDP.Command) !void {@@ -1030,7 +1030,7 @@ test "cdp.Network: setCacheDisabled" {.params = .{ .cacheDisabled = true },});try ctx.expectSentResult(null, .{ .id = 1 });- try testing.expect(client.cache == null);+ try testing.expectEqual(null, client.cache.active());}test "cdp.Network: configured CDP ignores setCacheDisabled" {@@ -1041,7 +1041,7 @@ test "cdp.Network: configured CDP ignores setCacheDisabled" {var cache: Cache = undefined;const client = &ctx.cdp().browser.http_client;client.cache = &cache;- defer client.cache = null;+ defer client.cache = &testing.base.test_app.network.cache;try ctx.processMessage(.{.id = 1,diff --git a/src/cdp/testing.zig b/src/cdp/testing.zigindex 8133f32bb..fd9edb2fe 100644--- a/src/cdp/testing.zig+++ b/src/cdp/testing.zig@@ -20,7 +20,7 @@ const std = @import("std");const CDP = @import("CDP.zig");-const base = @import("../testing.zig");+pub const base = @import("../testing.zig");const json = std.json;const posix = std.posix;diff --git a/src/lightpanda.zig b/src/lightpanda.zigindex f6fcd03c8..a8337ab2a 100644--- a/src/lightpanda.zig+++ b/src/lightpanda.zig@@ -60,8 +60,6 @@ pub const build_config = @import("build_config");pub const crash_handler = @import("crash_handler.zig");pub const core_dump = @import("core_dump.zig");-pub const Updater = @import("Updater.zig");-pub var metrics = @import("Metrics.zig"){};pub const IS_TEST = @import("builtin").is_test;@@ -395,14 +393,10 @@ fn dumpContent(app: *App, mode: Config.DumpFormat, dump_opts: dump.Opts, frame:}pub fn checkVersion(allocator: std.mem.Allocator, config: *const Config) !void {- var client = try Updater.init(allocator, config);- defer client.deinit();-const stdout = std.Io.File.stdout();var buf: [4096]u8 = undefined;var writer = stdout.writer(io, &buf);- const w = &writer.interface;- try client.inform(w);+ try @import("Updater.zig").inform(allocator, config, &writer.interface);}// Writes a single page's result object. Framing (the enclosing array and anydiff --git a/src/network/Certificates.zig b/src/network/Certificates.zignew file mode 100644index 000000000..b85658cc2--- /dev/null+++ b/src/network/Certificates.zig@@ -0,0 +1,139 @@+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)+//+// Francis Bouvier <[email protected]>+// Pierre Tachoire <[email protected]>+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++const std = @import("std");+const lp = @import("lightpanda");++const crypto = @import("../sys/libcrypto.zig");++const log = lp.log;+const Config = lp.Config;+const Allocator = std.mem.Allocator;++const Certificates = @This();++store: *crypto.X509_STORE,++pub fn init(allocator: Allocator, config: *const Config) !Certificates {+ const store = blk: {+ const custom_store = config.customCertStore();+ if (config.tlsVerifyHost() == false) {+ if (custom_store != null) {+ log.warn(.app, "custom CA ignored", .{+ .arg = "--ca-cert, --ca-path",+ .reason = "TLS verification disabled",+ });+ }+ break :blk crypto.X509_STORE_new() orelse return error.FailedToCreateX509Store;+ }+ break :blk custom_store orelse try storeFromSystemCA(allocator);+ };++ return .{ .store = store };+}++pub fn deinit(self: Certificates) void {+ crypto.X509_STORE_free(self.store);+}++fn storeFromSystemCA(allocator: Allocator) !*crypto.X509_STORE {+ const store = crypto.X509_STORE_new() orelse return error.FailedToCreateX509Store;+ errdefer crypto.X509_STORE_free(store);++ var count: usize = 0;+ defer {+ if (count == 0) {+ log.warn(.app, "No certificates loaded", .{});+ }+ }++ switch (comptime @import("builtin").os.tag) {+ .linux, .openbsd, .netbsd, .freebsd => blk: {+ // Iterate over known directories; this may or may not succeed.+ const cwd = std.Io.Dir.cwd();+ inline for ([_][]const u8{+ "/etc/ssl/certs", // Debian/Ubuntu/Gentoo/Alpine, SUSE+ "/etc/pki/tls/certs", // Fedora/RHEL+ }) |dir_path| {+ count += try loadFromDirectory(allocator, store, cwd, dir_path);+ if (count > 0) break :blk;+ }++ // Iterate over known files.+ inline for ([_][*:0]const u8{+ "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo+ "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6+ "/etc/ssl/ca-bundle.pem", // OpenSUSE+ "/etc/pki/tls/cacert.pem", // OpenELEC+ "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7+ "/etc/ssl/cert.pem", // Alpine, *BSD+ }) |file| {+ if (crypto.X509_STORE_load_locations(store, file, null) == 1) {+ count += 1;+ break :blk;+ }+ }+ },+ else => {+ // Prefer stdlib's cert scanner.+ var bundle: std.crypto.Certificate.Bundle = .empty;+ try bundle.rescan(allocator, lp.io, std.Io.Clock.now(.real, lp.io));+ defer bundle.deinit(allocator);++ const bytes = bundle.bytes.items;+ var it = bundle.map.valueIterator();+ while (it.next()) |index| {+ // d2i_X509 reads the cert's own DER length header to find its end and+ // advances `ptr` past it, so we just hand it the rest of the buffer.+ var ptr: [*]const u8 = bytes.ptr + index.*;+ const x509 = crypto.d2i_X509(null, &ptr, @intCast(bytes.len - index.*)) orelse {+ log.warn(.app, "Skipping unparseable system cert", .{});+ continue;+ };+ defer crypto.X509_free(x509); // add_cert takes its own ref; drop ours.++ const result = crypto.X509_STORE_add_cert(store, x509);+ if (result != 1) {+ log.warn(.app, "Failed to add X509 cert to store", .{});+ }+ count += 1;+ }+ },+ }++ return store;+}++fn loadFromDirectory(allocator: Allocator, store: *crypto.X509_STORE, cwd: std.Io.Dir, dir_path: []const u8) !usize {+ var count: usize = 0;+ var dir = cwd.openDir(lp.io, dir_path, .{ .iterate = true }) catch return count;+ defer dir.close(lp.io);++ var it = dir.iterate();+ while (it.next(lp.io) catch return count) |entry| {+ if (entry.kind != .file and entry.kind != .sym_link) continue;++ const path = try std.fs.path.joinZ(allocator, &.{ dir_path, entry.name });+ defer allocator.free(path);++ if (crypto.X509_STORE_load_locations(store, path, null) == 1) {+ count += 1;+ }+ }+ return count;+}diff --git a/src/network/CurlDebugAllocator.zig b/src/network/CurlDebugAllocator.zigdeleted file mode 100644index 7fc700e54..000000000--- a/src/network/CurlDebugAllocator.zig+++ /dev/null@@ -1,134 +0,0 @@-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)-//-// Francis Bouvier <[email protected]>-// Pierre Tachoire <[email protected]>-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--const std = @import("std");-const lp = @import("lightpanda");--const libcurl = @import("../sys/libcurl.zig");--const Allocator = std.mem.Allocator;--const CurlDebugAllocator = @This();--// C11 requires malloc to return memory aligned to max_align_t (16 bytes on x86_64).-// We match this guarantee since libcurl expects malloc-compatible alignment.-const alignment = 16;--var instance: ?CurlDebugAllocator = null;--allocator: Allocator,--pub fn init(allocator: Allocator) void {- lp.assert(instance == null, "Initialization of curl must happen only once", .{});- instance = .{ .allocator = allocator };-}--pub fn interface() libcurl.CurlAllocator {- return .{- .free = free,- .strdup = strdup,- .malloc = malloc,- .calloc = calloc,- .realloc = realloc,- };-}--fn _allocBlock(size: usize) ?*Block {- const slice = instance.?.allocator.alignedAlloc(u8, .fromByteUnits(alignment), Block.fullsize(size)) catch return null;- const block: *Block = @ptrCast(@alignCast(slice.ptr));- block.size = size;- return block;-}--fn _freeBlock(header: *Block) void {- instance.?.allocator.free(header.slice());-}--fn malloc(size: usize) ?*anyopaque {- const block = _allocBlock(size) orelse return null;- return @ptrCast(block.data());-}--fn calloc(nmemb: usize, size: usize) ?*anyopaque {- const total = nmemb * size;- const block = _allocBlock(total) orelse return null;- const ptr = block.data();- @memset(ptr[0..total], 0); // for historical reasons, calloc zeroes memory, but malloc does not.- return @ptrCast(ptr);-}--fn realloc(ptr: ?*anyopaque, size: usize) ?*anyopaque {- const p = ptr orelse return malloc(size);- const block = Block.fromPtr(p);-- const old_size = block.size;- if (size == old_size) return ptr;-- if (instance.?.allocator.resize(block.slice(), alignment + size)) {- block.size = size;- return ptr;- }-- const copy_size = @min(old_size, size);- const new_block = _allocBlock(size) orelse return null;- @memcpy(new_block.data()[0..copy_size], block.data()[0..copy_size]);- _freeBlock(block);- return @ptrCast(new_block.data());-}--fn free(ptr: ?*anyopaque) void {- const p = ptr orelse return;- _freeBlock(Block.fromPtr(p));-}--fn strdup(str: [*:0]const u8) ?[*:0]u8 {- const len = std.mem.len(str);- const header = _allocBlock(len + 1) orelse return null;- const ptr = header.data();- @memcpy(ptr[0..len], str[0..len]);- ptr[len] = 0;- return ptr[0..len :0];-}--const Block = extern struct {- size: usize = 0,- _padding: [alignment - @sizeOf(usize)]u8 = .{0} ** (alignment - @sizeOf(usize)),-- inline fn fullsize(bytes: usize) usize {- return alignment + bytes;- }-- inline fn fromPtr(ptr: *anyopaque) *Block {- const raw: [*]u8 = @ptrCast(ptr);- return @ptrCast(@alignCast(raw - @sizeOf(Block)));- }-- inline fn data(self: *Block) [*]u8 {- const ptr: [*]u8 = @ptrCast(self);- return ptr + @sizeOf(Block);- }-- inline fn slice(self: *Block) []align(alignment) u8 {- const base: [*]align(alignment) u8 = @ptrCast(@alignCast(self));- return base[0 .. alignment + self.size];- }-};--comptime {- std.debug.assert(@sizeOf(Block) == alignment);-}diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 041b00f64..06e437184 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -181,8 +181,7 @@ blocking_requests: std.AutoHashMapUnmanaged(u32, u32) = .empty,// heuristics add this in.intercepted: usize = 0,-// null or referencing network.cache-cache: ?*Cache,+cache: *Cache,// Cached config decisions, resolved once at init.serve_mode: bool,@@ -218,7 +217,7 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP).allocator = allocator,.cdp = cdp,.inbox = .{},- .cache = if (network.cache) |*c| c else null,+ .cache = &network.cache,.use_proxy = http_proxy != null,.http_proxy = http_proxy,@@ -265,6 +264,7 @@ pub fn deinit(self: *Client) void {self.blocking_requests.deinit(self.allocator);self.transfers.deinit(self.allocator);self.inbox.deinit();+ self.cache.maintenance(lp.datetime.timestamp(.real));}// Look up a live transfer by its id. Returns null if the transfer has been@@ -317,9 +317,9 @@ pub fn obeyRobots(self: *Client, enable: bool) !void {pub fn disableCache(self: *Client, disable: bool) void {if (disable) {- self.cache = null;+ self.cache = &Cache.noop;} else {- self.cache = if (self.network.cache) |*c| c else null;+ self.cache = &self.network.cache;}}@@ -984,7 +984,7 @@ fn findHeader(headers: []const http.Header, name: []const u8) ?[]const u8 {// the response; on an expired-with-validators entry the request becomes a// conditional revalidation.fn cacheLookup(self: *Client, transfer: *Transfer) !bool {- const cache = self.cache orelse return false;+ const cache = self.cache.active() orelse return false;const req = &transfer.req;if (req.method != .GET or req.streaming or req.skip_cache) {@@ -1067,7 +1067,7 @@ fn cacheRevalidated(self: *Client, transfer: *Transfer) !bool {return false;}// could have been disabled in-between- const cache = self.cache orelse return false;+ const cache = self.cache.active() orelse return false;const stale = transfer._cache_intent.revalidate;transfer._cache_intent = .none;@@ -1103,7 +1103,7 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {transfer._cache_intent = .none;// could have been disabled while waiting of the response- const cache = self.cache orelse return;+ const cache = self.cache.active() orelse return;const arena = transfer.arena;const rh = &(transfer.res.header orelse return);@@ -3582,7 +3582,7 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void {client.intercepted = 0;client.http_active = 0;client.ws_active = 0;- client.cache = null;+ client.cache = &Cache.noop;client.serve_mode = false;client.obey_robots = false;client.robots = .{@@ -3951,7 +3951,7 @@ test "HttpClient: fulfillIntercepted follows a 3xx redirect" {// Only network.config (httpMaxRedirects, which ignores its config),// network.cache and the (empty) connection pool are read on this path.var net: Network = undefined;- net.cache = null;+ net.cache = Cache.noop;net.adblocker = null;// An empty pool makes processTransfer queue the re-issued request// instead of putting it on the wire — the queue IS the capture.@@ -4309,7 +4309,7 @@ test "HttpClient: throttled navigations wait for their per-host slot" {defer pool.deinit();var net: Network = undefined;- net.cache = null;+ net.cache = Cache.noop;net.adblocker = null;net.web_bot_auth = null;// An empty pool makes processTransfer queue a started transfer insteaddiff --git a/src/network/Network.zig b/src/network/Network.zigindex 4087f2fda..2c532d010 100644--- a/src/network/Network.zig+++ b/src/network/Network.zig@@ -26,17 +26,15 @@ const Config = @import("../Config.zig");const CDP = @import("../cdp/CDP.zig");const sys_net = @import("../sys/net.zig");const libcurl = @import("../sys/libcurl.zig");-const crypto = @import("../sys/libcrypto.zig");const http = @import("http.zig");const IpFilter = @import("IpFilter.zig");const RobotStore = @import("Robots.zig").RobotStore;const WebBotAuth = @import("WebBotAuth.zig");const RateLimiter = @import("RateLimiter.zig");-const CurlDebugAllocator = @import("CurlDebugAllocator.zig");+const Certificates = @import("Certificates.zig");const Cache = @import("cache/Cache.zig");-const SqliteCache = @import("cache/SqliteCache.zig");const AdBlocker = @import("adblock/AdBlocker.zig");const log = lp.log;@@ -86,13 +84,12 @@ const PSEUDO_POLLFDS = 2;allocator: Allocator,app: *App,-cache: ?Cache,+cache: Cache,config: *const Config,-/// Holds certificate bundle.-x509_store: *crypto.X509_STORE,robot_store: RobotStore,web_bot_auth: ?WebBotAuth,rate_limiter: ?RateLimiter,+certificates: Certificates,/// Hostname dictionaries built from `--adblock-lists`. Parsed once here and/// never mutated afterwards, so every HttpClient can share this one copy.adblocker: ?AdBlocker,@@ -143,31 +140,11 @@ cdp_start: usize,/// Optional IP filter for blocking requests to private/internal networks (--block-private-networks).ip_filter: ?*IpFilter = null,-/// Calling `init` also calls this function; only marked public for situations-/// networking is needed without `App`.-pub fn globalInit(allocator: Allocator) void {- // Only route curl's own allocations through our allocator in Debug, so the- // leak detector sees them. In Release it'd just wrap c_allocator (curl's- // default malloc anyway) at the cost of a per-allocation header.- const curl_allocator = comptime if (lp.IS_DEBUG) CurlDebugAllocator.interface() else null;- if (comptime lp.IS_DEBUG) {- CurlDebugAllocator.init(allocator);- }-- libcurl.curl_global_init(.{ .ssl = true }, curl_allocator) catch |err| {+pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {+ libcurl.curl_global_init(.{ .ssl = true }, null) catch |err| {lp.assert(false, "curl global init", .{ .err = err });};-}--/// Calling `deinit` also calls this function; only marked public for situations-/// networking is needed without `App`.-pub fn globalDeinit() void {- libcurl.curl_global_cleanup();-}--pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {- globalInit(allocator);- errdefer globalDeinit();+ errdefer libcurl.curl_global_cleanup();const pipe = try sys_net.pipe2(.{ .NONBLOCK = true, .CLOEXEC = true });@@ -186,22 +163,8 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {@memset(pollfds, .{ .fd = -1, .events = 0, .revents = 0 });pollfds[0] = .{ .fd = pipe[0], .events = posix.POLL.IN, .revents = 0 };- const x509_store = blk: {- if (config.tlsVerifyHost()) {- break :blk try prepareX509Store(allocator, config);- }- // Verification is off, so the store is never consulted — but still- // take ownership of a user-supplied one so the flags compose and- // nothing leaks.- if (config.customCertStore()) |store| {- log.warn(.app, "custom CA ignored", .{ .arg = "--ca-cert, --ca-path", .reason = "TLS verification disabled" });- break :blk store;- }- break :blk crypto.X509_STORE_new() orelse {- return error.FailedToCreateX509Store;- };- };- errdefer crypto.X509_STORE_free(x509_store);+ const certificates = try Certificates.init(allocator, config);+ errdefer certificates.deinit();// IP filter for blocking requests to private/internal networks.const block_private = config.blockPrivateNetworks();@@ -227,7 +190,7 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {var available: DoublyLinkedList = .{};for (0..count) |i| {- connections[i] = try http.Connection.init(x509_store, config, ip_filter);+ connections[i] = try http.Connection.init(certificates, config, ip_filter);available.append(&connections[i].node);}@@ -240,30 +203,14 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {var adblocker = try AdBlocker.fromConfig(allocator, config);errdefer if (adblocker) |*blocker| blocker.deinit();- const cache = if (config.httpCacheDir()) |cache_dir_path|- Cache{- .kind = .{- .sqlite = SqliteCache.init(- allocator,- .{ .path = cache_dir_path },- config.httpCacheEntryLimit(),- ) catch |e| {- log.err(.cache, "failed to init", .{- .kind = "SqliteCache",- .path = cache_dir_path,- .err = e,- });- return e;- },- },- }- else- null;+ var cache = try Cache.init(allocator, config);+ errdefer cache.deinit();return .{- .allocator = allocator,+ .app = app,.config = config,- .x509_store = x509_store,+ .allocator = allocator,+ .certificates = certificates,.pollfds = pollfds,.wakeup_pipe = pipe,@@ -273,8 +220,6 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {.available = available,.connections = connections,- .app = app,-.cache = cache,.robot_store = RobotStore.init(allocator),.web_bot_auth = web_bot_auth,@@ -299,7 +244,7 @@ pub fn deinit(self: *Network) void {self.allocator.free(self.pollfds);self.allocator.free(self.cdp_poll_snapshot);- crypto.X509_STORE_free(self.x509_store);+ self.certificates.deinit();for (self.connections) |*conn| {conn.deinit();@@ -318,14 +263,14 @@ pub fn deinit(self: *Network) void {if (self.adblocker) |*blocker| blocker.deinit();- if (self.cache) |*cache| cache.deinit();+ self.cache.deinit();if (self.ip_filter) |f| {f.deinit(self.allocator);self.allocator.destroy(f);}- globalDeinit();+ libcurl.curl_global_cleanup();}pub fn bind(@@ -720,7 +665,7 @@ pub fn releaseConnection(self: *Network, conn: *http.Connection) void {self.ws_count -= 1;},else => {- conn.reset(self.config, self.x509_store, self.ip_filter) catch |err| {+ conn.reset(self.config, self.certificates, self.ip_filter) catch |err| {lp.assert(false, "couldn't reset curl easy", .{ .err = err });};self.conn_mutex.lockUncancelable(lp.io);@@ -745,7 +690,7 @@ pub fn newConnection(self: *Network) ?*http.Connection {};// don't do this under lock- conn.* = http.Connection.init(self.x509_store, self.config, self.ip_filter) catch {+ conn.* = http.Connection.init(self.certificates, self.config, self.ip_filter) catch {self.ws_mutex.lockUncancelable(lp.io);defer self.ws_mutex.unlock(lp.io);self.ws_pool.destroy(conn);@@ -757,110 +702,6 @@ pub fn newConnection(self: *Network) ?*http.Connection {return conn;}-/// NEVER give full ownership of store to `SSL_CTX`, always rely on ref counting.-/// Allocations made through passed `allocator` are freed before this function returns.-pub fn prepareX509Store(allocator: Allocator, config: *const Config) !*crypto.X509_STORE {- // A user-supplied store replaces system trust entirely.- if (config.customCertStore()) |store| {- return store;- }- return storeFromSystemCA(allocator);-}--/// Creates an X509_STORE from system root CA.-fn storeFromSystemCA(allocator: Allocator) !*crypto.X509_STORE {- const store = crypto.X509_STORE_new() orelse return error.FailedToCreateX509Store;- errdefer crypto.X509_STORE_free(store);-- var count: usize = 0;- defer {- if (count == 0) {- log.warn(.app, "No certificates loaded", .{});- }- }-- switch (comptime builtin.os.tag) {- .linux, .openbsd, .netbsd, .freebsd => blk: {- // Iterate over known directories; this may or may not succeed.- const cwd = std.Io.Dir.cwd();- inline for ([_][]const u8{- "/etc/ssl/certs", // Debian/Ubuntu/Gentoo/Alpine, SUSE- "/etc/pki/tls/certs", // Fedora/RHEL- }) |dir_path| {- count += try loadFromDirectory(allocator, store, cwd, dir_path);- if (count > 0) break :blk;- }-- // Iterate over known files.- inline for ([_][*:0]const u8{- "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo- "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6- "/etc/ssl/ca-bundle.pem", // OpenSUSE- "/etc/pki/tls/cacert.pem", // OpenELEC- "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7- "/etc/ssl/cert.pem", // Alpine, *BSD- }) |file| {- if (crypto.X509_STORE_load_locations(store, file, null) == 1) {- count += 1;- break :blk;- }- }- },- else => {- // Prefer stdlib's cert scanner.- var bundle: std.crypto.Certificate.Bundle = .empty;- try bundle.rescan(allocator, lp.io, std.Io.Clock.now(.real, lp.io));- defer bundle.deinit(allocator);-- const bytes = bundle.bytes.items;- var it = bundle.map.valueIterator();- while (it.next()) |index| {- // d2i_X509 reads the cert's own DER length header to find its end and- // advances `ptr` past it, so we just hand it the rest of the buffer.- var ptr: [*]const u8 = bytes.ptr + index.*;- const x509 = crypto.d2i_X509(null, &ptr, @intCast(bytes.len - index.*)) orelse {- log.warn(.app, "Skipping unparseable system cert", .{});- continue;- };- defer crypto.X509_free(x509); // add_cert takes its own ref; drop ours.-- const result = crypto.X509_STORE_add_cert(store, x509);- if (result != 1) {- log.warn(.app, "Failed to add X509 cert to store", .{});- }- count += 1;- }- },- }-- return store;-}--/// Loads certificates from given `path`; returning how many CA loaded.-fn loadFromDirectory(- allocator: Allocator,- store: *crypto.X509_STORE,- cwd: std.Io.Dir,- dir_path: []const u8,-) Allocator.Error!usize {- var count: usize = 0;- var dir = cwd.openDir(lp.io, dir_path, .{ .iterate = true }) catch return count;- defer dir.close(lp.io);-- var it = dir.iterate();- while (it.next(lp.io) catch return count) |entry| {- if (entry.kind != .file and entry.kind != .sym_link) continue;-- const path = try std.fs.path.joinZ(allocator, &.{ dir_path, entry.name });- defer allocator.free(path);-- if (crypto.X509_STORE_load_locations(store, path, null) == 1) {- count += 1;- }- }- return count;-}-pub fn HostHashMap(comptime V: type) type {return std.HashMapUnmanaged([]const u8, V, HostContext, 80);}diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zigindex 71fd819c7..8b0565881 100644--- a/src/network/cache/Cache.zig+++ b/src/network/cache/Cache.zig@@ -18,9 +18,13 @@const std = @import("std");const lp = @import("lightpanda");+const Http = @import("../http.zig");+const SqliteCache = @import("SqliteCache.zig");+pub var noop: Cache = .{ .kind = .noop };+const log = lp.log;/// A browser-wide cache for resources across the network.@@ -28,47 +32,85 @@ const log = lp.log;pub const Cache = @This();kind: union(enum) {+ noop: void,sqlite: SqliteCache,},+pub fn init(allocator: std.mem.Allocator, config: *const lp.Config) !Cache {+ const cache_path = config.httpCacheDir() orelse {+ return .{ .kind = .noop };+ };++ const sqlite = SqliteCache.init(+ allocator,+ .{ .path = cache_path },+ config.httpCacheEntryLimit(),+ ) catch |err| {+ log.err(.cache, "failed to init", .{+ .kind = "SqliteCache",+ .path = cache_path,+ .err = err,+ });+ return err;+ };++ return .{+ .kind = .{ .sqlite = sqlite },+ };+}+pub fn deinit(self: *Cache) void {return switch (self.kind) {+ .noop => {},inline else => |*c| c.deinit(),};}+pub fn active(self: *Cache) ?*Cache {+ return switch (self.kind) {+ .noop => null,+ inline else => self,+ };+}+pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheGetRequest) !CacheGetResult {return switch (self.kind) {+ .noop => .miss,inline else => |*c| c.get(arena, req),};}pub fn put(self: *Cache, req: CachePutRequest, body: []const u8) !void {return switch (self.kind) {+ .noop => {},inline else => |*c| c.put(req, body),};}pub fn evict(self: *Cache, url: []const u8) void {return switch (self.kind) {+ .noop => {},inline else => |*c| c.evict(url),};}pub fn renew(self: *Cache, arena: std.mem.Allocator, req: RenewResponse) !void {return switch (self.kind) {+ .noop => {},inline else => |*c| c.renew(arena, req),};}pub fn clear(self: *Cache) !void {return switch (self.kind) {+ .noop => {},inline else => |*c| c.clear(),};}pub fn maintenance(self: *Cache, now: u64) void {return switch (self.kind) {+ .noop => {},inline else => |*c| c.maintenance(now),};}diff --git a/src/network/http.zig b/src/network/http.zigindex 11c601bbe..2442b5806 100644--- a/src/network/http.zig+++ b/src/network/http.zig@@ -17,7 +17,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.const std = @import("std");-const posix = std.posix;+const lp = @import("lightpanda");const Config = @import("../Config.zig");const sys_net = @import("../sys/net.zig");@@ -25,8 +25,10 @@ const libcurl = @import("../sys/libcurl.zig");const crypto = @import("../sys/libcrypto.zig");const IpFilter = @import("IpFilter.zig");+const Certificates = @import("Certificates.zig");-const log = @import("lightpanda").log;+const log = lp.log;+const posix = std.posix;pub const ENABLE_DEBUG = false;@@ -280,7 +282,7 @@ pub const Connection = struct {};pub fn init(- x509_store: *crypto.X509_STORE,+ certificates: Certificates,config: *const Config,ip_filter: ?*const IpFilter,) !Connection {@@ -289,7 +291,7 @@ pub const Connection = struct {var self = Connection{ ._easy = easy, .transport = .none };errdefer self.deinit();- try self.reset(config, x509_store, ip_filter);+ try self.reset(config, certificates, ip_filter);return self;}@@ -443,7 +445,7 @@ pub const Connection = struct {pub fn reset(self: *Connection,config: *const Config,- x509_store: *crypto.X509_STORE,+ certificates: Certificates,ip_filter: ?*const IpFilter,) !void {libcurl.curl_easy_reset(self._easy);@@ -487,7 +489,7 @@ pub const Connection = struct {}}).wrap);// Pass our store to CURLOPT_SSL_CTX_FUNCTION.- try libcurl.curl_easy_setopt(self._easy, .ssl_ctx_data, x509_store);+ try libcurl.curl_easy_setopt(self._easy, .ssl_ctx_data, certificates.store);} else {try libcurl.curl_easy_setopt(self._easy, .ssl_verify_host, false);try libcurl.curl_easy_setopt(self._easy, .ssl_verify_peer, false);diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zigindex 4201ae22e..133dc1fcb 100644--- a/src/telemetry/lightpanda.zig+++ b/src/telemetry/lightpanda.zig@@ -132,7 +132,7 @@ pub fn send(self: *LightPanda, raw_event: telemetry.Event) !void {fn run(self: *LightPanda) void {// The connection is created, owned, and torn down entirely on this thread;// the network thread never sees it (Transport == .none).- var conn = http.Connection.init(self.network.x509_store, self.network.config, self.network.ip_filter) catch |err| {+ var conn = http.Connection.init(self.network.certificates, self.network.config, self.network.ip_filter) catch |err| {// Essentially OOM — the process is already in trouble. The thread// handle stays set so send() won't respawn; events drop at the cap.log.warn(.telemetry, "connection init", .{ .err = err });