Skip to content

Lightpanda: the headless browser designed for AI and automation

chore: Network cleanup

Karl Seguindecedca6e7bf
diff --git a/src/Updater.zig b/src/Updater.zig
index 837c8e42d..2c753a6a0 100644
--- a/src/Updater.zig
+++ b/src/Updater.zig
@@ -19,70 +19,31 @@
const std = @import("std");
const lp = @import("lightpanda");
-const Network = @import("network/Network.zig");
const http = @import("network/http.zig");
+const Certificates = @import("network/Certificates.zig");
const libcurl = @import("sys/libcurl.zig");
-const crypto = @import("sys/libcrypto.zig");
-const Config = @import("Config.zig");
const Allocator = std.mem.Allocator;
-/// Sole purpose of this client is to do updates; hence, its very minimal.
-const Updater = @This();
-x509_store: *crypto.X509_STORE,
-config: *const Config,
-
-/// Initializes the update client; meant to be used as singleton.
-pub fn init(allocator: Allocator, config: *const Config) !Updater {
- Network.globalInit(allocator);
- errdefer Network.globalDeinit();
- const x509_store = try Network.prepareX509Store(allocator, config);
+/// Sends running Lightpanda version to remote to get update information.
+/// Outputs directly to given `Writer`.
+pub fn inform(allocator: Allocator, config: *const lp.Config, writer: *std.Io.Writer) !void {
+ const certificates = try Certificates.init(allocator, config);
+ defer certificates.deinit();
- return .{
- .x509_store = x509_store,
- .config = config,
+ libcurl.curl_global_init(.{ .ssl = true }, null) catch |err| {
+ lp.assert(false, "curl global init", .{ .err = err });
};
-}
+ defer libcurl.curl_global_cleanup();
-pub fn deinit(self: *Updater) void {
- Network.globalDeinit();
- crypto.X509_STORE_free(self.x509_store);
-}
-
-/// Sends running Lightpanda version to remote to get update information.
-/// Outputs directly to given `Writer`.
-pub fn inform(self: *Updater, writer: *std.Io.Writer) !void {
- var conn = try http.Connection.init(self.x509_store, self.config, null);
+ var conn = try http.Connection.init(certificates, config, null);
defer conn.deinit();
- const url = std.fmt.comptimePrint("https://telemetry.lightpanda.io/v/{s}", .{lp.build_config.version});
- // Prepare the request.
- try conn.setURL(url);
+ try conn.setURL("https://telemetry.lightpanda.io/v/" ++ lp.build_config.version);
try conn.setGetMode();
try conn.setFollowLocation(true);
// Wraps everything needed to receive bytes.
- const ReceiverContext = struct {
- writer: *std.Io.Writer,
- err: std.Io.Writer.Error!void = {},
-
- /// curl -> writer.
- fn drain(
- buffer: [*]const u8,
- buf_count: usize,
- buf_len: usize,
- raw_ctx: *anyopaque,
- ) callconv(.c) usize {
- const ctx: *@This() = @ptrCast(@alignCast(raw_ctx));
- const chunk = buffer[0 .. buf_count * buf_len];
- ctx.writer.writeAll(chunk) catch |err| {
- ctx.err = err;
- return 0;
- };
-
- return chunk.len;
- }
- };
// Set receiver context.
var ctx = ReceiverContext{ .writer = writer };
@@ -97,3 +58,22 @@ pub fn inform(self: *Updater, writer: *std.Io.Writer) !void {
_ = status_int;
return writer.flush();
}
+
+const ReceiverContext = struct {
+ writer: *std.Io.Writer,
+ err: std.Io.Writer.Error!void = {},
+
+ fn drain(buffer: [*]const u8, buf_count: usize, buf_len: usize, ctx: *anyopaque) callconv(.c) usize {
+ // libcurl only ever sends 1 buffer
+ std.debug.assert(buf_count == 1);
+
+ const self: *ReceiverContext = @ptrCast(@alignCast(ctx));
+ const chunk = buffer[0..buf_len];
+ self.writer.writeAll(chunk) catch |err| {
+ self.err = err;
+ return 0;
+ };
+
+ return buf_len;
+ }
+};
diff --git a/src/browser/Browser.zig b/src/browser/Browser.zig
index f4cab1c65..3302a48bd 100644
--- a/src/browser/Browser.zig
+++ b/src/browser/Browser.zig
@@ -153,9 +153,6 @@ pub fn deinit(self: *Browser) void {
// fire — only now is it safe to free the pool backing their parameters.
self.fc_identity_pool.deinit(allocator);
self.page_pool.deinit(allocator);
- if (self.http_client.cache) |cache| {
- cache.maintenance(lp.datetime.timestamp(.real));
- }
self.http_client.deinit();
self.clearPermissions();
self.permissions.deinit(allocator);
diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig
index 99bf6cb37..c5fd99de7 100644
--- a/src/cdp/domains/network.zig
+++ b/src/cdp/domains/network.zig
@@ -216,14 +216,14 @@ fn clearBrowserCache(cmd: *CDP.Command) !void {
// Chrome accepts that and clears the jar; reject only on truly malformed JSON.
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
const network = bc.cdp.browser.http_client.network;
- if (network.cache) |*c| try c.clear();
+ try network.cache.clear();
return cmd.sendResult(null, .{});
}
fn canClearBrowserCache(cmd: *CDP.Command) !void {
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
const network = bc.cdp.browser.http_client.network;
- return cmd.sendResult(.{ .result = network.cache != null }, .{});
+ return cmd.sendResult(.{ .result = network.cache.active() != null }, .{});
}
fn clearBrowserCookies(cmd: *CDP.Command) !void {
@@ -1030,7 +1030,7 @@ test "cdp.Network: setCacheDisabled" {
.params = .{ .cacheDisabled = true },
});
try ctx.expectSentResult(null, .{ .id = 1 });
- try testing.expect(client.cache == null);
+ try testing.expectEqual(null, client.cache.active());
}
test "cdp.Network: configured CDP ignores setCacheDisabled" {
@@ -1041,7 +1041,7 @@ test "cdp.Network: configured CDP ignores setCacheDisabled" {
var cache: Cache = undefined;
const client = &ctx.cdp().browser.http_client;
client.cache = &cache;
- defer client.cache = null;
+ defer client.cache = &testing.base.test_app.network.cache;
try ctx.processMessage(.{
.id = 1,
diff --git a/src/cdp/testing.zig b/src/cdp/testing.zig
index 8133f32bb..fd9edb2fe 100644
--- a/src/cdp/testing.zig
+++ b/src/cdp/testing.zig
@@ -20,7 +20,7 @@ const std = @import("std");
const CDP = @import("CDP.zig");
-const base = @import("../testing.zig");
+pub const base = @import("../testing.zig");
const json = std.json;
const posix = std.posix;
diff --git a/src/lightpanda.zig b/src/lightpanda.zig
index f6fcd03c8..a8337ab2a 100644
--- a/src/lightpanda.zig
+++ b/src/lightpanda.zig
@@ -60,8 +60,6 @@ pub const build_config = @import("build_config");
pub const crash_handler = @import("crash_handler.zig");
pub const core_dump = @import("core_dump.zig");
-pub const Updater = @import("Updater.zig");
-
pub var metrics = @import("Metrics.zig"){};
pub const IS_TEST = @import("builtin").is_test;
@@ -395,14 +393,10 @@ fn dumpContent(app: *App, mode: Config.DumpFormat, dump_opts: dump.Opts, frame:
}
pub fn checkVersion(allocator: std.mem.Allocator, config: *const Config) !void {
- var client = try Updater.init(allocator, config);
- defer client.deinit();
-
const stdout = std.Io.File.stdout();
var buf: [4096]u8 = undefined;
var writer = stdout.writer(io, &buf);
- const w = &writer.interface;
- try client.inform(w);
+ try @import("Updater.zig").inform(allocator, config, &writer.interface);
}
// Writes a single page's result object. Framing (the enclosing array and any
diff --git a/src/network/Certificates.zig b/src/network/Certificates.zig
new file mode 100644
index 000000000..b85658cc2
--- /dev/null
+++ b/src/network/Certificates.zig
@@ -0,0 +1,139 @@
+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const lp = @import("lightpanda");
+
+const crypto = @import("../sys/libcrypto.zig");
+
+const log = lp.log;
+const Config = lp.Config;
+const Allocator = std.mem.Allocator;
+
+const Certificates = @This();
+
+store: *crypto.X509_STORE,
+
+pub fn init(allocator: Allocator, config: *const Config) !Certificates {
+ const store = blk: {
+ const custom_store = config.customCertStore();
+ if (config.tlsVerifyHost() == false) {
+ if (custom_store != null) {
+ log.warn(.app, "custom CA ignored", .{
+ .arg = "--ca-cert, --ca-path",
+ .reason = "TLS verification disabled",
+ });
+ }
+ break :blk crypto.X509_STORE_new() orelse return error.FailedToCreateX509Store;
+ }
+ break :blk custom_store orelse try storeFromSystemCA(allocator);
+ };
+
+ return .{ .store = store };
+}
+
+pub fn deinit(self: Certificates) void {
+ crypto.X509_STORE_free(self.store);
+}
+
+fn storeFromSystemCA(allocator: Allocator) !*crypto.X509_STORE {
+ const store = crypto.X509_STORE_new() orelse return error.FailedToCreateX509Store;
+ errdefer crypto.X509_STORE_free(store);
+
+ var count: usize = 0;
+ defer {
+ if (count == 0) {
+ log.warn(.app, "No certificates loaded", .{});
+ }
+ }
+
+ switch (comptime @import("builtin").os.tag) {
+ .linux, .openbsd, .netbsd, .freebsd => blk: {
+ // Iterate over known directories; this may or may not succeed.
+ const cwd = std.Io.Dir.cwd();
+ inline for ([_][]const u8{
+ "/etc/ssl/certs", // Debian/Ubuntu/Gentoo/Alpine, SUSE
+ "/etc/pki/tls/certs", // Fedora/RHEL
+ }) |dir_path| {
+ count += try loadFromDirectory(allocator, store, cwd, dir_path);
+ if (count > 0) break :blk;
+ }
+
+ // Iterate over known files.
+ inline for ([_][*:0]const u8{
+ "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo
+ "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6
+ "/etc/ssl/ca-bundle.pem", // OpenSUSE
+ "/etc/pki/tls/cacert.pem", // OpenELEC
+ "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7
+ "/etc/ssl/cert.pem", // Alpine, *BSD
+ }) |file| {
+ if (crypto.X509_STORE_load_locations(store, file, null) == 1) {
+ count += 1;
+ break :blk;
+ }
+ }
+ },
+ else => {
+ // Prefer stdlib's cert scanner.
+ var bundle: std.crypto.Certificate.Bundle = .empty;
+ try bundle.rescan(allocator, lp.io, std.Io.Clock.now(.real, lp.io));
+ defer bundle.deinit(allocator);
+
+ const bytes = bundle.bytes.items;
+ var it = bundle.map.valueIterator();
+ while (it.next()) |index| {
+ // d2i_X509 reads the cert's own DER length header to find its end and
+ // advances `ptr` past it, so we just hand it the rest of the buffer.
+ var ptr: [*]const u8 = bytes.ptr + index.*;
+ const x509 = crypto.d2i_X509(null, &ptr, @intCast(bytes.len - index.*)) orelse {
+ log.warn(.app, "Skipping unparseable system cert", .{});
+ continue;
+ };
+ defer crypto.X509_free(x509); // add_cert takes its own ref; drop ours.
+
+ const result = crypto.X509_STORE_add_cert(store, x509);
+ if (result != 1) {
+ log.warn(.app, "Failed to add X509 cert to store", .{});
+ }
+ count += 1;
+ }
+ },
+ }
+
+ return store;
+}
+
+fn loadFromDirectory(allocator: Allocator, store: *crypto.X509_STORE, cwd: std.Io.Dir, dir_path: []const u8) !usize {
+ var count: usize = 0;
+ var dir = cwd.openDir(lp.io, dir_path, .{ .iterate = true }) catch return count;
+ defer dir.close(lp.io);
+
+ var it = dir.iterate();
+ while (it.next(lp.io) catch return count) |entry| {
+ if (entry.kind != .file and entry.kind != .sym_link) continue;
+
+ const path = try std.fs.path.joinZ(allocator, &.{ dir_path, entry.name });
+ defer allocator.free(path);
+
+ if (crypto.X509_STORE_load_locations(store, path, null) == 1) {
+ count += 1;
+ }
+ }
+ return count;
+}
diff --git a/src/network/CurlDebugAllocator.zig b/src/network/CurlDebugAllocator.zig
deleted file mode 100644
index 7fc700e54..000000000
--- a/src/network/CurlDebugAllocator.zig
+++ /dev/null
@@ -1,134 +0,0 @@
-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
-//
-// Francis Bouvier <[email protected]>
-// Pierre Tachoire <[email protected]>
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-const std = @import("std");
-const lp = @import("lightpanda");
-
-const libcurl = @import("../sys/libcurl.zig");
-
-const Allocator = std.mem.Allocator;
-
-const CurlDebugAllocator = @This();
-
-// C11 requires malloc to return memory aligned to max_align_t (16 bytes on x86_64).
-// We match this guarantee since libcurl expects malloc-compatible alignment.
-const alignment = 16;
-
-var instance: ?CurlDebugAllocator = null;
-
-allocator: Allocator,
-
-pub fn init(allocator: Allocator) void {
- lp.assert(instance == null, "Initialization of curl must happen only once", .{});
- instance = .{ .allocator = allocator };
-}
-
-pub fn interface() libcurl.CurlAllocator {
- return .{
- .free = free,
- .strdup = strdup,
- .malloc = malloc,
- .calloc = calloc,
- .realloc = realloc,
- };
-}
-
-fn _allocBlock(size: usize) ?*Block {
- const slice = instance.?.allocator.alignedAlloc(u8, .fromByteUnits(alignment), Block.fullsize(size)) catch return null;
- const block: *Block = @ptrCast(@alignCast(slice.ptr));
- block.size = size;
- return block;
-}
-
-fn _freeBlock(header: *Block) void {
- instance.?.allocator.free(header.slice());
-}
-
-fn malloc(size: usize) ?*anyopaque {
- const block = _allocBlock(size) orelse return null;
- return @ptrCast(block.data());
-}
-
-fn calloc(nmemb: usize, size: usize) ?*anyopaque {
- const total = nmemb * size;
- const block = _allocBlock(total) orelse return null;
- const ptr = block.data();
- @memset(ptr[0..total], 0); // for historical reasons, calloc zeroes memory, but malloc does not.
- return @ptrCast(ptr);
-}
-
-fn realloc(ptr: ?*anyopaque, size: usize) ?*anyopaque {
- const p = ptr orelse return malloc(size);
- const block = Block.fromPtr(p);
-
- const old_size = block.size;
- if (size == old_size) return ptr;
-
- if (instance.?.allocator.resize(block.slice(), alignment + size)) {
- block.size = size;
- return ptr;
- }
-
- const copy_size = @min(old_size, size);
- const new_block = _allocBlock(size) orelse return null;
- @memcpy(new_block.data()[0..copy_size], block.data()[0..copy_size]);
- _freeBlock(block);
- return @ptrCast(new_block.data());
-}
-
-fn free(ptr: ?*anyopaque) void {
- const p = ptr orelse return;
- _freeBlock(Block.fromPtr(p));
-}
-
-fn strdup(str: [*:0]const u8) ?[*:0]u8 {
- const len = std.mem.len(str);
- const header = _allocBlock(len + 1) orelse return null;
- const ptr = header.data();
- @memcpy(ptr[0..len], str[0..len]);
- ptr[len] = 0;
- return ptr[0..len :0];
-}
-
-const Block = extern struct {
- size: usize = 0,
- _padding: [alignment - @sizeOf(usize)]u8 = .{0} ** (alignment - @sizeOf(usize)),
-
- inline fn fullsize(bytes: usize) usize {
- return alignment + bytes;
- }
-
- inline fn fromPtr(ptr: *anyopaque) *Block {
- const raw: [*]u8 = @ptrCast(ptr);
- return @ptrCast(@alignCast(raw - @sizeOf(Block)));
- }
-
- inline fn data(self: *Block) [*]u8 {
- const ptr: [*]u8 = @ptrCast(self);
- return ptr + @sizeOf(Block);
- }
-
- inline fn slice(self: *Block) []align(alignment) u8 {
- const base: [*]align(alignment) u8 = @ptrCast(@alignCast(self));
- return base[0 .. alignment + self.size];
- }
-};
-
-comptime {
- std.debug.assert(@sizeOf(Block) == alignment);
-}
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 041b00f64..06e437184 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -181,8 +181,7 @@ blocking_requests: std.AutoHashMapUnmanaged(u32, u32) = .empty,
// heuristics add this in.
intercepted: usize = 0,
-// null or referencing network.cache
-cache: ?*Cache,
+cache: *Cache,
// Cached config decisions, resolved once at init.
serve_mode: bool,
@@ -218,7 +217,7 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP)
.allocator = allocator,
.cdp = cdp,
.inbox = .{},
- .cache = if (network.cache) |*c| c else null,
+ .cache = &network.cache,
.use_proxy = http_proxy != null,
.http_proxy = http_proxy,
@@ -265,6 +264,7 @@ pub fn deinit(self: *Client) void {
self.blocking_requests.deinit(self.allocator);
self.transfers.deinit(self.allocator);
self.inbox.deinit();
+ self.cache.maintenance(lp.datetime.timestamp(.real));
}
// Look up a live transfer by its id. Returns null if the transfer has been
@@ -317,9 +317,9 @@ pub fn obeyRobots(self: *Client, enable: bool) !void {
pub fn disableCache(self: *Client, disable: bool) void {
if (disable) {
- self.cache = null;
+ self.cache = &Cache.noop;
} else {
- self.cache = if (self.network.cache) |*c| c else null;
+ self.cache = &self.network.cache;
}
}
@@ -984,7 +984,7 @@ fn findHeader(headers: []const http.Header, name: []const u8) ?[]const u8 {
// the response; on an expired-with-validators entry the request becomes a
// conditional revalidation.
fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
- const cache = self.cache orelse return false;
+ const cache = self.cache.active() orelse return false;
const req = &transfer.req;
if (req.method != .GET or req.streaming or req.skip_cache) {
@@ -1067,7 +1067,7 @@ fn cacheRevalidated(self: *Client, transfer: *Transfer) !bool {
return false;
}
// could have been disabled in-between
- const cache = self.cache orelse return false;
+ const cache = self.cache.active() orelse return false;
const stale = transfer._cache_intent.revalidate;
transfer._cache_intent = .none;
@@ -1103,7 +1103,7 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {
transfer._cache_intent = .none;
// could have been disabled while waiting of the response
- const cache = self.cache orelse return;
+ const cache = self.cache.active() orelse return;
const arena = transfer.arena;
const rh = &(transfer.res.header orelse return);
@@ -3582,7 +3582,7 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void {
client.intercepted = 0;
client.http_active = 0;
client.ws_active = 0;
- client.cache = null;
+ client.cache = &Cache.noop;
client.serve_mode = false;
client.obey_robots = false;
client.robots = .{
@@ -3951,7 +3951,7 @@ test "HttpClient: fulfillIntercepted follows a 3xx redirect" {
// Only network.config (httpMaxRedirects, which ignores its config),
// network.cache and the (empty) connection pool are read on this path.
var net: Network = undefined;
- net.cache = null;
+ net.cache = Cache.noop;
net.adblocker = null;
// An empty pool makes processTransfer queue the re-issued request
// instead of putting it on the wire — the queue IS the capture.
@@ -4309,7 +4309,7 @@ test "HttpClient: throttled navigations wait for their per-host slot" {
defer pool.deinit();
var net: Network = undefined;
- net.cache = null;
+ net.cache = Cache.noop;
net.adblocker = null;
net.web_bot_auth = null;
// An empty pool makes processTransfer queue a started transfer instead
diff --git a/src/network/Network.zig b/src/network/Network.zig
index 4087f2fda..2c532d010 100644
--- a/src/network/Network.zig
+++ b/src/network/Network.zig
@@ -26,17 +26,15 @@ const Config = @import("../Config.zig");
const CDP = @import("../cdp/CDP.zig");
const sys_net = @import("../sys/net.zig");
const libcurl = @import("../sys/libcurl.zig");
-const crypto = @import("../sys/libcrypto.zig");
const http = @import("http.zig");
const IpFilter = @import("IpFilter.zig");
const RobotStore = @import("Robots.zig").RobotStore;
const WebBotAuth = @import("WebBotAuth.zig");
const RateLimiter = @import("RateLimiter.zig");
-const CurlDebugAllocator = @import("CurlDebugAllocator.zig");
+const Certificates = @import("Certificates.zig");
const Cache = @import("cache/Cache.zig");
-const SqliteCache = @import("cache/SqliteCache.zig");
const AdBlocker = @import("adblock/AdBlocker.zig");
const log = lp.log;
@@ -86,13 +84,12 @@ const PSEUDO_POLLFDS = 2;
allocator: Allocator,
app: *App,
-cache: ?Cache,
+cache: Cache,
config: *const Config,
-/// Holds certificate bundle.
-x509_store: *crypto.X509_STORE,
robot_store: RobotStore,
web_bot_auth: ?WebBotAuth,
rate_limiter: ?RateLimiter,
+certificates: Certificates,
/// Hostname dictionaries built from `--adblock-lists`. Parsed once here and
/// never mutated afterwards, so every HttpClient can share this one copy.
adblocker: ?AdBlocker,
@@ -143,31 +140,11 @@ cdp_start: usize,
/// Optional IP filter for blocking requests to private/internal networks (--block-private-networks).
ip_filter: ?*IpFilter = null,
-/// Calling `init` also calls this function; only marked public for situations
-/// networking is needed without `App`.
-pub fn globalInit(allocator: Allocator) void {
- // Only route curl's own allocations through our allocator in Debug, so the
- // leak detector sees them. In Release it'd just wrap c_allocator (curl's
- // default malloc anyway) at the cost of a per-allocation header.
- const curl_allocator = comptime if (lp.IS_DEBUG) CurlDebugAllocator.interface() else null;
- if (comptime lp.IS_DEBUG) {
- CurlDebugAllocator.init(allocator);
- }
-
- libcurl.curl_global_init(.{ .ssl = true }, curl_allocator) catch |err| {
+pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
+ libcurl.curl_global_init(.{ .ssl = true }, null) catch |err| {
lp.assert(false, "curl global init", .{ .err = err });
};
-}
-
-/// Calling `deinit` also calls this function; only marked public for situations
-/// networking is needed without `App`.
-pub fn globalDeinit() void {
- libcurl.curl_global_cleanup();
-}
-
-pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
- globalInit(allocator);
- errdefer globalDeinit();
+ errdefer libcurl.curl_global_cleanup();
const pipe = try sys_net.pipe2(.{ .NONBLOCK = true, .CLOEXEC = true });
@@ -186,22 +163,8 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
@memset(pollfds, .{ .fd = -1, .events = 0, .revents = 0 });
pollfds[0] = .{ .fd = pipe[0], .events = posix.POLL.IN, .revents = 0 };
- const x509_store = blk: {
- if (config.tlsVerifyHost()) {
- break :blk try prepareX509Store(allocator, config);
- }
- // Verification is off, so the store is never consulted — but still
- // take ownership of a user-supplied one so the flags compose and
- // nothing leaks.
- if (config.customCertStore()) |store| {
- log.warn(.app, "custom CA ignored", .{ .arg = "--ca-cert, --ca-path", .reason = "TLS verification disabled" });
- break :blk store;
- }
- break :blk crypto.X509_STORE_new() orelse {
- return error.FailedToCreateX509Store;
- };
- };
- errdefer crypto.X509_STORE_free(x509_store);
+ const certificates = try Certificates.init(allocator, config);
+ errdefer certificates.deinit();
// IP filter for blocking requests to private/internal networks.
const block_private = config.blockPrivateNetworks();
@@ -227,7 +190,7 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
var available: DoublyLinkedList = .{};
for (0..count) |i| {
- connections[i] = try http.Connection.init(x509_store, config, ip_filter);
+ connections[i] = try http.Connection.init(certificates, config, ip_filter);
available.append(&connections[i].node);
}
@@ -240,30 +203,14 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
var adblocker = try AdBlocker.fromConfig(allocator, config);
errdefer if (adblocker) |*blocker| blocker.deinit();
- const cache = if (config.httpCacheDir()) |cache_dir_path|
- Cache{
- .kind = .{
- .sqlite = SqliteCache.init(
- allocator,
- .{ .path = cache_dir_path },
- config.httpCacheEntryLimit(),
- ) catch |e| {
- log.err(.cache, "failed to init", .{
- .kind = "SqliteCache",
- .path = cache_dir_path,
- .err = e,
- });
- return e;
- },
- },
- }
- else
- null;
+ var cache = try Cache.init(allocator, config);
+ errdefer cache.deinit();
return .{
- .allocator = allocator,
+ .app = app,
.config = config,
- .x509_store = x509_store,
+ .allocator = allocator,
+ .certificates = certificates,
.pollfds = pollfds,
.wakeup_pipe = pipe,
@@ -273,8 +220,6 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
.available = available,
.connections = connections,
- .app = app,
-
.cache = cache,
.robot_store = RobotStore.init(allocator),
.web_bot_auth = web_bot_auth,
@@ -299,7 +244,7 @@ pub fn deinit(self: *Network) void {
self.allocator.free(self.pollfds);
self.allocator.free(self.cdp_poll_snapshot);
- crypto.X509_STORE_free(self.x509_store);
+ self.certificates.deinit();
for (self.connections) |*conn| {
conn.deinit();
@@ -318,14 +263,14 @@ pub fn deinit(self: *Network) void {
if (self.adblocker) |*blocker| blocker.deinit();
- if (self.cache) |*cache| cache.deinit();
+ self.cache.deinit();
if (self.ip_filter) |f| {
f.deinit(self.allocator);
self.allocator.destroy(f);
}
- globalDeinit();
+ libcurl.curl_global_cleanup();
}
pub fn bind(
@@ -720,7 +665,7 @@ pub fn releaseConnection(self: *Network, conn: *http.Connection) void {
self.ws_count -= 1;
},
else => {
- conn.reset(self.config, self.x509_store, self.ip_filter) catch |err| {
+ conn.reset(self.config, self.certificates, self.ip_filter) catch |err| {
lp.assert(false, "couldn't reset curl easy", .{ .err = err });
};
self.conn_mutex.lockUncancelable(lp.io);
@@ -745,7 +690,7 @@ pub fn newConnection(self: *Network) ?*http.Connection {
};
// don't do this under lock
- conn.* = http.Connection.init(self.x509_store, self.config, self.ip_filter) catch {
+ conn.* = http.Connection.init(self.certificates, self.config, self.ip_filter) catch {
self.ws_mutex.lockUncancelable(lp.io);
defer self.ws_mutex.unlock(lp.io);
self.ws_pool.destroy(conn);
@@ -757,110 +702,6 @@ pub fn newConnection(self: *Network) ?*http.Connection {
return conn;
}
-/// NEVER give full ownership of store to `SSL_CTX`, always rely on ref counting.
-/// Allocations made through passed `allocator` are freed before this function returns.
-pub fn prepareX509Store(allocator: Allocator, config: *const Config) !*crypto.X509_STORE {
- // A user-supplied store replaces system trust entirely.
- if (config.customCertStore()) |store| {
- return store;
- }
- return storeFromSystemCA(allocator);
-}
-
-/// Creates an X509_STORE from system root CA.
-fn storeFromSystemCA(allocator: Allocator) !*crypto.X509_STORE {
- const store = crypto.X509_STORE_new() orelse return error.FailedToCreateX509Store;
- errdefer crypto.X509_STORE_free(store);
-
- var count: usize = 0;
- defer {
- if (count == 0) {
- log.warn(.app, "No certificates loaded", .{});
- }
- }
-
- switch (comptime builtin.os.tag) {
- .linux, .openbsd, .netbsd, .freebsd => blk: {
- // Iterate over known directories; this may or may not succeed.
- const cwd = std.Io.Dir.cwd();
- inline for ([_][]const u8{
- "/etc/ssl/certs", // Debian/Ubuntu/Gentoo/Alpine, SUSE
- "/etc/pki/tls/certs", // Fedora/RHEL
- }) |dir_path| {
- count += try loadFromDirectory(allocator, store, cwd, dir_path);
- if (count > 0) break :blk;
- }
-
- // Iterate over known files.
- inline for ([_][*:0]const u8{
- "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo
- "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6
- "/etc/ssl/ca-bundle.pem", // OpenSUSE
- "/etc/pki/tls/cacert.pem", // OpenELEC
- "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7
- "/etc/ssl/cert.pem", // Alpine, *BSD
- }) |file| {
- if (crypto.X509_STORE_load_locations(store, file, null) == 1) {
- count += 1;
- break :blk;
- }
- }
- },
- else => {
- // Prefer stdlib's cert scanner.
- var bundle: std.crypto.Certificate.Bundle = .empty;
- try bundle.rescan(allocator, lp.io, std.Io.Clock.now(.real, lp.io));
- defer bundle.deinit(allocator);
-
- const bytes = bundle.bytes.items;
- var it = bundle.map.valueIterator();
- while (it.next()) |index| {
- // d2i_X509 reads the cert's own DER length header to find its end and
- // advances `ptr` past it, so we just hand it the rest of the buffer.
- var ptr: [*]const u8 = bytes.ptr + index.*;
- const x509 = crypto.d2i_X509(null, &ptr, @intCast(bytes.len - index.*)) orelse {
- log.warn(.app, "Skipping unparseable system cert", .{});
- continue;
- };
- defer crypto.X509_free(x509); // add_cert takes its own ref; drop ours.
-
- const result = crypto.X509_STORE_add_cert(store, x509);
- if (result != 1) {
- log.warn(.app, "Failed to add X509 cert to store", .{});
- }
- count += 1;
- }
- },
- }
-
- return store;
-}
-
-/// Loads certificates from given `path`; returning how many CA loaded.
-fn loadFromDirectory(
- allocator: Allocator,
- store: *crypto.X509_STORE,
- cwd: std.Io.Dir,
- dir_path: []const u8,
-) Allocator.Error!usize {
- var count: usize = 0;
- var dir = cwd.openDir(lp.io, dir_path, .{ .iterate = true }) catch return count;
- defer dir.close(lp.io);
-
- var it = dir.iterate();
- while (it.next(lp.io) catch return count) |entry| {
- if (entry.kind != .file and entry.kind != .sym_link) continue;
-
- const path = try std.fs.path.joinZ(allocator, &.{ dir_path, entry.name });
- defer allocator.free(path);
-
- if (crypto.X509_STORE_load_locations(store, path, null) == 1) {
- count += 1;
- }
- }
- return count;
-}
-
pub fn HostHashMap(comptime V: type) type {
return std.HashMapUnmanaged([]const u8, V, HostContext, 80);
}
diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig
index 71fd819c7..8b0565881 100644
--- a/src/network/cache/Cache.zig
+++ b/src/network/cache/Cache.zig
@@ -18,9 +18,13 @@
const std = @import("std");
const lp = @import("lightpanda");
+
const Http = @import("../http.zig");
+
const SqliteCache = @import("SqliteCache.zig");
+pub var noop: Cache = .{ .kind = .noop };
+
const log = lp.log;
/// A browser-wide cache for resources across the network.
@@ -28,47 +32,85 @@ const log = lp.log;
pub const Cache = @This();
kind: union(enum) {
+ noop: void,
sqlite: SqliteCache,
},
+pub fn init(allocator: std.mem.Allocator, config: *const lp.Config) !Cache {
+ const cache_path = config.httpCacheDir() orelse {
+ return .{ .kind = .noop };
+ };
+
+ const sqlite = SqliteCache.init(
+ allocator,
+ .{ .path = cache_path },
+ config.httpCacheEntryLimit(),
+ ) catch |err| {
+ log.err(.cache, "failed to init", .{
+ .kind = "SqliteCache",
+ .path = cache_path,
+ .err = err,
+ });
+ return err;
+ };
+
+ return .{
+ .kind = .{ .sqlite = sqlite },
+ };
+}
+
pub fn deinit(self: *Cache) void {
return switch (self.kind) {
+ .noop => {},
inline else => |*c| c.deinit(),
};
}
+pub fn active(self: *Cache) ?*Cache {
+ return switch (self.kind) {
+ .noop => null,
+ inline else => self,
+ };
+}
+
pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheGetRequest) !CacheGetResult {
return switch (self.kind) {
+ .noop => .miss,
inline else => |*c| c.get(arena, req),
};
}
pub fn put(self: *Cache, req: CachePutRequest, body: []const u8) !void {
return switch (self.kind) {
+ .noop => {},
inline else => |*c| c.put(req, body),
};
}
pub fn evict(self: *Cache, url: []const u8) void {
return switch (self.kind) {
+ .noop => {},
inline else => |*c| c.evict(url),
};
}
pub fn renew(self: *Cache, arena: std.mem.Allocator, req: RenewResponse) !void {
return switch (self.kind) {
+ .noop => {},
inline else => |*c| c.renew(arena, req),
};
}
pub fn clear(self: *Cache) !void {
return switch (self.kind) {
+ .noop => {},
inline else => |*c| c.clear(),
};
}
pub fn maintenance(self: *Cache, now: u64) void {
return switch (self.kind) {
+ .noop => {},
inline else => |*c| c.maintenance(now),
};
}
diff --git a/src/network/http.zig b/src/network/http.zig
index 11c601bbe..2442b5806 100644
--- a/src/network/http.zig
+++ b/src/network/http.zig
@@ -17,7 +17,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
-const posix = std.posix;
+const lp = @import("lightpanda");
const Config = @import("../Config.zig");
const sys_net = @import("../sys/net.zig");
@@ -25,8 +25,10 @@ const libcurl = @import("../sys/libcurl.zig");
const crypto = @import("../sys/libcrypto.zig");
const IpFilter = @import("IpFilter.zig");
+const Certificates = @import("Certificates.zig");
-const log = @import("lightpanda").log;
+const log = lp.log;
+const posix = std.posix;
pub const ENABLE_DEBUG = false;
@@ -280,7 +282,7 @@ pub const Connection = struct {
};
pub fn init(
- x509_store: *crypto.X509_STORE,
+ certificates: Certificates,
config: *const Config,
ip_filter: ?*const IpFilter,
) !Connection {
@@ -289,7 +291,7 @@ pub const Connection = struct {
var self = Connection{ ._easy = easy, .transport = .none };
errdefer self.deinit();
- try self.reset(config, x509_store, ip_filter);
+ try self.reset(config, certificates, ip_filter);
return self;
}
@@ -443,7 +445,7 @@ pub const Connection = struct {
pub fn reset(
self: *Connection,
config: *const Config,
- x509_store: *crypto.X509_STORE,
+ certificates: Certificates,
ip_filter: ?*const IpFilter,
) !void {
libcurl.curl_easy_reset(self._easy);
@@ -487,7 +489,7 @@ pub const Connection = struct {
}
}).wrap);
// Pass our store to CURLOPT_SSL_CTX_FUNCTION.
- try libcurl.curl_easy_setopt(self._easy, .ssl_ctx_data, x509_store);
+ try libcurl.curl_easy_setopt(self._easy, .ssl_ctx_data, certificates.store);
} else {
try libcurl.curl_easy_setopt(self._easy, .ssl_verify_host, false);
try libcurl.curl_easy_setopt(self._easy, .ssl_verify_peer, false);
diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig
index 4201ae22e..133dc1fcb 100644
--- a/src/telemetry/lightpanda.zig
+++ b/src/telemetry/lightpanda.zig
@@ -132,7 +132,7 @@ pub fn send(self: *LightPanda, raw_event: telemetry.Event) !void {
fn run(self: *LightPanda) void {
// The connection is created, owned, and torn down entirely on this thread;
// the network thread never sees it (Transport == .none).
- var conn = http.Connection.init(self.network.x509_store, self.network.config, self.network.ip_filter) catch |err| {
+ var conn = http.Connection.init(self.network.certificates, self.network.config, self.network.ip_filter) catch |err| {
// Essentially OOM — the process is already in trouble. The thread
// handle stays set so send() won't respawn; events drop at the cap.
log.warn(.telemetry, "connection init", .{ .err = err });