Skip to content

Lightpanda: the headless browser designed for AI and automation

networking: replace `ca_blob` with `X509_STORE`

Halil Durak820f5684cbb7
diff --git a/src/network/Network.zig b/src/network/Network.zig
index dde76146e..4edc00aa7 100644
--- a/src/network/Network.zig
+++ b/src/network/Network.zig
@@ -25,6 +25,7 @@ const Config = @import("../Config.zig");
const CDP = @import("../cdp/CDP.zig");
const libcurl = @import("../sys/libcurl.zig");
+const crypto = @import("../sys/libcrypto.zig");
const http = @import("http.zig");
const IpFilter = @import("IpFilter.zig");
@@ -86,7 +87,8 @@ allocator: Allocator,
app: *App,
cache: ?Cache,
config: *const Config,
-ca_blob: ?http.Blob,
+/// Holds certificate bundle.
+x509_store: *crypto.X509_STORE,
robot_store: RobotStore,
web_bot_auth: ?WebBotAuth,
@@ -175,10 +177,15 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
@memset(pollfds, .{ .fd = -1, .events = 0, .revents = 0 });
pollfds[0] = .{ .fd = pipe[0], .events = posix.POLL.IN, .revents = 0 };
- var ca_blob: ?http.Blob = null;
- if (config.tlsVerifyHost()) {
- ca_blob = try loadCerts(allocator);
- }
+ const x509_store = blk: {
+ if (config.tlsVerifyHost()) {
+ break :blk try createX509Store(allocator);
+ }
+ break :blk crypto.X509_STORE_new() orelse {
+ return error.FailedToCreateX509Store;
+ };
+ };
+ errdefer crypto.X509_STORE_free(x509_store);
// IP filter for blocking requests to private/internal networks.
const block_private = config.blockPrivateNetworks();
@@ -204,7 +211,7 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
var available: DoublyLinkedList = .{};
for (0..count) |i| {
- connections[i] = try http.Connection.init(ca_blob, config, ip_filter);
+ connections[i] = try http.Connection.init(x509_store, config, ip_filter);
available.append(&connections[i].node);
}
@@ -232,7 +239,7 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
return .{
.allocator = allocator,
.config = config,
- .ca_blob = ca_blob,
+ .x509_store = x509_store,
.pollfds = pollfds,
.wakeup_pipe = pipe,
@@ -266,10 +273,7 @@ pub fn deinit(self: *Network) void {
self.allocator.free(self.pollfds);
self.allocator.free(self.cdp_poll_snapshot);
- if (self.ca_blob) |ca_blob| {
- const data: [*]u8 = @ptrCast(ca_blob.data);
- self.allocator.free(data[0..ca_blob.len]);
- }
+ crypto.X509_STORE_free(self.x509_store);
for (self.connections) |*conn| {
conn.deinit();
@@ -675,7 +679,7 @@ pub fn releaseConnection(self: *Network, conn: *http.Connection) void {
self.ws_count -= 1;
},
else => {
- conn.reset(self.config, self.ca_blob, self.ip_filter) catch |err| {
+ conn.reset(self.config, self.x509_store, self.ip_filter) catch |err| {
lp.assert(false, "couldn't reset curl easy", .{ .err = err });
};
self.conn_mutex.lock();
@@ -700,7 +704,7 @@ pub fn newConnection(self: *Network) ?*http.Connection {
};
// don't do this under lock
- conn.* = http.Connection.init(self.ca_blob, self.config, self.ip_filter) catch {
+ conn.* = http.Connection.init(self.x509_store, self.config, self.ip_filter) catch {
self.ws_mutex.lock();
defer self.ws_mutex.unlock();
self.ws_pool.destroy(conn);
@@ -712,89 +716,43 @@ pub fn newConnection(self: *Network) ?*http.Connection {
return conn;
}
-// Wraps lines @ 64 columns. A PEM is basically a base64 encoded DER (which is
-// what Zig has), with lines wrapped at 64 characters and with a basic header
-// and footer
-const LineWriter = struct {
- col: usize = 0,
- inner: std.ArrayList(u8).Writer,
-
- pub fn writeAll(self: *LineWriter, data: []const u8) !void {
- var writer = self.inner;
-
- var col = self.col;
- const len = 64 - col;
-
- var remain = data;
- if (remain.len > len) {
- col = 0;
- try writer.writeAll(data[0..len]);
- try writer.writeByte('\n');
- remain = data[len..];
- }
-
- while (remain.len > 64) {
- try writer.writeAll(remain[0..64]);
- try writer.writeByte('\n');
- remain = remain[64..];
- }
- try writer.writeAll(remain);
- self.col = col + remain.len;
- }
-};
+const CreateX509StoreError = std.crypto.Certificate.Bundle.RescanError || error{FailedToCreateX509Store};
// TODO: on BSD / Linux, we could just read the PEM file directly.
// This whole rescan + decode is really just needed for MacOS. On Linux
// bundle.rescan does find the .pem file(s) which could be in a few different
// places, so it's still useful, just not efficient.
-fn loadCerts(allocator: Allocator) !libcurl.CurlBlob {
+//
+/// NEVER give full ownership of store to SSL_CTX, always rely on ref counting.
+fn createX509Store(allocator: Allocator) CreateX509StoreError!*crypto.X509_STORE {
+ const store = crypto.X509_STORE_new() orelse return error.FailedToCreateX509Store;
+ errdefer crypto.X509_STORE_free(store);
+
var bundle: std.crypto.Certificate.Bundle = .{};
try bundle.rescan(allocator);
defer bundle.deinit(allocator);
const bytes = bundle.bytes.items;
if (bytes.len == 0) {
- lp.log.warn(.app, "No system certificates", .{});
- return .{
- .len = 0,
- .flags = 0,
- .data = bytes.ptr,
- };
+ log.warn(.app, "No system certificates", .{});
+ return store;
}
-
- const encoder = std.base64.standard.Encoder;
- var arr: std.ArrayList(u8) = .empty;
-
- const encoded_size = encoder.calcSize(bytes.len);
- const buffer_size = encoded_size +
- (bundle.map.count() * 75) + // start / end per certificate + extra, just in case
- (encoded_size / 64) // newline per 64 characters
- ;
- try arr.ensureTotalCapacity(allocator, buffer_size);
- errdefer arr.deinit(allocator);
- var writer = arr.writer(allocator);
-
var it = bundle.map.valueIterator();
while (it.next()) |index| {
- const cert = try std.crypto.Certificate.der.Element.parse(bytes, index.*);
-
- try writer.writeAll("-----BEGIN CERTIFICATE-----\n");
- var line_writer = LineWriter{ .inner = writer };
- try encoder.encodeWriter(&line_writer, bytes[index.*..cert.slice.end]);
- try writer.writeAll("\n-----END CERTIFICATE-----\n");
+ // d2i_X509 reads the cert's own DER length header to find its end and
+ // advances `ptr` past it, so we just hand it the rest of the buffer.
+ var ptr: [*]const u8 = bytes.ptr + index.*;
+ const x509 = crypto.d2i_X509(null, &ptr, @intCast(bytes.len - index.*)) orelse {
+ log.warn(.app, "Skipping unparseable system cert", .{});
+ continue;
+ };
+ defer crypto.X509_free(x509); // add_cert takes its own ref; drop ours.
+ // TODO: Handle error.
+ const result = crypto.X509_STORE_add_cert(store, x509);
+ if (result != 1) {
+ log.warn(.app, "Failed to add X509 cert to store", .{});
+ }
}
- // Final encoding should not be larger than our initial size estimate
- lp.assert(buffer_size > arr.items.len, "Http loadCerts", .{ .estimate = buffer_size, .len = arr.items.len });
-
- // Allocate exactly the size needed and copy the data
- const result = try allocator.dupe(u8, arr.items);
- // Free the original oversized allocation
- arr.deinit(allocator);
-
- return .{
- .len = result.len,
- .data = result.ptr,
- .flags = 0,
- };
+ return store;
}
diff --git a/src/network/http.zig b/src/network/http.zig
index 481a991a6..96c06e930 100644
--- a/src/network/http.zig
+++ b/src/network/http.zig
@@ -21,6 +21,7 @@ const posix = std.posix;
const Config = @import("../Config.zig");
const libcurl = @import("../sys/libcurl.zig");
+const crypto = @import("../sys/libcrypto.zig");
const IpFilter = @import("IpFilter.zig");
const log = @import("lightpanda").log;
@@ -368,7 +369,7 @@ pub const Connection = struct {
};
pub fn init(
- ca_blob: ?libcurl.CurlBlob,
+ x509_store: *crypto.X509_STORE,
config: *const Config,
ip_filter: ?*const IpFilter,
) !Connection {
@@ -377,7 +378,7 @@ pub const Connection = struct {
var self = Connection{ ._easy = easy, .in_use = false, .transport = .none };
errdefer self.deinit();
- try self.reset(config, ca_blob, ip_filter);
+ try self.reset(config, x509_store, ip_filter);
return self;
}
@@ -501,7 +502,7 @@ pub const Connection = struct {
pub fn reset(
self: *Connection,
config: *const Config,
- ca_blob: ?libcurl.CurlBlob,
+ x509_store: *crypto.X509_STORE,
ip_filter: ?*const IpFilter,
) !void {
libcurl.curl_easy_reset(self._easy);
@@ -524,15 +525,28 @@ pub const Connection = struct {
try libcurl.curl_easy_setopt(self._easy, .proxy, null);
}
- // tls
- if (ca_blob) |ca| {
- try libcurl.curl_easy_setopt(self._easy, .ca_info_blob, ca);
- if (http_proxy != null) {
- try libcurl.curl_easy_setopt(self._easy, .proxy_ca_info_blob, ca);
- }
+ // TLS.
+ if (config.tlsVerifyHost()) {
+ // Provide certificate store to connection's SSL_CTX.
+ try libcurl.curl_easy_setopt(self._easy, .ssl_ctx_function, &(struct {
+ fn wrap(
+ _: *libcurl.Curl,
+ raw_ssl_ctx: *anyopaque,
+ raw_x509_store: *anyopaque,
+ ) callconv(.c) libcurl.CurlCode {
+ const ssl_ctx: *crypto.SSL_CTX = @ptrCast(raw_ssl_ctx);
+ const store: *crypto.X509_STORE = @ptrCast(raw_x509_store);
+
+ const result = crypto.SSL_CTX_set1_verify_cert_store(ssl_ctx, store);
+ if (result != 1) {
+ return libcurl.CURLE.ABORTED_BY_CALLBACK;
+ }
+ return libcurl.CURLE.OK;
+ }
+ }).wrap);
+ // Pass our store to CURLOPT_SSL_CTX_FUNCTION.
+ try libcurl.curl_easy_setopt(self._easy, .ssl_ctx_data, x509_store);
} else {
- assert(config.tlsVerifyHost() == false, "Http.init tls_verify_host", .{});
-
try libcurl.curl_easy_setopt(self._easy, .ssl_verify_host, false);
try libcurl.curl_easy_setopt(self._easy, .ssl_verify_peer, false);
diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig
index 769ae9c11..c0a56c587 100644
--- a/src/telemetry/lightpanda.zig
+++ b/src/telemetry/lightpanda.zig
@@ -132,7 +132,7 @@ pub fn send(self: *LightPanda, raw_event: telemetry.Event) !void {
fn run(self: *LightPanda) void {
// The connection is created, owned, and torn down entirely on this thread;
// the network thread never sees it (Transport == .none).
- var conn = http.Connection.init(self.network.ca_blob, self.network.config, self.network.ip_filter) catch |err| {
+ var conn = http.Connection.init(self.network.x509_store, self.network.config, self.network.ip_filter) catch |err| {
// Essentially OOM — the process is already in trouble. The thread
// handle stays set so send() won't respawn; events drop at the cap.
log.warn(.telemetry, "connection init", .{ .err = err });