Lightpanda: the headless browser designed for AI and automation
networking: replace `ca_blob` with `X509_STORE`
Halil Durak820f5684cbb7
diff --git a/src/network/Network.zig b/src/network/Network.zigindex dde76146e..4edc00aa7 100644--- a/src/network/Network.zig+++ b/src/network/Network.zig@@ -25,6 +25,7 @@ const Config = @import("../Config.zig");const CDP = @import("../cdp/CDP.zig");const libcurl = @import("../sys/libcurl.zig");+const crypto = @import("../sys/libcrypto.zig");const http = @import("http.zig");const IpFilter = @import("IpFilter.zig");@@ -86,7 +87,8 @@ allocator: Allocator,app: *App,cache: ?Cache,config: *const Config,-ca_blob: ?http.Blob,+/// Holds certificate bundle.+x509_store: *crypto.X509_STORE,robot_store: RobotStore,web_bot_auth: ?WebBotAuth,@@ -175,10 +177,15 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {@memset(pollfds, .{ .fd = -1, .events = 0, .revents = 0 });pollfds[0] = .{ .fd = pipe[0], .events = posix.POLL.IN, .revents = 0 };- var ca_blob: ?http.Blob = null;- if (config.tlsVerifyHost()) {- ca_blob = try loadCerts(allocator);- }+ const x509_store = blk: {+ if (config.tlsVerifyHost()) {+ break :blk try createX509Store(allocator);+ }+ break :blk crypto.X509_STORE_new() orelse {+ return error.FailedToCreateX509Store;+ };+ };+ errdefer crypto.X509_STORE_free(x509_store);// IP filter for blocking requests to private/internal networks.const block_private = config.blockPrivateNetworks();@@ -204,7 +211,7 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {var available: DoublyLinkedList = .{};for (0..count) |i| {- connections[i] = try http.Connection.init(ca_blob, config, ip_filter);+ connections[i] = try http.Connection.init(x509_store, config, ip_filter);available.append(&connections[i].node);}@@ -232,7 +239,7 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {return .{.allocator = allocator,.config = config,- .ca_blob = ca_blob,+ .x509_store = x509_store,.pollfds = pollfds,.wakeup_pipe = pipe,@@ -266,10 +273,7 @@ pub fn deinit(self: *Network) void {self.allocator.free(self.pollfds);self.allocator.free(self.cdp_poll_snapshot);- if (self.ca_blob) |ca_blob| {- const data: [*]u8 = @ptrCast(ca_blob.data);- self.allocator.free(data[0..ca_blob.len]);- }+ crypto.X509_STORE_free(self.x509_store);for (self.connections) |*conn| {conn.deinit();@@ -675,7 +679,7 @@ pub fn releaseConnection(self: *Network, conn: *http.Connection) void {self.ws_count -= 1;},else => {- conn.reset(self.config, self.ca_blob, self.ip_filter) catch |err| {+ conn.reset(self.config, self.x509_store, self.ip_filter) catch |err| {lp.assert(false, "couldn't reset curl easy", .{ .err = err });};self.conn_mutex.lock();@@ -700,7 +704,7 @@ pub fn newConnection(self: *Network) ?*http.Connection {};// don't do this under lock- conn.* = http.Connection.init(self.ca_blob, self.config, self.ip_filter) catch {+ conn.* = http.Connection.init(self.x509_store, self.config, self.ip_filter) catch {self.ws_mutex.lock();defer self.ws_mutex.unlock();self.ws_pool.destroy(conn);@@ -712,89 +716,43 @@ pub fn newConnection(self: *Network) ?*http.Connection {return conn;}-// Wraps lines @ 64 columns. A PEM is basically a base64 encoded DER (which is-// what Zig has), with lines wrapped at 64 characters and with a basic header-// and footer-const LineWriter = struct {- col: usize = 0,- inner: std.ArrayList(u8).Writer,-- pub fn writeAll(self: *LineWriter, data: []const u8) !void {- var writer = self.inner;-- var col = self.col;- const len = 64 - col;-- var remain = data;- if (remain.len > len) {- col = 0;- try writer.writeAll(data[0..len]);- try writer.writeByte('\n');- remain = data[len..];- }-- while (remain.len > 64) {- try writer.writeAll(remain[0..64]);- try writer.writeByte('\n');- remain = remain[64..];- }- try writer.writeAll(remain);- self.col = col + remain.len;- }-};+const CreateX509StoreError = std.crypto.Certificate.Bundle.RescanError || error{FailedToCreateX509Store};// TODO: on BSD / Linux, we could just read the PEM file directly.// This whole rescan + decode is really just needed for MacOS. On Linux// bundle.rescan does find the .pem file(s) which could be in a few different// places, so it's still useful, just not efficient.-fn loadCerts(allocator: Allocator) !libcurl.CurlBlob {+//+/// NEVER give full ownership of store to SSL_CTX, always rely on ref counting.+fn createX509Store(allocator: Allocator) CreateX509StoreError!*crypto.X509_STORE {+ const store = crypto.X509_STORE_new() orelse return error.FailedToCreateX509Store;+ errdefer crypto.X509_STORE_free(store);+var bundle: std.crypto.Certificate.Bundle = .{};try bundle.rescan(allocator);defer bundle.deinit(allocator);const bytes = bundle.bytes.items;if (bytes.len == 0) {- lp.log.warn(.app, "No system certificates", .{});- return .{- .len = 0,- .flags = 0,- .data = bytes.ptr,- };+ log.warn(.app, "No system certificates", .{});+ return store;}-- const encoder = std.base64.standard.Encoder;- var arr: std.ArrayList(u8) = .empty;-- const encoded_size = encoder.calcSize(bytes.len);- const buffer_size = encoded_size +- (bundle.map.count() * 75) + // start / end per certificate + extra, just in case- (encoded_size / 64) // newline per 64 characters- ;- try arr.ensureTotalCapacity(allocator, buffer_size);- errdefer arr.deinit(allocator);- var writer = arr.writer(allocator);-var it = bundle.map.valueIterator();while (it.next()) |index| {- const cert = try std.crypto.Certificate.der.Element.parse(bytes, index.*);-- try writer.writeAll("-----BEGIN CERTIFICATE-----\n");- var line_writer = LineWriter{ .inner = writer };- try encoder.encodeWriter(&line_writer, bytes[index.*..cert.slice.end]);- try writer.writeAll("\n-----END CERTIFICATE-----\n");+ // d2i_X509 reads the cert's own DER length header to find its end and+ // advances `ptr` past it, so we just hand it the rest of the buffer.+ var ptr: [*]const u8 = bytes.ptr + index.*;+ const x509 = crypto.d2i_X509(null, &ptr, @intCast(bytes.len - index.*)) orelse {+ log.warn(.app, "Skipping unparseable system cert", .{});+ continue;+ };+ defer crypto.X509_free(x509); // add_cert takes its own ref; drop ours.+ // TODO: Handle error.+ const result = crypto.X509_STORE_add_cert(store, x509);+ if (result != 1) {+ log.warn(.app, "Failed to add X509 cert to store", .{});+ }}- // Final encoding should not be larger than our initial size estimate- lp.assert(buffer_size > arr.items.len, "Http loadCerts", .{ .estimate = buffer_size, .len = arr.items.len });-- // Allocate exactly the size needed and copy the data- const result = try allocator.dupe(u8, arr.items);- // Free the original oversized allocation- arr.deinit(allocator);-- return .{- .len = result.len,- .data = result.ptr,- .flags = 0,- };+ return store;}diff --git a/src/network/http.zig b/src/network/http.zigindex 481a991a6..96c06e930 100644--- a/src/network/http.zig+++ b/src/network/http.zig@@ -21,6 +21,7 @@ const posix = std.posix;const Config = @import("../Config.zig");const libcurl = @import("../sys/libcurl.zig");+const crypto = @import("../sys/libcrypto.zig");const IpFilter = @import("IpFilter.zig");const log = @import("lightpanda").log;@@ -368,7 +369,7 @@ pub const Connection = struct {};pub fn init(- ca_blob: ?libcurl.CurlBlob,+ x509_store: *crypto.X509_STORE,config: *const Config,ip_filter: ?*const IpFilter,) !Connection {@@ -377,7 +378,7 @@ pub const Connection = struct {var self = Connection{ ._easy = easy, .in_use = false, .transport = .none };errdefer self.deinit();- try self.reset(config, ca_blob, ip_filter);+ try self.reset(config, x509_store, ip_filter);return self;}@@ -501,7 +502,7 @@ pub const Connection = struct {pub fn reset(self: *Connection,config: *const Config,- ca_blob: ?libcurl.CurlBlob,+ x509_store: *crypto.X509_STORE,ip_filter: ?*const IpFilter,) !void {libcurl.curl_easy_reset(self._easy);@@ -524,15 +525,28 @@ pub const Connection = struct {try libcurl.curl_easy_setopt(self._easy, .proxy, null);}- // tls- if (ca_blob) |ca| {- try libcurl.curl_easy_setopt(self._easy, .ca_info_blob, ca);- if (http_proxy != null) {- try libcurl.curl_easy_setopt(self._easy, .proxy_ca_info_blob, ca);- }+ // TLS.+ if (config.tlsVerifyHost()) {+ // Provide certificate store to connection's SSL_CTX.+ try libcurl.curl_easy_setopt(self._easy, .ssl_ctx_function, &(struct {+ fn wrap(+ _: *libcurl.Curl,+ raw_ssl_ctx: *anyopaque,+ raw_x509_store: *anyopaque,+ ) callconv(.c) libcurl.CurlCode {+ const ssl_ctx: *crypto.SSL_CTX = @ptrCast(raw_ssl_ctx);+ const store: *crypto.X509_STORE = @ptrCast(raw_x509_store);++ const result = crypto.SSL_CTX_set1_verify_cert_store(ssl_ctx, store);+ if (result != 1) {+ return libcurl.CURLE.ABORTED_BY_CALLBACK;+ }+ return libcurl.CURLE.OK;+ }+ }).wrap);+ // Pass our store to CURLOPT_SSL_CTX_FUNCTION.+ try libcurl.curl_easy_setopt(self._easy, .ssl_ctx_data, x509_store);} else {- assert(config.tlsVerifyHost() == false, "Http.init tls_verify_host", .{});-try libcurl.curl_easy_setopt(self._easy, .ssl_verify_host, false);try libcurl.curl_easy_setopt(self._easy, .ssl_verify_peer, false);diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zigindex 769ae9c11..c0a56c587 100644--- a/src/telemetry/lightpanda.zig+++ b/src/telemetry/lightpanda.zig@@ -132,7 +132,7 @@ pub fn send(self: *LightPanda, raw_event: telemetry.Event) !void {fn run(self: *LightPanda) void {// The connection is created, owned, and torn down entirely on this thread;// the network thread never sees it (Transport == .none).- var conn = http.Connection.init(self.network.ca_blob, self.network.config, self.network.ip_filter) catch |err| {+ var conn = http.Connection.init(self.network.x509_store, self.network.config, self.network.ip_filter) catch |err| {// Essentially OOM — the process is already in trouble. The thread// handle stays set so send() won't respawn; events drop at the cap.log.warn(.telemetry, "connection init", .{ .err = err });