Lightpanda: the headless browser designed for AI and automation
http: Use a curl_share for DNS lookups
Karl Seguindf2469238f1f
diff --git a/src/network/Network.zig b/src/network/Network.zigindex b27dc4329..2b05f93cb 100644--- a/src/network/Network.zig+++ b/src/network/Network.zig@@ -59,6 +59,8 @@ ws_count: usize = 0,ws_max: u8,ws_mutex: std.Io.Mutex = .init,+dns_share: *http.DnsShare,+/// Optional IP filter for blocking requests to private/internal networks (--block-private-networks).ip_filter: ?*IpFilter = null,@@ -92,6 +94,9 @@ pub fn init(app: *App) !Network {allocator.destroy(f);};+ const dns_share = try http.DnsShare.init(allocator);+ errdefer dns_share.deinit(allocator);+const count: usize = config.httpMaxConcurrent();const connections = try allocator.alloc(http.Connection, count);errdefer allocator.free(connections);@@ -99,6 +104,7 @@ pub fn init(app: *App) !Network {var available: DoublyLinkedList = .{};for (0..count) |i| {connections[i] = try http.Connection.init(certificates, config, ip_filter);+ try connections[i].setDnsShare(dns_share);available.append(&connections[i].node);}@@ -133,6 +139,7 @@ pub fn init(app: *App) !Network {.ws_max = config.wsMaxConcurrent(),.ip_filter = ip_filter,+ .dns_share = dns_share,};}@@ -146,6 +153,9 @@ pub fn deinit(self: *Network) void {self.ws_pool.deinit(self.allocator);+ // after every easy that uses it+ self.dns_share.deinit(self.allocator);+self.robot_store.deinit();self.cors_store.deinit();@@ -180,10 +190,7 @@ pub fn releaseConnection(self: *Network, conn: *http.Connection) void {switch (conn.transport) {.websocket => {conn.deinit();- self.ws_mutex.lockUncancelable(lp.io);- defer self.ws_mutex.unlock(lp.io);- self.ws_pool.destroy(conn);- self.ws_count -= 1;+ self.destroyWsConnection(conn);},else => {conn.reset(self.config, self.certificates, self.ip_filter) catch |err| {@@ -212,17 +219,25 @@ pub fn newConnection(self: *Network) ?*http.Connection {// don't do this under lockconn.* = http.Connection.init(self.certificates, self.config, self.ip_filter) catch {- self.ws_mutex.lockUncancelable(lp.io);- defer self.ws_mutex.unlock(lp.io);- self.ws_pool.destroy(conn);- self.ws_count -= 1;-+ self.destroyWsConnection(conn);+ return null;+ };+ conn.setDnsShare(self.dns_share) catch {+ conn.deinit();+ self.destroyWsConnection(conn);return null;};return conn;}+fn destroyWsConnection(self: *Network, conn: *http.Connection) void {+ self.ws_mutex.lockUncancelable(lp.io);+ defer self.ws_mutex.unlock(lp.io);+ self.ws_pool.destroy(conn);+ self.ws_count -= 1;+}+pub fn HostHashMap(comptime V: type) type {return std.HashMapUnmanaged([]const u8, V, HostContext, 80);}@@ -252,3 +267,35 @@ const HostContext = struct {return std.ascii.eqlIgnoreCase(a, b);}};++const testing = @import("../testing.zig");+test "Network: easies on separate multis share one DNS cache" {+ const network = &testing.test_app.network;++ // curl_easy_perform drives each easy on its own internal multi, so without+ // the share each would have its own DNS cache.+ var teacher = try http.Connection.init(network.certificates, network.config, null);+ defer teacher.deinit();+ try teacher.setDnsShare(network.dns_share);++ var learner = try http.Connection.init(network.certificates, network.config, null);+ defer learner.deinit();+ try learner.setDnsShare(network.dns_share);++ var buf: [128]u8 = undefined;+ // .test never resolves, so the learner only connects if it reads the+ // entry the teacher's CURLOPT_RESOLVE put in the shared cache.+ const url = try std.fmt.bufPrintSentinel(&buf, "http://lpshare.test:{d}/json/version", .{testing.test_cdp_port}, 0);++ const entry = try std.fmt.allocPrintSentinel(testing.allocator, "lpshare.test:{d}:127.0.0.1", .{testing.test_cdp_port}, 0);+ defer testing.allocator.free(entry);+ const resolve = libcurl.curl_slist_append(null, entry) orelse return error.OutOfMemory;+ defer libcurl.curl_slist_free_all(resolve);++ try libcurl.curl_easy_setopt(teacher._easy, .resolve, resolve);+ try teacher.setURL(url);+ try testing.expectEqual(200, try teacher.perform());++ try learner.setURL(url);+ try testing.expectEqual(200, try learner.perform());+}diff --git a/src/network/http.zig b/src/network/http.zigindex d6f840f84..74c55034c 100644--- a/src/network/http.zig+++ b/src/network/http.zig@@ -30,6 +30,7 @@ const repeat = @import("../string.zig").repeat;const log = lp.log;const posix = std.posix;+const Allocator = std.mem.Allocator;// Tests run several processes, each with its own ephemeral test servers, while// fixtures keep addressing the fixed ports. Routes those to the real ones.@@ -74,7 +75,7 @@ pub const Header = struct {value: []const u8,};- pub fn normalize(self: Header, allocator: std.mem.Allocator) !Header {+ pub fn normalize(self: Header, allocator: Allocator) !Header {return .{.name = try std.ascii.allocLowerString(allocator, self.name),.value = try allocator.dupe(u8, self.value),@@ -281,6 +282,11 @@ pub const Connection = struct {return self;}+ // Survives curl_easy_reset, so once per easy is enough.+ pub fn setDnsShare(self: *const Connection, share: *const DnsShare) !void {+ try libcurl.curl_easy_setopt(self._easy, .share, share.handle);+ }+pub fn deinit(self: *Connection) void {self.clearHeaders();libcurl.curl_easy_cleanup(self._easy);@@ -348,7 +354,7 @@ pub const Connection = struct {// "name;" when the value is empty — curl reads "name:" as// remove-this-header and the ";" form as send-an-empty-value. curl// copies the string, so `allocator` only backs the transient join.- pub fn addHeader(self: *Connection, allocator: std.mem.Allocator, name: []const u8, value: []const u8) !void {+ pub fn addHeader(self: *Connection, allocator: Allocator, name: []const u8, value: []const u8) !void {const joined = if (value.len == 0)try allocator.printSentinel("{s};", .{name}, 0)else@@ -664,7 +670,7 @@ pub const Connection = struct {}// Copies the response headers, names lowercased, into `allocator`.- pub fn collectResponseHeaders(self: *const Connection, allocator: std.mem.Allocator) ![]const Header {+ pub fn collectResponseHeaders(self: *const Connection, allocator: Allocator) ![]const Header {var list: std.ArrayList(Header) = .empty;var prev: ?*libcurl.CurlHeader = null;while (libcurl.curl_easy_nextheader(self._easy, .header, -1, prev)) |h| {@@ -719,6 +725,60 @@ pub const Connection = struct {}};+// A DNS cache, using curl_share. Connection sharing isn't support and TLS+// is limited to 25 hosts (so caching it would be worse than then 25 host per+// multi we currently have).+pub const DnsShare = struct {+ handle: *libcurl.CurlSH,+ dns_mutex: std.Io.Mutex = .init,+ share_mutex: std.Io.Mutex = .init,++ pub fn init(allocator: Allocator) !*DnsShare {+ // needs to be on the heap, libcurl takes the address as user_data+ const self = try allocator.create(DnsShare);+ errdefer allocator.destroy(self);++ const handle = libcurl.curl_share_init() orelse return error.FailedToInitializeShare;+ errdefer libcurl.curl_share_cleanup(handle) catch {};++ self.* = .{ .handle = handle };+ try libcurl.curl_share_setopt(handle, .share, .dns);+ try libcurl.curl_share_setopt(handle, .lock_function, &lock);+ try libcurl.curl_share_setopt(handle, .unlock_function, &unlock);+ try libcurl.curl_share_setopt(handle, .user_data, self);+ return self;+ }++ // Every easy using the share must be cleaned up first.+ pub fn deinit(self: *DnsShare, allocator: Allocator) void {+ libcurl.curl_share_cleanup(self.handle) catch |err| {+ // An easy still points at us: leak rather than free under it.+ log.err(.http, "curl_share_cleanup", .{ .err = err });+ return;+ };+ allocator.destroy(self);+ }++ fn lock(_: ?*libcurl.Curl, data: libcurl.CurlLockData, _: libcurl.CurlLockAccess, ptr: ?*anyopaque) callconv(.c) void {+ const self: *DnsShare = @ptrCast(@alignCast(ptr));+ self.mutexFor(data).lockUncancelable(lp.io);+ }++ fn unlock(_: ?*libcurl.Curl, data: libcurl.CurlLockData, ptr: ?*anyopaque) callconv(.c) void {+ const self: *DnsShare = @ptrCast(@alignCast(ptr));+ self.mutexFor(data).unlock(lp.io);+ }++ fn mutexFor(self: *DnsShare, data: libcurl.CurlLockData) *std.Io.Mutex {+ return switch (data) {+ .dns => &self.dns_mutex,+ .share => &self.share_mutex,+ // curl only locks the types we share+ _ => unreachable,+ };+ }+};+pub const Handles = struct {multi: *libcurl.CurlM,diff --git a/src/sys/libcurl.zig b/src/sys/libcurl.zigindex 29e74e40f..60a69b6d8 100644--- a/src/sys/libcurl.zig+++ b/src/sys/libcurl.zig@@ -25,6 +25,7 @@ const c = @import("curl");pub const Curl = c.CURL;pub const CurlM = c.CURLM;+pub const CurlSH = c.CURLSH;pub const CurlCode = c.CURLcode;pub const CurlSList = c.curl_slist;pub const CurlHeader = c.curl_header;@@ -234,6 +235,8 @@ const CurlOption = enum(c.CURLoption) {ssl_ctx_data = c.CURLOPT_SSL_CTX_DATA,ssl_ec_curves = c.CURLOPT_SSL_EC_CURVES,connect_to = c.CURLOPT_CONNECT_TO,+ share = c.CURLOPT_SHARE,+ resolve = c.CURLOPT_RESOLVE,};pub const CurlHttpVersion = enum(c_long) {@@ -469,6 +472,15 @@ const ErrorMulti = error{Unknown,};+const ErrorShare = error{+ BadOption,+ InUse,+ Invalid,+ OutOfMemory,+ NotBuiltIn,+ Unknown,+};+const ErrorHeader = error{OutOfMemory,BadArgument,@@ -645,6 +657,7 @@ pub fn curl_easy_setopt(easy: *Curl, comptime option: CurlOption, value: anytype.http_header,.connect_to,+ .resolve,=> @as(?*CurlSList, value),.private,@@ -655,6 +668,7 @@ pub fn curl_easy_setopt(easy: *Curl, comptime option: CurlOption, value: anytype=> @as(?*anyopaque, @ptrCast(value)),.ssl_ctx_data => @as(*crypto.X509_STORE, value),+ .share => @as(?*CurlSH, value),.http_version => @as(c_long, @backingInt(@as(CurlHttpVersion, value))),@@ -755,6 +769,54 @@ pub fn curl_easy_nextheader(return @ptrCast(ptr);}+pub const CurlLockData = enum(c.curl_lock_data) {+ share = c.CURL_LOCK_DATA_SHARE,+ dns = c.CURL_LOCK_DATA_DNS,+ _,+};++pub const CurlLockAccess = c.curl_lock_access;+pub const CurlLockFunction = *const fn (?*Curl, CurlLockData, CurlLockAccess, ?*anyopaque) callconv(.c) void;+pub const CurlUnlockFunction = *const fn (?*Curl, CurlLockData, ?*anyopaque) callconv(.c) void;++const CurlSHOption = enum(c.CURLSHoption) {+ share = c.CURLSHOPT_SHARE,+ lock_function = c.CURLSHOPT_LOCKFUNC,+ unlock_function = c.CURLSHOPT_UNLOCKFUNC,+ user_data = c.CURLSHOPT_USERDATA,+};++pub fn curl_share_init() ?*CurlSH {+ return c.curl_share_init();+}++pub fn curl_share_cleanup(share: *CurlSH) ErrorShare!void {+ try errorSHCheck(c.curl_share_cleanup(share));+}++pub fn curl_share_setopt(share: *CurlSH, comptime option: CurlSHOption, value: anytype) ErrorShare!void {+ const opt: c.CURLSHoption = @backingInt(option);+ const code = switch (option) {+ .share => c.curl_share_setopt(share, opt, @as(c_int, @intCast(@backingInt(@as(CurlLockData, value))))),+ .lock_function => c.curl_share_setopt(share, opt, @as(CurlLockFunction, value)),+ .unlock_function => c.curl_share_setopt(share, opt, @as(CurlUnlockFunction, value)),+ .user_data => c.curl_share_setopt(share, opt, @as(?*anyopaque, @ptrCast(value))),+ };+ try errorSHCheck(code);+}++fn errorSHCheck(code: c.CURLSHcode) ErrorShare!void {+ return switch (code) {+ c.CURLSHE_OK => {},+ c.CURLSHE_BAD_OPTION => ErrorShare.BadOption,+ c.CURLSHE_IN_USE => ErrorShare.InUse,+ c.CURLSHE_INVALID => ErrorShare.Invalid,+ c.CURLSHE_NOMEM => ErrorShare.OutOfMemory,+ c.CURLSHE_NOT_BUILT_IN => ErrorShare.NotBuiltIn,+ else => ErrorShare.Unknown,+ };+}+pub fn curl_multi_init() ?*CurlM {return c.curl_multi_init();}