Skip to content

Lightpanda: the headless browser designed for AI and automation

http: Use a curl_share for DNS lookups

Karl Seguindf2469238f1f
diff --git a/src/network/Network.zig b/src/network/Network.zig
index b27dc4329..2b05f93cb 100644
--- a/src/network/Network.zig
+++ b/src/network/Network.zig
@@ -59,6 +59,8 @@ ws_count: usize = 0,
ws_max: u8,
ws_mutex: std.Io.Mutex = .init,
+dns_share: *http.DnsShare,
+
/// Optional IP filter for blocking requests to private/internal networks (--block-private-networks).
ip_filter: ?*IpFilter = null,
@@ -92,6 +94,9 @@ pub fn init(app: *App) !Network {
allocator.destroy(f);
};
+ const dns_share = try http.DnsShare.init(allocator);
+ errdefer dns_share.deinit(allocator);
+
const count: usize = config.httpMaxConcurrent();
const connections = try allocator.alloc(http.Connection, count);
errdefer allocator.free(connections);
@@ -99,6 +104,7 @@ pub fn init(app: *App) !Network {
var available: DoublyLinkedList = .{};
for (0..count) |i| {
connections[i] = try http.Connection.init(certificates, config, ip_filter);
+ try connections[i].setDnsShare(dns_share);
available.append(&connections[i].node);
}
@@ -133,6 +139,7 @@ pub fn init(app: *App) !Network {
.ws_max = config.wsMaxConcurrent(),
.ip_filter = ip_filter,
+ .dns_share = dns_share,
};
}
@@ -146,6 +153,9 @@ pub fn deinit(self: *Network) void {
self.ws_pool.deinit(self.allocator);
+ // after every easy that uses it
+ self.dns_share.deinit(self.allocator);
+
self.robot_store.deinit();
self.cors_store.deinit();
@@ -180,10 +190,7 @@ pub fn releaseConnection(self: *Network, conn: *http.Connection) void {
switch (conn.transport) {
.websocket => {
conn.deinit();
- self.ws_mutex.lockUncancelable(lp.io);
- defer self.ws_mutex.unlock(lp.io);
- self.ws_pool.destroy(conn);
- self.ws_count -= 1;
+ self.destroyWsConnection(conn);
},
else => {
conn.reset(self.config, self.certificates, self.ip_filter) catch |err| {
@@ -212,17 +219,25 @@ pub fn newConnection(self: *Network) ?*http.Connection {
// don't do this under lock
conn.* = http.Connection.init(self.certificates, self.config, self.ip_filter) catch {
- self.ws_mutex.lockUncancelable(lp.io);
- defer self.ws_mutex.unlock(lp.io);
- self.ws_pool.destroy(conn);
- self.ws_count -= 1;
-
+ self.destroyWsConnection(conn);
+ return null;
+ };
+ conn.setDnsShare(self.dns_share) catch {
+ conn.deinit();
+ self.destroyWsConnection(conn);
return null;
};
return conn;
}
+fn destroyWsConnection(self: *Network, conn: *http.Connection) void {
+ self.ws_mutex.lockUncancelable(lp.io);
+ defer self.ws_mutex.unlock(lp.io);
+ self.ws_pool.destroy(conn);
+ self.ws_count -= 1;
+}
+
pub fn HostHashMap(comptime V: type) type {
return std.HashMapUnmanaged([]const u8, V, HostContext, 80);
}
@@ -252,3 +267,35 @@ const HostContext = struct {
return std.ascii.eqlIgnoreCase(a, b);
}
};
+
+const testing = @import("../testing.zig");
+test "Network: easies on separate multis share one DNS cache" {
+ const network = &testing.test_app.network;
+
+ // curl_easy_perform drives each easy on its own internal multi, so without
+ // the share each would have its own DNS cache.
+ var teacher = try http.Connection.init(network.certificates, network.config, null);
+ defer teacher.deinit();
+ try teacher.setDnsShare(network.dns_share);
+
+ var learner = try http.Connection.init(network.certificates, network.config, null);
+ defer learner.deinit();
+ try learner.setDnsShare(network.dns_share);
+
+ var buf: [128]u8 = undefined;
+ // .test never resolves, so the learner only connects if it reads the
+ // entry the teacher's CURLOPT_RESOLVE put in the shared cache.
+ const url = try std.fmt.bufPrintSentinel(&buf, "http://lpshare.test:{d}/json/version", .{testing.test_cdp_port}, 0);
+
+ const entry = try std.fmt.allocPrintSentinel(testing.allocator, "lpshare.test:{d}:127.0.0.1", .{testing.test_cdp_port}, 0);
+ defer testing.allocator.free(entry);
+ const resolve = libcurl.curl_slist_append(null, entry) orelse return error.OutOfMemory;
+ defer libcurl.curl_slist_free_all(resolve);
+
+ try libcurl.curl_easy_setopt(teacher._easy, .resolve, resolve);
+ try teacher.setURL(url);
+ try testing.expectEqual(200, try teacher.perform());
+
+ try learner.setURL(url);
+ try testing.expectEqual(200, try learner.perform());
+}
diff --git a/src/network/http.zig b/src/network/http.zig
index d6f840f84..74c55034c 100644
--- a/src/network/http.zig
+++ b/src/network/http.zig
@@ -30,6 +30,7 @@ const repeat = @import("../string.zig").repeat;
const log = lp.log;
const posix = std.posix;
+const Allocator = std.mem.Allocator;
// Tests run several processes, each with its own ephemeral test servers, while
// fixtures keep addressing the fixed ports. Routes those to the real ones.
@@ -74,7 +75,7 @@ pub const Header = struct {
value: []const u8,
};
- pub fn normalize(self: Header, allocator: std.mem.Allocator) !Header {
+ pub fn normalize(self: Header, allocator: Allocator) !Header {
return .{
.name = try std.ascii.allocLowerString(allocator, self.name),
.value = try allocator.dupe(u8, self.value),
@@ -281,6 +282,11 @@ pub const Connection = struct {
return self;
}
+ // Survives curl_easy_reset, so once per easy is enough.
+ pub fn setDnsShare(self: *const Connection, share: *const DnsShare) !void {
+ try libcurl.curl_easy_setopt(self._easy, .share, share.handle);
+ }
+
pub fn deinit(self: *Connection) void {
self.clearHeaders();
libcurl.curl_easy_cleanup(self._easy);
@@ -348,7 +354,7 @@ pub const Connection = struct {
// "name;" when the value is empty — curl reads "name:" as
// remove-this-header and the ";" form as send-an-empty-value. curl
// copies the string, so `allocator` only backs the transient join.
- pub fn addHeader(self: *Connection, allocator: std.mem.Allocator, name: []const u8, value: []const u8) !void {
+ pub fn addHeader(self: *Connection, allocator: Allocator, name: []const u8, value: []const u8) !void {
const joined = if (value.len == 0)
try allocator.printSentinel("{s};", .{name}, 0)
else
@@ -664,7 +670,7 @@ pub const Connection = struct {
}
// Copies the response headers, names lowercased, into `allocator`.
- pub fn collectResponseHeaders(self: *const Connection, allocator: std.mem.Allocator) ![]const Header {
+ pub fn collectResponseHeaders(self: *const Connection, allocator: Allocator) ![]const Header {
var list: std.ArrayList(Header) = .empty;
var prev: ?*libcurl.CurlHeader = null;
while (libcurl.curl_easy_nextheader(self._easy, .header, -1, prev)) |h| {
@@ -719,6 +725,60 @@ pub const Connection = struct {
}
};
+// A DNS cache, using curl_share. Connection sharing isn't support and TLS
+// is limited to 25 hosts (so caching it would be worse than then 25 host per
+// multi we currently have).
+pub const DnsShare = struct {
+ handle: *libcurl.CurlSH,
+ dns_mutex: std.Io.Mutex = .init,
+ share_mutex: std.Io.Mutex = .init,
+
+ pub fn init(allocator: Allocator) !*DnsShare {
+ // needs to be on the heap, libcurl takes the address as user_data
+ const self = try allocator.create(DnsShare);
+ errdefer allocator.destroy(self);
+
+ const handle = libcurl.curl_share_init() orelse return error.FailedToInitializeShare;
+ errdefer libcurl.curl_share_cleanup(handle) catch {};
+
+ self.* = .{ .handle = handle };
+ try libcurl.curl_share_setopt(handle, .share, .dns);
+ try libcurl.curl_share_setopt(handle, .lock_function, &lock);
+ try libcurl.curl_share_setopt(handle, .unlock_function, &unlock);
+ try libcurl.curl_share_setopt(handle, .user_data, self);
+ return self;
+ }
+
+ // Every easy using the share must be cleaned up first.
+ pub fn deinit(self: *DnsShare, allocator: Allocator) void {
+ libcurl.curl_share_cleanup(self.handle) catch |err| {
+ // An easy still points at us: leak rather than free under it.
+ log.err(.http, "curl_share_cleanup", .{ .err = err });
+ return;
+ };
+ allocator.destroy(self);
+ }
+
+ fn lock(_: ?*libcurl.Curl, data: libcurl.CurlLockData, _: libcurl.CurlLockAccess, ptr: ?*anyopaque) callconv(.c) void {
+ const self: *DnsShare = @ptrCast(@alignCast(ptr));
+ self.mutexFor(data).lockUncancelable(lp.io);
+ }
+
+ fn unlock(_: ?*libcurl.Curl, data: libcurl.CurlLockData, ptr: ?*anyopaque) callconv(.c) void {
+ const self: *DnsShare = @ptrCast(@alignCast(ptr));
+ self.mutexFor(data).unlock(lp.io);
+ }
+
+ fn mutexFor(self: *DnsShare, data: libcurl.CurlLockData) *std.Io.Mutex {
+ return switch (data) {
+ .dns => &self.dns_mutex,
+ .share => &self.share_mutex,
+ // curl only locks the types we share
+ _ => unreachable,
+ };
+ }
+};
+
pub const Handles = struct {
multi: *libcurl.CurlM,
diff --git a/src/sys/libcurl.zig b/src/sys/libcurl.zig
index 29e74e40f..60a69b6d8 100644
--- a/src/sys/libcurl.zig
+++ b/src/sys/libcurl.zig
@@ -25,6 +25,7 @@ const c = @import("curl");
pub const Curl = c.CURL;
pub const CurlM = c.CURLM;
+pub const CurlSH = c.CURLSH;
pub const CurlCode = c.CURLcode;
pub const CurlSList = c.curl_slist;
pub const CurlHeader = c.curl_header;
@@ -234,6 +235,8 @@ const CurlOption = enum(c.CURLoption) {
ssl_ctx_data = c.CURLOPT_SSL_CTX_DATA,
ssl_ec_curves = c.CURLOPT_SSL_EC_CURVES,
connect_to = c.CURLOPT_CONNECT_TO,
+ share = c.CURLOPT_SHARE,
+ resolve = c.CURLOPT_RESOLVE,
};
pub const CurlHttpVersion = enum(c_long) {
@@ -469,6 +472,15 @@ const ErrorMulti = error{
Unknown,
};
+const ErrorShare = error{
+ BadOption,
+ InUse,
+ Invalid,
+ OutOfMemory,
+ NotBuiltIn,
+ Unknown,
+};
+
const ErrorHeader = error{
OutOfMemory,
BadArgument,
@@ -645,6 +657,7 @@ pub fn curl_easy_setopt(easy: *Curl, comptime option: CurlOption, value: anytype
.http_header,
.connect_to,
+ .resolve,
=> @as(?*CurlSList, value),
.private,
@@ -655,6 +668,7 @@ pub fn curl_easy_setopt(easy: *Curl, comptime option: CurlOption, value: anytype
=> @as(?*anyopaque, @ptrCast(value)),
.ssl_ctx_data => @as(*crypto.X509_STORE, value),
+ .share => @as(?*CurlSH, value),
.http_version => @as(c_long, @backingInt(@as(CurlHttpVersion, value))),
@@ -755,6 +769,54 @@ pub fn curl_easy_nextheader(
return @ptrCast(ptr);
}
+pub const CurlLockData = enum(c.curl_lock_data) {
+ share = c.CURL_LOCK_DATA_SHARE,
+ dns = c.CURL_LOCK_DATA_DNS,
+ _,
+};
+
+pub const CurlLockAccess = c.curl_lock_access;
+pub const CurlLockFunction = *const fn (?*Curl, CurlLockData, CurlLockAccess, ?*anyopaque) callconv(.c) void;
+pub const CurlUnlockFunction = *const fn (?*Curl, CurlLockData, ?*anyopaque) callconv(.c) void;
+
+const CurlSHOption = enum(c.CURLSHoption) {
+ share = c.CURLSHOPT_SHARE,
+ lock_function = c.CURLSHOPT_LOCKFUNC,
+ unlock_function = c.CURLSHOPT_UNLOCKFUNC,
+ user_data = c.CURLSHOPT_USERDATA,
+};
+
+pub fn curl_share_init() ?*CurlSH {
+ return c.curl_share_init();
+}
+
+pub fn curl_share_cleanup(share: *CurlSH) ErrorShare!void {
+ try errorSHCheck(c.curl_share_cleanup(share));
+}
+
+pub fn curl_share_setopt(share: *CurlSH, comptime option: CurlSHOption, value: anytype) ErrorShare!void {
+ const opt: c.CURLSHoption = @backingInt(option);
+ const code = switch (option) {
+ .share => c.curl_share_setopt(share, opt, @as(c_int, @intCast(@backingInt(@as(CurlLockData, value))))),
+ .lock_function => c.curl_share_setopt(share, opt, @as(CurlLockFunction, value)),
+ .unlock_function => c.curl_share_setopt(share, opt, @as(CurlUnlockFunction, value)),
+ .user_data => c.curl_share_setopt(share, opt, @as(?*anyopaque, @ptrCast(value))),
+ };
+ try errorSHCheck(code);
+}
+
+fn errorSHCheck(code: c.CURLSHcode) ErrorShare!void {
+ return switch (code) {
+ c.CURLSHE_OK => {},
+ c.CURLSHE_BAD_OPTION => ErrorShare.BadOption,
+ c.CURLSHE_IN_USE => ErrorShare.InUse,
+ c.CURLSHE_INVALID => ErrorShare.Invalid,
+ c.CURLSHE_NOMEM => ErrorShare.OutOfMemory,
+ c.CURLSHE_NOT_BUILT_IN => ErrorShare.NotBuiltIn,
+ else => ErrorShare.Unknown,
+ };
+}
+
pub fn curl_multi_init() ?*CurlM {
return c.curl_multi_init();
}