Skip to content

Lightpanda: the headless browser designed for AI and automation

42.8 KB
Raw Blame
// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
//
// Francis Bouvier <[email protected]>
// Pierre Tachoire <[email protected]>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

const std = @import("std");
const lp = @import("lightpanda");

const Config = @import("../Config.zig");
const sys_net = @import("../sys/net.zig");
const libcurl = @import("../sys/libcurl.zig");
const crypto = @import("../sys/libcrypto.zig");

const IpFilter = @import("IpFilter.zig");
const Certificates = @import("Certificates.zig");
const repeat = @import("../string.zig").repeat;

const log = lp.log;
const posix = std.posix;
const Allocator = std.mem.Allocator;

// Tests run several processes, each with its own ephemeral test servers, while
// fixtures keep addressing the fixed ports. Routes those to the real ones.
pub var test_connect_to: ?*libcurl.CurlSList = null;

pub const readfunc_pause = libcurl.curl_readfunc_pause;
pub const writefunc_error = libcurl.curl_writefunc_error;
pub const WsFrameType = libcurl.WsFrameType;

const Error = libcurl.Error;

pub fn curl_version() [*c]const u8 {
    return libcurl.curl_version();
}

pub const Method = enum(u8) {
    GET = 0,
    PUT = 1,
    POST = 2,
    DELETE = 3,
    HEAD = 4,
    OPTIONS = 5,
    PATCH = 6,
    PROPFIND = 7,

    // The safe methods of RFC 9110 9.2.1 (we have no TRACE). PROPFIND is
    // read-only in practice but isn't on that list.
    pub fn isSafe(self: Method) bool {
        return switch (self) {
            .GET, .HEAD, .OPTIONS => true,
            .PUT, .POST, .DELETE, .PATCH, .PROPFIND => false,
        };
    }
};

pub const Header = struct {
    name: []const u8,
    value: []const u8,

    pub const Param = struct {
        key: []const u8,
        value: []const u8,
    };

    pub fn normalize(self: Header, allocator: Allocator) !Header {
        return .{
            .name = try std.ascii.allocLowerString(allocator, self.name),
            .value = try allocator.dupe(u8, self.value),
        };
    }

    pub fn parse(header_str: []const u8) ?Header {
        const colon_pos = std.mem.findScalar(u8, header_str, ':') orelse return null;

        const name = std.mem.trim(u8, header_str[0..colon_pos], " \t");
        const value = std.mem.trim(u8, header_str[colon_pos + 1 ..], " \t");

        return .{ .name = name, .value = value };
    }

    // The header value up to the first ';', trimmed (e.g. "attachment" for a
    // Content-Disposition, "text/html" for a Content-Type).
    pub fn firstValue(self: Header) []const u8 {
        const end = std.mem.findScalar(u8, self.value, ';') orelse self.value.len;
        return std.mem.trim(u8, self.value[0..end], " \t");
    }

    // Iterates the `; key=value` parameters that follow the header's first value.
    pub fn params(self: Header) ParamIterator {
        const start = std.mem.findScalar(u8, self.value, ';') orelse self.value.len;
        return .{ .rest = self.value[start..] };
    }

    // Returns the (unquoted) value of the first non-empty `key=` parameter, if any.
    pub fn param(self: Header, key: []const u8) ?[]const u8 {
        var it = self.params();
        while (it.next()) |p| {
            if (p.value.len > 0 and std.ascii.eqlIgnoreCase(p.key, key)) {
                return p.value;
            }
        }
        return null;
    }

    const ParamIterator = struct {
        rest: []const u8,

        pub fn next(self: *ParamIterator) ?Param {
            while (self.rest.len > 0 and self.rest[0] == ';') {
                self.rest = self.rest[1..];
                const end = std.mem.findScalar(u8, self.rest, ';') orelse self.rest.len;
                const segment = self.rest[0..end];
                self.rest = self.rest[end..];

                const eq = std.mem.findScalar(u8, segment, '=') orelse continue;
                const key = std.mem.trim(u8, segment[0..eq], " \t");
                if (key.len == 0) continue;

                var value = std.mem.trim(u8, segment[eq + 1 ..], " \t");
                if (value.len >= 2 and value[0] == '"' and value[value.len - 1] == '"') {
                    value = value[1 .. value.len - 1];
                }
                return .{ .key = key, .value = value };
            }
            return null;
        }
    };
};

const HeaderValue = struct {
    value: []const u8,
    amount: usize,
};

pub const AuthChallenge = struct {
    const Source = enum { server, proxy };
    const Scheme = enum { basic, digest };

    status: u16,
    source: ?Source,
    scheme: ?Scheme,
    realm: ?[]const u8,

    pub fn parse(status: u16, source: Source, value: []const u8) !AuthChallenge {
        var ac: AuthChallenge = .{
            .status = status,
            .source = source,
            .realm = null,
            .scheme = null,
        };

        const challenge_value = std.mem.trim(u8, value, std.ascii.whitespace[0..]);
        const pos = std.mem.findPos(u8, challenge_value, 0, " ") orelse challenge_value.len;
        const _scheme = challenge_value[0..pos];
        if (std.ascii.eqlIgnoreCase(_scheme, "basic")) {
            ac.scheme = .basic;
        } else if (std.ascii.eqlIgnoreCase(_scheme, "digest")) {
            ac.scheme = .digest;
        } else {
            return error.UnknownAuthChallengeScheme;
        }

        return ac;
    }
};

// The actual reason phrase from the server, verbatim. HTTP/2 has none, so "".
pub const StatusText = struct {
    pub const MAX_LEN = 128;

    _len: ?u8 = null,
    _buf: [MAX_LEN]u8 = undefined,

    pub fn fromStatusLine(line: []const u8) StatusText {
        const trimmed = std.mem.trimEnd(u8, line, "\r\n");
        // HTTP-version SP status-code SP [ reason-phrase ]
        const sp1 = std.mem.findScalar(u8, trimmed, ' ') orelse return .{ ._len = 0 };
        const sp2 = std.mem.findScalarPos(u8, trimmed, sp1 + 1, ' ') orelse return .{ ._len = 0 };
        const phrase = trimmed[sp2 + 1 ..];
        const len = @min(phrase.len, MAX_LEN);

        var st: StatusText = .{ ._len = @intCast(len) };
        @memcpy(st._buf[0..len], phrase[0..len]);
        return st;
    }

    pub fn get(self: *const StatusText) ?[]const u8 {
        const len = self._len orelse return null;
        return self._buf[0..len];
    }
};

pub const ResponseHead = struct {
    // Matches Mime.parse's 255-byte cap
    pub const MAX_CONTENT_TYPE_LEN = 255;

    status: u16,
    url: [*c]const u8,
    redirect_count: u32,
    _content_type_len: usize = 0,
    _content_type: [MAX_CONTENT_TYPE_LEN]u8 = undefined,

    pub fn contentType(self: *ResponseHead) ?[]u8 {
        if (self._content_type_len == 0) {
            return null;
        }
        return self._content_type[0..self._content_type_len];
    }
};

/// Opensocket callback: blocks connections to private/internal IP ranges
/// before TCP SYN, regardless of request origin (JS, HTML resources, redirects, etc.).
/// Called by curl after DNS resolution, before the socket is created.
/// Returns CURL_SOCKET_BAD to block; otherwise creates and returns a real socket fd.
/// clientp is a *const IpFilter passed via CURLOPT_OPENSOCKETDATA.
fn opensocketCallback(
    clientp: ?*anyopaque,
    _: c_uint,
    addr: [*c]libcurl.CurlSockAddr,
) callconv(.c) libcurl.CurlSocket {
    const address: *libcurl.CurlSockAddr = @ptrCast(addr);
    const filter: *const IpFilter = @ptrCast(@alignCast(clientp orelse return libcurl.CURL_SOCKET_BAD));

    if (filter.isBlockedSockaddr(address)) {
        if (address.family == posix.AF.INET or address.family == posix.AF.INET6) {
            const ip = sys_net.addressFromSockaddr(@ptrCast(@alignCast(&address.addr)));
            log.debug(.http, "blocked by IP filter", .{ .ip = ip });
        } else {
            log.debug(.http, "blocked by IP filter", .{ .family = address.family });
        }
        return libcurl.CURL_SOCKET_BAD;
    }

    const fd = sys_net.socket(
        @intCast(address.family),
        @intCast(address.socktype),
        @intCast(address.protocol),
    ) catch return libcurl.CURL_SOCKET_BAD;
    return fd;
}

pub const Connection = struct {
    _easy: *libcurl.Curl,
    transport: Transport,
    node: std.DoublyLinkedList.Node = .{},

    // The curl_slist accumulated by addHeader/addRawHeader and handed to
    // curl by commitHeaders. Owned here because curl reads it during
    // perform; freed on reset/deinit or by the next clearHeaders.
    _header_list: ?*libcurl.CurlSList = null,

    pub const Transport = union(enum) {
        none, // used for cases that manage their own connection, e.g. telemetry
        http: *@import("HttpClient.zig").Transfer,
        websocket: *@import("../browser/webapi/net/WebSocket.zig"),
    };

    pub fn init(
        certificates: Certificates,
        config: *const Config,
        ip_filter: ?*const IpFilter,
    ) !Connection {
        const easy = libcurl.curl_easy_init() orelse return error.FailedToInitializeEasy;

        var self = Connection{ ._easy = easy, .transport = .none };
        errdefer self.deinit();

        try self.reset(config, certificates, ip_filter);
        return self;
    }

    // Survives curl_easy_reset, so once per easy is enough.
    pub fn setDnsShare(self: *const Connection, share: *const DnsShare) !void {
        try libcurl.curl_easy_setopt(self._easy, .share, share.handle);
    }

    pub fn deinit(self: *Connection) void {
        self.clearHeaders();
        libcurl.curl_easy_cleanup(self._easy);
    }

    pub fn setURL(self: *const Connection, url: [:0]const u8) !void {
        try libcurl.curl_easy_setopt(self._easy, .url, url.ptr);
    }

    pub fn setTimeout(self: *const Connection, timeout_ms: u32) !void {
        try libcurl.curl_easy_setopt(self._easy, .timeout_ms, timeout_ms);
    }

    // a libcurl request has 2 methods. The first is the method that
    // controls how libcurl behaves. This specifically influences how redirects
    // are handled. For example, if you do a POST and get a 301, libcurl will
    // change that to a GET. But if you do a POST and get a 308, libcurl will
    // keep the POST (and re-send the body).
    // The second method is the actual string that's included in the request
    // headers.
    // These two methods can be different - you can tell curl to behave as though
    // you made a GET, but include "POST" in the request header.
    //
    // Here, we're only concerned about the 2nd method. If we want, we'll set
    // the first one based on whether or not we have a body.
    //
    // It's important that, for each use of this connection, we set the 2nd
    // method. Else, if we make a HEAD request and re-use the connection, but
    // DON'T reset this, it'll keep making HEAD requests.
    // (I don't know if it's as important to reset the 1st method, or if libcurl
    // can infer that based on the presence of the body, but we also reset it
    // to be safe);
    pub fn setMethod(self: *const Connection, method: Method) !void {
        const easy = self._easy;
        const m: [:0]const u8 = switch (method) {
            .GET => "GET",
            .POST => "POST",
            .PUT => "PUT",
            .DELETE => "DELETE",
            .HEAD => "HEAD",
            .OPTIONS => "OPTIONS",
            .PATCH => "PATCH",
            .PROPFIND => "PROPFIND",
        };
        try libcurl.curl_easy_setopt(easy, .custom_request, m.ptr);
    }

    pub fn setBody(self: *const Connection, body: []const u8) !void {
        const easy = self._easy;
        try libcurl.curl_easy_setopt(easy, .post, true);
        try libcurl.curl_easy_setopt(easy, .post_field_size, body.len);
        try libcurl.curl_easy_setopt(easy, .copy_post_fields, body.ptr);
    }

    pub fn setNoBody(self: *const Connection) !void {
        const easy = self._easy;
        try libcurl.curl_easy_setopt(easy, .no_body, true);
    }

    pub fn setGetMode(self: *const Connection) !void {
        try libcurl.curl_easy_setopt(self._easy, .http_get, true);
    }

    // Appends "name: value" to the connection's pending header list, or
    // "name;" when the value is empty — curl reads "name:" as
    // remove-this-header and the ";" form as send-an-empty-value. curl
    // copies the string, so `allocator` only backs the transient join.
    pub fn addHeader(self: *Connection, allocator: Allocator, name: []const u8, value: []const u8) !void {
        const joined = if (value.len == 0)
            try allocator.printSentinel("{s};", .{name}, 0)
        else
            try allocator.printSentinel("{s}: {s}", .{ name, value }, 0);
        return self.addRawHeader(joined);
    }

    // `header` is raw curl header syntax, e.g. "Expect:" to suppress a
    // curl-generated header. curl copies the value.
    pub fn addRawHeader(self: *Connection, header: [*c]const u8) !void {
        const updated = libcurl.curl_slist_append(self._header_list, header);
        if (updated == null) {
            return error.OutOfMemory;
        }
        self._header_list = updated;
    }

    pub fn commitHeaders(self: *const Connection) !void {
        try libcurl.curl_easy_setopt(self._easy, .http_header, self._header_list);
    }

    pub fn clearHeaders(self: *Connection) void {
        if (self._header_list) |list| {
            libcurl.curl_slist_free_all(list);
            self._header_list = null;
        }
    }

    pub fn setCookies(self: *const Connection, cookies: [*c]const u8) !void {
        try libcurl.curl_easy_setopt(self._easy, .cookie, cookies);
    }

    pub fn setPrivate(self: *const Connection, ptr: *anyopaque) !void {
        try libcurl.curl_easy_setopt(self._easy, .private, ptr);
    }

    pub fn setProxyCredentials(self: *const Connection, creds: [:0]const u8) !void {
        try libcurl.curl_easy_setopt(self._easy, .proxy_user_pwd, creds.ptr);
    }

    pub fn setCredentials(self: *const Connection, creds: [:0]const u8) !void {
        try libcurl.curl_easy_setopt(self._easy, .user_pwd, creds.ptr);
    }

    pub fn setConnectOnly(self: *const Connection, connect_only: bool) !void {
        const value: c_long = if (connect_only) 2 else 0;
        try libcurl.curl_easy_setopt(self._easy, .connect_only, value);
    }

    // Keep a proxy's CONNECT response out of the header callback, so it only
    // sees the origin's.
    pub fn setSuppressConnectHeaders(self: *const Connection) !void {
        try libcurl.curl_easy_setopt(self._easy, .suppress_connect_headers, true);
    }

    // Close this connection when the transfer ends instead of returning it to
    // libcurl's keepalive pool. Read by libcurl when the transfer completes,
    // so it can be set while the response is being received.
    pub fn setForbidReuse(self: *const Connection) !void {
        try libcurl.curl_easy_setopt(self._easy, .forbid_reuse, true);
    }

    pub fn setWriteCallback(
        self: *Connection,
        comptime data_cb: libcurl.CurlWriteFunction,
    ) !void {
        try libcurl.curl_easy_setopt(self._easy, .write_data, self);
        try libcurl.curl_easy_setopt(self._easy, .write_function, data_cb);
    }

    pub fn setReadCallback(
        self: *Connection,
        comptime data_cb: libcurl.CurlReadFunction,
        upload: bool,
    ) !void {
        try libcurl.curl_easy_setopt(self._easy, .read_data, self);
        try libcurl.curl_easy_setopt(self._easy, .read_function, data_cb);
        if (upload) {
            try libcurl.curl_easy_setopt(self._easy, .upload, true);
        }
    }

    pub fn setHeaderCallback(
        self: *Connection,
        comptime data_cb: libcurl.CurlHeaderFunction,
    ) !void {
        try libcurl.curl_easy_setopt(self._easy, .header_data, self);
        try libcurl.curl_easy_setopt(self._easy, .header_function, data_cb);
    }

    pub fn pause(
        self: *Connection,
        flags: libcurl.CurlPauseFlags,
    ) !void {
        try libcurl.curl_easy_pause(self._easy, flags);
    }

    pub fn reset(
        self: *Connection,
        config: *const Config,
        certificates: Certificates,
        ip_filter: ?*const IpFilter,
    ) !void {
        libcurl.curl_easy_reset(self._easy);
        self.transport = .none;
        self.clearHeaders();

        // timeouts
        try libcurl.curl_easy_setopt(self._easy, .timeout_ms, config.httpTimeout());
        try libcurl.curl_easy_setopt(self._easy, .connect_timeout_ms, config.httpConnectTimeout());

        // Otherwise requests issued before ALPN settles each open a socket.
        try libcurl.curl_easy_setopt(self._easy, .pipewait, true);

        // compression, don't remove this. CloudFront will send gzip content
        // even if we don't support it, and then it won't be decompressed.
        // empty string means: use whatever's available
        try libcurl.curl_easy_setopt(self._easy, .accept_encoding, "");

        if (comptime lp.IS_TEST) {
            try libcurl.curl_easy_setopt(self._easy, .connect_to, test_connect_to);
        }

        // proxy
        const http_proxy = config.httpProxy();
        if (http_proxy) |proxy| {
            try libcurl.curl_easy_setopt(self._easy, .proxy, proxy.ptr);
        } else {
            try libcurl.curl_easy_setopt(self._easy, .proxy, null);
        }

        // TLS.
        if (config.tlsVerifyHost()) {
            // Provide certificate store to connection's SSL_CTX.
            try libcurl.curl_easy_setopt(self._easy, .ssl_ctx_function, &(struct {
                fn wrap(
                    _: *libcurl.Curl,
                    raw_ssl_ctx: *anyopaque,
                    raw_x509_store: *anyopaque,
                ) callconv(.c) libcurl.CurlCode {
                    const ssl_ctx: *crypto.SSL_CTX = @ptrCast(raw_ssl_ctx);
                    const store: *crypto.X509_STORE = @ptrCast(raw_x509_store);

                    const result = crypto.SSL_CTX_set1_verify_cert_store(ssl_ctx, store);
                    if (result != 1) {
                        return libcurl.CURLE.ABORTED_BY_CALLBACK;
                    }
                    return libcurl.CURLE.OK;
                }
            }).wrap);
            // Pass our store to CURLOPT_SSL_CTX_FUNCTION.
            try libcurl.curl_easy_setopt(self._easy, .ssl_ctx_data, certificates.store);
        } else {
            try libcurl.curl_easy_setopt(self._easy, .ssl_verify_host, false);
            try libcurl.curl_easy_setopt(self._easy, .ssl_verify_peer, false);

            if (http_proxy != null) {
                try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_host, false);
                try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_peer, false);
            }
        }

        if (config.httpCurves()) |list| {
            try libcurl.curl_easy_setopt(self._easy, .ssl_ec_curves, list.ptr);
        }

        if (config.httpDebug()) {
            try libcurl.curl_easy_setopt(self._easy, .verbose, true);
        }

        // Sometimes the default debug output hides some useful data. You can
        // uncomment the following line (BUT KEEP THE LINE ABOVE AS-IS), to
        // get more control over the data (specifically, the `CURLINFO_TEXT`
        // can include useful data).
        // try libcurl.curl_easy_setopt(easy, .debug_function, debugCallback);

        // default write callback to prevent libcurl from writing to stdout
        try self.setWriteCallback(discardBody);

        // IP filter: block private/internal network addresses
        if (ip_filter) |filter| {
            try libcurl.curl_easy_setopt(self._easy, .opensocket_function, opensocketCallback);
            try libcurl.curl_easy_setopt(self._easy, .opensocket_data, @constCast(filter));
        }
    }

    fn discardBody(_: [*]const u8, count: usize, len: usize, _: ?*anyopaque) callconv(.c) usize {
        return count * len;
    }

    pub fn setProxy(self: *const Connection, proxy: ?[:0]const u8) !void {
        try libcurl.curl_easy_setopt(self._easy, .proxy, if (proxy) |p| p.ptr else null);
    }

    pub fn setHttpVersion(self: *const Connection, version: Config.HttpVersion) !void {
        const v: libcurl.CurlHttpVersion = switch (version) {
            .auto => .none,
            .@"1.1" => .v1_1,
        };
        try libcurl.curl_easy_setopt(self._easy, .http_version, v);
    }

    pub fn setFollowLocation(self: *const Connection, follow: bool) !void {
        try libcurl.curl_easy_setopt(self._easy, .follow_location, @as(c_long, if (follow) 2 else 0));
    }

    pub fn setTlsVerify(self: *const Connection, verify: bool, use_proxy: bool) !void {
        try libcurl.curl_easy_setopt(self._easy, .ssl_verify_host, verify);
        try libcurl.curl_easy_setopt(self._easy, .ssl_verify_peer, verify);
        if (use_proxy) {
            try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_host, verify);
            try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_peer, verify);
        }
    }

    pub fn getEffectiveUrl(self: *const Connection) ![*c]const u8 {
        var url: [*c]u8 = undefined;
        try libcurl.curl_easy_getinfo(self._easy, .effective_url, &url);
        return url;
    }

    pub fn getConnectCode(self: *const Connection) !u16 {
        var status: c_long = undefined;
        try libcurl.curl_easy_getinfo(self._easy, .connect_code, &status);
        return inHttpRange(status);
    }

    pub fn getResponseCode(self: *const Connection) !u16 {
        var status: c_long = undefined;
        try libcurl.curl_easy_getinfo(self._easy, .response_code, &status);
        return inHttpRange(status);
    }

    /// 0 outside 100..599, which is curl's own value for having no status.
    /// Consumers cast this into `std.http.Status`, an enum(u10).
    fn inHttpRange(status: c_long) u16 {
        if (status < 100 or status > 599) return 0;
        return @intCast(status);
    }

    // -1 when the transfer used no connection.
    pub fn getConnId(self: *const Connection) !c_long {
        var conn_id: c_long = undefined;
        try libcurl.curl_easy_getinfo(self._easy, .conn_id, &conn_id);
        return conn_id;
    }

    pub fn isConnReused(self: *const Connection) !bool {
        var opened: c_long = undefined;
        try libcurl.curl_easy_getinfo(self._easy, .num_connects, &opened);
        return opened == 0;
    }

    // Total transfer time (name lookup to completion) in microseconds.
    pub fn getTotalTimeMicros(self: *const Connection) !c_long {
        var micros: c_long = undefined;
        try libcurl.curl_easy_getinfo(self._easy, .total_time_t, &micros);
        return micros;
    }

    // microseconds from the moment the transfer started. Reused connections
    // will report zero (or almost zero) for namelookup/connect/appconnect.
    pub const Timing = struct {
        queue: u64,
        namelookup: u64,
        connect: u64,
        appconnect: u64,
        pretransfer: u64,
        starttransfer: u64,
        total: u64,
    };

    pub fn getTiming(self: *const Connection) !Timing {
        return .{
            .queue = try self.getInfoMicros(.queue_time_t),
            .namelookup = try self.getInfoMicros(.namelookup_time_t),
            .connect = try self.getInfoMicros(.connect_time_t),
            .appconnect = try self.getInfoMicros(.appconnect_time_t),
            .pretransfer = try self.getInfoMicros(.pretransfer_time_t),
            .starttransfer = try self.getInfoMicros(.starttransfer_time_t),
            .total = try self.getInfoMicros(.total_time_t),
        };
    }

    fn getInfoMicros(self: *const Connection, comptime info: libcurl.CurlInfo) !u64 {
        var micros: c_long = undefined;
        try libcurl.curl_easy_getinfo(self._easy, info, &micros);
        return @intCast(@max(0, micros));
    }

    pub fn getDownloadSize(self: *const Connection) !u64 {
        var size: c_long = undefined;
        try libcurl.curl_easy_getinfo(self._easy, .size_download_t, &size);
        return @intCast(@max(0, size));
    }

    // Bytes of every response header received, status lines included.
    pub fn getHeaderSize(self: *const Connection) !u64 {
        var size: c_long = undefined;
        try libcurl.curl_easy_getinfo(self._easy, .header_size, &size);
        return @intCast(@max(0, size));
    }

    pub fn getHttpVersion(self: *const Connection) !libcurl.CurlHttpVersion {
        var version: c_long = undefined;
        try libcurl.curl_easy_getinfo(self._easy, .http_version, &version);
        return @fromBackingInt(@intCast(version));
    }

    pub fn getConnectHeader(self: *const Connection, name: [:0]const u8, index: usize) ?HeaderValue {
        var hdr: ?*libcurl.CurlHeader = null;
        libcurl.curl_easy_header(self._easy, name, index, .connect, -1, &hdr) catch |err| {
            // ErrorHeader includes OutOfMemory — rare but real errors from curl internals.
            // Logged and returned as null since callers don't expect errors.
            log.err(.http, "get response header", .{
                .name = name,
                .err = err,
            });
            return null;
        };
        const h = hdr orelse return null;
        return .{
            .amount = h.amount,
            .value = std.mem.span(h.value),
        };
    }

    // Copies the response headers, names lowercased, into `allocator`.
    pub fn collectResponseHeaders(self: *const Connection, allocator: Allocator) ![]const Header {
        var list: std.ArrayList(Header) = .empty;
        var prev: ?*libcurl.CurlHeader = null;
        while (libcurl.curl_easy_nextheader(self._easy, .header, -1, prev)) |h| {
            prev = h;
            const hdr: Header = .{
                .name = std.mem.span(h.name),
                .value = std.mem.span(h.value),
            };
            try list.append(allocator, try hdr.normalize(allocator));
        }
        return list.items;
    }

    pub fn getResponseHeader(self: *const Connection, name: [:0]const u8, index: usize) ?HeaderValue {
        var hdr: ?*libcurl.CurlHeader = null;
        libcurl.curl_easy_header(self._easy, name, index, .header, -1, &hdr) catch |err| {
            // ErrorHeader includes OutOfMemory — rare but real errors from curl internals.
            // Logged and returned as null since callers don't expect errors.
            log.err(.http, "get response header", .{
                .name = name,
                .err = err,
            });
            return null;
        };
        const h = hdr orelse return null;
        return .{
            .amount = h.amount,
            .value = std.mem.span(h.value),
        };
    }

    // These are headers that may not be send to the users for inteception.
    pub fn secretHeaders(self: *Connection, http_headers: *const Config.HttpHeaders) !void {
        if (http_headers.proxy_bearer_header) |hdr| {
            try self.addRawHeader(hdr);
        }
    }

    // Synchronous transfer that adds no request headers; callers that manage
    // their own connection (telemetry) use this leaner path.
    pub fn perform(self: *const Connection) !u16 {
        try libcurl.curl_easy_perform(self._easy);
        return self.getResponseCode();
    }

    pub fn wsStartFrame(self: *const Connection, frame_type: libcurl.WsFrameType, size: usize) !void {
        try libcurl.curl_ws_start_frame(self._easy, frame_type, @intCast(size));
    }

    pub fn wsMeta(self: *const Connection) ?libcurl.WsFrameMeta {
        return libcurl.curl_ws_meta(self._easy);
    }
};

// A DNS cache, using curl_share. Connection sharing isn't support and TLS
// is limited to 25 hosts (so caching it would be worse than then 25 host per
// multi we currently have).
pub const DnsShare = struct {
    handle: *libcurl.CurlSH,
    dns_mutex: std.Io.Mutex = .init,
    share_mutex: std.Io.Mutex = .init,

    pub fn init(allocator: Allocator) !*DnsShare {
        // needs to be on the heap, libcurl takes the address as user_data
        const self = try allocator.create(DnsShare);
        errdefer allocator.destroy(self);

        const handle = libcurl.curl_share_init() orelse return error.FailedToInitializeShare;
        errdefer libcurl.curl_share_cleanup(handle) catch {};

        self.* = .{ .handle = handle };
        try libcurl.curl_share_setopt(handle, .share, .dns);
        try libcurl.curl_share_setopt(handle, .lock_function, &lock);
        try libcurl.curl_share_setopt(handle, .unlock_function, &unlock);
        try libcurl.curl_share_setopt(handle, .user_data, self);
        return self;
    }

    // Every easy using the share must be cleaned up first.
    pub fn deinit(self: *DnsShare, allocator: Allocator) void {
        libcurl.curl_share_cleanup(self.handle) catch |err| {
            // An easy still points at us: leak rather than free under it.
            log.err(.http, "curl_share_cleanup", .{ .err = err });
            return;
        };
        allocator.destroy(self);
    }

    fn lock(_: ?*libcurl.Curl, data: libcurl.CurlLockData, _: libcurl.CurlLockAccess, ptr: ?*anyopaque) callconv(.c) void {
        const self: *DnsShare = @ptrCast(@alignCast(ptr));
        self.mutexFor(data).lockUncancelable(lp.io);
    }

    fn unlock(_: ?*libcurl.Curl, data: libcurl.CurlLockData, ptr: ?*anyopaque) callconv(.c) void {
        const self: *DnsShare = @ptrCast(@alignCast(ptr));
        self.mutexFor(data).unlock(lp.io);
    }

    fn mutexFor(self: *DnsShare, data: libcurl.CurlLockData) *std.Io.Mutex {
        return switch (data) {
            .dns => &self.dns_mutex,
            .share => &self.share_mutex,
            // curl only locks the types we share
            _ => unreachable,
        };
    }
};

pub const Handles = struct {
    multi: *libcurl.CurlM,

    pub fn init(config: *const Config) !Handles {
        const multi = libcurl.curl_multi_init() orelse return error.FailedToInitializeMulti;
        errdefer libcurl.curl_multi_cleanup(multi) catch {};

        try libcurl.curl_multi_setopt(multi, .max_host_connections, config.httpMaxHostOpen());
        // --http-max-concurrent is dynamic (based on --cdp-max-connections in
        // serve), so we don't want this to be based on that, since it could
        // result in an unexpected large # of keepalive connection. Instead, we
        // hard-code 160, which used to be the default when --cdp-max-connections
        // was fixed (and it seemed to be safe/work well, so why mess with it?)
        try libcurl.curl_multi_setopt(multi, .max_connects, 160);

        return .{ .multi = multi };
    }

    pub fn deinit(self: *Handles) void {
        libcurl.curl_multi_cleanup(self.multi) catch {};
    }

    pub fn add(self: *Handles, conn: *const Connection) !void {
        try libcurl.curl_multi_add_handle(self.multi, conn._easy);
    }

    pub fn remove(self: *Handles, conn: *const Connection) !void {
        try libcurl.curl_multi_remove_handle(self.multi, conn._easy);
    }

    pub fn perform(self: *Handles) !c_int {
        var running: c_int = undefined;
        try libcurl.curl_multi_perform(self.multi, &running);
        return running;
    }

    pub fn poll(self: *Handles, extra_fds: []libcurl.CurlWaitFd, timeout_ms: c_int) !void {
        try libcurl.curl_multi_poll(self.multi, extra_fds, timeout_ms, null);
    }

    // Thread-safe wake of a poll() in progress on this multi. Used by
    // the Network thread to nudge the worker out of curl_multi_poll
    // when it pushes work onto the worker's inbox.
    pub fn wakeup(self: *Handles) !void {
        try libcurl.curl_multi_wakeup(self.multi);
    }

    pub const MultiMessage = struct {
        conn: *Connection,
        err: ?Error,
    };

    pub fn readMessage(self: *Handles) !?MultiMessage {
        var messages_count: c_int = 0;
        const msg = libcurl.curl_multi_info_read(self.multi, &messages_count) orelse return null;
        return switch (msg.data) {
            .done => |err| {
                var private: *anyopaque = undefined;
                try libcurl.curl_easy_getinfo(msg.easy_handle, .private, &private);
                return .{
                    .conn = @ptrCast(@alignCast(private)),
                    .err = err,
                };
            },
            else => unreachable,
        };
    }
};

pub const StatusCategory = enum {
    @"2xx",
    @"4xx",
    @"5xx",
    other,
};

pub fn statusCategory(status: u16) StatusCategory {
    return switch (status) {
        200...299 => .@"2xx",
        400...499 => .@"4xx",
        500...599 => .@"5xx",
        else => .other,
    };
}

// https://fetch.spec.whatwg.org/#bad-port
pub fn isBadPort(port: u16) bool {
    return switch (port) {
        1, // tcpmux
        7, // echo
        9, // discard
        11, // systat
        13, // daytime
        15, // netstat
        17, // qotd
        19, // chargen
        20, // ftp-data
        21, // ftp
        22, // ssh
        23, // telnet
        25, // smtp
        37, // time
        42, // name
        43, // nicname
        53, // domain
        69, // tftp
        77, // priv-rjs
        79, // finger
        87, // ttylink
        95, // supdup
        101, // hostriame
        102, // iso-tsap
        103, // gppitnp
        104, // acr-nema
        109, // pop2
        110, // pop3
        111, // sunrpc
        113, // auth
        115, // sftp
        117, // uucp-path
        119, // nntp
        123, // ntp
        135, // loc-srv / epmap
        137, // netbios-ns
        139, // netbios-ssn
        143, // imap2
        161, // snmp
        179, // bgp
        389, // ldap
        427, // afp (alternate)
        465, // smtp (alternate)
        512, // print / exec
        513, // login
        514, // shell
        515, // printer
        526, // tempo
        530, // courier
        531, // chat
        532, // netnews
        540, // uucp
        548, // afp
        554, // rtsp
        556, // remotefs
        563, // nntp+ssl
        587, // smtp (outgoing)
        601, // syslog-conn
        636, // ldap+ssl
        989, // ftps-data
        990, // ftps
        993, // imap+ssl
        995, // pop3+ssl
        1719, // h323gatestat
        1720, // h323hostcall
        1723, // pptp
        2049, // nfs
        3659, // apple-sasl
        4045, // lockd
        4190, // sieve
        5060, // sip
        5061, // sips
        6000, // x11
        6566, // sane-port
        6665, // irc (alternate)
        6666, // irc (alternate)
        6667, // irc (default)
        6668, // irc (alternate)
        6669, // irc (alternate)
        6679, // osaut
        6697, // irc+tls
        10080, // amanda
        => true,
        else => false,
    };
}

// Coarse failure classes for the http_error metric. Anything we don't
// explicitly bucket lands in `other`.
pub const ErrorReason = enum {
    timeout,
    connect,
    tls,
    too_large,
    aborted,
    robots_blocked,
    bot_challenge,
    other,
};

pub fn errorReason(err: anyerror) ErrorReason {
    return switch (err) {
        error.OperationTimedout => .timeout,
        error.CouldntConnect,
        error.CouldntResolveHost,
        error.CouldntResolveProxy,
        error.NoConnectionAvailable,
        => .connect,
        error.SslConnectError,
        error.PeerFailedVerification,
        error.SslCertproblem,
        error.SslCacertBadfile,
        error.SslIssuerError,
        error.SslPinnedpubkeynotmatch,
        error.SslInvalidcertstatus,
        error.UseSslFailed,
        => .tls,
        error.ResponseTooLarge => .too_large,
        error.Abort,
        error.TransferCanceled,
        error.AbortedByCallback,
        error.AbortAuthChallenge,
        error.SyncWaitInterrupted,
        => .aborted,
        error.RobotsBlocked => .robots_blocked,
        error.BotChallenge => .bot_challenge,
        else => .other,
    };
}

fn debugCallback(_: *libcurl.Curl, msg_type: libcurl.CurlInfoType, raw: [*c]u8, len: usize, _: ?*anyopaque) callconv(.c) c_int {
    const data = raw[0..len];
    switch (msg_type) {
        .text => std.debug.print("libcurl [text]: {s}\n", .{data}),
        .header_out => std.debug.print("libcurl [req-h]: {s}\n", .{data}),
        .header_in => std.debug.print("libcurl [res-h]: {s}\n", .{data}),
        // .data_in => std.debug.print("libcurl [res-b]: {s}\n", .{data}),
        else => std.debug.print("libcurl ?? {d}\n", .{msg_type}),
    }
    return 0;
}

// ── Unit tests for opensocketCallback ────────────────────────────────────────

fn makeSockAddrV4(ip: [4]u8) libcurl.CurlSockAddr {
    var sa: posix.sockaddr.in = .{
        .port = 0,
        .addr = @bitCast(ip),
    };
    var curl_sa: libcurl.CurlSockAddr = .{
        .family = posix.AF.INET,
        .socktype = posix.SOCK.STREAM,
        .protocol = 0,
        .addrlen = @sizeOf(posix.sockaddr.in),
        .addr = undefined,
    };
    @memcpy(std.mem.asBytes(&curl_sa.addr)[0..@sizeOf(posix.sockaddr.in)], std.mem.asBytes(&sa));
    return curl_sa;
}

const testing = @import("../testing.zig");

test "isBadPort" {
    for ([_]u16{ 1, 22, 25, 143, 6697, 10080 }) |port| {
        try testing.expect(isBadPort(port));
    }
    for ([_]u16{ 0, 80, 443, 8000, 8080, 9584, 65535 }) |port| {
        try testing.expect(!isBadPort(port));
    }
}

test "Header.parse" {
    {
        const h = Header.parse("Content-Type: text/html; charset=utf-8").?;
        try testing.expectEqualSlices(u8, "Content-Type", h.name);
        try testing.expectEqualSlices(u8, "text/html; charset=utf-8", h.value);
    }
    {
        // no space after the colon
        const h = Header.parse("X-Custom:value").?;
        try testing.expectEqualSlices(u8, "X-Custom", h.name);
        try testing.expectEqualSlices(u8, "value", h.value);
    }
    {
        // name and value are trimmed of spaces and tabs
        const h = Header.parse(" \tAccept \t: \tapplication/json \t").?;
        try testing.expectEqualSlices(u8, "Accept", h.name);
        try testing.expectEqualSlices(u8, "application/json", h.value);
    }
    {
        // only the first colon splits; later colons stay in the value
        const h = Header.parse("Referer: http://example.com:8080/").?;
        try testing.expectEqualSlices(u8, "Referer", h.name);
        try testing.expectEqualSlices(u8, "http://example.com:8080/", h.value);
    }
    {
        // empty value
        const h = Header.parse("X-Empty:").?;
        try testing.expectEqualSlices(u8, "X-Empty", h.name);
        try testing.expectEqualSlices(u8, "", h.value);
    }

    {
        // Splitting is all `parse` does; a name that isn't an HTTP token still
        // parses. Callers validate what comes back.
        const h = Header.parse("Foo Bar: value").?;
        try testing.expectEqualSlices(u8, "Foo Bar", h.name);
        try testing.expectEqualSlices(u8, "value", h.value);
    }

    // no colon, no header
    try testing.expect(Header.parse("not-a-header") == null);
    try testing.expect(Header.parse("") == null);
}

test "Header.firstValue" {
    try testing.expectEqualSlices(u8, "attachment", (Header{ .name = "Content-Disposition", .value = "attachment" }).firstValue());
    // firstValue trims but preserves case (callers compare case-insensitively).
    try testing.expectEqualSlices(u8, "ATTACHMENT", (Header{ .name = "Content-Disposition", .value = "  ATTACHMENT ; filename=a.csv" }).firstValue());
    try testing.expectEqualSlices(u8, "text/html", (Header{ .name = "Content-Type", .value = "text/html; charset=utf-8" }).firstValue());
    try testing.expectEqualSlices(u8, "", (Header{ .name = "X", .value = "" }).firstValue());
}

test "Header.param" {
    const h: Header = .{ .name = "Content-Disposition", .value = "attachment; filename=\"r e.csv\"; filename*=UTF-8''e.txt" };
    try testing.expectEqualSlices(u8, "r e.csv", h.param("filename").?);
    try testing.expectEqualSlices(u8, "r e.csv", h.param("FILENAME").?);
    try testing.expectEqualSlices(u8, "UTF-8''e.txt", h.param("filename*").?);
    try testing.expect(h.param("missing") == null);
    // A bare value with no parameters has nothing to return.
    try testing.expect((Header{ .name = "Content-Disposition", .value = "attachment" }).param("filename") == null);
    // Empty values are skipped.
    try testing.expect((Header{ .name = "Content-Disposition", .value = "attachment; filename=\"\"" }).param("filename") == null);
}

test "StatusText.fromStatusLine" {
    try testing.expect((StatusText{}).get() == null);
    try testing.expectEqualSlices(u8, "OK", StatusText.fromStatusLine("HTTP/1.1 200 OK\r\n").get().?);
    try testing.expectEqualSlices(u8, "HOUSTON WE HAVE A", StatusText.fromStatusLine("HTTP/1.1 503 HOUSTON WE HAVE A\r\n").get().?);
    try testing.expectEqualSlices(u8, "lowercase", StatusText.fromStatusLine("HTTP/1.0 502 lowercase\r\n").get().?);
    // curl's synthesized HTTP/2 status line has no phrase
    try testing.expectEqualSlices(u8, "", StatusText.fromStatusLine("HTTP/2 200 \r\n").get().?);
    try testing.expectEqualSlices(u8, "", StatusText.fromStatusLine("HTTP/1.1 200\r\n").get().?);
    try testing.expectEqual(StatusText.MAX_LEN, StatusText.fromStatusLine("HTTP/1.1 200 " ++ repeat("x", 200)).get().?.len);
}

test "opensocketCallback: private IPv4 returns CURL_SOCKET_BAD" {
    testing.silenceLog(&.{.http});

    const filter = IpFilter.init(true, null);
    var sa = makeSockAddrV4(.{ 127, 0, 0, 1 });
    const result = opensocketCallback(@ptrCast(@constCast(&filter)), @backingInt(libcurl.CurlSockType.ipcxn), &sa);
    try testing.expectEqual(libcurl.CURL_SOCKET_BAD, result);
}

test "opensocketCallback: public IPv4 opens a real socket" {
    // 8.8.8.8 — not in any blocked range; callback should create a real socket
    const filter = IpFilter.init(true, null);
    var sa = makeSockAddrV4(.{ 8, 8, 8, 8 });

    const fd = opensocketCallback(@ptrCast(@constCast(&filter)), @backingInt(libcurl.CurlSockType.ipcxn), &sa);
    defer _ = std.c.close(fd);

    // A real fd is always >= 0
    try testing.expect(fd >= 0);
}

test "opensocketCallback: null clientp returns CURL_SOCKET_BAD (fail-closed)" {
    var sa = makeSockAddrV4(.{ 8, 8, 8, 8 });
    const result = opensocketCallback(null, @backingInt(libcurl.CurlSockType.ipcxn), &sa);
    try testing.expectEqual(libcurl.CURL_SOCKET_BAD, result);
}

test "opensocketCallback: block_private=false allows private IP" {
    // When block_private is false the filter blocks nothing
    const filter = IpFilter.init(false, null);
    var sa = makeSockAddrV4(.{ 127, 0, 0, 1 });
    const fd = opensocketCallback(@ptrCast(@constCast(&filter)), @backingInt(libcurl.CurlSockType.ipcxn), &sa);
    defer _ = std.c.close(fd);

    try testing.expect(fd >= 0);
}

test "Connection.inHttpRange: only a real status survives" {
    const kept = [_]c_long{ 100, 200, 404, 503, 599 };
    for (kept) |status| {
        try std.testing.expectEqual(@as(u16, @intCast(status)), Connection.inHttpRange(status));
    }
    // 9999 used to survive and then panic in every std.http.Status cast.
    const dropped = [_]c_long{ -1, 0, 99, 600, 1024, 9999, 65536 };
    for (dropped) |status| {
        try std.testing.expectEqual(0, Connection.inHttpRange(status));
    }
}