Lightpanda: the headless browser designed for AI and automation
refactor: rework how httpclient headers work
Karl Seguinfd2e6a8512f5
diff --git a/src/Config.zig b/src/Config.zigindex ce173a4b5..0bcaa1ac0 100644--- a/src/Config.zig+++ b/src/Config.zig@@ -786,7 +786,7 @@ pub const WaitUntil = enum {done,};-/// Pre-formatted HTTP headers for reuse across Http and Client.+/// HTTP header values shared across Http and Client./// Must be initialized with an allocator that outlives all HTTP connections.pub const HttpHeaders = struct {const user_agent_base: [:0]const u8 = "Lightpanda/1.0";@@ -804,9 +804,9 @@ pub const HttpHeaders = struct {};pub const sec_ch_ua: [:0]const u8 = blk: {- var out: [:0]const u8 = "Sec-Ch-Ua:";+ var out: [:0]const u8 = "";for (brands, 0..) |b, i| {- const sep = if (i == 0) " " else ", ";+ const sep = if (i == 0) "" else ", ";out = out ++ sep ++ "\"" ++ b.brand ++ "\";v=\"" ++ b.version ++ "\"";}break :blk out;@@ -816,13 +816,12 @@ pub const HttpHeaders = struct {// stream when a client sends Accept-Encoding without Accept-Language,// treating it as a bot signal. Ship a neutral default so we look like a// normal client.- pub const accept_language: [:0]const u8 = "Accept-Language: en-US,en;q=0.9";+ pub const accept_language: [:0]const u8 = "en-US,en;q=0.9";// Document-navigation Accept value Chrome sends.- pub const navigation_accept: [:0]const u8 = "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";+ pub const navigation_accept: [:0]const u8 = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";user_agent: [:0]const u8, // User agent value (e.g. "Lightpanda/1.0")- user_agent_header: [:0]const u8,proxy_bearer_header: ?[:0]const u8,@@ -835,9 +834,6 @@ pub const HttpHeaders = struct {user_agent_base;errdefer if (config.userAgent() != null or config.userAgentSuffix() != null) allocator.free(user_agent);- const user_agent_header = try std.fmt.allocPrintSentinel(allocator, "User-Agent: {s}", .{user_agent}, 0);- errdefer allocator.free(user_agent_header);-const proxy_bearer_header: ?[:0]const u8 = if (config.proxyBearerToken()) |token|try std.fmt.allocPrintSentinel(allocator, "Proxy-Authorization: Bearer {s}", .{token}, 0)else@@ -845,7 +841,6 @@ pub const HttpHeaders = struct {return .{.user_agent = user_agent,- .user_agent_header = user_agent_header,.proxy_bearer_header = proxy_bearer_header,};}@@ -854,7 +849,6 @@ pub const HttpHeaders = struct {if (self.proxy_bearer_header) |hdr| {allocator.free(hdr);}- allocator.free(self.user_agent_header);if (self.user_agent.ptr != user_agent_base.ptr) {allocator.free(self.user_agent);}diff --git a/src/Updater.zig b/src/Updater.zigindex da3122617..837c8e42d 100644--- a/src/Updater.zig+++ b/src/Updater.zig@@ -52,7 +52,7 @@ pub fn deinit(self: *Updater) void {/// Sends running Lightpanda version to remote to get update information./// Outputs directly to given `Writer`.pub fn inform(self: *Updater, writer: *std.Io.Writer) !void {- const conn = try http.Connection.init(self.x509_store, self.config, null);+ var conn = try http.Connection.init(self.x509_store, self.config, null);defer conn.deinit();const url = std.fmt.comptimePrint("https://telemetry.lightpanda.io/v/{s}", .{lp.build_config.version});diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex 542f9a4cf..e67e8c880 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -277,9 +277,6 @@ origin: ?[]const u8 = null,// If null the url must be used.base_url: ?[:0]const u8 = null,-// referer header cache.-referer_header: ?[:0]const u8 = null,-// Document charset (canonical name from encoding_rs, static lifetime)charset: []const u8 = "UTF-8",@@ -595,24 +592,9 @@ pub fn httpMetadata(self: *const Frame) HttpMetadata {// Add common headers for a request:// * referer-pub fn headersForRequest(self: *Frame, headers: *HttpClient.Headers) !void {- // Build the referer- const referer = blk: {- if (self.referer_header == null) {- // build the cache- if (std.mem.startsWith(u8, self.url, "http")) {- self.referer_header = try std.mem.concatWithSentinel(self.arena, u8, &.{ "Referer: ", self.url }, 0);- } else {- self.referer_header = "";- }- }-- break :blk self.referer_header.?;- };-- // If the referer is empty, ignore the header.- if (referer.len > 0) {- try headers.add(referer);+pub fn headersForRequest(self: *Frame, transfer: *HttpClient.Transfer) !void {+ if (std.mem.startsWith(u8, self.url, "http")) {+ try transfer.addHeader("Referer", self.url, .{});}}@@ -802,13 +784,15 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo{// Ours until submit; clean up if header setup fails.errdefer transfer.deinit();- try transfer.req.headers.add(lp.Config.HttpHeaders.navigation_accept);+ try transfer.addHeader("Accept", lp.Config.HttpHeaders.navigation_accept, .{});if (opts.header) |hdr| {- try transfer.req.headers.add(hdr);+ // Arrives pre-joined ("Name: Value"), e.g. from the CLI.+ if (HttpClient.Header.parse(hdr)) |parsed| {+ try transfer.addHeader(parsed.name, parsed.value, .{});+ }}if (opts.referer) |ref| {- const ref_header = try std.mem.concatWithSentinel(transfer.arena.allocator(), u8, &.{ "Referer: ", ref }, 0);- try transfer.req.headers.add(ref_header);+ try transfer.addHeader("Referer", ref, .{});}}@@ -1040,7 +1024,12 @@ fn canScheduleNavigation(self: *Frame, new_target_type: NavigationType) bool {}pub fn makeRequest(self: *Frame, req: HttpClient.Request) !void {- return self._session.browser.http_client.request(req, &self._http_owner);+ const transfer = try self._session.browser.http_client.newRequest(req, &self._http_owner);+ {+ errdefer transfer.deinit();+ try self.headersForRequest(transfer);+ }+ return transfer.submit();}// Two-phase variant; see HttpClient.newRequest for the ownership contract.@@ -2195,9 +2184,25 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co// the frame while it's registered (they'd run JS on the parser's stack)// and delivers them on the next tick after the sync fetch returns.- var headers = try http_client.newHeaders();- try headers.add("Accept: text/css,*/*;q=0.1");- try self.headersForRequest(&headers);+ const transfer = http_client.newRequest(.{+ .url = resolved,+ .method = .GET,+ .frame_id = self._frame_id,+ .loader_id = self._loader_id,+ .cookie_jar = &session.cookie_jar,+ .cookie_origin = self.url,+ .resource_type = .stylesheet,+ .notification = session.notification,+ .shutdown_callback = HttpClient.noopShutdown, // syncRequest installs its own+ }, &self._http_owner) catch |err| {+ log.warn(.http, "external stylesheet fetch", .{ .err = err, .url = resolved });+ return self.fireElementEvent(element, comptime .wrap("error"));+ };+ {+ errdefer transfer.deinit();+ try transfer.addHeader("Accept", "text/css,*/*;q=0.1", .{});+ try self.headersForRequest(transfer);+ }// Set the script-manager `is_evaluating` flag for the same reason// `ScriptManager.addFromElement` does: `syncRequest` pumps the CDP@@ -2211,18 +2216,7 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []cosm.is_evaluating = true;defer sm.endEvaluationWindow(was_evaluating);- var response = http_client.syncRequest(.{- .url = resolved,- .method = .GET,- .frame_id = self._frame_id,- .loader_id = self._loader_id,- .headers = headers,- .cookie_jar = &session.cookie_jar,- .cookie_origin = self.url,- .resource_type = .stylesheet,- .notification = session.notification,- .shutdown_callback = HttpClient.noopShutdown, // syncRequest installs its own- }, &self._http_owner) catch |err| {+ var response = http_client.syncRequest(transfer) catch |err| {log.warn(.http, "external stylesheet fetch", .{ .err = err, .url = resolved });return self.fireElementEvent(element, comptime .wrap("error"));};diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zigindex b0a91ba16..7d1dcce61 100644--- a/src/browser/ScriptManager.zig+++ b/src/browser/ScriptManager.zig@@ -97,10 +97,6 @@ pub fn tailHook(base: *ScriptManagerBase) void {}}-fn getHeaders(self: *ScriptManager) !HttpClient.Headers {- return self.base.getHeaders();-}-// Returns true when a fetch was started: the link's load/error event fires// when the fetch settles. false (duplicate hint) = no event will fire.// element is null when the hint came from the prescan rather than a <link>.@@ -140,7 +136,6 @@ pub fn preloadScript(self: *ScriptManager, element: ?*Element.Html, url: []const.method = .GET,.frame_id = frame._frame_id,.loader_id = frame._loader_id,- .headers = try self.base.getHeaders(),.cookie_jar = &frame._session.cookie_jar,.cookie_origin = frame.url,.resource_type = .script,@@ -352,18 +347,22 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_escript.status = pre.status;script.complete = true;} else {- const response = try self.base.client.syncRequest(.{+ const transfer = try self.base.client.newRequest(.{.url = remote_url,.method = .GET,.frame_id = frame._frame_id,.loader_id = frame._loader_id,- .headers = try self.getHeaders(),.cookie_jar = &frame._session.cookie_jar,.cookie_origin = frame.url,.resource_type = .script,.notification = frame._session.notification,.shutdown_callback = HttpClient.noopShutdown, // syncRequest installs its own}, &frame._http_owner);+ {+ errdefer transfer.deinit();+ try frame.headersForRequest(transfer);+ }+ const response = try self.base.client.syncRequest(transfer);// Take the body's arena rather than releasing it: `source`// has to outlive this call, up to script.deinit().@@ -400,7 +399,6 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e.method = .GET,.frame_id = frame._frame_id,.loader_id = frame._loader_id,- .headers = try self.getHeaders(),.cookie_jar = &frame._session.cookie_jar,.cookie_origin = frame.url,.resource_type = .script,diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zigindex e3edb7d1b..d471b23bd 100644--- a/src/browser/ScriptManagerBase.zig+++ b/src/browser/ScriptManagerBase.zig@@ -73,12 +73,6 @@ pub const Owner = union(enum) {};}- pub fn addHeaders(self: Owner, headers: *HttpClient.Headers) !void {- return switch (self) {- inline else => |g| g.headersForRequest(headers),- };- }-pub fn makeRequest(self: Owner, req: HttpClient.Request) !void {return switch (self) {inline else => |g| g.makeRequest(req),@@ -177,12 +171,6 @@ fn clearList(list: *std.DoublyLinkedList) void {}}-pub fn getHeaders(self: *ScriptManagerBase) !http.Headers {- var headers = try self.client.newHeaders();- try self.owner.addHeaders(&headers);- return headers;-}-fn acquireArena(self: *ScriptManagerBase, size_or_bucket: anytype, debug: []const u8) !*lp.Arena {return self.owner.session().getArena(size_or_bucket, debug);}@@ -276,7 +264,6 @@ pub fn preloadImport(self: *ScriptManagerBase, url: [:0]const u8, referrer: []co.method = .GET,.frame_id = owner.frameId(),.loader_id = owner.loaderId(),- .headers = try self.getHeaders(),.cookie_jar = &session.cookie_jar,.cookie_origin = owner.url(),.resource_type = .script,@@ -471,7 +458,6 @@ pub fn getAsyncImport(self: *ScriptManagerBase, url: [:0]const u8, cb: ImportAsy.method = .GET,.frame_id = owner.frameId(),.loader_id = owner.loaderId(),- .headers = try self.getHeaders(),.resource_type = .script,.cookie_jar = &session.cookie_jar,.cookie_origin = owner.url(),diff --git a/src/browser/js/Execution.zig b/src/browser/js/Execution.zigindex bcdfe04a6..a29e44167 100644--- a/src/browser/js/Execution.zig+++ b/src/browser/js/Execution.zig@@ -84,9 +84,9 @@ pub fn getPinnedArena(self: *const Execution, size_or_bucket: anytype, debug: []return self.page.getPinnedArena(size_or_bucket, debug);}-pub fn headersForRequest(self: *const Execution, headers: *HttpClient.Headers) !void {+pub fn headersForRequest(self: *const Execution, transfer: *HttpClient.Transfer) !void {return switch (self.js.global) {- inline else => |g| g.headersForRequest(headers),+ inline else => |g| g.headersForRequest(transfer),};}diff --git a/src/browser/webapi/WorkerGlobalScope.zig b/src/browser/webapi/WorkerGlobalScope.zigindex a845fa0fe..38c2e53a7 100644--- a/src/browser/webapi/WorkerGlobalScope.zig+++ b/src/browser/webapi/WorkerGlobalScope.zig@@ -256,8 +256,8 @@ pub fn hasDirectListeners(self: *WorkerGlobalScope, target: *EventTarget, typ: [// Workers don't have their own Referer; per spec, dedicated worker requests// use the parent document's URL. Delegate to the owning frame.-pub fn headersForRequest(self: *WorkerGlobalScope, headers: *HttpClient.Headers) !void {- return self._frame.headersForRequest(headers);+pub fn headersForRequest(self: *WorkerGlobalScope, transfer: *HttpClient.Transfer) !void {+ return self._frame.headersForRequest(transfer);}pub fn isSameOrigin(self: *const WorkerGlobalScope, url: [:0]const u8) bool {@@ -270,7 +270,12 @@ pub fn isSameOrigin(self: *const WorkerGlobalScope, url: [:0]const u8) bool {}pub fn makeRequest(self: *WorkerGlobalScope, req: HttpClient.Request) !void {- return self._session.browser.http_client.request(req, &self._http_owner);+ const transfer = try self._session.browser.http_client.newRequest(req, &self._http_owner);+ {+ errdefer transfer.deinit();+ try self.headersForRequest(transfer);+ }+ return transfer.submit();}// Two-phase variant; see HttpClient.newRequest for the ownership contract.@@ -416,16 +421,12 @@ fn importScript(self: *WorkerGlobalScope, arena: Allocator, url: [:0]const u8) !const http_client = &session.browser.http_client;- var headers = try http_client.newHeaders();- try self.headersForRequest(&headers);-- var response = http_client.syncRequest(.{+ const transfer = http_client.newRequest(.{.url = resolved_url,.method = .GET,.frame_id = self._frame_id,.document_frame_id = self._frame._frame_id,.loader_id = self._loader_id,- .headers = headers,.cookie_jar = &session.cookie_jar,.cookie_origin = self.url,.resource_type = .script,@@ -435,6 +436,15 @@ fn importScript(self: *WorkerGlobalScope, arena: Allocator, url: [:0]const u8) !log.warn(.http, "importScript", .{ .url = resolved_url, .err = err });return error.NetworkError;};+ {+ errdefer transfer.deinit();+ try self.headersForRequest(transfer);+ }++ var response = http_client.syncRequest(transfer) catch |err| {+ log.warn(.http, "importScript", .{ .url = resolved_url, .err = err });+ return error.NetworkError;+ };defer response.deinit();if (response.status != 200) {diff --git a/src/browser/webapi/net/EventSource.zig b/src/browser/webapi/net/EventSource.zigindex 2c1615d32..fadd69401 100644--- a/src/browser/webapi/net/EventSource.zig+++ b/src/browser/webapi/net/EventSource.zig@@ -163,7 +163,6 @@ fn asEventTarget(self: *EventSource) *EventTarget {fn connect(self: *EventSource) !void {const exec = self._exec;const session = exec.session;- const http_client = &session.browser.http_client;self._skip_lf = false;self._bom_checked = false;@@ -174,32 +173,13 @@ fn connect(self: *EventSource) !void {self._id_buf.clearRetainingCapacity();try self._id_buf.appendSlice(self._arena.allocator(), self._last_event_id.items);- var headers = try http_client.newHeaders();- try headers.add("Accept: text/event-stream");- try headers.add("Cache-Control: no-cache");- if (self._last_event_id.items.len > 0) {- // headers.add copies the value, so the local arena's lifetime is enough- const header = try std.fmt.allocPrintSentinel(exec.local_arena, "Last-Event-ID: {s}", .{self._last_event_id.items}, 0);- try headers.add(header);- }-const same_origin = exec.isSameOrigin(self._url);- if (!same_origin) {- // EventSource is a CORS request: cross-origin fetches carry the- // document's origin ("null" for opaque origins, like Chrome).- const origin = exec.origin() orelse "null";- const header = try std.fmt.allocPrintSentinel(exec.local_arena, "Origin: {s}", .{origin}, 0);- try headers.add(header);- }- try exec.headersForRequest(&headers);-const cookie_support = self._with_credentials or same_origin;const transfer = try exec.newRequest(.{.ctx = self,.url = self._url,.method = .GET,- .headers = headers,.frame_id = exec.frameId(),.loader_id = exec.loaderId(),.cookie_jar = if (cookie_support) &session.cookie_jar else null,@@ -214,6 +194,21 @@ fn connect(self: *EventSource) !void {.shutdown_callback = httpShutdownCallback,});+ {+ errdefer transfer.deinit();+ try transfer.addHeader("Accept", "text/event-stream", .{});+ try transfer.addHeader("Cache-Control", "no-cache", .{});+ if (self._last_event_id.items.len > 0) {+ try transfer.addHeader("Last-Event-ID", self._last_event_id.items, .{});+ }+ if (!same_origin) {+ // EventSource is a CORS request: cross-origin fetches carry the+ // document's origin ("null" for opaque origins, like Chrome).+ try transfer.addHeader("Origin", exec.origin() orelse "null", .{});+ }+ try exec.headersForRequest(transfer);+ }+self._transfer = transfer;// Failures inside submit are dispatched to httpErrorCallback.diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zigindex 6980ca3a6..0832c1aa8 100644--- a/src/browser/webapi/net/Fetch.zig+++ b/src/browser/webapi/net/Fetch.zig@@ -84,12 +84,6 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promis};const session = exec.session;- const http_client = &session.browser.http_client;- var headers = try http_client.newHeaders();- if (request._headers) |h| {- try h.populateHttpHeader(exec.call_arena, &headers);- }- try exec.headersForRequest(&headers);if (comptime lp.IS_DEBUG) {log.debug(.http, "fetch", .{ .url = request._url });@@ -108,7 +102,6 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promis.frame_id = exec.frameId(),.loader_id = exec.loaderId(),.body = request._body,- .headers = headers,.resource_type = .fetch,.cookie_jar = cookie_jar,.cookie_origin = exec.url.*,@@ -128,6 +121,14 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promisreturn resolver.promise();};+ {+ errdefer transfer.deinit();+ if (request._headers) |h| {+ try h.populateRequestHeaders(transfer);+ }+ try exec.headersForRequest(transfer);+ }+// Held for Response.deinit's abort; the error, shutdown and done// callbacks clear it.response._http_transfer = transfer;diff --git a/src/browser/webapi/net/Headers.zig b/src/browser/webapi/net/Headers.zigindex 5c6cadaa3..7bc7c8600 100644--- a/src/browser/webapi/net/Headers.zig+++ b/src/browser/webapi/net/Headers.zig@@ -8,7 +8,6 @@ const KeyValueList = @import("../KeyValueList.zig");const log = lp.log;const Execution = js.Execution;-const Allocator = std.mem.Allocator;const Headers = @This();@@ -99,12 +98,10 @@ pub fn forEach(self: *Headers, cb_: js.Function, js_this_: ?js.Object) !void {}}-// TODO: do we really need 2 different header structs??-const http = @import("../../../network/http.zig");-pub fn populateHttpHeader(self: *Headers, allocator: Allocator, http_headers: *http.Headers) !void {+const HttpClient = @import("../../../network/HttpClient.zig");+pub fn populateRequestHeaders(self: *Headers, transfer: *HttpClient.Transfer) !void {for (self._list._entries.items) |entry| {- const merged = try std.mem.concatWithSentinel(allocator, u8, &.{ entry.name.str(), ": ", entry.value.str() }, 0);- try http_headers.add(merged);+ try transfer.addHeader(entry.name.str(), entry.value.str(), .{ .source = .author });}}diff --git a/src/browser/webapi/net/WebSocket.zig b/src/browser/webapi/net/WebSocket.zigindex ad1ee7bd8..f119bc386 100644--- a/src/browser/webapi/net/WebSocket.zig+++ b/src/browser/webapi/net/WebSocket.zig@@ -56,7 +56,6 @@ _got_upgrade: bool = false,_conn: ?*http.Connection,_http_client: *HttpClient,-_req_headers: http.Headers,_owner_node: std.DoublyLinkedList.Node = .{},@@ -185,7 +184,6 @@ pub fn init(url: []const u8, protocols: [][]const u8, exec: *const Execution) !*._arena = arena,._proto = undefined,._url = resolved_url,- ._req_headers = .{ .headers = null },._http_client = http_client,});@@ -238,12 +236,13 @@ fn connect(self: *WebSocket, protocols: [][]const u8) !void {try conn.setWriteCallback(receivedDataCallback);try conn.setHeaderCallback(receivedHeaderCallback);- var headers = try http_client.newHeaders();- errdefer headers.deinit();+ const allocator = arena.allocator();+ for (http_client.baselineHeaders()) |hdr| {+ try conn.addHeader(allocator, hdr.name, hdr.value);+ }if (protocols.len > 0) {- const header = try std.fmt.allocPrintSentinel(arena.allocator(), "Sec-WebSocket-Protocol: {s}", .{try std.mem.join(arena.allocator(), ", ", protocols)}, 0);- try headers.add(header);+ try conn.addHeader(allocator, "Sec-WebSocket-Protocol", try std.mem.join(allocator, ", ", protocols));}{@@ -252,13 +251,12 @@ fn connect(self: *WebSocket, protocols: [][]const u8) !void {// protection on WS servers) reject upgrades that arrive without it.// Non-tuple origins (about:blank, data:) serialize to "null", like// Chrome sends for opaque origins.- const origin = (try URL.getOrigin(arena.allocator(), exec.url.*)) orelse "null";- const header = try std.fmt.allocPrintSentinel(arena.allocator(), "Origin: {s}", .{origin}, 0);- try headers.add(header);+ const origin = (try URL.getOrigin(allocator, exec.url.*)) orelse "null";+ try conn.addHeader(allocator, "Origin", origin);}{- var buf: std.Io.Writer.Allocating = .init(arena.allocator());+ var buf: std.Io.Writer.Allocating = .init(allocator);try exec.session.cookie_jar.forRequest(resolved_url, &buf.writer, .{.is_http = true,.is_navigation = false,@@ -271,13 +269,12 @@ fn connect(self: *WebSocket, protocols: [][]const u8) !void {}}- try conn.setHeaders(&headers);+ try conn.commitHeaders();conn.transport = .{ .websocket = self };try http_client.trackConn(conn);self._conn = conn;- self._req_headers = headers;}fn isBlockedPort(url: [:0]const u8) bool {@@ -454,14 +451,13 @@ fn deactivate(self: *WebSocket) void {self.releaseRef(self._exec.page);}-// Unlink the connection from the http client and free the request headers.-// Queued outgoing messages are kept: their arenas are released in deinit,-// and bufferedAmount keeps reporting them, as the spec wants.+// Unlink the connection from the http client. Queued outgoing messages are+// kept: their arenas are released in deinit, and bufferedAmount keeps+// reporting them, as the spec wants.fn releaseTransport(self: *WebSocket) void {const conn = self._conn orelse return;self._conn = null;self._http_client.removeConn(conn);- self._req_headers.deinit();}// Pump-side: record an event for delivery. Errors propagate to libcurl'sdiff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zigindex 66998c250..75275b429 100644--- a/src/browser/webapi/net/XMLHttpRequest.zig+++ b/src/browser/webapi/net/XMLHttpRequest.zig@@ -259,17 +259,10 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !vconst exec = self._exec;const session = exec.session;- const http_client = &session.browser.http_client;- var headers = try http_client.newHeaders();// Only add cookies for same-origin or when withCredentials is trueconst cookie_support = self._with_credentials or exec.isSameOrigin(self._url);- try self._request_headers.populateHttpHeader(exec.call_arena, &headers);- if (cookie_support) {- try exec.headersForRequest(&headers);- }-self.acquireRef();self._active_requests += 1;self._send_flag = true;@@ -278,7 +271,6 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !v.ctx = self,.url = self._url,.method = self._method,- .headers = headers,.frame_id = exec.frameId(),.loader_id = exec.loaderId(),.body = self._request_body,@@ -299,6 +291,18 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !vreturn err;};+ {+ errdefer {+ transfer.deinit();+ self.releaseSelfRef();+ self._send_flag = false;+ }+ try self._request_headers.populateRequestHeaders(transfer);+ if (cookie_support) {+ try exec.headersForRequest(transfer);+ }+ }+// Held for abort() / open() / deinit; the error, shutdown and done// callbacks clear it.self._http_transfer = transfer;diff --git a/src/cdp/CDP.zig b/src/cdp/CDP.zigindex 76264330f..2ed72c51b 100644--- a/src/cdp/CDP.zig+++ b/src/cdp/CDP.zig@@ -24,6 +24,7 @@ const Inbox = @import("../Inbox.zig");const Notification = @import("../Notification.zig");const WS = @import("../network/WS.zig");+const http = @import("../network/http.zig");const Network = @import("../network/Network.zig");const Transfer = @import("../network/HttpClient.zig").Transfer;@@ -568,7 +569,7 @@ pub const BrowserContext = struct {user_agent_changed: bool = false,// Extra headers to add to all requests.- extra_headers: std.ArrayList([*c]const u8) = .empty,+ extra_headers: std.ArrayList(http.Header) = .empty,intercept_state: InterceptState,@@ -1481,22 +1482,20 @@ test "cdp: syncRequest short-circuits after disconnect" {try testing.expectError(error.ClientDisconnected, client.tick(0));// A synchronous fetch attempted after the latch returns ClientDisconnected- // without starting the request. syncRequest also frees req.headers on this- // early-return path (it returns before request() takes ownership); that- // free isn't asserted here because curl_slist is C-allocated and escapes the- // per-test leak check, so it's verified by review. The latch check returns- // before any other req field is read, so the rest are placeholders.- const headers = try client.newHeaders();- try testing.expectError(error.ClientDisconnected, client.syncRequest(.{+ // without starting the request: syncRequest consumes (deinits) the+ // transfer on the early-return path, before any of the callbacks are+ // installed. The latch check runs before any req field is read, so the+ // rest are placeholders.+ const transfer = try client.newRequest(.{.frame_id = 0,.loader_id = 0,.method = .GET,.url = "http://127.0.0.1:9582/",- .headers = headers,.cookie_jar = null,.cookie_origin = "",.resource_type = .fetch,.notification = undefined,.shutdown_callback = @import("../network/HttpClient.zig").noopShutdown,- }, undefined));+ }, null);+ try testing.expectError(error.ClientDisconnected, client.syncRequest(transfer));}diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zigindex 5d25c4660..c32bf0d03 100644--- a/src/cdp/domains/fetch.zig+++ b/src/cdp/domains/fetch.zig@@ -264,17 +264,7 @@ fn continueRequest(cmd: *CDP.Command) !void {}if (params.headers) |headers| {- request.headers.deinit();-- var buf: std.ArrayList(u8) = .empty;- var new_headers = try bc.cdp.browser.http_client.newHeaders();- for (headers) |hdr| {- defer buf.clearRetainingCapacity();- try buf.print(cmd.arena, "{s}: {s}", .{ hdr.name, hdr.value });- try buf.append(cmd.arena, 0);- try new_headers.add(buf.items[0 .. buf.items.len - 1 :0]);- }- request.headers = new_headers;+ try transfer.replaceRequestHeaders(headers);}if (params.postData) |b| {diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zigindex 011c9d1ac..a08f85518 100644--- a/src/cdp/domains/network.zig+++ b/src/cdp/domains/network.zig@@ -130,21 +130,29 @@ fn setExtraHTTPHeaders(cmd: *CDP.Command) !void {const value = header.value_ptr.*;if (std.mem.indexOfAny(u8, key, "\r\n") != null or std.mem.indexOfAny(u8, value, "\r\n") != null) {- log.warn(.not_implemented, "network.setExtraHTTPHeaders", .{ .param = "header", .value = key, .info = "header name/value must not contain CR or LF" });+ log.warn(.cdp, "network.setExtraHTTPHeaders", .{ .param = "header", .value = key, .info = "header name/value must not contain CR or LF" });continue;}- const header_string = try std.fmt.allocPrintSentinel(arena, "{s}: {s}", .{ key, value }, 0);+ // A colon in the name would smuggle a different header name onto the+ // wire once the pair is joined ("User-Agent:Mozilla/5.0 (X" + "Y)"),+ // bypassing the User-Agent validation below.+ if (std.mem.indexOfScalar(u8, key, ':') != null) {+ log.warn(.cdp, "network.setExtraHTTPHeaders", .{ .param = "header", .value = key, .info = "header name must not contain a colon" });+ continue;+ }- if (Headers.parseHeader(header_string)) |parsed| {- if (std.ascii.eqlIgnoreCase(parsed.name, "user-agent")) {- Config.validateUserAgent(parsed.value) catch |err| {- log.warn(.not_implemented, "network.setExtraHTTPHeaders", .{ .param = "userAgent", .value = parsed.value, .err = err });- continue;- };- }+ if (std.ascii.eqlIgnoreCase(key, "user-agent")) {+ Config.validateUserAgent(value) catch |err| {+ log.warn(.cdp, "network.setExtraHTTPHeaders", .{ .param = "userAgent", .value = value, .err = err });+ continue;+ };}- extra_headers.appendAssumeCapacity(header_string);++ extra_headers.appendAssumeCapacity(.{+ .name = try arena.dupe(u8, key),+ .value = try arena.dupe(u8, value),+ });}return cmd.sendResult(null, .{});@@ -346,11 +354,11 @@ pub fn httpRequestStart(bc: *CDP.BrowserContext, msg: *const Notification.Requesconst frame_id = req.document_frame_id orelse req.frame_id;const frame = bc.session.findFrameByFrameId(frame_id) orelse return;- // Modify request with extra CDP headers. Use set (replace by name) so a- // caller-supplied header overrides a built-in default of the same name- // (e.g. User-Agent) instead of producing a duplicate libcurl drops.+ // Modify request with extra CDP headers. Use setHeader (replace by name)+ // so a caller-supplied header overrides a built-in default of the same+ // name (e.g. User-Agent) instead of producing a duplicate.for (bc.extra_headers.items) |extra| {- try req.headers.set(extra);+ try transfer.setHeader(extra.name, extra.value, .{});}// We're missing a bunch of fields, but, for now, this eems like enough@@ -464,8 +472,7 @@ pub const RequestWriter = struct {{try jws.objectField("headers");try jws.beginObject();- var it = request.headers.iterator();- while (it.next()) |hdr| {+ for (transfer.req_headers.items) |hdr| {try SafeString.writeObjectField(jws, hdr.name);try jws.write(SafeString.wrap(hdr.value));}@@ -652,7 +659,7 @@ test "cdp.network setExtraHTTPHeaders" {}test "cdp.network setExtraHTTPHeaders rejects non-printable User-Agent" {- testing.silenceLog(&.{.not_implemented});+ testing.silenceLog(&.{.cdp});var ctx = try testing.context();defer ctx.deinit();@@ -669,11 +676,12 @@ test "cdp.network setExtraHTTPHeaders rejects non-printable User-Agent" {});try testing.expectEqual(bc.extra_headers.items.len, 1);- try testing.expectEqual("x-custom: hi", std.mem.span(bc.extra_headers.items[0]));+ try testing.expectEqual("x-custom", bc.extra_headers.items[0].name);+ try testing.expectEqual("hi", bc.extra_headers.items[0].value);}test "cdp.network setExtraHTTPHeaders rejects a Mozilla User-Agent" {- testing.silenceLog(&.{.not_implemented});+ testing.silenceLog(&.{.cdp});var ctx = try testing.context();defer ctx.deinit();@@ -707,15 +715,15 @@ test "cdp.network setExtraHTTPHeaders accepts valid User-Agent" {}test "cdp.network setExtraHTTPHeaders rejects a Mozilla User-Agent smuggled via a colon in the key" {- testing.silenceLog(&.{.not_implemented});+ testing.silenceLog(&.{.cdp});var ctx = try testing.context();defer ctx.deinit();_ = try ctx.loadBrowserContext(.{ .id = "NID-UA4", .session_id = "NESI-UA4" });- // A colon in the key desyncs the raw key from the first-colon parse that- // req.headers.set/libcurl use: "User-Agent:Mozilla/5.0 (X: Y)" parses to+ // A colon in the key would desync the stored name from what lands on the+ // wire once the pair is joined: "User-Agent:Mozilla/5.0 (X: Y)" parses to// name="User-Agent", value="Mozilla/5.0 (X: Y)" on the wire.try ctx.processMessage(.{.id = 3,@@ -728,7 +736,7 @@ test "cdp.network setExtraHTTPHeaders rejects a Mozilla User-Agent smuggled via}test "cdp.network setExtraHTTPHeaders rejects a header that smuggles CRLF" {- testing.silenceLog(&.{.not_implemented});+ testing.silenceLog(&.{.cdp});var ctx = try testing.context();defer ctx.deinit();@@ -747,7 +755,8 @@ test "cdp.network setExtraHTTPHeaders rejects a header that smuggles CRLF" {});try testing.expectEqual(bc.extra_headers.items.len, 1);- try testing.expectEqual("x-keep: ok", std.mem.span(bc.extra_headers.items[0]));+ try testing.expectEqual("x-keep", bc.extra_headers.items[0].name);+ try testing.expectEqual("ok", bc.extra_headers.items[0].value);}test "cdp.Network: cookies" {diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 546ddbdb8..57da9a0f6 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -41,7 +41,6 @@ const Allocator = std.mem.Allocator;pub const Method = http.Method;pub const Header = http.Header;-pub const Headers = http.Headers;pub const HeaderIterator = http.HeaderIterator;// This is loosely tied to a browser Frame. Loading all the <scripts>, doing@@ -136,10 +135,9 @@ use_proxy: bool,tls_verify: bool = true,// User agent override set via CDP Emulation.setUserAgentOverride.-// When set, takes precedence over the config's http_headers values.-// Both fields are allocated from self.allocator when set, null otherwise.+// When set, takes precedence over the config's http_headers value.+// Allocated from self.allocator when set, null otherwise.user_agent_override: ?[:0]const u8 = null,-user_agent_header_override: ?[:0]const u8 = null,// The CDP layer we dispatch inbox messages to. Set in CDP.init for// `serve` mode; null in all other modes. Since this is set early, BEFORE the@@ -274,17 +272,10 @@ pub fn incrReqId(self: *Client) u32 {return id;}-// Set a user agent override. Both the raw UA string and the pre-formatted-// "User-Agent: <ua>" header string are allocated from self.allocator.+// Set a user agent override, allocated from self.allocator.pub fn setUserAgentOverride(self: *Client, ua: []const u8) !void {self.clearUserAgentOverride();-- const override = try self.allocator.dupeZ(u8, ua);- errdefer self.allocator.free(override);-- const header = try std.fmt.allocPrintSentinel(self.allocator, "User-Agent: {s}", .{ua}, 0);- self.user_agent_override = override;- self.user_agent_header_override = header;+ self.user_agent_override = try self.allocator.dupeZ(u8, ua);}// Clear any user agent override, restoring the default from config.@@ -293,10 +284,6 @@ pub fn clearUserAgentOverride(self: *Client) void {self.allocator.free(ua);self.user_agent_override = null;}- if (self.user_agent_header_override) |uah| {- self.allocator.free(uah);- self.user_agent_header_override = null;- }}// Enable TLS verification on all connections.@@ -387,15 +374,21 @@ fn isUrlBlocked(self: *const Client, url: []const u8, internal: bool) bool {return blocklist.isBlocked(url);}-pub fn newHeaders(self: *const Client) !http.Headers {- const ua_header = self.user_agent_header_override orelse self.network.config.http_headers.user_agent_header;- return http.Headers.init(ua_header);-}-pub fn getUserAgent(self: *const Client) [:0]const u8 {return self.user_agent_override orelse self.network.config.http_headers.user_agent;}+// Headers _all_ requests include.+pub fn baselineHeaders(self: *const Client) [3]http.Header {+ return .{+ .{ .name = "User-Agent", .value = self.getUserAgent() },+ .{ .name = "Sec-Ch-Ua", .value = lp.Config.HttpHeaders.sec_ch_ua },+ // Omitting Accept-Language triggers bot-protection on some CDNs+ // (Akamai) when Accept-Encoding is present.+ .{ .name = "Accept-Language", .value = lp.Config.HttpHeaders.accept_language },+ };+}+pub fn abort(self: *Client) void {// Snapshot before killing: kill() -> deinit removes entries from// self.transfers, which would invalidate a live iterator.@@ -522,35 +515,31 @@ pub fn activity(self: *const Client) Activity {// Cherry-pick only Fetch interception responsesconst DrainMode = enum { all, sync_wait };-// One-shot convenience: create and submit in a single call. HttpClient takes-// ownership of req.headers; do not pair with `errdefer headers.deinit()`.-// Callers that have no headers of their own can leave req.headers at its-// (empty) default — the client fills in its baseline headers (user agent,-// etc.).+// One-shot convenience: create and submit in a single call.pub fn request(self: *Client, req: Request, owner: ?*Owner) anyerror!void {const transfer = try self.newRequest(req, owner);return transfer.submit();}// Create a request without submitting it. The caller owns the returned-// transfer until transfer.submit(). On error, header is freed. On success,-// headers is owned by the transfer.+// transfer until transfer.submit() consumes it (or until syncRequest, which+// consumes it the same way): it can add headers and mutate transfer.req,+// scoped as:+//+// const transfer = try client.newRequest(.{...}, owner);+// {+// errdefer transfer.deinit();+// try transfer.addHeader("Blah", "x", .{});+// }+// try transfer.submit();+//+// The errdefer must cover the mutation only, never submit itself.pub fn newRequest(self: *Client, req: Request, owner: ?*Owner) anyerror!*Transfer {- const arena = self.arena_pool.acquire(.small, "Request.arena") catch |err| {- req.headers.deinit();- return err;- };+ const arena = try self.arena_pool.acquire(.small, "Request.arena");const transfer = blk: {var owned = req;- errdefer {- owned.headers.deinit();- arena.release();- }-- if (owned.headers.headers == null) {- owned.headers = try self.newHeaders();- }+ errdefer arena.release();// Most of the time, the req data will outlive the transfer. But not// always. The most problematic case is with a QueuedNavigation which@@ -572,16 +561,6 @@ pub fn newRequest(self: *Client, req: Request, owner: ?*Owner) anyerror!*Transfeif (req.body_outlives_request == false) {owned.body = try arena.dupe(u8, b);}-- // Browsers never send Expect: 100-continue; libcurl generates it- // for HTTP/1.1 requests whose body exceeds 1MB- // (EXPECT_100_THRESHOLD), which stalls the request ~1s against- // servers/proxies that never answer the interim response. An- // empty value ("Expect:") suppresses the generated header. Only- // requests with a body can trigger it, and over HTTP/2 curl never- // generates it, so the entry is inert there. Added here (not- // configureConn) so redirect/auth retries don't append duplicates.- try owned.headers.add("Expect:");}const t = try arena.create(Transfer);@@ -598,6 +577,7 @@ pub fn newRequest(self: *Client, req: Request, owner: ?*Owner) anyerror!*Transfe.owner = null,.owner_node = .{},};+ try t.seedHeaders();break :blk t;};@@ -834,7 +814,7 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {.start => {if (self.network.web_bot_auth) |wba| {const authority = URL.getHost(transfer.req.url);- try wba.signRequest(transfer.arena.allocator(), &transfer.req.headers, authority);+ try wba.signRequest(transfer, authority);}if (self.serve_mode) {@@ -920,13 +900,15 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {transfer._cache_key = req.url;const arena = transfer.arena;- var iter = req.headers.iterator();- const req_headers = try iter.collect(arena.allocator());+ const req_headers = try arena.alloc(http.Header, transfer.req_headers.items.len);+ for (transfer.req_headers.items, req_headers) |hdr, *out| {+ out.* = .{ .name = hdr.name, .value = hdr.value };+ }const cached = cache.get(arena.allocator(), .{.url = req.url,.timestamp = lp.datetime.timestamp(.real),- .request_headers = req_headers.items,+ .request_headers = req_headers,}) orelse {lp.metrics.http_cache.incr(.miss);transfer._cache_intent = .store;@@ -955,10 +937,10 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {.last_modified = cached.metadata.last_modified,});if (cached.metadata.etag) |etag| {- try req.headers.add(try std.fmt.allocPrintSentinel(arena.allocator(), "If-None-Match: {s}", .{etag}, 0));+ try transfer.addHeader("If-None-Match", etag, .{});}if (cached.metadata.last_modified) |lm| {- try req.headers.add(try std.fmt.allocPrintSentinel(arena.allocator(), "If-Modified-Since: {s}", .{lm}, 0));+ try transfer.addHeader("If-Modified-Since", lm, .{});}transfer._cache_intent = .{ .revalidate = cached };return false;@@ -1039,17 +1021,16 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {var vary_headers: std.ArrayList(http.Header) = .empty;if (vary) |vary_str| {- var req_it = transfer.req.headers.iterator();- while (req_it.next()) |hdr| {+ for (transfer.req_headers.items) |hdr| {var vary_iter = std.mem.splitScalar(u8, vary_str, ',');while (vary_iter.next()) |part| {const name = std.mem.trim(u8, part, &std.ascii.whitespace);if (std.ascii.eqlIgnoreCase(hdr.name, name)) {- const owned: http.Header = .{- .name = arena.dupe(u8, hdr.name) catch return,- .value = arena.dupe(u8, hdr.value) catch return,- };- vary_headers.append(arena.allocator(), owned) catch return;+ // name/value already live in transfer.arena+ vary_headers.append(arena.allocator(), .{+ .name = hdr.name,+ .value = hdr.value,+ }) catch return;}}}@@ -1125,34 +1106,33 @@ const SyncContext = struct {}};+// Synchronous submit for a transfer created with newRequest. Like `submit`,+// `syncRequest` consuems the transfer unconditionally.// Caller must deinit SyncResponse or otherwise take ownership of its optional arena-pub fn syncRequest(self: *Client, req: Request, owner: *Owner) !SyncResponse {+pub fn syncRequest(self: *Client, transfer: *Transfer) !SyncResponse {if (self.inbox.terminated) {- // request() takes ownership of req.headers on every path; we return- // before calling it, so free the curl_slist here to avoid leaking it.- req.deinit();+ transfer.deinit();return error.ClientDisconnected;}// A parser can start another blocking script/style fetch while unwinding// a previous interrupted fetch. The first tickSync below would fail- // anyway; bail before creating the transfer and notifying CDP.+ // anyway; bail before submitting and notifying CDP.if (self.hasPendingTeardown()) {- req.deinit();+ transfer.deinit();return error.SyncWaitInterrupted;}var sync_ctx = SyncContext{ .client = self, .body = .empty };errdefer if (sync_ctx.arena) |arena| arena.release();- var r = req;- r.sync = true;- r.ctx = &sync_ctx;- r.header_callback = SyncContext.headerCallback;- r.data_callback = SyncContext.dataCallback;- r.done_callback = SyncContext.doneCallback;- r.error_callback = SyncContext.errorCallback;- r.shutdown_callback = SyncContext.shutdownCallback;- const transfer = try self.newRequest(r, owner);+ const req = &transfer.req;+ req.sync = true;+ req.ctx = &sync_ctx;+ req.header_callback = SyncContext.headerCallback;+ req.data_callback = SyncContext.dataCallback;+ req.done_callback = SyncContext.doneCallback;+ req.error_callback = SyncContext.errorCallback;+ req.shutdown_callback = SyncContext.shutdownCallback;const frame_id = req.frame_id;self.blocking_requests.putNoClobber(self.allocator, frame_id, transfer.id) catch |err| {@@ -1646,9 +1626,6 @@ pub const Request = struct {loader_id: u32,method: Method,url: [:0]const u8,- // Empty by default; the client fills in its baseline headers (user- // agent, sec-ch-ua, accept-language) when none are supplied.- headers: http.Headers = .{ .headers = null },body: ?[]const u8 = null,cookie_jar: ?*CookieJar,cookie_origin: [:0]const u8,@@ -1711,10 +1688,6 @@ pub const Request = struct {const written = aw.written();return written.ptr[0 .. written.len - 1 :0];}-- pub fn deinit(self: *const Request) void {- self.headers.deinit();- }};pub const SyncResponse = struct {@@ -1919,6 +1892,8 @@ pub const Transfer = struct {res: Response = .{},client: *Client,+ req_headers: std.ArrayList(RequestHeader) = .empty,+start_time: u64,_notified_fail: bool = false,@@ -2145,7 +2120,6 @@ pub const Transfer = struct {// Any concurrent CDP lookup by id will now see this transfer as gone._ = self.client.transfers.remove(self.id);- self.req.deinit();if (self.owner) |o| {o.removeTransfer(self);}@@ -2496,9 +2470,26 @@ pub const Transfer = struct {try conn.setGetMode();}- var header_list = req.headers;- try conn.secretHeaders(&header_list, &client.network.config.http_headers);- try conn.setHeaders(&header_list);+ // Build the conn's curl_slist from req_headers, fresh on every+ // attempt so redirect/auth retries pick up mutations and never+ // accumulate duplicates.+ conn.clearHeaders();+ const arena = self.arena.allocator();+ for (self.req_headers.items) |hdr| {+ try conn.addHeader(arena, hdr.name, hdr.value);+ }+ if (req.body != null) {+ // Browsers never send Expect: 100-continue; libcurl generates it+ // for HTTP/1.1 requests whose body exceeds 1MB+ // (EXPECT_100_THRESHOLD), which stalls the request ~1s against+ // servers/proxies that never answer the interim response. An+ // empty value ("Expect:") suppresses the generated header. Only+ // requests with a body can trigger it, and over HTTP/2 curl never+ // generates it, so the entry is inert there.+ try conn.addRawHeader("Expect:");+ }+ try conn.secretHeaders(&client.network.config.http_headers);+ try conn.commitHeaders();// Add cookies from cookie jar.if (try self.req.getCookieString(self.arena.allocator())) |cookies| {@@ -2666,20 +2657,68 @@ pub const Transfer = struct {self.req.credentials = userpwd;}- pub fn replaceRequestHeaders(self: *Transfer, allocator: Allocator, headers: []const http.Header) !void {- self.req.headers.deinit();+ pub const RequestHeader = struct {+ name: []const u8,+ value: []const u8,+ source: HeaderSource = .user_agent,+ };++ // Who put the header on the request. CORS cares: only non-safelisted+ // author (i.e. script-set) headers trigger a preflight.+ pub const HeaderSource = enum { user_agent, author };- var buf: std.ArrayList(u8) = .empty;- var new_headers = try self.client.newHeaders();+ pub const HeaderOpts = struct {+ source: HeaderSource = .user_agent,+ };++ pub fn addHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {+ const arena = self.arena.allocator();+ try self.req_headers.append(arena, .{+ .name = try arena.dupe(u8, name),+ .value = try arena.dupe(u8, value),+ .source = opts.source,+ });+ }++ // Adds, replacing every existing header with the same case-insensitive name+ pub fn setHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {+ var found = false;+ var i: usize = 0;+ while (i < self.req_headers.items.len) {+ const hdr = &self.req_headers.items[i];+ if (std.ascii.eqlIgnoreCase(hdr.name, name) == false) {+ i += 1;+ continue;+ }+ if (found) {+ _ = self.req_headers.orderedRemove(i);+ continue;+ }+ found = true;+ hdr.value = try self.arena.allocator().dupe(u8, value);+ hdr.source = opts.source;+ i += 1;+ }+ if (!found) {+ try self.addHeader(name, value, opts);+ }+ }++ // The client's baseline headers, added to every request at creation.+ fn seedHeaders(self: *Transfer) !void {+ for (self.client.baselineHeaders()) |hdr| {+ try self.addHeader(hdr.name, hdr.value, .{});+ }+ }++ // CDP Fetch.continueRequest: the intercepting client supplies the+ // complete header set, replacing whatever the request carried.+ pub fn replaceRequestHeaders(self: *Transfer, headers: []const http.Header) !void {+ self.req_headers.clearRetainingCapacity();+ try self.seedHeaders();for (headers) |hdr| {- // safe to re-use this buffer, because Headers.add because curl copies- // the value we pass into curl_slist_append.- defer buf.clearRetainingCapacity();- try std.fmt.format(buf.writer(allocator), "{s}: {s}", .{ hdr.name, hdr.value });- try buf.append(allocator, 0); // null terminated- try new_headers.add(buf.items[0 .. buf.items.len - 1 :0]);+ try self.setHeader(hdr.name, hdr.value, .{});}- self.req.headers = new_headers;}// abortAuthChallenge is called when an auth challenge interception is@@ -3292,6 +3331,53 @@ test "HttpClient: URL blocking exempts internal transfers" {try testing.expect(!client.isUrlBlocked("https://example.test/robots.txt", true));}+test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {+ var pool = ArenaPool.init(testing.allocator, .{});+ defer pool.deinit();++ const arena = try pool.acquire(.small, "test");+ defer arena.release();++ var transfer = Transfer{+ .arena = arena,+ .owner = null,+ .req = .{+ .frame_id = 0,+ .loader_id = 0,+ .method = .GET,+ .url = "http://example.com/",+ .cookie_jar = null,+ .cookie_origin = "",+ .resource_type = .document,+ .notification = undefined,+ .shutdown_callback = noopShutdown,+ },+ .client = undefined,+ .id = 1,+ .start_time = 0,+ };++ try transfer.addHeader("User-Agent", "Lightpanda/1.0", .{});+ try transfer.addHeader("X-Twice", "a", .{});+ try transfer.addHeader("x-twice", "b", .{});++ // replaces in place, collapsing duplicates+ try transfer.setHeader("user-agent", "Custom/1.0", .{});+ try transfer.setHeader("X-TWICE", "c", .{ .source = .author });+ // no match: appends+ try transfer.setHeader("X-New", "yes", .{});++ const headers = transfer.req_headers.items;+ try testing.expectEqual(3, headers.len);+ try testing.expectEqual("User-Agent", headers[0].name);+ try testing.expectEqual("Custom/1.0", headers[0].value);+ try testing.expectEqual("X-Twice", headers[1].name);+ try testing.expectEqual("c", headers[1].value);+ try testing.expectEqual(.author, headers[1].source);+ try testing.expectEqual("X-New", headers[2].name);+ try testing.expectEqual("yes", headers[2].value);+}+test "HttpClient: fulfillIntercepted survives a done_callback that tears down the owner" {// Regression: the fulfilled response's done_callback runs JS which// navigates / closes the page, re-entrantly killing the transferdiff --git a/src/network/WebBotAuth.zig b/src/network/WebBotAuth.zigindex 68dd20858..aae9d1df1 100644--- a/src/network/WebBotAuth.zig+++ b/src/network/WebBotAuth.zig@@ -20,7 +20,8 @@ const std = @import("std");const lp = @import("lightpanda");const crypto = @import("../sys/libcrypto.zig");-const Http = @import("../network/http.zig");+const ArenaPool = @import("../ArenaPool.zig");+const Transfer = @import("HttpClient.zig").Transfer;const WebBotAuth = @This();@@ -90,28 +91,26 @@ pub fn fromConfig(allocator: std.mem.Allocator, config: *const Config) !WebBotAupub fn signRequest(self: *const WebBotAuth,- allocator: std.mem.Allocator,- headers: *Http.Headers,+ transfer: *Transfer,authority: []const u8,) !void {+ const arena = transfer.arena.allocator();const now = lp.datetime.timestamp(.real);const expires = now + 60;// build the signature-input value (without the sig1= label)const sig_input_value = try std.fmt.allocPrint(- allocator,+ arena,"(\"@authority\" \"signature-agent\");created={d};expires={d};keyid=\"{s}\";alg=\"ed25519\";tag=\"web-bot-auth\"",.{ now, expires, self.keyid },);- defer allocator.free(sig_input_value);// build the canonical string to signconst canonical = try std.fmt.allocPrint(- allocator,+ arena,"\"@authority\": {s}\n\"signature-agent\": \"{s}\"\n\"@signature-params\": {s}",.{ authority, self.directory_url, sig_input_value },);- defer allocator.free(canonical);// sign itvar sig: [64]u8 = undefined;@@ -119,38 +118,12 @@ pub fn signRequest(// base64 encodeconst encoded_len = std.base64.standard.Encoder.calcSize(sig.len);- const encoded = try allocator.alloc(u8, encoded_len);- defer allocator.free(encoded);+ const encoded = try arena.alloc(u8, encoded_len);_ = std.base64.standard.Encoder.encode(encoded, &sig);- // build the 3 headers and add them- const sig_agent = try std.fmt.allocPrintSentinel(- allocator,- "Signature-Agent: \"{s}\"",- .{self.directory_url},- 0,- );- defer allocator.free(sig_agent);-- const sig_input = try std.fmt.allocPrintSentinel(- allocator,- "Signature-Input: sig1={s}",- .{sig_input_value},- 0,- );- defer allocator.free(sig_input);-- const signature = try std.fmt.allocPrintSentinel(- allocator,- "Signature: sig1=:{s}:",- .{encoded},- 0,- );- defer allocator.free(signature);-- try headers.add(sig_agent);- try headers.add(sig_input);- try headers.add(signature);+ try transfer.addHeader("Signature-Agent", try std.fmt.allocPrint(arena, "\"{s}\"", .{self.directory_url}), .{});+ try transfer.addHeader("Signature-Input", try std.fmt.allocPrint(arena, "sig1={s}", .{sig_input_value}), .{});+ try transfer.addHeader("Signature", try std.fmt.allocPrint(arena, "sig1=:{s}:", .{encoded}), .{});}pub fn deinit(self: WebBotAuth, allocator: std.mem.Allocator) void {@@ -260,26 +233,41 @@ test "signRequest: adds headers with correct names" {};defer auth.deinit(allocator);- var headers = try Http.Headers.init("User-Agent: Test-Agent");- defer headers.deinit();-- try auth.signRequest(allocator, &headers, "example.com");-- var it = headers.iterator();- var found_sig_agent = false;- var found_sig_input = false;- var found_signature = false;- var count: usize = 0;+ var pool = ArenaPool.init(allocator, .{});+ defer pool.deinit();+ const arena = try pool.acquire(.small, "test");+ defer arena.release();++ var transfer = Transfer{+ .arena = arena,+ .owner = null,+ .req = .{+ .frame_id = 0,+ .loader_id = 0,+ .method = .GET,+ .url = "https://example.com/",+ .cookie_jar = null,+ .cookie_origin = "",+ .resource_type = .document,+ .notification = undefined,+ .shutdown_callback = @import("HttpClient.zig").noopShutdown,+ },+ .client = undefined,+ .id = 1,+ .start_time = 0,+ };- while (it.next()) |h| {- count += 1;- if (std.ascii.eqlIgnoreCase(h.name, "Signature-Agent")) found_sig_agent = true;- if (std.ascii.eqlIgnoreCase(h.name, "Signature-Input")) found_sig_input = true;- if (std.ascii.eqlIgnoreCase(h.name, "Signature")) found_signature = true;- }+ try auth.signRequest(&transfer, "example.com");- try std.testing.expect(count >= 3);- try std.testing.expect(found_sig_agent);- try std.testing.expect(found_sig_input);- try std.testing.expect(found_signature);+ const headers = transfer.req_headers.items;+ try std.testing.expectEqual(3, headers.len);+ try std.testing.expectEqualStrings("Signature-Agent", headers[0].name);+ try std.testing.expectEqualStrings(+ "\"https://example.com/.well-known/http-message-signatures-directory\"",+ headers[0].value,+ );+ try std.testing.expectEqualStrings("Signature-Input", headers[1].name);+ try std.testing.expect(std.mem.startsWith(u8, headers[1].value, "sig1=(\"@authority\" \"signature-agent\")"));+ try std.testing.expectEqualStrings("Signature", headers[2].name);+ try std.testing.expect(std.mem.startsWith(u8, headers[2].value, "sig1=:"));}diff --git a/src/network/http.zig b/src/network/http.zigindex be384566b..b2631dd04 100644--- a/src/network/http.zig+++ b/src/network/http.zig@@ -61,6 +61,15 @@ pub const Header = struct {value: []const u8,};+ pub fn parse(header_str: []const u8) ?Header {+ const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null;++ const name = std.mem.trim(u8, header_str[0..colon_pos], " \t");+ const value = std.mem.trim(u8, header_str[colon_pos + 1 ..], " \t");++ return .{ .name = name, .value = value };+ }+// The header value up to the first ';', trimmed (e.g. "attachment" for a// Content-Disposition, "text/html" for a Content-Type).pub fn firstValue(self: Header) []const u8 {@@ -110,98 +119,12 @@ pub const Header = struct {};};-pub const Headers = struct {- headers: ?*libcurl.CurlSList,-- pub fn init(user_agent: [:0]const u8) !Headers {- const header_list = libcurl.curl_slist_append(null, user_agent);- if (header_list == null) {- return error.OutOfMemory;- }- // libcurl leaves the list intact when curl_slist_append fails, so we own it.- errdefer libcurl.curl_slist_free_all(header_list);-- // Always add sec-CH-UA header- const with_sec_ch_ua = libcurl.curl_slist_append(header_list, Config.HttpHeaders.sec_ch_ua);- if (with_sec_ch_ua == null) {- return error.OutOfMemory;- }-- // Always add Accept-Language. Omitting it triggers bot-protection on- // some CDNs (Akamai) when Accept-Encoding is present.- const updated_headers = libcurl.curl_slist_append(with_sec_ch_ua, Config.HttpHeaders.accept_language);- if (updated_headers == null) {- return error.OutOfMemory;- }-- return .{ .headers = updated_headers };- }-- pub fn deinit(self: *const Headers) void {- if (self.headers) |hdr| {- libcurl.curl_slist_free_all(hdr);- }- }-- pub fn add(self: *Headers, header: [*c]const u8) !void {- // Copies the value- const updated_headers = libcurl.curl_slist_append(self.headers, header);- if (updated_headers == null) {- return error.OutOfMemory;- }-- self.headers = updated_headers;- }-- // Adds `header` ("Name: Value"), replacing any existing header with the- // same case-insensitive name. Caller-supplied headers (CDP- // Network.setExtraHTTPHeaders) must override built-in defaults like- // User-Agent; a plain append produces a duplicate that libcurl silently- // collapses to the first occurrence, so the override would be dropped.- pub fn set(self: *Headers, header: [*c]const u8) !void {- const new = parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(header)))) orelse {- // No colon: nothing to match against, fall back to append.- return self.add(header);- };-- var rebuilt: ?*libcurl.CurlSList = null;- errdefer libcurl.curl_slist_free_all(rebuilt);-- var node = self.headers;- while (node) |n| : (node = n.*.next) {- const data = @as([*:0]const u8, @ptrCast(n.*.data));- if (parseHeader(std.mem.span(data))) |existing| {- if (std.ascii.eqlIgnoreCase(existing.name, new.name)) continue;- }- rebuilt = libcurl.curl_slist_append(rebuilt, data) orelse return error.OutOfMemory;- }- rebuilt = libcurl.curl_slist_append(rebuilt, header) orelse return error.OutOfMemory;-- libcurl.curl_slist_free_all(self.headers);- self.headers = rebuilt;- }-- pub fn parseHeader(header_str: []const u8) ?Header {- const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null;-- const name = std.mem.trim(u8, header_str[0..colon_pos], " \t");- const value = std.mem.trim(u8, header_str[colon_pos + 1 ..], " \t");-- return .{ .name = name, .value = value };- }-- pub fn iterator(self: Headers) HeaderIterator {- return .{ .curl_slist = .{ .header = self.headers } };- }-};--// In normal cases, the header iterator comes from the curl linked list.+// In normal cases, the header iterator comes from the curl connection.// But it's also possible to inject a response, via `transfer.fulfill`. In that// case, the response headers are a list, []const Http.Header.// This union, is an iterator that exposes the same API for either case.pub const HeaderIterator = union(enum) {curl: CurlHeaderIterator,- curl_slist: CurlSListIterator,list: ListHeaderIterator,pub fn next(self: *HeaderIterator) ?Header {@@ -239,16 +162,6 @@ pub const HeaderIterator = union(enum) {}};- const CurlSListIterator = struct {- header: [*c]libcurl.CurlSList,-- pub fn next(self: *CurlSListIterator) ?Header {- const h = self.header orelse return null;- self.header = h.*.next;- return Headers.parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(h.*.data))));- }- };-const ListHeaderIterator = struct {index: usize = 0,list: []const Header,@@ -355,6 +268,11 @@ pub const Connection = struct {transport: Transport,node: std.DoublyLinkedList.Node = .{},+ // The curl_slist accumulated by addHeader/addRawHeader and handed to+ // curl by commitHeaders. Owned here because curl reads it during+ // perform; freed on reset/deinit or by the next clearHeaders.+ _header_list: ?*libcurl.CurlSList = null,+pub const Transport = union(enum) {none, // used for cases that manage their own connection, e.g. telemetryhttp: *@import("HttpClient.zig").Transfer,@@ -375,7 +293,8 @@ pub const Connection = struct {return self;}- pub fn deinit(self: *const Connection) void {+ pub fn deinit(self: *Connection) void {+ self.clearHeaders();libcurl.curl_easy_cleanup(self._easy);}@@ -432,8 +351,37 @@ pub const Connection = struct {try libcurl.curl_easy_setopt(self._easy, .http_get, true);}- pub fn setHeaders(self: *const Connection, headers: *Headers) !void {- try libcurl.curl_easy_setopt(self._easy, .http_header, headers.headers);+ // Appends "name: value" to the connection's pending header list, or+ // "name;" when the value is empty — curl reads "name:" as+ // remove-this-header and the ";" form as send-an-empty-value. curl+ // copies the string, so `allocator` only backs the transient join.+ pub fn addHeader(self: *Connection, allocator: std.mem.Allocator, name: []const u8, value: []const u8) !void {+ const joined = if (value.len == 0)+ try std.fmt.allocPrintSentinel(allocator, "{s};", .{name}, 0)+ else+ try std.fmt.allocPrintSentinel(allocator, "{s}: {s}", .{ name, value }, 0);+ return self.addRawHeader(joined);+ }++ // `header` is raw curl header syntax, e.g. "Expect:" to suppress a+ // curl-generated header. curl copies the value.+ pub fn addRawHeader(self: *Connection, header: [*c]const u8) !void {+ const updated = libcurl.curl_slist_append(self._header_list, header);+ if (updated == null) {+ return error.OutOfMemory;+ }+ self._header_list = updated;+ }++ pub fn commitHeaders(self: *const Connection) !void {+ try libcurl.curl_easy_setopt(self._easy, .http_header, self._header_list);+ }++ pub fn clearHeaders(self: *Connection) void {+ if (self._header_list) |list| {+ libcurl.curl_slist_free_all(list);+ self._header_list = null;+ }}pub fn setCookies(self: *const Connection, cookies: [*c]const u8) !void {@@ -500,6 +448,7 @@ pub const Connection = struct {) !void {libcurl.curl_easy_reset(self._easy);self.transport = .none;+ self.clearHeaders();// timeoutstry libcurl.curl_easy_setopt(self._easy, .timeout_ms, config.httpTimeout());@@ -679,25 +628,14 @@ pub const Connection = struct {}// These are headers that may not be send to the users for inteception.- pub fn secretHeaders(_: *const Connection, headers: *Headers, http_headers: *const Config.HttpHeaders) !void {+ pub fn secretHeaders(self: *Connection, http_headers: *const Config.HttpHeaders) !void {if (http_headers.proxy_bearer_header) |hdr| {- try headers.add(hdr);+ try self.addRawHeader(hdr);}}- pub fn request(self: *const Connection, http_headers: *const Config.HttpHeaders) !u16 {- var header_list = try Headers.init(http_headers.user_agent_header);- defer header_list.deinit();- try self.secretHeaders(&header_list, http_headers);- try self.setHeaders(&header_list);-- try libcurl.curl_easy_perform(self._easy);- return self.getResponseCode();- }-- // Synchronous transfer that adds no request headers. request() injects the- // browser User-Agent / sec-ch-ua machinery meant for page fetches; callers- // that manage their own connection (telemetry) use this leaner path.+ // Synchronous transfer that adds no request headers; callers that manage+ // their own connection (telemetry) use this leaner path.pub fn perform(self: *const Connection) !u16 {try libcurl.curl_easy_perform(self._easy);return self.getResponseCode();@@ -982,52 +920,6 @@ test "Header.param" {try testing.expect((Header{ .name = "Content-Disposition", .value = "attachment; filename=\"\"" }).param("filename") == null);}-fn findHeader(headers: Headers, name: []const u8) struct { count: usize, value: []const u8 } {- var count: usize = 0;- var value: []const u8 = "";- var it = headers.iterator();- while (it.next()) |h| {- if (std.ascii.eqlIgnoreCase(h.name, name)) {- count += 1;- value = h.value;- }- }- return .{ .count = count, .value = value };-}--test "Headers.set replaces an existing header instead of duplicating it" {- var headers = try Headers.init("User-Agent: Lightpanda/1.0");- defer headers.deinit();-- try headers.set("User-Agent: Custom/1.0");-- const ua = findHeader(headers, "User-Agent");- try testing.expectEqual(@as(usize, 1), ua.count);- try testing.expectString("Custom/1.0", ua.value);-}--test "Headers.set matches header names case-insensitively" {- var headers = try Headers.init("User-Agent: Lightpanda/1.0");- defer headers.deinit();-- try headers.set("user-agent: Custom/1.0");-- const ua = findHeader(headers, "User-Agent");- try testing.expectEqual(@as(usize, 1), ua.count);- try testing.expectString("Custom/1.0", ua.value);-}--test "Headers.set adds a new header and preserves defaults" {- var headers = try Headers.init("User-Agent: Lightpanda/1.0");- defer headers.deinit();-- try headers.set("X-Custom: yes");-- try testing.expectEqual(@as(usize, 1), findHeader(headers, "X-Custom").count);- try testing.expectEqual(@as(usize, 1), findHeader(headers, "User-Agent").count);- try testing.expectEqual(@as(usize, 1), findHeader(headers, "Accept-Language").count);-}-test "opensocketCallback: private IPv4 returns CURL_SOCKET_BAD" {testing.silenceLog(&.{.http});