Lightpanda: the headless browser designed for AI and automation
refactor: HttpClient
Karl Seguin2eab4d2630f5
diff --git a/src/Notification.zig b/src/Notification.zigindex 4a96b2382..c28a74c85 100644--- a/src/Notification.zig+++ b/src/Notification.zig@@ -21,8 +21,7 @@ const lp = @import("lightpanda");const js = @import("browser/js/js.zig");const Frame = @import("browser/Frame.zig");-const Transfer = @import("browser/HttpClient.zig").Transfer;-const Response = @import("browser/HttpClient.zig").Response;+const Transfer = @import("network/HttpClient.zig").Transfer;const log = lp.log;const Execution = js.Execution;@@ -224,7 +223,6 @@ pub const ResponseData = struct {pub const ResponseHeaderDone = struct {transfer: *Transfer,- response: *const Response,};pub const RequestDone = struct {diff --git a/src/browser/Browser.zig b/src/browser/Browser.zigindex e79834aa0..b767140ed 100644--- a/src/browser/Browser.zig+++ b/src/browser/Browser.zig@@ -28,7 +28,7 @@ const Watchdog = @import("../Watchdog.zig");const Session = @import("Session.zig");const Selector = @import("webapi/selector/Selector.zig");const Viewport = @import("Viewport.zig");-const HttpClient = @import("HttpClient.zig");+const HttpClient = @import("../network/HttpClient.zig");const PermissionState = @import("webapi/Permissions.zig").State;const ArenaPool = App.ArenaPool;diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex 23c2047c8..aa51e6eb6 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -60,7 +60,7 @@ const popover = @import("webapi/element/popover.zig");const slotting = @import("webapi/element/slotting.zig");const NavigationKind = @import("webapi/navigation/root.zig").NavigationKind;-const HttpClient = @import("HttpClient.zig");+const HttpClient = @import("../network/HttpClient.zig");const sys_url = @import("../sys/url.zig");const timestamp = @import("../datetime.zig").timestamp;@@ -597,16 +597,16 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !voself._load_state = .parsing;self._last_navigate_error = null;- const req_id = self._session.browser.http_client.nextReqId();log.info(.frame, "navigate", .{.url = request_url,.method = opts.method,.reason = opts.reason,.body = opts.body != null,- .req_id = req_id,.type = self._type,});+ const http_client = &session.browser.http_client;+// Handle synthetic navigations: about:blank and blob: URLsconst is_about_blank = std.mem.eql(u8, "about:blank", request_url);const is_blob = !is_about_blank and std.mem.startsWith(u8, request_url, "blob:");@@ -670,6 +670,10 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo};}+ // No real request is made, but CDP still correlates the navigation+ // events by request id, so consume one.+ const req_id = http_client.incrReqId();+session.notification.dispatch(.frame_navigate, &.{.opts = opts,.req_id = req_id,@@ -694,9 +698,6 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo.timestamp = timestamp(.monotonic),});- // force next request id manually b/c we won't create a real req.- _ = session.browser.http_client.incrReqId();-if (self.parent == null) {session.navigation._current_navigation_kind = opts.kind;try session.navigation.commitNavigation(self);@@ -706,8 +707,6 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !voreturn;}- const http_client = &session.browser.http_client;-self._http_status = null;self._http_headers = .empty;@@ -719,7 +718,6 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo};self.origin = try URL.getOrigin(self.arena, self.url);- self._req_id = req_id;self._navigated_options = .{.cdp_id = opts.cdp_id,.reason = opts.reason,@@ -728,14 +726,37 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo.header = if (opts.header) |h| try self.arena.dupeZ(u8, h) else null,};- var headers = try http_client.newHeaders();- try headers.add(lp.Config.HttpHeaders.navigation_accept);- if (opts.header) |hdr| {- try headers.add(hdr);- }- if (opts.referer) |ref| {- const ref_header = try std.mem.concatWithSentinel(self.arena, u8, &.{ "Referer: ", ref }, 0);- try headers.add(ref_header);+ const transfer = try http_client.newRequest(.{+ .ctx = self,+ .url = self.url,+ .frame_id = self._frame_id,+ .loader_id = self._loader_id,+ .method = opts.method,+ .body = opts.body,+ .cookie_jar = &session.cookie_jar,+ .cookie_origin = opts.initiator_url orelse self.url,+ .resource_type = .document,+ .notification = self._session.notification,+ .header_callback = frameHeaderDoneCallback,+ .data_callback = frameDataCallback,+ .done_callback = frameDoneCallback,+ .error_callback = frameErrorCallback,+ // The frame tracks its navigation by id (_req_id), never by pointer.+ .shutdown_callback = HttpClient.noopShutdown,+ }, &self._http_owner);+ self._req_id = transfer.id;++ {+ // Ours until submit; clean up if header setup fails.+ errdefer transfer.deinit();+ try transfer.req.headers.add(lp.Config.HttpHeaders.navigation_accept);+ if (opts.header) |hdr| {+ try transfer.req.headers.add(hdr);+ }+ if (opts.referer) |ref| {+ const ref_header = try std.mem.concatWithSentinel(transfer.arena, u8, &.{ "Referer: ", ref }, 0);+ try transfer.req.headers.add(ref_header);+ }}// A root navigation issued against a pending Page (i.e. one allocated by@@ -751,7 +772,7 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vosession.notification.dispatch(.frame_navigate, &.{.opts = opts,.url = self.url,- .req_id = req_id,+ .req_id = transfer.id,.frame_id = self._frame_id,.loader_id = self._loader_id,.timestamp = timestamp(.monotonic),@@ -763,23 +784,7 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vosession.navigation._current_navigation_kind = opts.kind;- self.makeRequest(.{- .ctx = self,- .url = self.url,- .frame_id = self._frame_id,- .loader_id = self._loader_id,- .method = opts.method,- .headers = headers,- .body = opts.body,- .cookie_jar = &session.cookie_jar,- .cookie_origin = opts.initiator_url orelse self.url,- .resource_type = .document,- .notification = self._session.notification,- .header_callback = frameHeaderDoneCallback,- .data_callback = frameDataCallback,- .done_callback = frameDoneCallback,- .error_callback = frameErrorCallback,- }) catch |err| {+ transfer.submit() catch |err| {log.err(.frame, "navigate request", .{ .url = self.url, .err = err, .type = self._type });return err;};@@ -962,6 +967,11 @@ pub fn makeRequest(self: *Frame, req: HttpClient.Request) !void {return self._session.browser.http_client.request(req, &self._http_owner);}+// Two-phase variant; see HttpClient.newRequest for the ownership contract.+pub fn newRequest(self: *Frame, req: HttpClient.Request) !*HttpClient.Transfer {+ return self._session.browser.http_client.newRequest(req, &self._http_owner);+}+// Synchronously abort every transfer and WebSocket owned by this frame// and all of its descendants.pub fn abortTransfers(self: *Frame) void {@@ -970,8 +980,6 @@ pub fn abortTransfers(self: *Frame) void {}const http_client = &self._session.browser.http_client;http_client.abortOwner(&self._http_owner);- // abortOwner misses deferred contexts whose transfer already completed.- http_client.deferring_layer.cancelFrame(self._frame_id);}pub fn documentIsLoaded(self: *Frame) void {@@ -1144,8 +1152,8 @@ fn notifyParentLoadComplete(self: *Frame) void {parent.iframeCompletedLoading(self.iframe.?, self._delays_parent_load);}-fn frameHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResult {- var self: *Frame = @ptrCast(@alignCast(response.ctx));+fn frameHeaderDoneCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.HeaderResult {+ var self: *Frame = @ptrCast(@alignCast(transfer.req.ctx));// Commit point for a pending root navigation. The session has been// holding the OLD page alive during the round-trip; now that response@@ -1157,7 +1165,7 @@ fn frameHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResutry self._session.commitPendingPage(self._page);}- const response_url = response.url();+ const response_url = transfer.req.url;if (std.mem.eql(u8, response_url, self.url) == false) {// would be different than self.url in the case of a redirectself.url = try self.arena.dupeZ(u8, response_url);@@ -1169,7 +1177,7 @@ fn frameHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResu// Page.reload doesn't re-POST form data to the redirect target. Conservative// default — 307/308 technically preserve the method per RFC 7231, but// resubmitting form data is the more dangerous failure mode.- if ((response.redirectCount() orelse 0) > 0) {+ if ((transfer.redirectCount() orelse 0) > 0) {if (self._navigated_options) |*no| {no.method = .GET;no.body = null;@@ -1186,14 +1194,14 @@ fn frameHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResuif (comptime IS_DEBUG) {log.debug(.frame, "navigate header", .{.url = self.url,- .status = response.status(),- .content_type = response.contentType(),+ .status = transfer.responseStatus(),+ .content_type = transfer.contentType(),.type = self._type,});}- self._http_status = response.status();- var it = response.headerIterator();+ self._http_status = transfer.responseStatus();+ var it = transfer.responseHeaderIterator();while (it.next()) |hdr| {try self._http_headers.append(self.arena, .{.name = try self.arena.dupe(u8, hdr.name),@@ -1218,7 +1226,7 @@ fn frameHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResu// If the response is a file download, stream its body to disk instead of// parsing it as a page. This sets _parse_state to .download, which the// data/done callbacks below special-case.- _ = try self.maybeStartDownload(response);+ _ = try self.maybeStartDownload(transfer);return .proceed;}@@ -1227,7 +1235,7 @@ fn frameHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResu// treated as a download when Browser.setDownloadBehavior opted in// (allow/allowAndName) and the response carries Content-Disposition: attachment.// See issue #2701.-fn maybeStartDownload(self: *Frame, response: HttpClient.Response) !bool {+fn maybeStartDownload(self: *Frame, transfer: *HttpClient.Transfer) !bool {const session = self._session;switch (session.download_behavior) {.allow, .allow_and_name => {},@@ -1235,7 +1243,7 @@ fn maybeStartDownload(self: *Frame, response: HttpClient.Response) !bool {}const disposition: HttpClient.Header = blk: {- var it = response.headerIterator();+ var it = transfer.responseHeaderIterator();while (it.next()) |hdr| {if (std.ascii.eqlIgnoreCase(hdr.name, "content-disposition")) {break :blk hdr;@@ -1281,7 +1289,7 @@ fn maybeStartDownload(self: *Frame, response: HttpClient.Response) !bool {return false;};- const total: ?u64 = if (response.contentLength()) |cl| cl else null;+ const total: ?u64 = if (transfer.getContentLength()) |cl| cl else null;self._parse_state = .{ .download = .{.guid = guid,@@ -1364,14 +1372,14 @@ fn isUtf16Encoding(charset: []const u8) bool {return std.mem.eql(u8, charset, "UTF-16LE") or std.mem.eql(u8, charset, "UTF-16BE");}-fn frameDataCallback(response: HttpClient.Response, data: []const u8) !void {- var self: *Frame = @ptrCast(@alignCast(response.ctx));+fn frameDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {+ var self: *Frame = @ptrCast(@alignCast(transfer.req.ctx));if (self._parse_state == .pre) {// we lazily do this, because we might need the first chunk of data// to sniff the content typevar mime: Mime = blk: {- if (response.contentType()) |ct| {+ if (transfer.contentType()) |ct| {break :blk try Mime.parse(ct);}break :blk Mime.sniff(data);@@ -2096,18 +2104,10 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []coconst http_client = &session.browser.http_client;- // `syncRequest` below registers a blocking request for this frame, which- // makes the DeferringLayer hold back the completion callbacks of every- // OTHER in-flight transfer for the frame (e.g. a `<script defer>` still- // loading) so they don't run JS while we're on the parser stack. Those- // deferred completions must be flushed once the sync fetch returns —- // otherwise a `<script defer>` whose fetch finishes during this window is- // left at `complete == false` forever, the deferred-script queue never- // drains, and `documentIsLoaded` (readyState -> "interactive",- // DOMContentLoaded, the load event) never fires. The blocking-`<script>`- // path (ScriptManager.addFromElement) and the worker path already flush;- // the external-stylesheet path must too.- defer http_client.deferring_layer.flushFrame(self._frame_id);+ // `syncRequest` below registers a blocking request for this frame; the+ // client's dispatcher holds back every OTHER transfer's callbacks for+ // the frame while it's registered (they'd run JS on the parser's stack)+ // and delivers them on the next tick after the sync fetch returns.var headers = try http_client.newHeaders();try headers.add("Accept: text/css,*/*;q=0.1");@@ -2135,6 +2135,7 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co.cookie_origin = self.url,.resource_type = .stylesheet,.notification = session.notification,+ .shutdown_callback = HttpClient.noopShutdown, // syncRequest installs its own}) catch |err| {log.warn(.http, "external stylesheet fetch", .{ .err = err, .url = resolved });return self.fireElementEvent(element, comptime .wrap("error"));diff --git a/src/browser/Runner.zig b/src/browser/Runner.zigindex a96adedf4..b9aad1a47 100644--- a/src/browser/Runner.zig+++ b/src/browser/Runner.zig@@ -22,7 +22,7 @@ const lp = @import("lightpanda");const js = @import("js/js.zig");const Browser = @import("Browser.zig");const Session = @import("Session.zig");-const HttpClient = @import("HttpClient.zig");+const HttpClient = @import("../network/HttpClient.zig");const Node = @import("webapi/Node.zig");const Selector = @import("webapi/selector/Selector.zig");@@ -221,12 +221,12 @@ fn _tick(self: *Runner, comptime is_cdp: bool, timeout_ms: u32, conditions: []Wa}const http_active = http_client.http_active;- const http_next_tick = http_client.next_tick_count;- const total_http_activity = http_active + http_next_tick + http_client.interception_layer.intercepted;+ const http_buffered = http_client.dispatch_count;+ const total_http_activity = http_active + http_buffered + http_client.intercepted;const total_network_activity = total_http_activity + http_client.ws_active;const ms_to_next_macrotask = browser.msToNextMacrotask();- const network_idle = total_network_activity == 0 and http_client.queue.first == null and http_client.ready_queue.first == null;+ const network_idle = total_network_activity == 0 and http_client.pending_queue.first == null and http_client.ready_queue.first == null;const is_done = ms_to_next_macrotask == null and network_idle;// _we_ have nothing to run, but v8 is working on background tasks. We'lldiff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zigindex 2605ba3c5..bda6be59e 100644--- a/src/browser/ScriptManager.zig+++ b/src/browser/ScriptManager.zig@@ -20,7 +20,7 @@ const std = @import("std");const lp = @import("lightpanda");const builtin = @import("builtin");-const HttpClient = @import("HttpClient.zig");+const HttpClient = @import("../network/HttpClient.zig");const js = @import("js/js.zig");const URL = @import("URL.zig");@@ -358,6 +358,7 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e.cookie_origin = frame.url,.resource_type = .script,.notification = frame._session.notification,+ .shutdown_callback = HttpClient.noopShutdown, // syncRequest installs its own});script.source = .{ .remote = response.body };@@ -386,6 +387,9 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e.data_callback = Script.dataCallback,.done_callback = Script.doneCallback,.error_callback = Script.errorCallback,+ // Nothing holds the transfer; teardown cleanup runs through+ // the manager's script lists.+ .shutdown_callback = HttpClient.noopShutdown,});}@@ -396,9 +400,6 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_ereturn;}- // This will flush any deferred scripts.- defer self.base.client.deferring_layer.flushFrame(self.base.owner.frameId());-if (script.status < 200 or script.status > 299) {log.info(.http, "script load error", .{ .status = script.status });script.executeCallback(comptime .wrap("error"));diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zigindex 277332a18..a62e269fa 100644--- a/src/browser/ScriptManagerBase.zig+++ b/src/browser/ScriptManagerBase.zig@@ -20,8 +20,8 @@ const std = @import("std");const lp = @import("lightpanda");const builtin = @import("builtin");-const HttpClient = @import("HttpClient.zig");const http = @import("../network/http.zig");+const HttpClient = @import("../network/HttpClient.zig");const js = @import("js/js.zig");const Session = @import("Session.zig");@@ -454,6 +454,7 @@ pub fn getAsyncImport(self: *ScriptManagerBase, url: [:0]const u8, cb: ImportAsy.data_callback = Script.dataCallback,.done_callback = Script.doneCallback,.error_callback = Script.errorCallback,+ .shutdown_callback = Script.shutdownCallback,}) catch |err| {self.async_scripts.remove(&script.node);return err;@@ -660,19 +661,19 @@ pub const Script = struct {self.manager.releaseArena(self.arena);}- pub fn startCallback(response: HttpClient.Response) !void {- log.debug(.http, "script fetch start", .{ .req = response });+ pub fn startCallback(transfer: *HttpClient.Transfer) !void {+ log.debug(.http, "script fetch start", .{ .req = transfer });}- pub fn headerCallback(response: HttpClient.Response) !HttpClient.HeaderResult {- const self: *Script = @ptrCast(@alignCast(response.ctx));+ pub fn headerCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.HeaderResult {+ const self: *Script = @ptrCast(@alignCast(transfer.req.ctx));- self.status = response.status().?;- if (response.status() != 200) {+ self.status = transfer.responseStatus().?;+ if (transfer.responseStatus() != 200) {log.info(.http, "script header", .{- .req = response,- .status = response.status(),- .content_type = response.contentType(),+ .req = transfer,+ .status = transfer.responseStatus(),+ .content_type = transfer.contentType(),});return .abort;@@ -680,64 +681,61 @@ pub const Script = struct {if (comptime IS_DEBUG) {log.debug(.http, "script header", .{- .req = response,- .status = response.status(),- .content_type = response.contentType(),+ .req = transfer,+ .status = transfer.responseStatus(),+ .content_type = transfer.contentType(),});}- switch (response.inner) {- .transfer => |transfer| {- // temp debug, trying to figure out why the next assert sometimes- // fails. Is the buffer just corrupt or is headerCallback really- // being called twice?- lp.assert(self.header_callback_called == false, "ScriptManagerBase.Header recall", .{- .m = @tagName(std.meta.activeTag(self.extra)),- .a1 = self.debug_transfer_id,- .a2 = self.debug_transfer_tries,- .a3 = self.debug_transfer_aborted,- .a4 = self.debug_transfer_bytes_received,- .a5 = self.debug_transfer_notified_fail,- .a8 = self.debug_transfer_auth_challenge,- .a9 = self.debug_transfer_easy_id,- .b1 = transfer.id,- .b2 = transfer._tries,- .b3 = transfer.state == .aborted,- .b4 = transfer.res.bytes_received,- .b5 = transfer._notified_fail,- .b8 = transfer._auth_challenge != null,- .b9 = if (transfer._conn) |c| @intFromPtr(c._easy) else 0,- });- self.header_callback_called = true;- self.debug_transfer_id = transfer.id;- self.debug_transfer_tries = transfer._tries;- self.debug_transfer_aborted = transfer.state == .aborted;- self.debug_transfer_bytes_received = transfer.res.bytes_received;- self.debug_transfer_notified_fail = transfer._notified_fail;- self.debug_transfer_auth_challenge = transfer._auth_challenge != null;- self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0;- },- else => {},+ {+ // temp debug, trying to figure out why the next assert sometimes+ // fails. Is the buffer just corrupt or is headerCallback really+ // being called twice?+ lp.assert(self.header_callback_called == false, "ScriptManagerBase.Header recall", .{+ .m = @tagName(std.meta.activeTag(self.extra)),+ .a1 = self.debug_transfer_id,+ .a2 = self.debug_transfer_tries,+ .a3 = self.debug_transfer_aborted,+ .a4 = self.debug_transfer_bytes_received,+ .a5 = self.debug_transfer_notified_fail,+ .a8 = self.debug_transfer_auth_challenge,+ .a9 = self.debug_transfer_easy_id,+ .b1 = transfer.id,+ .b2 = transfer._tries,+ .b3 = transfer.state == .aborted,+ .b4 = transfer.res.bytes_received,+ .b5 = transfer._notified_fail,+ .b8 = transfer._auth_challenge != null,+ .b9 = if (transfer._conn) |c| @intFromPtr(c._easy) else 0,+ });+ self.header_callback_called = true;+ self.debug_transfer_id = transfer.id;+ self.debug_transfer_tries = transfer._tries;+ self.debug_transfer_aborted = transfer.state == .aborted;+ self.debug_transfer_bytes_received = transfer.res.bytes_received;+ self.debug_transfer_notified_fail = transfer._notified_fail;+ self.debug_transfer_auth_challenge = transfer._auth_challenge != null;+ self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0;}lp.assert(self.source.remote.capacity == 0, "ScriptManagerBase.Header buffer", .{ .capacity = self.source.remote.capacity });var buffer: std.ArrayList(u8) = .empty;- if (response.contentLength()) |cl| {+ if (transfer.getContentLength()) |cl| {try buffer.ensureTotalCapacity(self.arena, cl);}self.source = .{ .remote = buffer };return .proceed;}- pub fn dataCallback(response: HttpClient.Response, data: []const u8) !void {- const self: *Script = @ptrCast(@alignCast(response.ctx));- self._dataCallback(response, data) catch |err| {- log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = response, .len = data.len });+ pub fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {+ const self: *Script = @ptrCast(@alignCast(transfer.req.ctx));+ self._dataCallback(transfer, data) catch |err| {+ log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = transfer, .len = data.len });return err;};}- fn _dataCallback(self: *Script, _: HttpClient.Response, data: []const u8) !void {+ fn _dataCallback(self: *Script, _: *HttpClient.Transfer, data: []const u8) !void {try self.source.remote.appendSlice(self.arena, data);}diff --git a/src/browser/js/Execution.zig b/src/browser/js/Execution.zigindex ee3abadf6..5efd164c6 100644--- a/src/browser/js/Execution.zig+++ b/src/browser/js/Execution.zig@@ -33,7 +33,7 @@ const Scheduler = @import("Scheduler.zig");const Page = @import("../Page.zig");const Session = @import("../Session.zig");const Factory = @import("../Factory.zig");-const HttpClient = @import("../HttpClient.zig");+const HttpClient = @import("../../network/HttpClient.zig");const EventManagerBase = @import("../EventManagerBase.zig");const Event = @import("../webapi/Event.zig");@@ -102,6 +102,13 @@ pub fn makeRequest(self: *const Execution, req: HttpClient.Request) !void {};}+// Two-phase variant; see HttpClient.newRequest for the ownership contract.+pub fn newRequest(self: *const Execution, req: HttpClient.Request) !*HttpClient.Transfer {+ return switch (self.js.global) {+ inline else => |g| g.newRequest(req),+ };+}+pub fn getBroadcastChannels(self: *const Execution) *std.DoublyLinkedList {return switch (self.js.global) {inline else => |g| &g._broadcast_channels,diff --git a/src/browser/webapi/Worker.zig b/src/browser/webapi/Worker.zigindex 9b2659ed7..82c5a1c25 100644--- a/src/browser/webapi/Worker.zig+++ b/src/browser/webapi/Worker.zig@@ -23,7 +23,7 @@ const js = @import("../js/js.zig");const URL = @import("../URL.zig");const Frame = @import("../Frame.zig");-const HttpClient = @import("../HttpClient.zig");+const Transfer = @import("../../network/HttpClient.zig").Transfer;const EventTarget = @import("EventTarget.zig");const MessageEvent = @import("event/MessageEvent.zig");@@ -56,7 +56,7 @@ _url: [:0]const u8,_type: WorkerType = .classic,_script_loaded: bool = false,_script_buffer: std.ArrayList(u8) = .empty,-_http_response: ?HttpClient.Response = null,+_http_transfer: ?*Transfer = null,// Event handlers_on_error: ?js.Function.Global = null,@@ -101,11 +101,9 @@ pub fn init(url: []const u8, options: ?WorkerOptions, frame: *Frame) !*Worker {return self;}- const headers = try session.browser.http_client.newHeaders();- frame.makeRequest(.{+ const transfer = frame.newRequest(.{.ctx = self,.method = .GET,- .headers = headers,.url = resolved_url,.frame_id = self._frame_id,.loader_id = self._loader_id,@@ -117,11 +115,24 @@ pub fn init(url: []const u8, options: ?WorkerOptions, frame: *Frame) !*Worker {.data_callback = httpDataCallback,.done_callback = httpDoneCallback,.error_callback = httpErrorCallback,+ .shutdown_callback = httpShutdownCallback,}) catch |err| {log.err(.browser, "Worker request", .{ .url = resolved_url, .err = err });frame.removeWorker(self);return err;};++ // Held for deinit's abort; the done, error and shutdown callbacks clear+ // it. The shutdown one matters: Frame.deinit's abortOwner kills the+ // transfer before it deinits this worker, and deinit must not abort a+ // freed transfer.+ self._http_transfer = transfer;++ transfer.submit() catch |err| {+ log.err(.browser, "Worker request", .{ .url = resolved_url, .err = err });+ frame.removeWorker(self);+ return err;+ };return self;}@@ -129,9 +140,9 @@ pub fn init(url: []const u8, options: ?WorkerOptions, frame: *Frame) !*Worker {// remove from the frame's worker list.pub fn deinit(self: *Worker) void {// No pending frame for workers, so we can abort all frames.- if (self._http_response) |res| {+ if (self._http_transfer) |res| {res.abort(error.Abort);- self._http_response = null;+ self._http_transfer = null;}self._worker_scope.deinit();self._frame._session.releaseArena(self._arena);@@ -141,10 +152,10 @@ pub fn asEventTarget(self: *Worker) *EventTarget {return self._proto;}-fn httpHeaderCallback(response: HttpClient.Response) !HttpClient.HeaderResult {- const self: *Worker = @ptrCast(@alignCast(response.ctx));+fn httpHeaderCallback(transfer: *Transfer) !Transfer.HeaderResult {+ const self: *Worker = @ptrCast(@alignCast(transfer.req.ctx));- const status = response.status() orelse return .abort;+ const status = transfer.responseStatus() orelse return .abort;if (status < 200 or status >= 300) {log.warn(.browser, "Worker status", .{.url = self._url,@@ -153,22 +164,21 @@ fn httpHeaderCallback(response: HttpClient.Response) !HttpClient.HeaderResult {return .abort;}- self._http_response = response;- if (response.contentLength()) |cl| {+ if (transfer.getContentLength()) |cl| {try self._script_buffer.ensureTotalCapacity(self._arena, cl);}return .proceed;}-fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void {- const self: *Worker = @ptrCast(@alignCast(response.ctx));+fn httpDataCallback(transfer: *Transfer, data: []const u8) !void {+ const self: *Worker = @ptrCast(@alignCast(transfer.req.ctx));try self._script_buffer.appendSlice(self._arena, data);}fn httpDoneCallback(ctx: *anyopaque) !void {const self: *Worker = @ptrCast(@alignCast(ctx));- self._http_response = null;+ self._http_transfer = null;const url = self._url;const script = self._script_buffer.items;@@ -245,9 +255,14 @@ fn loadInitialScript(self: *Worker, script: []const u8) !void {ls.local.runMacrotasks();}+fn httpShutdownCallback(ctx: *anyopaque) void {+ const self: *Worker = @ptrCast(@alignCast(ctx));+ self._http_transfer = null;+}+fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {const self: *Worker = @ptrCast(@alignCast(ctx));- self._http_response = null;+ self._http_transfer = null;log.err(.browser, "worker fetch error", .{.url = self._url,@@ -297,9 +312,9 @@ fn _fireErrorEvent(self: *Worker, message: []const u8, error_value: ?js.Value.Tepub fn terminate(self: *Worker) void {// Abort any pending script fetch- if (self._http_response) |resp| {+ if (self._http_transfer) |resp| {resp.abort(error.Abort);- self._http_response = null;+ self._http_transfer = null;}}diff --git a/src/browser/webapi/WorkerGlobalScope.zig b/src/browser/webapi/WorkerGlobalScope.zigindex 02ebb669f..d65a56ece 100644--- a/src/browser/webapi/WorkerGlobalScope.zig+++ b/src/browser/webapi/WorkerGlobalScope.zig@@ -29,7 +29,7 @@ const Page = @import("../Page.zig");const Frame = @import("../Frame.zig");const Factory = @import("../Factory.zig");const Session = @import("../Session.zig");-const HttpClient = @import("../HttpClient.zig");+const HttpClient = @import("../../network/HttpClient.zig");const EventManagerBase = @import("../EventManagerBase.zig");const ScriptManagerBase = @import("../ScriptManagerBase.zig");@@ -250,6 +250,11 @@ pub fn makeRequest(self: *WorkerGlobalScope, req: HttpClient.Request) !void {return self._session.browser.http_client.request(req, &self._http_owner);}+// Two-phase variant; see HttpClient.newRequest for the ownership contract.+pub fn newRequest(self: *WorkerGlobalScope, req: HttpClient.Request) !*HttpClient.Transfer {+ return self._session.browser.http_client.newRequest(req, &self._http_owner);+}+pub fn getSelf(self: *WorkerGlobalScope) *WorkerGlobalScope {return self;}@@ -390,6 +395,7 @@ fn importScript(self: *WorkerGlobalScope, arena: Allocator, url: [:0]const u8) !.cookie_origin = self.url,.resource_type = .script,.notification = session.notification,+ .shutdown_callback = HttpClient.noopShutdown, // syncRequest installs its own}) catch |err| {log.warn(.http, "importScript", .{ .url = resolved_url, .err = err });return error.NetworkError;@@ -400,8 +406,6 @@ fn importScript(self: *WorkerGlobalScope, arena: Allocator, url: [:0]const u8) !return error.NetworkError;}- defer http_client.deferring_layer.flushFrame(self._frame_id);-var ls: JS.Local.Scope = undefined;self.js.localScope(&ls);defer ls.deinit();diff --git a/src/browser/webapi/element/html/Link.zig b/src/browser/webapi/element/html/Link.zigindex af9472283..7eb2190f5 100644--- a/src/browser/webapi/element/html/Link.zig+++ b/src/browser/webapi/element/html/Link.zig@@ -290,10 +290,9 @@ test "WebApi: HTML.Link external stylesheet" {}// Regression: a synchronous external-stylesheet fetch must not strand the-// completion of an in-flight <script defer> (deferred by the blocking-request-// window). Otherwise the deferred-script queue never drains and the document-// is stuck at readyState "loading". See Frame.loadExternalStylesheet's-// flushFrame call.+// completion of an in-flight <script defer> (held back by the blocking-+// request gate during the sync window). Otherwise the deferred-script queue+// never drains and the document is stuck at readyState "loading".test "WebApi: HTML.Link deferred script then external stylesheet" {const filter: testing.LogFilter = .init(&.{.http});defer filter.deinit();diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zigindex 86e53ca4b..ba6849130 100644--- a/src/browser/webapi/net/Fetch.zig+++ b/src/browser/webapi/net/Fetch.zig@@ -18,10 +18,10 @@const std = @import("std");const lp = @import("lightpanda");-const HttpClient = @import("../../HttpClient.zig");const js = @import("../../js/js.zig");const URL = @import("../../URL.zig");+const Transfer = @import("../../../network/HttpClient.zig").Transfer;const Request = @import("Request.zig");const Response = @import("Response.zig");@@ -96,12 +96,7 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promis.@"same-origin" => if (exec.isSameOrigin(request._url)) &session.cookie_jar else null,};- // Synchronous failures from request layers (e.g. RobotsLayer returning- // RobotsBlocked when robots.txt is already cached) are dispatched to- // httpErrorCallback by Client.request, which rejects the promise and- // releases response._arena. Propagating the error from here would also- // fire the `errdefer response.deinit` above and double-free the arena.- exec.makeRequest(.{+ const transfer = exec.newRequest(.{.ctx = fetch,.url = request._url,.method = request._method,@@ -118,26 +113,30 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promis.@"error" => .@"error",},.notification = session.notification,- .start_callback = httpStartCallback,.header_callback = httpHeaderDoneCallback,.data_callback = httpDataCallback,.done_callback = httpDoneCallback,.error_callback = httpErrorCallback,.shutdown_callback = httpShutdownCallback,- }) catch {};- return resolver.promise();-}+ }) catch {+ // OOM-class; nothing was committed and no callback fired.+ return resolver.promise();+ };-fn httpStartCallback(response: HttpClient.Response) !void {- const self: *Fetch = @ptrCast(@alignCast(response.ctx));- if (comptime IS_DEBUG) {- log.debug(.http, "request start", .{ .url = self._url, .source = "fetch" });- }- self._response._http_response = response;+ // Held for Response.deinit's abort; the error, shutdown and done+ // callbacks clear it.+ response._http_transfer = transfer;++ // Failures inside submit are dispatched to httpErrorCallback, which+ // rejects the promise and releases response._arena. Propagating the+ // error from here would also fire the `errdefer response.deinit` above+ // and double-free the arena.+ transfer.submit() catch {};+ return resolver.promise();}-fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResult {- const self: *Fetch = @ptrCast(@alignCast(response.ctx));+fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {+ const self: *Fetch = @ptrCast(@alignCast(transfer.req.ctx));if (self._signal) |signal| {if (signal._aborted) {@@ -146,7 +145,7 @@ fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResul}const arena = self._response._arena;- if (response.contentLength()) |cl| {+ if (transfer.getContentLength()) |cl| {try self._buf.ensureTotalCapacity(arena, cl);}@@ -156,14 +155,14 @@ fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResullog.debug(.http, "request header", .{.source = "fetch",.url = self._url,- .status = response.status(),+ .status = transfer.responseStatus(),});}- res._status = response.status().?;- res._status_text = std.http.Status.phrase(@enumFromInt(response.status().?)) orelse "";- res._url = try arena.dupeZ(u8, response.url());- res._is_redirected = response.redirectCount().? > 0;+ res._status = transfer.responseStatus().?;+ res._status_text = std.http.Status.phrase(@enumFromInt(transfer.responseStatus().?)) orelse "";+ res._url = try arena.dupeZ(u8, transfer.req.url);+ res._is_redirected = transfer.redirectCount().? > 0;// redirect: "manual" surfaces the unfollowed 3xx as an opaque-redirect// filtered response: status 0, no headers, no body.@@ -195,7 +194,7 @@ fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResulres._type = .basic;}- var it = response.headerIterator();+ var it = transfer.responseHeaderIterator();while (it.next()) |hdr| {try res._headers.append(hdr.name, hdr.value, exec);}@@ -203,8 +202,8 @@ fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResulreturn .proceed;}-fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void {- const self: *Fetch = @ptrCast(@alignCast(response.ctx));+fn httpDataCallback(transfer: *Transfer, data: []const u8) !void {+ const self: *Fetch = @ptrCast(@alignCast(transfer.req.ctx));// Check if abortedif (self._signal) |signal| {@@ -219,7 +218,7 @@ fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void {fn httpDoneCallback(ctx: *anyopaque) !void {const self: *Fetch = @ptrCast(@alignCast(ctx));var response = self._response;- response._http_response = null;+ response._http_transfer = null;response._body = .{ .bytes = self._buf.items };log.info(.http, "request complete", .{@@ -249,7 +248,7 @@ fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {});var response = self._response;- response._http_response = null;+ response._http_transfer = null;// Capture this before we reject. Rejection could trigger httpShutdownCallback// (via a microtask callback). But if we're here, then we'll take care of@@ -277,7 +276,7 @@ fn httpShutdownCallback(ctx: *anyopaque) void {if (self._owns_response) {var response = self._response;- response._http_response = null;+ response._http_transfer = null;response.deinit(self._exec.page);// Do not access `self` after this point: the Fetch struct was// allocated from response._arena which has been released.diff --git a/src/browser/webapi/net/Response.zig b/src/browser/webapi/net/Response.zigindex 78851e067..fc34f6f19 100644--- a/src/browser/webapi/net/Response.zig+++ b/src/browser/webapi/net/Response.zig@@ -22,7 +22,7 @@ const lp = @import("lightpanda");const js = @import("../../js/js.zig");const URL = @import("../../URL.zig");const Page = @import("../../Page.zig");-const HttpClient = @import("../../HttpClient.zig");+const Transfer = @import("../../../network/HttpClient.zig").Transfer;const Blob = @import("../Blob.zig");const ReadableStream = @import("../streams/ReadableStream.zig");@@ -53,7 +53,7 @@ _type: Type,_status_text: []const u8,_url: [:0]const u8,_is_redirected: bool,-_http_response: ?HttpClient.Response = null,+_http_transfer: ?*Transfer = null,_body_used: bool = false,const Body = union(enum) {@@ -197,9 +197,9 @@ pub fn createJson(data: js.Value, opts_: ?InitOpts, exec: *const Execution) !*Re}pub fn deinit(self: *Response, page: *Page) void {- if (self._http_response) |resp| {+ if (self._http_transfer) |resp| {resp.abort(error.Abort);- self._http_response = null;+ self._http_transfer = null;}page.releaseArena(self._arena);}@@ -484,7 +484,7 @@ pub fn clone(self: *const Response, exec: *const Execution) !*Response {._type = self._type,._is_redirected = self._is_redirected,._headers = try Headers.init(.{ .obj = self._headers }, exec),- ._http_response = null,+ ._http_transfer = null,};return cloned;}diff --git a/src/browser/webapi/net/WebSocket.zig b/src/browser/webapi/net/WebSocket.zigindex 4d4268a88..a94d5fa3f 100644--- a/src/browser/webapi/net/WebSocket.zig+++ b/src/browser/webapi/net/WebSocket.zig@@ -26,7 +26,7 @@ const Blob = @import("../Blob.zig");const URL = @import("../../URL.zig");const Page = @import("../../Page.zig");-const HttpClient = @import("../../HttpClient.zig");+const HttpClient = @import("../../../network/HttpClient.zig");const Event = @import("../Event.zig");const EventTarget = @import("../EventTarget.zig");diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zigindex 278658efc..49714cc72 100644--- a/src/browser/webapi/net/XMLHttpRequest.zig+++ b/src/browser/webapi/net/XMLHttpRequest.zig@@ -20,8 +20,8 @@ const std = @import("std");const lp = @import("lightpanda");const js = @import("../../js/js.zig");-const HttpClient = @import("../../HttpClient.zig");const http = @import("../../../network/http.zig");+const Transfer = @import("../../../network/HttpClient.zig").Transfer;const URL = @import("../../URL.zig");const Mime = @import("../../Mime.zig");@@ -47,7 +47,7 @@ _exec: *const Execution,_proto: *XMLHttpRequestEventTarget,_upload: ?*XMLHttpRequestUpload = null,_arena: Allocator,-_http_response: ?HttpClient.Response = null,+_http_transfer: ?*Transfer = null,// number of inflight requests, we can have multiple, e.g. xhr calling its own// send from the onload callback@@ -111,9 +111,9 @@ pub fn init(exec: *const Execution) !*XMLHttpRequest {}pub fn deinit(self: *XMLHttpRequest, page: *Page) void {- if (self._http_response) |resp| {+ if (self._http_transfer) |resp| {resp.abort(error.Abort);- self._http_response = null;+ self._http_transfer = null;}if (self._on_ready_state_change) |func| {@@ -182,9 +182,9 @@ pub fn setTimeout(self: *XMLHttpRequest, value: u32) void {// TODO: url should be a union, as it can be multiple thingspub fn open(self: *XMLHttpRequest, method_: []const u8, url: [:0]const u8) !void {// Abort any in-progress request- if (self._http_response) |transfer| {+ if (self._http_transfer) |transfer| {transfer.abort(error.Abort);- self._http_response = null;+ self._http_transfer = null;}self._send_flag = false;@@ -263,7 +263,7 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !vself._active_requests += 1;self._send_flag = true;- exec.makeRequest(.{+ const transfer = exec.newRequest(.{.ctx = self,.url = self._url,.method = self._method,@@ -276,7 +276,6 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !v.resource_type = .xhr,.timeout_ms = self._timeout,.notification = session.notification,- .start_callback = httpStartCallback,.header_callback = httpHeaderDoneCallback,.data_callback = httpDataCallback,.done_callback = httpDoneCallback,@@ -288,6 +287,23 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !vself._send_flag = false;return err;};++ // Held for abort() / open() / deinit; the error, shutdown and done+ // callbacks clear it.+ self._http_transfer = transfer;++ if (comptime IS_DEBUG) {+ log.debug(.http, "request start", .{ .method = self._method, .url = self._url, .source = "xhr" });+ }++ transfer.submit() catch |err| {+ // If the failure aborted the transfer, httpErrorCallback already+ // released the self-ref and cleared _http_transfer, making this+ // release a guarded no-op.+ self.releaseSelfRef();+ self._send_flag = false;+ return err;+ };}// https://xhr.spec.whatwg.org/#the-upload-attribute@@ -453,32 +469,24 @@ pub fn getResponseXML(self: *XMLHttpRequest, exec: *const Execution) !?*Node.Doc}}-fn httpStartCallback(response: HttpClient.Response) !void {- const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));- if (comptime IS_DEBUG) {- log.debug(.http, "request start", .{ .method = self._method, .url = self._url, .source = "xhr" });- }- self._http_response = response;-}--fn httpHeaderCallback(response: HttpClient.Response, header: http.Header) !void {- const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));+fn httpHeaderCallback(transfer: *Transfer, header: http.Header) !void {+ const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.req.ctx));const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ header.name, header.value });try self._response_headers.append(self._arena, joined);}-fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResult {- const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));+fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {+ const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.req.ctx));if (comptime IS_DEBUG) {log.debug(.http, "request header", .{.source = "xhr",.url = self._url,- .status = response.status(),+ .status = transfer.responseStatus(),});}- if (response.contentType()) |ct| {+ if (transfer.contentType()) |ct| {self._response_mime = Mime.parse(ct) catch |e| {log.info(.http, "invalid content type", .{.content_Type = ct,@@ -489,18 +497,18 @@ fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResul};}- var it = response.headerIterator();+ var it = transfer.responseHeaderIterator();while (it.next()) |hdr| {const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ hdr.name, hdr.value });try self._response_headers.append(self._arena, joined);}- self._response_status = response.status().?;- if (response.contentLength()) |cl| {+ self._response_status = transfer.responseStatus().?;+ if (transfer.getContentLength()) |cl| {self._response_len = cl;try self._response_data.ensureTotalCapacity(self._arena, cl);}- self._response_url = try self._arena.dupeZ(u8, response.url());+ self._response_url = try self._arena.dupeZ(u8, transfer.req.url);const exec = self._exec;@@ -515,8 +523,8 @@ fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResulreturn .proceed;}-fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void {- const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));+fn httpDataCallback(transfer: *Transfer, data: []const u8) !void {+ const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.req.ctx));try self._response_data.appendSlice(self._arena, data);try self._proto.dispatch(.progress, .{@@ -537,7 +545,7 @@ fn httpDoneCallback(ctx: *anyopaque) !void {// Not that the request is done, the http/client will free the transfer// object. It isn't safe to keep it around.- self._http_response = null;+ self._http_transfer = null;const exec = self._exec;@@ -560,22 +568,22 @@ fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx));// http client will close it after an error, it isn't safe to keep aroundself.handleError(err);- if (self._http_response != null) {- self._http_response = null;+ if (self._http_transfer != null) {+ self._http_transfer = null;}self.releaseSelfRef();}fn httpShutdownCallback(ctx: *anyopaque) void {const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx));- self._http_response = null;+ self._http_transfer = null;self.releaseSelfRef();}pub fn abort(self: *XMLHttpRequest) void {self.handleError(error.Abort);- if (self._http_response) |resp| {- self._http_response = null;+ if (self._http_transfer) |resp| {+ self._http_transfer = null;resp.abort(error.Abort);}self.releaseSelfRef();diff --git a/src/cdp/CDP.zig b/src/cdp/CDP.zigindex ba3875bc8..346bc7af3 100644--- a/src/cdp/CDP.zig+++ b/src/cdp/CDP.zig@@ -21,9 +21,12 @@ const lp = @import("lightpanda");const App = @import("../App.zig");const Inbox = @import("../Inbox.zig");-const Network = @import("../network/Network.zig");const Notification = @import("../Notification.zig");+const WS = @import("../network/WS.zig");+const Network = @import("../network/Network.zig");+const Transfer = @import("../network/HttpClient.zig").Transfer;+const js = @import("../browser/js/js.zig");const Browser = @import("../browser/Browser.zig");const Session = @import("../browser/Session.zig");@@ -32,7 +35,6 @@ const Page = @import("../browser/Page.zig");const Mime = @import("../browser/Mime.zig");const Element = @import("../browser/webapi/Element.zig");const Label = @import("../browser/webapi/element/html/Label.zig");-const Transfer = @import("../browser/HttpClient.zig").Transfer;const Connection = @import("Connection.zig");const Incrementing = @import("id.zig").Incrementing;@@ -981,8 +983,7 @@ pub const BrowserContext = struct {// Encode the data in base64 by default, but don't encode// for well known content-type..must_encode = blk: {- const response = msg.response;- if (response.contentType()) |ct| {+ if (msg.transfer.contentType()) |ct| {const mime = try Mime.parse(ct);if (!mime.isText()) {@@ -1458,5 +1459,6 @@ test "cdp: syncRequest short-circuits after disconnect" {.cookie_origin = "",.resource_type = .fetch,.notification = undefined,+ .shutdown_callback = @import("../network/HttpClient.zig").noopShutdown,}));}diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zigindex cc62921dc..9d8e96ca5 100644--- a/src/cdp/domains/fetch.zig+++ b/src/cdp/domains/fetch.zig@@ -290,7 +290,7 @@ fn continueRequest(cmd: *CDP.Command) !void {request.body = body;}- try client.interception_layer.continueRequest(transfer);+ try client.continueIntercepted(transfer);return cmd.sendResult(null, .{});}@@ -396,7 +396,7 @@ fn fulfillRequest(cmd: *CDP.Command) !void {body = buf;}- try client.interception_layer.fulfillRequest(transfer, params.responseCode, params.responseHeaders orelse &.{}, body);+ try client.fulfillIntercepted(transfer, params.responseCode, params.responseHeaders orelse &.{}, body);return cmd.sendResult(null, .{});}@@ -417,7 +417,7 @@ fn failRequest(cmd: *CDP.Command) !void {return cmd.sendResult(null, .{});};- defer client.interception_layer.abortRequest(transfer);+ defer transfer.abortParked(error.Abort);log.info(.cdp, "request intercept", .{.state = "fail",diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zigindex 20be2d690..c5e07e2a0 100644--- a/src/cdp/domains/network.zig+++ b/src/cdp/domains/network.zig@@ -27,9 +27,9 @@ const URL = @import("../../browser/URL.zig");const Mime = @import("../../browser/Mime.zig");const Notification = @import("../../Notification.zig");const timestamp = @import("../../datetime.zig").timestamp;-const Headers = @import("../../browser/HttpClient.zig").Headers;-const Transfer = @import("../../browser/HttpClient.zig").Transfer;-const Response = @import("../../browser/HttpClient.zig").Response;++const Headers = @import("../../network/HttpClient.zig").Headers;+const Transfer = @import("../../network/HttpClient.zig").Transfer;const CdpStorage = @import("storage.zig");@@ -91,7 +91,7 @@ fn setCacheDisabled(cmd: *CDP.Command) !void {const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;const client = &bc.cdp.browser.http_client;- client.cache_layer.disabled = params.cacheDisabled;+ client.disableCache(params.cacheDisabled);return cmd.sendResult(null, .{});}@@ -362,7 +362,7 @@ pub fn httpResponseHeaderDone(arena: Allocator, bc: *CDP.BrowserContext, msg: *c.frameId = &id.toFrameId(req.frame_id),.requestId = &id.toRequestId(transfer),.loaderId = &id.toLoaderId(req.loader_id),- .response = ResponseWriter.init(arena, msg.response),+ .response = ResponseWriter.init(arena, msg.transfer),.hasExtraInfo = false, // TODO change after adding Network.responseReceivedExtraInfo}, .{ .session_id = session_id });}@@ -447,12 +447,12 @@ pub const RequestWriter = struct {const ResponseWriter = struct {arena: Allocator,- response: *const Response,+ transfer: *Transfer,- fn init(arena: Allocator, response: *const Response) ResponseWriter {+ fn init(arena: Allocator, transfer: *Transfer) ResponseWriter {return .{.arena = arena,- .response = response,+ .transfer = transfer,};}@@ -461,15 +461,15 @@ const ResponseWriter = struct {}fn _jsonStringify(self: *const ResponseWriter, jws: anytype) !void {- const response = self.response;+ const transfer = self.transfer;try jws.beginObject();{try jws.objectField("url");- try jws.write(response.url());+ try jws.write(transfer.req.url);}- if (response.status()) |status| {+ if (transfer.responseStatus()) |status| {try jws.objectField("status");try jws.write(status);@@ -479,7 +479,7 @@ const ResponseWriter = struct {{const mime: Mime = blk: {- if (response.contentType()) |ct| {+ if (transfer.contentType()) |ct| {break :blk try Mime.parse(ct);}break :blk .unknown;@@ -493,7 +493,7 @@ const ResponseWriter = struct {{try jws.objectField("fromDiskCache");- try jws.write(response.inner == .cached);+ try jws.write(transfer._from_cache);}{@@ -521,7 +521,7 @@ const ResponseWriter = struct {// common to get these from a server (e.g. for Cache-Control), but// Chrome joins these. So we have to too.const arena = self.arena;- var it = response.headerIterator();+ var it = transfer.responseHeaderIterator();var map: std.StringArrayHashMapUnmanaged([]const u8) = .empty;while (it.next()) |hdr| {const gop = try map.getOrPut(arena, hdr.name);@@ -866,7 +866,7 @@ test "cdp.Network: canClearBrowserCache" {try ctx.expectSentResult(.{ .result = false }, .{ .id = 1 });}-test "cdp.Network: setCacheDisabled disables cache" {+test "cdp.Network: setCacheDisabled" {var ctx = try testing.context();defer ctx.deinit();_ = try ctx.loadBrowserContext(.{ .id = "BID-CD1" });@@ -877,30 +877,4 @@ test "cdp.Network: setCacheDisabled disables cache" {.params = .{ .cacheDisabled = true },});try ctx.expectSentResult(null, .{ .id = 1 });-- const client = ctx.cdp().browser.http_client;- try testing.expectEqual(true, client.cache_layer.disabled);-}--test "cdp.Network: setCacheDisabled re-enables cache" {- var ctx = try testing.context();- defer ctx.deinit();- _ = try ctx.loadBrowserContext(.{ .id = "BID-CD2" });-- try ctx.processMessage(.{- .id = 1,- .method = "Network.setCacheDisabled",- .params = .{ .cacheDisabled = true },- });- try ctx.expectSentResult(null, .{ .id = 1 });-- try ctx.processMessage(.{- .id = 2,- .method = "Network.setCacheDisabled",- .params = .{ .cacheDisabled = false },- });- try ctx.expectSentResult(null, .{ .id = 2 });-- const client = ctx.cdp().browser.http_client;- try testing.expectEqual(false, client.cache_layer.disabled);}diff --git a/src/cdp/domains/page.zig b/src/cdp/domains/page.zigindex 355605585..cfeddd0a5 100644--- a/src/cdp/domains/page.zig+++ b/src/cdp/domains/page.zig@@ -149,8 +149,8 @@ fn setLifecycleEventsEnabled(cmd: *CDP.Command) !void {const http_client = frame._session.browser.http_client;const http_active = http_client.http_active;- const http_next_tick = http_client.next_tick_count;- const total_network_activity = http_active + http_next_tick + http_client.interception_layer.intercepted;+ const http_buffered = http_client.dispatch_count;+ const total_network_activity = http_active + http_buffered + http_client.intercepted;if (frame._notified_network_almost_idle.check(total_network_activity <= 2)) {try sendPageLifecycle(bc, "networkAlmostIdle", now, frame_id, loader_id);}diff --git a/src/cdp/id.zig b/src/cdp/id.zigindex c324da3ec..463e8eab4 100644--- a/src/cdp/id.zig+++ b/src/cdp/id.zig@@ -40,7 +40,7 @@ pub fn toLoaderId(id: u32) [14]u8 {// requestId has special requirements. If it's the main document navigation,// then it should match the loader id.-const Transfer = @import("../browser/HttpClient.zig").Transfer;+const Transfer = @import("../network/HttpClient.zig").Transfer;pub fn toRequestId(transfer: *const Transfer) [14]u8 {if (transfer.req.resource_type == .document) {return toLoaderId(transfer.req.loader_id);diff --git a/src/lightpanda.zig b/src/lightpanda.zigindex b9e9d9cc6..b81e34999 100644--- a/src/lightpanda.zig+++ b/src/lightpanda.zig@@ -43,7 +43,7 @@ pub const forms = @import("browser/forms.zig");pub const actions = @import("browser/actions.zig");pub const structured_data = @import("browser/structured_data.zig");pub const tools = @import("browser/tools.zig");-pub const HttpClient = @import("browser/HttpClient.zig");+pub const HttpClient = @import("network/HttpClient.zig");pub const mcp = @import("mcp.zig");pub const Agent = @import("agent/Agent.zig");diff --git a/src/browser/HttpClient.zig b/src/network/HttpClient.zigsimilarity index 63%rename from src/browser/HttpClient.zigrename to src/network/HttpClient.zigindex 842cc4787..6e05bb61c 100644--- a/src/browser/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -20,19 +20,20 @@ const std = @import("std");const lp = @import("lightpanda");const builtin = @import("builtin");+const Inbox = @import("../Inbox.zig");const ArenaPool = @import("../ArenaPool.zig");const Notification = @import("../Notification.zig");const timestamp = @import("../datetime.zig").timestamp;-const URL = @import("URL.zig");-const CookieJar = @import("webapi/storage/Cookie.zig").Jar;--const http = @import("../network/http.zig");-const Network = @import("../network/Network.zig");-const CDP = @import("../cdp/CDP.zig");-const Inbox = @import("../Inbox.zig");const Watchdog = @import("../Watchdog.zig");+const URL = @import("../browser/URL.zig");+const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar;++const http = @import("http.zig");+const Network = @import("Network.zig");+const Cache = @import("cache/Cache.zig");+const RobotsGate = @import("RobotsGate.zig");const log = lp.log;const Allocator = std.mem.Allocator;@@ -41,15 +42,7 @@ const IS_DEBUG = builtin.mode == .Debug;pub const Method = http.Method;pub const Header = http.Header;pub const Headers = http.Headers;-pub const ResponseHead = http.ResponseHead;pub const HeaderIterator = http.HeaderIterator;-const CachedResponse = @import("../network/cache/Cache.zig").CachedResponse;--pub const CacheLayer = @import("../network/layer/CacheLayer.zig");-pub const RobotsLayer = @import("../network/layer/RobotsLayer.zig");-pub const WebBotAuthLayer = @import("../network/layer/WebBotAuthLayer.zig");-pub const InterceptionLayer = @import("../network/layer/InterceptionLayer.zig");-pub const DeferringLayer = @import("../network/layer/DeferringLayer.zig");// This is loosely tied to a browser Frame. Loading all the <scripts>, doing// XHR requests, and loading imports all happens through here. Sine the app@@ -73,13 +66,16 @@ http_active: usize = 0,// Our curl multi handle.handles: http.Handles,-// Connections currently in this client's curl_multi.-in_use: std.DoublyLinkedList = .{},--// Connections that failed to be removed from curl_multi during perform.-dirty: std.DoublyLinkedList = .{},-// Whether we're currently inside a curl_multi_perform call.+//+// HTTP transfers no longer run user code under perform (the pump only+// buffers; dispatch delivers), so for them this flag is vestigial. It+// remains load-bearing solely because WebSocket dispatches JS (onopen /+// onmessage) from inside curl's callbacks: that JS can start requests+// (process), open sockets (trackConn -> ready_queue), close them+// (removeConn -> dirty), or abort transfers (detachOrDeinit) while the+// multi is live. If WS delivery ever moves onto the dispatch queue, this+// flag — and ready_queue, dirty, and the .inflight deferral — can go.performing: bool = false,// Watchdog instrumentation for this client's worker thread. Wraps the poll@@ -98,20 +94,31 @@ next_request_id: u32 = 0,// only valid for the lifetime of the entry.transfers: std.AutoHashMapUnmanaged(u32, *Transfer) = .empty,-// When handles has no more available easys, requests get queued.-queue: std.DoublyLinkedList = .{},+// Connections currently in this client's curl_multi.+in_use: std.DoublyLinkedList = .{},-// A queue for things that MUST happen on the next tick.-next_tick_queue: std.DoublyLinkedList = .{},-next_tick_count: usize = 0,+// Connections that failed to be removed from curl_multi during perform.+dirty: std.DoublyLinkedList = .{},++// Queue for request that are waiting an available connection (aka, easy)+pending_queue: std.DoublyLinkedList = .{},-// Queue is for Transfers that have no connection. ready_queue is for connections-// that were initiated when performing == true and thus need to wait until-// performing == false before being added. I'm hoping this is temporary and that-// we can unify the two queues. But HTTP is being changed a lot right now, and-// I'm trying to minimize the surface area.+// Queue for transfers that are ready to be performed.ready_queue: std.DoublyLinkedList = .{},+// Queue for completed transfers that haven't had their callbacks executed yet+dispatch_queue: std.DoublyLinkedList = .{},++// Completed transfers held back by a frame's blocking request. dispatch()+// parks them here on first sight so the gate is evaluated once per transfer+// per blocking window, not once per tick; releaseBlocking splices them+// back. Blocking requests are common (every bare <script src> is one), so+// this keeps dispatch O(new work) during page load.+gated_queue: std.DoublyLinkedList = .{},++// Undelivered transfers across dispatch_queue + gated_queue.+dispatch_count: usize = 0,+// The main app allocatorallocator: Allocator,@@ -168,43 +175,24 @@ inbox: Inbox,max_response_size: usize,+// While a frame has a blocking (synchronous) request in flight, dispatch+// holds back every other transfer for that frame so their callbacks can't+// run JS while the parser is on the stack. frame_id -> blocking transfer id.blocking_requests: std.AutoHashMapUnmanaged(u32, u32) = .empty,-cache_layer: CacheLayer,-robots_layer: RobotsLayer,-web_bot_auth_layer: WebBotAuthLayer,-interception_layer: InterceptionLayer,-deferring_layer: DeferringLayer,-entry_layer: Layer,+// Count of transfers parked for CDP interception (request or auth phase).+// They're off every other activity counter while parked; the network-idle+// heuristics add this in.+intercepted: usize = 0,-pub const Layer = struct {- ptr: *anyopaque,- vtable: *const VTable,+// null or referencing network.cache+cache: ?*Cache,- pub const VTable = struct {- request: *const fn (*anyopaque, *Transfer) anyerror!void,- };+// Cached config decisions, resolved once at init.+serve_mode: bool,+obey_robots: bool,- pub fn request(self: Layer, transfer: *Transfer) !void {- return self.vtable.request(self.ptr, transfer);- }-};--fn layerWith(self: anytype, next: Layer) Layer {- self.next = next;- return self.layer();-}--pub const NextTickNode = struct {- pub const Run =- *const fn (*Transfer, ?*anyopaque) void;- pub const Abort = *const fn (?*anyopaque) void;-- node: std.DoublyLinkedList.Node = .{},- ctx: ?*anyopaque,- run: Run,- abort: ?Abort = null,-};+robots: RobotsGate,pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP) !void {var handles = try http.Handles.init(network.config);@@ -218,42 +206,18 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP).allocator = allocator,.cdp = cdp,.inbox = .{},+ .cache = if (network.cache) |*c| c else null,.use_proxy = http_proxy != null,.http_proxy = http_proxy,.tls_verify = network.config.tlsVerifyHost(),.max_response_size = network.config.httpMaxResponseSize() orelse std.math.maxInt(u32),- .cache_layer = .{},- .robots_layer = .{ .allocator = allocator, .network = network },- .web_bot_auth_layer = .{},- .interception_layer = .{},- .deferring_layer = .{ .allocator = allocator, .network = network },- .entry_layer = undefined,+ .serve_mode = network.config.mode == .serve,+ .obey_robots = network.config.obeyRobots(),+ .robots = .{ .allocator = allocator, .network = network },.arena_pool = &network.app.arena_pool,};-- var next = self.layer();-- if (network.config.obeyRobots()) {- next = layerWith(&self.robots_layer, next);- }-- if (network.config.httpCacheDir() != null) {- next = layerWith(&self.cache_layer, next);- }-- if (network.config.mode == .serve) {- next = layerWith(&self.interception_layer, next);- }-- if (network.config.webBotAuth() != null) {- next = layerWith(&self.web_bot_auth_layer, next);- }-- next = layerWith(&self.deferring_layer, next);-- self.entry_layer = next;}pub fn deinit(self: *Client) void {@@ -261,9 +225,9 @@ pub fn deinit(self: *Client) void {if (comptime IS_DEBUG) {lp.assert(- self.next_tick_count == 0,- "next_tick_count must be 0",- .{ .value = self.next_tick_count },+ self.dispatch_count == 0,+ "dispatch_count must be 0",+ .{ .value = self.dispatch_count },);}@@ -274,8 +238,7 @@ pub fn deinit(self: *Client) void {self.allocator.free(owned);}- self.robots_layer.deinit(self.allocator);- self.deferring_layer.deinit();+ self.robots.deinit();self.blocking_requests.deinit(self.allocator);self.transfers.deinit(self.allocator);self.inbox.deinit(self.arena_pool);@@ -289,19 +252,20 @@ pub fn findTransfer(self: *Client, id: u32) ?*Transfer {return self.transfers.get(id);}-pub fn layer(self: *Client) Layer {- return .{- .ptr = self,- .vtable = &.{ .request = _request },- };+pub fn incrReqId(self: *Client) u32 {+ const id = self.next_request_id +% 1;+ self.next_request_id = id;+ return id;}// Set a user agent override. Both the raw UA string and the pre-formatted// "User-Agent: <ua>" header string are allocated from self.allocator.pub fn setUserAgentOverride(self: *Client, ua: []const u8) !void {self.clearUserAgentOverride();+const override = try self.allocator.dupeZ(u8, ua);errdefer self.allocator.free(override);+const header = try std.fmt.allocPrintSentinel(self.allocator, "User-Agent: {s}", .{ua}, 0);self.user_agent_override = override;self.user_agent_header_override = header;@@ -339,6 +303,14 @@ pub fn setTlsVerify(self: *Client, verify: bool) !void {self.tls_verify = verify;}+pub fn disableCache(self: *Client, disable: bool) void {+ if (disable) {+ self.cache = null;+ } else {+ self.cache = if (self.network.cache) |*c| c else null;+ }+}+// Restrictive since it'll only work if there are no inflight requests. In some// cases, the libcurl documentation is clear that changing settings while a// connection is inflight is undefined. It doesn't say anything about CURLOPT_PROXY,@@ -395,14 +367,16 @@ pub fn abort(self: *Client) void {// After the kill loop, every internal list should drain itself via// each transfer's deinit:// - self.transfers : transfers.remove(self.id)- // - self.queue : unlinked if _queued is set+ // - self.pending_queue : unlinked if _queued is set// - self.in_use / self.ready_queue : via removeConn// - self.dirty : drained at end of each perform; nothing left here// Any non-empty list means a transfer escaped cleanup — assert so we// catch the regression rather than silently leaking on next use.if (comptime IS_DEBUG) {std.debug.assert(self.transfers.size == 0);- std.debug.assert(self.queue.first == null);+ std.debug.assert(self.pending_queue.first == null);+ std.debug.assert(self.dispatch_queue.first == null);+ std.debug.assert(self.gated_queue.first == null);std.debug.assert(self.in_use.first == null);std.debug.assert(self.ready_queue.first == null);std.debug.assert(self.dirty.first == null);@@ -420,7 +394,7 @@ pub fn abortOwner(self: *Client, owner: *Owner) void {var n = owner.websockets.first;while (n) |node| {n = node.next;- const ws: *@import("webapi/net/WebSocket.zig") = @fieldParentPtr("_owner_node", node);+ const ws: *@import("../browser/webapi/net/WebSocket.zig") = @fieldParentPtr("_owner_node", node);ws.kill();}if (comptime IS_DEBUG) {@@ -455,116 +429,36 @@ pub fn abortRequests(_: *Client, owner: *Owner) void {// Cherry-pick only Fetch interception responsesconst DrainMode = enum { all, sync_wait };-pub fn tick(self: *Client, timeout_ms: u32, mode: DrainMode) !void {- if (self.inbox.terminated) {- return error.ClientDisconnected;- }-- try self.drainNextTickQueue();- try self.drainQueue();- try self.perform(@intCast(timeout_ms));- // perform/processMessages just released a batch of connections back to- // the pool. Drain again so queued transfers can use them this tick- // instead of waiting for the next runner iteration.- try self.drainQueue();- // Dispatch CDP messages here, not inside perform: perform recurses- // via processOneMessage's redirect path (perform → processMessages- // → processOneMessage → perform), and dispatching CDP from that- // nested call would fire CDP handlers mid-redirect, defeating the- // "safe points only" guarantee.- try self.drainInbox(mode);+// One-shot convenience: create and submit in a single call. HttpClient takes+// ownership of req.headers; do not pair with `errdefer headers.deinit()`.+// Callers that have no headers of their own can leave req.headers at its+// (empty) default — the client fills in its baseline headers (user agent,+// etc.).+pub fn request(self: *Client, req: Request, owner: ?*Owner) anyerror!void {+ const transfer = try self.newRequest(req, owner);+ return transfer.submit();}-pub fn runNextTick(- self: *Client,- transfer: *Transfer,- ctx: ?*anyopaque,- params: struct { run: NextTickNode.Run, abort: ?NextTickNode.Abort = null },-) !void {- transfer._next_tick_node = .{ .ctx = ctx, .run = params.run, .abort = params.abort };-- self.next_tick_count += 1;- self.next_tick_queue.append(&transfer._next_tick_node.?.node);-}--fn cancelNextTick(self: *Client, transfer: *Transfer) void {- if (transfer._next_tick_node) |*ntn| {- self.next_tick_queue.remove(&ntn.node);- self.next_tick_count -= 1;-- if (ntn.abort) |abort_cb| {- abort_cb(ntn.ctx);- }- }-}--fn drainNextTickQueue(self: *Client) !void {- var remaining = self.next_tick_count;- while (remaining > 0) : (remaining -= 1) {- const node = self.next_tick_queue.popFirst() orelse break;- defer self.next_tick_count -= 1;- const n: *NextTickNode = @fieldParentPtr("node", node);-- const transfer: *Transfer = @fieldParentPtr(- "_next_tick_node",- @as(*?NextTickNode, @ptrCast(n)),- );-- const ntn = n.*;- transfer._next_tick_node = null;- ntn.run(transfer, ntn.ctx);- }-}--fn drainQueue(self: *Client) !void {- while (self.queue.popFirst()) |queue_node| {- const transfer: *Transfer = @fieldParentPtr("_node", queue_node);- const conn = self.network.getConnection() orelse {- self.queue.prepend(queue_node);- return;- };- // Bridge state to .created so a failure inside makeRequest before- // any commit cleans up via the abort below. makeRequest flips to- // .inflight on a successful trackConn.- transfer.state = .created;- self.makeRequest(conn, transfer) catch |err| {- if (transfer.state == .created) {- transfer.abort(err);- }- return err;- };- }-}--// last layer-pub fn _request(_: *anyopaque, transfer: *Transfer) !void {- return transfer.client.process(transfer);-}--// HttpClient takes ownership of req.headers; do not pair with-// `errdefer headers.deinit()`-pub fn request(self: *Client, req: Request, owner: ?*Owner) !void {- _ = try self.requestT(req, owner);-}--// Like `request`, but returns the created `*Transfer`. The caller does not own-// the returned `*Transfer` and must thus use it with care. From the moment this-// function is entered, the HttpClient owns `req` — specifically `req.headers`-// On success, transfer.deinit eventually frees it. On any failure path inside-// this function, we free it before returning the error.-fn requestT(self: *Client, req: Request, owner: ?*Owner) !*Transfer {+// Create a request without submitting it. The caller owns the returned+// transfer until transfer.submit(). On error, header is freed. On success,+// headers is owned by the transfer.+pub fn newRequest(self: *Client, req: Request, owner: ?*Owner) anyerror!*Transfer {const arena = self.arena_pool.acquire(.small, "Request.arena") catch |err| {req.headers.deinit();return err;};const transfer = blk: {+ var owned = req;errdefer {- req.headers.deinit();+ owned.headers.deinit();self.arena_pool.release(arena);}- var owned = req;+ if (owned.headers.headers == null) {+ owned.headers = try self.newHeaders();+ }+// Most of the time, the req data will outlive the transfer. But not// always. The most problematic case is with a QueuedNavigation which// is freed quite quickly and would definetly not survive a queued@@ -614,13 +508,7 @@ fn requestT(self: *Client, req: Request, owner: ?*Owner) !*Transfer {break :blk t;};- // From here, transfer owns req+arena. Any subsequent failure flows- // through transfer.deinit (or transfer.abort), which handles headers- // via req.deinit. Do NOT free headers directly past this point.-- // Register for id-based lookup. putNoClobber would fail if request_id- // collides (i.e. we've wrapped through 2^32 requests and the old- // transfer is still alive — practically never).+ // From here the transfer owns `req` and `arena`self.transfers.putNoClobber(self.allocator, transfer.id, transfer) catch |err| {transfer.deinit();return err;@@ -631,115 +519,375 @@ fn requestT(self: *Client, req: Request, owner: ?*Owner) !*Transfer {transfer.owner = o;}- // From this point forward, the transfer owns `req` and `arena`. If the- // layer chain fails before any layer commits the transfer to an external- // owner (queue / multi handle / pending interception), we clean up here- // via transfer.abort which fires error_callback and deinits. `.created`- // means no commit happened — anything else is held by an owner that- // will clean up.+ return transfer;+}- // Synthetic schemes never touch the network or the layer chain — they skip- // robots/cache/interception and deliver on the next tick- if (Synthetic.isSynthetic(req.url)) {- // The 2nd transfer is the callback context. We don't actually use it,- // we're just sticking transfer in there to have something.- self.runNextTick(transfer, null, .{ .run = Synthetic.run }) catch |err| {- if (transfer.state == .created) {- transfer.abort(err);- }- return err;- };- return transfer;+pub fn tick(self: *Client, timeout_ms: u32, mode: DrainMode) !void {+ if (self.inbox.terminated) {+ return error.ClientDisconnected;}- self.entry_layer.request(transfer) catch |err| {- if (transfer.state == .created) {- transfer.abort(err);+ self.dispatchCompleted();++ // settle upfront so that we return any completed connections back to the pool+ try self.settleConns();++ try self.startPending();++ const running = try self.perform();++ // WebSocket can release connections during perform, so settle any+ // connections that are now free. TODO: move WebSocket to the same queue-+ // based approach as HTTP connections, and then this can be removed+ try self.settleConns();++ var has_processed = true;+ if (try self.processMessages() == false) {+ // No messages were processed. We need to wait for I/O. The network+ // layer will wake this up if there's acticity.+ if (running > 0 or self.cdp_link_active) {+ {+ self.heartbeat.enterWait();+ defer self.heartbeat.exitWait();+ try self.handles.poll(&.{}, @intCast(timeout_ms));+ }+ // poll only waits, so we do the perform -> process dance again+ _ = try self.perform();+ has_processed = try self.processMessages();}- return err;- };+ } else {+ // If we DO have processed messages, we don't wan to wait / poll. We+ // want to proceses completions and return to the caller ASAP so that it+ // can check progress, e.g. maybe the message we processed cause "load"+ // to fire, and that's what it was waiting for.+ }- return transfer;+ try self.startPending();++ self.dispatchCompleted();++ // dispatch CDP commands+ try self.drainInbox(mode);}-// Non-network URL schemes whose response is synthesized in-process rather than-// fetched, think blob data URLs.-const Synthetic = struct {- const data_url = @import("data_url.zig");+// get libcurl to make progress on the connections (easys) we've given it. This+// can mean starting a request or processing calllbacks. libcurl does not allow+// calling libcurl from callbacks, and that's something we never do (hence why+// Transfers have an event queue and dispatchCompleted is is own thing)...EXCEPT+// for WebSocket connections. So we need the performing guard to protect against+// invalid curl usage. TODO: Move WebSocket connections to work like HTTP+// connections.+fn perform(self: *Client) !c_int {+ self.performing = true;+ defer self.performing = false;+ return self.handles.perform();+}- fn isSynthetic(url: []const u8) bool {- return std.mem.startsWith(u8, url, "data:") or std.mem.startsWith(u8, url, "blob:");+// TODO: only needed because WebSocket connections can still open-close mid-perform+fn settleConns(self: *Client) !void {+ // Apply queued closes that happenened during "perform".+ while (self.dirty.popFirst()) |node| {+ const conn: *http.Connection = @fieldParentPtr("node", node);+ self.handles.remove(conn) catch |err| {+ lp.assert(false, "multi_remove_handle", .{+ .err = err,+ .in_use = conn.in_use,+ .added = conn.debug_added,+ .removed = conn.debug_removed,+ .remove_err = conn.debug_remove_err,+ });+ };+ conn.debug_removed = 2;+ self.releaseConn(conn);}- fn run(transfer: *Transfer, _: ?*anyopaque) void {- // prevents a callback that triggers a navigation queue from killing- // this transfer from under us.- transfer.state = .completing;- defer transfer.deinit();+ // Apply queued opens that happenened during "perform".+ while (self.ready_queue.popFirst()) |node| {+ const conn: *http.Connection = @fieldParentPtr("node", node);+ try self.trackConn(conn);+ }+}- const fulfilled = build(transfer) catch |err| {- transfer.req.error_callback(transfer.req.ctx, err);+// Deliver completed response. This is the ONLY place user callbacks run,+// so a callback is free to start new requests, abort transfers, or tear down+// its frame. The client is never inside libcurl here.+//+// This should be as simple as: iterate the `dispatch_queue` and dispatch each+// transfer...EXCEPT...a frame can have a blocking request. So, as we iterate+// we find out if a transfer is gated behind a blocking request and move it+// from the `dispatch_queue` to the `gated_queue`. That way, every call to+// `dispatchCompleted` doesn't keep checking the same gated transfers over and+// over.+fn dispatchCompleted(self: *Client) void {+ // pop is safest as it allows anything to manipulate the queue as necessary+ // (e.g. a frame could be aborted)s+ while (self.dispatch_queue.popFirst()) |n| {+ const transfer: *Transfer = @fieldParentPtr("_queue_node", n);+ if (self.isGated(transfer)) {+ transfer._gated = true;+ self.gated_queue.append(n);+ continue;+ }+ self.dispatch_count -= 1;+ transfer.deliver();+ }+}++// Is this transfer gated behind a blockig request+fn isGated(self: *const Client, transfer: *const Transfer) bool {+ if (transfer.req.internal) {+ // internal transfers are never blocked (e.g. robots.txt)+ return false;+ }+ if (self.blocking_requests.count() == 0) {+ // O(1) quick check.+ return false;+ }+ const blocking_id = self.blocking_requests.get(transfer.req.frame_id) orelse return false;+ return transfer.id != blocking_id;+}++fn startPending(self: *Client) !void {+ while (self.pending_queue.popFirst()) |queue_node| {+ const transfer: *Transfer = @fieldParentPtr("_node", queue_node);+ const conn = self.network.getConnection() orelse {+ self.pending_queue.prepend(queue_node);return;};- deliver(&transfer.req, &fulfilled) catch |err| {- transfer.req.error_callback(transfer.req.ctx, err);+ // Bridge state to .created so a failure inside makeRequest before+ // any commit cleans up via the abort below. makeRequest flips to+ // .inflight on a successful trackConn.+ transfer.state = .created;+ self.makeRequest(conn, transfer) catch |err| {+ if (transfer.state == .created) {+ transfer.abort(err);+ }+ return err;};}+}- fn build(transfer: *Transfer) !FulfilledResponse {- const arena = transfer.arena;- const url = transfer.req.url;+const SubmitFrom = enum { start, after_intercept, network };++// Process a transfer, passing it through our pipeline. A transfer an move off+// the pipeline(e.g. while parked waiting for a robots.txt check) and then+// pushed back onto it, which is what `from` helps us achieve.+fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {+ sw: switch (from) {+ .start => {+ if (self.network.web_bot_auth) |wba| {+ const authority = URL.getHost(transfer.req.url);+ try wba.signRequest(transfer.arena, &transfer.req.headers, authority);+ }- var body: []const u8 = "";- var content_type: []const u8 = "";+ if (self.serve_mode) {+ transfer._notify_cdp = true;+ transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });++ var wait_for_interception = false;+ transfer.req.notification.dispatch(.http_request_intercept, &.{+ .transfer = transfer,+ .wait_for_interception = &wait_for_interception,+ });+ if (wait_for_interception) {+ // The CDP listener stashed the transfer id and will+ // resolve it via continueIntercepted / fulfillIntercepted+ // / abortParked.+ self.intercepted += 1;+ transfer.park(.intercept_request);+ if (comptime IS_DEBUG) {+ log.debug(.http, "wait for interception", .{ .intercepted = self.intercepted });+ }+ return;+ }+ }+ continue :sw SubmitFrom.after_intercept;+ },+ .after_intercept => {+ if (try self.cacheLookup(transfer)) {+ // response came from the cache, we're done+ return;+ }+ if (self.obey_robots and !transfer.req.internal) {+ switch (try self.robots.check(transfer)) {+ .allowed => {},+ .blocked => return transfer.failAsync(error.RobotsBlocked),+ .pending => return,+ }+ }+ continue :sw SubmitFrom.network;+ },+ .network => try self.processTransfer(transfer),+ }+}- if (std.mem.startsWith(u8, url, "data:")) {- const parsed = try data_url.parse(arena, url);- content_type = parsed.content_type;- body = parsed.body;- } else {- // blob: — resolved against the owning frame's registry.- const owner = transfer.owner orelse return error.BlobNotFound;- const blob_urls = owner.blob_urls orelse return error.BlobNotFound;- const blob = blob_urls.get(url) orelse return error.BlobNotFound;- content_type = blob._mime;- body = blob._slice;+// RobotsGate resumption. The robots gate is the last step before the+// network, so an allowed transfer goes straight there.+pub fn resumeAfterRobots(self: *Client, transfer: *Transfer) !void {+ return self.pipeline(transfer, .network);+}++fn findHeader(headers: []const http.Header, name: []const u8) ?[]const u8 {+ for (headers) |hdr| {+ if (std.ascii.eqlIgnoreCase(hdr.name, name)) {+ return hdr.value;}+ }+ return null;+}- // A blob with no type yields no Content-Type header.- const headers = if (content_type.len > 0) blk: {- const h = try arena.alloc(http.Header, 1);- h[0] = .{ .name = "content-type", .value = content_type };- break :blk h;- } else &[_]http.Header{};+// Returns true if the request was served from the cache (response buffered+// for dispatch). On a miss the transfer is tagged so completion can store+// the response; on an expired-with-validators entry the request becomes a+// conditional revalidation.+fn cacheLookup(self: *Client, transfer: *Transfer) !bool {+ const cache = self.cache orelse return false;++ const req = &transfer.req;+ if (req.method != .GET) {+ return false;+ }++ const arena = transfer.arena;+ var iter = req.headers.iterator();+ const req_headers = try iter.collect(arena);++ const cached = cache.get(arena, .{+ .url = req.url,+ .timestamp = std.time.timestamp(),+ .request_headers = req_headers.items,+ }) orelse {+ transfer._cache_intent = .store;+ return false;+ };- return .{- .url = url,- .body = body,- .status = 200,- .headers = headers,- };+ if (cached.expired == false) {+ try transfer.bufferCached(cached);+ return true;}- fn deliver(req: *Request, fulfilled: *const FulfilledResponse) !void {- const response = Response.fromFulfilled(req.ctx, fulfilled);- if (req.start_callback) |cb| {- try cb(response);- }+ if (cached.metadata.hasValidators() == false) {+ // Expired and no validators+ cached.data.deinit();+ cache.evict(req.url);+ transfer._cache_intent = .store;+ return false;+ }- const result = try req.header_callback(response);- if (result == .abort) {- return error.Abort;- }+ // expired but with validators+ log.debug(.cache, "revalidate with etag", .{+ .url = req.url,+ .etag = cached.metadata.etag,+ .last_modified = cached.metadata.last_modified,+ });+ if (cached.metadata.etag) |etag| {+ try req.headers.add(try std.fmt.allocPrintSentinel(arena, "If-None-Match: {s}", .{etag}, 0));+ }+ if (cached.metadata.last_modified) |lm| {+ try req.headers.add(try std.fmt.allocPrintSentinel(arena, "If-Modified-Since: {s}", .{lm}, 0));+ }+ transfer._cache_intent = .{ .revalidate = cached };+ return false;+}++// 304 on a revalidation: renew the stored entry from the fresh headers and+// serve the stale copy. Returns true if the stale entry became the response.+fn cacheRevalidated(self: *Client, transfer: *Transfer) !bool {+ if (transfer._cache_intent != .revalidate) {+ return false;+ }+ const rh = transfer.res.header orelse return false;+ if (rh.status != 304) {+ return false;+ }+ // could have been disabled in-between+ const cache = self.cache orelse return false;++ const stale = transfer._cache_intent.revalidate;+ transfer._cache_intent = .none;++ cache.renew(transfer.arena, .{+ .url = transfer.req.url,+ .timestamp = std.time.timestamp(),+ .headers = transfer.res.headers,+ }) catch |err| {+ log.warn(.cache, "renew failed", .{ .err = err });+ };- if (fulfilled.body) |b| {- if (b.len > 0) {- try req.data_callback(response, b);+ try transfer.bufferCached(stale);+ return true;+}++// Store a cacheable response at completion, from the materialized response+// headers and the buffered body. Failures are logged, never fatal — the+// consumer gets its response either way.+fn cacheStore(self: *Client, transfer: *Transfer) void {+ switch (transfer._cache_intent) {+ .none => return,+ .store => {},+ .revalidate => |stale| {+ // Not a 304: the stored entry is out of date. Drop it and store+ // the fresh response below.+ stale.data.deinit();+ },+ }+ // could have been disabled while waiting of the response+ const cache = self.cache orelse return;++ transfer._cache_intent = .none;++ const arena = transfer.arena;+ const rh = &(transfer.res.header orelse return);+ const headers = transfer.res.headers;++ const vary = findHeader(headers, "vary");+ const maybe_cm = Cache.tryCache(+ arena,+ std.time.timestamp(),+ transfer.req.url,+ rh.status,+ rh.contentType(),+ findHeader(headers, "cache-control"),+ vary,+ findHeader(headers, "age"),+ findHeader(headers, "etag"),+ findHeader(headers, "last-modified"),+ findHeader(headers, "set-cookie") != null,+ findHeader(headers, "authorization") != null,+ ) catch |err| {+ log.warn(.http, "cache eligibility", .{ .err = err });+ return;+ };+ var metadata = maybe_cm orelse return;++ var vary_headers: std.ArrayList(http.Header) = .empty;+ if (vary) |vary_str| {+ var req_it = transfer.req.headers.iterator();+ while (req_it.next()) |hdr| {+ var vary_iter = std.mem.splitScalar(u8, vary_str, ',');+ while (vary_iter.next()) |part| {+ const name = std.mem.trim(u8, part, &std.ascii.whitespace);+ if (std.ascii.eqlIgnoreCase(hdr.name, name)) {+ const owned: http.Header = .{+ .name = arena.dupe(u8, hdr.name) catch return,+ .value = arena.dupe(u8, hdr.value) catch return,+ };+ vary_headers.append(arena, owned) catch return;+ }}}- try req.done_callback(req.ctx);}-};++ metadata.headers = headers;+ metadata.vary_headers = vary_headers.items;++ if (comptime IS_DEBUG) {+ log.debug(.browser, "http cache", .{ .key = transfer.req.url, .metadata = metadata });+ }+ cache.put(metadata, transfer.res.stream_buffer.items) catch |err| {+ log.warn(.http, "cache put failed", .{ .err = err });+ };+}const SyncContext = struct {allocator: Allocator,@@ -753,18 +901,18 @@ const SyncContext = struct {status: u16 = 0,body: std.ArrayList(u8),- fn headerCallback(response: Response) anyerror!HeaderResult {- const self: *SyncContext = @ptrCast(@alignCast(response.ctx));- lp.assert(response.status() != null, "HttpClient.SyncRequest.headerCallback", .{ .value = response.status() });- self.status = response.status().?;- if (response.contentLength()) |cl| {+ fn headerCallback(transfer: *Transfer) anyerror!Transfer.HeaderResult {+ const self: *SyncContext = @ptrCast(@alignCast(transfer.req.ctx));+ lp.assert(transfer.responseStatus() != null, "HttpClient.SyncRequest.headerCallback", .{ .value = transfer.responseStatus() });+ self.status = transfer.responseStatus().?;+ if (transfer.getContentLength()) |cl| {try self.body.ensureTotalCapacity(self.allocator, cl);}return .proceed;}- fn dataCallback(response: Response, data: []const u8) anyerror!void {- const self: *SyncContext = @ptrCast(@alignCast(response.ctx));+ fn dataCallback(transfer: *Transfer, data: []const u8) anyerror!void {+ const self: *SyncContext = @ptrCast(@alignCast(transfer.req.ctx));try self.body.appendSlice(self.allocator, data);}@@ -788,18 +936,13 @@ pub fn syncRequest(self: *Client, allocator: Allocator, req: Request) !SyncRespoif (self.inbox.terminated) {// request() takes ownership of req.headers on every path; we return// before calling it, so free the curl_slist here to avoid leaking it.- req.headers.deinit();+ req.deinit();return error.ClientDisconnected;}var sync_ctx = SyncContext{ .allocator = allocator, .body = .empty };errdefer sync_ctx.body.deinit(allocator);- const expected_id = self.nextReqId();- const frame_id = req.frame_id;- try self.blocking_requests.putNoClobber(self.allocator, frame_id, expected_id);- defer _ = self.blocking_requests.remove(frame_id);-var r = req;r.ctx = &sync_ctx;r.header_callback = SyncContext.headerCallback;@@ -807,7 +950,16 @@ pub fn syncRequest(self: *Client, allocator: Allocator, req: Request) !SyncRespor.done_callback = SyncContext.doneCallback;r.error_callback = SyncContext.errorCallback;r.shutdown_callback = SyncContext.shutdownCallback;- const transfer = try self.requestT(r, null);+ const transfer = try self.newRequest(r, null);++ const frame_id = req.frame_id;+ self.blocking_requests.putNoClobber(self.allocator, frame_id, transfer.id) catch |err| {+ transfer.deinit();+ return err;+ };+ defer self.releaseBlocking(frame_id);++ try transfer.submit();while (sync_ctx.completion == .in_progress) {self.tick(200, .sync_wait) catch |err| {@@ -839,10 +991,7 @@ pub fn syncRequest(self: *Client, allocator: Allocator, req: Request) !SyncRespo}}-// Above, request will not process if there's an interception request. In such-// cases, the interceptor is expected to call resume to continue the transfer-// or transfer.abort() to abort it.-fn process(self: *Client, transfer: *Transfer) !void {+fn processTransfer(self: *Client, transfer: *Transfer) !void {// libcurl doesn't allow recursive calls, if we're in a `perform()` operation// then we _have_ to queue this.if (self.performing == false) {@@ -851,27 +1000,30 @@ fn process(self: *Client, transfer: *Transfer) !void {}}- self.queue.append(&transfer._node);+ self.pending_queue.append(&transfer._node);transfer.state = .queued;}-pub fn nextReqId(self: *Client) u32 {- return self.next_request_id +% 1;-}--pub fn incrReqId(self: *Client) u32 {- const id = self.next_request_id +% 1;- self.next_request_id = id;- return id;-}--// Same restriction as changeProxy. Should be ok since this is only called on-// BrowserContext deinit.-pub fn restoreOriginalProxy(self: *Client) !void {- try self.ensureNoActiveConnection();-- self.http_proxy = self.network.config.httpProxy();- self.use_proxy = self.http_proxy != null;+// A blocking request is complete. Any completed transfer that was placed in the+// gated_queue because of it can now be placed back in the dispatch queue.+fn releaseBlocking(self: *Client, frame_id: u32) void {+ _ = self.blocking_requests.remove(frame_id);+ // items were added to the gate in order, so walking backwards restores that+ // order. (Order might not matter, but preserving it costs nothing)++ // No callback is executed here. Nothing mutates our queues, so iterating+ // it this way is safe.+ var node = self.gated_queue.last;+ while (node) |n| {+ node = n.prev;+ const transfer: *Transfer = @fieldParentPtr("_queue_node", n);+ if (transfer.req.frame_id != frame_id) {+ continue;+ }+ transfer._gated = false;+ self.gated_queue.remove(n);+ self.dispatch_queue.prepend(n);+ }}fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyerror!void {@@ -905,81 +1057,8 @@ fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyertransfer._conn = conn;transfer.state = .inflight;- if (transfer.req.start_callback) |cb| {- cb(Response.fromTransfer(transfer)) catch |err| {- // We're now committed to the multi. transfer.abort fires the- // error_callback and tears down (removeConn handles the- // already-in-multi case via the dirty queue).- transfer.abort(err);- return err;- };- }-- // Start the request (and move along any other request). This used to call- // self.perform(0) but that can also execute callbacks. Normally, that- // wouldn't be so bad. But curl can synchronously fire callbacks for the- // request we JUST added, which we do not want (it results in incorrect- // execution).- self.performing = true;- defer self.performing = false;- _ = try self.handles.perform();-}--fn perform(self: *Client, timeout_ms: c_int) anyerror!void {- const running = blk: {- self.performing = true;- defer self.performing = false;-- break :blk try self.handles.perform();- };-- // Process dirty connections — return them to Network pool.- while (self.dirty.popFirst()) |node| {- const conn: *http.Connection = @fieldParentPtr("node", node);- self.handles.remove(conn) catch |err| {- lp.assert(false, "multi_remove_handle", .{- .err = err,- .in_use = conn.in_use,- .added = conn.debug_added,- .removed = conn.debug_removed,- .remove_err = conn.debug_remove_err,- });- };- conn.debug_removed = 2;- self.releaseConn(conn);- }-- while (self.ready_queue.popFirst()) |node| {- const conn: *http.Connection = @fieldParentPtr("node", node);- try self.trackConn(conn);- }-- // We just processed completions; their done_callbacks may have- // scheduled microtasks (JS continuations) or queued new transfers.- // Return without polling so the caller (_tick) can run macrotasks- // and re-evaluate. Otherwise we'd sleep on cdp_link_active for up- // to timeout_ms while pending JS work sits idle.- if (try self.processMessages()) {- return;- }-- // Poll for HTTP I/O. The Network thread will call curl_multi_wakeup- // on our multi handle whenever it pushes to our inbox, so we drop- // out of poll promptly even when we have no curl handles in flight- // — but ONLY if a producer is actually wired up. `cdp_link_active`- // is set by Server.handleConnection once network.registerCdp has- // returned; in tests (which never register) and during the- // pre-handshake window the flag stays false and we don't waste a- // poll timeout waiting for a wakeup that won't arrive.- if (running > 0 or self.cdp_link_active) {- // when cdp_link_active == true, the network thread will unblock this- // by calling wakup on our multi.- self.heartbeat.enterWait();- defer self.heartbeat.exitWait();- try self.handles.poll(&.{}, timeout_ms);- }-- _ = try self.processMessages();+ // Start the request (and move along any other request).+ _ = try self.perform();}// Drain any CDP messages the Network thread pushed into our inbox@@ -1024,7 +1103,7 @@ fn drainInbox(self: *Client, mode: DrainMode) !void {// ping/close/disconnect (control frames must be observed). CDP data// messages are filtered: only the four Fetch interception methods// are safe to dispatch from inside a JS callback (they mutate-// transfer state via InterceptionLayer; they don't touch page /+// transfer state via the interception gate; they don't touch page /// session / V8 state). The check is exact on the parsed `method`// field — no substring matching against raw JSON.//@@ -1064,16 +1143,59 @@ fn isTeardownMethod(method: []const u8) bool {std.mem.eql(u8, method, "Page.close");}-pub fn isRedirectStatus(status: u16) bool {+fn isRedirectStatus(status: u16) bool {return switch (status) {301, 302, 303, 307, 308 => true,else => false,};}-fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Transfer) !bool {- // State at entry: .inflight = conn (multi just delivered a completion).+fn processMessages(self: *Client) !bool {+ var processed = false;+ while (try self.handles.readMessage()) |msg| {+ switch (msg.conn.transport) {+ .http => |transfer| {+ // On success, processOneMessage owns the transfer's fate: it+ // either buffered it for dispatch, parked it, or deinit'd it.+ // Only the throw path cleans up here.+ const done = self.processOneMessage(msg, transfer) catch |err| blk: {+ log.err(.http, "process_messages", .{ .err = err, .req = transfer });+ transfer.state = .completing;+ transfer.requestFailed(err);+ if (transfer._detached_conn) |c| {+ // Conn was removed from handles during redirect reconfiguration+ // but not re-added. Release it directly to avoid double-remove.+ self.in_use.remove(&c.node);+ self.http_active -= 1;+ self.releaseConn(c);+ transfer._detached_conn = null;+ }+ transfer.deinit();+ break :blk true;+ };+ if (done) {+ processed = true;+ }+ },+ .websocket => |ws| {+ // ws_active will be decremented through the call to disconnected+ if (msg.err) |err| switch (err) {+ error.GotNothing => ws.disconnected(null),+ else => ws.disconnected(err),+ } else {+ // Clean close - no error+ ws.disconnected(null);+ }++ processed = true;+ },+ .none => unreachable,+ }+ }+ return processed;+}+fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Transfer) !bool {// Workaround for libcurl Brotli trailing-byte rejection.//// Some CDNs (e.g. CloudFront serving Brave Search) emit a Brotli stream@@ -1094,6 +1216,15 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tbreak :blk err;} else null;+ if (transfer.state == .aborted) {+ // detachInPerform ran while this response was in flight; there is+ // nothing to deliver and nobody else owns the transfer anymore.+ self.removeConn(msg.conn);+ transfer._conn = null;+ transfer.deinit();+ return true;+ }+if (effective_err == null or effective_err.? == error.RecvError) {transfer.detectAuthChallenge(msg.conn);}@@ -1107,9 +1238,9 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T&.{ .transfer = transfer, .wait_for_interception = &wait_for_interception },);if (wait_for_interception) {- self.interception_layer.intercepted += 1;+ self.intercepted += 1;if (comptime IS_DEBUG) {- log.debug(.http, "wait for auth interception", .{ .intercepted = self.interception_layer.intercepted });+ log.debug(.http, "wait for auth interception", .{ .intercepted = self.intercepted });}// Whether or not this is a blocking request, we're not going@@ -1149,14 +1280,18 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tconn.debug_added = 2;transfer._detached_conn = null;- _ = try self.perform(0);+ // Get the redirect on the wire now. Any completion messages+ // this produces are picked up by the processMessages loop+ // we were called from — no recursion into the pump.+ _ = try self.perform();return false;},// error_callback surfaces this as a TypeError..@"error" => {- transfer.state = .completing;- transfer.requestFailed(error.RedirectNotAllowed, true);+ self.removeConn(msg.conn);+ transfer._conn = null;+ transfer.failAsync(error.RedirectNotAllowed);return true;},// Don't follow; fall through to deliver the 3xx as the final@@ -1166,8 +1301,9 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T}}- // Transfer is done (success or error). Caller (processMessages) owns deinit.- // Return true = done (caller will deinit), false = continues (redirect/auth).+ // Transfer is done (success or error). Materialize the response into the+ // transfer's arena, release the conn, and buffer the events — user+ // callbacks run later, from dispatch(), never from here.// When the server closes the TLS onnection without a close_notify alert,// BoringSSL reports RecvError. If we already received valid HTTP headers,@@ -1181,94 +1317,30 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tbreak :blk std.ascii.eqlIgnoreCase(hdr.value, "close");};- // Transition to .completing so re-entrant aborts from user callbacks- // defer their teardown to processMessages. (_conn carries through- // from .inflight; nothing to set here.)- transfer.state = .completing;-if (effective_err != null and !is_conn_close_recv) {- transfer.requestFailed(transfer.res.callback_error orelse effective_err.?, true);+ self.removeConn(msg.conn);+ transfer._conn = null;+ transfer.failAsync(transfer.res.callback_error orelse effective_err.?);return true;}- if (!transfer.res.header_done_called) {- // In case of request w/o data, we need to call the header done- // callback now.- const result = try transfer.headerDoneCallback(msg.conn);- switch (result) {- .proceed => {},- .handled => return true,- .abort => {- transfer.requestFailed(error.Abort, true);- return true;- },- }- }-- const body = transfer.res.stream_buffer.items;+ try transfer.materializeResponse(msg.conn);- // Replay buffered body through user's data_callback.- if (body.len > 0) {- try transfer.req.data_callback(Response.fromTransfer(transfer), body);-- if (transfer.state == .aborted) {- transfer.requestFailed(error.Abort, true);- return true;- }- }-- // release conn ASAP so that it's available; some done_callbacks- // will load more resources. State stays .completing — the- // processMessages caller still owns deinit.+ // Release the conn before any of this response's callbacks can run —+ // they'll want it for the next resource.self.removeConn(msg.conn);transfer._conn = null;- try transfer.req.done_callback(transfer.req.ctx);+ if (try self.cacheRevalidated(transfer)) {+ return true;+ }+ self.cacheStore(transfer);+ transfer._cdp_content_length = transfer.getContentLength() orelse 0;+ try transfer.bufferEvents(transfer.res.stream_buffer.items);return true;}-fn processMessages(self: *Client) !bool {- var processed = false;- while (try self.handles.readMessage()) |msg| {- switch (msg.conn.transport) {- .http => |transfer| {- const done = self.processOneMessage(msg, transfer) catch |err| blk: {- log.err(.http, "process_messages", .{ .err = err, .req = transfer });- transfer.requestFailed(err, true);- if (transfer._detached_conn) |c| {- // Conn was removed from handles during redirect reconfiguration- // but not re-added. Release it directly to avoid double-remove.- self.in_use.remove(&c.node);- self.http_active -= 1;- self.releaseConn(c);- transfer._detached_conn = null;- }- break :blk true;- };- if (done) {- transfer.deinit();- processed = true;- }- },- .websocket => |ws| {- // ws_active will be decremented through the call to disconnected- if (msg.err) |err| switch (err) {- error.GotNothing => ws.disconnected(null),- else => ws.disconnected(err),- } else {- // Clean close - no error- ws.disconnected(null);- }-- processed = true;- },- .none => unreachable,- }- }- return processed;-}-pub fn trackConn(self: *Client, conn: *http.Connection) !void {if (self.performing) {conn.in_use = false;@@ -1337,19 +1409,10 @@ fn ensureNoActiveConnection(self: *const Client) !void {}}-pub const HeaderResult = enum {- /// Continue processing normally.- proceed,- /// Caller took ownership of the response; stop w/o error or abort.- handled,- /// Abort the Transfer,- abort,-};-pub const Request = struct {- pub const StartCallback = *const fn (response: Response) anyerror!void;- pub const HeaderCallback = *const fn (response: Response) anyerror!HeaderResult;- pub const DataCallback = *const fn (response: Response, data: []const u8) anyerror!void;+ pub const StartCallback = *const fn (transfer: *Transfer) anyerror!void;+ pub const HeaderCallback = *const fn (transfer: *Transfer) anyerror!Transfer.HeaderResult;+ pub const DataCallback = *const fn (transfer: *Transfer, data: []const u8) anyerror!void;pub const DoneCallback = *const fn (ctx: *anyopaque) anyerror!void;pub const ErrorCallback = *const fn (ctx: *anyopaque, err: anyerror) void;pub const ShutdownCallback = *const fn (ctx: *anyopaque) void;@@ -1384,7 +1447,9 @@ pub const Request = struct {loader_id: u32,method: Method,url: [:0]const u8,- headers: http.Headers,+ // Empty by default; the client fills in its baseline headers (user+ // agent, sec-ch-ua, accept-language) when none are supplied.+ headers: http.Headers = .{ .headers = null },body: ?[]const u8 = null,cookie_jar: ?*CookieJar,cookie_origin: [:0]const u8,@@ -1410,7 +1475,14 @@ pub const Request = struct {data_callback: DataCallback = Noop.dataCallback,done_callback: DoneCallback = Noop.doneCallback,error_callback: ErrorCallback = Noop.errorCallback,- shutdown_callback: ?ShutdownCallback = null,++ // Fired exactly once if the transfer is torn down by its owner (frame /+ // worker teardown, a superseding navigation) before completing. It must+ // drop any stored *Transfer and must not run JS. Deliberately has no+ // default: anyone holding a *Transfer forgets this at their peril, so+ // every caller decides — pass `HttpClient.noopShutdown` to opt out,+ // knowingly.+ shutdown_callback: ShutdownCallback,pub fn getCookieString(self: *Request, arena: Allocator) !?[:0]const u8 {const jar = self.cookie_jar orelse return null;@@ -1433,159 +1505,148 @@ pub const Request = struct {}};-pub const FulfilledResponse = struct {+pub const SyncResponse = struct {status: u16,- url: [:0]const u8,- headers: []const http.Header,- body: ?[]const u8,+ body: std.ArrayList(u8),- pub fn contentType(self: *const FulfilledResponse) ?[]const u8 {- for (self.headers) |hdr| {- if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) return hdr.value;- }- return null;+ pub fn deinit(self: *SyncResponse, allocator: Allocator) void {+ self.body.deinit(allocator);}};-pub const StableResponse = struct {- ctx: *anyopaque,- status: u16,- url: [:0]const u8,- headers: []const http.Header,- body: ?[]const u8,+// CDP Fetch.continueWithAuth: resume a transfer parked on an auth+// challenge. The auth retry goes straight back to the network — it already+// passed the request-side pipeline on its first attempt.+pub fn continueTransfer(self: *Client, transfer: *Transfer) !void {+ if (comptime IS_DEBUG) {+ log.debug(.http, "continue transfer", .{ .intercepted = self.intercepted });+ }- pub fn contentType(self: *const StableResponse) ?[]const u8 {- for (self.headers) |hdr| {- if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) return hdr.value;+ transfer.unpark();+ self.processTransfer(transfer) catch |err| {+ if (transfer.state == .created) {+ transfer.abort(err);}- return null;+ return err;+ };+}++// CDP Fetch.continueRequest: resume a transfer parked at the interception+// gate, re-entering the pipeline at the step after interception. The CDP+// command may have mutated req (url / method / headers / body) first.+pub fn continueIntercepted(self: *Client, transfer: *Transfer) !void {+ if (comptime IS_DEBUG) {+ lp.assert(self.intercepted > 0, "Client.continueIntercepted", .{ .value = self.intercepted });}-};-pub const Response = struct {- ctx: *anyopaque,- inner: union(enum) {- transfer: *Transfer,- cached: *const CachedResponse,- fulfilled: *const FulfilledResponse,- stable: *const StableResponse,- },+ transfer.unpark();+ self.pipeline(transfer, .after_intercept) catch |err| {+ if (transfer.state == .created) {+ transfer.abort(err);+ }+ return err;+ };+}- pub fn fromTransfer(transfer: *Transfer) Response {- return .{ .ctx = transfer.req.ctx, .inner = .{ .transfer = transfer } };+// CDP Fetch.fulfillRequest: resolve a parked transfer with an interceptor-+// supplied response. A redirect status with a Location header is followed+// like a real network redirect; anything else is buffered for dispatch —+// the consumer's callbacks never run from the CDP command itself.+pub fn fulfillIntercepted(+ self: *Client,+ transfer: *Transfer,+ status: u16,+ headers: []const http.Header,+ body: ?[]const u8,+) !void {+ if (comptime IS_DEBUG) {+ lp.assert(self.intercepted > 0, "Client.fulfillIntercepted", .{ .value = self.intercepted });}- pub fn fromCached(ctx: *anyopaque, resp: *const CachedResponse) Response {- return .{ .ctx = ctx, .inner = .{ .cached = resp } };- }+ transfer.unpark();- pub fn fromFulfilled(ctx: *anyopaque, fulfilled: *const FulfilledResponse) Response {- return .{ .ctx = ctx, .inner = .{ .fulfilled = fulfilled } };- }+ const followed = blk: {+ if (isRedirectStatus(status) == false) {+ break :blk false;+ }+ const location = findHeader(headers, "location") orelse break :blk false;+ try self.fulfillRedirect(transfer, status, headers, location);+ break :blk true;+ };- pub fn fromStable(stable: *const StableResponse) Response {- return .{ .ctx = stable.ctx, .inner = .{ .stable = stable } };+ if (followed) {+ return;}- pub fn status(self: Response) ?u16 {- return switch (self.inner) {- .transfer => |t| if (t.res.header) |rh| rh.status else null,- .cached => |c| c.metadata.status,- .fulfilled => |f| f.status,- .stable => |s| s.status,- };- }+ transfer.bufferFulfilled(status, headers, body) catch |err| {+ if (transfer.state == .created) {+ transfer.abort(err);+ }+ return err;+ };+}- pub fn contentType(self: Response) ?[]const u8 {- return switch (self.inner) {- .transfer => |t| if (t.res.header) |*rh| rh.contentType() else null,- .cached => |c| c.metadata.content_type,- .fulfilled => |f| f.contentType(),- .stable => |s| s.contentType(),- };- }+fn fulfillRedirect(+ self: *Client,+ transfer: *Transfer,+ status: u16,+ headers: []const http.Header,+ location: []const u8,+) !void {+ errdefer |err| if (transfer.state == .created) transfer.abort(err);- pub fn contentLength(self: Response) ?u32 {- return switch (self.inner) {- .transfer => |t| t.getContentLength(),- .cached => |c| switch (c.data) {- .buffer => |buf| @intCast(buf.len),- .file => |f| @intCast(f.len),- },- .fulfilled => |f| if (f.body) |b| @intCast(b.len) else null,- .stable => |s| if (s.body) |b| @intCast(b.len) else null,- };+ // retrieve cookies from the fulfilled response's headers.+ if (transfer.req.cookie_jar) |jar| {+ for (headers) |hdr| {+ if (std.ascii.eqlIgnoreCase(hdr.name, "set-cookie")) {+ try jar.populateFromResponse(transfer.req.url, hdr.value);+ }+ }}- pub fn redirectCount(self: Response) ?u32 {- return switch (self.inner) {- .transfer => |t| if (t.res.header) |rh| rh.redirect_count else null,- .cached, .fulfilled, .stable => 0,- };- }+ try transfer.applyRedirectTarget(transfer.req.url, location, status);+ try self.pipeline(transfer, .after_intercept);+}- pub fn url(self: Response) [:0]const u8 {- return switch (self.inner) {- .transfer => |t| t.req.url,- .cached => |c| c.metadata.url,- .fulfilled => |f| f.url,- .stable => |s| s.url,- };+// The explicit opt-out for Request.shutdown_callback, for callers that hold+// no reference to the transfer (or whose cleanup runs through other paths).+pub fn noopShutdown(_: *anyopaque) void {}++const Noop = struct {+ fn headerCallback(_: *Transfer) !Transfer.HeaderResult {+ return .proceed;}+ fn dataCallback(_: *Transfer, _: []const u8) !void {}+ fn doneCallback(_: *anyopaque) !void {}+ fn errorCallback(_: *anyopaque, _: anyerror) void {}+};++// An opaque-from-the-outside handle that Frame / WorkerGlobalScope embed+// to track the HTTP transfers + WebSockets they own.+pub const Owner = struct {+ transfers: std.DoublyLinkedList = .{},+ websockets: std.DoublyLinkedList = .{},++ // The owning Frame's / WorkerGlobalScope's blob: registry,+ blob_urls: ?*const std.StringHashMapUnmanaged(*Blob) = null,- pub fn headerIterator(self: Response) HeaderIterator {- return switch (self.inner) {- .transfer => |t| t.responseHeaderIterator(),- .cached => |c| HeaderIterator{ .list = .{ .list = c.metadata.headers } },- .fulfilled => |f| HeaderIterator{ .list = .{ .list = f.headers } },- .stable => |s| HeaderIterator{ .list = .{ .list = s.headers } },- };- }+ const WebSocket = @import("../browser/webapi/net/WebSocket.zig");+ const Blob = @import("../browser/webapi/Blob.zig");- pub fn abort(self: Response, err: anyerror) void {- switch (self.inner) {- .transfer => |t| t.abort(err),- .cached, .fulfilled, .stable => {},- }+ pub fn addTransfer(self: *Owner, t: *Transfer) void {+ self.transfers.append(&t.owner_node);}- pub fn format(self: Response, writer: *std.Io.Writer) !void {- return switch (self.inner) {- .transfer => |t| try t.format(writer),- .cached => |c| try c.format(writer),- .fulfilled => |f| try writer.print("fulfilled {s}", .{f.url}),- .stable => |s| writer.print("stable {s}", .{s.url}),- };+ pub fn removeTransfer(self: *Owner, t: *Transfer) void {+ self.transfers.remove(&t.owner_node);}- pub fn toStable(self: Response, arena: std.mem.Allocator) !StableResponse {- const new_url = try arena.dupeZ(u8, self.url());-- var headers: std.ArrayListUnmanaged(http.Header) = .{};- var it = self.headerIterator();- while (it.next()) |hdr| {- try headers.append(arena, .{- .name = try arena.dupe(u8, hdr.name),- .value = try arena.dupe(u8, hdr.value),- });- }-- return .{- .ctx = self.ctx,- .status = self.status() orelse 0,- .url = new_url,- .headers = headers.items,- .body = null,- };+ pub fn addWS(self: *Owner, ws: *WebSocket) void {+ self.websockets.append(&ws._owner_node);}-};--pub const SyncResponse = struct {- status: u16,- body: std.ArrayList(u8),- pub fn deinit(self: *SyncResponse, allocator: Allocator) void {- self.body.deinit(allocator);+ pub fn removeWS(self: *Owner, ws: *WebSocket) void {+ self.websockets.remove(&ws._owner_node);}};@@ -1612,7 +1673,7 @@ pub const Transfer = struct {_conn: ?*http.Connection = null,req: Request,- res: Transfer.Response = .{},+ res: Response = .{},client: *Client,start_time: u64,@@ -1634,8 +1695,30 @@ pub const Transfer = struct {// for when a Transfer is queued in the client.queue_node: std.DoublyLinkedList.Node = .{},- // for when a Transfer is queued for the next tick.- _next_tick_node: ?NextTickNode = null,+ // Buffered response ordered events awaiting dispatch. Today, ArrayList is+ // overkill. _events will be at most len == 4 and almost always len == 4.+ // BUT, when WebSockets move to a dispatch mode AND we re-add streaming,+ // this will become more dynamic.+ _events: std.ArrayList(Event) = .{},++ // controls if _queue_node is in client.dispatch_queu (false) or+ // client.gated_queue (true)+ _gated: bool = false,++ _queue_node: std.DoublyLinkedList.Node = .{},++ // Mirror this transfer's lifecycle as CDP Network.* events (serve mode).+ _notify_cdp: bool = false,++ // Response came from the http cache; drives the served-from-cache+ // notification at delivery.+ _from_cache: bool = false,++ // What the cache wants from this transfer's response.+ _cache_intent: CacheIntent = .none,++ // Content length reported on the CDP loadingFinished event.+ _cdp_content_length: usize = 0,// Debug canary: set on the first deinit, so that if a second deinit on the// same instance is called, we have a double free. The memory _could_ be@@ -1653,6 +1736,11 @@ pub const Transfer = struct {// linked into client.queue.queued,+ // Response events are buffered on `_events`, waiting for dispatch+ // to deliver them. `_queue_node` is linked into+ // client.dispatch_queue. No conn is held.+ buffered,+// Conn (in `_conn`) is in the multi handle; libcurl owns the// lifecycle. processOneMessage will eventually fire callbacks// for us.@@ -1682,18 +1770,25 @@ pub const Transfer = struct {// waiting for continueWithAuth.intercept_auth,- // RobotsLayer holds the transfer pending a robots.txt fetch.+ // RobotsGate holds the transfer pending a robots.txt fetch.robots,};- // Layer-facing: park the transfer for an external owner. The caller+ pub const HeaderResult = enum {+ /// Continue processing normally.+ proceed,+ /// Abort the Transfer,+ abort,+ };++ // Gate-facing: park the transfer for an external owner. The caller// must be holding the transfer in the request flow (state == .created).pub fn park(self: *Transfer, by: ParkedBy) void {lp.assert(self.state == .created, "Transfer.park", .{ .state = self.state });self.state = .{ .parked = by };}- // Layer-facing: take the transfer out of .parked and return it to+ // Gate-facing: take the transfer out of .parked and return it to// the request flow (.created). This assumes pre-inflight handling (i.e. the// transfer was in .created before being parked). This is true today, but// could become false if Request Interception ever supports the "response"@@ -1714,13 +1809,41 @@ pub const Transfer = struct {switch (self.state.parked) {.robots => {},.intercept_request, .intercept_auth => {- const intercept_layer = &self.client.interception_layer;- lp.assert(intercept_layer.intercepted > 0, "Transfer.leaveIntercept", .{ .value = intercept_layer.intercepted });- intercept_layer.intercepted -= 1;+ lp.assert(self.client.intercepted > 0, "Transfer.leaveIntercept", .{ .value = self.client.intercepted });+ self.client.intercepted -= 1;},}}+ // Submit a transfer created with Client.newRequest, entering the+ // request pipeline. Consumes the caller's ownership unconditionally:+ // on failure the transfer is already cleaned up (error_callback fired,+ // memory freed) — do not deinit it, and do not touch it after this.+ pub fn submit(self: *Transfer) anyerror!void {+ // Synthetic schemes never touch the network — no robots, cache, or+ // interception. The response is materialized here and delivered by+ // the dispatcher like any other transfer.+ if (Synthetic.isSynthetic(self.req.url)) {+ Synthetic.buffer(self) catch |err| {+ // Buffer the failure too (blob-not-found is a normal+ // JS-visible TypeError). Delivered like any other error (e.g+ // asynchronously in deliver).+ self.failAsync(err);+ };+ return;+ }++ self.client.pipeline(self, .start) catch |err| {+ if (self.state == .created) {+ // pipeline failed before the Transfer was handed over to a+ // another owner (e.g. the multi, the pending queue, ...)+ // so we're still responsible for it.+ self.abort(err);+ }+ return err;+ };+ }+pub fn deinit(self: *Transfer) void {if (comptime IS_DEBUG) {lp.assert(self._deinited == false, "Transfer.deinit", .{ .id = self.id });@@ -1737,15 +1860,31 @@ pub const Transfer = struct {// abort during navigation) leaves a dangling _node in the queue// that the next tick would pop and hand to libcurl → UAF.if (self.state == .queued) {- self.client.queue.remove(&self._node);+ self.client.pending_queue.remove(&self._node);+ }++ // Same for the dispatch queue: a buffered transfer killed+ // out-of-band (owner teardown, abort) must not be delivered.+ if (self.state == .buffered) {+ if (self._gated) {+ self.client.gated_queue.remove(&self._queue_node);+ } else {+ self.client.dispatch_queue.remove(&self._queue_node);+ }+ self.client.dispatch_count -= 1;+ }++ // A pending revalidation entry owns cache resources (possibly an+ // open file). It's resolved at completion; if we die first, release.+ switch (self._cache_intent) {+ .revalidate => |stale| stale.data.deinit(),+ .none, .store => {},}// Drop the id→*Transfer index entry before freeing the memory.// Any concurrent CDP lookup by id will now see this transfer as gone._ = self.client.transfers.remove(self.id);- self.client.cancelNextTick(self);-self.req.deinit();if (self.owner) |o| {o.removeTransfer(self);@@ -1766,29 +1905,35 @@ pub const Transfer = struct {// a transfer. Don't reach for kill() or requestFailed() directly —// they're internal helpers.pub fn abort(self: *Transfer, err: anyerror) void {- self.requestFailed(err, true);+ self.requestFailed(err);self.detachOrDeinit();}// Abort a transfer that an external owner (CDP interception) is holding in a// .parked state. Unlike abort(), this is re-entrancy safe so that if// requestFailed causes a teardown/navigate, this won't be killed again- // Mirrors InterceptionLayer.fulfillRequest. unpark asserts the transfer is+ // Mirrors Client.fulfillIntercepted. unpark asserts the transfer is// actually parked.pub fn abortParked(self: *Transfer, err: anyerror) void {self.unpark();self.state = .completing;defer self.deinit();- self.requestFailed(err, true);+ self.requestFailed(err);}// Owner-driven teardown: fires shutdown_callback (not error_callback)// and otherwise behaves like abort. Called by Client.abortOwner /- // abortRequests when a Frame / WGS is being torn down.+ // abortRequests when a Frame / WGS is being torn down. Any buffered,+ // undelivered events are dropped — the consumer is going away with us.fn kill(self: *Transfer) void {- if (self.req.shutdown_callback) |cb| {- cb(self.req.ctx);+ if (self._notify_cdp and !self._notified_fail) {+ self._notified_fail = true;+ self.req.notification.dispatch(.http_request_fail, &.{+ .transfer = self,+ .err = error.Shutdown,+ });}+ self.req.shutdown_callback(self.req.ctx);self.detachOrDeinit();}@@ -1838,7 +1983,7 @@ pub const Transfer = struct {// releases it once libcurl is done.self.state = .aborted;self.req.start_callback = null;- self.req.shutdown_callback = null;+ self.req.shutdown_callback = noopShutdown;self.req.header_callback = Noop.headerCallback;self.req.data_callback = Noop.dataCallback;self.req.done_callback = Noop.doneCallback;@@ -1849,28 +1994,167 @@ pub const Transfer = struct {}}- // Internal failure-notification helper. Latches via _notified_fail so- // multiple paths racing to report the same failure only fire one- // notification. Goes through transfer.req — so layer wrappers- // (InterceptContext, CacheContext) see the failure and can propagate- // it up the chain.- //- // Not part of the external API: callers cancelling a transfer should- // use transfer.abort(err) instead, which goes through this and also- // handles the deinit / detach side. The internal HttpClient flow uses- // this directly (from processOneMessage) because it's already paired- // with the natural processMessages → transfer.deinit handoff.- //- // execute_callback=true → fires error_callback. false → fires- // shutdown_callback (used by Frame shutdown / WGS teardown).- fn requestFailed(self: *Transfer, err: anyerror, comptime execute_callback: bool) void {- if (self._notified_fail) return;+ // Some type of failure, either internal or external, either explicit or+ // implicit. transfer.abort() is the external and explicit path. There are+ // various internal callers, but the most common is a callback error.+ fn requestFailed(self: *Transfer, err: anyerror) void {+ if (self._notified_fail) {+ return;+ }self._notified_fail = true;- if (execute_callback) {- self.req.error_callback(self.req.ctx, err);- } else if (self.req.shutdown_callback) |cb| {- cb(self.req.ctx);+ if (self._notify_cdp) {+ self.req.notification.dispatch(.http_request_fail, &.{+ .transfer = self,+ .err = err,+ });+ }++ self.req.error_callback(self.req.ctx, err);+ }++ // A consumer callback failed mid-delivery: latch the failure, notify,+ // free. Only called from deliver().+ fn failDelivery(self: *Transfer, err: anyerror) void {+ log.err(.http, "delivery callback", .{ .err = err, .req = self });+ self.requestFailed(err);+ self.deinit();+ }++ // Fail the transfer asynchronously: the error is buffered and+ // error_callback fires from the dispatcher, never from the caller's+ // stack.+ pub fn failAsync(self: *Transfer, err: anyerror) void {+ self._events.append(self.arena, .{ .err = err }) catch {+ // Can't buffer (OOM): failing inline beats losing the error.+ return self.abort(err);+ };+ self.scheduleDispatch();+ }++ // Commit the transfer's buffered events for delivery. From here on the+ // dispatcher owns the transfer's fate; deinit unlinks it if it dies+ // out-of-band first.+ fn scheduleDispatch(self: *Transfer) void {+ if (comptime IS_DEBUG) {+ lp.assert(+ self.state == .created or self.state == .inflight,+ "Transfer.scheduleDispatch",+ .{ .state = self.state },+ );+ }+ self.state = .buffered;+ self.client.dispatch_queue.append(&self._queue_node);+ self.client.dispatch_count += 1;+ }++ // Buffer the standard success event sequence. `body` is either owned by+ // transfer.arena OR, through some other mechanism, outlives the transfer.+ fn bufferEvents(self: *Transfer, body: []const u8) !void {+ try self._events.ensureUnusedCapacity(self.arena, 4);+ self._events.appendAssumeCapacity(.start);+ self._events.appendAssumeCapacity(.header);+ if (body.len > 0) {+ self._events.appendAssumeCapacity(.{ .data = body });+ }+ self._events.appendAssumeCapacity(.done);+ self.scheduleDispatch();+ }++ fn setResponseHead(self: *Transfer, status: u16, content_type: ?[]const u8) void {+ self.res.header = .{+ .url = self.req.url.ptr,+ .status = status,+ .redirect_count = self._redirect_count,+ };+ if (content_type) |ct| {+ var hdr = &self.res.header.?;+ const len = @min(ct.len, http.ResponseHead.MAX_CONTENT_TYPE_LEN);+ hdr._content_type_len = len;+ @memcpy(hdr._content_type[0..len], ct[0..len]);+ }+ }++ // Serve a cache entry as this transfer's response. Takes ownership of+ // `cached` (file-backed bodies are read into the arena and closed).+ fn bufferCached(self: *Transfer, cached: Cache.CachedResponse) !void {+ const arena = self.arena;++ const body: []const u8 = switch (cached.data) {+ .buffer => |b| b,+ .file => |f| blk: {+ defer f.file.close();+ const buf = try arena.alloc(u8, f.len);+ const n = try f.file.preadAll(buf, f.offset);+ break :blk buf[0..n];+ },+ };++ self.setResponseHead(cached.metadata.status, cached.metadata.content_type);+ self.res.headers = cached.metadata.headers;+ self._from_cache = true;+ self._cdp_content_length = body.len;+ try self.bufferEvents(body);+ }++ // Materialize an interceptor-supplied response (CDP fulfillRequest).+ // `headers` and `body` are caller-owned; copy everything that must+ // survive until dispatch.+ fn bufferFulfilled(self: *Transfer, status: u16, headers: []const http.Header, body: ?[]const u8) !void {+ const arena = self.arena;++ const owned = try arena.alloc(http.Header, headers.len);+ var content_type: ?[]const u8 = null;+ for (headers, 0..) |hdr, i| {+ owned[i] = .{+ .name = try arena.dupe(u8, hdr.name),+ .value = try arena.dupe(u8, hdr.value),+ };+ if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) {+ content_type = owned[i].value;+ }+ }++ const owned_body: []const u8 = if (body) |b| try arena.dupe(u8, b) else "";++ self.setResponseHead(status, content_type);+ self.res.headers = owned;+ self._cdp_content_length = owned_body.len;+ try self.bufferEvents(owned_body);+ }++ // Copy everything the response needs out of the conn and into the+ // transfer arena. After this, delivery never touches libcurl state —+ // the conn can be released and reused before the consumer sees a byte.+ fn materializeResponse(self: *Transfer, conn: *http.Connection) !void {+ const arena = self.arena;++ if (self.res.header == null) {+ try self.buildResponseHeader(conn);+ }+ // buildResponseHeader stores a curl-owned url pointer; re-anchor it+ // in the arena so it survives the conn release.+ self.res.header.?.url = (try arena.dupeZ(u8, std.mem.span(self.res.header.?.url))).ptr;++ var it = HeaderIterator{ .curl = .{ .conn = conn } };+ const headers = try it.collect(arena);+ self.res.headers = headers.items;++ if (self.req.cookie_jar) |jar| {+ for (self.res.headers) |hdr| {+ if (std.ascii.eqlIgnoreCase(hdr.name, "set-cookie")) {+ jar.populateFromResponse(self.req.url, hdr.value) catch |err| {+ log.err(.http, "set cookie", .{ .err = err, .req = self });+ return err;+ };+ }+ }+ }++ if (self.getContentLength()) |cl| {+ if (cl > self.client.max_response_size) {+ return error.ResponseTooLarge;+ }}}@@ -1950,7 +2234,7 @@ pub const Transfer = struct {if (conn.getResponseHeader("content-type", 0)) |ct| {var hdr = &self.res.header.?;const value = ct.value;- const len = @min(value.len, ResponseHead.MAX_CONTENT_TYPE_LEN);+ const len = @min(value.len, http.ResponseHead.MAX_CONTENT_TYPE_LEN);hdr._content_type_len = len;@memcpy(hdr._content_type[0..len], value[0..len]);}@@ -2078,51 +2362,13 @@ pub const Transfer = struct {// before interception process.pub fn abortAuthChallenge(self: *Transfer) void {if (comptime IS_DEBUG) {- log.debug(.http, "abort auth transfer", .{ .intercepted = self.client.interception_layer.intercepted });+ log.debug(.http, "abort auth transfer", .{ .intercepted = self.client.intercepted });}// The transfer is still .parked(.intercept_auth)self.abortParked(error.AbortAuthChallenge);}- // headerDoneCallback is called once the headers have been read.- // It can be called either on dataCallback or once the request for those- // w/o body.- fn headerDoneCallback(transfer: *Transfer, conn: *const http.Connection) !HeaderResult {- lp.assert(transfer.res.header_done_called == false, "Transfer.headerDoneCallback", .{});- defer transfer.res.header_done_called = true;-- try transfer.buildResponseHeader(conn);-- if (transfer.req.cookie_jar) |jar| {- var i: usize = 0;- while (true) {- const ct = conn.getResponseHeader("set-cookie", i);- if (ct == null) break;- jar.populateFromResponse(transfer.req.url, ct.?.value) catch |err| {- log.err(.http, "set cookie", .{ .err = err, .req = transfer });- return err;- };- i += 1;- if (i >= ct.?.amount) break;- }- }-- if (transfer.getContentLength()) |cl| {- if (cl > transfer.client.max_response_size) {- return error.ResponseTooLarge;- }- }-- const result = transfer.req.header_callback(Client.Response.fromTransfer(transfer)) catch |err| {- log.err(.http, "header_callback", .{ .err = err, .req = transfer });- return err;- };-- if (result == .proceed and transfer.state == .aborted) return .abort;- return result;- }-fn dataCallback(buffer: [*]const u8, chunk_count: usize, chunk_len: usize, data: *anyopaque) callconv(.c) usize {// libcurl should only ever emit 1 chunk at a timeif (comptime IS_DEBUG) {@@ -2180,120 +2426,231 @@ pub const Transfer = struct {return @intCast(chunk_len);}+ // Response-view accessors. Callbacks receive the *Transfer once the+ // response is materialized, so these are always safe from a callback;+ // header/status are null before that point.+ pub fn responseStatus(self: *const Transfer) ?u16 {+ const rh = self.res.header orelse return null;+ return rh.status;+ }++ pub fn contentType(self: *Transfer) ?[]const u8 {+ if (self.res.header) |*rh| {+ return rh.contentType();+ }+ return null;+ }++ pub fn redirectCount(self: *const Transfer) ?u32 {+ const rh = self.res.header orelse return null;+ return rh.redirect_count;+ }+pub fn responseHeaderIterator(self: *Transfer) HeaderIterator {- // We always have a real curl request here. We handle injection up in InterceptionLayer.- const c = self._conn;- lp.assert(c != null, "Transfer.responseHeaderIterator", .{ .value = c != null });- // If we have a connection, than this is a real curl request and we- // iterate through the header that curl maintains.- return .{ .curl = .{ .conn = c.? } };+ if (self.res.headers.len > 0 or self._conn == null) {+ return .{ .list = .{ .list = self.res.headers } };+ }+ // Mid-stream (pump time): headers aren't materialized yet, read+ // them off the live conn.+ return .{ .curl = .{ .conn = self._conn.? } };}- // This function should be called during the dataCallback. Calling it after- // such as in the doneCallback is guaranteed to return null.pub fn getContentLength(self: *const Transfer) ?u32 {const cl = self.getContentLengthRawValue() orelse return null;return std.fmt.parseInt(u32, cl, 10) catch null;}fn getContentLengthRawValue(self: *const Transfer) ?[]const u8 {+ // Materialized headers (dispatch time, any source).+ for (self.res.headers) |hdr| {+ if (std.ascii.eqlIgnoreCase(hdr.name, "content-length")) {+ return hdr.value;+ }+ }++ // Mid-stream (curl's write callback): read from the live conn.if (self._conn) |c| {- // If we have a connection, than this is a normal request. We can get the- // header value from the connection.const cl = c.getResponseHeader("content-length", 0) orelse return null;return cl.value;}- // If we have no handle, then maybe this is being called after the- // doneCallback. OR, maybe this is a "fulfilled" request. Let's check- // the injected headers (if we have any).+ return null;+ }- const rh = self.res.header orelse return null;- for (rh._injected_headers) |hdr| {- if (std.ascii.eqlIgnoreCase(hdr.name, "content-length")) {- return hdr.value;- }+ // Only ever called from tick, delivers a completed event to its caller+ fn deliver(transfer: *Transfer) void {+ // Important. If a callback tries to kill the transfer (e.g. abort()),+ // this will cause the teardown to be deferred so that it doesn't get+ // freed from under us.+ transfer.state = .completing;++ const req = &transfer.req;+ if (transfer._from_cache) {+ req.notification.dispatch(+ .http_request_served_from_cache,+ &.{ .transfer = transfer },+ );}- return null;+ // The current design is that a Transfer will get one and only one+ // deliver(). So at the end of deliver() we can deinit. That also means+ // we can safely iterate transfer._events without risking a mutation to+ // it.+ //+ // When WebSockets and/or buffering is added this will no longer be true+ // and one transfer will be potentially get multiple deliver() calls.+ // When that happens, we need to adjust two things:+ // 1 - It might not be safe to iterate transfer._events like this,+ // something could append to it, making it invalid+ // 2 - Only terminal states (done, err, abort) should call deinit++ for (transfer._events.items) |event| {+ if (transfer.state == .aborted) {+ // the state can become aborted as events are processed, check+ // it on every iteration.+ break;+ }+ switch (event) {+ .start => {+ if (req.start_callback) |cb| {+ cb(transfer) catch |err| {+ return transfer.failDelivery(err);+ };+ }+ },+ .header => {+ if (transfer._notify_cdp) {+ req.notification.dispatch(.http_response_header_done, &.{+ .transfer = transfer,+ });+ }+ const result = req.header_callback(transfer) catch |err| {+ return transfer.failDelivery(err);+ };+ if (result == .abort) {+ return transfer.failDelivery(error.Abort);+ }+ },+ .data => |chunk| {+ if (transfer._notify_cdp) {+ req.notification.dispatch(.http_response_data, &.{+ .data = chunk,+ .transfer = transfer,+ });+ }+ req.data_callback(transfer, chunk) catch |err| {+ return transfer.failDelivery(err);+ };+ },+ .done => {+ if (transfer._notify_cdp) {+ req.notification.dispatch(.http_request_done, &.{+ .transfer = transfer,+ .content_length = transfer._cdp_content_length,+ });+ }+ req.done_callback(req.ctx) catch |err| {+ return transfer.failDelivery(err);+ };+ },+ .err => |err| {+ transfer.requestFailed(err);+ break;+ },+ }+ }+ transfer.deinit();}+};- // Response-state owned by this transfer's currently-in-flight response.- // Reset on every retry (auth retry, redirect) via Transfer.reset — only- // the cross-retry counters (_auth_challenge, _redirect_count) live on- // Transfer itself. `Transfer.Response` is the on-Transfer storage; the- // top-level `Client.Response` is the actual Response (which is a union, e.g.- // for a cached response)- const Response = struct {- header: ?ResponseHead = null,+// Response-state owned by this transfer's currently-in-flight response.+// Reset on every retry (auth retry, redirect) via Transfer.reset — only+// the cross-retry counters (_auth_challenge, _redirect_count) live on+// Transfer itself. Consumers read it through the accessors above+// (status / contentType / responseHeaderIterator / getContentLength).+const Response = struct {+ header: ?http.ResponseHead = null,- // total bytes received in the response, including the response status- // line, the headers, and the [encoded] body.- bytes_received: usize = 0,+ // Full response headers, materialized into the transfer arena at+ // completion (or set directly by cache / synthetic / fulfill).+ headers: []const http.Header = &.{},- // track if the header callbacks done have been called.- header_done_called: bool = false,+ // total bytes received in the response, including the response status+ // line, the headers, and the [encoded] body.+ bytes_received: usize = 0,- skip_body: bool = false,- first_data_received: bool = false,+ skip_body: bool = false,+ first_data_received: bool = false,- // Buffered response body. Filled by dataCallback, consumed in processMessages.- stream_buffer: std.ArrayList(u8) = .{},+ // Buffered response body. Filled by dataCallback, consumed in processMessages.+ stream_buffer: std.ArrayList(u8) = .{},- // Error captured in dataCallback to be reported in processMessages.- callback_error: ?anyerror = null,- };+ // Error captured in dataCallback to be reported in processMessages.+ callback_error: ?anyerror = null,};-pub fn continueTransfer(self: *Client, transfer: *Transfer) !void {- if (comptime IS_DEBUG) {- log.debug(.http, "continue transfer", .{ .intercepted = self.interception_layer.intercepted });- }-- transfer.unpark();- self.process(transfer) catch |err| {- if (transfer.state == .created) {- transfer.abort(err);- }- return err;- };-}--const Noop = struct {- fn headerCallback(_: Response) !HeaderResult {- return .proceed;- }- fn dataCallback(_: Response, _: []const u8) !void {}- fn doneCallback(_: *anyopaque) !void {}- fn errorCallback(_: *anyopaque, _: anyerror) void {}+// A single buffered response event. The payload of `data` has transfer-arena+// lifetime. `done`, and `err` are terminal — nothing follows them.+const Event = union(enum) {+ start,+ header,+ data: []const u8,+ done,+ err: anyerror,};-// An opaque-from-the-outside handle that Frame / WorkerGlobalScope embed-// to track the HTTP transfers + WebSockets they own.-pub const Owner = struct {- transfers: std.DoublyLinkedList = .{},- websockets: std.DoublyLinkedList = .{},+// What the http cache wants from a transfer's response.+const CacheIntent = union(enum) {+ none,- // The owning Frame's / WorkerGlobalScope's blob: registry,- blob_urls: ?*const std.StringHashMapUnmanaged(*Blob) = null,+ // Inspect the response at completion and store it if cacheable.+ store,- const WebSocket = @import("webapi/net/WebSocket.zig");- const Blob = @import("webapi/Blob.zig");+ // Conditional request in flight; on a 304 the stale entry is renewed+ // and served. Owns the entry (possibly an open file) until resolved.+ revalidate: Cache.CachedResponse,+};- pub fn addTransfer(self: *Owner, t: *Transfer) void {- self.transfers.append(&t.owner_node);- }+// Non-network URL schemes whose response is synthesized in-process rather than+// fetched, think blob data URLs.+const Synthetic = struct {+ const data_url = @import("../browser/data_url.zig");- pub fn removeTransfer(self: *Owner, t: *Transfer) void {- self.transfers.remove(&t.owner_node);+ fn isSynthetic(url: []const u8) bool {+ return std.mem.startsWith(u8, url, "data:") or std.mem.startsWith(u8, url, "blob:");}- pub fn addWS(self: *Owner, ws: *WebSocket) void {- self.websockets.append(&ws._owner_node);- }+ // Materialize the synthetic response onto the transfer and buffer it for+ // dispatch — delivered exactly like any other response.+ fn buffer(transfer: *Transfer) !void {+ const arena = transfer.arena;+ const url = transfer.req.url;- pub fn removeWS(self: *Owner, ws: *WebSocket) void {- self.websockets.remove(&ws._owner_node);+ var body: []const u8 = "";+ var content_type: []const u8 = "";++ if (std.mem.startsWith(u8, url, "data:")) {+ const parsed = try data_url.parse(arena, url);+ content_type = parsed.content_type;+ body = parsed.body;+ } else {+ // blob: — resolved against the owning frame's registry.+ const owner = transfer.owner orelse return error.BlobNotFound;+ const blob_urls = owner.blob_urls orelse return error.BlobNotFound;+ const blob = blob_urls.get(url) orelse return error.BlobNotFound;+ content_type = blob._mime;+ body = blob._slice;+ }++ const has_content_type = content_type.len > 0;+ transfer.setResponseHead(200, if (has_content_type) content_type else null);+ if (content_type.len > 0) {+ const h = try arena.alloc(http.Header, 1);+ h[0] = .{ .name = "content-type", .value = content_type };+ transfer.res.headers = h;+ }+ transfer._cdp_content_length = body.len;+ try transfer.bufferEvents(body);}};@@ -2434,28 +2791,33 @@ test "HttpClient: isSyncWaitInterrupt ignores ping and non-teardown CDP methods"}}-test "HttpClient: fulfillRequest survives a done_callback that tears down the owner" {- // Regression: Fetch.fulfillRequest runs the consumer's done_callback while- // the transfer is still parked for interception. A done_callback that runs- // JS which navigates / closes the page re-entrantly kills the transfer- // (abortOwner -> kill -> deinit). Previously the transfer was still- // `.parked` at that point, so the re-entrant teardown freed it- // synchronously and fulfillRequest's trailing deinit was a double-free +- // `intercepted` underflow (surfacing later as a Transfer.leaveIntercept- // assert at session teardown). fulfillRequest now moves the transfer to- // `.completing` first, so the teardown defers and there is exactly one free.+fn initTestClient(client: *Client, pool: *ArenaPool) void {+ client.* = undefined;+ client.allocator = testing.allocator;+ client.arena_pool = pool;+ client.transfers = .empty;+ client.blocking_requests = .empty;+ client.pending_queue = .{};+ client.dispatch_queue = .{};+ client.gated_queue = .{};+ client.dispatch_count = 0;+ client.performing = false;+ client.intercepted = 0;+ client.cache = null;+ client.serve_mode = false;+ client.obey_robots = false;+}++test "HttpClient: fulfillIntercepted survives a done_callback that tears down the owner" {+ // Regression: the fulfilled response's done_callback runs JS which+ // navigates / closes the page, re-entrantly killing the transfer+ // (abortOwner -> kill). deliver() holds the transfer in `.completing`,+ // so the teardown defers and there is exactly one free.var pool = ArenaPool.init(testing.allocator, .{});defer pool.deinit();var client: Client = undefined;- client.allocator = testing.allocator;- client.arena_pool = &pool;- client.transfers = .empty;- client.queue = .{};- client.next_tick_queue = .{};- client.next_tick_count = 0;- client.performing = false;- client.interception_layer = .{};+ initTestClient(&client, &pool);defer client.transfers.deinit(testing.allocator);var owner: Owner = .{};@@ -2485,11 +2847,11 @@ test "HttpClient: fulfillRequest survives a done_callback that tears down the ow.loader_id = 0,.method = .GET,.url = "http://example.com/",- .headers = .{ .headers = null },.cookie_jar = null,.cookie_origin = "",.resource_type = .document,.notification = undefined,+ .shutdown_callback = noopShutdown,.ctx = &ctx,.done_callback = Ctx.doneCallback,},@@ -2502,64 +2864,48 @@ test "HttpClient: fulfillRequest survives a done_callback that tears down the owowner.addTransfer(transfer);transfer.owner = &owner;- // Mirror InterceptionLayer.request committing the transfer to CDP.+ // Mirror the interception gate committing the transfer to CDP.transfer.park(.intercept_request);- client.interception_layer.intercepted += 1;+ client.intercepted += 1;- try client.interception_layer.fulfillRequest(transfer, 200, &.{}, "hello");+ try client.fulfillIntercepted(transfer, 200, &.{}, "hello");+ // The response is buffered, not delivered from the CDP command.+ try testing.expectEqual(false, ctx.done_called);+ try testing.expectEqual(1, client.dispatch_count);++ client.dispatchCompleted();try testing.expect(ctx.done_called);// The transfer was freed exactly once: counter back to 0, dropped from the// id index and the owner list. A double-free would have underflowed// `intercepted` (or tripped the leaveIntercept assert).- try testing.expectEqual(0, client.interception_layer.intercepted);+ try testing.expectEqual(0, client.intercepted);+ try testing.expectEqual(0, client.dispatch_count);try testing.expectEqual(0, client.transfers.count());try testing.expectEqual(null, owner.transfers.first);}-test "HttpClient: fulfillRequest follows a 3xx redirect" {+test "HttpClient: fulfillIntercepted follows a 3xx redirect" {// Regression for #2828: a CDP Fetch.fulfillRequest with a 3xx status + a// Location header must be followed like a real network redirect (re-issued- // down the chain to the resolved target), not delivered as a final response.+ // through the pipeline to the resolved target), not delivered as a final+ // response.var pool = ArenaPool.init(testing.allocator, .{});defer pool.deinit();- // Only network.config is read (httpMaxRedirects, which ignores its config),- // so a pointer to an otherwise-undefined Network is safe here.+ // Only network.config (httpMaxRedirects, which ignores its config) and+ // network.cache are read on this path.var net: Network = undefined;+ net.cache = null;var client: Client = undefined;- client.allocator = testing.allocator;- client.arena_pool = &pool;+ initTestClient(&client, &pool);client.network = &net;- client.transfers = .empty;- client.queue = .{};- client.next_tick_queue = .{};- client.next_tick_count = 0;- client.performing = false;- client.interception_layer = .{};+ // Forces process() to queue the re-issued request instead of asking the+ // (undefined) Network for a connection — the queue IS the capture.+ client.performing = true;defer client.transfers.deinit(testing.allocator);- // Capturing stub for interception_layer.next: records the re-issued request- // and returns without committing (transfer stays .created; we clean up).- const Captor = struct {- captured: bool = false,- url: []const u8 = "",- method: Method = undefined,-- fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {- const self: *@This() = @ptrCast(@alignCast(ptr));- self.captured = true;- self.url = transfer.req.url;- self.method = transfer.req.method;- }- };- var captor = Captor{};- client.interception_layer.next = .{- .ptr = &captor,- .vtable = &.{ .request = Captor.request },- };-// 302 with a relative Location: rewrite to GET, drop body, resolve target.{const arena = try pool.acquire(.small, "test");@@ -2573,11 +2919,11 @@ test "HttpClient: fulfillRequest follows a 3xx redirect" {.method = .POST,.url = "http://example.com/start",.body = "payload",- .headers = .{ .headers = null },.cookie_jar = null,.cookie_origin = "",.resource_type = .document,.notification = undefined,+ .shutdown_callback = noopShutdown,.ctx = undefined,},.client = &client,@@ -2587,24 +2933,24 @@ test "HttpClient: fulfillRequest follows a 3xx redirect" {try client.transfers.putNoClobber(testing.allocator, transfer.id, transfer);transfer.park(.intercept_request);- client.interception_layer.intercepted += 1;+ client.intercepted += 1;- try client.interception_layer.fulfillRequest(transfer, 302, &.{+ try client.fulfillIntercepted(transfer, 302, &.{.{ .name = "Location", .value = "/end" },}, null);- try testing.expect(captor.captured);- try testing.expectEqual("http://example.com/end", captor.url);- try testing.expectEqual(.GET, captor.method);+ // Re-issued (queued for the network), not delivered.+ try testing.expectEqual(true, transfer.state == .queued);+ try testing.expectEqual("http://example.com/end", transfer.req.url);+ try testing.expectEqual(.GET, transfer.req.method);try testing.expectEqual(null, transfer.req.body);- // Unparked exactly once; transfer is still alive (re-issued, not deinited).- try testing.expectEqual(0, client.interception_layer.intercepted);+ // Unparked exactly once; transfer is still alive.+ try testing.expectEqual(0, client.intercepted);try testing.expectEqual(1, client.transfers.count());transfer.deinit();}// 307 with an absolute Location: keep method and body.- captor = .{};{const arena = try pool.acquire(.small, "test");const transfer = try arena.create(Transfer);@@ -2617,11 +2963,11 @@ test "HttpClient: fulfillRequest follows a 3xx redirect" {.method = .POST,.url = "http://example.com/start",.body = "payload",- .headers = .{ .headers = null },.cookie_jar = null,.cookie_origin = "",.resource_type = .document,.notification = undefined,+ .shutdown_callback = noopShutdown,.ctx = undefined,},.client = &client,@@ -2631,40 +2977,41 @@ test "HttpClient: fulfillRequest follows a 3xx redirect" {try client.transfers.putNoClobber(testing.allocator, transfer.id, transfer);transfer.park(.intercept_request);- client.interception_layer.intercepted += 1;+ client.intercepted += 1;- try client.interception_layer.fulfillRequest(transfer, 307, &.{+ try client.fulfillIntercepted(transfer, 307, &.{.{ .name = "location", .value = "http://example.com/other" },}, null);- try testing.expect(captor.captured);- try testing.expectEqual("http://example.com/other", captor.url);- try testing.expectEqual(.POST, captor.method);+ try testing.expectEqual(true, transfer.state == .queued);+ try testing.expectEqual("http://example.com/other", transfer.req.url);+ try testing.expectEqual(.POST, transfer.req.method);try testing.expectEqual("payload", transfer.req.body.?);- try testing.expectEqual(0, client.interception_layer.intercepted);+ try testing.expectEqual(0, client.intercepted);transfer.deinit();}}-test "HttpClient: fulfillRequest delivers a 3xx without a Location as the response" {+test "HttpClient: fulfillIntercepted delivers a 3xx without a Location as the response" {// A redirect status with no Location header is not a redirect: the body is// delivered as the final response (matching the real-network path).var pool = ArenaPool.init(testing.allocator, .{});defer pool.deinit();var client: Client = undefined;- client.allocator = testing.allocator;- client.arena_pool = &pool;- client.transfers = .empty;- client.queue = .{};- client.next_tick_queue = .{};- client.next_tick_count = 0;- client.performing = false;- client.interception_layer = .{};+ initTestClient(&client, &pool);defer client.transfers.deinit(testing.allocator);const Ctx = struct {done_called: bool = false,+ status: ?u16 = null,++ fn headerCallback(transfer: *Transfer) !Transfer.HeaderResult {+ const self: *@This() = @ptrCast(@alignCast(transfer.req.ctx));+ self.status = transfer.responseStatus();+ return .proceed;+ }+fn doneCallback(ctx: *anyopaque) !void {const self: *@This() = @ptrCast(@alignCast(ctx));self.done_called = true;@@ -2682,12 +3029,13 @@ test "HttpClient: fulfillRequest delivers a 3xx without a Location as the respon.loader_id = 0,.method = .GET,.url = "http://example.com/",- .headers = .{ .headers = null },.cookie_jar = null,.cookie_origin = "",.resource_type = .document,.notification = undefined,+ .shutdown_callback = noopShutdown,.ctx = &ctx,+ .header_callback = Ctx.headerCallback,.done_callback = Ctx.doneCallback,},.client = &client,@@ -2697,13 +3045,15 @@ test "HttpClient: fulfillRequest delivers a 3xx without a Location as the respontry client.transfers.putNoClobber(testing.allocator, transfer.id, transfer);transfer.park(.intercept_request);- client.interception_layer.intercepted += 1;+ client.intercepted += 1;- try client.interception_layer.fulfillRequest(transfer, 302, &.{}, "body");+ try client.fulfillIntercepted(transfer, 302, &.{}, "body");+ client.dispatchCompleted();// Delivered (done_callback ran) and freed exactly once.try testing.expect(ctx.done_called);- try testing.expectEqual(0, client.interception_layer.intercepted);+ try testing.expectEqual(302, ctx.status.?);+ try testing.expectEqual(0, client.intercepted);try testing.expectEqual(0, client.transfers.count());}@@ -2716,14 +3066,7 @@ test "HttpClient: abortParked survives an error_callback that tears down the owndefer pool.deinit();var client: Client = undefined;- client.allocator = testing.allocator;- client.arena_pool = &pool;- client.transfers = .empty;- client.queue = .{};- client.next_tick_queue = .{};- client.next_tick_count = 0;- client.performing = false;- client.interception_layer = .{};+ initTestClient(&client, &pool);defer client.transfers.deinit(testing.allocator);var owner: Owner = .{};@@ -2751,11 +3094,11 @@ test "HttpClient: abortParked survives an error_callback that tears down the own.loader_id = 0,.method = .GET,.url = "http://example.com/",- .headers = .{ .headers = null },.cookie_jar = null,.cookie_origin = "",.resource_type = .document,.notification = undefined,+ .shutdown_callback = noopShutdown,.ctx = &ctx,.error_callback = Ctx.errorCallback,},@@ -2769,12 +3112,12 @@ test "HttpClient: abortParked survives an error_callback that tears down the owntransfer.owner = &owner;transfer.park(.intercept_request);- client.interception_layer.intercepted += 1;+ client.intercepted += 1;transfer.abortParked(error.Abort);try testing.expect(ctx.err_called);- try testing.expectEqual(0, client.interception_layer.intercepted);+ try testing.expectEqual(0, client.intercepted);try testing.expectEqual(0, client.transfers.count());try testing.expectEqual(null, owner.transfers.first);}diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zignew file mode 100644index 000000000..8cfa11162--- /dev/null+++ b/src/network/RobotsGate.zig@@ -0,0 +1,232 @@+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)+//+// Francis Bouvier <[email protected]>+// Pierre Tachoire <[email protected]>+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++// robots.txt gate for the HttpClient request pipeline. Answers allow/deny+// from the robot store; on a store miss it parks the transfer, coalesces+// concurrent requests for the same robots.txt behind a single internal+// fetch, and resumes (or fails) the parked transfers when it resolves.++const std = @import("std");+const lp = @import("lightpanda");++const URL = @import("../browser/URL.zig");++const Robots = @import("Robots.zig");+const Network = @import("Network.zig");+const Transfer = @import("HttpClient.zig").Transfer;++const log = lp.log;+const Allocator = std.mem.Allocator;++const RobotsGate = @This();++network: *Network,+allocator: Allocator,+pending: std.StringHashMapUnmanaged(std.ArrayList(*Transfer)) = .empty,++pub const Result = enum { allowed, blocked, pending };++pub fn deinit(self: *RobotsGate) void {+ var it = self.pending.iterator();+ while (it.next()) |entry| {+ entry.value_ptr.deinit(self.allocator);+ }+ self.pending.deinit(self.allocator);+}++pub fn check(self: *RobotsGate, transfer: *Transfer) !Result {+ const url = transfer.req.url;+ const robots_url = try URL.getRobotsUrl(transfer.arena, url);++ if (self.network.robot_store.get(robots_url)) |robot_entry| {+ switch (robot_entry) {+ .absent => return .allowed,+ .present => |robots| {+ if (robots.isAllowed(URL.getPathname(url))) {+ return .allowed;+ }+ log.warn(.http, "blocked by robots", .{ .url = url });+ return .blocked;+ },+ }+ unreachable;+ }++ try self.fetchThenResume(robots_url, transfer);+ return .pending;+}++fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Transfer) !void {+ const entry = try self.pending.getOrPut(self.allocator, robots_url);++ if (entry.found_existing == false) {+ // A fetch for this robots.txt is already in flight, queue behind it.+ try entry.value_ptr.append(self.allocator, transfer);+ transfer.park(.robots);+ return;+ }++ errdefer _ = self.pending.remove(robots_url);+ entry.value_ptr.* = .empty;++ try entry.value_ptr.append(self.allocator, transfer);+ transfer.park(.robots);+ errdefer {+ entry.value_ptr.deinit(self.allocator);+ transfer.unpark();+ }++ const robots_ctx = try transfer.arena.create(RobotsContext);+ robots_ctx.* = .{+ .gate = self,+ .buffer = .empty,+ .arena = transfer.arena,+ .robots_url = robots_url,+ };++ log.debug(.browser, "fetching robots.txt", .{ .robots_url = robots_url });++ // Only the parent's frame/loader ids (CDP correlation) and notification+ // carry over — no cookies, credentials, headers, or timeout.+ try transfer.client.request(.{+ .url = robots_url,+ .method = .GET,+ .internal = true,+ .resource_type = .fetch,+ .frame_id = transfer.req.frame_id,+ .loader_id = transfer.req.loader_id,+ .notification = transfer.req.notification,+ .cookie_jar = null,+ .cookie_origin = robots_url,+ .ctx = robots_ctx,+ .header_callback = RobotsContext.headerCallback,+ .data_callback = RobotsContext.dataCallback,+ .done_callback = RobotsContext.doneCallback,+ .error_callback = RobotsContext.errorCallback,+ .shutdown_callback = RobotsContext.shutdownCallback,+ }, transfer.owner);+}++fn flushPending(self: *RobotsGate, robots_url: [:0]const u8, allowed: bool) void {+ var queued = self.pending.fetchRemove(robots_url) orelse return;+ defer queued.value.deinit(self.allocator);++ for (queued.value.items) |transfer| {+ transfer.unpark();+ if (!allowed) {+ log.warn(.http, "blocked by robots", .{ .url = transfer.req.url });+ transfer.failAsync(error.RobotsBlocked);+ continue;+ }+ // Hand back to the pipeline; the robots gate is the last step+ // before the network. If it fails before committing, clean up here.+ transfer.client.resumeAfterRobots(transfer) catch |e| {+ if (transfer.state == .created) {+ transfer.abort(e);+ }+ };+ }+}++// shutdown_callback is only called on owner shutdown. And if this robot's fetch+// is being shutdown, than any transfers waiting for it will be shutdown too.+fn flushPendingShutdown(self: *RobotsGate, robots_url: [:0]const u8) void {+ var pending = self.pending.fetchRemove(robots_url) orelse return;+ pending.value.deinit(self.allocator);+}++const RobotsContext = struct {+ gate: *RobotsGate,+ arena: Allocator,+ robots_url: [:0]const u8,+ buffer: std.ArrayList(u8),+ status: u16 = 0,++ fn headerCallback(transfer: *Transfer) anyerror!Transfer.HeaderResult {+ const self: *RobotsContext = @ptrCast(@alignCast(transfer.req.ctx));+ if (transfer.res.header) |hdr| {+ log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = self.robots_url });+ self.status = hdr.status;+ }+ if (transfer.getContentLength()) |cl| {+ try self.buffer.ensureTotalCapacity(self.arena, cl);+ }+ return .proceed;+ }++ fn dataCallback(transfer: *Transfer, data: []const u8) anyerror!void {+ const self: *RobotsContext = @ptrCast(@alignCast(transfer.req.ctx));+ try self.buffer.appendSlice(self.arena, data);+ }++ fn doneCallback(ctx_ptr: *anyopaque) anyerror!void {+ const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));+ const gate = self.gate;+ const robots_url = self.robots_url;++ var allowed = true;+ const network = gate.network;++ switch (self.status) {+ 200 => {+ if (self.buffer.items.len > 0) {+ const robots: ?Robots = network.robot_store.robotsFromBytes(+ network.config.http_headers.user_agent,+ self.buffer.items,+ ) catch blk: {+ log.warn(.browser, "failed to parse robots", .{ .robots_url = robots_url });+ try network.robot_store.putAbsent(robots_url);+ break :blk null;+ };+ if (robots) |r| {+ try network.robot_store.put(robots_url, r);+ const path = URL.getPathname(gate.pending.get(robots_url).?.items[0].req.url);+ allowed = r.isAllowed(path);+ }+ }+ },+ 404 => {+ log.debug(.http, "robots not found", .{ .url = robots_url });+ try network.robot_store.putAbsent(robots_url);+ },+ else => {+ log.debug(.http, "unexpected status on robots", .{+ .url = robots_url,+ .status = self.status,+ });+ try network.robot_store.putAbsent(robots_url);+ },+ }++ gate.flushPending(robots_url, allowed);+ }++ fn errorCallback(ctx_ptr: *anyopaque, err: anyerror) void {+ const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));++ log.warn(.http, "robots fetch failed", .{ .err = err });+ self.gate.flushPending(self.robots_url, true);+ }++ fn shutdownCallback(ctx_ptr: *anyopaque) void {+ const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));++ log.debug(.http, "robots fetch shutdown", .{});+ self.gate.flushPendingShutdown(self.robots_url);+ }+};diff --git a/src/network/http.zig b/src/network/http.zigindex 8baeaf24a..b05206efa 100644--- a/src/network/http.zig+++ b/src/network/http.zig@@ -308,11 +308,6 @@ pub const ResponseHead = struct {redirect_count: u32,_content_type_len: usize = 0,_content_type: [MAX_CONTENT_TYPE_LEN]u8 = undefined,- // this is normally an empty list, but if the response is being injected- // than it'll be populated. It isn't meant to be used directly, but should- // be used through the transfer.responseHeaderIterator() which abstracts- // whether the headers are from a live curl easy handle, or injected.- _injected_headers: []const Header = &.{},pub fn contentType(self: *ResponseHead) ?[]u8 {if (self._content_type_len == 0) {@@ -364,7 +359,7 @@ pub const Connection = struct {pub const Transport = union(enum) {none, // used for cases that manage their own connection, e.g. telemetry- http: *@import("../browser/HttpClient.zig").Transfer,+ http: *@import("HttpClient.zig").Transfer,websocket: *@import("../browser/webapi/net/WebSocket.zig"),};diff --git a/src/network/layer/CacheLayer.zig b/src/network/layer/CacheLayer.zigdeleted file mode 100644index a65a3570e..000000000--- a/src/network/layer/CacheLayer.zig+++ /dev/null@@ -1,386 +0,0 @@-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)-//-// Francis Bouvier <[email protected]>-// Pierre Tachoire <[email protected]>-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--const std = @import("std");-const lp = @import("lightpanda");--const Layer = @import("../../browser/HttpClient.zig").Layer;-const Transfer = @import("../../browser/HttpClient.zig").Transfer;-const Response = @import("../../browser/HttpClient.zig").Response;--const Cache = @import("../cache/Cache.zig");-const CachedMetadata = @import("../cache/Cache.zig").CachedMetadata;-const CachedResponse = @import("../cache/Cache.zig").CachedResponse;--const HeaderResult = @import("../../browser/HttpClient.zig").HeaderResult;-const Forward = @import("Forward.zig");--const log = lp.log;-const IS_DEBUG = @import("builtin").mode == .Debug;--const CacheLayer = @This();--next: Layer = undefined,-disabled: bool = false,--pub fn layer(self: *CacheLayer) Layer {- return .{- .ptr = self,- .vtable = &.{- .request = request,- },- };-}--fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {- const self: *CacheLayer = @ptrCast(@alignCast(ptr));- const req = &transfer.req;-- if (self.disabled or req.method != .GET) {- return self.next.request(transfer);- }-- const arena = transfer.arena;-- var iter = req.headers.iterator();- const req_header_list = try iter.collect(arena);-- const cached = transfer.client.network.cache.?.get(arena, .{- .url = req.url,- .timestamp = std.time.timestamp(),- .request_headers = req_header_list.items,- }) orelse {- // Cache miss: install wrappers so we can inspect the response and decide- // whether to write the body into the cache when it's done.- try installCacheContext(arena, transfer, null);- return self.next.request(transfer);- };-- if (!cached.expired) {- const ctx = try arena.create(CachedResponse);- ctx.* = cached;-- try transfer.client.runNextTick(transfer, ctx, .{- .run = struct {- fn run(t: *Transfer, ctx_ptr: ?*anyopaque) void {- defer t.deinit();-- const c: *CachedResponse = @ptrCast(@alignCast(ctx_ptr.?));- serveFromCache(t, c) catch |err| {- t.req.error_callback(t.req.ctx, err);- };- }- }.run,- .abort = struct {- fn abort(ctx_ptr: ?*anyopaque) void {- const c: *CachedResponse = @ptrCast(@alignCast(ctx_ptr.?));- switch (c.data) {- .buffer => |_| {},- .file => |f| f.file.close(),- }- }- }.abort,- });- return;- }-- if (cached.metadata.hasValidators()) {- if (cached.metadata.etag) |etag| {- log.debug(.cache, "revalidate with etag", .{ .url = req.url, .etag = etag });- const header_value = try std.fmt.allocPrintSentinel(arena, "If-None-Match: {s}", .{etag}, 0);- try req.headers.add(header_value);- }-- if (cached.metadata.last_modified) |lm| {- log.debug(.cache, "revalidate with last-modified", .{ .url = req.url, .last_modified = lm });- const header_value = try std.fmt.allocPrintSentinel(arena, "If-Modified-Since: {s}", .{lm}, 0);- try req.headers.add(header_value);- }-- try installCacheContext(arena, transfer, cached);- } else {- defer cached.data.deinit();- // If it is expired w/o validators, evict from Cache.- transfer.client.network.cache.?.evict(req.url);- try installCacheContext(arena, transfer, null);- }-- return self.next.request(transfer);-}--fn installCacheContext(- arena: std.mem.Allocator,- transfer: *Transfer,- stale_entry: ?CachedResponse,-) !void {- const req = &transfer.req;- const ctx = try arena.create(CacheContext);- ctx.* = .{- .arena = arena,- .transfer = transfer,- .forward = Forward.capture(req),- .req_url = req.url,- .req_headers = req.headers,- .stale_entry = stale_entry,- };-- req.ctx = ctx;- req.header_callback = CacheContext.headerCallback;- req.data_callback = CacheContext.dataCallback;- req.done_callback = CacheContext.doneCallback;- req.error_callback = CacheContext.errorCallback;-- if (ctx.forward.start != null) req.start_callback = CacheContext.startCallback;- if (ctx.forward.shutdown != null) req.shutdown_callback = CacheContext.shutdownCallback;-}--fn forwardFromCache(- transfer: *Transfer,- forward: *Forward,- cached: *const CachedResponse,-) !void {- transfer.req.notification.dispatch(- .http_request_served_from_cache,- &.{ .transfer = transfer },- );-- const req = &transfer.req;- const response = Response.fromCached(req.ctx, cached);- defer cached.data.deinit();-- try forward.forwardStart(response);- const result = try forward.forwardHeader(response);- if (result == .abort) return error.Abort;-- switch (cached.data) {- .buffer => |data| {- if (data.len > 0) try forward.forwardData(response, data);- },- .file => |f| {- const file = f.file;- var buf: [1024]u8 = undefined;- var file_reader = file.reader(&buf);- try file_reader.seekTo(f.offset);- const reader = &file_reader.interface;- var read_buf: [1024]u8 = undefined;- var remaining = f.len;- while (remaining > 0) {- const read_len = @min(read_buf.len, remaining);- const n = try reader.readSliceShort(read_buf[0..read_len]);- if (n == 0) break;- remaining -= n;- try forward.forwardData(response, read_buf[0..n]);- }- },- }-- try forward.forwardDone();-}--fn serveFromCache(transfer: *Transfer, cached: *const CachedResponse) !void {- transfer.req.notification.dispatch(- .http_request_served_from_cache,- &.{ .transfer = transfer },- );-- const req = &transfer.req;- const response = Response.fromCached(req.ctx, cached);- defer cached.data.deinit();-- if (req.start_callback) |cb| {- try cb(response);- }-- const result = try req.header_callback(response);- if (result == .abort) {- return error.Abort;- }-- switch (cached.data) {- .buffer => |data| {- if (data.len > 0) {- try req.data_callback(response, data);- }- },- .file => |f| {- const file = f.file;- var buf: [1024]u8 = undefined;- var file_reader = file.reader(&buf);- try file_reader.seekTo(f.offset);- const reader = &file_reader.interface;- var read_buf: [1024]u8 = undefined;- var remaining = f.len;- while (remaining > 0) {- const read_len = @min(read_buf.len, remaining);- const n = try reader.readSliceShort(read_buf[0..read_len]);- if (n == 0) break;- remaining -= n;- try req.data_callback(response, read_buf[0..n]);- }- },- }-- try req.done_callback(req.ctx);-}--const CacheContext = struct {- arena: std.mem.Allocator,- transfer: *Transfer,- forward: Forward,- req_url: [:0]const u8,- req_headers: @import("../http.zig").Headers,- pending_metadata: ?*CachedMetadata = null,- stale_entry: ?CachedResponse = null,-- fn startCallback(response: Response) anyerror!void {- const self: *CacheContext = @ptrCast(@alignCast(response.ctx));- return self.forward.forwardStart(response);- }-- fn dataCallback(response: Response, chunk: []const u8) anyerror!void {- const self: *CacheContext = @ptrCast(@alignCast(response.ctx));- return self.forward.forwardData(response, chunk);- }-- fn headerCallback(response: Response) anyerror!HeaderResult {- const self: *CacheContext = @ptrCast(@alignCast(response.ctx));-- // For non-transfer responses (fulfilled by interception, or future- // cached-while-cached cases), there's nothing to inspect for caching- // decisions — just forward.- const transfer = switch (response.inner) {- .transfer => |t| t,- else => return self.forward.forwardHeader(response),- };-- const arena = self.arena;- const conn = transfer._conn.?;- var rh = &transfer.res.header.?;-- if (self.stale_entry != null and rh.status == 304) {- const stale = self.stale_entry.?;- self.stale_entry = null;-- var iter = response.headerIterator();- const headers = try iter.collect(arena);-- transfer.client.network.cache.?.renew(- arena,- .{- .url = self.req_url,- .timestamp = std.time.timestamp(),- .headers = headers.items,- },- ) catch |err| {- log.warn(.cache, "renew failed", .{ .err = err });- };-- try forwardFromCache(transfer, &self.forward, &stale);- return .handled;- }-- if (self.stale_entry) |stale| {- stale.data.deinit();- self.stale_entry = null;- }-- const vary = if (conn.getResponseHeader("vary", 0)) |h| h.value else null;- const maybe_cm = try Cache.tryCache(- arena,- std.time.timestamp(),- self.req_url,- rh.status,- rh.contentType(),- if (conn.getResponseHeader("cache-control", 0)) |h| h.value else null,- vary,- if (conn.getResponseHeader("age", 0)) |h| h.value else null,- if (conn.getResponseHeader("etag", 0)) |h| h.value else null,- if (conn.getResponseHeader("last-modified", 0)) |h| h.value else null,- conn.getResponseHeader("set-cookie", 0) != null,- conn.getResponseHeader("authorization", 0) != null,- );-- if (maybe_cm) |cm| {- var iter = transfer.responseHeaderIterator();- var header_list = try iter.collect(arena);- const end_of_response = header_list.items.len;-- if (vary) |vary_str| {- var req_it = self.req_headers.iterator();- while (req_it.next()) |hdr| {- var vary_iter = std.mem.splitScalar(u8, vary_str, ',');- while (vary_iter.next()) |part| {- const name = std.mem.trim(u8, part, &std.ascii.whitespace);- if (std.ascii.eqlIgnoreCase(hdr.name, name)) {- try header_list.append(arena, .{- .name = try arena.dupe(u8, hdr.name),- .value = try arena.dupe(u8, hdr.value),- });- }- }- }- }-- const metadata = try arena.create(CachedMetadata);- metadata.* = cm;- metadata.headers = header_list.items[0..end_of_response];- metadata.vary_headers = header_list.items[end_of_response..];- self.pending_metadata = metadata;- }-- return self.forward.forwardHeader(response);- }-- fn doneCallback(ctx: *anyopaque) anyerror!void {- const self: *CacheContext = @ptrCast(@alignCast(ctx));- const transfer = self.transfer;-- if (self.pending_metadata) |metadata| {- const cache = &transfer.client.network.cache.?;-- if (comptime IS_DEBUG) {- log.debug(.browser, "http cache", .{ .key = self.req_url, .metadata = metadata });- }- cache.put(metadata.*, transfer.res.stream_buffer.items) catch |err| {- log.warn(.http, "cache put failed", .{ .err = err });- };- }-- return self.forward.forwardDone();- }-- fn shutdownCallback(ctx: *anyopaque) void {- const self: *CacheContext = @ptrCast(@alignCast(ctx));- if (self.stale_entry) |entry| {- self.stale_entry = null;- entry.data.deinit();- }-- self.forward.forwardShutdown();- }-- fn errorCallback(ctx: *anyopaque, e: anyerror) void {- const self: *CacheContext = @ptrCast(@alignCast(ctx));- if (self.stale_entry) |entry| {- self.stale_entry = null;- entry.data.deinit();- }-- self.forward.forwardErr(e);- }-};diff --git a/src/network/layer/DeferringLayer.zig b/src/network/layer/DeferringLayer.zigdeleted file mode 100644index 0cc4c4717..000000000--- a/src/network/layer/DeferringLayer.zig+++ /dev/null@@ -1,361 +0,0 @@-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)-//-// Francis Bouvier <[email protected]>-// Pierre Tachoire <[email protected]>-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--const std = @import("std");-const lp = @import("lightpanda");-const log = lp.log;--const Network = @import("../Network.zig");-const Transfer = @import("../../browser/HttpClient.zig").Transfer;-const Response = @import("../../browser/HttpClient.zig").Response;-const Layer = @import("../../browser/HttpClient.zig").Layer;-const StableResponse = @import("../../browser/HttpClient.zig").StableResponse;-const Forward = @import("Forward.zig");-const HeaderResult = @import("../../browser/HttpClient.zig").HeaderResult;--const DeferringLayer = @This();--allocator: std.mem.Allocator,-network: *Network,-next: Layer = undefined,-active: std.DoublyLinkedList = .{},--pub fn layer(self: *DeferringLayer) Layer {- return .{- .ptr = self,- .vtable = &.{ .request = request },- };-}--pub fn deinit(self: *DeferringLayer) void {- self.drainAll();-}--fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {- const self: *DeferringLayer = @ptrCast(@alignCast(ptr));-- if (transfer.req.internal) {- return self.next.request(transfer);- }-- const arena = try self.network.app.arena_pool.acquire(.small, "DeferringContext");- errdefer self.network.app.arena_pool.release(arena);-- // this might outlive the transfer, we need to dupe eveyrthing we'll need to use- const ctx = try arena.create(DeferredContext);- ctx.* = .{- .arena = arena,- .layer = self,- .transfer = transfer,- .frame_id = transfer.req.frame_id,- .url = try arena.dupeZ(u8, transfer.req.url),- .forward = Forward.capture(&transfer.req),- };-- self.active.append(&ctx.node);- errdefer self.active.remove(&ctx.node);-- transfer.req.ctx = ctx;- transfer.req.start_callback = if (ctx.forward.start != null) DeferredContext.startCallback else null;- transfer.req.header_callback = DeferredContext.headerCallback;- transfer.req.data_callback = DeferredContext.dataCallback;- transfer.req.done_callback = DeferredContext.doneCallback;- transfer.req.error_callback = DeferredContext.errorCallback;- transfer.req.shutdown_callback = if (ctx.forward.shutdown != null) DeferredContext.shutdownCallback else null;-- return self.next.request(transfer);-}--pub fn flushFrame(self: *DeferringLayer, frame_id: u32) void {- // DeferredContext.fire() can re-enter flushFrame, so we'll capture- // ready items in this list, so that a reentrant flushFrame doesn't mutate- // self.active while we're iterating.- var ready: std.DoublyLinkedList = .{};-- var node = self.active.first;- while (node) |n| {- node = n.next;- const ctx: *DeferredContext = @fieldParentPtr("node", n);- if (!ctx.deferring) {- continue;- }-- // captured frame_id, not ctx.transfer: the transfer may be freed.- if (ctx.frame_id != frame_id) {- continue;- }-- if (ctx.terminal) {- self.active.remove(n);- ready.append(n);- } else {- ctx.firePartial();- ctx.deferring = false;- }- }-- // ready is local, ctx.fire() re-entering flushFrame can't invalidate it.- while (ready.popFirst()) |n| {- const ctx: *DeferredContext = @fieldParentPtr("node", n);- ctx.fire();- }-}--/// Drop orphaned deferred contexts for a frame that's going away. A `terminal`-/// context's transfer already completed while deferred, so it's been deinited-/// and unlinked from the owner — abortOwner can't reach it, yet it lingers in-/// `active` pointing at a forward target (the Fetch) whose arena page teardown-/// is about to free, and a later flushFrame would fire into it. Non-terminal-/// contexts still have a live transfer that cleans them up itself.-pub fn cancelFrame(self: *DeferringLayer, frame_id: u32) void {- var node = self.active.first;- while (node) |n| {- node = n.next;- const ctx: *DeferredContext = @fieldParentPtr("node", n);- if (ctx.frame_id != frame_id or !ctx.terminal) {- continue;- }- self.active.remove(n);- ctx.deinit();- }-}--pub fn drainAll(self: *DeferringLayer) void {- while (self.active.popFirst()) |node| {- const ctx: *DeferredContext = @fieldParentPtr("node", node);- ctx.deinit();- }-}--const DeferredContext = struct {- arena: std.mem.Allocator,- layer: *DeferringLayer,- transfer: *Transfer,- frame_id: u32,- url: [:0]const u8,- forward: Forward,- node: std.DoublyLinkedList.Node = .{},-- buffered: std.ArrayList(BufferedEvent) = .{},- done: bool = false,- deferring: bool = false,- terminal: bool = false,- stable_resp: ?StableResponse = null,-- const BufferedEvent = union(enum) {- start,- header,- data: []const u8,- done,- err: anyerror,- };-- fn deinit(self: *DeferredContext) void {- self.layer.network.app.arena_pool.release(self.arena);- }-- fn setStableResponse(self: *DeferredContext, response: Response) !void {- if (self.stable_resp == null) {- self.stable_resp = try Response.toStable(response, self.arena);- }- }-- fn shouldDefer(self: *DeferredContext) bool {- const req = self.transfer.req;- const blocking_id = self.transfer.client.blocking_requests.get(req.frame_id) orelse return false;- return self.transfer.id != blocking_id;- }-- fn startCallback(response: Response) anyerror!void {- const self: *DeferredContext = @ptrCast(@alignCast(response.ctx));-- if (!self.deferring and !self.shouldDefer()) {- return self.forward.forwardStart(response);- }-- log.debug(.http, "deferring start callback", .{ .url = self.url });- try self.setStableResponse(response);- self.deferring = true;- try self.buffered.append(self.arena, .start);- }-- fn headerCallback(response: Response) anyerror!HeaderResult {- const self: *DeferredContext = @ptrCast(@alignCast(response.ctx));-- if (!self.deferring and !self.shouldDefer()) {- return self.forward.forwardHeader(response);- }-- log.debug(.http, "deferring header callback", .{ .url = self.url });- try self.setStableResponse(response);- self.deferring = true;- try self.buffered.append(self.arena, .header);- return .proceed;- }-- fn dataCallback(response: Response, chunk: []const u8) anyerror!void {- const self: *DeferredContext = @ptrCast(@alignCast(response.ctx));-- if (!self.deferring and !self.shouldDefer()) {- return self.forward.forwardData(response, chunk);- }-- log.debug(.http, "deferring data callback", .{ .url = self.url });- try self.setStableResponse(response);- self.deferring = true;- try self.buffered.append(self.arena, .{ .data = try self.arena.dupe(u8, chunk) });- }-- fn doneCallback(ctx: *anyopaque) anyerror!void {- const self: *DeferredContext = @ptrCast(@alignCast(ctx));-- if (!self.deferring and !self.shouldDefer()) {- defer self.deinit();- self.done = true;- self.layer.active.remove(&self.node);- return self.forward.forwardDone();- }-- log.debug(.http, "deferring done callback", .{ .url = self.url });- self.deferring = true;- self.terminal = true;- try self.buffered.append(self.arena, .done);- }-- fn errorCallback(ctx: *anyopaque, err: anyerror) void {- const self: *DeferredContext = @ptrCast(@alignCast(ctx));-- if (!self.deferring and !self.shouldDefer()) {- defer self.deinit();- self.done = true;- self.layer.active.remove(&self.node);- self.forward.forwardErr(err);- return;- }-- log.debug(.http, "deferring error callback", .{ .url = self.url, .err = err });- self.deferring = true;- self.terminal = true;- self.buffered.append(self.arena, .{ .err = err }) catch {};- }-- fn shutdownCallback(ctx: *anyopaque) void {- const self: *DeferredContext = @ptrCast(@alignCast(ctx));- if (self.done) return;-- defer self.deinit();- self.done = true;- self.layer.active.remove(&self.node);-- log.debug(.http, "deferring shutdown callback", .{});- self.forward.forwardShutdown();- }-- fn fire(self: *DeferredContext) void {- defer self.deinit();-- for (self.buffered.items) |event| {- switch (event) {- .start => {- const stable_response = self.stable_resp orelse @panic("stable_resp must be set for start events");- const response = Response.fromStable(&stable_response);-- self.forward.forwardStart(response) catch |err| {- log.err(.http, "deferred start callback", .{ .err = err, .url = self.url });- self.forward.forwardErr(err);- return;- };- },- .header => {- const stable_response = self.stable_resp orelse @panic("stable_resp must be set for header events");- const response = Response.fromStable(&stable_response);-- const result = self.forward.forwardHeader(response) catch |err| {- log.err(.http, "deferred header callback", .{ .err = err, .url = self.url });- self.forward.forwardErr(err);- return;- };- if (result == .abort) {- self.forward.forwardErr(error.Abort);- return;- }- },- .data => |chunk| {- const stable_response = self.stable_resp orelse @panic("stable_resp must be set for data events");- const response = Response.fromStable(&stable_response);-- self.forward.forwardData(response, chunk) catch |err| {- log.err(.http, "deferred data callback", .{ .err = err, .url = self.url });- self.forward.forwardErr(err);- return;- };- },- .done => {- self.forward.forwardDone() catch |err| {- log.err(.http, "deferred done callback", .{ .err = err, .url = self.url });- self.forward.forwardErr(err);- };-- return;- },- .err => |err| {- self.forward.forwardErr(err);- return;- },- }- }- }-- fn firePartial(self: *DeferredContext) void {- const stable_response = self.stable_resp orelse @panic("stable_resp must be set for any of the partial fire events");- const response = Response.fromStable(&stable_response);-- for (self.buffered.items) |event| {- switch (event) {- .start => {- self.forward.forwardStart(response) catch |err| {- log.err(.http, "defer part start callback", .{ .err = err, .url = self.url });- self.forward.forwardErr(err);- return;- };- },- .header => {- const result = self.forward.forwardHeader(response) catch |err| {- log.err(.http, "defer part header callback", .{ .err = err, .url = self.url });- self.forward.forwardErr(err);- return;- };- if (result == .abort) {- self.forward.forwardErr(error.Abort);- return;- }- },- .data => |chunk| {- self.forward.forwardData(response, chunk) catch |err| {- log.err(.http, "defer part data callback", .{ .err = err, .url = self.url });- self.forward.forwardErr(err);- return;- };- },- .done, .err => @panic("firePartial cant fire terminal events"),- }- }-- self.buffered.clearRetainingCapacity();- }-};diff --git a/src/network/layer/Forward.zig b/src/network/layer/Forward.zigdeleted file mode 100644index 0f218f709..000000000--- a/src/network/layer/Forward.zig+++ /dev/null@@ -1,76 +0,0 @@-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)-//-// Francis Bouvier <[email protected]>-// Pierre Tachoire <[email protected]>-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--// A snapshot of the original ctx + callbacks from a Request, taken before a-// layer overwrites them with its own wrappers. The layer's wrapper callbacks-// call forwardX(...) to invoke the captured originals with the original ctx.-const Request = @import("../../browser/HttpClient.zig").Request;-const Response = @import("../../browser/HttpClient.zig").Response;-const HeaderResult = @import("../../browser/HttpClient.zig").HeaderResult;--const Forward = @This();--ctx: *anyopaque,-start: ?Request.StartCallback,-header: Request.HeaderCallback,-data: Request.DataCallback,-done: Request.DoneCallback,-err: Request.ErrorCallback,-shutdown: ?Request.ShutdownCallback,--pub fn capture(req: *const Request) Forward {- return .{- .ctx = req.ctx,- .start = req.start_callback,- .header = req.header_callback,- .data = req.data_callback,- .done = req.done_callback,- .err = req.error_callback,- .shutdown = req.shutdown_callback,- };-}--pub fn forwardStart(self: Forward, response: Response) anyerror!void {- var fwd = response;- fwd.ctx = self.ctx;- if (self.start) |cb| try cb(fwd);-}--pub fn forwardHeader(self: Forward, response: Response) anyerror!HeaderResult {- var fwd = response;- fwd.ctx = self.ctx;- return self.header(fwd);-}--pub fn forwardData(self: Forward, response: Response, chunk: []const u8) anyerror!void {- var fwd = response;- fwd.ctx = self.ctx;- return self.data(fwd, chunk);-}--pub fn forwardDone(self: Forward) anyerror!void {- return self.done(self.ctx);-}--pub fn forwardErr(self: Forward, e: anyerror) void {- self.err(self.ctx, e);-}--pub fn forwardShutdown(self: Forward) void {- if (self.shutdown) |cb| cb(self.ctx);-}diff --git a/src/network/layer/InterceptionLayer.zig b/src/network/layer/InterceptionLayer.zigdeleted file mode 100644index 785502606..000000000--- a/src/network/layer/InterceptionLayer.zig+++ /dev/null@@ -1,327 +0,0 @@-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)-//-// Francis Bouvier <[email protected]>-// Pierre Tachoire <[email protected]>-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--const std = @import("std");-const builtin = @import("builtin");-const lp = @import("lightpanda");-const log = lp.log;--const IS_DEBUG = builtin.mode == .Debug;--const http = @import("../http.zig");-const HttpClient = @import("../../browser/HttpClient.zig");-const Request = @import("../../browser/HttpClient.zig").Request;-const Transfer = @import("../../browser/HttpClient.zig").Transfer;-const Response = @import("../../browser/HttpClient.zig").Response;-const FulfilledResponse = @import("../../browser/HttpClient.zig").FulfilledResponse;-const Layer = @import("../../browser/HttpClient.zig").Layer;-const Forward = @import("Forward.zig");-const HeaderResult = @import("../../browser/HttpClient.zig").HeaderResult;--const InterceptionLayer = @This();--// Count of intercepted requests. The client doesn't track intercepted transfers-// on its own active counters: once intercepted, a transfer leaves the layer-// chain and waits for the interceptor (CDP) to call continue/abort/fulfill.-// We track them here so the network-idle / network-almost-idle CDP lifecycle-// events don't fire prematurely.-intercepted: usize = 0,--next: Layer = undefined,--pub fn layer(self: *InterceptionLayer) Layer {- return .{- .ptr = self,- .vtable = &.{ .request = request },- };-}--fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {- const self: *InterceptionLayer = @ptrCast(@alignCast(ptr));- const req = &transfer.req;-- const ctx = try transfer.arena.create(InterceptContext);- ctx.* = .{- .layer = self,- .transfer = transfer,- .forward = Forward.capture(req),- };-- // Install our wrappers on the transfer's request. The interceptor wants to- // observe every callback (start/header/data/done/err/shutdown) so it can- // mirror the Network.* CDP events.- req.ctx = ctx;- if (ctx.forward.start != null) req.start_callback = InterceptContext.startCallback;- req.header_callback = InterceptContext.headerCallback;- req.data_callback = InterceptContext.dataCallback;- req.done_callback = InterceptContext.doneCallback;- req.error_callback = InterceptContext.errorCallback;- if (ctx.forward.shutdown != null) req.shutdown_callback = InterceptContext.shutdownCallback;-- req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });-- var wait_for_interception = false;- req.notification.dispatch(.http_request_intercept, &.{- .transfer = transfer,- .wait_for_interception = &wait_for_interception,- });-- log.debug(.http, "interception check", .{- .wait_for_interception = wait_for_interception,- .intercepted = self.intercepted,- .url = req.url,- });-- if (!wait_for_interception) {- return self.next.request(transfer);- }-- // Paused: the CDP listener stashed `transfer` and will eventually call- // continueRequest / abortRequest / fulfillRequest. Until then, CDP owns- // the transfer's lifecycle. Park keeps the outer Client.request errdefer- // from tearing it down.- self.intercepted += 1;- transfer.park(.intercept_request);- if (comptime IS_DEBUG) {- log.debug(.http, "wait for interception", .{ .intercepted = self.intercepted });- }-}--pub const InterceptContext = struct {- layer: *InterceptionLayer,- transfer: *Transfer,- forward: Forward,- content_length: usize = 0,-- fn startCallback(response: Response) anyerror!void {- const self: *InterceptContext = @ptrCast(@alignCast(response.ctx));- log.debug(.http, "intercept start", .{ .url = self.transfer.req.url });- return self.forward.forwardStart(response);- }-- fn headerCallback(response: Response) anyerror!HeaderResult {- const self: *InterceptContext = @ptrCast(@alignCast(response.ctx));- log.debug(.http, "intercept header", .{- .url = self.transfer.req.url,- .status = response.status(),- .content_length = response.contentLength(),- });-- self.content_length = response.contentLength() orelse 0;-- self.transfer.req.notification.dispatch(.http_response_header_done, &.{- .transfer = self.transfer,- .response = &response,- });-- return self.forward.forwardHeader(response);- }-- fn dataCallback(response: Response, chunk: []const u8) anyerror!void {- const self: *InterceptContext = @ptrCast(@alignCast(response.ctx));- log.debug(.http, "intercept data", .{- .url = self.transfer.req.url,- .len = chunk.len,- });-- self.transfer.req.notification.dispatch(.http_response_data, &.{- .data = chunk,- .transfer = self.transfer,- });-- return self.forward.forwardData(response, chunk);- }-- fn doneCallback(ctx: *anyopaque) anyerror!void {- const self: *InterceptContext = @ptrCast(@alignCast(ctx));-- log.debug(.http, "intercept done", .{- .url = self.transfer.req.url,- .content_length = self.content_length,- });-- self.transfer.req.notification.dispatch(.http_request_done, &.{- .transfer = self.transfer,- .content_length = self.content_length,- });- return self.forward.forwardDone();- }-- fn errorCallback(ctx: *anyopaque, err: anyerror) void {- const self: *InterceptContext = @ptrCast(@alignCast(ctx));-- log.debug(.http, "intercept error", .{- .url = self.transfer.req.url,- .err = err,- });- self.transfer.req.notification.dispatch(.http_request_fail, &.{- .transfer = self.transfer,- .err = err,- });- self.forward.forwardErr(err);- }-- fn shutdownCallback(ctx: *anyopaque) void {- const self: *InterceptContext = @ptrCast(@alignCast(ctx));-- log.debug(.http, "intercept shutdown", .{ .url = self.transfer.req.url });- self.transfer.req.notification.dispatch(.http_request_fail, &.{- .transfer = self.transfer,- .err = error.Shutdown,- });- self.forward.forwardShutdown();- }-};--// CDP-driven resolution entry points. The transfer was paused inside `request`-// (state = .parked = .intercept_request). One of these three is called by CDP-// to resume / drop the transfer.--pub fn continueRequest(self: *InterceptionLayer, transfer: *Transfer) anyerror!void {- if (comptime IS_DEBUG) {- lp.assert(self.intercepted > 0, "InterceptionLayer.continueRequest", .{ .value = self.intercepted });- log.debug(.http, "continue transfer", .{ .intercepted = self.intercepted });- }-- // Resume the layer chain. Ownership is re-handed to whichever subsequent- // layer commits the transfer (queue, multi, or another park). If the- // chain fails before any commit, we clean up here — mirror the errdefer- // pattern in Client.request.- transfer.unpark();- self.next.request(transfer) catch |err| {- if (transfer.state == .created) {- transfer.abort(err);- }- return err;- };-}--pub fn abortRequest(self: *InterceptionLayer, transfer: *Transfer) void {- if (comptime IS_DEBUG) {- lp.assert(self.intercepted > 0, "InterceptionLayer.abortRequest", .{ .value = self.intercepted });- log.debug(.http, "abort transfer", .{ .intercepted = self.intercepted });- }- transfer.abortParked(error.Abort);-}--pub fn fulfillRequest(- self: *InterceptionLayer,- transfer: *Transfer,- status: u16,- headers: []const http.Header,- body: ?[]const u8,-) !void {- if (comptime IS_DEBUG) {- lp.assert(self.intercepted > 0, "InterceptionLayer.fulfillRequest", .{ .value = self.intercepted });- log.debug(.http, "fulfill transfer", .{ .intercepted = self.intercepted });- }-- // Leave the parked state (accounting `intercepted` exactly once) and move to- // .completing BEFORE running the user callbacks.- transfer.unpark();-- if (HttpClient.isRedirectStatus(status)) {- if (findLocation(headers)) |location| {- fulfilledRedirect(transfer, status, headers, location) catch |err| {- if (transfer.state == .created) {- transfer.abort(err);- }- return err;- };- self.next.request(transfer) catch |err| {- if (transfer.state == .created) {- transfer.abort(err);- }- return err;- };- return;- }- }-- // Not a redirect: move to .completing BEFORE running the user callbacks.- transfer.state = .completing;- defer transfer.deinit();-- // `done` flips true once we've called the user's done_callback. If- // done_callback itself throws, the user already saw their end-of-flow- // notification; suppress error_callback to avoid double-notify.- var done: bool = false;- fulfillInner(&transfer.req, status, headers, body, &done) catch |err| {- if (!done) {- // safe here despite the defer transfer.deinit() above since the- // state == .completing- transfer.abort(err);- }- return err;- };-}--fn fulfillInner(- req: *Request,- status: u16,- headers: []const http.Header,- body: ?[]const u8,- done: *bool,-) !void {- const fulfilled = FulfilledResponse{- .status = status,- .url = req.url,- .headers = headers,- .body = body,- };-- const response = Response.fromFulfilled(req.ctx, &fulfilled);-- if (req.start_callback) |cb| {- try cb(response);- }-- const result = try req.header_callback(response);- if (result == .abort) {- return error.Abort;- }-- if (body) |b| {- try req.data_callback(response, b);- }-- done.* = true;- try req.done_callback(req.ctx);-}--fn fulfilledRedirect(transfer: *Transfer, status: u16, headers: []const http.Header, location: []const u8) !void {- // retrieve cookies from the fulfilled response's headers.- if (transfer.req.cookie_jar) |jar| {- for (headers) |hdr| {- if (std.ascii.eqlIgnoreCase(hdr.name, "set-cookie")) {- try jar.populateFromResponse(transfer.req.url, hdr.value);- }- }- }-- try transfer.applyRedirectTarget(transfer.req.url, location, status);-}--fn findLocation(headers: []const http.Header) ?[]const u8 {- for (headers) |hdr| {- if (std.ascii.eqlIgnoreCase(hdr.name, "location")) {- return hdr.value;- }- }- return null;-}diff --git a/src/network/layer/RobotsLayer.zig b/src/network/layer/RobotsLayer.zigdeleted file mode 100644index eddccbb49..000000000--- a/src/network/layer/RobotsLayer.zig+++ /dev/null@@ -1,291 +0,0 @@-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)-//-// Francis Bouvier <[email protected]>-// Pierre Tachoire <[email protected]>-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--const std = @import("std");-const lp = @import("lightpanda");--const URL = @import("../../browser/URL.zig");-const Layer = @import("../../browser/HttpClient.zig").Layer;-const Transfer = @import("../../browser/HttpClient.zig").Transfer;-const Response = @import("../../browser/HttpClient.zig").Response;-const HeaderResult = @import("../../browser/HttpClient.zig").HeaderResult;--const Robots = @import("../Robots.zig");-const Network = @import("../Network.zig");--const log = lp.log;-const Allocator = std.mem.Allocator;--const RobotsLayer = @This();--next: Layer = undefined,-network: *Network,-allocator: Allocator,-pending: std.StringHashMapUnmanaged(std.ArrayList(*Transfer)) = .empty,--pub fn layer(self: *RobotsLayer) Layer {- return .{- .ptr = self,- .vtable = &.{- .request = request,- },- };-}--pub fn deinit(self: *RobotsLayer, allocator: Allocator) void {- var it = self.pending.iterator();- while (it.next()) |entry| {- entry.value_ptr.deinit(allocator);- }- self.pending.deinit(allocator);-}--fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {- const self: *RobotsLayer = @ptrCast(@alignCast(ptr));-- if (transfer.req.internal) {- return self.next.request(transfer);- }-- const url = transfer.req.url;- const robots_url = try URL.getRobotsUrl(transfer.arena, url);-- if (self.network.robot_store.get(robots_url)) |robot_entry| {- switch (robot_entry) {- .present => |robots| {- const path = URL.getPathname(url);-- if (!robots.isAllowed(path)) {- try transfer.client.runNextTick(- transfer,- null,- .{- .run = struct {- fn run(t: *Transfer, _: ?*anyopaque) void {- defer t.deinit();-- log.warn(.http, "blocked by robots", .{ .url = t.req.url });- t.req.error_callback(t.req.ctx, error.RobotsBlocked);- }- }.run,- },- );- return;- }- },- .absent => {},- }- return self.next.request(transfer);- }-- return self.fetchRobotsThenRequest(robots_url, transfer);-}--fn fetchRobotsThenRequest(- self: *RobotsLayer,- robots_url: [:0]const u8,- transfer: *Transfer,-) !void {- const entry = try self.pending.getOrPut(self.allocator, robots_url);-- if (!entry.found_existing) {- errdefer std.debug.assert(self.pending.remove(robots_url));- entry.value_ptr.* = .empty;-- try entry.value_ptr.append(self.allocator, transfer);- transfer.park(.robots);- errdefer {- entry.value_ptr.deinit(self.allocator);- transfer.unpark();- }-- const robots_ctx = try transfer.arena.create(RobotsContext);- robots_ctx.* = .{- .layer = self,- .buffer = .empty,- .arena = transfer.arena,- .robots_url = robots_url,- };-- // CRITICAL: build a fresh Headers for the inner robots fetch.- // We value-copy req from the parent, but Headers is a struct wrapping- // a *curl_slist — value copy shares the pointer. Letting Client.request- // take ownership of a shared headers list means both transfers will- // free it at deinit time -> double-free. The robots.txt fetch is a- // system-level GET anyway, no need to inherit the parent's user headers.- var new_req = transfer.req;- new_req.headers = try transfer.client.newHeaders();- errdefer new_req.headers.deinit();- new_req.method = .GET;- new_req.url = robots_url;- new_req.internal = true;- new_req.resource_type = .fetch;- new_req.body = null;- new_req.ctx = robots_ctx;- new_req.start_callback = null;- new_req.header_callback = RobotsContext.headerCallback;- new_req.data_callback = RobotsContext.dataCallback;- new_req.done_callback = RobotsContext.doneCallback;- new_req.error_callback = RobotsContext.errorCallback;- new_req.shutdown_callback = RobotsContext.shutdownCallback;-- log.debug(.browser, "fetching robots.txt", .{ .robots_url = robots_url });- try transfer.client.request(new_req, transfer.owner);- } else {- // Already one in flight, just queue behind.- try entry.value_ptr.append(self.allocator, transfer);-- // Parked: RobotsLayer owns destruction via flushPending / flushPendingShutdown- // until robots.txt resolves. Without this, Client.request's errdefer (or- // any caller's cleanup) would deinit a transfer that's still on the- // pending list, leaving flushPending with a dangling pointer.- transfer.park(.robots);- }-}--fn flushPending(self: *RobotsLayer, robots_url: [:0]const u8, allowed: bool) void {- var queued = self.pending.fetchRemove(robots_url) orelse return;- defer queued.value.deinit(self.allocator);-- for (queued.value.items) |transfer| {- if (!allowed) {- log.warn(.http, "blocked by robots", .{ .url = transfer.req.url });- transfer.abort(error.RobotsBlocked);- } else {- // Hand back to the layer chain. If a downstream layer commits- // (multi / queue / park), state advances past .created. If it- // fails before committing, we clean up here.- transfer.unpark();- self.next.request(transfer) catch |e| {- if (transfer.state == .created) {- transfer.abort(e);- }- };- }- }-}--// Invariant: shutdown_callback fires on a Transfer only via Transfer.kill,-// and the only callers of kill are Client.abortOwner / .abortRequests-// (owner-driven teardown). So if THIS robots fetch's shutdown_callback-// fired, the owner is being torn down — every parked transfer in this-// pending queue is on the same owner list and is already being killed by-// the same walk. We just need to drop the pending entry; the owner walk-// handles the rest. (If a future code path adds per-transfer kill-// without owner teardown, this assumption breaks — see comment above-// detachOrDeinit in HttpClient.zig.)-fn flushPendingShutdown(self: *RobotsLayer, robots_url: [:0]const u8) void {- var pending = self.pending.fetchRemove(robots_url) orelse return;- pending.value.deinit(self.allocator);-}--const RobotsContext = struct {- layer: *RobotsLayer,- arena: Allocator,- robots_url: [:0]const u8,- buffer: std.ArrayList(u8),- status: u16 = 0,-- fn deinit(self: *RobotsContext) void {- self.buffer.deinit(self.layer.allocator);- self.layer.allocator.destroy(self);- }-- fn headerCallback(response: Response) anyerror!HeaderResult {- const self: *RobotsContext = @ptrCast(@alignCast(response.ctx));- switch (response.inner) {- .transfer => |t| {- if (t.res.header) |hdr| {- log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = self.robots_url });- self.status = hdr.status;- }- if (t.getContentLength()) |cl| {- try self.buffer.ensureTotalCapacity(self.arena, cl);- }- },- else => {},- }-- return .proceed;- }-- fn dataCallback(response: Response, data: []const u8) anyerror!void {- const self: *RobotsContext = @ptrCast(@alignCast(response.ctx));- try self.buffer.appendSlice(self.arena, data);- }-- fn doneCallback(ctx_ptr: *anyopaque) anyerror!void {- const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));- const l = self.layer;- const robots_url = self.robots_url;-- var allowed = true;- const network = l.network;-- switch (self.status) {- 200 => {- if (self.buffer.items.len > 0) {- const robots: ?Robots = network.robot_store.robotsFromBytes(- network.config.http_headers.user_agent,- self.buffer.items,- ) catch blk: {- log.warn(.browser, "failed to parse robots", .{ .robots_url = robots_url });- try network.robot_store.putAbsent(robots_url);- break :blk null;- };- if (robots) |r| {- try network.robot_store.put(robots_url, r);- const path = URL.getPathname(l.pending.get(robots_url).?.items[0].req.url);- allowed = r.isAllowed(path);- }- }- },- 404 => {- log.debug(.http, "robots not found", .{ .url = robots_url });- try network.robot_store.putAbsent(robots_url);- },- else => {- log.debug(.http, "unexpected status on robots", .{- .url = robots_url,- .status = self.status,- });- try network.robot_store.putAbsent(robots_url);- },- }-- l.flushPending(robots_url, allowed);- }-- fn errorCallback(ctx_ptr: *anyopaque, err: anyerror) void {- const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));- const l = self.layer;- const robots_url = self.robots_url;-- log.warn(.http, "robots fetch failed", .{ .err = err });- l.flushPending(robots_url, true);- }-- fn shutdownCallback(ctx_ptr: *anyopaque) void {- const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));- const l = self.layer;- const robots_url = self.robots_url;-- log.debug(.http, "robots fetch shutdown", .{});- l.flushPendingShutdown(robots_url);- }-};diff --git a/src/network/layer/WebBotAuthLayer.zig b/src/network/layer/WebBotAuthLayer.zigdeleted file mode 100644index 810d94ef2..000000000--- a/src/network/layer/WebBotAuthLayer.zig+++ /dev/null@@ -1,43 +0,0 @@-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)-//-// Francis Bouvier <[email protected]>-// Pierre Tachoire <[email protected]>-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--const URL = @import("../../browser/URL.zig");-const Layer = @import("../../browser/HttpClient.zig").Layer;-const Transfer = @import("../../browser/HttpClient.zig").Transfer;--const WebBotAuthLayer = @This();--next: Layer = undefined,--pub fn layer(self: *WebBotAuthLayer) Layer {- return .{- .ptr = self,- .vtable = &.{ .request = request },- };-}--fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {- const self: *WebBotAuthLayer = @ptrCast(@alignCast(ptr));-- const wba = transfer.client.network.web_bot_auth orelse @panic("WebBotAuthLayer shouldn't be active without WebBotAuth");-- const authority = URL.getHost(transfer.req.url);- try wba.signRequest(transfer.arena, &transfer.req.headers, authority);-- return self.next.request(transfer);-}