Skip to content

Lightpanda: the headless browser designed for AI and automation

refactor: HttpClient

Karl Seguin2eab4d2630f5
diff --git a/src/Notification.zig b/src/Notification.zig
index 4a96b2382..c28a74c85 100644
--- a/src/Notification.zig
+++ b/src/Notification.zig
@@ -21,8 +21,7 @@ const lp = @import("lightpanda");
const js = @import("browser/js/js.zig");
const Frame = @import("browser/Frame.zig");
-const Transfer = @import("browser/HttpClient.zig").Transfer;
-const Response = @import("browser/HttpClient.zig").Response;
+const Transfer = @import("network/HttpClient.zig").Transfer;
const log = lp.log;
const Execution = js.Execution;
@@ -224,7 +223,6 @@ pub const ResponseData = struct {
pub const ResponseHeaderDone = struct {
transfer: *Transfer,
- response: *const Response,
};
pub const RequestDone = struct {
diff --git a/src/browser/Browser.zig b/src/browser/Browser.zig
index e79834aa0..b767140ed 100644
--- a/src/browser/Browser.zig
+++ b/src/browser/Browser.zig
@@ -28,7 +28,7 @@ const Watchdog = @import("../Watchdog.zig");
const Session = @import("Session.zig");
const Selector = @import("webapi/selector/Selector.zig");
const Viewport = @import("Viewport.zig");
-const HttpClient = @import("HttpClient.zig");
+const HttpClient = @import("../network/HttpClient.zig");
const PermissionState = @import("webapi/Permissions.zig").State;
const ArenaPool = App.ArenaPool;
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index 23c2047c8..aa51e6eb6 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -60,7 +60,7 @@ const popover = @import("webapi/element/popover.zig");
const slotting = @import("webapi/element/slotting.zig");
const NavigationKind = @import("webapi/navigation/root.zig").NavigationKind;
-const HttpClient = @import("HttpClient.zig");
+const HttpClient = @import("../network/HttpClient.zig");
const sys_url = @import("../sys/url.zig");
const timestamp = @import("../datetime.zig").timestamp;
@@ -597,16 +597,16 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
self._load_state = .parsing;
self._last_navigate_error = null;
- const req_id = self._session.browser.http_client.nextReqId();
log.info(.frame, "navigate", .{
.url = request_url,
.method = opts.method,
.reason = opts.reason,
.body = opts.body != null,
- .req_id = req_id,
.type = self._type,
});
+ const http_client = &session.browser.http_client;
+
// Handle synthetic navigations: about:blank and blob: URLs
const is_about_blank = std.mem.eql(u8, "about:blank", request_url);
const is_blob = !is_about_blank and std.mem.startsWith(u8, request_url, "blob:");
@@ -670,6 +670,10 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
};
}
+ // No real request is made, but CDP still correlates the navigation
+ // events by request id, so consume one.
+ const req_id = http_client.incrReqId();
+
session.notification.dispatch(.frame_navigate, &.{
.opts = opts,
.req_id = req_id,
@@ -694,9 +698,6 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
.timestamp = timestamp(.monotonic),
});
- // force next request id manually b/c we won't create a real req.
- _ = session.browser.http_client.incrReqId();
-
if (self.parent == null) {
session.navigation._current_navigation_kind = opts.kind;
try session.navigation.commitNavigation(self);
@@ -706,8 +707,6 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
return;
}
- const http_client = &session.browser.http_client;
-
self._http_status = null;
self._http_headers = .empty;
@@ -719,7 +718,6 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
};
self.origin = try URL.getOrigin(self.arena, self.url);
- self._req_id = req_id;
self._navigated_options = .{
.cdp_id = opts.cdp_id,
.reason = opts.reason,
@@ -728,14 +726,37 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
.header = if (opts.header) |h| try self.arena.dupeZ(u8, h) else null,
};
- var headers = try http_client.newHeaders();
- try headers.add(lp.Config.HttpHeaders.navigation_accept);
- if (opts.header) |hdr| {
- try headers.add(hdr);
- }
- if (opts.referer) |ref| {
- const ref_header = try std.mem.concatWithSentinel(self.arena, u8, &.{ "Referer: ", ref }, 0);
- try headers.add(ref_header);
+ const transfer = try http_client.newRequest(.{
+ .ctx = self,
+ .url = self.url,
+ .frame_id = self._frame_id,
+ .loader_id = self._loader_id,
+ .method = opts.method,
+ .body = opts.body,
+ .cookie_jar = &session.cookie_jar,
+ .cookie_origin = opts.initiator_url orelse self.url,
+ .resource_type = .document,
+ .notification = self._session.notification,
+ .header_callback = frameHeaderDoneCallback,
+ .data_callback = frameDataCallback,
+ .done_callback = frameDoneCallback,
+ .error_callback = frameErrorCallback,
+ // The frame tracks its navigation by id (_req_id), never by pointer.
+ .shutdown_callback = HttpClient.noopShutdown,
+ }, &self._http_owner);
+ self._req_id = transfer.id;
+
+ {
+ // Ours until submit; clean up if header setup fails.
+ errdefer transfer.deinit();
+ try transfer.req.headers.add(lp.Config.HttpHeaders.navigation_accept);
+ if (opts.header) |hdr| {
+ try transfer.req.headers.add(hdr);
+ }
+ if (opts.referer) |ref| {
+ const ref_header = try std.mem.concatWithSentinel(transfer.arena, u8, &.{ "Referer: ", ref }, 0);
+ try transfer.req.headers.add(ref_header);
+ }
}
// A root navigation issued against a pending Page (i.e. one allocated by
@@ -751,7 +772,7 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
session.notification.dispatch(.frame_navigate, &.{
.opts = opts,
.url = self.url,
- .req_id = req_id,
+ .req_id = transfer.id,
.frame_id = self._frame_id,
.loader_id = self._loader_id,
.timestamp = timestamp(.monotonic),
@@ -763,23 +784,7 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
session.navigation._current_navigation_kind = opts.kind;
- self.makeRequest(.{
- .ctx = self,
- .url = self.url,
- .frame_id = self._frame_id,
- .loader_id = self._loader_id,
- .method = opts.method,
- .headers = headers,
- .body = opts.body,
- .cookie_jar = &session.cookie_jar,
- .cookie_origin = opts.initiator_url orelse self.url,
- .resource_type = .document,
- .notification = self._session.notification,
- .header_callback = frameHeaderDoneCallback,
- .data_callback = frameDataCallback,
- .done_callback = frameDoneCallback,
- .error_callback = frameErrorCallback,
- }) catch |err| {
+ transfer.submit() catch |err| {
log.err(.frame, "navigate request", .{ .url = self.url, .err = err, .type = self._type });
return err;
};
@@ -962,6 +967,11 @@ pub fn makeRequest(self: *Frame, req: HttpClient.Request) !void {
return self._session.browser.http_client.request(req, &self._http_owner);
}
+// Two-phase variant; see HttpClient.newRequest for the ownership contract.
+pub fn newRequest(self: *Frame, req: HttpClient.Request) !*HttpClient.Transfer {
+ return self._session.browser.http_client.newRequest(req, &self._http_owner);
+}
+
// Synchronously abort every transfer and WebSocket owned by this frame
// and all of its descendants.
pub fn abortTransfers(self: *Frame) void {
@@ -970,8 +980,6 @@ pub fn abortTransfers(self: *Frame) void {
}
const http_client = &self._session.browser.http_client;
http_client.abortOwner(&self._http_owner);
- // abortOwner misses deferred contexts whose transfer already completed.
- http_client.deferring_layer.cancelFrame(self._frame_id);
}
pub fn documentIsLoaded(self: *Frame) void {
@@ -1144,8 +1152,8 @@ fn notifyParentLoadComplete(self: *Frame) void {
parent.iframeCompletedLoading(self.iframe.?, self._delays_parent_load);
}
-fn frameHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResult {
- var self: *Frame = @ptrCast(@alignCast(response.ctx));
+fn frameHeaderDoneCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.HeaderResult {
+ var self: *Frame = @ptrCast(@alignCast(transfer.req.ctx));
// Commit point for a pending root navigation. The session has been
// holding the OLD page alive during the round-trip; now that response
@@ -1157,7 +1165,7 @@ fn frameHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResu
try self._session.commitPendingPage(self._page);
}
- const response_url = response.url();
+ const response_url = transfer.req.url;
if (std.mem.eql(u8, response_url, self.url) == false) {
// would be different than self.url in the case of a redirect
self.url = try self.arena.dupeZ(u8, response_url);
@@ -1169,7 +1177,7 @@ fn frameHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResu
// Page.reload doesn't re-POST form data to the redirect target. Conservative
// default — 307/308 technically preserve the method per RFC 7231, but
// resubmitting form data is the more dangerous failure mode.
- if ((response.redirectCount() orelse 0) > 0) {
+ if ((transfer.redirectCount() orelse 0) > 0) {
if (self._navigated_options) |*no| {
no.method = .GET;
no.body = null;
@@ -1186,14 +1194,14 @@ fn frameHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResu
if (comptime IS_DEBUG) {
log.debug(.frame, "navigate header", .{
.url = self.url,
- .status = response.status(),
- .content_type = response.contentType(),
+ .status = transfer.responseStatus(),
+ .content_type = transfer.contentType(),
.type = self._type,
});
}
- self._http_status = response.status();
- var it = response.headerIterator();
+ self._http_status = transfer.responseStatus();
+ var it = transfer.responseHeaderIterator();
while (it.next()) |hdr| {
try self._http_headers.append(self.arena, .{
.name = try self.arena.dupe(u8, hdr.name),
@@ -1218,7 +1226,7 @@ fn frameHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResu
// If the response is a file download, stream its body to disk instead of
// parsing it as a page. This sets _parse_state to .download, which the
// data/done callbacks below special-case.
- _ = try self.maybeStartDownload(response);
+ _ = try self.maybeStartDownload(transfer);
return .proceed;
}
@@ -1227,7 +1235,7 @@ fn frameHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResu
// treated as a download when Browser.setDownloadBehavior opted in
// (allow/allowAndName) and the response carries Content-Disposition: attachment.
// See issue #2701.
-fn maybeStartDownload(self: *Frame, response: HttpClient.Response) !bool {
+fn maybeStartDownload(self: *Frame, transfer: *HttpClient.Transfer) !bool {
const session = self._session;
switch (session.download_behavior) {
.allow, .allow_and_name => {},
@@ -1235,7 +1243,7 @@ fn maybeStartDownload(self: *Frame, response: HttpClient.Response) !bool {
}
const disposition: HttpClient.Header = blk: {
- var it = response.headerIterator();
+ var it = transfer.responseHeaderIterator();
while (it.next()) |hdr| {
if (std.ascii.eqlIgnoreCase(hdr.name, "content-disposition")) {
break :blk hdr;
@@ -1281,7 +1289,7 @@ fn maybeStartDownload(self: *Frame, response: HttpClient.Response) !bool {
return false;
};
- const total: ?u64 = if (response.contentLength()) |cl| cl else null;
+ const total: ?u64 = if (transfer.getContentLength()) |cl| cl else null;
self._parse_state = .{ .download = .{
.guid = guid,
@@ -1364,14 +1372,14 @@ fn isUtf16Encoding(charset: []const u8) bool {
return std.mem.eql(u8, charset, "UTF-16LE") or std.mem.eql(u8, charset, "UTF-16BE");
}
-fn frameDataCallback(response: HttpClient.Response, data: []const u8) !void {
- var self: *Frame = @ptrCast(@alignCast(response.ctx));
+fn frameDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
+ var self: *Frame = @ptrCast(@alignCast(transfer.req.ctx));
if (self._parse_state == .pre) {
// we lazily do this, because we might need the first chunk of data
// to sniff the content type
var mime: Mime = blk: {
- if (response.contentType()) |ct| {
+ if (transfer.contentType()) |ct| {
break :blk try Mime.parse(ct);
}
break :blk Mime.sniff(data);
@@ -2096,18 +2104,10 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co
const http_client = &session.browser.http_client;
- // `syncRequest` below registers a blocking request for this frame, which
- // makes the DeferringLayer hold back the completion callbacks of every
- // OTHER in-flight transfer for the frame (e.g. a `<script defer>` still
- // loading) so they don't run JS while we're on the parser stack. Those
- // deferred completions must be flushed once the sync fetch returns —
- // otherwise a `<script defer>` whose fetch finishes during this window is
- // left at `complete == false` forever, the deferred-script queue never
- // drains, and `documentIsLoaded` (readyState -> "interactive",
- // DOMContentLoaded, the load event) never fires. The blocking-`<script>`
- // path (ScriptManager.addFromElement) and the worker path already flush;
- // the external-stylesheet path must too.
- defer http_client.deferring_layer.flushFrame(self._frame_id);
+ // `syncRequest` below registers a blocking request for this frame; the
+ // client's dispatcher holds back every OTHER transfer's callbacks for
+ // the frame while it's registered (they'd run JS on the parser's stack)
+ // and delivers them on the next tick after the sync fetch returns.
var headers = try http_client.newHeaders();
try headers.add("Accept: text/css,*/*;q=0.1");
@@ -2135,6 +2135,7 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co
.cookie_origin = self.url,
.resource_type = .stylesheet,
.notification = session.notification,
+ .shutdown_callback = HttpClient.noopShutdown, // syncRequest installs its own
}) catch |err| {
log.warn(.http, "external stylesheet fetch", .{ .err = err, .url = resolved });
return self.fireElementEvent(element, comptime .wrap("error"));
diff --git a/src/browser/Runner.zig b/src/browser/Runner.zig
index a96adedf4..b9aad1a47 100644
--- a/src/browser/Runner.zig
+++ b/src/browser/Runner.zig
@@ -22,7 +22,7 @@ const lp = @import("lightpanda");
const js = @import("js/js.zig");
const Browser = @import("Browser.zig");
const Session = @import("Session.zig");
-const HttpClient = @import("HttpClient.zig");
+const HttpClient = @import("../network/HttpClient.zig");
const Node = @import("webapi/Node.zig");
const Selector = @import("webapi/selector/Selector.zig");
@@ -221,12 +221,12 @@ fn _tick(self: *Runner, comptime is_cdp: bool, timeout_ms: u32, conditions: []Wa
}
const http_active = http_client.http_active;
- const http_next_tick = http_client.next_tick_count;
- const total_http_activity = http_active + http_next_tick + http_client.interception_layer.intercepted;
+ const http_buffered = http_client.dispatch_count;
+ const total_http_activity = http_active + http_buffered + http_client.intercepted;
const total_network_activity = total_http_activity + http_client.ws_active;
const ms_to_next_macrotask = browser.msToNextMacrotask();
- const network_idle = total_network_activity == 0 and http_client.queue.first == null and http_client.ready_queue.first == null;
+ const network_idle = total_network_activity == 0 and http_client.pending_queue.first == null and http_client.ready_queue.first == null;
const is_done = ms_to_next_macrotask == null and network_idle;
// _we_ have nothing to run, but v8 is working on background tasks. We'll
diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig
index 2605ba3c5..bda6be59e 100644
--- a/src/browser/ScriptManager.zig
+++ b/src/browser/ScriptManager.zig
@@ -20,7 +20,7 @@ const std = @import("std");
const lp = @import("lightpanda");
const builtin = @import("builtin");
-const HttpClient = @import("HttpClient.zig");
+const HttpClient = @import("../network/HttpClient.zig");
const js = @import("js/js.zig");
const URL = @import("URL.zig");
@@ -358,6 +358,7 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
.cookie_origin = frame.url,
.resource_type = .script,
.notification = frame._session.notification,
+ .shutdown_callback = HttpClient.noopShutdown, // syncRequest installs its own
});
script.source = .{ .remote = response.body };
@@ -386,6 +387,9 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
.data_callback = Script.dataCallback,
.done_callback = Script.doneCallback,
.error_callback = Script.errorCallback,
+ // Nothing holds the transfer; teardown cleanup runs through
+ // the manager's script lists.
+ .shutdown_callback = HttpClient.noopShutdown,
});
}
@@ -396,9 +400,6 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
return;
}
- // This will flush any deferred scripts.
- defer self.base.client.deferring_layer.flushFrame(self.base.owner.frameId());
-
if (script.status < 200 or script.status > 299) {
log.info(.http, "script load error", .{ .status = script.status });
script.executeCallback(comptime .wrap("error"));
diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zig
index 277332a18..a62e269fa 100644
--- a/src/browser/ScriptManagerBase.zig
+++ b/src/browser/ScriptManagerBase.zig
@@ -20,8 +20,8 @@ const std = @import("std");
const lp = @import("lightpanda");
const builtin = @import("builtin");
-const HttpClient = @import("HttpClient.zig");
const http = @import("../network/http.zig");
+const HttpClient = @import("../network/HttpClient.zig");
const js = @import("js/js.zig");
const Session = @import("Session.zig");
@@ -454,6 +454,7 @@ pub fn getAsyncImport(self: *ScriptManagerBase, url: [:0]const u8, cb: ImportAsy
.data_callback = Script.dataCallback,
.done_callback = Script.doneCallback,
.error_callback = Script.errorCallback,
+ .shutdown_callback = Script.shutdownCallback,
}) catch |err| {
self.async_scripts.remove(&script.node);
return err;
@@ -660,19 +661,19 @@ pub const Script = struct {
self.manager.releaseArena(self.arena);
}
- pub fn startCallback(response: HttpClient.Response) !void {
- log.debug(.http, "script fetch start", .{ .req = response });
+ pub fn startCallback(transfer: *HttpClient.Transfer) !void {
+ log.debug(.http, "script fetch start", .{ .req = transfer });
}
- pub fn headerCallback(response: HttpClient.Response) !HttpClient.HeaderResult {
- const self: *Script = @ptrCast(@alignCast(response.ctx));
+ pub fn headerCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.HeaderResult {
+ const self: *Script = @ptrCast(@alignCast(transfer.req.ctx));
- self.status = response.status().?;
- if (response.status() != 200) {
+ self.status = transfer.responseStatus().?;
+ if (transfer.responseStatus() != 200) {
log.info(.http, "script header", .{
- .req = response,
- .status = response.status(),
- .content_type = response.contentType(),
+ .req = transfer,
+ .status = transfer.responseStatus(),
+ .content_type = transfer.contentType(),
});
return .abort;
@@ -680,64 +681,61 @@ pub const Script = struct {
if (comptime IS_DEBUG) {
log.debug(.http, "script header", .{
- .req = response,
- .status = response.status(),
- .content_type = response.contentType(),
+ .req = transfer,
+ .status = transfer.responseStatus(),
+ .content_type = transfer.contentType(),
});
}
- switch (response.inner) {
- .transfer => |transfer| {
- // temp debug, trying to figure out why the next assert sometimes
- // fails. Is the buffer just corrupt or is headerCallback really
- // being called twice?
- lp.assert(self.header_callback_called == false, "ScriptManagerBase.Header recall", .{
- .m = @tagName(std.meta.activeTag(self.extra)),
- .a1 = self.debug_transfer_id,
- .a2 = self.debug_transfer_tries,
- .a3 = self.debug_transfer_aborted,
- .a4 = self.debug_transfer_bytes_received,
- .a5 = self.debug_transfer_notified_fail,
- .a8 = self.debug_transfer_auth_challenge,
- .a9 = self.debug_transfer_easy_id,
- .b1 = transfer.id,
- .b2 = transfer._tries,
- .b3 = transfer.state == .aborted,
- .b4 = transfer.res.bytes_received,
- .b5 = transfer._notified_fail,
- .b8 = transfer._auth_challenge != null,
- .b9 = if (transfer._conn) |c| @intFromPtr(c._easy) else 0,
- });
- self.header_callback_called = true;
- self.debug_transfer_id = transfer.id;
- self.debug_transfer_tries = transfer._tries;
- self.debug_transfer_aborted = transfer.state == .aborted;
- self.debug_transfer_bytes_received = transfer.res.bytes_received;
- self.debug_transfer_notified_fail = transfer._notified_fail;
- self.debug_transfer_auth_challenge = transfer._auth_challenge != null;
- self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0;
- },
- else => {},
+ {
+ // temp debug, trying to figure out why the next assert sometimes
+ // fails. Is the buffer just corrupt or is headerCallback really
+ // being called twice?
+ lp.assert(self.header_callback_called == false, "ScriptManagerBase.Header recall", .{
+ .m = @tagName(std.meta.activeTag(self.extra)),
+ .a1 = self.debug_transfer_id,
+ .a2 = self.debug_transfer_tries,
+ .a3 = self.debug_transfer_aborted,
+ .a4 = self.debug_transfer_bytes_received,
+ .a5 = self.debug_transfer_notified_fail,
+ .a8 = self.debug_transfer_auth_challenge,
+ .a9 = self.debug_transfer_easy_id,
+ .b1 = transfer.id,
+ .b2 = transfer._tries,
+ .b3 = transfer.state == .aborted,
+ .b4 = transfer.res.bytes_received,
+ .b5 = transfer._notified_fail,
+ .b8 = transfer._auth_challenge != null,
+ .b9 = if (transfer._conn) |c| @intFromPtr(c._easy) else 0,
+ });
+ self.header_callback_called = true;
+ self.debug_transfer_id = transfer.id;
+ self.debug_transfer_tries = transfer._tries;
+ self.debug_transfer_aborted = transfer.state == .aborted;
+ self.debug_transfer_bytes_received = transfer.res.bytes_received;
+ self.debug_transfer_notified_fail = transfer._notified_fail;
+ self.debug_transfer_auth_challenge = transfer._auth_challenge != null;
+ self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0;
}
lp.assert(self.source.remote.capacity == 0, "ScriptManagerBase.Header buffer", .{ .capacity = self.source.remote.capacity });
var buffer: std.ArrayList(u8) = .empty;
- if (response.contentLength()) |cl| {
+ if (transfer.getContentLength()) |cl| {
try buffer.ensureTotalCapacity(self.arena, cl);
}
self.source = .{ .remote = buffer };
return .proceed;
}
- pub fn dataCallback(response: HttpClient.Response, data: []const u8) !void {
- const self: *Script = @ptrCast(@alignCast(response.ctx));
- self._dataCallback(response, data) catch |err| {
- log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = response, .len = data.len });
+ pub fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
+ const self: *Script = @ptrCast(@alignCast(transfer.req.ctx));
+ self._dataCallback(transfer, data) catch |err| {
+ log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = transfer, .len = data.len });
return err;
};
}
- fn _dataCallback(self: *Script, _: HttpClient.Response, data: []const u8) !void {
+ fn _dataCallback(self: *Script, _: *HttpClient.Transfer, data: []const u8) !void {
try self.source.remote.appendSlice(self.arena, data);
}
diff --git a/src/browser/js/Execution.zig b/src/browser/js/Execution.zig
index ee3abadf6..5efd164c6 100644
--- a/src/browser/js/Execution.zig
+++ b/src/browser/js/Execution.zig
@@ -33,7 +33,7 @@ const Scheduler = @import("Scheduler.zig");
const Page = @import("../Page.zig");
const Session = @import("../Session.zig");
const Factory = @import("../Factory.zig");
-const HttpClient = @import("../HttpClient.zig");
+const HttpClient = @import("../../network/HttpClient.zig");
const EventManagerBase = @import("../EventManagerBase.zig");
const Event = @import("../webapi/Event.zig");
@@ -102,6 +102,13 @@ pub fn makeRequest(self: *const Execution, req: HttpClient.Request) !void {
};
}
+// Two-phase variant; see HttpClient.newRequest for the ownership contract.
+pub fn newRequest(self: *const Execution, req: HttpClient.Request) !*HttpClient.Transfer {
+ return switch (self.js.global) {
+ inline else => |g| g.newRequest(req),
+ };
+}
+
pub fn getBroadcastChannels(self: *const Execution) *std.DoublyLinkedList {
return switch (self.js.global) {
inline else => |g| &g._broadcast_channels,
diff --git a/src/browser/webapi/Worker.zig b/src/browser/webapi/Worker.zig
index 9b2659ed7..82c5a1c25 100644
--- a/src/browser/webapi/Worker.zig
+++ b/src/browser/webapi/Worker.zig
@@ -23,7 +23,7 @@ const js = @import("../js/js.zig");
const URL = @import("../URL.zig");
const Frame = @import("../Frame.zig");
-const HttpClient = @import("../HttpClient.zig");
+const Transfer = @import("../../network/HttpClient.zig").Transfer;
const EventTarget = @import("EventTarget.zig");
const MessageEvent = @import("event/MessageEvent.zig");
@@ -56,7 +56,7 @@ _url: [:0]const u8,
_type: WorkerType = .classic,
_script_loaded: bool = false,
_script_buffer: std.ArrayList(u8) = .empty,
-_http_response: ?HttpClient.Response = null,
+_http_transfer: ?*Transfer = null,
// Event handlers
_on_error: ?js.Function.Global = null,
@@ -101,11 +101,9 @@ pub fn init(url: []const u8, options: ?WorkerOptions, frame: *Frame) !*Worker {
return self;
}
- const headers = try session.browser.http_client.newHeaders();
- frame.makeRequest(.{
+ const transfer = frame.newRequest(.{
.ctx = self,
.method = .GET,
- .headers = headers,
.url = resolved_url,
.frame_id = self._frame_id,
.loader_id = self._loader_id,
@@ -117,11 +115,24 @@ pub fn init(url: []const u8, options: ?WorkerOptions, frame: *Frame) !*Worker {
.data_callback = httpDataCallback,
.done_callback = httpDoneCallback,
.error_callback = httpErrorCallback,
+ .shutdown_callback = httpShutdownCallback,
}) catch |err| {
log.err(.browser, "Worker request", .{ .url = resolved_url, .err = err });
frame.removeWorker(self);
return err;
};
+
+ // Held for deinit's abort; the done, error and shutdown callbacks clear
+ // it. The shutdown one matters: Frame.deinit's abortOwner kills the
+ // transfer before it deinits this worker, and deinit must not abort a
+ // freed transfer.
+ self._http_transfer = transfer;
+
+ transfer.submit() catch |err| {
+ log.err(.browser, "Worker request", .{ .url = resolved_url, .err = err });
+ frame.removeWorker(self);
+ return err;
+ };
return self;
}
@@ -129,9 +140,9 @@ pub fn init(url: []const u8, options: ?WorkerOptions, frame: *Frame) !*Worker {
// remove from the frame's worker list.
pub fn deinit(self: *Worker) void {
// No pending frame for workers, so we can abort all frames.
- if (self._http_response) |res| {
+ if (self._http_transfer) |res| {
res.abort(error.Abort);
- self._http_response = null;
+ self._http_transfer = null;
}
self._worker_scope.deinit();
self._frame._session.releaseArena(self._arena);
@@ -141,10 +152,10 @@ pub fn asEventTarget(self: *Worker) *EventTarget {
return self._proto;
}
-fn httpHeaderCallback(response: HttpClient.Response) !HttpClient.HeaderResult {
- const self: *Worker = @ptrCast(@alignCast(response.ctx));
+fn httpHeaderCallback(transfer: *Transfer) !Transfer.HeaderResult {
+ const self: *Worker = @ptrCast(@alignCast(transfer.req.ctx));
- const status = response.status() orelse return .abort;
+ const status = transfer.responseStatus() orelse return .abort;
if (status < 200 or status >= 300) {
log.warn(.browser, "Worker status", .{
.url = self._url,
@@ -153,22 +164,21 @@ fn httpHeaderCallback(response: HttpClient.Response) !HttpClient.HeaderResult {
return .abort;
}
- self._http_response = response;
- if (response.contentLength()) |cl| {
+ if (transfer.getContentLength()) |cl| {
try self._script_buffer.ensureTotalCapacity(self._arena, cl);
}
return .proceed;
}
-fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void {
- const self: *Worker = @ptrCast(@alignCast(response.ctx));
+fn httpDataCallback(transfer: *Transfer, data: []const u8) !void {
+ const self: *Worker = @ptrCast(@alignCast(transfer.req.ctx));
try self._script_buffer.appendSlice(self._arena, data);
}
fn httpDoneCallback(ctx: *anyopaque) !void {
const self: *Worker = @ptrCast(@alignCast(ctx));
- self._http_response = null;
+ self._http_transfer = null;
const url = self._url;
const script = self._script_buffer.items;
@@ -245,9 +255,14 @@ fn loadInitialScript(self: *Worker, script: []const u8) !void {
ls.local.runMacrotasks();
}
+fn httpShutdownCallback(ctx: *anyopaque) void {
+ const self: *Worker = @ptrCast(@alignCast(ctx));
+ self._http_transfer = null;
+}
+
fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {
const self: *Worker = @ptrCast(@alignCast(ctx));
- self._http_response = null;
+ self._http_transfer = null;
log.err(.browser, "worker fetch error", .{
.url = self._url,
@@ -297,9 +312,9 @@ fn _fireErrorEvent(self: *Worker, message: []const u8, error_value: ?js.Value.Te
pub fn terminate(self: *Worker) void {
// Abort any pending script fetch
- if (self._http_response) |resp| {
+ if (self._http_transfer) |resp| {
resp.abort(error.Abort);
- self._http_response = null;
+ self._http_transfer = null;
}
}
diff --git a/src/browser/webapi/WorkerGlobalScope.zig b/src/browser/webapi/WorkerGlobalScope.zig
index 02ebb669f..d65a56ece 100644
--- a/src/browser/webapi/WorkerGlobalScope.zig
+++ b/src/browser/webapi/WorkerGlobalScope.zig
@@ -29,7 +29,7 @@ const Page = @import("../Page.zig");
const Frame = @import("../Frame.zig");
const Factory = @import("../Factory.zig");
const Session = @import("../Session.zig");
-const HttpClient = @import("../HttpClient.zig");
+const HttpClient = @import("../../network/HttpClient.zig");
const EventManagerBase = @import("../EventManagerBase.zig");
const ScriptManagerBase = @import("../ScriptManagerBase.zig");
@@ -250,6 +250,11 @@ pub fn makeRequest(self: *WorkerGlobalScope, req: HttpClient.Request) !void {
return self._session.browser.http_client.request(req, &self._http_owner);
}
+// Two-phase variant; see HttpClient.newRequest for the ownership contract.
+pub fn newRequest(self: *WorkerGlobalScope, req: HttpClient.Request) !*HttpClient.Transfer {
+ return self._session.browser.http_client.newRequest(req, &self._http_owner);
+}
+
pub fn getSelf(self: *WorkerGlobalScope) *WorkerGlobalScope {
return self;
}
@@ -390,6 +395,7 @@ fn importScript(self: *WorkerGlobalScope, arena: Allocator, url: [:0]const u8) !
.cookie_origin = self.url,
.resource_type = .script,
.notification = session.notification,
+ .shutdown_callback = HttpClient.noopShutdown, // syncRequest installs its own
}) catch |err| {
log.warn(.http, "importScript", .{ .url = resolved_url, .err = err });
return error.NetworkError;
@@ -400,8 +406,6 @@ fn importScript(self: *WorkerGlobalScope, arena: Allocator, url: [:0]const u8) !
return error.NetworkError;
}
- defer http_client.deferring_layer.flushFrame(self._frame_id);
-
var ls: JS.Local.Scope = undefined;
self.js.localScope(&ls);
defer ls.deinit();
diff --git a/src/browser/webapi/element/html/Link.zig b/src/browser/webapi/element/html/Link.zig
index af9472283..7eb2190f5 100644
--- a/src/browser/webapi/element/html/Link.zig
+++ b/src/browser/webapi/element/html/Link.zig
@@ -290,10 +290,9 @@ test "WebApi: HTML.Link external stylesheet" {
}
// Regression: a synchronous external-stylesheet fetch must not strand the
-// completion of an in-flight <script defer> (deferred by the blocking-request
-// window). Otherwise the deferred-script queue never drains and the document
-// is stuck at readyState "loading". See Frame.loadExternalStylesheet's
-// flushFrame call.
+// completion of an in-flight <script defer> (held back by the blocking-
+// request gate during the sync window). Otherwise the deferred-script queue
+// never drains and the document is stuck at readyState "loading".
test "WebApi: HTML.Link deferred script then external stylesheet" {
const filter: testing.LogFilter = .init(&.{.http});
defer filter.deinit();
diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zig
index 86e53ca4b..ba6849130 100644
--- a/src/browser/webapi/net/Fetch.zig
+++ b/src/browser/webapi/net/Fetch.zig
@@ -18,10 +18,10 @@
const std = @import("std");
const lp = @import("lightpanda");
-const HttpClient = @import("../../HttpClient.zig");
const js = @import("../../js/js.zig");
const URL = @import("../../URL.zig");
+const Transfer = @import("../../../network/HttpClient.zig").Transfer;
const Request = @import("Request.zig");
const Response = @import("Response.zig");
@@ -96,12 +96,7 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promis
.@"same-origin" => if (exec.isSameOrigin(request._url)) &session.cookie_jar else null,
};
- // Synchronous failures from request layers (e.g. RobotsLayer returning
- // RobotsBlocked when robots.txt is already cached) are dispatched to
- // httpErrorCallback by Client.request, which rejects the promise and
- // releases response._arena. Propagating the error from here would also
- // fire the `errdefer response.deinit` above and double-free the arena.
- exec.makeRequest(.{
+ const transfer = exec.newRequest(.{
.ctx = fetch,
.url = request._url,
.method = request._method,
@@ -118,26 +113,30 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promis
.@"error" => .@"error",
},
.notification = session.notification,
- .start_callback = httpStartCallback,
.header_callback = httpHeaderDoneCallback,
.data_callback = httpDataCallback,
.done_callback = httpDoneCallback,
.error_callback = httpErrorCallback,
.shutdown_callback = httpShutdownCallback,
- }) catch {};
- return resolver.promise();
-}
+ }) catch {
+ // OOM-class; nothing was committed and no callback fired.
+ return resolver.promise();
+ };
-fn httpStartCallback(response: HttpClient.Response) !void {
- const self: *Fetch = @ptrCast(@alignCast(response.ctx));
- if (comptime IS_DEBUG) {
- log.debug(.http, "request start", .{ .url = self._url, .source = "fetch" });
- }
- self._response._http_response = response;
+ // Held for Response.deinit's abort; the error, shutdown and done
+ // callbacks clear it.
+ response._http_transfer = transfer;
+
+ // Failures inside submit are dispatched to httpErrorCallback, which
+ // rejects the promise and releases response._arena. Propagating the
+ // error from here would also fire the `errdefer response.deinit` above
+ // and double-free the arena.
+ transfer.submit() catch {};
+ return resolver.promise();
}
-fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResult {
- const self: *Fetch = @ptrCast(@alignCast(response.ctx));
+fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {
+ const self: *Fetch = @ptrCast(@alignCast(transfer.req.ctx));
if (self._signal) |signal| {
if (signal._aborted) {
@@ -146,7 +145,7 @@ fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResul
}
const arena = self._response._arena;
- if (response.contentLength()) |cl| {
+ if (transfer.getContentLength()) |cl| {
try self._buf.ensureTotalCapacity(arena, cl);
}
@@ -156,14 +155,14 @@ fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResul
log.debug(.http, "request header", .{
.source = "fetch",
.url = self._url,
- .status = response.status(),
+ .status = transfer.responseStatus(),
});
}
- res._status = response.status().?;
- res._status_text = std.http.Status.phrase(@enumFromInt(response.status().?)) orelse "";
- res._url = try arena.dupeZ(u8, response.url());
- res._is_redirected = response.redirectCount().? > 0;
+ res._status = transfer.responseStatus().?;
+ res._status_text = std.http.Status.phrase(@enumFromInt(transfer.responseStatus().?)) orelse "";
+ res._url = try arena.dupeZ(u8, transfer.req.url);
+ res._is_redirected = transfer.redirectCount().? > 0;
// redirect: "manual" surfaces the unfollowed 3xx as an opaque-redirect
// filtered response: status 0, no headers, no body.
@@ -195,7 +194,7 @@ fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResul
res._type = .basic;
}
- var it = response.headerIterator();
+ var it = transfer.responseHeaderIterator();
while (it.next()) |hdr| {
try res._headers.append(hdr.name, hdr.value, exec);
}
@@ -203,8 +202,8 @@ fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResul
return .proceed;
}
-fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void {
- const self: *Fetch = @ptrCast(@alignCast(response.ctx));
+fn httpDataCallback(transfer: *Transfer, data: []const u8) !void {
+ const self: *Fetch = @ptrCast(@alignCast(transfer.req.ctx));
// Check if aborted
if (self._signal) |signal| {
@@ -219,7 +218,7 @@ fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void {
fn httpDoneCallback(ctx: *anyopaque) !void {
const self: *Fetch = @ptrCast(@alignCast(ctx));
var response = self._response;
- response._http_response = null;
+ response._http_transfer = null;
response._body = .{ .bytes = self._buf.items };
log.info(.http, "request complete", .{
@@ -249,7 +248,7 @@ fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {
});
var response = self._response;
- response._http_response = null;
+ response._http_transfer = null;
// Capture this before we reject. Rejection could trigger httpShutdownCallback
// (via a microtask callback). But if we're here, then we'll take care of
@@ -277,7 +276,7 @@ fn httpShutdownCallback(ctx: *anyopaque) void {
if (self._owns_response) {
var response = self._response;
- response._http_response = null;
+ response._http_transfer = null;
response.deinit(self._exec.page);
// Do not access `self` after this point: the Fetch struct was
// allocated from response._arena which has been released.
diff --git a/src/browser/webapi/net/Response.zig b/src/browser/webapi/net/Response.zig
index 78851e067..fc34f6f19 100644
--- a/src/browser/webapi/net/Response.zig
+++ b/src/browser/webapi/net/Response.zig
@@ -22,7 +22,7 @@ const lp = @import("lightpanda");
const js = @import("../../js/js.zig");
const URL = @import("../../URL.zig");
const Page = @import("../../Page.zig");
-const HttpClient = @import("../../HttpClient.zig");
+const Transfer = @import("../../../network/HttpClient.zig").Transfer;
const Blob = @import("../Blob.zig");
const ReadableStream = @import("../streams/ReadableStream.zig");
@@ -53,7 +53,7 @@ _type: Type,
_status_text: []const u8,
_url: [:0]const u8,
_is_redirected: bool,
-_http_response: ?HttpClient.Response = null,
+_http_transfer: ?*Transfer = null,
_body_used: bool = false,
const Body = union(enum) {
@@ -197,9 +197,9 @@ pub fn createJson(data: js.Value, opts_: ?InitOpts, exec: *const Execution) !*Re
}
pub fn deinit(self: *Response, page: *Page) void {
- if (self._http_response) |resp| {
+ if (self._http_transfer) |resp| {
resp.abort(error.Abort);
- self._http_response = null;
+ self._http_transfer = null;
}
page.releaseArena(self._arena);
}
@@ -484,7 +484,7 @@ pub fn clone(self: *const Response, exec: *const Execution) !*Response {
._type = self._type,
._is_redirected = self._is_redirected,
._headers = try Headers.init(.{ .obj = self._headers }, exec),
- ._http_response = null,
+ ._http_transfer = null,
};
return cloned;
}
diff --git a/src/browser/webapi/net/WebSocket.zig b/src/browser/webapi/net/WebSocket.zig
index 4d4268a88..a94d5fa3f 100644
--- a/src/browser/webapi/net/WebSocket.zig
+++ b/src/browser/webapi/net/WebSocket.zig
@@ -26,7 +26,7 @@ const Blob = @import("../Blob.zig");
const URL = @import("../../URL.zig");
const Page = @import("../../Page.zig");
-const HttpClient = @import("../../HttpClient.zig");
+const HttpClient = @import("../../../network/HttpClient.zig");
const Event = @import("../Event.zig");
const EventTarget = @import("../EventTarget.zig");
diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zig
index 278658efc..49714cc72 100644
--- a/src/browser/webapi/net/XMLHttpRequest.zig
+++ b/src/browser/webapi/net/XMLHttpRequest.zig
@@ -20,8 +20,8 @@ const std = @import("std");
const lp = @import("lightpanda");
const js = @import("../../js/js.zig");
-const HttpClient = @import("../../HttpClient.zig");
const http = @import("../../../network/http.zig");
+const Transfer = @import("../../../network/HttpClient.zig").Transfer;
const URL = @import("../../URL.zig");
const Mime = @import("../../Mime.zig");
@@ -47,7 +47,7 @@ _exec: *const Execution,
_proto: *XMLHttpRequestEventTarget,
_upload: ?*XMLHttpRequestUpload = null,
_arena: Allocator,
-_http_response: ?HttpClient.Response = null,
+_http_transfer: ?*Transfer = null,
// number of inflight requests, we can have multiple, e.g. xhr calling its own
// send from the onload callback
@@ -111,9 +111,9 @@ pub fn init(exec: *const Execution) !*XMLHttpRequest {
}
pub fn deinit(self: *XMLHttpRequest, page: *Page) void {
- if (self._http_response) |resp| {
+ if (self._http_transfer) |resp| {
resp.abort(error.Abort);
- self._http_response = null;
+ self._http_transfer = null;
}
if (self._on_ready_state_change) |func| {
@@ -182,9 +182,9 @@ pub fn setTimeout(self: *XMLHttpRequest, value: u32) void {
// TODO: url should be a union, as it can be multiple things
pub fn open(self: *XMLHttpRequest, method_: []const u8, url: [:0]const u8) !void {
// Abort any in-progress request
- if (self._http_response) |transfer| {
+ if (self._http_transfer) |transfer| {
transfer.abort(error.Abort);
- self._http_response = null;
+ self._http_transfer = null;
}
self._send_flag = false;
@@ -263,7 +263,7 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !v
self._active_requests += 1;
self._send_flag = true;
- exec.makeRequest(.{
+ const transfer = exec.newRequest(.{
.ctx = self,
.url = self._url,
.method = self._method,
@@ -276,7 +276,6 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !v
.resource_type = .xhr,
.timeout_ms = self._timeout,
.notification = session.notification,
- .start_callback = httpStartCallback,
.header_callback = httpHeaderDoneCallback,
.data_callback = httpDataCallback,
.done_callback = httpDoneCallback,
@@ -288,6 +287,23 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !v
self._send_flag = false;
return err;
};
+
+ // Held for abort() / open() / deinit; the error, shutdown and done
+ // callbacks clear it.
+ self._http_transfer = transfer;
+
+ if (comptime IS_DEBUG) {
+ log.debug(.http, "request start", .{ .method = self._method, .url = self._url, .source = "xhr" });
+ }
+
+ transfer.submit() catch |err| {
+ // If the failure aborted the transfer, httpErrorCallback already
+ // released the self-ref and cleared _http_transfer, making this
+ // release a guarded no-op.
+ self.releaseSelfRef();
+ self._send_flag = false;
+ return err;
+ };
}
// https://xhr.spec.whatwg.org/#the-upload-attribute
@@ -453,32 +469,24 @@ pub fn getResponseXML(self: *XMLHttpRequest, exec: *const Execution) !?*Node.Doc
}
}
-fn httpStartCallback(response: HttpClient.Response) !void {
- const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
- if (comptime IS_DEBUG) {
- log.debug(.http, "request start", .{ .method = self._method, .url = self._url, .source = "xhr" });
- }
- self._http_response = response;
-}
-
-fn httpHeaderCallback(response: HttpClient.Response, header: http.Header) !void {
- const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
+fn httpHeaderCallback(transfer: *Transfer, header: http.Header) !void {
+ const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.req.ctx));
const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ header.name, header.value });
try self._response_headers.append(self._arena, joined);
}
-fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResult {
- const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
+fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {
+ const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.req.ctx));
if (comptime IS_DEBUG) {
log.debug(.http, "request header", .{
.source = "xhr",
.url = self._url,
- .status = response.status(),
+ .status = transfer.responseStatus(),
});
}
- if (response.contentType()) |ct| {
+ if (transfer.contentType()) |ct| {
self._response_mime = Mime.parse(ct) catch |e| {
log.info(.http, "invalid content type", .{
.content_Type = ct,
@@ -489,18 +497,18 @@ fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResul
};
}
- var it = response.headerIterator();
+ var it = transfer.responseHeaderIterator();
while (it.next()) |hdr| {
const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ hdr.name, hdr.value });
try self._response_headers.append(self._arena, joined);
}
- self._response_status = response.status().?;
- if (response.contentLength()) |cl| {
+ self._response_status = transfer.responseStatus().?;
+ if (transfer.getContentLength()) |cl| {
self._response_len = cl;
try self._response_data.ensureTotalCapacity(self._arena, cl);
}
- self._response_url = try self._arena.dupeZ(u8, response.url());
+ self._response_url = try self._arena.dupeZ(u8, transfer.req.url);
const exec = self._exec;
@@ -515,8 +523,8 @@ fn httpHeaderDoneCallback(response: HttpClient.Response) !HttpClient.HeaderResul
return .proceed;
}
-fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void {
- const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
+fn httpDataCallback(transfer: *Transfer, data: []const u8) !void {
+ const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.req.ctx));
try self._response_data.appendSlice(self._arena, data);
try self._proto.dispatch(.progress, .{
@@ -537,7 +545,7 @@ fn httpDoneCallback(ctx: *anyopaque) !void {
// Not that the request is done, the http/client will free the transfer
// object. It isn't safe to keep it around.
- self._http_response = null;
+ self._http_transfer = null;
const exec = self._exec;
@@ -560,22 +568,22 @@ fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {
const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx));
// http client will close it after an error, it isn't safe to keep around
self.handleError(err);
- if (self._http_response != null) {
- self._http_response = null;
+ if (self._http_transfer != null) {
+ self._http_transfer = null;
}
self.releaseSelfRef();
}
fn httpShutdownCallback(ctx: *anyopaque) void {
const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx));
- self._http_response = null;
+ self._http_transfer = null;
self.releaseSelfRef();
}
pub fn abort(self: *XMLHttpRequest) void {
self.handleError(error.Abort);
- if (self._http_response) |resp| {
- self._http_response = null;
+ if (self._http_transfer) |resp| {
+ self._http_transfer = null;
resp.abort(error.Abort);
}
self.releaseSelfRef();
diff --git a/src/cdp/CDP.zig b/src/cdp/CDP.zig
index ba3875bc8..346bc7af3 100644
--- a/src/cdp/CDP.zig
+++ b/src/cdp/CDP.zig
@@ -21,9 +21,12 @@ const lp = @import("lightpanda");
const App = @import("../App.zig");
const Inbox = @import("../Inbox.zig");
-const Network = @import("../network/Network.zig");
const Notification = @import("../Notification.zig");
+
const WS = @import("../network/WS.zig");
+const Network = @import("../network/Network.zig");
+const Transfer = @import("../network/HttpClient.zig").Transfer;
+
const js = @import("../browser/js/js.zig");
const Browser = @import("../browser/Browser.zig");
const Session = @import("../browser/Session.zig");
@@ -32,7 +35,6 @@ const Page = @import("../browser/Page.zig");
const Mime = @import("../browser/Mime.zig");
const Element = @import("../browser/webapi/Element.zig");
const Label = @import("../browser/webapi/element/html/Label.zig");
-const Transfer = @import("../browser/HttpClient.zig").Transfer;
const Connection = @import("Connection.zig");
const Incrementing = @import("id.zig").Incrementing;
@@ -981,8 +983,7 @@ pub const BrowserContext = struct {
// Encode the data in base64 by default, but don't encode
// for well known content-type.
.must_encode = blk: {
- const response = msg.response;
- if (response.contentType()) |ct| {
+ if (msg.transfer.contentType()) |ct| {
const mime = try Mime.parse(ct);
if (!mime.isText()) {
@@ -1458,5 +1459,6 @@ test "cdp: syncRequest short-circuits after disconnect" {
.cookie_origin = "",
.resource_type = .fetch,
.notification = undefined,
+ .shutdown_callback = @import("../network/HttpClient.zig").noopShutdown,
}));
}
diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zig
index cc62921dc..9d8e96ca5 100644
--- a/src/cdp/domains/fetch.zig
+++ b/src/cdp/domains/fetch.zig
@@ -290,7 +290,7 @@ fn continueRequest(cmd: *CDP.Command) !void {
request.body = body;
}
- try client.interception_layer.continueRequest(transfer);
+ try client.continueIntercepted(transfer);
return cmd.sendResult(null, .{});
}
@@ -396,7 +396,7 @@ fn fulfillRequest(cmd: *CDP.Command) !void {
body = buf;
}
- try client.interception_layer.fulfillRequest(transfer, params.responseCode, params.responseHeaders orelse &.{}, body);
+ try client.fulfillIntercepted(transfer, params.responseCode, params.responseHeaders orelse &.{}, body);
return cmd.sendResult(null, .{});
}
@@ -417,7 +417,7 @@ fn failRequest(cmd: *CDP.Command) !void {
return cmd.sendResult(null, .{});
};
- defer client.interception_layer.abortRequest(transfer);
+ defer transfer.abortParked(error.Abort);
log.info(.cdp, "request intercept", .{
.state = "fail",
diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig
index 20be2d690..c5e07e2a0 100644
--- a/src/cdp/domains/network.zig
+++ b/src/cdp/domains/network.zig
@@ -27,9 +27,9 @@ const URL = @import("../../browser/URL.zig");
const Mime = @import("../../browser/Mime.zig");
const Notification = @import("../../Notification.zig");
const timestamp = @import("../../datetime.zig").timestamp;
-const Headers = @import("../../browser/HttpClient.zig").Headers;
-const Transfer = @import("../../browser/HttpClient.zig").Transfer;
-const Response = @import("../../browser/HttpClient.zig").Response;
+
+const Headers = @import("../../network/HttpClient.zig").Headers;
+const Transfer = @import("../../network/HttpClient.zig").Transfer;
const CdpStorage = @import("storage.zig");
@@ -91,7 +91,7 @@ fn setCacheDisabled(cmd: *CDP.Command) !void {
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
const client = &bc.cdp.browser.http_client;
- client.cache_layer.disabled = params.cacheDisabled;
+ client.disableCache(params.cacheDisabled);
return cmd.sendResult(null, .{});
}
@@ -362,7 +362,7 @@ pub fn httpResponseHeaderDone(arena: Allocator, bc: *CDP.BrowserContext, msg: *c
.frameId = &id.toFrameId(req.frame_id),
.requestId = &id.toRequestId(transfer),
.loaderId = &id.toLoaderId(req.loader_id),
- .response = ResponseWriter.init(arena, msg.response),
+ .response = ResponseWriter.init(arena, msg.transfer),
.hasExtraInfo = false, // TODO change after adding Network.responseReceivedExtraInfo
}, .{ .session_id = session_id });
}
@@ -447,12 +447,12 @@ pub const RequestWriter = struct {
const ResponseWriter = struct {
arena: Allocator,
- response: *const Response,
+ transfer: *Transfer,
- fn init(arena: Allocator, response: *const Response) ResponseWriter {
+ fn init(arena: Allocator, transfer: *Transfer) ResponseWriter {
return .{
.arena = arena,
- .response = response,
+ .transfer = transfer,
};
}
@@ -461,15 +461,15 @@ const ResponseWriter = struct {
}
fn _jsonStringify(self: *const ResponseWriter, jws: anytype) !void {
- const response = self.response;
+ const transfer = self.transfer;
try jws.beginObject();
{
try jws.objectField("url");
- try jws.write(response.url());
+ try jws.write(transfer.req.url);
}
- if (response.status()) |status| {
+ if (transfer.responseStatus()) |status| {
try jws.objectField("status");
try jws.write(status);
@@ -479,7 +479,7 @@ const ResponseWriter = struct {
{
const mime: Mime = blk: {
- if (response.contentType()) |ct| {
+ if (transfer.contentType()) |ct| {
break :blk try Mime.parse(ct);
}
break :blk .unknown;
@@ -493,7 +493,7 @@ const ResponseWriter = struct {
{
try jws.objectField("fromDiskCache");
- try jws.write(response.inner == .cached);
+ try jws.write(transfer._from_cache);
}
{
@@ -521,7 +521,7 @@ const ResponseWriter = struct {
// common to get these from a server (e.g. for Cache-Control), but
// Chrome joins these. So we have to too.
const arena = self.arena;
- var it = response.headerIterator();
+ var it = transfer.responseHeaderIterator();
var map: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
while (it.next()) |hdr| {
const gop = try map.getOrPut(arena, hdr.name);
@@ -866,7 +866,7 @@ test "cdp.Network: canClearBrowserCache" {
try ctx.expectSentResult(.{ .result = false }, .{ .id = 1 });
}
-test "cdp.Network: setCacheDisabled disables cache" {
+test "cdp.Network: setCacheDisabled" {
var ctx = try testing.context();
defer ctx.deinit();
_ = try ctx.loadBrowserContext(.{ .id = "BID-CD1" });
@@ -877,30 +877,4 @@ test "cdp.Network: setCacheDisabled disables cache" {
.params = .{ .cacheDisabled = true },
});
try ctx.expectSentResult(null, .{ .id = 1 });
-
- const client = ctx.cdp().browser.http_client;
- try testing.expectEqual(true, client.cache_layer.disabled);
-}
-
-test "cdp.Network: setCacheDisabled re-enables cache" {
- var ctx = try testing.context();
- defer ctx.deinit();
- _ = try ctx.loadBrowserContext(.{ .id = "BID-CD2" });
-
- try ctx.processMessage(.{
- .id = 1,
- .method = "Network.setCacheDisabled",
- .params = .{ .cacheDisabled = true },
- });
- try ctx.expectSentResult(null, .{ .id = 1 });
-
- try ctx.processMessage(.{
- .id = 2,
- .method = "Network.setCacheDisabled",
- .params = .{ .cacheDisabled = false },
- });
- try ctx.expectSentResult(null, .{ .id = 2 });
-
- const client = ctx.cdp().browser.http_client;
- try testing.expectEqual(false, client.cache_layer.disabled);
}
diff --git a/src/cdp/domains/page.zig b/src/cdp/domains/page.zig
index 355605585..cfeddd0a5 100644
--- a/src/cdp/domains/page.zig
+++ b/src/cdp/domains/page.zig
@@ -149,8 +149,8 @@ fn setLifecycleEventsEnabled(cmd: *CDP.Command) !void {
const http_client = frame._session.browser.http_client;
const http_active = http_client.http_active;
- const http_next_tick = http_client.next_tick_count;
- const total_network_activity = http_active + http_next_tick + http_client.interception_layer.intercepted;
+ const http_buffered = http_client.dispatch_count;
+ const total_network_activity = http_active + http_buffered + http_client.intercepted;
if (frame._notified_network_almost_idle.check(total_network_activity <= 2)) {
try sendPageLifecycle(bc, "networkAlmostIdle", now, frame_id, loader_id);
}
diff --git a/src/cdp/id.zig b/src/cdp/id.zig
index c324da3ec..463e8eab4 100644
--- a/src/cdp/id.zig
+++ b/src/cdp/id.zig
@@ -40,7 +40,7 @@ pub fn toLoaderId(id: u32) [14]u8 {
// requestId has special requirements. If it's the main document navigation,
// then it should match the loader id.
-const Transfer = @import("../browser/HttpClient.zig").Transfer;
+const Transfer = @import("../network/HttpClient.zig").Transfer;
pub fn toRequestId(transfer: *const Transfer) [14]u8 {
if (transfer.req.resource_type == .document) {
return toLoaderId(transfer.req.loader_id);
diff --git a/src/lightpanda.zig b/src/lightpanda.zig
index b9e9d9cc6..b81e34999 100644
--- a/src/lightpanda.zig
+++ b/src/lightpanda.zig
@@ -43,7 +43,7 @@ pub const forms = @import("browser/forms.zig");
pub const actions = @import("browser/actions.zig");
pub const structured_data = @import("browser/structured_data.zig");
pub const tools = @import("browser/tools.zig");
-pub const HttpClient = @import("browser/HttpClient.zig");
+pub const HttpClient = @import("network/HttpClient.zig");
pub const mcp = @import("mcp.zig");
pub const Agent = @import("agent/Agent.zig");
diff --git a/src/browser/HttpClient.zig b/src/network/HttpClient.zig
similarity index 63%
rename from src/browser/HttpClient.zig
rename to src/network/HttpClient.zig
index 842cc4787..6e05bb61c 100644
--- a/src/browser/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -20,19 +20,20 @@ const std = @import("std");
const lp = @import("lightpanda");
const builtin = @import("builtin");
+const Inbox = @import("../Inbox.zig");
const ArenaPool = @import("../ArenaPool.zig");
const Notification = @import("../Notification.zig");
const timestamp = @import("../datetime.zig").timestamp;
-const URL = @import("URL.zig");
-const CookieJar = @import("webapi/storage/Cookie.zig").Jar;
-
-const http = @import("../network/http.zig");
-const Network = @import("../network/Network.zig");
-
const CDP = @import("../cdp/CDP.zig");
-const Inbox = @import("../Inbox.zig");
const Watchdog = @import("../Watchdog.zig");
+const URL = @import("../browser/URL.zig");
+const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar;
+
+const http = @import("http.zig");
+const Network = @import("Network.zig");
+const Cache = @import("cache/Cache.zig");
+const RobotsGate = @import("RobotsGate.zig");
const log = lp.log;
const Allocator = std.mem.Allocator;
@@ -41,15 +42,7 @@ const IS_DEBUG = builtin.mode == .Debug;
pub const Method = http.Method;
pub const Header = http.Header;
pub const Headers = http.Headers;
-pub const ResponseHead = http.ResponseHead;
pub const HeaderIterator = http.HeaderIterator;
-const CachedResponse = @import("../network/cache/Cache.zig").CachedResponse;
-
-pub const CacheLayer = @import("../network/layer/CacheLayer.zig");
-pub const RobotsLayer = @import("../network/layer/RobotsLayer.zig");
-pub const WebBotAuthLayer = @import("../network/layer/WebBotAuthLayer.zig");
-pub const InterceptionLayer = @import("../network/layer/InterceptionLayer.zig");
-pub const DeferringLayer = @import("../network/layer/DeferringLayer.zig");
// This is loosely tied to a browser Frame. Loading all the <scripts>, doing
// XHR requests, and loading imports all happens through here. Sine the app
@@ -73,13 +66,16 @@ http_active: usize = 0,
// Our curl multi handle.
handles: http.Handles,
-// Connections currently in this client's curl_multi.
-in_use: std.DoublyLinkedList = .{},
-
-// Connections that failed to be removed from curl_multi during perform.
-dirty: std.DoublyLinkedList = .{},
-
// Whether we're currently inside a curl_multi_perform call.
+//
+// HTTP transfers no longer run user code under perform (the pump only
+// buffers; dispatch delivers), so for them this flag is vestigial. It
+// remains load-bearing solely because WebSocket dispatches JS (onopen /
+// onmessage) from inside curl's callbacks: that JS can start requests
+// (process), open sockets (trackConn -> ready_queue), close them
+// (removeConn -> dirty), or abort transfers (detachOrDeinit) while the
+// multi is live. If WS delivery ever moves onto the dispatch queue, this
+// flag — and ready_queue, dirty, and the .inflight deferral — can go.
performing: bool = false,
// Watchdog instrumentation for this client's worker thread. Wraps the poll
@@ -98,20 +94,31 @@ next_request_id: u32 = 0,
// only valid for the lifetime of the entry.
transfers: std.AutoHashMapUnmanaged(u32, *Transfer) = .empty,
-// When handles has no more available easys, requests get queued.
-queue: std.DoublyLinkedList = .{},
+// Connections currently in this client's curl_multi.
+in_use: std.DoublyLinkedList = .{},
-// A queue for things that MUST happen on the next tick.
-next_tick_queue: std.DoublyLinkedList = .{},
-next_tick_count: usize = 0,
+// Connections that failed to be removed from curl_multi during perform.
+dirty: std.DoublyLinkedList = .{},
+
+// Queue for request that are waiting an available connection (aka, easy)
+pending_queue: std.DoublyLinkedList = .{},
-// Queue is for Transfers that have no connection. ready_queue is for connections
-// that were initiated when performing == true and thus need to wait until
-// performing == false before being added. I'm hoping this is temporary and that
-// we can unify the two queues. But HTTP is being changed a lot right now, and
-// I'm trying to minimize the surface area.
+// Queue for transfers that are ready to be performed.
ready_queue: std.DoublyLinkedList = .{},
+// Queue for completed transfers that haven't had their callbacks executed yet
+dispatch_queue: std.DoublyLinkedList = .{},
+
+// Completed transfers held back by a frame's blocking request. dispatch()
+// parks them here on first sight so the gate is evaluated once per transfer
+// per blocking window, not once per tick; releaseBlocking splices them
+// back. Blocking requests are common (every bare <script src> is one), so
+// this keeps dispatch O(new work) during page load.
+gated_queue: std.DoublyLinkedList = .{},
+
+// Undelivered transfers across dispatch_queue + gated_queue.
+dispatch_count: usize = 0,
+
// The main app allocator
allocator: Allocator,
@@ -168,43 +175,24 @@ inbox: Inbox,
max_response_size: usize,
+// While a frame has a blocking (synchronous) request in flight, dispatch
+// holds back every other transfer for that frame so their callbacks can't
+// run JS while the parser is on the stack. frame_id -> blocking transfer id.
blocking_requests: std.AutoHashMapUnmanaged(u32, u32) = .empty,
-cache_layer: CacheLayer,
-robots_layer: RobotsLayer,
-web_bot_auth_layer: WebBotAuthLayer,
-interception_layer: InterceptionLayer,
-deferring_layer: DeferringLayer,
-entry_layer: Layer,
+// Count of transfers parked for CDP interception (request or auth phase).
+// They're off every other activity counter while parked; the network-idle
+// heuristics add this in.
+intercepted: usize = 0,
-pub const Layer = struct {
- ptr: *anyopaque,
- vtable: *const VTable,
+// null or referencing network.cache
+cache: ?*Cache,
- pub const VTable = struct {
- request: *const fn (*anyopaque, *Transfer) anyerror!void,
- };
+// Cached config decisions, resolved once at init.
+serve_mode: bool,
+obey_robots: bool,
- pub fn request(self: Layer, transfer: *Transfer) !void {
- return self.vtable.request(self.ptr, transfer);
- }
-};
-
-fn layerWith(self: anytype, next: Layer) Layer {
- self.next = next;
- return self.layer();
-}
-
-pub const NextTickNode = struct {
- pub const Run =
- *const fn (*Transfer, ?*anyopaque) void;
- pub const Abort = *const fn (?*anyopaque) void;
-
- node: std.DoublyLinkedList.Node = .{},
- ctx: ?*anyopaque,
- run: Run,
- abort: ?Abort = null,
-};
+robots: RobotsGate,
pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP) !void {
var handles = try http.Handles.init(network.config);
@@ -218,42 +206,18 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP)
.allocator = allocator,
.cdp = cdp,
.inbox = .{},
+ .cache = if (network.cache) |*c| c else null,
.use_proxy = http_proxy != null,
.http_proxy = http_proxy,
.tls_verify = network.config.tlsVerifyHost(),
.max_response_size = network.config.httpMaxResponseSize() orelse std.math.maxInt(u32),
- .cache_layer = .{},
- .robots_layer = .{ .allocator = allocator, .network = network },
- .web_bot_auth_layer = .{},
- .interception_layer = .{},
- .deferring_layer = .{ .allocator = allocator, .network = network },
- .entry_layer = undefined,
+ .serve_mode = network.config.mode == .serve,
+ .obey_robots = network.config.obeyRobots(),
+ .robots = .{ .allocator = allocator, .network = network },
.arena_pool = &network.app.arena_pool,
};
-
- var next = self.layer();
-
- if (network.config.obeyRobots()) {
- next = layerWith(&self.robots_layer, next);
- }
-
- if (network.config.httpCacheDir() != null) {
- next = layerWith(&self.cache_layer, next);
- }
-
- if (network.config.mode == .serve) {
- next = layerWith(&self.interception_layer, next);
- }
-
- if (network.config.webBotAuth() != null) {
- next = layerWith(&self.web_bot_auth_layer, next);
- }
-
- next = layerWith(&self.deferring_layer, next);
-
- self.entry_layer = next;
}
pub fn deinit(self: *Client) void {
@@ -261,9 +225,9 @@ pub fn deinit(self: *Client) void {
if (comptime IS_DEBUG) {
lp.assert(
- self.next_tick_count == 0,
- "next_tick_count must be 0",
- .{ .value = self.next_tick_count },
+ self.dispatch_count == 0,
+ "dispatch_count must be 0",
+ .{ .value = self.dispatch_count },
);
}
@@ -274,8 +238,7 @@ pub fn deinit(self: *Client) void {
self.allocator.free(owned);
}
- self.robots_layer.deinit(self.allocator);
- self.deferring_layer.deinit();
+ self.robots.deinit();
self.blocking_requests.deinit(self.allocator);
self.transfers.deinit(self.allocator);
self.inbox.deinit(self.arena_pool);
@@ -289,19 +252,20 @@ pub fn findTransfer(self: *Client, id: u32) ?*Transfer {
return self.transfers.get(id);
}
-pub fn layer(self: *Client) Layer {
- return .{
- .ptr = self,
- .vtable = &.{ .request = _request },
- };
+pub fn incrReqId(self: *Client) u32 {
+ const id = self.next_request_id +% 1;
+ self.next_request_id = id;
+ return id;
}
// Set a user agent override. Both the raw UA string and the pre-formatted
// "User-Agent: <ua>" header string are allocated from self.allocator.
pub fn setUserAgentOverride(self: *Client, ua: []const u8) !void {
self.clearUserAgentOverride();
+
const override = try self.allocator.dupeZ(u8, ua);
errdefer self.allocator.free(override);
+
const header = try std.fmt.allocPrintSentinel(self.allocator, "User-Agent: {s}", .{ua}, 0);
self.user_agent_override = override;
self.user_agent_header_override = header;
@@ -339,6 +303,14 @@ pub fn setTlsVerify(self: *Client, verify: bool) !void {
self.tls_verify = verify;
}
+pub fn disableCache(self: *Client, disable: bool) void {
+ if (disable) {
+ self.cache = null;
+ } else {
+ self.cache = if (self.network.cache) |*c| c else null;
+ }
+}
+
// Restrictive since it'll only work if there are no inflight requests. In some
// cases, the libcurl documentation is clear that changing settings while a
// connection is inflight is undefined. It doesn't say anything about CURLOPT_PROXY,
@@ -395,14 +367,16 @@ pub fn abort(self: *Client) void {
// After the kill loop, every internal list should drain itself via
// each transfer's deinit:
// - self.transfers : transfers.remove(self.id)
- // - self.queue : unlinked if _queued is set
+ // - self.pending_queue : unlinked if _queued is set
// - self.in_use / self.ready_queue : via removeConn
// - self.dirty : drained at end of each perform; nothing left here
// Any non-empty list means a transfer escaped cleanup — assert so we
// catch the regression rather than silently leaking on next use.
if (comptime IS_DEBUG) {
std.debug.assert(self.transfers.size == 0);
- std.debug.assert(self.queue.first == null);
+ std.debug.assert(self.pending_queue.first == null);
+ std.debug.assert(self.dispatch_queue.first == null);
+ std.debug.assert(self.gated_queue.first == null);
std.debug.assert(self.in_use.first == null);
std.debug.assert(self.ready_queue.first == null);
std.debug.assert(self.dirty.first == null);
@@ -420,7 +394,7 @@ pub fn abortOwner(self: *Client, owner: *Owner) void {
var n = owner.websockets.first;
while (n) |node| {
n = node.next;
- const ws: *@import("webapi/net/WebSocket.zig") = @fieldParentPtr("_owner_node", node);
+ const ws: *@import("../browser/webapi/net/WebSocket.zig") = @fieldParentPtr("_owner_node", node);
ws.kill();
}
if (comptime IS_DEBUG) {
@@ -455,116 +429,36 @@ pub fn abortRequests(_: *Client, owner: *Owner) void {
// Cherry-pick only Fetch interception responses
const DrainMode = enum { all, sync_wait };
-pub fn tick(self: *Client, timeout_ms: u32, mode: DrainMode) !void {
- if (self.inbox.terminated) {
- return error.ClientDisconnected;
- }
-
- try self.drainNextTickQueue();
- try self.drainQueue();
- try self.perform(@intCast(timeout_ms));
- // perform/processMessages just released a batch of connections back to
- // the pool. Drain again so queued transfers can use them this tick
- // instead of waiting for the next runner iteration.
- try self.drainQueue();
- // Dispatch CDP messages here, not inside perform: perform recurses
- // via processOneMessage's redirect path (perform → processMessages
- // → processOneMessage → perform), and dispatching CDP from that
- // nested call would fire CDP handlers mid-redirect, defeating the
- // "safe points only" guarantee.
- try self.drainInbox(mode);
+// One-shot convenience: create and submit in a single call. HttpClient takes
+// ownership of req.headers; do not pair with `errdefer headers.deinit()`.
+// Callers that have no headers of their own can leave req.headers at its
+// (empty) default — the client fills in its baseline headers (user agent,
+// etc.).
+pub fn request(self: *Client, req: Request, owner: ?*Owner) anyerror!void {
+ const transfer = try self.newRequest(req, owner);
+ return transfer.submit();
}
-pub fn runNextTick(
- self: *Client,
- transfer: *Transfer,
- ctx: ?*anyopaque,
- params: struct { run: NextTickNode.Run, abort: ?NextTickNode.Abort = null },
-) !void {
- transfer._next_tick_node = .{ .ctx = ctx, .run = params.run, .abort = params.abort };
-
- self.next_tick_count += 1;
- self.next_tick_queue.append(&transfer._next_tick_node.?.node);
-}
-
-fn cancelNextTick(self: *Client, transfer: *Transfer) void {
- if (transfer._next_tick_node) |*ntn| {
- self.next_tick_queue.remove(&ntn.node);
- self.next_tick_count -= 1;
-
- if (ntn.abort) |abort_cb| {
- abort_cb(ntn.ctx);
- }
- }
-}
-
-fn drainNextTickQueue(self: *Client) !void {
- var remaining = self.next_tick_count;
- while (remaining > 0) : (remaining -= 1) {
- const node = self.next_tick_queue.popFirst() orelse break;
- defer self.next_tick_count -= 1;
- const n: *NextTickNode = @fieldParentPtr("node", node);
-
- const transfer: *Transfer = @fieldParentPtr(
- "_next_tick_node",
- @as(*?NextTickNode, @ptrCast(n)),
- );
-
- const ntn = n.*;
- transfer._next_tick_node = null;
- ntn.run(transfer, ntn.ctx);
- }
-}
-
-fn drainQueue(self: *Client) !void {
- while (self.queue.popFirst()) |queue_node| {
- const transfer: *Transfer = @fieldParentPtr("_node", queue_node);
- const conn = self.network.getConnection() orelse {
- self.queue.prepend(queue_node);
- return;
- };
- // Bridge state to .created so a failure inside makeRequest before
- // any commit cleans up via the abort below. makeRequest flips to
- // .inflight on a successful trackConn.
- transfer.state = .created;
- self.makeRequest(conn, transfer) catch |err| {
- if (transfer.state == .created) {
- transfer.abort(err);
- }
- return err;
- };
- }
-}
-
-// last layer
-pub fn _request(_: *anyopaque, transfer: *Transfer) !void {
- return transfer.client.process(transfer);
-}
-
-// HttpClient takes ownership of req.headers; do not pair with
-// `errdefer headers.deinit()`
-pub fn request(self: *Client, req: Request, owner: ?*Owner) !void {
- _ = try self.requestT(req, owner);
-}
-
-// Like `request`, but returns the created `*Transfer`. The caller does not own
-// the returned `*Transfer` and must thus use it with care. From the moment this
-// function is entered, the HttpClient owns `req` — specifically `req.headers`
-// On success, transfer.deinit eventually frees it. On any failure path inside
-// this function, we free it before returning the error.
-fn requestT(self: *Client, req: Request, owner: ?*Owner) !*Transfer {
+// Create a request without submitting it. The caller owns the returned
+// transfer until transfer.submit(). On error, header is freed. On success,
+// headers is owned by the transfer.
+pub fn newRequest(self: *Client, req: Request, owner: ?*Owner) anyerror!*Transfer {
const arena = self.arena_pool.acquire(.small, "Request.arena") catch |err| {
req.headers.deinit();
return err;
};
const transfer = blk: {
+ var owned = req;
errdefer {
- req.headers.deinit();
+ owned.headers.deinit();
self.arena_pool.release(arena);
}
- var owned = req;
+ if (owned.headers.headers == null) {
+ owned.headers = try self.newHeaders();
+ }
+
// Most of the time, the req data will outlive the transfer. But not
// always. The most problematic case is with a QueuedNavigation which
// is freed quite quickly and would definetly not survive a queued
@@ -614,13 +508,7 @@ fn requestT(self: *Client, req: Request, owner: ?*Owner) !*Transfer {
break :blk t;
};
- // From here, transfer owns req+arena. Any subsequent failure flows
- // through transfer.deinit (or transfer.abort), which handles headers
- // via req.deinit. Do NOT free headers directly past this point.
-
- // Register for id-based lookup. putNoClobber would fail if request_id
- // collides (i.e. we've wrapped through 2^32 requests and the old
- // transfer is still alive — practically never).
+ // From here the transfer owns `req` and `arena`
self.transfers.putNoClobber(self.allocator, transfer.id, transfer) catch |err| {
transfer.deinit();
return err;
@@ -631,115 +519,375 @@ fn requestT(self: *Client, req: Request, owner: ?*Owner) !*Transfer {
transfer.owner = o;
}
- // From this point forward, the transfer owns `req` and `arena`. If the
- // layer chain fails before any layer commits the transfer to an external
- // owner (queue / multi handle / pending interception), we clean up here
- // via transfer.abort which fires error_callback and deinits. `.created`
- // means no commit happened — anything else is held by an owner that
- // will clean up.
+ return transfer;
+}
- // Synthetic schemes never touch the network or the layer chain — they skip
- // robots/cache/interception and deliver on the next tick
- if (Synthetic.isSynthetic(req.url)) {
- // The 2nd transfer is the callback context. We don't actually use it,
- // we're just sticking transfer in there to have something.
- self.runNextTick(transfer, null, .{ .run = Synthetic.run }) catch |err| {
- if (transfer.state == .created) {
- transfer.abort(err);
- }
- return err;
- };
- return transfer;
+pub fn tick(self: *Client, timeout_ms: u32, mode: DrainMode) !void {
+ if (self.inbox.terminated) {
+ return error.ClientDisconnected;
}
- self.entry_layer.request(transfer) catch |err| {
- if (transfer.state == .created) {
- transfer.abort(err);
+ self.dispatchCompleted();
+
+ // settle upfront so that we return any completed connections back to the pool
+ try self.settleConns();
+
+ try self.startPending();
+
+ const running = try self.perform();
+
+ // WebSocket can release connections during perform, so settle any
+ // connections that are now free. TODO: move WebSocket to the same queue-
+ // based approach as HTTP connections, and then this can be removed
+ try self.settleConns();
+
+ var has_processed = true;
+ if (try self.processMessages() == false) {
+ // No messages were processed. We need to wait for I/O. The network
+ // layer will wake this up if there's acticity.
+ if (running > 0 or self.cdp_link_active) {
+ {
+ self.heartbeat.enterWait();
+ defer self.heartbeat.exitWait();
+ try self.handles.poll(&.{}, @intCast(timeout_ms));
+ }
+ // poll only waits, so we do the perform -> process dance again
+ _ = try self.perform();
+ has_processed = try self.processMessages();
}
- return err;
- };
+ } else {
+ // If we DO have processed messages, we don't wan to wait / poll. We
+ // want to proceses completions and return to the caller ASAP so that it
+ // can check progress, e.g. maybe the message we processed cause "load"
+ // to fire, and that's what it was waiting for.
+ }
- return transfer;
+ try self.startPending();
+
+ self.dispatchCompleted();
+
+ // dispatch CDP commands
+ try self.drainInbox(mode);
}
-// Non-network URL schemes whose response is synthesized in-process rather than
-// fetched, think blob data URLs.
-const Synthetic = struct {
- const data_url = @import("data_url.zig");
+// get libcurl to make progress on the connections (easys) we've given it. This
+// can mean starting a request or processing calllbacks. libcurl does not allow
+// calling libcurl from callbacks, and that's something we never do (hence why
+// Transfers have an event queue and dispatchCompleted is is own thing)...EXCEPT
+// for WebSocket connections. So we need the performing guard to protect against
+// invalid curl usage. TODO: Move WebSocket connections to work like HTTP
+// connections.
+fn perform(self: *Client) !c_int {
+ self.performing = true;
+ defer self.performing = false;
+ return self.handles.perform();
+}
- fn isSynthetic(url: []const u8) bool {
- return std.mem.startsWith(u8, url, "data:") or std.mem.startsWith(u8, url, "blob:");
+// TODO: only needed because WebSocket connections can still open-close mid-perform
+fn settleConns(self: *Client) !void {
+ // Apply queued closes that happenened during "perform".
+ while (self.dirty.popFirst()) |node| {
+ const conn: *http.Connection = @fieldParentPtr("node", node);
+ self.handles.remove(conn) catch |err| {
+ lp.assert(false, "multi_remove_handle", .{
+ .err = err,
+ .in_use = conn.in_use,
+ .added = conn.debug_added,
+ .removed = conn.debug_removed,
+ .remove_err = conn.debug_remove_err,
+ });
+ };
+ conn.debug_removed = 2;
+ self.releaseConn(conn);
}
- fn run(transfer: *Transfer, _: ?*anyopaque) void {
- // prevents a callback that triggers a navigation queue from killing
- // this transfer from under us.
- transfer.state = .completing;
- defer transfer.deinit();
+ // Apply queued opens that happenened during "perform".
+ while (self.ready_queue.popFirst()) |node| {
+ const conn: *http.Connection = @fieldParentPtr("node", node);
+ try self.trackConn(conn);
+ }
+}
- const fulfilled = build(transfer) catch |err| {
- transfer.req.error_callback(transfer.req.ctx, err);
+// Deliver completed response. This is the ONLY place user callbacks run,
+// so a callback is free to start new requests, abort transfers, or tear down
+// its frame. The client is never inside libcurl here.
+//
+// This should be as simple as: iterate the `dispatch_queue` and dispatch each
+// transfer...EXCEPT...a frame can have a blocking request. So, as we iterate
+// we find out if a transfer is gated behind a blocking request and move it
+// from the `dispatch_queue` to the `gated_queue`. That way, every call to
+// `dispatchCompleted` doesn't keep checking the same gated transfers over and
+// over.
+fn dispatchCompleted(self: *Client) void {
+ // pop is safest as it allows anything to manipulate the queue as necessary
+ // (e.g. a frame could be aborted)s
+ while (self.dispatch_queue.popFirst()) |n| {
+ const transfer: *Transfer = @fieldParentPtr("_queue_node", n);
+ if (self.isGated(transfer)) {
+ transfer._gated = true;
+ self.gated_queue.append(n);
+ continue;
+ }
+ self.dispatch_count -= 1;
+ transfer.deliver();
+ }
+}
+
+// Is this transfer gated behind a blockig request
+fn isGated(self: *const Client, transfer: *const Transfer) bool {
+ if (transfer.req.internal) {
+ // internal transfers are never blocked (e.g. robots.txt)
+ return false;
+ }
+ if (self.blocking_requests.count() == 0) {
+ // O(1) quick check.
+ return false;
+ }
+ const blocking_id = self.blocking_requests.get(transfer.req.frame_id) orelse return false;
+ return transfer.id != blocking_id;
+}
+
+fn startPending(self: *Client) !void {
+ while (self.pending_queue.popFirst()) |queue_node| {
+ const transfer: *Transfer = @fieldParentPtr("_node", queue_node);
+ const conn = self.network.getConnection() orelse {
+ self.pending_queue.prepend(queue_node);
return;
};
- deliver(&transfer.req, &fulfilled) catch |err| {
- transfer.req.error_callback(transfer.req.ctx, err);
+ // Bridge state to .created so a failure inside makeRequest before
+ // any commit cleans up via the abort below. makeRequest flips to
+ // .inflight on a successful trackConn.
+ transfer.state = .created;
+ self.makeRequest(conn, transfer) catch |err| {
+ if (transfer.state == .created) {
+ transfer.abort(err);
+ }
+ return err;
};
}
+}
- fn build(transfer: *Transfer) !FulfilledResponse {
- const arena = transfer.arena;
- const url = transfer.req.url;
+const SubmitFrom = enum { start, after_intercept, network };
+
+// Process a transfer, passing it through our pipeline. A transfer an move off
+// the pipeline(e.g. while parked waiting for a robots.txt check) and then
+// pushed back onto it, which is what `from` helps us achieve.
+fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {
+ sw: switch (from) {
+ .start => {
+ if (self.network.web_bot_auth) |wba| {
+ const authority = URL.getHost(transfer.req.url);
+ try wba.signRequest(transfer.arena, &transfer.req.headers, authority);
+ }
- var body: []const u8 = "";
- var content_type: []const u8 = "";
+ if (self.serve_mode) {
+ transfer._notify_cdp = true;
+ transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });
+
+ var wait_for_interception = false;
+ transfer.req.notification.dispatch(.http_request_intercept, &.{
+ .transfer = transfer,
+ .wait_for_interception = &wait_for_interception,
+ });
+ if (wait_for_interception) {
+ // The CDP listener stashed the transfer id and will
+ // resolve it via continueIntercepted / fulfillIntercepted
+ // / abortParked.
+ self.intercepted += 1;
+ transfer.park(.intercept_request);
+ if (comptime IS_DEBUG) {
+ log.debug(.http, "wait for interception", .{ .intercepted = self.intercepted });
+ }
+ return;
+ }
+ }
+ continue :sw SubmitFrom.after_intercept;
+ },
+ .after_intercept => {
+ if (try self.cacheLookup(transfer)) {
+ // response came from the cache, we're done
+ return;
+ }
+ if (self.obey_robots and !transfer.req.internal) {
+ switch (try self.robots.check(transfer)) {
+ .allowed => {},
+ .blocked => return transfer.failAsync(error.RobotsBlocked),
+ .pending => return,
+ }
+ }
+ continue :sw SubmitFrom.network;
+ },
+ .network => try self.processTransfer(transfer),
+ }
+}
- if (std.mem.startsWith(u8, url, "data:")) {
- const parsed = try data_url.parse(arena, url);
- content_type = parsed.content_type;
- body = parsed.body;
- } else {
- // blob: — resolved against the owning frame's registry.
- const owner = transfer.owner orelse return error.BlobNotFound;
- const blob_urls = owner.blob_urls orelse return error.BlobNotFound;
- const blob = blob_urls.get(url) orelse return error.BlobNotFound;
- content_type = blob._mime;
- body = blob._slice;
+// RobotsGate resumption. The robots gate is the last step before the
+// network, so an allowed transfer goes straight there.
+pub fn resumeAfterRobots(self: *Client, transfer: *Transfer) !void {
+ return self.pipeline(transfer, .network);
+}
+
+fn findHeader(headers: []const http.Header, name: []const u8) ?[]const u8 {
+ for (headers) |hdr| {
+ if (std.ascii.eqlIgnoreCase(hdr.name, name)) {
+ return hdr.value;
}
+ }
+ return null;
+}
- // A blob with no type yields no Content-Type header.
- const headers = if (content_type.len > 0) blk: {
- const h = try arena.alloc(http.Header, 1);
- h[0] = .{ .name = "content-type", .value = content_type };
- break :blk h;
- } else &[_]http.Header{};
+// Returns true if the request was served from the cache (response buffered
+// for dispatch). On a miss the transfer is tagged so completion can store
+// the response; on an expired-with-validators entry the request becomes a
+// conditional revalidation.
+fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
+ const cache = self.cache orelse return false;
+
+ const req = &transfer.req;
+ if (req.method != .GET) {
+ return false;
+ }
+
+ const arena = transfer.arena;
+ var iter = req.headers.iterator();
+ const req_headers = try iter.collect(arena);
+
+ const cached = cache.get(arena, .{
+ .url = req.url,
+ .timestamp = std.time.timestamp(),
+ .request_headers = req_headers.items,
+ }) orelse {
+ transfer._cache_intent = .store;
+ return false;
+ };
- return .{
- .url = url,
- .body = body,
- .status = 200,
- .headers = headers,
- };
+ if (cached.expired == false) {
+ try transfer.bufferCached(cached);
+ return true;
}
- fn deliver(req: *Request, fulfilled: *const FulfilledResponse) !void {
- const response = Response.fromFulfilled(req.ctx, fulfilled);
- if (req.start_callback) |cb| {
- try cb(response);
- }
+ if (cached.metadata.hasValidators() == false) {
+ // Expired and no validators
+ cached.data.deinit();
+ cache.evict(req.url);
+ transfer._cache_intent = .store;
+ return false;
+ }
- const result = try req.header_callback(response);
- if (result == .abort) {
- return error.Abort;
- }
+ // expired but with validators
+ log.debug(.cache, "revalidate with etag", .{
+ .url = req.url,
+ .etag = cached.metadata.etag,
+ .last_modified = cached.metadata.last_modified,
+ });
+ if (cached.metadata.etag) |etag| {
+ try req.headers.add(try std.fmt.allocPrintSentinel(arena, "If-None-Match: {s}", .{etag}, 0));
+ }
+ if (cached.metadata.last_modified) |lm| {
+ try req.headers.add(try std.fmt.allocPrintSentinel(arena, "If-Modified-Since: {s}", .{lm}, 0));
+ }
+ transfer._cache_intent = .{ .revalidate = cached };
+ return false;
+}
+
+// 304 on a revalidation: renew the stored entry from the fresh headers and
+// serve the stale copy. Returns true if the stale entry became the response.
+fn cacheRevalidated(self: *Client, transfer: *Transfer) !bool {
+ if (transfer._cache_intent != .revalidate) {
+ return false;
+ }
+ const rh = transfer.res.header orelse return false;
+ if (rh.status != 304) {
+ return false;
+ }
+ // could have been disabled in-between
+ const cache = self.cache orelse return false;
+
+ const stale = transfer._cache_intent.revalidate;
+ transfer._cache_intent = .none;
+
+ cache.renew(transfer.arena, .{
+ .url = transfer.req.url,
+ .timestamp = std.time.timestamp(),
+ .headers = transfer.res.headers,
+ }) catch |err| {
+ log.warn(.cache, "renew failed", .{ .err = err });
+ };
- if (fulfilled.body) |b| {
- if (b.len > 0) {
- try req.data_callback(response, b);
+ try transfer.bufferCached(stale);
+ return true;
+}
+
+// Store a cacheable response at completion, from the materialized response
+// headers and the buffered body. Failures are logged, never fatal — the
+// consumer gets its response either way.
+fn cacheStore(self: *Client, transfer: *Transfer) void {
+ switch (transfer._cache_intent) {
+ .none => return,
+ .store => {},
+ .revalidate => |stale| {
+ // Not a 304: the stored entry is out of date. Drop it and store
+ // the fresh response below.
+ stale.data.deinit();
+ },
+ }
+ // could have been disabled while waiting of the response
+ const cache = self.cache orelse return;
+
+ transfer._cache_intent = .none;
+
+ const arena = transfer.arena;
+ const rh = &(transfer.res.header orelse return);
+ const headers = transfer.res.headers;
+
+ const vary = findHeader(headers, "vary");
+ const maybe_cm = Cache.tryCache(
+ arena,
+ std.time.timestamp(),
+ transfer.req.url,
+ rh.status,
+ rh.contentType(),
+ findHeader(headers, "cache-control"),
+ vary,
+ findHeader(headers, "age"),
+ findHeader(headers, "etag"),
+ findHeader(headers, "last-modified"),
+ findHeader(headers, "set-cookie") != null,
+ findHeader(headers, "authorization") != null,
+ ) catch |err| {
+ log.warn(.http, "cache eligibility", .{ .err = err });
+ return;
+ };
+ var metadata = maybe_cm orelse return;
+
+ var vary_headers: std.ArrayList(http.Header) = .empty;
+ if (vary) |vary_str| {
+ var req_it = transfer.req.headers.iterator();
+ while (req_it.next()) |hdr| {
+ var vary_iter = std.mem.splitScalar(u8, vary_str, ',');
+ while (vary_iter.next()) |part| {
+ const name = std.mem.trim(u8, part, &std.ascii.whitespace);
+ if (std.ascii.eqlIgnoreCase(hdr.name, name)) {
+ const owned: http.Header = .{
+ .name = arena.dupe(u8, hdr.name) catch return,
+ .value = arena.dupe(u8, hdr.value) catch return,
+ };
+ vary_headers.append(arena, owned) catch return;
+ }
}
}
- try req.done_callback(req.ctx);
}
-};
+
+ metadata.headers = headers;
+ metadata.vary_headers = vary_headers.items;
+
+ if (comptime IS_DEBUG) {
+ log.debug(.browser, "http cache", .{ .key = transfer.req.url, .metadata = metadata });
+ }
+ cache.put(metadata, transfer.res.stream_buffer.items) catch |err| {
+ log.warn(.http, "cache put failed", .{ .err = err });
+ };
+}
const SyncContext = struct {
allocator: Allocator,
@@ -753,18 +901,18 @@ const SyncContext = struct {
status: u16 = 0,
body: std.ArrayList(u8),
- fn headerCallback(response: Response) anyerror!HeaderResult {
- const self: *SyncContext = @ptrCast(@alignCast(response.ctx));
- lp.assert(response.status() != null, "HttpClient.SyncRequest.headerCallback", .{ .value = response.status() });
- self.status = response.status().?;
- if (response.contentLength()) |cl| {
+ fn headerCallback(transfer: *Transfer) anyerror!Transfer.HeaderResult {
+ const self: *SyncContext = @ptrCast(@alignCast(transfer.req.ctx));
+ lp.assert(transfer.responseStatus() != null, "HttpClient.SyncRequest.headerCallback", .{ .value = transfer.responseStatus() });
+ self.status = transfer.responseStatus().?;
+ if (transfer.getContentLength()) |cl| {
try self.body.ensureTotalCapacity(self.allocator, cl);
}
return .proceed;
}
- fn dataCallback(response: Response, data: []const u8) anyerror!void {
- const self: *SyncContext = @ptrCast(@alignCast(response.ctx));
+ fn dataCallback(transfer: *Transfer, data: []const u8) anyerror!void {
+ const self: *SyncContext = @ptrCast(@alignCast(transfer.req.ctx));
try self.body.appendSlice(self.allocator, data);
}
@@ -788,18 +936,13 @@ pub fn syncRequest(self: *Client, allocator: Allocator, req: Request) !SyncRespo
if (self.inbox.terminated) {
// request() takes ownership of req.headers on every path; we return
// before calling it, so free the curl_slist here to avoid leaking it.
- req.headers.deinit();
+ req.deinit();
return error.ClientDisconnected;
}
var sync_ctx = SyncContext{ .allocator = allocator, .body = .empty };
errdefer sync_ctx.body.deinit(allocator);
- const expected_id = self.nextReqId();
- const frame_id = req.frame_id;
- try self.blocking_requests.putNoClobber(self.allocator, frame_id, expected_id);
- defer _ = self.blocking_requests.remove(frame_id);
-
var r = req;
r.ctx = &sync_ctx;
r.header_callback = SyncContext.headerCallback;
@@ -807,7 +950,16 @@ pub fn syncRequest(self: *Client, allocator: Allocator, req: Request) !SyncRespo
r.done_callback = SyncContext.doneCallback;
r.error_callback = SyncContext.errorCallback;
r.shutdown_callback = SyncContext.shutdownCallback;
- const transfer = try self.requestT(r, null);
+ const transfer = try self.newRequest(r, null);
+
+ const frame_id = req.frame_id;
+ self.blocking_requests.putNoClobber(self.allocator, frame_id, transfer.id) catch |err| {
+ transfer.deinit();
+ return err;
+ };
+ defer self.releaseBlocking(frame_id);
+
+ try transfer.submit();
while (sync_ctx.completion == .in_progress) {
self.tick(200, .sync_wait) catch |err| {
@@ -839,10 +991,7 @@ pub fn syncRequest(self: *Client, allocator: Allocator, req: Request) !SyncRespo
}
}
-// Above, request will not process if there's an interception request. In such
-// cases, the interceptor is expected to call resume to continue the transfer
-// or transfer.abort() to abort it.
-fn process(self: *Client, transfer: *Transfer) !void {
+fn processTransfer(self: *Client, transfer: *Transfer) !void {
// libcurl doesn't allow recursive calls, if we're in a `perform()` operation
// then we _have_ to queue this.
if (self.performing == false) {
@@ -851,27 +1000,30 @@ fn process(self: *Client, transfer: *Transfer) !void {
}
}
- self.queue.append(&transfer._node);
+ self.pending_queue.append(&transfer._node);
transfer.state = .queued;
}
-pub fn nextReqId(self: *Client) u32 {
- return self.next_request_id +% 1;
-}
-
-pub fn incrReqId(self: *Client) u32 {
- const id = self.next_request_id +% 1;
- self.next_request_id = id;
- return id;
-}
-
-// Same restriction as changeProxy. Should be ok since this is only called on
-// BrowserContext deinit.
-pub fn restoreOriginalProxy(self: *Client) !void {
- try self.ensureNoActiveConnection();
-
- self.http_proxy = self.network.config.httpProxy();
- self.use_proxy = self.http_proxy != null;
+// A blocking request is complete. Any completed transfer that was placed in the
+// gated_queue because of it can now be placed back in the dispatch queue.
+fn releaseBlocking(self: *Client, frame_id: u32) void {
+ _ = self.blocking_requests.remove(frame_id);
+ // items were added to the gate in order, so walking backwards restores that
+ // order. (Order might not matter, but preserving it costs nothing)
+
+ // No callback is executed here. Nothing mutates our queues, so iterating
+ // it this way is safe.
+ var node = self.gated_queue.last;
+ while (node) |n| {
+ node = n.prev;
+ const transfer: *Transfer = @fieldParentPtr("_queue_node", n);
+ if (transfer.req.frame_id != frame_id) {
+ continue;
+ }
+ transfer._gated = false;
+ self.gated_queue.remove(n);
+ self.dispatch_queue.prepend(n);
+ }
}
fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyerror!void {
@@ -905,81 +1057,8 @@ fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyer
transfer._conn = conn;
transfer.state = .inflight;
- if (transfer.req.start_callback) |cb| {
- cb(Response.fromTransfer(transfer)) catch |err| {
- // We're now committed to the multi. transfer.abort fires the
- // error_callback and tears down (removeConn handles the
- // already-in-multi case via the dirty queue).
- transfer.abort(err);
- return err;
- };
- }
-
- // Start the request (and move along any other request). This used to call
- // self.perform(0) but that can also execute callbacks. Normally, that
- // wouldn't be so bad. But curl can synchronously fire callbacks for the
- // request we JUST added, which we do not want (it results in incorrect
- // execution).
- self.performing = true;
- defer self.performing = false;
- _ = try self.handles.perform();
-}
-
-fn perform(self: *Client, timeout_ms: c_int) anyerror!void {
- const running = blk: {
- self.performing = true;
- defer self.performing = false;
-
- break :blk try self.handles.perform();
- };
-
- // Process dirty connections — return them to Network pool.
- while (self.dirty.popFirst()) |node| {
- const conn: *http.Connection = @fieldParentPtr("node", node);
- self.handles.remove(conn) catch |err| {
- lp.assert(false, "multi_remove_handle", .{
- .err = err,
- .in_use = conn.in_use,
- .added = conn.debug_added,
- .removed = conn.debug_removed,
- .remove_err = conn.debug_remove_err,
- });
- };
- conn.debug_removed = 2;
- self.releaseConn(conn);
- }
-
- while (self.ready_queue.popFirst()) |node| {
- const conn: *http.Connection = @fieldParentPtr("node", node);
- try self.trackConn(conn);
- }
-
- // We just processed completions; their done_callbacks may have
- // scheduled microtasks (JS continuations) or queued new transfers.
- // Return without polling so the caller (_tick) can run macrotasks
- // and re-evaluate. Otherwise we'd sleep on cdp_link_active for up
- // to timeout_ms while pending JS work sits idle.
- if (try self.processMessages()) {
- return;
- }
-
- // Poll for HTTP I/O. The Network thread will call curl_multi_wakeup
- // on our multi handle whenever it pushes to our inbox, so we drop
- // out of poll promptly even when we have no curl handles in flight
- // — but ONLY if a producer is actually wired up. `cdp_link_active`
- // is set by Server.handleConnection once network.registerCdp has
- // returned; in tests (which never register) and during the
- // pre-handshake window the flag stays false and we don't waste a
- // poll timeout waiting for a wakeup that won't arrive.
- if (running > 0 or self.cdp_link_active) {
- // when cdp_link_active == true, the network thread will unblock this
- // by calling wakup on our multi.
- self.heartbeat.enterWait();
- defer self.heartbeat.exitWait();
- try self.handles.poll(&.{}, timeout_ms);
- }
-
- _ = try self.processMessages();
+ // Start the request (and move along any other request).
+ _ = try self.perform();
}
// Drain any CDP messages the Network thread pushed into our inbox
@@ -1024,7 +1103,7 @@ fn drainInbox(self: *Client, mode: DrainMode) !void {
// ping/close/disconnect (control frames must be observed). CDP data
// messages are filtered: only the four Fetch interception methods
// are safe to dispatch from inside a JS callback (they mutate
-// transfer state via InterceptionLayer; they don't touch page /
+// transfer state via the interception gate; they don't touch page /
// session / V8 state). The check is exact on the parsed `method`
// field — no substring matching against raw JSON.
//
@@ -1064,16 +1143,59 @@ fn isTeardownMethod(method: []const u8) bool {
std.mem.eql(u8, method, "Page.close");
}
-pub fn isRedirectStatus(status: u16) bool {
+fn isRedirectStatus(status: u16) bool {
return switch (status) {
301, 302, 303, 307, 308 => true,
else => false,
};
}
-fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Transfer) !bool {
- // State at entry: .inflight = conn (multi just delivered a completion).
+fn processMessages(self: *Client) !bool {
+ var processed = false;
+ while (try self.handles.readMessage()) |msg| {
+ switch (msg.conn.transport) {
+ .http => |transfer| {
+ // On success, processOneMessage owns the transfer's fate: it
+ // either buffered it for dispatch, parked it, or deinit'd it.
+ // Only the throw path cleans up here.
+ const done = self.processOneMessage(msg, transfer) catch |err| blk: {
+ log.err(.http, "process_messages", .{ .err = err, .req = transfer });
+ transfer.state = .completing;
+ transfer.requestFailed(err);
+ if (transfer._detached_conn) |c| {
+ // Conn was removed from handles during redirect reconfiguration
+ // but not re-added. Release it directly to avoid double-remove.
+ self.in_use.remove(&c.node);
+ self.http_active -= 1;
+ self.releaseConn(c);
+ transfer._detached_conn = null;
+ }
+ transfer.deinit();
+ break :blk true;
+ };
+ if (done) {
+ processed = true;
+ }
+ },
+ .websocket => |ws| {
+ // ws_active will be decremented through the call to disconnected
+ if (msg.err) |err| switch (err) {
+ error.GotNothing => ws.disconnected(null),
+ else => ws.disconnected(err),
+ } else {
+ // Clean close - no error
+ ws.disconnected(null);
+ }
+
+ processed = true;
+ },
+ .none => unreachable,
+ }
+ }
+ return processed;
+}
+fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Transfer) !bool {
// Workaround for libcurl Brotli trailing-byte rejection.
//
// Some CDNs (e.g. CloudFront serving Brave Search) emit a Brotli stream
@@ -1094,6 +1216,15 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
break :blk err;
} else null;
+ if (transfer.state == .aborted) {
+ // detachInPerform ran while this response was in flight; there is
+ // nothing to deliver and nobody else owns the transfer anymore.
+ self.removeConn(msg.conn);
+ transfer._conn = null;
+ transfer.deinit();
+ return true;
+ }
+
if (effective_err == null or effective_err.? == error.RecvError) {
transfer.detectAuthChallenge(msg.conn);
}
@@ -1107,9 +1238,9 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
&.{ .transfer = transfer, .wait_for_interception = &wait_for_interception },
);
if (wait_for_interception) {
- self.interception_layer.intercepted += 1;
+ self.intercepted += 1;
if (comptime IS_DEBUG) {
- log.debug(.http, "wait for auth interception", .{ .intercepted = self.interception_layer.intercepted });
+ log.debug(.http, "wait for auth interception", .{ .intercepted = self.intercepted });
}
// Whether or not this is a blocking request, we're not going
@@ -1149,14 +1280,18 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
conn.debug_added = 2;
transfer._detached_conn = null;
- _ = try self.perform(0);
+ // Get the redirect on the wire now. Any completion messages
+ // this produces are picked up by the processMessages loop
+ // we were called from — no recursion into the pump.
+ _ = try self.perform();
return false;
},
// error_callback surfaces this as a TypeError.
.@"error" => {
- transfer.state = .completing;
- transfer.requestFailed(error.RedirectNotAllowed, true);
+ self.removeConn(msg.conn);
+ transfer._conn = null;
+ transfer.failAsync(error.RedirectNotAllowed);
return true;
},
// Don't follow; fall through to deliver the 3xx as the final
@@ -1166,8 +1301,9 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
}
}
- // Transfer is done (success or error). Caller (processMessages) owns deinit.
- // Return true = done (caller will deinit), false = continues (redirect/auth).
+ // Transfer is done (success or error). Materialize the response into the
+ // transfer's arena, release the conn, and buffer the events — user
+ // callbacks run later, from dispatch(), never from here.
// When the server closes the TLS onnection without a close_notify alert,
// BoringSSL reports RecvError. If we already received valid HTTP headers,
@@ -1181,94 +1317,30 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
break :blk std.ascii.eqlIgnoreCase(hdr.value, "close");
};
- // Transition to .completing so re-entrant aborts from user callbacks
- // defer their teardown to processMessages. (_conn carries through
- // from .inflight; nothing to set here.)
- transfer.state = .completing;
-
if (effective_err != null and !is_conn_close_recv) {
- transfer.requestFailed(transfer.res.callback_error orelse effective_err.?, true);
+ self.removeConn(msg.conn);
+ transfer._conn = null;
+ transfer.failAsync(transfer.res.callback_error orelse effective_err.?);
return true;
}
- if (!transfer.res.header_done_called) {
- // In case of request w/o data, we need to call the header done
- // callback now.
- const result = try transfer.headerDoneCallback(msg.conn);
- switch (result) {
- .proceed => {},
- .handled => return true,
- .abort => {
- transfer.requestFailed(error.Abort, true);
- return true;
- },
- }
- }
-
- const body = transfer.res.stream_buffer.items;
+ try transfer.materializeResponse(msg.conn);
- // Replay buffered body through user's data_callback.
- if (body.len > 0) {
- try transfer.req.data_callback(Response.fromTransfer(transfer), body);
-
- if (transfer.state == .aborted) {
- transfer.requestFailed(error.Abort, true);
- return true;
- }
- }
-
- // release conn ASAP so that it's available; some done_callbacks
- // will load more resources. State stays .completing — the
- // processMessages caller still owns deinit.
+ // Release the conn before any of this response's callbacks can run —
+ // they'll want it for the next resource.
self.removeConn(msg.conn);
transfer._conn = null;
- try transfer.req.done_callback(transfer.req.ctx);
+ if (try self.cacheRevalidated(transfer)) {
+ return true;
+ }
+ self.cacheStore(transfer);
+ transfer._cdp_content_length = transfer.getContentLength() orelse 0;
+ try transfer.bufferEvents(transfer.res.stream_buffer.items);
return true;
}
-fn processMessages(self: *Client) !bool {
- var processed = false;
- while (try self.handles.readMessage()) |msg| {
- switch (msg.conn.transport) {
- .http => |transfer| {
- const done = self.processOneMessage(msg, transfer) catch |err| blk: {
- log.err(.http, "process_messages", .{ .err = err, .req = transfer });
- transfer.requestFailed(err, true);
- if (transfer._detached_conn) |c| {
- // Conn was removed from handles during redirect reconfiguration
- // but not re-added. Release it directly to avoid double-remove.
- self.in_use.remove(&c.node);
- self.http_active -= 1;
- self.releaseConn(c);
- transfer._detached_conn = null;
- }
- break :blk true;
- };
- if (done) {
- transfer.deinit();
- processed = true;
- }
- },
- .websocket => |ws| {
- // ws_active will be decremented through the call to disconnected
- if (msg.err) |err| switch (err) {
- error.GotNothing => ws.disconnected(null),
- else => ws.disconnected(err),
- } else {
- // Clean close - no error
- ws.disconnected(null);
- }
-
- processed = true;
- },
- .none => unreachable,
- }
- }
- return processed;
-}
-
pub fn trackConn(self: *Client, conn: *http.Connection) !void {
if (self.performing) {
conn.in_use = false;
@@ -1337,19 +1409,10 @@ fn ensureNoActiveConnection(self: *const Client) !void {
}
}
-pub const HeaderResult = enum {
- /// Continue processing normally.
- proceed,
- /// Caller took ownership of the response; stop w/o error or abort.
- handled,
- /// Abort the Transfer,
- abort,
-};
-
pub const Request = struct {
- pub const StartCallback = *const fn (response: Response) anyerror!void;
- pub const HeaderCallback = *const fn (response: Response) anyerror!HeaderResult;
- pub const DataCallback = *const fn (response: Response, data: []const u8) anyerror!void;
+ pub const StartCallback = *const fn (transfer: *Transfer) anyerror!void;
+ pub const HeaderCallback = *const fn (transfer: *Transfer) anyerror!Transfer.HeaderResult;
+ pub const DataCallback = *const fn (transfer: *Transfer, data: []const u8) anyerror!void;
pub const DoneCallback = *const fn (ctx: *anyopaque) anyerror!void;
pub const ErrorCallback = *const fn (ctx: *anyopaque, err: anyerror) void;
pub const ShutdownCallback = *const fn (ctx: *anyopaque) void;
@@ -1384,7 +1447,9 @@ pub const Request = struct {
loader_id: u32,
method: Method,
url: [:0]const u8,
- headers: http.Headers,
+ // Empty by default; the client fills in its baseline headers (user
+ // agent, sec-ch-ua, accept-language) when none are supplied.
+ headers: http.Headers = .{ .headers = null },
body: ?[]const u8 = null,
cookie_jar: ?*CookieJar,
cookie_origin: [:0]const u8,
@@ -1410,7 +1475,14 @@ pub const Request = struct {
data_callback: DataCallback = Noop.dataCallback,
done_callback: DoneCallback = Noop.doneCallback,
error_callback: ErrorCallback = Noop.errorCallback,
- shutdown_callback: ?ShutdownCallback = null,
+
+ // Fired exactly once if the transfer is torn down by its owner (frame /
+ // worker teardown, a superseding navigation) before completing. It must
+ // drop any stored *Transfer and must not run JS. Deliberately has no
+ // default: anyone holding a *Transfer forgets this at their peril, so
+ // every caller decides — pass `HttpClient.noopShutdown` to opt out,
+ // knowingly.
+ shutdown_callback: ShutdownCallback,
pub fn getCookieString(self: *Request, arena: Allocator) !?[:0]const u8 {
const jar = self.cookie_jar orelse return null;
@@ -1433,159 +1505,148 @@ pub const Request = struct {
}
};
-pub const FulfilledResponse = struct {
+pub const SyncResponse = struct {
status: u16,
- url: [:0]const u8,
- headers: []const http.Header,
- body: ?[]const u8,
+ body: std.ArrayList(u8),
- pub fn contentType(self: *const FulfilledResponse) ?[]const u8 {
- for (self.headers) |hdr| {
- if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) return hdr.value;
- }
- return null;
+ pub fn deinit(self: *SyncResponse, allocator: Allocator) void {
+ self.body.deinit(allocator);
}
};
-pub const StableResponse = struct {
- ctx: *anyopaque,
- status: u16,
- url: [:0]const u8,
- headers: []const http.Header,
- body: ?[]const u8,
+// CDP Fetch.continueWithAuth: resume a transfer parked on an auth
+// challenge. The auth retry goes straight back to the network — it already
+// passed the request-side pipeline on its first attempt.
+pub fn continueTransfer(self: *Client, transfer: *Transfer) !void {
+ if (comptime IS_DEBUG) {
+ log.debug(.http, "continue transfer", .{ .intercepted = self.intercepted });
+ }
- pub fn contentType(self: *const StableResponse) ?[]const u8 {
- for (self.headers) |hdr| {
- if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) return hdr.value;
+ transfer.unpark();
+ self.processTransfer(transfer) catch |err| {
+ if (transfer.state == .created) {
+ transfer.abort(err);
}
- return null;
+ return err;
+ };
+}
+
+// CDP Fetch.continueRequest: resume a transfer parked at the interception
+// gate, re-entering the pipeline at the step after interception. The CDP
+// command may have mutated req (url / method / headers / body) first.
+pub fn continueIntercepted(self: *Client, transfer: *Transfer) !void {
+ if (comptime IS_DEBUG) {
+ lp.assert(self.intercepted > 0, "Client.continueIntercepted", .{ .value = self.intercepted });
}
-};
-pub const Response = struct {
- ctx: *anyopaque,
- inner: union(enum) {
- transfer: *Transfer,
- cached: *const CachedResponse,
- fulfilled: *const FulfilledResponse,
- stable: *const StableResponse,
- },
+ transfer.unpark();
+ self.pipeline(transfer, .after_intercept) catch |err| {
+ if (transfer.state == .created) {
+ transfer.abort(err);
+ }
+ return err;
+ };
+}
- pub fn fromTransfer(transfer: *Transfer) Response {
- return .{ .ctx = transfer.req.ctx, .inner = .{ .transfer = transfer } };
+// CDP Fetch.fulfillRequest: resolve a parked transfer with an interceptor-
+// supplied response. A redirect status with a Location header is followed
+// like a real network redirect; anything else is buffered for dispatch —
+// the consumer's callbacks never run from the CDP command itself.
+pub fn fulfillIntercepted(
+ self: *Client,
+ transfer: *Transfer,
+ status: u16,
+ headers: []const http.Header,
+ body: ?[]const u8,
+) !void {
+ if (comptime IS_DEBUG) {
+ lp.assert(self.intercepted > 0, "Client.fulfillIntercepted", .{ .value = self.intercepted });
}
- pub fn fromCached(ctx: *anyopaque, resp: *const CachedResponse) Response {
- return .{ .ctx = ctx, .inner = .{ .cached = resp } };
- }
+ transfer.unpark();
- pub fn fromFulfilled(ctx: *anyopaque, fulfilled: *const FulfilledResponse) Response {
- return .{ .ctx = ctx, .inner = .{ .fulfilled = fulfilled } };
- }
+ const followed = blk: {
+ if (isRedirectStatus(status) == false) {
+ break :blk false;
+ }
+ const location = findHeader(headers, "location") orelse break :blk false;
+ try self.fulfillRedirect(transfer, status, headers, location);
+ break :blk true;
+ };
- pub fn fromStable(stable: *const StableResponse) Response {
- return .{ .ctx = stable.ctx, .inner = .{ .stable = stable } };
+ if (followed) {
+ return;
}
- pub fn status(self: Response) ?u16 {
- return switch (self.inner) {
- .transfer => |t| if (t.res.header) |rh| rh.status else null,
- .cached => |c| c.metadata.status,
- .fulfilled => |f| f.status,
- .stable => |s| s.status,
- };
- }
+ transfer.bufferFulfilled(status, headers, body) catch |err| {
+ if (transfer.state == .created) {
+ transfer.abort(err);
+ }
+ return err;
+ };
+}
- pub fn contentType(self: Response) ?[]const u8 {
- return switch (self.inner) {
- .transfer => |t| if (t.res.header) |*rh| rh.contentType() else null,
- .cached => |c| c.metadata.content_type,
- .fulfilled => |f| f.contentType(),
- .stable => |s| s.contentType(),
- };
- }
+fn fulfillRedirect(
+ self: *Client,
+ transfer: *Transfer,
+ status: u16,
+ headers: []const http.Header,
+ location: []const u8,
+) !void {
+ errdefer |err| if (transfer.state == .created) transfer.abort(err);
- pub fn contentLength(self: Response) ?u32 {
- return switch (self.inner) {
- .transfer => |t| t.getContentLength(),
- .cached => |c| switch (c.data) {
- .buffer => |buf| @intCast(buf.len),
- .file => |f| @intCast(f.len),
- },
- .fulfilled => |f| if (f.body) |b| @intCast(b.len) else null,
- .stable => |s| if (s.body) |b| @intCast(b.len) else null,
- };
+ // retrieve cookies from the fulfilled response's headers.
+ if (transfer.req.cookie_jar) |jar| {
+ for (headers) |hdr| {
+ if (std.ascii.eqlIgnoreCase(hdr.name, "set-cookie")) {
+ try jar.populateFromResponse(transfer.req.url, hdr.value);
+ }
+ }
}
- pub fn redirectCount(self: Response) ?u32 {
- return switch (self.inner) {
- .transfer => |t| if (t.res.header) |rh| rh.redirect_count else null,
- .cached, .fulfilled, .stable => 0,
- };
- }
+ try transfer.applyRedirectTarget(transfer.req.url, location, status);
+ try self.pipeline(transfer, .after_intercept);
+}
- pub fn url(self: Response) [:0]const u8 {
- return switch (self.inner) {
- .transfer => |t| t.req.url,
- .cached => |c| c.metadata.url,
- .fulfilled => |f| f.url,
- .stable => |s| s.url,
- };
+// The explicit opt-out for Request.shutdown_callback, for callers that hold
+// no reference to the transfer (or whose cleanup runs through other paths).
+pub fn noopShutdown(_: *anyopaque) void {}
+
+const Noop = struct {
+ fn headerCallback(_: *Transfer) !Transfer.HeaderResult {
+ return .proceed;
}
+ fn dataCallback(_: *Transfer, _: []const u8) !void {}
+ fn doneCallback(_: *anyopaque) !void {}
+ fn errorCallback(_: *anyopaque, _: anyerror) void {}
+};
+
+// An opaque-from-the-outside handle that Frame / WorkerGlobalScope embed
+// to track the HTTP transfers + WebSockets they own.
+pub const Owner = struct {
+ transfers: std.DoublyLinkedList = .{},
+ websockets: std.DoublyLinkedList = .{},
+
+ // The owning Frame's / WorkerGlobalScope's blob: registry,
+ blob_urls: ?*const std.StringHashMapUnmanaged(*Blob) = null,
- pub fn headerIterator(self: Response) HeaderIterator {
- return switch (self.inner) {
- .transfer => |t| t.responseHeaderIterator(),
- .cached => |c| HeaderIterator{ .list = .{ .list = c.metadata.headers } },
- .fulfilled => |f| HeaderIterator{ .list = .{ .list = f.headers } },
- .stable => |s| HeaderIterator{ .list = .{ .list = s.headers } },
- };
- }
+ const WebSocket = @import("../browser/webapi/net/WebSocket.zig");
+ const Blob = @import("../browser/webapi/Blob.zig");
- pub fn abort(self: Response, err: anyerror) void {
- switch (self.inner) {
- .transfer => |t| t.abort(err),
- .cached, .fulfilled, .stable => {},
- }
+ pub fn addTransfer(self: *Owner, t: *Transfer) void {
+ self.transfers.append(&t.owner_node);
}
- pub fn format(self: Response, writer: *std.Io.Writer) !void {
- return switch (self.inner) {
- .transfer => |t| try t.format(writer),
- .cached => |c| try c.format(writer),
- .fulfilled => |f| try writer.print("fulfilled {s}", .{f.url}),
- .stable => |s| writer.print("stable {s}", .{s.url}),
- };
+ pub fn removeTransfer(self: *Owner, t: *Transfer) void {
+ self.transfers.remove(&t.owner_node);
}
- pub fn toStable(self: Response, arena: std.mem.Allocator) !StableResponse {
- const new_url = try arena.dupeZ(u8, self.url());
-
- var headers: std.ArrayListUnmanaged(http.Header) = .{};
- var it = self.headerIterator();
- while (it.next()) |hdr| {
- try headers.append(arena, .{
- .name = try arena.dupe(u8, hdr.name),
- .value = try arena.dupe(u8, hdr.value),
- });
- }
-
- return .{
- .ctx = self.ctx,
- .status = self.status() orelse 0,
- .url = new_url,
- .headers = headers.items,
- .body = null,
- };
+ pub fn addWS(self: *Owner, ws: *WebSocket) void {
+ self.websockets.append(&ws._owner_node);
}
-};
-
-pub const SyncResponse = struct {
- status: u16,
- body: std.ArrayList(u8),
- pub fn deinit(self: *SyncResponse, allocator: Allocator) void {
- self.body.deinit(allocator);
+ pub fn removeWS(self: *Owner, ws: *WebSocket) void {
+ self.websockets.remove(&ws._owner_node);
}
};
@@ -1612,7 +1673,7 @@ pub const Transfer = struct {
_conn: ?*http.Connection = null,
req: Request,
- res: Transfer.Response = .{},
+ res: Response = .{},
client: *Client,
start_time: u64,
@@ -1634,8 +1695,30 @@ pub const Transfer = struct {
// for when a Transfer is queued in the client.queue
_node: std.DoublyLinkedList.Node = .{},
- // for when a Transfer is queued for the next tick.
- _next_tick_node: ?NextTickNode = null,
+ // Buffered response ordered events awaiting dispatch. Today, ArrayList is
+ // overkill. _events will be at most len == 4 and almost always len == 4.
+ // BUT, when WebSockets move to a dispatch mode AND we re-add streaming,
+ // this will become more dynamic.
+ _events: std.ArrayList(Event) = .{},
+
+ // controls if _queue_node is in client.dispatch_queu (false) or
+ // client.gated_queue (true)
+ _gated: bool = false,
+
+ _queue_node: std.DoublyLinkedList.Node = .{},
+
+ // Mirror this transfer's lifecycle as CDP Network.* events (serve mode).
+ _notify_cdp: bool = false,
+
+ // Response came from the http cache; drives the served-from-cache
+ // notification at delivery.
+ _from_cache: bool = false,
+
+ // What the cache wants from this transfer's response.
+ _cache_intent: CacheIntent = .none,
+
+ // Content length reported on the CDP loadingFinished event.
+ _cdp_content_length: usize = 0,
// Debug canary: set on the first deinit, so that if a second deinit on the
// same instance is called, we have a double free. The memory _could_ be
@@ -1653,6 +1736,11 @@ pub const Transfer = struct {
// linked into client.queue.
queued,
+ // Response events are buffered on `_events`, waiting for dispatch
+ // to deliver them. `_queue_node` is linked into
+ // client.dispatch_queue. No conn is held.
+ buffered,
+
// Conn (in `_conn`) is in the multi handle; libcurl owns the
// lifecycle. processOneMessage will eventually fire callbacks
// for us.
@@ -1682,18 +1770,25 @@ pub const Transfer = struct {
// waiting for continueWithAuth.
intercept_auth,
- // RobotsLayer holds the transfer pending a robots.txt fetch.
+ // RobotsGate holds the transfer pending a robots.txt fetch.
robots,
};
- // Layer-facing: park the transfer for an external owner. The caller
+ pub const HeaderResult = enum {
+ /// Continue processing normally.
+ proceed,
+ /// Abort the Transfer,
+ abort,
+ };
+
+ // Gate-facing: park the transfer for an external owner. The caller
// must be holding the transfer in the request flow (state == .created).
pub fn park(self: *Transfer, by: ParkedBy) void {
lp.assert(self.state == .created, "Transfer.park", .{ .state = self.state });
self.state = .{ .parked = by };
}
- // Layer-facing: take the transfer out of .parked and return it to
+ // Gate-facing: take the transfer out of .parked and return it to
// the request flow (.created). This assumes pre-inflight handling (i.e. the
// transfer was in .created before being parked). This is true today, but
// could become false if Request Interception ever supports the "response"
@@ -1714,13 +1809,41 @@ pub const Transfer = struct {
switch (self.state.parked) {
.robots => {},
.intercept_request, .intercept_auth => {
- const intercept_layer = &self.client.interception_layer;
- lp.assert(intercept_layer.intercepted > 0, "Transfer.leaveIntercept", .{ .value = intercept_layer.intercepted });
- intercept_layer.intercepted -= 1;
+ lp.assert(self.client.intercepted > 0, "Transfer.leaveIntercept", .{ .value = self.client.intercepted });
+ self.client.intercepted -= 1;
},
}
}
+ // Submit a transfer created with Client.newRequest, entering the
+ // request pipeline. Consumes the caller's ownership unconditionally:
+ // on failure the transfer is already cleaned up (error_callback fired,
+ // memory freed) — do not deinit it, and do not touch it after this.
+ pub fn submit(self: *Transfer) anyerror!void {
+ // Synthetic schemes never touch the network — no robots, cache, or
+ // interception. The response is materialized here and delivered by
+ // the dispatcher like any other transfer.
+ if (Synthetic.isSynthetic(self.req.url)) {
+ Synthetic.buffer(self) catch |err| {
+ // Buffer the failure too (blob-not-found is a normal
+ // JS-visible TypeError). Delivered like any other error (e.g
+ // asynchronously in deliver).
+ self.failAsync(err);
+ };
+ return;
+ }
+
+ self.client.pipeline(self, .start) catch |err| {
+ if (self.state == .created) {
+ // pipeline failed before the Transfer was handed over to a
+ // another owner (e.g. the multi, the pending queue, ...)
+ // so we're still responsible for it.
+ self.abort(err);
+ }
+ return err;
+ };
+ }
+
pub fn deinit(self: *Transfer) void {
if (comptime IS_DEBUG) {
lp.assert(self._deinited == false, "Transfer.deinit", .{ .id = self.id });
@@ -1737,15 +1860,31 @@ pub const Transfer = struct {
// abort during navigation) leaves a dangling _node in the queue
// that the next tick would pop and hand to libcurl → UAF.
if (self.state == .queued) {
- self.client.queue.remove(&self._node);
+ self.client.pending_queue.remove(&self._node);
+ }
+
+ // Same for the dispatch queue: a buffered transfer killed
+ // out-of-band (owner teardown, abort) must not be delivered.
+ if (self.state == .buffered) {
+ if (self._gated) {
+ self.client.gated_queue.remove(&self._queue_node);
+ } else {
+ self.client.dispatch_queue.remove(&self._queue_node);
+ }
+ self.client.dispatch_count -= 1;
+ }
+
+ // A pending revalidation entry owns cache resources (possibly an
+ // open file). It's resolved at completion; if we die first, release.
+ switch (self._cache_intent) {
+ .revalidate => |stale| stale.data.deinit(),
+ .none, .store => {},
}
// Drop the id→*Transfer index entry before freeing the memory.
// Any concurrent CDP lookup by id will now see this transfer as gone.
_ = self.client.transfers.remove(self.id);
- self.client.cancelNextTick(self);
-
self.req.deinit();
if (self.owner) |o| {
o.removeTransfer(self);
@@ -1766,29 +1905,35 @@ pub const Transfer = struct {
// a transfer. Don't reach for kill() or requestFailed() directly —
// they're internal helpers.
pub fn abort(self: *Transfer, err: anyerror) void {
- self.requestFailed(err, true);
+ self.requestFailed(err);
self.detachOrDeinit();
}
// Abort a transfer that an external owner (CDP interception) is holding in a
// .parked state. Unlike abort(), this is re-entrancy safe so that if
// requestFailed causes a teardown/navigate, this won't be killed again
- // Mirrors InterceptionLayer.fulfillRequest. unpark asserts the transfer is
+ // Mirrors Client.fulfillIntercepted. unpark asserts the transfer is
// actually parked.
pub fn abortParked(self: *Transfer, err: anyerror) void {
self.unpark();
self.state = .completing;
defer self.deinit();
- self.requestFailed(err, true);
+ self.requestFailed(err);
}
// Owner-driven teardown: fires shutdown_callback (not error_callback)
// and otherwise behaves like abort. Called by Client.abortOwner /
- // abortRequests when a Frame / WGS is being torn down.
+ // abortRequests when a Frame / WGS is being torn down. Any buffered,
+ // undelivered events are dropped — the consumer is going away with us.
fn kill(self: *Transfer) void {
- if (self.req.shutdown_callback) |cb| {
- cb(self.req.ctx);
+ if (self._notify_cdp and !self._notified_fail) {
+ self._notified_fail = true;
+ self.req.notification.dispatch(.http_request_fail, &.{
+ .transfer = self,
+ .err = error.Shutdown,
+ });
}
+ self.req.shutdown_callback(self.req.ctx);
self.detachOrDeinit();
}
@@ -1838,7 +1983,7 @@ pub const Transfer = struct {
// releases it once libcurl is done.
self.state = .aborted;
self.req.start_callback = null;
- self.req.shutdown_callback = null;
+ self.req.shutdown_callback = noopShutdown;
self.req.header_callback = Noop.headerCallback;
self.req.data_callback = Noop.dataCallback;
self.req.done_callback = Noop.doneCallback;
@@ -1849,28 +1994,167 @@ pub const Transfer = struct {
}
}
- // Internal failure-notification helper. Latches via _notified_fail so
- // multiple paths racing to report the same failure only fire one
- // notification. Goes through transfer.req — so layer wrappers
- // (InterceptContext, CacheContext) see the failure and can propagate
- // it up the chain.
- //
- // Not part of the external API: callers cancelling a transfer should
- // use transfer.abort(err) instead, which goes through this and also
- // handles the deinit / detach side. The internal HttpClient flow uses
- // this directly (from processOneMessage) because it's already paired
- // with the natural processMessages → transfer.deinit handoff.
- //
- // execute_callback=true → fires error_callback. false → fires
- // shutdown_callback (used by Frame shutdown / WGS teardown).
- fn requestFailed(self: *Transfer, err: anyerror, comptime execute_callback: bool) void {
- if (self._notified_fail) return;
+ // Some type of failure, either internal or external, either explicit or
+ // implicit. transfer.abort() is the external and explicit path. There are
+ // various internal callers, but the most common is a callback error.
+ fn requestFailed(self: *Transfer, err: anyerror) void {
+ if (self._notified_fail) {
+ return;
+ }
self._notified_fail = true;
- if (execute_callback) {
- self.req.error_callback(self.req.ctx, err);
- } else if (self.req.shutdown_callback) |cb| {
- cb(self.req.ctx);
+ if (self._notify_cdp) {
+ self.req.notification.dispatch(.http_request_fail, &.{
+ .transfer = self,
+ .err = err,
+ });
+ }
+
+ self.req.error_callback(self.req.ctx, err);
+ }
+
+ // A consumer callback failed mid-delivery: latch the failure, notify,
+ // free. Only called from deliver().
+ fn failDelivery(self: *Transfer, err: anyerror) void {
+ log.err(.http, "delivery callback", .{ .err = err, .req = self });
+ self.requestFailed(err);
+ self.deinit();
+ }
+
+ // Fail the transfer asynchronously: the error is buffered and
+ // error_callback fires from the dispatcher, never from the caller's
+ // stack.
+ pub fn failAsync(self: *Transfer, err: anyerror) void {
+ self._events.append(self.arena, .{ .err = err }) catch {
+ // Can't buffer (OOM): failing inline beats losing the error.
+ return self.abort(err);
+ };
+ self.scheduleDispatch();
+ }
+
+ // Commit the transfer's buffered events for delivery. From here on the
+ // dispatcher owns the transfer's fate; deinit unlinks it if it dies
+ // out-of-band first.
+ fn scheduleDispatch(self: *Transfer) void {
+ if (comptime IS_DEBUG) {
+ lp.assert(
+ self.state == .created or self.state == .inflight,
+ "Transfer.scheduleDispatch",
+ .{ .state = self.state },
+ );
+ }
+ self.state = .buffered;
+ self.client.dispatch_queue.append(&self._queue_node);
+ self.client.dispatch_count += 1;
+ }
+
+ // Buffer the standard success event sequence. `body` is either owned by
+ // transfer.arena OR, through some other mechanism, outlives the transfer.
+ fn bufferEvents(self: *Transfer, body: []const u8) !void {
+ try self._events.ensureUnusedCapacity(self.arena, 4);
+ self._events.appendAssumeCapacity(.start);
+ self._events.appendAssumeCapacity(.header);
+ if (body.len > 0) {
+ self._events.appendAssumeCapacity(.{ .data = body });
+ }
+ self._events.appendAssumeCapacity(.done);
+ self.scheduleDispatch();
+ }
+
+ fn setResponseHead(self: *Transfer, status: u16, content_type: ?[]const u8) void {
+ self.res.header = .{
+ .url = self.req.url.ptr,
+ .status = status,
+ .redirect_count = self._redirect_count,
+ };
+ if (content_type) |ct| {
+ var hdr = &self.res.header.?;
+ const len = @min(ct.len, http.ResponseHead.MAX_CONTENT_TYPE_LEN);
+ hdr._content_type_len = len;
+ @memcpy(hdr._content_type[0..len], ct[0..len]);
+ }
+ }
+
+ // Serve a cache entry as this transfer's response. Takes ownership of
+ // `cached` (file-backed bodies are read into the arena and closed).
+ fn bufferCached(self: *Transfer, cached: Cache.CachedResponse) !void {
+ const arena = self.arena;
+
+ const body: []const u8 = switch (cached.data) {
+ .buffer => |b| b,
+ .file => |f| blk: {
+ defer f.file.close();
+ const buf = try arena.alloc(u8, f.len);
+ const n = try f.file.preadAll(buf, f.offset);
+ break :blk buf[0..n];
+ },
+ };
+
+ self.setResponseHead(cached.metadata.status, cached.metadata.content_type);
+ self.res.headers = cached.metadata.headers;
+ self._from_cache = true;
+ self._cdp_content_length = body.len;
+ try self.bufferEvents(body);
+ }
+
+ // Materialize an interceptor-supplied response (CDP fulfillRequest).
+ // `headers` and `body` are caller-owned; copy everything that must
+ // survive until dispatch.
+ fn bufferFulfilled(self: *Transfer, status: u16, headers: []const http.Header, body: ?[]const u8) !void {
+ const arena = self.arena;
+
+ const owned = try arena.alloc(http.Header, headers.len);
+ var content_type: ?[]const u8 = null;
+ for (headers, 0..) |hdr, i| {
+ owned[i] = .{
+ .name = try arena.dupe(u8, hdr.name),
+ .value = try arena.dupe(u8, hdr.value),
+ };
+ if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) {
+ content_type = owned[i].value;
+ }
+ }
+
+ const owned_body: []const u8 = if (body) |b| try arena.dupe(u8, b) else "";
+
+ self.setResponseHead(status, content_type);
+ self.res.headers = owned;
+ self._cdp_content_length = owned_body.len;
+ try self.bufferEvents(owned_body);
+ }
+
+ // Copy everything the response needs out of the conn and into the
+ // transfer arena. After this, delivery never touches libcurl state —
+ // the conn can be released and reused before the consumer sees a byte.
+ fn materializeResponse(self: *Transfer, conn: *http.Connection) !void {
+ const arena = self.arena;
+
+ if (self.res.header == null) {
+ try self.buildResponseHeader(conn);
+ }
+ // buildResponseHeader stores a curl-owned url pointer; re-anchor it
+ // in the arena so it survives the conn release.
+ self.res.header.?.url = (try arena.dupeZ(u8, std.mem.span(self.res.header.?.url))).ptr;
+
+ var it = HeaderIterator{ .curl = .{ .conn = conn } };
+ const headers = try it.collect(arena);
+ self.res.headers = headers.items;
+
+ if (self.req.cookie_jar) |jar| {
+ for (self.res.headers) |hdr| {
+ if (std.ascii.eqlIgnoreCase(hdr.name, "set-cookie")) {
+ jar.populateFromResponse(self.req.url, hdr.value) catch |err| {
+ log.err(.http, "set cookie", .{ .err = err, .req = self });
+ return err;
+ };
+ }
+ }
+ }
+
+ if (self.getContentLength()) |cl| {
+ if (cl > self.client.max_response_size) {
+ return error.ResponseTooLarge;
+ }
}
}
@@ -1950,7 +2234,7 @@ pub const Transfer = struct {
if (conn.getResponseHeader("content-type", 0)) |ct| {
var hdr = &self.res.header.?;
const value = ct.value;
- const len = @min(value.len, ResponseHead.MAX_CONTENT_TYPE_LEN);
+ const len = @min(value.len, http.ResponseHead.MAX_CONTENT_TYPE_LEN);
hdr._content_type_len = len;
@memcpy(hdr._content_type[0..len], value[0..len]);
}
@@ -2078,51 +2362,13 @@ pub const Transfer = struct {
// before interception process.
pub fn abortAuthChallenge(self: *Transfer) void {
if (comptime IS_DEBUG) {
- log.debug(.http, "abort auth transfer", .{ .intercepted = self.client.interception_layer.intercepted });
+ log.debug(.http, "abort auth transfer", .{ .intercepted = self.client.intercepted });
}
// The transfer is still .parked(.intercept_auth)
self.abortParked(error.AbortAuthChallenge);
}
- // headerDoneCallback is called once the headers have been read.
- // It can be called either on dataCallback or once the request for those
- // w/o body.
- fn headerDoneCallback(transfer: *Transfer, conn: *const http.Connection) !HeaderResult {
- lp.assert(transfer.res.header_done_called == false, "Transfer.headerDoneCallback", .{});
- defer transfer.res.header_done_called = true;
-
- try transfer.buildResponseHeader(conn);
-
- if (transfer.req.cookie_jar) |jar| {
- var i: usize = 0;
- while (true) {
- const ct = conn.getResponseHeader("set-cookie", i);
- if (ct == null) break;
- jar.populateFromResponse(transfer.req.url, ct.?.value) catch |err| {
- log.err(.http, "set cookie", .{ .err = err, .req = transfer });
- return err;
- };
- i += 1;
- if (i >= ct.?.amount) break;
- }
- }
-
- if (transfer.getContentLength()) |cl| {
- if (cl > transfer.client.max_response_size) {
- return error.ResponseTooLarge;
- }
- }
-
- const result = transfer.req.header_callback(Client.Response.fromTransfer(transfer)) catch |err| {
- log.err(.http, "header_callback", .{ .err = err, .req = transfer });
- return err;
- };
-
- if (result == .proceed and transfer.state == .aborted) return .abort;
- return result;
- }
-
fn dataCallback(buffer: [*]const u8, chunk_count: usize, chunk_len: usize, data: *anyopaque) callconv(.c) usize {
// libcurl should only ever emit 1 chunk at a time
if (comptime IS_DEBUG) {
@@ -2180,120 +2426,231 @@ pub const Transfer = struct {
return @intCast(chunk_len);
}
+ // Response-view accessors. Callbacks receive the *Transfer once the
+ // response is materialized, so these are always safe from a callback;
+ // header/status are null before that point.
+ pub fn responseStatus(self: *const Transfer) ?u16 {
+ const rh = self.res.header orelse return null;
+ return rh.status;
+ }
+
+ pub fn contentType(self: *Transfer) ?[]const u8 {
+ if (self.res.header) |*rh| {
+ return rh.contentType();
+ }
+ return null;
+ }
+
+ pub fn redirectCount(self: *const Transfer) ?u32 {
+ const rh = self.res.header orelse return null;
+ return rh.redirect_count;
+ }
+
pub fn responseHeaderIterator(self: *Transfer) HeaderIterator {
- // We always have a real curl request here. We handle injection up in InterceptionLayer.
- const c = self._conn;
- lp.assert(c != null, "Transfer.responseHeaderIterator", .{ .value = c != null });
- // If we have a connection, than this is a real curl request and we
- // iterate through the header that curl maintains.
- return .{ .curl = .{ .conn = c.? } };
+ if (self.res.headers.len > 0 or self._conn == null) {
+ return .{ .list = .{ .list = self.res.headers } };
+ }
+ // Mid-stream (pump time): headers aren't materialized yet, read
+ // them off the live conn.
+ return .{ .curl = .{ .conn = self._conn.? } };
}
- // This function should be called during the dataCallback. Calling it after
- // such as in the doneCallback is guaranteed to return null.
pub fn getContentLength(self: *const Transfer) ?u32 {
const cl = self.getContentLengthRawValue() orelse return null;
return std.fmt.parseInt(u32, cl, 10) catch null;
}
fn getContentLengthRawValue(self: *const Transfer) ?[]const u8 {
+ // Materialized headers (dispatch time, any source).
+ for (self.res.headers) |hdr| {
+ if (std.ascii.eqlIgnoreCase(hdr.name, "content-length")) {
+ return hdr.value;
+ }
+ }
+
+ // Mid-stream (curl's write callback): read from the live conn.
if (self._conn) |c| {
- // If we have a connection, than this is a normal request. We can get the
- // header value from the connection.
const cl = c.getResponseHeader("content-length", 0) orelse return null;
return cl.value;
}
- // If we have no handle, then maybe this is being called after the
- // doneCallback. OR, maybe this is a "fulfilled" request. Let's check
- // the injected headers (if we have any).
+ return null;
+ }
- const rh = self.res.header orelse return null;
- for (rh._injected_headers) |hdr| {
- if (std.ascii.eqlIgnoreCase(hdr.name, "content-length")) {
- return hdr.value;
- }
+ // Only ever called from tick, delivers a completed event to its caller
+ fn deliver(transfer: *Transfer) void {
+ // Important. If a callback tries to kill the transfer (e.g. abort()),
+ // this will cause the teardown to be deferred so that it doesn't get
+ // freed from under us.
+ transfer.state = .completing;
+
+ const req = &transfer.req;
+ if (transfer._from_cache) {
+ req.notification.dispatch(
+ .http_request_served_from_cache,
+ &.{ .transfer = transfer },
+ );
}
- return null;
+ // The current design is that a Transfer will get one and only one
+ // deliver(). So at the end of deliver() we can deinit. That also means
+ // we can safely iterate transfer._events without risking a mutation to
+ // it.
+ //
+ // When WebSockets and/or buffering is added this will no longer be true
+ // and one transfer will be potentially get multiple deliver() calls.
+ // When that happens, we need to adjust two things:
+ // 1 - It might not be safe to iterate transfer._events like this,
+ // something could append to it, making it invalid
+ // 2 - Only terminal states (done, err, abort) should call deinit
+
+ for (transfer._events.items) |event| {
+ if (transfer.state == .aborted) {
+ // the state can become aborted as events are processed, check
+ // it on every iteration.
+ break;
+ }
+ switch (event) {
+ .start => {
+ if (req.start_callback) |cb| {
+ cb(transfer) catch |err| {
+ return transfer.failDelivery(err);
+ };
+ }
+ },
+ .header => {
+ if (transfer._notify_cdp) {
+ req.notification.dispatch(.http_response_header_done, &.{
+ .transfer = transfer,
+ });
+ }
+ const result = req.header_callback(transfer) catch |err| {
+ return transfer.failDelivery(err);
+ };
+ if (result == .abort) {
+ return transfer.failDelivery(error.Abort);
+ }
+ },
+ .data => |chunk| {
+ if (transfer._notify_cdp) {
+ req.notification.dispatch(.http_response_data, &.{
+ .data = chunk,
+ .transfer = transfer,
+ });
+ }
+ req.data_callback(transfer, chunk) catch |err| {
+ return transfer.failDelivery(err);
+ };
+ },
+ .done => {
+ if (transfer._notify_cdp) {
+ req.notification.dispatch(.http_request_done, &.{
+ .transfer = transfer,
+ .content_length = transfer._cdp_content_length,
+ });
+ }
+ req.done_callback(req.ctx) catch |err| {
+ return transfer.failDelivery(err);
+ };
+ },
+ .err => |err| {
+ transfer.requestFailed(err);
+ break;
+ },
+ }
+ }
+ transfer.deinit();
}
+};
- // Response-state owned by this transfer's currently-in-flight response.
- // Reset on every retry (auth retry, redirect) via Transfer.reset — only
- // the cross-retry counters (_auth_challenge, _redirect_count) live on
- // Transfer itself. `Transfer.Response` is the on-Transfer storage; the
- // top-level `Client.Response` is the actual Response (which is a union, e.g.
- // for a cached response)
- const Response = struct {
- header: ?ResponseHead = null,
+// Response-state owned by this transfer's currently-in-flight response.
+// Reset on every retry (auth retry, redirect) via Transfer.reset — only
+// the cross-retry counters (_auth_challenge, _redirect_count) live on
+// Transfer itself. Consumers read it through the accessors above
+// (status / contentType / responseHeaderIterator / getContentLength).
+const Response = struct {
+ header: ?http.ResponseHead = null,
- // total bytes received in the response, including the response status
- // line, the headers, and the [encoded] body.
- bytes_received: usize = 0,
+ // Full response headers, materialized into the transfer arena at
+ // completion (or set directly by cache / synthetic / fulfill).
+ headers: []const http.Header = &.{},
- // track if the header callbacks done have been called.
- header_done_called: bool = false,
+ // total bytes received in the response, including the response status
+ // line, the headers, and the [encoded] body.
+ bytes_received: usize = 0,
- skip_body: bool = false,
- first_data_received: bool = false,
+ skip_body: bool = false,
+ first_data_received: bool = false,
- // Buffered response body. Filled by dataCallback, consumed in processMessages.
- stream_buffer: std.ArrayList(u8) = .{},
+ // Buffered response body. Filled by dataCallback, consumed in processMessages.
+ stream_buffer: std.ArrayList(u8) = .{},
- // Error captured in dataCallback to be reported in processMessages.
- callback_error: ?anyerror = null,
- };
+ // Error captured in dataCallback to be reported in processMessages.
+ callback_error: ?anyerror = null,
};
-pub fn continueTransfer(self: *Client, transfer: *Transfer) !void {
- if (comptime IS_DEBUG) {
- log.debug(.http, "continue transfer", .{ .intercepted = self.interception_layer.intercepted });
- }
-
- transfer.unpark();
- self.process(transfer) catch |err| {
- if (transfer.state == .created) {
- transfer.abort(err);
- }
- return err;
- };
-}
-
-const Noop = struct {
- fn headerCallback(_: Response) !HeaderResult {
- return .proceed;
- }
- fn dataCallback(_: Response, _: []const u8) !void {}
- fn doneCallback(_: *anyopaque) !void {}
- fn errorCallback(_: *anyopaque, _: anyerror) void {}
+// A single buffered response event. The payload of `data` has transfer-arena
+// lifetime. `done`, and `err` are terminal — nothing follows them.
+const Event = union(enum) {
+ start,
+ header,
+ data: []const u8,
+ done,
+ err: anyerror,
};
-// An opaque-from-the-outside handle that Frame / WorkerGlobalScope embed
-// to track the HTTP transfers + WebSockets they own.
-pub const Owner = struct {
- transfers: std.DoublyLinkedList = .{},
- websockets: std.DoublyLinkedList = .{},
+// What the http cache wants from a transfer's response.
+const CacheIntent = union(enum) {
+ none,
- // The owning Frame's / WorkerGlobalScope's blob: registry,
- blob_urls: ?*const std.StringHashMapUnmanaged(*Blob) = null,
+ // Inspect the response at completion and store it if cacheable.
+ store,
- const WebSocket = @import("webapi/net/WebSocket.zig");
- const Blob = @import("webapi/Blob.zig");
+ // Conditional request in flight; on a 304 the stale entry is renewed
+ // and served. Owns the entry (possibly an open file) until resolved.
+ revalidate: Cache.CachedResponse,
+};
- pub fn addTransfer(self: *Owner, t: *Transfer) void {
- self.transfers.append(&t.owner_node);
- }
+// Non-network URL schemes whose response is synthesized in-process rather than
+// fetched, think blob data URLs.
+const Synthetic = struct {
+ const data_url = @import("../browser/data_url.zig");
- pub fn removeTransfer(self: *Owner, t: *Transfer) void {
- self.transfers.remove(&t.owner_node);
+ fn isSynthetic(url: []const u8) bool {
+ return std.mem.startsWith(u8, url, "data:") or std.mem.startsWith(u8, url, "blob:");
}
- pub fn addWS(self: *Owner, ws: *WebSocket) void {
- self.websockets.append(&ws._owner_node);
- }
+ // Materialize the synthetic response onto the transfer and buffer it for
+ // dispatch — delivered exactly like any other response.
+ fn buffer(transfer: *Transfer) !void {
+ const arena = transfer.arena;
+ const url = transfer.req.url;
- pub fn removeWS(self: *Owner, ws: *WebSocket) void {
- self.websockets.remove(&ws._owner_node);
+ var body: []const u8 = "";
+ var content_type: []const u8 = "";
+
+ if (std.mem.startsWith(u8, url, "data:")) {
+ const parsed = try data_url.parse(arena, url);
+ content_type = parsed.content_type;
+ body = parsed.body;
+ } else {
+ // blob: — resolved against the owning frame's registry.
+ const owner = transfer.owner orelse return error.BlobNotFound;
+ const blob_urls = owner.blob_urls orelse return error.BlobNotFound;
+ const blob = blob_urls.get(url) orelse return error.BlobNotFound;
+ content_type = blob._mime;
+ body = blob._slice;
+ }
+
+ const has_content_type = content_type.len > 0;
+ transfer.setResponseHead(200, if (has_content_type) content_type else null);
+ if (content_type.len > 0) {
+ const h = try arena.alloc(http.Header, 1);
+ h[0] = .{ .name = "content-type", .value = content_type };
+ transfer.res.headers = h;
+ }
+ transfer._cdp_content_length = body.len;
+ try transfer.bufferEvents(body);
}
};
@@ -2434,28 +2791,33 @@ test "HttpClient: isSyncWaitInterrupt ignores ping and non-teardown CDP methods"
}
}
-test "HttpClient: fulfillRequest survives a done_callback that tears down the owner" {
- // Regression: Fetch.fulfillRequest runs the consumer's done_callback while
- // the transfer is still parked for interception. A done_callback that runs
- // JS which navigates / closes the page re-entrantly kills the transfer
- // (abortOwner -> kill -> deinit). Previously the transfer was still
- // `.parked` at that point, so the re-entrant teardown freed it
- // synchronously and fulfillRequest's trailing deinit was a double-free +
- // `intercepted` underflow (surfacing later as a Transfer.leaveIntercept
- // assert at session teardown). fulfillRequest now moves the transfer to
- // `.completing` first, so the teardown defers and there is exactly one free.
+fn initTestClient(client: *Client, pool: *ArenaPool) void {
+ client.* = undefined;
+ client.allocator = testing.allocator;
+ client.arena_pool = pool;
+ client.transfers = .empty;
+ client.blocking_requests = .empty;
+ client.pending_queue = .{};
+ client.dispatch_queue = .{};
+ client.gated_queue = .{};
+ client.dispatch_count = 0;
+ client.performing = false;
+ client.intercepted = 0;
+ client.cache = null;
+ client.serve_mode = false;
+ client.obey_robots = false;
+}
+
+test "HttpClient: fulfillIntercepted survives a done_callback that tears down the owner" {
+ // Regression: the fulfilled response's done_callback runs JS which
+ // navigates / closes the page, re-entrantly killing the transfer
+ // (abortOwner -> kill). deliver() holds the transfer in `.completing`,
+ // so the teardown defers and there is exactly one free.
var pool = ArenaPool.init(testing.allocator, .{});
defer pool.deinit();
var client: Client = undefined;
- client.allocator = testing.allocator;
- client.arena_pool = &pool;
- client.transfers = .empty;
- client.queue = .{};
- client.next_tick_queue = .{};
- client.next_tick_count = 0;
- client.performing = false;
- client.interception_layer = .{};
+ initTestClient(&client, &pool);
defer client.transfers.deinit(testing.allocator);
var owner: Owner = .{};
@@ -2485,11 +2847,11 @@ test "HttpClient: fulfillRequest survives a done_callback that tears down the ow
.loader_id = 0,
.method = .GET,
.url = "http://example.com/",
- .headers = .{ .headers = null },
.cookie_jar = null,
.cookie_origin = "",
.resource_type = .document,
.notification = undefined,
+ .shutdown_callback = noopShutdown,
.ctx = &ctx,
.done_callback = Ctx.doneCallback,
},
@@ -2502,64 +2864,48 @@ test "HttpClient: fulfillRequest survives a done_callback that tears down the ow
owner.addTransfer(transfer);
transfer.owner = &owner;
- // Mirror InterceptionLayer.request committing the transfer to CDP.
+ // Mirror the interception gate committing the transfer to CDP.
transfer.park(.intercept_request);
- client.interception_layer.intercepted += 1;
+ client.intercepted += 1;
- try client.interception_layer.fulfillRequest(transfer, 200, &.{}, "hello");
+ try client.fulfillIntercepted(transfer, 200, &.{}, "hello");
+ // The response is buffered, not delivered from the CDP command.
+ try testing.expectEqual(false, ctx.done_called);
+ try testing.expectEqual(1, client.dispatch_count);
+
+ client.dispatchCompleted();
try testing.expect(ctx.done_called);
// The transfer was freed exactly once: counter back to 0, dropped from the
// id index and the owner list. A double-free would have underflowed
// `intercepted` (or tripped the leaveIntercept assert).
- try testing.expectEqual(0, client.interception_layer.intercepted);
+ try testing.expectEqual(0, client.intercepted);
+ try testing.expectEqual(0, client.dispatch_count);
try testing.expectEqual(0, client.transfers.count());
try testing.expectEqual(null, owner.transfers.first);
}
-test "HttpClient: fulfillRequest follows a 3xx redirect" {
+test "HttpClient: fulfillIntercepted follows a 3xx redirect" {
// Regression for #2828: a CDP Fetch.fulfillRequest with a 3xx status + a
// Location header must be followed like a real network redirect (re-issued
- // down the chain to the resolved target), not delivered as a final response.
+ // through the pipeline to the resolved target), not delivered as a final
+ // response.
var pool = ArenaPool.init(testing.allocator, .{});
defer pool.deinit();
- // Only network.config is read (httpMaxRedirects, which ignores its config),
- // so a pointer to an otherwise-undefined Network is safe here.
+ // Only network.config (httpMaxRedirects, which ignores its config) and
+ // network.cache are read on this path.
var net: Network = undefined;
+ net.cache = null;
var client: Client = undefined;
- client.allocator = testing.allocator;
- client.arena_pool = &pool;
+ initTestClient(&client, &pool);
client.network = &net;
- client.transfers = .empty;
- client.queue = .{};
- client.next_tick_queue = .{};
- client.next_tick_count = 0;
- client.performing = false;
- client.interception_layer = .{};
+ // Forces process() to queue the re-issued request instead of asking the
+ // (undefined) Network for a connection — the queue IS the capture.
+ client.performing = true;
defer client.transfers.deinit(testing.allocator);
- // Capturing stub for interception_layer.next: records the re-issued request
- // and returns without committing (transfer stays .created; we clean up).
- const Captor = struct {
- captured: bool = false,
- url: []const u8 = "",
- method: Method = undefined,
-
- fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
- const self: *@This() = @ptrCast(@alignCast(ptr));
- self.captured = true;
- self.url = transfer.req.url;
- self.method = transfer.req.method;
- }
- };
- var captor = Captor{};
- client.interception_layer.next = .{
- .ptr = &captor,
- .vtable = &.{ .request = Captor.request },
- };
-
// 302 with a relative Location: rewrite to GET, drop body, resolve target.
{
const arena = try pool.acquire(.small, "test");
@@ -2573,11 +2919,11 @@ test "HttpClient: fulfillRequest follows a 3xx redirect" {
.method = .POST,
.url = "http://example.com/start",
.body = "payload",
- .headers = .{ .headers = null },
.cookie_jar = null,
.cookie_origin = "",
.resource_type = .document,
.notification = undefined,
+ .shutdown_callback = noopShutdown,
.ctx = undefined,
},
.client = &client,
@@ -2587,24 +2933,24 @@ test "HttpClient: fulfillRequest follows a 3xx redirect" {
try client.transfers.putNoClobber(testing.allocator, transfer.id, transfer);
transfer.park(.intercept_request);
- client.interception_layer.intercepted += 1;
+ client.intercepted += 1;
- try client.interception_layer.fulfillRequest(transfer, 302, &.{
+ try client.fulfillIntercepted(transfer, 302, &.{
.{ .name = "Location", .value = "/end" },
}, null);
- try testing.expect(captor.captured);
- try testing.expectEqual("http://example.com/end", captor.url);
- try testing.expectEqual(.GET, captor.method);
+ // Re-issued (queued for the network), not delivered.
+ try testing.expectEqual(true, transfer.state == .queued);
+ try testing.expectEqual("http://example.com/end", transfer.req.url);
+ try testing.expectEqual(.GET, transfer.req.method);
try testing.expectEqual(null, transfer.req.body);
- // Unparked exactly once; transfer is still alive (re-issued, not deinited).
- try testing.expectEqual(0, client.interception_layer.intercepted);
+ // Unparked exactly once; transfer is still alive.
+ try testing.expectEqual(0, client.intercepted);
try testing.expectEqual(1, client.transfers.count());
transfer.deinit();
}
// 307 with an absolute Location: keep method and body.
- captor = .{};
{
const arena = try pool.acquire(.small, "test");
const transfer = try arena.create(Transfer);
@@ -2617,11 +2963,11 @@ test "HttpClient: fulfillRequest follows a 3xx redirect" {
.method = .POST,
.url = "http://example.com/start",
.body = "payload",
- .headers = .{ .headers = null },
.cookie_jar = null,
.cookie_origin = "",
.resource_type = .document,
.notification = undefined,
+ .shutdown_callback = noopShutdown,
.ctx = undefined,
},
.client = &client,
@@ -2631,40 +2977,41 @@ test "HttpClient: fulfillRequest follows a 3xx redirect" {
try client.transfers.putNoClobber(testing.allocator, transfer.id, transfer);
transfer.park(.intercept_request);
- client.interception_layer.intercepted += 1;
+ client.intercepted += 1;
- try client.interception_layer.fulfillRequest(transfer, 307, &.{
+ try client.fulfillIntercepted(transfer, 307, &.{
.{ .name = "location", .value = "http://example.com/other" },
}, null);
- try testing.expect(captor.captured);
- try testing.expectEqual("http://example.com/other", captor.url);
- try testing.expectEqual(.POST, captor.method);
+ try testing.expectEqual(true, transfer.state == .queued);
+ try testing.expectEqual("http://example.com/other", transfer.req.url);
+ try testing.expectEqual(.POST, transfer.req.method);
try testing.expectEqual("payload", transfer.req.body.?);
- try testing.expectEqual(0, client.interception_layer.intercepted);
+ try testing.expectEqual(0, client.intercepted);
transfer.deinit();
}
}
-test "HttpClient: fulfillRequest delivers a 3xx without a Location as the response" {
+test "HttpClient: fulfillIntercepted delivers a 3xx without a Location as the response" {
// A redirect status with no Location header is not a redirect: the body is
// delivered as the final response (matching the real-network path).
var pool = ArenaPool.init(testing.allocator, .{});
defer pool.deinit();
var client: Client = undefined;
- client.allocator = testing.allocator;
- client.arena_pool = &pool;
- client.transfers = .empty;
- client.queue = .{};
- client.next_tick_queue = .{};
- client.next_tick_count = 0;
- client.performing = false;
- client.interception_layer = .{};
+ initTestClient(&client, &pool);
defer client.transfers.deinit(testing.allocator);
const Ctx = struct {
done_called: bool = false,
+ status: ?u16 = null,
+
+ fn headerCallback(transfer: *Transfer) !Transfer.HeaderResult {
+ const self: *@This() = @ptrCast(@alignCast(transfer.req.ctx));
+ self.status = transfer.responseStatus();
+ return .proceed;
+ }
+
fn doneCallback(ctx: *anyopaque) !void {
const self: *@This() = @ptrCast(@alignCast(ctx));
self.done_called = true;
@@ -2682,12 +3029,13 @@ test "HttpClient: fulfillRequest delivers a 3xx without a Location as the respon
.loader_id = 0,
.method = .GET,
.url = "http://example.com/",
- .headers = .{ .headers = null },
.cookie_jar = null,
.cookie_origin = "",
.resource_type = .document,
.notification = undefined,
+ .shutdown_callback = noopShutdown,
.ctx = &ctx,
+ .header_callback = Ctx.headerCallback,
.done_callback = Ctx.doneCallback,
},
.client = &client,
@@ -2697,13 +3045,15 @@ test "HttpClient: fulfillRequest delivers a 3xx without a Location as the respon
try client.transfers.putNoClobber(testing.allocator, transfer.id, transfer);
transfer.park(.intercept_request);
- client.interception_layer.intercepted += 1;
+ client.intercepted += 1;
- try client.interception_layer.fulfillRequest(transfer, 302, &.{}, "body");
+ try client.fulfillIntercepted(transfer, 302, &.{}, "body");
+ client.dispatchCompleted();
// Delivered (done_callback ran) and freed exactly once.
try testing.expect(ctx.done_called);
- try testing.expectEqual(0, client.interception_layer.intercepted);
+ try testing.expectEqual(302, ctx.status.?);
+ try testing.expectEqual(0, client.intercepted);
try testing.expectEqual(0, client.transfers.count());
}
@@ -2716,14 +3066,7 @@ test "HttpClient: abortParked survives an error_callback that tears down the own
defer pool.deinit();
var client: Client = undefined;
- client.allocator = testing.allocator;
- client.arena_pool = &pool;
- client.transfers = .empty;
- client.queue = .{};
- client.next_tick_queue = .{};
- client.next_tick_count = 0;
- client.performing = false;
- client.interception_layer = .{};
+ initTestClient(&client, &pool);
defer client.transfers.deinit(testing.allocator);
var owner: Owner = .{};
@@ -2751,11 +3094,11 @@ test "HttpClient: abortParked survives an error_callback that tears down the own
.loader_id = 0,
.method = .GET,
.url = "http://example.com/",
- .headers = .{ .headers = null },
.cookie_jar = null,
.cookie_origin = "",
.resource_type = .document,
.notification = undefined,
+ .shutdown_callback = noopShutdown,
.ctx = &ctx,
.error_callback = Ctx.errorCallback,
},
@@ -2769,12 +3112,12 @@ test "HttpClient: abortParked survives an error_callback that tears down the own
transfer.owner = &owner;
transfer.park(.intercept_request);
- client.interception_layer.intercepted += 1;
+ client.intercepted += 1;
transfer.abortParked(error.Abort);
try testing.expect(ctx.err_called);
- try testing.expectEqual(0, client.interception_layer.intercepted);
+ try testing.expectEqual(0, client.intercepted);
try testing.expectEqual(0, client.transfers.count());
try testing.expectEqual(null, owner.transfers.first);
}
diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zig
new file mode 100644
index 000000000..8cfa11162
--- /dev/null
+++ b/src/network/RobotsGate.zig
@@ -0,0 +1,232 @@
+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+// robots.txt gate for the HttpClient request pipeline. Answers allow/deny
+// from the robot store; on a store miss it parks the transfer, coalesces
+// concurrent requests for the same robots.txt behind a single internal
+// fetch, and resumes (or fails) the parked transfers when it resolves.
+
+const std = @import("std");
+const lp = @import("lightpanda");
+
+const URL = @import("../browser/URL.zig");
+
+const Robots = @import("Robots.zig");
+const Network = @import("Network.zig");
+const Transfer = @import("HttpClient.zig").Transfer;
+
+const log = lp.log;
+const Allocator = std.mem.Allocator;
+
+const RobotsGate = @This();
+
+network: *Network,
+allocator: Allocator,
+pending: std.StringHashMapUnmanaged(std.ArrayList(*Transfer)) = .empty,
+
+pub const Result = enum { allowed, blocked, pending };
+
+pub fn deinit(self: *RobotsGate) void {
+ var it = self.pending.iterator();
+ while (it.next()) |entry| {
+ entry.value_ptr.deinit(self.allocator);
+ }
+ self.pending.deinit(self.allocator);
+}
+
+pub fn check(self: *RobotsGate, transfer: *Transfer) !Result {
+ const url = transfer.req.url;
+ const robots_url = try URL.getRobotsUrl(transfer.arena, url);
+
+ if (self.network.robot_store.get(robots_url)) |robot_entry| {
+ switch (robot_entry) {
+ .absent => return .allowed,
+ .present => |robots| {
+ if (robots.isAllowed(URL.getPathname(url))) {
+ return .allowed;
+ }
+ log.warn(.http, "blocked by robots", .{ .url = url });
+ return .blocked;
+ },
+ }
+ unreachable;
+ }
+
+ try self.fetchThenResume(robots_url, transfer);
+ return .pending;
+}
+
+fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Transfer) !void {
+ const entry = try self.pending.getOrPut(self.allocator, robots_url);
+
+ if (entry.found_existing == false) {
+ // A fetch for this robots.txt is already in flight, queue behind it.
+ try entry.value_ptr.append(self.allocator, transfer);
+ transfer.park(.robots);
+ return;
+ }
+
+ errdefer _ = self.pending.remove(robots_url);
+ entry.value_ptr.* = .empty;
+
+ try entry.value_ptr.append(self.allocator, transfer);
+ transfer.park(.robots);
+ errdefer {
+ entry.value_ptr.deinit(self.allocator);
+ transfer.unpark();
+ }
+
+ const robots_ctx = try transfer.arena.create(RobotsContext);
+ robots_ctx.* = .{
+ .gate = self,
+ .buffer = .empty,
+ .arena = transfer.arena,
+ .robots_url = robots_url,
+ };
+
+ log.debug(.browser, "fetching robots.txt", .{ .robots_url = robots_url });
+
+ // Only the parent's frame/loader ids (CDP correlation) and notification
+ // carry over — no cookies, credentials, headers, or timeout.
+ try transfer.client.request(.{
+ .url = robots_url,
+ .method = .GET,
+ .internal = true,
+ .resource_type = .fetch,
+ .frame_id = transfer.req.frame_id,
+ .loader_id = transfer.req.loader_id,
+ .notification = transfer.req.notification,
+ .cookie_jar = null,
+ .cookie_origin = robots_url,
+ .ctx = robots_ctx,
+ .header_callback = RobotsContext.headerCallback,
+ .data_callback = RobotsContext.dataCallback,
+ .done_callback = RobotsContext.doneCallback,
+ .error_callback = RobotsContext.errorCallback,
+ .shutdown_callback = RobotsContext.shutdownCallback,
+ }, transfer.owner);
+}
+
+fn flushPending(self: *RobotsGate, robots_url: [:0]const u8, allowed: bool) void {
+ var queued = self.pending.fetchRemove(robots_url) orelse return;
+ defer queued.value.deinit(self.allocator);
+
+ for (queued.value.items) |transfer| {
+ transfer.unpark();
+ if (!allowed) {
+ log.warn(.http, "blocked by robots", .{ .url = transfer.req.url });
+ transfer.failAsync(error.RobotsBlocked);
+ continue;
+ }
+ // Hand back to the pipeline; the robots gate is the last step
+ // before the network. If it fails before committing, clean up here.
+ transfer.client.resumeAfterRobots(transfer) catch |e| {
+ if (transfer.state == .created) {
+ transfer.abort(e);
+ }
+ };
+ }
+}
+
+// shutdown_callback is only called on owner shutdown. And if this robot's fetch
+// is being shutdown, than any transfers waiting for it will be shutdown too.
+fn flushPendingShutdown(self: *RobotsGate, robots_url: [:0]const u8) void {
+ var pending = self.pending.fetchRemove(robots_url) orelse return;
+ pending.value.deinit(self.allocator);
+}
+
+const RobotsContext = struct {
+ gate: *RobotsGate,
+ arena: Allocator,
+ robots_url: [:0]const u8,
+ buffer: std.ArrayList(u8),
+ status: u16 = 0,
+
+ fn headerCallback(transfer: *Transfer) anyerror!Transfer.HeaderResult {
+ const self: *RobotsContext = @ptrCast(@alignCast(transfer.req.ctx));
+ if (transfer.res.header) |hdr| {
+ log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = self.robots_url });
+ self.status = hdr.status;
+ }
+ if (transfer.getContentLength()) |cl| {
+ try self.buffer.ensureTotalCapacity(self.arena, cl);
+ }
+ return .proceed;
+ }
+
+ fn dataCallback(transfer: *Transfer, data: []const u8) anyerror!void {
+ const self: *RobotsContext = @ptrCast(@alignCast(transfer.req.ctx));
+ try self.buffer.appendSlice(self.arena, data);
+ }
+
+ fn doneCallback(ctx_ptr: *anyopaque) anyerror!void {
+ const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));
+ const gate = self.gate;
+ const robots_url = self.robots_url;
+
+ var allowed = true;
+ const network = gate.network;
+
+ switch (self.status) {
+ 200 => {
+ if (self.buffer.items.len > 0) {
+ const robots: ?Robots = network.robot_store.robotsFromBytes(
+ network.config.http_headers.user_agent,
+ self.buffer.items,
+ ) catch blk: {
+ log.warn(.browser, "failed to parse robots", .{ .robots_url = robots_url });
+ try network.robot_store.putAbsent(robots_url);
+ break :blk null;
+ };
+ if (robots) |r| {
+ try network.robot_store.put(robots_url, r);
+ const path = URL.getPathname(gate.pending.get(robots_url).?.items[0].req.url);
+ allowed = r.isAllowed(path);
+ }
+ }
+ },
+ 404 => {
+ log.debug(.http, "robots not found", .{ .url = robots_url });
+ try network.robot_store.putAbsent(robots_url);
+ },
+ else => {
+ log.debug(.http, "unexpected status on robots", .{
+ .url = robots_url,
+ .status = self.status,
+ });
+ try network.robot_store.putAbsent(robots_url);
+ },
+ }
+
+ gate.flushPending(robots_url, allowed);
+ }
+
+ fn errorCallback(ctx_ptr: *anyopaque, err: anyerror) void {
+ const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));
+
+ log.warn(.http, "robots fetch failed", .{ .err = err });
+ self.gate.flushPending(self.robots_url, true);
+ }
+
+ fn shutdownCallback(ctx_ptr: *anyopaque) void {
+ const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));
+
+ log.debug(.http, "robots fetch shutdown", .{});
+ self.gate.flushPendingShutdown(self.robots_url);
+ }
+};
diff --git a/src/network/http.zig b/src/network/http.zig
index 8baeaf24a..b05206efa 100644
--- a/src/network/http.zig
+++ b/src/network/http.zig
@@ -308,11 +308,6 @@ pub const ResponseHead = struct {
redirect_count: u32,
_content_type_len: usize = 0,
_content_type: [MAX_CONTENT_TYPE_LEN]u8 = undefined,
- // this is normally an empty list, but if the response is being injected
- // than it'll be populated. It isn't meant to be used directly, but should
- // be used through the transfer.responseHeaderIterator() which abstracts
- // whether the headers are from a live curl easy handle, or injected.
- _injected_headers: []const Header = &.{},
pub fn contentType(self: *ResponseHead) ?[]u8 {
if (self._content_type_len == 0) {
@@ -364,7 +359,7 @@ pub const Connection = struct {
pub const Transport = union(enum) {
none, // used for cases that manage their own connection, e.g. telemetry
- http: *@import("../browser/HttpClient.zig").Transfer,
+ http: *@import("HttpClient.zig").Transfer,
websocket: *@import("../browser/webapi/net/WebSocket.zig"),
};
diff --git a/src/network/layer/CacheLayer.zig b/src/network/layer/CacheLayer.zig
deleted file mode 100644
index a65a3570e..000000000
--- a/src/network/layer/CacheLayer.zig
+++ /dev/null
@@ -1,386 +0,0 @@
-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
-//
-// Francis Bouvier <[email protected]>
-// Pierre Tachoire <[email protected]>
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-const std = @import("std");
-const lp = @import("lightpanda");
-
-const Layer = @import("../../browser/HttpClient.zig").Layer;
-const Transfer = @import("../../browser/HttpClient.zig").Transfer;
-const Response = @import("../../browser/HttpClient.zig").Response;
-
-const Cache = @import("../cache/Cache.zig");
-const CachedMetadata = @import("../cache/Cache.zig").CachedMetadata;
-const CachedResponse = @import("../cache/Cache.zig").CachedResponse;
-
-const HeaderResult = @import("../../browser/HttpClient.zig").HeaderResult;
-const Forward = @import("Forward.zig");
-
-const log = lp.log;
-const IS_DEBUG = @import("builtin").mode == .Debug;
-
-const CacheLayer = @This();
-
-next: Layer = undefined,
-disabled: bool = false,
-
-pub fn layer(self: *CacheLayer) Layer {
- return .{
- .ptr = self,
- .vtable = &.{
- .request = request,
- },
- };
-}
-
-fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
- const self: *CacheLayer = @ptrCast(@alignCast(ptr));
- const req = &transfer.req;
-
- if (self.disabled or req.method != .GET) {
- return self.next.request(transfer);
- }
-
- const arena = transfer.arena;
-
- var iter = req.headers.iterator();
- const req_header_list = try iter.collect(arena);
-
- const cached = transfer.client.network.cache.?.get(arena, .{
- .url = req.url,
- .timestamp = std.time.timestamp(),
- .request_headers = req_header_list.items,
- }) orelse {
- // Cache miss: install wrappers so we can inspect the response and decide
- // whether to write the body into the cache when it's done.
- try installCacheContext(arena, transfer, null);
- return self.next.request(transfer);
- };
-
- if (!cached.expired) {
- const ctx = try arena.create(CachedResponse);
- ctx.* = cached;
-
- try transfer.client.runNextTick(transfer, ctx, .{
- .run = struct {
- fn run(t: *Transfer, ctx_ptr: ?*anyopaque) void {
- defer t.deinit();
-
- const c: *CachedResponse = @ptrCast(@alignCast(ctx_ptr.?));
- serveFromCache(t, c) catch |err| {
- t.req.error_callback(t.req.ctx, err);
- };
- }
- }.run,
- .abort = struct {
- fn abort(ctx_ptr: ?*anyopaque) void {
- const c: *CachedResponse = @ptrCast(@alignCast(ctx_ptr.?));
- switch (c.data) {
- .buffer => |_| {},
- .file => |f| f.file.close(),
- }
- }
- }.abort,
- });
- return;
- }
-
- if (cached.metadata.hasValidators()) {
- if (cached.metadata.etag) |etag| {
- log.debug(.cache, "revalidate with etag", .{ .url = req.url, .etag = etag });
- const header_value = try std.fmt.allocPrintSentinel(arena, "If-None-Match: {s}", .{etag}, 0);
- try req.headers.add(header_value);
- }
-
- if (cached.metadata.last_modified) |lm| {
- log.debug(.cache, "revalidate with last-modified", .{ .url = req.url, .last_modified = lm });
- const header_value = try std.fmt.allocPrintSentinel(arena, "If-Modified-Since: {s}", .{lm}, 0);
- try req.headers.add(header_value);
- }
-
- try installCacheContext(arena, transfer, cached);
- } else {
- defer cached.data.deinit();
- // If it is expired w/o validators, evict from Cache.
- transfer.client.network.cache.?.evict(req.url);
- try installCacheContext(arena, transfer, null);
- }
-
- return self.next.request(transfer);
-}
-
-fn installCacheContext(
- arena: std.mem.Allocator,
- transfer: *Transfer,
- stale_entry: ?CachedResponse,
-) !void {
- const req = &transfer.req;
- const ctx = try arena.create(CacheContext);
- ctx.* = .{
- .arena = arena,
- .transfer = transfer,
- .forward = Forward.capture(req),
- .req_url = req.url,
- .req_headers = req.headers,
- .stale_entry = stale_entry,
- };
-
- req.ctx = ctx;
- req.header_callback = CacheContext.headerCallback;
- req.data_callback = CacheContext.dataCallback;
- req.done_callback = CacheContext.doneCallback;
- req.error_callback = CacheContext.errorCallback;
-
- if (ctx.forward.start != null) req.start_callback = CacheContext.startCallback;
- if (ctx.forward.shutdown != null) req.shutdown_callback = CacheContext.shutdownCallback;
-}
-
-fn forwardFromCache(
- transfer: *Transfer,
- forward: *Forward,
- cached: *const CachedResponse,
-) !void {
- transfer.req.notification.dispatch(
- .http_request_served_from_cache,
- &.{ .transfer = transfer },
- );
-
- const req = &transfer.req;
- const response = Response.fromCached(req.ctx, cached);
- defer cached.data.deinit();
-
- try forward.forwardStart(response);
- const result = try forward.forwardHeader(response);
- if (result == .abort) return error.Abort;
-
- switch (cached.data) {
- .buffer => |data| {
- if (data.len > 0) try forward.forwardData(response, data);
- },
- .file => |f| {
- const file = f.file;
- var buf: [1024]u8 = undefined;
- var file_reader = file.reader(&buf);
- try file_reader.seekTo(f.offset);
- const reader = &file_reader.interface;
- var read_buf: [1024]u8 = undefined;
- var remaining = f.len;
- while (remaining > 0) {
- const read_len = @min(read_buf.len, remaining);
- const n = try reader.readSliceShort(read_buf[0..read_len]);
- if (n == 0) break;
- remaining -= n;
- try forward.forwardData(response, read_buf[0..n]);
- }
- },
- }
-
- try forward.forwardDone();
-}
-
-fn serveFromCache(transfer: *Transfer, cached: *const CachedResponse) !void {
- transfer.req.notification.dispatch(
- .http_request_served_from_cache,
- &.{ .transfer = transfer },
- );
-
- const req = &transfer.req;
- const response = Response.fromCached(req.ctx, cached);
- defer cached.data.deinit();
-
- if (req.start_callback) |cb| {
- try cb(response);
- }
-
- const result = try req.header_callback(response);
- if (result == .abort) {
- return error.Abort;
- }
-
- switch (cached.data) {
- .buffer => |data| {
- if (data.len > 0) {
- try req.data_callback(response, data);
- }
- },
- .file => |f| {
- const file = f.file;
- var buf: [1024]u8 = undefined;
- var file_reader = file.reader(&buf);
- try file_reader.seekTo(f.offset);
- const reader = &file_reader.interface;
- var read_buf: [1024]u8 = undefined;
- var remaining = f.len;
- while (remaining > 0) {
- const read_len = @min(read_buf.len, remaining);
- const n = try reader.readSliceShort(read_buf[0..read_len]);
- if (n == 0) break;
- remaining -= n;
- try req.data_callback(response, read_buf[0..n]);
- }
- },
- }
-
- try req.done_callback(req.ctx);
-}
-
-const CacheContext = struct {
- arena: std.mem.Allocator,
- transfer: *Transfer,
- forward: Forward,
- req_url: [:0]const u8,
- req_headers: @import("../http.zig").Headers,
- pending_metadata: ?*CachedMetadata = null,
- stale_entry: ?CachedResponse = null,
-
- fn startCallback(response: Response) anyerror!void {
- const self: *CacheContext = @ptrCast(@alignCast(response.ctx));
- return self.forward.forwardStart(response);
- }
-
- fn dataCallback(response: Response, chunk: []const u8) anyerror!void {
- const self: *CacheContext = @ptrCast(@alignCast(response.ctx));
- return self.forward.forwardData(response, chunk);
- }
-
- fn headerCallback(response: Response) anyerror!HeaderResult {
- const self: *CacheContext = @ptrCast(@alignCast(response.ctx));
-
- // For non-transfer responses (fulfilled by interception, or future
- // cached-while-cached cases), there's nothing to inspect for caching
- // decisions — just forward.
- const transfer = switch (response.inner) {
- .transfer => |t| t,
- else => return self.forward.forwardHeader(response),
- };
-
- const arena = self.arena;
- const conn = transfer._conn.?;
- var rh = &transfer.res.header.?;
-
- if (self.stale_entry != null and rh.status == 304) {
- const stale = self.stale_entry.?;
- self.stale_entry = null;
-
- var iter = response.headerIterator();
- const headers = try iter.collect(arena);
-
- transfer.client.network.cache.?.renew(
- arena,
- .{
- .url = self.req_url,
- .timestamp = std.time.timestamp(),
- .headers = headers.items,
- },
- ) catch |err| {
- log.warn(.cache, "renew failed", .{ .err = err });
- };
-
- try forwardFromCache(transfer, &self.forward, &stale);
- return .handled;
- }
-
- if (self.stale_entry) |stale| {
- stale.data.deinit();
- self.stale_entry = null;
- }
-
- const vary = if (conn.getResponseHeader("vary", 0)) |h| h.value else null;
- const maybe_cm = try Cache.tryCache(
- arena,
- std.time.timestamp(),
- self.req_url,
- rh.status,
- rh.contentType(),
- if (conn.getResponseHeader("cache-control", 0)) |h| h.value else null,
- vary,
- if (conn.getResponseHeader("age", 0)) |h| h.value else null,
- if (conn.getResponseHeader("etag", 0)) |h| h.value else null,
- if (conn.getResponseHeader("last-modified", 0)) |h| h.value else null,
- conn.getResponseHeader("set-cookie", 0) != null,
- conn.getResponseHeader("authorization", 0) != null,
- );
-
- if (maybe_cm) |cm| {
- var iter = transfer.responseHeaderIterator();
- var header_list = try iter.collect(arena);
- const end_of_response = header_list.items.len;
-
- if (vary) |vary_str| {
- var req_it = self.req_headers.iterator();
- while (req_it.next()) |hdr| {
- var vary_iter = std.mem.splitScalar(u8, vary_str, ',');
- while (vary_iter.next()) |part| {
- const name = std.mem.trim(u8, part, &std.ascii.whitespace);
- if (std.ascii.eqlIgnoreCase(hdr.name, name)) {
- try header_list.append(arena, .{
- .name = try arena.dupe(u8, hdr.name),
- .value = try arena.dupe(u8, hdr.value),
- });
- }
- }
- }
- }
-
- const metadata = try arena.create(CachedMetadata);
- metadata.* = cm;
- metadata.headers = header_list.items[0..end_of_response];
- metadata.vary_headers = header_list.items[end_of_response..];
- self.pending_metadata = metadata;
- }
-
- return self.forward.forwardHeader(response);
- }
-
- fn doneCallback(ctx: *anyopaque) anyerror!void {
- const self: *CacheContext = @ptrCast(@alignCast(ctx));
- const transfer = self.transfer;
-
- if (self.pending_metadata) |metadata| {
- const cache = &transfer.client.network.cache.?;
-
- if (comptime IS_DEBUG) {
- log.debug(.browser, "http cache", .{ .key = self.req_url, .metadata = metadata });
- }
- cache.put(metadata.*, transfer.res.stream_buffer.items) catch |err| {
- log.warn(.http, "cache put failed", .{ .err = err });
- };
- }
-
- return self.forward.forwardDone();
- }
-
- fn shutdownCallback(ctx: *anyopaque) void {
- const self: *CacheContext = @ptrCast(@alignCast(ctx));
- if (self.stale_entry) |entry| {
- self.stale_entry = null;
- entry.data.deinit();
- }
-
- self.forward.forwardShutdown();
- }
-
- fn errorCallback(ctx: *anyopaque, e: anyerror) void {
- const self: *CacheContext = @ptrCast(@alignCast(ctx));
- if (self.stale_entry) |entry| {
- self.stale_entry = null;
- entry.data.deinit();
- }
-
- self.forward.forwardErr(e);
- }
-};
diff --git a/src/network/layer/DeferringLayer.zig b/src/network/layer/DeferringLayer.zig
deleted file mode 100644
index 0cc4c4717..000000000
--- a/src/network/layer/DeferringLayer.zig
+++ /dev/null
@@ -1,361 +0,0 @@
-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
-//
-// Francis Bouvier <[email protected]>
-// Pierre Tachoire <[email protected]>
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-const std = @import("std");
-const lp = @import("lightpanda");
-const log = lp.log;
-
-const Network = @import("../Network.zig");
-const Transfer = @import("../../browser/HttpClient.zig").Transfer;
-const Response = @import("../../browser/HttpClient.zig").Response;
-const Layer = @import("../../browser/HttpClient.zig").Layer;
-const StableResponse = @import("../../browser/HttpClient.zig").StableResponse;
-const Forward = @import("Forward.zig");
-const HeaderResult = @import("../../browser/HttpClient.zig").HeaderResult;
-
-const DeferringLayer = @This();
-
-allocator: std.mem.Allocator,
-network: *Network,
-next: Layer = undefined,
-active: std.DoublyLinkedList = .{},
-
-pub fn layer(self: *DeferringLayer) Layer {
- return .{
- .ptr = self,
- .vtable = &.{ .request = request },
- };
-}
-
-pub fn deinit(self: *DeferringLayer) void {
- self.drainAll();
-}
-
-fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
- const self: *DeferringLayer = @ptrCast(@alignCast(ptr));
-
- if (transfer.req.internal) {
- return self.next.request(transfer);
- }
-
- const arena = try self.network.app.arena_pool.acquire(.small, "DeferringContext");
- errdefer self.network.app.arena_pool.release(arena);
-
- // this might outlive the transfer, we need to dupe eveyrthing we'll need to use
- const ctx = try arena.create(DeferredContext);
- ctx.* = .{
- .arena = arena,
- .layer = self,
- .transfer = transfer,
- .frame_id = transfer.req.frame_id,
- .url = try arena.dupeZ(u8, transfer.req.url),
- .forward = Forward.capture(&transfer.req),
- };
-
- self.active.append(&ctx.node);
- errdefer self.active.remove(&ctx.node);
-
- transfer.req.ctx = ctx;
- transfer.req.start_callback = if (ctx.forward.start != null) DeferredContext.startCallback else null;
- transfer.req.header_callback = DeferredContext.headerCallback;
- transfer.req.data_callback = DeferredContext.dataCallback;
- transfer.req.done_callback = DeferredContext.doneCallback;
- transfer.req.error_callback = DeferredContext.errorCallback;
- transfer.req.shutdown_callback = if (ctx.forward.shutdown != null) DeferredContext.shutdownCallback else null;
-
- return self.next.request(transfer);
-}
-
-pub fn flushFrame(self: *DeferringLayer, frame_id: u32) void {
- // DeferredContext.fire() can re-enter flushFrame, so we'll capture
- // ready items in this list, so that a reentrant flushFrame doesn't mutate
- // self.active while we're iterating.
- var ready: std.DoublyLinkedList = .{};
-
- var node = self.active.first;
- while (node) |n| {
- node = n.next;
- const ctx: *DeferredContext = @fieldParentPtr("node", n);
- if (!ctx.deferring) {
- continue;
- }
-
- // captured frame_id, not ctx.transfer: the transfer may be freed.
- if (ctx.frame_id != frame_id) {
- continue;
- }
-
- if (ctx.terminal) {
- self.active.remove(n);
- ready.append(n);
- } else {
- ctx.firePartial();
- ctx.deferring = false;
- }
- }
-
- // ready is local, ctx.fire() re-entering flushFrame can't invalidate it.
- while (ready.popFirst()) |n| {
- const ctx: *DeferredContext = @fieldParentPtr("node", n);
- ctx.fire();
- }
-}
-
-/// Drop orphaned deferred contexts for a frame that's going away. A `terminal`
-/// context's transfer already completed while deferred, so it's been deinited
-/// and unlinked from the owner — abortOwner can't reach it, yet it lingers in
-/// `active` pointing at a forward target (the Fetch) whose arena page teardown
-/// is about to free, and a later flushFrame would fire into it. Non-terminal
-/// contexts still have a live transfer that cleans them up itself.
-pub fn cancelFrame(self: *DeferringLayer, frame_id: u32) void {
- var node = self.active.first;
- while (node) |n| {
- node = n.next;
- const ctx: *DeferredContext = @fieldParentPtr("node", n);
- if (ctx.frame_id != frame_id or !ctx.terminal) {
- continue;
- }
- self.active.remove(n);
- ctx.deinit();
- }
-}
-
-pub fn drainAll(self: *DeferringLayer) void {
- while (self.active.popFirst()) |node| {
- const ctx: *DeferredContext = @fieldParentPtr("node", node);
- ctx.deinit();
- }
-}
-
-const DeferredContext = struct {
- arena: std.mem.Allocator,
- layer: *DeferringLayer,
- transfer: *Transfer,
- frame_id: u32,
- url: [:0]const u8,
- forward: Forward,
- node: std.DoublyLinkedList.Node = .{},
-
- buffered: std.ArrayList(BufferedEvent) = .{},
- done: bool = false,
- deferring: bool = false,
- terminal: bool = false,
- stable_resp: ?StableResponse = null,
-
- const BufferedEvent = union(enum) {
- start,
- header,
- data: []const u8,
- done,
- err: anyerror,
- };
-
- fn deinit(self: *DeferredContext) void {
- self.layer.network.app.arena_pool.release(self.arena);
- }
-
- fn setStableResponse(self: *DeferredContext, response: Response) !void {
- if (self.stable_resp == null) {
- self.stable_resp = try Response.toStable(response, self.arena);
- }
- }
-
- fn shouldDefer(self: *DeferredContext) bool {
- const req = self.transfer.req;
- const blocking_id = self.transfer.client.blocking_requests.get(req.frame_id) orelse return false;
- return self.transfer.id != blocking_id;
- }
-
- fn startCallback(response: Response) anyerror!void {
- const self: *DeferredContext = @ptrCast(@alignCast(response.ctx));
-
- if (!self.deferring and !self.shouldDefer()) {
- return self.forward.forwardStart(response);
- }
-
- log.debug(.http, "deferring start callback", .{ .url = self.url });
- try self.setStableResponse(response);
- self.deferring = true;
- try self.buffered.append(self.arena, .start);
- }
-
- fn headerCallback(response: Response) anyerror!HeaderResult {
- const self: *DeferredContext = @ptrCast(@alignCast(response.ctx));
-
- if (!self.deferring and !self.shouldDefer()) {
- return self.forward.forwardHeader(response);
- }
-
- log.debug(.http, "deferring header callback", .{ .url = self.url });
- try self.setStableResponse(response);
- self.deferring = true;
- try self.buffered.append(self.arena, .header);
- return .proceed;
- }
-
- fn dataCallback(response: Response, chunk: []const u8) anyerror!void {
- const self: *DeferredContext = @ptrCast(@alignCast(response.ctx));
-
- if (!self.deferring and !self.shouldDefer()) {
- return self.forward.forwardData(response, chunk);
- }
-
- log.debug(.http, "deferring data callback", .{ .url = self.url });
- try self.setStableResponse(response);
- self.deferring = true;
- try self.buffered.append(self.arena, .{ .data = try self.arena.dupe(u8, chunk) });
- }
-
- fn doneCallback(ctx: *anyopaque) anyerror!void {
- const self: *DeferredContext = @ptrCast(@alignCast(ctx));
-
- if (!self.deferring and !self.shouldDefer()) {
- defer self.deinit();
- self.done = true;
- self.layer.active.remove(&self.node);
- return self.forward.forwardDone();
- }
-
- log.debug(.http, "deferring done callback", .{ .url = self.url });
- self.deferring = true;
- self.terminal = true;
- try self.buffered.append(self.arena, .done);
- }
-
- fn errorCallback(ctx: *anyopaque, err: anyerror) void {
- const self: *DeferredContext = @ptrCast(@alignCast(ctx));
-
- if (!self.deferring and !self.shouldDefer()) {
- defer self.deinit();
- self.done = true;
- self.layer.active.remove(&self.node);
- self.forward.forwardErr(err);
- return;
- }
-
- log.debug(.http, "deferring error callback", .{ .url = self.url, .err = err });
- self.deferring = true;
- self.terminal = true;
- self.buffered.append(self.arena, .{ .err = err }) catch {};
- }
-
- fn shutdownCallback(ctx: *anyopaque) void {
- const self: *DeferredContext = @ptrCast(@alignCast(ctx));
- if (self.done) return;
-
- defer self.deinit();
- self.done = true;
- self.layer.active.remove(&self.node);
-
- log.debug(.http, "deferring shutdown callback", .{});
- self.forward.forwardShutdown();
- }
-
- fn fire(self: *DeferredContext) void {
- defer self.deinit();
-
- for (self.buffered.items) |event| {
- switch (event) {
- .start => {
- const stable_response = self.stable_resp orelse @panic("stable_resp must be set for start events");
- const response = Response.fromStable(&stable_response);
-
- self.forward.forwardStart(response) catch |err| {
- log.err(.http, "deferred start callback", .{ .err = err, .url = self.url });
- self.forward.forwardErr(err);
- return;
- };
- },
- .header => {
- const stable_response = self.stable_resp orelse @panic("stable_resp must be set for header events");
- const response = Response.fromStable(&stable_response);
-
- const result = self.forward.forwardHeader(response) catch |err| {
- log.err(.http, "deferred header callback", .{ .err = err, .url = self.url });
- self.forward.forwardErr(err);
- return;
- };
- if (result == .abort) {
- self.forward.forwardErr(error.Abort);
- return;
- }
- },
- .data => |chunk| {
- const stable_response = self.stable_resp orelse @panic("stable_resp must be set for data events");
- const response = Response.fromStable(&stable_response);
-
- self.forward.forwardData(response, chunk) catch |err| {
- log.err(.http, "deferred data callback", .{ .err = err, .url = self.url });
- self.forward.forwardErr(err);
- return;
- };
- },
- .done => {
- self.forward.forwardDone() catch |err| {
- log.err(.http, "deferred done callback", .{ .err = err, .url = self.url });
- self.forward.forwardErr(err);
- };
-
- return;
- },
- .err => |err| {
- self.forward.forwardErr(err);
- return;
- },
- }
- }
- }
-
- fn firePartial(self: *DeferredContext) void {
- const stable_response = self.stable_resp orelse @panic("stable_resp must be set for any of the partial fire events");
- const response = Response.fromStable(&stable_response);
-
- for (self.buffered.items) |event| {
- switch (event) {
- .start => {
- self.forward.forwardStart(response) catch |err| {
- log.err(.http, "defer part start callback", .{ .err = err, .url = self.url });
- self.forward.forwardErr(err);
- return;
- };
- },
- .header => {
- const result = self.forward.forwardHeader(response) catch |err| {
- log.err(.http, "defer part header callback", .{ .err = err, .url = self.url });
- self.forward.forwardErr(err);
- return;
- };
- if (result == .abort) {
- self.forward.forwardErr(error.Abort);
- return;
- }
- },
- .data => |chunk| {
- self.forward.forwardData(response, chunk) catch |err| {
- log.err(.http, "defer part data callback", .{ .err = err, .url = self.url });
- self.forward.forwardErr(err);
- return;
- };
- },
- .done, .err => @panic("firePartial cant fire terminal events"),
- }
- }
-
- self.buffered.clearRetainingCapacity();
- }
-};
diff --git a/src/network/layer/Forward.zig b/src/network/layer/Forward.zig
deleted file mode 100644
index 0f218f709..000000000
--- a/src/network/layer/Forward.zig
+++ /dev/null
@@ -1,76 +0,0 @@
-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
-//
-// Francis Bouvier <[email protected]>
-// Pierre Tachoire <[email protected]>
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-// A snapshot of the original ctx + callbacks from a Request, taken before a
-// layer overwrites them with its own wrappers. The layer's wrapper callbacks
-// call forwardX(...) to invoke the captured originals with the original ctx.
-const Request = @import("../../browser/HttpClient.zig").Request;
-const Response = @import("../../browser/HttpClient.zig").Response;
-const HeaderResult = @import("../../browser/HttpClient.zig").HeaderResult;
-
-const Forward = @This();
-
-ctx: *anyopaque,
-start: ?Request.StartCallback,
-header: Request.HeaderCallback,
-data: Request.DataCallback,
-done: Request.DoneCallback,
-err: Request.ErrorCallback,
-shutdown: ?Request.ShutdownCallback,
-
-pub fn capture(req: *const Request) Forward {
- return .{
- .ctx = req.ctx,
- .start = req.start_callback,
- .header = req.header_callback,
- .data = req.data_callback,
- .done = req.done_callback,
- .err = req.error_callback,
- .shutdown = req.shutdown_callback,
- };
-}
-
-pub fn forwardStart(self: Forward, response: Response) anyerror!void {
- var fwd = response;
- fwd.ctx = self.ctx;
- if (self.start) |cb| try cb(fwd);
-}
-
-pub fn forwardHeader(self: Forward, response: Response) anyerror!HeaderResult {
- var fwd = response;
- fwd.ctx = self.ctx;
- return self.header(fwd);
-}
-
-pub fn forwardData(self: Forward, response: Response, chunk: []const u8) anyerror!void {
- var fwd = response;
- fwd.ctx = self.ctx;
- return self.data(fwd, chunk);
-}
-
-pub fn forwardDone(self: Forward) anyerror!void {
- return self.done(self.ctx);
-}
-
-pub fn forwardErr(self: Forward, e: anyerror) void {
- self.err(self.ctx, e);
-}
-
-pub fn forwardShutdown(self: Forward) void {
- if (self.shutdown) |cb| cb(self.ctx);
-}
diff --git a/src/network/layer/InterceptionLayer.zig b/src/network/layer/InterceptionLayer.zig
deleted file mode 100644
index 785502606..000000000
--- a/src/network/layer/InterceptionLayer.zig
+++ /dev/null
@@ -1,327 +0,0 @@
-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
-//
-// Francis Bouvier <[email protected]>
-// Pierre Tachoire <[email protected]>
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-const std = @import("std");
-const builtin = @import("builtin");
-const lp = @import("lightpanda");
-const log = lp.log;
-
-const IS_DEBUG = builtin.mode == .Debug;
-
-const http = @import("../http.zig");
-const HttpClient = @import("../../browser/HttpClient.zig");
-const Request = @import("../../browser/HttpClient.zig").Request;
-const Transfer = @import("../../browser/HttpClient.zig").Transfer;
-const Response = @import("../../browser/HttpClient.zig").Response;
-const FulfilledResponse = @import("../../browser/HttpClient.zig").FulfilledResponse;
-const Layer = @import("../../browser/HttpClient.zig").Layer;
-const Forward = @import("Forward.zig");
-const HeaderResult = @import("../../browser/HttpClient.zig").HeaderResult;
-
-const InterceptionLayer = @This();
-
-// Count of intercepted requests. The client doesn't track intercepted transfers
-// on its own active counters: once intercepted, a transfer leaves the layer
-// chain and waits for the interceptor (CDP) to call continue/abort/fulfill.
-// We track them here so the network-idle / network-almost-idle CDP lifecycle
-// events don't fire prematurely.
-intercepted: usize = 0,
-
-next: Layer = undefined,
-
-pub fn layer(self: *InterceptionLayer) Layer {
- return .{
- .ptr = self,
- .vtable = &.{ .request = request },
- };
-}
-
-fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
- const self: *InterceptionLayer = @ptrCast(@alignCast(ptr));
- const req = &transfer.req;
-
- const ctx = try transfer.arena.create(InterceptContext);
- ctx.* = .{
- .layer = self,
- .transfer = transfer,
- .forward = Forward.capture(req),
- };
-
- // Install our wrappers on the transfer's request. The interceptor wants to
- // observe every callback (start/header/data/done/err/shutdown) so it can
- // mirror the Network.* CDP events.
- req.ctx = ctx;
- if (ctx.forward.start != null) req.start_callback = InterceptContext.startCallback;
- req.header_callback = InterceptContext.headerCallback;
- req.data_callback = InterceptContext.dataCallback;
- req.done_callback = InterceptContext.doneCallback;
- req.error_callback = InterceptContext.errorCallback;
- if (ctx.forward.shutdown != null) req.shutdown_callback = InterceptContext.shutdownCallback;
-
- req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });
-
- var wait_for_interception = false;
- req.notification.dispatch(.http_request_intercept, &.{
- .transfer = transfer,
- .wait_for_interception = &wait_for_interception,
- });
-
- log.debug(.http, "interception check", .{
- .wait_for_interception = wait_for_interception,
- .intercepted = self.intercepted,
- .url = req.url,
- });
-
- if (!wait_for_interception) {
- return self.next.request(transfer);
- }
-
- // Paused: the CDP listener stashed `transfer` and will eventually call
- // continueRequest / abortRequest / fulfillRequest. Until then, CDP owns
- // the transfer's lifecycle. Park keeps the outer Client.request errdefer
- // from tearing it down.
- self.intercepted += 1;
- transfer.park(.intercept_request);
- if (comptime IS_DEBUG) {
- log.debug(.http, "wait for interception", .{ .intercepted = self.intercepted });
- }
-}
-
-pub const InterceptContext = struct {
- layer: *InterceptionLayer,
- transfer: *Transfer,
- forward: Forward,
- content_length: usize = 0,
-
- fn startCallback(response: Response) anyerror!void {
- const self: *InterceptContext = @ptrCast(@alignCast(response.ctx));
- log.debug(.http, "intercept start", .{ .url = self.transfer.req.url });
- return self.forward.forwardStart(response);
- }
-
- fn headerCallback(response: Response) anyerror!HeaderResult {
- const self: *InterceptContext = @ptrCast(@alignCast(response.ctx));
- log.debug(.http, "intercept header", .{
- .url = self.transfer.req.url,
- .status = response.status(),
- .content_length = response.contentLength(),
- });
-
- self.content_length = response.contentLength() orelse 0;
-
- self.transfer.req.notification.dispatch(.http_response_header_done, &.{
- .transfer = self.transfer,
- .response = &response,
- });
-
- return self.forward.forwardHeader(response);
- }
-
- fn dataCallback(response: Response, chunk: []const u8) anyerror!void {
- const self: *InterceptContext = @ptrCast(@alignCast(response.ctx));
- log.debug(.http, "intercept data", .{
- .url = self.transfer.req.url,
- .len = chunk.len,
- });
-
- self.transfer.req.notification.dispatch(.http_response_data, &.{
- .data = chunk,
- .transfer = self.transfer,
- });
-
- return self.forward.forwardData(response, chunk);
- }
-
- fn doneCallback(ctx: *anyopaque) anyerror!void {
- const self: *InterceptContext = @ptrCast(@alignCast(ctx));
-
- log.debug(.http, "intercept done", .{
- .url = self.transfer.req.url,
- .content_length = self.content_length,
- });
-
- self.transfer.req.notification.dispatch(.http_request_done, &.{
- .transfer = self.transfer,
- .content_length = self.content_length,
- });
- return self.forward.forwardDone();
- }
-
- fn errorCallback(ctx: *anyopaque, err: anyerror) void {
- const self: *InterceptContext = @ptrCast(@alignCast(ctx));
-
- log.debug(.http, "intercept error", .{
- .url = self.transfer.req.url,
- .err = err,
- });
- self.transfer.req.notification.dispatch(.http_request_fail, &.{
- .transfer = self.transfer,
- .err = err,
- });
- self.forward.forwardErr(err);
- }
-
- fn shutdownCallback(ctx: *anyopaque) void {
- const self: *InterceptContext = @ptrCast(@alignCast(ctx));
-
- log.debug(.http, "intercept shutdown", .{ .url = self.transfer.req.url });
- self.transfer.req.notification.dispatch(.http_request_fail, &.{
- .transfer = self.transfer,
- .err = error.Shutdown,
- });
- self.forward.forwardShutdown();
- }
-};
-
-// CDP-driven resolution entry points. The transfer was paused inside `request`
-// (state = .parked = .intercept_request). One of these three is called by CDP
-// to resume / drop the transfer.
-
-pub fn continueRequest(self: *InterceptionLayer, transfer: *Transfer) anyerror!void {
- if (comptime IS_DEBUG) {
- lp.assert(self.intercepted > 0, "InterceptionLayer.continueRequest", .{ .value = self.intercepted });
- log.debug(.http, "continue transfer", .{ .intercepted = self.intercepted });
- }
-
- // Resume the layer chain. Ownership is re-handed to whichever subsequent
- // layer commits the transfer (queue, multi, or another park). If the
- // chain fails before any commit, we clean up here — mirror the errdefer
- // pattern in Client.request.
- transfer.unpark();
- self.next.request(transfer) catch |err| {
- if (transfer.state == .created) {
- transfer.abort(err);
- }
- return err;
- };
-}
-
-pub fn abortRequest(self: *InterceptionLayer, transfer: *Transfer) void {
- if (comptime IS_DEBUG) {
- lp.assert(self.intercepted > 0, "InterceptionLayer.abortRequest", .{ .value = self.intercepted });
- log.debug(.http, "abort transfer", .{ .intercepted = self.intercepted });
- }
- transfer.abortParked(error.Abort);
-}
-
-pub fn fulfillRequest(
- self: *InterceptionLayer,
- transfer: *Transfer,
- status: u16,
- headers: []const http.Header,
- body: ?[]const u8,
-) !void {
- if (comptime IS_DEBUG) {
- lp.assert(self.intercepted > 0, "InterceptionLayer.fulfillRequest", .{ .value = self.intercepted });
- log.debug(.http, "fulfill transfer", .{ .intercepted = self.intercepted });
- }
-
- // Leave the parked state (accounting `intercepted` exactly once) and move to
- // .completing BEFORE running the user callbacks.
- transfer.unpark();
-
- if (HttpClient.isRedirectStatus(status)) {
- if (findLocation(headers)) |location| {
- fulfilledRedirect(transfer, status, headers, location) catch |err| {
- if (transfer.state == .created) {
- transfer.abort(err);
- }
- return err;
- };
- self.next.request(transfer) catch |err| {
- if (transfer.state == .created) {
- transfer.abort(err);
- }
- return err;
- };
- return;
- }
- }
-
- // Not a redirect: move to .completing BEFORE running the user callbacks.
- transfer.state = .completing;
- defer transfer.deinit();
-
- // `done` flips true once we've called the user's done_callback. If
- // done_callback itself throws, the user already saw their end-of-flow
- // notification; suppress error_callback to avoid double-notify.
- var done: bool = false;
- fulfillInner(&transfer.req, status, headers, body, &done) catch |err| {
- if (!done) {
- // safe here despite the defer transfer.deinit() above since the
- // state == .completing
- transfer.abort(err);
- }
- return err;
- };
-}
-
-fn fulfillInner(
- req: *Request,
- status: u16,
- headers: []const http.Header,
- body: ?[]const u8,
- done: *bool,
-) !void {
- const fulfilled = FulfilledResponse{
- .status = status,
- .url = req.url,
- .headers = headers,
- .body = body,
- };
-
- const response = Response.fromFulfilled(req.ctx, &fulfilled);
-
- if (req.start_callback) |cb| {
- try cb(response);
- }
-
- const result = try req.header_callback(response);
- if (result == .abort) {
- return error.Abort;
- }
-
- if (body) |b| {
- try req.data_callback(response, b);
- }
-
- done.* = true;
- try req.done_callback(req.ctx);
-}
-
-fn fulfilledRedirect(transfer: *Transfer, status: u16, headers: []const http.Header, location: []const u8) !void {
- // retrieve cookies from the fulfilled response's headers.
- if (transfer.req.cookie_jar) |jar| {
- for (headers) |hdr| {
- if (std.ascii.eqlIgnoreCase(hdr.name, "set-cookie")) {
- try jar.populateFromResponse(transfer.req.url, hdr.value);
- }
- }
- }
-
- try transfer.applyRedirectTarget(transfer.req.url, location, status);
-}
-
-fn findLocation(headers: []const http.Header) ?[]const u8 {
- for (headers) |hdr| {
- if (std.ascii.eqlIgnoreCase(hdr.name, "location")) {
- return hdr.value;
- }
- }
- return null;
-}
diff --git a/src/network/layer/RobotsLayer.zig b/src/network/layer/RobotsLayer.zig
deleted file mode 100644
index eddccbb49..000000000
--- a/src/network/layer/RobotsLayer.zig
+++ /dev/null
@@ -1,291 +0,0 @@
-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
-//
-// Francis Bouvier <[email protected]>
-// Pierre Tachoire <[email protected]>
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-const std = @import("std");
-const lp = @import("lightpanda");
-
-const URL = @import("../../browser/URL.zig");
-const Layer = @import("../../browser/HttpClient.zig").Layer;
-const Transfer = @import("../../browser/HttpClient.zig").Transfer;
-const Response = @import("../../browser/HttpClient.zig").Response;
-const HeaderResult = @import("../../browser/HttpClient.zig").HeaderResult;
-
-const Robots = @import("../Robots.zig");
-const Network = @import("../Network.zig");
-
-const log = lp.log;
-const Allocator = std.mem.Allocator;
-
-const RobotsLayer = @This();
-
-next: Layer = undefined,
-network: *Network,
-allocator: Allocator,
-pending: std.StringHashMapUnmanaged(std.ArrayList(*Transfer)) = .empty,
-
-pub fn layer(self: *RobotsLayer) Layer {
- return .{
- .ptr = self,
- .vtable = &.{
- .request = request,
- },
- };
-}
-
-pub fn deinit(self: *RobotsLayer, allocator: Allocator) void {
- var it = self.pending.iterator();
- while (it.next()) |entry| {
- entry.value_ptr.deinit(allocator);
- }
- self.pending.deinit(allocator);
-}
-
-fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
- const self: *RobotsLayer = @ptrCast(@alignCast(ptr));
-
- if (transfer.req.internal) {
- return self.next.request(transfer);
- }
-
- const url = transfer.req.url;
- const robots_url = try URL.getRobotsUrl(transfer.arena, url);
-
- if (self.network.robot_store.get(robots_url)) |robot_entry| {
- switch (robot_entry) {
- .present => |robots| {
- const path = URL.getPathname(url);
-
- if (!robots.isAllowed(path)) {
- try transfer.client.runNextTick(
- transfer,
- null,
- .{
- .run = struct {
- fn run(t: *Transfer, _: ?*anyopaque) void {
- defer t.deinit();
-
- log.warn(.http, "blocked by robots", .{ .url = t.req.url });
- t.req.error_callback(t.req.ctx, error.RobotsBlocked);
- }
- }.run,
- },
- );
- return;
- }
- },
- .absent => {},
- }
- return self.next.request(transfer);
- }
-
- return self.fetchRobotsThenRequest(robots_url, transfer);
-}
-
-fn fetchRobotsThenRequest(
- self: *RobotsLayer,
- robots_url: [:0]const u8,
- transfer: *Transfer,
-) !void {
- const entry = try self.pending.getOrPut(self.allocator, robots_url);
-
- if (!entry.found_existing) {
- errdefer std.debug.assert(self.pending.remove(robots_url));
- entry.value_ptr.* = .empty;
-
- try entry.value_ptr.append(self.allocator, transfer);
- transfer.park(.robots);
- errdefer {
- entry.value_ptr.deinit(self.allocator);
- transfer.unpark();
- }
-
- const robots_ctx = try transfer.arena.create(RobotsContext);
- robots_ctx.* = .{
- .layer = self,
- .buffer = .empty,
- .arena = transfer.arena,
- .robots_url = robots_url,
- };
-
- // CRITICAL: build a fresh Headers for the inner robots fetch.
- // We value-copy req from the parent, but Headers is a struct wrapping
- // a *curl_slist — value copy shares the pointer. Letting Client.request
- // take ownership of a shared headers list means both transfers will
- // free it at deinit time -> double-free. The robots.txt fetch is a
- // system-level GET anyway, no need to inherit the parent's user headers.
- var new_req = transfer.req;
- new_req.headers = try transfer.client.newHeaders();
- errdefer new_req.headers.deinit();
- new_req.method = .GET;
- new_req.url = robots_url;
- new_req.internal = true;
- new_req.resource_type = .fetch;
- new_req.body = null;
- new_req.ctx = robots_ctx;
- new_req.start_callback = null;
- new_req.header_callback = RobotsContext.headerCallback;
- new_req.data_callback = RobotsContext.dataCallback;
- new_req.done_callback = RobotsContext.doneCallback;
- new_req.error_callback = RobotsContext.errorCallback;
- new_req.shutdown_callback = RobotsContext.shutdownCallback;
-
- log.debug(.browser, "fetching robots.txt", .{ .robots_url = robots_url });
- try transfer.client.request(new_req, transfer.owner);
- } else {
- // Already one in flight, just queue behind.
- try entry.value_ptr.append(self.allocator, transfer);
-
- // Parked: RobotsLayer owns destruction via flushPending / flushPendingShutdown
- // until robots.txt resolves. Without this, Client.request's errdefer (or
- // any caller's cleanup) would deinit a transfer that's still on the
- // pending list, leaving flushPending with a dangling pointer.
- transfer.park(.robots);
- }
-}
-
-fn flushPending(self: *RobotsLayer, robots_url: [:0]const u8, allowed: bool) void {
- var queued = self.pending.fetchRemove(robots_url) orelse return;
- defer queued.value.deinit(self.allocator);
-
- for (queued.value.items) |transfer| {
- if (!allowed) {
- log.warn(.http, "blocked by robots", .{ .url = transfer.req.url });
- transfer.abort(error.RobotsBlocked);
- } else {
- // Hand back to the layer chain. If a downstream layer commits
- // (multi / queue / park), state advances past .created. If it
- // fails before committing, we clean up here.
- transfer.unpark();
- self.next.request(transfer) catch |e| {
- if (transfer.state == .created) {
- transfer.abort(e);
- }
- };
- }
- }
-}
-
-// Invariant: shutdown_callback fires on a Transfer only via Transfer.kill,
-// and the only callers of kill are Client.abortOwner / .abortRequests
-// (owner-driven teardown). So if THIS robots fetch's shutdown_callback
-// fired, the owner is being torn down — every parked transfer in this
-// pending queue is on the same owner list and is already being killed by
-// the same walk. We just need to drop the pending entry; the owner walk
-// handles the rest. (If a future code path adds per-transfer kill
-// without owner teardown, this assumption breaks — see comment above
-// detachOrDeinit in HttpClient.zig.)
-fn flushPendingShutdown(self: *RobotsLayer, robots_url: [:0]const u8) void {
- var pending = self.pending.fetchRemove(robots_url) orelse return;
- pending.value.deinit(self.allocator);
-}
-
-const RobotsContext = struct {
- layer: *RobotsLayer,
- arena: Allocator,
- robots_url: [:0]const u8,
- buffer: std.ArrayList(u8),
- status: u16 = 0,
-
- fn deinit(self: *RobotsContext) void {
- self.buffer.deinit(self.layer.allocator);
- self.layer.allocator.destroy(self);
- }
-
- fn headerCallback(response: Response) anyerror!HeaderResult {
- const self: *RobotsContext = @ptrCast(@alignCast(response.ctx));
- switch (response.inner) {
- .transfer => |t| {
- if (t.res.header) |hdr| {
- log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = self.robots_url });
- self.status = hdr.status;
- }
- if (t.getContentLength()) |cl| {
- try self.buffer.ensureTotalCapacity(self.arena, cl);
- }
- },
- else => {},
- }
-
- return .proceed;
- }
-
- fn dataCallback(response: Response, data: []const u8) anyerror!void {
- const self: *RobotsContext = @ptrCast(@alignCast(response.ctx));
- try self.buffer.appendSlice(self.arena, data);
- }
-
- fn doneCallback(ctx_ptr: *anyopaque) anyerror!void {
- const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));
- const l = self.layer;
- const robots_url = self.robots_url;
-
- var allowed = true;
- const network = l.network;
-
- switch (self.status) {
- 200 => {
- if (self.buffer.items.len > 0) {
- const robots: ?Robots = network.robot_store.robotsFromBytes(
- network.config.http_headers.user_agent,
- self.buffer.items,
- ) catch blk: {
- log.warn(.browser, "failed to parse robots", .{ .robots_url = robots_url });
- try network.robot_store.putAbsent(robots_url);
- break :blk null;
- };
- if (robots) |r| {
- try network.robot_store.put(robots_url, r);
- const path = URL.getPathname(l.pending.get(robots_url).?.items[0].req.url);
- allowed = r.isAllowed(path);
- }
- }
- },
- 404 => {
- log.debug(.http, "robots not found", .{ .url = robots_url });
- try network.robot_store.putAbsent(robots_url);
- },
- else => {
- log.debug(.http, "unexpected status on robots", .{
- .url = robots_url,
- .status = self.status,
- });
- try network.robot_store.putAbsent(robots_url);
- },
- }
-
- l.flushPending(robots_url, allowed);
- }
-
- fn errorCallback(ctx_ptr: *anyopaque, err: anyerror) void {
- const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));
- const l = self.layer;
- const robots_url = self.robots_url;
-
- log.warn(.http, "robots fetch failed", .{ .err = err });
- l.flushPending(robots_url, true);
- }
-
- fn shutdownCallback(ctx_ptr: *anyopaque) void {
- const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));
- const l = self.layer;
- const robots_url = self.robots_url;
-
- log.debug(.http, "robots fetch shutdown", .{});
- l.flushPendingShutdown(robots_url);
- }
-};
diff --git a/src/network/layer/WebBotAuthLayer.zig b/src/network/layer/WebBotAuthLayer.zig
deleted file mode 100644
index 810d94ef2..000000000
--- a/src/network/layer/WebBotAuthLayer.zig
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
-//
-// Francis Bouvier <[email protected]>
-// Pierre Tachoire <[email protected]>
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-const URL = @import("../../browser/URL.zig");
-const Layer = @import("../../browser/HttpClient.zig").Layer;
-const Transfer = @import("../../browser/HttpClient.zig").Transfer;
-
-const WebBotAuthLayer = @This();
-
-next: Layer = undefined,
-
-pub fn layer(self: *WebBotAuthLayer) Layer {
- return .{
- .ptr = self,
- .vtable = &.{ .request = request },
- };
-}
-
-fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
- const self: *WebBotAuthLayer = @ptrCast(@alignCast(ptr));
-
- const wba = transfer.client.network.web_bot_auth orelse @panic("WebBotAuthLayer shouldn't be active without WebBotAuth");
-
- const authority = URL.getHost(transfer.req.url);
- try wba.signRequest(transfer.arena, &transfer.req.headers, authority);
-
- return self.next.request(transfer);
-}