Skip to content

Lightpanda: the headless browser designed for AI and automation

cdp: pause requests at the Fetch response stage

Thawindu Angesh Tuto41ecc246d1c8
diff --git a/src/Notification.zig b/src/Notification.zig
index e4be273e4..26d48abf9 100644
--- a/src/Notification.zig
+++ b/src/Notification.zig
@@ -89,6 +89,7 @@ const EventListeners = struct {
http_request_intercept: List = .{},
http_request_done: List = .{},
http_request_auth_required: List = .{},
+ http_response_intercept: List = .{},
http_request_served_from_cache: List = .{},
http_response_data: List = .{},
http_response_header_done: List = .{},
@@ -121,6 +122,7 @@ const Events = union(enum) {
http_request_start: *const RequestStart,
http_request_intercept: *const RequestIntercept,
http_request_auth_required: *const RequestAuthRequired,
+ http_response_intercept: *const ResponseIntercept,
http_request_served_from_cache: *const RequestServedFromCache,
http_request_done: *const RequestDone,
http_response_data: *const ResponseData,
@@ -247,6 +249,13 @@ pub const RequestAuthRequired = struct {
wait_for_interception: *bool,
};
+// A final response is complete (status, headers and the whole body are on the
+// transfer) and has not been delivered to its consumer yet.
+pub const ResponseIntercept = struct {
+ transfer: *Transfer,
+ wait_for_interception: *bool,
+};
+
pub const ResponseData = struct {
data: []const u8,
transfer: *Transfer,
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 776c67c58..241621794 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -1690,6 +1690,8 @@ fn isTerminal(msg: *Inbox.Message) bool {
fn isFetchInterceptionMethod(method: []const u8) bool {
return std.mem.eql(u8, method, "Fetch.continueRequest") or
+ std.mem.eql(u8, method, "Fetch.continueResponse") or
+ std.mem.eql(u8, method, "Fetch.getResponseBody") or
std.mem.eql(u8, method, "Fetch.failRequest") or
std.mem.eql(u8, method, "Fetch.fulfillRequest") or
std.mem.eql(u8, method, "Fetch.continueWithAuth");
@@ -1876,22 +1878,22 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
try transfer.materializeResponse(msg.conn, .{ .check_content_length = false });
if (self.enforceCorsResponse(msg, transfer)) return true;
- try transfer.handleRedirect(location.value);
-
- if (!transfer.req.internal) {
- lp.metrics.http_redirects.incr();
- }
+ // location.value is curl-owned and dies with the conn.
+ const owned_location = try transfer.arena.dupe(u8, location.value);
// The multi should keep this alive. Releasing it now makes
// it so we can re-enter the pipeline and keeps the code
- // streamlined.
+ // streamlined. Everything the redirect needs is
+ // materialized on the transfer.
self.removeConn(msg.conn);
transfer._conn = null;
- transfer.state = .created;
- // We have to restart (.redirect == .start) the whole
- // pipeline, rechecking the cache/robot/ratelimit/...
- try self.pipeline(transfer, .redirect);
+ if (self.interceptResponse(transfer, .{ .redirect = owned_location })) {
+ return false;
+ }
+
+ transfer.state = .created;
+ try self.followRedirect(transfer, owned_location);
return true;
},
// error_callback surfaces this as a TypeError.
@@ -1914,6 +1916,9 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
if (effective_err != null and !is_conn_close_recv) {
const err = transfer.res.callback_error orelse effective_err.?;
+ // A failure the consumer caused (abort, cancel) is not a network
+ // failure to offer for interception.
+ const offer = http.errorReason(err) != .aborted;
if (try transfer.deliversHeaderBeforeError(msg.conn)) {
try transfer.materializeResponse(msg.conn, .{ .check_content_length = false });
if (self.enforceCorsResponse(msg, transfer)) {
@@ -1921,11 +1926,17 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
}
self.removeConn(msg.conn);
transfer._conn = null;
+ if (offer and self.interceptResponse(transfer, .{ .failed = .{ .err = err, .header_first = true } })) {
+ return false;
+ }
try transfer.bufferHeaderThenError(err);
return true;
}
self.removeConn(msg.conn);
transfer._conn = null;
+ if (offer and self.interceptResponse(transfer, .{ .failed = .{ .err = err, .header_first = false } })) {
+ return false;
+ }
transfer.failAsync(err);
return true;
}
@@ -1946,14 +1957,58 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
self.removeConn(msg.conn);
transfer._conn = null;
+ // A revalidation (304) is not offered: what it delivers is the cached
+ // body, which the response on the transfer does not carry.
+ if (transfer._cache_intent != .revalidate and self.interceptResponse(transfer, .final)) {
+ return false;
+ }
+
+ try self.deliverResponse(transfer);
+ return true;
+}
+
+// CDP Fetch response-stage interception: offer a completed transfer (its conn
+// already released) to CDP. True when CDP took it: the transfer is parked
+// until continueInterceptedResponse, fulfillIntercepted or abortParked.
+fn interceptResponse(self: *Client, transfer: *Transfer, pause: Transfer.ResponsePause) bool {
+ transfer._response_pause = pause;
+ var wait_for_interception = false;
+ transfer.notify(
+ .http_response_intercept,
+ &.{ .transfer = transfer, .wait_for_interception = &wait_for_interception },
+ );
+ if (!wait_for_interception) {
+ return false;
+ }
+ self.intercepted += 1;
+ transfer.state = .{ .parked = .intercept_response };
+ return true;
+}
+
+// Follow a materialized redirect response: point the transfer at the
+// redirect target and restart the whole pipeline (.redirect == .start),
+// rechecking the cache/robot/ratelimit/...
+fn followRedirect(self: *Client, transfer: *Transfer, location: []const u8) !void {
+ try transfer.handleRedirect(location);
+
+ if (!transfer.req.internal) {
+ lp.metrics.http_redirects.incr();
+ }
+
+ try self.pipeline(transfer, .redirect);
+}
+
+// The tail of a completed, materialized response: settle the cache and buffer
+// the events for dispatch. Runs straight from processOneMessage, or from
+// continueInterceptedResponse once CDP releases a response-stage pause.
+fn deliverResponse(self: *Client, transfer: *Transfer) !void {
if (try self.cacheRevalidated(transfer)) {
- return true;
+ return;
}
self.cacheStore(transfer);
transfer._content_length = transfer.getContentLength() orelse 0;
try transfer.bufferEvents(transfer.res.buffer.items);
- return true;
}
// Commit a configured conn to the multi handle. On error the conn is left
@@ -2193,6 +2248,36 @@ pub fn continueTransfer(self: *Client, transfer: *Transfer) !void {
};
}
+// CDP Fetch.continueResponse (or continueRequest on a response-stage pause):
+// release what the transfer is holding, unchanged — deliver the response,
+// follow the redirect, or report the network failure.
+pub fn continueInterceptedResponse(self: *Client, transfer: *Transfer) !void {
+ if (comptime lp.IS_DEBUG) {
+ lp.assert(self.intercepted > 0, "Client.continueInterceptedResponse", .{ .value = self.intercepted });
+ }
+
+ transfer.unpark();
+ switch (transfer._response_pause) {
+ .final => self.deliverResponse(transfer) catch |err| {
+ transfer.abortPipelineError(err);
+ return err;
+ },
+ .redirect => |location| self.followRedirect(transfer, location) catch |err| {
+ transfer.abortPipelineError(err);
+ return err;
+ },
+ .failed => |failure| {
+ if (!failure.header_first) {
+ return transfer.failAsync(failure.err);
+ }
+ transfer.bufferHeaderThenError(failure.err) catch |err| {
+ transfer.abortPipelineError(err);
+ return err;
+ };
+ },
+ }
+}
+
// CDP Fetch.continueRequest: resume a transfer parked at the interception
// gate, re-entering the pipeline at the step after interception. The CDP
// command may have mutated req (url / method / headers / body) first.
@@ -2493,6 +2578,11 @@ pub const Transfer = struct {
// What the cache wants from this transfer's response.
_cache_intent: CacheIntent = .none,
+ // What a response-stage CDP pause (.intercept_response) is holding, and so
+ // what releasing it unchanged does: deliver the response, follow the
+ // redirect, or fail with the network error.
+ _response_pause: ResponsePause = .final,
+
// Cache key captured at lookup time (req.url before any redirect
// rewrote it). Only meaningful while _cache_intent != .none.
_cache_key: [:0]const u8 = "",
@@ -2611,6 +2701,12 @@ pub const Transfer = struct {
// waiting for continueWithAuth.
intercept_auth,
+ // CDP Fetch interception, response phase: processOneMessage completed
+ // the transfer (headers and body are materialized, the conn is
+ // released) and holds it before delivery, waiting for
+ // continueResponse / continueRequest / fulfillRequest / failRequest.
+ intercept_response,
+
// RobotsGate holds the transfer pending a robots.txt fetch.
robots,
@@ -2618,6 +2714,24 @@ pub const Transfer = struct {
cors,
};
+ pub const ResponsePause = union(enum) {
+ // A final response, delivered to the consumer on continue.
+ final,
+ // A redirect response; the arena-owned Location is followed on continue.
+ redirect: []const u8,
+ // A network failure; on continue it is reported exactly as it would
+ // have been without the pause.
+ failed: Failure,
+
+ pub const Failure = struct {
+ err: anyerror,
+ // The consumer asked for the header before a body error
+ // (Request.header_before_body_error) and the header arrived:
+ // deliver it, then the error.
+ header_first: bool,
+ };
+ };
+
pub const HeaderResult = enum {
/// Continue processing normally.
proceed,
@@ -2633,11 +2747,11 @@ pub const Transfer = struct {
}
// Gate-facing: take the transfer out of .parked and return it to
- // the request flow (.created). This assumes pre-inflight handling (i.e. the
- // transfer was in .created before being parked). This is true today, but
- // could become false if Request Interception ever supports the "response"
- // requestStage (although, to support this, I think the safety of transfers
- // post-perform would need to be improved),
+ // the request flow (.created). A transfer parked before it went inflight
+ // re-enters the pipeline from here. One parked after completion
+ // (.intercept_auth, .intercept_response) holds no conn: its response is
+ // already materialized, so resuming it only buffers events, which
+ // scheduleDispatch accepts from .created.
pub fn unpark(self: *Transfer) void {
lp.assert(self.state == .parked, "Transfer.unpark", .{ .state = self.state });
self.leaveIntercept();
@@ -2652,7 +2766,7 @@ pub const Transfer = struct {
}
switch (self.state.parked) {
.robots, .cors => {},
- .intercept_request, .intercept_auth => {
+ .intercept_request, .intercept_auth, .intercept_response => {
lp.assert(self.client.intercepted > 0, "Transfer.leaveIntercept", .{ .value = self.client.intercepted });
self.client.intercepted -= 1;
},
@@ -2796,7 +2910,7 @@ pub const Transfer = struct {
switch (self.state.parked) {
.cors => self.client.cors.remove(self),
.robots => self.client.robots.remove(self),
- .intercept_auth, .intercept_request => {},
+ .intercept_auth, .intercept_request, .intercept_response => {},
}
}
@@ -3621,9 +3735,12 @@ pub const Transfer = struct {
self.req.url = url;
}
+ // Reads only the materialized redirect response (res.header, res.headers),
+ // never the conn, so it runs the same whether the conn was just released
+ // or the transfer sat parked for a CDP response-stage interception.
fn handleRedirect(transfer: *Transfer, location: []const u8) !void {
const req = &transfer.req;
- const conn = transfer._conn.?;
+ const head = transfer.res.header orelse return error.MissingResponseHead;
// Cookies were applied while materializing the redirect response.
@@ -3631,23 +3748,20 @@ pub const Transfer = struct {
// remaining hops.
if (req.referrer_policy != null) {
// referrer-policy is re-applied on every hop.
- var i: usize = 0;
- while (conn.getResponseHeader("referrer-policy", i)) |hdr| : (i += 1) {
+ for (transfer.res.headers) |hdr| {
+ if (!std.mem.eql(u8, hdr.name, "referrer-policy")) {
+ continue;
+ }
if (referrer.parseHeader(hdr.value)) |policy| {
req.referrer_policy = policy;
}
- if (i >= hdr.amount) {
- break;
- }
}
}
- // base_url and location are owned by curl; applyRedirectTarget resolves a
- // fresh arena-owned copy that gets stored in transfer.req.url.
- const base_url = try conn.getEffectiveUrl();
- const status = try conn.getResponseCode();
+ // applyRedirectTarget resolves a fresh arena-owned copy of the target
+ // that gets stored in transfer.req.url.
transfer.redirectTiming(transfer.res.headers);
- try transfer.applyRedirectTarget(std.mem.span(base_url), location, status);
+ try transfer.applyRedirectTarget(std.mem.span(head.url), location, head.status);
}
// Called above (in handleRedirect) and by a CDP fulfill request which redirects
@@ -4561,8 +4675,10 @@ const AdBlocker = @import("adblock/AdBlocker.zig");
// exercises are ever set, by initTestClient or by the test itself.
var test_network: Network = undefined;
-test "HttpClient: isFetchInterceptionMethod matches the four Fetch methods" {
+test "HttpClient: isFetchInterceptionMethod matches the Fetch interception methods" {
try testing.expect(isFetchInterceptionMethod("Fetch.continueRequest"));
+ try testing.expect(isFetchInterceptionMethod("Fetch.continueResponse"));
+ try testing.expect(isFetchInterceptionMethod("Fetch.getResponseBody"));
try testing.expect(isFetchInterceptionMethod("Fetch.failRequest"));
try testing.expect(isFetchInterceptionMethod("Fetch.fulfillRequest"));
try testing.expect(isFetchInterceptionMethod("Fetch.continueWithAuth"));
diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zig
index 13faa5fb5..60a2d8010 100644
--- a/src/server/cdp/CDP.zig
+++ b/src/server/cdp/CDP.zig
@@ -872,6 +872,7 @@ pub const BrowserContext = struct {
try self.intercept_state.setPatterns(patterns);
self.fetch_session_id = session_id;
try self.notification.register(.http_request_intercept, self, onHttpRequestIntercept);
+ try self.notification.register(.http_response_intercept, self, onHttpResponseIntercept);
if (authRequests) {
try self.notification.register(.http_request_auth_required, self, onHttpRequestAuthRequired);
}
@@ -918,6 +919,7 @@ pub const BrowserContext = struct {
fn fetchDisable(self: *BrowserContext) void {
self.notification.unregister(.http_request_intercept, self);
+ self.notification.unregister(.http_response_intercept, self);
self.notification.unregister(.http_request_auth_required, self);
self.intercept_state.clearPatterns();
self.fetch_session_id = null;
@@ -1078,6 +1080,13 @@ pub const BrowserContext = struct {
try @import("domains/fetch.zig").requestIntercept(arena, self, msg);
}
+ fn onHttpResponseIntercept(ctx: *anyopaque, msg: *const Notification.ResponseIntercept) !void {
+ const self: *BrowserContext = @ptrCast(@alignCast(ctx));
+ const arena = self.acquireNotificationArena();
+ defer self.releaseNotificationArena();
+ try @import("domains/fetch.zig").responseIntercept(arena, self, msg);
+ }
+
fn onHttpRequestFail(ctx: *anyopaque, msg: *const Notification.RequestFail) !void {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
return network_domain.httpRequestFail(self, msg);
diff --git a/src/server/cdp/domains/fetch.zig b/src/server/cdp/domains/fetch.zig
index 91964255a..279aabd1e 100644
--- a/src/server/cdp/domains/fetch.zig
+++ b/src/server/cdp/domains/fetch.zig
@@ -36,15 +36,19 @@ pub fn processMessage(cmd: *CDP.Command) !void {
disable,
enable,
continueRequest,
+ continueResponse,
failRequest,
fulfillRequest,
continueWithAuth,
+ getResponseBody,
}, cmd.input.action) orelse return error.UnknownMethod;
switch (action) {
.disable => return disable(cmd),
.enable => return enable(cmd),
.continueRequest => return continueRequest(cmd),
+ .continueResponse => return continueResponse(cmd),
+ .getResponseBody => return getResponseBody(cmd),
.continueWithAuth => return continueWithAuth(cmd),
.failRequest => return failRequest(cmd),
.fulfillRequest => return fulfillRequest(cmd),
@@ -120,7 +124,7 @@ pub fn requestIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *c
const session_id = bc.fetch_session_id orelse return;
const transfer = intercept.transfer;
- if (!bc.intercept_state.matches(transfer.req.url, transfer.req.resource_type)) {
+ if (!bc.intercept_state.matches(.Request, transfer.req.url, transfer.req.resource_type)) {
// Leaves wait_for_interception false: the request proceeds untouched.
return;
}
@@ -149,6 +153,126 @@ pub fn requestIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *c
intercept.wait_for_interception.* = true;
}
+pub fn responseIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *const Notification.ResponseIntercept) !void {
+ const session_id = bc.fetch_session_id orelse return;
+
+ const transfer = intercept.transfer;
+ if (!bc.intercept_state.matches(.Response, transfer.req.url, transfer.req.resource_type)) {
+ // Leaves wait_for_interception false: the response is delivered untouched.
+ return;
+ }
+
+ const intercept_id = try bc.intercept_state.put(transfer.id);
+ errdefer _ = bc.intercept_state.remove(intercept_id);
+
+ // The stage is told by the fields present: a network failure carries
+ // responseErrorReason, a response (final or redirect) its status and headers.
+ switch (transfer._response_pause) {
+ .failed => |failure| try bc.cdp.sendEvent("Fetch.requestPaused", .{
+ .requestId = &id.toInterceptId(intercept_id),
+ .frameId = &id.toFrameId(transfer.req.frame_id),
+ .request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data),
+ .resourceType = transfer.req.resource_type.string(),
+ .responseErrorReason = cdpErrorReason(failure.err),
+ .networkId = &id.toRequestId(transfer), // matches the Network REQ-ID
+ }, .{ .session_id = session_id }),
+ .final, .redirect => {
+ const head = transfer.res.header orelse return error.MissingResponseHead;
+ try bc.cdp.sendEvent("Fetch.requestPaused", .{
+ .requestId = &id.toInterceptId(intercept_id),
+ .frameId = &id.toFrameId(transfer.req.frame_id),
+ .request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data),
+ .resourceType = transfer.req.resource_type.string(),
+ .responseStatusCode = head.status,
+ .responseStatusText = transfer.statusText() orelse "",
+ .responseHeaders = transfer.responseHeaders(),
+ .networkId = &id.toRequestId(transfer), // matches the Network REQ-ID
+ }, .{ .session_id = session_id });
+ },
+ }
+
+ log.debug(.cdp, "response intercept", .{
+ .state = "paused",
+ .id = transfer.id,
+ .url = transfer.req.url,
+ .pause = std.meta.activeTag(transfer._response_pause),
+ });
+ // Await continueResponse, continueRequest, fulfillRequest or failRequest
+
+ intercept.wait_for_interception.* = true;
+}
+
+// The transfer a response-stage intercept id holds, left in the intercept
+// state. Null when the id is unknown, the transfer is gone, or the pause is
+// not at the response stage.
+fn pausedResponse(bc: *CDP.BrowserContext, request_id: []const u8) !?*HttpClient.Transfer {
+ const intercept_id = try idFromRequestId(request_id);
+ const transfer_id = bc.intercept_state.get(intercept_id) orelse return null;
+ const transfer = bc.cdp.browser.http_client.findTransfer(transfer_id) orelse return null;
+ if (transfer.state != .parked or transfer.state.parked != .intercept_response) {
+ return null;
+ }
+ return transfer;
+}
+
+fn continueResponse(cmd: *CDP.Command) !void {
+ const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+ const params = (try cmd.params(struct {
+ requestId: []const u8, // "INT-{d}"
+ responseCode: ?u16 = null,
+ responsePhrase: ?[]const u8 = null,
+ responseHeaders: ?[]const http.Header = null,
+ binaryResponseHeaders: ?[]const u8 = null,
+ })) orelse return error.InvalidParams;
+
+ if (params.responseCode != null or params.responsePhrase != null or params.responseHeaders != null or params.binaryResponseHeaders != null) {
+ log.debug(.not_implemented, "Fetch.continueResponse", .{ .params = "response overrides" });
+ return error.NotImplemented;
+ }
+
+ const transfer = (try pausedResponse(bc, params.requestId)) orelse {
+ return cmd.sendError(-32000, "Invalid InterceptionId.", .{});
+ };
+ _ = bc.intercept_state.remove(try idFromRequestId(params.requestId));
+
+ log.debug(.cdp, "response intercept", .{
+ .state = "continue",
+ .id = transfer.id,
+ .url = transfer.req.url,
+ });
+
+ try bc.cdp.browser.http_client.continueInterceptedResponse(transfer);
+ return cmd.sendResult(null, .{});
+}
+
+fn getResponseBody(cmd: *CDP.Command) !void {
+ const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+ const params = (try cmd.params(struct {
+ requestId: []const u8, // "INT-{d}"
+ })) orelse return error.InvalidParams;
+
+ const transfer = (try pausedResponse(bc, params.requestId)) orelse {
+ return cmd.sendError(-32000, "Can only get response body on requests captured after headers received.", .{});
+ };
+ switch (transfer._response_pause) {
+ .final => {},
+ .redirect => return cmd.sendError(-32000, "Response body is not available for redirects.", .{}),
+ .failed => return cmd.sendError(-32000, "Response body is not available for a failed request.", .{}),
+ }
+
+ const body = transfer.res.buffer.items;
+ if (std.unicode.utf8ValidateSlice(body)) {
+ return cmd.sendResult(.{
+ .body = body,
+ .base64Encoded = false,
+ }, .{ .size_hint = body.len });
+ }
+ return cmd.sendResult(.{
+ .body = network.Base64Body{ .data = body },
+ .base64Encoded = true,
+ }, .{ .size_hint = std.base64.standard.Encoder.calcSize(body.len) });
+}
+
fn continueRequest(cmd: *CDP.Command) !void {
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
const params = (try cmd.params(struct {
@@ -164,6 +288,16 @@ fn continueRequest(cmd: *CDP.Command) !void {
return error.NotImplemented;
}
+ if (try pausedResponse(bc, params.requestId)) |transfer| {
+ // The request has already been sent; only an unchanged continue is meaningful.
+ if (params.url != null or params.method != null or params.postData != null or params.headers != null) {
+ return cmd.sendError(-32602, "Cannot override request at the response stage.", .{});
+ }
+ _ = bc.intercept_state.remove(try idFromRequestId(params.requestId));
+ try bc.cdp.browser.http_client.continueInterceptedResponse(transfer);
+ return cmd.sendResult(null, .{});
+ }
+
const client = &bc.cdp.browser.http_client;
var intercept_state = &bc.intercept_state;
const intercept_id = try idFromRequestId(params.requestId);
@@ -347,7 +481,7 @@ pub fn requestAuthRequired(arena: Allocator, bc: *CDP.BrowserContext, intercept:
// Auth challenges are only surfaced for requests the patterns select.
const transfer = intercept.transfer;
- if (!bc.intercept_state.matches(transfer.req.url, transfer.req.resource_type)) {
+ if (!bc.intercept_state.matches(.Request, transfer.req.url, transfer.req.resource_type)) {
return;
}
@@ -385,6 +519,20 @@ pub fn requestAuthRequired(arena: Allocator, bc: *CDP.BrowserContext, intercept:
intercept.wait_for_interception.* = true;
}
+// The CDP Network.ErrorReason for a network failure paused at the response
+// stage. Failed is CDP's own generic reason, for failures it has no narrower
+// value for (TLS, protocol and size errors among them).
+fn cdpErrorReason(err: anyerror) ErrorReason {
+ return switch (err) {
+ error.OperationTimedout => .TimedOut,
+ error.CouldntResolveHost, error.CouldntResolveProxy => .NameNotResolved,
+ error.CouldntConnect => .ConnectionFailed,
+ error.GotNothing => .ConnectionClosed,
+ error.SendError, error.RecvError => .ConnectionReset,
+ else => .Failed,
+ };
+}
+
// Get u32 from requestId which is formatted as: "INT-{d}"
fn idFromRequestId(request_id: []const u8) !u32 {
if (!std.mem.startsWith(u8, request_id, "INT-")) {
@@ -408,6 +556,7 @@ pub const InterceptState = struct {
const Pattern = struct {
url: []const u8,
resource_type: ?ResourceType,
+ stage: RequestStage,
};
pub fn init(allocator: Allocator) !InterceptState {
@@ -424,6 +573,11 @@ pub const InterceptState = struct {
return intercept_id;
}
+ // The transfer id an intercept id holds, leaving the entry in place.
+ pub fn get(self: *const InterceptState, intercept_id: u32) ?u32 {
+ return self.waiting.get(intercept_id);
+ }
+
// Returns the transfer id if the intercept id was present and removed.
pub fn remove(self: *InterceptState, intercept_id: u32) ?u32 {
const entry = self.waiting.fetchSwapRemove(intercept_id) orelse return null;
@@ -442,23 +596,9 @@ pub const InterceptState = struct {
pub fn setPatterns(self: *InterceptState, patterns: ?[]const RequestPattern) !void {
self.clearPatterns();
- var has_response_stage = false;
const source: []const RequestPattern = patterns orelse &.{.{}};
- var count: usize = 0;
- for (source) |pattern| {
- if (pattern.requestStage == .Response) {
- has_response_stage = true;
- continue;
- }
- count += 1;
- }
-
- if (has_response_stage) {
- log.debug(.not_implemented, "Fetch.enable", .{ .params = "requestStage=Response" });
- }
-
- const owned = try self.allocator.alloc(Pattern, count);
+ const owned = try self.allocator.alloc(Pattern, source.len);
var initialized: usize = 0;
errdefer {
for (owned[0..initialized]) |pattern| {
@@ -467,12 +607,10 @@ pub const InterceptState = struct {
self.allocator.free(owned);
}
for (source) |pattern| {
- if (pattern.requestStage != .Request) {
- continue;
- }
owned[initialized] = .{
.url = try self.allocator.dupe(u8, pattern.urlPattern),
.resource_type = pattern.resourceType,
+ .stage = pattern.requestStage,
};
initialized += 1;
}
@@ -487,8 +625,11 @@ pub const InterceptState = struct {
self.patterns = &.{};
}
- pub fn matches(self: *const InterceptState, url: []const u8, resource_type: HttpClient.Request.ResourceType) bool {
+ pub fn matches(self: *const InterceptState, stage: RequestStage, url: []const u8, resource_type: HttpClient.Request.ResourceType) bool {
for (self.patterns) |pattern| {
+ if (pattern.stage != stage) {
+ continue;
+ }
if (pattern.resource_type) |rt| {
if (rt != cdpResourceType(resource_type)) {
continue;
@@ -569,6 +710,7 @@ pub const InterceptState = struct {
};
const testing = @import("../testing.zig");
+const repeat = @import("../../../string.zig").repeat;
test "cdp.Fetch: interception events belong to the enabling session" {
var ctx = try testing.context();
defer ctx.deinit();
@@ -660,30 +802,32 @@ test "cdp.Fetch: InterceptState patterns" {
defer state.deinit();
// disabled: nothing is paused
- try testing.expectEqual(false, state.matches("http://h/", .document));
+ try testing.expectEqual(false, state.matches(.Request, "http://h/", .document));
// omitted patterns = every request
try state.setPatterns(null);
- try testing.expect(state.matches("http://h/", .document));
- try testing.expect(state.matches("http://h/x.js", .script));
+ try testing.expect(state.matches(.Request, "http://h/", .document));
+ try testing.expect(state.matches(.Request, "http://h/x.js", .script));
// an explicit empty list = no request
try state.setPatterns(&.{});
- try testing.expectEqual(false, state.matches("http://h/", .document));
+ try testing.expectEqual(false, state.matches(.Request, "http://h/", .document));
try state.setPatterns(&.{
.{ .urlPattern = "*/api/*", .resourceType = .XHR },
.{ .urlPattern = "*.css", .requestStage = .Response },
});
- try testing.expectEqual(1, state.patterns.len);
- try testing.expect(state.matches("http://h/api/x", .xhr));
- try testing.expectEqual(false, state.matches("http://h/api/x", .fetch));
- try testing.expectEqual(false, state.matches("http://h/other", .xhr));
- // Response-stage entries are dropped
- try testing.expectEqual(false, state.matches("http://h/a.css", .stylesheet));
+ try testing.expectEqual(2, state.patterns.len);
+ try testing.expect(state.matches(.Request, "http://h/api/x", .xhr));
+ try testing.expectEqual(false, state.matches(.Request, "http://h/api/x", .fetch));
+ try testing.expectEqual(false, state.matches(.Request, "http://h/other", .xhr));
+ // A pattern pauses only at its own stage
+ try testing.expect(state.matches(.Response, "http://h/a.css", .stylesheet));
+ try testing.expectEqual(false, state.matches(.Request, "http://h/a.css", .stylesheet));
+ try testing.expectEqual(false, state.matches(.Response, "http://h/api/x", .xhr));
state.clearPatterns();
- try testing.expectEqual(false, state.matches("http://h/api/x", .xhr));
+ try testing.expectEqual(false, state.matches(.Request, "http://h/api/x", .xhr));
}
test "cdp.Fetch: enable compiles patterns, disable clears them" {
@@ -705,24 +849,337 @@ test "cdp.Fetch: enable compiles patterns, disable clears them" {
});
try ctx.expectSentResult(null, .{ .id = 1, .session_id = "SID-P" });
try testing.expectEqual(2, bc.intercept_state.patterns.len);
- try testing.expect(bc.intercept_state.matches("http://h/json/a", .fetch));
- try testing.expectEqual(false, bc.intercept_state.matches("http://h/json/a", .xhr));
- try testing.expect(bc.intercept_state.matches("http://h/exact", .xhr));
- try testing.expectEqual(false, bc.intercept_state.matches("http://h/", .document));
+ try testing.expect(bc.intercept_state.matches(.Request, "http://h/json/a", .fetch));
+ try testing.expectEqual(false, bc.intercept_state.matches(.Request, "http://h/json/a", .xhr));
+ try testing.expect(bc.intercept_state.matches(.Request, "http://h/exact", .xhr));
+ try testing.expectEqual(false, bc.intercept_state.matches(.Request, "http://h/", .document));
// re-enable without patterns = catch-all
try ctx.processMessage(.{ .id = 2, .method = "Fetch.enable", .sessionId = "SID-P" });
try ctx.expectSentResult(null, .{ .id = 2, .session_id = "SID-P" });
- try testing.expect(bc.intercept_state.matches("http://h/", .document));
+ try testing.expect(bc.intercept_state.matches(.Request, "http://h/", .document));
// explicit empty list = pause nothing; rejected when auth handling is on
try ctx.processMessage(.{ .id = 3, .method = "Fetch.enable", .params = .{ .patterns = .{} }, .sessionId = "SID-P" });
try ctx.expectSentResult(null, .{ .id = 3, .session_id = "SID-P" });
- try testing.expectEqual(false, bc.intercept_state.matches("http://h/", .document));
+ try testing.expectEqual(false, bc.intercept_state.matches(.Request, "http://h/", .document));
try ctx.processMessage(.{ .id = 4, .method = "Fetch.enable", .params = .{ .patterns = .{}, .handleAuthRequests = true }, .sessionId = "SID-P" });
try ctx.expectSentError(-32602, "Can't specify empty patterns with handleAuth set", .{ .id = 4 });
try ctx.processMessage(.{ .id = 5, .method = "Fetch.disable", .sessionId = "SID-P" });
try testing.expectEqual(0, bc.intercept_state.patterns.len);
- try testing.expectEqual(false, bc.intercept_state.matches("http://h/", .document));
+ try testing.expectEqual(false, bc.intercept_state.matches(.Request, "http://h/", .document));
+}
+
+// Response-stage tests drive real requests against the test server and read
+// the outcome through the request's own callbacks.
+const ResponseStageProbe = struct {
+ body: std.ArrayList(u8) = .empty,
+ done: bool = false,
+ err: ?anyerror = null,
+
+ fn deinit(self: *ResponseStageProbe) void {
+ self.body.deinit(testing.allocator);
+ }
+
+ fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
+ const self: *ResponseStageProbe = @ptrCast(@alignCast(transfer.req.ctx));
+ try self.body.appendSlice(testing.allocator, data);
+ }
+
+ fn doneCallback(raw: *anyopaque) !void {
+ const self: *ResponseStageProbe = @ptrCast(@alignCast(raw));
+ self.done = true;
+ }
+
+ fn errorCallback(raw: *anyopaque, err: anyerror) void {
+ const self: *ResponseStageProbe = @ptrCast(@alignCast(raw));
+ self.err = err;
+ }
+
+ fn start(self: *ResponseStageProbe, bc: *CDP.BrowserContext, url: [:0]const u8) !void {
+ const page = try bc.session.createPage();
+ try bc.cdp.browser.http_client.request(.{
+ .frame_id = page.frame_id,
+ .loader_id = 7,
+ .method = .GET,
+ .url = url,
+ .origin = bc.security_origin,
+ .credentials_mode = .omit,
+ .request_mode = .no_cors,
+ .resource_type = .script,
+ .notification = bc.session.notification,
+ .ctx = self,
+ .data_callback = dataCallback,
+ .done_callback = doneCallback,
+ .error_callback = errorCallback,
+ .shutdown_callback = HttpClient.noopShutdown,
+ }, null);
+ }
+
+ // Ticks the client until `count` transfers are parked for interception.
+ fn tickUntilPaused(bc: *CDP.BrowserContext, count: usize) !void {
+ const client = &bc.cdp.browser.http_client;
+ for (0..100) |_| {
+ if (client.intercepted >= count) return;
+ _ = try client.tick(20);
+ }
+ return error.NeverPaused;
+ }
+
+ // Ticks the client until the request's done or error callback has run.
+ fn tickUntilSettled(self: *const ResponseStageProbe, bc: *CDP.BrowserContext) !void {
+ const client = &bc.cdp.browser.http_client;
+ for (0..100) |_| {
+ if (self.done or self.err != null) return;
+ _ = try client.tick(20);
+ }
+ return error.NeverSettled;
+ }
+};
+
+// The params of the Fetch.requestPaused event carrying `request_id`.
+fn sentPausedParams(ctx: anytype, request_id: []const u8) !std.json.ObjectMap {
+ for (ctx.received.items) |message| {
+ const method = message.object.get("method") orelse continue;
+ if (!std.mem.eql(u8, method.string, "Fetch.requestPaused")) continue;
+ const params = message.object.get("params").?.object;
+ if (std.mem.eql(u8, params.get("requestId").?.string, request_id)) return params;
+ }
+ return error.PausedEventNotSent;
+}
+
+fn enableResponseStage(ctx: anytype) !void {
+ try ctx.processMessage(.{
+ .id = 1,
+ .method = "Fetch.enable",
+ .params = .{ .patterns = .{.{ .urlPattern = "*", .requestStage = "Response" }} },
+ });
+ try ctx.expectSentResult(null, .{ .id = 1 });
+}
+
+test "cdp.Fetch: response stage pauses a final response and continueResponse delivers it" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });
+ try enableResponseStage(&ctx);
+
+ var probe: ResponseStageProbe = .{};
+ defer probe.deinit();
+ try probe.start(bc, "http://127.0.0.1:9582/xhr");
+ try ResponseStageProbe.tickUntilPaused(bc, 1);
+
+ try ctx.expectSentEvent("Fetch.requestPaused", .{
+ .requestId = "INT-0000000001",
+ .request = .{ .url = "http://127.0.0.1:9582/xhr" },
+ .resourceType = "Script",
+ .responseStatusCode = 200,
+ }, .{ .session_id = "SID-RS" });
+ const paused = try sentPausedParams(&ctx, "INT-0000000001");
+ try testing.expectEqual(null, paused.get("responseErrorReason"));
+ var has_content_type = false;
+ for (paused.get("responseHeaders").?.array.items) |header| {
+ if (std.mem.eql(u8, header.object.get("name").?.string, "content-type")) {
+ has_content_type = std.mem.eql(u8, header.object.get("value").?.string, "text/html; charset=utf-8");
+ }
+ }
+ try testing.expect(has_content_type);
+
+ // Paused means not delivered: the consumer has seen nothing yet.
+ try testing.expectEqual(false, probe.done);
+ try testing.expectEqual(0, probe.body.items.len);
+
+ try ctx.processMessage(.{ .id = 2, .method = "Fetch.getResponseBody", .params = .{ .requestId = "INT-0000000001" } });
+ try ctx.expectSentResult(.{ .body = repeat("1234567890", 10), .base64Encoded = false }, .{ .id = 2 });
+
+ try ctx.processMessage(.{ .id = 3, .method = "Fetch.continueResponse", .params = .{ .requestId = "INT-0000000001" } });
+ try ctx.expectSentResult(null, .{ .id = 3 });
+
+ try probe.tickUntilSettled(bc);
+ try testing.expectEqual(null, probe.err);
+ try testing.expect(probe.done);
+ try testing.expectEqualSlices(u8, repeat("1234567890", 10), probe.body.items);
+ try testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);
+}
+
+test "cdp.Fetch: response stage failRequest fails the request without delivering it" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });
+ try enableResponseStage(&ctx);
+
+ var probe: ResponseStageProbe = .{};
+ defer probe.deinit();
+ try probe.start(bc, "http://127.0.0.1:9582/xhr");
+ try ResponseStageProbe.tickUntilPaused(bc, 1);
+
+ try ctx.processMessage(.{ .id = 2, .method = "Fetch.failRequest", .params = .{ .requestId = "INT-0000000001", .errorReason = "Aborted" } });
+ try ctx.expectSentResult(null, .{ .id = 2 });
+
+ try probe.tickUntilSettled(bc);
+ try testing.expectEqual(error.TransferCanceled, probe.err.?);
+ try testing.expectEqual(false, probe.done);
+ try testing.expectEqual(0, probe.body.items.len);
+ try testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);
+}
+
+test "cdp.Fetch: response stage fulfillRequest replaces the response" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });
+ try enableResponseStage(&ctx);
+
+ var probe: ResponseStageProbe = .{};
+ defer probe.deinit();
+ try probe.start(bc, "http://127.0.0.1:9582/xhr");
+ try ResponseStageProbe.tickUntilPaused(bc, 1);
+
+ try ctx.processMessage(.{
+ .id = 2,
+ .method = "Fetch.fulfillRequest",
+ .params = .{
+ .requestId = "INT-0000000001",
+ .responseCode = 200,
+ .responseHeaders = &[_]http.Header{.{ .name = "content-type", .value = "text/plain" }},
+ .body = "ZnVsZmlsbGVk", // "fulfilled"
+ },
+ });
+ try ctx.expectSentResult(null, .{ .id = 2 });
+
+ try probe.tickUntilSettled(bc);
+ try testing.expectEqual(null, probe.err);
+ try testing.expectEqualSlices(u8, "fulfilled", probe.body.items);
+ try testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);
+}
+
+test "cdp.Fetch: response stage getResponseBody base64-encodes a non-UTF-8 body" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });
+ try enableResponseStage(&ctx);
+
+ var probe: ResponseStageProbe = .{};
+ defer probe.deinit();
+ try probe.start(bc, "http://127.0.0.1:9582/images/small.png");
+ try ResponseStageProbe.tickUntilPaused(bc, 1);
+
+ // The bytes the test server sends for /images/small.png.
+ var png: [1024]u8 = undefined;
+ @memset(&png, 'x');
+ @memcpy(png[0..24], "\x89PNG\r\n\x1a\n\x00\x00\x00\x0dIHDR\x00\x00\x01\x40\x00\x00\x00\xf0");
+ var encoded: [std.base64.standard.Encoder.calcSize(png.len)]u8 = undefined;
+ _ = std.base64.standard.Encoder.encode(&encoded, &png);
+
+ try ctx.processMessage(.{ .id = 2, .method = "Fetch.getResponseBody", .params = .{ .requestId = "INT-0000000001" } });
+ try ctx.expectSentResult(.{ .body = &encoded, .base64Encoded = true }, .{ .id = 2 });
+
+ try ctx.processMessage(.{ .id = 3, .method = "Fetch.continueResponse", .params = .{ .requestId = "INT-0000000001" } });
+ try ctx.expectSentResult(null, .{ .id = 3 });
+ try probe.tickUntilSettled(bc);
+ try testing.expectEqualSlices(u8, &png, probe.body.items);
+}
+
+test "cdp.Fetch: response stage pauses a redirect response, then the response it leads to" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });
+ try enableResponseStage(&ctx);
+
+ var probe: ResponseStageProbe = .{};
+ defer probe.deinit();
+ try probe.start(bc, "http://127.0.0.1:9582/xhr/redirect");
+ try ResponseStageProbe.tickUntilPaused(bc, 1);
+
+ try ctx.expectSentEvent("Fetch.requestPaused", .{
+ .requestId = "INT-0000000001",
+ .request = .{ .url = "http://127.0.0.1:9582/xhr/redirect" },
+ .responseStatusCode = 302,
+ }, .{ .session_id = "SID-RS" });
+ var location: ?[]const u8 = null;
+ for ((try sentPausedParams(&ctx, "INT-0000000001")).get("responseHeaders").?.array.items) |header| {
+ if (std.mem.eql(u8, header.object.get("name").?.string, "location")) {
+ location = header.object.get("value").?.string;
+ }
+ }
+ try testing.expectEqualSlices(u8, "http://127.0.0.1:9582/xhr", location.?);
+
+ try ctx.processMessage(.{ .id = 2, .method = "Fetch.getResponseBody", .params = .{ .requestId = "INT-0000000001" } });
+ try ctx.expectSentError(-32000, "Response body is not available for redirects.", .{ .id = 2 });
+
+ // Releasing the redirect follows it; the redirected request's response pauses in turn.
+ try ctx.processMessage(.{ .id = 3, .method = "Fetch.continueResponse", .params = .{ .requestId = "INT-0000000001" } });
+ try ctx.expectSentResult(null, .{ .id = 3 });
+ try ResponseStageProbe.tickUntilPaused(bc, 1);
+ try ctx.expectSentEvent("Fetch.requestPaused", .{
+ .requestId = "INT-0000000002",
+ .request = .{ .url = "http://127.0.0.1:9582/xhr" },
+ .responseStatusCode = 200,
+ }, .{ .session_id = "SID-RS" });
+
+ try ctx.processMessage(.{ .id = 4, .method = "Fetch.continueResponse", .params = .{ .requestId = "INT-0000000002" } });
+ try ctx.expectSentResult(null, .{ .id = 4 });
+ try probe.tickUntilSettled(bc);
+ try testing.expectEqual(null, probe.err);
+ try testing.expectEqualSlices(u8, repeat("1234567890", 10), probe.body.items);
+ try testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);
+}
+
+test "cdp.Fetch: response stage pauses a network failure with responseErrorReason" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });
+ try enableResponseStage(&ctx);
+
+ var probe: ResponseStageProbe = .{};
+ defer probe.deinit();
+ // Nothing listens on port 1: the connection is refused.
+ try probe.start(bc, "http://127.0.0.1:1/");
+ try ResponseStageProbe.tickUntilPaused(bc, 1);
+
+ try ctx.expectSentEvent("Fetch.requestPaused", .{
+ .requestId = "INT-0000000001",
+ .request = .{ .url = "http://127.0.0.1:1/" },
+ .responseErrorReason = "ConnectionFailed",
+ }, .{ .session_id = "SID-RS" });
+ const paused = try sentPausedParams(&ctx, "INT-0000000001");
+ try testing.expectEqual(null, paused.get("responseStatusCode"));
+ try testing.expectEqual(null, paused.get("responseHeaders"));
+
+ try ctx.processMessage(.{ .id = 2, .method = "Fetch.getResponseBody", .params = .{ .requestId = "INT-0000000001" } });
+ try ctx.expectSentError(-32000, "Response body is not available for a failed request.", .{ .id = 2 });
+
+ // Releasing it unchanged reports the original failure.
+ try ctx.processMessage(.{ .id = 3, .method = "Fetch.continueResponse", .params = .{ .requestId = "INT-0000000001" } });
+ try ctx.expectSentResult(null, .{ .id = 3 });
+ try probe.tickUntilSettled(bc);
+ try testing.expectEqual(error.CouldntConnect, probe.err.?);
+ try testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);
+}
+
+test "cdp.Fetch: closing the browser context releases a response-stage pause" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });
+ try enableResponseStage(&ctx);
+
+ var probe: ResponseStageProbe = .{};
+ defer probe.deinit();
+ try probe.start(bc, "http://127.0.0.1:9582/xhr");
+ try ResponseStageProbe.tickUntilPaused(bc, 1);
+
+ // Never released by CDP: disposing the context must abort it, and the
+ // testing allocator fails the test if anything it held leaks.
+ ctx.cdp().disposeBrowserContext();
+ try testing.expectEqual(false, probe.done);
+}
+
+test "cdp.Fetch: cdpErrorReason" {
+ try testing.expectEqual(ErrorReason.TimedOut, cdpErrorReason(error.OperationTimedout));
+ try testing.expectEqual(ErrorReason.NameNotResolved, cdpErrorReason(error.CouldntResolveHost));
+ try testing.expectEqual(ErrorReason.NameNotResolved, cdpErrorReason(error.CouldntResolveProxy));
+ try testing.expectEqual(ErrorReason.ConnectionFailed, cdpErrorReason(error.CouldntConnect));
+ try testing.expectEqual(ErrorReason.ConnectionClosed, cdpErrorReason(error.GotNothing));
+ try testing.expectEqual(ErrorReason.ConnectionReset, cdpErrorReason(error.RecvError));
+ try testing.expectEqual(ErrorReason.ConnectionReset, cdpErrorReason(error.SendError));
+ try testing.expectEqual(ErrorReason.Failed, cdpErrorReason(error.SslConnectError));
}
diff --git a/src/server/cdp/domains/network.zig b/src/server/cdp/domains/network.zig
index 6912ae503..5eb22942e 100644
--- a/src/server/cdp/domains/network.zig
+++ b/src/server/cdp/domains/network.zig
@@ -445,7 +445,7 @@ fn getResponseBody(cmd: *CDP.Command) !void {
}
// Encodes straight into the outgoing message, no intermediate buffer.
-const Base64Body = struct {
+pub const Base64Body = struct {
data: []const u8,
pub fn jsonStringify(self: Base64Body, jws: *std.json.Stringify) std.Io.Writer.Error!void {