Lightpanda: the headless browser designed for AI and automation
cdp: pause requests at the Fetch response stage
Thawindu Angesh Tuto41ecc246d1c8
diff --git a/src/Notification.zig b/src/Notification.zigindex e4be273e4..26d48abf9 100644--- a/src/Notification.zig+++ b/src/Notification.zig@@ -89,6 +89,7 @@ const EventListeners = struct {http_request_intercept: List = .{},http_request_done: List = .{},http_request_auth_required: List = .{},+ http_response_intercept: List = .{},http_request_served_from_cache: List = .{},http_response_data: List = .{},http_response_header_done: List = .{},@@ -121,6 +122,7 @@ const Events = union(enum) {http_request_start: *const RequestStart,http_request_intercept: *const RequestIntercept,http_request_auth_required: *const RequestAuthRequired,+ http_response_intercept: *const ResponseIntercept,http_request_served_from_cache: *const RequestServedFromCache,http_request_done: *const RequestDone,http_response_data: *const ResponseData,@@ -247,6 +249,13 @@ pub const RequestAuthRequired = struct {wait_for_interception: *bool,};+// A final response is complete (status, headers and the whole body are on the+// transfer) and has not been delivered to its consumer yet.+pub const ResponseIntercept = struct {+ transfer: *Transfer,+ wait_for_interception: *bool,+};+pub const ResponseData = struct {data: []const u8,transfer: *Transfer,diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 776c67c58..241621794 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1690,6 +1690,8 @@ fn isTerminal(msg: *Inbox.Message) bool {fn isFetchInterceptionMethod(method: []const u8) bool {return std.mem.eql(u8, method, "Fetch.continueRequest") or+ std.mem.eql(u8, method, "Fetch.continueResponse") or+ std.mem.eql(u8, method, "Fetch.getResponseBody") orstd.mem.eql(u8, method, "Fetch.failRequest") orstd.mem.eql(u8, method, "Fetch.fulfillRequest") orstd.mem.eql(u8, method, "Fetch.continueWithAuth");@@ -1876,22 +1878,22 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Ttry transfer.materializeResponse(msg.conn, .{ .check_content_length = false });if (self.enforceCorsResponse(msg, transfer)) return true;- try transfer.handleRedirect(location.value);-- if (!transfer.req.internal) {- lp.metrics.http_redirects.incr();- }+ // location.value is curl-owned and dies with the conn.+ const owned_location = try transfer.arena.dupe(u8, location.value);// The multi should keep this alive. Releasing it now makes// it so we can re-enter the pipeline and keeps the code- // streamlined.+ // streamlined. Everything the redirect needs is+ // materialized on the transfer.self.removeConn(msg.conn);transfer._conn = null;- transfer.state = .created;- // We have to restart (.redirect == .start) the whole- // pipeline, rechecking the cache/robot/ratelimit/...- try self.pipeline(transfer, .redirect);+ if (self.interceptResponse(transfer, .{ .redirect = owned_location })) {+ return false;+ }++ transfer.state = .created;+ try self.followRedirect(transfer, owned_location);return true;},// error_callback surfaces this as a TypeError.@@ -1914,6 +1916,9 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tif (effective_err != null and !is_conn_close_recv) {const err = transfer.res.callback_error orelse effective_err.?;+ // A failure the consumer caused (abort, cancel) is not a network+ // failure to offer for interception.+ const offer = http.errorReason(err) != .aborted;if (try transfer.deliversHeaderBeforeError(msg.conn)) {try transfer.materializeResponse(msg.conn, .{ .check_content_length = false });if (self.enforceCorsResponse(msg, transfer)) {@@ -1921,11 +1926,17 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T}self.removeConn(msg.conn);transfer._conn = null;+ if (offer and self.interceptResponse(transfer, .{ .failed = .{ .err = err, .header_first = true } })) {+ return false;+ }try transfer.bufferHeaderThenError(err);return true;}self.removeConn(msg.conn);transfer._conn = null;+ if (offer and self.interceptResponse(transfer, .{ .failed = .{ .err = err, .header_first = false } })) {+ return false;+ }transfer.failAsync(err);return true;}@@ -1946,14 +1957,58 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tself.removeConn(msg.conn);transfer._conn = null;+ // A revalidation (304) is not offered: what it delivers is the cached+ // body, which the response on the transfer does not carry.+ if (transfer._cache_intent != .revalidate and self.interceptResponse(transfer, .final)) {+ return false;+ }++ try self.deliverResponse(transfer);+ return true;+}++// CDP Fetch response-stage interception: offer a completed transfer (its conn+// already released) to CDP. True when CDP took it: the transfer is parked+// until continueInterceptedResponse, fulfillIntercepted or abortParked.+fn interceptResponse(self: *Client, transfer: *Transfer, pause: Transfer.ResponsePause) bool {+ transfer._response_pause = pause;+ var wait_for_interception = false;+ transfer.notify(+ .http_response_intercept,+ &.{ .transfer = transfer, .wait_for_interception = &wait_for_interception },+ );+ if (!wait_for_interception) {+ return false;+ }+ self.intercepted += 1;+ transfer.state = .{ .parked = .intercept_response };+ return true;+}++// Follow a materialized redirect response: point the transfer at the+// redirect target and restart the whole pipeline (.redirect == .start),+// rechecking the cache/robot/ratelimit/...+fn followRedirect(self: *Client, transfer: *Transfer, location: []const u8) !void {+ try transfer.handleRedirect(location);++ if (!transfer.req.internal) {+ lp.metrics.http_redirects.incr();+ }++ try self.pipeline(transfer, .redirect);+}++// The tail of a completed, materialized response: settle the cache and buffer+// the events for dispatch. Runs straight from processOneMessage, or from+// continueInterceptedResponse once CDP releases a response-stage pause.+fn deliverResponse(self: *Client, transfer: *Transfer) !void {if (try self.cacheRevalidated(transfer)) {- return true;+ return;}self.cacheStore(transfer);transfer._content_length = transfer.getContentLength() orelse 0;try transfer.bufferEvents(transfer.res.buffer.items);- return true;}// Commit a configured conn to the multi handle. On error the conn is left@@ -2193,6 +2248,36 @@ pub fn continueTransfer(self: *Client, transfer: *Transfer) !void {};}+// CDP Fetch.continueResponse (or continueRequest on a response-stage pause):+// release what the transfer is holding, unchanged — deliver the response,+// follow the redirect, or report the network failure.+pub fn continueInterceptedResponse(self: *Client, transfer: *Transfer) !void {+ if (comptime lp.IS_DEBUG) {+ lp.assert(self.intercepted > 0, "Client.continueInterceptedResponse", .{ .value = self.intercepted });+ }++ transfer.unpark();+ switch (transfer._response_pause) {+ .final => self.deliverResponse(transfer) catch |err| {+ transfer.abortPipelineError(err);+ return err;+ },+ .redirect => |location| self.followRedirect(transfer, location) catch |err| {+ transfer.abortPipelineError(err);+ return err;+ },+ .failed => |failure| {+ if (!failure.header_first) {+ return transfer.failAsync(failure.err);+ }+ transfer.bufferHeaderThenError(failure.err) catch |err| {+ transfer.abortPipelineError(err);+ return err;+ };+ },+ }+}+// CDP Fetch.continueRequest: resume a transfer parked at the interception// gate, re-entering the pipeline at the step after interception. The CDP// command may have mutated req (url / method / headers / body) first.@@ -2493,6 +2578,11 @@ pub const Transfer = struct {// What the cache wants from this transfer's response._cache_intent: CacheIntent = .none,+ // What a response-stage CDP pause (.intercept_response) is holding, and so+ // what releasing it unchanged does: deliver the response, follow the+ // redirect, or fail with the network error.+ _response_pause: ResponsePause = .final,+// Cache key captured at lookup time (req.url before any redirect// rewrote it). Only meaningful while _cache_intent != .none._cache_key: [:0]const u8 = "",@@ -2611,6 +2701,12 @@ pub const Transfer = struct {// waiting for continueWithAuth.intercept_auth,+ // CDP Fetch interception, response phase: processOneMessage completed+ // the transfer (headers and body are materialized, the conn is+ // released) and holds it before delivery, waiting for+ // continueResponse / continueRequest / fulfillRequest / failRequest.+ intercept_response,+// RobotsGate holds the transfer pending a robots.txt fetch.robots,@@ -2618,6 +2714,24 @@ pub const Transfer = struct {cors,};+ pub const ResponsePause = union(enum) {+ // A final response, delivered to the consumer on continue.+ final,+ // A redirect response; the arena-owned Location is followed on continue.+ redirect: []const u8,+ // A network failure; on continue it is reported exactly as it would+ // have been without the pause.+ failed: Failure,++ pub const Failure = struct {+ err: anyerror,+ // The consumer asked for the header before a body error+ // (Request.header_before_body_error) and the header arrived:+ // deliver it, then the error.+ header_first: bool,+ };+ };+pub const HeaderResult = enum {/// Continue processing normally.proceed,@@ -2633,11 +2747,11 @@ pub const Transfer = struct {}// Gate-facing: take the transfer out of .parked and return it to- // the request flow (.created). This assumes pre-inflight handling (i.e. the- // transfer was in .created before being parked). This is true today, but- // could become false if Request Interception ever supports the "response"- // requestStage (although, to support this, I think the safety of transfers- // post-perform would need to be improved),+ // the request flow (.created). A transfer parked before it went inflight+ // re-enters the pipeline from here. One parked after completion+ // (.intercept_auth, .intercept_response) holds no conn: its response is+ // already materialized, so resuming it only buffers events, which+ // scheduleDispatch accepts from .created.pub fn unpark(self: *Transfer) void {lp.assert(self.state == .parked, "Transfer.unpark", .{ .state = self.state });self.leaveIntercept();@@ -2652,7 +2766,7 @@ pub const Transfer = struct {}switch (self.state.parked) {.robots, .cors => {},- .intercept_request, .intercept_auth => {+ .intercept_request, .intercept_auth, .intercept_response => {lp.assert(self.client.intercepted > 0, "Transfer.leaveIntercept", .{ .value = self.client.intercepted });self.client.intercepted -= 1;},@@ -2796,7 +2910,7 @@ pub const Transfer = struct {switch (self.state.parked) {.cors => self.client.cors.remove(self),.robots => self.client.robots.remove(self),- .intercept_auth, .intercept_request => {},+ .intercept_auth, .intercept_request, .intercept_response => {},}}@@ -3621,9 +3735,12 @@ pub const Transfer = struct {self.req.url = url;}+ // Reads only the materialized redirect response (res.header, res.headers),+ // never the conn, so it runs the same whether the conn was just released+ // or the transfer sat parked for a CDP response-stage interception.fn handleRedirect(transfer: *Transfer, location: []const u8) !void {const req = &transfer.req;- const conn = transfer._conn.?;+ const head = transfer.res.header orelse return error.MissingResponseHead;// Cookies were applied while materializing the redirect response.@@ -3631,23 +3748,20 @@ pub const Transfer = struct {// remaining hops.if (req.referrer_policy != null) {// referrer-policy is re-applied on every hop.- var i: usize = 0;- while (conn.getResponseHeader("referrer-policy", i)) |hdr| : (i += 1) {+ for (transfer.res.headers) |hdr| {+ if (!std.mem.eql(u8, hdr.name, "referrer-policy")) {+ continue;+ }if (referrer.parseHeader(hdr.value)) |policy| {req.referrer_policy = policy;}- if (i >= hdr.amount) {- break;- }}}- // base_url and location are owned by curl; applyRedirectTarget resolves a- // fresh arena-owned copy that gets stored in transfer.req.url.- const base_url = try conn.getEffectiveUrl();- const status = try conn.getResponseCode();+ // applyRedirectTarget resolves a fresh arena-owned copy of the target+ // that gets stored in transfer.req.url.transfer.redirectTiming(transfer.res.headers);- try transfer.applyRedirectTarget(std.mem.span(base_url), location, status);+ try transfer.applyRedirectTarget(std.mem.span(head.url), location, head.status);}// Called above (in handleRedirect) and by a CDP fulfill request which redirects@@ -4561,8 +4675,10 @@ const AdBlocker = @import("adblock/AdBlocker.zig");// exercises are ever set, by initTestClient or by the test itself.var test_network: Network = undefined;-test "HttpClient: isFetchInterceptionMethod matches the four Fetch methods" {+test "HttpClient: isFetchInterceptionMethod matches the Fetch interception methods" {try testing.expect(isFetchInterceptionMethod("Fetch.continueRequest"));+ try testing.expect(isFetchInterceptionMethod("Fetch.continueResponse"));+ try testing.expect(isFetchInterceptionMethod("Fetch.getResponseBody"));try testing.expect(isFetchInterceptionMethod("Fetch.failRequest"));try testing.expect(isFetchInterceptionMethod("Fetch.fulfillRequest"));try testing.expect(isFetchInterceptionMethod("Fetch.continueWithAuth"));diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zigindex 13faa5fb5..60a2d8010 100644--- a/src/server/cdp/CDP.zig+++ b/src/server/cdp/CDP.zig@@ -872,6 +872,7 @@ pub const BrowserContext = struct {try self.intercept_state.setPatterns(patterns);self.fetch_session_id = session_id;try self.notification.register(.http_request_intercept, self, onHttpRequestIntercept);+ try self.notification.register(.http_response_intercept, self, onHttpResponseIntercept);if (authRequests) {try self.notification.register(.http_request_auth_required, self, onHttpRequestAuthRequired);}@@ -918,6 +919,7 @@ pub const BrowserContext = struct {fn fetchDisable(self: *BrowserContext) void {self.notification.unregister(.http_request_intercept, self);+ self.notification.unregister(.http_response_intercept, self);self.notification.unregister(.http_request_auth_required, self);self.intercept_state.clearPatterns();self.fetch_session_id = null;@@ -1078,6 +1080,13 @@ pub const BrowserContext = struct {try @import("domains/fetch.zig").requestIntercept(arena, self, msg);}+ fn onHttpResponseIntercept(ctx: *anyopaque, msg: *const Notification.ResponseIntercept) !void {+ const self: *BrowserContext = @ptrCast(@alignCast(ctx));+ const arena = self.acquireNotificationArena();+ defer self.releaseNotificationArena();+ try @import("domains/fetch.zig").responseIntercept(arena, self, msg);+ }+fn onHttpRequestFail(ctx: *anyopaque, msg: *const Notification.RequestFail) !void {const self: *BrowserContext = @ptrCast(@alignCast(ctx));return network_domain.httpRequestFail(self, msg);diff --git a/src/server/cdp/domains/fetch.zig b/src/server/cdp/domains/fetch.zigindex 91964255a..279aabd1e 100644--- a/src/server/cdp/domains/fetch.zig+++ b/src/server/cdp/domains/fetch.zig@@ -36,15 +36,19 @@ pub fn processMessage(cmd: *CDP.Command) !void {disable,enable,continueRequest,+ continueResponse,failRequest,fulfillRequest,continueWithAuth,+ getResponseBody,}, cmd.input.action) orelse return error.UnknownMethod;switch (action) {.disable => return disable(cmd),.enable => return enable(cmd),.continueRequest => return continueRequest(cmd),+ .continueResponse => return continueResponse(cmd),+ .getResponseBody => return getResponseBody(cmd),.continueWithAuth => return continueWithAuth(cmd),.failRequest => return failRequest(cmd),.fulfillRequest => return fulfillRequest(cmd),@@ -120,7 +124,7 @@ pub fn requestIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *cconst session_id = bc.fetch_session_id orelse return;const transfer = intercept.transfer;- if (!bc.intercept_state.matches(transfer.req.url, transfer.req.resource_type)) {+ if (!bc.intercept_state.matches(.Request, transfer.req.url, transfer.req.resource_type)) {// Leaves wait_for_interception false: the request proceeds untouched.return;}@@ -149,6 +153,126 @@ pub fn requestIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *cintercept.wait_for_interception.* = true;}+pub fn responseIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *const Notification.ResponseIntercept) !void {+ const session_id = bc.fetch_session_id orelse return;++ const transfer = intercept.transfer;+ if (!bc.intercept_state.matches(.Response, transfer.req.url, transfer.req.resource_type)) {+ // Leaves wait_for_interception false: the response is delivered untouched.+ return;+ }++ const intercept_id = try bc.intercept_state.put(transfer.id);+ errdefer _ = bc.intercept_state.remove(intercept_id);++ // The stage is told by the fields present: a network failure carries+ // responseErrorReason, a response (final or redirect) its status and headers.+ switch (transfer._response_pause) {+ .failed => |failure| try bc.cdp.sendEvent("Fetch.requestPaused", .{+ .requestId = &id.toInterceptId(intercept_id),+ .frameId = &id.toFrameId(transfer.req.frame_id),+ .request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data),+ .resourceType = transfer.req.resource_type.string(),+ .responseErrorReason = cdpErrorReason(failure.err),+ .networkId = &id.toRequestId(transfer), // matches the Network REQ-ID+ }, .{ .session_id = session_id }),+ .final, .redirect => {+ const head = transfer.res.header orelse return error.MissingResponseHead;+ try bc.cdp.sendEvent("Fetch.requestPaused", .{+ .requestId = &id.toInterceptId(intercept_id),+ .frameId = &id.toFrameId(transfer.req.frame_id),+ .request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data),+ .resourceType = transfer.req.resource_type.string(),+ .responseStatusCode = head.status,+ .responseStatusText = transfer.statusText() orelse "",+ .responseHeaders = transfer.responseHeaders(),+ .networkId = &id.toRequestId(transfer), // matches the Network REQ-ID+ }, .{ .session_id = session_id });+ },+ }++ log.debug(.cdp, "response intercept", .{+ .state = "paused",+ .id = transfer.id,+ .url = transfer.req.url,+ .pause = std.meta.activeTag(transfer._response_pause),+ });+ // Await continueResponse, continueRequest, fulfillRequest or failRequest++ intercept.wait_for_interception.* = true;+}++// The transfer a response-stage intercept id holds, left in the intercept+// state. Null when the id is unknown, the transfer is gone, or the pause is+// not at the response stage.+fn pausedResponse(bc: *CDP.BrowserContext, request_id: []const u8) !?*HttpClient.Transfer {+ const intercept_id = try idFromRequestId(request_id);+ const transfer_id = bc.intercept_state.get(intercept_id) orelse return null;+ const transfer = bc.cdp.browser.http_client.findTransfer(transfer_id) orelse return null;+ if (transfer.state != .parked or transfer.state.parked != .intercept_response) {+ return null;+ }+ return transfer;+}++fn continueResponse(cmd: *CDP.Command) !void {+ const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;+ const params = (try cmd.params(struct {+ requestId: []const u8, // "INT-{d}"+ responseCode: ?u16 = null,+ responsePhrase: ?[]const u8 = null,+ responseHeaders: ?[]const http.Header = null,+ binaryResponseHeaders: ?[]const u8 = null,+ })) orelse return error.InvalidParams;++ if (params.responseCode != null or params.responsePhrase != null or params.responseHeaders != null or params.binaryResponseHeaders != null) {+ log.debug(.not_implemented, "Fetch.continueResponse", .{ .params = "response overrides" });+ return error.NotImplemented;+ }++ const transfer = (try pausedResponse(bc, params.requestId)) orelse {+ return cmd.sendError(-32000, "Invalid InterceptionId.", .{});+ };+ _ = bc.intercept_state.remove(try idFromRequestId(params.requestId));++ log.debug(.cdp, "response intercept", .{+ .state = "continue",+ .id = transfer.id,+ .url = transfer.req.url,+ });++ try bc.cdp.browser.http_client.continueInterceptedResponse(transfer);+ return cmd.sendResult(null, .{});+}++fn getResponseBody(cmd: *CDP.Command) !void {+ const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;+ const params = (try cmd.params(struct {+ requestId: []const u8, // "INT-{d}"+ })) orelse return error.InvalidParams;++ const transfer = (try pausedResponse(bc, params.requestId)) orelse {+ return cmd.sendError(-32000, "Can only get response body on requests captured after headers received.", .{});+ };+ switch (transfer._response_pause) {+ .final => {},+ .redirect => return cmd.sendError(-32000, "Response body is not available for redirects.", .{}),+ .failed => return cmd.sendError(-32000, "Response body is not available for a failed request.", .{}),+ }++ const body = transfer.res.buffer.items;+ if (std.unicode.utf8ValidateSlice(body)) {+ return cmd.sendResult(.{+ .body = body,+ .base64Encoded = false,+ }, .{ .size_hint = body.len });+ }+ return cmd.sendResult(.{+ .body = network.Base64Body{ .data = body },+ .base64Encoded = true,+ }, .{ .size_hint = std.base64.standard.Encoder.calcSize(body.len) });+}+fn continueRequest(cmd: *CDP.Command) !void {const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;const params = (try cmd.params(struct {@@ -164,6 +288,16 @@ fn continueRequest(cmd: *CDP.Command) !void {return error.NotImplemented;}+ if (try pausedResponse(bc, params.requestId)) |transfer| {+ // The request has already been sent; only an unchanged continue is meaningful.+ if (params.url != null or params.method != null or params.postData != null or params.headers != null) {+ return cmd.sendError(-32602, "Cannot override request at the response stage.", .{});+ }+ _ = bc.intercept_state.remove(try idFromRequestId(params.requestId));+ try bc.cdp.browser.http_client.continueInterceptedResponse(transfer);+ return cmd.sendResult(null, .{});+ }+const client = &bc.cdp.browser.http_client;var intercept_state = &bc.intercept_state;const intercept_id = try idFromRequestId(params.requestId);@@ -347,7 +481,7 @@ pub fn requestAuthRequired(arena: Allocator, bc: *CDP.BrowserContext, intercept:// Auth challenges are only surfaced for requests the patterns select.const transfer = intercept.transfer;- if (!bc.intercept_state.matches(transfer.req.url, transfer.req.resource_type)) {+ if (!bc.intercept_state.matches(.Request, transfer.req.url, transfer.req.resource_type)) {return;}@@ -385,6 +519,20 @@ pub fn requestAuthRequired(arena: Allocator, bc: *CDP.BrowserContext, intercept:intercept.wait_for_interception.* = true;}+// The CDP Network.ErrorReason for a network failure paused at the response+// stage. Failed is CDP's own generic reason, for failures it has no narrower+// value for (TLS, protocol and size errors among them).+fn cdpErrorReason(err: anyerror) ErrorReason {+ return switch (err) {+ error.OperationTimedout => .TimedOut,+ error.CouldntResolveHost, error.CouldntResolveProxy => .NameNotResolved,+ error.CouldntConnect => .ConnectionFailed,+ error.GotNothing => .ConnectionClosed,+ error.SendError, error.RecvError => .ConnectionReset,+ else => .Failed,+ };+}+// Get u32 from requestId which is formatted as: "INT-{d}"fn idFromRequestId(request_id: []const u8) !u32 {if (!std.mem.startsWith(u8, request_id, "INT-")) {@@ -408,6 +556,7 @@ pub const InterceptState = struct {const Pattern = struct {url: []const u8,resource_type: ?ResourceType,+ stage: RequestStage,};pub fn init(allocator: Allocator) !InterceptState {@@ -424,6 +573,11 @@ pub const InterceptState = struct {return intercept_id;}+ // The transfer id an intercept id holds, leaving the entry in place.+ pub fn get(self: *const InterceptState, intercept_id: u32) ?u32 {+ return self.waiting.get(intercept_id);+ }+// Returns the transfer id if the intercept id was present and removed.pub fn remove(self: *InterceptState, intercept_id: u32) ?u32 {const entry = self.waiting.fetchSwapRemove(intercept_id) orelse return null;@@ -442,23 +596,9 @@ pub const InterceptState = struct {pub fn setPatterns(self: *InterceptState, patterns: ?[]const RequestPattern) !void {self.clearPatterns();- var has_response_stage = false;const source: []const RequestPattern = patterns orelse &.{.{}};- var count: usize = 0;- for (source) |pattern| {- if (pattern.requestStage == .Response) {- has_response_stage = true;- continue;- }- count += 1;- }-- if (has_response_stage) {- log.debug(.not_implemented, "Fetch.enable", .{ .params = "requestStage=Response" });- }-- const owned = try self.allocator.alloc(Pattern, count);+ const owned = try self.allocator.alloc(Pattern, source.len);var initialized: usize = 0;errdefer {for (owned[0..initialized]) |pattern| {@@ -467,12 +607,10 @@ pub const InterceptState = struct {self.allocator.free(owned);}for (source) |pattern| {- if (pattern.requestStage != .Request) {- continue;- }owned[initialized] = .{.url = try self.allocator.dupe(u8, pattern.urlPattern),.resource_type = pattern.resourceType,+ .stage = pattern.requestStage,};initialized += 1;}@@ -487,8 +625,11 @@ pub const InterceptState = struct {self.patterns = &.{};}- pub fn matches(self: *const InterceptState, url: []const u8, resource_type: HttpClient.Request.ResourceType) bool {+ pub fn matches(self: *const InterceptState, stage: RequestStage, url: []const u8, resource_type: HttpClient.Request.ResourceType) bool {for (self.patterns) |pattern| {+ if (pattern.stage != stage) {+ continue;+ }if (pattern.resource_type) |rt| {if (rt != cdpResourceType(resource_type)) {continue;@@ -569,6 +710,7 @@ pub const InterceptState = struct {};const testing = @import("../testing.zig");+const repeat = @import("../../../string.zig").repeat;test "cdp.Fetch: interception events belong to the enabling session" {var ctx = try testing.context();defer ctx.deinit();@@ -660,30 +802,32 @@ test "cdp.Fetch: InterceptState patterns" {defer state.deinit();// disabled: nothing is paused- try testing.expectEqual(false, state.matches("http://h/", .document));+ try testing.expectEqual(false, state.matches(.Request, "http://h/", .document));// omitted patterns = every requesttry state.setPatterns(null);- try testing.expect(state.matches("http://h/", .document));- try testing.expect(state.matches("http://h/x.js", .script));+ try testing.expect(state.matches(.Request, "http://h/", .document));+ try testing.expect(state.matches(.Request, "http://h/x.js", .script));// an explicit empty list = no requesttry state.setPatterns(&.{});- try testing.expectEqual(false, state.matches("http://h/", .document));+ try testing.expectEqual(false, state.matches(.Request, "http://h/", .document));try state.setPatterns(&.{.{ .urlPattern = "*/api/*", .resourceType = .XHR },.{ .urlPattern = "*.css", .requestStage = .Response },});- try testing.expectEqual(1, state.patterns.len);- try testing.expect(state.matches("http://h/api/x", .xhr));- try testing.expectEqual(false, state.matches("http://h/api/x", .fetch));- try testing.expectEqual(false, state.matches("http://h/other", .xhr));- // Response-stage entries are dropped- try testing.expectEqual(false, state.matches("http://h/a.css", .stylesheet));+ try testing.expectEqual(2, state.patterns.len);+ try testing.expect(state.matches(.Request, "http://h/api/x", .xhr));+ try testing.expectEqual(false, state.matches(.Request, "http://h/api/x", .fetch));+ try testing.expectEqual(false, state.matches(.Request, "http://h/other", .xhr));+ // A pattern pauses only at its own stage+ try testing.expect(state.matches(.Response, "http://h/a.css", .stylesheet));+ try testing.expectEqual(false, state.matches(.Request, "http://h/a.css", .stylesheet));+ try testing.expectEqual(false, state.matches(.Response, "http://h/api/x", .xhr));state.clearPatterns();- try testing.expectEqual(false, state.matches("http://h/api/x", .xhr));+ try testing.expectEqual(false, state.matches(.Request, "http://h/api/x", .xhr));}test "cdp.Fetch: enable compiles patterns, disable clears them" {@@ -705,24 +849,337 @@ test "cdp.Fetch: enable compiles patterns, disable clears them" {});try ctx.expectSentResult(null, .{ .id = 1, .session_id = "SID-P" });try testing.expectEqual(2, bc.intercept_state.patterns.len);- try testing.expect(bc.intercept_state.matches("http://h/json/a", .fetch));- try testing.expectEqual(false, bc.intercept_state.matches("http://h/json/a", .xhr));- try testing.expect(bc.intercept_state.matches("http://h/exact", .xhr));- try testing.expectEqual(false, bc.intercept_state.matches("http://h/", .document));+ try testing.expect(bc.intercept_state.matches(.Request, "http://h/json/a", .fetch));+ try testing.expectEqual(false, bc.intercept_state.matches(.Request, "http://h/json/a", .xhr));+ try testing.expect(bc.intercept_state.matches(.Request, "http://h/exact", .xhr));+ try testing.expectEqual(false, bc.intercept_state.matches(.Request, "http://h/", .document));// re-enable without patterns = catch-alltry ctx.processMessage(.{ .id = 2, .method = "Fetch.enable", .sessionId = "SID-P" });try ctx.expectSentResult(null, .{ .id = 2, .session_id = "SID-P" });- try testing.expect(bc.intercept_state.matches("http://h/", .document));+ try testing.expect(bc.intercept_state.matches(.Request, "http://h/", .document));// explicit empty list = pause nothing; rejected when auth handling is ontry ctx.processMessage(.{ .id = 3, .method = "Fetch.enable", .params = .{ .patterns = .{} }, .sessionId = "SID-P" });try ctx.expectSentResult(null, .{ .id = 3, .session_id = "SID-P" });- try testing.expectEqual(false, bc.intercept_state.matches("http://h/", .document));+ try testing.expectEqual(false, bc.intercept_state.matches(.Request, "http://h/", .document));try ctx.processMessage(.{ .id = 4, .method = "Fetch.enable", .params = .{ .patterns = .{}, .handleAuthRequests = true }, .sessionId = "SID-P" });try ctx.expectSentError(-32602, "Can't specify empty patterns with handleAuth set", .{ .id = 4 });try ctx.processMessage(.{ .id = 5, .method = "Fetch.disable", .sessionId = "SID-P" });try testing.expectEqual(0, bc.intercept_state.patterns.len);- try testing.expectEqual(false, bc.intercept_state.matches("http://h/", .document));+ try testing.expectEqual(false, bc.intercept_state.matches(.Request, "http://h/", .document));+}++// Response-stage tests drive real requests against the test server and read+// the outcome through the request's own callbacks.+const ResponseStageProbe = struct {+ body: std.ArrayList(u8) = .empty,+ done: bool = false,+ err: ?anyerror = null,++ fn deinit(self: *ResponseStageProbe) void {+ self.body.deinit(testing.allocator);+ }++ fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {+ const self: *ResponseStageProbe = @ptrCast(@alignCast(transfer.req.ctx));+ try self.body.appendSlice(testing.allocator, data);+ }++ fn doneCallback(raw: *anyopaque) !void {+ const self: *ResponseStageProbe = @ptrCast(@alignCast(raw));+ self.done = true;+ }++ fn errorCallback(raw: *anyopaque, err: anyerror) void {+ const self: *ResponseStageProbe = @ptrCast(@alignCast(raw));+ self.err = err;+ }++ fn start(self: *ResponseStageProbe, bc: *CDP.BrowserContext, url: [:0]const u8) !void {+ const page = try bc.session.createPage();+ try bc.cdp.browser.http_client.request(.{+ .frame_id = page.frame_id,+ .loader_id = 7,+ .method = .GET,+ .url = url,+ .origin = bc.security_origin,+ .credentials_mode = .omit,+ .request_mode = .no_cors,+ .resource_type = .script,+ .notification = bc.session.notification,+ .ctx = self,+ .data_callback = dataCallback,+ .done_callback = doneCallback,+ .error_callback = errorCallback,+ .shutdown_callback = HttpClient.noopShutdown,+ }, null);+ }++ // Ticks the client until `count` transfers are parked for interception.+ fn tickUntilPaused(bc: *CDP.BrowserContext, count: usize) !void {+ const client = &bc.cdp.browser.http_client;+ for (0..100) |_| {+ if (client.intercepted >= count) return;+ _ = try client.tick(20);+ }+ return error.NeverPaused;+ }++ // Ticks the client until the request's done or error callback has run.+ fn tickUntilSettled(self: *const ResponseStageProbe, bc: *CDP.BrowserContext) !void {+ const client = &bc.cdp.browser.http_client;+ for (0..100) |_| {+ if (self.done or self.err != null) return;+ _ = try client.tick(20);+ }+ return error.NeverSettled;+ }+};++// The params of the Fetch.requestPaused event carrying `request_id`.+fn sentPausedParams(ctx: anytype, request_id: []const u8) !std.json.ObjectMap {+ for (ctx.received.items) |message| {+ const method = message.object.get("method") orelse continue;+ if (!std.mem.eql(u8, method.string, "Fetch.requestPaused")) continue;+ const params = message.object.get("params").?.object;+ if (std.mem.eql(u8, params.get("requestId").?.string, request_id)) return params;+ }+ return error.PausedEventNotSent;+}++fn enableResponseStage(ctx: anytype) !void {+ try ctx.processMessage(.{+ .id = 1,+ .method = "Fetch.enable",+ .params = .{ .patterns = .{.{ .urlPattern = "*", .requestStage = "Response" }} },+ });+ try ctx.expectSentResult(null, .{ .id = 1 });+}++test "cdp.Fetch: response stage pauses a final response and continueResponse delivers it" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });+ try enableResponseStage(&ctx);++ var probe: ResponseStageProbe = .{};+ defer probe.deinit();+ try probe.start(bc, "http://127.0.0.1:9582/xhr");+ try ResponseStageProbe.tickUntilPaused(bc, 1);++ try ctx.expectSentEvent("Fetch.requestPaused", .{+ .requestId = "INT-0000000001",+ .request = .{ .url = "http://127.0.0.1:9582/xhr" },+ .resourceType = "Script",+ .responseStatusCode = 200,+ }, .{ .session_id = "SID-RS" });+ const paused = try sentPausedParams(&ctx, "INT-0000000001");+ try testing.expectEqual(null, paused.get("responseErrorReason"));+ var has_content_type = false;+ for (paused.get("responseHeaders").?.array.items) |header| {+ if (std.mem.eql(u8, header.object.get("name").?.string, "content-type")) {+ has_content_type = std.mem.eql(u8, header.object.get("value").?.string, "text/html; charset=utf-8");+ }+ }+ try testing.expect(has_content_type);++ // Paused means not delivered: the consumer has seen nothing yet.+ try testing.expectEqual(false, probe.done);+ try testing.expectEqual(0, probe.body.items.len);++ try ctx.processMessage(.{ .id = 2, .method = "Fetch.getResponseBody", .params = .{ .requestId = "INT-0000000001" } });+ try ctx.expectSentResult(.{ .body = repeat("1234567890", 10), .base64Encoded = false }, .{ .id = 2 });++ try ctx.processMessage(.{ .id = 3, .method = "Fetch.continueResponse", .params = .{ .requestId = "INT-0000000001" } });+ try ctx.expectSentResult(null, .{ .id = 3 });++ try probe.tickUntilSettled(bc);+ try testing.expectEqual(null, probe.err);+ try testing.expect(probe.done);+ try testing.expectEqualSlices(u8, repeat("1234567890", 10), probe.body.items);+ try testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);+}++test "cdp.Fetch: response stage failRequest fails the request without delivering it" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });+ try enableResponseStage(&ctx);++ var probe: ResponseStageProbe = .{};+ defer probe.deinit();+ try probe.start(bc, "http://127.0.0.1:9582/xhr");+ try ResponseStageProbe.tickUntilPaused(bc, 1);++ try ctx.processMessage(.{ .id = 2, .method = "Fetch.failRequest", .params = .{ .requestId = "INT-0000000001", .errorReason = "Aborted" } });+ try ctx.expectSentResult(null, .{ .id = 2 });++ try probe.tickUntilSettled(bc);+ try testing.expectEqual(error.TransferCanceled, probe.err.?);+ try testing.expectEqual(false, probe.done);+ try testing.expectEqual(0, probe.body.items.len);+ try testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);+}++test "cdp.Fetch: response stage fulfillRequest replaces the response" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });+ try enableResponseStage(&ctx);++ var probe: ResponseStageProbe = .{};+ defer probe.deinit();+ try probe.start(bc, "http://127.0.0.1:9582/xhr");+ try ResponseStageProbe.tickUntilPaused(bc, 1);++ try ctx.processMessage(.{+ .id = 2,+ .method = "Fetch.fulfillRequest",+ .params = .{+ .requestId = "INT-0000000001",+ .responseCode = 200,+ .responseHeaders = &[_]http.Header{.{ .name = "content-type", .value = "text/plain" }},+ .body = "ZnVsZmlsbGVk", // "fulfilled"+ },+ });+ try ctx.expectSentResult(null, .{ .id = 2 });++ try probe.tickUntilSettled(bc);+ try testing.expectEqual(null, probe.err);+ try testing.expectEqualSlices(u8, "fulfilled", probe.body.items);+ try testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);+}++test "cdp.Fetch: response stage getResponseBody base64-encodes a non-UTF-8 body" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });+ try enableResponseStage(&ctx);++ var probe: ResponseStageProbe = .{};+ defer probe.deinit();+ try probe.start(bc, "http://127.0.0.1:9582/images/small.png");+ try ResponseStageProbe.tickUntilPaused(bc, 1);++ // The bytes the test server sends for /images/small.png.+ var png: [1024]u8 = undefined;+ @memset(&png, 'x');+ @memcpy(png[0..24], "\x89PNG\r\n\x1a\n\x00\x00\x00\x0dIHDR\x00\x00\x01\x40\x00\x00\x00\xf0");+ var encoded: [std.base64.standard.Encoder.calcSize(png.len)]u8 = undefined;+ _ = std.base64.standard.Encoder.encode(&encoded, &png);++ try ctx.processMessage(.{ .id = 2, .method = "Fetch.getResponseBody", .params = .{ .requestId = "INT-0000000001" } });+ try ctx.expectSentResult(.{ .body = &encoded, .base64Encoded = true }, .{ .id = 2 });++ try ctx.processMessage(.{ .id = 3, .method = "Fetch.continueResponse", .params = .{ .requestId = "INT-0000000001" } });+ try ctx.expectSentResult(null, .{ .id = 3 });+ try probe.tickUntilSettled(bc);+ try testing.expectEqualSlices(u8, &png, probe.body.items);+}++test "cdp.Fetch: response stage pauses a redirect response, then the response it leads to" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });+ try enableResponseStage(&ctx);++ var probe: ResponseStageProbe = .{};+ defer probe.deinit();+ try probe.start(bc, "http://127.0.0.1:9582/xhr/redirect");+ try ResponseStageProbe.tickUntilPaused(bc, 1);++ try ctx.expectSentEvent("Fetch.requestPaused", .{+ .requestId = "INT-0000000001",+ .request = .{ .url = "http://127.0.0.1:9582/xhr/redirect" },+ .responseStatusCode = 302,+ }, .{ .session_id = "SID-RS" });+ var location: ?[]const u8 = null;+ for ((try sentPausedParams(&ctx, "INT-0000000001")).get("responseHeaders").?.array.items) |header| {+ if (std.mem.eql(u8, header.object.get("name").?.string, "location")) {+ location = header.object.get("value").?.string;+ }+ }+ try testing.expectEqualSlices(u8, "http://127.0.0.1:9582/xhr", location.?);++ try ctx.processMessage(.{ .id = 2, .method = "Fetch.getResponseBody", .params = .{ .requestId = "INT-0000000001" } });+ try ctx.expectSentError(-32000, "Response body is not available for redirects.", .{ .id = 2 });++ // Releasing the redirect follows it; the redirected request's response pauses in turn.+ try ctx.processMessage(.{ .id = 3, .method = "Fetch.continueResponse", .params = .{ .requestId = "INT-0000000001" } });+ try ctx.expectSentResult(null, .{ .id = 3 });+ try ResponseStageProbe.tickUntilPaused(bc, 1);+ try ctx.expectSentEvent("Fetch.requestPaused", .{+ .requestId = "INT-0000000002",+ .request = .{ .url = "http://127.0.0.1:9582/xhr" },+ .responseStatusCode = 200,+ }, .{ .session_id = "SID-RS" });++ try ctx.processMessage(.{ .id = 4, .method = "Fetch.continueResponse", .params = .{ .requestId = "INT-0000000002" } });+ try ctx.expectSentResult(null, .{ .id = 4 });+ try probe.tickUntilSettled(bc);+ try testing.expectEqual(null, probe.err);+ try testing.expectEqualSlices(u8, repeat("1234567890", 10), probe.body.items);+ try testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);+}++test "cdp.Fetch: response stage pauses a network failure with responseErrorReason" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });+ try enableResponseStage(&ctx);++ var probe: ResponseStageProbe = .{};+ defer probe.deinit();+ // Nothing listens on port 1: the connection is refused.+ try probe.start(bc, "http://127.0.0.1:1/");+ try ResponseStageProbe.tickUntilPaused(bc, 1);++ try ctx.expectSentEvent("Fetch.requestPaused", .{+ .requestId = "INT-0000000001",+ .request = .{ .url = "http://127.0.0.1:1/" },+ .responseErrorReason = "ConnectionFailed",+ }, .{ .session_id = "SID-RS" });+ const paused = try sentPausedParams(&ctx, "INT-0000000001");+ try testing.expectEqual(null, paused.get("responseStatusCode"));+ try testing.expectEqual(null, paused.get("responseHeaders"));++ try ctx.processMessage(.{ .id = 2, .method = "Fetch.getResponseBody", .params = .{ .requestId = "INT-0000000001" } });+ try ctx.expectSentError(-32000, "Response body is not available for a failed request.", .{ .id = 2 });++ // Releasing it unchanged reports the original failure.+ try ctx.processMessage(.{ .id = 3, .method = "Fetch.continueResponse", .params = .{ .requestId = "INT-0000000001" } });+ try ctx.expectSentResult(null, .{ .id = 3 });+ try probe.tickUntilSettled(bc);+ try testing.expectEqual(error.CouldntConnect, probe.err.?);+ try testing.expectEqual(0, bc.cdp.browser.http_client.intercepted);+}++test "cdp.Fetch: closing the browser context releases a response-stage pause" {+ var ctx = try testing.context();+ defer ctx.deinit();+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RS", .session_id = "SID-RS" });+ try enableResponseStage(&ctx);++ var probe: ResponseStageProbe = .{};+ defer probe.deinit();+ try probe.start(bc, "http://127.0.0.1:9582/xhr");+ try ResponseStageProbe.tickUntilPaused(bc, 1);++ // Never released by CDP: disposing the context must abort it, and the+ // testing allocator fails the test if anything it held leaks.+ ctx.cdp().disposeBrowserContext();+ try testing.expectEqual(false, probe.done);+}++test "cdp.Fetch: cdpErrorReason" {+ try testing.expectEqual(ErrorReason.TimedOut, cdpErrorReason(error.OperationTimedout));+ try testing.expectEqual(ErrorReason.NameNotResolved, cdpErrorReason(error.CouldntResolveHost));+ try testing.expectEqual(ErrorReason.NameNotResolved, cdpErrorReason(error.CouldntResolveProxy));+ try testing.expectEqual(ErrorReason.ConnectionFailed, cdpErrorReason(error.CouldntConnect));+ try testing.expectEqual(ErrorReason.ConnectionClosed, cdpErrorReason(error.GotNothing));+ try testing.expectEqual(ErrorReason.ConnectionReset, cdpErrorReason(error.RecvError));+ try testing.expectEqual(ErrorReason.ConnectionReset, cdpErrorReason(error.SendError));+ try testing.expectEqual(ErrorReason.Failed, cdpErrorReason(error.SslConnectError));}diff --git a/src/server/cdp/domains/network.zig b/src/server/cdp/domains/network.zigindex 6912ae503..5eb22942e 100644--- a/src/server/cdp/domains/network.zig+++ b/src/server/cdp/domains/network.zig@@ -445,7 +445,7 @@ fn getResponseBody(cmd: *CDP.Command) !void {}// Encodes straight into the outgoing message, no intermediate buffer.-const Base64Body = struct {+pub const Base64Body = struct {data: []const u8,pub fn jsonStringify(self: Base64Body, jws: *std.json.Stringify) std.Io.Writer.Error!void {