Lightpanda: the headless browser designed for AI and automation
cdp: send request information (RI) notifications on redirect
Karl Seguin7c89a462eb11
diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zigindex 5d09752b5..c94ccb277 100644--- a/src/cdp/domains/fetch.zig+++ b/src/cdp/domains/fetch.zig@@ -50,15 +50,15 @@ pub fn processMessage(cmd: *CDP.Command) !void {}}-// Stored in CDP. Holds *transfer ids* (not *Transfer pointers) of paused-// transfers waiting for CDP continueRequest/fulfillRequest/failRequest/-// continueWithAuth. Anyone resolving an entry must look the transfer up via-// `Client.findTransfer(id)` — if the transfer has been destroyed out-of-band-// (e.g. frame shutdown), the lookup returns null and the CDP command should-// no-op rather than UAF.+// Stored in CDP. Maps intercept ids to *transfer ids* (not *Transfer+// pointers) of paused transfers waiting for CDP continueRequest/+// fulfillRequest/failRequest/continueWithAuth. Anyone resolving an entry must+// look the transfer up via `Client.findTransfer(id)` — if the transfer has been+// destroyed out-of-band (e.g. frame shutdown), the lookup returns null.pub const InterceptState = struct {allocator: Allocator,- waiting: std.AutoArrayHashMapUnmanaged(u32, void),+ next_id: u32 = 1,+ waiting: std.AutoArrayHashMapUnmanaged(u32, u32),pub fn init(allocator: Allocator) !InterceptState {return .{@@ -67,17 +67,17 @@ pub const InterceptState = struct {};}- pub fn empty(self: *const InterceptState) bool {- return self.waiting.count() == 0;+ pub fn put(self: *InterceptState, transfer_id: u32) !u32 {+ const intercept_id = self.next_id;+ try self.waiting.put(self.allocator, intercept_id, transfer_id);+ self.next_id +%= 1;+ return intercept_id;}- pub fn put(self: *InterceptState, transfer_id: u32) !void {- return self.waiting.put(self.allocator, transfer_id, {});- }-- // Returns true if the id was present and removed, false otherwise.- pub fn remove(self: *InterceptState, transfer_id: u32) bool {- return self.waiting.swapRemove(transfer_id);+ // Returns the transfer id if the intercept id was present and removed.+ pub fn remove(self: *InterceptState, intercept_id: u32) ?u32 {+ const entry = self.waiting.fetchSwapRemove(intercept_id) orelse return null;+ return entry.value;}pub fn deinit(self: *InterceptState) void {@@ -85,7 +85,7 @@ pub const InterceptState = struct {}pub fn pendingIntercepts(self: *const InterceptState) []u32 {- return self.waiting.keys();+ return self.waiting.values();}};@@ -203,11 +203,11 @@ pub fn requestIntercept(bc: *CDP.BrowserContext, intercept: *const Notification.// TODO: What to do when receiving replies for a previous frame's requests?const transfer = intercept.transfer;- try bc.intercept_state.put(transfer.id);- errdefer _ = bc.intercept_state.remove(transfer.id);+ const intercept_id = try bc.intercept_state.put(transfer.id);+ errdefer _ = bc.intercept_state.remove(intercept_id);try bc.cdp.sendEvent("Fetch.requestPaused", .{- .requestId = &id.toInterceptId(transfer.id),+ .requestId = &id.toInterceptId(intercept_id),.frameId = &id.toFrameId(transfer.req.frame_id),.request = network.RequestWriter.init(transfer),.resourceType = transfer.req.resource_type.string(),@@ -241,9 +241,9 @@ fn continueRequest(cmd: *CDP.Command) !void {const client = &bc.cdp.browser.http_client;var intercept_state = &bc.intercept_state;- const transfer_id = try idFromRequestId(params.requestId);+ const intercept_id = try idFromRequestId(params.requestId);- if (!intercept_state.remove(transfer_id)) return error.RequestNotFound;+ const transfer_id = intercept_state.remove(intercept_id) orelse return error.RequestNotFound;// Transfer may have been destroyed out-of-band between pause and now// (e.g. frame shutdown). Treat as a no-op rather than an error — the CDP// client's view of "this request still exists" is just stale.@@ -304,9 +304,9 @@ fn continueWithAuth(cmd: *CDP.Command) !void {const client = &bc.cdp.browser.http_client;var intercept_state = &bc.intercept_state;- const transfer_id = try idFromRequestId(params.requestId);+ const intercept_id = try idFromRequestId(params.requestId);- if (!intercept_state.remove(transfer_id)) return error.RequestNotFound;+ const transfer_id = intercept_state.remove(intercept_id) orelse return error.RequestNotFound;const transfer = client.findTransfer(transfer_id) orelse {log.debug(.cdp, "intercept lookup miss", .{ .id = transfer_id, .op = "auth" });return cmd.sendResult(null, .{});@@ -362,9 +362,9 @@ fn fulfillRequest(cmd: *CDP.Command) !void {const client = &bc.cdp.browser.http_client;var intercept_state = &bc.intercept_state;- const transfer_id = try idFromRequestId(params.requestId);+ const intercept_id = try idFromRequestId(params.requestId);- if (!intercept_state.remove(transfer_id)) return error.RequestNotFound;+ const transfer_id = intercept_state.remove(intercept_id) orelse return error.RequestNotFound;const transfer = client.findTransfer(transfer_id) orelse {log.debug(.cdp, "intercept lookup miss", .{ .id = transfer_id, .op = "fulfill" });return cmd.sendResult(null, .{});@@ -399,9 +399,9 @@ fn failRequest(cmd: *CDP.Command) !void {const client = &bc.cdp.browser.http_client;var intercept_state = &bc.intercept_state;- const transfer_id = try idFromRequestId(params.requestId);+ const intercept_id = try idFromRequestId(params.requestId);- if (!intercept_state.remove(transfer_id)) return error.RequestNotFound;+ const transfer_id = intercept_state.remove(intercept_id) orelse return error.RequestNotFound;const transfer = client.findTransfer(transfer_id) orelse {log.debug(.cdp, "intercept lookup miss", .{ .id = transfer_id, .op = "fail" });return cmd.sendResult(null, .{});@@ -426,14 +426,14 @@ pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notificati// TODO: What to do when receiving replies for a previous frame's requests?const transfer = intercept.transfer;- try bc.intercept_state.put(transfer.id);- errdefer _ = bc.intercept_state.remove(transfer.id);+ const intercept_id = try bc.intercept_state.put(transfer.id);+ errdefer _ = bc.intercept_state.remove(intercept_id);const request = &transfer.req;const challenge = transfer._auth_challenge orelse return error.NullAuthChallenge;try bc.cdp.sendEvent("Fetch.authRequired", .{- .requestId = &id.toInterceptId(transfer.id),+ .requestId = &id.toInterceptId(intercept_id),.frameId = &id.toFrameId(request.frame_id),.request = network.RequestWriter.init(transfer),.resourceType = request.resource_type.string(),@@ -465,7 +465,6 @@ fn idFromRequestId(request_id: []const u8) !u32 {}const testing = @import("../testing.zig");-test "cdp.Fetch: interception events belong to the enabling session" {var ctx = try testing.context();defer ctx.deinit();@@ -500,3 +499,19 @@ test "cdp.Fetch: interception events belong to the enabling session" {});try testing.expectEqual(null, bc.fetch_session_id);}++test "cdp.Fetch: InterceptState issues a fresh id per pause" {+ var state = try InterceptState.init(testing.allocator);+ defer state.deinit();++ const first = try state.put(7);+ const second = try state.put(7);+ try testing.expectEqual(false, first == second);+ try testing.expectEqual(2, state.pendingIntercepts().len);+ try testing.expectEqual(7, state.pendingIntercepts()[0]);++ try testing.expectEqual(7, state.remove(first).?);+ try testing.expectEqual(null, state.remove(first));+ try testing.expectEqual(7, state.remove(second).?);+ try testing.expectEqual(null, state.remove(second));+}diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex f88f9f6bc..bdf981bc8 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1465,6 +1465,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tif (msg.conn.getResponseHeader("location", 0)) |location| switch (transfer.req.redirect) {.follow => {try transfer.handleRedirect(location.value);+ transfer.restoreInterceptHeaders();if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {log.warn(.http, "blocked url", .{ .url = transfer.req.url });@@ -1476,6 +1477,28 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tif (!transfer.req.internal) lp.metrics.http_redirects.incr();+ if (self.serve_mode) { // e.g. cdp+ var wait_for_interception = false;+ transfer.req.notification.dispatch(.http_request_intercept, &.{+ .transfer = transfer,+ .wait_for_interception = &wait_for_interception,+ });++ if (wait_for_interception) {+ transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });++ // Same shape as the auth-interception park above:+ // give up the connection, wait for the CDP client.+ self.removeConn(msg.conn);+ transfer._conn = null;+ transfer.reset();+ transfer.state = .created;+ self.intercepted += 1;+ transfer.park(.intercept_request);+ return false;+ }+ }+const conn = transfer._conn.?;try self.handles.remove(conn);@@ -1913,6 +1936,10 @@ pub const Transfer = struct {_tries: u8 = 0,_redirect_count: u8 = 0,+ // Fetch.continueRequest header overrides apply to a single network hop. We+ // need to restore (and hence capture) the original headers.+ _intercept_original_headers: ?[]const RequestHeader = null,+// Linked into client.pending_queue while .queued; reused to link the// retired transfer into client.graveyard (deinit unlinks it from the// pending queue first, so the node is always free by then).@@ -2709,6 +2736,8 @@ pub const Transfer = struct {// CDP Fetch.continueRequest: the intercepting client supplies the// complete header set, replacing whatever the request carried.pub fn replaceRequestHeaders(self: *Transfer, headers: []const http.Header) !void {+ lp.assert(self._intercept_original_headers == null, "Transfer.replaceRequestHeaders", .{ .id = self.id });+ self._intercept_original_headers = try self.arena.allocator().dupe(RequestHeader, self.req_headers.items);self.req_headers.clearRetainingCapacity();try self.seedHeaders();for (headers) |hdr| {@@ -2716,6 +2745,15 @@ pub const Transfer = struct {}}+ fn restoreInterceptHeaders(self: *Transfer) void {+ const headers = self._intercept_original_headers orelse return;+ self.req_headers.clearRetainingCapacity();+ // _intercept_original_headers.items.len MIGHT be larger than self.req_headers.items.len+ // but the capacity never shrank from when _intercept_original_headers WAS req_headers.+ self.req_headers.appendSliceAssumeCapacity(headers);+ self._intercept_original_headers = null;+ }+// abortAuthChallenge is called when an auth challenge interception is// abort. We don't call self.releaseConn here b/c it has been done// before interception process.@@ -3373,6 +3411,33 @@ test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {try testing.expectEqual("yes", headers[2].value);}+test "HttpClient: Fetch header overrides restore after one hop" {+ const original = [_]Transfer.RequestHeader{+ .{ .name = "User-Agent", .value = "original" },+ .{ .name = "X-Original", .value = "yes" },+ };+ var overridden = [_]Transfer.RequestHeader{+ .{ .name = "User-Agent", .value = "override" },+ .{ .name = "X-Override", .value = "yes" },+ };++ var transfer: Transfer = undefined;+ transfer.req_headers = .{ .items = &overridden, .capacity = overridden.len };+ transfer._intercept_original_headers = &original;+ transfer.restoreInterceptHeaders();++ try testing.expectEqual(2, transfer.req_headers.items.len);+ try testing.expectEqual("User-Agent", transfer.req_headers.items[0].name);+ try testing.expectEqual("original", transfer.req_headers.items[0].value);+ try testing.expectEqual("X-Original", transfer.req_headers.items[1].name);+ try testing.expectEqual("yes", transfer.req_headers.items[1].value);+ try testing.expectEqual(null, transfer._intercept_original_headers);++ // idempotent once restored+ transfer.restoreInterceptHeaders();+ try testing.expectEqual(2, transfer.req_headers.items.len);+}+test "HttpClient: fulfillIntercepted survives a done_callback that tears down the owner" {// Regression: the fulfilled response's done_callback runs JS which// navigates / closes the page, re-entrantly killing the transfer