Skip to content

Lightpanda: the headless browser designed for AI and automation

cdp: send request information (RI) notifications on redirect

Karl Seguin7c89a462eb11
diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zig
index 5d09752b5..c94ccb277 100644
--- a/src/cdp/domains/fetch.zig
+++ b/src/cdp/domains/fetch.zig
@@ -50,15 +50,15 @@ pub fn processMessage(cmd: *CDP.Command) !void {
}
}
-// Stored in CDP. Holds *transfer ids* (not *Transfer pointers) of paused
-// transfers waiting for CDP continueRequest/fulfillRequest/failRequest/
-// continueWithAuth. Anyone resolving an entry must look the transfer up via
-// `Client.findTransfer(id)` — if the transfer has been destroyed out-of-band
-// (e.g. frame shutdown), the lookup returns null and the CDP command should
-// no-op rather than UAF.
+// Stored in CDP. Maps intercept ids to *transfer ids* (not *Transfer
+// pointers) of paused transfers waiting for CDP continueRequest/
+// fulfillRequest/failRequest/continueWithAuth. Anyone resolving an entry must
+// look the transfer up via `Client.findTransfer(id)` — if the transfer has been
+// destroyed out-of-band (e.g. frame shutdown), the lookup returns null.
pub const InterceptState = struct {
allocator: Allocator,
- waiting: std.AutoArrayHashMapUnmanaged(u32, void),
+ next_id: u32 = 1,
+ waiting: std.AutoArrayHashMapUnmanaged(u32, u32),
pub fn init(allocator: Allocator) !InterceptState {
return .{
@@ -67,17 +67,17 @@ pub const InterceptState = struct {
};
}
- pub fn empty(self: *const InterceptState) bool {
- return self.waiting.count() == 0;
+ pub fn put(self: *InterceptState, transfer_id: u32) !u32 {
+ const intercept_id = self.next_id;
+ try self.waiting.put(self.allocator, intercept_id, transfer_id);
+ self.next_id +%= 1;
+ return intercept_id;
}
- pub fn put(self: *InterceptState, transfer_id: u32) !void {
- return self.waiting.put(self.allocator, transfer_id, {});
- }
-
- // Returns true if the id was present and removed, false otherwise.
- pub fn remove(self: *InterceptState, transfer_id: u32) bool {
- return self.waiting.swapRemove(transfer_id);
+ // Returns the transfer id if the intercept id was present and removed.
+ pub fn remove(self: *InterceptState, intercept_id: u32) ?u32 {
+ const entry = self.waiting.fetchSwapRemove(intercept_id) orelse return null;
+ return entry.value;
}
pub fn deinit(self: *InterceptState) void {
@@ -85,7 +85,7 @@ pub const InterceptState = struct {
}
pub fn pendingIntercepts(self: *const InterceptState) []u32 {
- return self.waiting.keys();
+ return self.waiting.values();
}
};
@@ -203,11 +203,11 @@ pub fn requestIntercept(bc: *CDP.BrowserContext, intercept: *const Notification.
// TODO: What to do when receiving replies for a previous frame's requests?
const transfer = intercept.transfer;
- try bc.intercept_state.put(transfer.id);
- errdefer _ = bc.intercept_state.remove(transfer.id);
+ const intercept_id = try bc.intercept_state.put(transfer.id);
+ errdefer _ = bc.intercept_state.remove(intercept_id);
try bc.cdp.sendEvent("Fetch.requestPaused", .{
- .requestId = &id.toInterceptId(transfer.id),
+ .requestId = &id.toInterceptId(intercept_id),
.frameId = &id.toFrameId(transfer.req.frame_id),
.request = network.RequestWriter.init(transfer),
.resourceType = transfer.req.resource_type.string(),
@@ -241,9 +241,9 @@ fn continueRequest(cmd: *CDP.Command) !void {
const client = &bc.cdp.browser.http_client;
var intercept_state = &bc.intercept_state;
- const transfer_id = try idFromRequestId(params.requestId);
+ const intercept_id = try idFromRequestId(params.requestId);
- if (!intercept_state.remove(transfer_id)) return error.RequestNotFound;
+ const transfer_id = intercept_state.remove(intercept_id) orelse return error.RequestNotFound;
// Transfer may have been destroyed out-of-band between pause and now
// (e.g. frame shutdown). Treat as a no-op rather than an error — the CDP
// client's view of "this request still exists" is just stale.
@@ -304,9 +304,9 @@ fn continueWithAuth(cmd: *CDP.Command) !void {
const client = &bc.cdp.browser.http_client;
var intercept_state = &bc.intercept_state;
- const transfer_id = try idFromRequestId(params.requestId);
+ const intercept_id = try idFromRequestId(params.requestId);
- if (!intercept_state.remove(transfer_id)) return error.RequestNotFound;
+ const transfer_id = intercept_state.remove(intercept_id) orelse return error.RequestNotFound;
const transfer = client.findTransfer(transfer_id) orelse {
log.debug(.cdp, "intercept lookup miss", .{ .id = transfer_id, .op = "auth" });
return cmd.sendResult(null, .{});
@@ -362,9 +362,9 @@ fn fulfillRequest(cmd: *CDP.Command) !void {
const client = &bc.cdp.browser.http_client;
var intercept_state = &bc.intercept_state;
- const transfer_id = try idFromRequestId(params.requestId);
+ const intercept_id = try idFromRequestId(params.requestId);
- if (!intercept_state.remove(transfer_id)) return error.RequestNotFound;
+ const transfer_id = intercept_state.remove(intercept_id) orelse return error.RequestNotFound;
const transfer = client.findTransfer(transfer_id) orelse {
log.debug(.cdp, "intercept lookup miss", .{ .id = transfer_id, .op = "fulfill" });
return cmd.sendResult(null, .{});
@@ -399,9 +399,9 @@ fn failRequest(cmd: *CDP.Command) !void {
const client = &bc.cdp.browser.http_client;
var intercept_state = &bc.intercept_state;
- const transfer_id = try idFromRequestId(params.requestId);
+ const intercept_id = try idFromRequestId(params.requestId);
- if (!intercept_state.remove(transfer_id)) return error.RequestNotFound;
+ const transfer_id = intercept_state.remove(intercept_id) orelse return error.RequestNotFound;
const transfer = client.findTransfer(transfer_id) orelse {
log.debug(.cdp, "intercept lookup miss", .{ .id = transfer_id, .op = "fail" });
return cmd.sendResult(null, .{});
@@ -426,14 +426,14 @@ pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notificati
// TODO: What to do when receiving replies for a previous frame's requests?
const transfer = intercept.transfer;
- try bc.intercept_state.put(transfer.id);
- errdefer _ = bc.intercept_state.remove(transfer.id);
+ const intercept_id = try bc.intercept_state.put(transfer.id);
+ errdefer _ = bc.intercept_state.remove(intercept_id);
const request = &transfer.req;
const challenge = transfer._auth_challenge orelse return error.NullAuthChallenge;
try bc.cdp.sendEvent("Fetch.authRequired", .{
- .requestId = &id.toInterceptId(transfer.id),
+ .requestId = &id.toInterceptId(intercept_id),
.frameId = &id.toFrameId(request.frame_id),
.request = network.RequestWriter.init(transfer),
.resourceType = request.resource_type.string(),
@@ -465,7 +465,6 @@ fn idFromRequestId(request_id: []const u8) !u32 {
}
const testing = @import("../testing.zig");
-
test "cdp.Fetch: interception events belong to the enabling session" {
var ctx = try testing.context();
defer ctx.deinit();
@@ -500,3 +499,19 @@ test "cdp.Fetch: interception events belong to the enabling session" {
});
try testing.expectEqual(null, bc.fetch_session_id);
}
+
+test "cdp.Fetch: InterceptState issues a fresh id per pause" {
+ var state = try InterceptState.init(testing.allocator);
+ defer state.deinit();
+
+ const first = try state.put(7);
+ const second = try state.put(7);
+ try testing.expectEqual(false, first == second);
+ try testing.expectEqual(2, state.pendingIntercepts().len);
+ try testing.expectEqual(7, state.pendingIntercepts()[0]);
+
+ try testing.expectEqual(7, state.remove(first).?);
+ try testing.expectEqual(null, state.remove(first));
+ try testing.expectEqual(7, state.remove(second).?);
+ try testing.expectEqual(null, state.remove(second));
+}
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index f88f9f6bc..bdf981bc8 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -1465,6 +1465,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
if (msg.conn.getResponseHeader("location", 0)) |location| switch (transfer.req.redirect) {
.follow => {
try transfer.handleRedirect(location.value);
+ transfer.restoreInterceptHeaders();
if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {
log.warn(.http, "blocked url", .{ .url = transfer.req.url });
@@ -1476,6 +1477,28 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
if (!transfer.req.internal) lp.metrics.http_redirects.incr();
+ if (self.serve_mode) { // e.g. cdp
+ var wait_for_interception = false;
+ transfer.req.notification.dispatch(.http_request_intercept, &.{
+ .transfer = transfer,
+ .wait_for_interception = &wait_for_interception,
+ });
+
+ if (wait_for_interception) {
+ transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });
+
+ // Same shape as the auth-interception park above:
+ // give up the connection, wait for the CDP client.
+ self.removeConn(msg.conn);
+ transfer._conn = null;
+ transfer.reset();
+ transfer.state = .created;
+ self.intercepted += 1;
+ transfer.park(.intercept_request);
+ return false;
+ }
+ }
+
const conn = transfer._conn.?;
try self.handles.remove(conn);
@@ -1913,6 +1936,10 @@ pub const Transfer = struct {
_tries: u8 = 0,
_redirect_count: u8 = 0,
+ // Fetch.continueRequest header overrides apply to a single network hop. We
+ // need to restore (and hence capture) the original headers.
+ _intercept_original_headers: ?[]const RequestHeader = null,
+
// Linked into client.pending_queue while .queued; reused to link the
// retired transfer into client.graveyard (deinit unlinks it from the
// pending queue first, so the node is always free by then).
@@ -2709,6 +2736,8 @@ pub const Transfer = struct {
// CDP Fetch.continueRequest: the intercepting client supplies the
// complete header set, replacing whatever the request carried.
pub fn replaceRequestHeaders(self: *Transfer, headers: []const http.Header) !void {
+ lp.assert(self._intercept_original_headers == null, "Transfer.replaceRequestHeaders", .{ .id = self.id });
+ self._intercept_original_headers = try self.arena.allocator().dupe(RequestHeader, self.req_headers.items);
self.req_headers.clearRetainingCapacity();
try self.seedHeaders();
for (headers) |hdr| {
@@ -2716,6 +2745,15 @@ pub const Transfer = struct {
}
}
+ fn restoreInterceptHeaders(self: *Transfer) void {
+ const headers = self._intercept_original_headers orelse return;
+ self.req_headers.clearRetainingCapacity();
+ // _intercept_original_headers.items.len MIGHT be larger than self.req_headers.items.len
+ // but the capacity never shrank from when _intercept_original_headers WAS req_headers.
+ self.req_headers.appendSliceAssumeCapacity(headers);
+ self._intercept_original_headers = null;
+ }
+
// abortAuthChallenge is called when an auth challenge interception is
// abort. We don't call self.releaseConn here b/c it has been done
// before interception process.
@@ -3373,6 +3411,33 @@ test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {
try testing.expectEqual("yes", headers[2].value);
}
+test "HttpClient: Fetch header overrides restore after one hop" {
+ const original = [_]Transfer.RequestHeader{
+ .{ .name = "User-Agent", .value = "original" },
+ .{ .name = "X-Original", .value = "yes" },
+ };
+ var overridden = [_]Transfer.RequestHeader{
+ .{ .name = "User-Agent", .value = "override" },
+ .{ .name = "X-Override", .value = "yes" },
+ };
+
+ var transfer: Transfer = undefined;
+ transfer.req_headers = .{ .items = &overridden, .capacity = overridden.len };
+ transfer._intercept_original_headers = &original;
+ transfer.restoreInterceptHeaders();
+
+ try testing.expectEqual(2, transfer.req_headers.items.len);
+ try testing.expectEqual("User-Agent", transfer.req_headers.items[0].name);
+ try testing.expectEqual("original", transfer.req_headers.items[0].value);
+ try testing.expectEqual("X-Original", transfer.req_headers.items[1].name);
+ try testing.expectEqual("yes", transfer.req_headers.items[1].value);
+ try testing.expectEqual(null, transfer._intercept_original_headers);
+
+ // idempotent once restored
+ transfer.restoreInterceptHeaders();
+ try testing.expectEqual(2, transfer.req_headers.items.len);
+}
+
test "HttpClient: fulfillIntercepted survives a done_callback that tears down the owner" {
// Regression: the fulfilled response's done_callback runs JS which
// navigates / closes the page, re-entrantly killing the transfer