Skip to content

Lightpanda: the headless browser designed for AI and automation

cdp: report redirect chains before interception

Scott Taylorff975b106f82
diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig
index d8e334ac9..708e014d1 100644
--- a/src/cdp/domains/network.zig
+++ b/src/cdp/domains/network.zig
@@ -385,6 +385,10 @@ pub fn httpRequestStart(arena: Allocator, bc: *CDP.BrowserContext, msg: *const N
.documentURL = frame.url,
.request = RequestWriter.init(arena, transfer),
.initiator = .{ .type = "other" },
+ .redirectResponse = if (transfer._redirect_count > 0)
+ ResponseWriter.init(arena, transfer)
+ else
+ null,
.redirectHasExtraInfo = false, // TODO change after adding Network.requestWillBeSentExtraInfo
.hasUserGesture = false,
.timestamp = lp.datetime.timestamp(.boot),
@@ -536,11 +540,15 @@ const ResponseWriter = struct {
fn _jsonStringify(self: *const ResponseWriter, jws: anytype) !void {
const transfer = self.transfer;
+ const response_url = if (transfer.res.header) |header|
+ std.mem.span(header.url)
+ else
+ transfer.req.url;
try jws.beginObject();
{
try jws.objectField("url");
- try jws.write(transfer.req.url);
+ try jws.write(response_url);
}
if (transfer.responseStatus()) |status| {
@@ -580,7 +588,7 @@ const ResponseWriter = struct {
try jws.objectField("encodedDataLength");
try jws.write(transfer._content_length);
try jws.objectField("securityState");
- try jws.write(securityState(transfer.req.url));
+ try jws.write(securityState(response_url));
}
{
@@ -1173,6 +1181,123 @@ test "cdp.Network: POST body exposed as postData" {
try ctx.expectSentError(-31998, "RequestNotFound", .{ .id = 3 });
}
+test "cdp.Network: redirect hop precedes Fetch pause and carries redirectResponse" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-REDIRECT", .session_id = "SID-REDIRECT" });
+ const page = try bc.session.createPage();
+ const client = &bc.cdp.browser.http_client;
+
+ try ctx.processMessage(.{ .id = 1, .method = "Network.enable" });
+ try ctx.expectSentResult(null, .{ .id = 1 });
+ try ctx.processMessage(.{ .id = 2, .method = "Fetch.enable" });
+ try ctx.expectSentResult(null, .{ .id = 2 });
+
+ const CallbackContext = struct {
+ body: [64]u8 = undefined,
+ body_len: usize = 0,
+ done: bool = false,
+ err: ?anyerror = null,
+
+ fn dataCallback(transfer: *Transfer, data: []const u8) !void {
+ const self: *@This() = @ptrCast(@alignCast(transfer.req.ctx));
+ @memcpy(self.body[self.body_len..][0..data.len], data);
+ self.body_len += data.len;
+ }
+
+ fn doneCallback(raw: *anyopaque) !void {
+ const self: *@This() = @ptrCast(@alignCast(raw));
+ self.done = true;
+ }
+
+ fn errorCallback(raw: *anyopaque, err: anyerror) void {
+ const self: *@This() = @ptrCast(@alignCast(raw));
+ self.err = err;
+ }
+ };
+ var callback_context: CallbackContext = .{};
+
+ const start_url = "http://127.0.0.1:9582/redirect-cross-origin-x-hop";
+ const target_url = "http://localhost:9582/echo-x-hop";
+ var request_id: [14]u8 = undefined;
+ _ = std.fmt.bufPrint(&request_id, "REQ-{d:0>10}", .{client.next_request_id +% 1}) catch unreachable;
+
+ try client.request(.{
+ .frame_id = page.frame_id,
+ .loader_id = 7,
+ .method = .GET,
+ .url = start_url,
+ .cookie_jar = null,
+ .cookie_origin = start_url,
+ .resource_type = .script,
+ .notification = bc.session.notification,
+ .ctx = &callback_context,
+ .data_callback = CallbackContext.dataCallback,
+ .done_callback = CallbackContext.doneCallback,
+ .error_callback = CallbackContext.errorCallback,
+ .shutdown_callback = HttpClient.noopShutdown,
+ }, null);
+
+ try ctx.expectSentEvent("Network.requestWillBeSent", .{
+ .requestId = &request_id,
+ .loaderId = &id.toLoaderId(7),
+ .type = "Script",
+ .request = .{ .url = start_url },
+ }, .{ .index = 2, .session_id = "SID-REDIRECT" });
+ try ctx.expectSentEvent("Fetch.requestPaused", .{
+ .requestId = "INT-0000000001",
+ .networkId = &request_id,
+ .request = .{ .url = start_url },
+ }, .{ .index = 3, .session_id = "SID-REDIRECT" });
+
+ try ctx.processMessage(.{
+ .id = 3,
+ .method = "Fetch.continueRequest",
+ .params = .{
+ .requestId = "INT-0000000001",
+ .headers = &[_]HttpClient.Header{.{ .name = "x-hop", .value = "127.0.0.1:9582" }},
+ },
+ });
+ try ctx.expectSentResult(null, .{ .id = 3, .index = 4 });
+
+ // Playwright requires requestWillBeSent first so it can create the new
+ // redirected Request before pairing the fresh Fetch pause with it.
+ try ctx.expectSentEvent("Network.requestWillBeSent", .{
+ .requestId = &request_id,
+ .loaderId = &id.toLoaderId(7),
+ .type = "Script",
+ .request = .{ .url = target_url },
+ .redirectResponse = .{
+ .url = start_url,
+ .status = 302,
+ },
+ }, .{ .index = 5, .session_id = "SID-REDIRECT" });
+ try ctx.expectSentEvent("Fetch.requestPaused", .{
+ .requestId = "INT-0000000002",
+ .networkId = &request_id,
+ .request = .{ .url = target_url },
+ }, .{ .index = 6, .session_id = "SID-REDIRECT" });
+
+ try ctx.processMessage(.{
+ .id = 4,
+ .method = "Fetch.continueRequest",
+ .params = .{
+ .requestId = "INT-0000000002",
+ .headers = &[_]HttpClient.Header{.{ .name = "x-hop", .value = "localhost:9582" }},
+ },
+ });
+ try ctx.expectSentResult(null, .{ .id = 4, .index = 7 });
+
+ for (0..50) |_| {
+ if (callback_context.done or callback_context.err != null) break;
+ _ = try client.tick(20);
+ }
+ try testing.expectEqual(null, callback_context.err);
+ try testing.expect(callback_context.done);
+ try testing.expectEqualSlices(u8, "localhost:9582", callback_context.body[0..callback_context.body_len]);
+}
+
test "cdp.Network: worker requests emit network events" {
var ctx = try testing.context();
defer ctx.deinit();
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 90a2c4eab..1d0294a23 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -1466,8 +1466,21 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
if (msg.conn.getResponseHeader("location", 0)) |location| switch (transfer.req.redirect) {
.follow => {
transfer.restoreInterceptHeaders();
+ // Preserve the completed 3xx response until the redirected
+ // requestWillBeSent event has been serialized. reset() below
+ // clears it before the next network attempt.
+ try transfer.materializeResponse(msg.conn, .{ .check_content_length = false });
try transfer.handleRedirect(location.value);
+ if (!transfer.req.internal) lp.metrics.http_redirects.incr();
+
+ if (self.serve_mode) { // e.g. cdp
+ // Chromium announces each redirect hop before pausing it
+ // for Fetch interception. Playwright uses redirectResponse
+ // to pair the new pause with a new Request.
+ transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });
+ }
+
if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {
log.warn(.http, "blocked url", .{ .url = transfer.req.url });
self.removeConn(msg.conn);
@@ -1476,8 +1489,6 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
return true;
}
- if (!transfer.req.internal) lp.metrics.http_redirects.incr();
-
if (self.serve_mode) { // e.g. cdp
var wait_for_interception = false;
transfer.req.notification.dispatch(.http_request_intercept, &.{
@@ -1486,8 +1497,6 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
});
if (wait_for_interception) {
- transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });
-
// Same shape as the auth-interception park above:
// give up the connection, wait for the CDP client.
self.removeConn(msg.conn);
@@ -1545,7 +1554,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
return true;
}
- try transfer.materializeResponse(msg.conn);
+ try transfer.materializeResponse(msg.conn, .{});
// Latency is only meaningful for responses that hit the network (cache
// and synthetic responses never reach processOneMessage).
@@ -2433,7 +2442,11 @@ pub const Transfer = struct {
// Copy everything the response needs out of the conn and into the
// transfer arena. After this, delivery never touches libcurl state —
// the conn can be released and reused before the consumer sees a byte.
- fn materializeResponse(self: *Transfer, conn: *http.Connection) !void {
+ const MaterializeResponseOpts = struct {
+ check_content_length: bool = true,
+ };
+
+ fn materializeResponse(self: *Transfer, conn: *http.Connection, opts: MaterializeResponseOpts) !void {
if (self.res.stream.started) {
// Streaming: already materialized at the first delivered chunk.
return;
@@ -2469,9 +2482,11 @@ pub const Transfer = struct {
}
}
- if (self.getContentLength()) |cl| {
- if (cl > self.client.max_response_size) {
- return error.ResponseTooLarge;
+ if (opts.check_content_length) {
+ if (self.getContentLength()) |cl| {
+ if (cl > self.client.max_response_size) {
+ return error.ResponseTooLarge;
+ }
}
}
}
@@ -2594,17 +2609,7 @@ pub const Transfer = struct {
const req = &transfer.req;
const conn = transfer._conn.?;
- // retrieve cookies from the redirect's response.
- if (req.cookie_jar) |jar| {
- var i: usize = 0;
- while (conn.getResponseHeader("set-cookie", i)) |ct| : (i += 1) {
- try jar.populateFromResponse(transfer.req.url, ct.value);
-
- if (i >= ct.amount) {
- break;
- }
- }
- }
+ // Cookies were applied while materializing the redirect response.
// A Referrer-Policy header on a redirect response applies to the
// remaining hops.
@@ -2901,7 +2906,7 @@ pub const Transfer = struct {
const res = &self.res;
if (res.stream.started == false) {
// we haven't delivered the start/header events yet
- try self.materializeResponse(conn);
+ try self.materializeResponse(conn, .{});
try self._events.ensureUnusedCapacity(self.arena.allocator(), 3);
self._events.appendAssumeCapacity(.start);
self._events.appendAssumeCapacity(.header);
diff --git a/src/testing.zig b/src/testing.zig
index b62a5a327..daac8eded 100644
--- a/src/testing.zig
+++ b/src/testing.zig
@@ -681,6 +681,31 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {
});
}
+ if (std.mem.eql(u8, path, "/redirect-cross-origin-x-hop")) {
+ return req.respond("", .{
+ .status = .found,
+ .extra_headers = &.{
+ .{ .name = "Location", .value = "http://localhost:9582/echo-x-hop" },
+ },
+ });
+ }
+
+ if (std.mem.eql(u8, path, "/echo-x-hop")) {
+ var it = req.iterateHeaders();
+ var value: []const u8 = "NONE";
+ while (it.next()) |header| {
+ if (std.ascii.eqlIgnoreCase(header.name, "x-hop")) {
+ value = header.value;
+ break;
+ }
+ }
+ return req.respond(value, .{
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = "text/plain; charset=utf-8" },
+ },
+ });
+ }
+
if (std.mem.eql(u8, path, "/redirect-target")) {
return req.respond("<!DOCTYPE html><title>landed</title>", .{
.extra_headers = &.{