Lightpanda: the headless browser designed for AI and automation
cdp: report redirect chains before interception
Scott Taylorff975b106f82
diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zigindex d8e334ac9..708e014d1 100644--- a/src/cdp/domains/network.zig+++ b/src/cdp/domains/network.zig@@ -385,6 +385,10 @@ pub fn httpRequestStart(arena: Allocator, bc: *CDP.BrowserContext, msg: *const N.documentURL = frame.url,.request = RequestWriter.init(arena, transfer),.initiator = .{ .type = "other" },+ .redirectResponse = if (transfer._redirect_count > 0)+ ResponseWriter.init(arena, transfer)+ else+ null,.redirectHasExtraInfo = false, // TODO change after adding Network.requestWillBeSentExtraInfo.hasUserGesture = false,.timestamp = lp.datetime.timestamp(.boot),@@ -536,11 +540,15 @@ const ResponseWriter = struct {fn _jsonStringify(self: *const ResponseWriter, jws: anytype) !void {const transfer = self.transfer;+ const response_url = if (transfer.res.header) |header|+ std.mem.span(header.url)+ else+ transfer.req.url;try jws.beginObject();{try jws.objectField("url");- try jws.write(transfer.req.url);+ try jws.write(response_url);}if (transfer.responseStatus()) |status| {@@ -580,7 +588,7 @@ const ResponseWriter = struct {try jws.objectField("encodedDataLength");try jws.write(transfer._content_length);try jws.objectField("securityState");- try jws.write(securityState(transfer.req.url));+ try jws.write(securityState(response_url));}{@@ -1173,6 +1181,123 @@ test "cdp.Network: POST body exposed as postData" {try ctx.expectSentError(-31998, "RequestNotFound", .{ .id = 3 });}+test "cdp.Network: redirect hop precedes Fetch pause and carries redirectResponse" {+ var ctx = try testing.context();+ defer ctx.deinit();++ const bc = try ctx.loadBrowserContext(.{ .id = "BID-REDIRECT", .session_id = "SID-REDIRECT" });+ const page = try bc.session.createPage();+ const client = &bc.cdp.browser.http_client;++ try ctx.processMessage(.{ .id = 1, .method = "Network.enable" });+ try ctx.expectSentResult(null, .{ .id = 1 });+ try ctx.processMessage(.{ .id = 2, .method = "Fetch.enable" });+ try ctx.expectSentResult(null, .{ .id = 2 });++ const CallbackContext = struct {+ body: [64]u8 = undefined,+ body_len: usize = 0,+ done: bool = false,+ err: ?anyerror = null,++ fn dataCallback(transfer: *Transfer, data: []const u8) !void {+ const self: *@This() = @ptrCast(@alignCast(transfer.req.ctx));+ @memcpy(self.body[self.body_len..][0..data.len], data);+ self.body_len += data.len;+ }++ fn doneCallback(raw: *anyopaque) !void {+ const self: *@This() = @ptrCast(@alignCast(raw));+ self.done = true;+ }++ fn errorCallback(raw: *anyopaque, err: anyerror) void {+ const self: *@This() = @ptrCast(@alignCast(raw));+ self.err = err;+ }+ };+ var callback_context: CallbackContext = .{};++ const start_url = "http://127.0.0.1:9582/redirect-cross-origin-x-hop";+ const target_url = "http://localhost:9582/echo-x-hop";+ var request_id: [14]u8 = undefined;+ _ = std.fmt.bufPrint(&request_id, "REQ-{d:0>10}", .{client.next_request_id +% 1}) catch unreachable;++ try client.request(.{+ .frame_id = page.frame_id,+ .loader_id = 7,+ .method = .GET,+ .url = start_url,+ .cookie_jar = null,+ .cookie_origin = start_url,+ .resource_type = .script,+ .notification = bc.session.notification,+ .ctx = &callback_context,+ .data_callback = CallbackContext.dataCallback,+ .done_callback = CallbackContext.doneCallback,+ .error_callback = CallbackContext.errorCallback,+ .shutdown_callback = HttpClient.noopShutdown,+ }, null);++ try ctx.expectSentEvent("Network.requestWillBeSent", .{+ .requestId = &request_id,+ .loaderId = &id.toLoaderId(7),+ .type = "Script",+ .request = .{ .url = start_url },+ }, .{ .index = 2, .session_id = "SID-REDIRECT" });+ try ctx.expectSentEvent("Fetch.requestPaused", .{+ .requestId = "INT-0000000001",+ .networkId = &request_id,+ .request = .{ .url = start_url },+ }, .{ .index = 3, .session_id = "SID-REDIRECT" });++ try ctx.processMessage(.{+ .id = 3,+ .method = "Fetch.continueRequest",+ .params = .{+ .requestId = "INT-0000000001",+ .headers = &[_]HttpClient.Header{.{ .name = "x-hop", .value = "127.0.0.1:9582" }},+ },+ });+ try ctx.expectSentResult(null, .{ .id = 3, .index = 4 });++ // Playwright requires requestWillBeSent first so it can create the new+ // redirected Request before pairing the fresh Fetch pause with it.+ try ctx.expectSentEvent("Network.requestWillBeSent", .{+ .requestId = &request_id,+ .loaderId = &id.toLoaderId(7),+ .type = "Script",+ .request = .{ .url = target_url },+ .redirectResponse = .{+ .url = start_url,+ .status = 302,+ },+ }, .{ .index = 5, .session_id = "SID-REDIRECT" });+ try ctx.expectSentEvent("Fetch.requestPaused", .{+ .requestId = "INT-0000000002",+ .networkId = &request_id,+ .request = .{ .url = target_url },+ }, .{ .index = 6, .session_id = "SID-REDIRECT" });++ try ctx.processMessage(.{+ .id = 4,+ .method = "Fetch.continueRequest",+ .params = .{+ .requestId = "INT-0000000002",+ .headers = &[_]HttpClient.Header{.{ .name = "x-hop", .value = "localhost:9582" }},+ },+ });+ try ctx.expectSentResult(null, .{ .id = 4, .index = 7 });++ for (0..50) |_| {+ if (callback_context.done or callback_context.err != null) break;+ _ = try client.tick(20);+ }+ try testing.expectEqual(null, callback_context.err);+ try testing.expect(callback_context.done);+ try testing.expectEqualSlices(u8, "localhost:9582", callback_context.body[0..callback_context.body_len]);+}+test "cdp.Network: worker requests emit network events" {var ctx = try testing.context();defer ctx.deinit();diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 90a2c4eab..1d0294a23 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -1466,8 +1466,21 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tif (msg.conn.getResponseHeader("location", 0)) |location| switch (transfer.req.redirect) {.follow => {transfer.restoreInterceptHeaders();+ // Preserve the completed 3xx response until the redirected+ // requestWillBeSent event has been serialized. reset() below+ // clears it before the next network attempt.+ try transfer.materializeResponse(msg.conn, .{ .check_content_length = false });try transfer.handleRedirect(location.value);+ if (!transfer.req.internal) lp.metrics.http_redirects.incr();++ if (self.serve_mode) { // e.g. cdp+ // Chromium announces each redirect hop before pausing it+ // for Fetch interception. Playwright uses redirectResponse+ // to pair the new pause with a new Request.+ transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });+ }+if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {log.warn(.http, "blocked url", .{ .url = transfer.req.url });self.removeConn(msg.conn);@@ -1476,8 +1489,6 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Treturn true;}- if (!transfer.req.internal) lp.metrics.http_redirects.incr();-if (self.serve_mode) { // e.g. cdpvar wait_for_interception = false;transfer.req.notification.dispatch(.http_request_intercept, &.{@@ -1486,8 +1497,6 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T});if (wait_for_interception) {- transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });-// Same shape as the auth-interception park above:// give up the connection, wait for the CDP client.self.removeConn(msg.conn);@@ -1545,7 +1554,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Treturn true;}- try transfer.materializeResponse(msg.conn);+ try transfer.materializeResponse(msg.conn, .{});// Latency is only meaningful for responses that hit the network (cache// and synthetic responses never reach processOneMessage).@@ -2433,7 +2442,11 @@ pub const Transfer = struct {// Copy everything the response needs out of the conn and into the// transfer arena. After this, delivery never touches libcurl state —// the conn can be released and reused before the consumer sees a byte.- fn materializeResponse(self: *Transfer, conn: *http.Connection) !void {+ const MaterializeResponseOpts = struct {+ check_content_length: bool = true,+ };++ fn materializeResponse(self: *Transfer, conn: *http.Connection, opts: MaterializeResponseOpts) !void {if (self.res.stream.started) {// Streaming: already materialized at the first delivered chunk.return;@@ -2469,9 +2482,11 @@ pub const Transfer = struct {}}- if (self.getContentLength()) |cl| {- if (cl > self.client.max_response_size) {- return error.ResponseTooLarge;+ if (opts.check_content_length) {+ if (self.getContentLength()) |cl| {+ if (cl > self.client.max_response_size) {+ return error.ResponseTooLarge;+ }}}}@@ -2594,17 +2609,7 @@ pub const Transfer = struct {const req = &transfer.req;const conn = transfer._conn.?;- // retrieve cookies from the redirect's response.- if (req.cookie_jar) |jar| {- var i: usize = 0;- while (conn.getResponseHeader("set-cookie", i)) |ct| : (i += 1) {- try jar.populateFromResponse(transfer.req.url, ct.value);-- if (i >= ct.amount) {- break;- }- }- }+ // Cookies were applied while materializing the redirect response.// A Referrer-Policy header on a redirect response applies to the// remaining hops.@@ -2901,7 +2906,7 @@ pub const Transfer = struct {const res = &self.res;if (res.stream.started == false) {// we haven't delivered the start/header events yet- try self.materializeResponse(conn);+ try self.materializeResponse(conn, .{});try self._events.ensureUnusedCapacity(self.arena.allocator(), 3);self._events.appendAssumeCapacity(.start);self._events.appendAssumeCapacity(.header);diff --git a/src/testing.zig b/src/testing.zigindex b62a5a327..daac8eded 100644--- a/src/testing.zig+++ b/src/testing.zig@@ -681,6 +681,31 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {});}+ if (std.mem.eql(u8, path, "/redirect-cross-origin-x-hop")) {+ return req.respond("", .{+ .status = .found,+ .extra_headers = &.{+ .{ .name = "Location", .value = "http://localhost:9582/echo-x-hop" },+ },+ });+ }++ if (std.mem.eql(u8, path, "/echo-x-hop")) {+ var it = req.iterateHeaders();+ var value: []const u8 = "NONE";+ while (it.next()) |header| {+ if (std.ascii.eqlIgnoreCase(header.name, "x-hop")) {+ value = header.value;+ break;+ }+ }+ return req.respond(value, .{+ .extra_headers = &.{+ .{ .name = "Content-Type", .value = "text/plain; charset=utf-8" },+ },+ });+ }+if (std.mem.eql(u8, path, "/redirect-target")) {return req.respond("<!DOCTYPE html><title>landed</title>", .{.extra_headers = &.{