Lightpanda: the headless browser designed for AI and automation
cdp: add request post data
Karl Seguin93bcf5c93a41
diff --git a/src/cdp/CDP.zig b/src/cdp/CDP.zigindex 7703058e7..660096936 100644--- a/src/cdp/CDP.zig+++ b/src/cdp/CDP.zig@@ -509,14 +509,14 @@ pub const BrowserContext = struct {data: std.ArrayList(u8),};- // Key for `captured_responses`. Documents are keyed by `loader_id`,- // everything else by `request_id` — the two id-spaces are independent- // counters and overlap numerically (loader 1 / request 1, loader 2 /- // request 2, ...), so the map key has to carry the namespace or- // entries collide. The wire-format prefix (`LID-` / `REQ-`) provides- // the same disambiguation on lookup; see `idFromRequestId` in- // domains/network.zig.- pub const CapturedResponseKey = struct {+ // Key for `captured_responses` / `captured_requests`. Documents are+ // keyed by `loader_id`, everything else by `request_id` — the two+ // id-spaces are independent counters and overlap numerically (loader 1+ // / request 1, loader 2 / request 2, ...), so the map key has to carry+ // the namespace or entries collide. The wire-format prefix (`LID-` /+ // `REQ-`) provides the same disambiguation on lookup; see+ // `idFromRequestId` in domains/network.zig.+ pub const CapturedKey = struct {kind: enum { request, loader },id: u32,};@@ -592,6 +592,10 @@ pub const BrowserContext = struct {intercept_state: InterceptState,fetch_session_id: ?[]const u8 = null,+ // Request bodies retained for Network.getRequestPostData, which can be+ // called after the transfer is gone. Capped at max_post_data_size.+ captured_requests: std.AutoHashMapUnmanaged(CapturedKey, []const u8),+// When network is enabled, we'll capture the transfer.id -> body// This is awfully memory intensive, but our underlying http client and// its users (script manager and frame) correctly do not hold the body@@ -599,7 +603,7 @@ pub const BrowserContext = struct {// ever streamed. So if CDP is the only thing that needs bodies in// memory for an arbitrary amount of time, then that's where we're going// to store the,- captured_responses: std.AutoHashMapUnmanaged(CapturedResponseKey, CapturedResponse),+ captured_responses: std.AutoHashMapUnmanaged(CapturedKey, CapturedResponse),notification: *Notification,@@ -651,6 +655,7 @@ pub const BrowserContext = struct {.arena = cdp.browser_context_arena.allocator(),.notification_arena = cdp.notification_arena.allocator(),.intercept_state = try InterceptState.init(allocator),+ .captured_requests = .empty,.captured_responses = .empty,.notification = notification,};@@ -991,12 +996,24 @@ pub const BrowserContext = struct {pub fn onHttpRequestStart(ctx: *anyopaque, msg: *const Notification.RequestStart) !void {const self: *BrowserContext = @ptrCast(@alignCast(ctx));- try @import("domains/network.zig").httpRequestStart(self, msg);+ {+ // capture the request+ const transfer = msg.transfer;+ const key = keyFromTransfer(transfer);+ const body = transfer.req.body orelse "";+ if (body.len == 0 or body.len > @import("domains/network.zig").max_post_data_size) {+ _ = self.captured_requests.remove(key);+ } else {+ const owned_body = try self.frame_arena.dupe(u8, body);+ try self.captured_requests.put(self.frame_arena, key, owned_body);+ }+ }+ try @import("domains/network.zig").httpRequestStart(self.notification_arena, self, msg);}pub fn onHttpRequestIntercept(ctx: *anyopaque, msg: *const Notification.RequestIntercept) !void {const self: *BrowserContext = @ptrCast(@alignCast(ctx));- try @import("domains/fetch.zig").requestIntercept(self, msg);+ try @import("domains/fetch.zig").requestIntercept(self.notification_arena, self, msg);}pub fn onHttpRequestFail(ctx: *anyopaque, msg: *const Notification.RequestFail) !void {@@ -1019,7 +1036,7 @@ pub const BrowserContext = struct {return @import("domains/page.zig").javascriptDialogOpening(self, msg);}- fn keyFromTransfer(transfer: *const Transfer) CDP.BrowserContext.CapturedResponseKey {+ fn keyFromTransfer(transfer: *const Transfer) CDP.BrowserContext.CapturedKey {return if (transfer.req.resource_type == .document).{ .kind = .loader, .id = transfer.req.loader_id }else@@ -1078,7 +1095,7 @@ pub const BrowserContext = struct {pub fn onHttpRequestAuthRequired(ctx: *anyopaque, data: *const Notification.RequestAuthRequired) !void {const self: *BrowserContext = @ptrCast(@alignCast(ctx));defer self.resetNotificationArena();- try @import("domains/fetch.zig").requestAuthRequired(self, data);+ try @import("domains/fetch.zig").requestAuthRequired(self.notification_arena, self, data);}pub fn onHttpRequestServedFromCache(ctx: *anyopaque, msg: *const Notification.RequestServedFromCache) !void {diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zigindex c94ccb277..733dd8b77 100644--- a/src/cdp/domains/fetch.zig+++ b/src/cdp/domains/fetch.zig@@ -195,7 +195,7 @@ fn arePatternsSupported(patterns: []RequestPattern) bool {return true;}-pub fn requestIntercept(bc: *CDP.BrowserContext, intercept: *const Notification.RequestIntercept) !void {+pub fn requestIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *const Notification.RequestIntercept) !void {// The session that enabled Fetch owns its interception events.const session_id = bc.fetch_session_id orelse return;@@ -209,7 +209,7 @@ pub fn requestIntercept(bc: *CDP.BrowserContext, intercept: *const Notification.try bc.cdp.sendEvent("Fetch.requestPaused", .{.requestId = &id.toInterceptId(intercept_id),.frameId = &id.toFrameId(transfer.req.frame_id),- .request = network.RequestWriter.init(transfer),+ .request = network.RequestWriter.init(arena, transfer),.resourceType = transfer.req.resource_type.string(),.networkId = &id.toRequestId(transfer), // matches the Network REQ-ID}, .{ .session_id = session_id });@@ -418,7 +418,7 @@ fn failRequest(cmd: *CDP.Command) !void {return cmd.sendResult(null, .{});}-pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notification.RequestAuthRequired) !void {+pub fn requestAuthRequired(arena: Allocator, bc: *CDP.BrowserContext, intercept: *const Notification.RequestAuthRequired) !void {const session_id = bc.fetch_session_id orelse return;// We keep it around to wait for modifications to the request.@@ -435,7 +435,7 @@ pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notificatitry bc.cdp.sendEvent("Fetch.authRequired", .{.requestId = &id.toInterceptId(intercept_id),.frameId = &id.toFrameId(request.frame_id),- .request = network.RequestWriter.init(transfer),+ .request = network.RequestWriter.init(arena, transfer),.resourceType = request.resource_type.string(),.authChallenge = .{.origin = "", // TODO get origin, could be the proxy address for example.diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zigindex a08f85518..dce037291 100644--- a/src/cdp/domains/network.zig+++ b/src/cdp/domains/network.zig@@ -37,6 +37,7 @@ const CdpStorage = @import("storage.zig");const log = lp.log;const Allocator = std.mem.Allocator;+pub const max_post_data_size = 64 * 1024;pub fn processMessage(cmd: *CDP.Command) !void {const action = std.meta.stringToEnum(enum {@@ -55,6 +56,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {getCookies,getAllCookies,getResponseBody,+ getRequestPostData,}, cmd.input.action) orelse return error.UnknownMethod;switch (action) {@@ -73,6 +75,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {.getCookies => return getCookies(cmd),.getAllCookies => return getAllCookies(cmd),.getResponseBody => return getResponseBody(cmd),+ .getRequestPostData => return getRequestPostData(cmd),}}@@ -322,6 +325,18 @@ fn getResponseBody(cmd: *CDP.Command) !void {}, .{});}+fn getRequestPostData(cmd: *CDP.Command) !void {+ const params = (try cmd.params(struct {+ requestId: []const u8, // "REQ-{d}" or "LID-{d}"+ })) orelse return error.InvalidParams;++ const key = try keyFromRequestId(params.requestId);+ const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;+ const body = bc.captured_requests.get(key) orelse return error.RequestNotFound;++ return cmd.sendResult(.{ .postData = SafeString.wrap(body) }, .{});+}+pub fn httpRequestFail(bc: *CDP.BrowserContext, msg: *const Notification.RequestFail) !void {// It's possible that the request failed because we aborted when the client// sent Target.closeTarget. In that case, bc.session_id will be cleared@@ -344,7 +359,7 @@ pub fn httpRequestFail(bc: *CDP.BrowserContext, msg: *const Notification.Request}, .{ .session_id = session_id });}-pub fn httpRequestStart(bc: *CDP.BrowserContext, msg: *const Notification.RequestStart) !void {+pub fn httpRequestStart(arena: Allocator, bc: *CDP.BrowserContext, msg: *const Notification.RequestStart) !void {// detachTarget could be called, in which case, we still have a frame doing// things, but no session.const session_id = bc.session_id orelse return;@@ -368,7 +383,7 @@ pub fn httpRequestStart(bc: *CDP.BrowserContext, msg: *const Notification.Reques.loaderId = &id.toLoaderId(req.loader_id),.type = req.resource_type.string(),.documentURL = frame.url,- .request = RequestWriter.init(transfer),+ .request = RequestWriter.init(arena, transfer),.initiator = .{ .type = "other" },.redirectHasExtraInfo = false, // TODO change after adding Network.requestWillBeSentExtraInfo.hasUserGesture = false,@@ -418,10 +433,12 @@ pub fn httpServedFromCache(bc: *CDP.BrowserContext, msg: *const Notification.Req}pub const RequestWriter = struct {+ arena: Allocator,transfer: *Transfer,- pub fn init(transfer: *Transfer) RequestWriter {+ pub fn init(arena: Allocator, transfer: *Transfer) RequestWriter {return .{+ .arena = arena,.transfer = transfer,};}@@ -458,6 +475,22 @@ pub const RequestWriter = struct {try jws.write(request.body != null);}+ if (request.body) |body| {+ if (body.len <= max_post_data_size) {+ try jws.objectField("postData");+ try jws.write(SafeString.wrap(body));++ // postDataEntries is the binary-safe representation+ // (postData is lossy for non-UTF-8 bodies).+ const encoder = std.base64.standard.Encoder;+ const encoded = try self.arena.alloc(u8, encoder.calcSize(body.len));+ try jws.objectField("postDataEntries");+ try jws.write(&[_]struct { bytes: []const u8 }{+ .{ .bytes = encoder.encode(encoded, body) },+ });+ }+ }+{try jws.objectField("initialPriority");try jws.write(initialPriority(request.resource_type));@@ -625,7 +658,7 @@ fn securityState(url: [:0]const u8) []const u8 {return "unknown";}-fn keyFromRequestId(request_id: []const u8) !CDP.BrowserContext.CapturedResponseKey {+fn keyFromRequestId(request_id: []const u8) !CDP.BrowserContext.CapturedKey {const key = std.fmt.parseInt(u32, request_id[4..], 10) catch return error.InvalidParams;return if (std.mem.startsWith(u8, request_id, "LID-"))@@ -1078,6 +1111,64 @@ test "cdp.Network: setBlockedURLs blocks requests with inspector reason" {try testing.expectEqual(error.UrlBlocked, error_context.err.?);}+test "cdp.Network: POST body exposed as postData" {+ var ctx = try testing.context();+ defer ctx.deinit();++ const bc = try ctx.loadBrowserContext(.{ .id = "BID-PD", .session_id = "SID-PD" });+ const page = try bc.session.createPage();+ const client = &bc.cdp.browser.http_client;++ try ctx.processMessage(.{ .id = 1, .method = "Network.enable" });+ try ctx.expectSentResult(null, .{ .id = 1 });++ var request_id: [14]u8 = undefined;+ _ = std.fmt.bufPrint(&request_id, "REQ-{d:0>10}", .{client.next_request_id +% 1}) catch unreachable;++ // \xE9 exercises the Latin-1 -> UTF-8 transcode in postData;+ // postDataEntries carry the raw bytes in base64.+ const body = "name=Zig¬e=caf\xE9";++ try client.request(.{+ .frame_id = page.frame_id,+ .loader_id = 1,+ .method = .POST,+ .url = "http://127.0.0.1:9582/echo_body",+ .body = body,+ .cookie_jar = null,+ .cookie_origin = "http://127.0.0.1:9582/",+ .resource_type = .fetch,+ .notification = bc.session.notification,+ .shutdown_callback = HttpClient.noopShutdown,+ }, null);++ try ctx.expectSentEvent("Network.requestWillBeSent", .{+ .requestId = &request_id,+ .request = .{+ .method = "POST",+ .hasPostData = true,+ .postData = "name=Zig¬e=café",+ .postDataEntries = &[_]struct { bytes: []const u8 }{+ .{ .bytes = "bmFtZT1aaWcmbm90ZT1jYWbp" },+ },+ },+ }, .{ .session_id = "SID-PD" });++ try ctx.processMessage(.{+ .id = 2,+ .method = "Network.getRequestPostData",+ .params = .{ .requestId = &request_id },+ });+ try ctx.expectSentResult(.{ .postData = "name=Zig¬e=café" }, .{ .id = 2 });++ try ctx.processMessage(.{+ .id = 3,+ .method = "Network.getRequestPostData",+ .params = .{ .requestId = "REQ-4294967295" },+ });+ try ctx.expectSentError(-31998, "RequestNotFound", .{ .id = 3 });+}+test "cdp.Network: worker requests emit network events" {var ctx = try testing.context();defer ctx.deinit();diff --git a/src/cdp/domains/page.zig b/src/cdp/domains/page.zigindex 9b334c43f..0b3e89338 100644--- a/src/cdp/domains/page.zig+++ b/src/cdp/domains/page.zig@@ -542,11 +542,12 @@ pub fn frameCreated(bc: *CDP.BrowserContext, frame: *Frame) !void {_ = try isolated_world.createContext(frame);}- if (!in_commit) {+ if (in_commit == false) {// Only retain captured responses until a navigation event. In CDP// terms, this is called a "renderer" and the cache-duration can be// controlled via Network.configureDurableMessages (which we don't// support).+ bc.captured_requests = .empty;bc.captured_responses = .empty;}}