Skip to content

Lightpanda: the headless browser designed for AI and automation

cdp: add request post data

Karl Seguin93bcf5c93a41
diff --git a/src/cdp/CDP.zig b/src/cdp/CDP.zig
index 7703058e7..660096936 100644
--- a/src/cdp/CDP.zig
+++ b/src/cdp/CDP.zig
@@ -509,14 +509,14 @@ pub const BrowserContext = struct {
data: std.ArrayList(u8),
};
- // Key for `captured_responses`. Documents are keyed by `loader_id`,
- // everything else by `request_id` — the two id-spaces are independent
- // counters and overlap numerically (loader 1 / request 1, loader 2 /
- // request 2, ...), so the map key has to carry the namespace or
- // entries collide. The wire-format prefix (`LID-` / `REQ-`) provides
- // the same disambiguation on lookup; see `idFromRequestId` in
- // domains/network.zig.
- pub const CapturedResponseKey = struct {
+ // Key for `captured_responses` / `captured_requests`. Documents are
+ // keyed by `loader_id`, everything else by `request_id` — the two
+ // id-spaces are independent counters and overlap numerically (loader 1
+ // / request 1, loader 2 / request 2, ...), so the map key has to carry
+ // the namespace or entries collide. The wire-format prefix (`LID-` /
+ // `REQ-`) provides the same disambiguation on lookup; see
+ // `idFromRequestId` in domains/network.zig.
+ pub const CapturedKey = struct {
kind: enum { request, loader },
id: u32,
};
@@ -592,6 +592,10 @@ pub const BrowserContext = struct {
intercept_state: InterceptState,
fetch_session_id: ?[]const u8 = null,
+ // Request bodies retained for Network.getRequestPostData, which can be
+ // called after the transfer is gone. Capped at max_post_data_size.
+ captured_requests: std.AutoHashMapUnmanaged(CapturedKey, []const u8),
+
// When network is enabled, we'll capture the transfer.id -> body
// This is awfully memory intensive, but our underlying http client and
// its users (script manager and frame) correctly do not hold the body
@@ -599,7 +603,7 @@ pub const BrowserContext = struct {
// ever streamed. So if CDP is the only thing that needs bodies in
// memory for an arbitrary amount of time, then that's where we're going
// to store the,
- captured_responses: std.AutoHashMapUnmanaged(CapturedResponseKey, CapturedResponse),
+ captured_responses: std.AutoHashMapUnmanaged(CapturedKey, CapturedResponse),
notification: *Notification,
@@ -651,6 +655,7 @@ pub const BrowserContext = struct {
.arena = cdp.browser_context_arena.allocator(),
.notification_arena = cdp.notification_arena.allocator(),
.intercept_state = try InterceptState.init(allocator),
+ .captured_requests = .empty,
.captured_responses = .empty,
.notification = notification,
};
@@ -991,12 +996,24 @@ pub const BrowserContext = struct {
pub fn onHttpRequestStart(ctx: *anyopaque, msg: *const Notification.RequestStart) !void {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
- try @import("domains/network.zig").httpRequestStart(self, msg);
+ {
+ // capture the request
+ const transfer = msg.transfer;
+ const key = keyFromTransfer(transfer);
+ const body = transfer.req.body orelse "";
+ if (body.len == 0 or body.len > @import("domains/network.zig").max_post_data_size) {
+ _ = self.captured_requests.remove(key);
+ } else {
+ const owned_body = try self.frame_arena.dupe(u8, body);
+ try self.captured_requests.put(self.frame_arena, key, owned_body);
+ }
+ }
+ try @import("domains/network.zig").httpRequestStart(self.notification_arena, self, msg);
}
pub fn onHttpRequestIntercept(ctx: *anyopaque, msg: *const Notification.RequestIntercept) !void {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
- try @import("domains/fetch.zig").requestIntercept(self, msg);
+ try @import("domains/fetch.zig").requestIntercept(self.notification_arena, self, msg);
}
pub fn onHttpRequestFail(ctx: *anyopaque, msg: *const Notification.RequestFail) !void {
@@ -1019,7 +1036,7 @@ pub const BrowserContext = struct {
return @import("domains/page.zig").javascriptDialogOpening(self, msg);
}
- fn keyFromTransfer(transfer: *const Transfer) CDP.BrowserContext.CapturedResponseKey {
+ fn keyFromTransfer(transfer: *const Transfer) CDP.BrowserContext.CapturedKey {
return if (transfer.req.resource_type == .document)
.{ .kind = .loader, .id = transfer.req.loader_id }
else
@@ -1078,7 +1095,7 @@ pub const BrowserContext = struct {
pub fn onHttpRequestAuthRequired(ctx: *anyopaque, data: *const Notification.RequestAuthRequired) !void {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
defer self.resetNotificationArena();
- try @import("domains/fetch.zig").requestAuthRequired(self, data);
+ try @import("domains/fetch.zig").requestAuthRequired(self.notification_arena, self, data);
}
pub fn onHttpRequestServedFromCache(ctx: *anyopaque, msg: *const Notification.RequestServedFromCache) !void {
diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zig
index c94ccb277..733dd8b77 100644
--- a/src/cdp/domains/fetch.zig
+++ b/src/cdp/domains/fetch.zig
@@ -195,7 +195,7 @@ fn arePatternsSupported(patterns: []RequestPattern) bool {
return true;
}
-pub fn requestIntercept(bc: *CDP.BrowserContext, intercept: *const Notification.RequestIntercept) !void {
+pub fn requestIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *const Notification.RequestIntercept) !void {
// The session that enabled Fetch owns its interception events.
const session_id = bc.fetch_session_id orelse return;
@@ -209,7 +209,7 @@ pub fn requestIntercept(bc: *CDP.BrowserContext, intercept: *const Notification.
try bc.cdp.sendEvent("Fetch.requestPaused", .{
.requestId = &id.toInterceptId(intercept_id),
.frameId = &id.toFrameId(transfer.req.frame_id),
- .request = network.RequestWriter.init(transfer),
+ .request = network.RequestWriter.init(arena, transfer),
.resourceType = transfer.req.resource_type.string(),
.networkId = &id.toRequestId(transfer), // matches the Network REQ-ID
}, .{ .session_id = session_id });
@@ -418,7 +418,7 @@ fn failRequest(cmd: *CDP.Command) !void {
return cmd.sendResult(null, .{});
}
-pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notification.RequestAuthRequired) !void {
+pub fn requestAuthRequired(arena: Allocator, bc: *CDP.BrowserContext, intercept: *const Notification.RequestAuthRequired) !void {
const session_id = bc.fetch_session_id orelse return;
// We keep it around to wait for modifications to the request.
@@ -435,7 +435,7 @@ pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notificati
try bc.cdp.sendEvent("Fetch.authRequired", .{
.requestId = &id.toInterceptId(intercept_id),
.frameId = &id.toFrameId(request.frame_id),
- .request = network.RequestWriter.init(transfer),
+ .request = network.RequestWriter.init(arena, transfer),
.resourceType = request.resource_type.string(),
.authChallenge = .{
.origin = "", // TODO get origin, could be the proxy address for example.
diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig
index a08f85518..dce037291 100644
--- a/src/cdp/domains/network.zig
+++ b/src/cdp/domains/network.zig
@@ -37,6 +37,7 @@ const CdpStorage = @import("storage.zig");
const log = lp.log;
const Allocator = std.mem.Allocator;
+pub const max_post_data_size = 64 * 1024;
pub fn processMessage(cmd: *CDP.Command) !void {
const action = std.meta.stringToEnum(enum {
@@ -55,6 +56,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {
getCookies,
getAllCookies,
getResponseBody,
+ getRequestPostData,
}, cmd.input.action) orelse return error.UnknownMethod;
switch (action) {
@@ -73,6 +75,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {
.getCookies => return getCookies(cmd),
.getAllCookies => return getAllCookies(cmd),
.getResponseBody => return getResponseBody(cmd),
+ .getRequestPostData => return getRequestPostData(cmd),
}
}
@@ -322,6 +325,18 @@ fn getResponseBody(cmd: *CDP.Command) !void {
}, .{});
}
+fn getRequestPostData(cmd: *CDP.Command) !void {
+ const params = (try cmd.params(struct {
+ requestId: []const u8, // "REQ-{d}" or "LID-{d}"
+ })) orelse return error.InvalidParams;
+
+ const key = try keyFromRequestId(params.requestId);
+ const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+ const body = bc.captured_requests.get(key) orelse return error.RequestNotFound;
+
+ return cmd.sendResult(.{ .postData = SafeString.wrap(body) }, .{});
+}
+
pub fn httpRequestFail(bc: *CDP.BrowserContext, msg: *const Notification.RequestFail) !void {
// It's possible that the request failed because we aborted when the client
// sent Target.closeTarget. In that case, bc.session_id will be cleared
@@ -344,7 +359,7 @@ pub fn httpRequestFail(bc: *CDP.BrowserContext, msg: *const Notification.Request
}, .{ .session_id = session_id });
}
-pub fn httpRequestStart(bc: *CDP.BrowserContext, msg: *const Notification.RequestStart) !void {
+pub fn httpRequestStart(arena: Allocator, bc: *CDP.BrowserContext, msg: *const Notification.RequestStart) !void {
// detachTarget could be called, in which case, we still have a frame doing
// things, but no session.
const session_id = bc.session_id orelse return;
@@ -368,7 +383,7 @@ pub fn httpRequestStart(bc: *CDP.BrowserContext, msg: *const Notification.Reques
.loaderId = &id.toLoaderId(req.loader_id),
.type = req.resource_type.string(),
.documentURL = frame.url,
- .request = RequestWriter.init(transfer),
+ .request = RequestWriter.init(arena, transfer),
.initiator = .{ .type = "other" },
.redirectHasExtraInfo = false, // TODO change after adding Network.requestWillBeSentExtraInfo
.hasUserGesture = false,
@@ -418,10 +433,12 @@ pub fn httpServedFromCache(bc: *CDP.BrowserContext, msg: *const Notification.Req
}
pub const RequestWriter = struct {
+ arena: Allocator,
transfer: *Transfer,
- pub fn init(transfer: *Transfer) RequestWriter {
+ pub fn init(arena: Allocator, transfer: *Transfer) RequestWriter {
return .{
+ .arena = arena,
.transfer = transfer,
};
}
@@ -458,6 +475,22 @@ pub const RequestWriter = struct {
try jws.write(request.body != null);
}
+ if (request.body) |body| {
+ if (body.len <= max_post_data_size) {
+ try jws.objectField("postData");
+ try jws.write(SafeString.wrap(body));
+
+ // postDataEntries is the binary-safe representation
+ // (postData is lossy for non-UTF-8 bodies).
+ const encoder = std.base64.standard.Encoder;
+ const encoded = try self.arena.alloc(u8, encoder.calcSize(body.len));
+ try jws.objectField("postDataEntries");
+ try jws.write(&[_]struct { bytes: []const u8 }{
+ .{ .bytes = encoder.encode(encoded, body) },
+ });
+ }
+ }
+
{
try jws.objectField("initialPriority");
try jws.write(initialPriority(request.resource_type));
@@ -625,7 +658,7 @@ fn securityState(url: [:0]const u8) []const u8 {
return "unknown";
}
-fn keyFromRequestId(request_id: []const u8) !CDP.BrowserContext.CapturedResponseKey {
+fn keyFromRequestId(request_id: []const u8) !CDP.BrowserContext.CapturedKey {
const key = std.fmt.parseInt(u32, request_id[4..], 10) catch return error.InvalidParams;
return if (std.mem.startsWith(u8, request_id, "LID-"))
@@ -1078,6 +1111,64 @@ test "cdp.Network: setBlockedURLs blocks requests with inspector reason" {
try testing.expectEqual(error.UrlBlocked, error_context.err.?);
}
+test "cdp.Network: POST body exposed as postData" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-PD", .session_id = "SID-PD" });
+ const page = try bc.session.createPage();
+ const client = &bc.cdp.browser.http_client;
+
+ try ctx.processMessage(.{ .id = 1, .method = "Network.enable" });
+ try ctx.expectSentResult(null, .{ .id = 1 });
+
+ var request_id: [14]u8 = undefined;
+ _ = std.fmt.bufPrint(&request_id, "REQ-{d:0>10}", .{client.next_request_id +% 1}) catch unreachable;
+
+ // \xE9 exercises the Latin-1 -> UTF-8 transcode in postData;
+ // postDataEntries carry the raw bytes in base64.
+ const body = "name=Zig&note=caf\xE9";
+
+ try client.request(.{
+ .frame_id = page.frame_id,
+ .loader_id = 1,
+ .method = .POST,
+ .url = "http://127.0.0.1:9582/echo_body",
+ .body = body,
+ .cookie_jar = null,
+ .cookie_origin = "http://127.0.0.1:9582/",
+ .resource_type = .fetch,
+ .notification = bc.session.notification,
+ .shutdown_callback = HttpClient.noopShutdown,
+ }, null);
+
+ try ctx.expectSentEvent("Network.requestWillBeSent", .{
+ .requestId = &request_id,
+ .request = .{
+ .method = "POST",
+ .hasPostData = true,
+ .postData = "name=Zig&note=café",
+ .postDataEntries = &[_]struct { bytes: []const u8 }{
+ .{ .bytes = "bmFtZT1aaWcmbm90ZT1jYWbp" },
+ },
+ },
+ }, .{ .session_id = "SID-PD" });
+
+ try ctx.processMessage(.{
+ .id = 2,
+ .method = "Network.getRequestPostData",
+ .params = .{ .requestId = &request_id },
+ });
+ try ctx.expectSentResult(.{ .postData = "name=Zig&note=café" }, .{ .id = 2 });
+
+ try ctx.processMessage(.{
+ .id = 3,
+ .method = "Network.getRequestPostData",
+ .params = .{ .requestId = "REQ-4294967295" },
+ });
+ try ctx.expectSentError(-31998, "RequestNotFound", .{ .id = 3 });
+}
+
test "cdp.Network: worker requests emit network events" {
var ctx = try testing.context();
defer ctx.deinit();
diff --git a/src/cdp/domains/page.zig b/src/cdp/domains/page.zig
index 9b334c43f..0b3e89338 100644
--- a/src/cdp/domains/page.zig
+++ b/src/cdp/domains/page.zig
@@ -542,11 +542,12 @@ pub fn frameCreated(bc: *CDP.BrowserContext, frame: *Frame) !void {
_ = try isolated_world.createContext(frame);
}
- if (!in_commit) {
+ if (in_commit == false) {
// Only retain captured responses until a navigation event. In CDP
// terms, this is called a "renderer" and the cache-duration can be
// controlled via Network.configureDurableMessages (which we don't
// support).
+ bc.captured_requests = .empty;
bc.captured_responses = .empty;
}
}