Skip to content

Lightpanda: the headless browser designed for AI and automation

cdp: optimize Network.getResponseBody

Karl Seguin57a263b2119b
diff --git a/src/server/Link.zig b/src/server/Link.zig
index 8d7348c67..f5be9b110 100644
--- a/src/server/Link.zig
+++ b/src/server/Link.zig
@@ -196,13 +196,18 @@ pub fn sendPong(self: *Link, data: []const u8) !void {
// Websocket frames have a variable-length header (2-10 bytes server->client).
// We serialize into a buffer whose first 10 bytes are reserved, then
// backfill the header right-aligned and send the slice.
-pub fn sendJSON(self: *Link, message: anytype, opts: std.json.Stringify.Options) !void {
+pub const SendOpts = struct {
+ stringify: std.json.Stringify.Options = .{},
+ size_hint: usize = 0,
+};
+pub fn sendJSON(self: *Link, message: anytype, opts: SendOpts) !void {
const allocator = self.acquireSendArena();
defer self.releaseSendArena();
- var aw = try std.Io.Writer.Allocating.initCapacity(allocator, 512);
+ // 512 covers the envelope (id, sessionId, field names) around the payload.
+ var aw = try std.Io.Writer.Allocating.initCapacity(allocator, 512 + opts.size_hint);
try aw.writer.writeAll(&[_]u8{0} ** 10);
- try std.json.Stringify.value(message, opts, &aw.writer);
+ try std.json.Stringify.value(message, opts.stringify, &aw.writer);
const framed = WS.fillHeader(aw.toArrayList());
return self.send(framed);
}
diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zig
index 9f59fec21..5fc29e8cf 100644
--- a/src/server/cdp/CDP.zig
+++ b/src/server/cdp/CDP.zig
@@ -169,8 +169,14 @@ pub fn processMessage(self: *CDP, msg: []const u8) !void {
return self.dispatch(arena.allocator(), .{ .cdp = self }, msg);
}
-pub fn sendJSON(self: *CDP, message: anytype) !void {
- try self.link.sendJSON(message, .{ .emit_null_optional_fields = false });
+pub const SendJSONOpts = struct {
+ size_hint: usize = 0,
+};
+pub fn sendJSON(self: *CDP, message: anytype, opts: SendJSONOpts) !void {
+ try self.link.sendJSON(message, .{
+ .size_hint = opts.size_hint,
+ .stringify = .{ .emit_null_optional_fields = false },
+ });
}
// Parse-then-dispatch entry point. Used by:
@@ -370,7 +376,7 @@ pub fn sendEvent(self: *CDP, method: []const u8, p: anytype, opts: SendEventOpts
.method = method,
.params = if (comptime @typeInfo(@TypeOf(p)) == .null) struct {}{} else p,
.sessionId = opts.session_id,
- });
+ }, .{});
}
pub const BrowserContext = struct {
@@ -1450,9 +1456,9 @@ pub const Command = struct {
cdp: *CDP,
capture: *std.Io.Writer,
- pub fn sendJSON(self: Sender, message: anytype) !void {
+ pub fn sendJSON(self: Sender, message: anytype, opts: SendJSONOpts) !void {
switch (self) {
- .cdp => |cdp| return cdp.sendJSON(message),
+ .cdp => |cdp| return cdp.sendJSON(message, opts),
.capture => |writer| {
return std.json.Stringify.value(message, .{
.emit_null_optional_fields = false,
@@ -1480,13 +1486,15 @@ pub const Command = struct {
return self.browser_context.?;
}
- const SendResultOpts = struct {};
- pub fn sendResult(self: *Command, result: anytype, _: SendResultOpts) !void {
+ const SendResultOpts = struct {
+ size_hint: usize = 0, // hint about the final serialized size
+ };
+ pub fn sendResult(self: *Command, result: anytype, opts: SendResultOpts) !void {
return self.sender.sendJSON(.{
.id = self.input.id,
.result = if (comptime @typeInfo(@TypeOf(result)) == .null) struct {}{} else result,
.sessionId = self.input.session_id,
- });
+ }, .{ .size_hint = opts.size_hint });
}
pub fn sendEvent(self: *Command, method: []const u8, p: anytype, opts: SendEventOpts) !void {
@@ -1500,7 +1508,7 @@ pub const Command = struct {
.id = self.input.id,
.@"error" = .{ .code = code, .message = message },
.sessionId = self.input.session_id,
- });
+ }, .{});
}
const Input = struct {
diff --git a/src/server/cdp/domains/network.zig b/src/server/cdp/domains/network.zig
index d7f1b54d7..28228f3ba 100644
--- a/src/server/cdp/domains/network.zig
+++ b/src/server/cdp/domains/network.zig
@@ -22,6 +22,7 @@ const lp = @import("lightpanda");
const id = @import("../id.zig");
const CDP = @import("../CDP.zig");
const SafeString = @import("../SafeString.zig");
+const Base64Writer = @import("../../../Base64Writer.zig");
const Config = @import("../../../Config.zig");
const URL = @import("../../../browser/URL.zig");
@@ -359,19 +360,30 @@ fn getResponseBody(cmd: *CDP.Command) !void {
return cmd.sendResult(.{
.body = data.items,
.base64Encoded = false,
- }, .{});
+ }, .{ .size_hint = data.items.len });
}
- const encoded_len = std.base64.standard.Encoder.calcSize(data.items.len);
- const encoded = try cmd.arena.alloc(u8, encoded_len);
- _ = std.base64.standard.Encoder.encode(encoded, data.items);
-
return cmd.sendResult(.{
- .body = encoded,
+ .body = Base64Body{ .data = data.items },
.base64Encoded = true,
- }, .{});
+ }, .{ .size_hint = std.base64.standard.Encoder.calcSize(data.items.len) });
}
+// Encodes straight into the outgoing message, no intermediate buffer.
+const Base64Body = struct {
+ data: []const u8,
+
+ pub fn jsonStringify(self: Base64Body, jws: *std.json.Stringify) std.Io.Writer.Error!void {
+ try jws.beginWriteRaw();
+ try jws.writer.writeByte('"');
+ var b64 = Base64Writer.init(jws.writer, .standard);
+ try b64.writer.writeAll(self.data);
+ try b64.finish();
+ try jws.writer.writeByte('"');
+ jws.endWriteRaw();
+ }
+};
+
fn getRequestPostData(cmd: *CDP.Command) !void {
const params = (try cmd.params(struct {
requestId: []const u8, // "REQ-{d}" or "LID-{d}"
@@ -1398,6 +1410,25 @@ test "cdp.Network: getResponseBody omits a partial body" {
try testing.expectEqual(0, bc.captured_responses_size);
}
+test "cdp.Network: getResponseBody base64-encodes a non-UTF-8 body" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-B64", .session_id = "SID-B64" });
+ const page = try bc.session.createPage();
+
+ try ctx.processMessage(.{ .id = 1, .method = "Network.enable" });
+ try ctx.expectSentResult(null, .{ .id = 1 });
+
+ const request_id = try EchoDriver.run(bc, page.frame_id, "\xffab\xfe", null);
+ try ctx.processMessage(.{
+ .id = 2,
+ .method = "Network.getResponseBody",
+ .params = .{ .requestId = &request_id },
+ });
+ try ctx.expectSentResult(.{ .body = "/2Fi/g==", .base64Encoded = true }, .{ .id = 2 });
+}
+
test "cdp.Network: enable maxTotalBufferSize evicts oldest bodies first" {
var ctx = try testing.context();
defer ctx.deinit();
diff --git a/src/server/cdp/domains/page.zig b/src/server/cdp/domains/page.zig
index 97f039979..835a1f1bb 100644
--- a/src/server/cdp/domains/page.zig
+++ b/src/server/cdp/domains/page.zig
@@ -623,7 +623,7 @@ pub fn frameNavigateFailed(bc: *CDP.BrowserContext, event: *const Notification.F
},
},
.sessionId = session_id,
- });
+ }, .{});
}
// Fired from Frame.deinit while the frame's JS is still alive.
@@ -677,7 +677,7 @@ pub fn frameNavigated(arena: Allocator, bc: *CDP.BrowserContext, event: *const N
.loaderId = loader_id,
},
.sessionId = session_id,
- });
+ }, .{});
}
if (bc.page_life_cycle_events) {