Lightpanda: the headless browser designed for AI and automation
cdp: optimize Network.getResponseBody
Karl Seguin57a263b2119b
diff --git a/src/server/Link.zig b/src/server/Link.zigindex 8d7348c67..f5be9b110 100644--- a/src/server/Link.zig+++ b/src/server/Link.zig@@ -196,13 +196,18 @@ pub fn sendPong(self: *Link, data: []const u8) !void {// Websocket frames have a variable-length header (2-10 bytes server->client).// We serialize into a buffer whose first 10 bytes are reserved, then// backfill the header right-aligned and send the slice.-pub fn sendJSON(self: *Link, message: anytype, opts: std.json.Stringify.Options) !void {+pub const SendOpts = struct {+ stringify: std.json.Stringify.Options = .{},+ size_hint: usize = 0,+};+pub fn sendJSON(self: *Link, message: anytype, opts: SendOpts) !void {const allocator = self.acquireSendArena();defer self.releaseSendArena();- var aw = try std.Io.Writer.Allocating.initCapacity(allocator, 512);+ // 512 covers the envelope (id, sessionId, field names) around the payload.+ var aw = try std.Io.Writer.Allocating.initCapacity(allocator, 512 + opts.size_hint);try aw.writer.writeAll(&[_]u8{0} ** 10);- try std.json.Stringify.value(message, opts, &aw.writer);+ try std.json.Stringify.value(message, opts.stringify, &aw.writer);const framed = WS.fillHeader(aw.toArrayList());return self.send(framed);}diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zigindex 9f59fec21..5fc29e8cf 100644--- a/src/server/cdp/CDP.zig+++ b/src/server/cdp/CDP.zig@@ -169,8 +169,14 @@ pub fn processMessage(self: *CDP, msg: []const u8) !void {return self.dispatch(arena.allocator(), .{ .cdp = self }, msg);}-pub fn sendJSON(self: *CDP, message: anytype) !void {- try self.link.sendJSON(message, .{ .emit_null_optional_fields = false });+pub const SendJSONOpts = struct {+ size_hint: usize = 0,+};+pub fn sendJSON(self: *CDP, message: anytype, opts: SendJSONOpts) !void {+ try self.link.sendJSON(message, .{+ .size_hint = opts.size_hint,+ .stringify = .{ .emit_null_optional_fields = false },+ });}// Parse-then-dispatch entry point. Used by:@@ -370,7 +376,7 @@ pub fn sendEvent(self: *CDP, method: []const u8, p: anytype, opts: SendEventOpts.method = method,.params = if (comptime @typeInfo(@TypeOf(p)) == .null) struct {}{} else p,.sessionId = opts.session_id,- });+ }, .{});}pub const BrowserContext = struct {@@ -1450,9 +1456,9 @@ pub const Command = struct {cdp: *CDP,capture: *std.Io.Writer,- pub fn sendJSON(self: Sender, message: anytype) !void {+ pub fn sendJSON(self: Sender, message: anytype, opts: SendJSONOpts) !void {switch (self) {- .cdp => |cdp| return cdp.sendJSON(message),+ .cdp => |cdp| return cdp.sendJSON(message, opts),.capture => |writer| {return std.json.Stringify.value(message, .{.emit_null_optional_fields = false,@@ -1480,13 +1486,15 @@ pub const Command = struct {return self.browser_context.?;}- const SendResultOpts = struct {};- pub fn sendResult(self: *Command, result: anytype, _: SendResultOpts) !void {+ const SendResultOpts = struct {+ size_hint: usize = 0, // hint about the final serialized size+ };+ pub fn sendResult(self: *Command, result: anytype, opts: SendResultOpts) !void {return self.sender.sendJSON(.{.id = self.input.id,.result = if (comptime @typeInfo(@TypeOf(result)) == .null) struct {}{} else result,.sessionId = self.input.session_id,- });+ }, .{ .size_hint = opts.size_hint });}pub fn sendEvent(self: *Command, method: []const u8, p: anytype, opts: SendEventOpts) !void {@@ -1500,7 +1508,7 @@ pub const Command = struct {.id = self.input.id,.@"error" = .{ .code = code, .message = message },.sessionId = self.input.session_id,- });+ }, .{});}const Input = struct {diff --git a/src/server/cdp/domains/network.zig b/src/server/cdp/domains/network.zigindex d7f1b54d7..28228f3ba 100644--- a/src/server/cdp/domains/network.zig+++ b/src/server/cdp/domains/network.zig@@ -22,6 +22,7 @@ const lp = @import("lightpanda");const id = @import("../id.zig");const CDP = @import("../CDP.zig");const SafeString = @import("../SafeString.zig");+const Base64Writer = @import("../../../Base64Writer.zig");const Config = @import("../../../Config.zig");const URL = @import("../../../browser/URL.zig");@@ -359,19 +360,30 @@ fn getResponseBody(cmd: *CDP.Command) !void {return cmd.sendResult(.{.body = data.items,.base64Encoded = false,- }, .{});+ }, .{ .size_hint = data.items.len });}- const encoded_len = std.base64.standard.Encoder.calcSize(data.items.len);- const encoded = try cmd.arena.alloc(u8, encoded_len);- _ = std.base64.standard.Encoder.encode(encoded, data.items);-return cmd.sendResult(.{- .body = encoded,+ .body = Base64Body{ .data = data.items },.base64Encoded = true,- }, .{});+ }, .{ .size_hint = std.base64.standard.Encoder.calcSize(data.items.len) });}+// Encodes straight into the outgoing message, no intermediate buffer.+const Base64Body = struct {+ data: []const u8,++ pub fn jsonStringify(self: Base64Body, jws: *std.json.Stringify) std.Io.Writer.Error!void {+ try jws.beginWriteRaw();+ try jws.writer.writeByte('"');+ var b64 = Base64Writer.init(jws.writer, .standard);+ try b64.writer.writeAll(self.data);+ try b64.finish();+ try jws.writer.writeByte('"');+ jws.endWriteRaw();+ }+};+fn getRequestPostData(cmd: *CDP.Command) !void {const params = (try cmd.params(struct {requestId: []const u8, // "REQ-{d}" or "LID-{d}"@@ -1398,6 +1410,25 @@ test "cdp.Network: getResponseBody omits a partial body" {try testing.expectEqual(0, bc.captured_responses_size);}+test "cdp.Network: getResponseBody base64-encodes a non-UTF-8 body" {+ var ctx = try testing.context();+ defer ctx.deinit();++ const bc = try ctx.loadBrowserContext(.{ .id = "BID-B64", .session_id = "SID-B64" });+ const page = try bc.session.createPage();++ try ctx.processMessage(.{ .id = 1, .method = "Network.enable" });+ try ctx.expectSentResult(null, .{ .id = 1 });++ const request_id = try EchoDriver.run(bc, page.frame_id, "\xffab\xfe", null);+ try ctx.processMessage(.{+ .id = 2,+ .method = "Network.getResponseBody",+ .params = .{ .requestId = &request_id },+ });+ try ctx.expectSentResult(.{ .body = "/2Fi/g==", .base64Encoded = true }, .{ .id = 2 });+}+test "cdp.Network: enable maxTotalBufferSize evicts oldest bodies first" {var ctx = try testing.context();defer ctx.deinit();diff --git a/src/server/cdp/domains/page.zig b/src/server/cdp/domains/page.zigindex 97f039979..835a1f1bb 100644--- a/src/server/cdp/domains/page.zig+++ b/src/server/cdp/domains/page.zig@@ -623,7 +623,7 @@ pub fn frameNavigateFailed(bc: *CDP.BrowserContext, event: *const Notification.F},},.sessionId = session_id,- });+ }, .{});}// Fired from Frame.deinit while the frame's JS is still alive.@@ -677,7 +677,7 @@ pub fn frameNavigated(arena: Allocator, bc: *CDP.BrowserContext, event: *const N.loaderId = loader_id,},.sessionId = session_id,- });+ }, .{});}if (bc.page_life_cycle_events) {