Skip to content

Lightpanda: the headless browser designed for AI and automation

cdp: preserve message storage across reentrant notifications

Scott Taylorc284fce11aae
diff --git a/src/server/Link.zig b/src/server/Link.zig
index ffe97fa38..9bc1ffc1b 100644
--- a/src/server/Link.zig
+++ b/src/server/Link.zig
@@ -56,6 +56,8 @@ socket: posix.socket_t,
protocol: Driver.Protocol,
reader: WS.Reader,
send_arena: ArenaAllocator,
+// Nested serialization must not reset an outer message's storage.
+send_depth: u32,
send_timeout_ms: i32,
max_inbox_backlog: usize,
@@ -86,6 +88,7 @@ pub fn init(
.arena_pool = &app.arena_pool,
.reader = try .init(allocator, config.cdpMaxMessageSize()),
.send_arena = ArenaAllocator.init(allocator),
+ .send_depth = 0,
.send_timeout_ms = SEND_TIMEOUT_MS,
.max_inbox_backlog = @as(usize, config.cdpMaxMessageSize()) * INBOX_BACKLOG_MESSAGES,
};
@@ -115,10 +118,22 @@ pub fn destroy(self: *Link) void {
allocator.destroy(self);
}
+// Pair every call with sendDone.
+pub fn sendAllocator(self: *Link) Allocator {
+ self.send_depth += 1;
+ return self.send_arena.allocator();
+}
+
+pub fn sendDone(self: *Link) void {
+ self.send_depth -= 1;
+ if (self.send_depth == 0) {
+ _ = self.send_arena.reset(.{ .retain_with_limit = 1024 * 32 });
+ }
+}
+
pub fn send(self: *Link, data: []const u8) !void {
var pos: usize = 0;
const socket = self.socket;
- defer _ = self.send_arena.reset(.{ .retain_with_limit = 1024 * 32 });
while (pos < data.len) {
const written = sys_net.write(socket, data[pos..]) catch |err| switch (err) {
@@ -159,7 +174,8 @@ pub fn sendPong(self: *Link, data: []const u8) !void {
var header_buf: [10]u8 = undefined;
const header = WS.frameHeader(&header_buf, .pong, data.len);
- const allocator = self.send_arena.allocator();
+ const allocator = self.sendAllocator();
+ defer self.sendDone();
const framed = try allocator.alloc(u8, header.len + data.len);
@memcpy(framed[0..header.len], header);
@memcpy(framed[header.len..], data);
@@ -170,7 +186,8 @@ pub fn sendPong(self: *Link, data: []const u8) !void {
// We serialize into a buffer whose first 10 bytes are reserved, then
// backfill the header right-aligned and send the slice.
pub fn sendJSON(self: *Link, message: anytype, opts: std.json.Stringify.Options) !void {
- const allocator = self.send_arena.allocator();
+ const allocator = self.sendAllocator();
+ defer self.sendDone();
var aw = try std.Io.Writer.Allocating.initCapacity(allocator, 512);
try aw.writer.writeAll(&[_]u8{0} ** 10);
@@ -360,6 +377,44 @@ test "link: send gives up when the peer stops reading" {
try testing.expect(after & nonblocking != 0);
}
+test "link: nested serialization preserves complete frames and recovers from errors" {
+ const Nested = struct {
+ link: *Link,
+ fail: bool,
+
+ pub fn jsonStringify(self: @This(), w: anytype) error{WriteFailed}!void {
+ try w.beginObject();
+ try w.objectField("head");
+ try w.write("outer head");
+ self.link.sendJSON(.{ .nested = "x" ** 64 }, .{}) catch return error.WriteFailed;
+ self.link.sendPong("ping") catch return error.WriteFailed;
+ self.link.sendJSON(.{ .nested = "y" ** 64 }, .{}) catch return error.WriteFailed;
+ if (self.fail) return error.WriteFailed;
+ try w.objectField("tail");
+ try w.write("outer tail" ** 128);
+ try w.endObject();
+ }
+ };
+
+ var ctx = try @import("cdp/testing.zig").context();
+ defer ctx.deinit();
+ const link = &ctx.cdp().link;
+
+ try link.sendJSON(Nested{ .link = link, .fail = false }, .{});
+ try ctx.expectSent(.{ .nested = "x" ** 64 }, .{ .index = 0 });
+ try ctx.expectSent(.{ .nested = "y" ** 64 }, .{ .index = 1 });
+ try ctx.expectSent(.{ .head = "outer head", .tail = "outer tail" ** 128 }, .{ .index = 2 });
+ try testing.expectEqual(0, link.send_depth);
+
+ try testing.expectError(error.WriteFailed, link.sendJSON(Nested{ .link = link, .fail = true }, .{}));
+ try testing.expectEqual(0, link.send_depth);
+ try link.sendJSON(.{ .recovered = true }, .{});
+ try testing.expectJson(.{ .nested = "x" ** 64 }, (try ctx.getSentMessage(3)).?);
+ try testing.expectJson(.{ .nested = "y" ** 64 }, (try ctx.getSentMessage(4)).?);
+ try ctx.expectSent(.{ .recovered = true }, .{ .index = 5 });
+ try testing.expectEqual(6, ctx.received.items.len);
+}
+
test "link: stops reading once the worker's inbox backs up" {
var pair: [2]posix.socket_t = undefined;
if (std.c.socketpair(posix.AF.LOCAL, posix.SOCK.STREAM, 0, &pair) != 0) {
diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zig
index 14be21be9..1f1549ad4 100644
--- a/src/server/cdp/CDP.zig
+++ b/src/server/cdp/CDP.zig
@@ -83,8 +83,9 @@ disable_set_cache_disabled: bool = false,
// one message at a time.
message_arena: std.heap.ArenaAllocator,
-// Used for processing notifications within a browser context.
+// Retained until the outermost notification handler returns.
notification_arena: std.heap.ArenaAllocator,
+notification_depth: u32 = 0,
// Valid for 1 frame navigation (what CDP calls a "renderer")
frame_arena: std.heap.ArenaAllocator,
@@ -936,8 +937,9 @@ pub const BrowserContext = struct {
fn onFrameNavigated(ctx: *anyopaque, msg: *const Notification.FrameNavigated) !void {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
- defer self.resetNotificationArena();
- return @import("domains/page.zig").frameNavigated(self.notification_arena, self, msg);
+ const arena = self.acquireNotificationArena();
+ defer self.releaseNotificationArena();
+ return @import("domains/page.zig").frameNavigated(arena, self, msg);
}
fn onFrameNavigateFailed(ctx: *anyopaque, msg: *const Notification.FrameNavigateFailed) !void {
@@ -984,14 +986,16 @@ pub const BrowserContext = struct {
try self.captured_requests.put(self.frame_arena, key, owned_body);
}
}
- defer self.resetNotificationArena();
- try network_domain.httpRequestStart(self.notification_arena, self, msg);
+ const arena = self.acquireNotificationArena();
+ defer self.releaseNotificationArena();
+ try network_domain.httpRequestStart(arena, self, msg);
}
fn onHttpRequestIntercept(ctx: *anyopaque, msg: *const Notification.RequestIntercept) !void {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
- defer self.resetNotificationArena();
- try @import("domains/fetch.zig").requestIntercept(self.notification_arena, self, msg);
+ const arena = self.acquireNotificationArena();
+ defer self.releaseNotificationArena();
+ try @import("domains/fetch.zig").requestIntercept(arena, self, msg);
}
fn onHttpRequestFail(ctx: *anyopaque, msg: *const Notification.RequestFail) !void {
@@ -1023,7 +1027,8 @@ pub const BrowserContext = struct {
fn onHttpResponseHeadersDone(ctx: *anyopaque, msg: *const Notification.ResponseHeaderDone) !void {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
- defer self.resetNotificationArena();
+ const arena = self.acquireNotificationArena();
+ defer self.releaseNotificationArena();
// Prepare the captured response value.
const key = keyFromTransfer(msg.transfer);
@@ -1056,7 +1061,7 @@ pub const BrowserContext = struct {
};
}
- return network_domain.httpResponseHeaderDone(self.notification_arena, self, msg);
+ return network_domain.httpResponseHeaderDone(arena, self, msg);
}
fn onHttpRequestDone(ctx: *anyopaque, msg: *const Notification.RequestDone) !void {
@@ -1087,8 +1092,9 @@ pub const BrowserContext = struct {
fn onHttpRequestAuthRequired(ctx: *anyopaque, data: *const Notification.RequestAuthRequired) !void {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
- defer self.resetNotificationArena();
- try @import("domains/fetch.zig").requestAuthRequired(self.notification_arena, self, data);
+ const arena = self.acquireNotificationArena();
+ defer self.releaseNotificationArena();
+ try @import("domains/fetch.zig").requestAuthRequired(arena, self, data);
}
fn onHttpRequestServedFromCache(ctx: *anyopaque, msg: *const Notification.RequestServedFromCache) !void {
@@ -1098,18 +1104,28 @@ pub const BrowserContext = struct {
fn onConsoleMessage(ctx: *anyopaque, msg: *const Notification.ConsoleMessage) !void {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
- defer self.resetNotificationArena();
- return @import("domains/console.zig").consoleMessage(self.notification_arena, self, msg);
+ const arena = self.acquireNotificationArena();
+ defer self.releaseNotificationArena();
+ return @import("domains/console.zig").consoleMessage(arena, self, msg);
}
fn onRuntimeConsoleMessage(ctx: *anyopaque, msg: *const Notification.ConsoleMessage) !void {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
- defer self.resetNotificationArena();
- return @import("domains/runtime.zig").consoleMessage(self.notification_arena, self, msg);
+ const arena = self.acquireNotificationArena();
+ defer self.releaseNotificationArena();
+ return @import("domains/runtime.zig").consoleMessage(arena, self, msg);
}
- fn resetNotificationArena(self: *BrowserContext) void {
- defer _ = self.cdp.notification_arena.reset(.{ .retain_with_limit = 1024 * 64 });
+ fn acquireNotificationArena(self: *BrowserContext) Allocator {
+ self.cdp.notification_depth += 1;
+ return self.notification_arena;
+ }
+
+ fn releaseNotificationArena(self: *BrowserContext) void {
+ self.cdp.notification_depth -= 1;
+ if (self.cdp.notification_depth == 0) {
+ _ = self.cdp.notification_arena.reset(.{ .retain_with_limit = 1024 * 64 });
+ }
}
pub fn callInspector(self: *const BrowserContext, msg: []const u8) void {
@@ -1151,7 +1167,8 @@ pub const BrowserContext = struct {
};
const cdp = self.cdp;
- const allocator = cdp.link.send_arena.allocator();
+ const allocator = cdp.link.sendAllocator();
+ defer cdp.link.sendDone();
const field = ",\"sessionId\":\"";
diff --git a/src/server/cdp/domains/runtime.zig b/src/server/cdp/domains/runtime.zig
index c9a640f9b..a8aaf1658 100644
--- a/src/server/cdp/domains/runtime.zig
+++ b/src/server/cdp/domains/runtime.zig
@@ -83,13 +83,40 @@ fn sendInspector(cmd: *CDP.Command) !void {
bc.callInspector(cmd.input.json);
}
+// Object arguments stay remote handles; serializing them would execute page JS.
const RemoteObject = struct {
type: []const u8,
subtype: ?[]const u8,
className: ?[]const u8,
description: ?[]const u8,
objectId: ?[]const u8,
- value: js.Value,
+ value: ?std.json.Value = null,
+ unserializableValue: ?[]const u8 = null,
+
+ fn setPrimitive(self: *RemoteObject, arena: Allocator, value: js.Value) !void {
+ if (value.isString()) |str| {
+ self.value = .{ .string = try str.toSliceWithAlloc(arena) };
+ } else if (value.isBoolean()) {
+ self.value = .{ .bool = value.isTrue() };
+ } else if (value.isNull()) {
+ self.value = .null;
+ } else if (value.isNumber()) {
+ const n = try value.toF64();
+ if (std.math.isNan(n)) {
+ self.unserializableValue = "NaN";
+ } else if (std.math.isInf(n)) {
+ self.unserializableValue = if (n > 0) "Infinity" else "-Infinity";
+ } else if (n == 0 and std.math.signbit(n)) {
+ self.unserializableValue = "-0";
+ } else if (value.isInt32()) {
+ self.value = .{ .integer = @intFromFloat(n) };
+ } else {
+ self.value = .{ .float = n };
+ }
+ } else if (value.isBigInt()) {
+ self.unserializableValue = try std.fmt.allocPrint(arena, "{s}n", .{try value.toStringSliceWithAlloc(arena)});
+ }
+ }
};
const ConsoleMessage = struct {
@@ -118,14 +145,15 @@ pub fn consoleMessage(arena: Allocator, bc: *CDP.BrowserContext, event: *const N
);
defer remote_object.deinit();
- try args.append(arena, .{
+ const arg = try args.addOne(arena);
+ arg.* = .{
.type = try remote_object.getType(arena),
.subtype = try remote_object.getSubtype(arena),
.className = try remote_object.getClassName(arena),
.description = try remote_object.getDescription(arena),
.objectId = try remote_object.getObjectId(arena),
- .value = value,
- });
+ };
+ try arg.setPrimitive(arena, value);
}
return bc.cdp.sendEvent("Runtime.consoleAPICalled", ConsoleMessage{
@@ -176,3 +204,102 @@ test "cdp.runtime: consoleAPICalled type matches the console method" {
try ctx.expectSentEvent("Runtime.consoleAPICalled", .{ .type = "error" }, .{});
try ctx.expectSentEvent("Runtime.consoleAPICalled", .{ .type = "debug" }, .{});
}
+
+test "cdp.runtime: consoleAPICalled only carries values for primitives" {
+ testing.silenceLog(&.{.js});
+
+ var ctx = try testing.context();
+ defer ctx.deinit();
+
+ var bc = try ctx.loadBrowserContext(.{ .id = "BID-CONS", .url = "hi.html", .target_id = "FID-0000000CON".* });
+ try ctx.processMessage(.{ .id = 60, .method = "Runtime.enable" });
+
+ const frame = bc.mainFrame() orelse unreachable;
+ var ls: js.Local.Scope = undefined;
+ frame.js.localScope(&ls);
+ defer ls.deinit();
+ _ = try ls.local.exec(
+ \\globalThis.probed = 0;
+ \\const div = document.createElement('div');
+ \\Object.defineProperty(div, 'id', { get() { globalThis.probed++; return 'x'; }, enumerable: true });
+ \\const obj = { toJSON() { globalThis.probed++; return 'x'; } };
+ \\const cycle = {}; cycle.self = cycle;
+ \\console.log(div, obj, 'str', 12, 1.5, true, null, undefined, NaN, -0, 10n,
+ \\ Infinity, -Infinity, false, 0, 2 ** 40, -9007199254740993n, Symbol('test'),
+ \\ function named() {}, cycle, new Number(42));
+ , null);
+
+ try ctx.expectSentEvent("Runtime.consoleAPICalled", .{ .type = "log", .args = .{
+ .{ .type = "object", .className = "HTMLDivElement" },
+ .{ .type = "object", .className = "Object" },
+ .{ .type = "string", .value = "str" },
+ .{ .type = "number", .value = 12 },
+ .{ .type = "number", .value = 1.5 },
+ .{ .type = "boolean", .value = true },
+ .{ .type = "object", .subtype = "null", .value = null },
+ .{ .type = "undefined" },
+ .{ .type = "number", .unserializableValue = "NaN" },
+ .{ .type = "number", .unserializableValue = "-0" },
+ .{ .type = "bigint", .unserializableValue = "10n" },
+ .{ .type = "number", .unserializableValue = "Infinity" },
+ .{ .type = "number", .unserializableValue = "-Infinity" },
+ .{ .type = "boolean", .value = false },
+ .{ .type = "number", .value = 0 },
+ .{ .type = "number", .value = @as(f64, 1099511627776) },
+ .{ .type = "bigint", .unserializableValue = "-9007199254740993n" },
+ .{ .type = "symbol" },
+ .{ .type = "function" },
+ .{ .type = "object" },
+ .{ .type = "object", .className = "Number" },
+ } }, .{});
+
+ const event = ctx.received.items[ctx.received.items.len - 1];
+ const args = event.object.get("params").?.object.get("args").?.array.items;
+ try testing.expectEqual(21, args.len);
+ try testing.expect(args[6].object.get("value").? == .null);
+ for ([_]usize{ 0, 1, 7, 8, 9, 10, 11, 12, 16, 17, 18, 19, 20 }) |i| {
+ try testing.expectEqual(null, args[i].object.get("value"));
+ }
+ for ([_]usize{ 0, 1, 18, 19, 20 }) |i| {
+ try testing.expect(args[i].object.contains("objectId"));
+ }
+ for ([_]usize{ 0, 1, 2, 3, 4, 5, 6, 7, 13, 14, 15, 17, 18, 19, 20 }) |i| {
+ try testing.expectEqual(null, args[i].object.get("unserializableValue"));
+ }
+
+ const probed = try ls.local.exec("globalThis.probed", null);
+ try testing.expectEqual(0, try probed.toF64());
+}
+
+test "cdp.runtime: console calls made while a notification is being built" {
+ testing.silenceLog(&.{.js});
+
+ var ctx = try testing.context();
+ defer ctx.deinit();
+
+ var bc = try ctx.loadBrowserContext(.{ .id = "BID-CONS", .url = "hi.html", .target_id = "FID-0000000CON".* });
+ try ctx.processMessage(.{ .id = 60, .method = "Runtime.enable" });
+ try ctx.processMessage(.{ .id = 61, .method = "Console.enable" });
+
+ const frame = bc.mainFrame() orelse unreachable;
+ var ls: js.Local.Scope = undefined;
+ frame.js.localScope(&ls);
+ defer ls.deinit();
+ // Legacy Console formatting can re-enter the notification handlers.
+ _ = try ls.local.exec(
+ \\const inner = 'x'.repeat(120);
+ \\const probe = { toString() { console.log(inner); console.log(inner); return 'outer'; } };
+ \\console.log('head-marker', probe, 'tail-marker-'.repeat(20));
+ , null);
+
+ const inner = "x" ** 120;
+ const tail = "tail-marker-" ** 20;
+ try ctx.expectSentEvent("Console.messageAdded", .{ .level = "log", .text = inner }, .{});
+ try ctx.expectSentEvent("Console.messageAdded", .{ .level = "log", .text = "head-marker outer " ++ tail }, .{});
+ try ctx.expectSentEvent("Runtime.consoleAPICalled", .{ .type = "log", .args = .{.{ .type = "string", .value = inner }} }, .{});
+ try ctx.expectSentEvent("Runtime.consoleAPICalled", .{ .type = "log", .args = .{ .{ .type = "string", .value = "head-marker" }, .{ .type = "object", .className = "Object" }, .{ .type = "string", .value = tail } } }, .{});
+ try testing.expectEqual(0, ctx.cdp().notification_depth);
+ try testing.expectEqual(0, ctx.cdp().link.send_depth);
+ try ctx.processMessage(.{ .id = 62, .method = "Runtime.evaluate", .params = .{ .expression = "6 * 7" } });
+ try ctx.expectSentResult(.{ .result = .{ .type = "number", .value = 42 } }, .{ .id = 62 });
+}