Lightpanda: the headless browser designed for AI and automation
cdp: preserve message storage across reentrant notifications
Scott Taylorc284fce11aae
diff --git a/src/server/Link.zig b/src/server/Link.zigindex ffe97fa38..9bc1ffc1b 100644--- a/src/server/Link.zig+++ b/src/server/Link.zig@@ -56,6 +56,8 @@ socket: posix.socket_t,protocol: Driver.Protocol,reader: WS.Reader,send_arena: ArenaAllocator,+// Nested serialization must not reset an outer message's storage.+send_depth: u32,send_timeout_ms: i32,max_inbox_backlog: usize,@@ -86,6 +88,7 @@ pub fn init(.arena_pool = &app.arena_pool,.reader = try .init(allocator, config.cdpMaxMessageSize()),.send_arena = ArenaAllocator.init(allocator),+ .send_depth = 0,.send_timeout_ms = SEND_TIMEOUT_MS,.max_inbox_backlog = @as(usize, config.cdpMaxMessageSize()) * INBOX_BACKLOG_MESSAGES,};@@ -115,10 +118,22 @@ pub fn destroy(self: *Link) void {allocator.destroy(self);}+// Pair every call with sendDone.+pub fn sendAllocator(self: *Link) Allocator {+ self.send_depth += 1;+ return self.send_arena.allocator();+}++pub fn sendDone(self: *Link) void {+ self.send_depth -= 1;+ if (self.send_depth == 0) {+ _ = self.send_arena.reset(.{ .retain_with_limit = 1024 * 32 });+ }+}+pub fn send(self: *Link, data: []const u8) !void {var pos: usize = 0;const socket = self.socket;- defer _ = self.send_arena.reset(.{ .retain_with_limit = 1024 * 32 });while (pos < data.len) {const written = sys_net.write(socket, data[pos..]) catch |err| switch (err) {@@ -159,7 +174,8 @@ pub fn sendPong(self: *Link, data: []const u8) !void {var header_buf: [10]u8 = undefined;const header = WS.frameHeader(&header_buf, .pong, data.len);- const allocator = self.send_arena.allocator();+ const allocator = self.sendAllocator();+ defer self.sendDone();const framed = try allocator.alloc(u8, header.len + data.len);@memcpy(framed[0..header.len], header);@memcpy(framed[header.len..], data);@@ -170,7 +186,8 @@ pub fn sendPong(self: *Link, data: []const u8) !void {// We serialize into a buffer whose first 10 bytes are reserved, then// backfill the header right-aligned and send the slice.pub fn sendJSON(self: *Link, message: anytype, opts: std.json.Stringify.Options) !void {- const allocator = self.send_arena.allocator();+ const allocator = self.sendAllocator();+ defer self.sendDone();var aw = try std.Io.Writer.Allocating.initCapacity(allocator, 512);try aw.writer.writeAll(&[_]u8{0} ** 10);@@ -360,6 +377,44 @@ test "link: send gives up when the peer stops reading" {try testing.expect(after & nonblocking != 0);}+test "link: nested serialization preserves complete frames and recovers from errors" {+ const Nested = struct {+ link: *Link,+ fail: bool,++ pub fn jsonStringify(self: @This(), w: anytype) error{WriteFailed}!void {+ try w.beginObject();+ try w.objectField("head");+ try w.write("outer head");+ self.link.sendJSON(.{ .nested = "x" ** 64 }, .{}) catch return error.WriteFailed;+ self.link.sendPong("ping") catch return error.WriteFailed;+ self.link.sendJSON(.{ .nested = "y" ** 64 }, .{}) catch return error.WriteFailed;+ if (self.fail) return error.WriteFailed;+ try w.objectField("tail");+ try w.write("outer tail" ** 128);+ try w.endObject();+ }+ };++ var ctx = try @import("cdp/testing.zig").context();+ defer ctx.deinit();+ const link = &ctx.cdp().link;++ try link.sendJSON(Nested{ .link = link, .fail = false }, .{});+ try ctx.expectSent(.{ .nested = "x" ** 64 }, .{ .index = 0 });+ try ctx.expectSent(.{ .nested = "y" ** 64 }, .{ .index = 1 });+ try ctx.expectSent(.{ .head = "outer head", .tail = "outer tail" ** 128 }, .{ .index = 2 });+ try testing.expectEqual(0, link.send_depth);++ try testing.expectError(error.WriteFailed, link.sendJSON(Nested{ .link = link, .fail = true }, .{}));+ try testing.expectEqual(0, link.send_depth);+ try link.sendJSON(.{ .recovered = true }, .{});+ try testing.expectJson(.{ .nested = "x" ** 64 }, (try ctx.getSentMessage(3)).?);+ try testing.expectJson(.{ .nested = "y" ** 64 }, (try ctx.getSentMessage(4)).?);+ try ctx.expectSent(.{ .recovered = true }, .{ .index = 5 });+ try testing.expectEqual(6, ctx.received.items.len);+}+test "link: stops reading once the worker's inbox backs up" {var pair: [2]posix.socket_t = undefined;if (std.c.socketpair(posix.AF.LOCAL, posix.SOCK.STREAM, 0, &pair) != 0) {diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zigindex 14be21be9..1f1549ad4 100644--- a/src/server/cdp/CDP.zig+++ b/src/server/cdp/CDP.zig@@ -83,8 +83,9 @@ disable_set_cache_disabled: bool = false,// one message at a time.message_arena: std.heap.ArenaAllocator,-// Used for processing notifications within a browser context.+// Retained until the outermost notification handler returns.notification_arena: std.heap.ArenaAllocator,+notification_depth: u32 = 0,// Valid for 1 frame navigation (what CDP calls a "renderer")frame_arena: std.heap.ArenaAllocator,@@ -936,8 +937,9 @@ pub const BrowserContext = struct {fn onFrameNavigated(ctx: *anyopaque, msg: *const Notification.FrameNavigated) !void {const self: *BrowserContext = @ptrCast(@alignCast(ctx));- defer self.resetNotificationArena();- return @import("domains/page.zig").frameNavigated(self.notification_arena, self, msg);+ const arena = self.acquireNotificationArena();+ defer self.releaseNotificationArena();+ return @import("domains/page.zig").frameNavigated(arena, self, msg);}fn onFrameNavigateFailed(ctx: *anyopaque, msg: *const Notification.FrameNavigateFailed) !void {@@ -984,14 +986,16 @@ pub const BrowserContext = struct {try self.captured_requests.put(self.frame_arena, key, owned_body);}}- defer self.resetNotificationArena();- try network_domain.httpRequestStart(self.notification_arena, self, msg);+ const arena = self.acquireNotificationArena();+ defer self.releaseNotificationArena();+ try network_domain.httpRequestStart(arena, self, msg);}fn onHttpRequestIntercept(ctx: *anyopaque, msg: *const Notification.RequestIntercept) !void {const self: *BrowserContext = @ptrCast(@alignCast(ctx));- defer self.resetNotificationArena();- try @import("domains/fetch.zig").requestIntercept(self.notification_arena, self, msg);+ const arena = self.acquireNotificationArena();+ defer self.releaseNotificationArena();+ try @import("domains/fetch.zig").requestIntercept(arena, self, msg);}fn onHttpRequestFail(ctx: *anyopaque, msg: *const Notification.RequestFail) !void {@@ -1023,7 +1027,8 @@ pub const BrowserContext = struct {fn onHttpResponseHeadersDone(ctx: *anyopaque, msg: *const Notification.ResponseHeaderDone) !void {const self: *BrowserContext = @ptrCast(@alignCast(ctx));- defer self.resetNotificationArena();+ const arena = self.acquireNotificationArena();+ defer self.releaseNotificationArena();// Prepare the captured response value.const key = keyFromTransfer(msg.transfer);@@ -1056,7 +1061,7 @@ pub const BrowserContext = struct {};}- return network_domain.httpResponseHeaderDone(self.notification_arena, self, msg);+ return network_domain.httpResponseHeaderDone(arena, self, msg);}fn onHttpRequestDone(ctx: *anyopaque, msg: *const Notification.RequestDone) !void {@@ -1087,8 +1092,9 @@ pub const BrowserContext = struct {fn onHttpRequestAuthRequired(ctx: *anyopaque, data: *const Notification.RequestAuthRequired) !void {const self: *BrowserContext = @ptrCast(@alignCast(ctx));- defer self.resetNotificationArena();- try @import("domains/fetch.zig").requestAuthRequired(self.notification_arena, self, data);+ const arena = self.acquireNotificationArena();+ defer self.releaseNotificationArena();+ try @import("domains/fetch.zig").requestAuthRequired(arena, self, data);}fn onHttpRequestServedFromCache(ctx: *anyopaque, msg: *const Notification.RequestServedFromCache) !void {@@ -1098,18 +1104,28 @@ pub const BrowserContext = struct {fn onConsoleMessage(ctx: *anyopaque, msg: *const Notification.ConsoleMessage) !void {const self: *BrowserContext = @ptrCast(@alignCast(ctx));- defer self.resetNotificationArena();- return @import("domains/console.zig").consoleMessage(self.notification_arena, self, msg);+ const arena = self.acquireNotificationArena();+ defer self.releaseNotificationArena();+ return @import("domains/console.zig").consoleMessage(arena, self, msg);}fn onRuntimeConsoleMessage(ctx: *anyopaque, msg: *const Notification.ConsoleMessage) !void {const self: *BrowserContext = @ptrCast(@alignCast(ctx));- defer self.resetNotificationArena();- return @import("domains/runtime.zig").consoleMessage(self.notification_arena, self, msg);+ const arena = self.acquireNotificationArena();+ defer self.releaseNotificationArena();+ return @import("domains/runtime.zig").consoleMessage(arena, self, msg);}- fn resetNotificationArena(self: *BrowserContext) void {- defer _ = self.cdp.notification_arena.reset(.{ .retain_with_limit = 1024 * 64 });+ fn acquireNotificationArena(self: *BrowserContext) Allocator {+ self.cdp.notification_depth += 1;+ return self.notification_arena;+ }++ fn releaseNotificationArena(self: *BrowserContext) void {+ self.cdp.notification_depth -= 1;+ if (self.cdp.notification_depth == 0) {+ _ = self.cdp.notification_arena.reset(.{ .retain_with_limit = 1024 * 64 });+ }}pub fn callInspector(self: *const BrowserContext, msg: []const u8) void {@@ -1151,7 +1167,8 @@ pub const BrowserContext = struct {};const cdp = self.cdp;- const allocator = cdp.link.send_arena.allocator();+ const allocator = cdp.link.sendAllocator();+ defer cdp.link.sendDone();const field = ",\"sessionId\":\"";diff --git a/src/server/cdp/domains/runtime.zig b/src/server/cdp/domains/runtime.zigindex c9a640f9b..a8aaf1658 100644--- a/src/server/cdp/domains/runtime.zig+++ b/src/server/cdp/domains/runtime.zig@@ -83,13 +83,40 @@ fn sendInspector(cmd: *CDP.Command) !void {bc.callInspector(cmd.input.json);}+// Object arguments stay remote handles; serializing them would execute page JS.const RemoteObject = struct {type: []const u8,subtype: ?[]const u8,className: ?[]const u8,description: ?[]const u8,objectId: ?[]const u8,- value: js.Value,+ value: ?std.json.Value = null,+ unserializableValue: ?[]const u8 = null,++ fn setPrimitive(self: *RemoteObject, arena: Allocator, value: js.Value) !void {+ if (value.isString()) |str| {+ self.value = .{ .string = try str.toSliceWithAlloc(arena) };+ } else if (value.isBoolean()) {+ self.value = .{ .bool = value.isTrue() };+ } else if (value.isNull()) {+ self.value = .null;+ } else if (value.isNumber()) {+ const n = try value.toF64();+ if (std.math.isNan(n)) {+ self.unserializableValue = "NaN";+ } else if (std.math.isInf(n)) {+ self.unserializableValue = if (n > 0) "Infinity" else "-Infinity";+ } else if (n == 0 and std.math.signbit(n)) {+ self.unserializableValue = "-0";+ } else if (value.isInt32()) {+ self.value = .{ .integer = @intFromFloat(n) };+ } else {+ self.value = .{ .float = n };+ }+ } else if (value.isBigInt()) {+ self.unserializableValue = try std.fmt.allocPrint(arena, "{s}n", .{try value.toStringSliceWithAlloc(arena)});+ }+ }};const ConsoleMessage = struct {@@ -118,14 +145,15 @@ pub fn consoleMessage(arena: Allocator, bc: *CDP.BrowserContext, event: *const N);defer remote_object.deinit();- try args.append(arena, .{+ const arg = try args.addOne(arena);+ arg.* = .{.type = try remote_object.getType(arena),.subtype = try remote_object.getSubtype(arena),.className = try remote_object.getClassName(arena),.description = try remote_object.getDescription(arena),.objectId = try remote_object.getObjectId(arena),- .value = value,- });+ };+ try arg.setPrimitive(arena, value);}return bc.cdp.sendEvent("Runtime.consoleAPICalled", ConsoleMessage{@@ -176,3 +204,102 @@ test "cdp.runtime: consoleAPICalled type matches the console method" {try ctx.expectSentEvent("Runtime.consoleAPICalled", .{ .type = "error" }, .{});try ctx.expectSentEvent("Runtime.consoleAPICalled", .{ .type = "debug" }, .{});}++test "cdp.runtime: consoleAPICalled only carries values for primitives" {+ testing.silenceLog(&.{.js});++ var ctx = try testing.context();+ defer ctx.deinit();++ var bc = try ctx.loadBrowserContext(.{ .id = "BID-CONS", .url = "hi.html", .target_id = "FID-0000000CON".* });+ try ctx.processMessage(.{ .id = 60, .method = "Runtime.enable" });++ const frame = bc.mainFrame() orelse unreachable;+ var ls: js.Local.Scope = undefined;+ frame.js.localScope(&ls);+ defer ls.deinit();+ _ = try ls.local.exec(+ \\globalThis.probed = 0;+ \\const div = document.createElement('div');+ \\Object.defineProperty(div, 'id', { get() { globalThis.probed++; return 'x'; }, enumerable: true });+ \\const obj = { toJSON() { globalThis.probed++; return 'x'; } };+ \\const cycle = {}; cycle.self = cycle;+ \\console.log(div, obj, 'str', 12, 1.5, true, null, undefined, NaN, -0, 10n,+ \\ Infinity, -Infinity, false, 0, 2 ** 40, -9007199254740993n, Symbol('test'),+ \\ function named() {}, cycle, new Number(42));+ , null);++ try ctx.expectSentEvent("Runtime.consoleAPICalled", .{ .type = "log", .args = .{+ .{ .type = "object", .className = "HTMLDivElement" },+ .{ .type = "object", .className = "Object" },+ .{ .type = "string", .value = "str" },+ .{ .type = "number", .value = 12 },+ .{ .type = "number", .value = 1.5 },+ .{ .type = "boolean", .value = true },+ .{ .type = "object", .subtype = "null", .value = null },+ .{ .type = "undefined" },+ .{ .type = "number", .unserializableValue = "NaN" },+ .{ .type = "number", .unserializableValue = "-0" },+ .{ .type = "bigint", .unserializableValue = "10n" },+ .{ .type = "number", .unserializableValue = "Infinity" },+ .{ .type = "number", .unserializableValue = "-Infinity" },+ .{ .type = "boolean", .value = false },+ .{ .type = "number", .value = 0 },+ .{ .type = "number", .value = @as(f64, 1099511627776) },+ .{ .type = "bigint", .unserializableValue = "-9007199254740993n" },+ .{ .type = "symbol" },+ .{ .type = "function" },+ .{ .type = "object" },+ .{ .type = "object", .className = "Number" },+ } }, .{});++ const event = ctx.received.items[ctx.received.items.len - 1];+ const args = event.object.get("params").?.object.get("args").?.array.items;+ try testing.expectEqual(21, args.len);+ try testing.expect(args[6].object.get("value").? == .null);+ for ([_]usize{ 0, 1, 7, 8, 9, 10, 11, 12, 16, 17, 18, 19, 20 }) |i| {+ try testing.expectEqual(null, args[i].object.get("value"));+ }+ for ([_]usize{ 0, 1, 18, 19, 20 }) |i| {+ try testing.expect(args[i].object.contains("objectId"));+ }+ for ([_]usize{ 0, 1, 2, 3, 4, 5, 6, 7, 13, 14, 15, 17, 18, 19, 20 }) |i| {+ try testing.expectEqual(null, args[i].object.get("unserializableValue"));+ }++ const probed = try ls.local.exec("globalThis.probed", null);+ try testing.expectEqual(0, try probed.toF64());+}++test "cdp.runtime: console calls made while a notification is being built" {+ testing.silenceLog(&.{.js});++ var ctx = try testing.context();+ defer ctx.deinit();++ var bc = try ctx.loadBrowserContext(.{ .id = "BID-CONS", .url = "hi.html", .target_id = "FID-0000000CON".* });+ try ctx.processMessage(.{ .id = 60, .method = "Runtime.enable" });+ try ctx.processMessage(.{ .id = 61, .method = "Console.enable" });++ const frame = bc.mainFrame() orelse unreachable;+ var ls: js.Local.Scope = undefined;+ frame.js.localScope(&ls);+ defer ls.deinit();+ // Legacy Console formatting can re-enter the notification handlers.+ _ = try ls.local.exec(+ \\const inner = 'x'.repeat(120);+ \\const probe = { toString() { console.log(inner); console.log(inner); return 'outer'; } };+ \\console.log('head-marker', probe, 'tail-marker-'.repeat(20));+ , null);++ const inner = "x" ** 120;+ const tail = "tail-marker-" ** 20;+ try ctx.expectSentEvent("Console.messageAdded", .{ .level = "log", .text = inner }, .{});+ try ctx.expectSentEvent("Console.messageAdded", .{ .level = "log", .text = "head-marker outer " ++ tail }, .{});+ try ctx.expectSentEvent("Runtime.consoleAPICalled", .{ .type = "log", .args = .{.{ .type = "string", .value = inner }} }, .{});+ try ctx.expectSentEvent("Runtime.consoleAPICalled", .{ .type = "log", .args = .{ .{ .type = "string", .value = "head-marker" }, .{ .type = "object", .className = "Object" }, .{ .type = "string", .value = tail } } }, .{});+ try testing.expectEqual(0, ctx.cdp().notification_depth);+ try testing.expectEqual(0, ctx.cdp().link.send_depth);+ try ctx.processMessage(.{ .id = 62, .method = "Runtime.evaluate", .params = .{ .expression = "6 * 7" } });+ try ctx.expectSentResult(.{ .result = .{ .type = "number", .value = 42 } }, .{ .id = 62 });+}