Lightpanda: the headless browser designed for AI and automation
CDP: enforce Network.enable limits
Karl Seguindad2ed42c9b3
diff --git a/src/cdp/CDP.zig b/src/cdp/CDP.zigindex 593f6c732..b2e29c49b 100644--- a/src/cdp/CDP.zig+++ b/src/cdp/CDP.zig@@ -39,6 +39,7 @@ const Label = @import("../browser/webapi/element/html/Label.zig");const Connection = @import("Connection.zig");const Incrementing = @import("id.zig").Incrementing;+const network_domain = @import("domains/network.zig");const InterceptState = @import("domains/fetch.zig").InterceptState;const log = lp.log;@@ -420,7 +421,7 @@ fn dispatchCommand(command: *Command, method: []const u8) !void {7 => switch (@as(u56, @bitCast(domain[0..7].*))) {asUint(u56, "Browser") => return @import("domains/browser.zig").processMessage(command),asUint(u56, "Runtime") => return @import("domains/runtime.zig").processMessage(command),- asUint(u56, "Network") => return @import("domains/network.zig").processMessage(command),+ asUint(u56, "Network") => return network_domain.processMessage(command),asUint(u56, "Storage") => return @import("domains/storage.zig").processMessage(command),asUint(u56, "Console") => return @import("domains/console.zig").processMessage(command),else => {},@@ -515,7 +516,9 @@ pub const BrowserContext = struct {const CapturedResponse = struct {must_encode: bool,- data: std.ArrayList(u8),+ // null once evicted (size limit). We keep the map entry so that we+ // can give an "evicted" error instead of "unknown"+ data: ?std.ArrayList(u8),};// Key for `captured_responses` / `captured_requests`. Documents are@@ -616,8 +619,10 @@ pub const BrowserContext = struct {// memory longer than they have to. In fact, the main request is only// ever streamed. So if CDP is the only thing that needs bodies in// memory for an arbitrary amount of time, then that's where we're going- // to store the,- captured_responses: std.AutoHashMapUnmanaged(CapturedKey, CapturedResponse),+ // to store them.+ captured_responses_size: usize = 0,+ captured_responses: std.AutoArrayHashMapUnmanaged(CapturedKey, CapturedResponse),+ network_limits: network_domain.BufferLimits = .{},notification: *Notification,@@ -723,6 +728,7 @@ pub const BrowserContext = struct {// rely on those notifications to do our normal cleanup?)self.notification.unregisterAll(self);+ self.clearCapturedResponses();// If the session has a frame, we need to clear it first. The page// context is always nested inside of the isolated world context,@@ -852,7 +858,22 @@ pub const BrowserContext = struct {};}- pub fn networkEnable(self: *BrowserContext) !void {+ pub fn networkEnable(self: *BrowserContext, limits: network_domain.BufferLimits) !void {+ const previous = self.network_limits;+ self.network_limits = limits;+ if (limits.resource < previous.resource) {+ // evict responses which are over the new small limit+ for (self.captured_responses.values()) |*resp| {+ const data = resp.data orelse continue;+ if (data.items.len > limits.resource) {+ self.evictCapturedResponse(resp);+ }+ }+ }+ if (limits.total < previous.total) {+ self.ensureCapturedSpace(0);+ }+try self.notification.register(.http_request_fail, self, onHttpRequestFail);try self.notification.register(.http_request_start, self, onHttpRequestStart);try self.notification.register(.http_request_done, self, onHttpRequestDone);@@ -868,6 +889,41 @@ pub const BrowserContext = struct {self.notification.unregister(.http_response_data, self);self.notification.unregister(.http_response_header_done, self);self.notification.unregister(.http_request_served_from_cache, self);+ self.clearCapturedResponses();+ }++ pub fn clearCapturedResponses(self: *BrowserContext) void {+ const allocator = self.cdp.allocator;+ for (self.captured_responses.values()) |*resp| {+ if (resp.data) |*data| {+ data.deinit(allocator);+ }+ }+ self.captured_responses.deinit(allocator);+ self.captured_responses = .empty;+ self.captured_responses_size = 0;+ }++ fn evictCapturedResponse(self: *BrowserContext, resp: *CapturedResponse) void {+ if (resp.data) |*data| {+ self.captured_responses_size -= data.items.len;+ data.deinit(self.cdp.allocator);+ resp.data = null;+ }+ }++ // Evict captured responses, ordered by age, until we can fit `size` more bytes+ fn ensureCapturedSpace(self: *BrowserContext, size: usize) void {+ for (self.captured_responses.values()) |*resp| {+ if (self.captured_responses_size + size <= self.network_limits.total) {+ return;+ }+ const data = resp.data orelse continue;+ if (data.items.len == 0) {+ continue;+ }+ self.evictCapturedResponse(resp);+ }}pub fn fetchEnable(self: *BrowserContext, authRequests: bool, session_id: []const u8) !void {@@ -1015,7 +1071,7 @@ pub const BrowserContext = struct {const transfer = msg.transfer;const key = keyFromTransfer(transfer);const body = transfer.req.body orelse "";- if (body.len == 0 or body.len > @import("domains/network.zig").max_post_data_size) {+ if (body.len == 0 or body.len > network_domain.max_post_data_size) {_ = self.captured_requests.remove(key);} else {const owned_body = try self.frame_arena.dupe(u8, body);@@ -1023,7 +1079,7 @@ pub const BrowserContext = struct {}}defer self.resetNotificationArena();- try @import("domains/network.zig").httpRequestStart(self.notification_arena, self, msg);+ try network_domain.httpRequestStart(self.notification_arena, self, msg);}pub fn onHttpRequestIntercept(ctx: *anyopaque, msg: *const Notification.RequestIntercept) !void {@@ -1034,7 +1090,7 @@ pub const BrowserContext = struct {pub fn onHttpRequestFail(ctx: *anyopaque, msg: *const Notification.RequestFail) !void {const self: *BrowserContext = @ptrCast(@alignCast(ctx));- return @import("domains/network.zig").httpRequestFail(self, msg);+ return network_domain.httpRequestFail(self, msg);}pub fn onFrameDOMContentLoaded(ctx: *anyopaque, msg: *const Notification.FrameDOMContentLoaded) !void {@@ -1063,14 +1119,18 @@ pub const BrowserContext = struct {const self: *BrowserContext = @ptrCast(@alignCast(ctx));defer self.resetNotificationArena();- const arena = self.frame_arena;-// Prepare the captured response value.const key = keyFromTransfer(msg.transfer);- const gop = try self.captured_responses.getOrPut(arena, key);+ const gop = try self.captured_responses.getOrPut(self.cdp.allocator, key);if (!gop.found_existing) {gop.value_ptr.* = .{- .data = .empty,+ .data = blk: {+ const cl = msg.transfer.getContentLength() orelse break :blk .empty;+ if (cl > self.network_limits.resource) {+ break :blk null;+ }+ break :blk try std.ArrayList(u8).initCapacity(self.cdp.allocator, cl);+ },// Encode the data in base64 by default, but don't encode// for well known content-type..must_encode = blk: {@@ -1081,7 +1141,7 @@ pub const BrowserContext = struct {break :blk true;}- if (std.mem.eql(u8, "UTF-8", mime.charsetString())) {+ if (std.ascii.eqlIgnoreCase("UTF-8", mime.charsetString())) {break :blk false;}}@@ -1090,22 +1150,33 @@ pub const BrowserContext = struct {};}- return @import("domains/network.zig").httpResponseHeaderDone(self.notification_arena, self, msg);+ return network_domain.httpResponseHeaderDone(self.notification_arena, self, msg);}pub fn onHttpRequestDone(ctx: *anyopaque, msg: *const Notification.RequestDone) !void {const self: *BrowserContext = @ptrCast(@alignCast(ctx));- return @import("domains/network.zig").httpRequestDone(self, msg);+ return network_domain.httpRequestDone(self, msg);}pub fn onHttpResponseData(ctx: *anyopaque, msg: *const Notification.ResponseData) !void {const self: *BrowserContext = @ptrCast(@alignCast(ctx));- const arena = self.frame_arena;const key = keyFromTransfer(msg.transfer);const resp = self.captured_responses.getPtr(key) orelse lp.assert(false, "onHttpResponseData missing captured response", .{});- return resp.data.appendSlice(arena, msg.data);+ const data = &(resp.data orelse return);+ const chunk = msg.data;+ const limits = &self.network_limits;+ if (data.items.len + chunk.len > limits.resource or chunk.len > limits.total) {+ return self.evictCapturedResponse(resp);+ }+ // Can evict `resp` itself when it's the oldest, hence the re-check.+ self.ensureCapturedSpace(chunk.len);+ if (resp.data == null) {+ return;+ }+ try data.appendSlice(self.cdp.allocator, chunk);+ self.captured_responses_size += chunk.len;}pub fn onHttpRequestAuthRequired(ctx: *anyopaque, data: *const Notification.RequestAuthRequired) !void {@@ -1116,7 +1187,7 @@ pub const BrowserContext = struct {pub fn onHttpRequestServedFromCache(ctx: *anyopaque, msg: *const Notification.RequestServedFromCache) !void {const self: *BrowserContext = @ptrCast(@alignCast(ctx));- return @import("domains/network.zig").httpServedFromCache(self, msg);+ return network_domain.httpServedFromCache(self, msg);}pub fn onConsoleMessage(ctx: *anyopaque, msg: *const Notification.ConsoleMessage) !void {diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zigindex 733dd8b77..b5a7179c1 100644--- a/src/cdp/domains/fetch.zig+++ b/src/cdp/domains/fetch.zig@@ -209,7 +209,7 @@ pub fn requestIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *ctry bc.cdp.sendEvent("Fetch.requestPaused", .{.requestId = &id.toInterceptId(intercept_id),.frameId = &id.toFrameId(transfer.req.frame_id),- .request = network.RequestWriter.init(arena, transfer),+ .request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data),.resourceType = transfer.req.resource_type.string(),.networkId = &id.toRequestId(transfer), // matches the Network REQ-ID}, .{ .session_id = session_id });@@ -435,7 +435,7 @@ pub fn requestAuthRequired(arena: Allocator, bc: *CDP.BrowserContext, intercept:try bc.cdp.sendEvent("Fetch.authRequired", .{.requestId = &id.toInterceptId(intercept_id),.frameId = &id.toFrameId(request.frame_id),- .request = network.RequestWriter.init(arena, transfer),+ .request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data),.resourceType = request.resource_type.string(),.authChallenge = .{.origin = "", // TODO get origin, could be the proxy address for example.diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zigindex 9271ee9ce..113e6de0e 100644--- a/src/cdp/domains/network.zig+++ b/src/cdp/domains/network.zig@@ -39,6 +39,15 @@ const log = lp.log;const Allocator = std.mem.Allocator;pub const max_post_data_size = 64 * 1024;+// Network.enable's buffer parameters.+pub const BufferLimits = struct {+ // total bytes that we'll capture before evicting older entries+ total: usize = 200 * 1000 * 1000,+ // max bytes-per-capture that we'll retain+ resource: usize = 20 * 1000 * 1000,+ post_data: usize = max_post_data_size,+};+pub fn processMessage(cmd: *CDP.Command) !void {const action = std.meta.stringToEnum(enum {enable,@@ -103,8 +112,26 @@ fn emulateNetworkConditions(cmd: *CDP.Command) !void {}fn enable(cmd: *CDP.Command) !void {+ const Params = struct {+ maxTotalBufferSize: ?u32 = null,+ maxResourceBufferSize: ?u32 = null,+ maxPostDataSize: ?u32 = null,+ };+ const params = (try cmd.params(Params)) orelse Params{};const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;- try bc.networkEnable();++ var limits: BufferLimits = .{};+ if (params.maxTotalBufferSize) |max| {+ limits.total = max;+ }+ if (params.maxResourceBufferSize) |max| {+ limits.resource = max;+ }+ if (params.maxPostDataSize) |max| {+ // 0 is Chrome's "no limit".+ limits.post_data = if (max == 0) std.math.maxInt(usize) else max;+ }+ try bc.networkEnable(limits);return cmd.sendResult(null, .{});}@@ -325,19 +352,22 @@ fn getResponseBody(cmd: *CDP.Command) !void {const key = try keyFromRequestId(params.requestId);const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;const resp = bc.captured_responses.getPtr(key) orelse return error.RequestNotFound;+ const data = resp.data orelse {+ return cmd.sendError(-32000, "Request content was evicted from inspector cache", .{});+ };// must_encode trusts the declared charset; a server can declare UTF-8 and// still send invalid bytes.- if (!resp.must_encode and std.unicode.utf8ValidateSlice(resp.data.items)) {+ if (!resp.must_encode and std.unicode.utf8ValidateSlice(data.items)) {return cmd.sendResult(.{- .body = resp.data.items,+ .body = data.items,.base64Encoded = false,}, .{});}- const encoded_len = std.base64.standard.Encoder.calcSize(resp.data.items.len);+ const encoded_len = std.base64.standard.Encoder.calcSize(data.items.len);const encoded = try cmd.arena.alloc(u8, encoded_len);- _ = std.base64.standard.Encoder.encode(encoded, resp.data.items);+ _ = std.base64.standard.Encoder.encode(encoded, data.items);return cmd.sendResult(.{.body = encoded,@@ -403,7 +433,7 @@ pub fn httpRequestStart(arena: Allocator, bc: *CDP.BrowserContext, msg: *const N.loaderId = &id.toLoaderId(req.loader_id),.type = req.resource_type.string(),.documentURL = frame.url,- .request = RequestWriter.init(arena, transfer),+ .request = RequestWriter.init(arena, transfer, bc.network_limits.post_data),.initiator = .{ .type = "other" },.redirectResponse = if (msg.redirect_response)ResponseWriter.init(arena, transfer)@@ -459,11 +489,13 @@ pub fn httpServedFromCache(bc: *CDP.BrowserContext, msg: *const Notification.Reqpub const RequestWriter = struct {arena: Allocator,transfer: *Transfer,+ max_post_data: usize,- pub fn init(arena: Allocator, transfer: *Transfer) RequestWriter {+ pub fn init(arena: Allocator, transfer: *Transfer, max_post_data: usize) RequestWriter {return .{.arena = arena,.transfer = transfer,+ .max_post_data = max_post_data,};}@@ -500,7 +532,7 @@ pub const RequestWriter = struct {}if (request.body) |body| {- if (body.len <= max_post_data_size) {+ if (body.len <= self.max_post_data) {try jws.objectField("postData");try jws.write(SafeString.wrap(body));@@ -1242,6 +1274,190 @@ test "cdp.Network: POST body exposed as postData" {try ctx.expectSentError(-31998, "RequestNotFound", .{ .id = 3 });}+// Drives POST /echo_body (the response echoes the body, so its size is the+// request body's) to completion and returns the wire requestId.+const EchoDriver = struct {+ done: bool = false,+ err: ?anyerror = null,++ fn doneCallback(raw: *anyopaque) !void {+ const self: *EchoDriver = @ptrCast(@alignCast(raw));+ self.done = true;+ }++ fn errorCallback(raw: *anyopaque, err: anyerror) void {+ const self: *EchoDriver = @ptrCast(@alignCast(raw));+ self.err = err;+ }++ fn run(bc: *CDP.BrowserContext, frame_id: u32, body: []const u8) ![14]u8 {+ const client = &bc.cdp.browser.http_client;+ var request_id: [14]u8 = undefined;+ _ = std.fmt.bufPrint(&request_id, "REQ-{d:0>10}", .{client.next_request_id +% 1}) catch unreachable;++ var driver: EchoDriver = .{};+ try client.request(.{+ .frame_id = frame_id,+ .loader_id = 1,+ .method = .POST,+ .url = "http://127.0.0.1:9582/echo_body",+ .body = body,+ .cookie_jar = null,+ .cookie_origin = "http://127.0.0.1:9582/",+ .resource_type = .fetch,+ .notification = bc.session.notification,+ .ctx = &driver,+ .done_callback = doneCallback,+ .error_callback = errorCallback,+ .shutdown_callback = HttpClient.noopShutdown,+ }, null);++ for (0..50) |_| {+ if (driver.done or driver.err != null) break;+ _ = try client.tick(20);+ }+ try testing.expectEqual(null, driver.err);+ try testing.expect(driver.done);+ return request_id;+ }+};++test "cdp.Network: enable maxResourceBufferSize evicts oversized bodies" {+ var ctx = try testing.context();+ defer ctx.deinit();++ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RBS", .session_id = "SID-RBS" });+ const page = try bc.session.createPage();++ try ctx.processMessage(.{+ .id = 1,+ .method = "Network.enable",+ .params = .{ .maxResourceBufferSize = 4 },+ });+ try ctx.expectSentResult(null, .{ .id = 1 });++ const big = try EchoDriver.run(bc, page.frame_id, "12345678");+ const small = try EchoDriver.run(bc, page.frame_id, "123");++ try ctx.processMessage(.{+ .id = 2,+ .method = "Network.getResponseBody",+ .params = .{ .requestId = &big },+ });+ try ctx.expectSentError(-32000, "Request content was evicted from inspector cache", .{ .id = 2 });++ try ctx.processMessage(.{+ .id = 3,+ .method = "Network.getResponseBody",+ .params = .{ .requestId = &small },+ });+ try ctx.expectSentResult(.{ .body = "123", .base64Encoded = false }, .{ .id = 3 });+ try testing.expectEqual(3, bc.captured_responses_size);++ // Re-enabling with a tighter limit evicts what's already captured.+ try ctx.processMessage(.{+ .id = 4,+ .method = "Network.enable",+ .params = .{ .maxResourceBufferSize = 2 },+ });+ try ctx.expectSentResult(null, .{ .id = 4 });+ try ctx.processMessage(.{+ .id = 5,+ .method = "Network.getResponseBody",+ .params = .{ .requestId = &small },+ });+ try ctx.expectSentError(-32000, "Request content was evicted from inspector cache", .{ .id = 5 });+ try testing.expectEqual(0, bc.captured_responses_size);+}++test "cdp.Network: enable maxTotalBufferSize evicts oldest bodies first" {+ var ctx = try testing.context();+ defer ctx.deinit();++ const bc = try ctx.loadBrowserContext(.{ .id = "BID-TBS", .session_id = "SID-TBS" });+ const page = try bc.session.createPage();++ try ctx.processMessage(.{+ .id = 1,+ .method = "Network.enable",+ .params = .{ .maxTotalBufferSize = 10 },+ });+ try ctx.expectSentResult(null, .{ .id = 1 });++ const first = try EchoDriver.run(bc, page.frame_id, "aaaaaa");+ const second = try EchoDriver.run(bc, page.frame_id, "bbbbbb");++ try ctx.processMessage(.{+ .id = 2,+ .method = "Network.getResponseBody",+ .params = .{ .requestId = &first },+ });+ try ctx.expectSentError(-32000, "Request content was evicted from inspector cache", .{ .id = 2 });++ try ctx.processMessage(.{+ .id = 3,+ .method = "Network.getResponseBody",+ .params = .{ .requestId = &second },+ });+ try ctx.expectSentResult(.{ .body = "bbbbbb", .base64Encoded = false }, .{ .id = 3 });+ try testing.expectEqual(6, bc.captured_responses_size);++ // Network.disable releases everything retained.+ try ctx.processMessage(.{ .id = 4, .method = "Network.disable" });+ try ctx.expectSentResult(null, .{ .id = 4 });+ try testing.expectEqual(0, bc.captured_responses.count());+ try testing.expectEqual(0, bc.captured_responses_size);+ try ctx.processMessage(.{+ .id = 5,+ .method = "Network.getResponseBody",+ .params = .{ .requestId = &second },+ });+ try ctx.expectSentError(-31998, "RequestNotFound", .{ .id = 5 });+}++test "cdp.Network: enable maxPostDataSize omits inline postData" {+ var ctx = try testing.context();+ defer ctx.deinit();++ const bc = try ctx.loadBrowserContext(.{ .id = "BID-PDS", .session_id = "SID-PDS" });+ const page = try bc.session.createPage();++ try ctx.processMessage(.{+ .id = 1,+ .method = "Network.enable",+ .params = .{ .maxPostDataSize = 8 },+ });+ try ctx.expectSentResult(null, .{ .id = 1 });++ const body = "{\"source\":\"xhr\",\"pageSize\":100}";+ const request_id = try EchoDriver.run(bc, page.frame_id, body);++ try ctx.expectSentEvent("Network.requestWillBeSent", .{+ .requestId = &request_id,+ .request = .{ .method = "POST", .hasPostData = true },+ }, .{ .session_id = "SID-PDS" });++ // The subset matcher can't assert absence; look at the event directly.+ var seen = false;+ for (ctx.received.items) |received| {+ const method = received.object.get("method") orelse continue;+ if (!std.mem.eql(u8, method.string, "Network.requestWillBeSent")) continue;+ const request = received.object.get("params").?.object.get("request").?.object;+ try testing.expectEqual(null, request.get("postData"));+ try testing.expectEqual(null, request.get("postDataEntries"));+ seen = true;+ }+ try testing.expect(seen);++ // Retention is independent of the inline limit.+ try ctx.processMessage(.{+ .id = 2,+ .method = "Network.getRequestPostData",+ .params = .{ .requestId = &request_id },+ });+ try ctx.expectSentResult(.{ .postData = body }, .{ .id = 2 });+}+test "cdp.Network: redirect hop precedes Fetch pause and carries redirectResponse" {var ctx = try testing.context();defer ctx.deinit();diff --git a/src/cdp/domains/page.zig b/src/cdp/domains/page.zigindex b65960baf..248bc2972 100644--- a/src/cdp/domains/page.zig+++ b/src/cdp/domains/page.zig@@ -551,7 +551,7 @@ pub fn frameCreated(bc: *CDP.BrowserContext, frame: *Frame) !void {// controlled via Network.configureDurableMessages (which we don't// support).bc.captured_requests = .empty;- bc.captured_responses = .empty;+ bc.clearCapturedResponses();}}