Skip to content

Lightpanda: the headless browser designed for AI and automation

CDP: enforce Network.enable limits

Karl Seguindad2ed42c9b3
diff --git a/src/cdp/CDP.zig b/src/cdp/CDP.zig
index 593f6c732..b2e29c49b 100644
--- a/src/cdp/CDP.zig
+++ b/src/cdp/CDP.zig
@@ -39,6 +39,7 @@ const Label = @import("../browser/webapi/element/html/Label.zig");
const Connection = @import("Connection.zig");
const Incrementing = @import("id.zig").Incrementing;
+const network_domain = @import("domains/network.zig");
const InterceptState = @import("domains/fetch.zig").InterceptState;
const log = lp.log;
@@ -420,7 +421,7 @@ fn dispatchCommand(command: *Command, method: []const u8) !void {
7 => switch (@as(u56, @bitCast(domain[0..7].*))) {
asUint(u56, "Browser") => return @import("domains/browser.zig").processMessage(command),
asUint(u56, "Runtime") => return @import("domains/runtime.zig").processMessage(command),
- asUint(u56, "Network") => return @import("domains/network.zig").processMessage(command),
+ asUint(u56, "Network") => return network_domain.processMessage(command),
asUint(u56, "Storage") => return @import("domains/storage.zig").processMessage(command),
asUint(u56, "Console") => return @import("domains/console.zig").processMessage(command),
else => {},
@@ -515,7 +516,9 @@ pub const BrowserContext = struct {
const CapturedResponse = struct {
must_encode: bool,
- data: std.ArrayList(u8),
+ // null once evicted (size limit). We keep the map entry so that we
+ // can give an "evicted" error instead of "unknown"
+ data: ?std.ArrayList(u8),
};
// Key for `captured_responses` / `captured_requests`. Documents are
@@ -616,8 +619,10 @@ pub const BrowserContext = struct {
// memory longer than they have to. In fact, the main request is only
// ever streamed. So if CDP is the only thing that needs bodies in
// memory for an arbitrary amount of time, then that's where we're going
- // to store the,
- captured_responses: std.AutoHashMapUnmanaged(CapturedKey, CapturedResponse),
+ // to store them.
+ captured_responses_size: usize = 0,
+ captured_responses: std.AutoArrayHashMapUnmanaged(CapturedKey, CapturedResponse),
+ network_limits: network_domain.BufferLimits = .{},
notification: *Notification,
@@ -723,6 +728,7 @@ pub const BrowserContext = struct {
// rely on those notifications to do our normal cleanup?)
self.notification.unregisterAll(self);
+ self.clearCapturedResponses();
// If the session has a frame, we need to clear it first. The page
// context is always nested inside of the isolated world context,
@@ -852,7 +858,22 @@ pub const BrowserContext = struct {
};
}
- pub fn networkEnable(self: *BrowserContext) !void {
+ pub fn networkEnable(self: *BrowserContext, limits: network_domain.BufferLimits) !void {
+ const previous = self.network_limits;
+ self.network_limits = limits;
+ if (limits.resource < previous.resource) {
+ // evict responses which are over the new small limit
+ for (self.captured_responses.values()) |*resp| {
+ const data = resp.data orelse continue;
+ if (data.items.len > limits.resource) {
+ self.evictCapturedResponse(resp);
+ }
+ }
+ }
+ if (limits.total < previous.total) {
+ self.ensureCapturedSpace(0);
+ }
+
try self.notification.register(.http_request_fail, self, onHttpRequestFail);
try self.notification.register(.http_request_start, self, onHttpRequestStart);
try self.notification.register(.http_request_done, self, onHttpRequestDone);
@@ -868,6 +889,41 @@ pub const BrowserContext = struct {
self.notification.unregister(.http_response_data, self);
self.notification.unregister(.http_response_header_done, self);
self.notification.unregister(.http_request_served_from_cache, self);
+ self.clearCapturedResponses();
+ }
+
+ pub fn clearCapturedResponses(self: *BrowserContext) void {
+ const allocator = self.cdp.allocator;
+ for (self.captured_responses.values()) |*resp| {
+ if (resp.data) |*data| {
+ data.deinit(allocator);
+ }
+ }
+ self.captured_responses.deinit(allocator);
+ self.captured_responses = .empty;
+ self.captured_responses_size = 0;
+ }
+
+ fn evictCapturedResponse(self: *BrowserContext, resp: *CapturedResponse) void {
+ if (resp.data) |*data| {
+ self.captured_responses_size -= data.items.len;
+ data.deinit(self.cdp.allocator);
+ resp.data = null;
+ }
+ }
+
+ // Evict captured responses, ordered by age, until we can fit `size` more bytes
+ fn ensureCapturedSpace(self: *BrowserContext, size: usize) void {
+ for (self.captured_responses.values()) |*resp| {
+ if (self.captured_responses_size + size <= self.network_limits.total) {
+ return;
+ }
+ const data = resp.data orelse continue;
+ if (data.items.len == 0) {
+ continue;
+ }
+ self.evictCapturedResponse(resp);
+ }
}
pub fn fetchEnable(self: *BrowserContext, authRequests: bool, session_id: []const u8) !void {
@@ -1015,7 +1071,7 @@ pub const BrowserContext = struct {
const transfer = msg.transfer;
const key = keyFromTransfer(transfer);
const body = transfer.req.body orelse "";
- if (body.len == 0 or body.len > @import("domains/network.zig").max_post_data_size) {
+ if (body.len == 0 or body.len > network_domain.max_post_data_size) {
_ = self.captured_requests.remove(key);
} else {
const owned_body = try self.frame_arena.dupe(u8, body);
@@ -1023,7 +1079,7 @@ pub const BrowserContext = struct {
}
}
defer self.resetNotificationArena();
- try @import("domains/network.zig").httpRequestStart(self.notification_arena, self, msg);
+ try network_domain.httpRequestStart(self.notification_arena, self, msg);
}
pub fn onHttpRequestIntercept(ctx: *anyopaque, msg: *const Notification.RequestIntercept) !void {
@@ -1034,7 +1090,7 @@ pub const BrowserContext = struct {
pub fn onHttpRequestFail(ctx: *anyopaque, msg: *const Notification.RequestFail) !void {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
- return @import("domains/network.zig").httpRequestFail(self, msg);
+ return network_domain.httpRequestFail(self, msg);
}
pub fn onFrameDOMContentLoaded(ctx: *anyopaque, msg: *const Notification.FrameDOMContentLoaded) !void {
@@ -1063,14 +1119,18 @@ pub const BrowserContext = struct {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
defer self.resetNotificationArena();
- const arena = self.frame_arena;
-
// Prepare the captured response value.
const key = keyFromTransfer(msg.transfer);
- const gop = try self.captured_responses.getOrPut(arena, key);
+ const gop = try self.captured_responses.getOrPut(self.cdp.allocator, key);
if (!gop.found_existing) {
gop.value_ptr.* = .{
- .data = .empty,
+ .data = blk: {
+ const cl = msg.transfer.getContentLength() orelse break :blk .empty;
+ if (cl > self.network_limits.resource) {
+ break :blk null;
+ }
+ break :blk try std.ArrayList(u8).initCapacity(self.cdp.allocator, cl);
+ },
// Encode the data in base64 by default, but don't encode
// for well known content-type.
.must_encode = blk: {
@@ -1081,7 +1141,7 @@ pub const BrowserContext = struct {
break :blk true;
}
- if (std.mem.eql(u8, "UTF-8", mime.charsetString())) {
+ if (std.ascii.eqlIgnoreCase("UTF-8", mime.charsetString())) {
break :blk false;
}
}
@@ -1090,22 +1150,33 @@ pub const BrowserContext = struct {
};
}
- return @import("domains/network.zig").httpResponseHeaderDone(self.notification_arena, self, msg);
+ return network_domain.httpResponseHeaderDone(self.notification_arena, self, msg);
}
pub fn onHttpRequestDone(ctx: *anyopaque, msg: *const Notification.RequestDone) !void {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
- return @import("domains/network.zig").httpRequestDone(self, msg);
+ return network_domain.httpRequestDone(self, msg);
}
pub fn onHttpResponseData(ctx: *anyopaque, msg: *const Notification.ResponseData) !void {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
- const arena = self.frame_arena;
const key = keyFromTransfer(msg.transfer);
const resp = self.captured_responses.getPtr(key) orelse lp.assert(false, "onHttpResponseData missing captured response", .{});
- return resp.data.appendSlice(arena, msg.data);
+ const data = &(resp.data orelse return);
+ const chunk = msg.data;
+ const limits = &self.network_limits;
+ if (data.items.len + chunk.len > limits.resource or chunk.len > limits.total) {
+ return self.evictCapturedResponse(resp);
+ }
+ // Can evict `resp` itself when it's the oldest, hence the re-check.
+ self.ensureCapturedSpace(chunk.len);
+ if (resp.data == null) {
+ return;
+ }
+ try data.appendSlice(self.cdp.allocator, chunk);
+ self.captured_responses_size += chunk.len;
}
pub fn onHttpRequestAuthRequired(ctx: *anyopaque, data: *const Notification.RequestAuthRequired) !void {
@@ -1116,7 +1187,7 @@ pub const BrowserContext = struct {
pub fn onHttpRequestServedFromCache(ctx: *anyopaque, msg: *const Notification.RequestServedFromCache) !void {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
- return @import("domains/network.zig").httpServedFromCache(self, msg);
+ return network_domain.httpServedFromCache(self, msg);
}
pub fn onConsoleMessage(ctx: *anyopaque, msg: *const Notification.ConsoleMessage) !void {
diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zig
index 733dd8b77..b5a7179c1 100644
--- a/src/cdp/domains/fetch.zig
+++ b/src/cdp/domains/fetch.zig
@@ -209,7 +209,7 @@ pub fn requestIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *c
try bc.cdp.sendEvent("Fetch.requestPaused", .{
.requestId = &id.toInterceptId(intercept_id),
.frameId = &id.toFrameId(transfer.req.frame_id),
- .request = network.RequestWriter.init(arena, transfer),
+ .request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data),
.resourceType = transfer.req.resource_type.string(),
.networkId = &id.toRequestId(transfer), // matches the Network REQ-ID
}, .{ .session_id = session_id });
@@ -435,7 +435,7 @@ pub fn requestAuthRequired(arena: Allocator, bc: *CDP.BrowserContext, intercept:
try bc.cdp.sendEvent("Fetch.authRequired", .{
.requestId = &id.toInterceptId(intercept_id),
.frameId = &id.toFrameId(request.frame_id),
- .request = network.RequestWriter.init(arena, transfer),
+ .request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data),
.resourceType = request.resource_type.string(),
.authChallenge = .{
.origin = "", // TODO get origin, could be the proxy address for example.
diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig
index 9271ee9ce..113e6de0e 100644
--- a/src/cdp/domains/network.zig
+++ b/src/cdp/domains/network.zig
@@ -39,6 +39,15 @@ const log = lp.log;
const Allocator = std.mem.Allocator;
pub const max_post_data_size = 64 * 1024;
+// Network.enable's buffer parameters.
+pub const BufferLimits = struct {
+ // total bytes that we'll capture before evicting older entries
+ total: usize = 200 * 1000 * 1000,
+ // max bytes-per-capture that we'll retain
+ resource: usize = 20 * 1000 * 1000,
+ post_data: usize = max_post_data_size,
+};
+
pub fn processMessage(cmd: *CDP.Command) !void {
const action = std.meta.stringToEnum(enum {
enable,
@@ -103,8 +112,26 @@ fn emulateNetworkConditions(cmd: *CDP.Command) !void {
}
fn enable(cmd: *CDP.Command) !void {
+ const Params = struct {
+ maxTotalBufferSize: ?u32 = null,
+ maxResourceBufferSize: ?u32 = null,
+ maxPostDataSize: ?u32 = null,
+ };
+ const params = (try cmd.params(Params)) orelse Params{};
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
- try bc.networkEnable();
+
+ var limits: BufferLimits = .{};
+ if (params.maxTotalBufferSize) |max| {
+ limits.total = max;
+ }
+ if (params.maxResourceBufferSize) |max| {
+ limits.resource = max;
+ }
+ if (params.maxPostDataSize) |max| {
+ // 0 is Chrome's "no limit".
+ limits.post_data = if (max == 0) std.math.maxInt(usize) else max;
+ }
+ try bc.networkEnable(limits);
return cmd.sendResult(null, .{});
}
@@ -325,19 +352,22 @@ fn getResponseBody(cmd: *CDP.Command) !void {
const key = try keyFromRequestId(params.requestId);
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
const resp = bc.captured_responses.getPtr(key) orelse return error.RequestNotFound;
+ const data = resp.data orelse {
+ return cmd.sendError(-32000, "Request content was evicted from inspector cache", .{});
+ };
// must_encode trusts the declared charset; a server can declare UTF-8 and
// still send invalid bytes.
- if (!resp.must_encode and std.unicode.utf8ValidateSlice(resp.data.items)) {
+ if (!resp.must_encode and std.unicode.utf8ValidateSlice(data.items)) {
return cmd.sendResult(.{
- .body = resp.data.items,
+ .body = data.items,
.base64Encoded = false,
}, .{});
}
- const encoded_len = std.base64.standard.Encoder.calcSize(resp.data.items.len);
+ const encoded_len = std.base64.standard.Encoder.calcSize(data.items.len);
const encoded = try cmd.arena.alloc(u8, encoded_len);
- _ = std.base64.standard.Encoder.encode(encoded, resp.data.items);
+ _ = std.base64.standard.Encoder.encode(encoded, data.items);
return cmd.sendResult(.{
.body = encoded,
@@ -403,7 +433,7 @@ pub fn httpRequestStart(arena: Allocator, bc: *CDP.BrowserContext, msg: *const N
.loaderId = &id.toLoaderId(req.loader_id),
.type = req.resource_type.string(),
.documentURL = frame.url,
- .request = RequestWriter.init(arena, transfer),
+ .request = RequestWriter.init(arena, transfer, bc.network_limits.post_data),
.initiator = .{ .type = "other" },
.redirectResponse = if (msg.redirect_response)
ResponseWriter.init(arena, transfer)
@@ -459,11 +489,13 @@ pub fn httpServedFromCache(bc: *CDP.BrowserContext, msg: *const Notification.Req
pub const RequestWriter = struct {
arena: Allocator,
transfer: *Transfer,
+ max_post_data: usize,
- pub fn init(arena: Allocator, transfer: *Transfer) RequestWriter {
+ pub fn init(arena: Allocator, transfer: *Transfer, max_post_data: usize) RequestWriter {
return .{
.arena = arena,
.transfer = transfer,
+ .max_post_data = max_post_data,
};
}
@@ -500,7 +532,7 @@ pub const RequestWriter = struct {
}
if (request.body) |body| {
- if (body.len <= max_post_data_size) {
+ if (body.len <= self.max_post_data) {
try jws.objectField("postData");
try jws.write(SafeString.wrap(body));
@@ -1242,6 +1274,190 @@ test "cdp.Network: POST body exposed as postData" {
try ctx.expectSentError(-31998, "RequestNotFound", .{ .id = 3 });
}
+// Drives POST /echo_body (the response echoes the body, so its size is the
+// request body's) to completion and returns the wire requestId.
+const EchoDriver = struct {
+ done: bool = false,
+ err: ?anyerror = null,
+
+ fn doneCallback(raw: *anyopaque) !void {
+ const self: *EchoDriver = @ptrCast(@alignCast(raw));
+ self.done = true;
+ }
+
+ fn errorCallback(raw: *anyopaque, err: anyerror) void {
+ const self: *EchoDriver = @ptrCast(@alignCast(raw));
+ self.err = err;
+ }
+
+ fn run(bc: *CDP.BrowserContext, frame_id: u32, body: []const u8) ![14]u8 {
+ const client = &bc.cdp.browser.http_client;
+ var request_id: [14]u8 = undefined;
+ _ = std.fmt.bufPrint(&request_id, "REQ-{d:0>10}", .{client.next_request_id +% 1}) catch unreachable;
+
+ var driver: EchoDriver = .{};
+ try client.request(.{
+ .frame_id = frame_id,
+ .loader_id = 1,
+ .method = .POST,
+ .url = "http://127.0.0.1:9582/echo_body",
+ .body = body,
+ .cookie_jar = null,
+ .cookie_origin = "http://127.0.0.1:9582/",
+ .resource_type = .fetch,
+ .notification = bc.session.notification,
+ .ctx = &driver,
+ .done_callback = doneCallback,
+ .error_callback = errorCallback,
+ .shutdown_callback = HttpClient.noopShutdown,
+ }, null);
+
+ for (0..50) |_| {
+ if (driver.done or driver.err != null) break;
+ _ = try client.tick(20);
+ }
+ try testing.expectEqual(null, driver.err);
+ try testing.expect(driver.done);
+ return request_id;
+ }
+};
+
+test "cdp.Network: enable maxResourceBufferSize evicts oversized bodies" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-RBS", .session_id = "SID-RBS" });
+ const page = try bc.session.createPage();
+
+ try ctx.processMessage(.{
+ .id = 1,
+ .method = "Network.enable",
+ .params = .{ .maxResourceBufferSize = 4 },
+ });
+ try ctx.expectSentResult(null, .{ .id = 1 });
+
+ const big = try EchoDriver.run(bc, page.frame_id, "12345678");
+ const small = try EchoDriver.run(bc, page.frame_id, "123");
+
+ try ctx.processMessage(.{
+ .id = 2,
+ .method = "Network.getResponseBody",
+ .params = .{ .requestId = &big },
+ });
+ try ctx.expectSentError(-32000, "Request content was evicted from inspector cache", .{ .id = 2 });
+
+ try ctx.processMessage(.{
+ .id = 3,
+ .method = "Network.getResponseBody",
+ .params = .{ .requestId = &small },
+ });
+ try ctx.expectSentResult(.{ .body = "123", .base64Encoded = false }, .{ .id = 3 });
+ try testing.expectEqual(3, bc.captured_responses_size);
+
+ // Re-enabling with a tighter limit evicts what's already captured.
+ try ctx.processMessage(.{
+ .id = 4,
+ .method = "Network.enable",
+ .params = .{ .maxResourceBufferSize = 2 },
+ });
+ try ctx.expectSentResult(null, .{ .id = 4 });
+ try ctx.processMessage(.{
+ .id = 5,
+ .method = "Network.getResponseBody",
+ .params = .{ .requestId = &small },
+ });
+ try ctx.expectSentError(-32000, "Request content was evicted from inspector cache", .{ .id = 5 });
+ try testing.expectEqual(0, bc.captured_responses_size);
+}
+
+test "cdp.Network: enable maxTotalBufferSize evicts oldest bodies first" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-TBS", .session_id = "SID-TBS" });
+ const page = try bc.session.createPage();
+
+ try ctx.processMessage(.{
+ .id = 1,
+ .method = "Network.enable",
+ .params = .{ .maxTotalBufferSize = 10 },
+ });
+ try ctx.expectSentResult(null, .{ .id = 1 });
+
+ const first = try EchoDriver.run(bc, page.frame_id, "aaaaaa");
+ const second = try EchoDriver.run(bc, page.frame_id, "bbbbbb");
+
+ try ctx.processMessage(.{
+ .id = 2,
+ .method = "Network.getResponseBody",
+ .params = .{ .requestId = &first },
+ });
+ try ctx.expectSentError(-32000, "Request content was evicted from inspector cache", .{ .id = 2 });
+
+ try ctx.processMessage(.{
+ .id = 3,
+ .method = "Network.getResponseBody",
+ .params = .{ .requestId = &second },
+ });
+ try ctx.expectSentResult(.{ .body = "bbbbbb", .base64Encoded = false }, .{ .id = 3 });
+ try testing.expectEqual(6, bc.captured_responses_size);
+
+ // Network.disable releases everything retained.
+ try ctx.processMessage(.{ .id = 4, .method = "Network.disable" });
+ try ctx.expectSentResult(null, .{ .id = 4 });
+ try testing.expectEqual(0, bc.captured_responses.count());
+ try testing.expectEqual(0, bc.captured_responses_size);
+ try ctx.processMessage(.{
+ .id = 5,
+ .method = "Network.getResponseBody",
+ .params = .{ .requestId = &second },
+ });
+ try ctx.expectSentError(-31998, "RequestNotFound", .{ .id = 5 });
+}
+
+test "cdp.Network: enable maxPostDataSize omits inline postData" {
+ var ctx = try testing.context();
+ defer ctx.deinit();
+
+ const bc = try ctx.loadBrowserContext(.{ .id = "BID-PDS", .session_id = "SID-PDS" });
+ const page = try bc.session.createPage();
+
+ try ctx.processMessage(.{
+ .id = 1,
+ .method = "Network.enable",
+ .params = .{ .maxPostDataSize = 8 },
+ });
+ try ctx.expectSentResult(null, .{ .id = 1 });
+
+ const body = "{\"source\":\"xhr\",\"pageSize\":100}";
+ const request_id = try EchoDriver.run(bc, page.frame_id, body);
+
+ try ctx.expectSentEvent("Network.requestWillBeSent", .{
+ .requestId = &request_id,
+ .request = .{ .method = "POST", .hasPostData = true },
+ }, .{ .session_id = "SID-PDS" });
+
+ // The subset matcher can't assert absence; look at the event directly.
+ var seen = false;
+ for (ctx.received.items) |received| {
+ const method = received.object.get("method") orelse continue;
+ if (!std.mem.eql(u8, method.string, "Network.requestWillBeSent")) continue;
+ const request = received.object.get("params").?.object.get("request").?.object;
+ try testing.expectEqual(null, request.get("postData"));
+ try testing.expectEqual(null, request.get("postDataEntries"));
+ seen = true;
+ }
+ try testing.expect(seen);
+
+ // Retention is independent of the inline limit.
+ try ctx.processMessage(.{
+ .id = 2,
+ .method = "Network.getRequestPostData",
+ .params = .{ .requestId = &request_id },
+ });
+ try ctx.expectSentResult(.{ .postData = body }, .{ .id = 2 });
+}
+
test "cdp.Network: redirect hop precedes Fetch pause and carries redirectResponse" {
var ctx = try testing.context();
defer ctx.deinit();
diff --git a/src/cdp/domains/page.zig b/src/cdp/domains/page.zig
index b65960baf..248bc2972 100644
--- a/src/cdp/domains/page.zig
+++ b/src/cdp/domains/page.zig
@@ -551,7 +551,7 @@ pub fn frameCreated(bc: *CDP.BrowserContext, frame: *Frame) !void {
// controlled via Network.configureDurableMessages (which we don't
// support).
bc.captured_requests = .empty;
- bc.captured_responses = .empty;
+ bc.clearCapturedResponses();
}
}