Skip to content

Lightpanda: the headless browser designed for AI and automation

cdp: configurable max websocket and http message size

Karl Seguin297027f056ed
diff --git a/src/Config.zig b/src/Config.zig
index 1fba522b6..749722057 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -30,13 +30,6 @@ const WebBotAuthConfig = @import("network/WebBotAuth.zig").Config;
const Allocator = std.mem.Allocator;
-pub const CDP_MAX_HTTP_REQUEST_SIZE = 4096;
-
-// max message size
-// +14 for max websocket payload overhead
-// +140 for the max control packet that might be interleaved in a message
-pub const CDP_MAX_MESSAGE_SIZE = 512 * 1024 + 14 + 140;
-
// TCP keepalive parameters applied to accepted CDP connections.
// Detection window ≈ IDLE + CNT * INTVL = 4 + 3*2 = 10s.
pub const CDP_KEEPALIVE_IDLE_S: c_int = 4;
@@ -184,6 +177,9 @@ const Commands = cli.Builder(.{
.{ .name = "timeout", .type = ?u31 },
.{ .name = "cdp_max_connections", .type = u16, .default = 16 },
.{ .name = "cdp_max_pending_connections", .type = u16, .default = 128 },
+ .{ .name = "cdp_max_message_size", .type = u32, .default = 1024 * 1024 },
+ // Don't widen this without growing the reader buffer in the HTTP path.
+ .{ .name = "cdp_max_http_message_size", .type = u14, .default = 4096 },
},
.shared_options = CommonOptions,
},
@@ -522,6 +518,20 @@ pub fn maxPendingConnections(self: *const Config) u31 {
};
}
+pub fn cdpMaxMessageSize(self: *const Config) u32 {
+ return switch (self.mode) {
+ .serve => |opts| opts.cdp_max_message_size,
+ else => unreachable,
+ };
+}
+
+pub fn cdpMaxHTTPMessageSize(self: *const Config) u14 {
+ return switch (self.mode) {
+ .serve => |opts| opts.cdp_max_http_message_size,
+ else => unreachable,
+ };
+}
+
pub fn storageEngine(self: *const Config) ?Storage.EngineType {
return switch (self.mode) {
inline .serve, .fetch, .mcp, .agent => |opts| opts.storage_engine,
diff --git a/src/Server.zig b/src/Server.zig
index 43b1a2bc9..242a42e17 100644
--- a/src/Server.zig
+++ b/src/Server.zig
@@ -313,6 +313,9 @@ test "server: buildJSONVersionResponse" {
}
test "Client: http invalid request" {
+ const filter: testing.LogFilter = .init(&.{.cdp});
+ defer filter.deinit();
+
var c = try createTestClient();
defer c.deinit();
@@ -427,8 +430,12 @@ test "Client: read invalid websocket message" {
);
}
- // length of message is 0000 0810, i.e: 1024 * 512 + 265
- try assertWebSocketError(1009, &.{ 129, 255, 0, 0, 0, 0, 0, 8, 1, 0, 'm', 'a', 's', 'k' });
+ {
+ const filter: testing.LogFilter = .init(&.{.cdp});
+ defer filter.deinit();
+ // length of message is 0, 0, 0, 0, 0, 16, 0, 1 i.e: 1024 * 1024 + 1
+ try assertWebSocketError(1009, &.{ 129, 255, 0, 0, 0, 0, 0, 16, 0, 1, 'm', 'a', 's', 'k' });
+ }
// continuation type message must come after a normal message
// even when not a fin frame
@@ -632,6 +639,7 @@ fn createTestClient() !TestClient {
return .{
.stream = stream,
.reader = .{
+ .max_message_size = 1024,
.allocator = testing.allocator,
.buf = try testing.allocator.alloc(u8, 1024 * 16),
},
@@ -641,7 +649,7 @@ fn createTestClient() !TestClient {
const TestClient = struct {
stream: std.net.Stream,
buf: [1024]u8 = undefined,
- reader: WS.Reader(false, 1024),
+ reader: WS.Reader(false),
const WS = @import("network/WS.zig");
diff --git a/src/cdp/CDP.zig b/src/cdp/CDP.zig
index f8d529ced..88e1f1345 100644
--- a/src/cdp/CDP.zig
+++ b/src/cdp/CDP.zig
@@ -111,7 +111,7 @@ pub fn init(
try self.browser.init(app, .{ .env = .{ .with_inspector = true } }, self);
const http_client = &self.browser.http_client;
- try self.conn.init(allocator, socket, json_version_response, &http_client.inbox, &app.arena_pool);
+ try self.conn.init(app, socket, json_version_response, &http_client.inbox);
errdefer self.conn.deinit();
self.link = .{
diff --git a/src/cdp/Connection.zig b/src/cdp/Connection.zig
index f42f2cf7e..69b984463 100644
--- a/src/cdp/Connection.zig
+++ b/src/cdp/Connection.zig
@@ -22,6 +22,7 @@ const builtin = @import("builtin");
const CDP = @import("CDP.zig");
+const App = @import("../App.zig");
const Inbox = @import("../Inbox.zig");
const Config = @import("../Config.zig");
const WS = @import("../network/WS.zig");
@@ -42,17 +43,17 @@ arena_pool: *ArenaPool,
socket: posix.socket_t,
socket_flags: usize,
state: State = .handshaking,
-reader: WS.Reader(true, Config.CDP_MAX_MESSAGE_SIZE),
+reader: WS.Reader(true),
send_arena: ArenaAllocator,
+max_http_message_size: usize,
json_version_response: []const u8,
pub fn init(
self: *Connection,
- allocator: Allocator,
+ app: *App,
socket: posix.socket_t,
json_version_response: []const u8,
inbox: *Inbox,
- arena_pool: *ArenaPool,
) !void {
const socket_flags = try posix.fcntl(socket, posix.F.GETFL, 0);
const nonblocking = @as(u32, @bitCast(posix.O{ .NONBLOCK = true }));
@@ -60,12 +61,16 @@ pub fn init(
lp.assert(socket_flags & nonblocking == nonblocking, "Connection.init blocking", .{});
}
+ const config = app.config;
+ const allocator = app.allocator;
+
self.* = .{
.inbox = inbox,
.socket = socket,
- .arena_pool = arena_pool,
+ .arena_pool = &app.arena_pool,
.socket_flags = socket_flags,
- .reader = try .init(allocator),
+ .max_http_message_size = config.cdpMaxHTTPMessageSize(),
+ .reader = try .init(allocator, config.cdpMaxMessageSize()),
.send_arena = ArenaAllocator.init(allocator),
.json_version_response = json_version_response,
};
@@ -222,7 +227,8 @@ fn processHttpRequest(self: *Connection) !HttpResult {
lp.assert(self.reader.pos == 0, "Connection.HTTP pos", .{ .pos = self.reader.pos });
const request = self.reader.buf[0..self.reader.len];
- if (request.len > Config.CDP_MAX_HTTP_REQUEST_SIZE) {
+ if (request.len > self.max_http_message_size) {
+ log.warn(.cdp, "CDP message too big", .{ .type = "HTTP", .len = request.len, .hint = "See the --cdp-max-http-message-size <bytes>" });
self.sendHttpError(413, "Request too large");
return error.RequestTooLarge;
}
diff --git a/src/help.zon b/src/help.zon
index f180a11f8..af72d0d63 100644
--- a/src/help.zon
+++ b/src/help.zon
@@ -35,6 +35,12 @@
\\ --cdp-max-pending-connections <INT>
\\ Maximum pending connections in the accept queue.
\\ Defaults to 128.
+ \\ --cdp-max-message-size <INT>
+ \\ Maximum allowed incoming websocket message size.
+ \\ Defaults to 1048576 (1MB)
+ \\ --cdp-max-http-message-size <INT>
+ \\ Maximum allowed HTTP request size
+ \\ Defaults to 4096 (maximum allowed: 16383)
\\ --cookie <PATH>
\\ Path to a JSON file to load cookies from (read-only).
\\ Defaults to no cookie loading.
diff --git a/src/network/WS.zig b/src/network/WS.zig
index 5678c8047..c144993e3 100644
--- a/src/network/WS.zig
+++ b/src/network/WS.zig
@@ -58,7 +58,7 @@ pub const OpCode = enum(u8) {
// WebSocket message reader. Given websocket message, acts as an iterator that
// can return zero or more Messages. When next returns null, any incomplete
// message will remain in reader.data
-pub fn Reader(comptime EXPECT_MASK: bool, MAX_MESSAGE_SIZE: usize) type {
+pub fn Reader(comptime EXPECT_MASK: bool) type {
return struct {
allocator: Allocator,
@@ -69,19 +69,20 @@ pub fn Reader(comptime EXPECT_MASK: bool, MAX_MESSAGE_SIZE: usize) type {
// (any new reads must be placed after this)
len: usize = 0,
- // we add 140 to allow 1 control message (ping/pong/close) to be
- // fragmented into a normal message.
+ max_message_size: usize,
+
buf: []u8,
fragments: ?Fragments = null,
const Self = @This();
- pub fn init(allocator: Allocator) !Self {
+ pub fn init(allocator: Allocator, max_message_size: usize) !Self {
const buf = try allocator.alloc(u8, 16 * 1024);
return .{
.buf = buf,
.allocator = allocator,
+ .max_message_size = max_message_size,
};
}
@@ -154,7 +155,8 @@ pub fn Reader(comptime EXPECT_MASK: bool, MAX_MESSAGE_SIZE: usize) type {
if (message_len > 125) {
return error.ControlTooLarge;
}
- } else if (message_len > MAX_MESSAGE_SIZE) {
+ } else if (message_len > self.max_message_size) {
+ lp.log.warn(.cdp, "CDP message too big", .{ .type = "WS", .len = message_len, .hint = "See the --cdp-max-message-size <bytes>" });
return error.TooLarge;
} else if (message_len > self.buf.len) {
const len = self.buf.len;
@@ -182,7 +184,9 @@ pub fn Reader(comptime EXPECT_MASK: bool, MAX_MESSAGE_SIZE: usize) type {
if (is_continuation) {
const fragments = &(self.fragments orelse return error.InvalidContinuation);
- if (fragments.message.items.len + message_len > MAX_MESSAGE_SIZE) {
+ const full_len = fragments.message.items.len + message_len;
+ if (full_len > self.max_message_size) {
+ lp.log.warn(.cdp, "CDP message too big", .{ .type = "WS", .len = full_len, .hint = "See the --cdp-max-message-size <bytes>" });
return error.TooLarge;
}