Lightpanda: the headless browser designed for AI and automation
cdp: configurable max websocket and http message size
Karl Seguin297027f056ed
diff --git a/src/Config.zig b/src/Config.zigindex 1fba522b6..749722057 100644--- a/src/Config.zig+++ b/src/Config.zig@@ -30,13 +30,6 @@ const WebBotAuthConfig = @import("network/WebBotAuth.zig").Config;const Allocator = std.mem.Allocator;-pub const CDP_MAX_HTTP_REQUEST_SIZE = 4096;--// max message size-// +14 for max websocket payload overhead-// +140 for the max control packet that might be interleaved in a message-pub const CDP_MAX_MESSAGE_SIZE = 512 * 1024 + 14 + 140;-// TCP keepalive parameters applied to accepted CDP connections.// Detection window ≈ IDLE + CNT * INTVL = 4 + 3*2 = 10s.pub const CDP_KEEPALIVE_IDLE_S: c_int = 4;@@ -184,6 +177,9 @@ const Commands = cli.Builder(.{.{ .name = "timeout", .type = ?u31 },.{ .name = "cdp_max_connections", .type = u16, .default = 16 },.{ .name = "cdp_max_pending_connections", .type = u16, .default = 128 },+ .{ .name = "cdp_max_message_size", .type = u32, .default = 1024 * 1024 },+ // Don't widen this without growing the reader buffer in the HTTP path.+ .{ .name = "cdp_max_http_message_size", .type = u14, .default = 4096 },},.shared_options = CommonOptions,},@@ -522,6 +518,20 @@ pub fn maxPendingConnections(self: *const Config) u31 {};}+pub fn cdpMaxMessageSize(self: *const Config) u32 {+ return switch (self.mode) {+ .serve => |opts| opts.cdp_max_message_size,+ else => unreachable,+ };+}++pub fn cdpMaxHTTPMessageSize(self: *const Config) u14 {+ return switch (self.mode) {+ .serve => |opts| opts.cdp_max_http_message_size,+ else => unreachable,+ };+}+pub fn storageEngine(self: *const Config) ?Storage.EngineType {return switch (self.mode) {inline .serve, .fetch, .mcp, .agent => |opts| opts.storage_engine,diff --git a/src/Server.zig b/src/Server.zigindex 43b1a2bc9..242a42e17 100644--- a/src/Server.zig+++ b/src/Server.zig@@ -313,6 +313,9 @@ test "server: buildJSONVersionResponse" {}test "Client: http invalid request" {+ const filter: testing.LogFilter = .init(&.{.cdp});+ defer filter.deinit();+var c = try createTestClient();defer c.deinit();@@ -427,8 +430,12 @@ test "Client: read invalid websocket message" {);}- // length of message is 0000 0810, i.e: 1024 * 512 + 265- try assertWebSocketError(1009, &.{ 129, 255, 0, 0, 0, 0, 0, 8, 1, 0, 'm', 'a', 's', 'k' });+ {+ const filter: testing.LogFilter = .init(&.{.cdp});+ defer filter.deinit();+ // length of message is 0, 0, 0, 0, 0, 16, 0, 1 i.e: 1024 * 1024 + 1+ try assertWebSocketError(1009, &.{ 129, 255, 0, 0, 0, 0, 0, 16, 0, 1, 'm', 'a', 's', 'k' });+ }// continuation type message must come after a normal message// even when not a fin frame@@ -632,6 +639,7 @@ fn createTestClient() !TestClient {return .{.stream = stream,.reader = .{+ .max_message_size = 1024,.allocator = testing.allocator,.buf = try testing.allocator.alloc(u8, 1024 * 16),},@@ -641,7 +649,7 @@ fn createTestClient() !TestClient {const TestClient = struct {stream: std.net.Stream,buf: [1024]u8 = undefined,- reader: WS.Reader(false, 1024),+ reader: WS.Reader(false),const WS = @import("network/WS.zig");diff --git a/src/cdp/CDP.zig b/src/cdp/CDP.zigindex f8d529ced..88e1f1345 100644--- a/src/cdp/CDP.zig+++ b/src/cdp/CDP.zig@@ -111,7 +111,7 @@ pub fn init(try self.browser.init(app, .{ .env = .{ .with_inspector = true } }, self);const http_client = &self.browser.http_client;- try self.conn.init(allocator, socket, json_version_response, &http_client.inbox, &app.arena_pool);+ try self.conn.init(app, socket, json_version_response, &http_client.inbox);errdefer self.conn.deinit();self.link = .{diff --git a/src/cdp/Connection.zig b/src/cdp/Connection.zigindex f42f2cf7e..69b984463 100644--- a/src/cdp/Connection.zig+++ b/src/cdp/Connection.zig@@ -22,6 +22,7 @@ const builtin = @import("builtin");const CDP = @import("CDP.zig");+const App = @import("../App.zig");const Inbox = @import("../Inbox.zig");const Config = @import("../Config.zig");const WS = @import("../network/WS.zig");@@ -42,17 +43,17 @@ arena_pool: *ArenaPool,socket: posix.socket_t,socket_flags: usize,state: State = .handshaking,-reader: WS.Reader(true, Config.CDP_MAX_MESSAGE_SIZE),+reader: WS.Reader(true),send_arena: ArenaAllocator,+max_http_message_size: usize,json_version_response: []const u8,pub fn init(self: *Connection,- allocator: Allocator,+ app: *App,socket: posix.socket_t,json_version_response: []const u8,inbox: *Inbox,- arena_pool: *ArenaPool,) !void {const socket_flags = try posix.fcntl(socket, posix.F.GETFL, 0);const nonblocking = @as(u32, @bitCast(posix.O{ .NONBLOCK = true }));@@ -60,12 +61,16 @@ pub fn init(lp.assert(socket_flags & nonblocking == nonblocking, "Connection.init blocking", .{});}+ const config = app.config;+ const allocator = app.allocator;+self.* = .{.inbox = inbox,.socket = socket,- .arena_pool = arena_pool,+ .arena_pool = &app.arena_pool,.socket_flags = socket_flags,- .reader = try .init(allocator),+ .max_http_message_size = config.cdpMaxHTTPMessageSize(),+ .reader = try .init(allocator, config.cdpMaxMessageSize()),.send_arena = ArenaAllocator.init(allocator),.json_version_response = json_version_response,};@@ -222,7 +227,8 @@ fn processHttpRequest(self: *Connection) !HttpResult {lp.assert(self.reader.pos == 0, "Connection.HTTP pos", .{ .pos = self.reader.pos });const request = self.reader.buf[0..self.reader.len];- if (request.len > Config.CDP_MAX_HTTP_REQUEST_SIZE) {+ if (request.len > self.max_http_message_size) {+ log.warn(.cdp, "CDP message too big", .{ .type = "HTTP", .len = request.len, .hint = "See the --cdp-max-http-message-size <bytes>" });self.sendHttpError(413, "Request too large");return error.RequestTooLarge;}diff --git a/src/help.zon b/src/help.zonindex f180a11f8..af72d0d63 100644--- a/src/help.zon+++ b/src/help.zon@@ -35,6 +35,12 @@\\ --cdp-max-pending-connections <INT>\\ Maximum pending connections in the accept queue.\\ Defaults to 128.+ \\ --cdp-max-message-size <INT>+ \\ Maximum allowed incoming websocket message size.+ \\ Defaults to 1048576 (1MB)+ \\ --cdp-max-http-message-size <INT>+ \\ Maximum allowed HTTP request size+ \\ Defaults to 4096 (maximum allowed: 16383)\\ --cookie <PATH>\\ Path to a JSON file to load cookies from (read-only).\\ Defaults to no cookie loading.diff --git a/src/network/WS.zig b/src/network/WS.zigindex 5678c8047..c144993e3 100644--- a/src/network/WS.zig+++ b/src/network/WS.zig@@ -58,7 +58,7 @@ pub const OpCode = enum(u8) {// WebSocket message reader. Given websocket message, acts as an iterator that// can return zero or more Messages. When next returns null, any incomplete// message will remain in reader.data-pub fn Reader(comptime EXPECT_MASK: bool, MAX_MESSAGE_SIZE: usize) type {+pub fn Reader(comptime EXPECT_MASK: bool) type {return struct {allocator: Allocator,@@ -69,19 +69,20 @@ pub fn Reader(comptime EXPECT_MASK: bool, MAX_MESSAGE_SIZE: usize) type {// (any new reads must be placed after this)len: usize = 0,- // we add 140 to allow 1 control message (ping/pong/close) to be- // fragmented into a normal message.+ max_message_size: usize,+buf: []u8,fragments: ?Fragments = null,const Self = @This();- pub fn init(allocator: Allocator) !Self {+ pub fn init(allocator: Allocator, max_message_size: usize) !Self {const buf = try allocator.alloc(u8, 16 * 1024);return .{.buf = buf,.allocator = allocator,+ .max_message_size = max_message_size,};}@@ -154,7 +155,8 @@ pub fn Reader(comptime EXPECT_MASK: bool, MAX_MESSAGE_SIZE: usize) type {if (message_len > 125) {return error.ControlTooLarge;}- } else if (message_len > MAX_MESSAGE_SIZE) {+ } else if (message_len > self.max_message_size) {+ lp.log.warn(.cdp, "CDP message too big", .{ .type = "WS", .len = message_len, .hint = "See the --cdp-max-message-size <bytes>" });return error.TooLarge;} else if (message_len > self.buf.len) {const len = self.buf.len;@@ -182,7 +184,9 @@ pub fn Reader(comptime EXPECT_MASK: bool, MAX_MESSAGE_SIZE: usize) type {if (is_continuation) {const fragments = &(self.fragments orelse return error.InvalidContinuation);- if (fragments.message.items.len + message_len > MAX_MESSAGE_SIZE) {+ const full_len = fragments.message.items.len + message_len;+ if (full_len > self.max_message_size) {+ lp.log.warn(.cdp, "CDP message too big", .{ .type = "WS", .len = full_len, .hint = "See the --cdp-max-message-size <bytes>" });return error.TooLarge;}