Skip to content

Lightpanda: the headless browser designed for AI and automation

cdp: Base --http-max-concurrent default on --cdp-max-connections

Karl Seguin01d1aed1d761
diff --git a/src/Config.zig b/src/Config.zig
index 650dec382..de940858b 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -259,7 +259,7 @@ const CommonOptions = .{
.{ .name = "cors_store_entry_limit", .type = ?u32, .default = 1000 },
.{ .name = "proxy_bearer_token", .type = ?[:0]const u8 },
.{ .name = "http_proxy", .type = ?[:0]const u8 },
- .{ .name = "http_max_concurrent", .type = ?u8 },
+ .{ .name = "http_max_concurrent", .type = ?u16 },
.{ .name = "http_max_host_open", .type = ?u8 },
.{ .name = "http_nav_delay", .type = ?u32 },
.{ .name = "http_nav_burst", .type = ?u32 },
@@ -683,9 +683,10 @@ fn proxyBearerToken(self: *const Config) ?[:0]const u8 {
};
}
-pub fn httpMaxConcurrent(self: *const Config) u8 {
+pub fn httpMaxConcurrent(self: *const Config) u16 {
return switch (self.mode) {
- inline .serve, .fetch, .mcp, .agent => |opts| opts.http_max_concurrent orelse 40,
+ .serve => |opts| opts.http_max_concurrent orelse std.math.clamp(opts.cdp_max_connections *| 8, 40, 256),
+ inline .fetch, .mcp, .agent => |opts| opts.http_max_concurrent orelse 40,
else => unreachable,
};
}
@@ -1482,6 +1483,27 @@ test "Config: parseArgs --http-session-timeout" {
}
}
+test "Config: httpMaxConcurrent default scales with --cdp-max-connections on serve" {
+ var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+ defer arena.deinit();
+
+ const cases = [_]struct { argv: []const [*:0]const u8, expected: u16 }{
+ .{ .argv = &.{ "lightpanda", "serve" }, .expected = 128 },
+ .{ .argv = &.{ "lightpanda", "serve", "--cdp-max-connections", "2" }, .expected = 40 },
+ .{ .argv = &.{ "lightpanda", "serve", "--cdp-max-connections", "100" }, .expected = 256 },
+ .{ .argv = &.{ "lightpanda", "serve", "--cdp-max-connections", "65535" }, .expected = 256 },
+ .{ .argv = &.{ "lightpanda", "serve", "--cdp-max-connections", "100", "--http-max-concurrent", "1000" }, .expected = 1000 },
+ .{ .argv = &.{ "lightpanda", "serve", "--http-max-concurrent", "10" }, .expected = 10 },
+ .{ .argv = &.{ "lightpanda", "fetch", "https://example.com" }, .expected = 40 },
+ .{ .argv = &.{ "lightpanda", "mcp" }, .expected = 40 },
+ };
+ for (cases) |case| {
+ const proc_args: std.process.Args = .{ .vector = case.argv };
+ const config = try parseArgs(arena.allocator(), proc_args);
+ try std.testing.expectEqual(case.expected, config.httpMaxConcurrent());
+ }
+}
+
test "Config: parseArgs --dump tells a url from a misspelt format" {
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena.deinit();
diff --git a/src/help.zon b/src/help.zon
index 1896ef568..081ab831f 100644
--- a/src/help.zon
+++ b/src/help.zon
@@ -532,8 +532,10 @@
\\ perfectly well what you are doing.
\\ Script set headers via CDP or JS can't override a CLI set header.
\\ --http-max-concurrent <INT>
- \\ Maximum number of concurrent HTTP requests.
- \\ Defaults to 40.
+ \\ Maximum number of concurrent HTTP requests, shared by every
+ \\ session.
+ \\ Defaults to 40. For serve, defaults to 8 per
+ \\ --cdp-max-connections with a min of 40 and a max of 256.
\\ --http-max-host-open <INT>
\\ Maximum open connections to a given host:port.
\\ Defaults to 6.
diff --git a/src/network/http.zig b/src/network/http.zig
index d6f840f84..4a1e85fb0 100644
--- a/src/network/http.zig
+++ b/src/network/http.zig
@@ -727,9 +727,12 @@ pub const Handles = struct {
errdefer libcurl.curl_multi_cleanup(multi) catch {};
try libcurl.curl_multi_setopt(multi, .max_host_connections, config.httpMaxHostOpen());
- // Default is 4x the attached easy handles, i.e. ~0 between page loads,
- // so keepalive connections were evicted on every cross-site navigation.
- try libcurl.curl_multi_setopt(multi, .max_connects, 4 * @as(u32, config.httpMaxConcurrent()));
+ // --http-max-concurrent is dynamic (based on --cdp-max-connections in
+ // serve), so we don't want this to be based on that, since it could
+ // result in an unexpected large # of keepalive connection. Instead, we
+ // hard-code 160, which used to be the default when --cdp-max-connections
+ // was fixed (and it seemed to be safe/work well, so why mess with it?)
+ try libcurl.curl_multi_setopt(multi, .max_connects, 160);
return .{ .multi = multi };
}