Lightpanda: the headless browser designed for AI and automation
Hardening
Karl Seguin17c2aa7d1f84
diff --git a/src/Inbox.zig b/src/Inbox.zigindex d5a4092d6..24cdd80dc 100644--- a/src/Inbox.zig+++ b/src/Inbox.zig@@ -37,6 +37,10 @@ const Inbox = @This();mutex: std.Io.Mutex = .init,queue: DoublyLinkedList = .{},+// Payload bytes sitting in the queue. Used to disconnect a client if we've+// fallen too far behind (largely to protect against a misbehaving client)+queued_bytes: usize = 0,+// One-way latch, set by the worker's drainInbox the first time it// observes a .disconnect (or .close) and never cleared. Ensures that, on// multiple drains, the terminated state is preserved / communicated. This is@@ -51,6 +55,13 @@ pub fn deinit(self: *Inbox) void {const msg: *Message = @fieldParentPtr("node", node);msg.deinit();}+ self.queued_bytes = 0;+}++pub fn queuedBytes(self: *Inbox) usize {+ self.mutex.lockUncancelable(lp.io);+ defer self.mutex.unlock(lp.io);+ return self.queued_bytes;}pub fn push(self: *Inbox, arena: *lp.Arena, payload: Message.Payload) void {@@ -61,6 +72,7 @@ pub fn push(self: *Inbox, arena: *lp.Arena, payload: Message.Payload) void {msg.* = .{ .payload = payload, .arena = arena };self.mutex.lockUncancelable(lp.io);defer self.mutex.unlock(lp.io);+ self.queued_bytes += payload.size();self.queue.append(&msg.node);}@@ -68,7 +80,9 @@ pub fn pop(self: *Inbox) ?*Message {self.mutex.lockUncancelable(lp.io);defer self.mutex.unlock(lp.io);const node = self.queue.popFirst() orelse return null;- return @fieldParentPtr("node", node);+ const msg: *Message = @fieldParentPtr("node", node);+ self.queued_bytes -= msg.payload.size();+ return msg;}// Peek for a message matching `predicate` without removing it. Used by@@ -99,6 +113,7 @@ pub fn popIf(self: *Inbox, predicate: *const fn (*Message) bool) ?*Message {const msg: *Message = @fieldParentPtr("node", node);if (predicate(msg)) {self.queue.remove(node);+ self.queued_bytes -= msg.payload.size();return msg;}}@@ -141,6 +156,14 @@ pub const Message = struct {// pushes this on peer EOF, fatal WS framing error, or// (now) JSON parse failure.disconnect: ?anyerror,++ pub fn size(self: Payload) usize {+ return switch (self) {+ .cdp => |c| c.raw.len,+ .bidi, .ping => |b| b.len,+ .close, .disconnect => 0,+ };+ }};pub const Cdp = struct {@@ -346,3 +369,48 @@ test "Inbox: popIf picks first match in FIFO order" {defer m.deinit();try testing.expectEqual("first", m.payload.ping);}++test "Inbox: queued bytes track the payloads" {+ const arena_pool = &testing.test_app.arena_pool;++ var inbox = Inbox{};+ defer inbox.deinit();++ try testing.expectEqual(0, inbox.queuedBytes());++ {+ const arena = try arena_pool.acquire(.tiny, "inbox test");+ inbox.push(arena, .{ .ping = try arena.dupe(u8, "12345") });+ }+ try testing.expectEqual(5, inbox.queuedBytes());++ {+ // control payloads are free; only what the peer sends counts+ const arena = try arena_pool.acquire(.tiny, "inbox test");+ inbox.push(arena, .{ .disconnect = null });+ }+ try testing.expectEqual(5, inbox.queuedBytes());++ {+ const arena = try arena_pool.acquire(.tiny, "inbox test");+ inbox.push(arena, .{ .bidi = try arena.dupe(u8, "abc") });+ }+ try testing.expectEqual(8, inbox.queuedBytes());++ // popIf cherry-picks out of the middle, and has to pay the same toll+ {+ const m = inbox.popIf(struct {+ fn f(msg: *Message) bool {+ return msg.payload == .bidi;+ }+ }.f).?;+ defer m.deinit();+ }+ try testing.expectEqual(5, inbox.queuedBytes());++ {+ const m = inbox.pop().?;+ defer m.deinit();+ }+ try testing.expectEqual(0, inbox.queuedBytes());+}diff --git a/src/Metrics.zig b/src/Metrics.zigindex eff9fc6f7..89a351495 100644--- a/src/Metrics.zig+++ b/src/Metrics.zig@@ -23,7 +23,8 @@ const Metrics = @This();const Driver = @import("server/Driver.zig").Protocol;serve_http_requests: CounterEnum("status", @import("network/http.zig").StatusCategory) = .{},-serve_http_evictions: CounterEnum("reason", enum { first_request, idle }) = .{},+serve_http_evictions: Counter = .{},+serve_inbox_backlog: Counter = .{},serve_connections: CounterEnum("driver", Driver) = .{},serve_connection_limit: Counter = .{},serve_active_connections: GaugeEnum("driver", Driver) = .{},@@ -95,7 +96,8 @@ robots_access: CounterEnum("result", enum { allow, deny }) = .{},// compile error.const help = .{.serve_http_requests = "HTTP responses sent, by status category (includes the pre-parse 400/413 rejections)",- .serve_http_evictions = "HTTP connections closed for exceeding a deadline; first_request never completed a request, idle is a keepalive connection that went quiet",+ .serve_http_evictions = "HTTP connections closed for sitting past their deadline without completing a request",+ .serve_inbox_backlog = "Websocket connections closed for queueing more unprocessed messages than the worker could drain",.serve_connections = "Websocket connections accepted, by driver protocol",.serve_connection_limit = "Accepts deferred because the connection budget was full: the listener pauses until a slot frees (counted before any handshake, so no driver label)",.serve_active_connections = "Currently connected clients, by driver protocol",diff --git a/src/server/Link.zig b/src/server/Link.zigindex 621fb9212..faeed6459 100644--- a/src/server/Link.zig+++ b/src/server/Link.zig@@ -39,6 +39,16 @@ const ArenaAllocator = std.heap.ArenaAllocator;// inbox's own.const Link = @This();+// A send that has flipped to blocking (below) waits at most this long for the+// peer to take more bytes.+const SEND_TIMEOUT_S = 5;++// The loop reads as fast as it can. The worker _can_ be slow to process its+// inbox (e.g. stuck in a syncRequest). Still, we want _some_ limit on how much+// data is queued. This is 32 * the configured max message size. Which should+// be plenty for a well-behaving client.+const INBOX_BACKLOG_MESSAGES = 32;+inbox: *Inbox,arena_pool: *ArenaPool,socket: posix.socket_t,@@ -46,6 +56,7 @@ socket_flags: usize,protocol: Driver.Protocol,reader: WS.Reader,send_arena: ArenaAllocator,+max_inbox_backlog: usize,pub fn init(self: *Link,@@ -60,6 +71,9 @@ pub fn init(lp.assert(socket_flags & nonblocking == nonblocking, "Link.init blocking", .{});}+ const timeout = std.mem.toBytes(posix.timeval{ .sec = SEND_TIMEOUT_S, .usec = 0 });+ try posix.setsockopt(socket, posix.SOL.SOCKET, posix.SO.SNDTIMEO, &timeout);+const config = app.config;const allocator = app.allocator;@@ -71,6 +85,7 @@ pub fn init(.socket_flags = socket_flags,.reader = try .init(allocator, config.cdpMaxMessageSize()),.send_arena = ArenaAllocator.init(allocator),+ .max_inbox_backlog = @as(usize, config.cdpMaxMessageSize()) * INBOX_BACKLOG_MESSAGES,};}@@ -98,11 +113,18 @@ pub fn send(self: *Link, data: []const u8) !void {// queue with its own allocations. On WouldBlock we flip the// socket to blocking for this write and flip it back after.// Should virtually never happen.- lp.assert(changed_to_blocking == false, "Link double block", .{});+ if (changed_to_blocking) {+ // We already flipped, so this is SO_SNDTIMEO firing: the+ // peer has stopped draining entirely. Treat it as gone+ // rather than parking the worker on it.+ return error.Timeout;+ }changed_to_blocking = true;_ = try sys_net.fcntl(self.socket, posix.F.SETFL, self.socket_flags & ~@as(u32, @bitCast(posix.O{ .NONBLOCK = true })));continue :LOOP;},+ // a signal landed mid-write; nothing was written+ error.Interrupted => continue :LOOP,else => return err,};@@ -156,6 +178,11 @@ pub const Read = struct {// Server loop. The socket is readablepub fn readAvailable(self: *Link, budget: usize) !Read {+ if (self.inbox.queuedBytes() >= self.max_inbox_backlog) {+ lp.metrics.serve_inbox_backlog.incr();+ return error.InboxBacklog;+ }+var pushed = false;var remaining = budget;while (remaining > 0) {@@ -267,7 +294,95 @@ fn pushBiDi(self: *Link, bytes: []const u8, pushed: *bool) !bool {return true;}-// Called from the worker (Driver.shutdown) to break the loop's read.+// Server loop, closing only the read side. The worker can still send a message+// (e.g. a close frame).pub fn shutdown(self: *Link) void {sys_net.shutdown(self.socket, .recv) catch {};}++const testing = @import("../testing.zig");++test "link: send gives up when the peer stops reading" {+ var pair: [2]posix.socket_t = undefined;+ if (std.c.socketpair(posix.AF.LOCAL, posix.SOCK.STREAM, 0, &pair) != 0) {+ return error.SocketPairFailed;+ }+ defer sys_net.close(pair[0]);+ defer sys_net.close(pair[1]);++ const small = std.mem.toBytes(@as(c_int, 4096));+ try posix.setsockopt(pair[0], posix.SOL.SOCKET, posix.SO.RCVBUF, &small);+ try posix.setsockopt(pair[1], posix.SOL.SOCKET, posix.SO.SNDBUF, &small);++ const nonblocking = @as(u32, @bitCast(posix.O{ .NONBLOCK = true }));+ const flags = try sys_net.fcntl(pair[1], posix.F.GETFL, 0);+ _ = try sys_net.fcntl(pair[1], posix.F.SETFL, flags | nonblocking);++ var inbox: Inbox = .{};+ defer inbox.deinit();++ var link: Link = undefined;+ try link.init(testing.test_app, pair[1], .cdp, &inbox);+ defer link.deinit();++ // shorten the blocking window so the test doesn't sit out the real one+ const timeout = std.mem.toBytes(posix.timeval{ .sec = 0, .usec = 50_000 });+ try posix.setsockopt(pair[1], posix.SOL.SOCKET, posix.SO.SNDTIMEO, &timeout);++ const payload = try testing.allocator.alloc(u8, 1024 * 1024);+ defer testing.allocator.free(payload);+ @memset(payload, 'a');++ // Nobody drains pair[0]. send() flips to blocking to finish the write;+ // unbounded, that parks the worker forever and, because shutdown only+ // half-closes the read side, hangs the whole process on SIGINT.+ try testing.expectError(error.Timeout, link.send(payload));++ // and the socket has to be non-blocking again for the run loop's reads+ try testing.expectEqual(flags | nonblocking, try sys_net.fcntl(pair[1], posix.F.GETFL, 0));+}++test "link: stops reading once the worker's inbox backs up" {+ var pair: [2]posix.socket_t = undefined;+ if (std.c.socketpair(posix.AF.LOCAL, posix.SOCK.STREAM, 0, &pair) != 0) {+ return error.SocketPairFailed;+ }+ defer sys_net.close(pair[0]);+ defer sys_net.close(pair[1]);++ const nonblocking = @as(u32, @bitCast(posix.O{ .NONBLOCK = true }));+ const flags = try sys_net.fcntl(pair[1], posix.F.GETFL, 0);+ _ = try sys_net.fcntl(pair[1], posix.F.SETFL, flags | nonblocking);++ var inbox: Inbox = .{};+ defer inbox.deinit();++ var link: Link = undefined;+ try link.init(testing.test_app, pair[1], .cdp, &inbox);+ defer link.deinit();++ // a ceiling below one max-size message would refuse what was configured+ try testing.expect(link.max_inbox_backlog > testing.test_app.config.cdpMaxMessageSize());++ // an empty inbox reads normally (nothing pending, so nothing pushed)+ const read = try link.readAvailable(1024);+ try testing.expectEqual(true, read.keep);+ try testing.expectEqual(false, read.pushed);++ // a worker that has fallen this far behind isn't going to catch up+ {+ const arena = try testing.test_app.arena_pool.acquire(link.max_inbox_backlog, "backlog test");+ const payload = try arena.allocator().alloc(u8, link.max_inbox_backlog);+ inbox.push(arena, .{ .bidi = payload });+ }+ try testing.expectEqual(link.max_inbox_backlog, inbox.queuedBytes());+ try testing.expectError(error.InboxBacklog, link.readAvailable(1024));++ // and it recovers once the worker drains+ {+ const msg = inbox.pop().?;+ defer msg.deinit();+ }+ try testing.expectEqual(0, inbox.queuedBytes());+ _ = try link.readAvailable(1024);+}diff --git a/src/server/Server.zig b/src/server/Server.zigindex 34b781a00..ad26fc98e 100644--- a/src/server/Server.zig+++ b/src/server/Server.zig@@ -280,47 +280,64 @@ pub fn run(self: *Server) void {return;};- while (true) {- const deadline = blk: {- // self.http_connections is ordered by deadline- const node = self.http_connections.first orelse break :blk null;- const conn: *Connection = @fieldParentPtr("node", node);- break :blk conn.deadline -| lp.datetime.milliTimestamp(.boot);- };+ while (self.runOnce()) {}+}- var events = self.io_engine.wait(deadline);- const now = lp.datetime.milliTimestamp(.boot);- while (events.next()) |event| {- switch (event) {- .accept => self.accept(now) catch |err| log.err(.serve, "accept", .{ .err = err }),- .read_write => |rw| switch (rw.target) {- .http => |conn| http.processEvent(self, conn, rw, now),- .ws => |ws| self.processWebSocketEvent(ws, rw),- },- .signal => self.drainWorkerQueue(),- .shutdown => self.beginShutdown(),- }- }+fn runOnce(self: *Server) bool {+ const deadline = blk: {+ // self.http_connections is ordered by deadline+ const node = self.http_connections.first orelse break :blk null;+ const conn: *Connection = @fieldParentPtr("node", node);+ break :blk conn.deadline -| lp.datetime.milliTimestamp(.boot);+ };- // evict http connections that have passed their deadline- while (self.http_connections.first) |node| {- const conn: *Connection = @fieldParentPtr("node", node);- if (conn.deadline > now) {- // self.http_connections is ordered by deadline, so as soon as we find one- // that hasn't reach its deadline, none of the ones after can.- break;- }- lp.metrics.serve_http_evictions.incr(if (conn.served) .idle else .first_request);- http.disconnect(self, conn);+ var events = self.io_engine.wait(deadline);+ const now = lp.datetime.milliTimestamp(.boot);++ var pending_accept = false;+ var pending_signal = false;+ var pending_shutdown = false;+ while (events.next()) |event| {+ switch (event) {+ .accept => pending_accept = true,+ .read_write => |rw| switch (rw.target) {+ .http => |conn| http.processEvent(self, conn, rw, now),+ .ws => |ws| self.processWebSocketEvent(ws, rw),+ },+ .signal => pending_signal = true,+ .shutdown => pending_shutdown = true,}+ }- // Keep looping until every worker has released; their terminate was- // requested in beginShutdown. Only then is it safe to return (deinit- // frees state the workers reference).- if (self.shutdown_begun and self.websocket_pool.live == 0) {- return;+ // signal first: a worker that released frees a slot the accept can use.+ if (pending_signal) {+ self.drainWorkerQueue();+ }+ if (pending_accept) {+ self.accept(now) catch |err| log.err(.serve, "accept", .{ .err = err });+ }+ // shutdown last: it's terminal, and draining the queue first keeps it from+ // walking a websocket whose worker has already gone.+ if (pending_shutdown) {+ self.beginShutdown();+ }++ // evict http connections that have passed their deadline+ while (self.http_connections.first) |node| {+ const conn: *Connection = @fieldParentPtr("node", node);+ if (conn.deadline > now) {+ // self.http_connections is ordered by deadline, so as soon as we find one+ // that hasn't reach its deadline, none of the ones after can.+ break;}+ lp.metrics.serve_http_evictions.incr();+ http.disconnect(self, conn);}++ if (self.shutdown_begun and self.websocket_pool.live == 0) {+ return false;+ }+ return true;}fn accept(self: *Server, now: u64) !void {@@ -349,6 +366,8 @@ fn accept(self: *Server, now: u64) !void {}};errdefer sys_net.close(socket);+ configureSocket(socket);+const peer = sys_net.addressFromSockaddr(@ptrCast(&address));if (comptime lp.IS_DEBUG) {log.debug(.serve, "client connected", .{ .address = peer });@@ -360,7 +379,7 @@ fn accept(self: *Server, now: u64) !void {conn.address = peer;try self.io_engine.monitorHTTP(conn);- conn.deadline = now + http.FIRST_TIMEOUT_MS;+ conn.deadline = now + http.IDLE_TIMEOUT_MS;self.http_connections.append(&conn.node);if (self.liveConnections() == self.max_connections) {@@ -369,6 +388,33 @@ fn accept(self: *Server, now: u64) !void {}}+// A peer that goes away without a FIN (a killed VM, a NAT timeout) leaves an+// upgraded connection holding a worker, a browser and a --cdp-max-connections+// slot; nothing above the socket notices. Keepalive is what ends it, and+// Driver.tick's wait cadence is built on that. Best effort: a socket we can't+// configure is still a usable socket.+fn configureSocket(socket: posix.socket_t) void {+ setSocketOption(socket, posix.SOL.SOCKET, posix.SO.KEEPALIVE, @as(c_int, 1), "SO_KEEPALIVE");++ const idle_opt = switch (builtin.os.tag) {+ .macos, .ios => posix.TCP.KEEPALIVE,+ else => posix.TCP.KEEPIDLE,+ };+ setSocketOption(socket, posix.IPPROTO.TCP, idle_opt, Config.CDP_KEEPALIVE_IDLE_S, "TCP_KEEPIDLE");+ setSocketOption(socket, posix.IPPROTO.TCP, posix.TCP.KEEPINTVL, Config.CDP_KEEPALIVE_INTVL_S, "TCP_KEEPINTVL");+ setSocketOption(socket, posix.IPPROTO.TCP, posix.TCP.KEEPCNT, Config.CDP_KEEPALIVE_CNT, "TCP_KEEPCNT");++ if (comptime builtin.os.tag == .linux) {+ setSocketOption(socket, posix.IPPROTO.TCP, std.os.linux.TCP.USER_TIMEOUT, Config.CDP_TCP_USER_TIMEOUT_MS, "TCP_USER_TIMEOUT");+ }+}++fn setSocketOption(socket: posix.socket_t, level: i32, option: u32, value: anytype, comptime name: []const u8) void {+ posix.setsockopt(socket, level, option, &std.mem.toBytes(value)) catch |err| {+ log.warn(.serve, "setsockopt", .{ .err = err, .option = name });+ };+}+fn liveConnections(self: *const Server) usize {return self.http_connection_pool.live + self.websocket_pool.live;}@@ -399,8 +445,8 @@ fn saturated(self: *Server) !void {fn processWebSocketEvent(self: *Server, ws: *WebSocket, rw: IOEvent.ReadWrite) void {if (ws.monitored == false) {- // can have an event that comes in during the same batch as a release- // and there's no guarantee about the order that we process them in.+ // only attachWorker puts a websocket in the poll set, and only once+ // the driver is set; an unmonitored slot has no business here.return;}@@ -516,15 +562,17 @@ fn attachWorker(self: *Server, ws: *WebSocket, driver: Driver) void {fn releaseWorker(self: *Server, ws: *WebSocket, notify: *std.Io.Event) void {if (ws.monitored) {+ ws.monitored = false;self.io_engine.remove(ws.socket);}self.releaseWebSocket(ws);+ // The worker is free to deinit its driver and close the fd from here.notify.set(lp.io);}// Frees the slot once the loop is done with the fd. The fd itself is closed// by whoever owns the end of its life: the worker after its driver's deinit,-// or openWebSocket when there never was a worker.+// or upgradeConnection when there never was a worker.fn releaseWebSocket(self: *Server, ws: *WebSocket) void {self.websockets.remove(&ws.node);lp.metrics.serve_active_connections.decr(ws.protocol);@@ -585,8 +633,9 @@ fn fdBudget(config: *const Config) usize {};break :blk limit.cur;};- // unlimited is really "as many as we'd care to hold 4KB buffers for"- const budget = @min(soft, 1 << 16) -| reserve;+ // put some limit incase of a unlimited or very large rlimit+ const ceiling = (64 * 1024 * 1024) / @max(@as(u64, config.cdpMaxHTTPMessageSize()), 1);+ const budget = @min(soft, ceiling) -| reserve;return @intCast(@max(budget, 8));}@@ -640,11 +689,14 @@ const Worker = struct {// The loop is done with us once releaseConnection returns; the// driver's deinit may still tick the client, without a producer.defer driver.browser.http_client.driver_link_active = false;- // It's possible the terminate flag is set. Our teardown (e.g. cdp.deinit()- // and bidi.deinit() in our caller) might need V8 in a usable state. So- // clear the terminate flag- driver.browser.env.cancelTerminate();+ // Release first: until the loop has let go of this websocket it can+ // still drop the link, and onLinkDisconnect requests a terminate. Doing+ // it the other way round left that request landing after the cancel,+ // so the teardown below ran with a pending terminate -- which is the+ // one thing the cancel is here to prevent (cdp.deinit() and+ // bidi.deinit() in our caller need V8 in a usable state).Worker.releaseConnection(server, ws);+ driver.browser.env.cancelTerminate();}// Worker -> loop: synchronous release. Blocks until the loop has dropped the@@ -2036,3 +2088,220 @@ const TestClient = struct {}}};++// A server of our own, bound to an ephemeral port and never run(): these+// tests drive its handlers by hand to reproduce what one event batch does.+// The real test server (port 9583) is shared and can't be torn down.+const LoopTest = struct {+ server: *Server,+ address: sys_net.IpAddress,++ fn init() !LoopTest {+ const server = try Server.init(testing.test_app, .{ .ip4 = .loopback(0) });+ errdefer server.deinit();+ server.protocols = .{ .cdp = true, .webdriver = true };+ // run() does this; runOnce() on its own would never see an accept+ try server.io_engine.monitorListener(server.listener);++ var bound: posix.sockaddr.storage = undefined;+ var bound_len: posix.socklen_t = @sizeOf(posix.sockaddr.storage);+ try sys_net.getsockname(server.listener, @ptrCast(&bound), &bound_len);++ return .{ .server = server, .address = sys_net.addressFromSockaddr(@ptrCast(&bound)) };+ }++ fn deinit(self: *LoopTest) void {+ self.server.deinit();+ }++ fn expectResponse(self: *const LoopTest, client: posix.socket_t, prefix: []const u8) !void {+ _ = self;+ var buf: [512]u8 = undefined;+ const n = try posix.read(client, &buf);+ try testing.expect(std.mem.startsWith(u8, buf[0..n], prefix));+ }++ // Connects a client and runs the accept the loop would have run,+ // returning the client's end and the Connection the loop now owns.+ fn accept(self: *LoopTest) !struct { posix.socket_t, *Connection } {+ const client = try sys_net.connect(&self.address);+ errdefer sys_net.close(client);+ // never block the suite on a response that isn't coming+ const timeout = std.mem.toBytes(posix.timeval{ .sec = 5, .usec = 0 });+ try posix.setsockopt(client, posix.SOL.SOCKET, posix.SO.RCVTIMEO, &timeout);++ const before = self.server.http_connections.last;+ try self.server.accept(lp.datetime.milliTimestamp(.boot));++ const node = self.server.http_connections.last orelse return error.NotAccepted;+ if (node == before) {+ return error.NotAccepted;+ }+ return .{ client, @fieldParentPtr("node", node) };+ }+};++// epoll and kqueue both report in the order things became ready, so making the+// deferred event ready first and the socket readable second puts the batch in+// the order that used to be fatal: the recycle before the event that names it.+test "server: a shutdown in the same batch as a readable connection" {+ var lt = try LoopTest.init();+ defer lt.deinit();++ const client, _ = try lt.accept();+ defer sys_net.close(client);++ // shutdown first...+ lt.server.shutdown();+ // ...then the request, so it lands behind it in the batch+ try sys_net.writeAll(client, "GET /json/version HTTP/1.1\r\n\r\n");++ // beginShutdown drops every http connection, so running it where it+ // arrived left the rest of the batch pointing at a recycled (or freed)+ // Connection. The request has to be answered first.+ try testing.expectEqual(false, lt.server.runOnce());+ try lt.expectResponse(client, "HTTP/1.1 200 OK\r\n");+ try testing.expect(lt.server.shutdown_begun);+}++test "server: an accept at the connection limit in the same batch as a readable connection" {+ var lt = try LoopTest.init();+ defer lt.deinit();++ const client, _ = try lt.accept();+ defer sys_net.close(client);++ // one connection, and no room for another: the next accept has to make+ // room by disconnecting an idle connection+ lt.server.max_connections = 1;+ try testing.expectEqual(1, lt.server.liveConnections());++ // the accept first...+ const second = try sys_net.connect(<.address);+ defer sys_net.close(second);+ // ...then the request, so it lands behind it in the batch+ try sys_net.writeAll(client, "GET /json/version HTTP/1.1\r\n\r\n");++ // saturated() picks the idle connection to drop, which is the one the+ // batch is still holding a pointer to. Its request comes first.+ _ = lt.server.runOnce();+ try lt.expectResponse(client, "HTTP/1.1 200 OK\r\n");+}++// Why the ordering matters rather than a flag on the connection: past the+// pool's retain count a release doesn't recycle, it destroys, so a stale+// pointer isn't merely pointing at the wrong client -- it's dangling.+test "server: releasing past the pool's retain destroys the connection" {+ var lt = try LoopTest.init();+ defer lt.deinit();++ const pool = <.server.http_connection_pool;+ const retain = pool.retain;++ const clients = try testing.allocator.alloc(posix.socket_t, retain + 1);+ defer testing.allocator.free(clients);+ var doomed: *Connection = undefined;+ for (clients, 0..) |*client, i| {+ client.*, const conn = try lt.accept();+ if (i == clients.len - 1) {+ doomed = conn;+ }+ }+ defer for (clients) |client| sys_net.close(client);+ try testing.expectEqual(retain + 1, pool.live);+ try testing.expectEqual(0, pool.free_count);++ while (lt.server.http_connections.first) |node| {+ http.disconnect(lt.server, @fieldParentPtr("node", node));+ }++ // retain + 1 released but only retain came back, and `doomed` is not among+ // them: it was destroyed, not pooled.+ try testing.expectEqual(0, pool.live);+ try testing.expectEqual(retain, pool.free_count);+ var node = pool.free.first;+ while (node) |n| : (node = n.next) {+ try testing.expect(@as(*Connection, @fieldParentPtr("node", n)) != doomed);+ }+}++test "server: the connection budget is bounded by buffer memory" {+ const opts = &testing.test_config.mode.serve;+ const original = opts.cdp_max_http_message_size;+ defer opts.cdp_max_http_message_size = original;++ // whatever NOFILE happens to be, we never sign up for more read buffers+ // than fdBudget's ceiling pays for (kept in step with it by hand)+ const ceiling = 64 * 1024 * 1024;+ for ([_]u14{ 1024, 4096, 16383 }) |size| {+ opts.cdp_max_http_message_size = size;+ const budget = fdBudget(testing.test_app.config);+ try testing.expect(budget * size <= ceiling);+ try testing.expect(budget >= 8);+ }+}++test "server: accepted sockets get TCP keepalive" {+ var lt = try LoopTest.init();+ defer lt.deinit();++ const client, const conn = try lt.accept();+ defer sys_net.close(client);++ // Driver.tick leans on this for liveness: without it a peer that goes+ // away without a FIN holds a worker and a connection slot forever.+ var value: c_int = 0;+ var len: posix.socklen_t = @sizeOf(c_int);+ try testing.expectEqual(0, std.c.getsockopt(conn.socket, posix.SOL.SOCKET, posix.SO.KEEPALIVE, &value, &len));+ try testing.expectEqual(1, value);++ http.disconnect(lt.server, conn);+}++test "server: the http read buffer is sized by --cdp-max-http-message-size" {+ // the pool is built in Server.init, so this has to move first+ const opts = &testing.test_config.mode.serve;+ const original = opts.cdp_max_http_message_size;+ defer opts.cdp_max_http_message_size = original;+ opts.cdp_max_http_message_size = 8192;++ var lt = try LoopTest.init();+ defer lt.deinit();++ const client, const conn = try lt.accept();+ defer sys_net.close(client);++ try testing.expectEqual(8192, conn.buffer.buf.len);++ http.disconnect(lt.server, conn);+}++test "server: http connections stay ordered by deadline" {+ var lt = try LoopTest.init();+ defer lt.deinit();++ // A connects and completes a request, so it gets the served deadline...+ const client_a, const conn_a = try lt.accept();+ defer sys_net.close(client_a);+ try sys_net.writeAll(client_a, "GET /json/version HTTP/1.1\r\n\r\n");+ const readable: IOEvent.ReadWrite = .{ .target = .{ .http = conn_a }, .readable = true, .writable = false, .hangup = false };+ http.processEvent(lt.server, conn_a, readable, lp.datetime.milliTimestamp(.boot));++ // ...and B connects after it, so it sits behind A in the list.+ const client_b, const conn_b = try lt.accept();+ defer sys_net.close(client_b);++ // run() reads the wait timeout off the head and the eviction sweep stops+ // at the first unexpired entry, so a later node may never hold an earlier+ // deadline.+ var node = lt.server.http_connections.first;+ var previous: u64 = 0;+ while (node) |n| : (node = n.next) {+ const conn: *Connection = @fieldParentPtr("node", n);+ try testing.expect(conn.deadline >= previous);+ previous = conn.deadline;+ }++ http.disconnect(lt.server, conn_a);+ http.disconnect(lt.server, conn_b);+}diff --git a/src/server/WS.zig b/src/server/WS.zigindex 9f99faa35..1ca441840 100644--- a/src/server/WS.zig+++ b/src/server/WS.zig@@ -223,7 +223,9 @@ fn ReaderM(comptime EXPECT_MASK: bool) type {buf = self.buf[0..len];// we need more datareturn null;- } else if (buf.len < message_len) {+ }++ if (buf.len < message_len) {// we need more datareturn null;}@@ -396,7 +398,7 @@ fn ReaderM(comptime EXPECT_MASK: bool) type {// don't need to narrow it first; unrecognized errors return null.pub fn errorReply(err: anyerror) ?[]const u8 {return switch (err) {- error.TooLarge => &CLOSE_TOO_BIG,+ error.TooLarge, error.InboxBacklog => &CLOSE_TOO_BIG,error.Masked,error.NotMasked,error.ReservedFlags,@@ -580,3 +582,31 @@ test "reader: reclaims buffer after a run of small messages" {try testing.expect(reader.buf.len > RECLAIM_TO);try testing.expectEqual(@as(usize, 0), reader.small_message_streak);}++test "reader: control frame arriving in pieces" {+ const allocator = testing.allocator;+ var reader = try ReaderNoMask.init(allocator, 1024 * 1024);+ defer reader.deinit();++ // A ping with a 114 byte payload; only the header and 50 bytes of it+ // have arrived. The control branch skips the "is the whole frame here"+ // check that the data branches do, so next() used to slice past len.+ var frame: [2 + 114]u8 = undefined;+ frame[0] = 128 | 9; // FIN + ping+ frame[1] = 114;+ @memset(frame[2..], 'a');++ const partial = frame[0 .. 2 + 50];+ @memcpy(reader.readBuf()[0..partial.len], partial);+ reader.len += partial.len;+ try testing.expectEqual(@as(?Message, null), try reader.next());++ // the rest arrives+ const rest = frame[2 + 50 ..];+ @memcpy(reader.readBuf()[0..rest.len], rest);+ reader.len += rest.len;++ const msg = (try reader.next()) orelse return error.NoMessage;+ try testing.expectEqual(.ping, msg.type);+ try testing.expectEqual(114, msg.data.len);+}diff --git a/src/server/http.zig b/src/server/http.zigindex acc666d34..2390e638a 100644--- a/src/server/http.zig+++ b/src/server/http.zig@@ -44,10 +44,6 @@ pub const Connection = struct {// When a keepalive (or just connected) connection should be closeddeadline: u64,- // Whether at least one request has been answered. A deadline miss before- // that is a client that connected and never spoke; after, an idle keepalive.- served: bool,-// Response that couldn't be sent without blocking. Socket will switch to// "write-mode" until it's drained.pending: ?Writing,@@ -236,6 +232,7 @@ pub const Connection = struct {live: usize, // acquired and not yet releasedretain: usize, // min # to keepfree_count: usize, // # of connections available in free+ buffer_size: usize, // --cdp-max-http-message-sizepub fn init(app: *App) !Pool {const retain = app.config.maxConnections();@@ -245,6 +242,7 @@ pub const Connection = struct {.free_count = 0,.retain = retain,.allocator = app.allocator,+ .buffer_size = app.config.cdpMaxHTTPMessageSize(),};errdefer self.deinit();@@ -288,7 +286,6 @@ pub const Connection = struct {conn.socket = -1;conn.address = .{ .ip4 = .unspecified(0) };conn.deadline = 0;- conn.served = false;conn.pending = null;conn.buffer.len = 0;conn.state = .header;@@ -306,10 +303,9 @@ pub const Connection = struct {.socket = -1,.address = .{ .ip4 = .unspecified(0) },.deadline = 0,- .served = false,.pending = null,.state = .header,- .buffer = try .init(allocator, 4096),+ .buffer = try .init(allocator, self.buffer_size),};return conn;}@@ -321,11 +317,8 @@ pub const Connection = struct {};};-// How long a keepalive connection may sit without a complete request before-// we close it.-const IDLE_TIMEOUT_MS = 10_000;--pub const FIRST_TIMEOUT_MS = 5_000;+// How long a connection may sit without completing a request before we close it.+pub const IDLE_TIMEOUT_MS = 10_000;pub fn processEvent(server: *Server, conn: *Connection, rw: Server.IOEvent.ReadWrite, now: u64) void {if (conn.pending != null) {@@ -402,6 +395,7 @@ fn processHTTP(server: *Server, conn: *Connection, now: u64) !bool {if (try serveHTTP(server, conn, req) == .upgraded) {// The fd moved to a WebSocket (and out of server.http); all// that's left of this Connection is to recycle it.+ // upgradeConnection already took it out of http_connections.recycle(server, conn);return true;}@@ -796,7 +790,6 @@ fn recycle(server: *Server, conn: *Connection) void {}fn touch(server: *Server, conn: *Connection, now: u64) void {- conn.served = true;conn.deadline = now + IDLE_TIMEOUT_MS;const node = &conn.node;if (server.http_connections.last == node) {diff --git a/src/testing.zig b/src/testing.zigindex 8fdeac8c1..bfc8c7f50 100644--- a/src/testing.zig+++ b/src/testing.zig@@ -524,7 +524,7 @@ var test_ws_server_thread: ?std.Thread = null;var sse_flag = std.atomic.Value(bool).init(false);var sse_reconnect_hits = std.atomic.Value(usize).init(0);-var test_config: Config = undefined;+pub var test_config: Config = undefined;test "tests:beforeAll" {log.opts.level = .warn;