Skip to content

Lightpanda: the headless browser designed for AI and automation

Hardening

Karl Seguin17c2aa7d1f84
diff --git a/src/Inbox.zig b/src/Inbox.zig
index d5a4092d6..24cdd80dc 100644
--- a/src/Inbox.zig
+++ b/src/Inbox.zig
@@ -37,6 +37,10 @@ const Inbox = @This();
mutex: std.Io.Mutex = .init,
queue: DoublyLinkedList = .{},
+// Payload bytes sitting in the queue. Used to disconnect a client if we've
+// fallen too far behind (largely to protect against a misbehaving client)
+queued_bytes: usize = 0,
+
// One-way latch, set by the worker's drainInbox the first time it
// observes a .disconnect (or .close) and never cleared. Ensures that, on
// multiple drains, the terminated state is preserved / communicated. This is
@@ -51,6 +55,13 @@ pub fn deinit(self: *Inbox) void {
const msg: *Message = @fieldParentPtr("node", node);
msg.deinit();
}
+ self.queued_bytes = 0;
+}
+
+pub fn queuedBytes(self: *Inbox) usize {
+ self.mutex.lockUncancelable(lp.io);
+ defer self.mutex.unlock(lp.io);
+ return self.queued_bytes;
}
pub fn push(self: *Inbox, arena: *lp.Arena, payload: Message.Payload) void {
@@ -61,6 +72,7 @@ pub fn push(self: *Inbox, arena: *lp.Arena, payload: Message.Payload) void {
msg.* = .{ .payload = payload, .arena = arena };
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
+ self.queued_bytes += payload.size();
self.queue.append(&msg.node);
}
@@ -68,7 +80,9 @@ pub fn pop(self: *Inbox) ?*Message {
self.mutex.lockUncancelable(lp.io);
defer self.mutex.unlock(lp.io);
const node = self.queue.popFirst() orelse return null;
- return @fieldParentPtr("node", node);
+ const msg: *Message = @fieldParentPtr("node", node);
+ self.queued_bytes -= msg.payload.size();
+ return msg;
}
// Peek for a message matching `predicate` without removing it. Used by
@@ -99,6 +113,7 @@ pub fn popIf(self: *Inbox, predicate: *const fn (*Message) bool) ?*Message {
const msg: *Message = @fieldParentPtr("node", node);
if (predicate(msg)) {
self.queue.remove(node);
+ self.queued_bytes -= msg.payload.size();
return msg;
}
}
@@ -141,6 +156,14 @@ pub const Message = struct {
// pushes this on peer EOF, fatal WS framing error, or
// (now) JSON parse failure.
disconnect: ?anyerror,
+
+ pub fn size(self: Payload) usize {
+ return switch (self) {
+ .cdp => |c| c.raw.len,
+ .bidi, .ping => |b| b.len,
+ .close, .disconnect => 0,
+ };
+ }
};
pub const Cdp = struct {
@@ -346,3 +369,48 @@ test "Inbox: popIf picks first match in FIFO order" {
defer m.deinit();
try testing.expectEqual("first", m.payload.ping);
}
+
+test "Inbox: queued bytes track the payloads" {
+ const arena_pool = &testing.test_app.arena_pool;
+
+ var inbox = Inbox{};
+ defer inbox.deinit();
+
+ try testing.expectEqual(0, inbox.queuedBytes());
+
+ {
+ const arena = try arena_pool.acquire(.tiny, "inbox test");
+ inbox.push(arena, .{ .ping = try arena.dupe(u8, "12345") });
+ }
+ try testing.expectEqual(5, inbox.queuedBytes());
+
+ {
+ // control payloads are free; only what the peer sends counts
+ const arena = try arena_pool.acquire(.tiny, "inbox test");
+ inbox.push(arena, .{ .disconnect = null });
+ }
+ try testing.expectEqual(5, inbox.queuedBytes());
+
+ {
+ const arena = try arena_pool.acquire(.tiny, "inbox test");
+ inbox.push(arena, .{ .bidi = try arena.dupe(u8, "abc") });
+ }
+ try testing.expectEqual(8, inbox.queuedBytes());
+
+ // popIf cherry-picks out of the middle, and has to pay the same toll
+ {
+ const m = inbox.popIf(struct {
+ fn f(msg: *Message) bool {
+ return msg.payload == .bidi;
+ }
+ }.f).?;
+ defer m.deinit();
+ }
+ try testing.expectEqual(5, inbox.queuedBytes());
+
+ {
+ const m = inbox.pop().?;
+ defer m.deinit();
+ }
+ try testing.expectEqual(0, inbox.queuedBytes());
+}
diff --git a/src/Metrics.zig b/src/Metrics.zig
index eff9fc6f7..89a351495 100644
--- a/src/Metrics.zig
+++ b/src/Metrics.zig
@@ -23,7 +23,8 @@ const Metrics = @This();
const Driver = @import("server/Driver.zig").Protocol;
serve_http_requests: CounterEnum("status", @import("network/http.zig").StatusCategory) = .{},
-serve_http_evictions: CounterEnum("reason", enum { first_request, idle }) = .{},
+serve_http_evictions: Counter = .{},
+serve_inbox_backlog: Counter = .{},
serve_connections: CounterEnum("driver", Driver) = .{},
serve_connection_limit: Counter = .{},
serve_active_connections: GaugeEnum("driver", Driver) = .{},
@@ -95,7 +96,8 @@ robots_access: CounterEnum("result", enum { allow, deny }) = .{},
// compile error.
const help = .{
.serve_http_requests = "HTTP responses sent, by status category (includes the pre-parse 400/413 rejections)",
- .serve_http_evictions = "HTTP connections closed for exceeding a deadline; first_request never completed a request, idle is a keepalive connection that went quiet",
+ .serve_http_evictions = "HTTP connections closed for sitting past their deadline without completing a request",
+ .serve_inbox_backlog = "Websocket connections closed for queueing more unprocessed messages than the worker could drain",
.serve_connections = "Websocket connections accepted, by driver protocol",
.serve_connection_limit = "Accepts deferred because the connection budget was full: the listener pauses until a slot frees (counted before any handshake, so no driver label)",
.serve_active_connections = "Currently connected clients, by driver protocol",
diff --git a/src/server/Link.zig b/src/server/Link.zig
index 621fb9212..faeed6459 100644
--- a/src/server/Link.zig
+++ b/src/server/Link.zig
@@ -39,6 +39,16 @@ const ArenaAllocator = std.heap.ArenaAllocator;
// inbox's own.
const Link = @This();
+// A send that has flipped to blocking (below) waits at most this long for the
+// peer to take more bytes.
+const SEND_TIMEOUT_S = 5;
+
+// The loop reads as fast as it can. The worker _can_ be slow to process its
+// inbox (e.g. stuck in a syncRequest). Still, we want _some_ limit on how much
+// data is queued. This is 32 * the configured max message size. Which should
+// be plenty for a well-behaving client.
+const INBOX_BACKLOG_MESSAGES = 32;
+
inbox: *Inbox,
arena_pool: *ArenaPool,
socket: posix.socket_t,
@@ -46,6 +56,7 @@ socket_flags: usize,
protocol: Driver.Protocol,
reader: WS.Reader,
send_arena: ArenaAllocator,
+max_inbox_backlog: usize,
pub fn init(
self: *Link,
@@ -60,6 +71,9 @@ pub fn init(
lp.assert(socket_flags & nonblocking == nonblocking, "Link.init blocking", .{});
}
+ const timeout = std.mem.toBytes(posix.timeval{ .sec = SEND_TIMEOUT_S, .usec = 0 });
+ try posix.setsockopt(socket, posix.SOL.SOCKET, posix.SO.SNDTIMEO, &timeout);
+
const config = app.config;
const allocator = app.allocator;
@@ -71,6 +85,7 @@ pub fn init(
.socket_flags = socket_flags,
.reader = try .init(allocator, config.cdpMaxMessageSize()),
.send_arena = ArenaAllocator.init(allocator),
+ .max_inbox_backlog = @as(usize, config.cdpMaxMessageSize()) * INBOX_BACKLOG_MESSAGES,
};
}
@@ -98,11 +113,18 @@ pub fn send(self: *Link, data: []const u8) !void {
// queue with its own allocations. On WouldBlock we flip the
// socket to blocking for this write and flip it back after.
// Should virtually never happen.
- lp.assert(changed_to_blocking == false, "Link double block", .{});
+ if (changed_to_blocking) {
+ // We already flipped, so this is SO_SNDTIMEO firing: the
+ // peer has stopped draining entirely. Treat it as gone
+ // rather than parking the worker on it.
+ return error.Timeout;
+ }
changed_to_blocking = true;
_ = try sys_net.fcntl(self.socket, posix.F.SETFL, self.socket_flags & ~@as(u32, @bitCast(posix.O{ .NONBLOCK = true })));
continue :LOOP;
},
+ // a signal landed mid-write; nothing was written
+ error.Interrupted => continue :LOOP,
else => return err,
};
@@ -156,6 +178,11 @@ pub const Read = struct {
// Server loop. The socket is readable
pub fn readAvailable(self: *Link, budget: usize) !Read {
+ if (self.inbox.queuedBytes() >= self.max_inbox_backlog) {
+ lp.metrics.serve_inbox_backlog.incr();
+ return error.InboxBacklog;
+ }
+
var pushed = false;
var remaining = budget;
while (remaining > 0) {
@@ -267,7 +294,95 @@ fn pushBiDi(self: *Link, bytes: []const u8, pushed: *bool) !bool {
return true;
}
-// Called from the worker (Driver.shutdown) to break the loop's read.
+// Server loop, closing only the read side. The worker can still send a message
+// (e.g. a close frame).
pub fn shutdown(self: *Link) void {
sys_net.shutdown(self.socket, .recv) catch {};
}
+
+const testing = @import("../testing.zig");
+
+test "link: send gives up when the peer stops reading" {
+ var pair: [2]posix.socket_t = undefined;
+ if (std.c.socketpair(posix.AF.LOCAL, posix.SOCK.STREAM, 0, &pair) != 0) {
+ return error.SocketPairFailed;
+ }
+ defer sys_net.close(pair[0]);
+ defer sys_net.close(pair[1]);
+
+ const small = std.mem.toBytes(@as(c_int, 4096));
+ try posix.setsockopt(pair[0], posix.SOL.SOCKET, posix.SO.RCVBUF, &small);
+ try posix.setsockopt(pair[1], posix.SOL.SOCKET, posix.SO.SNDBUF, &small);
+
+ const nonblocking = @as(u32, @bitCast(posix.O{ .NONBLOCK = true }));
+ const flags = try sys_net.fcntl(pair[1], posix.F.GETFL, 0);
+ _ = try sys_net.fcntl(pair[1], posix.F.SETFL, flags | nonblocking);
+
+ var inbox: Inbox = .{};
+ defer inbox.deinit();
+
+ var link: Link = undefined;
+ try link.init(testing.test_app, pair[1], .cdp, &inbox);
+ defer link.deinit();
+
+ // shorten the blocking window so the test doesn't sit out the real one
+ const timeout = std.mem.toBytes(posix.timeval{ .sec = 0, .usec = 50_000 });
+ try posix.setsockopt(pair[1], posix.SOL.SOCKET, posix.SO.SNDTIMEO, &timeout);
+
+ const payload = try testing.allocator.alloc(u8, 1024 * 1024);
+ defer testing.allocator.free(payload);
+ @memset(payload, 'a');
+
+ // Nobody drains pair[0]. send() flips to blocking to finish the write;
+ // unbounded, that parks the worker forever and, because shutdown only
+ // half-closes the read side, hangs the whole process on SIGINT.
+ try testing.expectError(error.Timeout, link.send(payload));
+
+ // and the socket has to be non-blocking again for the run loop's reads
+ try testing.expectEqual(flags | nonblocking, try sys_net.fcntl(pair[1], posix.F.GETFL, 0));
+}
+
+test "link: stops reading once the worker's inbox backs up" {
+ var pair: [2]posix.socket_t = undefined;
+ if (std.c.socketpair(posix.AF.LOCAL, posix.SOCK.STREAM, 0, &pair) != 0) {
+ return error.SocketPairFailed;
+ }
+ defer sys_net.close(pair[0]);
+ defer sys_net.close(pair[1]);
+
+ const nonblocking = @as(u32, @bitCast(posix.O{ .NONBLOCK = true }));
+ const flags = try sys_net.fcntl(pair[1], posix.F.GETFL, 0);
+ _ = try sys_net.fcntl(pair[1], posix.F.SETFL, flags | nonblocking);
+
+ var inbox: Inbox = .{};
+ defer inbox.deinit();
+
+ var link: Link = undefined;
+ try link.init(testing.test_app, pair[1], .cdp, &inbox);
+ defer link.deinit();
+
+ // a ceiling below one max-size message would refuse what was configured
+ try testing.expect(link.max_inbox_backlog > testing.test_app.config.cdpMaxMessageSize());
+
+ // an empty inbox reads normally (nothing pending, so nothing pushed)
+ const read = try link.readAvailable(1024);
+ try testing.expectEqual(true, read.keep);
+ try testing.expectEqual(false, read.pushed);
+
+ // a worker that has fallen this far behind isn't going to catch up
+ {
+ const arena = try testing.test_app.arena_pool.acquire(link.max_inbox_backlog, "backlog test");
+ const payload = try arena.allocator().alloc(u8, link.max_inbox_backlog);
+ inbox.push(arena, .{ .bidi = payload });
+ }
+ try testing.expectEqual(link.max_inbox_backlog, inbox.queuedBytes());
+ try testing.expectError(error.InboxBacklog, link.readAvailable(1024));
+
+ // and it recovers once the worker drains
+ {
+ const msg = inbox.pop().?;
+ defer msg.deinit();
+ }
+ try testing.expectEqual(0, inbox.queuedBytes());
+ _ = try link.readAvailable(1024);
+}
diff --git a/src/server/Server.zig b/src/server/Server.zig
index 34b781a00..ad26fc98e 100644
--- a/src/server/Server.zig
+++ b/src/server/Server.zig
@@ -280,47 +280,64 @@ pub fn run(self: *Server) void {
return;
};
- while (true) {
- const deadline = blk: {
- // self.http_connections is ordered by deadline
- const node = self.http_connections.first orelse break :blk null;
- const conn: *Connection = @fieldParentPtr("node", node);
- break :blk conn.deadline -| lp.datetime.milliTimestamp(.boot);
- };
+ while (self.runOnce()) {}
+}
- var events = self.io_engine.wait(deadline);
- const now = lp.datetime.milliTimestamp(.boot);
- while (events.next()) |event| {
- switch (event) {
- .accept => self.accept(now) catch |err| log.err(.serve, "accept", .{ .err = err }),
- .read_write => |rw| switch (rw.target) {
- .http => |conn| http.processEvent(self, conn, rw, now),
- .ws => |ws| self.processWebSocketEvent(ws, rw),
- },
- .signal => self.drainWorkerQueue(),
- .shutdown => self.beginShutdown(),
- }
- }
+fn runOnce(self: *Server) bool {
+ const deadline = blk: {
+ // self.http_connections is ordered by deadline
+ const node = self.http_connections.first orelse break :blk null;
+ const conn: *Connection = @fieldParentPtr("node", node);
+ break :blk conn.deadline -| lp.datetime.milliTimestamp(.boot);
+ };
- // evict http connections that have passed their deadline
- while (self.http_connections.first) |node| {
- const conn: *Connection = @fieldParentPtr("node", node);
- if (conn.deadline > now) {
- // self.http_connections is ordered by deadline, so as soon as we find one
- // that hasn't reach its deadline, none of the ones after can.
- break;
- }
- lp.metrics.serve_http_evictions.incr(if (conn.served) .idle else .first_request);
- http.disconnect(self, conn);
+ var events = self.io_engine.wait(deadline);
+ const now = lp.datetime.milliTimestamp(.boot);
+
+ var pending_accept = false;
+ var pending_signal = false;
+ var pending_shutdown = false;
+ while (events.next()) |event| {
+ switch (event) {
+ .accept => pending_accept = true,
+ .read_write => |rw| switch (rw.target) {
+ .http => |conn| http.processEvent(self, conn, rw, now),
+ .ws => |ws| self.processWebSocketEvent(ws, rw),
+ },
+ .signal => pending_signal = true,
+ .shutdown => pending_shutdown = true,
}
+ }
- // Keep looping until every worker has released; their terminate was
- // requested in beginShutdown. Only then is it safe to return (deinit
- // frees state the workers reference).
- if (self.shutdown_begun and self.websocket_pool.live == 0) {
- return;
+ // signal first: a worker that released frees a slot the accept can use.
+ if (pending_signal) {
+ self.drainWorkerQueue();
+ }
+ if (pending_accept) {
+ self.accept(now) catch |err| log.err(.serve, "accept", .{ .err = err });
+ }
+ // shutdown last: it's terminal, and draining the queue first keeps it from
+ // walking a websocket whose worker has already gone.
+ if (pending_shutdown) {
+ self.beginShutdown();
+ }
+
+ // evict http connections that have passed their deadline
+ while (self.http_connections.first) |node| {
+ const conn: *Connection = @fieldParentPtr("node", node);
+ if (conn.deadline > now) {
+ // self.http_connections is ordered by deadline, so as soon as we find one
+ // that hasn't reach its deadline, none of the ones after can.
+ break;
}
+ lp.metrics.serve_http_evictions.incr();
+ http.disconnect(self, conn);
}
+
+ if (self.shutdown_begun and self.websocket_pool.live == 0) {
+ return false;
+ }
+ return true;
}
fn accept(self: *Server, now: u64) !void {
@@ -349,6 +366,8 @@ fn accept(self: *Server, now: u64) !void {
}
};
errdefer sys_net.close(socket);
+ configureSocket(socket);
+
const peer = sys_net.addressFromSockaddr(@ptrCast(&address));
if (comptime lp.IS_DEBUG) {
log.debug(.serve, "client connected", .{ .address = peer });
@@ -360,7 +379,7 @@ fn accept(self: *Server, now: u64) !void {
conn.address = peer;
try self.io_engine.monitorHTTP(conn);
- conn.deadline = now + http.FIRST_TIMEOUT_MS;
+ conn.deadline = now + http.IDLE_TIMEOUT_MS;
self.http_connections.append(&conn.node);
if (self.liveConnections() == self.max_connections) {
@@ -369,6 +388,33 @@ fn accept(self: *Server, now: u64) !void {
}
}
+// A peer that goes away without a FIN (a killed VM, a NAT timeout) leaves an
+// upgraded connection holding a worker, a browser and a --cdp-max-connections
+// slot; nothing above the socket notices. Keepalive is what ends it, and
+// Driver.tick's wait cadence is built on that. Best effort: a socket we can't
+// configure is still a usable socket.
+fn configureSocket(socket: posix.socket_t) void {
+ setSocketOption(socket, posix.SOL.SOCKET, posix.SO.KEEPALIVE, @as(c_int, 1), "SO_KEEPALIVE");
+
+ const idle_opt = switch (builtin.os.tag) {
+ .macos, .ios => posix.TCP.KEEPALIVE,
+ else => posix.TCP.KEEPIDLE,
+ };
+ setSocketOption(socket, posix.IPPROTO.TCP, idle_opt, Config.CDP_KEEPALIVE_IDLE_S, "TCP_KEEPIDLE");
+ setSocketOption(socket, posix.IPPROTO.TCP, posix.TCP.KEEPINTVL, Config.CDP_KEEPALIVE_INTVL_S, "TCP_KEEPINTVL");
+ setSocketOption(socket, posix.IPPROTO.TCP, posix.TCP.KEEPCNT, Config.CDP_KEEPALIVE_CNT, "TCP_KEEPCNT");
+
+ if (comptime builtin.os.tag == .linux) {
+ setSocketOption(socket, posix.IPPROTO.TCP, std.os.linux.TCP.USER_TIMEOUT, Config.CDP_TCP_USER_TIMEOUT_MS, "TCP_USER_TIMEOUT");
+ }
+}
+
+fn setSocketOption(socket: posix.socket_t, level: i32, option: u32, value: anytype, comptime name: []const u8) void {
+ posix.setsockopt(socket, level, option, &std.mem.toBytes(value)) catch |err| {
+ log.warn(.serve, "setsockopt", .{ .err = err, .option = name });
+ };
+}
+
fn liveConnections(self: *const Server) usize {
return self.http_connection_pool.live + self.websocket_pool.live;
}
@@ -399,8 +445,8 @@ fn saturated(self: *Server) !void {
fn processWebSocketEvent(self: *Server, ws: *WebSocket, rw: IOEvent.ReadWrite) void {
if (ws.monitored == false) {
- // can have an event that comes in during the same batch as a release
- // and there's no guarantee about the order that we process them in.
+ // only attachWorker puts a websocket in the poll set, and only once
+ // the driver is set; an unmonitored slot has no business here.
return;
}
@@ -516,15 +562,17 @@ fn attachWorker(self: *Server, ws: *WebSocket, driver: Driver) void {
fn releaseWorker(self: *Server, ws: *WebSocket, notify: *std.Io.Event) void {
if (ws.monitored) {
+ ws.monitored = false;
self.io_engine.remove(ws.socket);
}
self.releaseWebSocket(ws);
+ // The worker is free to deinit its driver and close the fd from here.
notify.set(lp.io);
}
// Frees the slot once the loop is done with the fd. The fd itself is closed
// by whoever owns the end of its life: the worker after its driver's deinit,
-// or openWebSocket when there never was a worker.
+// or upgradeConnection when there never was a worker.
fn releaseWebSocket(self: *Server, ws: *WebSocket) void {
self.websockets.remove(&ws.node);
lp.metrics.serve_active_connections.decr(ws.protocol);
@@ -585,8 +633,9 @@ fn fdBudget(config: *const Config) usize {
};
break :blk limit.cur;
};
- // unlimited is really "as many as we'd care to hold 4KB buffers for"
- const budget = @min(soft, 1 << 16) -| reserve;
+ // put some limit incase of a unlimited or very large rlimit
+ const ceiling = (64 * 1024 * 1024) / @max(@as(u64, config.cdpMaxHTTPMessageSize()), 1);
+ const budget = @min(soft, ceiling) -| reserve;
return @intCast(@max(budget, 8));
}
@@ -640,11 +689,14 @@ const Worker = struct {
// The loop is done with us once releaseConnection returns; the
// driver's deinit may still tick the client, without a producer.
defer driver.browser.http_client.driver_link_active = false;
- // It's possible the terminate flag is set. Our teardown (e.g. cdp.deinit()
- // and bidi.deinit() in our caller) might need V8 in a usable state. So
- // clear the terminate flag
- driver.browser.env.cancelTerminate();
+ // Release first: until the loop has let go of this websocket it can
+ // still drop the link, and onLinkDisconnect requests a terminate. Doing
+ // it the other way round left that request landing after the cancel,
+ // so the teardown below ran with a pending terminate -- which is the
+ // one thing the cancel is here to prevent (cdp.deinit() and
+ // bidi.deinit() in our caller need V8 in a usable state).
Worker.releaseConnection(server, ws);
+ driver.browser.env.cancelTerminate();
}
// Worker -> loop: synchronous release. Blocks until the loop has dropped the
@@ -2036,3 +2088,220 @@ const TestClient = struct {
}
}
};
+
+// A server of our own, bound to an ephemeral port and never run(): these
+// tests drive its handlers by hand to reproduce what one event batch does.
+// The real test server (port 9583) is shared and can't be torn down.
+const LoopTest = struct {
+ server: *Server,
+ address: sys_net.IpAddress,
+
+ fn init() !LoopTest {
+ const server = try Server.init(testing.test_app, .{ .ip4 = .loopback(0) });
+ errdefer server.deinit();
+ server.protocols = .{ .cdp = true, .webdriver = true };
+ // run() does this; runOnce() on its own would never see an accept
+ try server.io_engine.monitorListener(server.listener);
+
+ var bound: posix.sockaddr.storage = undefined;
+ var bound_len: posix.socklen_t = @sizeOf(posix.sockaddr.storage);
+ try sys_net.getsockname(server.listener, @ptrCast(&bound), &bound_len);
+
+ return .{ .server = server, .address = sys_net.addressFromSockaddr(@ptrCast(&bound)) };
+ }
+
+ fn deinit(self: *LoopTest) void {
+ self.server.deinit();
+ }
+
+ fn expectResponse(self: *const LoopTest, client: posix.socket_t, prefix: []const u8) !void {
+ _ = self;
+ var buf: [512]u8 = undefined;
+ const n = try posix.read(client, &buf);
+ try testing.expect(std.mem.startsWith(u8, buf[0..n], prefix));
+ }
+
+ // Connects a client and runs the accept the loop would have run,
+ // returning the client's end and the Connection the loop now owns.
+ fn accept(self: *LoopTest) !struct { posix.socket_t, *Connection } {
+ const client = try sys_net.connect(&self.address);
+ errdefer sys_net.close(client);
+ // never block the suite on a response that isn't coming
+ const timeout = std.mem.toBytes(posix.timeval{ .sec = 5, .usec = 0 });
+ try posix.setsockopt(client, posix.SOL.SOCKET, posix.SO.RCVTIMEO, &timeout);
+
+ const before = self.server.http_connections.last;
+ try self.server.accept(lp.datetime.milliTimestamp(.boot));
+
+ const node = self.server.http_connections.last orelse return error.NotAccepted;
+ if (node == before) {
+ return error.NotAccepted;
+ }
+ return .{ client, @fieldParentPtr("node", node) };
+ }
+};
+
+// epoll and kqueue both report in the order things became ready, so making the
+// deferred event ready first and the socket readable second puts the batch in
+// the order that used to be fatal: the recycle before the event that names it.
+test "server: a shutdown in the same batch as a readable connection" {
+ var lt = try LoopTest.init();
+ defer lt.deinit();
+
+ const client, _ = try lt.accept();
+ defer sys_net.close(client);
+
+ // shutdown first...
+ lt.server.shutdown();
+ // ...then the request, so it lands behind it in the batch
+ try sys_net.writeAll(client, "GET /json/version HTTP/1.1\r\n\r\n");
+
+ // beginShutdown drops every http connection, so running it where it
+ // arrived left the rest of the batch pointing at a recycled (or freed)
+ // Connection. The request has to be answered first.
+ try testing.expectEqual(false, lt.server.runOnce());
+ try lt.expectResponse(client, "HTTP/1.1 200 OK\r\n");
+ try testing.expect(lt.server.shutdown_begun);
+}
+
+test "server: an accept at the connection limit in the same batch as a readable connection" {
+ var lt = try LoopTest.init();
+ defer lt.deinit();
+
+ const client, _ = try lt.accept();
+ defer sys_net.close(client);
+
+ // one connection, and no room for another: the next accept has to make
+ // room by disconnecting an idle connection
+ lt.server.max_connections = 1;
+ try testing.expectEqual(1, lt.server.liveConnections());
+
+ // the accept first...
+ const second = try sys_net.connect(&lt.address);
+ defer sys_net.close(second);
+ // ...then the request, so it lands behind it in the batch
+ try sys_net.writeAll(client, "GET /json/version HTTP/1.1\r\n\r\n");
+
+ // saturated() picks the idle connection to drop, which is the one the
+ // batch is still holding a pointer to. Its request comes first.
+ _ = lt.server.runOnce();
+ try lt.expectResponse(client, "HTTP/1.1 200 OK\r\n");
+}
+
+// Why the ordering matters rather than a flag on the connection: past the
+// pool's retain count a release doesn't recycle, it destroys, so a stale
+// pointer isn't merely pointing at the wrong client -- it's dangling.
+test "server: releasing past the pool's retain destroys the connection" {
+ var lt = try LoopTest.init();
+ defer lt.deinit();
+
+ const pool = &lt.server.http_connection_pool;
+ const retain = pool.retain;
+
+ const clients = try testing.allocator.alloc(posix.socket_t, retain + 1);
+ defer testing.allocator.free(clients);
+ var doomed: *Connection = undefined;
+ for (clients, 0..) |*client, i| {
+ client.*, const conn = try lt.accept();
+ if (i == clients.len - 1) {
+ doomed = conn;
+ }
+ }
+ defer for (clients) |client| sys_net.close(client);
+ try testing.expectEqual(retain + 1, pool.live);
+ try testing.expectEqual(0, pool.free_count);
+
+ while (lt.server.http_connections.first) |node| {
+ http.disconnect(lt.server, @fieldParentPtr("node", node));
+ }
+
+ // retain + 1 released but only retain came back, and `doomed` is not among
+ // them: it was destroyed, not pooled.
+ try testing.expectEqual(0, pool.live);
+ try testing.expectEqual(retain, pool.free_count);
+ var node = pool.free.first;
+ while (node) |n| : (node = n.next) {
+ try testing.expect(@as(*Connection, @fieldParentPtr("node", n)) != doomed);
+ }
+}
+
+test "server: the connection budget is bounded by buffer memory" {
+ const opts = &testing.test_config.mode.serve;
+ const original = opts.cdp_max_http_message_size;
+ defer opts.cdp_max_http_message_size = original;
+
+ // whatever NOFILE happens to be, we never sign up for more read buffers
+ // than fdBudget's ceiling pays for (kept in step with it by hand)
+ const ceiling = 64 * 1024 * 1024;
+ for ([_]u14{ 1024, 4096, 16383 }) |size| {
+ opts.cdp_max_http_message_size = size;
+ const budget = fdBudget(testing.test_app.config);
+ try testing.expect(budget * size <= ceiling);
+ try testing.expect(budget >= 8);
+ }
+}
+
+test "server: accepted sockets get TCP keepalive" {
+ var lt = try LoopTest.init();
+ defer lt.deinit();
+
+ const client, const conn = try lt.accept();
+ defer sys_net.close(client);
+
+ // Driver.tick leans on this for liveness: without it a peer that goes
+ // away without a FIN holds a worker and a connection slot forever.
+ var value: c_int = 0;
+ var len: posix.socklen_t = @sizeOf(c_int);
+ try testing.expectEqual(0, std.c.getsockopt(conn.socket, posix.SOL.SOCKET, posix.SO.KEEPALIVE, &value, &len));
+ try testing.expectEqual(1, value);
+
+ http.disconnect(lt.server, conn);
+}
+
+test "server: the http read buffer is sized by --cdp-max-http-message-size" {
+ // the pool is built in Server.init, so this has to move first
+ const opts = &testing.test_config.mode.serve;
+ const original = opts.cdp_max_http_message_size;
+ defer opts.cdp_max_http_message_size = original;
+ opts.cdp_max_http_message_size = 8192;
+
+ var lt = try LoopTest.init();
+ defer lt.deinit();
+
+ const client, const conn = try lt.accept();
+ defer sys_net.close(client);
+
+ try testing.expectEqual(8192, conn.buffer.buf.len);
+
+ http.disconnect(lt.server, conn);
+}
+
+test "server: http connections stay ordered by deadline" {
+ var lt = try LoopTest.init();
+ defer lt.deinit();
+
+ // A connects and completes a request, so it gets the served deadline...
+ const client_a, const conn_a = try lt.accept();
+ defer sys_net.close(client_a);
+ try sys_net.writeAll(client_a, "GET /json/version HTTP/1.1\r\n\r\n");
+ const readable: IOEvent.ReadWrite = .{ .target = .{ .http = conn_a }, .readable = true, .writable = false, .hangup = false };
+ http.processEvent(lt.server, conn_a, readable, lp.datetime.milliTimestamp(.boot));
+
+ // ...and B connects after it, so it sits behind A in the list.
+ const client_b, const conn_b = try lt.accept();
+ defer sys_net.close(client_b);
+
+ // run() reads the wait timeout off the head and the eviction sweep stops
+ // at the first unexpired entry, so a later node may never hold an earlier
+ // deadline.
+ var node = lt.server.http_connections.first;
+ var previous: u64 = 0;
+ while (node) |n| : (node = n.next) {
+ const conn: *Connection = @fieldParentPtr("node", n);
+ try testing.expect(conn.deadline >= previous);
+ previous = conn.deadline;
+ }
+
+ http.disconnect(lt.server, conn_a);
+ http.disconnect(lt.server, conn_b);
+}
diff --git a/src/server/WS.zig b/src/server/WS.zig
index 9f99faa35..1ca441840 100644
--- a/src/server/WS.zig
+++ b/src/server/WS.zig
@@ -223,7 +223,9 @@ fn ReaderM(comptime EXPECT_MASK: bool) type {
buf = self.buf[0..len];
// we need more data
return null;
- } else if (buf.len < message_len) {
+ }
+
+ if (buf.len < message_len) {
// we need more data
return null;
}
@@ -396,7 +398,7 @@ fn ReaderM(comptime EXPECT_MASK: bool) type {
// don't need to narrow it first; unrecognized errors return null.
pub fn errorReply(err: anyerror) ?[]const u8 {
return switch (err) {
- error.TooLarge => &CLOSE_TOO_BIG,
+ error.TooLarge, error.InboxBacklog => &CLOSE_TOO_BIG,
error.Masked,
error.NotMasked,
error.ReservedFlags,
@@ -580,3 +582,31 @@ test "reader: reclaims buffer after a run of small messages" {
try testing.expect(reader.buf.len > RECLAIM_TO);
try testing.expectEqual(@as(usize, 0), reader.small_message_streak);
}
+
+test "reader: control frame arriving in pieces" {
+ const allocator = testing.allocator;
+ var reader = try ReaderNoMask.init(allocator, 1024 * 1024);
+ defer reader.deinit();
+
+ // A ping with a 114 byte payload; only the header and 50 bytes of it
+ // have arrived. The control branch skips the "is the whole frame here"
+ // check that the data branches do, so next() used to slice past len.
+ var frame: [2 + 114]u8 = undefined;
+ frame[0] = 128 | 9; // FIN + ping
+ frame[1] = 114;
+ @memset(frame[2..], 'a');
+
+ const partial = frame[0 .. 2 + 50];
+ @memcpy(reader.readBuf()[0..partial.len], partial);
+ reader.len += partial.len;
+ try testing.expectEqual(@as(?Message, null), try reader.next());
+
+ // the rest arrives
+ const rest = frame[2 + 50 ..];
+ @memcpy(reader.readBuf()[0..rest.len], rest);
+ reader.len += rest.len;
+
+ const msg = (try reader.next()) orelse return error.NoMessage;
+ try testing.expectEqual(.ping, msg.type);
+ try testing.expectEqual(114, msg.data.len);
+}
diff --git a/src/server/http.zig b/src/server/http.zig
index acc666d34..2390e638a 100644
--- a/src/server/http.zig
+++ b/src/server/http.zig
@@ -44,10 +44,6 @@ pub const Connection = struct {
// When a keepalive (or just connected) connection should be closed
deadline: u64,
- // Whether at least one request has been answered. A deadline miss before
- // that is a client that connected and never spoke; after, an idle keepalive.
- served: bool,
-
// Response that couldn't be sent without blocking. Socket will switch to
// "write-mode" until it's drained.
pending: ?Writing,
@@ -236,6 +232,7 @@ pub const Connection = struct {
live: usize, // acquired and not yet released
retain: usize, // min # to keep
free_count: usize, // # of connections available in free
+ buffer_size: usize, // --cdp-max-http-message-size
pub fn init(app: *App) !Pool {
const retain = app.config.maxConnections();
@@ -245,6 +242,7 @@ pub const Connection = struct {
.free_count = 0,
.retain = retain,
.allocator = app.allocator,
+ .buffer_size = app.config.cdpMaxHTTPMessageSize(),
};
errdefer self.deinit();
@@ -288,7 +286,6 @@ pub const Connection = struct {
conn.socket = -1;
conn.address = .{ .ip4 = .unspecified(0) };
conn.deadline = 0;
- conn.served = false;
conn.pending = null;
conn.buffer.len = 0;
conn.state = .header;
@@ -306,10 +303,9 @@ pub const Connection = struct {
.socket = -1,
.address = .{ .ip4 = .unspecified(0) },
.deadline = 0,
- .served = false,
.pending = null,
.state = .header,
- .buffer = try .init(allocator, 4096),
+ .buffer = try .init(allocator, self.buffer_size),
};
return conn;
}
@@ -321,11 +317,8 @@ pub const Connection = struct {
};
};
-// How long a keepalive connection may sit without a complete request before
-// we close it.
-const IDLE_TIMEOUT_MS = 10_000;
-
-pub const FIRST_TIMEOUT_MS = 5_000;
+// How long a connection may sit without completing a request before we close it.
+pub const IDLE_TIMEOUT_MS = 10_000;
pub fn processEvent(server: *Server, conn: *Connection, rw: Server.IOEvent.ReadWrite, now: u64) void {
if (conn.pending != null) {
@@ -402,6 +395,7 @@ fn processHTTP(server: *Server, conn: *Connection, now: u64) !bool {
if (try serveHTTP(server, conn, req) == .upgraded) {
// The fd moved to a WebSocket (and out of server.http); all
// that's left of this Connection is to recycle it.
+ // upgradeConnection already took it out of http_connections.
recycle(server, conn);
return true;
}
@@ -796,7 +790,6 @@ fn recycle(server: *Server, conn: *Connection) void {
}
fn touch(server: *Server, conn: *Connection, now: u64) void {
- conn.served = true;
conn.deadline = now + IDLE_TIMEOUT_MS;
const node = &conn.node;
if (server.http_connections.last == node) {
diff --git a/src/testing.zig b/src/testing.zig
index 8fdeac8c1..bfc8c7f50 100644
--- a/src/testing.zig
+++ b/src/testing.zig
@@ -524,7 +524,7 @@ var test_ws_server_thread: ?std.Thread = null;
var sse_flag = std.atomic.Value(bool).init(false);
var sse_reconnect_hits = std.atomic.Value(usize).init(0);
-var test_config: Config = undefined;
+pub var test_config: Config = undefined;
test "tests:beforeAll" {
log.opts.level = .warn;