Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: replace Socket.IO destroyUpgrade timer with PortOS upgrade fallback (#12069)

Adam Eivya2f350a89708
diff --git a/server/index.js b/server/index.js
index 959d43b920..a695c683b3 100644
--- a/server/index.js
+++ b/server/index.js
@@ -17,6 +17,7 @@ import { getBuildIdentity } from './lib/buildIdentity.js';
import { httpCompression } from './lib/httpCompression.js';
import { PORTS } from './lib/ports.js';
import { isSmokeBoot } from './lib/runtimeEnv.js';
+import { installUpgradeFallback } from './lib/upgradeFallback.js';
import alertsRoutes from './routes/alerts.js';
import appleHealthRoutes from './routes/appleHealth.js';
@@ -216,6 +217,7 @@ setHttpsEnabledAtBoot(httpsEnabled);
// Socket.IO with relative path support for Tailscale
const io = new Server(httpServer, {
+ destroyUpgrade: false,
maxHttpBufferSize: SOCKET_MAX_PAYLOAD_BYTES,
// CORS headers are reflected only for handshakes allowRequest admits: the
// same browser-relay guard as the HTTP gate refuses a foreign Origin or a
@@ -240,6 +242,11 @@ remoteDesktopBroker.mountWebSocket(localHttpServer);
// host starts). Upgrade for `/ws` and `/eidoverse-host/**` on both listeners.
mountEidoverseOnServer(null, [httpServer, localHttpServer]);
+// Close unclaimed upgrade sockets after a bounded timeout (replaces engine.io's
+// destroyUpgrade timer so reverse proxies with slow upstreams aren't cut short).
+installUpgradeFallback(httpServer);
+installUpgradeFallback(localHttpServer);
+
// Auth gate for Socket.IO — when settings.secrets.auth.enabled is true the
// handshake must carry a valid token cookie or Authorization: Bearer header
// (set by POST /api/auth/login). No-op when auth is off.
diff --git a/server/lib/upgradeFallback.js b/server/lib/upgradeFallback.js
new file mode 100644
index 0000000000..bce2d38956
--- /dev/null
+++ b/server/lib/upgradeFallback.js
@@ -0,0 +1,86 @@
+/**
+ * Upgrade fallback for HTTP servers sharing a port with Socket.IO.
+ *
+ * Socket.IO's engine.io server attaches an `upgrade` listener that defaults
+ * to destroying any upgrade socket that does not match Socket.IO's path after
+ * 1000ms if bytesWritten <= 0 (`destroyUpgrade: true`).
+ *
+ * For PortOS reverse proxies (e.g. Eidoverse `/ws` proxy) whose upstream
+ * handshake can take longer than 1 second, engine.io's timer prematurely
+ * ends the client socket before the upstream 101 can arrive.
+ *
+ * Setting `destroyUpgrade: false` on Socket.IO prevents engine.io from racing
+ * legitimate reverse proxies. This PortOS-owned fallback ensures unclaimed
+ * upgrade sockets still terminate after a bounded timeout rather than leaking
+ * open indefinitely.
+ */
+
+const mountedUpgradeFallbackServers = new WeakSet();
+
+export const DEFAULT_UPGRADE_TIMEOUT_MS = 1000;
+
+/**
+ * Mark a socket as claimed by a PortOS upgrade handler.
+ * Sockets marked claimed will not be destroyed by the fallback timer.
+ *
+ * @param {import('node:net').Socket} socket
+ */
+export function markUpgradeClaimed(socket) {
+ if (socket) {
+ socket.portosUpgradeClaimed = true;
+ }
+}
+
+/**
+ * Install an upgrade fallback listener on an HTTP/HTTPS server.
+ * Unclaimed sockets that receive no response after `timeoutMs` are ended.
+ *
+ * @param {import('node:http').Server|import('node:https').Server|null|undefined} server
+ * @param {object} [options]
+ * @param {number} [options.timeoutMs=1000]
+ * @returns {(() => void)} Cleanup function to detach listener
+ */
+export function installUpgradeFallback(server, { timeoutMs = DEFAULT_UPGRADE_TIMEOUT_MS } = {}) {
+ if (!server || typeof server.on !== 'function') {
+ return () => {};
+ }
+ if (mountedUpgradeFallbackServers.has(server)) {
+ return () => {};
+ }
+ mountedUpgradeFallbackServers.add(server);
+
+ const onUpgrade = (_req, socket) => {
+ if (socket.portosUpgradeClaimed) return;
+
+ let timer = null;
+ const cleanup = () => {
+ if (timer) {
+ clearTimeout(timer);
+ timer = null;
+ }
+ };
+ socket.once('close', cleanup);
+ socket.once('error', cleanup);
+
+ timer = setTimeout(() => {
+ socket.off('close', cleanup);
+ socket.off('error', cleanup);
+ if (socket.portosUpgradeClaimed) return;
+ if (!socket.writable || socket.destroyed) return;
+ if (socket.bytesWritten > 0) return;
+ socket.on('error', () => {});
+ socket.end();
+ }, timeoutMs);
+
+ timer.unref?.();
+ };
+
+ server.on('upgrade', onUpgrade);
+
+ return () => {
+ mountedUpgradeFallbackServers.delete(server);
+ server.off('upgrade', onUpgrade);
+ };
+}
+
+export const mountUpgradeFallback = installUpgradeFallback;
diff --git a/server/lib/upgradeFallback.test.js b/server/lib/upgradeFallback.test.js
new file mode 100644
index 0000000000..c9f1dfe1cc
--- /dev/null
+++ b/server/lib/upgradeFallback.test.js
@@ -0,0 +1,154 @@
+import { once } from 'node:events';
+import { createServer, request as httpRequest } from 'node:http';
+import { afterEach, describe, expect, it } from 'vitest';
+import {
+ DEFAULT_UPGRADE_TIMEOUT_MS,
+ installUpgradeFallback,
+ markUpgradeClaimed,
+ mountUpgradeFallback,
+} from './upgradeFallback.js';
+
+describe('upgradeFallback', () => {
+ const servers = [];
+ const sockets = [];
+
+ const listen = async (server) => {
+ server.on('connection', (s) => sockets.push(s));
+ server.listen(0, '127.0.0.1');
+ await once(server, 'listening');
+ servers.push(server);
+ return server.address().port;
+ };
+
+ afterEach(async () => {
+ while (sockets.length > 0) {
+ const socket = sockets.pop();
+ try {
+ socket?.destroy();
+ } catch {}
+ }
+ while (servers.length > 0) {
+ const server = servers.pop();
+ server.closeAllConnections?.();
+ await new Promise((resolve) => {
+ if (!server.listening) {
+ resolve();
+ return;
+ }
+ server.close(() => resolve());
+ });
+ }
+ });
+
+ const sendRawUpgrade = (port, path = '/unclaimed') => {
+ const req = httpRequest({
+ hostname: '127.0.0.1',
+ port,
+ path,
+ headers: {
+ Connection: 'Upgrade',
+ Upgrade: 'websocket',
+ },
+ });
+ req.on('upgrade', (_res, socket) => {
+ sockets.push(socket);
+ });
+ req.on('socket', (socket) => {
+ sockets.push(socket);
+ });
+ req.on('error', () => {});
+ req.end();
+ return req;
+ };
+
+ it('exports default timeout and aliases', () => {
+ expect(DEFAULT_UPGRADE_TIMEOUT_MS).toBe(1000);
+ expect(mountUpgradeFallback).toBe(installUpgradeFallback);
+ });
+
+ it('handles null or invalid server gracefully', () => {
+ const cleanup1 = installUpgradeFallback(null);
+ expect(typeof cleanup1).toBe('function');
+ cleanup1();
+
+ const cleanup2 = installUpgradeFallback({});
+ expect(typeof cleanup2).toBe('function');
+ cleanup2();
+ });
+
+ it('ends an unclaimed upgrade socket after timeout', async () => {
+ const server = createServer();
+ installUpgradeFallback(server, { timeoutMs: 50 });
+ const port = await listen(server);
+
+ let clientSocket = null;
+ await new Promise((resolve) => {
+ const req = httpRequest({
+ hostname: '127.0.0.1',
+ port,
+ path: '/unknown-upgrade',
+ headers: {
+ Connection: 'Upgrade',
+ Upgrade: 'websocket',
+ },
+ });
+ req.on('socket', (s) => {
+ clientSocket = s;
+ sockets.push(s);
+ });
+ req.on('close', resolve);
+ req.on('error', resolve);
+ req.end();
+ });
+
+ await new Promise((resolve) => setTimeout(resolve, 80));
+ expect(clientSocket?.destroyed ?? true).toBe(true);
+ });
+
+ it('does not end a socket marked portosUpgradeClaimed', async () => {
+ const server = createServer();
+ let serverSocket = null;
+ server.on('upgrade', (_req, socket) => {
+ serverSocket = socket;
+ markUpgradeClaimed(socket);
+ });
+ installUpgradeFallback(server, { timeoutMs: 50 });
+ const port = await listen(server);
+
+ sendRawUpgrade(port, '/claimed');
+
+ await new Promise((resolve) => setTimeout(resolve, 80));
+ expect(serverSocket).not.toBeNull();
+ expect(serverSocket.destroyed).toBe(false);
+ expect(serverSocket.writable).toBe(true);
+ });
+
+ it('does not end a socket that has written bytes', async () => {
+ const server = createServer();
+ let serverSocket = null;
+ server.on('upgrade', (_req, socket) => {
+ serverSocket = socket;
+ socket.write('HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n');
+ });
+ installUpgradeFallback(server, { timeoutMs: 50 });
+ const port = await listen(server);
+
+ sendRawUpgrade(port, '/socket.io');
+
+ await new Promise((resolve) => setTimeout(resolve, 80));
+ expect(serverSocket.bytesWritten).toBeGreaterThan(0);
+ expect(serverSocket.destroyed).toBe(false);
+ });
+
+ it('prevents duplicate installation and cleans up properly', async () => {
+ const server = createServer();
+ const cleanup1 = installUpgradeFallback(server);
+ const countBefore = server.listenerCount('upgrade');
+ const cleanup2 = installUpgradeFallback(server);
+ expect(server.listenerCount('upgrade')).toBe(countBefore);
+
+ cleanup1();
+ expect(server.listenerCount('upgrade')).toBe(countBefore - 1);
+ cleanup2();
+ });
+});
diff --git a/server/services/eidoverseProxy.js b/server/services/eidoverseProxy.js
index cf633277b8..03e10f505a 100644
--- a/server/services/eidoverseProxy.js
+++ b/server/services/eidoverseProxy.js
@@ -184,6 +184,7 @@ export const proxyUpgradeToEidoverse = ({
pathOverride,
trackSocket = (socket) => socket,
}) => {
+ clientSocket.portosUpgradeClaimed = true;
trackSocket(clientSocket);
const upstreamSocket = trackSocket(createConnection({ host: targetHost, port: targetPort }));
let connected = false;
@@ -311,6 +312,7 @@ export function mountEidoverseWebSocket(httpServer, {
pathOverride = req.url;
}
+ clientSocket.portosUpgradeClaimed = true;
const protocol = typeof getProtocol === 'function' ? getProtocol(req) : getProtocol;
proxyUpgradeToEidoverse({
req,
diff --git a/server/services/eidoverseProxy.test.js b/server/services/eidoverseProxy.test.js
index 538a067031..1ddb47c4db 100644
--- a/server/services/eidoverseProxy.test.js
+++ b/server/services/eidoverseProxy.test.js
@@ -3,6 +3,9 @@ import { createServer, request as httpRequest } from 'node:http';
import express from 'express';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { WebSocket, WebSocketServer } from 'ws';
+import { Server as SocketIoServer } from 'socket.io';
+import { io as SocketIoClient } from 'socket.io-client';
+import { installUpgradeFallback } from '../lib/upgradeFallback.js';
import {
createEidoverseExpressMiddleware,
embedParentOrigin,
@@ -25,6 +28,7 @@ vi.mock('./eidoverseWorld.js', () => ({
const servers = [];
const webSocketServers = [];
const webSocketClients = [];
+const ioServers = [];
const listen = async (server) => {
server.listen(0, '127.0.0.1');
@@ -34,6 +38,10 @@ const listen = async (server) => {
};
const closeServer = (server) => new Promise((resolve, reject) => {
+ if (!server.listening) {
+ resolve();
+ return;
+ }
server.close((error) => (error ? reject(error) : resolve()));
});
@@ -51,6 +59,12 @@ const rawGet = (port, path, hostHeader) => new Promise((settle, reject) => {
afterEach(async () => {
webSocketClients.splice(0).forEach((client) => client.terminate());
await Promise.all(webSocketServers.splice(0).map((server) => new Promise((resolve) => server.close(resolve))));
+ while (ioServers.length > 0) {
+ const io = ioServers.pop();
+ try {
+ io?.close();
+ } catch {}
+ }
await Promise.all(servers.splice(0).map(closeServer));
});
@@ -218,4 +232,105 @@ describe('main-server eidoverse reverse-proxy', () => {
expect(receivedBody).toBe(JSON.stringify(payload));
expect(Number(receivedHeaders['content-length'])).toBe(Buffer.byteLength(JSON.stringify(payload)));
});
+
+ it('completes a /ws upgrade whose upstream answers after 1.5 s when Socket.IO is attached with destroyUpgrade: false', async () => {
+ const upstream = createServer();
+ const upstreamWss = new WebSocketServer({ noServer: true });
+ webSocketServers.push(upstreamWss);
+ upstream.on('upgrade', (req, socket, head) => {
+ const timer = setTimeout(() => {
+ upstreamWss.handleUpgrade(req, socket, head, (ws) => {
+ ws.send('slow-ready');
+ });
+ }, 1500);
+ timer.unref?.();
+ });
+ const upstreamPort = await listen(upstream);
+
+ const app = express();
+ app.use(createEidoverseExpressMiddleware({
+ isActive: () => true,
+ targetHost: '127.0.0.1',
+ targetPort: upstreamPort,
+ getProtocol: () => 'http',
+ }));
+ const server = createServer(app);
+ const io = new SocketIoServer(server, {
+ destroyUpgrade: false,
+ path: '/socket.io',
+ });
+ ioServers.push(io);
+ mountEidoverseWebSocket(server, {
+ isActive: () => true,
+ targetHost: '127.0.0.1',
+ targetPort: upstreamPort,
+ getProtocol: () => 'http',
+ });
+ installUpgradeFallback(server);
+ const port = await listen(server);
+
+ const client = new WebSocket(`ws://127.0.0.1:${port}/ws`);
+ webSocketClients.push(client);
+ const first = once(client, 'message');
+ await once(client, 'open');
+ expect(String((await first)[0])).toBe('slow-ready');
+ client.close();
+ await once(client, 'close');
+ webSocketClients.pop();
+ });
+
+ it('ends an upgrade request for an unclaimed path after a bounded timeout', async () => {
+ const app = express();
+ const server = createServer(app);
+ const io = new SocketIoServer(server, {
+ destroyUpgrade: false,
+ path: '/socket.io',
+ });
+ ioServers.push(io);
+ mountEidoverseWebSocket(server, {
+ isActive: () => true,
+ targetHost: '127.0.0.1',
+ targetPort: 5556,
+ getProtocol: () => 'http',
+ });
+ installUpgradeFallback(server, { timeoutMs: 100 });
+ const port = await listen(server);
+
+ const client = new WebSocket(`ws://127.0.0.1:${port}/unclaimed-upgrade-path`);
+ webSocketClients.push(client);
+ const closed = await new Promise((resolve) => {
+ client.on('close', () => resolve(true));
+ client.on('error', () => resolve(true));
+ });
+ expect(closed).toBe(true);
+ expect(client.readyState).toBeGreaterThanOrEqual(WebSocket.CLOSING);
+ webSocketClients.pop();
+ });
+
+ it('preserves Socket.IO upgrades on the same server when destroyUpgrade: false is set', async () => {
+ const app = express();
+ const server = createServer(app);
+ const io = new SocketIoServer(server, {
+ destroyUpgrade: false,
+ path: '/socket.io',
+ });
+ ioServers.push(io);
+ mountEidoverseWebSocket(server, {
+ isActive: () => true,
+ targetHost: '127.0.0.1',
+ targetPort: 5556,
+ getProtocol: () => 'http',
+ });
+ installUpgradeFallback(server);
+ const port = await listen(server);
+
+ const client = SocketIoClient(`http://127.0.0.1:${port}`, {
+ path: '/socket.io',
+ transports: ['websocket'],
+ });
+ await once(client, 'connect');
+ expect(client.connected).toBe(true);
+ client.disconnect();
+ });
+ });
});
diff --git a/server/services/remoteDesktop.js b/server/services/remoteDesktop.js
index 20e64d1e96..4281a47bee 100644
--- a/server/services/remoteDesktop.js
+++ b/server/services/remoteDesktop.js
@@ -108,6 +108,7 @@ export const createRemoteDesktopBroker = ({
httpServer.on('upgrade', (request, socket, head) => {
if (request.url?.split('?', 1)[0] !== '/remote-desktop/ws') return;
+ socket.portosUpgradeClaimed = true;
let url;
try {
url = new URL(request.url, 'http://portos.invalid');