Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix(ingress): handle socket errors so ECONNRESET can't crash the proxy (#280)

Robert Brennan31302f9282d9
diff --git a/__tests__/scripts/ingress.test.ts b/__tests__/scripts/ingress.test.ts
index 317e09556e..a3085c6cf9 100644
--- a/__tests__/scripts/ingress.test.ts
+++ b/__tests__/scripts/ingress.test.ts
@@ -1,4 +1,6 @@
import { createServer, type Server } from "node:http";
+import { connect as netConnect, type Socket } from "node:net";
+import type { Duplex } from "node:stream";
import { spawn, type ChildProcess } from "node:child_process";
import { once } from "node:events";
import path from "node:path";
@@ -303,3 +305,147 @@ describe("ingress route matching", () => {
expect(response.status).toBe(503);
});
});
+
+describe("ingress socket-error resilience", () => {
+ // Regression coverage for crashes like:
+ // Error: read ECONNRESET ... Emitted 'error' event on Socket instance
+ // which previously took down the whole ingress process when a WebSocket's
+ // underlying TCP socket reset.
+ let upstream: Server;
+ let upstreamSockets: Duplex[];
+ let ingressProcess: ChildProcess;
+ let ingressStderr: string;
+ const upstreamPort = 29111;
+ const ingressPort = 29110;
+
+ beforeAll(async () => {
+ upstreamSockets = [];
+
+ upstream = createServer((req, res) => {
+ res.writeHead(200, { "Content-Type": "text/plain" });
+ res.end("ok");
+ });
+
+ // Accept WebSocket upgrades, immediately RST the upstream socket on the
+ // next tick. This reproduces the production crash without requiring a
+ // real WebSocket handshake handler.
+ upstream.on("upgrade", (_req, socket) => {
+ upstreamSockets.push(socket);
+ socket.write(
+ "HTTP/1.1 101 Switching Protocols\r\n" +
+ "Upgrade: websocket\r\n" +
+ "Connection: Upgrade\r\n\r\n",
+ );
+ // Force a TCP RST instead of a clean FIN to mirror ECONNRESET.
+ setImmediate(() => {
+ const s = socket as Duplex & { resetAndDestroy?: () => void };
+ if (typeof s.resetAndDestroy === "function") {
+ s.resetAndDestroy();
+ } else {
+ s.destroy(
+ Object.assign(new Error("forced reset"), { code: "ECONNRESET" }),
+ );
+ }
+ });
+ });
+
+ await new Promise<void>((resolve) => upstream.listen(upstreamPort, resolve));
+
+ ingressStderr = "";
+ ingressProcess = spawn(
+ process.execPath,
+ [
+ ingressScript,
+ "--port",
+ ingressPort.toString(),
+ "--default",
+ `http://localhost:${upstreamPort}`,
+ ],
+ {
+ cwd: repoRoot,
+ stdio: ["ignore", "pipe", "pipe"],
+ },
+ );
+ ingressProcess.stderr?.on("data", (chunk) => {
+ ingressStderr += chunk.toString();
+ });
+
+ await delay(800);
+ });
+
+ afterAll(async () => {
+ ingressProcess?.kill("SIGTERM");
+ for (const s of upstreamSockets) {
+ try {
+ s.destroy();
+ } catch {
+ // ignore
+ }
+ }
+ await new Promise<void>((resolve) => upstream?.close(() => resolve()));
+ });
+
+ function openWebSocketHandshake(port: number): Promise<Socket> {
+ return new Promise((resolve, reject) => {
+ const sock = netConnect({ host: "127.0.0.1", port }, () => {
+ sock.write(
+ "GET /sockets/events/test HTTP/1.1\r\n" +
+ `Host: 127.0.0.1:${port}\r\n` +
+ "Upgrade: websocket\r\n" +
+ "Connection: Upgrade\r\n" +
+ "Sec-WebSocket-Version: 13\r\n" +
+ "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n",
+ );
+ resolve(sock);
+ });
+ // The upstream RST will surface here as ECONNRESET; we just want the
+ // handshake to be initiated, so swallow any error after that.
+ sock.on("error", () => {});
+ sock.once("error", reject);
+ });
+ }
+
+ it("survives upstream WebSocket ECONNRESET without crashing", async () => {
+ // Trigger the bug repeatedly to make sure no path crashes the proxy.
+ for (let i = 0; i < 5; i++) {
+ const client = await openWebSocketHandshake(ingressPort);
+ // Wait long enough for the upstream RST to propagate through the proxy.
+ await delay(150);
+ client.destroy();
+ await delay(50);
+ }
+
+ // Process must still be alive.
+ expect(ingressProcess.exitCode).toBeNull();
+ expect(ingressProcess.signalCode).toBeNull();
+
+ // And it must still be serving HTTP traffic.
+ const response = await fetch(`http://localhost:${ingressPort}/health`);
+ expect(response.status).toBe(200);
+
+ // The unhandled-error crash signature must not appear in stderr.
+ expect(ingressStderr).not.toContain("Unhandled 'error' event");
+ expect(ingressStderr).not.toMatch(/throw er;/);
+ });
+
+ it("survives client aborting an in-flight HTTP request", async () => {
+ // Open and abruptly destroy a TCP connection mid-request to make sure
+ // req/res 'error' events on the client side are handled.
+ const client = netConnect({ host: "127.0.0.1", port: ingressPort }, () => {
+ client.write(
+ "GET /something HTTP/1.1\r\n" +
+ `Host: 127.0.0.1:${ingressPort}\r\n` +
+ "Connection: close\r\n\r\n",
+ );
+ // Reset before the upstream finishes responding.
+ setImmediate(() => client.destroy());
+ });
+ client.on("error", () => {});
+
+ await delay(200);
+
+ expect(ingressProcess.exitCode).toBeNull();
+ expect(ingressProcess.signalCode).toBeNull();
+ expect(ingressStderr).not.toContain("Unhandled 'error' event");
+ });
+});
diff --git a/scripts/ingress.mjs b/scripts/ingress.mjs
index fdbb205875..d32dfea090 100644
--- a/scripts/ingress.mjs
+++ b/scripts/ingress.mjs
@@ -154,6 +154,19 @@ function parseBackendUrl(backendUrl) {
// Proxy
// ═══════════════════════════════════════════════════════════════════════════
+// Errors that are normal during connection teardown and should not be logged
+// at error level (clients/upstreams routinely reset long-lived connections).
+const BENIGN_SOCKET_ERRORS = new Set([
+ "ECONNRESET",
+ "EPIPE",
+ "ECONNABORTED",
+ "ERR_STREAM_PREMATURE_CLOSE",
+]);
+
+function isBenignSocketError(err) {
+ return Boolean(err && BENIGN_SOCKET_ERRORS.has(err.code));
+}
+
function proxyRequest(req, res, backendUrl) {
const backend = parseBackendUrl(backendUrl);
@@ -169,16 +182,50 @@ function proxyRequest(req, res, backendUrl) {
};
const proxyReq = httpRequest(options, (proxyRes) => {
+ proxyRes.on("error", (err) => {
+ if (!isBenignSocketError(err)) {
+ console.error(`Upstream response error for ${req.url}:`, err.message);
+ }
+ if (!res.headersSent) {
+ res.writeHead(502);
+ res.end(`Bad Gateway: ${err.message}`);
+ } else {
+ res.destroy();
+ }
+ });
+
res.writeHead(proxyRes.statusCode, proxyRes.headers);
proxyRes.pipe(res, { end: true });
});
proxyReq.on("error", (err) => {
- console.error(`Proxy error for ${req.url}:`, err.message);
+ if (!isBenignSocketError(err)) {
+ console.error(`Proxy error for ${req.url}:`, err.message);
+ }
if (!res.headersSent) {
res.writeHead(502);
res.end(`Bad Gateway: ${err.message}`);
+ } else {
+ res.destroy();
+ }
+ });
+
+ // If the client disconnects mid-request, abort the upstream call so we
+ // don't leak connections or emit unhandled 'error' events.
+ req.on("error", (err) => {
+ if (!isBenignSocketError(err)) {
+ console.error(`Client request error for ${req.url}:`, err.message);
}
+ proxyReq.destroy();
+ });
+
+ // The outbound HTTP response object can also emit 'error' (e.g. EPIPE if
+ // the client socket is gone). Without a listener Node would crash.
+ res.on("error", (err) => {
+ if (!isBenignSocketError(err)) {
+ console.error(`Client response error for ${req.url}:`, err.message);
+ }
+ proxyReq.destroy();
});
req.pipe(proxyReq, { end: true });
@@ -200,7 +247,36 @@ function proxyWebSocket(req, socket, head, backendUrl) {
const proxyReq = httpRequest(options);
+ // Always attach an 'error' handler to the client socket immediately. The
+ // client can drop the connection before the upstream upgrade completes
+ // (e.g. during a slow DNS / connect), and an unhandled 'error' on the raw
+ // TCP socket would crash the entire ingress process.
+ socket.on("error", (err) => {
+ if (!isBenignSocketError(err)) {
+ console.error(`WebSocket client socket error for ${req.url}:`, err.message);
+ }
+ proxyReq.destroy();
+ });
+
proxyReq.on("upgrade", (proxyRes, proxySocket, proxyHead) => {
+ // Once the upgrade is established, errors on either raw TCP socket must
+ // be handled or Node will throw. ECONNRESET / EPIPE on long-lived
+ // WebSocket connections is routine (browser tab closed, NAT timeout,
+ // mobile network handoff, …) and should never crash the proxy.
+ const teardown = (label) => (err) => {
+ if (err && !isBenignSocketError(err)) {
+ console.error(`WebSocket ${label} error for ${req.url}:`, err.message);
+ }
+ socket.destroy();
+ proxySocket.destroy();
+ };
+ proxySocket.on("error", teardown("upstream socket"));
+ // Note: socket already has an 'error' listener from above; add a second
+ // one that also destroys proxySocket so we don't leak the upstream half.
+ socket.on("error", () => proxySocket.destroy());
+ socket.on("close", () => proxySocket.destroy());
+ proxySocket.on("close", () => socket.destroy());
+
socket.write(
`HTTP/${proxyRes.httpVersion} ${proxyRes.statusCode} ${proxyRes.statusMessage}\r\n`
);
@@ -218,7 +294,9 @@ function proxyWebSocket(req, socket, head, backendUrl) {
});
proxyReq.on("error", (err) => {
- console.error(`WebSocket proxy error for ${req.url}:`, err.message);
+ if (!isBenignSocketError(err)) {
+ console.error(`WebSocket proxy error for ${req.url}:`, err.message);
+ }
socket.destroy();
});
@@ -256,6 +334,19 @@ function startIngress(config) {
proxyWebSocket(req, socket, head, backend);
});
+ // Built-in protection against malformed client requests that can otherwise
+ // bubble up as unhandled errors on the underlying TCP socket.
+ server.on("clientError", (err, socket) => {
+ if (!isBenignSocketError(err)) {
+ console.error("Client error:", err.message);
+ }
+ if (socket.writable) {
+ socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");
+ } else {
+ socket.destroy();
+ }
+ });
+
server.listen(config.port, () => {
console.log("");
console.log("╔═══════════════════════════════════════════════════════════════╗");
@@ -301,6 +392,17 @@ if (Object.keys(config.routes).length === 0 && !config.defaultBackend) {
startIngress(config);
+// Last-resort safety net: a benign socket reset on a stream we forgot to wire
+// up should never crash the proxy. Anything else is re-thrown so real bugs
+// stay visible.
+process.on("uncaughtException", (err) => {
+ if (isBenignSocketError(err)) {
+ console.warn(`Ignoring socket reset: ${err.code} ${err.message}`);
+ return;
+ }
+ throw err;
+});
+
// Handle graceful shutdown
process.on("SIGINT", () => {
console.log("\nShutting down...");