π OpenHands: AI-Driven Development
fix(ingress): handle socket errors so ECONNRESET can't crash the proxy (#280)
Robert Brennan31302f9282d9
diff --git a/__tests__/scripts/ingress.test.ts b/__tests__/scripts/ingress.test.tsindex 317e09556e..a3085c6cf9 100644--- a/__tests__/scripts/ingress.test.ts+++ b/__tests__/scripts/ingress.test.ts@@ -1,4 +1,6 @@import { createServer, type Server } from "node:http";+import { connect as netConnect, type Socket } from "node:net";+import type { Duplex } from "node:stream";import { spawn, type ChildProcess } from "node:child_process";import { once } from "node:events";import path from "node:path";@@ -303,3 +305,147 @@ describe("ingress route matching", () => {expect(response.status).toBe(503);});});++describe("ingress socket-error resilience", () => {+ // Regression coverage for crashes like:+ // Error: read ECONNRESET ... Emitted 'error' event on Socket instance+ // which previously took down the whole ingress process when a WebSocket's+ // underlying TCP socket reset.+ let upstream: Server;+ let upstreamSockets: Duplex[];+ let ingressProcess: ChildProcess;+ let ingressStderr: string;+ const upstreamPort = 29111;+ const ingressPort = 29110;++ beforeAll(async () => {+ upstreamSockets = [];++ upstream = createServer((req, res) => {+ res.writeHead(200, { "Content-Type": "text/plain" });+ res.end("ok");+ });++ // Accept WebSocket upgrades, immediately RST the upstream socket on the+ // next tick. This reproduces the production crash without requiring a+ // real WebSocket handshake handler.+ upstream.on("upgrade", (_req, socket) => {+ upstreamSockets.push(socket);+ socket.write(+ "HTTP/1.1 101 Switching Protocols\r\n" ++ "Upgrade: websocket\r\n" ++ "Connection: Upgrade\r\n\r\n",+ );+ // Force a TCP RST instead of a clean FIN to mirror ECONNRESET.+ setImmediate(() => {+ const s = socket as Duplex & { resetAndDestroy?: () => void };+ if (typeof s.resetAndDestroy === "function") {+ s.resetAndDestroy();+ } else {+ s.destroy(+ Object.assign(new Error("forced reset"), { code: "ECONNRESET" }),+ );+ }+ });+ });++ await new Promise<void>((resolve) => upstream.listen(upstreamPort, resolve));++ ingressStderr = "";+ ingressProcess = spawn(+ process.execPath,+ [+ ingressScript,+ "--port",+ ingressPort.toString(),+ "--default",+ `http://localhost:${upstreamPort}`,+ ],+ {+ cwd: repoRoot,+ stdio: ["ignore", "pipe", "pipe"],+ },+ );+ ingressProcess.stderr?.on("data", (chunk) => {+ ingressStderr += chunk.toString();+ });++ await delay(800);+ });++ afterAll(async () => {+ ingressProcess?.kill("SIGTERM");+ for (const s of upstreamSockets) {+ try {+ s.destroy();+ } catch {+ // ignore+ }+ }+ await new Promise<void>((resolve) => upstream?.close(() => resolve()));+ });++ function openWebSocketHandshake(port: number): Promise<Socket> {+ return new Promise((resolve, reject) => {+ const sock = netConnect({ host: "127.0.0.1", port }, () => {+ sock.write(+ "GET /sockets/events/test HTTP/1.1\r\n" ++ `Host: 127.0.0.1:${port}\r\n` ++ "Upgrade: websocket\r\n" ++ "Connection: Upgrade\r\n" ++ "Sec-WebSocket-Version: 13\r\n" ++ "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n",+ );+ resolve(sock);+ });+ // The upstream RST will surface here as ECONNRESET; we just want the+ // handshake to be initiated, so swallow any error after that.+ sock.on("error", () => {});+ sock.once("error", reject);+ });+ }++ it("survives upstream WebSocket ECONNRESET without crashing", async () => {+ // Trigger the bug repeatedly to make sure no path crashes the proxy.+ for (let i = 0; i < 5; i++) {+ const client = await openWebSocketHandshake(ingressPort);+ // Wait long enough for the upstream RST to propagate through the proxy.+ await delay(150);+ client.destroy();+ await delay(50);+ }++ // Process must still be alive.+ expect(ingressProcess.exitCode).toBeNull();+ expect(ingressProcess.signalCode).toBeNull();++ // And it must still be serving HTTP traffic.+ const response = await fetch(`http://localhost:${ingressPort}/health`);+ expect(response.status).toBe(200);++ // The unhandled-error crash signature must not appear in stderr.+ expect(ingressStderr).not.toContain("Unhandled 'error' event");+ expect(ingressStderr).not.toMatch(/throw er;/);+ });++ it("survives client aborting an in-flight HTTP request", async () => {+ // Open and abruptly destroy a TCP connection mid-request to make sure+ // req/res 'error' events on the client side are handled.+ const client = netConnect({ host: "127.0.0.1", port: ingressPort }, () => {+ client.write(+ "GET /something HTTP/1.1\r\n" ++ `Host: 127.0.0.1:${ingressPort}\r\n` ++ "Connection: close\r\n\r\n",+ );+ // Reset before the upstream finishes responding.+ setImmediate(() => client.destroy());+ });+ client.on("error", () => {});++ await delay(200);++ expect(ingressProcess.exitCode).toBeNull();+ expect(ingressProcess.signalCode).toBeNull();+ expect(ingressStderr).not.toContain("Unhandled 'error' event");+ });+});diff --git a/scripts/ingress.mjs b/scripts/ingress.mjsindex fdbb205875..d32dfea090 100644--- a/scripts/ingress.mjs+++ b/scripts/ingress.mjs@@ -154,6 +154,19 @@ function parseBackendUrl(backendUrl) {// Proxy// βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+// Errors that are normal during connection teardown and should not be logged+// at error level (clients/upstreams routinely reset long-lived connections).+const BENIGN_SOCKET_ERRORS = new Set([+ "ECONNRESET",+ "EPIPE",+ "ECONNABORTED",+ "ERR_STREAM_PREMATURE_CLOSE",+]);++function isBenignSocketError(err) {+ return Boolean(err && BENIGN_SOCKET_ERRORS.has(err.code));+}+function proxyRequest(req, res, backendUrl) {const backend = parseBackendUrl(backendUrl);@@ -169,16 +182,50 @@ function proxyRequest(req, res, backendUrl) {};const proxyReq = httpRequest(options, (proxyRes) => {+ proxyRes.on("error", (err) => {+ if (!isBenignSocketError(err)) {+ console.error(`Upstream response error for ${req.url}:`, err.message);+ }+ if (!res.headersSent) {+ res.writeHead(502);+ res.end(`Bad Gateway: ${err.message}`);+ } else {+ res.destroy();+ }+ });+res.writeHead(proxyRes.statusCode, proxyRes.headers);proxyRes.pipe(res, { end: true });});proxyReq.on("error", (err) => {- console.error(`Proxy error for ${req.url}:`, err.message);+ if (!isBenignSocketError(err)) {+ console.error(`Proxy error for ${req.url}:`, err.message);+ }if (!res.headersSent) {res.writeHead(502);res.end(`Bad Gateway: ${err.message}`);+ } else {+ res.destroy();+ }+ });++ // If the client disconnects mid-request, abort the upstream call so we+ // don't leak connections or emit unhandled 'error' events.+ req.on("error", (err) => {+ if (!isBenignSocketError(err)) {+ console.error(`Client request error for ${req.url}:`, err.message);}+ proxyReq.destroy();+ });++ // The outbound HTTP response object can also emit 'error' (e.g. EPIPE if+ // the client socket is gone). Without a listener Node would crash.+ res.on("error", (err) => {+ if (!isBenignSocketError(err)) {+ console.error(`Client response error for ${req.url}:`, err.message);+ }+ proxyReq.destroy();});req.pipe(proxyReq, { end: true });@@ -200,7 +247,36 @@ function proxyWebSocket(req, socket, head, backendUrl) {const proxyReq = httpRequest(options);+ // Always attach an 'error' handler to the client socket immediately. The+ // client can drop the connection before the upstream upgrade completes+ // (e.g. during a slow DNS / connect), and an unhandled 'error' on the raw+ // TCP socket would crash the entire ingress process.+ socket.on("error", (err) => {+ if (!isBenignSocketError(err)) {+ console.error(`WebSocket client socket error for ${req.url}:`, err.message);+ }+ proxyReq.destroy();+ });+proxyReq.on("upgrade", (proxyRes, proxySocket, proxyHead) => {+ // Once the upgrade is established, errors on either raw TCP socket must+ // be handled or Node will throw. ECONNRESET / EPIPE on long-lived+ // WebSocket connections is routine (browser tab closed, NAT timeout,+ // mobile network handoff, β¦) and should never crash the proxy.+ const teardown = (label) => (err) => {+ if (err && !isBenignSocketError(err)) {+ console.error(`WebSocket ${label} error for ${req.url}:`, err.message);+ }+ socket.destroy();+ proxySocket.destroy();+ };+ proxySocket.on("error", teardown("upstream socket"));+ // Note: socket already has an 'error' listener from above; add a second+ // one that also destroys proxySocket so we don't leak the upstream half.+ socket.on("error", () => proxySocket.destroy());+ socket.on("close", () => proxySocket.destroy());+ proxySocket.on("close", () => socket.destroy());+socket.write(`HTTP/${proxyRes.httpVersion} ${proxyRes.statusCode} ${proxyRes.statusMessage}\r\n`);@@ -218,7 +294,9 @@ function proxyWebSocket(req, socket, head, backendUrl) {});proxyReq.on("error", (err) => {- console.error(`WebSocket proxy error for ${req.url}:`, err.message);+ if (!isBenignSocketError(err)) {+ console.error(`WebSocket proxy error for ${req.url}:`, err.message);+ }socket.destroy();});@@ -256,6 +334,19 @@ function startIngress(config) {proxyWebSocket(req, socket, head, backend);});+ // Built-in protection against malformed client requests that can otherwise+ // bubble up as unhandled errors on the underlying TCP socket.+ server.on("clientError", (err, socket) => {+ if (!isBenignSocketError(err)) {+ console.error("Client error:", err.message);+ }+ if (socket.writable) {+ socket.end("HTTP/1.1 400 Bad Request\r\n\r\n");+ } else {+ socket.destroy();+ }+ });+server.listen(config.port, () => {console.log("");console.log("βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ");@@ -301,6 +392,17 @@ if (Object.keys(config.routes).length === 0 && !config.defaultBackend) {startIngress(config);+// Last-resort safety net: a benign socket reset on a stream we forgot to wire+// up should never crash the proxy. Anything else is re-thrown so real bugs+// stay visible.+process.on("uncaughtException", (err) => {+ if (isBenignSocketError(err)) {+ console.warn(`Ignoring socket reset: ${err.code} ${err.message}`);+ return;+ }+ throw err;+});+// Handle graceful shutdownprocess.on("SIGINT", () => {console.log("\nShutting down...");