🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): drain in-flight requests on fatal error before exiting (#16996)
Niklas Semmlerd690ca430803
diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.tsindex 8b56a4ce87..ce1e6a7796 100644--- a/packages/shared/src/server/index.ts+++ b/packages/shared/src/server/index.ts@@ -138,6 +138,7 @@ export * from "./filterToPrisma";export * from "./prismaFilter";export * from "./instrumentation";export * from "./logger";+export * from "./processErrorHandlers";export * from "./headerPropagation";export * from "./queries";export * from "./queries/clickhouse-sql/orderby-factory";diff --git a/packages/shared/src/server/processErrorHandlers.test.ts b/packages/shared/src/server/processErrorHandlers.test.tsnew file mode 100644index 0000000000..59e2c8b91a--- /dev/null+++ b/packages/shared/src/server/processErrorHandlers.test.ts@@ -0,0 +1,186 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++const recordIncrement = vi.fn();++vi.mock("./instrumentation", () => ({+ recordIncrement: (...args: unknown[]) => recordIncrement(...args),+}));+vi.mock("./logger", () => ({+ logger: { error: vi.fn() },+}));++import type { ProcessErrorHandlerOptions } from "./processErrorHandlers";++const OUR_LISTENER_NAMES = ["onUnhandledRejection", "onUncaughtException"];++type ProcessEvent = "unhandledRejection" | "uncaughtException";++// process.listeners is typed against NodeJS.Signals; cast so a plain event+// name variable is accepted. The runtime value is the real event string.+const listenersFor = (+ event: ProcessEvent,+): Array<(...args: unknown[]) => void> =>+ process.listeners(event as NodeJS.Signals) as unknown as Array<+ (...args: unknown[]) => void+ >;++const removeOurListeners = () => {+ for (const event of ["unhandledRejection", "uncaughtException"] as const) {+ for (const listener of listenersFor(event)) {+ if (OUR_LISTENER_NAMES.includes(listener.name)) {+ process.removeListener(event as NodeJS.Signals, listener as never);+ }+ }+ }+};++// Freshly load the module (its `installed`/`draining` state is module-level)+// and install with the given options.+async function freshInstall(options: ProcessErrorHandlerOptions) {+ vi.resetModules();+ const mod = await import("./processErrorHandlers.js");+ mod.installProcessErrorHandlers(options);+ return mod;+}++// Invoke our registered handler directly instead of process.emit(), so the+// test never risks killing the runner or tripping the framework's own+// unhandledRejection/uncaughtException listeners.+const fire = (event: ProcessEvent, reason: unknown) => {+ const name =+ event === "unhandledRejection"+ ? "onUnhandledRejection"+ : "onUncaughtException";+ const listener = listenersFor(event).find((l) => l.name === name);+ if (!listener) {+ throw new Error(`no ${name} listener registered`);+ }+ listener(reason, Promise.resolve());+};++const flush = async () => {+ await Promise.resolve();+ await Promise.resolve();+ await new Promise<void>((resolve) => setImmediate(resolve));+};++describe("installProcessErrorHandlers", () => {+ beforeEach(() => {+ recordIncrement.mockClear();+ });++ afterEach(() => {+ removeOurListeners();+ });++ it("registers each listener only once", async () => {+ const mod = await freshInstall({ exit: vi.fn() });+ const count = (event: ProcessEvent) =>+ listenersFor(event).filter((l) => OUR_LISTENER_NAMES.includes(l.name))+ .length;++ expect(count("unhandledRejection")).toBe(1);+ expect(count("uncaughtException")).toBe(1);++ mod.installProcessErrorHandlers({ exit: vi.fn() });++ expect(count("unhandledRejection")).toBe(1);+ expect(count("uncaughtException")).toBe(1);+ });++ it("drains then exits non-zero on an unhandled rejection", async () => {+ const exit = vi.fn();+ const onFatal = vi.fn().mockResolvedValue(undefined);+ await freshInstall({ onFatal, exit });++ fire("unhandledRejection", new Error("leaked"));+ await flush();++ expect(onFatal).toHaveBeenCalledTimes(1);+ expect(onFatal.mock.calls[0][0]).toMatchObject({+ source: "unhandledRejection",+ });+ expect(exit).toHaveBeenCalledWith(1);+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.process.unhandled_rejection",+ 1,+ );+ });++ it("drains then exits non-zero on an uncaught exception", async () => {+ const exit = vi.fn();+ const onFatal = vi.fn().mockResolvedValue(undefined);+ await freshInstall({ onFatal, exit });++ fire("uncaughtException", new Error("boom"));+ await flush();++ expect(onFatal).toHaveBeenCalledTimes(1);+ expect(onFatal.mock.calls[0][0]).toMatchObject({+ source: "uncaughtException",+ });+ expect(exit).toHaveBeenCalledWith(1);+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.process.uncaught_exception",+ 1,+ );+ });++ it("still exits once when the drain itself rejects", async () => {+ const exit = vi.fn();+ const onFatal = vi.fn().mockRejectedValue(new Error("drain failed"));+ await freshInstall({ onFatal, exit });++ fire("unhandledRejection", new Error("leaked"));+ await flush();++ expect(exit).toHaveBeenCalledWith(1);+ expect(exit).toHaveBeenCalledTimes(1);+ });++ it("force-exits immediately on a repeated fatal while still draining", async () => {+ const exit = vi.fn();+ let releaseDrain: () => void = () => {};+ const onFatal = vi.fn(+ () =>+ new Promise<void>((resolve) => {+ releaseDrain = resolve;+ }),+ );+ await freshInstall({ onFatal, exit });++ // First fatal: drain starts, no exit yet.+ fire("unhandledRejection", new Error("first"));+ await flush();+ expect(onFatal).toHaveBeenCalledTimes(1);+ expect(exit).not.toHaveBeenCalled();++ // Second fatal mid-drain: force exit immediately, without re-draining.+ fire("uncaughtException", new Error("second"));+ expect(exit).toHaveBeenCalledWith(1);+ expect(onFatal).toHaveBeenCalledTimes(1);+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.process.forced_exit",+ 1,+ );++ releaseDrain();+ });++ it("force-exits when the drain exceeds the timeout budget", async () => {+ vi.useFakeTimers();+ try {+ const exit = vi.fn();+ const onFatal = vi.fn(() => new Promise<void>(() => {})); // never resolves+ await freshInstall({ onFatal, exit, drainTimeoutMs: 1_000 });++ fire("unhandledRejection", new Error("leaked"));+ expect(exit).not.toHaveBeenCalled();++ vi.advanceTimersByTime(1_000);+ expect(exit).toHaveBeenCalledWith(1);+ } finally {+ vi.useRealTimers();+ }+ });+});diff --git a/packages/shared/src/server/processErrorHandlers.ts b/packages/shared/src/server/processErrorHandlers.tsnew file mode 100644index 0000000000..f1495649f2--- /dev/null+++ b/packages/shared/src/server/processErrorHandlers.ts@@ -0,0 +1,94 @@+import { recordIncrement } from "./instrumentation";+import { logger } from "./logger";++type FatalSource = "uncaughtException" | "unhandledRejection";++export type ProcessErrorHandlerOptions = {+ /**+ * Runtime graceful drain: flip readiness to unhealthy so the load balancer+ * stops routing new traffic, let in-flight work finish, and close+ * connections. Must NOT call process.exit — this module owns the exit so it+ * can enforce the repeated-error safety gate. Omit to exit immediately with+ * no drain (for runtimes that have nothing to drain).+ */+ onFatal?: (info: { source: FatalSource; reason: unknown }) => Promise<void>;+ /**+ * Backstop: force exit if the drain never resolves (e.g. a wedged connection+ * close). Not the primary safety gate — that is the repeated-error kill.+ */+ drainTimeoutMs?: number;+ /** Injectable process exit, for tests. */+ exit?: (code: number) => void;+};++let installed = false;+let draining = false;++const DEFAULT_DRAIN_TIMEOUT_MS = 130_000;++/**+ * Install process-level handlers for unhandled rejections and uncaught+ * exceptions. Both are fatal: the first one drains in-flight requests via+ * `onFatal` and then exits non-zero so the orchestrator restarts a clean+ * process. A self-initiated shutdown never receives a follow-up SIGKILL from+ * the orchestrator, so we must exit ourselves.+ *+ * Safety gate: if a second fatal error arrives while the drain is still+ * running, the process is wedged — abandon the drain and exit immediately.+ *+ * Idempotent. Call after dd.init.+ */+export function installProcessErrorHandlers(+ options: ProcessErrorHandlerOptions = {},+): void {+ if (installed) {+ return;+ }+ installed = true;++ const exit = options.exit ?? ((code: number) => process.exit(code));+ const drainTimeoutMs = options.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;++ const onFatalError = (source: FatalSource, reason: unknown): void => {+ if (draining) {+ // Safety gate: a second fatal error mid-drain means draining will not+ // recover the process. Stop waiting and die now.+ logger.error(+ `Repeated ${source} during shutdown drain; exiting now`,+ reason,+ );+ recordIncrement("langfuse.process.forced_exit", 1);+ exit(1);+ return;+ }+ draining = true;+ logger.error(+ `Fatal ${source}; draining in-flight requests before exit`,+ reason,+ );++ const backstop = setTimeout(() => {+ logger.error("Shutdown drain exceeded budget; forcing exit");+ exit(1);+ }, drainTimeoutMs);+ backstop.unref?.();++ Promise.resolve()+ .then(() => options.onFatal?.({ source, reason }))+ .catch((err) => logger.error("Shutdown drain failed", err))+ .finally(() => {+ clearTimeout(backstop);+ exit(1);+ });+ };++ process.on("unhandledRejection", function onUnhandledRejection(reason) {+ recordIncrement("langfuse.process.unhandled_rejection", 1);+ onFatalError("unhandledRejection", reason);+ });++ process.on("uncaughtException", function onUncaughtException(err) {+ recordIncrement("langfuse.process.uncaught_exception", 1);+ onFatalError("uncaughtException", err);+ });+}diff --git a/web/src/instrumentation.ts b/web/src/instrumentation.tsindex 27279e596c..36b824f130 100644--- a/web/src/instrumentation.ts+++ b/web/src/instrumentation.ts@@ -7,9 +7,31 @@ export async function register() {? process.env.NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT === "true": true;- if (process.env.NEXT_RUNTIME === "nodejs" && isInitLoadingEnabled) {+ const isNodeRuntime = process.env.NEXT_RUNTIME === "nodejs";++ if (isNodeRuntime && isInitLoadingEnabled) {console.log("Running init scripts...");await import("./observability.config");+ }++ // Install after dd.init when init runs. Keep the dynamic import so AWS SDK+ // is not loaded before dd-trace wraps it. Install even when init is skipped+ // (secondary replicas set NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=false). On a+ // fatal error, drain in-flight requests before exiting instead of dying+ // abruptly and 5xx-ing them; the drain is imported lazily so its heavy+ // dependencies load only when a fatal actually fires.+ if (isNodeRuntime) {+ const { installProcessErrorHandlers } =+ await import("@langfuse/shared/src/server");+ installProcessErrorHandlers({+ onFatal: async () => {+ const { drainAndClose } = await import("./utils/shutdown");+ await drainAndClose();+ },+ });+ }++ if (isNodeRuntime && isInitLoadingEnabled) {await import("./initialize");}}diff --git a/web/src/pages/_app.tsx b/web/src/pages/_app.tsxindex f67c69a0f1..5ccb0a77b2 100644--- a/web/src/pages/_app.tsx+++ b/web/src/pages/_app.tsx@@ -312,8 +312,15 @@ if (process.env.NEXT_MANUAL_SIG_HANDLE) {const { shutdown } = await import("@/src/utils/shutdown");- prexit(async (signal) => {- console.log("Signal: ", signal);- return await shutdown(signal);- });+ // uncaughtException is intentionally omitted: it is handled by+ // installProcessErrorHandlers (registered in instrumentation.ts), which+ // drains in-flight requests and then exits, rather than prexit exiting+ // abruptly without draining.+ prexit(+ ["exit", "beforeExit", "SIGTSTP", "SIGQUIT", "SIGHUP", "SIGTERM", "SIGINT"],+ async (signal) => {+ console.log("Signal: ", signal);+ return await shutdown(signal);+ },+ );}diff --git a/web/src/utils/shutdown.ts b/web/src/utils/shutdown.tsindex 356b51df81..33d21efe8f 100644--- a/web/src/utils/shutdown.ts+++ b/web/src/utils/shutdown.ts@@ -28,36 +28,48 @@ const setSigtermReceived = () => {export const isSigtermReceived = () =>Boolean(process.env.NEXT_MANUAL_SIG_HANDLE) && globalThis.sigtermReceived;+let drainPromise: Promise<void> | null = null;++// Flip readiness to unhealthy so the load balancer stops routing new traffic,+// wait TIMEOUT for in-flight requests to drain, then close backend+// connections. Shared by the SIGTERM/SIGINT path and the fatal-error path;+// memoized so concurrent triggers reuse one drain instead of closing+// connections twice.+export const drainAndClose = (): Promise<void> => {+ if (drainPromise) {+ return drainPromise;+ }+ setSigtermReceived();++ drainPromise = new Promise<void>((resolve) => {+ setTimeout(async () => {+ RateLimitService.shutdown();++ await ClickHouseClientManager.getInstance().closeAllConnections();++ logger.info(`Redis status ${redis?.status}`);+ if (redis && redis.status !== "end") {+ redis.disconnect();+ } else {+ logger.info("Redis connection already closed");+ }++ await prisma.$disconnect();+ logger.info("Prisma connection has been closed.");++ logger.info("Shutdown complete");+ resolve();+ }, TIMEOUT);+ });++ return drainPromise;+};+export const shutdown = async (signal: PrexitSignal) => {if (signal === "SIGTERM" || signal === "SIGINT") {console.log(`SIGTERM / SIGINT received. Shutting down in ${TIMEOUT / 1000} seconds.`,);- setSigtermReceived();-- return await new Promise<void>((resolve) => {- setTimeout(async () => {- RateLimitService.shutdown();-- // Shutdown clickhouse connections- await ClickHouseClientManager.getInstance().closeAllConnections();-- logger.info(`Redis status ${redis?.status}`);- if (!redis) {- return;- }- if (redis.status === "end") {- logger.info("Redis connection already closed");- return;- }- redis?.disconnect();-- await prisma.$disconnect();- logger.info("Prisma connection has been closed.");-- logger.info("Shutdown complete");- resolve();- }, TIMEOUT);- });+ await drainAndClose();}};diff --git a/worker/src/app.ts b/worker/src/app.tsindex 9103040570..3563a43b70 100644--- a/worker/src/app.ts+++ b/worker/src/app.ts@@ -15,7 +15,8 @@ import {} from "./queues/evalQueue";import { codeEvalExecutionQueueProcessorBuilder } from "./queues/codeEvalQueue";import { batchExportQueueProcessor } from "./queues/batchExportQueue";-import { onShutdown } from "./utils/shutdown";+import { drainAndClose, onShutdown } from "./utils/shutdown";+import { installProcessErrorHandlers } from "@langfuse/shared/src/server";import helmet from "helmet";import { cloudUsageMeteringQueueProcessor } from "./queues/cloudUsageMeteringQueue";import { cloudSpendAlertQueueProcessor } from "./queues/cloudSpendAlertQueue";@@ -821,4 +822,13 @@ if (env.LANGFUSE_MONITOR_SCHEDULER_ENABLED === "true") {process.on("SIGINT", () => onShutdown("SIGINT"));process.on("SIGTERM", () => onShutdown("SIGTERM"));+// On a fatal error (uncaught exception / unhandled rejection), drain in-flight+// jobs and flush pending writes before exiting instead of dying abruptly. A+// repeated fatal mid-drain forces an immediate exit.+installProcessErrorHandlers({+ onFatal: async () => {+ await drainAndClose();+ },+});+export default app;diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.tsindex 509370870c..3717e49d6f 100644--- a/worker/src/utils/shutdown.ts+++ b/worker/src/utils/shutdown.ts@@ -28,9 +28,25 @@ import {export const onShutdown: NodeJS.SignalsListener = async (signal) => {logger.info(`Received ${signal}, closing server...`);+ await drainAndClose();+};++let drainPromise: Promise<void> | null = null;++// Flip readiness to unhealthy, stop accepting new work, drain in-flight jobs+// and flush pending writes, then close connections. Shared by the+// SIGTERM/SIGINT path and the fatal-error path; memoized so concurrent+// triggers reuse one drain instead of closing workers/connections twice.+export const drainAndClose = (): Promise<void> => {+ if (!drainPromise) {+ drainPromise = runDrainAndClose();+ }+ return drainPromise;+};++const runDrainAndClose = async () => {setSigtermReceived();- // Stop accepting new connectionsserver?.close();logger.info("Server has been closed.");