Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): drain in-flight requests on fatal error before exiting (#16996)

Niklas Semmlerd690ca430803
diff --git a/packages/shared/src/server/index.ts b/packages/shared/src/server/index.ts
index 8b56a4ce87..ce1e6a7796 100644
--- a/packages/shared/src/server/index.ts
+++ b/packages/shared/src/server/index.ts
@@ -138,6 +138,7 @@ export * from "./filterToPrisma";
export * from "./prismaFilter";
export * from "./instrumentation";
export * from "./logger";
+export * from "./processErrorHandlers";
export * from "./headerPropagation";
export * from "./queries";
export * from "./queries/clickhouse-sql/orderby-factory";
diff --git a/packages/shared/src/server/processErrorHandlers.test.ts b/packages/shared/src/server/processErrorHandlers.test.ts
new file mode 100644
index 0000000000..59e2c8b91a
--- /dev/null
+++ b/packages/shared/src/server/processErrorHandlers.test.ts
@@ -0,0 +1,186 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const recordIncrement = vi.fn();
+
+vi.mock("./instrumentation", () => ({
+ recordIncrement: (...args: unknown[]) => recordIncrement(...args),
+}));
+vi.mock("./logger", () => ({
+ logger: { error: vi.fn() },
+}));
+
+import type { ProcessErrorHandlerOptions } from "./processErrorHandlers";
+
+const OUR_LISTENER_NAMES = ["onUnhandledRejection", "onUncaughtException"];
+
+type ProcessEvent = "unhandledRejection" | "uncaughtException";
+
+// process.listeners is typed against NodeJS.Signals; cast so a plain event
+// name variable is accepted. The runtime value is the real event string.
+const listenersFor = (
+ event: ProcessEvent,
+): Array<(...args: unknown[]) => void> =>
+ process.listeners(event as NodeJS.Signals) as unknown as Array<
+ (...args: unknown[]) => void
+ >;
+
+const removeOurListeners = () => {
+ for (const event of ["unhandledRejection", "uncaughtException"] as const) {
+ for (const listener of listenersFor(event)) {
+ if (OUR_LISTENER_NAMES.includes(listener.name)) {
+ process.removeListener(event as NodeJS.Signals, listener as never);
+ }
+ }
+ }
+};
+
+// Freshly load the module (its `installed`/`draining` state is module-level)
+// and install with the given options.
+async function freshInstall(options: ProcessErrorHandlerOptions) {
+ vi.resetModules();
+ const mod = await import("./processErrorHandlers.js");
+ mod.installProcessErrorHandlers(options);
+ return mod;
+}
+
+// Invoke our registered handler directly instead of process.emit(), so the
+// test never risks killing the runner or tripping the framework's own
+// unhandledRejection/uncaughtException listeners.
+const fire = (event: ProcessEvent, reason: unknown) => {
+ const name =
+ event === "unhandledRejection"
+ ? "onUnhandledRejection"
+ : "onUncaughtException";
+ const listener = listenersFor(event).find((l) => l.name === name);
+ if (!listener) {
+ throw new Error(`no ${name} listener registered`);
+ }
+ listener(reason, Promise.resolve());
+};
+
+const flush = async () => {
+ await Promise.resolve();
+ await Promise.resolve();
+ await new Promise<void>((resolve) => setImmediate(resolve));
+};
+
+describe("installProcessErrorHandlers", () => {
+ beforeEach(() => {
+ recordIncrement.mockClear();
+ });
+
+ afterEach(() => {
+ removeOurListeners();
+ });
+
+ it("registers each listener only once", async () => {
+ const mod = await freshInstall({ exit: vi.fn() });
+ const count = (event: ProcessEvent) =>
+ listenersFor(event).filter((l) => OUR_LISTENER_NAMES.includes(l.name))
+ .length;
+
+ expect(count("unhandledRejection")).toBe(1);
+ expect(count("uncaughtException")).toBe(1);
+
+ mod.installProcessErrorHandlers({ exit: vi.fn() });
+
+ expect(count("unhandledRejection")).toBe(1);
+ expect(count("uncaughtException")).toBe(1);
+ });
+
+ it("drains then exits non-zero on an unhandled rejection", async () => {
+ const exit = vi.fn();
+ const onFatal = vi.fn().mockResolvedValue(undefined);
+ await freshInstall({ onFatal, exit });
+
+ fire("unhandledRejection", new Error("leaked"));
+ await flush();
+
+ expect(onFatal).toHaveBeenCalledTimes(1);
+ expect(onFatal.mock.calls[0][0]).toMatchObject({
+ source: "unhandledRejection",
+ });
+ expect(exit).toHaveBeenCalledWith(1);
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.process.unhandled_rejection",
+ 1,
+ );
+ });
+
+ it("drains then exits non-zero on an uncaught exception", async () => {
+ const exit = vi.fn();
+ const onFatal = vi.fn().mockResolvedValue(undefined);
+ await freshInstall({ onFatal, exit });
+
+ fire("uncaughtException", new Error("boom"));
+ await flush();
+
+ expect(onFatal).toHaveBeenCalledTimes(1);
+ expect(onFatal.mock.calls[0][0]).toMatchObject({
+ source: "uncaughtException",
+ });
+ expect(exit).toHaveBeenCalledWith(1);
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.process.uncaught_exception",
+ 1,
+ );
+ });
+
+ it("still exits once when the drain itself rejects", async () => {
+ const exit = vi.fn();
+ const onFatal = vi.fn().mockRejectedValue(new Error("drain failed"));
+ await freshInstall({ onFatal, exit });
+
+ fire("unhandledRejection", new Error("leaked"));
+ await flush();
+
+ expect(exit).toHaveBeenCalledWith(1);
+ expect(exit).toHaveBeenCalledTimes(1);
+ });
+
+ it("force-exits immediately on a repeated fatal while still draining", async () => {
+ const exit = vi.fn();
+ let releaseDrain: () => void = () => {};
+ const onFatal = vi.fn(
+ () =>
+ new Promise<void>((resolve) => {
+ releaseDrain = resolve;
+ }),
+ );
+ await freshInstall({ onFatal, exit });
+
+ // First fatal: drain starts, no exit yet.
+ fire("unhandledRejection", new Error("first"));
+ await flush();
+ expect(onFatal).toHaveBeenCalledTimes(1);
+ expect(exit).not.toHaveBeenCalled();
+
+ // Second fatal mid-drain: force exit immediately, without re-draining.
+ fire("uncaughtException", new Error("second"));
+ expect(exit).toHaveBeenCalledWith(1);
+ expect(onFatal).toHaveBeenCalledTimes(1);
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.process.forced_exit",
+ 1,
+ );
+
+ releaseDrain();
+ });
+
+ it("force-exits when the drain exceeds the timeout budget", async () => {
+ vi.useFakeTimers();
+ try {
+ const exit = vi.fn();
+ const onFatal = vi.fn(() => new Promise<void>(() => {})); // never resolves
+ await freshInstall({ onFatal, exit, drainTimeoutMs: 1_000 });
+
+ fire("unhandledRejection", new Error("leaked"));
+ expect(exit).not.toHaveBeenCalled();
+
+ vi.advanceTimersByTime(1_000);
+ expect(exit).toHaveBeenCalledWith(1);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+});
diff --git a/packages/shared/src/server/processErrorHandlers.ts b/packages/shared/src/server/processErrorHandlers.ts
new file mode 100644
index 0000000000..f1495649f2
--- /dev/null
+++ b/packages/shared/src/server/processErrorHandlers.ts
@@ -0,0 +1,94 @@
+import { recordIncrement } from "./instrumentation";
+import { logger } from "./logger";
+
+type FatalSource = "uncaughtException" | "unhandledRejection";
+
+export type ProcessErrorHandlerOptions = {
+ /**
+ * Runtime graceful drain: flip readiness to unhealthy so the load balancer
+ * stops routing new traffic, let in-flight work finish, and close
+ * connections. Must NOT call process.exit — this module owns the exit so it
+ * can enforce the repeated-error safety gate. Omit to exit immediately with
+ * no drain (for runtimes that have nothing to drain).
+ */
+ onFatal?: (info: { source: FatalSource; reason: unknown }) => Promise<void>;
+ /**
+ * Backstop: force exit if the drain never resolves (e.g. a wedged connection
+ * close). Not the primary safety gate — that is the repeated-error kill.
+ */
+ drainTimeoutMs?: number;
+ /** Injectable process exit, for tests. */
+ exit?: (code: number) => void;
+};
+
+let installed = false;
+let draining = false;
+
+const DEFAULT_DRAIN_TIMEOUT_MS = 130_000;
+
+/**
+ * Install process-level handlers for unhandled rejections and uncaught
+ * exceptions. Both are fatal: the first one drains in-flight requests via
+ * `onFatal` and then exits non-zero so the orchestrator restarts a clean
+ * process. A self-initiated shutdown never receives a follow-up SIGKILL from
+ * the orchestrator, so we must exit ourselves.
+ *
+ * Safety gate: if a second fatal error arrives while the drain is still
+ * running, the process is wedged — abandon the drain and exit immediately.
+ *
+ * Idempotent. Call after dd.init.
+ */
+export function installProcessErrorHandlers(
+ options: ProcessErrorHandlerOptions = {},
+): void {
+ if (installed) {
+ return;
+ }
+ installed = true;
+
+ const exit = options.exit ?? ((code: number) => process.exit(code));
+ const drainTimeoutMs = options.drainTimeoutMs ?? DEFAULT_DRAIN_TIMEOUT_MS;
+
+ const onFatalError = (source: FatalSource, reason: unknown): void => {
+ if (draining) {
+ // Safety gate: a second fatal error mid-drain means draining will not
+ // recover the process. Stop waiting and die now.
+ logger.error(
+ `Repeated ${source} during shutdown drain; exiting now`,
+ reason,
+ );
+ recordIncrement("langfuse.process.forced_exit", 1);
+ exit(1);
+ return;
+ }
+ draining = true;
+ logger.error(
+ `Fatal ${source}; draining in-flight requests before exit`,
+ reason,
+ );
+
+ const backstop = setTimeout(() => {
+ logger.error("Shutdown drain exceeded budget; forcing exit");
+ exit(1);
+ }, drainTimeoutMs);
+ backstop.unref?.();
+
+ Promise.resolve()
+ .then(() => options.onFatal?.({ source, reason }))
+ .catch((err) => logger.error("Shutdown drain failed", err))
+ .finally(() => {
+ clearTimeout(backstop);
+ exit(1);
+ });
+ };
+
+ process.on("unhandledRejection", function onUnhandledRejection(reason) {
+ recordIncrement("langfuse.process.unhandled_rejection", 1);
+ onFatalError("unhandledRejection", reason);
+ });
+
+ process.on("uncaughtException", function onUncaughtException(err) {
+ recordIncrement("langfuse.process.uncaught_exception", 1);
+ onFatalError("uncaughtException", err);
+ });
+}
diff --git a/web/src/instrumentation.ts b/web/src/instrumentation.ts
index 27279e596c..36b824f130 100644
--- a/web/src/instrumentation.ts
+++ b/web/src/instrumentation.ts
@@ -7,9 +7,31 @@ export async function register() {
? process.env.NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT === "true"
: true;
- if (process.env.NEXT_RUNTIME === "nodejs" && isInitLoadingEnabled) {
+ const isNodeRuntime = process.env.NEXT_RUNTIME === "nodejs";
+
+ if (isNodeRuntime && isInitLoadingEnabled) {
console.log("Running init scripts...");
await import("./observability.config");
+ }
+
+ // Install after dd.init when init runs. Keep the dynamic import so AWS SDK
+ // is not loaded before dd-trace wraps it. Install even when init is skipped
+ // (secondary replicas set NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=false). On a
+ // fatal error, drain in-flight requests before exiting instead of dying
+ // abruptly and 5xx-ing them; the drain is imported lazily so its heavy
+ // dependencies load only when a fatal actually fires.
+ if (isNodeRuntime) {
+ const { installProcessErrorHandlers } =
+ await import("@langfuse/shared/src/server");
+ installProcessErrorHandlers({
+ onFatal: async () => {
+ const { drainAndClose } = await import("./utils/shutdown");
+ await drainAndClose();
+ },
+ });
+ }
+
+ if (isNodeRuntime && isInitLoadingEnabled) {
await import("./initialize");
}
}
diff --git a/web/src/pages/_app.tsx b/web/src/pages/_app.tsx
index f67c69a0f1..5ccb0a77b2 100644
--- a/web/src/pages/_app.tsx
+++ b/web/src/pages/_app.tsx
@@ -312,8 +312,15 @@ if (
process.env.NEXT_MANUAL_SIG_HANDLE
) {
const { shutdown } = await import("@/src/utils/shutdown");
- prexit(async (signal) => {
- console.log("Signal: ", signal);
- return await shutdown(signal);
- });
+ // uncaughtException is intentionally omitted: it is handled by
+ // installProcessErrorHandlers (registered in instrumentation.ts), which
+ // drains in-flight requests and then exits, rather than prexit exiting
+ // abruptly without draining.
+ prexit(
+ ["exit", "beforeExit", "SIGTSTP", "SIGQUIT", "SIGHUP", "SIGTERM", "SIGINT"],
+ async (signal) => {
+ console.log("Signal: ", signal);
+ return await shutdown(signal);
+ },
+ );
}
diff --git a/web/src/utils/shutdown.ts b/web/src/utils/shutdown.ts
index 356b51df81..33d21efe8f 100644
--- a/web/src/utils/shutdown.ts
+++ b/web/src/utils/shutdown.ts
@@ -28,36 +28,48 @@ const setSigtermReceived = () => {
export const isSigtermReceived = () =>
Boolean(process.env.NEXT_MANUAL_SIG_HANDLE) && globalThis.sigtermReceived;
+let drainPromise: Promise<void> | null = null;
+
+// Flip readiness to unhealthy so the load balancer stops routing new traffic,
+// wait TIMEOUT for in-flight requests to drain, then close backend
+// connections. Shared by the SIGTERM/SIGINT path and the fatal-error path;
+// memoized so concurrent triggers reuse one drain instead of closing
+// connections twice.
+export const drainAndClose = (): Promise<void> => {
+ if (drainPromise) {
+ return drainPromise;
+ }
+ setSigtermReceived();
+
+ drainPromise = new Promise<void>((resolve) => {
+ setTimeout(async () => {
+ RateLimitService.shutdown();
+
+ await ClickHouseClientManager.getInstance().closeAllConnections();
+
+ logger.info(`Redis status ${redis?.status}`);
+ if (redis && redis.status !== "end") {
+ redis.disconnect();
+ } else {
+ logger.info("Redis connection already closed");
+ }
+
+ await prisma.$disconnect();
+ logger.info("Prisma connection has been closed.");
+
+ logger.info("Shutdown complete");
+ resolve();
+ }, TIMEOUT);
+ });
+
+ return drainPromise;
+};
+
export const shutdown = async (signal: PrexitSignal) => {
if (signal === "SIGTERM" || signal === "SIGINT") {
console.log(
`SIGTERM / SIGINT received. Shutting down in ${TIMEOUT / 1000} seconds.`,
);
- setSigtermReceived();
-
- return await new Promise<void>((resolve) => {
- setTimeout(async () => {
- RateLimitService.shutdown();
-
- // Shutdown clickhouse connections
- await ClickHouseClientManager.getInstance().closeAllConnections();
-
- logger.info(`Redis status ${redis?.status}`);
- if (!redis) {
- return;
- }
- if (redis.status === "end") {
- logger.info("Redis connection already closed");
- return;
- }
- redis?.disconnect();
-
- await prisma.$disconnect();
- logger.info("Prisma connection has been closed.");
-
- logger.info("Shutdown complete");
- resolve();
- }, TIMEOUT);
- });
+ await drainAndClose();
}
};
diff --git a/worker/src/app.ts b/worker/src/app.ts
index 9103040570..3563a43b70 100644
--- a/worker/src/app.ts
+++ b/worker/src/app.ts
@@ -15,7 +15,8 @@ import {
} from "./queues/evalQueue";
import { codeEvalExecutionQueueProcessorBuilder } from "./queues/codeEvalQueue";
import { batchExportQueueProcessor } from "./queues/batchExportQueue";
-import { onShutdown } from "./utils/shutdown";
+import { drainAndClose, onShutdown } from "./utils/shutdown";
+import { installProcessErrorHandlers } from "@langfuse/shared/src/server";
import helmet from "helmet";
import { cloudUsageMeteringQueueProcessor } from "./queues/cloudUsageMeteringQueue";
import { cloudSpendAlertQueueProcessor } from "./queues/cloudSpendAlertQueue";
@@ -821,4 +822,13 @@ if (env.LANGFUSE_MONITOR_SCHEDULER_ENABLED === "true") {
process.on("SIGINT", () => onShutdown("SIGINT"));
process.on("SIGTERM", () => onShutdown("SIGTERM"));
+// On a fatal error (uncaught exception / unhandled rejection), drain in-flight
+// jobs and flush pending writes before exiting instead of dying abruptly. A
+// repeated fatal mid-drain forces an immediate exit.
+installProcessErrorHandlers({
+ onFatal: async () => {
+ await drainAndClose();
+ },
+});
+
export default app;
diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.ts
index 509370870c..3717e49d6f 100644
--- a/worker/src/utils/shutdown.ts
+++ b/worker/src/utils/shutdown.ts
@@ -28,9 +28,25 @@ import {
export const onShutdown: NodeJS.SignalsListener = async (signal) => {
logger.info(`Received ${signal}, closing server...`);
+ await drainAndClose();
+};
+
+let drainPromise: Promise<void> | null = null;
+
+// Flip readiness to unhealthy, stop accepting new work, drain in-flight jobs
+// and flush pending writes, then close connections. Shared by the
+// SIGTERM/SIGINT path and the fatal-error path; memoized so concurrent
+// triggers reuse one drain instead of closing workers/connections twice.
+export const drainAndClose = (): Promise<void> => {
+ if (!drainPromise) {
+ drainPromise = runDrainAndClose();
+ }
+ return drainPromise;
+};
+
+const runDrainAndClose = async () => {
setSigtermReceived();
- // Stop accepting new connections
server?.close();
logger.info("Server has been closed.");