Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): guard token-count worker pool against shutdown races (#17300)

Niklas Semmler36852553de96
diff --git a/worker/src/__tests__/tokenCountWorkerShutdown.unit.test.ts b/worker/src/__tests__/tokenCountWorkerShutdown.unit.test.ts
new file mode 100644
index 0000000000..387f98376a
--- /dev/null
+++ b/worker/src/__tests__/tokenCountWorkerShutdown.unit.test.ts
@@ -0,0 +1,100 @@
+import { describe, expect, it, vi, beforeEach } from "vitest";
+import { type Model } from "@langfuse/shared";
+
+// Stand in for a real worker_threads Worker so the pool spins up without
+// spawning threads. postMessage echoes a successful result synchronously
+// while `autoRespond` is set, which is enough to prove the live path
+// resolves; disabling it leaves a request in flight so the shutdown drop
+// can be observed. Hoisted so the vi.mock factory can close over the class,
+// the instance registry, and the response toggle.
+const { workerInstances, FakeWorker, control, recordIncrement } = vi.hoisted(
+ () => {
+ const instances: any[] = [];
+ const control = { autoRespond: true };
+ const recordIncrement = vi.fn();
+ class FakeWorker {
+ handlers: Record<string, (arg: any) => void> = {};
+ postMessage = vi.fn((msg: { id: string }) => {
+ if (control.autoRespond) {
+ this.handlers["message"]?.({ id: msg.id, result: 7, error: null });
+ }
+ });
+ terminate = vi.fn(async () => {});
+ constructor() {
+ instances.push(this);
+ }
+ on(event: string, cb: (arg: any) => void) {
+ this.handlers[event] = cb;
+ }
+ }
+ return { workerInstances: instances, FakeWorker, control, recordIncrement };
+ },
+);
+
+vi.mock("worker_threads", () => ({ Worker: FakeWorker }));
+vi.mock("@langfuse/shared/src/server", () => ({
+ logger: { error: vi.fn(), warn: vi.fn(), info: vi.fn() },
+ recordIncrement,
+}));
+vi.mock("../env", () => ({
+ env: { LANGFUSE_TOKEN_COUNT_WORKER_POOL_SIZE: 2 },
+}));
+
+import { TokenCountWorkerManager } from "../features/tokenisation/async-usage";
+
+const model = { id: "m", tokenizerId: "openai" } as unknown as Model;
+
+const totalPostMessages = () =>
+ workerInstances.reduce((n, w) => n + w.postMessage.mock.calls.length, 0);
+
+describe("TokenCountWorkerManager shutdown guard", () => {
+ beforeEach(() => {
+ workerInstances.length = 0;
+ control.autoRespond = true;
+ recordIncrement.mockClear();
+ });
+
+ it("returns the worker result while the pool is live", async () => {
+ const manager = new TokenCountWorkerManager(2);
+
+ await expect(manager.tokenCount({ model, text: "hi" })).resolves.toBe(7);
+ });
+
+ it("drops the count and records the metric for calls made after terminate", async () => {
+ const manager = new TokenCountWorkerManager(2);
+ await manager.terminate();
+ recordIncrement.mockClear();
+
+ const postsBefore = totalPostMessages();
+ await expect(
+ manager.tokenCount({ model, text: "hi" }),
+ ).resolves.toBeUndefined();
+ expect(totalPostMessages()).toBe(postsBefore);
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.tokenisation.skipped_on_shutdown",
+ 1,
+ );
+ });
+
+ it("resolves in-flight requests as undefined and records one summary metric on terminate", async () => {
+ const manager = new TokenCountWorkerManager(2);
+ control.autoRespond = false;
+
+ const inFlight = manager.tokenCount({ model, text: "hi" });
+ await manager.terminate();
+
+ await expect(inFlight).resolves.toBeUndefined();
+ expect(recordIncrement).toHaveBeenCalledTimes(1);
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.tokenisation.skipped_on_shutdown",
+ 1,
+ );
+ });
+
+ it("does not record the shutdown metric when nothing is in flight", async () => {
+ const manager = new TokenCountWorkerManager(2);
+ await manager.terminate();
+
+ expect(recordIncrement).not.toHaveBeenCalled();
+ });
+});
diff --git a/worker/src/features/tokenisation/async-usage.ts b/worker/src/features/tokenisation/async-usage.ts
index c6d9ed7fe8..cc6caca599 100644
--- a/worker/src/features/tokenisation/async-usage.ts
+++ b/worker/src/features/tokenisation/async-usage.ts
@@ -1,6 +1,6 @@
import { Worker } from "worker_threads";
import { Model } from "@langfuse/shared";
-import { logger } from "@langfuse/shared/src/server";
+import { logger, recordIncrement } from "@langfuse/shared/src/server";
import path from "path";
import { env } from "../../env";
@@ -17,11 +17,12 @@ interface TokenCountWorkerPool {
>;
}
-class TokenCountWorkerManager {
+export class TokenCountWorkerManager {
private pool: TokenCountWorkerPool;
private readonly workerPath: string;
private readonly poolSize: number;
private requestCounter = 0;
+ private isShuttingDown = false;
constructor(poolSize: number) {
this.poolSize = poolSize;
@@ -71,12 +72,15 @@ class TokenCountWorkerManager {
);
worker.on("error", (error) => {
+ // Terminating workers surface as errors/non-zero exits; don't respawn
+ // into a pool we are tearing down.
+ if (this.isShuttingDown) return;
logger.error("Worker thread error:", error);
- // Recreate worker on error
this.replaceWorker(worker);
});
worker.on("exit", (code) => {
+ if (this.isShuttingDown) return;
if (code !== 0) {
logger.error(`Worker stopped with exit code ${code}`);
this.replaceWorker(worker);
@@ -106,7 +110,10 @@ class TokenCountWorkerManager {
}
}
- private getNextWorker(): Worker {
+ private getNextWorker(): Worker | undefined {
+ if (this.pool.workers.length === 0) {
+ return undefined;
+ }
const worker = this.pool.workers[this.pool.currentWorkerIndex];
this.pool.currentWorkerIndex =
(this.pool.currentWorkerIndex + 1) % this.poolSize;
@@ -117,10 +124,24 @@ class TokenCountWorkerManager {
params: { model: Model; text: unknown },
timeoutMs = 30000,
): Promise<number | undefined> {
+ // Once the pool is terminating, drop the count instead of posting to a
+ // terminating or already-gone worker. undefined means usage unknown, which
+ // the caller commits as absent rather than as a zero; count the drop so it
+ // stays observable.
+ if (this.isShuttingDown) {
+ recordIncrement("langfuse.tokenisation.skipped_on_shutdown", 1);
+ return undefined;
+ }
+
return new Promise((resolve, reject) => {
const id = `token-count-${++this.requestCounter}-${Date.now()}`;
const worker = this.getNextWorker();
+ if (!worker) {
+ resolve(undefined);
+ return;
+ }
+
const timeout = setTimeout(() => {
this.pool.pendingRequests.delete(id);
reject(
@@ -142,14 +163,30 @@ class TokenCountWorkerManager {
}
async terminate() {
- // Clear all pending requests
+ // Set before touching the pool so concurrent tokenCount() calls and the
+ // workers' own exit/error events see the shutdown and stop early.
+ this.isShuttingDown = true;
+
+ // In-flight requests can no longer get a response from the terminating
+ // workers. Resolve them with undefined (usage unknown) rather than
+ // rejecting, and count the drop so it stays observable without the caller
+ // logging each one as a tokenization failure.
+ const droppedOnShutdown = this.pool.pendingRequests.size;
+ if (droppedOnShutdown > 0) {
+ recordIncrement(
+ "langfuse.tokenisation.skipped_on_shutdown",
+ droppedOnShutdown,
+ );
+ logger.info(
+ `Dropped ${droppedOnShutdown} in-flight token-count request(s) on shutdown.`,
+ );
+ }
for (const [, request] of this.pool.pendingRequests.entries()) {
clearTimeout(request.timeout);
- request.reject(new Error("Worker pool is terminating"));
+ request.resolve(undefined);
}
this.pool.pendingRequests.clear();
- // Terminate all workers
await Promise.all(this.pool.workers.map((worker) => worker.terminate()));
this.pool.workers = [];
}