🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): guard token-count worker pool against shutdown races (#17300)
Niklas Semmler36852553de96
diff --git a/worker/src/__tests__/tokenCountWorkerShutdown.unit.test.ts b/worker/src/__tests__/tokenCountWorkerShutdown.unit.test.tsnew file mode 100644index 0000000000..387f98376a--- /dev/null+++ b/worker/src/__tests__/tokenCountWorkerShutdown.unit.test.ts@@ -0,0 +1,100 @@+import { describe, expect, it, vi, beforeEach } from "vitest";+import { type Model } from "@langfuse/shared";++// Stand in for a real worker_threads Worker so the pool spins up without+// spawning threads. postMessage echoes a successful result synchronously+// while `autoRespond` is set, which is enough to prove the live path+// resolves; disabling it leaves a request in flight so the shutdown drop+// can be observed. Hoisted so the vi.mock factory can close over the class,+// the instance registry, and the response toggle.+const { workerInstances, FakeWorker, control, recordIncrement } = vi.hoisted(+ () => {+ const instances: any[] = [];+ const control = { autoRespond: true };+ const recordIncrement = vi.fn();+ class FakeWorker {+ handlers: Record<string, (arg: any) => void> = {};+ postMessage = vi.fn((msg: { id: string }) => {+ if (control.autoRespond) {+ this.handlers["message"]?.({ id: msg.id, result: 7, error: null });+ }+ });+ terminate = vi.fn(async () => {});+ constructor() {+ instances.push(this);+ }+ on(event: string, cb: (arg: any) => void) {+ this.handlers[event] = cb;+ }+ }+ return { workerInstances: instances, FakeWorker, control, recordIncrement };+ },+);++vi.mock("worker_threads", () => ({ Worker: FakeWorker }));+vi.mock("@langfuse/shared/src/server", () => ({+ logger: { error: vi.fn(), warn: vi.fn(), info: vi.fn() },+ recordIncrement,+}));+vi.mock("../env", () => ({+ env: { LANGFUSE_TOKEN_COUNT_WORKER_POOL_SIZE: 2 },+}));++import { TokenCountWorkerManager } from "../features/tokenisation/async-usage";++const model = { id: "m", tokenizerId: "openai" } as unknown as Model;++const totalPostMessages = () =>+ workerInstances.reduce((n, w) => n + w.postMessage.mock.calls.length, 0);++describe("TokenCountWorkerManager shutdown guard", () => {+ beforeEach(() => {+ workerInstances.length = 0;+ control.autoRespond = true;+ recordIncrement.mockClear();+ });++ it("returns the worker result while the pool is live", async () => {+ const manager = new TokenCountWorkerManager(2);++ await expect(manager.tokenCount({ model, text: "hi" })).resolves.toBe(7);+ });++ it("drops the count and records the metric for calls made after terminate", async () => {+ const manager = new TokenCountWorkerManager(2);+ await manager.terminate();+ recordIncrement.mockClear();++ const postsBefore = totalPostMessages();+ await expect(+ manager.tokenCount({ model, text: "hi" }),+ ).resolves.toBeUndefined();+ expect(totalPostMessages()).toBe(postsBefore);+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.tokenisation.skipped_on_shutdown",+ 1,+ );+ });++ it("resolves in-flight requests as undefined and records one summary metric on terminate", async () => {+ const manager = new TokenCountWorkerManager(2);+ control.autoRespond = false;++ const inFlight = manager.tokenCount({ model, text: "hi" });+ await manager.terminate();++ await expect(inFlight).resolves.toBeUndefined();+ expect(recordIncrement).toHaveBeenCalledTimes(1);+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.tokenisation.skipped_on_shutdown",+ 1,+ );+ });++ it("does not record the shutdown metric when nothing is in flight", async () => {+ const manager = new TokenCountWorkerManager(2);+ await manager.terminate();++ expect(recordIncrement).not.toHaveBeenCalled();+ });+});diff --git a/worker/src/features/tokenisation/async-usage.ts b/worker/src/features/tokenisation/async-usage.tsindex c6d9ed7fe8..cc6caca599 100644--- a/worker/src/features/tokenisation/async-usage.ts+++ b/worker/src/features/tokenisation/async-usage.ts@@ -1,6 +1,6 @@import { Worker } from "worker_threads";import { Model } from "@langfuse/shared";-import { logger } from "@langfuse/shared/src/server";+import { logger, recordIncrement } from "@langfuse/shared/src/server";import path from "path";import { env } from "../../env";@@ -17,11 +17,12 @@ interface TokenCountWorkerPool {>;}-class TokenCountWorkerManager {+export class TokenCountWorkerManager {private pool: TokenCountWorkerPool;private readonly workerPath: string;private readonly poolSize: number;private requestCounter = 0;+ private isShuttingDown = false;constructor(poolSize: number) {this.poolSize = poolSize;@@ -71,12 +72,15 @@ class TokenCountWorkerManager {);worker.on("error", (error) => {+ // Terminating workers surface as errors/non-zero exits; don't respawn+ // into a pool we are tearing down.+ if (this.isShuttingDown) return;logger.error("Worker thread error:", error);- // Recreate worker on errorthis.replaceWorker(worker);});worker.on("exit", (code) => {+ if (this.isShuttingDown) return;if (code !== 0) {logger.error(`Worker stopped with exit code ${code}`);this.replaceWorker(worker);@@ -106,7 +110,10 @@ class TokenCountWorkerManager {}}- private getNextWorker(): Worker {+ private getNextWorker(): Worker | undefined {+ if (this.pool.workers.length === 0) {+ return undefined;+ }const worker = this.pool.workers[this.pool.currentWorkerIndex];this.pool.currentWorkerIndex =(this.pool.currentWorkerIndex + 1) % this.poolSize;@@ -117,10 +124,24 @@ class TokenCountWorkerManager {params: { model: Model; text: unknown },timeoutMs = 30000,): Promise<number | undefined> {+ // Once the pool is terminating, drop the count instead of posting to a+ // terminating or already-gone worker. undefined means usage unknown, which+ // the caller commits as absent rather than as a zero; count the drop so it+ // stays observable.+ if (this.isShuttingDown) {+ recordIncrement("langfuse.tokenisation.skipped_on_shutdown", 1);+ return undefined;+ }+return new Promise((resolve, reject) => {const id = `token-count-${++this.requestCounter}-${Date.now()}`;const worker = this.getNextWorker();+ if (!worker) {+ resolve(undefined);+ return;+ }+const timeout = setTimeout(() => {this.pool.pendingRequests.delete(id);reject(@@ -142,14 +163,30 @@ class TokenCountWorkerManager {}async terminate() {- // Clear all pending requests+ // Set before touching the pool so concurrent tokenCount() calls and the+ // workers' own exit/error events see the shutdown and stop early.+ this.isShuttingDown = true;++ // In-flight requests can no longer get a response from the terminating+ // workers. Resolve them with undefined (usage unknown) rather than+ // rejecting, and count the drop so it stays observable without the caller+ // logging each one as a tokenization failure.+ const droppedOnShutdown = this.pool.pendingRequests.size;+ if (droppedOnShutdown > 0) {+ recordIncrement(+ "langfuse.tokenisation.skipped_on_shutdown",+ droppedOnShutdown,+ );+ logger.info(+ `Dropped ${droppedOnShutdown} in-flight token-count request(s) on shutdown.`,+ );+ }for (const [, request] of this.pool.pendingRequests.entries()) {clearTimeout(request.timeout);- request.reject(new Error("Worker pool is terminating"));+ request.resolve(undefined);}this.pool.pendingRequests.clear();- // Terminate all workersawait Promise.all(this.pool.workers.map((worker) => worker.terminate()));this.pool.workers = [];}