🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { Worker } from "worker_threads";
import { Model } from "@langfuse/shared";
import { logger, recordIncrement } from "@langfuse/shared/src/server";
import path from "path";
import { env } from "../../env";
interface TokenCountWorkerPool {
workers: Worker[];
currentWorkerIndex: number;
pendingRequests: Map<
string,
{
resolve: (value: number | undefined) => void;
reject: (error: Error) => void;
timeout: NodeJS.Timeout;
}
>;
}
export class TokenCountWorkerManager {
private pool: TokenCountWorkerPool;
private readonly workerPath: string;
private readonly poolSize: number;
private requestCounter = 0;
private isShuttingDown = false;
constructor(poolSize: number) {
this.poolSize = poolSize;
// Use compiled JavaScript file
this.workerPath = path.join(__dirname, "worker-thread.js");
this.pool = {
workers: [],
currentWorkerIndex: 0,
pendingRequests: new Map(),
};
this.initializeWorkers();
}
private initializeWorkers() {
for (let i = 0; i < this.poolSize; i++) {
this.createWorker();
}
}
private createWorker() {
const worker = this.createWorkerWithListeners();
this.pool.workers.push(worker);
}
private createWorkerWithListeners(): Worker {
const worker = new Worker(this.workerPath);
worker.on(
"message",
(data: {
id: string;
result: number | undefined;
error: string | null;
}) => {
const request = this.pool.pendingRequests.get(data.id);
if (request) {
clearTimeout(request.timeout);
this.pool.pendingRequests.delete(data.id);
if (data.error) {
request.reject(new Error(data.error));
} else {
request.resolve(data.result);
}
}
},
);
worker.on("error", (error) => {
// Terminating workers surface as errors/non-zero exits; don't respawn
// into a pool we are tearing down.
if (this.isShuttingDown) return;
logger.error("Worker thread error:", error);
this.replaceWorker(worker);
});
worker.on("exit", (code) => {
if (this.isShuttingDown) return;
if (code !== 0) {
logger.error(`Worker stopped with exit code ${code}`);
this.replaceWorker(worker);
}
});
return worker;
}
private cleanupPendingRequests() {
// Reject all pending requests to provide faster error feedback
for (const [, request] of this.pool.pendingRequests.entries()) {
clearTimeout(request.timeout);
request.reject(new Error("Worker failed and is being replaced"));
}
this.pool.pendingRequests.clear();
}
private replaceWorker(deadWorker: Worker) {
const index = this.pool.workers.indexOf(deadWorker);
if (index !== -1) {
// Clean up any pending requests for the dead worker
this.cleanupPendingRequests();
// Create a new worker with proper event listeners
this.pool.workers[index] = this.createWorkerWithListeners();
}
}
private getNextWorker(): Worker | undefined {
if (this.pool.workers.length === 0) {
return undefined;
}
const worker = this.pool.workers[this.pool.currentWorkerIndex];
this.pool.currentWorkerIndex =
(this.pool.currentWorkerIndex + 1) % this.poolSize;
return worker;
}
async tokenCount(
params: { model: Model; text: unknown },
timeoutMs = 30000,
): Promise<number | undefined> {
// Once the pool is terminating, drop the count instead of posting to a
// terminating or already-gone worker. undefined means usage unknown, which
// the caller commits as absent rather than as a zero; count the drop so it
// stays observable.
if (this.isShuttingDown) {
recordIncrement("langfuse.tokenisation.skipped_on_shutdown", 1);
return undefined;
}
return new Promise((resolve, reject) => {
const id = `token-count-${++this.requestCounter}-${Date.now()}`;
const worker = this.getNextWorker();
if (!worker) {
resolve(undefined);
return;
}
const timeout = setTimeout(() => {
this.pool.pendingRequests.delete(id);
reject(
new Error(`Token count operation timed out after ${timeoutMs}ms`),
);
}, timeoutMs);
this.pool.pendingRequests.set(id, { resolve, reject, timeout });
// Serialize the data to ensure no complex objects like Decimal are passed
const serializedParams = {
model: JSON.parse(JSON.stringify(params.model)),
text: params.text,
id,
};
worker.postMessage(serializedParams);
});
}
async terminate() {
// Set before touching the pool so concurrent tokenCount() calls and the
// workers' own exit/error events see the shutdown and stop early.
this.isShuttingDown = true;
// In-flight requests can no longer get a response from the terminating
// workers. Resolve them with undefined (usage unknown) rather than
// rejecting, and count the drop so it stays observable without the caller
// logging each one as a tokenization failure.
const droppedOnShutdown = this.pool.pendingRequests.size;
if (droppedOnShutdown > 0) {
recordIncrement(
"langfuse.tokenisation.skipped_on_shutdown",
droppedOnShutdown,
);
logger.info(
`Dropped ${droppedOnShutdown} in-flight token-count request(s) on shutdown.`,
);
}
for (const [, request] of this.pool.pendingRequests.entries()) {
clearTimeout(request.timeout);
request.resolve(undefined);
}
this.pool.pendingRequests.clear();
await Promise.all(this.pool.workers.map((worker) => worker.terminate()));
this.pool.workers = [];
}
}
// Singleton instance
let workerManager: TokenCountWorkerManager | null = null;
export function getTokenCountWorkerManager(
poolSize?: number,
): TokenCountWorkerManager {
if (!workerManager) {
workerManager = new TokenCountWorkerManager(
poolSize ?? env.LANGFUSE_TOKEN_COUNT_WORKER_POOL_SIZE,
);
}
return workerManager;
}
export async function tokenCountAsync(
params: {
model: Model;
text: unknown;
},
timeoutMs?: number,
): Promise<number | undefined> {
const manager = getTokenCountWorkerManager();
return manager.tokenCount(params, timeoutMs);
}