🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { Job, Processor, Worker, WorkerOptions } from "bullmq";
import { context as otelContext } from "@opentelemetry/api";
import {
convertQueueNameToMetricName,
contextWithLangfuseProps,
createBullMQWorkerOptionsWithRedis,
logger,
QueueName,
recordDistribution,
recordIncrement,
traceException,
} from "@langfuse/shared/src/server";
import {
markQueueJobActivity,
markQueueWorkerRegistered,
} from "../features/health/queueConsumption";
import { WORKER_HOST_ID } from "../utils/hostId";
import { logRetryableJobFailure } from "./jobFailureLog";
import { SHARDED_QUEUE_BASE_NAMES } from "./shardedQueueRegistry";
import { classifyJobFailure } from "./jobFailureReason";
export class WorkerManager {
private static workers: { [key: string]: Worker } = {};
private static extractProjectId(job: Job): string | undefined {
const data = job.data as
| {
payload?: {
projectId?: unknown;
authCheck?: { scope?: { projectId?: unknown } };
};
}
| undefined;
const candidates = [
data?.payload?.projectId,
data?.payload?.authCheck?.scope?.projectId,
];
return candidates.find((candidate): candidate is string => {
return typeof candidate === "string" && candidate.length > 0;
});
}
private static resolveMetricInfo(queueName: QueueName): {
baseMetric: string;
shardTag: { shard: string } | undefined;
} {
for (const base of SHARDED_QUEUE_BASE_NAMES) {
if (queueName.startsWith(base)) {
return {
baseMetric: convertQueueNameToMetricName(base),
shardTag: { shard: queueName },
};
}
}
return {
baseMetric: convertQueueNameToMetricName(queueName),
shardTag: undefined,
};
}
// Empty failed set emits 0 so monitors see the gauge reset after a DLQ
// drain.
public static computeDlqOldestAgeMs(
jobs: (Job | undefined)[],
nowMs: number,
): number {
const oldest = jobs.find(Boolean);
return oldest ? nowMs - (oldest.finishedOn ?? oldest.timestamp) : 0;
}
/**
* Queue wait before the current pickup: first attempts measure from
* timestamp + opts.delay; retries and stall re-runs return undefined because
* BullMQ keeps no record of when they became ready again.
*/
public static computeWaitTimeMs(
job: Pick<
Job,
| "timestamp"
| "opts"
| "attemptsMade"
| "attemptsStarted"
| "stalledCounter"
>,
nowMs: number,
): { attempt: "first" | "retry"; waitMs: number | undefined } {
const isRetry =
(job.attemptsMade ?? 0) > 0 ||
(job.attemptsStarted ?? 0) > 1 ||
(job.stalledCounter ?? 0) > 0;
if (isRetry) {
return { attempt: "retry", waitMs: undefined };
}
const readyAt = job.timestamp + Math.max(0, job.opts?.delay ?? 0);
return { attempt: "first", waitMs: Math.max(0, nowMs - readyAt) };
}
private static metricWrapper(
processor: Processor,
queueName: QueueName,
): Processor {
const { baseMetric, shardTag } = WorkerManager.resolveMetricInfo(queueName);
return async (job: Job) => {
const startTime = Date.now();
const { attempt, waitMs } = WorkerManager.computeWaitTimeMs(
job,
startTime,
);
recordIncrement(baseMetric + ".rate", 1, {
type: "request",
attempt,
...shardTag,
});
if (waitMs !== undefined) {
recordDistribution(baseMetric + ".time_distribution", waitMs, {
type: "wait",
attempt,
unit: "milliseconds",
...shardTag,
});
}
const clickHouseCtx = contextWithLangfuseProps({
projectId: WorkerManager.extractProjectId(job),
clickhouse: {
surface: "worker",
route: baseMetric,
},
});
const result = await otelContext.with(clickHouseCtx, () =>
processor(job),
);
const processingTime = Date.now() - startTime;
recordDistribution(baseMetric + ".time_distribution", processingTime, {
type: "processing",
unit: "milliseconds",
...shardTag,
});
return result;
};
}
public static async closeWorkers(): Promise<void> {
await Promise.all(
Object.values(WorkerManager.workers).map((worker) => worker.close()),
);
logger.info("All workers have been closed.");
}
public static getWorker(queueName: QueueName): Worker | undefined {
return WorkerManager.workers[queueName];
}
public static getRegisteredQueueNames(): string[] {
return Object.keys(WorkerManager.workers);
}
public static register(
queueName: QueueName,
processor: Processor,
additionalOptions: Partial<WorkerOptions> = {},
): void {
if (WorkerManager.workers[queueName]) {
logger.info(`Worker ${queueName} is already registered`);
return;
}
const workerOptionsWithRedis =
createBullMQWorkerOptionsWithRedis(queueName);
if (!workerOptionsWithRedis) {
logger.error("Failed to initialize redis connection");
return;
}
// Register worker
const worker = new Worker(
queueName,
WorkerManager.metricWrapper(processor, queueName),
{
...workerOptionsWithRedis,
...additionalOptions,
},
);
WorkerManager.workers[queueName] = worker;
markQueueWorkerRegistered();
logger.info(`${queueName} executor started: ${worker.isRunning()}`);
const { baseMetric, shardTag } = WorkerManager.resolveMetricInfo(queueName);
// Liveness signal for the ?failIfQueueConsumptionStuck=true health check.
// "active" and "completed" prove this container's consumption loop is
// alive; "failed" is excluded because the stalled-checker emits it for
// jobs this container never picked up.
worker.on("active", markQueueJobActivity);
// No "active" counter: metricWrapper already records "request" on pickup.
worker.on("completed", () => {
markQueueJobActivity();
recordIncrement(baseMetric + ".rate", 1, {
type: "completed",
...shardTag,
});
});
// Add error handling
worker.on("failed", (job: Job | undefined, err: Error) => {
const reason = classifyJobFailure(err);
logRetryableJobFailure({
message: `Queue job ${job?.name} with id ${job?.id} in ${queueName} failed`,
error: err,
job,
attemptsIncludeCurrentFailure: true,
fields: {
projectId: job ? WorkerManager.extractProjectId(job) : undefined,
jobId: job?.id,
failureReason: reason,
// true once BullMQ stops retrying: the job's events are dropped.
terminal: Boolean(job?.finishedOn),
},
});
recordIncrement(baseMetric + ".rate", 1, {
type: "failed",
reason,
...shardTag,
});
// BullMQ sets finishedOn only when it moves the job to the failed set
// instead of scheduling a retry.
if (job?.finishedOn) {
recordIncrement(baseMetric + ".rate", 1, {
type: "failed_terminal",
reason,
...shardTag,
});
}
});
worker.on("error", (failedReason: Error) => {
logger.error(
`Queue job ${queueName} errored: ${failedReason}`,
failedReason,
);
traceException(failedReason);
recordIncrement(baseMetric + ".rate", 1, {
type: "error",
...shardTag,
});
});
// Counts intermediate re-enqueues (LFE-10063), not just the terminal
// "stalled more than allowable limit" the "failed" handler catches.
worker.on("stalled", (jobId: string) => {
// detectedOnHost: the stall-checker pod, which may differ from the pod
// whose lock expired.
logger.warn(
`Queue job ${jobId} in ${queueName} stalled (lock expired, re-enqueued) detectedOnHost=${WORKER_HOST_ID}`,
);
recordIncrement(baseMetric + ".rate", 1, {
type: "stalled",
...shardTag,
});
});
}
}