Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

7.9 KB
Raw Blame
import { Job, Processor, Worker, WorkerOptions } from "bullmq";
import { context as otelContext } from "@opentelemetry/api";
import {
  convertQueueNameToMetricName,
  contextWithLangfuseProps,
  createBullMQWorkerOptionsWithRedis,
  logger,
  QueueName,
  recordDistribution,
  recordIncrement,
  traceException,
} from "@langfuse/shared/src/server";
import {
  markQueueJobActivity,
  markQueueWorkerRegistered,
} from "../features/health/queueConsumption";
import { WORKER_HOST_ID } from "../utils/hostId";
import { logRetryableJobFailure } from "./jobFailureLog";
import { SHARDED_QUEUE_BASE_NAMES } from "./shardedQueueRegistry";
import { classifyJobFailure } from "./jobFailureReason";

export class WorkerManager {
  private static workers: { [key: string]: Worker } = {};

  private static extractProjectId(job: Job): string | undefined {
    const data = job.data as
      | {
          payload?: {
            projectId?: unknown;
            authCheck?: { scope?: { projectId?: unknown } };
          };
        }
      | undefined;

    const candidates = [
      data?.payload?.projectId,
      data?.payload?.authCheck?.scope?.projectId,
    ];

    return candidates.find((candidate): candidate is string => {
      return typeof candidate === "string" && candidate.length > 0;
    });
  }

  private static resolveMetricInfo(queueName: QueueName): {
    baseMetric: string;
    shardTag: { shard: string } | undefined;
  } {
    for (const base of SHARDED_QUEUE_BASE_NAMES) {
      if (queueName.startsWith(base)) {
        return {
          baseMetric: convertQueueNameToMetricName(base),
          shardTag: { shard: queueName },
        };
      }
    }
    return {
      baseMetric: convertQueueNameToMetricName(queueName),
      shardTag: undefined,
    };
  }

  // Empty failed set emits 0 so monitors see the gauge reset after a DLQ
  // drain.
  public static computeDlqOldestAgeMs(
    jobs: (Job | undefined)[],
    nowMs: number,
  ): number {
    const oldest = jobs.find(Boolean);
    return oldest ? nowMs - (oldest.finishedOn ?? oldest.timestamp) : 0;
  }

  /**
   * Queue wait before the current pickup: first attempts measure from
   * timestamp + opts.delay; retries and stall re-runs return undefined because
   * BullMQ keeps no record of when they became ready again.
   */
  public static computeWaitTimeMs(
    job: Pick<
      Job,
      | "timestamp"
      | "opts"
      | "attemptsMade"
      | "attemptsStarted"
      | "stalledCounter"
    >,
    nowMs: number,
  ): { attempt: "first" | "retry"; waitMs: number | undefined } {
    const isRetry =
      (job.attemptsMade ?? 0) > 0 ||
      (job.attemptsStarted ?? 0) > 1 ||
      (job.stalledCounter ?? 0) > 0;
    if (isRetry) {
      return { attempt: "retry", waitMs: undefined };
    }
    const readyAt = job.timestamp + Math.max(0, job.opts?.delay ?? 0);
    return { attempt: "first", waitMs: Math.max(0, nowMs - readyAt) };
  }

  private static metricWrapper(
    processor: Processor,
    queueName: QueueName,
  ): Processor {
    const { baseMetric, shardTag } = WorkerManager.resolveMetricInfo(queueName);

    return async (job: Job) => {
      const startTime = Date.now();
      const { attempt, waitMs } = WorkerManager.computeWaitTimeMs(
        job,
        startTime,
      );

      recordIncrement(baseMetric + ".rate", 1, {
        type: "request",
        attempt,
        ...shardTag,
      });

      if (waitMs !== undefined) {
        recordDistribution(baseMetric + ".time_distribution", waitMs, {
          type: "wait",
          attempt,
          unit: "milliseconds",
          ...shardTag,
        });
      }

      const clickHouseCtx = contextWithLangfuseProps({
        projectId: WorkerManager.extractProjectId(job),
        clickhouse: {
          surface: "worker",
          route: baseMetric,
        },
      });
      const result = await otelContext.with(clickHouseCtx, () =>
        processor(job),
      );

      const processingTime = Date.now() - startTime;
      recordDistribution(baseMetric + ".time_distribution", processingTime, {
        type: "processing",
        unit: "milliseconds",
        ...shardTag,
      });

      return result;
    };
  }

  public static async closeWorkers(): Promise<void> {
    await Promise.all(
      Object.values(WorkerManager.workers).map((worker) => worker.close()),
    );
    logger.info("All workers have been closed.");
  }

  public static getWorker(queueName: QueueName): Worker | undefined {
    return WorkerManager.workers[queueName];
  }

  public static getRegisteredQueueNames(): string[] {
    return Object.keys(WorkerManager.workers);
  }

  public static register(
    queueName: QueueName,
    processor: Processor,
    additionalOptions: Partial<WorkerOptions> = {},
  ): void {
    if (WorkerManager.workers[queueName]) {
      logger.info(`Worker ${queueName} is already registered`);
      return;
    }

    const workerOptionsWithRedis =
      createBullMQWorkerOptionsWithRedis(queueName);
    if (!workerOptionsWithRedis) {
      logger.error("Failed to initialize redis connection");
      return;
    }

    // Register worker
    const worker = new Worker(
      queueName,
      WorkerManager.metricWrapper(processor, queueName),
      {
        ...workerOptionsWithRedis,
        ...additionalOptions,
      },
    );
    WorkerManager.workers[queueName] = worker;
    markQueueWorkerRegistered();
    logger.info(`${queueName} executor started: ${worker.isRunning()}`);

    const { baseMetric, shardTag } = WorkerManager.resolveMetricInfo(queueName);

    // Liveness signal for the ?failIfQueueConsumptionStuck=true health check.
    // "active" and "completed" prove this container's consumption loop is
    // alive; "failed" is excluded because the stalled-checker emits it for
    // jobs this container never picked up.
    worker.on("active", markQueueJobActivity);
    // No "active" counter: metricWrapper already records "request" on pickup.
    worker.on("completed", () => {
      markQueueJobActivity();
      recordIncrement(baseMetric + ".rate", 1, {
        type: "completed",
        ...shardTag,
      });
    });

    // Add error handling
    worker.on("failed", (job: Job | undefined, err: Error) => {
      const reason = classifyJobFailure(err);
      logRetryableJobFailure({
        message: `Queue job ${job?.name} with id ${job?.id} in ${queueName} failed`,
        error: err,
        job,
        attemptsIncludeCurrentFailure: true,
        fields: {
          projectId: job ? WorkerManager.extractProjectId(job) : undefined,
          jobId: job?.id,
          failureReason: reason,
          // true once BullMQ stops retrying: the job's events are dropped.
          terminal: Boolean(job?.finishedOn),
        },
      });
      recordIncrement(baseMetric + ".rate", 1, {
        type: "failed",
        reason,
        ...shardTag,
      });
      // BullMQ sets finishedOn only when it moves the job to the failed set
      // instead of scheduling a retry.
      if (job?.finishedOn) {
        recordIncrement(baseMetric + ".rate", 1, {
          type: "failed_terminal",
          reason,
          ...shardTag,
        });
      }
    });
    worker.on("error", (failedReason: Error) => {
      logger.error(
        `Queue job ${queueName} errored: ${failedReason}`,
        failedReason,
      );
      traceException(failedReason);
      recordIncrement(baseMetric + ".rate", 1, {
        type: "error",
        ...shardTag,
      });
    });
    // Counts intermediate re-enqueues (LFE-10063), not just the terminal
    // "stalled more than allowable limit" the "failed" handler catches.
    worker.on("stalled", (jobId: string) => {
      // detectedOnHost: the stall-checker pod, which may differ from the pod
      // whose lock expired.
      logger.warn(
        `Queue job ${jobId} in ${queueName} stalled (lock expired, re-enqueued) detectedOnHost=${WORKER_HOST_ID}`,
      );
      recordIncrement(baseMetric + ".rate", 1, {
        type: "stalled",
        ...shardTag,
      });
    });
  }
}