Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

5.5 KB
Raw Blame
import { Queue } from "bullmq";
import { QueueName, TQueueJobTypes } from "../queues";
import { createBullMQQueueOptionsWithRedis } from "./redis";
import { logger } from "../logger";
import { getShardIndex } from "./sharding";
import { env } from "../../env";

export class IngestionQueue {
  private static instances: Map<
    number,
    Queue<TQueueJobTypes[QueueName.IngestionQueue]> | null
  > = new Map();

  public static getShardNames() {
    return Array.from(
      { length: env.LANGFUSE_INGESTION_QUEUE_SHARD_COUNT },
      (_, i) => `${QueueName.IngestionQueue}${i > 0 ? `-${i}` : ""}`,
    );
  }

  static getShardIndexFromShardName(
    shardName: string | undefined,
  ): number | null {
    if (!shardName) return null;

    // Extract shard index from shard name
    const shardIndex =
      shardName === QueueName.IngestionQueue
        ? 0
        : parseInt(shardName.replace(`${QueueName.IngestionQueue}-`, ""), 10);

    if (isNaN(shardIndex)) return null;
    return shardIndex;
  }

  /**
   * Get the ingestion queue instance for the given sharding key or shard name.
   * @param shardingKey - ShardingKey is being hashed and randomly allocated to a shard. Should be `projectId-eventBodyId`.
   * @param shardName - Name of the shard. Should be `ingestion-queue-${shardIndex}` or plainly `ingestion-queue` for the first shard.
   */
  public static getInstance({
    shardingKey,
    shardName,
  }: {
    shardingKey?: string;
    shardName?: string;
  }): Queue<TQueueJobTypes[QueueName.IngestionQueue]> | null {
    const shardIndex =
      IngestionQueue.getShardIndexFromShardName(shardName) ??
      (env.REDIS_CLUSTER_ENABLED === "true" && shardingKey
        ? getShardIndex(shardingKey, env.LANGFUSE_INGESTION_QUEUE_SHARD_COUNT)
        : 0);

    // Check if we already have an instance for this shard
    if (IngestionQueue.instances.has(shardIndex)) {
      return IngestionQueue.instances.get(shardIndex) || null;
    }

    const name = `${QueueName.IngestionQueue}${shardIndex > 0 ? `-${shardIndex}` : ""}`;
    const queueOptionsWithRedis = createBullMQQueueOptionsWithRedis(name);
    const queueInstance = queueOptionsWithRedis
      ? new Queue<TQueueJobTypes[QueueName.IngestionQueue]>(name, {
          ...queueOptionsWithRedis,
          defaultJobOptions: {
            removeOnComplete: true,
            removeOnFail: 100_000,
            attempts: 6,
            backoff: {
              type: "exponential",
              delay: 5000,
            },
          },
        })
      : null;

    queueInstance?.on("error", (err) => {
      logger.error(`IngestionQueue shard ${shardIndex} error`, err);
    });

    IngestionQueue.instances.set(shardIndex, queueInstance);

    return queueInstance;
  }
}

export class SecondaryIngestionQueue {
  private static instances: Map<
    number,
    Queue<TQueueJobTypes[QueueName.IngestionSecondaryQueue]> | null
  > = new Map();

  public static getShardNames() {
    return Array.from(
      { length: env.LANGFUSE_INGESTION_SECONDARY_QUEUE_SHARD_COUNT },
      (_, i) => `${QueueName.IngestionSecondaryQueue}${i > 0 ? `-${i}` : ""}`,
    );
  }

  static getShardIndexFromShardName(
    shardName: string | undefined,
  ): number | null {
    if (!shardName) return null;

    const shardIndex =
      shardName === QueueName.IngestionSecondaryQueue
        ? 0
        : parseInt(
            shardName.replace(`${QueueName.IngestionSecondaryQueue}-`, ""),
            10,
          );

    if (isNaN(shardIndex)) return null;
    return shardIndex;
  }

  /**
   * Get the secondary ingestion queue instance for the given sharding key or shard name.
   * @param shardingKey - ShardingKey is being hashed and randomly allocated to a shard. Should be `projectId-eventBodyId`.
   * @param shardName - Name of the shard. Should be `secondary-ingestion-queue-${shardIndex}` or plainly `secondary-ingestion-queue` for the first shard.
   */
  public static getInstance({
    shardingKey,
    shardName,
  }: {
    shardingKey?: string;
    shardName?: string;
  } = {}): Queue<TQueueJobTypes[QueueName.IngestionSecondaryQueue]> | null {
    const shardIndex =
      SecondaryIngestionQueue.getShardIndexFromShardName(shardName) ??
      (env.REDIS_CLUSTER_ENABLED === "true" && shardingKey
        ? getShardIndex(
            shardingKey,
            env.LANGFUSE_INGESTION_SECONDARY_QUEUE_SHARD_COUNT,
          )
        : 0);

    if (SecondaryIngestionQueue.instances.has(shardIndex)) {
      return SecondaryIngestionQueue.instances.get(shardIndex) || null;
    }

    const name = `${QueueName.IngestionSecondaryQueue}${shardIndex > 0 ? `-${shardIndex}` : ""}`;
    const queueOptionsWithRedis = createBullMQQueueOptionsWithRedis(name);
    const queueInstance = queueOptionsWithRedis
      ? new Queue<TQueueJobTypes[QueueName.IngestionSecondaryQueue]>(name, {
          ...queueOptionsWithRedis,
          defaultJobOptions: {
            removeOnComplete: true,
            removeOnFail: 100_000,
            attempts: 8,
            backoff: {
              type: "exponential",
              delay: 5000,
              // Spreads retries of jobs that failed together (e.g. one S3
              // throttling episode) so they don't hit the prefix in lockstep.
              jitter: 0.5,
            },
          },
        })
      : null;

    queueInstance?.on("error", (err) => {
      logger.error(`SecondaryIngestionQueue shard ${shardIndex} error`, err);
    });

    SecondaryIngestionQueue.instances.set(shardIndex, queueInstance);

    return queueInstance;
  }
}