🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { Queue } from "bullmq";
import { QueueName, TQueueJobTypes } from "../queues";
import { createBullMQQueueOptionsWithRedis } from "./redis";
import { logger } from "../logger";
import { getShardIndex } from "./sharding";
import { env } from "../../env";
export class IngestionQueue {
private static instances: Map<
number,
Queue<TQueueJobTypes[QueueName.IngestionQueue]> | null
> = new Map();
public static getShardNames() {
return Array.from(
{ length: env.LANGFUSE_INGESTION_QUEUE_SHARD_COUNT },
(_, i) => `${QueueName.IngestionQueue}${i > 0 ? `-${i}` : ""}`,
);
}
static getShardIndexFromShardName(
shardName: string | undefined,
): number | null {
if (!shardName) return null;
// Extract shard index from shard name
const shardIndex =
shardName === QueueName.IngestionQueue
? 0
: parseInt(shardName.replace(`${QueueName.IngestionQueue}-`, ""), 10);
if (isNaN(shardIndex)) return null;
return shardIndex;
}
/**
* Get the ingestion queue instance for the given sharding key or shard name.
* @param shardingKey - ShardingKey is being hashed and randomly allocated to a shard. Should be `projectId-eventBodyId`.
* @param shardName - Name of the shard. Should be `ingestion-queue-${shardIndex}` or plainly `ingestion-queue` for the first shard.
*/
public static getInstance({
shardingKey,
shardName,
}: {
shardingKey?: string;
shardName?: string;
}): Queue<TQueueJobTypes[QueueName.IngestionQueue]> | null {
const shardIndex =
IngestionQueue.getShardIndexFromShardName(shardName) ??
(env.REDIS_CLUSTER_ENABLED === "true" && shardingKey
? getShardIndex(shardingKey, env.LANGFUSE_INGESTION_QUEUE_SHARD_COUNT)
: 0);
// Check if we already have an instance for this shard
if (IngestionQueue.instances.has(shardIndex)) {
return IngestionQueue.instances.get(shardIndex) || null;
}
const name = `${QueueName.IngestionQueue}${shardIndex > 0 ? `-${shardIndex}` : ""}`;
const queueOptionsWithRedis = createBullMQQueueOptionsWithRedis(name);
const queueInstance = queueOptionsWithRedis
? new Queue<TQueueJobTypes[QueueName.IngestionQueue]>(name, {
...queueOptionsWithRedis,
defaultJobOptions: {
removeOnComplete: true,
removeOnFail: 100_000,
attempts: 6,
backoff: {
type: "exponential",
delay: 5000,
},
},
})
: null;
queueInstance?.on("error", (err) => {
logger.error(`IngestionQueue shard ${shardIndex} error`, err);
});
IngestionQueue.instances.set(shardIndex, queueInstance);
return queueInstance;
}
}
export class SecondaryIngestionQueue {
private static instances: Map<
number,
Queue<TQueueJobTypes[QueueName.IngestionSecondaryQueue]> | null
> = new Map();
public static getShardNames() {
return Array.from(
{ length: env.LANGFUSE_INGESTION_SECONDARY_QUEUE_SHARD_COUNT },
(_, i) => `${QueueName.IngestionSecondaryQueue}${i > 0 ? `-${i}` : ""}`,
);
}
static getShardIndexFromShardName(
shardName: string | undefined,
): number | null {
if (!shardName) return null;
const shardIndex =
shardName === QueueName.IngestionSecondaryQueue
? 0
: parseInt(
shardName.replace(`${QueueName.IngestionSecondaryQueue}-`, ""),
10,
);
if (isNaN(shardIndex)) return null;
return shardIndex;
}
/**
* Get the secondary ingestion queue instance for the given sharding key or shard name.
* @param shardingKey - ShardingKey is being hashed and randomly allocated to a shard. Should be `projectId-eventBodyId`.
* @param shardName - Name of the shard. Should be `secondary-ingestion-queue-${shardIndex}` or plainly `secondary-ingestion-queue` for the first shard.
*/
public static getInstance({
shardingKey,
shardName,
}: {
shardingKey?: string;
shardName?: string;
} = {}): Queue<TQueueJobTypes[QueueName.IngestionSecondaryQueue]> | null {
const shardIndex =
SecondaryIngestionQueue.getShardIndexFromShardName(shardName) ??
(env.REDIS_CLUSTER_ENABLED === "true" && shardingKey
? getShardIndex(
shardingKey,
env.LANGFUSE_INGESTION_SECONDARY_QUEUE_SHARD_COUNT,
)
: 0);
if (SecondaryIngestionQueue.instances.has(shardIndex)) {
return SecondaryIngestionQueue.instances.get(shardIndex) || null;
}
const name = `${QueueName.IngestionSecondaryQueue}${shardIndex > 0 ? `-${shardIndex}` : ""}`;
const queueOptionsWithRedis = createBullMQQueueOptionsWithRedis(name);
const queueInstance = queueOptionsWithRedis
? new Queue<TQueueJobTypes[QueueName.IngestionSecondaryQueue]>(name, {
...queueOptionsWithRedis,
defaultJobOptions: {
removeOnComplete: true,
removeOnFail: 100_000,
attempts: 8,
backoff: {
type: "exponential",
delay: 5000,
// Spreads retries of jobs that failed together (e.g. one S3
// throttling episode) so they don't hit the prefix in lockstep.
jitter: 0.5,
},
},
})
: null;
queueInstance?.on("error", (err) => {
logger.error(`SecondaryIngestionQueue shard ${shardIndex} error`, err);
});
SecondaryIngestionQueue.instances.set(shardIndex, queueInstance);
return queueInstance;
}
}