🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { Job } from "bullmq";
import {
ExperimentCreateQueue,
QueueJobs,
QueueName,
TQueueJobTypes,
classifyEvaluatorLlmError,
logger,
traceException,
} from "@langfuse/shared/src/server";
import { retryLLMRateLimitError } from "../features/utils/retry-handler";
import { createExperimentJobClickhouse } from "../features/experiments/experimentServiceClickhouse";
import { isUnrecoverableError } from "../errors/UnrecoverableError";
export const experimentCreateQueueProcessor = async (
job: Job<TQueueJobTypes[QueueName.ExperimentCreate]>,
) => {
try {
await createExperimentJobClickhouse({
event: job.data.payload,
});
return true;
} catch (e) {
const llmError = classifyEvaluatorLlmError(e);
if (llmError?.isRetryable) {
const retryResult = await retryLLMRateLimitError(job, {
table: "dataset_runs",
idField: "runId",
queue: ExperimentCreateQueue.getInstance(),
queueName: QueueName.ExperimentCreate,
jobName: QueueJobs.ExperimentCreateJob,
});
if (retryResult.outcome === "scheduled") return;
if (retryResult.outcome === "queue_unavailable") throw e;
}
if (llmError || isUnrecoverableError(e)) return;
logger.error(
`Failed to process experiment create job for project: ${job.data.payload.projectId}`,
e,
);
traceException(e);
// Retry job by rethrowing error
throw e;
}
};