Skip to content

πŸͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

15.1 KB
Raw Blame
import { Job, Processor } from "bullmq";
import { EvalTemplateType, JobExecutionStatus } from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
import {
  QueueName,
  TQueueJobTypes,
  logger,
  traceException,
  EvalExecutionQueue,
  SecondaryEvalExecutionQueue,
  LLMAsJudgeExecutionQueue,
  QueueJobs,
  getCurrentSpan,
  classifyEvaluatorLlmError,
  type EvaluatorLlmErrorClassification,
} from "@langfuse/shared/src/server";
import { createEvalJobs, evaluate } from "../features/evaluation/evalService";
import { processObservationEval } from "../features/evaluation/observationEval/observationEvalProcessor";
import { createW3CTraceId } from "../features/utils/utilities";
import { retryLLMRateLimitError } from "../features/utils/retry-handler";
import { isUnrecoverableError } from "../errors/UnrecoverableError";
import { retryObservationNotFound } from "../features/evaluation/retryObservationNotFound";
import { isObservationNotFoundError } from "../errors/ObservationNotFoundError";
import { env } from "../env";
import {
  getLlmEvalTerminalErrorAttribution,
  recordEvalTerminalOutcome,
  recordEvalTimeToFirstAttempt,
} from "../features/evaluation/evalExecutionMetrics";

export const evalJobTraceCreatorQueueProcessor = async (
  job: Job<TQueueJobTypes[QueueName.TraceUpsert]>,
) => {
  try {
    await createEvalJobs({
      sourceEventType: "trace-upsert",
      event: job.data.payload,
      jobTimestamp: job.data.timestamp,
      enforcedJobTimeScope: "NEW", // we must not execute evals which are intended for existing data only.
    });
    return true;
  } catch (e) {
    logger.error(
      `Failed job Evaluation for traceId ${job.data.payload.traceId}`,
      e,
    );
    traceException(e);
    throw e;
  }
};

export const evalJobDatasetCreatorQueueProcessor = async (
  job: Job<TQueueJobTypes[QueueName.DatasetRunItemUpsert]>,
) => {
  try {
    await createEvalJobs({
      sourceEventType: "dataset-run-item-upsert",
      event: job.data.payload,
      jobTimestamp: job.data.timestamp,
      enforcedJobTimeScope: "NEW", // we must not execute evals which are intended for existing data only.
    });
    return true;
  } catch (e) {
    // Handle observation-not-found errors with manual retry
    if (isObservationNotFoundError(e)) {
      const shouldRetry = await retryObservationNotFound(e, {
        data: {
          projectId: job.data.payload.projectId,
          datasetItemId: job.data.payload.datasetItemId,
          traceId: job.data.payload.traceId,
          observationId: job.data.payload.observationId,
          retryBaggage: job.data.retryBaggage,
        },
      });

      if (shouldRetry) {
        // Retry was scheduled, complete this job successfully
        return true;
      }

      // Max attempts reached, log warning and complete successfully
      logger.warn(
        `Observation not found after max retries. Completing job without creating eval.`,
        {
          projectId: job.data.payload.projectId,
          datasetItemId: job.data.payload.datasetItemId,
          observationId: job.data.payload.observationId,
          traceId: job.data.payload.traceId,
        },
      );
      return true;
    }

    // All other errors should be logged and propagated for BullMQ retry
    logger.error(
      `Failed job Evaluation for dataset item: ${job.data.payload.datasetItemId}`,
      e,
    );
    traceException(e);
    throw e;
  }
};

export const evalJobCreatorQueueProcessor = async (
  job: Job<TQueueJobTypes[QueueName.CreateEvalQueue]>,
) => {
  try {
    await createEvalJobs({
      sourceEventType: "ui-create-eval",
      event: job.data.payload,
      jobTimestamp: job.data.timestamp,
    });
    return true;
  } catch (e) {
    logger.error(
      `Failed to create evaluation jobs: ${JSON.stringify(job.data.payload)}`,
      e,
    );
    traceException(e);
    throw e;
  }
};

export const evalJobExecutorQueueProcessorBuilder = (
  enableRedirectToSecondaryQueue: boolean,
  queueName: string,
): Processor => {
  const projectIdsToRedirectToSecondaryQueue =
    env.LANGFUSE_SECONDARY_EVAL_EXECUTION_QUEUE_ENABLED_PROJECT_IDS?.split(
      ",",
    ) ?? [];

  return async (job: Job<TQueueJobTypes[QueueName.EvaluationExecution]>) => {
    try {
      logger.info("Executing Evaluation Execution Job", job.data);

      // Redirect selected projects to the secondary queue from the primary consumer.
      if (enableRedirectToSecondaryQueue) {
        const projectId = job.data.payload.projectId;
        const shouldRedirectToSecondaryQueue =
          projectIdsToRedirectToSecondaryQueue.includes(projectId);

        if (shouldRedirectToSecondaryQueue) {
          logger.debug(
            `Redirecting evaluation execution job to secondary queue for project ${projectId}`,
          );
          const shardingKey = `${projectId}-${job.data.payload.jobExecutionId}`;
          const secondaryQueue = SecondaryEvalExecutionQueue.getInstance({
            shardingKey,
          });
          if (!secondaryQueue) {
            throw new Error(
              "Secondary evaluation execution queue is not available",
            );
          }

          await secondaryQueue.add(
            QueueName.EvaluationExecutionSecondaryQueue,
            job.data,
          );
          return;
        }
      }

      const span = getCurrentSpan();

      if (span) {
        span.setAttribute(
          "messaging.bullmq.job.input.jobExecutionId",
          job.data.payload.jobExecutionId,
        );
        span.setAttribute(
          "messaging.bullmq.job.input.projectId",
          job.data.payload.projectId,
        );
        span.setAttribute(
          "messaging.bullmq.job.input.retryBaggage.attempt",
          job.data.retryBaggage?.attempt ?? 0,
        );
      }

      await evaluate({ event: job.data.payload });
      return true;
    } catch (e) {
      const llmError = classifyEvaluatorLlmError(e);
      // β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”
      // β”‚   Job Fails with Error  β”‚
      // β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜
      //             β”‚
      //             β–Ό
      // β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”
      // β”‚ Is it a retryable native AI SDK        β”‚
      // β”‚ provider error?                        β”‚
      // β””β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”˜
      //       β”‚ Yes                          β”‚ No
      //       β–Ό                              β–Ό
      // β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”       β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”
      // β”‚ Is job inside itsβ”‚       β”‚ Is it retryable?      β”‚
      // β”‚ retry budget?    β”‚       β”‚ (shouldRetryJob)      β”‚
      // β””β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜       β””β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”˜
      //   Yes β”‚      β”‚ No             Yesβ”‚             β”‚No
      //       β–Ό      β–Ό                Yesβ”‚             β”‚No
      // β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”          β–Ό             β–Ό
      // β”‚Set:     β”‚ β”‚Set:    β”‚    β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”  β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”
      // β”‚DELAYED  β”‚ β”‚ERROR   β”‚    β”‚BullMQ   β”‚  β”‚Set:      β”‚
      // β”‚Retry by β”‚ β”‚Stop    β”‚    β”‚retry    β”‚  β”‚ERROR     β”‚
      // β”‚120 min  β”‚ β”‚        β”‚    β”‚w/ exp.  β”‚  β”‚Done      β”‚
      // β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”˜    β”‚backoff  β”‚  β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜
      //                           β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜

      const executionTraceId = createW3CTraceId(
        job.data.payload.jobExecutionId,
      );

      if (llmError?.isRetryable) {
        const queue = queueName.startsWith(
          QueueName.EvaluationExecutionSecondaryQueue,
        )
          ? SecondaryEvalExecutionQueue.getInstance({ shardName: queueName })
          : EvalExecutionQueue.getInstance({ shardName: queueName });

        const retryResult = await retryLLMRateLimitError(job, {
          table: "job_executions",
          idField: "jobExecutionId",
          queue,
          queueName,
          jobName: QueueJobs.EvaluationExecution,
        });

        if (retryResult.outcome === "scheduled") {
          // Use the deterministic execution trace ID to update the job execution
          await prisma.jobExecution.update({
            where: {
              id: job.data.payload.jobExecutionId,
              projectId: job.data.payload.projectId,
            },
            data: {
              status: JobExecutionStatus.DELAYED,
              executionTraceId,
            },
          });

          // Return early as we have already scheduled a delayed retry
          return;
        }
      }

      // At this point only terminal LLM failures and application errors remain.
      await prisma.jobExecution.update({
        where: {
          id: job.data.payload.jobExecutionId,
          projectId: job.data.payload.projectId,
        },
        data: {
          status: JobExecutionStatus.ERROR,
          endTime: new Date(),
          // Show user-facing error messages (LLM and config errors)
          error:
            llmError || isUnrecoverableError(e)
              ? (llmError?.message ?? (e as Error).message)
              : "An internal error occurred",
          executionTraceId,
        },
      });

      if (llmError || isUnrecoverableError(e)) {
        logger.warn(
          `${queueName} job ${job.data.payload.jobExecutionId} failed terminally`,
          getTerminalEvalErrorLogContext(job.data.payload, e, llmError),
        );
        return;
      }

      traceException(e);
      logger.error(
        `Failed ${queueName} job for id ${job.data.payload.jobExecutionId}`,
        e,
      );

      // Retry job by rethrowing error
      throw e;
    }
  };
};

export const llmAsJudgeExecutionQueueProcessorBuilder =
  (queueName: string): Processor =>
  async (job: Job<TQueueJobTypes[QueueName.LLMAsJudgeExecution]>) => {
    const retryAttempt = job.data.retryBaggage?.attempt ?? 0;
    if (job.attemptsStarted === 1 && retryAttempt === 0) {
      recordEvalTimeToFirstAttempt(
        EvalTemplateType.LLM_AS_JUDGE,
        Math.max(0, Date.now() - job.timestamp),
      );
    }

    try {
      logger.debug(
        "Executing LLM-as-Judge Observation Evaluation Job",
        job.data,
      );

      const span = getCurrentSpan();

      if (span) {
        span.setAttribute(
          "messaging.bullmq.job.input.jobExecutionId",
          job.data.payload.jobExecutionId,
        );
        span.setAttribute(
          "messaging.bullmq.job.input.projectId",
          job.data.payload.projectId,
        );
        span.setAttribute(
          "messaging.bullmq.job.input.retryBaggage.attempt",
          retryAttempt,
        );
      }

      const outcome = await processObservationEval({
        event: job.data.payload,
        executionType: EvalTemplateType.LLM_AS_JUDGE,
      });

      if (outcome === "completed") {
        recordEvalTerminalOutcome(EvalTemplateType.LLM_AS_JUDGE, "success");
      } else if (outcome === "cancelled") {
        recordEvalTerminalOutcome(EvalTemplateType.LLM_AS_JUDGE, "cancelled");
      }

      return true;
    } catch (e) {
      const llmError = classifyEvaluatorLlmError(e);
      // Media may still be uploading when evaluation starts after ingestion.
      // Retry only this evaluator job instead of making the error globally retryable.
      const isMediaNotFoundError =
        llmError?.kind === "validation" &&
        llmError.validationError?.code === "media-not-found";
      const executionTraceId = createW3CTraceId(
        job.data.payload.jobExecutionId,
      );

      if (llmError?.isRetryable) {
        const queue = LLMAsJudgeExecutionQueue.getInstance({
          shardName: queueName,
        });
        const retryResult = await retryLLMRateLimitError(job, {
          table: "job_executions",
          idField: "jobExecutionId",
          queue,
          queueName,
          jobName: QueueJobs.LLMAsJudgeExecution,
        });

        if (retryResult.outcome === "scheduled") {
          await prisma.jobExecution.update({
            where: {
              id: job.data.payload.jobExecutionId,
              projectId: job.data.payload.projectId,
            },
            data: {
              status: JobExecutionStatus.DELAYED,
              executionTraceId,
            },
          });

          return;
        }
      }

      const isUnrecoverable = isUnrecoverableError(e);
      const isTerminalError =
        (Boolean(llmError) && !isMediaNotFoundError) || isUnrecoverable;
      const totalAttempts = job.opts.attempts ?? 1;
      const isFinalAttempt = job.attemptsMade + 1 >= totalAttempts;

      // Only persist the terminal ERROR state when there will be no more
      // retries; otherwise observationEvalProcessor would short-circuit the
      // retry attempts because it skips jobs already in ERROR status.
      if (isTerminalError || isFinalAttempt) {
        await prisma.jobExecution.update({
          where: {
            id: job.data.payload.jobExecutionId,
            projectId: job.data.payload.projectId,
          },
          data: {
            status: JobExecutionStatus.ERROR,
            endTime: new Date(),
            error:
              llmError || isUnrecoverable
                ? (llmError?.message ?? (e as Error).message)
                : "An internal error occurred",
            executionTraceId,
          },
        });
        const attribution = getLlmEvalTerminalErrorAttribution({
          error: e,
          classification: llmError,
          isUnrecoverable,
        });
        recordEvalTerminalOutcome(
          EvalTemplateType.LLM_AS_JUDGE,
          attribution.outcome,
          attribution.reason,
        );

        if (isTerminalError || isMediaNotFoundError) {
          logger.warn(
            `LLM-as-Judge execution job ${job.data.payload.jobExecutionId} failed terminally`,
            {
              ...getTerminalEvalErrorLogContext(job.data.payload, e, llmError),
              outcome: attribution.outcome,
              reason: attribution.reason,
            },
          );
        }
      }

      if (isTerminalError || (isMediaNotFoundError && isFinalAttempt)) return;

      traceException(e);
      logger.error(
        `Failed LLM-as-Judge execution job for id ${job.data.payload.jobExecutionId}`,
        e,
      );

      throw e;
    }
  };

function getTerminalEvalErrorLogContext(
  payload: { projectId: string; jobExecutionId: string },
  error: unknown,
  llmError: EvaluatorLlmErrorClassification | null,
) {
  return {
    projectId: payload.projectId,
    jobExecutionId: payload.jobExecutionId,
    errorName: error instanceof Error ? error.name : typeof error,
    errorMessage: (error instanceof Error
      ? error.message
      : String(error)
    ).slice(0, 500),
    llmErrorKind: llmError?.kind,
    llmStatusCode: llmError?.statusCode,
  };
}