Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

3.4 KB
Raw Blame
import { Job, Processor } from "bullmq";
import { EvalTemplateType, JobExecutionStatus } from "@prisma/client";
import { prisma } from "@langfuse/shared/src/db";
import {
  CodeEvalExecutionError,
  getCodeEvalUserVisibleError,
  getCurrentSpan,
  logger,
  QueueName,
  TQueueJobTypes,
  traceException,
} from "@langfuse/shared/src/server";
import { processObservationEval } from "../features/evaluation/observationEval/observationEvalProcessor";
import { createW3CTraceId } from "../features/utils/utilities";
import { isUnrecoverableError } from "../errors/UnrecoverableError";
import {
  getCodeEvalTerminalErrorOutcome,
  recordEvalTerminalOutcome,
  recordEvalTimeToFirstAttempt,
} from "../features/evaluation/evalExecutionMetrics";

export const codeEvalExecutionQueueProcessorBuilder = (
  _queueName: string,
): Processor => {
  return async (job: Job<TQueueJobTypes[QueueName.CodeEvalExecution]>) => {
    if (job.attemptsStarted === 1) {
      recordEvalTimeToFirstAttempt(
        EvalTemplateType.CODE,
        Math.max(0, Date.now() - job.timestamp),
      );
    }

    try {
      logger.debug("Executing Code Evaluation Observation Job", job.data);

      const span = getCurrentSpan();

      if (span) {
        span.setAttribute(
          "messaging.bullmq.job.input.jobExecutionId",
          job.data.payload.jobExecutionId,
        );
        span.setAttribute(
          "messaging.bullmq.job.input.projectId",
          job.data.payload.projectId,
        );
      }

      const outcome = await processObservationEval({
        event: job.data.payload,
        executionType: EvalTemplateType.CODE,
      });

      if (outcome === "completed") {
        recordEvalTerminalOutcome(EvalTemplateType.CODE, "success");
      } else if (outcome === "cancelled") {
        recordEvalTerminalOutcome(EvalTemplateType.CODE, "cancelled");
      }

      return true;
    } catch (e) {
      const executionTraceId = createW3CTraceId(
        job.data.payload.jobExecutionId,
      );

      const isTerminalError =
        isUnrecoverableError(e) ||
        (e instanceof CodeEvalExecutionError && !e.retryable);
      const totalAttempts = job.opts.attempts ?? 1;
      const isFinalAttempt = job.attemptsMade + 1 >= totalAttempts;

      // Only persist the terminal ERROR state when there will be no more
      // retries; otherwise observationEvalProcessor would short-circuit the
      // retry attempts because it skips jobs already in ERROR status.
      if (isTerminalError || isFinalAttempt) {
        await prisma.jobExecution.update({
          where: {
            id: job.data.payload.jobExecutionId,
            projectId: job.data.payload.projectId,
          },
          data: {
            status: JobExecutionStatus.ERROR,
            endTime: new Date(),
            error: getJobExecutionErrorMessage(e),
            executionTraceId,
          },
        });
        recordEvalTerminalOutcome(
          EvalTemplateType.CODE,
          getCodeEvalTerminalErrorOutcome(e),
        );
      }

      if (isTerminalError) return;

      traceException(e);
      logger.error(
        `Failed code eval execution job for id ${job.data.payload.jobExecutionId}`,
        e,
      );

      throw e;
    }
  };
};

function getJobExecutionErrorMessage(e: unknown): string {
  if (isUnrecoverableError(e)) return e.message;
  if (e instanceof CodeEvalExecutionError) return e.message;

  return getCodeEvalUserVisibleError(e).message;
}