🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(evals): add evaluator execution SLO metrics (#16589)
Hassieb Pakzadadd3eee1b60a
diff --git a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.tsindex 01ae0c870f..163b6723dc 100644--- a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts+++ b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts@@ -9,7 +9,6 @@ import {parseDispatchResult,} from "../../../../../packages/shared/src/server/evals/codeEvalDispatcherTypes";import { CodeEvalExecutionError } from "../../../../../packages/shared/src/server/evals/codeEvalExecution";-import { UnrecoverableError } from "../../../errors/UnrecoverableError";const mocks = vi.hoisted(() => ({dispatcher: {@@ -490,7 +489,11 @@ describe("executeCodeBasedEvaluation", () => {executionMetadata: { job_execution_id: "job-1" },});- await expect(promise).rejects.toThrow(UnrecoverableError);+ await expect(promise).rejects.toBeInstanceOf(CodeEvalExecutionError);+ await expect(promise).rejects.toMatchObject({+ code: CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,+ retryable: false,+ });await expect(promise).rejects.toThrow("runner exploded");expect(mocks.writeInternalTrace).toHaveBeenCalledTimes(1);@@ -560,7 +563,11 @@ describe("executeCodeBasedEvaluation", () => {executionMetadata: { job_execution_id: "job-1" },});- await expect(promise).rejects.toThrow(UnrecoverableError);+ await expect(promise).rejects.toBeInstanceOf(CodeEvalExecutionError);+ await expect(promise).rejects.toMatchObject({+ code: CodeEvalDispatcherErrorCodes.INVALID_RESULT,+ retryable: false,+ });await expect(promise).rejects.toThrow("The evaluator returned an invalid result.",);@@ -669,7 +676,11 @@ describe("executeCodeBasedEvaluation", () => {executionMetadata: { job_execution_id: "job-1" },});- await expect(promise).rejects.toThrow(UnrecoverableError);+ await expect(promise).rejects.toBeInstanceOf(CodeEvalExecutionError);+ await expect(promise).rejects.toMatchObject({+ code: "INTERNAL_ERROR",+ retryable: false,+ });await expect(promise).rejects.toThrow("An internal error occurred");expect(mocks.writeInternalTrace).toHaveBeenCalledTimes(1);diff --git a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.ts b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.tsindex 06c73440bf..d818e8e164 100644--- a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.ts+++ b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.ts@@ -77,10 +77,6 @@ export async function executeCodeBasedEvaluation(params: {});if (!dispatchOutcome.success) {- if (dispatchOutcome.error.retryable === false) {- throw new UnrecoverableError(dispatchOutcome.error.message);- }-throw new CodeEvalExecutionError(dispatchOutcome.error);}diff --git a/worker/src/features/evaluation/evalExecutionMetrics.test.ts b/worker/src/features/evaluation/evalExecutionMetrics.test.tsnew file mode 100644index 0000000000..3ca04f3083--- /dev/null+++ b/worker/src/features/evaluation/evalExecutionMetrics.test.ts@@ -0,0 +1,129 @@+import { EvalTemplateType } from "@langfuse/shared";+import {+ CodeEvalDispatcherErrorCodes,+ CodeEvalExecutionError,+ recordDistribution,+ recordIncrement,+ type EvaluatorLlmErrorClassification,+} from "@langfuse/shared/src/server";+import { beforeEach, describe, expect, it, vi } from "vitest";+import {+ getCodeEvalTerminalErrorOutcome,+ getLlmEvalTerminalErrorOutcome,+ recordEvalTerminalOutcome,+ recordEvalTimeToFirstAttempt,+} from "./evalExecutionMetrics";++vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),+ recordDistribution: vi.fn(),+ recordIncrement: vi.fn(),+}));++describe("evaluation execution metrics", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("records the bounded metric contract", () => {+ recordEvalTimeToFirstAttempt(EvalTemplateType.LLM_AS_JUDGE, 1_234);+ recordEvalTerminalOutcome(EvalTemplateType.CODE, "success");++ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.time_to_first_attempt_ms",+ 1_234,+ {+ evaluator_type: "llm_as_judge",+ unit: "milliseconds",+ },+ );+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ 1,+ { evaluator_type: "code_as_judge", outcome: "success" },+ );+ });++ it.each([+ {+ classification: null,+ expected: "platform_error",+ },+ {+ classification: {+ kind: "validation",+ message: "invalid connection",+ isRetryable: false,+ error: new Error("invalid connection"),+ blockReason: null,+ } satisfies EvaluatorLlmErrorClassification,+ expected: "customer_error",+ },+ {+ classification: {+ kind: "provider",+ message: "invalid request",+ statusCode: 400,+ isRetryable: false,+ error: new Error("invalid request"),+ blockReason: null,+ } satisfies EvaluatorLlmErrorClassification,+ expected: "customer_error",+ },+ {+ classification: {+ kind: "provider",+ message: "rate limited",+ statusCode: 429,+ isRetryable: true,+ error: new Error("rate limited"),+ blockReason: null,+ } satisfies EvaluatorLlmErrorClassification,+ expected: "upstream_error",+ },+ {+ classification: {+ kind: "timeout",+ message: "provider timed out",+ isRetryable: false,+ error: new Error("provider timed out"),+ blockReason: null,+ } satisfies EvaluatorLlmErrorClassification,+ expected: "upstream_error",+ },+ ])(+ "classifies LLM terminal errors as $expected",+ ({ classification, expected }) => {+ expect(getLlmEvalTerminalErrorOutcome(classification)).toBe(expected);+ },+ );++ it.each([+ {+ code: CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,+ expected: "customer_error",+ },+ {+ code: CodeEvalDispatcherErrorCodes.TIMEOUT,+ expected: "customer_error",+ },+ {+ code: CodeEvalDispatcherErrorCodes.LAMBDA_CONCURRENCY_LIMIT,+ expected: "platform_error",+ },+ {+ code: CodeEvalDispatcherErrorCodes.LAMBDA_CONFIGURATION_ERROR,+ expected: "platform_error",+ },+ ])("classifies code terminal errors as $expected", ({ code, expected }) => {+ expect(+ getCodeEvalTerminalErrorOutcome(+ new CodeEvalExecutionError({+ code,+ message: code,+ retryable: false,+ }),+ ),+ ).toBe(expected);+ });+});diff --git a/worker/src/features/evaluation/evalExecutionMetrics.ts b/worker/src/features/evaluation/evalExecutionMetrics.tsnew file mode 100644index 0000000000..0332258473--- /dev/null+++ b/worker/src/features/evaluation/evalExecutionMetrics.ts@@ -0,0 +1,103 @@+import { EvalTemplateType } from "@langfuse/shared";+import {+ CodeEvalDispatcherErrorCodes,+ CodeEvalExecutionError,+ recordDistribution,+ recordIncrement,+ type EvaluatorLlmErrorClassification,+} from "@langfuse/shared/src/server";++export type ObservationEvalExecutionType =+ | typeof EvalTemplateType.LLM_AS_JUDGE+ | typeof EvalTemplateType.CODE;++export type EvalExecutionTerminalOutcome =+ | "success"+ | "platform_error"+ | "upstream_error"+ | "customer_error"+ | "cancelled";++const CODE_EVAL_CUSTOMER_ERROR_CODES: ReadonlySet<string> = new Set([+ CodeEvalDispatcherErrorCodes.INVALID_RESULT,+ CodeEvalDispatcherErrorCodes.INVALID_SOURCE,+ CodeEvalDispatcherErrorCodes.PAYLOAD_TOO_LARGE,+ CodeEvalDispatcherErrorCodes.RESULT_TOO_LARGE,+ CodeEvalDispatcherErrorCodes.SOURCE_TOO_LARGE,+ CodeEvalDispatcherErrorCodes.TIMEOUT,+ CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,+]);++export function recordEvalTimeToFirstAttempt(+ executionType: ObservationEvalExecutionType,+ durationMs: number,+): void {+ recordDistribution(+ "langfuse.evaluation.execution.time_to_first_attempt_ms",+ durationMs,+ {+ evaluator_type: getEvaluatorTypeTag(executionType),+ unit: "milliseconds",+ },+ );+}++export function recordEvalTerminalOutcome(+ executionType: ObservationEvalExecutionType,+ outcome: EvalExecutionTerminalOutcome,+): void {+ recordIncrement("langfuse.evaluation.execution.terminal", 1, {+ evaluator_type: getEvaluatorTypeTag(executionType),+ outcome,+ });+}++export function getLlmEvalTerminalErrorOutcome(+ classification: EvaluatorLlmErrorClassification | null,+): Extract<+ EvalExecutionTerminalOutcome,+ "platform_error" | "upstream_error" | "customer_error"+> {+ if (!classification) return "platform_error";++ if (+ classification.blockReason !== null ||+ classification.kind === "evaluator-policy" ||+ classification.kind === "validation"+ ) {+ return "customer_error";+ }++ if (classification.kind === "provider") {+ const statusCode = classification.statusCode;+ return statusCode !== undefined &&+ statusCode >= 400 &&+ statusCode < 500 &&+ statusCode !== 408 &&+ statusCode !== 429+ ? "customer_error"+ : "upstream_error";+ }++ if (classification.kind === "timeout") return "upstream_error";++ return "platform_error";+}++export function getCodeEvalTerminalErrorOutcome(+ error: unknown,+): Extract<EvalExecutionTerminalOutcome, "platform_error" | "customer_error"> {+ if (!(error instanceof CodeEvalExecutionError)) return "platform_error";++ return CODE_EVAL_CUSTOMER_ERROR_CODES.has(error.code)+ ? "customer_error"+ : "platform_error";+}++function getEvaluatorTypeTag(+ executionType: ObservationEvalExecutionType,+): "llm_as_judge" | "code_as_judge" {+ return executionType === EvalTemplateType.LLM_AS_JUDGE+ ? "llm_as_judge"+ : "code_as_judge";+}diff --git a/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts b/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.tsindex 18a56bf29d..cf9152925e 100644--- a/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts+++ b/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts@@ -183,6 +183,31 @@ describe("processObservationEval", () => {);});+ it.each([+ JobExecutionStatus.CANCELLED,+ JobExecutionStatus.ERROR,+ JobExecutionStatus.COMPLETED,+ ])("skips an already terminal %s execution", async (status) => {+ (prisma.jobExecution.findFirst as Mock).mockResolvedValue(+ createMockJobExecution({+ id: jobExecutionId,+ projectId,+ status,+ }),+ );++ const outcome = await processObservationEval({+ event: baseEvent,+ executionType: EvalTemplateType.LLM_AS_JUDGE,+ deps: createMockProcessorDeps(),+ });++ expect(outcome).toBe("skipped");+ expect(runLLMAsJudgeEvaluation).not.toHaveBeenCalled();+ expect(executeCodeBasedEvaluation).not.toHaveBeenCalled();+ expect(prisma.jobExecution.update).not.toHaveBeenCalled();+ });+describe("v2 execution resolution", () => {const rule = {id: "rule-123",diff --git a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts b/worker/src/features/evaluation/observationEval/observationEvalProcessor.tsindex da66b422fc..a3869f29ae 100644--- a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts+++ b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts@@ -71,10 +71,15 @@ function createObservationEvalProcessorDeps(): ObservationEvalProcessorDeps {* 4. Executes the evaluator-specific implementation* 5. Completes the eval execution with shared score persistence*/-type ObservationEvalExecutionType =+export type ObservationEvalExecutionType =| typeof EvalTemplateType.LLM_AS_JUDGE| typeof EvalTemplateType.CODE;+export type ObservationEvalProcessorOutcome =+ | "completed"+ | "cancelled"+ | "skipped";+type ProcessObservationEvalParams = {event: z.infer<typeof ObservationEvalExecutionEventSchema>;executionType: ObservationEvalExecutionType;@@ -83,7 +88,7 @@ type ProcessObservationEvalParams = {export async function processObservationEval(params: ProcessObservationEvalParams,-): Promise<void> {+): Promise<ObservationEvalProcessorOutcome> {const { event, deps = createObservationEvalProcessorDeps() } = params;logger.debug(`Processing observation eval job ${event.jobExecutionId} for project ${event.projectId}`,@@ -102,18 +107,19 @@ export async function processObservationEval(`Job execution ${event.jobExecutionId} not found. It may have been deleted.`,);- return;+ return "skipped";}- // Observation eval executions may already be CANCELLED if the evaluator was- // blocked after scheduling, or ERROR if a previous attempt already failed and- // the processor retried the same queue job.- if (job.status === "CANCELLED" || job.status === "ERROR") {- logger.debug(- `Job execution ${event.jobExecutionId} was cancelled or has an error.`,- );+ // Observation eval executions may already be terminal if the evaluator was+ // blocked after scheduling or the queue job was redelivered after processing.+ if (+ job.status === "CANCELLED" ||+ job.status === "ERROR" ||+ job.status === "COMPLETED"+ ) {+ logger.debug(`Job execution ${event.jobExecutionId} is already terminal.`);- return;+ return "skipped";}const resolved = await resolveObservationEvalExecution({@@ -123,7 +129,7 @@ export async function processObservationEval(});if (resolved.type === "cancelled") {await cancelJobExecution(job, event.projectId, resolved.reason);- return;+ return "cancelled";}const { config: evalJobConfig, template } = resolved;@@ -143,7 +149,7 @@ export async function processObservationEval(},});- return;+ return "cancelled";}// Download observation data from S3@@ -201,7 +207,7 @@ export async function processObservationEval(data: { status: JobExecutionStatus.CANCELLED, endTime: new Date() },});- return;+ return "cancelled";}// Extract variables from observation@@ -279,6 +285,8 @@ export async function processObservationEval(deps: executionParams.deps,result: executionResult,});++ return "completed";}async function cancelJobExecution(diff --git a/worker/src/queues/__tests__/codeEvalExecutionQueueProcessor.test.ts b/worker/src/queues/__tests__/codeEvalExecutionQueueProcessor.test.tsindex f34a93eec2..6c59fef07c 100644--- a/worker/src/queues/__tests__/codeEvalExecutionQueueProcessor.test.ts+++ b/worker/src/queues/__tests__/codeEvalExecutionQueueProcessor.test.ts@@ -54,6 +54,8 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {QueueName: {CodeEvalExecution: "code-eval-execution-queue",},+ recordDistribution: vi.fn(),+ recordIncrement: vi.fn(),traceException: vi.fn(),};});@@ -72,7 +74,11 @@ vi.mock("../../errors/UnrecoverableError", async () => {import { prisma } from "@langfuse/shared/src/db";import { processObservationEval } from "../../features/evaluation/observationEval";-import { traceException } from "@langfuse/shared/src/server";+import {+ recordDistribution,+ recordIncrement,+ traceException,+} from "@langfuse/shared/src/server";import { CodeEvalDispatcherErrorCodes } from "../../../../packages/shared/src/server/evals/codeEvalDispatcherTypes";import { CodeEvalExecutionError } from "../../../../packages/shared/src/server/evals/codeEvalExecution";import { isUnrecoverableError } from "../../errors/UnrecoverableError";@@ -88,7 +94,9 @@ describe("codeEvalExecutionQueueProcessor", () => {overrides: {data?: Record<string, unknown>;attemptsMade?: number;+ attemptsStarted?: number;opts?: { attempts?: number };+ timestamp?: number;} = {},): Job<any> =>({@@ -104,7 +112,9 @@ describe("codeEvalExecutionQueueProcessor", () => {retryBaggage: { attempt: 0 },...overrides.data,},+ timestamp: overrides.timestamp ?? Date.now(),attemptsMade: overrides.attemptsMade ?? 0,+ attemptsStarted: overrides.attemptsStarted ?? 1,opts: overrides.opts ?? { attempts: 10 },}) as Job<any>;@@ -118,11 +128,10 @@ describe("codeEvalExecutionQueueProcessor", () => {beforeEach(() => {vi.clearAllMocks();(isUnrecoverableError as unknown as Mock).mockReturnValue(false);+ (processObservationEval as Mock).mockResolvedValue("completed");});it("should process code observation evals through the code template gate", async () => {- (processObservationEval as Mock).mockResolvedValue(undefined);-const result = await codeEvalExecutionQueueProcessor(createMockJob());expect(result).toBe(true);@@ -136,6 +145,67 @@ describe("codeEvalExecutionQueueProcessor", () => {});});+ it("records schedule-to-first-attempt latency only for the initial attempt", async () => {+ const now = vi.spyOn(Date, "now").mockReturnValue(10_000);++ await codeEvalExecutionQueueProcessor(createMockJob({ timestamp: 8_500 }));++ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.time_to_first_attempt_ms",+ 1_500,+ {+ evaluator_type: "code_as_judge",+ unit: "milliseconds",+ },+ );++ vi.mocked(recordDistribution).mockClear();+ await codeEvalExecutionQueueProcessor(+ createMockJob({ attemptsMade: 0, attemptsStarted: 2, timestamp: 8_000 }),+ );+ expect(recordDistribution).not.toHaveBeenCalled();++ now.mockRestore();+ });++ it("records completed and cancelled terminal outcomes", async () => {+ await codeEvalExecutionQueueProcessor(createMockJob());+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ 1,+ { evaluator_type: "code_as_judge", outcome: "success" },+ );++ vi.mocked(recordIncrement).mockClear();+ (processObservationEval as Mock).mockResolvedValue("cancelled");+ await codeEvalExecutionQueueProcessor(createMockJob());+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ 1,+ { evaluator_type: "code_as_judge", outcome: "cancelled" },+ );+ });++ it("classifies non-retryable user-code failures as customer errors", async () => {+ const error = new CodeEvalExecutionError({+ code: CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,+ message: "user code failed",+ retryable: false,+ });+ (processObservationEval as Mock).mockRejectedValue(error);++ await expect(+ codeEvalExecutionQueueProcessor(createMockJob()),+ ).resolves.toBeUndefined();++ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ 1,+ { evaluator_type: "code_as_judge", outcome: "customer_error" },+ );+ expect(traceException).not.toHaveBeenCalled();+ });+it("should treat unrecoverable code eval errors as terminal", async () => {const error = new UnrecoverableError("The evaluator returned an invalid result.",@@ -156,6 +226,11 @@ describe("codeEvalExecutionQueueProcessor", () => {},});expect(traceException).not.toHaveBeenCalled();+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ 1,+ { evaluator_type: "code_as_judge", outcome: "platform_error" },+ );});it("should persist already-masked internal code eval errors", async () => {@@ -212,6 +287,11 @@ describe("codeEvalExecutionQueueProcessor", () => {expect(prisma.jobExecution.update).not.toHaveBeenCalled();expect(traceException).toHaveBeenCalledWith(error);+ expect(recordIncrement).not.toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ expect.anything(),+ expect.anything(),+ );});it("should mark the job as ERROR with a masked message on the final retry attempt", async () => {@@ -237,6 +317,11 @@ describe("codeEvalExecutionQueueProcessor", () => {},});expect(traceException).toHaveBeenCalledWith(error);+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ 1,+ { evaluator_type: "code_as_judge", outcome: "platform_error" },+ );});it("should preserve retryable code eval timeout messages on the final retry attempt", async () => {@@ -268,5 +353,10 @@ describe("codeEvalExecutionQueueProcessor", () => {},});expect(traceException).toHaveBeenCalledWith(error);+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ 1,+ { evaluator_type: "code_as_judge", outcome: "customer_error" },+ );});});diff --git a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.tsindex 35efd76fad..154ea6a84c 100644--- a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts+++ b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts@@ -51,12 +51,18 @@ vi.mock("../../features/evaluation/observationEval", () => ({}));// Mock logger and span-vi.mock("@langfuse/shared/src/server", () => {+vi.mock("@langfuse/shared/src/server", async () => {const getQueueInstance = vi.fn().mockReturnValue({add: vi.fn(),});+ const { CodeEvalExecutionError } =+ await import("../../../../packages/shared/src/server/evals/codeEvalExecution");+ const { CodeEvalDispatcherErrorCodes } =+ await import("../../../../packages/shared/src/server/evals/codeEvalDispatcherTypes");return {+ CodeEvalDispatcherErrorCodes,+ CodeEvalExecutionError,QueueName: {LLMAsJudgeExecution: "llm-as-a-judge-execution-queue",EvaluationExecutionSecondaryQueue: "evaluation-execution-secondary-queue",@@ -85,6 +91,8 @@ vi.mock("@langfuse/shared/src/server", () => {getInstance: getQueueInstance,},classifyEvaluatorLlmError: vi.fn(),+ recordDistribution: vi.fn(),+ recordIncrement: vi.fn(),};});@@ -108,6 +116,8 @@ import { processObservationEval } from "../../features/evaluation/observationEvaimport {LLMAsJudgeExecutionQueue,classifyEvaluatorLlmError,+ recordDistribution,+ recordIncrement,traceException,} from "@langfuse/shared/src/server";import { retryLLMRateLimitError } from "../../features/utils";@@ -143,7 +153,9 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {overrides: {data?: Record<string, unknown>;attemptsMade?: number;+ attemptsStarted?: number;opts?: { attempts?: number };+ timestamp?: number;} = {},): Job<any> => {return {@@ -159,7 +171,9 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {retryBaggage: { attempt: 0 },...overrides.data,},+ timestamp: overrides.timestamp ?? Date.now(),attemptsMade: overrides.attemptsMade ?? 0,+ attemptsStarted: overrides.attemptsStarted ?? 1,opts: overrides.opts ?? { attempts: 10 },} as Job<any>;};@@ -175,12 +189,11 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {vi.clearAllMocks();vi.mocked(classifyEvaluatorLlmError).mockReturnValue(null);vi.mocked(isUnrecoverableError).mockReturnValue(false);+ (processObservationEval as Mock).mockResolvedValue("completed");});describe("successful processing", () => {it("should process observation eval successfully and return true", async () => {- (processObservationEval as Mock).mockResolvedValue(undefined);-const job = createMockJob();const result = await llmAsJudgeExecutionQueueProcessor(job);@@ -199,8 +212,6 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {const mockSpan = { setAttribute: vi.fn() };const { getCurrentSpan } = await import("@langfuse/shared/src/server");(getCurrentSpan as Mock).mockReturnValue(mockSpan);- (processObservationEval as Mock).mockResolvedValue(undefined);-const job = createMockJob();await llmAsJudgeExecutionQueueProcessor(job);@@ -213,6 +224,58 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {projectId,);});++ it("records schedule-to-first-attempt latency only for the logical first attempt", async () => {+ const now = vi.spyOn(Date, "now").mockReturnValue(10_000);++ await llmAsJudgeExecutionQueueProcessor(+ createMockJob({ timestamp: 8_500 }),+ );+ expect(recordDistribution).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.time_to_first_attempt_ms",+ 1_500,+ {+ evaluator_type: "llm_as_judge",+ unit: "milliseconds",+ },+ );++ vi.mocked(recordDistribution).mockClear();+ await llmAsJudgeExecutionQueueProcessor(+ createMockJob({+ attemptsMade: 0,+ attemptsStarted: 2,+ timestamp: 8_000,+ }),+ );+ await llmAsJudgeExecutionQueueProcessor(+ createMockJob({+ data: { retryBaggage: { attempt: 1 } },+ timestamp: 8_000,+ }),+ );+ expect(recordDistribution).not.toHaveBeenCalled();++ now.mockRestore();+ });++ it("records completed and cancelled terminal outcomes", async () => {+ await llmAsJudgeExecutionQueueProcessor(createMockJob());+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ 1,+ { evaluator_type: "llm_as_judge", outcome: "success" },+ );++ vi.mocked(recordIncrement).mockClear();+ (processObservationEval as Mock).mockResolvedValue("cancelled");+ await llmAsJudgeExecutionQueueProcessor(createMockJob());+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ 1,+ { evaluator_type: "llm_as_judge", outcome: "cancelled" },+ );+ });});describe("LLM rate limit errors (retryable)", () => {@@ -249,6 +312,11 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {executionTraceId: "test-trace-id",}),});+ expect(recordIncrement).not.toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ expect.anything(),+ expect.anything(),+ );});it("should not rethrow error after scheduling retry", async () => {@@ -291,6 +359,11 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {executionTraceId: "test-trace-id",}),});+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ 1,+ { evaluator_type: "llm_as_judge", outcome: "upstream_error" },+ );});it("should set ERROR when the retry queue is unavailable", async () => {@@ -419,6 +492,11 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {);expect(prisma.jobExecution.update).not.toHaveBeenCalled();+ expect(recordIncrement).not.toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ expect.anything(),+ expect.anything(),+ );});it("should call traceException for unexpected errors", async () => {@@ -469,6 +547,11 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {executionTraceId: "test-trace-id",},});+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ 1,+ { evaluator_type: "llm_as_judge", outcome: "platform_error" },+ );});});@@ -492,8 +575,6 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {const mockSpan = { setAttribute: vi.fn() };const { getCurrentSpan } = await import("@langfuse/shared/src/server");(getCurrentSpan as Mock).mockReturnValue(mockSpan);- (processObservationEval as Mock).mockResolvedValue(undefined);-const job = createMockJob({data: { retryBaggage: { attempt: 3 } },});@@ -509,8 +590,6 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {const mockSpan = { setAttribute: vi.fn() };const { getCurrentSpan } = await import("@langfuse/shared/src/server");(getCurrentSpan as Mock).mockReturnValue(mockSpan);- (processObservationEval as Mock).mockResolvedValue(undefined);-const job = createMockJob();delete (job.data as { retryBaggage?: unknown }).retryBaggage;@@ -527,8 +606,6 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {it("should handle null span gracefully", async () => {const { getCurrentSpan } = await import("@langfuse/shared/src/server");(getCurrentSpan as Mock).mockReturnValue(null);- (processObservationEval as Mock).mockResolvedValue(undefined);-const job = createMockJob();// Should not throwdiff --git a/worker/src/queues/codeEvalQueue.ts b/worker/src/queues/codeEvalQueue.tsindex 167c97d1f5..75bb85d31a 100644--- a/worker/src/queues/codeEvalQueue.ts+++ b/worker/src/queues/codeEvalQueue.ts@@ -13,11 +13,23 @@ import {import { processObservationEval } from "../features/evaluation/observationEval";import { createW3CTraceId } from "../features/utils";import { isUnrecoverableError } from "../errors/UnrecoverableError";+import {+ getCodeEvalTerminalErrorOutcome,+ recordEvalTerminalOutcome,+ recordEvalTimeToFirstAttempt,+} from "../features/evaluation/evalExecutionMetrics";export const codeEvalExecutionQueueProcessorBuilder = (_queueName: string,): Processor => {return async (job: Job<TQueueJobTypes[QueueName.CodeEvalExecution]>) => {+ if (job.attemptsStarted === 1) {+ recordEvalTimeToFirstAttempt(+ EvalTemplateType.CODE,+ Math.max(0, Date.now() - job.timestamp),+ );+ }+try {logger.debug("Executing Code Evaluation Observation Job", job.data);@@ -34,18 +46,26 @@ export const codeEvalExecutionQueueProcessorBuilder = ();}- await processObservationEval({+ const outcome = await processObservationEval({event: job.data.payload,executionType: EvalTemplateType.CODE,});+ if (outcome === "completed") {+ recordEvalTerminalOutcome(EvalTemplateType.CODE, "success");+ } else if (outcome === "cancelled") {+ recordEvalTerminalOutcome(EvalTemplateType.CODE, "cancelled");+ }+return true;} catch (e) {const executionTraceId = createW3CTraceId(job.data.payload.jobExecutionId,);- const isTerminalError = isUnrecoverableError(e);+ const isTerminalError =+ isUnrecoverableError(e) ||+ (e instanceof CodeEvalExecutionError && !e.retryable);const totalAttempts = job.opts.attempts ?? 1;const isFinalAttempt = job.attemptsMade + 1 >= totalAttempts;@@ -65,6 +85,10 @@ export const codeEvalExecutionQueueProcessorBuilder = (executionTraceId,},});+ recordEvalTerminalOutcome(+ EvalTemplateType.CODE,+ getCodeEvalTerminalErrorOutcome(e),+ );}if (isTerminalError) return;diff --git a/worker/src/queues/evalQueue.ts b/worker/src/queues/evalQueue.tsindex 9cde4faf9c..24e6097ac7 100644--- a/worker/src/queues/evalQueue.ts+++ b/worker/src/queues/evalQueue.ts@@ -20,6 +20,11 @@ import { isUnrecoverableError } from "../errors/UnrecoverableError";import { retryObservationNotFound } from "../features/evaluation/retryObservationNotFound";import { isObservationNotFoundError } from "../errors/ObservationNotFoundError";import { env } from "../env";+import {+ getLlmEvalTerminalErrorOutcome,+ recordEvalTerminalOutcome,+ recordEvalTimeToFirstAttempt,+} from "../features/evaluation/evalExecutionMetrics";export const evalJobTraceCreatorQueueProcessor = async (job: Job<TQueueJobTypes[QueueName.TraceUpsert]>,@@ -273,6 +278,14 @@ export const evalJobExecutorQueueProcessorBuilder = (export const llmAsJudgeExecutionQueueProcessorBuilder =(queueName: string): Processor =>async (job: Job<TQueueJobTypes[QueueName.LLMAsJudgeExecution]>) => {+ const retryAttempt = job.data.retryBaggage?.attempt ?? 0;+ if (job.attemptsStarted === 1 && retryAttempt === 0) {+ recordEvalTimeToFirstAttempt(+ EvalTemplateType.LLM_AS_JUDGE,+ Math.max(0, Date.now() - job.timestamp),+ );+ }+try {logger.debug("Executing LLM-as-Judge Observation Evaluation Job",@@ -292,14 +305,21 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =);span.setAttribute("messaging.bullmq.job.input.retryBaggage.attempt",- job.data.retryBaggage?.attempt ?? 0,+ retryAttempt,);}- await processObservationEval({+ const outcome = await processObservationEval({event: job.data.payload,executionType: EvalTemplateType.LLM_AS_JUDGE,});++ if (outcome === "completed") {+ recordEvalTerminalOutcome(EvalTemplateType.LLM_AS_JUDGE, "success");+ } else if (outcome === "cancelled") {+ recordEvalTerminalOutcome(EvalTemplateType.LLM_AS_JUDGE, "cancelled");+ }+return true;} catch (e) {const llmError = classifyEvaluatorLlmError(e);@@ -357,6 +377,10 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =executionTraceId,},});+ recordEvalTerminalOutcome(+ EvalTemplateType.LLM_AS_JUDGE,+ getLlmEvalTerminalErrorOutcome(llmError),+ );}if (isTerminalError) return;