Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(evals): add evaluator execution SLO metrics (#16589)

Hassieb Pakzadadd3eee1b60a
diff --git a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts
index 01ae0c870f..163b6723dc 100644
--- a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts
+++ b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.test.ts
@@ -9,7 +9,6 @@ import {
parseDispatchResult,
} from "../../../../../packages/shared/src/server/evals/codeEvalDispatcherTypes";
import { CodeEvalExecutionError } from "../../../../../packages/shared/src/server/evals/codeEvalExecution";
-import { UnrecoverableError } from "../../../errors/UnrecoverableError";
const mocks = vi.hoisted(() => ({
dispatcher: {
@@ -490,7 +489,11 @@ describe("executeCodeBasedEvaluation", () => {
executionMetadata: { job_execution_id: "job-1" },
});
- await expect(promise).rejects.toThrow(UnrecoverableError);
+ await expect(promise).rejects.toBeInstanceOf(CodeEvalExecutionError);
+ await expect(promise).rejects.toMatchObject({
+ code: CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,
+ retryable: false,
+ });
await expect(promise).rejects.toThrow("runner exploded");
expect(mocks.writeInternalTrace).toHaveBeenCalledTimes(1);
@@ -560,7 +563,11 @@ describe("executeCodeBasedEvaluation", () => {
executionMetadata: { job_execution_id: "job-1" },
});
- await expect(promise).rejects.toThrow(UnrecoverableError);
+ await expect(promise).rejects.toBeInstanceOf(CodeEvalExecutionError);
+ await expect(promise).rejects.toMatchObject({
+ code: CodeEvalDispatcherErrorCodes.INVALID_RESULT,
+ retryable: false,
+ });
await expect(promise).rejects.toThrow(
"The evaluator returned an invalid result.",
);
@@ -669,7 +676,11 @@ describe("executeCodeBasedEvaluation", () => {
executionMetadata: { job_execution_id: "job-1" },
});
- await expect(promise).rejects.toThrow(UnrecoverableError);
+ await expect(promise).rejects.toBeInstanceOf(CodeEvalExecutionError);
+ await expect(promise).rejects.toMatchObject({
+ code: "INTERNAL_ERROR",
+ retryable: false,
+ });
await expect(promise).rejects.toThrow("An internal error occurred");
expect(mocks.writeInternalTrace).toHaveBeenCalledTimes(1);
diff --git a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.ts b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.ts
index 06c73440bf..d818e8e164 100644
--- a/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.ts
+++ b/worker/src/features/evaluation/codeBased/executeCodeBasedEvaluation.ts
@@ -77,10 +77,6 @@ export async function executeCodeBasedEvaluation(params: {
});
if (!dispatchOutcome.success) {
- if (dispatchOutcome.error.retryable === false) {
- throw new UnrecoverableError(dispatchOutcome.error.message);
- }
-
throw new CodeEvalExecutionError(dispatchOutcome.error);
}
diff --git a/worker/src/features/evaluation/evalExecutionMetrics.test.ts b/worker/src/features/evaluation/evalExecutionMetrics.test.ts
new file mode 100644
index 0000000000..3ca04f3083
--- /dev/null
+++ b/worker/src/features/evaluation/evalExecutionMetrics.test.ts
@@ -0,0 +1,129 @@
+import { EvalTemplateType } from "@langfuse/shared";
+import {
+ CodeEvalDispatcherErrorCodes,
+ CodeEvalExecutionError,
+ recordDistribution,
+ recordIncrement,
+ type EvaluatorLlmErrorClassification,
+} from "@langfuse/shared/src/server";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ getCodeEvalTerminalErrorOutcome,
+ getLlmEvalTerminalErrorOutcome,
+ recordEvalTerminalOutcome,
+ recordEvalTimeToFirstAttempt,
+} from "./evalExecutionMetrics";
+
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
+ recordDistribution: vi.fn(),
+ recordIncrement: vi.fn(),
+}));
+
+describe("evaluation execution metrics", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it("records the bounded metric contract", () => {
+ recordEvalTimeToFirstAttempt(EvalTemplateType.LLM_AS_JUDGE, 1_234);
+ recordEvalTerminalOutcome(EvalTemplateType.CODE, "success");
+
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.time_to_first_attempt_ms",
+ 1_234,
+ {
+ evaluator_type: "llm_as_judge",
+ unit: "milliseconds",
+ },
+ );
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ 1,
+ { evaluator_type: "code_as_judge", outcome: "success" },
+ );
+ });
+
+ it.each([
+ {
+ classification: null,
+ expected: "platform_error",
+ },
+ {
+ classification: {
+ kind: "validation",
+ message: "invalid connection",
+ isRetryable: false,
+ error: new Error("invalid connection"),
+ blockReason: null,
+ } satisfies EvaluatorLlmErrorClassification,
+ expected: "customer_error",
+ },
+ {
+ classification: {
+ kind: "provider",
+ message: "invalid request",
+ statusCode: 400,
+ isRetryable: false,
+ error: new Error("invalid request"),
+ blockReason: null,
+ } satisfies EvaluatorLlmErrorClassification,
+ expected: "customer_error",
+ },
+ {
+ classification: {
+ kind: "provider",
+ message: "rate limited",
+ statusCode: 429,
+ isRetryable: true,
+ error: new Error("rate limited"),
+ blockReason: null,
+ } satisfies EvaluatorLlmErrorClassification,
+ expected: "upstream_error",
+ },
+ {
+ classification: {
+ kind: "timeout",
+ message: "provider timed out",
+ isRetryable: false,
+ error: new Error("provider timed out"),
+ blockReason: null,
+ } satisfies EvaluatorLlmErrorClassification,
+ expected: "upstream_error",
+ },
+ ])(
+ "classifies LLM terminal errors as $expected",
+ ({ classification, expected }) => {
+ expect(getLlmEvalTerminalErrorOutcome(classification)).toBe(expected);
+ },
+ );
+
+ it.each([
+ {
+ code: CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,
+ expected: "customer_error",
+ },
+ {
+ code: CodeEvalDispatcherErrorCodes.TIMEOUT,
+ expected: "customer_error",
+ },
+ {
+ code: CodeEvalDispatcherErrorCodes.LAMBDA_CONCURRENCY_LIMIT,
+ expected: "platform_error",
+ },
+ {
+ code: CodeEvalDispatcherErrorCodes.LAMBDA_CONFIGURATION_ERROR,
+ expected: "platform_error",
+ },
+ ])("classifies code terminal errors as $expected", ({ code, expected }) => {
+ expect(
+ getCodeEvalTerminalErrorOutcome(
+ new CodeEvalExecutionError({
+ code,
+ message: code,
+ retryable: false,
+ }),
+ ),
+ ).toBe(expected);
+ });
+});
diff --git a/worker/src/features/evaluation/evalExecutionMetrics.ts b/worker/src/features/evaluation/evalExecutionMetrics.ts
new file mode 100644
index 0000000000..0332258473
--- /dev/null
+++ b/worker/src/features/evaluation/evalExecutionMetrics.ts
@@ -0,0 +1,103 @@
+import { EvalTemplateType } from "@langfuse/shared";
+import {
+ CodeEvalDispatcherErrorCodes,
+ CodeEvalExecutionError,
+ recordDistribution,
+ recordIncrement,
+ type EvaluatorLlmErrorClassification,
+} from "@langfuse/shared/src/server";
+
+export type ObservationEvalExecutionType =
+ | typeof EvalTemplateType.LLM_AS_JUDGE
+ | typeof EvalTemplateType.CODE;
+
+export type EvalExecutionTerminalOutcome =
+ | "success"
+ | "platform_error"
+ | "upstream_error"
+ | "customer_error"
+ | "cancelled";
+
+const CODE_EVAL_CUSTOMER_ERROR_CODES: ReadonlySet<string> = new Set([
+ CodeEvalDispatcherErrorCodes.INVALID_RESULT,
+ CodeEvalDispatcherErrorCodes.INVALID_SOURCE,
+ CodeEvalDispatcherErrorCodes.PAYLOAD_TOO_LARGE,
+ CodeEvalDispatcherErrorCodes.RESULT_TOO_LARGE,
+ CodeEvalDispatcherErrorCodes.SOURCE_TOO_LARGE,
+ CodeEvalDispatcherErrorCodes.TIMEOUT,
+ CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,
+]);
+
+export function recordEvalTimeToFirstAttempt(
+ executionType: ObservationEvalExecutionType,
+ durationMs: number,
+): void {
+ recordDistribution(
+ "langfuse.evaluation.execution.time_to_first_attempt_ms",
+ durationMs,
+ {
+ evaluator_type: getEvaluatorTypeTag(executionType),
+ unit: "milliseconds",
+ },
+ );
+}
+
+export function recordEvalTerminalOutcome(
+ executionType: ObservationEvalExecutionType,
+ outcome: EvalExecutionTerminalOutcome,
+): void {
+ recordIncrement("langfuse.evaluation.execution.terminal", 1, {
+ evaluator_type: getEvaluatorTypeTag(executionType),
+ outcome,
+ });
+}
+
+export function getLlmEvalTerminalErrorOutcome(
+ classification: EvaluatorLlmErrorClassification | null,
+): Extract<
+ EvalExecutionTerminalOutcome,
+ "platform_error" | "upstream_error" | "customer_error"
+> {
+ if (!classification) return "platform_error";
+
+ if (
+ classification.blockReason !== null ||
+ classification.kind === "evaluator-policy" ||
+ classification.kind === "validation"
+ ) {
+ return "customer_error";
+ }
+
+ if (classification.kind === "provider") {
+ const statusCode = classification.statusCode;
+ return statusCode !== undefined &&
+ statusCode >= 400 &&
+ statusCode < 500 &&
+ statusCode !== 408 &&
+ statusCode !== 429
+ ? "customer_error"
+ : "upstream_error";
+ }
+
+ if (classification.kind === "timeout") return "upstream_error";
+
+ return "platform_error";
+}
+
+export function getCodeEvalTerminalErrorOutcome(
+ error: unknown,
+): Extract<EvalExecutionTerminalOutcome, "platform_error" | "customer_error"> {
+ if (!(error instanceof CodeEvalExecutionError)) return "platform_error";
+
+ return CODE_EVAL_CUSTOMER_ERROR_CODES.has(error.code)
+ ? "customer_error"
+ : "platform_error";
+}
+
+function getEvaluatorTypeTag(
+ executionType: ObservationEvalExecutionType,
+): "llm_as_judge" | "code_as_judge" {
+ return executionType === EvalTemplateType.LLM_AS_JUDGE
+ ? "llm_as_judge"
+ : "code_as_judge";
+}
diff --git a/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts b/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts
index 18a56bf29d..cf9152925e 100644
--- a/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts
+++ b/worker/src/features/evaluation/observationEval/__tests__/observationEvalProcessor.test.ts
@@ -183,6 +183,31 @@ describe("processObservationEval", () => {
);
});
+ it.each([
+ JobExecutionStatus.CANCELLED,
+ JobExecutionStatus.ERROR,
+ JobExecutionStatus.COMPLETED,
+ ])("skips an already terminal %s execution", async (status) => {
+ (prisma.jobExecution.findFirst as Mock).mockResolvedValue(
+ createMockJobExecution({
+ id: jobExecutionId,
+ projectId,
+ status,
+ }),
+ );
+
+ const outcome = await processObservationEval({
+ event: baseEvent,
+ executionType: EvalTemplateType.LLM_AS_JUDGE,
+ deps: createMockProcessorDeps(),
+ });
+
+ expect(outcome).toBe("skipped");
+ expect(runLLMAsJudgeEvaluation).not.toHaveBeenCalled();
+ expect(executeCodeBasedEvaluation).not.toHaveBeenCalled();
+ expect(prisma.jobExecution.update).not.toHaveBeenCalled();
+ });
+
describe("v2 execution resolution", () => {
const rule = {
id: "rule-123",
diff --git a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
index da66b422fc..a3869f29ae 100644
--- a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
+++ b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
@@ -71,10 +71,15 @@ function createObservationEvalProcessorDeps(): ObservationEvalProcessorDeps {
* 4. Executes the evaluator-specific implementation
* 5. Completes the eval execution with shared score persistence
*/
-type ObservationEvalExecutionType =
+export type ObservationEvalExecutionType =
| typeof EvalTemplateType.LLM_AS_JUDGE
| typeof EvalTemplateType.CODE;
+export type ObservationEvalProcessorOutcome =
+ | "completed"
+ | "cancelled"
+ | "skipped";
+
type ProcessObservationEvalParams = {
event: z.infer<typeof ObservationEvalExecutionEventSchema>;
executionType: ObservationEvalExecutionType;
@@ -83,7 +88,7 @@ type ProcessObservationEvalParams = {
export async function processObservationEval(
params: ProcessObservationEvalParams,
-): Promise<void> {
+): Promise<ObservationEvalProcessorOutcome> {
const { event, deps = createObservationEvalProcessorDeps() } = params;
logger.debug(
`Processing observation eval job ${event.jobExecutionId} for project ${event.projectId}`,
@@ -102,18 +107,19 @@ export async function processObservationEval(
`Job execution ${event.jobExecutionId} not found. It may have been deleted.`,
);
- return;
+ return "skipped";
}
- // Observation eval executions may already be CANCELLED if the evaluator was
- // blocked after scheduling, or ERROR if a previous attempt already failed and
- // the processor retried the same queue job.
- if (job.status === "CANCELLED" || job.status === "ERROR") {
- logger.debug(
- `Job execution ${event.jobExecutionId} was cancelled or has an error.`,
- );
+ // Observation eval executions may already be terminal if the evaluator was
+ // blocked after scheduling or the queue job was redelivered after processing.
+ if (
+ job.status === "CANCELLED" ||
+ job.status === "ERROR" ||
+ job.status === "COMPLETED"
+ ) {
+ logger.debug(`Job execution ${event.jobExecutionId} is already terminal.`);
- return;
+ return "skipped";
}
const resolved = await resolveObservationEvalExecution({
@@ -123,7 +129,7 @@ export async function processObservationEval(
});
if (resolved.type === "cancelled") {
await cancelJobExecution(job, event.projectId, resolved.reason);
- return;
+ return "cancelled";
}
const { config: evalJobConfig, template } = resolved;
@@ -143,7 +149,7 @@ export async function processObservationEval(
},
});
- return;
+ return "cancelled";
}
// Download observation data from S3
@@ -201,7 +207,7 @@ export async function processObservationEval(
data: { status: JobExecutionStatus.CANCELLED, endTime: new Date() },
});
- return;
+ return "cancelled";
}
// Extract variables from observation
@@ -279,6 +285,8 @@ export async function processObservationEval(
deps: executionParams.deps,
result: executionResult,
});
+
+ return "completed";
}
async function cancelJobExecution(
diff --git a/worker/src/queues/__tests__/codeEvalExecutionQueueProcessor.test.ts b/worker/src/queues/__tests__/codeEvalExecutionQueueProcessor.test.ts
index f34a93eec2..6c59fef07c 100644
--- a/worker/src/queues/__tests__/codeEvalExecutionQueueProcessor.test.ts
+++ b/worker/src/queues/__tests__/codeEvalExecutionQueueProcessor.test.ts
@@ -54,6 +54,8 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
QueueName: {
CodeEvalExecution: "code-eval-execution-queue",
},
+ recordDistribution: vi.fn(),
+ recordIncrement: vi.fn(),
traceException: vi.fn(),
};
});
@@ -72,7 +74,11 @@ vi.mock("../../errors/UnrecoverableError", async () => {
import { prisma } from "@langfuse/shared/src/db";
import { processObservationEval } from "../../features/evaluation/observationEval";
-import { traceException } from "@langfuse/shared/src/server";
+import {
+ recordDistribution,
+ recordIncrement,
+ traceException,
+} from "@langfuse/shared/src/server";
import { CodeEvalDispatcherErrorCodes } from "../../../../packages/shared/src/server/evals/codeEvalDispatcherTypes";
import { CodeEvalExecutionError } from "../../../../packages/shared/src/server/evals/codeEvalExecution";
import { isUnrecoverableError } from "../../errors/UnrecoverableError";
@@ -88,7 +94,9 @@ describe("codeEvalExecutionQueueProcessor", () => {
overrides: {
data?: Record<string, unknown>;
attemptsMade?: number;
+ attemptsStarted?: number;
opts?: { attempts?: number };
+ timestamp?: number;
} = {},
): Job<any> =>
({
@@ -104,7 +112,9 @@ describe("codeEvalExecutionQueueProcessor", () => {
retryBaggage: { attempt: 0 },
...overrides.data,
},
+ timestamp: overrides.timestamp ?? Date.now(),
attemptsMade: overrides.attemptsMade ?? 0,
+ attemptsStarted: overrides.attemptsStarted ?? 1,
opts: overrides.opts ?? { attempts: 10 },
}) as Job<any>;
@@ -118,11 +128,10 @@ describe("codeEvalExecutionQueueProcessor", () => {
beforeEach(() => {
vi.clearAllMocks();
(isUnrecoverableError as unknown as Mock).mockReturnValue(false);
+ (processObservationEval as Mock).mockResolvedValue("completed");
});
it("should process code observation evals through the code template gate", async () => {
- (processObservationEval as Mock).mockResolvedValue(undefined);
-
const result = await codeEvalExecutionQueueProcessor(createMockJob());
expect(result).toBe(true);
@@ -136,6 +145,67 @@ describe("codeEvalExecutionQueueProcessor", () => {
});
});
+ it("records schedule-to-first-attempt latency only for the initial attempt", async () => {
+ const now = vi.spyOn(Date, "now").mockReturnValue(10_000);
+
+ await codeEvalExecutionQueueProcessor(createMockJob({ timestamp: 8_500 }));
+
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.time_to_first_attempt_ms",
+ 1_500,
+ {
+ evaluator_type: "code_as_judge",
+ unit: "milliseconds",
+ },
+ );
+
+ vi.mocked(recordDistribution).mockClear();
+ await codeEvalExecutionQueueProcessor(
+ createMockJob({ attemptsMade: 0, attemptsStarted: 2, timestamp: 8_000 }),
+ );
+ expect(recordDistribution).not.toHaveBeenCalled();
+
+ now.mockRestore();
+ });
+
+ it("records completed and cancelled terminal outcomes", async () => {
+ await codeEvalExecutionQueueProcessor(createMockJob());
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ 1,
+ { evaluator_type: "code_as_judge", outcome: "success" },
+ );
+
+ vi.mocked(recordIncrement).mockClear();
+ (processObservationEval as Mock).mockResolvedValue("cancelled");
+ await codeEvalExecutionQueueProcessor(createMockJob());
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ 1,
+ { evaluator_type: "code_as_judge", outcome: "cancelled" },
+ );
+ });
+
+ it("classifies non-retryable user-code failures as customer errors", async () => {
+ const error = new CodeEvalExecutionError({
+ code: CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,
+ message: "user code failed",
+ retryable: false,
+ });
+ (processObservationEval as Mock).mockRejectedValue(error);
+
+ await expect(
+ codeEvalExecutionQueueProcessor(createMockJob()),
+ ).resolves.toBeUndefined();
+
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ 1,
+ { evaluator_type: "code_as_judge", outcome: "customer_error" },
+ );
+ expect(traceException).not.toHaveBeenCalled();
+ });
+
it("should treat unrecoverable code eval errors as terminal", async () => {
const error = new UnrecoverableError(
"The evaluator returned an invalid result.",
@@ -156,6 +226,11 @@ describe("codeEvalExecutionQueueProcessor", () => {
},
});
expect(traceException).not.toHaveBeenCalled();
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ 1,
+ { evaluator_type: "code_as_judge", outcome: "platform_error" },
+ );
});
it("should persist already-masked internal code eval errors", async () => {
@@ -212,6 +287,11 @@ describe("codeEvalExecutionQueueProcessor", () => {
expect(prisma.jobExecution.update).not.toHaveBeenCalled();
expect(traceException).toHaveBeenCalledWith(error);
+ expect(recordIncrement).not.toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ expect.anything(),
+ expect.anything(),
+ );
});
it("should mark the job as ERROR with a masked message on the final retry attempt", async () => {
@@ -237,6 +317,11 @@ describe("codeEvalExecutionQueueProcessor", () => {
},
});
expect(traceException).toHaveBeenCalledWith(error);
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ 1,
+ { evaluator_type: "code_as_judge", outcome: "platform_error" },
+ );
});
it("should preserve retryable code eval timeout messages on the final retry attempt", async () => {
@@ -268,5 +353,10 @@ describe("codeEvalExecutionQueueProcessor", () => {
},
});
expect(traceException).toHaveBeenCalledWith(error);
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ 1,
+ { evaluator_type: "code_as_judge", outcome: "customer_error" },
+ );
});
});
diff --git a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts
index 35efd76fad..154ea6a84c 100644
--- a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts
+++ b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts
@@ -51,12 +51,18 @@ vi.mock("../../features/evaluation/observationEval", () => ({
}));
// Mock logger and span
-vi.mock("@langfuse/shared/src/server", () => {
+vi.mock("@langfuse/shared/src/server", async () => {
const getQueueInstance = vi.fn().mockReturnValue({
add: vi.fn(),
});
+ const { CodeEvalExecutionError } =
+ await import("../../../../packages/shared/src/server/evals/codeEvalExecution");
+ const { CodeEvalDispatcherErrorCodes } =
+ await import("../../../../packages/shared/src/server/evals/codeEvalDispatcherTypes");
return {
+ CodeEvalDispatcherErrorCodes,
+ CodeEvalExecutionError,
QueueName: {
LLMAsJudgeExecution: "llm-as-a-judge-execution-queue",
EvaluationExecutionSecondaryQueue: "evaluation-execution-secondary-queue",
@@ -85,6 +91,8 @@ vi.mock("@langfuse/shared/src/server", () => {
getInstance: getQueueInstance,
},
classifyEvaluatorLlmError: vi.fn(),
+ recordDistribution: vi.fn(),
+ recordIncrement: vi.fn(),
};
});
@@ -108,6 +116,8 @@ import { processObservationEval } from "../../features/evaluation/observationEva
import {
LLMAsJudgeExecutionQueue,
classifyEvaluatorLlmError,
+ recordDistribution,
+ recordIncrement,
traceException,
} from "@langfuse/shared/src/server";
import { retryLLMRateLimitError } from "../../features/utils";
@@ -143,7 +153,9 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
overrides: {
data?: Record<string, unknown>;
attemptsMade?: number;
+ attemptsStarted?: number;
opts?: { attempts?: number };
+ timestamp?: number;
} = {},
): Job<any> => {
return {
@@ -159,7 +171,9 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
retryBaggage: { attempt: 0 },
...overrides.data,
},
+ timestamp: overrides.timestamp ?? Date.now(),
attemptsMade: overrides.attemptsMade ?? 0,
+ attemptsStarted: overrides.attemptsStarted ?? 1,
opts: overrides.opts ?? { attempts: 10 },
} as Job<any>;
};
@@ -175,12 +189,11 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
vi.clearAllMocks();
vi.mocked(classifyEvaluatorLlmError).mockReturnValue(null);
vi.mocked(isUnrecoverableError).mockReturnValue(false);
+ (processObservationEval as Mock).mockResolvedValue("completed");
});
describe("successful processing", () => {
it("should process observation eval successfully and return true", async () => {
- (processObservationEval as Mock).mockResolvedValue(undefined);
-
const job = createMockJob();
const result = await llmAsJudgeExecutionQueueProcessor(job);
@@ -199,8 +212,6 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
const mockSpan = { setAttribute: vi.fn() };
const { getCurrentSpan } = await import("@langfuse/shared/src/server");
(getCurrentSpan as Mock).mockReturnValue(mockSpan);
- (processObservationEval as Mock).mockResolvedValue(undefined);
-
const job = createMockJob();
await llmAsJudgeExecutionQueueProcessor(job);
@@ -213,6 +224,58 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
projectId,
);
});
+
+ it("records schedule-to-first-attempt latency only for the logical first attempt", async () => {
+ const now = vi.spyOn(Date, "now").mockReturnValue(10_000);
+
+ await llmAsJudgeExecutionQueueProcessor(
+ createMockJob({ timestamp: 8_500 }),
+ );
+ expect(recordDistribution).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.time_to_first_attempt_ms",
+ 1_500,
+ {
+ evaluator_type: "llm_as_judge",
+ unit: "milliseconds",
+ },
+ );
+
+ vi.mocked(recordDistribution).mockClear();
+ await llmAsJudgeExecutionQueueProcessor(
+ createMockJob({
+ attemptsMade: 0,
+ attemptsStarted: 2,
+ timestamp: 8_000,
+ }),
+ );
+ await llmAsJudgeExecutionQueueProcessor(
+ createMockJob({
+ data: { retryBaggage: { attempt: 1 } },
+ timestamp: 8_000,
+ }),
+ );
+ expect(recordDistribution).not.toHaveBeenCalled();
+
+ now.mockRestore();
+ });
+
+ it("records completed and cancelled terminal outcomes", async () => {
+ await llmAsJudgeExecutionQueueProcessor(createMockJob());
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ 1,
+ { evaluator_type: "llm_as_judge", outcome: "success" },
+ );
+
+ vi.mocked(recordIncrement).mockClear();
+ (processObservationEval as Mock).mockResolvedValue("cancelled");
+ await llmAsJudgeExecutionQueueProcessor(createMockJob());
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ 1,
+ { evaluator_type: "llm_as_judge", outcome: "cancelled" },
+ );
+ });
});
describe("LLM rate limit errors (retryable)", () => {
@@ -249,6 +312,11 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
executionTraceId: "test-trace-id",
}),
});
+ expect(recordIncrement).not.toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ expect.anything(),
+ expect.anything(),
+ );
});
it("should not rethrow error after scheduling retry", async () => {
@@ -291,6 +359,11 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
executionTraceId: "test-trace-id",
}),
});
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ 1,
+ { evaluator_type: "llm_as_judge", outcome: "upstream_error" },
+ );
});
it("should set ERROR when the retry queue is unavailable", async () => {
@@ -419,6 +492,11 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
);
expect(prisma.jobExecution.update).not.toHaveBeenCalled();
+ expect(recordIncrement).not.toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ expect.anything(),
+ expect.anything(),
+ );
});
it("should call traceException for unexpected errors", async () => {
@@ -469,6 +547,11 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
executionTraceId: "test-trace-id",
},
});
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ 1,
+ { evaluator_type: "llm_as_judge", outcome: "platform_error" },
+ );
});
});
@@ -492,8 +575,6 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
const mockSpan = { setAttribute: vi.fn() };
const { getCurrentSpan } = await import("@langfuse/shared/src/server");
(getCurrentSpan as Mock).mockReturnValue(mockSpan);
- (processObservationEval as Mock).mockResolvedValue(undefined);
-
const job = createMockJob({
data: { retryBaggage: { attempt: 3 } },
});
@@ -509,8 +590,6 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
const mockSpan = { setAttribute: vi.fn() };
const { getCurrentSpan } = await import("@langfuse/shared/src/server");
(getCurrentSpan as Mock).mockReturnValue(mockSpan);
- (processObservationEval as Mock).mockResolvedValue(undefined);
-
const job = createMockJob();
delete (job.data as { retryBaggage?: unknown }).retryBaggage;
@@ -527,8 +606,6 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
it("should handle null span gracefully", async () => {
const { getCurrentSpan } = await import("@langfuse/shared/src/server");
(getCurrentSpan as Mock).mockReturnValue(null);
- (processObservationEval as Mock).mockResolvedValue(undefined);
-
const job = createMockJob();
// Should not throw
diff --git a/worker/src/queues/codeEvalQueue.ts b/worker/src/queues/codeEvalQueue.ts
index 167c97d1f5..75bb85d31a 100644
--- a/worker/src/queues/codeEvalQueue.ts
+++ b/worker/src/queues/codeEvalQueue.ts
@@ -13,11 +13,23 @@ import {
import { processObservationEval } from "../features/evaluation/observationEval";
import { createW3CTraceId } from "../features/utils";
import { isUnrecoverableError } from "../errors/UnrecoverableError";
+import {
+ getCodeEvalTerminalErrorOutcome,
+ recordEvalTerminalOutcome,
+ recordEvalTimeToFirstAttempt,
+} from "../features/evaluation/evalExecutionMetrics";
export const codeEvalExecutionQueueProcessorBuilder = (
_queueName: string,
): Processor => {
return async (job: Job<TQueueJobTypes[QueueName.CodeEvalExecution]>) => {
+ if (job.attemptsStarted === 1) {
+ recordEvalTimeToFirstAttempt(
+ EvalTemplateType.CODE,
+ Math.max(0, Date.now() - job.timestamp),
+ );
+ }
+
try {
logger.debug("Executing Code Evaluation Observation Job", job.data);
@@ -34,18 +46,26 @@ export const codeEvalExecutionQueueProcessorBuilder = (
);
}
- await processObservationEval({
+ const outcome = await processObservationEval({
event: job.data.payload,
executionType: EvalTemplateType.CODE,
});
+ if (outcome === "completed") {
+ recordEvalTerminalOutcome(EvalTemplateType.CODE, "success");
+ } else if (outcome === "cancelled") {
+ recordEvalTerminalOutcome(EvalTemplateType.CODE, "cancelled");
+ }
+
return true;
} catch (e) {
const executionTraceId = createW3CTraceId(
job.data.payload.jobExecutionId,
);
- const isTerminalError = isUnrecoverableError(e);
+ const isTerminalError =
+ isUnrecoverableError(e) ||
+ (e instanceof CodeEvalExecutionError && !e.retryable);
const totalAttempts = job.opts.attempts ?? 1;
const isFinalAttempt = job.attemptsMade + 1 >= totalAttempts;
@@ -65,6 +85,10 @@ export const codeEvalExecutionQueueProcessorBuilder = (
executionTraceId,
},
});
+ recordEvalTerminalOutcome(
+ EvalTemplateType.CODE,
+ getCodeEvalTerminalErrorOutcome(e),
+ );
}
if (isTerminalError) return;
diff --git a/worker/src/queues/evalQueue.ts b/worker/src/queues/evalQueue.ts
index 9cde4faf9c..24e6097ac7 100644
--- a/worker/src/queues/evalQueue.ts
+++ b/worker/src/queues/evalQueue.ts
@@ -20,6 +20,11 @@ import { isUnrecoverableError } from "../errors/UnrecoverableError";
import { retryObservationNotFound } from "../features/evaluation/retryObservationNotFound";
import { isObservationNotFoundError } from "../errors/ObservationNotFoundError";
import { env } from "../env";
+import {
+ getLlmEvalTerminalErrorOutcome,
+ recordEvalTerminalOutcome,
+ recordEvalTimeToFirstAttempt,
+} from "../features/evaluation/evalExecutionMetrics";
export const evalJobTraceCreatorQueueProcessor = async (
job: Job<TQueueJobTypes[QueueName.TraceUpsert]>,
@@ -273,6 +278,14 @@ export const evalJobExecutorQueueProcessorBuilder = (
export const llmAsJudgeExecutionQueueProcessorBuilder =
(queueName: string): Processor =>
async (job: Job<TQueueJobTypes[QueueName.LLMAsJudgeExecution]>) => {
+ const retryAttempt = job.data.retryBaggage?.attempt ?? 0;
+ if (job.attemptsStarted === 1 && retryAttempt === 0) {
+ recordEvalTimeToFirstAttempt(
+ EvalTemplateType.LLM_AS_JUDGE,
+ Math.max(0, Date.now() - job.timestamp),
+ );
+ }
+
try {
logger.debug(
"Executing LLM-as-Judge Observation Evaluation Job",
@@ -292,14 +305,21 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =
);
span.setAttribute(
"messaging.bullmq.job.input.retryBaggage.attempt",
- job.data.retryBaggage?.attempt ?? 0,
+ retryAttempt,
);
}
- await processObservationEval({
+ const outcome = await processObservationEval({
event: job.data.payload,
executionType: EvalTemplateType.LLM_AS_JUDGE,
});
+
+ if (outcome === "completed") {
+ recordEvalTerminalOutcome(EvalTemplateType.LLM_AS_JUDGE, "success");
+ } else if (outcome === "cancelled") {
+ recordEvalTerminalOutcome(EvalTemplateType.LLM_AS_JUDGE, "cancelled");
+ }
+
return true;
} catch (e) {
const llmError = classifyEvaluatorLlmError(e);
@@ -357,6 +377,10 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =
executionTraceId,
},
});
+ recordEvalTerminalOutcome(
+ EvalTemplateType.LLM_AS_JUDGE,
+ getLlmEvalTerminalErrorOutcome(llmError),
+ );
}
if (isTerminalError) return;