🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import {
beforeAll,
beforeEach,
describe,
expect,
it,
vi,
type Mock,
} from "vitest";
import { Job } from "bullmq";
import {
UnrecoverableError,
isUnrecoverableError,
} from "../../errors/UnrecoverableError";
import {
EvalFailureError,
EvalFailureReason,
UnrecoverableEvalFailureError,
} from "../../features/evaluation/evalFailureErrors";
const QueueName = {
LLMAsJudgeExecution: "llm-as-a-judge-execution-queue",
EvaluationExecutionSecondaryQueue: "evaluation-execution-secondary-queue",
} as const;
const QueueJobs = {
LLMAsJudgeExecution: "llm-as-a-judge-execution-job",
EvaluationExecution: "evaluation-execution-job",
} as const;
const JobExecutionStatus = {
DELAYED: "DELAYED",
ERROR: "ERROR",
} as const;
vi.mock("@langfuse/shared", () => ({
removeEmptyEnvVariables: <T>(value: T) => value,
EvalTemplateType: {
LLM_AS_JUDGE: "LLM_AS_JUDGE",
},
JobExecutionStatus: {
DELAYED: "DELAYED",
ERROR: "ERROR",
},
}));
// Mock prisma
vi.mock("@langfuse/shared/src/db", () => ({
prisma: {
jobExecution: {
update: vi.fn(),
},
},
}));
// Mock processObservationEval
vi.mock(
"../../features/evaluation/observationEval/observationEvalProcessor",
() => ({
processObservationEval: vi.fn(),
}),
);
// Mock logger and span
vi.mock("@langfuse/shared/src/server", async () => {
const getQueueInstance = vi.fn().mockReturnValue({
add: vi.fn(),
});
const { CodeEvalExecutionError } =
await import("../../../../packages/shared/src/server/evals/codeEvalExecution");
const { CodeEvalDispatcherErrorCodes } =
await import("../../../../packages/shared/src/server/evals/codeEvalDispatcherTypes");
return {
CodeEvalDispatcherErrorCodes,
CodeEvalExecutionError,
QueueName: {
LLMAsJudgeExecution: "llm-as-a-judge-execution-queue",
EvaluationExecutionSecondaryQueue: "evaluation-execution-secondary-queue",
},
QueueJobs: {
LLMAsJudgeExecution: "llm-as-a-judge-execution-job",
EvaluationExecution: "evaluation-execution-job",
},
logger: {
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
},
traceException: vi.fn(),
getCurrentSpan: vi.fn().mockReturnValue({
setAttribute: vi.fn(),
}),
EvalExecutionQueue: {
getInstance: vi.fn(),
},
SecondaryEvalExecutionQueue: {
getInstance: vi.fn(),
},
LLMAsJudgeExecutionQueue: {
getInstance: getQueueInstance,
},
classifyEvaluatorLlmError: vi.fn(),
recordDistribution: vi.fn(),
recordIncrement: vi.fn(),
};
});
// Mock retryLLMRateLimitError
vi.mock("../../features/utils/utilities", () => ({
createW3CTraceId: vi.fn().mockReturnValue("test-trace-id"),
retryLLMRateLimitError: vi.fn(),
}));
vi.mock("../../features/utils/retry-handler", () => ({
createW3CTraceId: vi.fn().mockReturnValue("test-trace-id"),
retryLLMRateLimitError: vi.fn(),
}));
// Mock isUnrecoverableError
vi.mock("../../errors/UnrecoverableError", async () => {
const actual = await vi.importActual("../../errors/UnrecoverableError");
return {
...actual,
isUnrecoverableError: vi.fn(),
};
});
import { prisma } from "@langfuse/shared/src/db";
import { processObservationEval } from "../../features/evaluation/observationEval/observationEvalProcessor";
import {
LLMAsJudgeExecutionQueue,
classifyEvaluatorLlmError,
logger,
recordDistribution,
recordIncrement,
traceException,
} from "@langfuse/shared/src/server";
import { retryLLMRateLimitError } from "../../features/utils/retry-handler";
function mockLlmError(error: Error, isRetryable: boolean): void {
vi.mocked(classifyEvaluatorLlmError).mockReturnValue({
kind: "provider",
message: error.message,
isRetryable,
error,
blockReason: null,
});
}
describe("llmAsJudgeExecutionQueueProcessor", () => {
const projectId = "test-project-123";
const jobExecutionId = "job-exec-456";
const observationS3Path = "evals/test/observation.json";
const queueName = `${QueueName.LLMAsJudgeExecution}-1`;
let llmAsJudgeExecutionQueueProcessor: (
job: Job<{
payload: {
projectId: string;
jobExecutionId: string;
observationS3Path: string;
};
retryBaggage?: { attempt: number };
}>,
) => Promise<unknown>;
const createMockJob = (
overrides: {
data?: Record<string, unknown>;
attemptsMade?: number;
attemptsStarted?: number;
opts?: { attempts?: number };
timestamp?: number;
} = {},
): Job<any> => {
return {
data: {
id: "queue-job-123",
name: "LLMAsJudgeExecution",
timestamp: new Date(),
payload: {
projectId,
jobExecutionId,
observationS3Path,
},
retryBaggage: { attempt: 0 },
...overrides.data,
},
timestamp: overrides.timestamp ?? Date.now(),
attemptsMade: overrides.attemptsMade ?? 0,
attemptsStarted: overrides.attemptsStarted ?? 1,
opts: overrides.opts ?? { attempts: 10 },
} as Job<any>;
};
beforeAll(async () => {
const { llmAsJudgeExecutionQueueProcessorBuilder } =
await import("../evalQueue");
llmAsJudgeExecutionQueueProcessor =
llmAsJudgeExecutionQueueProcessorBuilder(queueName);
});
beforeEach(() => {
vi.clearAllMocks();
vi.mocked(classifyEvaluatorLlmError).mockReturnValue(null);
vi.mocked(isUnrecoverableError).mockReturnValue(false);
(processObservationEval as Mock).mockResolvedValue("completed");
});
describe("successful processing", () => {
it("should process observation eval successfully and return true", async () => {
const job = createMockJob();
const result = await llmAsJudgeExecutionQueueProcessor(job);
expect(result).toBe(true);
expect(processObservationEval).toHaveBeenCalledWith({
event: {
projectId,
jobExecutionId,
observationS3Path,
},
executionType: "LLM_AS_JUDGE",
});
});
it("should set span attributes for tracing", async () => {
const mockSpan = { setAttribute: vi.fn() };
const { getCurrentSpan } = await import("@langfuse/shared/src/server");
(getCurrentSpan as Mock).mockReturnValue(mockSpan);
const job = createMockJob();
await llmAsJudgeExecutionQueueProcessor(job);
expect(mockSpan.setAttribute).toHaveBeenCalledWith(
"messaging.bullmq.job.input.jobExecutionId",
jobExecutionId,
);
expect(mockSpan.setAttribute).toHaveBeenCalledWith(
"messaging.bullmq.job.input.projectId",
projectId,
);
});
it("records schedule-to-first-attempt latency only for the logical first attempt", async () => {
const now = vi.spyOn(Date, "now").mockReturnValue(10_000);
await llmAsJudgeExecutionQueueProcessor(
createMockJob({ timestamp: 8_500 }),
);
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.evaluation.execution.time_to_first_attempt_ms",
1_500,
{
evaluator_type: "llm_as_judge",
unit: "milliseconds",
},
);
vi.mocked(recordDistribution).mockClear();
await llmAsJudgeExecutionQueueProcessor(
createMockJob({
attemptsMade: 0,
attemptsStarted: 2,
timestamp: 8_000,
}),
);
await llmAsJudgeExecutionQueueProcessor(
createMockJob({
data: { retryBaggage: { attempt: 1 } },
timestamp: 8_000,
}),
);
expect(recordDistribution).not.toHaveBeenCalled();
now.mockRestore();
});
it("records completed and cancelled terminal outcomes", async () => {
await llmAsJudgeExecutionQueueProcessor(createMockJob());
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
1,
{ evaluator_type: "llm_as_judge", outcome: "success" },
);
vi.mocked(recordIncrement).mockClear();
(processObservationEval as Mock).mockResolvedValue("cancelled");
await llmAsJudgeExecutionQueueProcessor(createMockJob());
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
1,
{ evaluator_type: "llm_as_judge", outcome: "cancelled" },
);
});
});
describe("LLM rate limit errors (retryable)", () => {
it("should schedule retry and set DELAYED status for retryable LLM errors", async () => {
const rateLimitError = new Error("Rate limit exceeded");
(processObservationEval as Mock).mockRejectedValue(rateLimitError);
mockLlmError(rateLimitError, true);
(retryLLMRateLimitError as Mock).mockResolvedValue({
outcome: "scheduled",
});
const job = createMockJob();
await llmAsJudgeExecutionQueueProcessor(job);
expect(retryLLMRateLimitError).toHaveBeenCalledWith(
job,
expect.objectContaining({
table: "job_executions",
idField: "jobExecutionId",
queueName,
}),
);
expect(LLMAsJudgeExecutionQueue.getInstance).toHaveBeenCalledWith({
shardName: queueName,
});
expect(prisma.jobExecution.update).toHaveBeenCalledWith({
where: {
id: jobExecutionId,
projectId,
},
data: expect.objectContaining({
status: JobExecutionStatus.DELAYED,
executionTraceId: "test-trace-id",
}),
});
expect(recordIncrement).not.toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
expect.anything(),
expect.anything(),
);
});
it("should not rethrow error after scheduling retry", async () => {
const rateLimitError = new Error("Rate limit exceeded");
(processObservationEval as Mock).mockRejectedValue(rateLimitError);
mockLlmError(rateLimitError, true);
(retryLLMRateLimitError as Mock).mockResolvedValue({
outcome: "scheduled",
});
const job = createMockJob();
// Should not throw
await expect(
llmAsJudgeExecutionQueueProcessor(job),
).resolves.not.toThrow();
});
it("should set ERROR when retryable LLM errors are not re-enqueued", async () => {
const rateLimitError = new Error("Rate limit exceeded");
(processObservationEval as Mock).mockRejectedValue(rateLimitError);
mockLlmError(rateLimitError, true);
(retryLLMRateLimitError as Mock).mockResolvedValue({
outcome: "skipped",
reason: "too_old",
});
const job = createMockJob();
await llmAsJudgeExecutionQueueProcessor(job);
expect(prisma.jobExecution.update).toHaveBeenCalledWith({
where: {
id: jobExecutionId,
projectId,
},
data: expect.objectContaining({
status: JobExecutionStatus.ERROR,
endTime: expect.any(Date),
error: "Rate limit exceeded",
executionTraceId: "test-trace-id",
}),
});
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
1,
{ evaluator_type: "llm_as_judge", outcome: "upstream_error" },
);
});
it("should set ERROR when the retry queue is unavailable", async () => {
const rateLimitError = new Error("Rate limit exceeded");
(processObservationEval as Mock).mockRejectedValue(rateLimitError);
mockLlmError(rateLimitError, true);
(retryLLMRateLimitError as Mock).mockResolvedValue({
outcome: "queue_unavailable",
});
const job = createMockJob();
await llmAsJudgeExecutionQueueProcessor(job);
expect(prisma.jobExecution.update).toHaveBeenCalledWith({
where: {
id: jobExecutionId,
projectId,
},
data: expect.objectContaining({
status: JobExecutionStatus.ERROR,
endTime: expect.any(Date),
error: "Rate limit exceeded",
executionTraceId: "test-trace-id",
}),
});
});
});
describe("LLM completion errors (non-retryable)", () => {
it("retries media-not-found validation errors until the final attempt", async () => {
const mediaNotFoundError = new Error(
"Media asset image-1 was not found in this project",
);
(processObservationEval as Mock).mockRejectedValue(mediaNotFoundError);
vi.mocked(classifyEvaluatorLlmError).mockReturnValue({
kind: "validation",
message: mediaNotFoundError.message,
statusCode: 400,
isRetryable: false,
error: mediaNotFoundError,
validationError: {
code: "media-not-found",
},
blockReason: null,
} as ReturnType<typeof classifyEvaluatorLlmError>);
const job = createMockJob();
await expect(llmAsJudgeExecutionQueueProcessor(job)).rejects.toThrow(
mediaNotFoundError,
);
expect(prisma.jobExecution.update).not.toHaveBeenCalled();
vi.mocked(traceException).mockClear();
await expect(
llmAsJudgeExecutionQueueProcessor(
createMockJob({ attemptsMade: 9, opts: { attempts: 10 } }),
),
).resolves.toBeUndefined();
expect(prisma.jobExecution.update).toHaveBeenCalledWith({
where: {
id: jobExecutionId,
projectId,
},
data: expect.objectContaining({
status: JobExecutionStatus.ERROR,
error: mediaNotFoundError.message,
}),
});
expect(traceException).not.toHaveBeenCalled();
});
it("should set ERROR status for non-retryable LLM errors", async () => {
const llmError = new Error("Invalid API key");
(processObservationEval as Mock).mockRejectedValue(llmError);
mockLlmError(llmError, false);
const job = createMockJob();
await llmAsJudgeExecutionQueueProcessor(job);
expect(prisma.jobExecution.update).toHaveBeenCalledWith({
where: {
id: jobExecutionId,
projectId,
},
data: expect.objectContaining({
status: JobExecutionStatus.ERROR,
endTime: expect.any(Date),
error: "Invalid API key",
executionTraceId: "test-trace-id",
}),
});
});
it("should not rethrow non-retryable LLM errors", async () => {
const llmError = new Error("Invalid API key");
(processObservationEval as Mock).mockRejectedValue(llmError);
mockLlmError(llmError, false);
const job = createMockJob();
// Should not throw - error is handled gracefully
await expect(
llmAsJudgeExecutionQueueProcessor(job),
).resolves.not.toThrow();
});
});
describe("UnrecoverableError handling", () => {
it("should set ERROR status with user-facing message for UnrecoverableError", async () => {
const unrecoverableError = new UnrecoverableError(
"Job configuration not found",
);
(processObservationEval as Mock).mockRejectedValue(unrecoverableError);
vi.mocked(isUnrecoverableError).mockReturnValue(true);
const job = createMockJob();
await llmAsJudgeExecutionQueueProcessor(job);
expect(prisma.jobExecution.update).toHaveBeenCalledWith({
where: {
id: jobExecutionId,
projectId,
},
data: expect.objectContaining({
status: JobExecutionStatus.ERROR,
endTime: expect.any(Date),
error: "Job configuration not found",
executionTraceId: "test-trace-id",
}),
});
});
it("should not rethrow UnrecoverableError", async () => {
const unrecoverableError = new UnrecoverableError("Config not found");
(processObservationEval as Mock).mockRejectedValue(unrecoverableError);
vi.mocked(isUnrecoverableError).mockReturnValue(true);
const job = createMockJob();
// Should not throw
await expect(
llmAsJudgeExecutionQueueProcessor(job),
).resolves.not.toThrow();
});
it("should not call traceException for UnrecoverableError", async () => {
const unrecoverableError = new UnrecoverableError("Config not found");
(processObservationEval as Mock).mockRejectedValue(unrecoverableError);
vi.mocked(isUnrecoverableError).mockReturnValue(true);
const job = createMockJob();
await llmAsJudgeExecutionQueueProcessor(job);
expect(traceException).not.toHaveBeenCalled();
});
it.each([
{
error: new UnrecoverableEvalFailureError(
"invalid model configuration",
EvalFailureReason.InvalidModelConfig,
),
tags: { outcome: "customer_error" },
},
{
error: new UnrecoverableEvalFailureError(
"invalid JSON",
EvalFailureReason.InvalidObservationData,
),
tags: { outcome: "platform_error", reason: "invalid_observation_data" },
},
{
error: new UnrecoverableError("Evaluator identity missing"),
tags: { outcome: "platform_error", reason: "unrecoverable" },
},
])(
"records and warns terminal failures as $tags.outcome",
async ({ error, tags }) => {
(processObservationEval as Mock).mockRejectedValue(error);
vi.mocked(isUnrecoverableError).mockReturnValue(true);
await llmAsJudgeExecutionQueueProcessor(createMockJob());
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
1,
{ evaluator_type: "llm_as_judge", ...tags },
);
expect(logger.warn).toHaveBeenCalledWith(
expect.any(String),
expect.objectContaining({ projectId, errorName: error.name }),
);
},
);
});
describe("unexpected errors (retryable by BullMQ)", () => {
it("should rethrow unexpected errors without persisting ERROR while BullMQ retries remain", async () => {
const unexpectedError = new Error("Database connection failed");
(processObservationEval as Mock).mockRejectedValue(unexpectedError);
const job = createMockJob();
// Should rethrow for BullMQ retry
await expect(llmAsJudgeExecutionQueueProcessor(job)).rejects.toThrow(
"Database connection failed",
);
expect(prisma.jobExecution.update).not.toHaveBeenCalled();
expect(recordIncrement).not.toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
expect.anything(),
expect.anything(),
);
});
it("should call traceException for unexpected errors", async () => {
const unexpectedError = new Error("Unexpected failure");
(processObservationEval as Mock).mockRejectedValue(unexpectedError);
const job = createMockJob();
try {
await llmAsJudgeExecutionQueueProcessor(job);
} catch {
// Expected to throw
}
expect(traceException).toHaveBeenCalledWith(unexpectedError);
});
it("should rethrow unexpected errors for BullMQ retry", async () => {
const unexpectedError = new Error("Network timeout");
(processObservationEval as Mock).mockRejectedValue(unexpectedError);
const job = createMockJob();
await expect(llmAsJudgeExecutionQueueProcessor(job)).rejects.toThrow(
"Network timeout",
);
});
it.each([
{ error: new Error("Database connection failed"), reason: "unexpected" },
{
error: new EvalFailureError(
"Failed to write score",
EvalFailureReason.ScorePersistence,
),
reason: "score_persistence",
},
])(
"should persist ERROR on the final BullMQ attempt with reason $reason",
async ({ error, reason }) => {
(processObservationEval as Mock).mockRejectedValue(error);
await expect(
llmAsJudgeExecutionQueueProcessor(
createMockJob({ attemptsMade: 9, opts: { attempts: 10 } }),
),
).rejects.toThrow(error);
expect(prisma.jobExecution.update).toHaveBeenCalledWith({
where: {
id: jobExecutionId,
projectId,
},
data: {
status: JobExecutionStatus.ERROR,
endTime: expect.any(Date),
error: "An internal error occurred",
executionTraceId: "test-trace-id",
},
});
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
1,
{
evaluator_type: "llm_as_judge",
outcome: "platform_error",
reason,
},
);
},
);
});
describe("execution trace ID", () => {
it("should generate deterministic trace ID from job execution ID", async () => {
const error = new UnrecoverableError("Test error");
(processObservationEval as Mock).mockRejectedValue(error);
vi.mocked(isUnrecoverableError).mockReturnValue(true);
const { createW3CTraceId } =
await import("../../features/utils/utilities");
const job = createMockJob();
await llmAsJudgeExecutionQueueProcessor(job);
expect(createW3CTraceId).toHaveBeenCalledWith(jobExecutionId);
});
});
describe("retry baggage tracking", () => {
it("should track retry attempt in span attributes", async () => {
const mockSpan = { setAttribute: vi.fn() };
const { getCurrentSpan } = await import("@langfuse/shared/src/server");
(getCurrentSpan as Mock).mockReturnValue(mockSpan);
const job = createMockJob({
data: { retryBaggage: { attempt: 3 } },
});
await llmAsJudgeExecutionQueueProcessor(job as Job<any>);
expect(mockSpan.setAttribute).toHaveBeenCalledWith(
"messaging.bullmq.job.input.retryBaggage.attempt",
3,
);
});
it("should default to 0 when retry baggage is missing", async () => {
const mockSpan = { setAttribute: vi.fn() };
const { getCurrentSpan } = await import("@langfuse/shared/src/server");
(getCurrentSpan as Mock).mockReturnValue(mockSpan);
const job = createMockJob();
delete (job.data as { retryBaggage?: unknown }).retryBaggage;
await llmAsJudgeExecutionQueueProcessor(job as Job<any>);
expect(mockSpan.setAttribute).toHaveBeenCalledWith(
"messaging.bullmq.job.input.retryBaggage.attempt",
0,
);
});
});
describe("null span handling", () => {
it("should handle null span gracefully", async () => {
const { getCurrentSpan } = await import("@langfuse/shared/src/server");
(getCurrentSpan as Mock).mockReturnValue(null);
const job = createMockJob();
// Should not throw
await expect(
llmAsJudgeExecutionQueueProcessor(job),
).resolves.not.toThrow();
expect(processObservationEval).toHaveBeenCalled();
});
});
});