🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import {
beforeAll,
beforeEach,
describe,
expect,
it,
vi,
type Mock,
} from "vitest";
import { Job } from "bullmq";
import { EvalTemplateType } from "@prisma/client";
import {
UnrecoverableError,
isUnrecoverableError,
} from "../../errors/UnrecoverableError";
const QueueName = {
CodeEvalExecution: "code-eval-execution-queue",
} as const;
const JobExecutionStatus = {
ERROR: "ERROR",
} as const;
vi.mock("@langfuse/shared/src/db", () => ({
prisma: {
jobExecution: {
update: vi.fn(),
},
},
}));
vi.mock(
"../../features/evaluation/observationEval/observationEvalProcessor",
() => ({
processObservationEval: vi.fn(),
}),
);
vi.mock(
"../../features/evaluation/codeBased/executeCodeBasedEvaluation",
() => ({
executeCodeBasedEvaluation: vi.fn(),
}),
);
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
const actual =
await importOriginal<typeof import("@langfuse/shared/src/server")>();
const { CodeEvalExecutionError } =
await import("../../../../packages/shared/src/server/evals/codeEvalExecution");
return {
...actual,
CodeEvalExecutionError,
getCurrentSpan: vi.fn().mockReturnValue({
setAttribute: vi.fn(),
}),
logger: {
debug: vi.fn(),
error: vi.fn(),
},
QueueName: {
CodeEvalExecution: "code-eval-execution-queue",
},
recordDistribution: vi.fn(),
recordIncrement: vi.fn(),
traceException: vi.fn(),
};
});
vi.mock("../../features/utils/utilities", () => ({
createW3CTraceId: vi.fn().mockReturnValue("test-trace-id"),
}));
vi.mock("../../errors/UnrecoverableError", async () => {
const actual = await vi.importActual("../../errors/UnrecoverableError");
return {
...actual,
isUnrecoverableError: vi.fn(),
};
});
import { prisma } from "@langfuse/shared/src/db";
import { processObservationEval } from "../../features/evaluation/observationEval/observationEvalProcessor";
import {
recordDistribution,
recordIncrement,
traceException,
} from "@langfuse/shared/src/server";
import { CodeEvalDispatcherErrorCodes } from "../../../../packages/shared/src/server/evals/codeEvalDispatcherTypes";
import { CodeEvalExecutionError } from "../../../../packages/shared/src/server/evals/codeEvalExecution";
describe("codeEvalExecutionQueueProcessor", () => {
const projectId = "test-project-123";
const jobExecutionId = "job-exec-456";
const observationS3Path = "evals/test/observation.json";
const queueName = `${QueueName.CodeEvalExecution}-1`;
let codeEvalExecutionQueueProcessor: (job: Job<any>) => Promise<unknown>;
const createMockJob = (
overrides: {
data?: Record<string, unknown>;
attemptsMade?: number;
attemptsStarted?: number;
opts?: { attempts?: number };
timestamp?: number;
} = {},
): Job<any> =>
({
data: {
id: "queue-job-123",
name: "CodeEvalExecution",
timestamp: new Date(),
payload: {
projectId,
jobExecutionId,
observationS3Path,
},
retryBaggage: { attempt: 0 },
...overrides.data,
},
timestamp: overrides.timestamp ?? Date.now(),
attemptsMade: overrides.attemptsMade ?? 0,
attemptsStarted: overrides.attemptsStarted ?? 1,
opts: overrides.opts ?? { attempts: 10 },
}) as Job<any>;
beforeAll(async () => {
const { codeEvalExecutionQueueProcessorBuilder } =
await import("../codeEvalQueue");
codeEvalExecutionQueueProcessor =
codeEvalExecutionQueueProcessorBuilder(queueName);
});
beforeEach(() => {
vi.clearAllMocks();
(isUnrecoverableError as unknown as Mock).mockReturnValue(false);
(processObservationEval as Mock).mockResolvedValue("completed");
});
it("should process code observation evals through the code template gate", async () => {
const result = await codeEvalExecutionQueueProcessor(createMockJob());
expect(result).toBe(true);
expect(processObservationEval).toHaveBeenCalledWith({
event: {
projectId,
jobExecutionId,
observationS3Path,
},
executionType: EvalTemplateType.CODE,
});
});
it("records schedule-to-first-attempt latency only for the initial attempt", async () => {
const now = vi.spyOn(Date, "now").mockReturnValue(10_000);
await codeEvalExecutionQueueProcessor(createMockJob({ timestamp: 8_500 }));
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.evaluation.execution.time_to_first_attempt_ms",
1_500,
{
evaluator_type: "code_as_judge",
unit: "milliseconds",
},
);
vi.mocked(recordDistribution).mockClear();
await codeEvalExecutionQueueProcessor(
createMockJob({ attemptsMade: 0, attemptsStarted: 2, timestamp: 8_000 }),
);
expect(recordDistribution).not.toHaveBeenCalled();
now.mockRestore();
});
it("records completed and cancelled terminal outcomes", async () => {
await codeEvalExecutionQueueProcessor(createMockJob());
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
1,
{ evaluator_type: "code_as_judge", outcome: "success" },
);
vi.mocked(recordIncrement).mockClear();
(processObservationEval as Mock).mockResolvedValue("cancelled");
await codeEvalExecutionQueueProcessor(createMockJob());
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
1,
{ evaluator_type: "code_as_judge", outcome: "cancelled" },
);
});
it("classifies non-retryable user-code failures as customer errors", async () => {
const error = new CodeEvalExecutionError({
code: CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,
message: "user code failed",
retryable: false,
});
(processObservationEval as Mock).mockRejectedValue(error);
await expect(
codeEvalExecutionQueueProcessor(createMockJob()),
).resolves.toBeUndefined();
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
1,
{ evaluator_type: "code_as_judge", outcome: "customer_error" },
);
expect(traceException).not.toHaveBeenCalled();
});
it("should treat unrecoverable code eval errors as terminal", async () => {
const error = new UnrecoverableError(
"The evaluator returned an invalid result.",
);
(processObservationEval as Mock).mockRejectedValue(error);
(isUnrecoverableError as unknown as Mock).mockReturnValue(true);
const result = await codeEvalExecutionQueueProcessor(createMockJob());
expect(result).toBeUndefined();
expect(prisma.jobExecution.update).toHaveBeenCalledWith({
where: { id: jobExecutionId, projectId },
data: {
status: JobExecutionStatus.ERROR,
endTime: expect.any(Date),
error: "The evaluator returned an invalid result.",
executionTraceId: "test-trace-id",
},
});
expect(traceException).not.toHaveBeenCalled();
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
1,
{ evaluator_type: "code_as_judge", outcome: "platform_error" },
);
});
it("should persist already-masked internal code eval errors", async () => {
const error = new UnrecoverableError("An internal error occurred");
(processObservationEval as Mock).mockRejectedValue(error);
(isUnrecoverableError as unknown as Mock).mockReturnValue(true);
const result = await codeEvalExecutionQueueProcessor(createMockJob());
expect(result).toBeUndefined();
expect(prisma.jobExecution.update).toHaveBeenCalledWith({
where: { id: jobExecutionId, projectId },
data: {
status: JobExecutionStatus.ERROR,
endTime: expect.any(Date),
error: "An internal error occurred",
executionTraceId: "test-trace-id",
},
});
});
it("should mark unrecoverable skeleton errors as terminal", async () => {
const error = new UnrecoverableError(
"Code-based eval execution is not implemented yet",
);
(processObservationEval as Mock).mockRejectedValue(error);
(isUnrecoverableError as unknown as Mock).mockReturnValue(true);
const result = await codeEvalExecutionQueueProcessor(createMockJob());
expect(result).toBeUndefined();
expect(prisma.jobExecution.update).toHaveBeenCalledWith({
where: {
id: jobExecutionId,
projectId,
},
data: {
status: JobExecutionStatus.ERROR,
endTime: expect.any(Date),
error: "Code-based eval execution is not implemented yet",
executionTraceId: "test-trace-id",
},
});
expect(traceException).not.toHaveBeenCalled();
});
it("should rethrow retryable errors without writing ERROR while retries remain", async () => {
const error = new Error("temporary dispatcher failure");
(processObservationEval as Mock).mockRejectedValue(error);
await expect(
codeEvalExecutionQueueProcessor(createMockJob()),
).rejects.toThrow(error);
expect(prisma.jobExecution.update).not.toHaveBeenCalled();
expect(traceException).toHaveBeenCalledWith(error);
expect(recordIncrement).not.toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
expect.anything(),
expect.anything(),
);
});
it("should mark the job as ERROR with a masked message on the final retry attempt", async () => {
const error = new Error("temporary dispatcher failure");
(processObservationEval as Mock).mockRejectedValue(error);
await expect(
codeEvalExecutionQueueProcessor(
createMockJob({ attemptsMade: 9, opts: { attempts: 10 } }),
),
).rejects.toThrow(error);
expect(prisma.jobExecution.update).toHaveBeenCalledWith({
where: {
id: jobExecutionId,
projectId,
},
data: {
status: JobExecutionStatus.ERROR,
endTime: expect.any(Date),
error: "An internal error occurred",
executionTraceId: "test-trace-id",
},
});
expect(traceException).toHaveBeenCalledWith(error);
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
1,
{ evaluator_type: "code_as_judge", outcome: "platform_error" },
);
});
it("should treat code eval timeouts as terminal without retrying", async () => {
const timeoutMessage =
"Evaluator timed out. Code-based evaluators must complete within the configured runtime limit. Long executions can be caused by network calls, which are forbidden and may never complete. Remove network calls, optimize your evaluator code, and try again. See https://langfuse.com/docs/evaluation/evaluation-methods/code-evaluators for details.";
const error = new CodeEvalExecutionError({
code: CodeEvalDispatcherErrorCodes.TIMEOUT,
message: timeoutMessage,
retryable: false,
});
(processObservationEval as Mock).mockRejectedValue(error);
await expect(
codeEvalExecutionQueueProcessor(createMockJob()),
).resolves.toBeUndefined();
expect(prisma.jobExecution.update).toHaveBeenCalledWith({
where: {
id: jobExecutionId,
projectId,
},
data: {
status: JobExecutionStatus.ERROR,
endTime: expect.any(Date),
error: timeoutMessage,
executionTraceId: "test-trace-id",
},
});
expect(traceException).not.toHaveBeenCalled();
expect(recordIncrement).toHaveBeenCalledWith(
"langfuse.evaluation.execution.terminal",
1,
{ evaluator_type: "code_as_judge", outcome: "customer_error" },
);
});
});