🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { beforeEach, describe, expect, it, vi, type Mock } from "vitest";
import {
BatchActionStatus,
EvalTargetObject,
EvalTemplateType,
JobConfigState,
} from "@langfuse/shared";
import type {
ObservationEvalRule,
ObservationEvalSchedulerDeps,
} from "../evaluation/observationEval/types";
vi.mock("@langfuse/shared/src/db", () => ({
prisma: {
batchAction: {
update: vi.fn().mockResolvedValue(undefined),
},
},
}));
vi.mock("../evaluation/observationEval/createSchedulerDeps", () => ({
createObservationEvalSchedulerDeps: vi.fn(() => ({ deps: true })),
}));
vi.mock("../evaluation/observationEval/scheduleObservationEvals", () => ({
scheduleObservationEvals: vi.fn().mockResolvedValue(undefined),
}));
import { prisma } from "@langfuse/shared/src/db";
import { createObservationEvalSchedulerDeps } from "../evaluation/observationEval/createSchedulerDeps";
import { scheduleObservationEvals } from "../evaluation/observationEval/scheduleObservationEvals";
import { processBatchedObservationEval } from "./processBatchedObservationEval";
describe("processBatchedObservationEval", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("passes evaluators with match-all filter and sampling to the scheduler", async () => {
const projectId = "project-1";
const batchActionId = "batch-action-1";
// Evaluators should already have filter=[] and sampling=1 set by the
// caller (handleBatchActionJob), so the scheduler applies its normal
// targeting logic and every observation matches.
const evaluators: ObservationEvalRule[] = [
{
id: "config-1",
projectId,
filter: [],
sampling: { toNumber: () => 1 } as ObservationEvalRule["sampling"],
evalTemplateId: "template-1",
evalTemplate: { type: EvalTemplateType.LLM_AS_JUDGE },
scoreName: "quality",
targetObject: EvalTargetObject.EVENT,
variableMapping: [],
status: JobConfigState.ACTIVE,
blockedAt: null,
},
];
const remappedRow: Record<string, unknown> = {
span_id: "obs-1",
trace_id: "trace-1",
project_id: projectId,
parent_span_id: null,
type: "GENERATION",
name: "test",
usage_details: {},
cost_details: {},
provided_usage_details: {},
provided_cost_details: {},
tags: [],
input: "input",
output: "output",
metadata: {},
};
const observationStream = (async function* () {
yield remappedRow;
})();
await processBatchedObservationEval({
projectId,
batchActionId,
evaluators,
observationStream,
});
expect(scheduleObservationEvals).toHaveBeenCalledTimes(1);
expect(scheduleObservationEvals).toHaveBeenCalledWith(
expect.objectContaining({
configs: evaluators,
executionMode: "MANUAL",
executionScopeId: batchActionId,
}),
);
expect(
(prisma.batchAction.update as Mock).mock.calls.length,
).toBeGreaterThan(0);
});
it("passes stable evaluator assignments to the scheduler for v2 batches", async () => {
const projectId = "project-1";
const evaluators: ObservationEvalRule[] = [
{
id: "evaluator-1",
// A standalone evaluator falls back to its own id as the execution anchor.
ruleId: null,
projectId,
filter: [],
sampling: { toNumber: () => 1 } as ObservationEvalRule["sampling"],
status: JobConfigState.ACTIVE,
targetObject: EvalTargetObject.EVENT,
assignments: [
{
id: "evaluator-1",
evaluatorId: "evaluator-1",
variableMapping: null,
evaluator: {
id: "evaluator-1",
projectId,
type: EvalTemplateType.LLM_AS_JUDGE,
blockedAt: null,
versions: [{ id: "evaluator-version-1", variableMapping: [] }],
},
},
],
},
];
const observationStream = (async function* () {
yield {
span_id: "obs-v2",
trace_id: "trace-v2",
project_id: projectId,
parent_span_id: null,
type: "GENERATION",
name: "v2-test",
usage_details: {},
cost_details: {},
provided_usage_details: {},
provided_cost_details: {},
tags: [],
input: "input",
output: "output",
metadata: {},
};
})();
await processBatchedObservationEval({
projectId,
batchActionId: "batch-action-v2",
evaluators,
evaluatorLabels: ["Quality"],
observationStream,
});
expect(scheduleObservationEvals).toHaveBeenCalledWith(
expect.objectContaining({
configs: evaluators,
executionMode: "MANUAL",
executionScopeId: "batch-action-v2",
}),
);
});
it("derives tool_call_count from tool_call_names in batch records", async () => {
const projectId = "project-1";
const batchActionId = "batch-action-2";
const evaluators: ObservationEvalRule[] = [
{
id: "config-1",
projectId,
filter: [],
sampling: { toNumber: () => 1 } as ObservationEvalRule["sampling"],
evalTemplateId: "template-1",
evalTemplate: { type: EvalTemplateType.LLM_AS_JUDGE },
scoreName: "quality",
targetObject: EvalTargetObject.EVENT,
variableMapping: [],
status: JobConfigState.ACTIVE,
blockedAt: null,
},
];
const rowWithToolCalls: Record<string, unknown> = {
span_id: "obs-2",
trace_id: "trace-2",
project_id: projectId,
parent_span_id: null,
type: "GENERATION",
name: "tool-test",
usage_details: {},
cost_details: {},
provided_usage_details: {},
provided_cost_details: {},
tags: [],
input: "input",
output: "output",
metadata: {},
tool_call_names: ["search", "calculator", "fetch"],
};
const rowWithoutToolCalls: Record<string, unknown> = {
span_id: "obs-3",
trace_id: "trace-3",
project_id: projectId,
parent_span_id: null,
type: "GENERATION",
name: "no-tool-test",
usage_details: {},
cost_details: {},
provided_usage_details: {},
provided_cost_details: {},
tags: [],
input: "input",
output: "output",
metadata: {},
};
const observationStream = (async function* () {
yield rowWithToolCalls;
yield rowWithoutToolCalls;
})();
await processBatchedObservationEval({
projectId,
batchActionId,
evaluators,
observationStream,
});
expect(scheduleObservationEvals).toHaveBeenCalledTimes(2);
const firstCall = (scheduleObservationEvals as Mock).mock.calls[0][0];
expect(firstCall.observation.tool_call_count).toBe(3);
expect(firstCall.observation.tool_call_names).toEqual([
"search",
"calculator",
"fetch",
]);
const secondCall = (scheduleObservationEvals as Mock).mock.calls[1][0];
expect(secondCall.observation.tool_call_count).toBe(0);
});
it("marks an observation as failed when its evaluator job cannot be enqueued", async () => {
const projectId = "project-1";
const enqueueError = new Error("Simulated queue enqueue failure");
const schedulerDeps: ObservationEvalSchedulerDeps = {
uploadObservationToS3: vi.fn().mockResolvedValue("observation.json"),
upsertJobExecution: vi.fn().mockResolvedValue({ id: "execution-1" }),
enqueueEvalJob: vi.fn().mockRejectedValue(enqueueError),
};
const actualModule = await vi.importActual<
typeof import("../evaluation/observationEval/scheduleObservationEvals")
>("../evaluation/observationEval/scheduleObservationEvals");
vi.mocked(createObservationEvalSchedulerDeps).mockReturnValueOnce(
schedulerDeps,
);
vi.mocked(scheduleObservationEvals).mockImplementationOnce(
actualModule.scheduleObservationEvals,
);
await processBatchedObservationEval({
projectId,
batchActionId: "batch-action-failed-enqueue",
evaluators: [
{
id: "config-1",
projectId,
filter: [],
sampling: { toNumber: () => 1 } as ObservationEvalRule["sampling"],
evalTemplateId: "template-1",
evalTemplate: { type: EvalTemplateType.LLM_AS_JUDGE },
scoreName: "quality",
targetObject: EvalTargetObject.EVENT,
variableMapping: [],
status: JobConfigState.ACTIVE,
blockedAt: null,
},
],
observationStream: (async function* () {
yield {
span_id: "obs-failed-enqueue",
trace_id: "trace-1",
project_id: projectId,
parent_span_id: null,
type: "GENERATION",
name: "test",
usage_details: {},
cost_details: {},
provided_usage_details: {},
provided_cost_details: {},
tags: [],
input: "input",
output: "output",
metadata: {},
};
})(),
});
expect(prisma.batchAction.update).toHaveBeenLastCalledWith({
where: {
id: "batch-action-failed-enqueue",
projectId,
},
data: expect.objectContaining({
status: BatchActionStatus.Failed,
totalCount: 1,
processedCount: 0,
failedCount: 1,
log: expect.stringContaining(enqueueError.message),
}),
});
});
});