🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { describe, it, expect, vi, beforeEach } from "vitest";
import {
convertEventRecordToObservationForEval,
LangfuseInternalTraceEnvironment,
} from "@langfuse/shared";
import {
buildInternalTraceEventInputs,
materializeInternalTrace,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import type { ObservationForEval } from "../../evaluation/observationEval/types";
import { IngestionService } from "../../../services/IngestionService";
import * as clickhouseWriterExports from "../../../services/ClickhouseWriter";
import { scheduleExperimentObservationEvals } from "../scheduleExperimentEvals";
const {
mockAddToClickhouseWriter,
mockFetchObservationEvalRules,
mockCreateObservationEvalSchedulerDeps,
mockScheduleObservationEvals,
} = vi.hoisted(() => ({
mockAddToClickhouseWriter: vi.fn(),
mockFetchObservationEvalRules: vi.fn(),
mockCreateObservationEvalSchedulerDeps: vi.fn(),
mockScheduleObservationEvals: vi.fn(),
}));
vi.mock("../../../services/ClickhouseWriter", async (importOriginal) => {
const original = (await importOriginal()) as object;
return {
...original,
ClickhouseWriter: {
getInstance: () => ({
addToQueue: mockAddToClickhouseWriter,
}),
},
};
});
vi.mock(
"../../evaluation/observationEval/fetchObservationEvalRules",
async (importOriginal) => {
const original = (await importOriginal()) as object;
return {
...original,
fetchObservationEvalRules: mockFetchObservationEvalRules,
createObservationEvalSchedulerDeps:
mockCreateObservationEvalSchedulerDeps,
scheduleObservationEvals: mockScheduleObservationEvals,
};
},
);
vi.mock(
"../../evaluation/observationEval/createSchedulerDeps",
async (importOriginal) => {
const original = (await importOriginal()) as object;
return {
...original,
fetchObservationEvalRules: mockFetchObservationEvalRules,
createObservationEvalSchedulerDeps:
mockCreateObservationEvalSchedulerDeps,
scheduleObservationEvals: mockScheduleObservationEvals,
};
},
);
vi.mock(
"../../evaluation/observationEval/scheduleObservationEvals",
async (importOriginal) => {
const original = (await importOriginal()) as object;
return {
...original,
fetchObservationEvalRules: mockFetchObservationEvalRules,
createObservationEvalSchedulerDeps:
mockCreateObservationEvalSchedulerDeps,
scheduleObservationEvals: mockScheduleObservationEvals,
};
},
);
const mockClickhouseClient = {
query: async () => ({
json: async () => [],
query_id: "test-query-id",
response_headers: { "x-clickhouse-summary": "[]" },
}),
};
const ingestionService = new IngestionService(
null as any,
prisma,
clickhouseWriterExports.ClickhouseWriter.getInstance() as any,
mockClickhouseClient as any,
);
const traceId = "4a56b6d74bbdffd08b9a2485f3315eeb";
const generationId = "d6f103ed-6ef6-4f83-a520-ab746e717360";
const blockedParserSpanId = "8bd2ff9f-1d45-4240-9581-45a7190c77fd";
const prompt = { name: "Capital guesser", version: 1 };
const datasetItem = {
id: "f0c467a1-539b-4e25-b41b-8db3ae399ef4",
datasetId: "cml1yj2ag0001xsej1fcv6vz8",
validFrom: new Date("2026-01-31T06:57:38.646Z"),
expectedOutput: { capital: "Berlin" },
metadata: {
region: "EU",
difficulty: "easy",
},
input: {
country: "Germany",
},
} as any;
const config = {
runId: "run-456",
datasetRun: {
name: "Prompt Capital guesser-v1 on dataset countries - 2026-01-31T06:57:38.646Z",
description: "Prompt experiment run",
metadata: {
prompt_id: "prompt-123",
provider: "openai",
model: "gpt-4.1",
model_params: {},
experiment_name: "Prompt Capital guesser-v1 on dataset countries",
experiment_run_name:
"Prompt Capital guesser-v1 on dataset countries - 2026-01-31T06:57:38.646Z",
},
},
} as any;
// Real events captured from an experiment run.
const realExperimentEvents = [
{
id: "669353e4-e6a5-451e-9eab-1b33c9e396d5",
type: "trace-create",
timestamp: "2026-01-31T06:58:02.221Z",
body: {
id: traceId,
timestamp: "2026-01-31T06:58:02.218Z",
environment: "langfuse-prompt-experiment",
name: "dataset-run-item-008e4",
metadata: {
dataset_id: "cml1yj2ag0001xsej1fcv6vz8",
dataset_item_id: "f0c467a1-539b-4e25-b41b-8db3ae399ef4",
experiment_name: "Prompt Capital guesser-v1 on dataset countries",
experiment_run_name:
"Prompt Capital guesser-v1 on dataset countries - 2026-01-31T06:57:38.646Z",
},
input: [
{
content: "You are a euro capital guesser.",
role: "system",
},
{
content: "What is the capital of Germany?",
role: "user",
},
],
},
},
{
id: "632bcb55-2130-4b67-8a00-575106ab979e",
type: "span-create",
timestamp: "2026-01-31T06:58:02.221Z",
body: {
id: traceId,
startTime: "2026-01-31T06:58:02.218Z",
environment: "langfuse-prompt-experiment",
traceId,
name: "dataset-run-item-008e4",
metadata: {
dataset_id: "cml1yj2ag0001xsej1fcv6vz8",
dataset_item_id: "f0c467a1-539b-4e25-b41b-8db3ae399ef4",
experiment_name: "Prompt Capital guesser-v1 on dataset countries",
experiment_run_name:
"Prompt Capital guesser-v1 on dataset countries - 2026-01-31T06:57:38.646Z",
},
input: [
{
content: "You are a euro capital guesser.",
role: "system",
},
{
content: "What is the capital of Germany?",
role: "user",
},
],
},
},
{
id: "423fb5f6-1aa2-4285-ac05-03801ce32def",
type: "generation-create",
timestamp: "2026-01-31T06:58:02.222Z",
body: {
id: generationId,
startTime: "2026-01-31T06:58:02.220Z",
environment: "langfuse-prompt-experiment",
traceId,
name: "ChatOpenAI",
metadata: {
tags: ["seq:step:1"],
dataset_id: "cml1yj2ag0001xsej1fcv6vz8",
dataset_item_id: "f0c467a1-539b-4e25-b41b-8db3ae399ef4",
experiment_name: "Prompt Capital guesser-v1 on dataset countries",
experiment_run_name:
"Prompt Capital guesser-v1 on dataset countries - 2026-01-31T06:57:38.646Z",
ls_provider: "openai",
ls_model_name: "gpt-4.1",
ls_model_type: "chat",
},
parentObservationId: traceId,
input: [
{
content: "You are a euro capital guesser.",
role: "system",
},
{
content: "What is the capital of Germany?",
role: "user",
},
],
model: "gpt-4.1",
modelParameters: {},
},
},
{
id: "89688bd4-ec5c-4df3-830d-1588c6faa189",
type: "generation-create",
timestamp: "2026-01-31T06:58:02.230Z",
body: {
id: generationId,
startTime: "2026-01-31T06:58:02.220Z",
environment: "langfuse-prompt-experiment",
traceId,
name: "ChatOpenAI",
metadata: {
tags: ["seq:step:1"],
dataset_id: "cml1yj2ag0001xsej1fcv6vz8",
dataset_item_id: "f0c467a1-539b-4e25-b41b-8db3ae399ef4",
experiment_name: "Prompt Capital guesser-v1 on dataset countries",
experiment_run_name:
"Prompt Capital guesser-v1 on dataset countries - 2026-01-31T06:57:38.646Z",
ls_provider: "openai",
ls_model_name: "gpt-4.1",
ls_model_type: "chat",
},
parentObservationId: traceId,
input: [
{
content: "You are a euro capital guesser.",
role: "system",
},
{
content: "What is the capital of Germany?",
role: "user",
},
],
model: "gpt-4.1",
modelParameters: {},
},
},
{
id: "a7ec0f62-7e7b-4ed7-a179-099e588e4f88",
type: "generation-update",
timestamp: "2026-01-31T06:58:03.708Z",
body: {
id: generationId,
model: "gpt-4.1-2025-04-14",
traceId,
output: {
content: "The capital of Germany is Berlin.",
role: "assistant",
},
endTime: "2026-01-31T06:58:03.707Z",
usage: {
input: 26,
output: 7,
total: 33,
input_audio: 0,
input_cache_read: 0,
output_audio: 0,
output_reasoning: 0,
},
usageDetails: {
input: 26,
output: 7,
total: 33,
input_audio: 0,
input_cache_read: 0,
output_audio: 0,
output_reasoning: 0,
},
},
},
{
id: "2ae04b82-00bc-4dcb-a838-3857b87d5a48",
type: "generation-update",
timestamp: "2026-01-31T06:58:03.708Z",
body: {
id: generationId,
model: "gpt-4.1-2025-04-14",
traceId,
output: {
content: "The capital of Germany is Berlin.",
role: "assistant",
},
endTime: "2026-01-31T06:58:03.707Z",
usage: {
input: 26,
output: 7,
total: 33,
input_audio: 0,
input_cache_read: 0,
output_audio: 0,
output_reasoning: 0,
},
usageDetails: {
input: 26,
output: 7,
total: 33,
input_audio: 0,
input_cache_read: 0,
output_audio: 0,
output_reasoning: 0,
},
},
},
{
id: "8439a06a-4879-4b39-8014-1966c6a9b54d",
type: "span-create",
timestamp: "2026-01-31T06:58:03.709Z",
body: {
id: blockedParserSpanId,
startTime: "2026-01-31T06:58:03.708Z",
environment: "langfuse-prompt-experiment",
traceId,
parentObservationId: traceId,
name: "StrOutputParser",
metadata: {
tags: ["seq:step:2"],
dataset_id: "cml1yj2ag0001xsej1fcv6vz8",
dataset_item_id: "f0c467a1-539b-4e25-b41b-8db3ae399ef4",
experiment_name: "Prompt Capital guesser-v1 on dataset countries",
experiment_run_name:
"Prompt Capital guesser-v1 on dataset countries - 2026-01-31T06:57:38.646Z",
},
input: "The capital of Germany is Berlin.",
},
},
{
id: "de27a791-209f-4008-bc16-617eb59c9914",
type: "span-update",
timestamp: "2026-01-31T06:58:03.709Z",
body: {
id: blockedParserSpanId,
traceId,
output: "The capital of Germany is Berlin.",
endTime: "2026-01-31T06:58:03.709Z",
},
},
{
id: "98f391e0-b74d-46c5-ae73-5899215b7915",
type: "span-update",
timestamp: "2026-01-31T06:58:03.710Z",
body: {
id: traceId,
traceId,
output: "The capital of Germany is Berlin.",
endTime: "2026-01-31T06:58:03.709Z",
},
},
{
id: "f86aecdf-9a52-4c37-8ef1-0f5ab450eab1",
type: "trace-create",
timestamp: "2026-01-31T06:58:03.710Z",
body: {
id: traceId,
timestamp: "2026-01-31T06:58:03.709Z",
environment: "langfuse-prompt-experiment",
output: "The capital of Germany is Berlin.",
tags: [],
},
},
];
function getProcessedExperimentEvents() {
return realExperimentEvents
.filter((event) => event.body.id !== blockedParserSpanId)
.map((event) => ({
...event,
body: {
...event.body,
environment: LangfuseInternalTraceEnvironment.PromptExperiments,
...(event.type === "generation-create"
? { promptName: prompt.name, promptVersion: prompt.version }
: {}),
},
}));
}
function getExperimentContext() {
return {
id: config.runId,
name: config.datasetRun.name,
metadata: config.datasetRun.metadata,
description: config.datasetRun.description,
datasetId: datasetItem.datasetId,
itemId: datasetItem.id,
itemVersion: datasetItem.validFrom
.toISOString()
.replace("T", " ")
.replace("Z", ""),
itemExpectedOutput: datasetItem.expectedOutput,
itemMetadata: datasetItem.metadata,
};
}
describe("prompt experiment direct-write materialization", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("should use the trace root as the experiment root", () => {
const processedEvents = getProcessedExperimentEvents();
const { rootSpanId, snapshots } = materializeInternalTrace({
processedEvents,
traceId,
});
expect(rootSpanId).toBe(traceId);
expect(snapshots).toHaveLength(2);
const rootSnapshot = snapshots.find(
(snapshot) => snapshot.spanId === traceId,
);
const generationSnapshot = snapshots.find(
(snapshot) => snapshot.spanId === generationId,
);
expect(rootSnapshot).toMatchObject({
spanId: traceId,
traceId,
type: "SPAN",
name: "dataset-run-item-008e4",
environment: LangfuseInternalTraceEnvironment.PromptExperiments,
input: [
{ content: "You are a euro capital guesser.", role: "system" },
{ content: "What is the capital of Germany?", role: "user" },
],
output: "The capital of Germany is Berlin.",
metadata: {
dataset_id: "cml1yj2ag0001xsej1fcv6vz8",
dataset_item_id: "f0c467a1-539b-4e25-b41b-8db3ae399ef4",
experiment_name: "Prompt Capital guesser-v1 on dataset countries",
experiment_run_name:
"Prompt Capital guesser-v1 on dataset countries - 2026-01-31T06:57:38.646Z",
},
endTimeISO: "2026-01-31T06:58:03.709Z",
});
expect(generationSnapshot).toMatchObject({
spanId: generationId,
traceId,
parentSpanId: traceId,
type: "GENERATION",
name: "ChatOpenAI",
promptName: "Capital guesser",
promptVersion: "1",
modelName: "gpt-4.1-2025-04-14",
});
});
it("should build trace-rooted event inputs with experiment metadata", async () => {
const { rootSpanId, eventInputs } = buildInternalTraceEventInputs({
processedEvents: getProcessedExperimentEvents(),
traceId,
projectId: "project-123",
experimentContext: getExperimentContext(),
});
// Direct write uses original IDs (no t- prefix remapping).
// The experiment backfill job skips traces already in events_core via LEFT ANTI JOIN.
expect(rootSpanId).toBe(traceId);
expect(eventInputs).toHaveLength(2);
const rootEventInput = eventInputs.find(
(eventInput) => eventInput.spanId === traceId,
);
const generationEventInput = eventInputs.find(
(eventInput) => eventInput.spanId === generationId,
);
expect(rootEventInput).toMatchObject({
projectId: "project-123",
traceId,
spanId: traceId,
parentSpanId: undefined,
type: "SPAN",
experimentId: "run-456",
experimentName:
"Prompt Capital guesser-v1 on dataset countries - 2026-01-31T06:57:38.646Z",
experimentItemRootSpanId: traceId,
input: JSON.stringify([
{ content: "You are a euro capital guesser.", role: "system" },
{ content: "What is the capital of Germany?", role: "user" },
]),
output: "The capital of Germany is Berlin.",
source: "ingestion-api-dual-write-experiments",
});
expect(generationEventInput).toMatchObject({
traceId,
spanId: generationId,
parentSpanId: traceId,
type: "GENERATION",
promptName: "Capital guesser",
promptVersion: "1",
modelName: "gpt-4.1-2025-04-14",
});
const rootEventRecord = await ingestionService.createEventRecord(
rootEventInput!,
"",
);
const rootObservation =
convertEventRecordToObservationForEval(rootEventRecord);
expect(rootObservation.span_id).toBe(traceId);
expect(rootObservation.experiment_item_root_span_id).toBe(traceId);
expect(rootObservation.type).toBe("SPAN");
expect(rootObservation.name).toBe("dataset-run-item-008e4");
expect(rootObservation.experiment_id).toBe("run-456");
expect(rootObservation.experiment_name).toBe(
"Prompt Capital guesser-v1 on dataset countries - 2026-01-31T06:57:38.646Z",
);
expect(rootObservation.experiment_item_expected_output).toBe(
JSON.stringify({ capital: "Berlin" }),
);
});
it("should build non-experiment internal traces without experiment columns", () => {
const { eventInputs } = buildInternalTraceEventInputs({
processedEvents: getProcessedExperimentEvents(),
traceId,
projectId: "project-123",
});
const rootEventInput = eventInputs.find(
(eventInput) => eventInput.spanId === traceId,
);
expect(rootEventInput).toMatchObject({
spanId: traceId,
source: "ingestion-api-dual-write",
experimentId: undefined,
experimentItemRootSpanId: undefined,
});
});
});
describe("scheduleExperimentObservationEvals", () => {
beforeEach(() => {
vi.clearAllMocks();
mockFetchObservationEvalRules.mockResolvedValue([{ id: "config-1" }]);
mockCreateObservationEvalSchedulerDeps.mockReturnValue({
uploadObservationToS3: vi.fn(),
upsertJobExecution: vi.fn(),
enqueueEvalJob: vi.fn(),
});
mockScheduleObservationEvals.mockResolvedValue(undefined);
});
it("should schedule evals against the trace-root observation", async () => {
const { eventInputs } = buildInternalTraceEventInputs({
processedEvents: getProcessedExperimentEvents(),
traceId,
projectId: "project-123",
experimentContext: getExperimentContext(),
});
const rootEventInput = eventInputs.find(
(eventInput) => eventInput.spanId === traceId,
);
const rootEventRecord = await ingestionService.createEventRecord(
rootEventInput!,
"",
);
const observation = convertEventRecordToObservationForEval(rootEventRecord);
await scheduleExperimentObservationEvals({ observation });
expect(mockFetchObservationEvalRules).toHaveBeenCalledWith("project-123");
expect(mockScheduleObservationEvals).toHaveBeenCalledWith({
observation,
configs: [{ id: "config-1" }],
schedulerDeps: expect.any(Object),
});
expect(observation.span_id).toBe(traceId);
expect(observation.experiment_item_root_span_id).toBe(traceId);
});
it("should skip scheduling when no configs exist", async () => {
mockFetchObservationEvalRules.mockResolvedValue([]);
const observation = {
span_id: traceId,
trace_id: traceId,
project_id: "project-123",
type: "SPAN",
name: "dataset-run-item-008e4",
environment: LangfuseInternalTraceEnvironment.PromptExperiments,
level: "DEFAULT",
experiment_item_root_span_id: traceId,
tags: [],
provided_usage_details: {},
provided_cost_details: {},
usage_details: {},
cost_details: {},
tool_definitions: {},
tool_calls: [],
tool_call_names: [],
} as ObservationForEval;
await scheduleExperimentObservationEvals({ observation });
expect(mockScheduleObservationEvals).not.toHaveBeenCalled();
});
});