🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import {
ApiError,
LLMAdapter,
ObservationType,
variableMappingList,
EvalTargetObject,
compileTemplateString,
} from "@langfuse/shared";
import { encrypt } from "@langfuse/shared/encryption";
import { Prisma, prisma } from "@langfuse/shared/src/db";
import {
convertDateToClickhouseDateTime,
createObservation,
createObservationsCh,
createTrace,
createTracesCh,
upsertObservation,
upsertTrace,
createDatasetRunItemsCh,
createDatasetRunItem,
createOrgProjectAndApiKey,
LangfuseInternalTraceEnvironment,
} from "@langfuse/shared/src/server";
import { randomUUID } from "crypto";
import Decimal from "decimal.js";
import { afterEach } from "node:test";
import { afterAll, beforeAll, describe, expect, test, vi } from "vitest";
import { OpenAIServer } from "./network";
import {
createEvalJobs,
evaluate,
extractVariablesFromTracingData,
} from "../features/evaluation/evalService";
import { requiresDatabaseLookup } from "../features/evaluation/traceFilterUtils";
// Mock the shared evaluator model call with default passthrough behavior.
vi.mock("@langfuse/shared/src/server", async () => {
const actual = await vi.importActual<
typeof import("@langfuse/shared/src/server")
>("@langfuse/shared/src/server");
return {
...actual,
generateLlmEvaluatorOutput: vi.fn(actual.generateLlmEvaluatorOutput),
};
});
// Import the mocked function
import {
EvalExecutionQueue,
generateLlmEvaluatorOutput,
} from "@langfuse/shared/src/server";
import { UnrecoverableError } from "../errors/UnrecoverableError";
import {
EvalFailureReason,
UnrecoverableEvalFailureError,
} from "../features/evaluation/evalFailureErrors";
let OPENAI_API_KEY = process.env.OPENAI_API_KEY;
// Check for both OPENAI_API_KEY and LANGFUSE_LLM_CONNECTION_OPENAI_KEY
// to avoid interfering with llmConnections tests that use the latter
const hasActiveKey = Boolean(
OPENAI_API_KEY || process.env.LANGFUSE_LLM_CONNECTION_OPENAI_KEY,
);
if (!hasActiveKey) {
OPENAI_API_KEY = "sk-test_not_used_as_network_mocks_are_activated";
}
const openAIServer = new OpenAIServer({
hasActiveKey,
useDefaultResponse: false,
});
const jobTimestamp = new Date();
async function createMigratedEvalConfig(
args: Parameters<typeof prisma.jobConfiguration.create>[0],
) {
const config = await prisma.jobConfiguration.create(args);
const template = config.evalTemplateId
? await prisma.evalTemplate.findUnique({
where: { id: config.evalTemplateId },
})
: null;
const evaluator = await prisma.evaluator.create({
data: {
projectId: config.projectId,
name: config.scoreName,
type: "LLM_AS_JUDGE",
blockedAt: config.blockedAt,
blockReason: config.blockReason,
blockMessage: config.blockMessage,
versions: {
create: {
version: 1,
prompt: template?.prompt,
partner: template?.partner,
model: template?.model,
provider: template?.provider,
modelParams: template?.modelParams ?? undefined,
vars: template?.vars,
variableMapping: config.variableMapping,
outputDefinition: template?.outputDefinition ?? undefined,
sourceCode: template?.sourceCode,
sourceCodeLanguage: template?.sourceCodeLanguage,
},
},
},
});
await prisma.evaluationRule.create({
data: {
id: config.id,
projectId: config.projectId,
name: config.scoreName,
status: config.status,
targetObject: config.targetObject,
filter: config.filter as Prisma.InputJsonValue,
sampling: config.sampling,
delay: config.delay,
timeScope: config.timeScope,
assignments: {
create: {
projectId: config.projectId,
evaluatorId: evaluator.id,
variableMapping: config.variableMapping as Prisma.InputJsonValue,
},
},
},
});
return config;
}
async function createMigratedEvalConfigs(
args: Parameters<typeof prisma.jobConfiguration.createMany>[0],
) {
const configs = Array.isArray(args.data) ? args.data : [args.data];
await Promise.all(configs.map((data) => createMigratedEvalConfig({ data })));
}
beforeAll(openAIServer.setup);
beforeAll(async () => {
openAIServer.respondWithDefault();
});
afterEach(openAIServer.reset);
afterAll(openAIServer.teardown);
describe("eval service tests", () => {
describe("compile prompt", () => {
test("compile template string with basic variables", async () => {
const template = "Please evaluate toxicity {{input}} {{output}}";
const compiledString = compileTemplateString(template, {
input: "foo",
output: "bar",
});
expect(compiledString).toBe("Please evaluate toxicity foo bar");
});
test("compile template string with single variable", async () => {
const template = "Hello {{name}}!";
const compiledString = compileTemplateString(template, {
name: "World",
});
expect(compiledString).toBe("Hello World!");
});
test("compile template string with no variables", async () => {
const template = "This is a plain string without variables";
const compiledString = compileTemplateString(template, {
input: "foo",
});
expect(compiledString).toBe("This is a plain string without variables");
});
test("compile template string with empty context keeps original variables", async () => {
const template = "Hello {{name}}!";
const compiledString = compileTemplateString(template, {});
expect(compiledString).toBe("Hello {{name}}!");
});
test("compile template string with missing variable keeps original variable", async () => {
const template = "Hello {{name}}, your score is {{score}}";
const compiledString = compileTemplateString(template, {
name: "Alice",
});
expect(compiledString).toBe("Hello Alice, your score is {{score}}");
});
test("compile template string with all variables missing keeps all original", async () => {
const template = "{{greeting}} {{name}}, welcome to {{place}}!";
const compiledString = compileTemplateString(template, {});
expect(compiledString).toBe(
"{{greeting}} {{name}}, welcome to {{place}}!",
);
});
test("compile template string with whitespace variable missing keeps original with whitespace", async () => {
const template = "Hello {{ name }}!";
const compiledString = compileTemplateString(template, {});
expect(compiledString).toBe("Hello {{ name }}!");
});
test("compile template string with null value returns empty string", async () => {
const template = "Value: {{value}}";
const compiledString = compileTemplateString(template, {
value: null,
});
expect(compiledString).toBe("Value: ");
});
test("compile template string with undefined value returns empty string", async () => {
const template = "Value: {{value}}";
const compiledString = compileTemplateString(template, {
value: undefined,
});
expect(compiledString).toBe("Value: ");
});
test("compile template string with numeric value", async () => {
const template = "Score: {{score}}";
const compiledString = compileTemplateString(template, {
score: 42,
});
expect(compiledString).toBe("Score: 42");
});
test("compile template string with zero value", async () => {
const template = "Count: {{count}}";
const compiledString = compileTemplateString(template, {
count: 0,
});
expect(compiledString).toBe("Count: 0");
});
test("compile template string with boolean true value", async () => {
const template = "Active: {{active}}";
const compiledString = compileTemplateString(template, {
active: true,
});
expect(compiledString).toBe("Active: true");
});
test("compile template string with boolean false value", async () => {
const template = "Active: {{active}}";
const compiledString = compileTemplateString(template, {
active: false,
});
expect(compiledString).toBe("Active: false");
});
test("compile template string with object value stringifies object", async () => {
const template = "Data: {{data}}";
const compiledString = compileTemplateString(template, {
data: { key: "value" },
});
expect(compiledString).toBe("Data: [object Object]");
});
test("compile template string with array value stringifies array", async () => {
const template = "Items: {{items}}";
const compiledString = compileTemplateString(template, {
items: [1, 2, 3],
});
expect(compiledString).toBe("Items: 1,2,3");
});
test("compile template string with whitespace around variable name", async () => {
const template = "Hello {{ name }}!";
const compiledString = compileTemplateString(template, {
name: "World",
});
expect(compiledString).toBe("Hello World!");
});
test("compile template string with extra whitespace around variable name", async () => {
const template = "Hello {{ name }}!";
const compiledString = compileTemplateString(template, {
name: "World",
});
expect(compiledString).toBe("Hello World!");
});
test("compile template string with tabs around variable name", async () => {
const template = "Hello {{\tname\t}}!";
const compiledString = compileTemplateString(template, {
name: "World",
});
expect(compiledString).toBe("Hello World!");
});
test("compile template string with mixed whitespace around variable name", async () => {
const template = "Hello {{ \t name \t }}!";
const compiledString = compileTemplateString(template, {
name: "World",
});
expect(compiledString).toBe("Hello World!");
});
test("compile template string with newline inside", async () => {
const template = "Hello {{\nname\n}}!";
const compiledString = compileTemplateString(template, {
name: "World",
});
// Newlines are not matched by \s* in the regex pattern (which only matches spaces)
expect(compiledString).toBe("Hello World!");
});
test("compile template string with leading whitespace only", async () => {
const template = "Hello {{ name}}!";
const compiledString = compileTemplateString(template, {
name: "World",
});
expect(compiledString).toBe("Hello World!");
});
test("compile template string with trailing whitespace only", async () => {
const template = "Hello {{name }}!";
const compiledString = compileTemplateString(template, {
name: "World",
});
expect(compiledString).toBe("Hello World!");
});
test("compile template string with excessive whitespace", async () => {
const template = "Hello {{ name }}!";
const compiledString = compileTemplateString(template, {
name: "World",
});
expect(compiledString).toBe("Hello World!");
});
test("compile template string whitespace variations with multiple variables", async () => {
const template = "{{ input }} and {{output}} and {{ expected }}";
const compiledString = compileTemplateString(template, {
input: "A",
output: "B",
expected: "C",
});
expect(compiledString).toBe("A and B and C");
});
test("compile template string with multiple occurrences of same variable", async () => {
const template = "{{name}} said hello to {{name}}";
const compiledString = compileTemplateString(template, {
name: "Alice",
});
expect(compiledString).toBe("Alice said hello to Alice");
});
test("compile template string with special characters in value", async () => {
const template = "Query: {{query}}";
const compiledString = compileTemplateString(template, {
query: "SELECT * FROM users WHERE name = 'test'",
});
expect(compiledString).toBe(
"Query: SELECT * FROM users WHERE name = 'test'",
);
});
test("compile template string with HTML in value (no escaping)", async () => {
const template = "Content: {{content}}";
const compiledString = compileTemplateString(template, {
content: "<script>alert('xss')</script>",
});
expect(compiledString).toBe("Content: <script>alert('xss')</script>");
});
test("compile template string with newlines in value", async () => {
const template = "Message: {{message}}";
const compiledString = compileTemplateString(template, {
message: "Line 1\nLine 2\nLine 3",
});
expect(compiledString).toBe("Message: Line 1\nLine 2\nLine 3");
});
test("compile template string with empty string value", async () => {
const template = "Name: {{name}}";
const compiledString = compileTemplateString(template, {
name: "",
});
expect(compiledString).toBe("Name: ");
});
test("compile template string with JSON string value", async () => {
const template = "Data: {{data}}";
const jsonData = JSON.stringify({ input: "test", output: "result" });
const compiledString = compileTemplateString(template, {
data: jsonData,
});
expect(compiledString).toBe('Data: {"input":"test","output":"result"}');
});
test("compile template string with underscore in variable name", async () => {
const template = "Value: {{my_variable}}";
const compiledString = compileTemplateString(template, {
my_variable: "test",
});
expect(compiledString).toBe("Value: test");
});
test("compile template string with numbers in variable name", async () => {
const template = "Value: {{var1}}";
const compiledString = compileTemplateString(template, {
var1: "test",
});
expect(compiledString).toBe("Value: test");
});
test("compile template string with dot notation in variable name", async () => {
const template = "Value: {{user.name}}";
const compiledString = compileTemplateString(template, {
"user.name": "Alice",
});
expect(compiledString).toBe("Value: Alice");
});
test("compile template string preserves curly braces that are not variables", async () => {
const template = "JSON: {key: value} and {{variable}}";
const compiledString = compileTemplateString(template, {
variable: "test",
});
expect(compiledString).toBe("JSON: {key: value} and test");
});
test("compile template string with triple braces preserves outer braces", async () => {
const template = "{{{variable}}}";
const compiledString = compileTemplateString(template, {
variable: "test",
});
expect(compiledString).toBe("{test}");
});
test("compile template string with complex LLM eval prompt", async () => {
const template = `You are an expert evaluator. Please evaluate the following:
Input: {{input}}
Output: {{output}}
Expected: {{expected}}
Provide a score from 0 to 1 based on correctness.`;
const compiledString = compileTemplateString(template, {
input: "What is 2+2?",
output: "4",
expected: "4",
});
expect(compiledString)
.toBe(`You are an expert evaluator. Please evaluate the following:
Input: What is 2+2?
Output: 4
Expected: 4
Provide a score from 0 to 1 based on correctness.`);
});
test("compile template string with very long value", async () => {
const template = "Content: {{content}}";
const longValue = "a".repeat(10000);
const compiledString = compileTemplateString(template, {
content: longValue,
});
expect(compiledString).toBe(`Content: ${longValue}`);
});
test("compile template string with unicode characters in value", async () => {
const template = "Message: {{message}}";
const compiledString = compileTemplateString(template, {
message: "Hello 世界 🌍 مرحبا",
});
expect(compiledString).toBe("Message: Hello 世界 🌍 مرحبا");
});
test("compile template string with unicode characters in template", async () => {
const template = "你好 {{name}} 🎉";
const compiledString = compileTemplateString(template, {
name: "World",
});
expect(compiledString).toBe("你好 World 🎉");
});
test("compile template string returns original on regex error", async () => {
// This test verifies error handling - the function should gracefully handle errors
const template = "Test {{var}}";
// Normal context should work
const compiledString = compileTemplateString(template, {
var: "value",
});
expect(compiledString).toBe("Test value");
});
test("compile template string with many variables", async () => {
const template =
"{{a}} {{b}} {{c}} {{d}} {{e}} {{f}} {{g}} {{h}} {{i}} {{j}}";
const compiledString = compileTemplateString(template, {
a: "1",
b: "2",
c: "3",
d: "4",
e: "5",
f: "6",
g: "7",
h: "8",
i: "9",
j: "10",
});
expect(compiledString).toBe("1 2 3 4 5 6 7 8 9 10");
});
test("compile template string with adjacent variables", async () => {
const template = "{{first}}{{second}}{{third}}";
const compiledString = compileTemplateString(template, {
first: "A",
second: "B",
third: "C",
});
expect(compiledString).toBe("ABC");
});
test("compile template string with variable at start", async () => {
const template = "{{greeting}} there!";
const compiledString = compileTemplateString(template, {
greeting: "Hello",
});
expect(compiledString).toBe("Hello there!");
});
test("compile template string with variable at end", async () => {
const template = "Hello {{name}}";
const compiledString = compileTemplateString(template, {
name: "World",
});
expect(compiledString).toBe("Hello World");
});
test("compile template string with only variable", async () => {
const template = "{{value}}";
const compiledString = compileTemplateString(template, {
value: "Just this",
});
expect(compiledString).toBe("Just this");
});
test("compile template string with empty template", async () => {
const template = "";
const compiledString = compileTemplateString(template, {
value: "test",
});
expect(compiledString).toBe("");
});
test("compile template string with float value", async () => {
const template = "Score: {{score}}";
const compiledString = compileTemplateString(template, {
score: 0.95,
});
expect(compiledString).toBe("Score: 0.95");
});
test("compile template string with negative number", async () => {
const template = "Temperature: {{temp}}";
const compiledString = compileTemplateString(template, {
temp: -15,
});
expect(compiledString).toBe("Temperature: -15");
});
test("compile template string with Infinity value", async () => {
const template = "Value: {{val}}";
const compiledString = compileTemplateString(template, {
val: Infinity,
});
expect(compiledString).toBe("Value: Infinity");
});
test("compile template string with NaN value", async () => {
const template = "Value: {{val}}";
const compiledString = compileTemplateString(template, {
val: NaN,
});
expect(compiledString).toBe("Value: NaN");
});
test("compile template string escapes regex special characters in values", async () => {
const template = "Pattern: {{pattern}}";
const compiledString = compileTemplateString(template, {
pattern: ".*+?^${}()|[]\\",
});
expect(compiledString).toBe("Pattern: .*+?^${}()|[]\\");
});
test("compile template string with backslash in value", async () => {
const template = "Path: {{path}}";
const compiledString = compileTemplateString(template, {
path: "C:\\Users\\test",
});
expect(compiledString).toBe("Path: C:\\Users\\test");
});
test("compile template string with dollar sign in value", async () => {
const template = "Price: {{price}}";
const compiledString = compileTemplateString(template, {
price: "$100",
});
expect(compiledString).toBe("Price: $100");
});
test("compile template string does not process nested braces as variables", async () => {
const template = "Code: {{ {{nested}} }}";
const compiledString = compileTemplateString(template, {
nested: "value",
});
// The outer braces should match the inner {{nested}} first
expect(compiledString).toBe("Code: {{ value }}");
});
test("compile template string handles real-world eval template", async () => {
const template = `You are evaluating an AI assistant's response.
## Task
Evaluate how well the assistant's response matches the expected output.
## Input
The user asked: {{input}}
## Assistant's Response
{{output}}
## Expected Response
{{expected_output}}
## Instructions
1. Compare the assistant's response with the expected output
2. Consider semantic similarity, not just exact matching
3. Provide a score from 0.0 to 1.0
Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;
const compiledString = compileTemplateString(template, {
input: "What is the capital of France?",
output: "The capital of France is Paris.",
expected_output: "Paris is the capital of France.",
});
expect(compiledString).toContain("What is the capital of France?");
expect(compiledString).toContain("The capital of France is Paris.");
expect(compiledString).toContain("Paris is the capital of France.");
expect(compiledString).not.toContain("{{input}}");
expect(compiledString).not.toContain("{{output}}");
expect(compiledString).not.toContain("{{expected_output}}");
});
});
describe("create eval jobs", () => {
test("schedules a migrated rule only once when its legacy row still exists", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const configId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await prisma.jobConfiguration.create({
data: {
id: configId,
projectId,
filter: [],
jobType: "EVAL",
delay: 0,
sampling: new Decimal(1),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: [],
},
});
const evaluator = await prisma.evaluator.create({
data: {
projectId,
name: "score",
type: "LLM_AS_JUDGE",
versions: { create: { version: 1 } },
},
});
await prisma.evaluationRule.create({
data: {
id: configId,
projectId,
name: "score",
filter: [],
sampling: new Decimal(1),
delay: 0,
targetObject: EvalTargetObject.TRACE,
assignments: {
create: {
projectId,
evaluatorId: evaluator.id,
},
},
},
});
await createEvalJobs({
sourceEventType: "trace-upsert",
event: { projectId, traceId },
jobTimestamp,
});
await expect(
prisma.jobExecution.count({ where: { projectId } }),
).resolves.toBe(1);
});
test("creates new 'trace' eval job", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
});
const payload = {
projectId,
traceId: traceId,
};
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
});
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(1);
expect(jobs[0].projectId).toBe(projectId);
expect(jobs[0].jobInputTraceId).toBe(traceId);
expect(jobs[0].status.toString()).toBe("PENDING");
expect(jobs[0].startTime).not.toBeNull();
}, 10_000);
test("creates a single eval job when two producers race for the same trace", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
});
const payload = { projectId, traceId };
// Two shard workers observing the same trace concurrently. Both pass the
// dedup existence check before either inserts.
await Promise.all([
createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
}),
createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
}),
]);
const jobs = await prisma.jobExecution.findMany({ where: { projectId } });
expect(jobs.length).toBe(1);
}, 10_000);
test("does not leave a PENDING eval job behind when enqueueing fails", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
});
const getInstanceSpy = vi
.spyOn(EvalExecutionQueue, "getInstance")
.mockReturnValue({
add: vi.fn().mockRejectedValue(new Error("redis is down")),
} as never);
try {
await expect(
createEvalJobs({
sourceEventType: "trace-upsert",
event: { projectId, traceId },
jobTimestamp,
}),
).rejects.toThrow("redis is down");
} finally {
getInstanceSpy.mockRestore();
}
// The row must be gone, otherwise the BullMQ redelivery hits the dedup
// check and the execution stays PENDING forever.
await expect(
prisma.jobExecution.count({ where: { projectId } }),
).resolves.toBe(0);
// The retry recreates and enqueues it.
await createEvalJobs({
sourceEventType: "trace-upsert",
event: { projectId, traceId },
jobTimestamp,
});
await expect(
prisma.jobExecution.count({ where: { projectId } }),
).resolves.toBe(1);
}, 10_000);
test("creates new 'dataset' eval job", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const observationId = randomUUID();
const datasetId = randomUUID();
const datasetItemId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await upsertObservation({
id: observationId,
trace_id: traceId,
project_id: projectId,
type: "GENERATION",
start_time: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await prisma.dataset.create({
data: {
id: datasetId,
projectId,
name: "test-dataset",
},
});
await prisma.datasetItem.create({
data: {
id: datasetItemId,
projectId,
datasetId,
sourceTraceId: traceId,
sourceObservationId: observationId,
},
});
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.DATASET,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
});
const payload = {
projectId,
traceId: traceId,
datasetItemId: datasetItemId,
observationId: observationId,
};
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
});
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(1);
expect(jobs[0].projectId).toBe(projectId);
expect(jobs[0].jobInputTraceId).toBe(traceId);
expect(jobs[0].jobInputObservationId).toBe(observationId);
expect(jobs[0].jobInputDatasetItemId).toBe(datasetItemId);
expect(jobs[0].status.toString()).toBe("PENDING");
expect(jobs[0].startTime).not.toBeNull();
}, 10_000);
test("creates new eval job for a dataset on upsert of the trace", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const datasetId = randomUUID();
const datasetItemId = randomUUID();
const datasetRunId = randomUUID();
await prisma.dataset.create({
data: {
id: datasetId,
projectId,
name: "test-dataset",
},
});
await prisma.datasetItem.create({
data: {
id: datasetItemId,
projectId,
datasetId,
},
});
await prisma.datasetRuns.create({
data: {
id: datasetRunId,
name: randomUUID(),
datasetId,
projectId,
},
});
// Create a clickhouse run item
await createDatasetRunItemsCh([
createDatasetRunItem({
project_id: projectId,
dataset_id: datasetId,
dataset_run_id: datasetRunId,
dataset_item_id: datasetItemId,
trace_id: traceId,
}),
]);
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.DATASET,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
});
const payloadDataset = {
projectId,
traceId: traceId,
datasetItemId: datasetItemId,
};
// This should exit early without an error as there is no trace yet.
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payloadDataset,
jobTimestamp,
});
const jobsAfterDataset = await prisma.jobExecution.findMany({
where: { projectId },
});
// No jobs should have been created.
expect(jobsAfterDataset.length).toBe(0);
// Now upsert the trace and validate that the job was created.
await upsertTrace({
id: traceId,
project_id: projectId,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
const payloadTrace = {
projectId,
traceId: traceId,
};
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payloadTrace,
jobTimestamp,
});
const jobsAfterTrace = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobsAfterTrace.length).toBe(1);
expect(jobsAfterTrace[0].projectId).toBe(projectId);
expect(jobsAfterTrace[0].jobInputTraceId).toBe(traceId);
expect(jobsAfterTrace[0].jobInputDatasetItemId).toBe(datasetItemId);
expect(jobsAfterTrace[0].status.toString()).toBe("PENDING");
expect(jobsAfterTrace[0].startTime).not.toBeNull();
}, 10_000);
test("creates a new eval job for a dataset only if trace _and_ dataset are available", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const datasetId = randomUUID();
const datasetItemId = randomUUID();
await prisma.dataset.create({
data: {
id: datasetId,
projectId,
name: "test-dataset",
},
});
// Create the trace and send the trace event. No job should be created
await upsertTrace({
id: traceId,
project_id: projectId,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.DATASET,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
});
const payloadTrace = {
projectId,
traceId,
};
// This should exit early without an error as there is no trace yet.
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payloadTrace,
jobTimestamp,
});
const jobsAfterDataset = await prisma.jobExecution.findMany({
where: { projectId },
});
// No jobs should have been created.
expect(jobsAfterDataset.length).toBe(0);
// Now, create the dataset item and validate that the job was created.
await prisma.datasetItem.create({
data: {
id: datasetItemId,
projectId,
datasetId,
sourceTraceId: traceId,
},
});
const payloadDataset = {
projectId,
traceId,
datasetItemId,
};
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payloadDataset,
jobTimestamp,
});
const jobsAfterTrace = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobsAfterTrace.length).toBe(1);
expect(jobsAfterTrace[0].projectId).toBe(projectId);
expect(jobsAfterTrace[0].jobInputTraceId).toBe(traceId);
expect(jobsAfterTrace[0].jobInputDatasetItemId).toBe(datasetItemId);
expect(jobsAfterTrace[0].status.toString()).toBe("PENDING");
expect(jobsAfterTrace[0].startTime).not.toBeNull();
}, 10_000);
test("does not create job for 'event' config", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.EVENT,
scoreName: "score",
variableMapping: JSON.parse("[]"),
status: "INACTIVE",
},
});
const payload = {
projectId,
traceId: traceId,
};
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
});
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(0);
}, 10_000);
test("does not create job for 'experiment' config", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.EXPERIMENT,
scoreName: "score",
variableMapping: JSON.parse("[]"),
status: "INACTIVE",
},
});
const payload = {
projectId,
traceId: traceId,
};
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
});
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(0);
}, 10_000);
test("does not create job for inactive config", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
status: "INACTIVE",
},
});
const payload = {
projectId,
traceId: traceId,
};
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
});
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(0);
}, 10_000);
test("does not create eval job for existing job execution", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await prisma.llmApiKeys.create({
data: {
id: randomUUID(),
projectId,
secretKey: encrypt(String(OPENAI_API_KEY)),
provider: "openai",
adapter: LLMAdapter.OpenAI,
customModels: [],
displaySecretKey: "123456",
},
});
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
});
const payload = {
projectId,
traceId: traceId,
};
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
});
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
}); // calling it twice to check it is only generated once
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(1);
expect(jobs[0].projectId).toBe(projectId);
expect(jobs[0].jobInputTraceId).toBe(traceId);
expect(jobs[0].status.toString()).toBe("PENDING");
expect(jobs[0].startTime).not.toBeNull();
expect(jobs[0].endTime).to.be.null;
}, 10_000);
test("does not create eval job for 0 sample rate", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await prisma.llmApiKeys.create({
data: {
id: randomUUID(),
projectId,
secretKey: encrypt(String(OPENAI_API_KEY)),
provider: "openai",
adapter: LLMAdapter.OpenAI,
customModels: [],
displaySecretKey: "123456",
},
});
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("0"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
});
const payload = {
projectId,
traceId: traceId,
};
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
});
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(0);
}, 10_000);
test("deterministically creates nested samples across configs", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = "trace-456";
const sampledOutConfigId = randomUUID();
const sampledInConfigId = randomUUID();
await createMigratedEvalConfigs({
data: [
{
id: sampledOutConfigId,
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("0.5"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
{
id: sampledInConfigId,
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("0.9"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
],
});
await createEvalJobs({
sourceEventType: "trace-upsert",
event: { projectId, traceId },
jobTimestamp,
});
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs).toHaveLength(1);
expect(jobs[0].jobConfigurationId).toBe(sampledInConfigId);
}, 10_000);
test("does not create eval job for existing traces if time scope is EXISTING but handler enforces NEW only", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const trace = createTrace({
project_id: projectId,
id: traceId,
});
await createTracesCh([trace]);
const jobConfiguration = await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
timeScope: ["EXISTING"],
},
});
// this one should not be selected for eval as it was not provided via the event.
const jobConfiguration2 = await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
timeScope: ["NEW"],
},
});
const payload = {
projectId,
traceId: traceId,
configId: jobConfiguration.id,
};
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
enforcedJobTimeScope: "NEW", // the config must contain NEW
});
const jobs = await prisma.jobExecution.findMany({
where: {
projectId,
jobConfigurationId: {
in: [jobConfiguration.id, jobConfiguration2.id],
},
jobInputTraceId: traceId,
},
});
expect(jobs.length).toBe(0);
}, 10_000);
test("does create eval for trace which is way in the past if timestamp is provided", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const timestamp = new Date(Date.now() - 1000 * 60 * 60 * 24 * 365 * 1);
const trace = createTrace({
project_id: projectId,
id: traceId,
timestamp: timestamp.getTime(),
});
await createTracesCh([trace]);
const jobConfiguration = await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
timeScope: ["NEW"],
},
});
const payload = {
projectId,
traceId: traceId,
configId: jobConfiguration.id,
timestamp: timestamp,
};
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
enforcedJobTimeScope: "NEW", // the config must contain NEW
});
const jobs = await prisma.jobExecution.findMany({
where: {
projectId,
jobConfigurationId: { in: [jobConfiguration.id] },
jobInputTraceId: traceId,
},
});
expect(jobs.length).toBe(1);
}, 10_000);
test("create eval for trace with timestamp in the near future", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
timeScope: ["NEW"],
},
});
const trace = createTrace({
project_id: projectId,
id: traceId,
timestamp: new Date(Date.now() + 1000 * 60 * 60 * 24).getTime(),
});
await createTracesCh([trace]);
await createEvalJobs({
sourceEventType: "trace-upsert",
event: {
projectId,
traceId: traceId,
},
jobTimestamp,
});
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(1);
}, 10_000);
test("creates dataset eval job with cached dataset item filtering - positive match", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const datasetId1 = randomUUID();
const datasetId2 = randomUUID();
const datasetId3 = randomUUID();
const datasetRunId = randomUUID();
const datasetItemId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
// Create three datasets
await prisma.dataset.createMany({
data: [
{
id: datasetId1,
projectId,
name: "dataset-alpha",
},
{
id: datasetId2,
projectId,
name: "dataset-beta",
},
{
id: datasetId3,
projectId,
name: "dataset-gamma",
},
],
});
// Create dataset item that matches the second dataset filter
await prisma.datasetItem.create({
data: {
id: datasetItemId,
projectId,
datasetId: datasetId2,
sourceTraceId: traceId,
},
});
// Used if ClickHouse reads are disabled.
await prisma.datasetRuns.create({
data: {
id: datasetRunId,
projectId,
name: randomUUID(),
datasetId: datasetId2,
},
});
// Create a clickhouse run item that references dataset 2 and the new trace.
await createDatasetRunItemsCh([
createDatasetRunItem({
project_id: projectId,
dataset_id: datasetId2,
dataset_item_id: datasetItemId,
trace_id: traceId,
}),
]);
// Create three job configurations, each filtering for a specific dataset
await createMigratedEvalConfigs({
data: [
{
id: randomUUID(),
projectId,
filter: [
{
type: "stringOptions",
value: [datasetId1],
column: "Dataset",
operator: "any of",
},
],
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.DATASET,
scoreName: "score-alpha",
variableMapping: JSON.parse("[]"),
},
{
id: randomUUID(),
projectId,
filter: [
{
type: "stringOptions",
value: [datasetId2],
column: "Dataset",
operator: "any of",
},
],
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.DATASET,
scoreName: "score-beta",
variableMapping: JSON.parse("[]"),
},
{
id: randomUUID(),
projectId,
filter: [
{
type: "stringOptions",
value: [datasetId3],
column: "Dataset",
operator: "any of",
},
],
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.DATASET,
scoreName: "score-gamma",
variableMapping: JSON.parse("[]"),
},
],
});
await createEvalJobs({
sourceEventType: "trace-upsert",
event: {
projectId,
traceId,
},
jobTimestamp,
});
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
// Should create exactly one job for the matching dataset (dataset-beta)
expect(jobs.length).toBe(1);
expect(jobs[0].projectId).toBe(projectId);
expect(jobs[0].jobInputTraceId).toBe(traceId);
expect(jobs[0].jobInputDatasetItemId).toBe(datasetItemId);
expect(jobs[0].status.toString()).toBe("PENDING");
// Verify it's the correct config by checking the score name
const config = await prisma.jobConfiguration.findFirstOrThrow({
select: { scoreName: true },
where: { id: jobs[0].jobConfigurationId },
});
expect(config.scoreName).toBe("score-beta");
}, 10_000);
test("creates no dataset eval jobs with cached dataset item filtering - negative match", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const datasetId1 = randomUUID();
const datasetId2 = randomUUID();
const datasetItemId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
// Create four datasets
await prisma.dataset.createMany({
data: [
{
id: datasetId1,
projectId,
name: "dataset-alpha",
},
{
id: datasetId2,
projectId,
name: "dataset-beta",
},
],
});
// Create dataset item that matches none of the config filters (dataset-delta)
await prisma.datasetItem.create({
data: {
id: datasetItemId,
projectId,
datasetId: datasetId1,
sourceTraceId: traceId,
},
});
// Create a clickhouse run item that references a non-existing dataset and the new trace.
await createDatasetRunItemsCh([
createDatasetRunItem({
dataset_id: randomUUID(),
dataset_item_id: datasetItemId,
trace_id: traceId,
}),
]);
// Create three job configurations, each filtering for specific datasets (but not delta)
await createMigratedEvalConfigs({
data: [
{
id: randomUUID(),
projectId,
filter: [
{
type: "stringOptions",
value: [datasetId1],
column: "Dataset",
operator: "any of",
},
],
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.DATASET,
scoreName: "score-alpha",
variableMapping: JSON.parse("[]"),
},
{
id: randomUUID(),
projectId,
filter: [
{
type: "stringOptions",
value: [datasetId2],
column: "Dataset",
operator: "any of",
},
],
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.DATASET,
scoreName: "score-beta",
variableMapping: JSON.parse("[]"),
},
],
});
await createEvalJobs({
sourceEventType: "trace-upsert",
event: {
projectId,
traceId,
},
jobTimestamp,
});
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
// Should create no jobs since dataset-delta doesn't match any filter
expect(jobs.length).toBe(0);
}, 10_000);
});
describe("execute evals", () => {
test("evals a valid 'trace' event", async () => {
const { projectId } = await createOrgProjectAndApiKey();
openAIServer.respondWithDefault();
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
user_id: "a",
input: JSON.stringify({ input: "This is a great prompt" }),
output: JSON.stringify({ output: "This is a great response" }),
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
const templateId = randomUUID();
await prisma.evalTemplate.create({
data: {
id: templateId,
projectId,
name: "test-template",
version: 1,
prompt: "Please evaluate toxicity {{input}} {{output}}",
model: "gpt-3.5-turbo",
provider: "openai",
modelParams: {},
outputDefinition: {
reasoning: "Please explain your reasoning",
score: "Please provide a score between 0 and 1",
},
},
});
const jobConfiguration = await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: [
{
type: "string",
value: "a",
column: "User ID",
operator: "contains",
},
],
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
evalTemplateId: templateId,
},
});
const jobExecutionId = randomUUID();
await prisma.jobExecution.create({
data: {
id: jobExecutionId,
projectId,
jobConfigurationId: jobConfiguration.id,
status: "PENDING",
startTime: new Date(),
jobInputTraceId: traceId,
},
});
await prisma.llmApiKeys.create({
data: {
id: randomUUID(),
projectId,
secretKey: encrypt(String(OPENAI_API_KEY)),
provider: "openai",
adapter: LLMAdapter.OpenAI,
customModels: [],
displaySecretKey: "123456",
},
});
const payload = {
projectId,
jobExecutionId: jobExecutionId,
delay: 1000,
};
await evaluate({ event: payload });
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(1);
expect(jobs[0].projectId).toBe(projectId);
expect(jobs[0].jobInputTraceId).toBe(traceId);
expect(jobs[0].status.toString()).toBe("COMPLETED");
expect(jobs[0].startTime).not.toBeNull();
expect(jobs[0].endTime).not.toBeNull();
}, 50_000);
test("fails to eval without llm api key", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const templateId = randomUUID();
await prisma.evalTemplate.create({
data: {
id: templateId,
projectId,
name: "test-template",
version: 1,
prompt: "Please evaluate toxicity {{input}} {{output}}",
model: "gpt-3.5-turbo",
provider: "openai",
modelParams: {},
outputDefinition: {
reasoning: "Please explain your reasoning",
score: "Please provide a score between 0 and 1",
},
},
});
const jobConfiguration = await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: [
{
type: "string",
value: "a",
column: "User ID",
operator: "contains",
},
],
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
evalTemplateId: templateId,
},
});
const jobExecutionId = randomUUID();
await prisma.jobExecution.create({
data: {
id: jobExecutionId,
projectId,
jobConfigurationId: jobConfiguration.id,
status: "PENDING",
startTime: new Date(),
jobInputTraceId: traceId,
},
});
const payload = {
projectId,
jobExecutionId: jobExecutionId,
delay: 1000,
};
await expect(evaluate({ event: payload })).rejects.toThrowError(
new UnrecoverableEvalFailureError(
`Invalid model configuration for job ${jobExecutionId}: API key for provider "openai" not found in project ${projectId}`,
EvalFailureReason.InvalidModelConfig,
),
);
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(1);
expect(jobs[0].projectId).toBe(projectId);
expect(jobs[0].jobInputTraceId).toBe(traceId);
// the job will be failed when the exception is caught in the worker consumer
expect(jobs[0].status.toString()).toBe("PENDING");
}, 10_000);
test("fails to eval on openai error", async () => {
const { projectId } = await createOrgProjectAndApiKey();
openAIServer.respondWithError(401, "Not authorized");
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
user_id: "a",
input: JSON.stringify({ input: "This is a great prompt" }),
output: JSON.stringify({ output: "This is a great response" }),
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
const templateId = randomUUID();
await prisma.evalTemplate.create({
data: {
id: templateId,
projectId,
name: "test-template",
version: 1,
prompt: "Please evaluate toxicity {{input}} {{output}}",
model: "gpt-3.5-turbo",
provider: "openai",
modelParams: {},
outputDefinition: {
reasoning: "Please explain your reasoning",
score: "Please provide a score between 0 and 1",
},
},
});
const jobConfiguration = await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: [
{
type: "string",
value: "a",
column: "User ID",
operator: "contains",
},
],
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
evalTemplateId: templateId,
},
});
const jobExecutionId = randomUUID();
await prisma.jobExecution.create({
data: {
id: jobExecutionId,
projectId,
jobConfigurationId: jobConfiguration.id,
status: "PENDING",
startTime: new Date(),
jobInputTraceId: traceId,
},
});
await prisma.llmApiKeys.create({
data: {
id: randomUUID(),
projectId,
secretKey: encrypt(String(OPENAI_API_KEY)),
provider: "openai",
adapter: LLMAdapter.OpenAI,
customModels: [],
displaySecretKey: "123456",
},
});
const payload = {
projectId,
jobExecutionId: jobExecutionId,
delay: 1000,
};
await expect(evaluate({ event: payload })).rejects.toMatchObject({
name: "AI_APICallError",
statusCode: 401,
isRetryable: false,
});
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(1);
expect(jobs[0].projectId).toBe(projectId);
expect(jobs[0].jobInputTraceId).toBe(traceId);
// the job will be failed when the exception is caught in the worker consumer
expect(jobs[0].status.toString()).toBe("PENDING");
}, 10_000);
test("blocks the migrated evaluator for a job queued without v2 identity", async () => {
const { projectId } = await createOrgProjectAndApiKey();
openAIServer.respondWithError(401, "Not authorized");
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
user_id: "a",
input: JSON.stringify({ input: "This is a great prompt" }),
output: JSON.stringify({ output: "This is a great response" }),
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
const template = await prisma.evalTemplate.create({
data: {
projectId,
name: "migrated-template",
version: 1,
prompt: "Please evaluate toxicity {{input}} {{output}}",
model: "gpt-3.5-turbo",
provider: "openai",
modelParams: {},
vars: ["input", "output"],
outputDefinition: {
reasoning: "Please explain your reasoning",
score: "Please provide a score between 0 and 1",
},
},
});
const jobConfiguration = await createMigratedEvalConfig({
data: {
projectId,
filter: [],
jobType: "EVAL",
delay: 0,
sampling: new Decimal(1),
targetObject: EvalTargetObject.TRACE,
scoreName: "migrated-score",
variableMapping: [
{
templateVariable: "input",
langfuseObject: "trace",
selectedColumnId: "input",
},
{
templateVariable: "output",
langfuseObject: "trace",
selectedColumnId: "output",
},
],
evalTemplateId: template.id,
},
});
const assignment =
await prisma.evaluationRuleEvaluatorAssignment.findFirstOrThrow({
where: { evaluationRuleId: jobConfiguration.id },
include: { evaluator: { include: { versions: true } } },
});
await prisma.evaluatorVersion.update({
where: { id: assignment.evaluator.versions[0].id },
data: {
prompt: template.prompt,
model: template.model,
provider: template.provider,
modelParams: template.modelParams ?? Prisma.DbNull,
vars: template.vars,
variableMapping: jobConfiguration.variableMapping,
outputDefinition: template.outputDefinition ?? Prisma.DbNull,
},
});
const jobExecutionId = randomUUID();
await prisma.jobExecution.create({
data: {
id: jobExecutionId,
projectId,
jobConfigurationId: jobConfiguration.id,
status: "PENDING",
startTime: new Date(),
jobInputTraceId: traceId,
},
});
await prisma.llmApiKeys.create({
data: {
id: randomUUID(),
projectId,
secretKey: encrypt(String(OPENAI_API_KEY)),
provider: "openai",
adapter: LLMAdapter.OpenAI,
customModels: [],
displaySecretKey: "123456",
},
});
await expect(
evaluate({
event: { projectId, jobExecutionId, delay: 1000 },
}),
).rejects.toMatchObject({ statusCode: 401 });
await expect(
prisma.evaluator.findUniqueOrThrow({
where: { id: assignment.evaluatorId },
select: { blockedAt: true, blockReason: true },
}),
).resolves.toMatchObject({
blockedAt: expect.any(Date),
blockReason: "LLM_CONNECTION_AUTH_INVALID",
});
await expect(
prisma.jobConfiguration.findUniqueOrThrow({
where: { id: jobConfiguration.id },
select: { blockedAt: true },
}),
).resolves.toEqual({ blockedAt: null });
}, 10_000);
test("evals should cancel if job is cancelled", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const templateId = randomUUID();
await prisma.evalTemplate.create({
data: {
id: templateId,
projectId,
name: "test-template",
version: 1,
prompt: "Please evaluate toxicity {{input}} {{output}}",
model: "gpt-3.5-turbo",
provider: "openai",
modelParams: {},
outputDefinition: {
reasoning: "Please explain your reasoning",
score: "Please provide a score between 0 and 1",
},
},
});
const jobConfiguration = await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: [
{
type: "string",
value: "a",
column: "User ID",
operator: "contains",
},
],
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
evalTemplateId: templateId,
},
});
const jobExecutionId = randomUUID();
await prisma.jobExecution.create({
data: {
id: jobExecutionId,
projectId,
jobConfigurationId: jobConfiguration.id,
status: "CANCELLED",
startTime: new Date(),
jobInputTraceId: traceId,
},
});
const payload = {
projectId,
jobExecutionId: jobExecutionId,
delay: 1000,
};
await evaluate({ event: payload });
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(0);
}, 10_000);
test("evals a valid 'trace' event and inserts score to ingestion pipeline", async () => {
const { projectId } = await createOrgProjectAndApiKey();
openAIServer.respondWithDefault();
const traceId = randomUUID();
const templateId = randomUUID();
await prisma.evalTemplate.create({
data: {
id: templateId,
projectId,
name: "test-template",
version: 1,
prompt: "Please evaluate toxicity {{input}} {{output}}",
model: "gpt-3.5-turbo",
provider: "openai",
modelParams: {},
outputDefinition: {
reasoning: "Please explain your reasoning",
score: "Please provide a score between 0 and 1",
},
},
});
const jobConfiguration = await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: [
{
type: "string",
value: "a",
column: "User ID",
operator: "contains",
},
],
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
evalTemplateId: templateId,
},
});
const jobExecutionId = randomUUID();
await prisma.jobExecution.create({
data: {
id: jobExecutionId,
projectId,
jobConfigurationId: jobConfiguration.id,
status: "PENDING",
startTime: new Date(),
jobInputTraceId: traceId,
},
});
await prisma.llmApiKeys.create({
data: {
id: randomUUID(),
projectId,
secretKey: encrypt(String(OPENAI_API_KEY)),
provider: "openai",
adapter: LLMAdapter.OpenAI,
customModels: [],
displaySecretKey: "123456",
},
});
const payload = {
projectId,
jobExecutionId: jobExecutionId,
delay: 1000,
};
await evaluate({ event: payload });
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(1);
expect(jobs[0].projectId).toBe(projectId);
expect(jobs[0].jobInputTraceId).toBe(traceId);
expect(jobs[0].status.toString()).toBe("COMPLETED");
expect(jobs[0].startTime).not.toBeNull();
expect(jobs[0].endTime).not.toBeNull();
}, 20_000);
test("handles LLM timeout gracefully", async () => {
const { projectId } = await createOrgProjectAndApiKey();
// Set up the mock to simulate timeout for this test only
const mockGenerateLlmEvaluatorOutput = vi.mocked(
generateLlmEvaluatorOutput,
);
mockGenerateLlmEvaluatorOutput.mockRejectedValueOnce(
new ApiError("Request timeout after 120000ms", 500),
);
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
user_id: "a",
input: JSON.stringify({ input: "This is a great prompt" }),
output: JSON.stringify({ output: "This is a great response" }),
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
const templateId = randomUUID();
await prisma.evalTemplate.create({
data: {
id: templateId,
projectId,
name: "test-template",
version: 1,
prompt: "Please evaluate toxicity {{input}} {{output}}",
model: "gpt-3.5-turbo",
provider: "openai",
modelParams: {},
outputDefinition: {
reasoning: "Please explain your reasoning",
score: "Please provide a score between 0 and 1",
},
},
});
const jobConfiguration = await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: [
{
type: "string",
value: "a",
column: "User ID",
operator: "contains",
},
],
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
evalTemplateId: templateId,
},
});
const jobExecutionId = randomUUID();
await prisma.jobExecution.create({
data: {
id: jobExecutionId,
projectId,
jobConfigurationId: jobConfiguration.id,
status: "PENDING",
startTime: new Date(),
jobInputTraceId: traceId,
},
});
await prisma.llmApiKeys.create({
data: {
id: randomUUID(),
projectId,
secretKey: encrypt(String(OPENAI_API_KEY)),
provider: "openai",
adapter: LLMAdapter.OpenAI,
customModels: [],
displaySecretKey: "123456",
},
});
const payload = {
projectId,
jobExecutionId: jobExecutionId,
delay: 1000,
};
// Test that timeout error is thrown
await expect(evaluate({ event: payload })).rejects.toThrowError(
/timeout/i,
);
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(1);
expect(jobs[0].projectId).toBe(projectId);
expect(jobs[0].jobInputTraceId).toBe(traceId);
// Job should still be PENDING because the error will be handled by the queue processor
expect(jobs[0].status.toString()).toBe("PENDING");
// Clean up the mock after this test
mockGenerateLlmEvaluatorOutput.mockReset();
}, 15_000);
});
describe("test variable extraction", () => {
test("extracts variables from a dataset item", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const datasetId = randomUUID();
const datasetItemId = randomUUID();
const traceId = randomUUID();
await prisma.dataset.create({
data: {
id: datasetId,
projectId,
name: "test-dataset",
},
});
await prisma.datasetItem.create({
data: {
id: datasetItemId,
input: { input: "This is a great prompt" },
expectedOutput: { expected_output: "This is a great response" },
projectId,
datasetId,
},
});
const variableMapping = variableMappingList.parse([
{
langfuseObject: "dataset_item",
selectedColumnId: "input",
templateVariable: "input",
},
{
langfuseObject: "dataset_item",
selectedColumnId: "expected_output",
templateVariable: "output",
},
]);
const result = await extractVariablesFromTracingData({
projectId,
variables: ["input", "output"],
traceId: traceId,
datasetItemId: datasetItemId,
variableMapping: variableMapping,
});
expect(result).toEqual([
{
value: { input: "This is a great prompt" },
var: "input",
},
{
value: { expected_output: "This is a great response" },
var: "output",
},
]);
}, 10_000);
test("extracts variables from a trace", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
user_id: "a",
environment: "production",
input: JSON.stringify({ input: "This is a great prompt" }),
output: JSON.stringify({ output: "This is a great response" }),
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
const variableMapping = variableMappingList.parse([
{
langfuseObject: "trace",
selectedColumnId: "input",
templateVariable: "input",
},
{
langfuseObject: "trace",
selectedColumnId: "output",
templateVariable: "output",
},
]);
const result = await extractVariablesFromTracingData({
projectId,
variables: ["input", "output"],
traceId: traceId,
variableMapping: variableMapping,
});
expect(result).toEqual([
{
value: { input: "This is a great prompt" },
var: "input",
environment: "production",
},
{
value: { output: "This is a great response" },
var: "output",
environment: "production",
},
]);
}, 10_000);
test("extracts variables from a observation", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
user_id: "a",
input: JSON.stringify({ input: "This is a great prompt" }),
output: JSON.stringify({ output: "This is a great response" }),
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await upsertObservation({
id: randomUUID(),
trace_id: traceId,
project_id: projectId,
name: "great-llm-name",
type: "GENERATION",
environment: "production",
input: JSON.stringify({ huhu: "This is a great prompt" }),
output: JSON.stringify({ haha: "This is a great response" }),
start_time: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
const variableMapping = variableMappingList.parse([
{
langfuseObject: "generation",
selectedColumnId: "input",
templateVariable: "input",
objectName: "great-llm-name",
},
{
langfuseObject: "generation",
selectedColumnId: "output",
templateVariable: "output",
objectName: "great-llm-name",
},
]);
const result = await extractVariablesFromTracingData({
projectId,
variables: ["input", "output"],
traceId: traceId,
variableMapping: variableMapping,
});
expect(result).toEqual([
{
value: { huhu: "This is a great prompt" },
var: "input",
environment: "production",
},
{
value: { haha: "This is a great response" },
var: "output",
environment: "production",
},
]);
}, 10_000);
test("fails if observation is not present", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const variableMapping = variableMappingList.parse([
{
langfuseObject: "generation",
selectedColumnId: "input",
templateVariable: "input",
objectName: "great-llm-name",
},
{
langfuseObject: "generation",
selectedColumnId: "output",
templateVariable: "output",
objectName: "great-llm-name",
},
]);
await expect(
extractVariablesFromTracingData({
projectId,
variables: ["input", "output"],
traceId: traceId,
variableMapping: variableMapping,
}),
).rejects.toThrowError(
new UnrecoverableError(
`Observation great-llm-name for trace ${traceId} not found. Please ensure the mapped data exists and consider extending the job delay.`,
),
);
}, 10_000);
test("does not fail if observation data is null", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
user_id: "a",
input: JSON.stringify({ input: "This is a great prompt" }),
output: JSON.stringify({ output: "This is a great response" }),
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
// fetching input and output for an observation which has NULL values
await upsertObservation({
id: randomUUID(),
trace_id: traceId,
project_id: projectId,
name: "great-llm-name",
type: "GENERATION",
start_time: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
const variableMapping = variableMappingList.parse([
{
langfuseObject: "generation",
selectedColumnId: "input",
templateVariable: "input",
objectName: "great-llm-name",
},
{
langfuseObject: "generation",
selectedColumnId: "output",
templateVariable: "output",
objectName: "great-llm-name",
},
]);
const result = await extractVariablesFromTracingData({
projectId,
variables: ["input", "output"],
traceId: traceId,
variableMapping: variableMapping,
});
expect(result).toEqual([
{
environment: "default",
value: null,
var: "input",
},
{
environment: "default",
value: null,
var: "output",
},
]);
}, 10_000);
test("extracts variables from a youngest observation", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
user_id: "a",
input: JSON.stringify({ input: "This is a great prompt" }),
output: JSON.stringify({ output: "This is a great response" }),
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await upsertObservation({
id: randomUUID(),
trace_id: traceId,
project_id: projectId,
name: "great-llm-name",
type: "GENERATION",
input: JSON.stringify({ huhu: "This is a great prompt" }),
output: JSON.stringify({ haha: "This is a great response" }),
start_time: convertDateToClickhouseDateTime(
new Date("2022-01-01T00:00:00.000Z"),
),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await upsertObservation({
id: randomUUID(),
trace_id: traceId,
project_id: projectId,
name: "great-llm-name",
type: "GENERATION",
input: JSON.stringify({ huhu: "This is a great prompt again" }),
output: JSON.stringify({ haha: "This is a great response again" }),
start_time: convertDateToClickhouseDateTime(
new Date("2022-01-02T00:00:00.000Z"),
),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
const variableMapping = variableMappingList.parse([
{
langfuseObject: "generation",
selectedColumnId: "input",
templateVariable: "input",
objectName: "great-llm-name",
},
{
langfuseObject: "generation",
selectedColumnId: "output",
templateVariable: "output",
objectName: "great-llm-name",
},
]);
const result = await extractVariablesFromTracingData({
projectId,
variables: ["input", "output"],
traceId: traceId,
variableMapping: variableMapping,
});
expect(result).toEqual([
{
environment: "default",
value: { huhu: "This is a great prompt again" },
var: "input",
},
{
environment: "default",
value: { haha: "This is a great response again" },
var: "output",
},
]);
}, 10_000);
test.each(
Object.values(ObservationType).filter(
(type) => !["SPAN", "EVENT", "GENERATION"].includes(type),
),
)(
"extracts variables from a %s observation",
async (observationType) => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
user_id: "a",
input: JSON.stringify({ input: "This is a great prompt" }),
output: JSON.stringify({ output: "This is a great response" }),
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await upsertObservation({
id: randomUUID(),
trace_id: traceId,
project_id: projectId,
name: `great-${observationType.toLowerCase()}-name`,
type: observationType,
environment: "production",
input: JSON.stringify({ huhu: "This is a great prompt" }),
output: JSON.stringify({ haha: "This is a great response" }),
start_time: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
const variableMapping = variableMappingList.parse([
{
langfuseObject: observationType.toLowerCase(),
selectedColumnId: "input",
templateVariable: "input",
objectName: `great-${observationType.toLowerCase()}-name`,
},
{
langfuseObject: observationType.toLowerCase(),
selectedColumnId: "output",
templateVariable: "output",
objectName: `great-${observationType.toLowerCase()}-name`,
},
]);
const result = await extractVariablesFromTracingData({
projectId,
variables: ["input", "output"],
traceId: traceId,
variableMapping: variableMapping,
});
expect(result).toEqual([
{
value: { huhu: "This is a great prompt" },
var: "input",
environment: "production",
},
{
value: { haha: "This is a great response" },
var: "output",
environment: "production",
},
]);
},
10_000,
);
test("extracts variables from a versioned dataset item using datasetItemValidFrom", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const datasetId = randomUUID();
const datasetItemId = randomUUID();
const validFromV1 = new Date("2024-01-01T00:00:00.000Z");
const validFromV2 = new Date("2024-06-01T00:00:00.000Z");
await upsertTrace({
id: traceId,
project_id: projectId,
user_id: "a",
environment: "production",
input: JSON.stringify({ input: "trace input" }),
output: JSON.stringify({ output: "trace output" }),
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await prisma.dataset.create({
data: {
id: datasetId,
name: `dataset-versioned-${randomUUID()}`,
projectId,
},
});
// Create version 1 (superseded) with validTo set
await prisma.datasetItem.create({
data: {
id: datasetItemId,
input: { input: "v1 input" },
expectedOutput: { expected_output: "v1 expected output" },
projectId,
datasetId,
validFrom: validFromV1,
validTo: validFromV2,
},
});
// Create version 2 (current) with validTo = null
await prisma.datasetItem.create({
data: {
id: datasetItemId,
input: { input: "v2 input" },
expectedOutput: { expected_output: "v2 expected output" },
projectId,
datasetId,
validFrom: validFromV2,
validTo: null,
},
});
const variableMapping = variableMappingList.parse([
{
langfuseObject: "dataset_item",
selectedColumnId: "expected_output",
templateVariable: "output",
},
]);
// When datasetItemValidFrom is provided, it should return the exact version
const resultV1 = await extractVariablesFromTracingData({
projectId,
variables: ["output"],
traceId,
datasetItemId,
datasetItemValidFrom: validFromV1,
variableMapping,
});
expect(resultV1).toEqual([
{
value: { expected_output: "v1 expected output" },
var: "output",
},
]);
// When datasetItemValidFrom is NOT provided, it should return the latest (validTo = null)
const resultLatest = await extractVariablesFromTracingData({
projectId,
variables: ["output"],
traceId,
datasetItemId,
variableMapping,
});
expect(resultLatest).toEqual([
{
value: { expected_output: "v2 expected output" },
var: "output",
},
]);
}, 10_000);
test("extracts variables from a dataset item using jsonSelector", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const datasetId = randomUUID();
const datasetItemId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
user_id: "a",
environment: "production",
input: JSON.stringify({ input: "trace input" }),
output: JSON.stringify({ output: "trace output" }),
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await prisma.dataset.create({
data: {
id: datasetId,
name: `dataset-json-${randomUUID()}`,
projectId,
},
});
await prisma.datasetItem.create({
data: {
id: datasetItemId,
input: {
messages: [
{ role: "system", content: "You are a helpful assistant" },
{ role: "user", content: "Hello world" },
],
},
expectedOutput: {
nested: {
score: 0.95,
label: "positive",
},
},
projectId,
datasetId,
},
});
// Test jsonSelector on input - extract nested array element
const variableMappingInput = variableMappingList.parse([
{
langfuseObject: "dataset_item",
selectedColumnId: "input",
templateVariable: "user_message",
jsonSelector: "$.messages[1].content",
},
]);
const resultInput = await extractVariablesFromTracingData({
projectId,
variables: ["user_message"],
traceId,
datasetItemId,
variableMapping: variableMappingInput,
});
expect(resultInput).toEqual([
{
value: "Hello world",
var: "user_message",
},
]);
// Test jsonSelector on expected_output - extract nested field
const variableMappingOutput = variableMappingList.parse([
{
langfuseObject: "dataset_item",
selectedColumnId: "expected_output",
templateVariable: "expected_label",
jsonSelector: "$.nested.label",
},
]);
const resultOutput = await extractVariablesFromTracingData({
projectId,
variables: ["expected_label"],
traceId,
datasetItemId,
variableMapping: variableMappingOutput,
});
expect(resultOutput).toEqual([
{
value: "positive",
var: "expected_label",
},
]);
}, 10_000);
test("extracts variables from a trace using jsonSelector", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
user_id: "a",
environment: "production",
input: JSON.stringify({
messages: [{ role: "user", content: "What is the weather?" }],
}),
output: JSON.stringify({
response: { text: "It is sunny", confidence: 0.9 },
}),
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
const variableMapping = variableMappingList.parse([
{
langfuseObject: "trace",
selectedColumnId: "input",
templateVariable: "user_input",
jsonSelector: "$.messages[0].content",
},
{
langfuseObject: "trace",
selectedColumnId: "output",
templateVariable: "response_text",
jsonSelector: "$.response.text",
},
]);
const result = await extractVariablesFromTracingData({
projectId,
variables: ["user_input", "response_text"],
traceId,
variableMapping,
});
expect(result).toEqual([
{
value: "What is the weather?",
var: "user_input",
environment: "production",
},
{
value: "It is sunny",
var: "response_text",
environment: "production",
},
]);
}, 10_000);
});
test("requiresDatabaseLookup correctly identifies complex filters", () => {
// Simple filters that can be evaluated with trace data only
const simpleFilters = [
{
column: "traceName",
type: "string",
operator: "=",
value: "test-trace",
},
{
column: "environment",
type: "string",
operator: "=",
value: "production",
},
{
column: "traceTags",
type: "arrayOptions",
operator: "any of",
value: ["prod"],
},
{ column: "bookmarked", type: "boolean", operator: "=", value: true },
];
expect(!requiresDatabaseLookup(simpleFilters)).toBe(true);
// Complex filters that require observation data
const complexFilters = [
{ column: "level", type: "string", operator: "=", value: "ERROR" },
];
expect(!requiresDatabaseLookup(complexFilters)).toBe(false);
// Mixed filters - should return false if any filter requires observation data
const mixedFilters = [
{
column: "traceName",
type: "string",
operator: "=",
value: "test-trace",
},
{ column: "level", type: "string", operator: "=", value: "ERROR" }, // This requires observation data
];
expect(!requiresDatabaseLookup(mixedFilters)).toBe(false);
});
describe("eval execution tracing", () => {
test("creates trace for eval execution and stores trace ID in score", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const jobExecutionId = randomUUID();
const templateId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await prisma.llmApiKeys.create({
data: {
id: randomUUID(),
projectId,
secretKey: encrypt(String(OPENAI_API_KEY)),
provider: "openai",
adapter: LLMAdapter.OpenAI,
customModels: [],
displaySecretKey: "123456",
},
});
await prisma.evalTemplate.create({
data: {
id: templateId,
name: "test-evaluator",
projectId,
model: "gpt-3.5-turbo",
provider: "openai",
prompt: "Please evaluate: {{input}}",
version: 1,
vars: ["input"],
outputDefinition: {
score: "score",
reasoning: "reasoning",
},
},
});
const configId = randomUUID();
await createMigratedEvalConfig({
data: {
id: configId,
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "test-score",
variableMapping: JSON.parse(
'[{"langfuseObject":"trace","selectedColumnId":"input","templateVariable":"input"}]',
),
status: "ACTIVE",
evalTemplateId: templateId,
},
});
await prisma.jobExecution.create({
data: {
id: jobExecutionId,
projectId,
jobConfigurationId: configId,
jobInputTraceId: traceId,
jobTemplateId: templateId,
status: "PENDING",
startTime: new Date(),
},
});
// Mock the LLM runtime to capture the trace parameters.
let capturedTraceSinkParams: any = null;
vi.mocked(generateLlmEvaluatorOutput).mockImplementationOnce((async (
params: any,
) => {
capturedTraceSinkParams = params.trace;
return {
output: { score: 0.8, reasoning: "Good response" },
usage: {},
providerMessages: [],
modelFacingOutputSchema: params.outputResultSchema,
};
}) as any);
await evaluate({
event: {
projectId,
jobExecutionId,
},
});
// Verify trace parameters were passed to the shared LLM runtime.
expect(capturedTraceSinkParams).toBeDefined();
expect(capturedTraceSinkParams.targetProjectId).toBe(projectId);
expect(capturedTraceSinkParams.traceId).toMatch(/^[a-f0-9]{32}$/);
expect(capturedTraceSinkParams.traceName).toBe(
"Execute evaluator: test-score",
);
expect(capturedTraceSinkParams.environment).toBe(
LangfuseInternalTraceEnvironment.LLMJudge,
);
expect(capturedTraceSinkParams.metadata).toMatchObject({
job_execution_id: jobExecutionId,
job_configuration_id: configId,
target_trace_id: traceId,
});
}, 15_000);
});
describe("legacy eval output definition backward compatibility", () => {
test("executes a legacy numeric outputDefinition template", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const jobExecutionId = randomUUID();
const templateId = randomUUID();
vi.mocked(generateLlmEvaluatorOutput).mockResolvedValueOnce({
output: {
score: 0,
reasoning: "The response is safe and not toxic.",
},
usage: {},
providerMessages: [],
modelFacingOutputSchema: {},
} as any);
await upsertTrace({
id: traceId,
project_id: projectId,
user_id: "a",
input: JSON.stringify({ input: "This is a great prompt" }),
output: JSON.stringify({ output: "This is a great response" }),
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
await prisma.evalTemplate.create({
data: {
id: templateId,
name: "legacy-output-definition-template",
projectId,
model: "gpt-3.5-turbo",
provider: "openai",
prompt: "Please evaluate toxicity {{input}} {{output}}",
version: 1,
vars: ["input", "output"],
outputDefinition: {
score: "Please provide a score between 0 and 1",
reasoning: "Please explain your reasoning",
},
},
});
const jobConfiguration = await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "legacy-score",
variableMapping: JSON.parse("[]"),
status: "ACTIVE",
evalTemplateId: templateId,
},
});
await prisma.jobExecution.create({
data: {
id: jobExecutionId,
projectId,
jobConfigurationId: jobConfiguration.id,
jobInputTraceId: traceId,
jobTemplateId: templateId,
status: "PENDING",
startTime: new Date(),
},
});
await prisma.llmApiKeys.create({
data: {
id: randomUUID(),
projectId,
secretKey: encrypt(String(OPENAI_API_KEY)),
provider: "openai",
adapter: LLMAdapter.OpenAI,
customModels: [],
displaySecretKey: "123456",
},
});
await evaluate({
event: {
projectId,
jobExecutionId,
},
});
const updatedJobExecution = await prisma.jobExecution.findUnique({
where: {
id: jobExecutionId,
projectId,
},
});
expect(updatedJobExecution?.status).toBe("COMPLETED");
expect(updatedJobExecution?.jobOutputScoreId).toBeTruthy();
expect(updatedJobExecution?.executionTraceId).toBeTruthy();
expect(updatedJobExecution?.endTime).not.toBeNull();
}, 20_000);
});
describe("internal trace environment filtering", () => {
test.each([LangfuseInternalTraceEnvironment.LLMJudge, "llm-as-a-judge"])(
"does not create eval jobs for trace-upsert with %s environment",
async (environment) => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
await upsertTrace({
id: traceId,
project_id: projectId,
environment,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
// Create an active eval configuration
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
});
const payload = {
projectId,
traceId,
traceEnvironment: environment,
};
// Attempt to create eval jobs
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
});
// Verify no eval jobs were created
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(0);
},
10_000,
);
test("does not create eval jobs for trace-upsert with PromptExperiments environment", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
// Create trace with PromptExperiments environment
await upsertTrace({
id: traceId,
project_id: projectId,
environment: LangfuseInternalTraceEnvironment.PromptExperiments,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
// Create an active eval configuration
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
});
const payload = {
projectId,
traceId,
traceEnvironment: LangfuseInternalTraceEnvironment.PromptExperiments,
};
// Attempt to create eval jobs
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
});
// Verify no eval jobs were created
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(0);
}, 10_000);
test("creates eval jobs for dataset-run-item-upsert with PromptExperiments environment", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
const datasetId = randomUUID();
const datasetItemId = randomUUID();
const datasetRunId = randomUUID();
// Create dataset infrastructure
await prisma.dataset.create({
data: {
id: datasetId,
projectId,
name: "test-dataset",
},
});
await prisma.datasetItem.create({
data: {
id: datasetItemId,
projectId,
datasetId,
},
});
await prisma.datasetRuns.create({
data: {
id: datasetRunId,
name: randomUUID(),
datasetId,
projectId,
},
});
// Create clickhouse run item
await createDatasetRunItemsCh([
createDatasetRunItem({
project_id: projectId,
dataset_id: datasetId,
dataset_run_id: datasetRunId,
dataset_item_id: datasetItemId,
trace_id: traceId,
}),
]);
// Create trace with PromptExperiments environment
await upsertTrace({
id: traceId,
project_id: projectId,
environment: LangfuseInternalTraceEnvironment.PromptExperiments,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
// Create an active dataset eval configuration
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.DATASET,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
});
const payload = {
projectId,
traceId,
datasetItemId,
traceEnvironment: LangfuseInternalTraceEnvironment.PromptExperiments,
};
// Attempt to create eval jobs via dataset-run-item-upsert
await createEvalJobs({
sourceEventType: "dataset-run-item-upsert",
event: payload,
jobTimestamp,
});
// Verify eval jobs WERE created (experiments need this)
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(1);
expect(jobs[0].jobInputTraceId).toBe(traceId);
expect(jobs[0].jobInputDatasetItemId).toBe(datasetItemId);
}, 10_000);
test("creates eval jobs for trace-upsert with production environment", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
// Create trace with production environment
await upsertTrace({
id: traceId,
project_id: projectId,
environment: "production",
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
// Create an active eval configuration
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
});
const payload = {
projectId,
traceId,
traceEnvironment: "production",
};
// Attempt to create eval jobs
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
});
// Verify eval jobs WERE created (normal traces should be evaluated)
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(1);
expect(jobs[0].jobInputTraceId).toBe(traceId);
}, 10_000);
test("creates eval jobs for trace-upsert with undefined environment", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
// Create trace with undefined environment
await upsertTrace({
id: traceId,
project_id: projectId,
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
// Create an active eval configuration
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
});
const payload = {
projectId,
traceId,
// traceEnvironment intentionally omitted
};
// Attempt to create eval jobs
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
});
// Verify eval jobs WERE created (traces without environment should be evaluated)
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(1);
expect(jobs[0].jobInputTraceId).toBe(traceId);
}, 10_000);
test("does not create eval jobs for trace-upsert with 'langfuse' environment without hyphen", async () => {
const { projectId } = await createOrgProjectAndApiKey();
const traceId = randomUUID();
// Create trace with "langfuse" environment (no hyphen)
await upsertTrace({
id: traceId,
project_id: projectId,
environment: "langfuse",
timestamp: convertDateToClickhouseDateTime(new Date()),
created_at: convertDateToClickhouseDateTime(new Date()),
updated_at: convertDateToClickhouseDateTime(new Date()),
});
// Create an active eval configuration
await createMigratedEvalConfig({
data: {
id: randomUUID(),
projectId,
filter: JSON.parse("[]"),
jobType: "EVAL",
delay: 0,
sampling: new Decimal("1"),
targetObject: EvalTargetObject.TRACE,
scoreName: "score",
variableMapping: JSON.parse("[]"),
},
});
const payload = {
projectId,
traceId,
traceEnvironment: "langfuse",
};
// Attempt to create eval jobs
await createEvalJobs({
sourceEventType: "trace-upsert",
event: payload,
jobTimestamp,
});
// Verify eval jobs WERE created (only "langfuse-" prefix is blocked)
const jobs = await prisma.jobExecution.findMany({
where: { projectId },
});
expect(jobs.length).toBe(0);
}, 10_000);
});
});