🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import {
createDatasetRunItemsCh,
createObservationsCh,
createScoresCh,
createTracesCh,
getDatasetRunItemsByDatasetIdCh,
getDatasetRunsTableMetricsCh,
getScoresForExperiments,
getTraceScoresForDatasetRuns,
getDatasetRunItemsWithoutIOByItemIds,
createDatasetRunItem,
getDatasetItemIdsWithRunData,
createDatasetItem,
createManyDatasetItems,
createObservation,
createTraceScore,
createTrace,
} from "@langfuse/shared/src/server";
import { v4 } from "uuid";
import { prisma } from "@langfuse/shared/src/db";
import {
enrichAndMapToDatasetItemId,
getRunItemsByRunIdOrItemId,
} from "@/src/features/datasets/server/service";
import {
aggregateScores,
composeAggregateScoreKey,
} from "@/src/features/scores/lib/aggregateScores";
const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
process.env.LANGFUSE_DATASET_SERVICE_READ_FROM_VERSIONED_IMPLEMENTATION =
"true";
process.env.LANGFUSE_DATASET_SERVICE_WRITE_TO_VERSIONED_IMPLEMENTATION = "true";
describe("Fetch datasets for UI presentation", () => {
it("should fetch dataset runs for UI", async () => {
const datasetId = v4();
await prisma.dataset.create({
data: {
id: datasetId,
name: v4(),
projectId: projectId,
},
});
const datasetRunId = v4();
const datasetRun2Id = v4();
const now = new Date();
const oneMinuteAgo = new Date(now.getTime() - 60 * 1000);
await prisma.datasetRuns.create({
data: {
id: datasetRunId,
name: "run1",
datasetId,
metadata: {},
projectId,
createdAt: now,
},
});
await prisma.datasetRuns.create({
data: {
id: datasetRun2Id,
name: "run2",
datasetId,
metadata: {},
projectId,
createdAt: oneMinuteAgo,
},
});
const datasetItemId = v4();
const datasetItemId2 = v4();
const datasetItemId3 = v4();
const datasetItemId4 = v4();
await createManyDatasetItems({
projectId,
items: [
{ id: datasetItemId, datasetId, metadata: {} },
{ id: datasetItemId2, datasetId, metadata: {} },
{ id: datasetItemId3, datasetId, metadata: {} },
{ id: datasetItemId4, datasetId, metadata: {} },
],
});
const datasetRunItemId = v4();
const datasetRunItemId2 = v4();
const datasetRunItemId3 = v4();
const datasetRunItemId4 = v4();
const observationId = v4();
const traceId = v4();
const traceId2 = v4();
const traceId3 = v4();
const traceId4 = v4();
const scoreId = v4();
const scoreId2 = v4();
const scoreId3 = v4();
const scoreName = v4();
const datasetRunItem1 = createDatasetRunItem({
id: datasetRunItemId,
dataset_run_id: datasetRunId,
observation_id: observationId,
trace_id: traceId,
project_id: projectId,
dataset_item_id: datasetItemId,
dataset_id: datasetId,
dataset_run_name: "run1",
dataset_run_created_at: now.getTime(),
});
const datasetRunItem2 = createDatasetRunItem({
id: datasetRunItemId2,
dataset_run_id: datasetRunId,
observation_id: null,
trace_id: traceId2,
project_id: projectId,
dataset_item_id: datasetItemId2,
dataset_id: datasetId,
dataset_run_name: "run1",
dataset_run_created_at: now.getTime(),
});
const datasetRunItem3 = createDatasetRunItem({
id: datasetRunItemId3,
dataset_run_id: datasetRunId,
observation_id: null,
trace_id: traceId3,
project_id: projectId,
dataset_item_id: datasetItemId3,
dataset_id: datasetId,
dataset_run_name: "run1",
dataset_run_created_at: now.getTime(),
});
const datasetRunItem4 = createDatasetRunItem({
id: datasetRunItemId4,
dataset_run_id: datasetRun2Id,
observation_id: null,
trace_id: traceId4,
project_id: projectId,
dataset_item_id: datasetItemId4,
dataset_id: datasetId,
dataset_run_name: "run2",
dataset_run_created_at: oneMinuteAgo.getTime(),
});
await createDatasetRunItemsCh([
datasetRunItem1,
datasetRunItem2,
datasetRunItem3,
datasetRunItem4,
]);
const observation = createObservation({
id: observationId,
trace_id: traceId,
project_id: projectId,
start_time: new Date().getTime() - 1000 * 60 * 60, // minus 1 min
end_time: new Date().getTime(),
});
const observation2 = createObservation({
trace_id: traceId2,
project_id: projectId,
start_time: new Date().getTime() - 1000 * 60 * 60, // minus 1 min
end_time: new Date().getTime(),
});
const observation3 = createObservation({
trace_id: traceId2,
project_id: projectId,
start_time: new Date().getTime() - 1000 * 60 * 30,
end_time: new Date().getTime(),
total_cost: 200,
});
const observation4 = createObservation({
trace_id: traceId3,
project_id: projectId,
start_time: new Date().getTime() - 1000 * 60 * 300,
end_time: new Date().getTime(),
total_cost: 50,
});
const observation5 = createObservation({
trace_id: traceId4,
project_id: projectId,
start_time: new Date().getTime() - 1000,
end_time: new Date().getTime(),
});
const observation6 = createObservation({
trace_id: traceId,
project_id: projectId,
start_time: new Date().getTime() - 1000 * 60 * 60, // minus 1 min
end_time: new Date().getTime(),
});
await createObservationsCh([
observation,
observation2,
observation3,
observation4,
observation5,
observation6,
]);
const score = createTraceScore({
id: scoreId,
observation_id: observationId,
trace_id: traceId,
project_id: projectId,
name: scoreName,
});
const score2 = createTraceScore({
id: scoreId2,
observation_id: null,
trace_id: traceId,
project_id: projectId,
name: scoreName,
value: 1,
comment: "some other comment",
});
const observationId2 = v4(); // this one is not related to a run
const anotherScoreName = v4();
const score3 = createTraceScore({
id: scoreId3,
observation_id: observationId2,
trace_id: traceId,
project_id: projectId,
name: anotherScoreName,
value: 1,
comment: "some other comment for non run related score",
});
await createScoresCh([score, score2, score3]);
// Get runs that have metrics (only runs with dataset_run_items_rmt)
const runsWithMetrics = await getDatasetRunsTableMetricsCh({
projectId: projectId,
datasetId: datasetId,
runIds: [datasetRunId, datasetRun2Id],
filter: [],
});
// Only fetch scores for runs that have metrics (runs without dataset_run_items_rmt won't have trace scores)
const runsWithMetricsIds = runsWithMetrics.map((run) => run.id);
const [traceScores, runScores] = await Promise.all([
runsWithMetricsIds.length > 0
? getTraceScoresForDatasetRuns(projectId, runsWithMetricsIds)
: [],
getScoresForExperiments({
projectId: projectId,
runIds: runsWithMetrics.map((run) => run.id),
includeHasMetadata: true,
excludeMetadata: false,
}),
]);
// Merge all runs: use metrics where available, defaults otherwise
const allRuns = runsWithMetrics.map((run) => {
return {
id: run.id,
name: run.name,
// Use ClickHouse metrics if available, otherwise use defaults for runs without dataset_run_items_rmt
countRunItems: run.countRunItems ?? 0,
avgTotalCost: run.avgTotalCost ?? null,
totalCost: run.totalCost ?? null,
avgLatency: run.avgLatency ?? null,
scores: aggregateScores(
traceScores.filter((s) => s.datasetRunId === run.id),
),
runScores: aggregateScores(
runScores.filter((s) => s.datasetRunId === run.id),
),
};
});
expect(allRuns).toHaveLength(2);
// Verify runs are returned in chronological order (oldest first, most recent last)
expect(allRuns[0].id).toEqual(datasetRunId); // First run (older)
expect(allRuns[1].id).toEqual(datasetRun2Id); // Second run (more recent)
expect(allRuns[0].name).toEqual("run1");
expect(allRuns[1].name).toEqual("run2");
const firstRun = allRuns.find((run) => run.id === datasetRunId);
expect(firstRun).toBeDefined();
if (!firstRun) {
throw new Error("first run is not defined");
}
expect(firstRun.id).toEqual(datasetRunId);
expect(firstRun.avgLatency).toBeGreaterThanOrEqual(10800);
expect(firstRun.avgTotalCost?.toString()).toStrictEqual("275");
expect(firstRun.totalCost?.toString()).toStrictEqual("550");
const expectedObject = {
[`${scoreName.replaceAll("-", "_")}-API-NUMERIC`]: {
type: "NUMERIC",
values: expect.arrayContaining([1, 100.5]),
average: 50.75,
id: undefined,
comment: undefined,
hasMetadata: undefined,
timestamp: undefined,
},
[`${anotherScoreName.replaceAll("-", "_")}-API-NUMERIC`]: {
id: score3.id,
type: "NUMERIC",
values: expect.arrayContaining([1]),
average: 1,
comment: "some other comment for non run related score",
hasMetadata: true,
timestamp: expect.any(Date),
// single-value aggregate carries the score's executionTraceId (null for API scores)
executionTraceId: null,
},
};
expect(firstRun.scores).toEqual(expectedObject);
const secondRun = allRuns.find((run) => run.id === datasetRun2Id);
expect(secondRun).toBeDefined();
if (!secondRun) {
throw new Error("second run is not defined");
}
expect(secondRun.id).toEqual(datasetRun2Id);
expect(secondRun.avgLatency).toBeGreaterThanOrEqual(1);
expect(secondRun.avgLatency).toBeLessThanOrEqual(1.002);
expect(secondRun.avgTotalCost?.toString()).toStrictEqual("300");
expect(JSON.stringify(secondRun.scores)).toEqual(JSON.stringify({}));
});
it("should test that dataset runs can link to the same traces", async () => {
const datasetId = v4();
await prisma.dataset.create({
data: {
id: datasetId,
name: v4(),
projectId: projectId,
},
});
const datasetRunId = v4();
const datasetRun2Id = v4();
await prisma.datasetRuns.create({
data: {
id: datasetRunId,
name: v4(),
datasetId,
metadata: {},
projectId,
},
});
await prisma.datasetRuns.create({
data: {
id: datasetRun2Id,
name: v4(),
datasetId,
metadata: {},
projectId,
},
});
const res = await createDatasetItem({
datasetId,
metadata: {},
projectId,
});
if (!res.success) {
throw new Error("Failed to create dataset item");
}
const datasetItemId = res.datasetItem.id;
const datasetRunItemId = v4();
const datasetRunItemId2 = v4();
const traceId = v4();
const scoreId = v4();
const datasetRunItem1 = createDatasetRunItem({
id: datasetRunItemId,
dataset_run_id: datasetRunId,
observation_id: null,
trace_id: traceId,
project_id: projectId,
dataset_item_id: datasetItemId,
dataset_id: datasetId,
});
const datasetRunItem2 = createDatasetRunItem({
id: datasetRunItemId2,
dataset_run_id: datasetRun2Id,
observation_id: null,
trace_id: traceId,
project_id: projectId,
dataset_item_id: datasetItemId,
dataset_id: datasetId,
});
await createDatasetRunItemsCh([datasetRunItem1, datasetRunItem2]);
const scoreName = v4();
const score = createTraceScore({
id: scoreId,
observation_id: null,
trace_id: traceId,
project_id: projectId,
name: scoreName,
});
await createScoresCh([score]);
// Get runs that have metrics (only runs with dataset_run_items_rmt)
const runsWithMetrics = await getDatasetRunsTableMetricsCh({
projectId: projectId,
datasetId: datasetId,
runIds: [datasetRunId, datasetRun2Id],
filter: [],
});
// Only fetch scores for runs that have metrics (runs without dataset_run_items_rmt won't have trace scores)
const runsWithMetricsIds = runsWithMetrics.map((run) => run.id);
const [traceScores, runScores] = await Promise.all([
runsWithMetricsIds.length > 0
? getTraceScoresForDatasetRuns(projectId, runsWithMetricsIds)
: [],
getScoresForExperiments({
projectId: projectId,
runIds: runsWithMetrics.map((run) => run.id),
includeHasMetadata: true,
excludeMetadata: false,
}),
]);
// Merge all runs: use metrics where available, defaults otherwise
const allRuns = runsWithMetrics.map((run) => {
return {
id: run.id,
name: run.name,
// Use ClickHouse metrics if available, otherwise use defaults for runs without dataset_run_items_rmt
countRunItems: run.countRunItems ?? 0,
avgTotalCost: run.avgTotalCost ?? null,
totalCost: run.totalCost ?? null,
avgLatency: run.avgLatency ?? null,
scores: aggregateScores(
traceScores.filter((s) => s.datasetRunId === run.id),
),
runScores: aggregateScores(
runScores.filter((s) => s.datasetRunId === run.id),
),
};
});
expect(allRuns).toHaveLength(2);
const firstRun = allRuns.find((run) => run.id === datasetRunId);
expect(firstRun).toBeDefined();
if (!firstRun) {
throw new Error("first run is not defined");
}
expect(firstRun.id).toEqual(datasetRunId);
expect(firstRun.avgLatency).toBeGreaterThanOrEqual(0);
expect(firstRun.avgTotalCost?.toString()).toStrictEqual("0");
const expectedObject = {
[`${scoreName.replaceAll("-", "_")}-API-NUMERIC`]: {
id: score.id,
type: "NUMERIC",
values: expect.arrayContaining([100.5]),
average: 100.5,
comment: "comment",
// createScore adds metadata to the score
hasMetadata: true,
timestamp: expect.any(Date),
executionTraceId: null,
},
};
expect(firstRun.scores).toEqual(expectedObject);
const secondRun = allRuns.find((run) => run.id === datasetRun2Id);
expect(secondRun).toBeDefined();
if (!secondRun) {
throw new Error("second run is not defined");
}
expect(secondRun.id).toEqual(datasetRun2Id);
expect(secondRun.avgLatency).toEqual(0);
expect(secondRun.avgTotalCost?.toString()).toStrictEqual("0");
expect(firstRun.scores).toEqual(expectedObject);
});
describe("Dataset Run Score Filtering", () => {
it("should filter dataset runs by numeric scores", async () => {
const datasetId = v4();
await prisma.dataset.create({
data: {
id: datasetId,
name: v4(),
projectId: projectId,
},
});
// Create dataset runs
const highScoreRunId = v4();
const lowScoreRunId = v4();
const noScoreRunId = v4();
await prisma.datasetRuns.create({
data: {
id: highScoreRunId,
name: "high-accuracy-run",
datasetId,
metadata: {},
projectId,
},
});
await prisma.datasetRuns.create({
data: {
id: lowScoreRunId,
name: "low-accuracy-run",
datasetId,
metadata: {},
projectId,
},
});
await prisma.datasetRuns.create({
data: {
id: noScoreRunId,
name: "no-score-run",
datasetId,
metadata: {},
projectId,
},
});
// Create dataset items
const itemId1 = v4();
const itemId2 = v4();
const itemId3 = v4();
await createManyDatasetItems({
projectId,
items: [
{ id: itemId1, datasetId, metadata: {} },
{ id: itemId2, datasetId, metadata: {} },
{ id: itemId3, datasetId, metadata: {} },
],
});
// Create traces and observations
const traceId1 = v4();
const traceId2 = v4();
const traceId3 = v4();
// Create dataset run items
const runItem1 = createDatasetRunItem({
id: v4(),
dataset_run_id: highScoreRunId,
trace_id: traceId1,
project_id: projectId,
dataset_item_id: itemId1,
dataset_id: datasetId,
dataset_run_name: "high-accuracy-run",
});
const runItem2 = createDatasetRunItem({
id: v4(),
dataset_run_id: lowScoreRunId,
trace_id: traceId2,
project_id: projectId,
dataset_item_id: itemId2,
dataset_id: datasetId,
dataset_run_name: "low-accuracy-run",
});
const runItem3 = createDatasetRunItem({
id: v4(),
dataset_run_id: noScoreRunId,
trace_id: traceId3,
project_id: projectId,
dataset_item_id: itemId3,
dataset_id: datasetId,
dataset_run_name: "no-score-run",
});
await createDatasetRunItemsCh([runItem1, runItem2, runItem3]);
// Create observations for latency calculation
const observation1 = createObservation({
trace_id: traceId1,
project_id: projectId,
start_time: new Date().getTime() - 2000,
end_time: new Date().getTime() - 1000,
});
const observation2 = createObservation({
trace_id: traceId2,
project_id: projectId,
start_time: new Date().getTime() - 3000,
end_time: new Date().getTime() - 1000,
});
const observation3 = createObservation({
trace_id: traceId3,
project_id: projectId,
start_time: new Date().getTime() - 1000,
end_time: new Date().getTime(),
});
await createObservationsCh([observation1, observation2, observation3]);
// Create scores with different values for the same score name
const highAccuracyScore = createTraceScore({
id: v4(),
trace_id: traceId1,
project_id: projectId,
name: "accuracy",
value: 0.95, // High accuracy
});
const lowAccuracyScore = createTraceScore({
id: v4(),
trace_id: traceId2,
project_id: projectId,
name: "accuracy",
value: 0.65, // Low accuracy
});
const highPrecisionScore = createTraceScore({
id: v4(),
trace_id: traceId1,
project_id: projectId,
name: "precision",
value: 0.88,
});
// No scores for traceId3 to test filtering behavior
await createScoresCh([
highAccuracyScore,
lowAccuracyScore,
highPrecisionScore,
]);
// Test 1: Filter for runs with accuracy > 0.8
const highAccuracyRuns = await getDatasetRunsTableMetricsCh({
projectId: projectId,
datasetId: datasetId,
runIds: [highScoreRunId, lowScoreRunId, noScoreRunId],
filter: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">" as const,
value: 0.8,
},
],
});
expect(highAccuracyRuns).toHaveLength(1);
expect(highAccuracyRuns[0].id).toEqual(highScoreRunId);
expect(highAccuracyRuns[0].name).toEqual("high-accuracy-run");
// Test 2: Filter for runs with accuracy >= 0.65 (should include both)
const mediumAccuracyRuns = await getDatasetRunsTableMetricsCh({
projectId: projectId,
datasetId: datasetId,
runIds: [highScoreRunId, lowScoreRunId, noScoreRunId],
filter: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">=" as const,
value: 0.65,
},
],
});
expect(mediumAccuracyRuns).toHaveLength(2);
const runIds = mediumAccuracyRuns.map((run) => run.id);
expect(runIds).toContain(highScoreRunId);
expect(runIds).toContain(lowScoreRunId);
// Test 3: Filter for runs with precision = 0.88
const precisionRuns = await getDatasetRunsTableMetricsCh({
projectId: projectId,
datasetId: datasetId,
runIds: [highScoreRunId, lowScoreRunId, noScoreRunId],
filter: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "precision",
operator: "=" as const,
value: 0.88,
},
],
});
expect(precisionRuns).toHaveLength(1);
expect(precisionRuns[0].id).toEqual(highScoreRunId);
});
it("should filter dataset run items by boolean scores", async () => {
const datasetId = v4();
const scoreName = `passes_guardrail_${v4()}`;
await prisma.dataset.create({
data: {
id: datasetId,
name: v4(),
projectId: projectId,
},
});
const runId = v4();
await prisma.datasetRuns.create({
data: {
id: runId,
name: "boolean-score-run",
datasetId,
metadata: {},
projectId,
},
});
// Three items: one trace scored True, one scored False, one unscored
const itemIdWithTrueScore = v4();
const itemIdWithFalseScore = v4();
const itemIdWithoutScore = v4();
await createManyDatasetItems({
projectId,
items: [
{ id: itemIdWithTrueScore, datasetId, metadata: {} },
{ id: itemIdWithFalseScore, datasetId, metadata: {} },
{ id: itemIdWithoutScore, datasetId, metadata: {} },
],
});
const traceIdWithTrueScore = v4();
const traceIdWithFalseScore = v4();
const traceIdWithoutScore = v4();
await createDatasetRunItemsCh([
createDatasetRunItem({
id: v4(),
dataset_run_id: runId,
trace_id: traceIdWithTrueScore,
project_id: projectId,
dataset_item_id: itemIdWithTrueScore,
dataset_id: datasetId,
dataset_run_name: "boolean-score-run",
}),
createDatasetRunItem({
id: v4(),
dataset_run_id: runId,
trace_id: traceIdWithFalseScore,
project_id: projectId,
dataset_item_id: itemIdWithFalseScore,
dataset_id: datasetId,
dataset_run_name: "boolean-score-run",
}),
createDatasetRunItem({
id: v4(),
dataset_run_id: runId,
trace_id: traceIdWithoutScore,
project_id: projectId,
dataset_item_id: itemIdWithoutScore,
dataset_id: datasetId,
dataset_run_name: "boolean-score-run",
}),
]);
await createScoresCh([
createTraceScore({
id: v4(),
trace_id: traceIdWithTrueScore,
project_id: projectId,
name: scoreName,
value: 1,
string_value: "True",
data_type: "BOOLEAN",
}),
createTraceScore({
id: v4(),
trace_id: traceIdWithFalseScore,
project_id: projectId,
name: scoreName,
value: 0,
string_value: "False",
data_type: "BOOLEAN",
}),
]);
const runItems = await getDatasetRunItemsByDatasetIdCh({
projectId,
datasetId,
filter: [
{
type: "booleanObject" as const,
column: "agg_score_booleans",
key: scoreName,
operator: "=" as const,
value: true,
},
],
limit: 10,
offset: 0,
});
expect(runItems).toHaveLength(1);
expect(runItems[0].traceId).toEqual(traceIdWithTrueScore);
expect(runItems[0].datasetItemId).toEqual(itemIdWithTrueScore);
});
it("should filter dataset runs by categorical scores", async () => {
const datasetId = v4();
await prisma.dataset.create({
data: {
id: datasetId,
name: v4(),
projectId: projectId,
},
});
// Create dataset runs
const excellentRunId = v4();
const goodRunId = v4();
const poorRunId = v4();
await prisma.datasetRuns.createMany({
data: [
{
id: excellentRunId,
name: "excellent-quality-run",
datasetId,
metadata: {},
projectId,
},
{
id: goodRunId,
name: "good-quality-run",
datasetId,
metadata: {},
projectId,
},
{
id: poorRunId,
name: "poor-quality-run",
datasetId,
metadata: {},
projectId,
},
],
});
// Create dataset items and traces
const itemIds = [v4(), v4(), v4()];
const traceIds = [v4(), v4(), v4()];
await createManyDatasetItems({
projectId,
items: itemIds.map((id) => ({
id,
datasetId,
metadata: {},
})),
});
// Create dataset run items
const runItems = [
createDatasetRunItem({
id: v4(),
dataset_run_id: excellentRunId,
trace_id: traceIds[0],
project_id: projectId,
dataset_item_id: itemIds[0],
dataset_id: datasetId,
dataset_run_name: "excellent-quality-run",
}),
createDatasetRunItem({
id: v4(),
dataset_run_id: goodRunId,
trace_id: traceIds[1],
project_id: projectId,
dataset_item_id: itemIds[1],
dataset_id: datasetId,
dataset_run_name: "good-quality-run",
}),
createDatasetRunItem({
id: v4(),
dataset_run_id: poorRunId,
trace_id: traceIds[2],
project_id: projectId,
dataset_item_id: itemIds[2],
dataset_id: datasetId,
dataset_run_name: "poor-quality-run",
}),
];
await createDatasetRunItemsCh(runItems);
// Create observations for each trace
const observations = traceIds.map((traceId) =>
createObservation({
trace_id: traceId,
project_id: projectId,
start_time: new Date().getTime() - 1000,
end_time: new Date().getTime(),
}),
);
await createObservationsCh(observations);
// Create categorical scores
const qualityScores = [
createTraceScore({
id: v4(),
trace_id: traceIds[0],
project_id: projectId,
name: "quality",
data_type: "CATEGORICAL",
string_value: "excellent",
}),
createTraceScore({
id: v4(),
trace_id: traceIds[1],
project_id: projectId,
name: "quality",
data_type: "CATEGORICAL",
string_value: "good",
}),
createTraceScore({
id: v4(),
trace_id: traceIds[2],
project_id: projectId,
name: "quality",
data_type: "CATEGORICAL",
string_value: "poor",
}),
];
// Add sentiment scores for more complex filtering
const sentimentScores = [
createTraceScore({
id: v4(),
trace_id: traceIds[0],
project_id: projectId,
name: "sentiment",
data_type: "CATEGORICAL",
string_value: "positive",
}),
createTraceScore({
id: v4(),
trace_id: traceIds[1],
project_id: projectId,
name: "sentiment",
data_type: "CATEGORICAL",
string_value: "neutral",
}),
createTraceScore({
id: v4(),
trace_id: traceIds[2],
project_id: projectId,
name: "sentiment",
data_type: "CATEGORICAL",
string_value: "negative",
}),
];
await createScoresCh([...qualityScores, ...sentimentScores]);
// Test: Filter for high quality runs (excellent or good)
const highQualityRuns = await getDatasetRunsTableMetricsCh({
projectId: projectId,
datasetId: datasetId,
runIds: [excellentRunId, goodRunId, poorRunId],
filter: [
{
type: "categoryOptions" as const,
column: "agg_score_categories",
key: "quality",
operator: "any of" as const,
value: ["excellent", "good"],
},
],
});
expect(highQualityRuns).toHaveLength(2);
const highQualityIds = highQualityRuns.map((run) => run.id);
expect(highQualityIds).toContain(excellentRunId);
expect(highQualityIds).toContain(goodRunId);
// Test: Filter for runs that are NOT poor quality
const notPoorRuns = await getDatasetRunsTableMetricsCh({
projectId: projectId,
datasetId: datasetId,
runIds: [excellentRunId, goodRunId, poorRunId],
filter: [
{
type: "categoryOptions" as const,
column: "agg_score_categories",
key: "quality",
operator: "none of" as const,
value: ["poor"],
},
],
});
expect(notPoorRuns).toHaveLength(2);
const notPoorIds = notPoorRuns.map((run) => run.id);
expect(notPoorIds).toContain(excellentRunId);
expect(notPoorIds).toContain(goodRunId);
// Test: Filter for positive sentiment
const positiveSentimentRuns = await getDatasetRunsTableMetricsCh({
projectId: projectId,
datasetId: datasetId,
runIds: [excellentRunId, goodRunId, poorRunId],
filter: [
{
type: "categoryOptions" as const,
column: "agg_score_categories",
key: "sentiment",
operator: "any of" as const,
value: ["positive"],
},
],
});
expect(positiveSentimentRuns).toHaveLength(1);
expect(positiveSentimentRuns[0].id).toEqual(excellentRunId);
});
it("should filter dataset runs by boolean scores", async () => {
const datasetId = v4();
await prisma.dataset.create({
data: {
id: datasetId,
name: v4(),
projectId: projectId,
},
});
const passingRunId = v4();
const failingRunId = v4();
await prisma.datasetRuns.createMany({
data: [
{
id: passingRunId,
name: "passing-run",
datasetId,
metadata: {},
projectId,
},
{
id: failingRunId,
name: "failing-run",
datasetId,
metadata: {},
projectId,
},
],
});
const itemIds = [v4(), v4()];
const traceIds = [v4(), v4()];
await createManyDatasetItems({
projectId,
items: itemIds.map((id) => ({ id, datasetId, metadata: {} })),
});
await createDatasetRunItemsCh([
createDatasetRunItem({
id: v4(),
dataset_run_id: passingRunId,
trace_id: traceIds[0],
project_id: projectId,
dataset_item_id: itemIds[0],
dataset_id: datasetId,
dataset_run_name: "passing-run",
}),
createDatasetRunItem({
id: v4(),
dataset_run_id: failingRunId,
trace_id: traceIds[1],
project_id: projectId,
dataset_item_id: itemIds[1],
dataset_id: datasetId,
dataset_run_name: "failing-run",
}),
]);
await createScoresCh([
createTraceScore({
id: v4(),
trace_id: traceIds[0],
project_id: projectId,
name: "passes_guardrail",
value: 1,
string_value: "True",
data_type: "BOOLEAN",
}),
createTraceScore({
id: v4(),
trace_id: traceIds[1],
project_id: projectId,
name: "passes_guardrail",
value: 0,
string_value: "False",
data_type: "BOOLEAN",
}),
]);
const passingRuns = await getDatasetRunsTableMetricsCh({
projectId: projectId,
datasetId: datasetId,
runIds: [passingRunId, failingRunId],
filter: [
{
type: "booleanObject" as const,
column: "agg_score_booleans",
key: "passes_guardrail",
operator: "=" as const,
value: true,
},
],
});
expect(passingRuns).toHaveLength(1);
expect(passingRuns[0].id).toEqual(passingRunId);
});
it("should combine score filters with other filters", async () => {
const datasetId = v4();
await prisma.dataset.create({
data: {
id: datasetId,
name: v4(),
projectId: projectId,
},
});
// Create dataset runs with specific names and times
const recentRunId = v4();
const oldRunId = v4();
const recentDate = new Date();
const oldDate = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); // 7 days ago
await prisma.datasetRuns.create({
data: {
id: recentRunId,
name: "recent-high-accuracy",
datasetId,
metadata: { version: "2.0" },
projectId,
createdAt: recentDate,
},
});
await prisma.datasetRuns.create({
data: {
id: oldRunId,
name: "old-high-accuracy",
datasetId,
metadata: { version: "1.0" },
projectId,
createdAt: oldDate,
},
});
// Create dataset items and traces
const itemIds = [v4(), v4()];
const traceIds = [v4(), v4()];
await createManyDatasetItems({
projectId,
items: itemIds.map((id) => ({
id,
datasetId,
metadata: {},
})),
});
// Create dataset run items
const runItems = [
createDatasetRunItem({
id: v4(),
dataset_run_id: recentRunId,
trace_id: traceIds[0],
project_id: projectId,
dataset_item_id: itemIds[0],
dataset_id: datasetId,
dataset_run_name: "recent-high-accuracy",
created_at: recentDate.getTime(),
}),
createDatasetRunItem({
id: v4(),
dataset_run_id: oldRunId,
trace_id: traceIds[1],
project_id: projectId,
dataset_item_id: itemIds[1],
dataset_id: datasetId,
dataset_run_name: "old-high-accuracy",
created_at: oldDate.getTime(),
}),
];
await createDatasetRunItemsCh(runItems);
// Create observations with different costs
const observations = [
createObservation({
trace_id: traceIds[0],
project_id: projectId,
start_time: new Date().getTime() - 1000,
end_time: new Date().getTime(),
total_cost: 100, // Higher cost
}),
createObservation({
trace_id: traceIds[1],
project_id: projectId,
start_time: new Date().getTime() - 1000,
end_time: new Date().getTime(),
total_cost: 50, // Lower cost
}),
];
await createObservationsCh(observations);
// Create high accuracy scores for both runs
const scores = [
createTraceScore({
id: v4(),
trace_id: traceIds[0],
project_id: projectId,
name: "accuracy",
value: 0.92,
}),
createTraceScore({
id: v4(),
trace_id: traceIds[1],
project_id: projectId,
name: "accuracy",
value: 0.94,
}),
createTraceScore({
id: v4(),
trace_id: traceIds[0],
project_id: projectId,
name: "sentiment",
data_type: "CATEGORICAL",
string_value: "positive",
}),
];
await createScoresCh(scores);
// Test: Combine numeric score filter with category filter
const recentHighAccuracyRuns = await getDatasetRunsTableMetricsCh({
projectId: projectId,
datasetId: datasetId,
runIds: [recentRunId, oldRunId],
filter: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">=" as const,
value: 0.9,
},
{
type: "categoryOptions" as const,
column: "agg_score_categories",
key: "sentiment",
operator: "any of" as const,
value: ["positive"],
},
],
});
expect(recentHighAccuracyRuns).toHaveLength(1);
expect(recentHighAccuracyRuns[0].id).toEqual(recentRunId);
expect(recentHighAccuracyRuns[0].name).toEqual("recent-high-accuracy");
// Test: All high accuracy runs (should return both)
const allHighAccuracyRuns = await getDatasetRunsTableMetricsCh({
projectId: projectId,
datasetId: datasetId,
runIds: [recentRunId, oldRunId],
filter: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">=" as const,
value: 0.9,
},
],
});
expect(allHighAccuracyRuns).toHaveLength(2);
const allIds = allHighAccuracyRuns.map((run) => run.id);
expect(allIds).toContain(recentRunId);
expect(allIds).toContain(oldRunId);
});
});
it("should fetch dataset run items for UI", async () => {
const datasetId = v4();
await prisma.dataset.create({
data: {
id: datasetId,
name: v4(),
projectId: projectId,
},
});
const datasetRunId = v4();
await prisma.datasetRuns.create({
data: {
id: datasetRunId,
name: v4(),
datasetId,
metadata: {},
projectId,
},
});
const res = await createDatasetItem({
datasetId,
metadata: {},
projectId,
});
const res2 = await createDatasetItem({
datasetId,
metadata: {},
projectId,
});
if (!res.success || !res2.success) {
throw new Error("Failed to create dataset item");
}
const datasetItemId = res.datasetItem.id;
const datasetItemId2 = res2.datasetItem.id;
const datasetRunItemId1 = v4();
const traceId1 = v4();
const datasetRunItem1 = createDatasetRunItem({
id: datasetRunItemId1,
dataset_run_id: datasetRunId,
observation_id: null,
trace_id: traceId1,
project_id: projectId,
dataset_item_id: datasetItemId,
dataset_id: datasetId,
});
const traceId2 = v4();
const observationId = v4();
const datasetRunItemId2 = v4();
const datasetRunItem2 = createDatasetRunItem({
id: datasetRunItemId2,
dataset_run_id: datasetRunId,
observation_id: observationId,
trace_id: traceId2,
project_id: projectId,
dataset_item_id: datasetItemId2,
dataset_id: datasetId,
});
await createDatasetRunItemsCh([datasetRunItem1, datasetRunItem2]);
const trace1 = createTrace({
id: traceId1,
project_id: projectId,
});
const trace2 = createTrace({
id: traceId2,
project_id: projectId,
});
await createTracesCh([trace1, trace2]);
const observation = createObservation({
id: observationId,
trace_id: traceId2,
project_id: projectId,
start_time: new Date().getTime() - 1000 * 60 * 60, // minus 1 min
end_time: new Date().getTime(),
});
const observation2 = createObservation({
trace_id: traceId1,
});
await createObservationsCh([observation]);
const score = createTraceScore({
observation_id: observation2.id,
trace_id: traceId2,
project_id: projectId,
});
await createScoresCh([score]);
const runItems = await getDatasetRunItemsByDatasetIdCh({
projectId: projectId,
datasetId: datasetId,
filter: [],
orderBy: [
{
column: "createdAt",
order: "ASC",
},
],
limit: 10,
offset: 0,
});
const enrichedRunItems = await getRunItemsByRunIdOrItemId(
projectId,
runItems,
);
expect(enrichedRunItems).toHaveLength(2);
const firstRunItem = enrichedRunItems.find(
(runItem) => runItem.id === datasetRunItemId1,
);
expect(firstRunItem).toBeDefined();
if (!firstRunItem) {
throw new Error("first run item is not defined");
}
expect(firstRunItem.id).toEqual(datasetRunItemId1);
expect(firstRunItem.datasetItemId).toEqual(datasetItemId);
expect(firstRunItem.observation).toBeUndefined();
expect(firstRunItem.trace).toBeDefined();
expect(firstRunItem.trace?.id).toEqual(traceId1);
const secondRunItem = enrichedRunItems.find(
(runItem) => runItem.id === datasetRunItemId2,
);
expect(secondRunItem).toBeDefined();
if (!secondRunItem) {
throw new Error("second run item is not defined");
}
expect(secondRunItem.id).toEqual(datasetRunItemId2);
expect(secondRunItem.datasetItemId).toEqual(datasetItemId2);
expect(secondRunItem.trace?.id).toEqual(traceId2);
expect(secondRunItem.observation?.id).toEqual(observationId);
const expectedObject = {
[`${score.name.replaceAll("-", "_")}-API-NUMERIC`]: {
id: score.id,
type: "NUMERIC",
values: expect.arrayContaining([100.5]),
average: 100.5,
comment: "comment",
timestamp: expect.any(Date),
// createScore adds metadata to the score
hasMetadata: true,
executionTraceId: null,
},
};
expect(secondRunItem.scores).toEqual(expectedObject);
});
describe("Dataset Run Item Compare Data", () => {
describe("Compare data without filters", () => {
let datasetId: string;
let run1Id: string;
let run2Id: string;
let run3Id: string;
let itemIds: string[];
let traceIds: string[];
const accuracyKey = composeAggregateScoreKey({
name: "accuracy",
source: "API",
dataType: "NUMERIC",
});
const relevanceKey = composeAggregateScoreKey({
name: "relevance",
source: "API",
dataType: "NUMERIC",
});
beforeEach(async () => {
datasetId = v4();
run1Id = v4();
run2Id = v4();
run3Id = v4();
itemIds = [v4(), v4(), v4(), v4()];
traceIds = [v4(), v4(), v4(), v4()];
// Create dataset
await prisma.dataset.create({
data: {
id: datasetId,
name: `compare-test-dataset-${datasetId}`,
projectId: projectId,
},
});
// Create dataset runs
await prisma.datasetRuns.createMany({
data: [
{
id: run1Id,
name: "run-1",
datasetId,
metadata: { purpose: "baseline" },
projectId,
createdAt: new Date("2024-01-01T00:00:00Z"),
},
{
id: run2Id,
name: "run-2",
datasetId,
metadata: { purpose: "experiment" },
projectId,
createdAt: new Date("2024-01-02T00:00:00Z"),
},
{
id: run3Id,
name: "run-3",
datasetId,
metadata: { purpose: "validation" },
projectId,
createdAt: new Date("2024-01-03T00:00:00Z"),
},
],
});
// Create dataset items with same timestamp but different IDs for pagination testing
await createManyDatasetItems({
projectId,
items: itemIds.map((id, index) => ({
id,
datasetId,
input: { prompt: `Test input ${index + 1}` },
expectedOutput: { response: `Expected output ${index + 1}` },
metadata: {
category: index < 2 ? "category-a" : "category-b",
difficulty: index % 2 === 0 ? "easy" : "hard",
},
})),
});
// Create traces
const traces = traceIds.map((traceId, index) =>
createTrace({
id: traceId,
project_id: projectId,
name: `trace-${index + 1}`,
timestamp: new Date().getTime() - (4 - index) * 1000,
metadata: { test: `trace-${index + 1}` },
}),
);
await createTracesCh(traces);
// Create observations for some traces (mix of trace-level and observation-level linkage)
const parentObsId = v4();
const childObs1Id = v4();
const childObs2Id = v4();
const observations = [
// First trace: parent observation with 2 children
createObservation({
id: parentObsId,
trace_id: traceIds[0],
project_id: projectId,
type: "GENERATION",
name: "parent-generation",
parent_observation_id: null,
start_time: new Date().getTime() - 3500,
end_time: new Date().getTime() - 2500,
metadata: { model: "gpt-4" },
cost_details: { total: 0 }, // Parent has no direct cost
}),
createObservation({
id: childObs1Id,
trace_id: traceIds[0],
project_id: projectId,
type: "GENERATION",
name: "child-generation-1",
parent_observation_id: parentObsId,
start_time: new Date().getTime() - 3400,
end_time: new Date().getTime() - 3000,
cost_details: { total: 0.005 }, // Child 1 cost
}),
createObservation({
id: childObs2Id,
trace_id: traceIds[0],
project_id: projectId,
type: "GENERATION",
name: "child-generation-2",
parent_observation_id: parentObsId,
start_time: new Date().getTime() - 3000,
end_time: new Date().getTime() - 2600,
cost_details: { total: 0.008765 }, // Child 2 cost (total = 0.013765)
}),
createObservation({
trace_id: traceIds[1],
project_id: projectId,
type: "GENERATION",
name: "generation-2",
start_time: new Date().getTime() - 2500,
end_time: new Date().getTime() - 1500,
metadata: { model: "gpt-3.5" },
}),
// Third trace has observation but run item will link to trace (trace-level linkage)
createObservation({
trace_id: traceIds[2],
project_id: projectId,
type: "GENERATION",
name: "generation-3",
start_time: new Date().getTime() - 1500,
end_time: new Date().getTime() - 500,
metadata: { model: "claude" },
}),
// Fourth trace: no observation (trace-level linkage only)
];
await createObservationsCh(observations);
// Create dataset run items - mix of trace-level and observation-level linkage
const runItems = [
// Run 1: All items, mix of trace and observation linkage
createDatasetRunItem({
id: v4(),
dataset_run_id: run1Id,
dataset_item_id: itemIds[0],
trace_id: traceIds[0],
observation_id: parentObsId, // Link to parent observation (has children)
project_id: projectId,
dataset_id: datasetId,
dataset_run_name: "run-1",
}),
createDatasetRunItem({
id: v4(),
dataset_run_id: run1Id,
dataset_item_id: itemIds[1],
trace_id: traceIds[1],
observation_id: null, // Trace-level linkage
project_id: projectId,
dataset_id: datasetId,
dataset_run_name: "run-1",
}),
createDatasetRunItem({
id: v4(),
dataset_run_id: run1Id,
dataset_item_id: itemIds[2],
trace_id: traceIds[2],
observation_id: null, // Trace-level linkage
project_id: projectId,
dataset_id: datasetId,
dataset_run_name: "run-1",
}),
// Run 2: Partial items, observation-level linkage
createDatasetRunItem({
id: v4(),
dataset_run_id: run2Id,
dataset_item_id: itemIds[0],
trace_id: traceIds[0],
observation_id: observations[0].id, // Observation-level linkage
project_id: projectId,
dataset_id: datasetId,
dataset_run_name: "run-2",
}),
createDatasetRunItem({
id: v4(),
dataset_run_id: run2Id,
dataset_item_id: itemIds[3],
trace_id: traceIds[3],
observation_id: null, // Trace-level linkage (no observation exists)
project_id: projectId,
dataset_id: datasetId,
dataset_run_name: "run-2",
}),
// Run 3: Single item, trace-level linkage
createDatasetRunItem({
id: v4(),
dataset_run_id: run3Id,
dataset_item_id: itemIds[1],
trace_id: traceIds[1],
observation_id: null, // Trace-level linkage
project_id: projectId,
dataset_id: datasetId,
dataset_run_name: "run-3",
}),
];
await createDatasetRunItemsCh(runItems);
// Create scores for traces
const scores = [
createTraceScore({
id: v4(),
trace_id: traceIds[0],
project_id: projectId,
name: "accuracy",
value: 0.95,
source: "API",
}),
createTraceScore({
id: v4(),
trace_id: traceIds[0],
project_id: projectId,
name: "relevance",
value: 0.88,
source: "API",
}),
createTraceScore({
id: v4(),
trace_id: traceIds[1],
project_id: projectId,
name: "accuracy",
value: 0.72,
source: "API",
}),
createTraceScore({
id: v4(),
trace_id: traceIds[2],
project_id: projectId,
name: "accuracy",
value: 0.81,
source: "API",
}),
createTraceScore({
id: v4(),
trace_id: traceIds[3],
project_id: projectId,
name: "relevance",
value: 0.65,
source: "API",
}),
];
await createScoresCh(scores);
});
it("should return correct data structure with no pagination", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [],
limit: 100,
offset: 0,
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Should be a record, not an array
expect(typeof result).toBe("object");
expect(Array.isArray(result)).toBe(false);
// Should have the correct nested structure: datasetItemId -> runId -> enriched data
expect(Array.from(result.keys())).toEqual(
expect.arrayContaining(itemIds),
);
// Check specific dataset items have the correct run data
// Item 0: Should have data from run1 and run2
expect(result.get(itemIds[0])).toBeDefined();
expect(Object.keys(result.get(itemIds[0]) ?? {})).toEqual(
expect.arrayContaining([run1Id, run2Id]),
);
expect(result.get(itemIds[0])?.[run3Id]).toBeUndefined(); // Not in run3
// Item 1: Should have data from run1 and run3
expect(result.get(itemIds[1])).toBeDefined();
expect(Object.keys(result.get(itemIds[1]) ?? {})).toEqual(
expect.arrayContaining([run1Id, run3Id]),
);
expect(result.get(itemIds[1])?.[run2Id]).toBeUndefined(); // Not in run2
// Item 2: Should have data from run1 only
expect(result.get(itemIds[2])).toBeDefined();
expect(Object.keys(result.get(itemIds[2]) ?? {})).toEqual([run1Id]);
// Item 3: Should have data from run2 only
expect(result.get(itemIds[3])).toBeDefined();
expect(Object.keys(result.get(itemIds[3]) ?? {})).toEqual([run2Id]);
});
it("should include correct enriched data (scores, latency, costs)", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [],
limit: 100,
offset: 0,
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Test enriched data for item 0, run 1 (has observation-level linkage + scores)
const item0Run1 = result.get(itemIds[0])?.[run1Id];
expect(item0Run1).toBeDefined();
expect(item0Run1?.datasetRunId).toBe(run1Id);
expect(item0Run1?.datasetItemId).toBe(itemIds[0]);
// Should have observation data (observation-level linkage)
expect(item0Run1?.observation).toBeDefined();
expect(item0Run1?.observation?.latency).toBeDefined();
expect(typeof item0Run1?.observation?.latency).toBe("number");
expect(item0Run1?.observation?.calculatedTotalCost).toBeDefined();
// Should calculate recursive cost (parent + 2 children: 0 + 0.005 + 0.008765 = 0.013765)
expect(
item0Run1?.observation?.calculatedTotalCost?.toNumber(),
).toBeCloseTo(0.013765, 6);
// Should have trace data
expect(item0Run1?.trace).toBeDefined();
expect(item0Run1?.trace?.id).toBe(traceIds[0]);
expect(typeof item0Run1?.trace?.duration).toBe("number");
expect(typeof item0Run1?.trace?.totalCost).toBe("number");
// Should have scores (accuracy: 0.95, relevance: 0.88)
expect(item0Run1?.scores).toBeDefined();
expect(item0Run1?.scores?.[accuracyKey]).toBeDefined();
expect((item0Run1?.scores?.[accuracyKey] as any)?.average).toBe(0.95);
expect(item0Run1?.scores?.[relevanceKey]).toBeDefined();
expect((item0Run1?.scores?.[relevanceKey] as any)?.average).toBe(0.88);
});
it("should handle trace-level vs observation-level linkage correctly", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [],
limit: 100,
offset: 0,
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Item 0, Run 1: Observation-level linkage (should have observation data)
const item0Run1 = result.get(itemIds[0])?.[run1Id];
expect(item0Run1?.observation).toBeDefined();
expect(item0Run1?.observation?.id).toBeDefined();
// Item 1, Run 1: Trace-level linkage (should not have observation data)
const item1Run1 = result.get(itemIds[1])?.[run1Id];
expect(item1Run1?.observation).toBeUndefined();
// But should still have trace data
expect(item1Run1?.trace).toBeDefined();
expect(item1Run1?.trace?.id).toBe(traceIds[1]);
// Item 3, Run 2: Trace-level linkage, no observation exists for this trace
const item3Run2 = result.get(itemIds[3])?.[run2Id];
expect(item3Run2?.observation).toBeUndefined();
expect(item3Run2?.trace).toBeDefined();
expect(item3Run2?.trace?.id).toBe(traceIds[3]);
});
it("should include correct scores for different traces", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [],
limit: 100,
offset: 0,
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Trace 0: accuracy: 0.95, relevance: 0.88
const item0Run1 = result.get(itemIds[0])?.[run1Id];
expect((item0Run1?.scores?.[accuracyKey] as any)?.average).toBe(0.95);
expect((item0Run1?.scores?.[relevanceKey] as any)?.average).toBe(0.88);
// Trace 1: accuracy: 0.72
const item1Run1 = result.get(itemIds[1])?.[run1Id];
expect((item1Run1?.scores?.[accuracyKey] as any)?.average).toBe(0.72);
expect(item1Run1?.scores?.[relevanceKey]).toBeUndefined();
// Trace 2: accuracy: 0.81
const item2Run1 = result.get(itemIds[2])?.[run1Id];
expect((item2Run1?.scores?.[accuracyKey] as any)?.average).toBe(0.81);
// Trace 3: relevance: 0.65
const item3Run2 = result.get(itemIds[3])?.[run2Id];
expect((item3Run2?.scores?.[relevanceKey] as any)?.average).toBe(0.65);
expect(item3Run2?.scores?.[accuracyKey]).toBeUndefined();
});
it("should handle latency calculations correctly", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [],
limit: 100,
offset: 0,
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Test observation-level latency (should be based on observation start/end time)
const item0Run1 = result.get(itemIds[0])?.[run1Id];
expect(item0Run1?.observation?.latency).toBeDefined();
// Observation 1: 3500ms - 2500ms = 1000ms latency, which translates to 1 second
expect(item0Run1?.observation?.latency).toBeCloseTo(1);
const item0Run2 = result.get(itemIds[0])?.[run2Id];
expect(item0Run2?.observation?.latency).toBeDefined();
expect(item0Run2?.observation?.latency).toBeCloseTo(1); // Same observation
// Test trace-level latency (calculated from trace timestamps and observations)
const item1Run1 = result.get(itemIds[1])?.[run1Id];
expect(item1Run1?.trace?.duration).toBeDefined();
expect(typeof item1Run1?.trace?.duration).toBe("number");
});
it("should handle cost calculations correctly", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [],
limit: 100,
offset: 0,
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Test that cost fields are present and properly typed
result.forEach((runData) => {
Object.values(runData).forEach((enrichedItem) => {
// Trace costs
expect(typeof enrichedItem?.trace?.totalCost).toBe("number");
// Observation costs (if observation exists)
if (enrichedItem?.observation) {
expect(
enrichedItem?.observation?.calculatedTotalCost,
).toBeDefined();
expect(
enrichedItem?.observation?.calculatedTotalCost?.constructor
?.name,
).toBe("Decimal");
}
});
});
});
it("should filter by specific runs correctly", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [],
limit: 100,
offset: 0,
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id], // Only first two runs
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Should not have any data from run3 across all dataset items
for (const datasetItemToRunData of result.values()) {
for (const runId of Object.keys(datasetItemToRunData)) {
expect(runId).not.toBe(run3Id);
}
}
// But should have data from run1 and run2 where applicable
expect(result.get(itemIds[0])?.[run1Id]).toBeDefined();
expect(result.get(itemIds[0])?.[run2Id]).toBeDefined();
expect(result.get(itemIds[1])?.[run1Id]).toBeDefined();
expect(result.get(itemIds[1])?.[run2Id]).toBeUndefined(); // Item 1 not in run 2
});
it("should handle empty dataset item list", async () => {
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId,
datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds: [], // Empty list
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
expect(result).toEqual(new Map());
});
it("should handle non-existent dataset items", async () => {
const nonExistentItemId = v4();
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId,
datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds: [nonExistentItemId],
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
expect(result).toEqual(new Map());
});
it("should handle non-existent runs", async () => {
const nonExistentRunId = v4();
const itemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId: datasetId,
runIds: [nonExistentRunId],
filterByRun: [],
limit: 100,
offset: 0,
});
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId,
datasetId,
runIds: [nonExistentRunId],
datasetItemIds: itemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
expect(result).toEqual(new Map());
});
it("should preserve created timestamps and metadata", async () => {
const itemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [],
limit: 100,
offset: 0,
});
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId,
datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds: itemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Test that timestamps are preserved
result.forEach((runData) => {
Object.values(runData).forEach((enrichedItem) => {
expect(enrichedItem.createdAt).toBeInstanceOf(Date);
expect(enrichedItem.id).toBeDefined();
expect(typeof enrichedItem.id).toBe("string");
expect(enrichedItem.datasetItemId).toBeDefined();
expect(enrichedItem.datasetRunId).toBeDefined();
});
});
});
});
describe("Compare data with filters", () => {
let datasetId: string;
let run1Id: string;
let run2Id: string;
let run3Id: string;
let itemIds: string[];
let run1TraceIds: string[];
let run2TraceIds: string[];
let run3TraceIds: string[];
const accuracyKey = composeAggregateScoreKey({
name: "accuracy",
source: "API",
dataType: "NUMERIC",
});
const relevanceKey = composeAggregateScoreKey({
name: "relevance",
source: "API",
dataType: "NUMERIC",
});
beforeEach(async () => {
datasetId = v4();
run1Id = v4();
run2Id = v4();
run3Id = v4();
itemIds = [v4(), v4(), v4(), v4(), v4()];
run1TraceIds = [v4(), v4(), v4(), v4(), v4()];
run2TraceIds = [v4(), v4(), v4(), v4(), v4()];
run3TraceIds = [v4(), v4(), v4(), v4(), v4()];
// Create dataset
await prisma.dataset.create({
data: {
id: datasetId,
name: `filtered-compare-test-dataset-${datasetId}`,
projectId: projectId,
},
});
// Create dataset runs
await prisma.datasetRuns.createMany({
data: [
{
id: run1Id,
name: "high-accuracy-run",
datasetId,
metadata: { model: "gpt-4" },
projectId,
createdAt: new Date("2024-01-01T00:00:00Z"),
},
{
id: run2Id,
name: "mixed-performance-run",
datasetId,
metadata: { model: "gpt-3.5" },
projectId,
createdAt: new Date("2024-01-02T00:00:00Z"),
},
{
id: run3Id,
name: "quality-focused-run",
datasetId,
metadata: { model: "claude" },
projectId,
createdAt: new Date("2024-01-03T00:00:00Z"),
},
],
});
// Create dataset items
await createManyDatasetItems({
projectId,
items: itemIds.map((id, index) => ({
id,
datasetId,
input: { prompt: `Test prompt ${index + 1}` },
expectedOutput: { response: `Expected response ${index + 1}` },
metadata: { category: `category-${index % 3}` },
})),
});
// Create traces
const traces = [...run1TraceIds, ...run2TraceIds, ...run3TraceIds].map(
(traceId, index) =>
createTrace({
id: traceId,
project_id: projectId,
name: `filtered-trace-${index + 1}`,
timestamp: new Date().getTime() - (5 - index) * 1000,
}),
);
await createTracesCh(traces);
// Create observations for latency calculations
const observations = [
...run1TraceIds,
...run2TraceIds,
...run3TraceIds,
].map((traceId, index) =>
createObservation({
trace_id: traceId,
project_id: projectId,
type: "GENERATION",
start_time: new Date().getTime() - (index + 1) * 2000,
end_time: new Date().getTime() - (index + 1) * 1000,
total_cost: (index + 1) * 10,
}),
);
await createObservationsCh(observations);
// Create dataset run items - ALL runs have ALL 5 items
const runItems = [
// Run 1: All 5 items (high accuracy run)
...itemIds.map((itemId, index) =>
createDatasetRunItem({
id: v4(),
dataset_run_id: run1Id,
dataset_item_id: itemId,
trace_id: run1TraceIds[index],
observation_id: null,
project_id: projectId,
dataset_id: datasetId,
dataset_run_name: "high-accuracy-run",
}),
),
// Run 2: All 5 items (mixed performance)
...itemIds.map((itemId, index) =>
createDatasetRunItem({
id: v4(),
dataset_run_id: run2Id,
dataset_item_id: itemId,
trace_id: run2TraceIds[index],
observation_id: null,
project_id: projectId,
dataset_id: datasetId,
dataset_run_name: "mixed-performance-run",
}),
),
// Run 3: All 5 items (quality focused)
...itemIds.map((itemId, index) =>
createDatasetRunItem({
id: v4(),
dataset_run_id: run3Id,
dataset_item_id: itemId,
trace_id: run3TraceIds[index],
observation_id: null,
project_id: projectId,
dataset_id: datasetId,
dataset_run_name: "quality-focused-run",
}),
),
];
await createDatasetRunItemsCh(runItems);
// Create scores with different patterns for filtering
const scores: any[] = [];
const runs = [
{ id: run1Id, traceIds: run1TraceIds },
{ id: run2Id, traceIds: run2TraceIds },
{ id: run3Id, traceIds: run3TraceIds },
];
runs.forEach((run, runIndex) => {
const traceSubset = run.traceIds; // Use all 5 traces for each run
// Add accuracy scores for each run with different base values
const accuracyBase = 0.6 + runIndex * 0.15; // 0.6, 0.75, 0.9
traceSubset.forEach((traceId, traceIndex) => {
scores.push(
createTraceScore({
id: v4(),
trace_id: traceId,
project_id: projectId,
name: "accuracy",
value: accuracyBase + traceIndex * 0.05, // Incremental within run
source: "API",
}),
);
});
// Add secondary numeric scores with variation per run
const secondaryScores = ["relevance", "precision", "fluency"];
const secondaryBase = 0.5 + runIndex * 0.1; // 0.5, 0.6, 0.7
traceSubset.forEach((traceId, traceIndex) => {
scores.push(
createTraceScore({
id: v4(),
trace_id: traceId,
project_id: projectId,
name: secondaryScores[runIndex],
value: secondaryBase + traceIndex * 0.08,
source: "API",
}),
);
});
// Add categorical scores with different categories per run
const categoricalConfigs = [
{
name: "quality",
values: ["excellent", "good", "average", "good", "excellent"],
},
{
name: "sentiment",
values: [
"positive",
"neutral",
"negative",
"positive",
"neutral",
],
},
{
name: "language",
values: ["formal", "casual", "technical", "formal", "casual"],
},
];
traceSubset.forEach((traceId, traceIndex) => {
scores.push(
createTraceScore({
id: v4(),
trace_id: traceId,
project_id: projectId,
name: categoricalConfigs[runIndex].name,
data_type: "CATEGORICAL",
string_value: categoricalConfigs[runIndex].values[traceIndex],
source: "API",
}),
);
});
});
await createScoresCh(scores);
});
it("should return intersection of items across runs with different filters (intersection complexity)", async () => {
// Test case 1: Filter run1 for high accuracy items, filter run2 for specific items
// Should only return items that exist in both runs AND meet both filter criteria
// Step 1: Return dataset item ids for which the run items match the filters
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id],
filterByRun: [
{
runId: run1Id,
filters: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">=" as const,
value: 0.65,
},
],
},
{
runId: run2Id,
filters: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">=" as const,
value: 0.8,
},
],
},
],
limit: 100,
offset: 0,
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id],
datasetItemIds: datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Run1 has items 0,1,2,3,4 with accuracy 0.6,0.65,0.7,0.75,0.8 - filter >= 0.65 includes items 1,2,3,4
// Run2 has items 0,1,2,3,4 with accuracy 0.75,0.8,0.85,0.9,0.95 - filter >= 0.8 includes items 1,2,3,4
// Intersection: items that exist in BOTH runs AND meet BOTH criteria = items 1,2,3,4
expect(Array.from(result.keys())).toHaveLength(4);
expect(Array.from(result.keys())).toEqual(
expect.arrayContaining([
itemIds[1],
itemIds[2],
itemIds[3],
itemIds[4],
]),
);
// Verify each item has data from both runs
result.forEach((runData) => {
expect(Object.keys(runData)).toHaveLength(2);
expect(Object.keys(runData)).toEqual(
expect.arrayContaining([run1Id, run2Id]),
);
});
// Verify accuracy values meet filter criteria
const item1Run1 = result.get(itemIds[1])?.[run1Id];
const item1Run2 = result.get(itemIds[1])?.[run2Id];
expect(
(item1Run1?.scores?.[accuracyKey] as any)?.average,
).toBeGreaterThanOrEqual(0.65);
expect(
(item1Run2?.scores?.[accuracyKey] as any)?.average,
).toBeGreaterThanOrEqual(0.8);
});
it("should return empty result when intersection is empty (no items meet all run filter criteria)", async () => {
// Test case: Apply very strict filters that have no intersection
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
limit: 100,
offset: 0,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [
{
runId: run1Id,
filters: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">=" as const,
value: 0.8, // Only item 4 in run1
},
],
},
{
runId: run2Id,
filters: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">=" as const,
value: 0.85, // Items 2,3 in run2
},
],
},
{
runId: run3Id,
filters: [
{
type: "categoryOptions" as const,
column: "agg_score_categories",
key: "language",
operator: "any of" as const,
value: ["technical"], // Only item 2 in run3
},
],
},
],
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Run1 accuracy >= 0.8: only item 4 (accuracy 0.8)
// Run2 accuracy >= 0.85: items 2,3,4 (accuracy 0.85, 0.9, 0.95)
// Run3 language = "technical": only item 2 (language "technical")
// Intersection: no item meets ALL three criteria
expect(result).toEqual(new Map());
});
it("should handle intersection with mixed filter types across runs", async () => {
// Test case: Combine numeric, categorical, and empty filters across multiple runs
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
limit: 100,
offset: 0,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [
{
runId: run1Id,
filters: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">=" as const,
value: 0.65, // Items 1,2,3,4 meet this
},
],
},
{
runId: run2Id,
filters: [], // No filters - all items in run2 qualify (items 0,1,2,3)
},
{
runId: run3Id,
filters: [
{
type: "categoryOptions" as const,
column: "agg_score_categories",
key: "language",
operator: "none of" as const,
value: ["technical"], // Exclude item 2, so items 0,1 qualify
},
],
},
],
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Intersection analysis:
// Run1 qualifies: items 1,2,3,4 (accuracy >= 0.65: 0.65, 0.7, 0.75, 0.8)
// Run2 qualifies: items 0,1,2,3,4 (no filter - all items)
// Run3 qualifies: items 0,1,3,4 (language != "technical": excludes item 2 which is "technical")
// Intersection: items 1,3,4 (items that exist in ALL runs AND meet ALL criteria)
expect(Array.from(result.keys())).toHaveLength(3);
expect(Array.from(result.keys())).toEqual(
expect.arrayContaining([itemIds[1], itemIds[3], itemIds[4]]),
);
// Verify each qualifying item has data from all three runs
result.forEach((runData) => {
expect(Object.keys(runData)).toHaveLength(3);
expect(Object.keys(runData)).toEqual(
expect.arrayContaining([run1Id, run2Id, run3Id]),
);
});
});
it("should handle intersection with partially overlapping runs (some runs have items, others don't)", async () => {
// Test case: Create scenario where some runs have certain items and others don't
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
limit: 100,
offset: 0,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [
{
runId: run1Id,
filters: [], // No filters - all items qualify (items 0,1,2,3,4)
},
{
runId: run2Id,
filters: [], // No filters - all items qualify (items 0,1,2,3)
},
{
runId: run3Id,
filters: [], // No filters - all items qualify (items 0,1,2)
},
],
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// All items exist in ALL three runs now: items 0,1,2,3,4
expect(Array.from(result.keys())).toHaveLength(5);
expect(Array.from(result.keys())).toEqual(
expect.arrayContaining([
itemIds[0],
itemIds[1],
itemIds[2],
itemIds[3],
itemIds[4],
]),
);
// Verify each qualifying item has data from all three runs
result.forEach((runData) => {
expect(Object.keys(runData)).toHaveLength(3);
expect(Object.keys(runData)).toEqual(
expect.arrayContaining([run1Id, run2Id, run3Id]),
);
});
});
it("should properly paginate intersection results", async () => {
// Test case: Verify that pagination works correctly with intersection logic
const firstPageDatasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
limit: 2,
offset: 0,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [
{ runId: run1Id, filters: [] },
{ runId: run2Id, filters: [] },
],
});
const secondPageDatasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
limit: 2,
offset: 2,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [
{ runId: run1Id, filters: [] },
{ runId: run2Id, filters: [] },
],
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItemsFirstPage =
await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds: firstPageDatasetItemIds,
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItemsSecondPage =
await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds: secondPageDatasetItemIds,
});
const firstPageResult = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItemsFirstPage,
);
const secondPageResult = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItemsSecondPage,
);
// Check actual counts - pagination test uses all 3 runs (all have 5 items, no filters)
expect(Array.from(firstPageResult.keys())).toHaveLength(2);
expect(Array.from(secondPageResult.keys())).toHaveLength(2); // Second page: offset=2, limit=2
});
it("should filter single run with single numeric filter", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
limit: 1000,
offset: 0,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [
{
runId: run1Id,
filters: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">=" as const,
value: 0.7,
},
],
},
],
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Should only return items where accuracy >= 0.7 in run1
expect(Array.from(result.keys())).toHaveLength(3);
expect(Array.from(result.keys())).toEqual(
expect.arrayContaining([itemIds[2], itemIds[3], itemIds[4]]),
);
// Verify the returned data
const item2Data = result.get(itemIds[2])?.[run1Id];
expect(item2Data).toBeDefined();
expect((item2Data?.scores?.[accuracyKey] as any)?.average).toBe(0.7);
const item4Data = result.get(itemIds[4])?.[run1Id];
expect(item4Data).toBeDefined();
expect((item4Data?.scores?.[accuracyKey] as any)?.average).toBe(0.8);
});
it("should filter multiple runs with different numeric filters", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
limit: 100,
offset: 0,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [
{
runId: run1Id,
filters: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">=" as const,
value: 0.7,
},
],
},
{
runId: run2Id,
filters: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">=" as const,
value: 0.75,
},
],
},
],
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Run 1: Items 2, 3, 4 (accuracy >= 0.7: 0.7, 0.75, 0.8)
// Run 2: Items 0, 1, 2, 3, 4 (accuracy >= 0.75: 0.75, 0.8, 0.85, 0.9, 0.95)
// Intersection: items 2, 3, 4 (meet both criteria)
expect(Array.from(result.keys())).toEqual(
expect.arrayContaining([itemIds[2], itemIds[3], itemIds[4]]),
);
// Check run1 data (items 2, 3, 4)
expect(result.get(itemIds[2])?.[run1Id]).toBeDefined();
expect(result.get(itemIds[3])?.[run1Id]).toBeDefined();
expect(result.get(itemIds[4])?.[run1Id]).toBeDefined();
// Check run2 data (items 2, 3, 4 should have run2 data)
expect(result.get(itemIds[2])?.[run2Id]).toBeDefined();
expect(result.get(itemIds[3])?.[run2Id]).toBeDefined();
expect(result.get(itemIds[4])?.[run2Id]).toBeDefined();
// Verify accuracy values
expect(
(result.get(itemIds[2])?.[run1Id]?.scores?.[accuracyKey] as any)
?.average,
).toBe(0.7);
expect(
(result.get(itemIds[2])?.[run2Id]?.scores?.[accuracyKey] as any)
?.average,
).toBe(0.85);
});
it("should filter single run with multiple numeric filters (AND condition)", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
limit: 100,
offset: 0,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [
{
runId: run1Id,
filters: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">=" as const,
value: 0.75,
},
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "relevance",
operator: ">=" as const,
value: 0.74,
},
],
},
],
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Should only return items where accuracy >= 0.75 AND relevance >= 0.74
// Run1 accuracy: 0.6, 0.65, 0.7, 0.75, 0.8 (items 3, 4 meet accuracy >= 0.75)
// Run1 relevance: 0.5, 0.58, 0.66, 0.74, 0.82 (items 3, 4 meet relevance >= 0.74)
expect(Array.from(result.keys())).toHaveLength(2);
expect(Array.from(result.keys())).toEqual(
expect.arrayContaining([itemIds[3], itemIds[4]]),
);
// Verify both conditions are met
const item3Data = result.get(itemIds[3])?.[run1Id];
expect((item3Data?.scores?.[accuracyKey] as any)?.average).toBe(0.75);
expect((item3Data?.scores?.[relevanceKey] as any)?.average).toBe(0.74);
const item4Data = result.get(itemIds[4])?.[run1Id];
expect((item4Data?.scores?.[accuracyKey] as any)?.average).toBe(0.8);
expect((item4Data?.scores?.[relevanceKey] as any)?.average).toBeCloseTo(
0.82,
);
});
it("should filter multiple runs with multiple filters each", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
limit: 100,
offset: 0,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [
{
runId: run1Id,
filters: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">=" as const,
value: 0.75,
},
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "relevance",
operator: ">=" as const,
value: 0.74,
},
],
},
{
runId: run3Id,
filters: [
{
type: "categoryOptions" as const,
column: "agg_score_categories",
key: "language",
operator: "any of" as const,
value: ["formal", "casual"],
},
],
},
],
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Run 1: Items where accuracy >= 0.75 AND relevance >= 0.74 (items 3, 4)
// Run 2: No filter - all items qualify (items 0, 1, 2, 3, 4)
// Run 3: Items where language is "formal" or "casual" (items 0, 1, 3, 4)
// Intersection: items 3, 4 (items that meet ALL criteria across ALL runs)
expect(Array.from(result.keys())).toHaveLength(2);
expect(Array.from(result.keys())).toEqual(
expect.arrayContaining([itemIds[3], itemIds[4]]),
);
});
it("should filter with categorical filters", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
limit: 100,
offset: 0,
runIds: [run1Id, run2Id, run3Id], // All runs for intersection
filterByRun: [
{
runId: run3Id,
filters: [
{
type: "categoryOptions" as const,
column: "agg_score_categories",
key: "language",
operator: "none of" as const,
value: ["technical"],
},
],
},
],
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Intersection logic: items must exist in ALL runs AND meet filter criteria
// Run1: no filter → items 0,1,2,3,4 qualify
// Run2: no filter → items 0,1,2,3,4 qualify
// Run3: language != "technical" → items 0,1,3,4 qualify (excludes item 2 with "technical")
// Intersection: items 0,1,3,4 (4 items)
expect(Array.from(result.keys())).toHaveLength(4);
expect(Array.from(result.keys())).toEqual(
expect.arrayContaining([
itemIds[0],
itemIds[1],
itemIds[3],
itemIds[4],
]),
);
expect(result.get(itemIds[2])).toBeUndefined(); // Has "technical" language, excluded by run3 filter
});
it("should handle filters that match no items", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
limit: 100,
offset: 0,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [
{
runId: run1Id,
filters: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: ">=" as const,
value: 1.0, // No scores are this high
},
],
},
],
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
expect(result).toEqual(new Map());
});
it("should handle filters with non-existent score names", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
limit: 100,
offset: 0,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [
{
runId: run1Id,
filters: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "nonexistent_score",
operator: ">=" as const,
value: 0.5,
},
],
},
],
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
expect(result).toEqual(new Map());
});
it("should handle empty filter list for a run", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
limit: 100,
offset: 0,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [
{
runId: run1Id,
filters: [], // Empty filters should return all items
},
],
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Should return intersection of all runs (items that exist in ALL runs)
// Run1: 5 items (0-4), Run2: 5 items (0-4), Run3: 5 items (0-4)
// Intersection: all items 0, 1, 2, 3, 4 (5 items)
expect(Array.from(result.keys())).toHaveLength(5);
expect(Array.from(result.keys())).toEqual(
expect.arrayContaining([
itemIds[0],
itemIds[1],
itemIds[2],
itemIds[3],
itemIds[4],
]),
);
// Verify each item has data from all three runs
result.forEach((runData) => {
expect(Object.keys(runData)).toHaveLength(3);
expect(Object.keys(runData)).toEqual(
expect.arrayContaining([run1Id, run2Id, run3Id]),
);
});
});
it("should maintain data integrity during complex intersection filtering", async () => {
const datasetItemIds = await getDatasetItemIdsWithRunData({
projectId,
datasetId,
limit: 100,
offset: 0,
runIds: [run1Id, run2Id, run3Id],
filterByRun: [
{
runId: run1Id,
filters: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: "=" as const,
value: 0.7,
},
],
},
{
runId: run2Id,
filters: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: "accuracy",
operator: "=" as const,
value: 0.85,
},
],
},
],
});
// Step 2: Given dataset item ids, lookup dataset run items in clickhouse
// Note: for each unique dataset item id and dataset run id combination, we will retrieve a dataset run item
const datasetRunItems = await getDatasetRunItemsWithoutIOByItemIds({
projectId: projectId,
datasetId: datasetId,
runIds: [run1Id, run2Id, run3Id],
datasetItemIds,
});
const result = await enrichAndMapToDatasetItemId(
projectId,
datasetRunItems,
);
// Verify that the intersection logic preserves correct data for matching items
const item2Run1 = result.get(itemIds[2])?.[run1Id];
const item2Run2 = result.get(itemIds[2])?.[run2Id];
expect(item2Run1).toBeDefined();
expect(item2Run2).toBeDefined();
// Verify accuracy scores match filter exactly (intersection requirement)
expect((item2Run1?.scores?.[accuracyKey] as any)?.average).toBe(0.7);
expect((item2Run2?.scores?.[accuracyKey] as any)?.average).toBe(0.85);
// Verify run metadata is correct for intersected data
expect(item2Run1?.datasetRunId).toBe(run1Id);
expect(item2Run2?.datasetRunId).toBe(run2Id);
// Verify trace data is preserved across intersection
expect(item2Run1?.trace?.id).toBe(run1TraceIds[2]);
expect(item2Run2?.trace?.id).toBe(run2TraceIds[2]);
// Verify that only item 2 is returned (the only one meeting both filters)
expect(Array.from(result.keys())).toEqual([itemIds[2]]);
// Verify intersection logic: item must exist in ALL specified runs with their respective filters
expect(Array.from(result.keys())).toHaveLength(1);
});
});
});
// Regression test for https://github.com/langfuse/langfuse/issues/13788
// The scores_aggregated CTE was scanning the full project scores table (no trace_id filter),
// causing resource limit errors on projects with millions of scores from unrelated datasets.
describe("scores subquery isolation across datasets", () => {
it("getDatasetRunsTableMetricsCh should not include scores from unrelated dataset traces", async () => {
const datasetAId = v4();
const datasetBId = v4();
await prisma.dataset.createMany({
data: [
{ id: datasetAId, name: v4(), projectId },
{ id: datasetBId, name: v4(), projectId },
],
});
const runAId = v4();
const runBId = v4();
await prisma.datasetRuns.createMany({
data: [
{
id: runAId,
name: "run-a",
datasetId: datasetAId,
metadata: {},
projectId,
},
{
id: runBId,
name: "run-b",
datasetId: datasetBId,
metadata: {},
projectId,
},
],
});
const itemAId = v4();
const itemBId = v4();
await prisma.datasetItem.createMany({
data: [
{ id: itemAId, datasetId: datasetAId, metadata: {}, projectId },
{ id: itemBId, datasetId: datasetBId, metadata: {}, projectId },
],
});
const traceAId = v4();
const traceBId = v4();
const scoreName = v4();
await createDatasetRunItemsCh([
createDatasetRunItem({
id: v4(),
dataset_run_id: runAId,
trace_id: traceAId,
project_id: projectId,
dataset_item_id: itemAId,
dataset_id: datasetAId,
dataset_run_name: "run-a",
}),
createDatasetRunItem({
id: v4(),
dataset_run_id: runBId,
trace_id: traceBId,
project_id: projectId,
dataset_item_id: itemBId,
dataset_id: datasetBId,
dataset_run_name: "run-b",
}),
]);
// Score on Dataset A's trace — should appear in Dataset A query
const scoreForA = createTraceScore({
id: v4(),
trace_id: traceAId,
project_id: projectId,
name: scoreName,
value: 0.9,
});
// Score on Dataset B's trace — must NOT bleed into Dataset A query
const scoreForB = createTraceScore({
id: v4(),
trace_id: traceBId,
project_id: projectId,
name: scoreName,
value: 0.1,
});
await createScoresCh([scoreForA, scoreForB]);
const runsA = await getDatasetRunsTableMetricsCh({
projectId,
datasetId: datasetAId,
runIds: [runAId],
filter: [],
});
expect(runsA).toHaveLength(1);
expect(runsA[0].id).toEqual(runAId);
// Assert directly on aggScoresAvg — the field populated by the scores_aggregated CTE
// that was modified. If the CTE regresses to a full-project scan the join still works
// correctly (UUIDs are unique), but if it ever incorrectly scopes to wrong traces this
// will catch it. Value should be 0.9 (Dataset A only), not averaged with B's 0.1.
const scoreEntry = runsA[0].aggScoresAvg.find(
([name]) => name === scoreName,
);
expect(scoreEntry).toBeDefined();
expect(scoreEntry?.[1]).toBeCloseTo(0.9);
});
it("getDatasetRunItemsByDatasetIdCh should not include scores from unrelated dataset traces", async () => {
const datasetAId = v4();
const datasetBId = v4();
await prisma.dataset.createMany({
data: [
{ id: datasetAId, name: v4(), projectId },
{ id: datasetBId, name: v4(), projectId },
],
});
const runAId = v4();
const runBId = v4();
await prisma.datasetRuns.createMany({
data: [
{
id: runAId,
name: "run-a",
datasetId: datasetAId,
metadata: {},
projectId,
},
{
id: runBId,
name: "run-b",
datasetId: datasetBId,
metadata: {},
projectId,
},
],
});
const itemAId = v4();
const itemBId = v4();
await prisma.datasetItem.createMany({
data: [
{ id: itemAId, datasetId: datasetAId, metadata: {}, projectId },
{ id: itemBId, datasetId: datasetBId, metadata: {}, projectId },
],
});
const traceAId = v4();
const traceBId = v4();
const scoreName = v4();
await createDatasetRunItemsCh([
createDatasetRunItem({
id: v4(),
dataset_run_id: runAId,
trace_id: traceAId,
project_id: projectId,
dataset_item_id: itemAId,
dataset_id: datasetAId,
dataset_run_name: "run-a",
}),
createDatasetRunItem({
id: v4(),
dataset_run_id: runBId,
trace_id: traceBId,
project_id: projectId,
dataset_item_id: itemBId,
dataset_id: datasetBId,
dataset_run_name: "run-b",
}),
]);
const scoreForA = createTraceScore({
id: v4(),
trace_id: traceAId,
project_id: projectId,
name: scoreName,
value: 0.9,
});
const scoreForB = createTraceScore({
id: v4(),
trace_id: traceBId,
project_id: projectId,
name: scoreName,
value: 0.1,
});
await createScoresCh([scoreForA, scoreForB]);
// Filter by score name to exercise the scores_aggregated CTE in getDatasetRunItemsTableInternal
const items = await getDatasetRunItemsByDatasetIdCh({
projectId,
datasetId: datasetAId,
filter: [
{
type: "numberObject" as const,
column: "agg_scores_avg",
key: scoreName,
operator: ">=" as const,
value: 0.8,
},
],
});
// Only Dataset A's run item (score 0.9) should match; B's item (score 0.1) must not appear
expect(items).toHaveLength(1);
expect(items[0].traceId).toEqual(traceAId);
});
});
});