🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { z } from "zod/v4";
import { randomUUID } from "crypto";
import { addDays } from "date-fns";
import {
type ExperimentMetadata,
createDatasetItemFilterState,
ExperimentCreateQueue,
getCategoricalScoresGroupedByName,
getBooleanScoresGroupedByName,
getDatasetItemsCount,
countDatasetItemVariableMatches,
getEventsGroupedByExperimentDatasetId,
getExperimentsCountFromEvents,
getExperimentsFromEvents,
getExperimentItemsBatchIO,
getExperimentItemsCountFromEvents,
getExperimentItemsFromEvents,
getExperimentMetricsFromEvents,
getNumericScoresGroupedByName,
getScoresForExperimentItems,
getScoresForExperiments,
PromptService,
QueueJobs,
QueueName,
redis,
ZodModelConfig,
getScoresForObservations,
getScoresForTraces,
traceException,
getExperimentNamesFromEvents,
getExperimentItemsFilterOptions,
getExperimentScoreOptions,
} from "@langfuse/shared/src/server";
import {
createTRPCRouter,
protectedProjectProcedure,
} from "@/src/server/api/trpc";
import {
extractVariables,
isBaseError,
UnauthorizedError,
PromptType,
extractPlaceholderNames,
type PromptMessage,
singleFilterList,
type FilterState,
orderBy,
paginationZod,
timeFilter,
AGGREGATABLE_SCORE_TYPES,
filterAndValidateDbScoreList,
hasPromptToolStructuredOutputConflict,
InvalidRequestError,
parsePromptToolConfig,
PROMPT_TOOL_STRUCTURED_OUTPUT_CONFLICT_MESSAGE,
} from "@langfuse/shared";
import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
import { aggregateScores } from "@/src/features/scores/lib/aggregateScores";
import { describeVariableMismatch } from "@/src/features/experiments/fns/describeVariableMismatch";
const ExperimentFilterOptions = z.object({
projectId: z.string(),
filter: singleFilterList.nullable(),
orderBy: orderBy,
...paginationZod,
});
/**
* Lookback for `mostRecent`: wide enough to cover a project whose last run is
* months old, still bounded so the query prunes partitions instead of scanning
* the whole project. Deliberately wider than any table date-range preset.
*/
const MOST_RECENT_LOOKBACK_DAYS = 365;
const ValidConfigResponse = z.object({
isValid: z.literal(true),
totalItems: z.number(),
variablesMap: z.record(z.string(), z.number()),
});
const InvalidConfigResponse = z.object({
isValid: z.literal(false),
message: z.string(),
});
const ConfigResponse = z.discriminatedUnion("isValid", [
ValidConfigResponse,
InvalidConfigResponse,
]);
export const experimentsRouter = createTRPCRouter({
validateConfig: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
datasetId: z.string(),
promptId: z.string(),
datasetVersion: z.coerce.date().optional(),
}),
)
.output(ConfigResponse)
.query(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "promptExperiments:CUD",
});
const prompt = await ctx.prisma.prompt.findFirst({
where: {
id: input.promptId,
projectId: input.projectId,
},
});
if (!prompt) {
return {
isValid: false,
message: "Selected prompt not found.",
};
}
const promptService = new PromptService(ctx.prisma, redis);
let resolvedPrompt;
try {
resolvedPrompt = await promptService.resolvePrompt(prompt);
} catch (error) {
if (
error instanceof SyntaxError ||
(isBaseError(error) && error.isUserError())
) {
return {
isValid: false,
message: isBaseError(error)
? error.message
: "Selected prompt could not be resolved.",
};
}
throw error;
}
if (!resolvedPrompt) {
return {
isValid: false,
message: "Selected prompt not found.",
};
}
const extractedVariables = extractVariables(
resolvedPrompt.type === PromptType.Text
? (resolvedPrompt.prompt?.toString() ?? "")
: JSON.stringify(resolvedPrompt.prompt ?? ""),
);
const promptMessages =
resolvedPrompt?.type === PromptType.Chat &&
Array.isArray(resolvedPrompt?.prompt)
? resolvedPrompt.prompt
: [];
const placeholderNames = extractPlaceholderNames(
promptMessages as PromptMessage[],
);
const allVariables = [...extractedVariables, ...placeholderNames];
if (!Boolean(allVariables.length)) {
return {
isValid: false,
message: "Selected prompt has no variables or placeholders.",
};
}
const filterState = createDatasetItemFilterState({
datasetIds: [input.datasetId],
status: "ACTIVE",
});
const totalItems = await getDatasetItemsCount({
projectId: input.projectId,
filterState,
version: input.datasetVersion,
});
if (!Boolean(totalItems)) {
return {
isValid: false,
message: "Selected dataset is empty or all items are inactive.",
};
}
const variablesMap = await countDatasetItemVariableMatches({
projectId: input.projectId,
filterState,
version: input.datasetVersion,
variables: allVariables,
});
if (!Boolean(Object.keys(variablesMap).length)) {
return {
isValid: false,
message: describeVariableMismatch(allVariables),
};
}
return {
isValid: true,
totalItems,
variablesMap,
};
}),
createExperiment: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
name: z.string().min(1, "Please enter an experiment name"),
runName: z.string().min(1, "Run name is required"),
promptId: z.string().min(1, "Please select a prompt"),
datasetId: z.string().min(1, "Please select a dataset"),
datasetVersion: z.coerce.date().optional(),
description: z.string().max(1000).optional(),
modelConfig: z.object({
provider: z.string().min(1, "Please select a provider"),
model: z.string().min(1, "Please select a model"),
modelParams: ZodModelConfig,
}),
structuredOutputSchema: z.record(z.string(), z.any()).optional(),
}),
)
.mutation(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "promptExperiments:CUD",
});
if (input.structuredOutputSchema) {
const prompt = await ctx.prisma.prompt.findUnique({
where: { id: input.promptId, projectId: input.projectId },
select: { config: true },
});
const toolConfig = parsePromptToolConfig(prompt?.config);
if (hasPromptToolStructuredOutputConflict(toolConfig, true)) {
throw new InvalidRequestError(
PROMPT_TOOL_STRUCTURED_OUTPUT_CONFLICT_MESSAGE,
);
}
}
if (!redis) {
throw new UnauthorizedError("Experiment creation failed");
}
const metadata: ExperimentMetadata = {
prompt_id: input.promptId,
provider: input.modelConfig.provider,
model: input.modelConfig.model,
model_params: input.modelConfig.modelParams,
...(input.structuredOutputSchema && {
structured_output_schema: input.structuredOutputSchema,
}),
...(input.datasetVersion && {
dataset_version: input.datasetVersion,
}),
};
const datasetRun = await ctx.prisma.datasetRuns.create({
data: {
name: input.runName,
description: input.description,
datasetId: input.datasetId,
metadata: {
...metadata,
experiment_name: input.name,
experiment_run_name: input.runName,
},
projectId: input.projectId,
},
});
const queue = ExperimentCreateQueue.getInstance();
if (queue) {
await queue.add(QueueName.ExperimentCreate, {
name: QueueJobs.ExperimentCreateJob,
id: randomUUID(),
timestamp: new Date(),
payload: {
projectId: input.projectId,
datasetId: input.datasetId,
runId: datasetRun.id,
description: input.description,
},
retryBaggage: {
originalJobTimestamp: new Date(),
attempt: 0,
},
});
}
return {
success: true,
datasetId: input.datasetId,
runId: datasetRun.id,
runName: input.runName,
};
}),
all: protectedProjectProcedure
.input(ExperimentFilterOptions)
.query(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "promptExperiments:read",
});
const experiments = await getExperimentsFromEvents({
projectId: input.projectId,
filter: input.filter ?? [],
orderBy: input.orderBy,
page: input.page,
limit: input.limit,
});
return {
data: experiments,
};
}),
/**
* The most recent runs regardless of the selected time range, for the empty
* window on the experiments list ("if there's nothing in the last X days, I
* still want to see the last ones"). Same scoping and filters as `all`, only
* the start-time bounds are replaced.
*/
mostRecent: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
filter: singleFilterList.nullable(),
limit: z.number().int().min(1).max(50),
}),
)
.query(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "promptExperiments:read",
});
// Return the server-resolved window so widgets share it without adding
// a render-time timestamp to the fallback query input.
const to = new Date();
const from = addDays(to, -MOST_RECENT_LOOKBACK_DAYS);
const filter: FilterState = [
// The selected window is the one that came back empty; every other
// filter the user applied still holds.
...(input.filter ?? []).filter((f) => f.column !== "startTime"),
{
column: "startTime",
type: "datetime",
operator: ">=",
value: from,
},
];
const experiments = await getExperimentsFromEvents({
projectId: input.projectId,
filter,
orderBy: { column: "startTime", order: "DESC" },
page: 0,
limit: input.limit,
});
return { data: experiments, dateRange: { from, to } };
}),
byId: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
experimentId: z.string(),
}),
)
.query(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "promptExperiments:read",
});
const experiments = await getExperimentsFromEvents({
projectId: input.projectId,
filter: [
{
type: "string",
column: "id",
operator: "=",
value: input.experimentId,
},
],
orderBy: undefined,
page: 0,
limit: 1,
});
if (experiments.length === 0) {
return null;
}
return experiments[0];
}),
countAll: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
filter: singleFilterList.nullable(),
}),
)
.query(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "promptExperiments:read",
});
const count = await getExperimentsCountFromEvents({
projectId: input.projectId,
filter: input.filter ?? [],
});
return {
count: count,
};
}),
metrics: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
experimentIds: z.array(z.string()),
filter: singleFilterList.nullable(),
}),
)
.query(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "promptExperiments:read",
});
if (input.experimentIds.length === 0) {
return [];
}
// Fetch metrics (cost, latency) and both score types in parallel
const [metrics, itemScores, experimentScores] = await Promise.all([
getExperimentMetricsFromEvents({
projectId: input.projectId,
experimentIds: input.experimentIds,
}),
getScoresForExperimentItems(input.projectId, input.experimentIds),
getScoresForExperiments({
projectId: input.projectId,
runIds: input.experimentIds, // experiment_id === dataset_run_id
excludeMetadata: true,
includeHasMetadata: true,
}),
]);
return metrics.map((metric) => {
// Filter item scores for this experiment
const experimentItemScores = itemScores.filter(
(s) => s.experimentId === metric.id,
);
return {
id: metric.id,
totalCost: metric.totalCost,
latencyAvg: metric.latencyAvg,
// Trace-level item scores (observation_id is null/empty)
traceItemScores: aggregateScores(
experimentItemScores.filter((s) => !s.observationId),
),
// Observation-level item scores (observation_id is present)
observationItemScores: aggregateScores(
experimentItemScores.filter((s) => Boolean(s.observationId)),
),
// Experiment-level scores (direct dataset_run_id match)
experimentScores: aggregateScores(
experimentScores.filter((s) => s.datasetRunId === metric.id),
),
};
});
}),
filterOptions: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
startTimeFilter: z.array(timeFilter).optional(),
}),
)
.query(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "promptExperiments:read",
});
// Map startTimeFilter to Timestamp column for trace queries
const traceTimestampFilters =
input.startTimeFilter && input.startTimeFilter.length > 0
? input.startTimeFilter.map((f) => ({
column: "Timestamp" as const,
operator: f.operator,
value: f.value,
type: "datetime" as const,
}))
: [];
// Offered-set == matchable-set: the level-agnostic score filters match a
// score only if it rolls up into a trace, so discovery is scoped the same
// way. Session- and dataset-run scores can never match, and offering them
// would advertise a filter that cannot fire.
const nullCondition = (
column: "traceId" | "observationId",
operator: "is null" | "is not null",
) => ({ type: "null" as const, column, operator, value: "" as const });
const traceScopedFilter = [
nullCondition("traceId", "is not null"),
...(traceTimestampFilters ?? []),
];
// Discovered per level as well, to tag each offered name with the level(s)
// it exists at rather than duplicating the facets per level.
const observationScopedFilter = [
...traceScopedFilter,
nullCondition("observationId", "is not null"),
];
const traceLevelScopedFilter = [
...traceScopedFilter,
nullCondition("observationId", "is null"),
];
const [
numericScoreNames,
categoricalScoreNames,
booleanScoreNames,
observationLevelNumeric,
observationLevelCategorical,
observationLevelBoolean,
traceLevelNumeric,
traceLevelCategorical,
traceLevelBoolean,
experimentDatasetIds,
] = await Promise.all([
getNumericScoresGroupedByName(input.projectId, traceScopedFilter),
getCategoricalScoresGroupedByName(input.projectId, traceScopedFilter),
getBooleanScoresGroupedByName(input.projectId, traceScopedFilter),
getNumericScoresGroupedByName(input.projectId, observationScopedFilter),
getCategoricalScoresGroupedByName(
input.projectId,
observationScopedFilter,
),
getBooleanScoresGroupedByName(input.projectId, observationScopedFilter),
getNumericScoresGroupedByName(input.projectId, traceLevelScopedFilter),
getCategoricalScoresGroupedByName(
input.projectId,
traceLevelScopedFilter,
),
getBooleanScoresGroupedByName(input.projectId, traceLevelScopedFilter),
getEventsGroupedByExperimentDatasetId(
input.projectId,
input.startTimeFilter ?? [],
),
]);
/**
* Split per data type: a name can be reused across types at different
* levels (a numeric observation-level "accuracy" beside an unrelated
* categorical trace-level one), and a name-only map would mislabel both.
*/
const levelsOf = (
observationNames: string[],
traceNames: string[],
): Record<string, ("observation" | "trace")[]> => {
const out: Record<string, ("observation" | "trace")[]> = {};
for (const name of observationNames) {
if (out[name] === undefined) {
out[name] = [];
}
out[name].push("observation");
}
for (const name of traceNames) {
if (out[name] === undefined) {
out[name] = [];
}
out[name].push("trace");
}
return out;
};
const experimentDatasetIdSet = new Set<string>();
for (const { experimentDatasetId } of experimentDatasetIds) {
if (experimentDatasetId !== null) {
experimentDatasetIdSet.add(experimentDatasetId);
}
}
// Return score options for both observation-level and trace-level filters
// The same score names are available at both levels
const numericScoreOptions = numericScoreNames.map((score) => score.name);
const booleanScoreOptions = booleanScoreNames.map((score) => score.name);
return {
// What the three score facets offer, with the level(s) each name exists
// at so the picker can tag them.
scores_avg: numericScoreOptions,
score_categories: categoricalScoreNames,
score_booleans: booleanScoreOptions,
score_name_levels_numeric: levelsOf(
observationLevelNumeric.map((score) => score.name),
traceLevelNumeric.map((score) => score.name),
),
score_name_levels_categorical: levelsOf(
observationLevelCategorical.map((score) => score.label),
traceLevelCategorical.map((score) => score.label),
),
score_name_levels_boolean: levelsOf(
observationLevelBoolean.map((score) => score.name),
traceLevelBoolean.map((score) => score.name),
),
experimentDatasetIds: Array.from(experimentDatasetIdSet),
};
}),
items: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
baseExperimentId: z.string().nullish(),
compExperimentIds: z.array(z.string()),
filterByExperiment: z
.array(
z.object({
experimentId: z.string(),
filters: singleFilterList,
}),
)
.nullish(),
orderBy: orderBy,
itemVisibility: z
.enum(["baseline-only", "all"])
.optional()
.default("baseline-only"),
...paginationZod,
}),
)
.query(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "promptExperiments:read",
});
const items = await getExperimentItemsFromEvents({
projectId: input.projectId,
baseExperimentId: input.baseExperimentId ?? undefined,
compExperimentIds: input.compExperimentIds,
filterByExperiment: input.filterByExperiment ?? [],
offset: input.page * input.limit,
limit: input.limit,
config: {
requireBaselinePresence: input.itemVisibility === "baseline-only",
},
});
const observationIds = Array.from(
new Set(
items.flatMap((item) => item.experiments.map((i) => i.observationId)),
),
);
const traceIds = Array.from(
new Set(
items.flatMap((item) => item.experiments.map((i) => i.traceId)),
),
);
const [observationScores, traceScores] = await Promise.all([
getScoresForObservations({
projectId: input.projectId,
observationIds,
excludeMetadata: true,
includeHasMetadata: true,
preferredClickhouseService: "ReadOnly",
}),
getScoresForTraces({
projectId: input.projectId,
traceIds,
level: "trace",
excludeMetadata: true,
includeHasMetadata: true,
}),
]);
const validatedObservationScores = filterAndValidateDbScoreList({
scores: observationScores,
dataTypes: AGGREGATABLE_SCORE_TYPES,
includeHasMetadata: true,
onParseError: traceException,
});
const validatedTraceScores = filterAndValidateDbScoreList({
scores: traceScores,
dataTypes: AGGREGATABLE_SCORE_TYPES,
includeHasMetadata: true,
onParseError: traceException,
});
const scoresByObservationId = new Map<
string,
Array<(typeof validatedObservationScores)[number]>
>();
for (const score of validatedObservationScores) {
if (!score.observationId) continue;
const existingScores = scoresByObservationId.get(score.observationId);
if (existingScores) {
existingScores.push(score);
} else {
scoresByObservationId.set(score.observationId, [score]);
}
}
const scoresByTraceId = new Map<
string,
Array<(typeof validatedTraceScores)[number]>
>();
for (const score of validatedTraceScores) {
if (!score.traceId) continue;
const existingScores = scoresByTraceId.get(score.traceId);
if (existingScores) {
existingScores.push(score);
} else {
scoresByTraceId.set(score.traceId, [score]);
}
}
return {
data: items.map(({ itemId, experiments }) => ({
itemId,
experiments: experiments.map(
({
observationId,
traceId,
experimentId,
level,
startTime,
totalCost,
latencyMs,
}) => ({
observationId,
traceId,
experimentId,
level,
startTime,
totalCost,
latencyMs,
observationScores: aggregateScores(
scoresByObservationId.get(observationId) ?? [],
),
traceScores: aggregateScores(scoresByTraceId.get(traceId) ?? []),
}),
),
})),
};
}),
itemsCount: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
baseExperimentId: z.string().nullish(),
compExperimentIds: z.array(z.string()),
filterByExperiment: z
.array(
z.object({
experimentId: z.string(),
filters: singleFilterList,
}),
)
.nullish(),
itemVisibility: z
.enum(["baseline-only", "all"])
.optional()
.default("baseline-only"),
}),
)
.query(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "promptExperiments:read",
});
const count = await getExperimentItemsCountFromEvents({
projectId: input.projectId,
baseExperimentId: input.baseExperimentId ?? undefined,
compExperimentIds: input.compExperimentIds,
filterByExperiment: input.filterByExperiment ?? [],
config: {
requireBaselinePresence: input.itemVisibility === "baseline-only",
},
});
return {
count,
};
}),
itemsFilterOptions: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
experimentIds: z.array(z.string()).min(1),
}),
)
.query(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "promptExperiments:read",
});
return getExperimentItemsFilterOptions({
projectId: input.projectId,
experimentIds: input.experimentIds,
});
}),
scoreOptions: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
experimentIds: z.array(z.string()).min(1),
}),
)
.query(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "promptExperiments:read",
});
return getExperimentScoreOptions({
projectId: input.projectId,
experimentIds: input.experimentIds,
});
}),
batchIO: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
itemIds: z.array(z.string()),
baseExperimentId: z.string().nullish(),
compExperimentIds: z.array(z.string()),
ioCharLimit: z.number().int().positive().max(10_000).optional(),
}),
)
.query(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "promptExperiments:read",
});
if (input.itemIds.length === 0) {
return [];
}
const batchIO = await getExperimentItemsBatchIO({
projectId: input.projectId,
itemIds: input.itemIds,
baseExperimentId: input.baseExperimentId ?? undefined,
compExperimentIds: input.compExperimentIds,
ioCharLimit: input.ioCharLimit,
});
return batchIO;
}),
byProjectId: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
}),
)
.query(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "promptExperiments:read",
});
const experiments = await getExperimentNamesFromEvents({
projectId: input.projectId,
});
// Dataset names live in Postgres only — ClickHouse carries the id — so the
// display name is resolved here rather than in the projection, and the
// pickers never have to render a raw dataset id.
const datasetIds = [
...new Set(
experiments
.map((experiment) => experiment.datasetId)
.filter((datasetId): datasetId is string => Boolean(datasetId)),
),
];
const datasets = datasetIds.length
? await ctx.prisma.dataset.findMany({
where: { projectId: input.projectId, id: { in: datasetIds } },
select: { id: true, name: true },
})
: [];
const datasetNameById = new Map(datasets.map((d) => [d.id, d.name]));
return {
experimentNames: experiments.map((experiment) => ({
experimentId: experiment.experimentId,
experimentName: experiment.experimentName,
startTime: experiment.startTime,
datasetId: experiment.datasetId,
datasetName: experiment.datasetId
? (datasetNameById.get(experiment.datasetId) ?? null)
: null,
})),
};
}),
});