🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { z } from "zod";
import { auditLog } from "@/src/features/audit-logs/auditLog";
import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
import { aggregateScores } from "@/src/features/scores/lib/aggregateScores";
import {
applyCommentFilters,
traceException,
getTracesTable,
getTracesTableCount,
getScoresForTraces,
getNumericScoresGroupedByName,
getBooleanScoresGroupedByName,
getTracesGroupedByName,
getTracesGroupedByTags,
getObservationsForTrace,
getTraceById,
logger,
upsertTrace,
convertTraceDomainToClickhouse,
hasAnyTracingData,
traceDeletionProcessor,
getTracesTableMetrics,
getCategoricalScoresGroupedByName,
convertDateToClickhouseDateTime,
getAgentGraphData,
tracesTableUiColumnDefinitions,
getTracesGroupedByUsers,
getTracesGroupedBySessionId,
updateEvents,
getScoresAndCorrectionsForTraces,
} from "@langfuse/shared/src/server";
import {
createTRPCRouter,
protectedGetTraceProcedure,
protectedProjectProcedure,
} from "@/src/server/api/trpc";
import {
isoDateTime,
BatchActionQuerySchema,
BatchTableNames,
BatchExportTableName,
BatchActionType,
ActionId,
filterAndValidateDbScoreList,
normalizeOrderByForTable,
orderBy,
paginationZod,
singleFilterList,
timeFilter,
type Observation,
hasValidTracingSearchTypes,
TRACING_SEARCH_TYPE_REQUIRED_MESSAGE,
TracingSearchType,
type ScoreDomain,
ScoreDataTypeArray,
ScoreDataTypeEnum,
LISTABLE_SCORE_TYPES,
LangfuseNotFoundError,
} from "@langfuse/shared";
import { TRPCError } from "@trpc/server";
import { createBatchActionJob } from "@/src/features/table/server/createBatchActionJob";
import { throwIfNoEntitlement } from "@/src/features/entitlements/server/hasEntitlement";
import { sanitizeLegacyTracingSearch } from "@/src/features/traces/server/legacyIoSearch";
import {
type AgentGraphDataResponse,
AgentGraphDataSchema,
} from "@/src/features/trace-graph-view/types";
import { env } from "@/src/env.mjs";
import {
toDomainWithStringifiedMetadata,
toDomainArrayWithStringifiedMetadata,
} from "@/src/utils/clientSideDomainTypes";
import { scoreFilters } from "@/src/features/scores/lib/scoreColumns";
import partition from "lodash/partition";
const TraceCountOptions = z
.object({
projectId: z.string(), // Required for protectedProjectProcedure
searchQuery: z.string().nullable(),
searchType: z.array(TracingSearchType),
filter: singleFilterList.nullable(),
orderBy: orderBy,
})
.refine(hasValidTracingSearchTypes, {
message: TRACING_SEARCH_TYPE_REQUIRED_MESSAGE,
path: ["searchType"],
});
const TraceFilterOptions = TraceCountOptions.safeExtend({
...paginationZod,
});
type TraceFilterOptions = z.infer<typeof TraceFilterOptions>;
export type ObservationReturnTypeWithMetadata = Omit<
Observation,
"input" | "output" | "metadata"
> & {
traceId: string;
metadata: string | null;
traceName?: string | null;
traceTags?: string[];
// optional, because in v4 an observation can have those properties
userId?: string | null;
sessionId?: string | null;
release?: string | null;
};
export type ObservationReturnType = Omit<
ObservationReturnTypeWithMetadata,
"metadata"
>;
export const traceRouter = createTRPCRouter({
hasTracingConfigured: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
}),
)
.query(async ({ input, ctx }) => {
// Check if there is any tracing data in the database
const hasTraces = await hasAnyTracingData(input.projectId);
if (hasTraces) {
return true;
}
// If no traces, check if data retention is configured
// This indicates the user has configured tracing even if data retention cleaned all traces
const project = await ctx.prisma.project.findUnique({
where: {
id: input.projectId,
},
select: {
retentionDays: true,
},
});
return !!(project?.retentionDays && project.retentionDays > 0);
}),
all: protectedProjectProcedure
.input(TraceFilterOptions)
.query(async ({ input, ctx }) => {
const search = sanitizeLegacyTracingSearch({
searchQuery: input.searchQuery,
searchType: input.searchType,
tableName: BatchTableNames.Traces,
});
const { filterState, hasNoMatches } = await applyCommentFilters({
filterState: input.filter ?? [],
prisma: ctx.prisma,
projectId: ctx.session.projectId,
objectType: "TRACE",
});
if (hasNoMatches) {
return { traces: [] };
}
const traces = await getTracesTable({
projectId: ctx.session.projectId,
filter: filterState,
searchQuery: search.searchQuery,
searchType: search.searchType ?? ["id"],
orderBy: normalizeOrderByForTable({
orderBy: input.orderBy,
expectedTimeColumn: "timestamp",
}),
limit: input.limit,
page: input.page,
});
return { traces };
}),
countAll: protectedProjectProcedure
.input(TraceCountOptions)
.query(async ({ input, ctx }) => {
const search = sanitizeLegacyTracingSearch({
searchQuery: input.searchQuery,
searchType: input.searchType,
tableName: BatchTableNames.Traces,
});
const { filterState, hasNoMatches } = await applyCommentFilters({
filterState: input.filter ?? [],
prisma: ctx.prisma,
projectId: ctx.session.projectId,
objectType: "TRACE",
});
if (hasNoMatches) {
return { totalCount: 0 };
}
const count = await getTracesTableCount({
projectId: ctx.session.projectId,
filter: filterState,
searchType: search.searchType ?? ["id"],
searchQuery: search.searchQuery,
limit: 1,
page: 0,
});
return {
totalCount: count,
};
}),
metrics: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
traceIds: z.array(z.string()),
filter: singleFilterList.nullable(),
}),
)
.query(async ({ input, ctx }) => {
if (input.traceIds.length === 0) return [];
const { filterState, hasNoMatches, matchingIds } =
await applyCommentFilters({
filterState: input.filter ?? [],
prisma: ctx.prisma,
projectId: ctx.session.projectId,
objectType: "TRACE",
});
if (hasNoMatches) {
return [];
}
// If comment filters returned matching IDs, intersect with input.traceIds
let filteredTraceIds = input.traceIds;
if (matchingIds !== null) {
filteredTraceIds = input.traceIds.filter((id) =>
matchingIds.includes(id),
);
if (filteredTraceIds.length === 0) {
return [];
}
}
// Remove the comment filter's ID injection and use filteredTraceIds instead
const filterWithoutCommentIds = filterState.filter(
(f) =>
!(
f.type === "stringOptions" &&
f.column === "id" &&
f.operator === "any of"
),
);
const res = await getTracesTableMetrics({
projectId: ctx.session.projectId,
filter: [
...filterWithoutCommentIds,
{
type: "stringOptions",
operator: "any of",
column: "ID",
value: filteredTraceIds,
},
],
orderBy: { column: "timestamp", order: "DESC" },
});
const traceScores = await getScoresForTraces({
projectId: ctx.session.projectId,
traceIds: res.map((r) => r.id),
excludeMetadata: true,
includeHasMetadata: true,
});
const validatedScores = filterAndValidateDbScoreList({
scores: traceScores,
dataTypes: LISTABLE_SCORE_TYPES,
includeHasMetadata: true,
onParseError: traceException,
});
return res.map((row) => ({
...row,
scores: aggregateScores(
validatedScores.filter((s) => s.traceId === row.id),
),
}));
}),
filterOptions: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
timestampFilter: z.array(timeFilter).optional(),
}),
)
.query(async ({ input }) => {
const { timestampFilter } = input;
// Trace table filters/columns operate on trace-scoped aggregates: any
// score row attached to the trace, whether it lives on the trace itself
// or on one of the trace's observations.
const traceScopedScoreFilters = [
...(timestampFilter ?? []),
...scoreFilters.forTraceScopedAggregates(),
];
const [
numericScoreNames,
categoricalScoreNames,
booleanScoreNames,
traceNames,
tags,
userIds,
sessionIds,
] = await Promise.all([
getNumericScoresGroupedByName(input.projectId, traceScopedScoreFilters),
getCategoricalScoresGroupedByName(
input.projectId,
traceScopedScoreFilters,
),
getBooleanScoresGroupedByName(input.projectId, traceScopedScoreFilters),
getTracesGroupedByName(
input.projectId,
tracesTableUiColumnDefinitions,
timestampFilter ?? [],
),
getTracesGroupedByTags({
projectId: input.projectId,
filter: timestampFilter ?? [],
}),
getTracesGroupedByUsers(
input.projectId,
timestampFilter ?? [],
undefined,
100,
0,
),
getTracesGroupedBySessionId(
input.projectId,
timestampFilter ?? [],
undefined,
100,
0,
),
]);
return {
name: traceNames.map((n) => ({ value: n.name, count: n.count })),
scores_avg: numericScoreNames.map((s) => s.name),
score_categories: categoricalScoreNames,
score_booleans: booleanScoreNames.map((s) => s.name),
tags: tags,
users: userIds.map((u) => ({
value: u.user,
count: u.count,
})),
sessions: sessionIds.map((s) => ({
value: s.session_id,
count: s.count,
})),
};
}),
byId: protectedGetTraceProcedure
.input(
z.object({
traceId: z.string(), // used for security check
projectId: z.string(), // used for security check
timestamp: z.date().nullish(), // timestamp of the trace. Used to query CH more efficiently
fromTimestamp: z.date().nullish(), // min timestamp of the trace. Used to query CH more efficiently
verbosity: z.enum(["compact", "truncated", "full"]).default("full"),
}),
)
.query(async ({ ctx }) => {
if (!ctx.trace) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Trace not found",
});
}
return {
...ctx.trace,
input: ctx.trace.input as string,
output: ctx.trace.output as string,
metadata: ctx.trace.metadata
? JSON.stringify(ctx.trace.metadata)
: undefined,
};
}),
byIdWithObservationsAndScores: protectedGetTraceProcedure
.input(
z.object({
traceId: z.string(), // used for security check
timestamp: z.date().nullish(), // timestamp of the trace. Used to query CH more efficiently
fromTimestamp: z.date().nullish(), // min timestamp of the trace. Used to query CH more efficiently
projectId: z.string(), // used for security check
}),
)
.query(async ({ input, ctx }) => {
if (!ctx.trace) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Trace not found",
});
}
const [observations, traceScores] = await Promise.all([
getObservationsForTrace({
traceId: input.traceId,
projectId: input.projectId,
timestamp: input.timestamp ?? input.fromTimestamp ?? undefined,
includeIO: false,
}),
getScoresAndCorrectionsForTraces({
projectId: input.projectId,
traceIds: [input.traceId],
timestamp: input.timestamp ?? input.fromTimestamp ?? undefined,
}),
]);
const validatedScores = filterAndValidateDbScoreList({
scores: traceScores,
dataTypes: [...ScoreDataTypeArray],
onParseError: traceException,
});
const [corrections, scores] = partition(
validatedScores,
(s) => s.dataType === ScoreDataTypeEnum.CORRECTION,
);
const obsStartTimes = observations
.map((o) => o.startTime)
.sort((a, b) => a.getTime() - b.getTime());
const obsEndTimes = observations
.map((o) => o.endTime)
.filter((t) => t)
.sort((a, b) => (a as Date).getTime() - (b as Date).getTime());
const latencyMs = (() => {
if (obsStartTimes.length > 0) {
if (obsEndTimes.length > 0) {
return (
(obsEndTimes[obsEndTimes.length - 1] as Date).getTime() -
obsStartTimes[0]!.getTime()
);
}
if (obsStartTimes.length > 1) {
return (
obsStartTimes[obsStartTimes.length - 1]!.getTime() -
obsStartTimes[0]!.getTime()
);
}
return undefined;
}
return undefined;
})();
const scoresDomain =
toDomainArrayWithStringifiedMetadata<ScoreDomain>(scores);
return {
...toDomainWithStringifiedMetadata(ctx.trace),
input: ctx.trace.input ? JSON.stringify(ctx.trace.input) : null,
output: ctx.trace.output ? JSON.stringify(ctx.trace.output) : null,
scores: scoresDomain,
corrections,
latency: latencyMs !== undefined ? latencyMs / 1000 : undefined,
observations: observations.map((o) => ({
...toDomainWithStringifiedMetadata(o),
output: undefined,
input: undefined, // this is not queried above.
})) as ObservationReturnTypeWithMetadata[],
};
}),
deleteMany: protectedProjectProcedure
.input(
z
.object({
traceIds: z.array(z.string()),
projectId: z.string(),
query: BatchActionQuerySchema.optional(),
isBatchAction: z.boolean().default(false),
})
// Batch actions delete by query and ignore traceIds, so an empty list
// is valid there (paging/refetch can drain the visible selection while
// select-all is armed); only id-based deletes need at least one id.
.refine((input) => input.isBatchAction || input.traceIds.length > 0, {
message: "Minimum 1 traceId is required.",
path: ["traceIds"],
})
.refine((input) => !input.isBatchAction || input.query !== undefined, {
message: "Batch actions require a query.",
path: ["query"],
}),
)
.mutation(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "traces:delete",
});
throwIfNoEntitlement({
entitlement: "trace-deletion",
projectId: input.projectId,
sessionUser: ctx.session.user,
});
if (input.isBatchAction && input.query) {
// Comment filters (commentCount/commentContent in both the v3 traces
// and v4 events views) resolve via Postgres lookups at read time
// (applyCommentFilters); when the worker translates the stored
// filters into ClickHouse SQL, these columns map to a nonexistent
// "comments" table and would deterministically fail every batch, so
// reject them at dispatch.
const hasCommentFilter = (input.query.filter ?? []).some(
(f) => f.column === "commentCount" || f.column === "commentContent",
);
if (hasCommentFilter) {
throw new TRPCError({
code: "BAD_REQUEST",
message:
"Batch deletion does not support comment filters. Remove the comment filter and try again.",
});
}
// Decide here whether this delete reads from the events table: the
// v4 events view sets query.useEventsTable: true, which we honor
// after checking the events view is actually available to this user
// (v4 beta flag, or the instance-wide preview opt-in). In every
// other case createBatchActionJob infers the choice from the user's
// v4 beta flag.
const declaresEventsTable = input.query.useEventsTable === true;
if (declaresEventsTable) {
const eventsSurfaceAvailable =
ctx.session.user.v4BetaEnabled === true ||
env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true";
if (!eventsSurfaceAvailable) {
throw new TRPCError({
code: "BAD_REQUEST",
message:
"Events-backed batch deletion is not available for this user on this instance.",
});
}
}
await createBatchActionJob({
projectId: input.projectId,
actionId: ActionId.TraceDelete,
actionType: BatchActionType.Delete,
tableName: BatchExportTableName.Traces,
session: ctx.session,
query: input.query,
useEventsTableOverride: declaresEventsTable ? true : undefined,
});
} else {
await Promise.all(
input.traceIds.map((traceId) =>
auditLog({
resourceType: "trace",
resourceId: traceId,
action: "delete",
session: ctx.session,
}),
),
);
await traceDeletionProcessor(input.projectId, input.traceIds, {
actor: { type: "USER", userId: ctx.session.user.id },
});
}
}),
bookmark: protectedProjectProcedure
.input(
z.object({
traceId: z.string(),
projectId: z.string(),
bookmarked: z.boolean(),
}),
)
.mutation(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "objects:bookmark",
});
try {
await auditLog({
session: ctx.session,
resourceType: "trace",
resourceId: input.traceId,
action: "bookmark",
after: input.bookmarked,
});
let trace;
// eslint-disable-next-line @typescript-eslint/no-deprecated
const clickhouseTrace = await getTraceById({
traceId: input.traceId,
projectId: input.projectId,
});
if (clickhouseTrace) {
trace = clickhouseTrace;
clickhouseTrace.bookmarked = input.bookmarked;
const promises = [
upsertTrace(convertTraceDomainToClickhouse(clickhouseTrace)),
];
if (env.LANGFUSE_MIGRATION_V4_WRITE_MODE !== "legacy") {
promises.push(
updateEvents(
input.projectId,
{ traceIds: [clickhouseTrace.id], rootOnly: true },
{ bookmarked: input.bookmarked },
),
);
}
await Promise.all(promises);
} else {
logger.error(
`Trace not found in Clickhouse: ${input.traceId}. Skipping bookmark.`,
);
}
return trace;
} catch (error) {
logger.error("Failed to call traces.bookmark", error);
throw new TRPCError({
code: "INTERNAL_SERVER_ERROR",
});
}
}),
publish: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
traceId: z.string(),
public: z.boolean(),
}),
)
.mutation(async ({ input, ctx }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "objects:publish",
});
await auditLog({
session: ctx.session,
resourceType: "trace",
resourceId: input.traceId,
action: "publish",
after: input.public,
});
// eslint-disable-next-line @typescript-eslint/no-deprecated
const clickhouseTrace = await getTraceById({
traceId: input.traceId,
projectId: input.projectId,
});
if (!clickhouseTrace) {
throw new LangfuseNotFoundError("Trace not found");
}
clickhouseTrace.public = input.public;
const promises = [
upsertTrace(convertTraceDomainToClickhouse(clickhouseTrace)),
];
if (env.LANGFUSE_MIGRATION_V4_WRITE_MODE !== "legacy") {
promises.push(
updateEvents(
input.projectId,
{ traceIds: [clickhouseTrace.id] },
{ public: input.public },
),
);
}
await Promise.all(promises);
return clickhouseTrace;
}),
getAgentGraphData: protectedGetTraceProcedure
.input(
z.object({
projectId: z.string(),
traceId: z.string(),
minStartTime: isoDateTime({ offset: true }),
maxStartTime: isoDateTime({ offset: true }),
// Optional fields for enforceTraceAccess middleware (supports public traces)
timestamp: z.date().nullish(),
fromTimestamp: z.date().nullish(),
}),
)
.query(async ({ input }): Promise<Required<AgentGraphDataResponse>[]> => {
const { traceId, projectId, minStartTime, maxStartTime } = input;
const chMinStartTime = convertDateToClickhouseDateTime(
new Date(minStartTime),
);
const chMaxStartTime = convertDateToClickhouseDateTime(
new Date(maxStartTime),
);
const records = await getAgentGraphData({
projectId,
traceId,
chMinStartTime,
chMaxStartTime,
});
const result = records
.map((r) => {
const parsed = AgentGraphDataSchema.safeParse(r);
if (!parsed.success) {
return null;
}
const data = parsed.data;
const hasLangGraphData = data.step != null && data.node != null;
const hasAgentData = data.type !== "EVENT"; // Include all types except EVENT
if (hasLangGraphData) {
return {
id: data.id,
node: data.node,
step: data.step,
parentObservationId: data.parent_observation_id || null,
name: data.name,
startTime: data.start_time,
endTime: data.end_time || undefined,
observationType: data.type,
};
} else if (hasAgentData) {
return {
id: data.id,
node: data.name,
step: 0,
parentObservationId: data.parent_observation_id || null,
name: data.name,
startTime: data.start_time,
endTime: data.end_time || undefined,
observationType: data.type,
};
}
return null;
})
.filter((r) => Boolean(r)) as Required<AgentGraphDataResponse>[];
return result;
}),
});