🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { z } from "zod";
import {
createTRPCRouter,
protectedProjectProcedure,
} from "@/src/server/api/trpc";
import {
filterInterface,
sqlInterface,
type DatabaseRow,
} from "@/src/server/api/services/sqlInterface";
import { createHistogramData } from "@/src/features/dashboard/lib/score-analytics-utils";
import { TRPCError } from "@trpc/server";
import { env } from "@/src/env.mjs";
import {
getScoreAggregate,
getNumericScoreHistogram,
extractFromAndToTimestampsFromFilter,
logger,
getObservationCostByTypeByTime,
getObservationUsageByTypeByTime,
DashboardService,
DashboardDefinitionSchema,
escapeSqlLikePattern,
} from "@langfuse/shared/src/server";
import { executeQuery } from "@langfuse/shared/query/server";
import {
query as customQuery,
validateQuery,
viewVersions,
type QueryType,
} from "@langfuse/shared/query";
import { mapLegacyUiTableFilterToView } from "@/src/features/dashboard/lib/dashboardUiTableToViewMapping";
import {
paginationZod,
orderBy,
StringNoHTML,
InvalidRequestError,
singleFilterList,
LANGFUSE_HOME_DASHBOARD_ID,
type FilterState,
} from "@langfuse/shared";
import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
// Define the dashboard list input schema
const ListDashboardsInput = z.object({
projectId: z.string(),
...paginationZod,
// The Home-dashboard picker and clone detection fetch the whole list in one
// page, so allow a higher ceiling than the default table pagination.
limit: z.coerce.number().int().positive().lte(500).default(50),
orderBy: orderBy.refine(
(value) =>
value === null ||
["name", "createdAt", "updatedAt"].includes(value.column),
{ message: "Invalid dashboard sort column", path: ["column"] },
),
});
// Get dashboard by ID input schema
const GetDashboardInput = z.object({
projectId: z.string(),
dashboardId: z.string(),
});
// Update dashboard definition input schema
const UpdateDashboardDefinitionInput = z.object({
projectId: z.string(),
dashboardId: z.string(),
definition: DashboardDefinitionSchema,
});
// Update dashboard input schema
const UpdateDashboardInput = z.object({
projectId: z.string(),
dashboardId: z.string(),
name: StringNoHTML.min(1, "Dashboard name is required"),
description: StringNoHTML,
});
// Create dashboard input schema
const CreateDashboardInput = z.object({
projectId: z.string(),
name: StringNoHTML.min(1, "Dashboard name is required"),
description: StringNoHTML,
});
// Clone dashboard input schema
const CloneDashboardInput = z.object({
projectId: z.string(),
dashboardId: z.string(),
// Optional definition override so an edit attempted on a read-only
// dashboard (e.g. a tile moved on the curated Home) carries into the clone.
definition: DashboardDefinitionSchema.optional(),
// Set the clone as the project's home dashboard in the same gesture.
setAsHome: z.boolean().optional(),
});
// Set home dashboard input schema
const SetHomeDashboardInput = z.object({
projectId: z.string(),
// null resets to the Langfuse-curated default
dashboardId: z.string().min(1).nullable(),
});
// Update dashboard filters input schema
const UpdateDashboardFiltersInput = z.object({
projectId: z.string(),
dashboardId: z.string(),
filters: singleFilterList,
});
/**
* First free clone name: "ABC (Clone)", then "ABC (Clone 2)", "ABC (Clone 3)", …
*/
function nextCloneName(sourceName: string, existingNames: string[]): string {
const taken = new Set(existingNames);
const base = `${sourceName} (Clone)`;
if (!taken.has(base)) return base;
let n = 2;
while (taken.has(`${sourceName} (Clone ${n})`)) n++;
return `${sourceName} (Clone ${n})`;
}
// Map camelCase legacy column names (used by scoreHistogram component)
// to view-native field names before passing through the general mapper.
const LEGACY_CAMEL_CASE_MAP: Record<string, string> = {
scoreName: "name",
scoreSource: "source",
scoreDataType: "dataType",
};
/**
* Shared filter preparation for scores-numeric v2 queries.
* Extracts time boundaries, strips time filters, and maps legacy UI column
* names to view-native field names.
*/
function prepareScoresNumericV2Params(filter: FilterState) {
const [from, to] = extractFromAndToTimestampsFromFilter(filter);
// Preserve the existing default range. DateTimeFilter now accepts epoch as a
// lower bound, but widening dashboard queries is a separate behavior change.
const fromIso = from?.value
? new Date(from.value as Date).toISOString()
: new Date("2000-01-01T00:00:00.000Z").toISOString();
const toIso = to?.value
? new Date(to.value as Date).toISOString()
: new Date().toISOString();
const nonTimeFilters = filter.filter(
(f) => f.column !== "scoreTimestamp" && f.column !== "startTime",
);
const normalizedFilters = nonTimeFilters.map((f) => {
const mapped = LEGACY_CAMEL_CASE_MAP[f.column];
return mapped ? { ...f, column: mapped } : f;
});
const mappedFilters = mapLegacyUiTableFilterToView(
"scores-numeric",
normalizedFilters,
);
return { fromIso, toIso, mappedFilters };
}
/**
* Converts ClickHouse histogram(N)(...) output to the { chartData, chartLabels }
* shape returned by createHistogramData (used by the NumericScoreHistogram component).
*
* ClickHouse histogram() returns an Array(Tuple(Float64, Float64, Float64))
* where each tuple is (lower_bound, upper_bound, count).
* The result column is named "histogram_value" by QueryBuilder
* (pattern: `${aggregation}_${alias}`).
*/
function clickhouseHistogramToChartData(
result: Array<Record<string, unknown>>,
): {
chartData: Array<{ binLabel: string; count: number }>;
chartLabels: string[];
} {
if (result.length > 0 && !("histogram_value" in result[0])) {
throw new Error(
`Expected histogram_value column in QueryBuilder result, got: ${Object.keys(result[0]).join(", ")}`,
);
}
const histogramBins = result[0]?.histogram_value as
| Array<[number, number, number]>
| undefined;
if (!histogramBins?.length) return { chartData: [], chartLabels: [] };
const round = (v: number) => parseFloat(v.toFixed(2));
return {
chartLabels: ["count"],
chartData: histogramBins.map(([lower, upper, count]) => ({
binLabel: `[${round(lower)}, ${round(upper)}]`,
count: Math.round(count),
})),
};
}
async function getScoreAggregateV2({
projectId,
filter,
}: {
projectId: string;
filter: FilterState;
}): Promise<DatabaseRow[]> {
// prepareScoresNumericV2Params also applies LEGACY_CAMEL_CASE_MAP for
// scoreHistogram callers. For score-aggregate calls, the only camelCase
// column is "scoreTimestamp" which is stripped as a time filter before
// LEGACY_CAMEL_CASE_MAP runs, making the normalization step a no-op here.
const { fromIso, toIso, mappedFilters } =
prepareScoresNumericV2Params(filter);
// Non-time filters in their original form — used for categorical query
// filter mapping and value filter detection below.
const nonTimeFilters = filter.filter(
(f) => f.column !== "scoreTimestamp" && f.column !== "startTime",
);
const baseQuery = {
dimensions: [{ field: "name" }, { field: "source" }, { field: "dataType" }],
timeDimension: null,
fromTimestamp: fromIso,
toTimestamp: toIso,
orderBy: [{ field: "sum_count", direction: "desc" as const }],
};
const numericQuery: QueryType = {
...baseQuery,
view: "scores-numeric",
metrics: [
{ measure: "count", aggregation: "sum" },
{ measure: "value", aggregation: "avg" },
],
filters: mappedFilters,
};
// The scores-categorical view has no "value" dimension, so we handle value
// filters manually: categorical scores always have value=0 in ClickHouse, so
// value=0 should include all categoricals, while any other value filter
// (e.g. value=1) should exclude them. This matches v1 behavior where numeric
// and categorical scores are queried together in a single SQL statement.
const valueFilter = nonTimeFilters.find((f) => f.column === "value");
const skipCategorical =
valueFilter && "value" in valueFilter && valueFilter.value !== 0;
const categoricalFilters = nonTimeFilters.filter((f) => f.column !== "value");
const categoricalQuery: QueryType = {
...baseQuery,
view: "scores-categorical",
metrics: [{ measure: "count", aggregation: "sum" }],
filters: mapLegacyUiTableFilterToView(
"scores-categorical",
categoricalFilters,
),
};
const [numericResults, categoricalResults] = await Promise.all([
executeQuery(projectId, numericQuery, "v2"),
skipCategorical
? Promise.resolve([])
: executeQuery(projectId, categoricalQuery, "v2"),
]);
const merged = [
...numericResults.map((r) => ({
scoreName: String(r.name),
countScoreId: Number(r.sum_count ?? 0),
avgValue: Number(r.avg_value ?? 0),
scoreSource: String(r.source),
scoreDataType: String(r.dataType),
})),
...categoricalResults.map((r) => ({
scoreName: String(r.name),
countScoreId: Number(r.sum_count ?? 0),
avgValue: 0,
scoreSource: String(r.source),
scoreDataType: String(r.dataType),
})),
].sort((a, b) => b.countScoreId - a.countScoreId);
return merged as DatabaseRow[];
}
async function getObservationsByTypeV2(params: {
projectId: string;
filter: FilterState;
dimensionField: "costType" | "usageType";
metricMeasure: "costByType" | "usageByType";
}): Promise<DatabaseRow[]> {
const { projectId, filter, dimensionField, metricMeasure } = params;
const [from, to] = extractFromAndToTimestampsFromFilter(filter);
if (!from?.value || !to?.value) {
throw new TRPCError({
code: "BAD_REQUEST",
message: "Time filter required",
});
}
// Filter normalisation for the executeQuery (v2) path:
//
const nonDatetimeFilters = filter.filter((f) => f.type !== "datetime");
// mapLegacyUiTableFilterToView normalizes legacy dashboard filters whether
// they arrive as display labels, uiTableIds, or explicit aliases.
const viewFilters = mapLegacyUiTableFilterToView(
"observations",
nonDatetimeFilters,
);
const q: QueryType = {
view: "observations",
dimensions: [{ field: dimensionField }],
metrics: [{ measure: metricMeasure, aggregation: "sum" }],
filters: viewFilters,
timeDimension: { granularity: "auto" },
fromTimestamp: new Date(from.value as Date).toISOString(),
toTimestamp: new Date(to.value as Date).toISOString(),
orderBy: null,
};
const rows = await executeQuery(projectId, q, "v2", true);
// Transform flat rows to { intervalStart, key, sum } expected by the component.
const sumField = `sum_${metricMeasure}`;
return rows.map((row) => ({
intervalStart: new Date(row["time_dimension"] as string),
key: row[dimensionField] as string,
sum: Number(row[sumField] ?? 0),
})) as DatabaseRow[];
}
export const dashboardRouter = createTRPCRouter({
chart: protectedProjectProcedure
.input(
sqlInterface.extend({
projectId: z.string(),
filter: filterInterface.optional(),
version: viewVersions.optional().default("v1"),
queryName: z
.enum([
// Current score table is weird and does not fit into new model. Keep around as is until we decide what to do with it.
"score-aggregate",
"observations-usage-by-type-timeseries",
"observations-cost-by-type-timeseries",
])
.nullish(),
}),
)
.query(async ({ input }) => {
const [from, to] = extractFromAndToTimestampsFromFilter(input.filter);
if (from.value && to.value && from.value > to.value) {
logger.error(
`from > to, returning empty result: from=${from}, to=${to}`,
);
return [];
}
switch (input.queryName) {
case "score-aggregate":
if (input.version === "v2") {
return getScoreAggregateV2({
projectId: input.projectId,
filter: input.filter ?? [],
});
}
const scores = await getScoreAggregate(
input.projectId,
input.filter ?? [],
);
return scores.map((row) => ({
scoreName: row.name,
scoreSource: row.source,
scoreDataType: row.data_type,
avgValue: row.avg_value,
countScoreId: Number(row.count),
})) as DatabaseRow[];
case "observations-usage-by-type-timeseries":
if (input.version === "v2") {
return getObservationsByTypeV2({
projectId: input.projectId,
filter: input.filter ?? [],
dimensionField: "usageType",
metricMeasure: "usageByType",
});
}
const rowsObsType = await getObservationUsageByTypeByTime(
input.projectId,
input.filter ?? [],
);
return rowsObsType as DatabaseRow[];
case "observations-cost-by-type-timeseries":
if (input.version === "v2") {
return getObservationsByTypeV2({
projectId: input.projectId,
filter: input.filter ?? [],
dimensionField: "costType",
metricMeasure: "costByType",
});
}
const rowsObsCostByType = await getObservationCostByTypeByTime(
input.projectId,
input.filter ?? [],
);
return rowsObsCostByType as DatabaseRow[];
default:
throw new TRPCError({
code: "NOT_FOUND",
message: "Query not found",
});
}
}),
scoreHistogram: protectedProjectProcedure
.input(
sqlInterface.extend({
projectId: z.string(),
filter: filterInterface.optional(),
version: viewVersions.optional().default("v1"),
}),
)
.query(async ({ input }) => {
if (input.version === "v2") {
// v2: ClickHouse histogram() aggregates all matching rows server-side.
// `input.limit` is ignored — no row-level cap is needed.
const { fromIso, toIso, mappedFilters } = prepareScoresNumericV2Params(
input.filter ?? [],
);
const histogramQuery: QueryType = {
view: "scores-numeric",
dimensions: [],
metrics: [{ measure: "value", aggregation: "histogram" }],
filters: mappedFilters,
fromTimestamp: fromIso,
toTimestamp: toIso,
timeDimension: null,
orderBy: null,
chartConfig: { type: "HISTOGRAM", bins: 10 },
};
const result = await executeQuery(
input.projectId,
histogramQuery,
"v2",
);
return clickhouseHistogramToChartData(result);
}
const data = await getNumericScoreHistogram(
input.projectId,
input.filter ?? [],
input.limit ?? 10000,
);
return createHistogramData(data);
}),
executeQuery: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
query: customQuery,
// Required on purpose: a client must decide the read path explicitly.
// An implicit v1 default let unresolved-session clients silently fire
// legacy-table queries for v4 users.
version: viewVersions,
}),
)
.query(async ({ input }) => {
// Deployment-level guard, not a per-user one: v2 reads the events
// views, which exist whenever the deployment writes them. Monitors
// previews and shared v2-widgets legitimately query v2 for users whose
// own read path is still v3; only a legacy deployment has nothing for
// v2 to read. v1 stays open (saved v1 widgets render for v4 users).
if (
input.version === "v2" &&
env.LANGFUSE_MIGRATION_V4_WRITE_MODE === "legacy"
) {
throw new TRPCError({
code: "PRECONDITION_FAILED",
message: "v2 queries are not available on this deployment",
});
}
try {
const validation = validateQuery(input.query, input.version);
if (!validation.valid) {
throw new InvalidRequestError(validation.reason);
}
return executeQuery(
input.projectId,
input.query,
input.version,
input.version === "v2",
);
} catch (error) {
if (error instanceof InvalidRequestError) {
logger.warn("Bad request in query execution", error, {
projectId: input.projectId,
query: input.query,
});
throw error;
}
logger.error("Error executing query", error, {
projectId: input.projectId,
query: input.query,
});
throw error;
}
}),
allDashboards: protectedProjectProcedure
.input(ListDashboardsInput)
.query(async ({ ctx, input }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "dashboards:read",
});
const result = await DashboardService.listDashboards({
projectId: input.projectId,
limit: input.limit,
page: input.page,
orderBy: input.orderBy,
});
return result;
}),
getDashboard: protectedProjectProcedure
.input(GetDashboardInput)
.query(async ({ ctx, input }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "dashboards:read",
});
const dashboard = await DashboardService.getDashboard(
input.dashboardId,
input.projectId,
);
if (!dashboard) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Dashboard not found",
});
}
return dashboard;
}),
createDashboard: protectedProjectProcedure
.input(CreateDashboardInput)
.mutation(async ({ ctx, input }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "dashboards:CUD",
});
const dashboard = await DashboardService.createDashboard(
input.projectId,
input.name,
input.description,
ctx.session.user.id,
);
return dashboard;
}),
updateDashboardDefinition: protectedProjectProcedure
.input(UpdateDashboardDefinitionInput)
.mutation(async ({ ctx, input }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "dashboards:CUD",
});
const dashboard = await DashboardService.updateDashboardDefinition(
input.dashboardId,
input.projectId,
input.definition,
ctx.session.user.id,
);
return dashboard;
}),
updateDashboardMetadata: protectedProjectProcedure
.input(UpdateDashboardInput)
.mutation(async ({ ctx, input }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "dashboards:CUD",
});
const dashboard = await DashboardService.updateDashboard(
input.dashboardId,
input.projectId,
input.name,
input.description,
ctx.session.user.id,
);
return dashboard;
}),
cloneDashboard: protectedProjectProcedure
.input(CloneDashboardInput)
.mutation(async ({ ctx, input }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "dashboards:CUD",
});
// Get the source dashboard
const sourceDashboard = await DashboardService.getDashboard(
input.dashboardId,
input.projectId,
);
if (!sourceDashboard) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Source dashboard not found",
});
}
// Create a new dashboard with the same data but a numbered clone name
const existingClones = await ctx.prisma.dashboard.findMany({
where: {
projectId: input.projectId,
name: {
startsWith: `${escapeSqlLikePattern(sourceDashboard.name)} (Clone`,
},
},
select: { name: true },
});
const clonedDashboard = await DashboardService.createDashboard(
input.projectId,
nextCloneName(
sourceDashboard.name,
existingClones.map((d) => d.name),
),
sourceDashboard.description,
ctx.session.user.id,
input.definition ?? sourceDashboard.definition,
);
if (input.setAsHome) {
await ctx.prisma.project.update({
where: { id: input.projectId },
data: { homeDashboardId: clonedDashboard.id },
});
}
return clonedDashboard;
}),
getHomeDashboard: protectedProjectProcedure
.input(z.object({ projectId: z.string() }))
.query(async ({ ctx, input }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "dashboards:read",
});
const project = await ctx.prisma.project.findUnique({
where: { id: input.projectId },
select: { homeDashboardId: true },
});
// Resolve the pointer; a missing/foreign target silently falls back to
// the Langfuse-curated default (like an unset pointer).
const pointedDashboard = project?.homeDashboardId
? await DashboardService.getDashboard(
project.homeDashboardId,
input.projectId,
)
: null;
const dashboard =
pointedDashboard ??
(await DashboardService.getDashboard(
LANGFUSE_HOME_DASHBOARD_ID,
input.projectId,
));
return {
// null only when the curated row is also absent — the client then
// renders from the shared constant.
dashboard,
homeDashboardId: pointedDashboard
? (project?.homeDashboardId ?? null)
: null,
};
}),
setHomeDashboard: protectedProjectProcedure
.input(SetHomeDashboardInput)
.mutation(async ({ ctx, input }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "dashboards:CUD",
});
if (input.dashboardId) {
const dashboard = await DashboardService.getDashboard(
input.dashboardId,
input.projectId,
);
if (!dashboard) {
throw new TRPCError({
code: "NOT_FOUND",
message: "Dashboard not found",
});
}
}
await ctx.prisma.project.update({
where: { id: input.projectId },
data: { homeDashboardId: input.dashboardId },
});
return { success: true };
}),
updateDashboardFilters: protectedProjectProcedure
.input(UpdateDashboardFiltersInput)
.mutation(async ({ ctx, input }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "dashboards:CUD",
});
const dashboard = await DashboardService.updateDashboardFilters(
input.dashboardId,
input.projectId,
input.filters,
ctx.session.user.id,
);
return dashboard;
}),
// Delete dashboard input schema
delete: protectedProjectProcedure
.input(
z.object({
projectId: z.string(),
dashboardId: z.string(),
}),
)
.mutation(async ({ ctx, input }) => {
throwIfNoProjectAccess({
session: ctx.session,
projectId: input.projectId,
scope: "dashboards:CUD",
});
await DashboardService.deleteDashboard(
input.dashboardId,
input.projectId,
);
return { success: true };
}),
});