🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(datasets): read v4 experiment metrics from events (#16814)
Tobias Wochinger05ecd219ddb8
diff --git a/packages/shared/src/server/repositories/experiments.ts b/packages/shared/src/server/repositories/experiments.tsindex f036474239..2670b1b084 100644--- a/packages/shared/src/server/repositories/experiments.ts+++ b/packages/shared/src/server/repositories/experiments.ts@@ -54,6 +54,54 @@ export type ExperimentMetricsReturnType = {latency_avg: number | null;};+type DatasetExperimentMetricsReturnType = {+ experiment_dataset_id: string;+ count_dataset_runs: string;+ last_run_at: string;+};++export const getDatasetExperimentMetricsFromEvents = async (props: {+ projectId: string;+ datasetIds: string[];+}) => {+ if (props.datasetIds.length === 0) {+ return [];+ }++ const experimentsQuery = eventsExperimentsAggregation({+ projectId: props.projectId,+ fieldSet: "count",+ })+ .selectRaw(+ "nullIf(any(e.experiment_dataset_id), '') AS experiment_dataset_id",+ "min(e.start_time) AS start_time",+ )+ .whereRaw("e.experiment_dataset_id IN ({datasetIds: Array(String)})", {+ datasetIds: props.datasetIds,+ })+ .buildWithParams();++ const rows = await queryClickhouse<DatasetExperimentMetricsReturnType>({+ query: `+ SELECT+ experiment_dataset_id,+ count() AS count_dataset_runs,+ max(start_time) AS last_run_at+ FROM (${experimentsQuery.query}) experiments+ GROUP BY experiment_dataset_id+ `,+ params: experimentsQuery.params,+ tags: { projectId: props.projectId },+ preferredClickhouseService: "EventsReadOnly",+ });++ return rows.map((row) => ({+ datasetId: row.experiment_dataset_id,+ countDatasetRuns: Number(row.count_dataset_runs),+ lastRunAt: parseClickhouseUTCDateTimeFormat(row.last_run_at),+ }));+};+const experimentScoreCTE = (params: {projectId: string;startTimeFrom?: string | null;diff --git a/web/src/__tests__/server/datasets-metrics-events-only.servertest.ts b/web/src/__tests__/server/datasets-metrics-events-only.servertest.tsnew file mode 100644index 0000000000..a37ee27d60--- /dev/null+++ b/web/src/__tests__/server/datasets-metrics-events-only.servertest.ts@@ -0,0 +1,146 @@+import { vi } from "vitest";++const eventsTableAvailable = vi.hoisted(() => {+ const enabled =+ process.env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true";+ process.env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";+ process.env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN = "true";+ return enabled;+});++import type { Session } from "next-auth";+import { randomUUID } from "crypto";+import { prisma } from "@langfuse/shared/src/db";+import { createEvent, createEventsCh } from "@langfuse/shared/src/server";+import { env } from "@langfuse/shared/src/env";+import { appRouter } from "@/src/server/api/root";+import { createInnerTRPCContext } from "@/src/server/api/trpc";++const maybe = eventsTableAvailable ? describe : describe.skip;++describe("datasets metrics events-only liveness", () => {+ it("does not hang when the events table is unavailable", () => {});+});++maybe("datasets.allDatasetsMetrics in events_only write mode", () => {+ const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";++ const session: Session = {+ expires: "1",+ user: {+ id: "user-1",+ canCreateOrganizations: true,+ name: "Demo User",+ organizations: [+ {+ id: "seed-org-id",+ name: "Test Organization",+ role: "OWNER",+ plan: "cloud:hobby",+ cloudConfig: undefined,+ metadata: {},+ aiFeaturesEnabled: false,+ aiTelemetryEnabled: false,+ projects: [+ {+ id: projectId,+ role: "ADMIN",+ retentionDays: 30,+ deletedAt: null,+ name: "Test Project",+ hasTraces: true,+ metadata: {},+ createdAt: new Date().toISOString(),+ },+ ],+ },+ ],+ featureFlags: {+ excludeClickhouseRead: false,+ templateFlag: true,+ searchBar: false,+ v4BetaToggleVisible: false,+ observationEvals: false,+ experimentsV4Enabled: false,+ },+ v4BetaEnabled: true,+ admin: true,+ },+ environment: {} as any,+ };++ const ctx = createInnerTRPCContext({ session, headers: {} });+ const caller = appRouter.createCaller({ ...ctx, prisma });++ it("reads experiment count and latest start time from events", async () => {+ expect(env.LANGFUSE_MIGRATION_V4_WRITE_MODE).toBe("events_only");++ const datasetId = randomUUID();+ const olderExperimentId = randomUUID();+ const latestExperimentId = randomUUID();+ const olderStart = new Date("2026-08-28T10:00:00.000Z");+ const latestStart = new Date("2026-08-29T11:00:00.000Z");++ await prisma.dataset.create({+ data: { id: datasetId, projectId, name: `events-only-${datasetId}` },+ });++ const makeEvent = ({+ experimentId,+ startTime,+ }: {+ experimentId: string;+ startTime: Date;+ }) => {+ const spanId = randomUUID();+ return createEvent({+ id: spanId,+ span_id: spanId,+ project_id: projectId,+ trace_id: randomUUID(),+ type: "GENERATION",+ experiment_id: experimentId,+ experiment_name: `experiment-${experimentId}`,+ experiment_dataset_id: datasetId,+ experiment_item_id: randomUUID(),+ experiment_item_root_span_id: spanId,+ start_time: startTime.getTime() * 1000,+ });+ };++ await createEventsCh([+ makeEvent({+ experimentId: olderExperimentId,+ startTime: olderStart,+ }),+ makeEvent({+ experimentId: latestExperimentId,+ startTime: latestStart,+ }),+ makeEvent({+ experimentId: latestExperimentId,+ startTime: new Date(latestStart.getTime() + 60_000),+ }),+ ]);++ expect(+ await prisma.datasetRuns.count({+ where: { projectId, datasetId },+ }),+ ).toBe(0);++ const result = await caller.datasets.allDatasetsMetrics({+ projectId,+ datasetIds: [datasetId],+ });++ expect(result.metrics).toEqual([+ {+ id: datasetId,+ countDatasetItems: 0,+ countDatasetRuns: 2,+ lastRunAt: latestStart,+ },+ ]);+ });+});diff --git a/web/src/__tests__/server/datasets-metrics.servertest.ts b/web/src/__tests__/server/datasets-metrics.servertest.tsindex 63f69db08c..a6cbfbac3a 100644--- a/web/src/__tests__/server/datasets-metrics.servertest.ts+++ b/web/src/__tests__/server/datasets-metrics.servertest.ts@@ -44,6 +44,7 @@ const session: Session = {experimentsV4Enabled: false,searchBar: false,},+ v4BetaEnabled: true,admin: true,},environment: {} as any,diff --git a/web/src/__tests__/server/repositories/experiment-repository.servertest.ts b/web/src/__tests__/server/repositories/experiment-repository.servertest.tsindex 0ece681b65..b6f408f726 100644--- a/web/src/__tests__/server/repositories/experiment-repository.servertest.ts+++ b/web/src/__tests__/server/repositories/experiment-repository.servertest.ts@@ -5,6 +5,7 @@ import {createEventsCh,getExperimentsCountFromEvents,getExperimentsFromEvents,+ getDatasetExperimentMetricsFromEvents,getExperimentMetricsFromEvents,getExperimentItemsFilterOptions,getExperimentNamesFromEvents,@@ -39,6 +40,85 @@ describe("Clickhouse Experiment Repository Test", () => {expect(count).toBe(0);});+ it("should return experiment count and latest start time per dataset", async () => {+ const datasetId1 = randomUUID();+ const datasetId2 = randomUUID();+ const experimentId1 = randomUUID();+ const experimentId2 = randomUUID();+ const experimentId3 = randomUUID();+ const olderStart = new Date("2026-08-27T10:00:00.000Z");+ const latestStart = new Date("2026-08-29T12:00:00.000Z");++ const makeEvent = ({+ datasetId,+ experimentId,+ startTime,+ }: {+ datasetId: string;+ experimentId: string;+ startTime: Date;+ }) => {+ const spanId = randomUUID();+ return createEvent({+ id: spanId,+ span_id: spanId,+ project_id: projectId,+ trace_id: randomUUID(),+ type: "GENERATION",+ experiment_id: experimentId,+ experiment_name: `experiment-${experimentId}`,+ experiment_dataset_id: datasetId,+ experiment_item_id: randomUUID(),+ experiment_item_root_span_id: spanId,+ start_time: startTime.getTime() * 1000,+ });+ };++ await createEventsCh([+ makeEvent({+ datasetId: datasetId1,+ experimentId: experimentId1,+ startTime: olderStart,+ }),+ makeEvent({+ datasetId: datasetId1,+ experimentId: experimentId2,+ startTime: latestStart,+ }),+ makeEvent({+ datasetId: datasetId1,+ experimentId: experimentId2,+ startTime: new Date(latestStart.getTime() + 60_000),+ }),+ makeEvent({+ datasetId: datasetId2,+ experimentId: experimentId3,+ startTime: olderStart,+ }),+ ]);++ const result = await getDatasetExperimentMetricsFromEvents({+ projectId,+ datasetIds: [datasetId1, datasetId2],+ });++ expect(result).toHaveLength(2);+ expect(result).toEqual(+ expect.arrayContaining([+ {+ datasetId: datasetId1,+ countDatasetRuns: 2,+ lastRunAt: latestStart,+ },+ {+ datasetId: datasetId2,+ countDatasetRuns: 1,+ lastRunAt: olderStart,+ },+ ]),+ );+ });+it("should return one experiment row with two item rows", async () => {const traceId = randomUUID();const generationId = randomUUID();diff --git a/web/src/features/datasets/server/dataset-router.ts b/web/src/features/datasets/server/dataset-router.tsindex 97791bb4b3..0bb1776192 100644--- a/web/src/features/datasets/server/dataset-router.ts+++ b/web/src/features/datasets/server/dataset-router.ts@@ -83,6 +83,7 @@ import {getDatasetItemVersionHistory,getDatasetItemChangesSinceVersion,getDatasetItemsCountGrouped,+ getDatasetExperimentMetricsFromEvents,getDatasetVersionForRun,escapeSqlLikePattern,fetchWithSecureRedirects,@@ -490,31 +491,40 @@ export const datasetRouter = createTRPCRouter({if (input.datasetIds.length === 0) return { metrics: [] };- // Get dataset runs metrics- const runsMetrics = await ctx.prisma.$queryRaw<- Array<{- id: string;- countDatasetRuns: number;- lastRunAt: Date | null;- }>- >`- SELECT d.id, COUNT(DISTINCT dr.id) AS "countDatasetRuns", MAX(dr.created_at) AS "lastRunAt"- FROM datasets d- LEFT JOIN dataset_runs dr ON d.id = dr.dataset_id AND dr.project_id = ${input.projectId}- WHERE d.project_id = ${input.projectId}- AND d.id IN (${Prisma.join(input.datasetIds)})- GROUP BY d.id- `;-- // Get dataset items count for all datasets- const itemsCounts = await getDatasetItemsCountGrouped({- projectId: input.projectId,- datasetIds: input.datasetIds,- });+ const [runsMetrics, itemsCounts] = await Promise.all([+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE === "events_only"+ ? getDatasetExperimentMetricsFromEvents({+ projectId: input.projectId,+ datasetIds: input.datasetIds,+ })+ : ctx.prisma.$queryRaw<+ Array<{+ datasetId: string;+ countDatasetRuns: bigint;+ lastRunAt: Date | null;+ }>+ >`+ SELECT d.id AS "datasetId", COUNT(DISTINCT dr.id) AS "countDatasetRuns", MAX(dr.created_at) AS "lastRunAt"+ FROM datasets d+ LEFT JOIN dataset_runs dr ON d.id = dr.dataset_id AND dr.project_id = ${input.projectId}+ WHERE d.project_id = ${input.projectId}+ AND d.id IN (${Prisma.join(input.datasetIds)})+ GROUP BY d.id+ `.then((rows) =>+ rows.map((row) => ({+ ...row,+ countDatasetRuns: Number(row.countDatasetRuns),+ })),+ ),+ getDatasetItemsCountGrouped({+ projectId: input.projectId,+ datasetIds: input.datasetIds,+ }),+ ]);// Merge the metricsconst metrics = input.datasetIds.map((datasetId) => {- const runsMetric = runsMetrics.find((m) => m.id === datasetId);+ const runsMetric = runsMetrics.find((m) => m.datasetId === datasetId);const itemsCount = itemsCounts.find((m) => m.datasetId === datasetId);return {