Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(datasets): read v4 experiment metrics from events (#16814)

Tobias Wochinger05ecd219ddb8
diff --git a/packages/shared/src/server/repositories/experiments.ts b/packages/shared/src/server/repositories/experiments.ts
index f036474239..2670b1b084 100644
--- a/packages/shared/src/server/repositories/experiments.ts
+++ b/packages/shared/src/server/repositories/experiments.ts
@@ -54,6 +54,54 @@ export type ExperimentMetricsReturnType = {
latency_avg: number | null;
};
+type DatasetExperimentMetricsReturnType = {
+ experiment_dataset_id: string;
+ count_dataset_runs: string;
+ last_run_at: string;
+};
+
+export const getDatasetExperimentMetricsFromEvents = async (props: {
+ projectId: string;
+ datasetIds: string[];
+}) => {
+ if (props.datasetIds.length === 0) {
+ return [];
+ }
+
+ const experimentsQuery = eventsExperimentsAggregation({
+ projectId: props.projectId,
+ fieldSet: "count",
+ })
+ .selectRaw(
+ "nullIf(any(e.experiment_dataset_id), '') AS experiment_dataset_id",
+ "min(e.start_time) AS start_time",
+ )
+ .whereRaw("e.experiment_dataset_id IN ({datasetIds: Array(String)})", {
+ datasetIds: props.datasetIds,
+ })
+ .buildWithParams();
+
+ const rows = await queryClickhouse<DatasetExperimentMetricsReturnType>({
+ query: `
+ SELECT
+ experiment_dataset_id,
+ count() AS count_dataset_runs,
+ max(start_time) AS last_run_at
+ FROM (${experimentsQuery.query}) experiments
+ GROUP BY experiment_dataset_id
+ `,
+ params: experimentsQuery.params,
+ tags: { projectId: props.projectId },
+ preferredClickhouseService: "EventsReadOnly",
+ });
+
+ return rows.map((row) => ({
+ datasetId: row.experiment_dataset_id,
+ countDatasetRuns: Number(row.count_dataset_runs),
+ lastRunAt: parseClickhouseUTCDateTimeFormat(row.last_run_at),
+ }));
+};
+
const experimentScoreCTE = (params: {
projectId: string;
startTimeFrom?: string | null;
diff --git a/web/src/__tests__/server/datasets-metrics-events-only.servertest.ts b/web/src/__tests__/server/datasets-metrics-events-only.servertest.ts
new file mode 100644
index 0000000000..a37ee27d60
--- /dev/null
+++ b/web/src/__tests__/server/datasets-metrics-events-only.servertest.ts
@@ -0,0 +1,146 @@
+import { vi } from "vitest";
+
+const eventsTableAvailable = vi.hoisted(() => {
+ const enabled =
+ process.env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN === "true";
+ process.env.LANGFUSE_MIGRATION_V4_WRITE_MODE = "events_only";
+ process.env.LANGFUSE_MIGRATION_V4_ALLOW_PREVIEW_OPT_IN = "true";
+ return enabled;
+});
+
+import type { Session } from "next-auth";
+import { randomUUID } from "crypto";
+import { prisma } from "@langfuse/shared/src/db";
+import { createEvent, createEventsCh } from "@langfuse/shared/src/server";
+import { env } from "@langfuse/shared/src/env";
+import { appRouter } from "@/src/server/api/root";
+import { createInnerTRPCContext } from "@/src/server/api/trpc";
+
+const maybe = eventsTableAvailable ? describe : describe.skip;
+
+describe("datasets metrics events-only liveness", () => {
+ it("does not hang when the events table is unavailable", () => {});
+});
+
+maybe("datasets.allDatasetsMetrics in events_only write mode", () => {
+ const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+
+ const session: Session = {
+ expires: "1",
+ user: {
+ id: "user-1",
+ canCreateOrganizations: true,
+ name: "Demo User",
+ organizations: [
+ {
+ id: "seed-org-id",
+ name: "Test Organization",
+ role: "OWNER",
+ plan: "cloud:hobby",
+ cloudConfig: undefined,
+ metadata: {},
+ aiFeaturesEnabled: false,
+ aiTelemetryEnabled: false,
+ projects: [
+ {
+ id: projectId,
+ role: "ADMIN",
+ retentionDays: 30,
+ deletedAt: null,
+ name: "Test Project",
+ hasTraces: true,
+ metadata: {},
+ createdAt: new Date().toISOString(),
+ },
+ ],
+ },
+ ],
+ featureFlags: {
+ excludeClickhouseRead: false,
+ templateFlag: true,
+ searchBar: false,
+ v4BetaToggleVisible: false,
+ observationEvals: false,
+ experimentsV4Enabled: false,
+ },
+ v4BetaEnabled: true,
+ admin: true,
+ },
+ environment: {} as any,
+ };
+
+ const ctx = createInnerTRPCContext({ session, headers: {} });
+ const caller = appRouter.createCaller({ ...ctx, prisma });
+
+ it("reads experiment count and latest start time from events", async () => {
+ expect(env.LANGFUSE_MIGRATION_V4_WRITE_MODE).toBe("events_only");
+
+ const datasetId = randomUUID();
+ const olderExperimentId = randomUUID();
+ const latestExperimentId = randomUUID();
+ const olderStart = new Date("2026-08-28T10:00:00.000Z");
+ const latestStart = new Date("2026-08-29T11:00:00.000Z");
+
+ await prisma.dataset.create({
+ data: { id: datasetId, projectId, name: `events-only-${datasetId}` },
+ });
+
+ const makeEvent = ({
+ experimentId,
+ startTime,
+ }: {
+ experimentId: string;
+ startTime: Date;
+ }) => {
+ const spanId = randomUUID();
+ return createEvent({
+ id: spanId,
+ span_id: spanId,
+ project_id: projectId,
+ trace_id: randomUUID(),
+ type: "GENERATION",
+ experiment_id: experimentId,
+ experiment_name: `experiment-${experimentId}`,
+ experiment_dataset_id: datasetId,
+ experiment_item_id: randomUUID(),
+ experiment_item_root_span_id: spanId,
+ start_time: startTime.getTime() * 1000,
+ });
+ };
+
+ await createEventsCh([
+ makeEvent({
+ experimentId: olderExperimentId,
+ startTime: olderStart,
+ }),
+ makeEvent({
+ experimentId: latestExperimentId,
+ startTime: latestStart,
+ }),
+ makeEvent({
+ experimentId: latestExperimentId,
+ startTime: new Date(latestStart.getTime() + 60_000),
+ }),
+ ]);
+
+ expect(
+ await prisma.datasetRuns.count({
+ where: { projectId, datasetId },
+ }),
+ ).toBe(0);
+
+ const result = await caller.datasets.allDatasetsMetrics({
+ projectId,
+ datasetIds: [datasetId],
+ });
+
+ expect(result.metrics).toEqual([
+ {
+ id: datasetId,
+ countDatasetItems: 0,
+ countDatasetRuns: 2,
+ lastRunAt: latestStart,
+ },
+ ]);
+ });
+});
diff --git a/web/src/__tests__/server/datasets-metrics.servertest.ts b/web/src/__tests__/server/datasets-metrics.servertest.ts
index 63f69db08c..a6cbfbac3a 100644
--- a/web/src/__tests__/server/datasets-metrics.servertest.ts
+++ b/web/src/__tests__/server/datasets-metrics.servertest.ts
@@ -44,6 +44,7 @@ const session: Session = {
experimentsV4Enabled: false,
searchBar: false,
},
+ v4BetaEnabled: true,
admin: true,
},
environment: {} as any,
diff --git a/web/src/__tests__/server/repositories/experiment-repository.servertest.ts b/web/src/__tests__/server/repositories/experiment-repository.servertest.ts
index 0ece681b65..b6f408f726 100644
--- a/web/src/__tests__/server/repositories/experiment-repository.servertest.ts
+++ b/web/src/__tests__/server/repositories/experiment-repository.servertest.ts
@@ -5,6 +5,7 @@ import {
createEventsCh,
getExperimentsCountFromEvents,
getExperimentsFromEvents,
+ getDatasetExperimentMetricsFromEvents,
getExperimentMetricsFromEvents,
getExperimentItemsFilterOptions,
getExperimentNamesFromEvents,
@@ -39,6 +40,85 @@ describe("Clickhouse Experiment Repository Test", () => {
expect(count).toBe(0);
});
+ it("should return experiment count and latest start time per dataset", async () => {
+ const datasetId1 = randomUUID();
+ const datasetId2 = randomUUID();
+ const experimentId1 = randomUUID();
+ const experimentId2 = randomUUID();
+ const experimentId3 = randomUUID();
+ const olderStart = new Date("2026-08-27T10:00:00.000Z");
+ const latestStart = new Date("2026-08-29T12:00:00.000Z");
+
+ const makeEvent = ({
+ datasetId,
+ experimentId,
+ startTime,
+ }: {
+ datasetId: string;
+ experimentId: string;
+ startTime: Date;
+ }) => {
+ const spanId = randomUUID();
+ return createEvent({
+ id: spanId,
+ span_id: spanId,
+ project_id: projectId,
+ trace_id: randomUUID(),
+ type: "GENERATION",
+ experiment_id: experimentId,
+ experiment_name: `experiment-${experimentId}`,
+ experiment_dataset_id: datasetId,
+ experiment_item_id: randomUUID(),
+ experiment_item_root_span_id: spanId,
+ start_time: startTime.getTime() * 1000,
+ });
+ };
+
+ await createEventsCh([
+ makeEvent({
+ datasetId: datasetId1,
+ experimentId: experimentId1,
+ startTime: olderStart,
+ }),
+ makeEvent({
+ datasetId: datasetId1,
+ experimentId: experimentId2,
+ startTime: latestStart,
+ }),
+ makeEvent({
+ datasetId: datasetId1,
+ experimentId: experimentId2,
+ startTime: new Date(latestStart.getTime() + 60_000),
+ }),
+ makeEvent({
+ datasetId: datasetId2,
+ experimentId: experimentId3,
+ startTime: olderStart,
+ }),
+ ]);
+
+ const result = await getDatasetExperimentMetricsFromEvents({
+ projectId,
+ datasetIds: [datasetId1, datasetId2],
+ });
+
+ expect(result).toHaveLength(2);
+ expect(result).toEqual(
+ expect.arrayContaining([
+ {
+ datasetId: datasetId1,
+ countDatasetRuns: 2,
+ lastRunAt: latestStart,
+ },
+ {
+ datasetId: datasetId2,
+ countDatasetRuns: 1,
+ lastRunAt: olderStart,
+ },
+ ]),
+ );
+ });
+
it("should return one experiment row with two item rows", async () => {
const traceId = randomUUID();
const generationId = randomUUID();
diff --git a/web/src/features/datasets/server/dataset-router.ts b/web/src/features/datasets/server/dataset-router.ts
index 97791bb4b3..0bb1776192 100644
--- a/web/src/features/datasets/server/dataset-router.ts
+++ b/web/src/features/datasets/server/dataset-router.ts
@@ -83,6 +83,7 @@ import {
getDatasetItemVersionHistory,
getDatasetItemChangesSinceVersion,
getDatasetItemsCountGrouped,
+ getDatasetExperimentMetricsFromEvents,
getDatasetVersionForRun,
escapeSqlLikePattern,
fetchWithSecureRedirects,
@@ -490,31 +491,40 @@ export const datasetRouter = createTRPCRouter({
if (input.datasetIds.length === 0) return { metrics: [] };
- // Get dataset runs metrics
- const runsMetrics = await ctx.prisma.$queryRaw<
- Array<{
- id: string;
- countDatasetRuns: number;
- lastRunAt: Date | null;
- }>
- >`
- SELECT d.id, COUNT(DISTINCT dr.id) AS "countDatasetRuns", MAX(dr.created_at) AS "lastRunAt"
- FROM datasets d
- LEFT JOIN dataset_runs dr ON d.id = dr.dataset_id AND dr.project_id = ${input.projectId}
- WHERE d.project_id = ${input.projectId}
- AND d.id IN (${Prisma.join(input.datasetIds)})
- GROUP BY d.id
- `;
-
- // Get dataset items count for all datasets
- const itemsCounts = await getDatasetItemsCountGrouped({
- projectId: input.projectId,
- datasetIds: input.datasetIds,
- });
+ const [runsMetrics, itemsCounts] = await Promise.all([
+ env.LANGFUSE_MIGRATION_V4_WRITE_MODE === "events_only"
+ ? getDatasetExperimentMetricsFromEvents({
+ projectId: input.projectId,
+ datasetIds: input.datasetIds,
+ })
+ : ctx.prisma.$queryRaw<
+ Array<{
+ datasetId: string;
+ countDatasetRuns: bigint;
+ lastRunAt: Date | null;
+ }>
+ >`
+ SELECT d.id AS "datasetId", COUNT(DISTINCT dr.id) AS "countDatasetRuns", MAX(dr.created_at) AS "lastRunAt"
+ FROM datasets d
+ LEFT JOIN dataset_runs dr ON d.id = dr.dataset_id AND dr.project_id = ${input.projectId}
+ WHERE d.project_id = ${input.projectId}
+ AND d.id IN (${Prisma.join(input.datasetIds)})
+ GROUP BY d.id
+ `.then((rows) =>
+ rows.map((row) => ({
+ ...row,
+ countDatasetRuns: Number(row.countDatasetRuns),
+ })),
+ ),
+ getDatasetItemsCountGrouped({
+ projectId: input.projectId,
+ datasetIds: input.datasetIds,
+ }),
+ ]);
// Merge the metrics
const metrics = input.datasetIds.map((datasetId) => {
- const runsMetric = runsMetrics.find((m) => m.id === datasetId);
+ const runsMetric = runsMetrics.find((m) => m.datasetId === datasetId);
const itemsCount = itemsCounts.find((m) => m.datasetId === datasetId);
return {