🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(v4): merge legacy API usage across ClickHouse services (#16106)
Max Deichmann7cc6d2c0b925
diff --git a/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts b/web/src/__tests__/server/unit/v4TransitionRouter.servertest.tsindex b86f8b54af..50bfa8753b 100644--- a/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts+++ b/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts@@ -99,6 +99,14 @@ const sharedServerMock = vi.hoisted(() => ({},}));+const sharedEnvMock = vi.hoisted(() => ({+ CLICKHOUSE_URL: "https://clickhouse-main.example.com",+ CLICKHOUSE_READ_ONLY_URL: "https://clickhouse-read.example.com",+ CLICKHOUSE_EVENTS_READ_ONLY_URL: "https://clickhouse-main.example.com",+}));++vi.mock("@langfuse/shared/src/env", () => ({ env: sharedEnvMock }));+vi.mock("@langfuse/shared/src/server", async () => {const { ROOT_CONTEXT } = await import("@opentelemetry/api");@@ -370,6 +378,10 @@ const accessibleProjectsFindManyArgs = {describe("v4TransitionRouter", () => {beforeEach(() => {+ sharedEnvMock.CLICKHOUSE_READ_ONLY_URL =+ "https://clickhouse-read.example.com";+ sharedEnvMock.CLICKHOUSE_EVENTS_READ_ONLY_URL =+ sharedEnvMock.CLICKHOUSE_URL;mockedQueryClickhouse.mockResolvedValue([{projectId,@@ -385,6 +397,29 @@ describe("v4TransitionRouter", () => {});it("summarizes legacy public API usage for a project with route classification", async () => {+ mockedQueryClickhouse+ .mockResolvedValueOnce([+ {+ projectId,+ entrypoint: "publicapi: GET /api/public/traces/{id}",+ count: "0.6666666666666666",+ lastSeen: "2026-06-24T12:34:56.789123Z",+ },+ {+ projectId,+ entrypoint: "publicapi: GET /api/public/datasets/{datasetName}/runs",+ count: "1",+ lastSeen: "2026-06-24T14:00:00.000000Z",+ },+ ])+ .mockResolvedValueOnce([+ {+ projectId,+ entrypoint: "publicapi: GET /api/public/traces/{id}",+ count: "1.3333333333333335",+ lastSeen: "2026-06-24T15:00:00.000000Z",+ },+ ]);const caller = createCaller();const rows = await caller.legacyApiUsageSummary({@@ -394,15 +429,21 @@ describe("v4TransitionRouter", () => {});expect(rows).toEqual([+ {+ projectId,+ entrypoint: "publicapi: GET /api/public/datasets/{datasetName}/runs",+ count: 1,+ lastSeen: "2026-06-24T14:00:00.000000Z",+ },{projectId,entrypoint: "publicapi: GET /api/public/traces/{id}",- count: 0.6666666666666666,- lastSeen: "2026-06-24T12:34:56.789123Z",+ count: 2,+ lastSeen: "2026-06-24T15:00:00.000000Z",},]);- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);const clickhouseQuery = mockedQueryClickhouse.mock.calls[0]?.[0];expect(clickhouseQuery?.query).toContain("FROM clusterAllReplicas('test-cluster', 'system.query_log')",@@ -434,6 +475,7 @@ describe("v4TransitionRouter", () => {expect(clickhouseQuery?.query).toContain("GROUP BY project_id, legacy_route",);+ expect(clickhouseQuery?.query).not.toContain("hostName()");expect(clickhouseQuery?.params).toMatchObject({projectIds: [projectId],});@@ -443,6 +485,12 @@ describe("v4TransitionRouter", () => {expect(clickhouseQuery?.clickhouseSettings).toEqual({skip_unavailable_shards: 1,});+ expect(mockedQueryClickhouse.mock.calls[0]?.[0]).toMatchObject({+ preferredClickhouseService: "ReadOnly",+ });+ expect(mockedQueryClickhouse.mock.calls[1]?.[0]).toMatchObject({+ preferredClickhouseService: "ReadWrite",+ });["GET /api/public/spans",@@ -463,12 +511,16 @@ describe("v4TransitionRouter", () => {"GET /api/public/observations/{id}","GET /api/public/scores/{id}","GET /api/public/v2/scores/{id}",+ "GET /api/public/datasets/{datasetName}/runs","GET /api/public/datasets/{datasetName}/runs/{runName}",].forEach((route) => expect(clickhouseQuery?.query).toContain(route));+ expect(clickhouseQuery?.query).toContain(+ "match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 'GET /api/public/datasets/{datasetName}/runs'",+ );expect(clickhouseQuery?.query).toContain("match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 1",);-+ expect(clickhouseQuery?.query).not.toContain("DELETE /api/public/datasets");expect(clickhouseQuery?.query).toContain("'GET /api/public/traces',\n 'GET /api/public/observations',\n 'GET /api/public/scores',\n 'GET /api/public/v2/scores',\n 'GET /api/public/metrics/daily',\n 'GET /api/public/dataset-run-items'\n ), 2",);@@ -494,6 +546,89 @@ describe("v4TransitionRouter", () => {expect(mockedQueryClickhouse).not.toHaveBeenCalled();});+ it("queries the main service once when no separate read replica is configured", async () => {+ sharedEnvMock.CLICKHOUSE_READ_ONLY_URL = sharedEnvMock.CLICKHOUSE_URL;+ const caller = createCaller();++ await caller.legacyApiUsageSummary({+ projectId,+ fromTimestamp: new Date("2026-06-24T00:00:00Z"),+ toTimestamp: new Date("2026-06-25T00:00:00Z"),+ });++ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);+ expect(mockedQueryClickhouse).toHaveBeenCalledWith(+ expect.objectContaining({ preferredClickhouseService: "ReadWrite" }),+ );+ });++ it("returns legacy API usage when one ClickHouse service is unavailable", async () => {+ mockedQueryClickhouse+ .mockRejectedValueOnce(new Error("read replica unavailable"))+ .mockResolvedValueOnce([+ {+ projectId,+ entrypoint: "publicapi: GET /api/public/traces/{id}",+ count: "1",+ lastSeen: "2026-06-24T15:00:00.000000Z",+ },+ ]);+ const caller = createCaller();++ await expect(+ caller.legacyApiUsageSummary({+ projectId,+ fromTimestamp: new Date("2026-06-24T00:00:00Z"),+ toTimestamp: new Date("2026-06-25T00:00:00Z"),+ }),+ ).resolves.toEqual([+ {+ projectId,+ entrypoint: "publicapi: GET /api/public/traces/{id}",+ count: 1,+ lastSeen: "2026-06-24T15:00:00.000000Z",+ },+ ]);++ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);+ });++ it("does not double-count identical results from two ClickHouse services", async () => {+ mockedQueryClickhouse+ .mockResolvedValueOnce([+ {+ projectId,+ entrypoint: "publicapi: GET /api/public/traces/{id}",+ count: "2",+ lastSeen: "2026-06-24T14:00:00.000000Z",+ },+ ])+ .mockResolvedValueOnce([+ {+ projectId,+ entrypoint: "publicapi: GET /api/public/traces/{id}",+ count: "2",+ lastSeen: "2026-06-24T14:00:00.000000Z",+ },+ ]);+ const caller = createCaller();++ await expect(+ caller.legacyApiUsageSummary({+ projectId,+ fromTimestamp: new Date("2026-06-24T00:00:00Z"),+ toTimestamp: new Date("2026-06-25T00:00:00Z"),+ }),+ ).resolves.toEqual([+ {+ projectId,+ entrypoint: "publicapi: GET /api/public/traces/{id}",+ count: 2,+ lastSeen: "2026-06-24T14:00:00.000000Z",+ },+ ]);+ });+it("queries SDK usage for only the authorized project", async () => {mockedQueryClickhouse.mockResolvedValueOnce([@@ -1271,6 +1406,7 @@ describe("v4TransitionRouter", () => {expect(experimentUsageQuery?.query).toContain("event_date >= toDate({fromTimestamp: DateTime64(3)})",);+ expect(experimentUsageQuery?.query).not.toContain("hostName()");expect(experimentUsageQuery?.params).toMatchObject({projectIds: [projectId, secondProjectId],fromTimestamp: "2026-06-24 00:00:00.000",@@ -1281,6 +1417,45 @@ describe("v4TransitionRouter", () => {});});+ it("finds dataset-run-items POST usage when only the main service has it", async () => {+ sharedEnvMock.CLICKHOUSE_EVENTS_READ_ONLY_URL =+ "https://clickhouse-events-read.example.com";+ mockedQueryClickhouse+ .mockResolvedValueOnce([+ mockSdkUsageRow({+ projectId,+ source: "otel",+ sdkName: "unknown",+ sdkVersion: "unknown",+ }),+ ])+ .mockRejectedValueOnce(new Error("events read replica unavailable"))+ .mockResolvedValueOnce([{ projectId, count: "1" }]);+ const caller = createCaller({+ project: {+ findMany: vi.fn().mockResolvedValue([{ id: projectId }]),+ },+ });++ const [summary] = await caller.sdkUsageSummaryByProject({+ orgId,+ fromTimestamp: new Date("2026-06-24T00:00:00Z"),+ toTimestamp: new Date("2026-06-25T00:00:00Z"),+ });++ expect(summary?.experimentInstrumentationMigration).toEqual({+ status: "required",+ upgradePath: "api",+ });+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(3);+ expect(mockedQueryClickhouse.mock.calls[1]?.[0]).toMatchObject({+ preferredClickhouseService: "EventsReadOnly",+ });+ expect(mockedQueryClickhouse.mock.calls[2]?.[0]).toMatchObject({+ preferredClickhouseService: "ReadWrite",+ });+ });+it("summarizes SDK usage for a single project with exactly that projectId", async () => {mockedQueryClickhouse.mockResolvedValueOnce([@@ -1562,20 +1737,35 @@ describe("v4TransitionRouter", () => {});it("summarizes legacy public API usage by organization project", async () => {- mockedQueryClickhouse.mockResolvedValue([- {- projectId,- entrypoint: "publicapi: GET /api/public/traces/{id}",- count: "0.6666666666666666",- lastSeen: "2026-06-24T12:34:56.789123Z",- },- {- projectId: secondProjectId,- entrypoint: "publicapi: GET /api/public/metrics",- count: 3,- lastSeen: "2026-06-24T15:00:00.000000Z",- },- ]);+ mockedQueryClickhouse+ .mockResolvedValueOnce([+ {+ projectId,+ entrypoint: "publicapi: GET /api/public/traces/{id}",+ count: "0.6666666666666666",+ lastSeen: "2026-06-24T12:34:56.789123Z",+ },+ {+ projectId: secondProjectId,+ entrypoint: "publicapi: GET /api/public/metrics",+ count: 3,+ lastSeen: "2026-06-24T15:00:00.000000Z",+ },+ ])+ .mockResolvedValueOnce([+ {+ projectId,+ entrypoint: "publicapi: GET /api/public/traces/{id}",+ count: "0.3333333333333334",+ lastSeen: "2026-06-24T14:00:00.000000Z",+ },+ {+ projectId: secondProjectId,+ entrypoint: "publicapi: GET /api/public/metrics",+ count: 2,+ lastSeen: "2026-06-24T16:00:00.000000Z",+ },+ ]);const mockPrisma = {project: {findMany: vi.fn().mockResolvedValue([@@ -1596,21 +1786,21 @@ describe("v4TransitionRouter", () => {{projectId,entrypoint: "publicapi: GET /api/public/traces/{id}",- count: 0.6666666666666666,- lastSeen: "2026-06-24T12:34:56.789123Z",+ count: 1,+ lastSeen: "2026-06-24T14:00:00.000000Z",},{projectId: secondProjectId,entrypoint: "publicapi: GET /api/public/metrics",- count: 3,- lastSeen: "2026-06-24T15:00:00.000000Z",+ count: 5,+ lastSeen: "2026-06-24T16:00:00.000000Z",},]);expect(mockPrisma.project.findMany).toHaveBeenCalledWith(accessibleProjectsFindManyArgs,);- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);const clickhouseQuery = mockedQueryClickhouse.mock.calls[0]?.[0];expect(clickhouseQuery?.query).not.toContain("toStartOfInterval");expect(clickhouseQuery?.query).not.toContain("bucket_time");@@ -1633,5 +1823,11 @@ describe("v4TransitionRouter", () => {expect(clickhouseQuery?.tags).toEqual({route: "v4-legacy-api-usage-summary",});+ expect(mockedQueryClickhouse.mock.calls[0]?.[0]).toMatchObject({+ preferredClickhouseService: "ReadOnly",+ });+ expect(mockedQueryClickhouse.mock.calls[1]?.[0]).toMatchObject({+ preferredClickhouseService: "ReadWrite",+ });});});diff --git a/web/src/features/v4/server/v4TransitionRouter.ts b/web/src/features/v4/server/v4TransitionRouter.tsindex 25218268fa..5616068032 100644--- a/web/src/features/v4/server/v4TransitionRouter.ts+++ b/web/src/features/v4/server/v4TransitionRouter.ts@@ -10,6 +10,7 @@ import {type PrismaClient,} from "@langfuse/shared/src/db";import type { Session } from "next-auth";+import { env } from "@langfuse/shared/src/env";import {convertDateToClickhouseDateTime,INTERNAL_INGESTION_SDK_NAMES,@@ -18,6 +19,7 @@ import {queryClickhouse,systemTableRef,type IngestionSdkAttributionStatus,+ type PreferredClickhouseService,} from "@langfuse/shared/src/server";import { getSdkVersionCapabilityStatus } from "@/src/features/sdk-version/lib/sdkVersionCapabilities";const MAX_DETECTION_RANGE_MS = 30 * 24 * 60 * 60 * 1000;@@ -174,6 +176,59 @@ type DatasetRunItemsPostUsageByProjectRow = {count: string | number;};+type SystemQueryLogReadService = "ReadOnly" | "EventsReadOnly";++const getSystemQueryLogServices = ({+ readService,+ readUrl,+}: {+ readService: SystemQueryLogReadService;+ readUrl: string | undefined;+}): PreferredClickhouseService[] =>+ readUrl &&+ new URL(readUrl).toString() !== new URL(env.CLICKHOUSE_URL).toString()+ ? [readService, "ReadWrite"]+ : ["ReadWrite"];++const querySystemQueryLogAcrossServices = async <T>({+ preferredClickhouseServices,+ queryService,+ failureMessage,+}: {+ preferredClickhouseServices: PreferredClickhouseService[];+ queryService: (service: PreferredClickhouseService) => Promise<T[]>;+ failureMessage: string;+}): Promise<T[][]> => {+ const settledServiceRows = await Promise.allSettled(+ preferredClickhouseServices.map(queryService),+ );+ const firstSuccessfulResult = settledServiceRows.find(+ (result) => result.status === "fulfilled",+ );+ if (!firstSuccessfulResult) {+ const firstFailure = settledServiceRows.find(+ (result) => result.status === "rejected",+ );+ throw firstFailure?.reason ?? new Error("ClickHouse services unavailable");+ }++ settledServiceRows.forEach((result, index) => {+ if (result.status === "rejected") {+ logger.warn(failureMessage, {+ preferredClickhouseService: preferredClickhouseServices[index],+ error:+ result.reason instanceof Error+ ? result.reason.message+ : "Unknown error",+ });+ }+ });++ return settledServiceRows.flatMap((result) =>+ result.status === "fulfilled" ? [result.value] : [],+ );+};+const toBoolean = (value: boolean | string | number): boolean =>value === true || value === 1 || value === "1";@@ -354,6 +409,12 @@ const getSdkUsageSummaries = async ({}): Promise<SdkUsageSummaryByProjectResultRow[]> => {if (projectIds.length === 0) return [];+ const systemQueryLogServices = getSystemQueryLogServices({+ readService: "EventsReadOnly",+ readUrl:+ env.CLICKHOUSE_EVENTS_READ_ONLY_URL ?? env.CLICKHOUSE_READ_ONLY_URL,+ });+const [rows, datasetRunItemsPostUsageResult] = await Promise.all([queryClickhouse<SdkUsageSummaryByProjectRow>({query: `@@ -482,8 +543,11 @@ ORDER BYtags: { route: "v4-sdk-usage-summary" },preferredClickhouseService: "EventsReadOnly",}),- queryClickhouse<DatasetRunItemsPostUsageByProjectRow>({- query: `+ querySystemQueryLogAcrossServices({+ preferredClickhouseServices: systemQueryLogServices,+ queryService: (preferredClickhouseService) =>+ queryClickhouse<DatasetRunItemsPostUsageByProjectRow>({+ query: `SELECTJSONExtractString(log_comment, 'projectId') AS projectId,count() AS count@@ -501,16 +565,22 @@ WHEREGROUP BY projectIdSETTINGS skip_unavailable_shards = 1`,- params: {- projectIds,- fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp),- toTimestamp: convertDateToClickhouseDateTime(toTimestamp),- },- tags: { route: "v4-experiment-instrumentation-summary" },- preferredClickhouseService: "EventsReadOnly",- clickhouseSettings: { skip_unavailable_shards: 1 },+ params: {+ projectIds,+ fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp),+ toTimestamp: convertDateToClickhouseDateTime(toTimestamp),+ },+ tags: { route: "v4-experiment-instrumentation-summary" },+ preferredClickhouseService,+ clickhouseSettings: { skip_unavailable_shards: 1 },+ }),+ failureMessage:+ "Failed to query dataset-run-items POST usage from ClickHouse service",})- .then((rows) => ({ status: "success" as const, rows }))+ .then((serviceRows) => ({+ status: "success" as const,+ rows: serviceRows.flat(),+ })).catch((error: unknown) => {logger.warn("Failed to query dataset-run-items POST usage for v4 migration",@@ -604,14 +674,16 @@ SETTINGS skip_unavailable_shards = 1});};-const getLegacyApiUsageSummaries = async ({+const getLegacyApiUsageSummariesForService = async ({projectIds,fromTimestamp,toTimestamp,+ preferredClickhouseService,}: {projectIds: string[];fromTimestamp: Date;toTimestamp: Date;+ preferredClickhouseService: PreferredClickhouseService;}): Promise<LegacyApiUsageSummaryByProjectResultRow[]> => {if (projectIds.length === 0) return [];@@ -655,6 +727,7 @@ classified AS (match(route_path, '^GET /api/public/observations/[^/?#]+$'), 'GET /api/public/observations/{id}',match(route_path, '^GET /api/public/scores/[^/?#]+$'), 'GET /api/public/scores/{id}',match(route_path, '^GET /api/public/v2/scores/[^/?#]+$'), 'GET /api/public/v2/scores/{id}',+ match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 'GET /api/public/datasets/{datasetName}/runs',match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 'GET /api/public/datasets/{datasetName}/runs/{runName}',NULL) AS legacy_route,@@ -678,6 +751,7 @@ classified AS (match(route_path, '^GET /api/public/observations/[^/?#]+$'), 1,match(route_path, '^GET /api/public/scores/[^/?#]+$'), 1,match(route_path, '^GET /api/public/v2/scores/[^/?#]+$'), 1,+ match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 1,match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 1,NULL) AS clickhouse_queries_per_api_call@@ -702,7 +776,7 @@ SETTINGS skip_unavailable_shards = 1toTimestamp: convertDateToClickhouseDateTime(toTimestamp),},tags: { route: "v4-legacy-api-usage-summary" },- preferredClickhouseService: "ReadOnly",+ preferredClickhouseService,clickhouseSettings: { skip_unavailable_shards: 1 },});@@ -714,6 +788,79 @@ SETTINGS skip_unavailable_shards = 1}));};+const getLegacyApiUsageSummaries = async ({+ projectIds,+ fromTimestamp,+ toTimestamp,+}: {+ projectIds: string[];+ fromTimestamp: Date;+ toTimestamp: Date;+}): Promise<LegacyApiUsageSummaryByProjectResultRow[]> => {+ if (projectIds.length === 0) return [];++ const preferredClickhouseServices = getSystemQueryLogServices({+ readService: "ReadOnly",+ readUrl: env.CLICKHOUSE_READ_ONLY_URL,+ });+ const serviceResultSets = await querySystemQueryLogAcrossServices({+ preferredClickhouseServices,+ queryService: (preferredClickhouseService) =>+ getLegacyApiUsageSummariesForService({+ projectIds,+ fromTimestamp,+ toTimestamp,+ preferredClickhouseService,+ }),+ failureMessage: "Failed to query legacy API usage from ClickHouse service",+ });++ const uniqueServiceResultSets = new Map<+ string,+ LegacyApiUsageSummaryByProjectResultRow[]+ >();+ for (const rows of serviceResultSets) {+ const sortedRows = [...rows].sort(+ (left, right) =>+ left.projectId.localeCompare(right.projectId) ||+ left.entrypoint.localeCompare(right.entrypoint),+ );+ uniqueServiceResultSets.set(JSON.stringify(sortedRows), sortedRows);+ }+ const deduplicatedServiceRows = Array.from(+ uniqueServiceResultSets.values(),+ ).flat();++ const rowsByProjectAndEntrypoint = new Map<+ string,+ LegacyApiUsageSummaryByProjectResultRow+ >();++ for (const row of deduplicatedServiceRows) {+ const key = `${row.projectId}\u0000${row.entrypoint}`;+ const existing = rowsByProjectAndEntrypoint.get(key);+ rowsByProjectAndEntrypoint.set(+ key,+ existing+ ? {+ ...existing,+ count: existing.count + row.count,+ lastSeen:+ existing.lastSeen > row.lastSeen+ ? existing.lastSeen+ : row.lastSeen,+ }+ : row,+ );+ }++ return Array.from(rowsByProjectAndEntrypoint.values()).sort(+ (left, right) =>+ left.projectId.localeCompare(right.projectId) ||+ left.entrypoint.localeCompare(right.entrypoint),+ );+};+export const v4TransitionRouter = createTRPCRouter({forceV3Experience: protectedProjectProcedure.input(z.object({ projectId: z.string() }))