Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(v4): merge legacy API usage across ClickHouse services (#16106)

Max Deichmann7cc6d2c0b925
diff --git a/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts b/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts
index b86f8b54af..50bfa8753b 100644
--- a/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts
+++ b/web/src/__tests__/server/unit/v4TransitionRouter.servertest.ts
@@ -99,6 +99,14 @@ const sharedServerMock = vi.hoisted(() => ({
},
}));
+const sharedEnvMock = vi.hoisted(() => ({
+ CLICKHOUSE_URL: "https://clickhouse-main.example.com",
+ CLICKHOUSE_READ_ONLY_URL: "https://clickhouse-read.example.com",
+ CLICKHOUSE_EVENTS_READ_ONLY_URL: "https://clickhouse-main.example.com",
+}));
+
+vi.mock("@langfuse/shared/src/env", () => ({ env: sharedEnvMock }));
+
vi.mock("@langfuse/shared/src/server", async () => {
const { ROOT_CONTEXT } = await import("@opentelemetry/api");
@@ -370,6 +378,10 @@ const accessibleProjectsFindManyArgs = {
describe("v4TransitionRouter", () => {
beforeEach(() => {
+ sharedEnvMock.CLICKHOUSE_READ_ONLY_URL =
+ "https://clickhouse-read.example.com";
+ sharedEnvMock.CLICKHOUSE_EVENTS_READ_ONLY_URL =
+ sharedEnvMock.CLICKHOUSE_URL;
mockedQueryClickhouse.mockResolvedValue([
{
projectId,
@@ -385,6 +397,29 @@ describe("v4TransitionRouter", () => {
});
it("summarizes legacy public API usage for a project with route classification", async () => {
+ mockedQueryClickhouse
+ .mockResolvedValueOnce([
+ {
+ projectId,
+ entrypoint: "publicapi: GET /api/public/traces/{id}",
+ count: "0.6666666666666666",
+ lastSeen: "2026-06-24T12:34:56.789123Z",
+ },
+ {
+ projectId,
+ entrypoint: "publicapi: GET /api/public/datasets/{datasetName}/runs",
+ count: "1",
+ lastSeen: "2026-06-24T14:00:00.000000Z",
+ },
+ ])
+ .mockResolvedValueOnce([
+ {
+ projectId,
+ entrypoint: "publicapi: GET /api/public/traces/{id}",
+ count: "1.3333333333333335",
+ lastSeen: "2026-06-24T15:00:00.000000Z",
+ },
+ ]);
const caller = createCaller();
const rows = await caller.legacyApiUsageSummary({
@@ -394,15 +429,21 @@ describe("v4TransitionRouter", () => {
});
expect(rows).toEqual([
+ {
+ projectId,
+ entrypoint: "publicapi: GET /api/public/datasets/{datasetName}/runs",
+ count: 1,
+ lastSeen: "2026-06-24T14:00:00.000000Z",
+ },
{
projectId,
entrypoint: "publicapi: GET /api/public/traces/{id}",
- count: 0.6666666666666666,
- lastSeen: "2026-06-24T12:34:56.789123Z",
+ count: 2,
+ lastSeen: "2026-06-24T15:00:00.000000Z",
},
]);
- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);
+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);
const clickhouseQuery = mockedQueryClickhouse.mock.calls[0]?.[0];
expect(clickhouseQuery?.query).toContain(
"FROM clusterAllReplicas('test-cluster', 'system.query_log')",
@@ -434,6 +475,7 @@ describe("v4TransitionRouter", () => {
expect(clickhouseQuery?.query).toContain(
"GROUP BY project_id, legacy_route",
);
+ expect(clickhouseQuery?.query).not.toContain("hostName()");
expect(clickhouseQuery?.params).toMatchObject({
projectIds: [projectId],
});
@@ -443,6 +485,12 @@ describe("v4TransitionRouter", () => {
expect(clickhouseQuery?.clickhouseSettings).toEqual({
skip_unavailable_shards: 1,
});
+ expect(mockedQueryClickhouse.mock.calls[0]?.[0]).toMatchObject({
+ preferredClickhouseService: "ReadOnly",
+ });
+ expect(mockedQueryClickhouse.mock.calls[1]?.[0]).toMatchObject({
+ preferredClickhouseService: "ReadWrite",
+ });
[
"GET /api/public/spans",
@@ -463,12 +511,16 @@ describe("v4TransitionRouter", () => {
"GET /api/public/observations/{id}",
"GET /api/public/scores/{id}",
"GET /api/public/v2/scores/{id}",
+ "GET /api/public/datasets/{datasetName}/runs",
"GET /api/public/datasets/{datasetName}/runs/{runName}",
].forEach((route) => expect(clickhouseQuery?.query).toContain(route));
+ expect(clickhouseQuery?.query).toContain(
+ "match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 'GET /api/public/datasets/{datasetName}/runs'",
+ );
expect(clickhouseQuery?.query).toContain(
"match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 1",
);
-
+ expect(clickhouseQuery?.query).not.toContain("DELETE /api/public/datasets");
expect(clickhouseQuery?.query).toContain(
"'GET /api/public/traces',\n 'GET /api/public/observations',\n 'GET /api/public/scores',\n 'GET /api/public/v2/scores',\n 'GET /api/public/metrics/daily',\n 'GET /api/public/dataset-run-items'\n ), 2",
);
@@ -494,6 +546,89 @@ describe("v4TransitionRouter", () => {
expect(mockedQueryClickhouse).not.toHaveBeenCalled();
});
+ it("queries the main service once when no separate read replica is configured", async () => {
+ sharedEnvMock.CLICKHOUSE_READ_ONLY_URL = sharedEnvMock.CLICKHOUSE_URL;
+ const caller = createCaller();
+
+ await caller.legacyApiUsageSummary({
+ projectId,
+ fromTimestamp: new Date("2026-06-24T00:00:00Z"),
+ toTimestamp: new Date("2026-06-25T00:00:00Z"),
+ });
+
+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);
+ expect(mockedQueryClickhouse).toHaveBeenCalledWith(
+ expect.objectContaining({ preferredClickhouseService: "ReadWrite" }),
+ );
+ });
+
+ it("returns legacy API usage when one ClickHouse service is unavailable", async () => {
+ mockedQueryClickhouse
+ .mockRejectedValueOnce(new Error("read replica unavailable"))
+ .mockResolvedValueOnce([
+ {
+ projectId,
+ entrypoint: "publicapi: GET /api/public/traces/{id}",
+ count: "1",
+ lastSeen: "2026-06-24T15:00:00.000000Z",
+ },
+ ]);
+ const caller = createCaller();
+
+ await expect(
+ caller.legacyApiUsageSummary({
+ projectId,
+ fromTimestamp: new Date("2026-06-24T00:00:00Z"),
+ toTimestamp: new Date("2026-06-25T00:00:00Z"),
+ }),
+ ).resolves.toEqual([
+ {
+ projectId,
+ entrypoint: "publicapi: GET /api/public/traces/{id}",
+ count: 1,
+ lastSeen: "2026-06-24T15:00:00.000000Z",
+ },
+ ]);
+
+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);
+ });
+
+ it("does not double-count identical results from two ClickHouse services", async () => {
+ mockedQueryClickhouse
+ .mockResolvedValueOnce([
+ {
+ projectId,
+ entrypoint: "publicapi: GET /api/public/traces/{id}",
+ count: "2",
+ lastSeen: "2026-06-24T14:00:00.000000Z",
+ },
+ ])
+ .mockResolvedValueOnce([
+ {
+ projectId,
+ entrypoint: "publicapi: GET /api/public/traces/{id}",
+ count: "2",
+ lastSeen: "2026-06-24T14:00:00.000000Z",
+ },
+ ]);
+ const caller = createCaller();
+
+ await expect(
+ caller.legacyApiUsageSummary({
+ projectId,
+ fromTimestamp: new Date("2026-06-24T00:00:00Z"),
+ toTimestamp: new Date("2026-06-25T00:00:00Z"),
+ }),
+ ).resolves.toEqual([
+ {
+ projectId,
+ entrypoint: "publicapi: GET /api/public/traces/{id}",
+ count: 2,
+ lastSeen: "2026-06-24T14:00:00.000000Z",
+ },
+ ]);
+ });
+
it("queries SDK usage for only the authorized project", async () => {
mockedQueryClickhouse
.mockResolvedValueOnce([
@@ -1271,6 +1406,7 @@ describe("v4TransitionRouter", () => {
expect(experimentUsageQuery?.query).toContain(
"event_date >= toDate({fromTimestamp: DateTime64(3)})",
);
+ expect(experimentUsageQuery?.query).not.toContain("hostName()");
expect(experimentUsageQuery?.params).toMatchObject({
projectIds: [projectId, secondProjectId],
fromTimestamp: "2026-06-24 00:00:00.000",
@@ -1281,6 +1417,45 @@ describe("v4TransitionRouter", () => {
});
});
+ it("finds dataset-run-items POST usage when only the main service has it", async () => {
+ sharedEnvMock.CLICKHOUSE_EVENTS_READ_ONLY_URL =
+ "https://clickhouse-events-read.example.com";
+ mockedQueryClickhouse
+ .mockResolvedValueOnce([
+ mockSdkUsageRow({
+ projectId,
+ source: "otel",
+ sdkName: "unknown",
+ sdkVersion: "unknown",
+ }),
+ ])
+ .mockRejectedValueOnce(new Error("events read replica unavailable"))
+ .mockResolvedValueOnce([{ projectId, count: "1" }]);
+ const caller = createCaller({
+ project: {
+ findMany: vi.fn().mockResolvedValue([{ id: projectId }]),
+ },
+ });
+
+ const [summary] = await caller.sdkUsageSummaryByProject({
+ orgId,
+ fromTimestamp: new Date("2026-06-24T00:00:00Z"),
+ toTimestamp: new Date("2026-06-25T00:00:00Z"),
+ });
+
+ expect(summary?.experimentInstrumentationMigration).toEqual({
+ status: "required",
+ upgradePath: "api",
+ });
+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(3);
+ expect(mockedQueryClickhouse.mock.calls[1]?.[0]).toMatchObject({
+ preferredClickhouseService: "EventsReadOnly",
+ });
+ expect(mockedQueryClickhouse.mock.calls[2]?.[0]).toMatchObject({
+ preferredClickhouseService: "ReadWrite",
+ });
+ });
+
it("summarizes SDK usage for a single project with exactly that projectId", async () => {
mockedQueryClickhouse
.mockResolvedValueOnce([
@@ -1562,20 +1737,35 @@ describe("v4TransitionRouter", () => {
});
it("summarizes legacy public API usage by organization project", async () => {
- mockedQueryClickhouse.mockResolvedValue([
- {
- projectId,
- entrypoint: "publicapi: GET /api/public/traces/{id}",
- count: "0.6666666666666666",
- lastSeen: "2026-06-24T12:34:56.789123Z",
- },
- {
- projectId: secondProjectId,
- entrypoint: "publicapi: GET /api/public/metrics",
- count: 3,
- lastSeen: "2026-06-24T15:00:00.000000Z",
- },
- ]);
+ mockedQueryClickhouse
+ .mockResolvedValueOnce([
+ {
+ projectId,
+ entrypoint: "publicapi: GET /api/public/traces/{id}",
+ count: "0.6666666666666666",
+ lastSeen: "2026-06-24T12:34:56.789123Z",
+ },
+ {
+ projectId: secondProjectId,
+ entrypoint: "publicapi: GET /api/public/metrics",
+ count: 3,
+ lastSeen: "2026-06-24T15:00:00.000000Z",
+ },
+ ])
+ .mockResolvedValueOnce([
+ {
+ projectId,
+ entrypoint: "publicapi: GET /api/public/traces/{id}",
+ count: "0.3333333333333334",
+ lastSeen: "2026-06-24T14:00:00.000000Z",
+ },
+ {
+ projectId: secondProjectId,
+ entrypoint: "publicapi: GET /api/public/metrics",
+ count: 2,
+ lastSeen: "2026-06-24T16:00:00.000000Z",
+ },
+ ]);
const mockPrisma = {
project: {
findMany: vi.fn().mockResolvedValue([
@@ -1596,21 +1786,21 @@ describe("v4TransitionRouter", () => {
{
projectId,
entrypoint: "publicapi: GET /api/public/traces/{id}",
- count: 0.6666666666666666,
- lastSeen: "2026-06-24T12:34:56.789123Z",
+ count: 1,
+ lastSeen: "2026-06-24T14:00:00.000000Z",
},
{
projectId: secondProjectId,
entrypoint: "publicapi: GET /api/public/metrics",
- count: 3,
- lastSeen: "2026-06-24T15:00:00.000000Z",
+ count: 5,
+ lastSeen: "2026-06-24T16:00:00.000000Z",
},
]);
expect(mockPrisma.project.findMany).toHaveBeenCalledWith(
accessibleProjectsFindManyArgs,
);
- expect(mockedQueryClickhouse).toHaveBeenCalledTimes(1);
+ expect(mockedQueryClickhouse).toHaveBeenCalledTimes(2);
const clickhouseQuery = mockedQueryClickhouse.mock.calls[0]?.[0];
expect(clickhouseQuery?.query).not.toContain("toStartOfInterval");
expect(clickhouseQuery?.query).not.toContain("bucket_time");
@@ -1633,5 +1823,11 @@ describe("v4TransitionRouter", () => {
expect(clickhouseQuery?.tags).toEqual({
route: "v4-legacy-api-usage-summary",
});
+ expect(mockedQueryClickhouse.mock.calls[0]?.[0]).toMatchObject({
+ preferredClickhouseService: "ReadOnly",
+ });
+ expect(mockedQueryClickhouse.mock.calls[1]?.[0]).toMatchObject({
+ preferredClickhouseService: "ReadWrite",
+ });
});
});
diff --git a/web/src/features/v4/server/v4TransitionRouter.ts b/web/src/features/v4/server/v4TransitionRouter.ts
index 25218268fa..5616068032 100644
--- a/web/src/features/v4/server/v4TransitionRouter.ts
+++ b/web/src/features/v4/server/v4TransitionRouter.ts
@@ -10,6 +10,7 @@ import {
type PrismaClient,
} from "@langfuse/shared/src/db";
import type { Session } from "next-auth";
+import { env } from "@langfuse/shared/src/env";
import {
convertDateToClickhouseDateTime,
INTERNAL_INGESTION_SDK_NAMES,
@@ -18,6 +19,7 @@ import {
queryClickhouse,
systemTableRef,
type IngestionSdkAttributionStatus,
+ type PreferredClickhouseService,
} from "@langfuse/shared/src/server";
import { getSdkVersionCapabilityStatus } from "@/src/features/sdk-version/lib/sdkVersionCapabilities";
const MAX_DETECTION_RANGE_MS = 30 * 24 * 60 * 60 * 1000;
@@ -174,6 +176,59 @@ type DatasetRunItemsPostUsageByProjectRow = {
count: string | number;
};
+type SystemQueryLogReadService = "ReadOnly" | "EventsReadOnly";
+
+const getSystemQueryLogServices = ({
+ readService,
+ readUrl,
+}: {
+ readService: SystemQueryLogReadService;
+ readUrl: string | undefined;
+}): PreferredClickhouseService[] =>
+ readUrl &&
+ new URL(readUrl).toString() !== new URL(env.CLICKHOUSE_URL).toString()
+ ? [readService, "ReadWrite"]
+ : ["ReadWrite"];
+
+const querySystemQueryLogAcrossServices = async <T>({
+ preferredClickhouseServices,
+ queryService,
+ failureMessage,
+}: {
+ preferredClickhouseServices: PreferredClickhouseService[];
+ queryService: (service: PreferredClickhouseService) => Promise<T[]>;
+ failureMessage: string;
+}): Promise<T[][]> => {
+ const settledServiceRows = await Promise.allSettled(
+ preferredClickhouseServices.map(queryService),
+ );
+ const firstSuccessfulResult = settledServiceRows.find(
+ (result) => result.status === "fulfilled",
+ );
+ if (!firstSuccessfulResult) {
+ const firstFailure = settledServiceRows.find(
+ (result) => result.status === "rejected",
+ );
+ throw firstFailure?.reason ?? new Error("ClickHouse services unavailable");
+ }
+
+ settledServiceRows.forEach((result, index) => {
+ if (result.status === "rejected") {
+ logger.warn(failureMessage, {
+ preferredClickhouseService: preferredClickhouseServices[index],
+ error:
+ result.reason instanceof Error
+ ? result.reason.message
+ : "Unknown error",
+ });
+ }
+ });
+
+ return settledServiceRows.flatMap((result) =>
+ result.status === "fulfilled" ? [result.value] : [],
+ );
+};
+
const toBoolean = (value: boolean | string | number): boolean =>
value === true || value === 1 || value === "1";
@@ -354,6 +409,12 @@ const getSdkUsageSummaries = async ({
}): Promise<SdkUsageSummaryByProjectResultRow[]> => {
if (projectIds.length === 0) return [];
+ const systemQueryLogServices = getSystemQueryLogServices({
+ readService: "EventsReadOnly",
+ readUrl:
+ env.CLICKHOUSE_EVENTS_READ_ONLY_URL ?? env.CLICKHOUSE_READ_ONLY_URL,
+ });
+
const [rows, datasetRunItemsPostUsageResult] = await Promise.all([
queryClickhouse<SdkUsageSummaryByProjectRow>({
query: `
@@ -482,8 +543,11 @@ ORDER BY
tags: { route: "v4-sdk-usage-summary" },
preferredClickhouseService: "EventsReadOnly",
}),
- queryClickhouse<DatasetRunItemsPostUsageByProjectRow>({
- query: `
+ querySystemQueryLogAcrossServices({
+ preferredClickhouseServices: systemQueryLogServices,
+ queryService: (preferredClickhouseService) =>
+ queryClickhouse<DatasetRunItemsPostUsageByProjectRow>({
+ query: `
SELECT
JSONExtractString(log_comment, 'projectId') AS projectId,
count() AS count
@@ -501,16 +565,22 @@ WHERE
GROUP BY projectId
SETTINGS skip_unavailable_shards = 1
`,
- params: {
- projectIds,
- fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp),
- toTimestamp: convertDateToClickhouseDateTime(toTimestamp),
- },
- tags: { route: "v4-experiment-instrumentation-summary" },
- preferredClickhouseService: "EventsReadOnly",
- clickhouseSettings: { skip_unavailable_shards: 1 },
+ params: {
+ projectIds,
+ fromTimestamp: convertDateToClickhouseDateTime(fromTimestamp),
+ toTimestamp: convertDateToClickhouseDateTime(toTimestamp),
+ },
+ tags: { route: "v4-experiment-instrumentation-summary" },
+ preferredClickhouseService,
+ clickhouseSettings: { skip_unavailable_shards: 1 },
+ }),
+ failureMessage:
+ "Failed to query dataset-run-items POST usage from ClickHouse service",
})
- .then((rows) => ({ status: "success" as const, rows }))
+ .then((serviceRows) => ({
+ status: "success" as const,
+ rows: serviceRows.flat(),
+ }))
.catch((error: unknown) => {
logger.warn(
"Failed to query dataset-run-items POST usage for v4 migration",
@@ -604,14 +674,16 @@ SETTINGS skip_unavailable_shards = 1
});
};
-const getLegacyApiUsageSummaries = async ({
+const getLegacyApiUsageSummariesForService = async ({
projectIds,
fromTimestamp,
toTimestamp,
+ preferredClickhouseService,
}: {
projectIds: string[];
fromTimestamp: Date;
toTimestamp: Date;
+ preferredClickhouseService: PreferredClickhouseService;
}): Promise<LegacyApiUsageSummaryByProjectResultRow[]> => {
if (projectIds.length === 0) return [];
@@ -655,6 +727,7 @@ classified AS (
match(route_path, '^GET /api/public/observations/[^/?#]+$'), 'GET /api/public/observations/{id}',
match(route_path, '^GET /api/public/scores/[^/?#]+$'), 'GET /api/public/scores/{id}',
match(route_path, '^GET /api/public/v2/scores/[^/?#]+$'), 'GET /api/public/v2/scores/{id}',
+ match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 'GET /api/public/datasets/{datasetName}/runs',
match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 'GET /api/public/datasets/{datasetName}/runs/{runName}',
NULL
) AS legacy_route,
@@ -678,6 +751,7 @@ classified AS (
match(route_path, '^GET /api/public/observations/[^/?#]+$'), 1,
match(route_path, '^GET /api/public/scores/[^/?#]+$'), 1,
match(route_path, '^GET /api/public/v2/scores/[^/?#]+$'), 1,
+ match(route_path, '^GET /api/public/datasets/[^/?#]+/runs$'), 1,
match(route_path, '^GET /api/public/datasets/[^/?#]+/runs/[^/?#]+$'), 1,
NULL
) AS clickhouse_queries_per_api_call
@@ -702,7 +776,7 @@ SETTINGS skip_unavailable_shards = 1
toTimestamp: convertDateToClickhouseDateTime(toTimestamp),
},
tags: { route: "v4-legacy-api-usage-summary" },
- preferredClickhouseService: "ReadOnly",
+ preferredClickhouseService,
clickhouseSettings: { skip_unavailable_shards: 1 },
});
@@ -714,6 +788,79 @@ SETTINGS skip_unavailable_shards = 1
}));
};
+const getLegacyApiUsageSummaries = async ({
+ projectIds,
+ fromTimestamp,
+ toTimestamp,
+}: {
+ projectIds: string[];
+ fromTimestamp: Date;
+ toTimestamp: Date;
+}): Promise<LegacyApiUsageSummaryByProjectResultRow[]> => {
+ if (projectIds.length === 0) return [];
+
+ const preferredClickhouseServices = getSystemQueryLogServices({
+ readService: "ReadOnly",
+ readUrl: env.CLICKHOUSE_READ_ONLY_URL,
+ });
+ const serviceResultSets = await querySystemQueryLogAcrossServices({
+ preferredClickhouseServices,
+ queryService: (preferredClickhouseService) =>
+ getLegacyApiUsageSummariesForService({
+ projectIds,
+ fromTimestamp,
+ toTimestamp,
+ preferredClickhouseService,
+ }),
+ failureMessage: "Failed to query legacy API usage from ClickHouse service",
+ });
+
+ const uniqueServiceResultSets = new Map<
+ string,
+ LegacyApiUsageSummaryByProjectResultRow[]
+ >();
+ for (const rows of serviceResultSets) {
+ const sortedRows = [...rows].sort(
+ (left, right) =>
+ left.projectId.localeCompare(right.projectId) ||
+ left.entrypoint.localeCompare(right.entrypoint),
+ );
+ uniqueServiceResultSets.set(JSON.stringify(sortedRows), sortedRows);
+ }
+ const deduplicatedServiceRows = Array.from(
+ uniqueServiceResultSets.values(),
+ ).flat();
+
+ const rowsByProjectAndEntrypoint = new Map<
+ string,
+ LegacyApiUsageSummaryByProjectResultRow
+ >();
+
+ for (const row of deduplicatedServiceRows) {
+ const key = `${row.projectId}\u0000${row.entrypoint}`;
+ const existing = rowsByProjectAndEntrypoint.get(key);
+ rowsByProjectAndEntrypoint.set(
+ key,
+ existing
+ ? {
+ ...existing,
+ count: existing.count + row.count,
+ lastSeen:
+ existing.lastSeen > row.lastSeen
+ ? existing.lastSeen
+ : row.lastSeen,
+ }
+ : row,
+ );
+ }
+
+ return Array.from(rowsByProjectAndEntrypoint.values()).sort(
+ (left, right) =>
+ left.projectId.localeCompare(right.projectId) ||
+ left.entrypoint.localeCompare(right.entrypoint),
+ );
+};
+
export const v4TransitionRouter = createTRPCRouter({
forceV3Experience: protectedProjectProcedure
.input(z.object({ projectId: z.string() }))