Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(batch-export): route batch export ClickHouse reads to a read replica (#16076)

Niklas Semmler55f9fc78fe70
diff --git a/packages/shared/src/server/repositories/dataset-run-items.ts b/packages/shared/src/server/repositories/dataset-run-items.ts
index 25f18607e7..cfe38ee1f4 100644
--- a/packages/shared/src/server/repositories/dataset-run-items.ts
+++ b/packages/shared/src/server/repositories/dataset-run-items.ts
@@ -23,7 +23,10 @@ import { env } from "../../env";
import { commandClickhouse } from "./clickhouse";
import Decimal from "decimal.js";
import { ClickHouseClientConfigOptions } from "@clickhouse/client";
-import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import {
+ convertDateToClickhouseDateTime,
+ type PreferredClickhouseService,
+} from "../clickhouse/client";
import { ScoreAggregate } from "../../features/scores";
type DatasetItemIdsByTraceIdQuery = {
@@ -541,6 +544,7 @@ type GetDatasetRunItemsTableOpts<IncludeIO extends boolean> =
DatasetRunItemsTableQuery & {
select: "count" | "rows";
includeIO?: IncludeIO;
+ preferredClickhouseService?: PreferredClickhouseService;
};
// Phase 1: Find dataset item IDs or count that satisfy conditions across ALL runs
@@ -902,13 +906,16 @@ const getDatasetRunItemsTableInternal = async <
},
tags: { projectId },
clickhouseConfigs: opts.clickhouseConfigs,
+ preferredClickhouseService: opts.preferredClickhouseService,
});
return res;
};
export const getDatasetRunItemsCh = async (
- opts: DatasetRunItemsTableQuery,
+ opts: DatasetRunItemsTableQuery & {
+ preferredClickhouseService?: PreferredClickhouseService;
+ },
): Promise<DatasetRunItemDomain[]> => {
const rows = await getDatasetRunItemsTableInternal<DatasetRunItemRecord>({
...opts,
diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.ts
index 8e0d6c7031..216ef25039 100644
--- a/packages/shared/src/server/repositories/scores.ts
+++ b/packages/shared/src/server/repositories/scores.ts
@@ -1225,11 +1225,13 @@ export async function getScoresUiTable<
clickhouseConfigs?: ClickHouseClientConfigOptions;
excludeMetadata?: ExcludeMetadata;
includeHasMetadataFlag?: IncludeHasMetadata;
+ preferredClickhouseService?: PreferredClickhouseService;
}) {
const {
excludeMetadata = false,
includeHasMetadataFlag = false,
clickhouseConfigs,
+ preferredClickhouseService,
...rest
} = props;
@@ -1273,6 +1275,7 @@ export async function getScoresUiTable<
excludeMetadata,
includeHasMetadataFlag,
clickhouseConfigs,
+ preferredClickhouseService,
...rest,
});
@@ -1310,6 +1313,7 @@ const getScoresUiGeneric = async <T>(props: {
clickhouseConfigs?: ClickHouseClientConfigOptions;
excludeMetadata?: boolean;
includeHasMetadataFlag?: boolean;
+ preferredClickhouseService?: PreferredClickhouseService;
}): Promise<T[]> => {
const {
projectId,
@@ -1318,6 +1322,7 @@ const getScoresUiGeneric = async <T>(props: {
limit,
offset,
clickhouseConfigs,
+ preferredClickhouseService,
excludeMetadata = false,
includeHasMetadataFlag = false,
} = props;
@@ -1406,6 +1411,7 @@ const getScoresUiGeneric = async <T>(props: {
params: input.params,
tags: input.tags,
clickhouseConfigs,
+ preferredClickhouseService,
});
};
@@ -2245,9 +2251,15 @@ export const getDistinctScoreNames = async (
projectId: string;
cutoffCreatedAt: Date;
clickhouseConfigs?: ClickHouseClientConfigOptions | undefined;
+ preferredClickhouseService?: PreferredClickhouseService;
} & DistinctScoreNamesTimestampSource,
) => {
- const { projectId, cutoffCreatedAt, clickhouseConfigs } = p;
+ const {
+ projectId,
+ cutoffCreatedAt,
+ clickhouseConfigs,
+ preferredClickhouseService,
+ } = p;
const startTimeFrom = getDistinctScoreNamesStartTimeFrom(p);
const query = ` SELECT DISTINCT
@@ -2273,6 +2285,7 @@ export const getDistinctScoreNames = async (
},
tags: { projectId },
clickhouseConfigs,
+ preferredClickhouseService,
});
return rows.map((row) => row.name);
diff --git a/packages/shared/src/server/services/sessions-ui-table-service.ts b/packages/shared/src/server/services/sessions-ui-table-service.ts
index 77e050bec2..90a5cf8757 100644
--- a/packages/shared/src/server/services/sessions-ui-table-service.ts
+++ b/packages/shared/src/server/services/sessions-ui-table-service.ts
@@ -4,7 +4,10 @@ import { sessionCols } from "../tableMappings/mapSessionTable";
import { FilterState } from "../../types";
import { sessionsViewCols } from "../../tableDefinitions/sessionsView";
import { findUiColumnMapping } from "../../tableDefinitions";
-import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import {
+ convertDateToClickhouseDateTime,
+ type PreferredClickhouseService,
+} from "../clickhouse/client";
import { scoreBooleansAggregation } from "../queries/clickhouse-sql/query-fragments";
import { DateTimeFilter, FilterList, orderByToClickhouseSql } from "../queries";
import {
@@ -91,6 +94,7 @@ export const getSessionsWithMetrics = async (props: {
limit?: number;
page?: number;
clickhouseConfigs?: ClickHouseClientConfigOptions | undefined;
+ preferredClickhouseService?: PreferredClickhouseService;
}) => {
const rows = await getSessionsTableGeneric<SessionWithMetricsReturnType>({
select: "metrics",
@@ -100,6 +104,7 @@ export const getSessionsWithMetrics = async (props: {
limit: props.limit,
page: props.page,
clickhouseConfigs: props.clickhouseConfigs,
+ preferredClickhouseService: props.preferredClickhouseService,
});
return rows.map((row) => ({
@@ -119,11 +124,20 @@ export type FetchSessionsTableProps = {
page?: number;
tags?: Record<string, string>;
clickhouseConfigs?: ClickHouseClientConfigOptions | undefined;
+ preferredClickhouseService?: PreferredClickhouseService;
};
const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
- const { select, projectId, filter, orderBy, limit, page, clickhouseConfigs } =
- props;
+ const {
+ select,
+ projectId,
+ filter,
+ orderBy,
+ limit,
+ page,
+ clickhouseConfigs,
+ preferredClickhouseService,
+ } = props;
let sqlSelect: string;
switch (select) {
@@ -402,5 +416,6 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
params: input.params,
tags: input.tags,
clickhouseConfigs,
+ preferredClickhouseService,
});
};
diff --git a/worker/src/__tests__/batchExportReadReplica.test.ts b/worker/src/__tests__/batchExportReadReplica.test.ts
new file mode 100644
index 0000000000..c9cb40adcd
--- /dev/null
+++ b/worker/src/__tests__/batchExportReadReplica.test.ts
@@ -0,0 +1,226 @@
+/**
+ * ClickHouse service routing for the batch export read path, observed by
+ * pointing each service URL at a recording proxy before the shared env module
+ * is parsed. Two things this rules out: mutating the shared `env` object at
+ * runtime is a no-op (the test and the built shared code hold different module
+ * instances), and mocking `queryClickhouse` on the shared barrel cannot see the
+ * queries `shared` issues internally.
+ */
+import http from "http";
+import { randomUUID } from "crypto";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+
+const harness = vi.hoisted(() => {
+ const upstream = new URL(process.env.CLICKHOUSE_URL as string);
+ const base = 21000 + Math.floor(Math.random() * 1000) * 4;
+ const ports = { primary: base, readOnly: base + 1, eventsReadOnly: base + 2 };
+ process.env.CLICKHOUSE_URL = `http://127.0.0.1:${ports.primary}`;
+ process.env.CLICKHOUSE_READ_ONLY_URL = `http://127.0.0.1:${ports.readOnly}`;
+ process.env.CLICKHOUSE_EVENTS_READ_ONLY_URL = `http://127.0.0.1:${ports.eventsReadOnly}`;
+
+ return {
+ upstreamHost: upstream.hostname,
+ upstreamPort: Number(upstream.port || 8123),
+ ports,
+ requests: {
+ primary: [] as string[],
+ readOnly: [] as string[],
+ eventsReadOnly: [] as string[],
+ },
+ };
+});
+
+const {
+ createObservation,
+ createObservationsCh,
+ createOrgProjectAndApiKey,
+ createScoresCh,
+ createTrace,
+ createTraceScore,
+ createTracesCh,
+ queryClickhouse,
+} = await import("@langfuse/shared/src/server");
+const { BatchExportTableName } = await import("@langfuse/shared");
+const { getDatabaseReadStreamPaginated } =
+ await import("../features/database-read-stream/getDatabaseReadStream");
+const { getObservationStream } =
+ await import("../features/database-read-stream/observation-stream");
+const { getTraceStream } =
+ await import("../features/database-read-stream/trace-stream");
+
+const startRecordingProxy = (port: number, sink: string[]) =>
+ new Promise<http.Server>((resolve, reject) => {
+ const server = http.createServer((req, res) => {
+ const chunks: Buffer[] = [];
+ req.on("data", (c) => chunks.push(c));
+ req.on("end", () => {
+ const body = Buffer.concat(chunks).toString("utf8");
+ sink.push(body.replace(/\s+/g, " ").slice(0, 120));
+ const headers = { ...req.headers };
+ delete headers.host;
+ delete headers["content-length"];
+ const upstreamReq = http.request(
+ {
+ host: harness.upstreamHost,
+ port: harness.upstreamPort,
+ method: req.method,
+ path: req.url,
+ headers,
+ },
+ (upstreamRes) => {
+ res.writeHead(upstreamRes.statusCode ?? 500, upstreamRes.headers);
+ upstreamRes.pipe(res);
+ },
+ );
+ upstreamReq.on("error", (error) => {
+ res.writeHead(502);
+ res.end(String(error));
+ });
+ upstreamReq.end(body);
+ });
+ });
+ server.timeout = 0;
+ server.requestTimeout = 0;
+ server.once("error", reject);
+ server.listen(port, "127.0.0.1", () => resolve(server));
+ });
+
+let servers: http.Server[] = [];
+
+const resetRecordings = () =>
+ Object.values(harness.requests).forEach((sink) => (sink.length = 0));
+
+const servicesUsed = () =>
+ Object.entries(harness.requests)
+ .filter(([, sink]) => sink.length > 0)
+ .map(([service]) => service);
+
+const replicaQueries = () => [
+ ...harness.requests.readOnly,
+ ...harness.requests.eventsReadOnly,
+];
+
+const drain = async (stream: AsyncIterable<unknown>) => {
+ const rows: any[] = [];
+ for await (const row of stream) rows.push(row);
+ return rows;
+};
+
+const cutoffCreatedAt = new Date(Date.now() + 1000 * 60 * 60 * 24);
+const scoreName = "quality";
+
+let projectId: string;
+
+beforeAll(async () => {
+ servers = await Promise.all(
+ Object.entries(harness.ports).map(([service, port]) =>
+ startRecordingProxy(
+ port,
+ harness.requests[service as keyof typeof harness.requests],
+ ),
+ ),
+ );
+
+ ({ projectId } = await createOrgProjectAndApiKey());
+
+ const traceId = randomUUID();
+ await createTracesCh([createTrace({ project_id: projectId, id: traceId })]);
+ await createObservationsCh([
+ createObservation({ project_id: projectId, trace_id: traceId }),
+ ]);
+ await createScoresCh([
+ createTraceScore({
+ project_id: projectId,
+ trace_id: traceId,
+ name: scoreName,
+ value: 1,
+ observation_id: null,
+ }),
+ ]);
+}, 120_000);
+
+afterAll(async () => {
+ await Promise.all(
+ servers.map((s) => new Promise<void>((r) => s.close(() => r()))),
+ );
+});
+
+describe("batch export ClickHouse routing", () => {
+ it("records each service on its own sink", async () => {
+ resetRecordings();
+
+ for (const preferredClickhouseService of [
+ undefined,
+ "ReadOnly",
+ "EventsReadOnly",
+ ] as const) {
+ await queryClickhouse({
+ query: "SELECT 1 AS x",
+ params: {},
+ tags: {},
+ preferredClickhouseService,
+ });
+ }
+
+ expect(servicesUsed()).toEqual(["primary", "readOnly", "eventsReadOnly"]);
+ });
+
+ // A score whose name is missing from the name list is dropped from the row, so
+ // a name read that lags behind the value read silently removes a whole column
+ // from the export. Both reads must therefore resolve to the same service,
+ // whichever one that is.
+ it.each([
+ [
+ "traces",
+ () => getTraceStream({ projectId, cutoffCreatedAt, filter: [] }),
+ ],
+ [
+ "observations",
+ () =>
+ getObservationStream({
+ projectId,
+ cutoffCreatedAt,
+ filter: [],
+ preferredClickhouseService: "ReadOnly",
+ }),
+ ],
+ ])(
+ "%s export reads score names and values from one service",
+ async (_, openStream) => {
+ resetRecordings();
+
+ const rows = await drain(await openStream());
+
+ expect(Object.keys(rows[0])).toContain(scoreName);
+ expect(servicesUsed()).toHaveLength(1);
+ },
+ );
+
+ // Batch actions call these two with no service to enumerate a selection the
+ // user made against the primary. A replica default would make "add all to
+ // dataset" silently act on fewer rows than the user selected.
+ it.each([
+ [
+ "getObservationStream",
+ () => getObservationStream({ projectId, cutoffCreatedAt, filter: [] }),
+ ],
+ [
+ "getDatabaseReadStreamPaginated",
+ () =>
+ getDatabaseReadStreamPaginated({
+ projectId,
+ tableName: BatchExportTableName.Scores,
+ cutoffCreatedAt,
+ filter: [],
+ orderBy: { column: "timestamp", order: "DESC" },
+ }),
+ ],
+ ])("%s reads the primary when passed no service", async (_, openStream) => {
+ resetRecordings();
+
+ await drain(await openStream());
+
+ expect(replicaQueries()).toEqual([]);
+ expect(harness.requests.primary.length).toBeGreaterThan(0);
+ });
+});
diff --git a/worker/src/features/batchExport/handleBatchExportJob.ts b/worker/src/features/batchExport/handleBatchExportJob.ts
index 6116c25726..8de7a9b4f8 100644
--- a/worker/src/features/batchExport/handleBatchExportJob.ts
+++ b/worker/src/features/batchExport/handleBatchExportJob.ts
@@ -17,6 +17,7 @@ import {
getCurrentSpan,
applyCommentFilters,
type CommentObjectType,
+ type PreferredClickhouseService,
} from "@langfuse/shared/src/server";
import { env } from "../../env";
import { getDatabaseReadStreamPaginated } from "../database-read-stream/getDatabaseReadStream";
@@ -32,6 +33,8 @@ const tableToCommentType: Record<string, CommentObjectType | undefined> = {
sessions: "SESSION",
};
+const BATCH_EXPORT_CLICKHOUSE_SERVICE: PreferredClickhouseService = "ReadOnly";
+
export const handleBatchExportJob = async (
batchExportJob: BatchExportJobType,
) => {
@@ -180,6 +183,7 @@ export const handleBatchExportJob = async (
...parsedQuery.data,
filter: processedFilter,
fileFormat: jobDetails.format as BatchExportFileFormat,
+ preferredClickhouseService: BATCH_EXPORT_CLICKHOUSE_SERVICE,
})
: parsedQuery.data.tableName === BatchExportTableName.Traces
? await getTraceStream({
@@ -200,6 +204,7 @@ export const handleBatchExportJob = async (
cutoffCreatedAt: jobDetails.createdAt,
...parsedQuery.data,
filter: processedFilter,
+ preferredClickhouseService: BATCH_EXPORT_CLICKHOUSE_SERVICE,
});
// Transform data to desired format
diff --git a/worker/src/features/database-read-stream/event-stream.ts b/worker/src/features/database-read-stream/event-stream.ts
index 3e31624167..1782a1f4e7 100644
--- a/worker/src/features/database-read-stream/event-stream.ts
+++ b/worker/src/features/database-read-stream/event-stream.ts
@@ -76,11 +76,13 @@ export const getEventsStream = async (props: {
const { query, params: queryParams } = queryBuilder.buildWithParams();
// Get distinct score names for empty columns
+ // Same service as the values read below: a missing name is silently dropped.
const distinctScoreNames = await getDistinctScoreNames({
projectId,
cutoffCreatedAt,
startTimeFrom,
clickhouseConfigs,
+ preferredClickhouseService: "EventsReadOnly",
});
const emptyScoreColumns = distinctScoreNames.reduce(
diff --git a/worker/src/features/database-read-stream/getDatabaseReadStream.ts b/worker/src/features/database-read-stream/getDatabaseReadStream.ts
index a60b747247..9d4dde4992 100644
--- a/worker/src/features/database-read-stream/getDatabaseReadStream.ts
+++ b/worker/src/features/database-read-stream/getDatabaseReadStream.ts
@@ -28,6 +28,7 @@ import {
getTracesByIds,
getScoresForTraces,
getDatasetItems,
+ type PreferredClickhouseService,
} from "@langfuse/shared/src/server";
import Decimal from "decimal.js";
import { env } from "../../env";
@@ -153,6 +154,7 @@ export const getDatabaseReadStreamPaginated = async ({
searchQuery,
searchType,
useEventsTable,
+ preferredClickhouseService,
rowLimit = env.BATCH_EXPORT_ROW_LIMIT,
}: {
projectId: string;
@@ -160,6 +162,7 @@ export const getDatabaseReadStreamPaginated = async ({
searchQuery?: string;
searchType?: TracingSearchType[];
rowLimit?: number;
+ preferredClickhouseService?: PreferredClickhouseService;
} & BatchExportQueryType): Promise<DatabaseReadStream<unknown>> => {
// Set createdAt cutoff to prevent exporting data that was created after the job was queued
const createdAtCutoffFilter: FilterCondition = {
@@ -214,6 +217,7 @@ export const getDatabaseReadStreamPaginated = async ({
limit: pageSize,
offset,
clickhouseConfigs,
+ preferredClickhouseService,
});
// Get author user info for scores
@@ -293,6 +297,7 @@ export const getDatabaseReadStreamPaginated = async ({
limit: pageSize,
page: Math.floor(offset / pageSize),
clickhouseConfigs,
+ preferredClickhouseService,
});
const prismaSessionInfo = await prisma.traceSession.findMany({
@@ -561,6 +566,7 @@ export const getDatabaseReadStreamPaginated = async ({
},
offset,
clickhouseConfigs,
+ preferredClickhouseService,
});
// fetch all project dataset names
diff --git a/worker/src/features/database-read-stream/observation-stream.ts b/worker/src/features/database-read-stream/observation-stream.ts
index fea0e74492..f25f156c47 100644
--- a/worker/src/features/database-read-stream/observation-stream.ts
+++ b/worker/src/features/database-read-stream/observation-stream.ts
@@ -29,6 +29,7 @@ import {
eventsTracesScoresAggregation,
toLevelAgnosticScoreFilter,
scoreBooleansAggregation,
+ type PreferredClickhouseService,
} from "@langfuse/shared/src/server";
import { Readable } from "stream";
import { env } from "../../env";
@@ -53,6 +54,7 @@ type ObservationStreamProps = {
// BatchExportQuerySchema). When true, read from the ClickHouse events table
// instead of the legacy observations/traces tables.
useEventsTable?: boolean;
+ preferredClickhouseService?: PreferredClickhouseService;
};
export const getObservationStream = async (
@@ -106,6 +108,7 @@ export const getObservationStream = async (
return filter.column === "Start Time" && filter.type === "datetime";
},
clickhouseConfigs,
+ preferredClickhouseService: props.preferredClickhouseService,
});
const scoresFilter = new FilterList([
@@ -280,6 +283,7 @@ export const getObservationStream = async (
},
clickhouseConfigs,
tags: { projectId },
+ preferredClickhouseService: props.preferredClickhouseService,
});
// Helper function to process a single observation row
@@ -482,6 +486,8 @@ const getObservationStreamFromEvents = async (
): filterItem is TimeFilter =>
filterItem.column === "Start Time" && filterItem.type === "datetime",
clickhouseConfigs,
+ // Same service as the values read below: a missing name is silently dropped.
+ preferredClickhouseService: "EventsReadOnly",
});
const emptyScoreColumns = distinctScoreNames.reduce(
diff --git a/worker/src/features/database-read-stream/trace-stream.ts b/worker/src/features/database-read-stream/trace-stream.ts
index 011aa3fa13..6b359a65ff 100644
--- a/worker/src/features/database-read-stream/trace-stream.ts
+++ b/worker/src/features/database-read-stream/trace-stream.ts
@@ -72,6 +72,7 @@ export const getTraceStream = async (props: {
filter: traceOnlyFilters,
isTimestampFilter: isTraceTimestampFilter,
clickhouseConfigs,
+ preferredClickhouseService: "ReadOnly",
});
const emptyScoreColumns = distinctScoreNames.reduce(
@@ -224,6 +225,7 @@ export const getTraceStream = async (props: {
},
clickhouseConfigs,
tags: { projectId },
+ preferredClickhouseService: "ReadOnly",
});
// Helper function to process a single trace row