🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(batch-export): route batch export ClickHouse reads to a read replica (#16076)
Niklas Semmler55f9fc78fe70
diff --git a/packages/shared/src/server/repositories/dataset-run-items.ts b/packages/shared/src/server/repositories/dataset-run-items.tsindex 25f18607e7..cfe38ee1f4 100644--- a/packages/shared/src/server/repositories/dataset-run-items.ts+++ b/packages/shared/src/server/repositories/dataset-run-items.ts@@ -23,7 +23,10 @@ import { env } from "../../env";import { commandClickhouse } from "./clickhouse";import Decimal from "decimal.js";import { ClickHouseClientConfigOptions } from "@clickhouse/client";-import { convertDateToClickhouseDateTime } from "../clickhouse/client";+import {+ convertDateToClickhouseDateTime,+ type PreferredClickhouseService,+} from "../clickhouse/client";import { ScoreAggregate } from "../../features/scores";type DatasetItemIdsByTraceIdQuery = {@@ -541,6 +544,7 @@ type GetDatasetRunItemsTableOpts<IncludeIO extends boolean> =DatasetRunItemsTableQuery & {select: "count" | "rows";includeIO?: IncludeIO;+ preferredClickhouseService?: PreferredClickhouseService;};// Phase 1: Find dataset item IDs or count that satisfy conditions across ALL runs@@ -902,13 +906,16 @@ const getDatasetRunItemsTableInternal = async <},tags: { projectId },clickhouseConfigs: opts.clickhouseConfigs,+ preferredClickhouseService: opts.preferredClickhouseService,});return res;};export const getDatasetRunItemsCh = async (- opts: DatasetRunItemsTableQuery,+ opts: DatasetRunItemsTableQuery & {+ preferredClickhouseService?: PreferredClickhouseService;+ },): Promise<DatasetRunItemDomain[]> => {const rows = await getDatasetRunItemsTableInternal<DatasetRunItemRecord>({...opts,diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.tsindex 8e0d6c7031..216ef25039 100644--- a/packages/shared/src/server/repositories/scores.ts+++ b/packages/shared/src/server/repositories/scores.ts@@ -1225,11 +1225,13 @@ export async function getScoresUiTable<clickhouseConfigs?: ClickHouseClientConfigOptions;excludeMetadata?: ExcludeMetadata;includeHasMetadataFlag?: IncludeHasMetadata;+ preferredClickhouseService?: PreferredClickhouseService;}) {const {excludeMetadata = false,includeHasMetadataFlag = false,clickhouseConfigs,+ preferredClickhouseService,...rest} = props;@@ -1273,6 +1275,7 @@ export async function getScoresUiTable<excludeMetadata,includeHasMetadataFlag,clickhouseConfigs,+ preferredClickhouseService,...rest,});@@ -1310,6 +1313,7 @@ const getScoresUiGeneric = async <T>(props: {clickhouseConfigs?: ClickHouseClientConfigOptions;excludeMetadata?: boolean;includeHasMetadataFlag?: boolean;+ preferredClickhouseService?: PreferredClickhouseService;}): Promise<T[]> => {const {projectId,@@ -1318,6 +1322,7 @@ const getScoresUiGeneric = async <T>(props: {limit,offset,clickhouseConfigs,+ preferredClickhouseService,excludeMetadata = false,includeHasMetadataFlag = false,} = props;@@ -1406,6 +1411,7 @@ const getScoresUiGeneric = async <T>(props: {params: input.params,tags: input.tags,clickhouseConfigs,+ preferredClickhouseService,});};@@ -2245,9 +2251,15 @@ export const getDistinctScoreNames = async (projectId: string;cutoffCreatedAt: Date;clickhouseConfigs?: ClickHouseClientConfigOptions | undefined;+ preferredClickhouseService?: PreferredClickhouseService;} & DistinctScoreNamesTimestampSource,) => {- const { projectId, cutoffCreatedAt, clickhouseConfigs } = p;+ const {+ projectId,+ cutoffCreatedAt,+ clickhouseConfigs,+ preferredClickhouseService,+ } = p;const startTimeFrom = getDistinctScoreNamesStartTimeFrom(p);const query = ` SELECT DISTINCT@@ -2273,6 +2285,7 @@ export const getDistinctScoreNames = async (},tags: { projectId },clickhouseConfigs,+ preferredClickhouseService,});return rows.map((row) => row.name);diff --git a/packages/shared/src/server/services/sessions-ui-table-service.ts b/packages/shared/src/server/services/sessions-ui-table-service.tsindex 77e050bec2..90a5cf8757 100644--- a/packages/shared/src/server/services/sessions-ui-table-service.ts+++ b/packages/shared/src/server/services/sessions-ui-table-service.ts@@ -4,7 +4,10 @@ import { sessionCols } from "../tableMappings/mapSessionTable";import { FilterState } from "../../types";import { sessionsViewCols } from "../../tableDefinitions/sessionsView";import { findUiColumnMapping } from "../../tableDefinitions";-import { convertDateToClickhouseDateTime } from "../clickhouse/client";+import {+ convertDateToClickhouseDateTime,+ type PreferredClickhouseService,+} from "../clickhouse/client";import { scoreBooleansAggregation } from "../queries/clickhouse-sql/query-fragments";import { DateTimeFilter, FilterList, orderByToClickhouseSql } from "../queries";import {@@ -91,6 +94,7 @@ export const getSessionsWithMetrics = async (props: {limit?: number;page?: number;clickhouseConfigs?: ClickHouseClientConfigOptions | undefined;+ preferredClickhouseService?: PreferredClickhouseService;}) => {const rows = await getSessionsTableGeneric<SessionWithMetricsReturnType>({select: "metrics",@@ -100,6 +104,7 @@ export const getSessionsWithMetrics = async (props: {limit: props.limit,page: props.page,clickhouseConfigs: props.clickhouseConfigs,+ preferredClickhouseService: props.preferredClickhouseService,});return rows.map((row) => ({@@ -119,11 +124,20 @@ export type FetchSessionsTableProps = {page?: number;tags?: Record<string, string>;clickhouseConfigs?: ClickHouseClientConfigOptions | undefined;+ preferredClickhouseService?: PreferredClickhouseService;};const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {- const { select, projectId, filter, orderBy, limit, page, clickhouseConfigs } =- props;+ const {+ select,+ projectId,+ filter,+ orderBy,+ limit,+ page,+ clickhouseConfigs,+ preferredClickhouseService,+ } = props;let sqlSelect: string;switch (select) {@@ -402,5 +416,6 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {params: input.params,tags: input.tags,clickhouseConfigs,+ preferredClickhouseService,});};diff --git a/worker/src/__tests__/batchExportReadReplica.test.ts b/worker/src/__tests__/batchExportReadReplica.test.tsnew file mode 100644index 0000000000..c9cb40adcd--- /dev/null+++ b/worker/src/__tests__/batchExportReadReplica.test.ts@@ -0,0 +1,226 @@+/**+ * ClickHouse service routing for the batch export read path, observed by+ * pointing each service URL at a recording proxy before the shared env module+ * is parsed. Two things this rules out: mutating the shared `env` object at+ * runtime is a no-op (the test and the built shared code hold different module+ * instances), and mocking `queryClickhouse` on the shared barrel cannot see the+ * queries `shared` issues internally.+ */+import http from "http";+import { randomUUID } from "crypto";+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";++const harness = vi.hoisted(() => {+ const upstream = new URL(process.env.CLICKHOUSE_URL as string);+ const base = 21000 + Math.floor(Math.random() * 1000) * 4;+ const ports = { primary: base, readOnly: base + 1, eventsReadOnly: base + 2 };+ process.env.CLICKHOUSE_URL = `http://127.0.0.1:${ports.primary}`;+ process.env.CLICKHOUSE_READ_ONLY_URL = `http://127.0.0.1:${ports.readOnly}`;+ process.env.CLICKHOUSE_EVENTS_READ_ONLY_URL = `http://127.0.0.1:${ports.eventsReadOnly}`;++ return {+ upstreamHost: upstream.hostname,+ upstreamPort: Number(upstream.port || 8123),+ ports,+ requests: {+ primary: [] as string[],+ readOnly: [] as string[],+ eventsReadOnly: [] as string[],+ },+ };+});++const {+ createObservation,+ createObservationsCh,+ createOrgProjectAndApiKey,+ createScoresCh,+ createTrace,+ createTraceScore,+ createTracesCh,+ queryClickhouse,+} = await import("@langfuse/shared/src/server");+const { BatchExportTableName } = await import("@langfuse/shared");+const { getDatabaseReadStreamPaginated } =+ await import("../features/database-read-stream/getDatabaseReadStream");+const { getObservationStream } =+ await import("../features/database-read-stream/observation-stream");+const { getTraceStream } =+ await import("../features/database-read-stream/trace-stream");++const startRecordingProxy = (port: number, sink: string[]) =>+ new Promise<http.Server>((resolve, reject) => {+ const server = http.createServer((req, res) => {+ const chunks: Buffer[] = [];+ req.on("data", (c) => chunks.push(c));+ req.on("end", () => {+ const body = Buffer.concat(chunks).toString("utf8");+ sink.push(body.replace(/\s+/g, " ").slice(0, 120));+ const headers = { ...req.headers };+ delete headers.host;+ delete headers["content-length"];+ const upstreamReq = http.request(+ {+ host: harness.upstreamHost,+ port: harness.upstreamPort,+ method: req.method,+ path: req.url,+ headers,+ },+ (upstreamRes) => {+ res.writeHead(upstreamRes.statusCode ?? 500, upstreamRes.headers);+ upstreamRes.pipe(res);+ },+ );+ upstreamReq.on("error", (error) => {+ res.writeHead(502);+ res.end(String(error));+ });+ upstreamReq.end(body);+ });+ });+ server.timeout = 0;+ server.requestTimeout = 0;+ server.once("error", reject);+ server.listen(port, "127.0.0.1", () => resolve(server));+ });++let servers: http.Server[] = [];++const resetRecordings = () =>+ Object.values(harness.requests).forEach((sink) => (sink.length = 0));++const servicesUsed = () =>+ Object.entries(harness.requests)+ .filter(([, sink]) => sink.length > 0)+ .map(([service]) => service);++const replicaQueries = () => [+ ...harness.requests.readOnly,+ ...harness.requests.eventsReadOnly,+];++const drain = async (stream: AsyncIterable<unknown>) => {+ const rows: any[] = [];+ for await (const row of stream) rows.push(row);+ return rows;+};++const cutoffCreatedAt = new Date(Date.now() + 1000 * 60 * 60 * 24);+const scoreName = "quality";++let projectId: string;++beforeAll(async () => {+ servers = await Promise.all(+ Object.entries(harness.ports).map(([service, port]) =>+ startRecordingProxy(+ port,+ harness.requests[service as keyof typeof harness.requests],+ ),+ ),+ );++ ({ projectId } = await createOrgProjectAndApiKey());++ const traceId = randomUUID();+ await createTracesCh([createTrace({ project_id: projectId, id: traceId })]);+ await createObservationsCh([+ createObservation({ project_id: projectId, trace_id: traceId }),+ ]);+ await createScoresCh([+ createTraceScore({+ project_id: projectId,+ trace_id: traceId,+ name: scoreName,+ value: 1,+ observation_id: null,+ }),+ ]);+}, 120_000);++afterAll(async () => {+ await Promise.all(+ servers.map((s) => new Promise<void>((r) => s.close(() => r()))),+ );+});++describe("batch export ClickHouse routing", () => {+ it("records each service on its own sink", async () => {+ resetRecordings();++ for (const preferredClickhouseService of [+ undefined,+ "ReadOnly",+ "EventsReadOnly",+ ] as const) {+ await queryClickhouse({+ query: "SELECT 1 AS x",+ params: {},+ tags: {},+ preferredClickhouseService,+ });+ }++ expect(servicesUsed()).toEqual(["primary", "readOnly", "eventsReadOnly"]);+ });++ // A score whose name is missing from the name list is dropped from the row, so+ // a name read that lags behind the value read silently removes a whole column+ // from the export. Both reads must therefore resolve to the same service,+ // whichever one that is.+ it.each([+ [+ "traces",+ () => getTraceStream({ projectId, cutoffCreatedAt, filter: [] }),+ ],+ [+ "observations",+ () =>+ getObservationStream({+ projectId,+ cutoffCreatedAt,+ filter: [],+ preferredClickhouseService: "ReadOnly",+ }),+ ],+ ])(+ "%s export reads score names and values from one service",+ async (_, openStream) => {+ resetRecordings();++ const rows = await drain(await openStream());++ expect(Object.keys(rows[0])).toContain(scoreName);+ expect(servicesUsed()).toHaveLength(1);+ },+ );++ // Batch actions call these two with no service to enumerate a selection the+ // user made against the primary. A replica default would make "add all to+ // dataset" silently act on fewer rows than the user selected.+ it.each([+ [+ "getObservationStream",+ () => getObservationStream({ projectId, cutoffCreatedAt, filter: [] }),+ ],+ [+ "getDatabaseReadStreamPaginated",+ () =>+ getDatabaseReadStreamPaginated({+ projectId,+ tableName: BatchExportTableName.Scores,+ cutoffCreatedAt,+ filter: [],+ orderBy: { column: "timestamp", order: "DESC" },+ }),+ ],+ ])("%s reads the primary when passed no service", async (_, openStream) => {+ resetRecordings();++ await drain(await openStream());++ expect(replicaQueries()).toEqual([]);+ expect(harness.requests.primary.length).toBeGreaterThan(0);+ });+});diff --git a/worker/src/features/batchExport/handleBatchExportJob.ts b/worker/src/features/batchExport/handleBatchExportJob.tsindex 6116c25726..8de7a9b4f8 100644--- a/worker/src/features/batchExport/handleBatchExportJob.ts+++ b/worker/src/features/batchExport/handleBatchExportJob.ts@@ -17,6 +17,7 @@ import {getCurrentSpan,applyCommentFilters,type CommentObjectType,+ type PreferredClickhouseService,} from "@langfuse/shared/src/server";import { env } from "../../env";import { getDatabaseReadStreamPaginated } from "../database-read-stream/getDatabaseReadStream";@@ -32,6 +33,8 @@ const tableToCommentType: Record<string, CommentObjectType | undefined> = {sessions: "SESSION",};+const BATCH_EXPORT_CLICKHOUSE_SERVICE: PreferredClickhouseService = "ReadOnly";+export const handleBatchExportJob = async (batchExportJob: BatchExportJobType,) => {@@ -180,6 +183,7 @@ export const handleBatchExportJob = async (...parsedQuery.data,filter: processedFilter,fileFormat: jobDetails.format as BatchExportFileFormat,+ preferredClickhouseService: BATCH_EXPORT_CLICKHOUSE_SERVICE,}): parsedQuery.data.tableName === BatchExportTableName.Traces? await getTraceStream({@@ -200,6 +204,7 @@ export const handleBatchExportJob = async (cutoffCreatedAt: jobDetails.createdAt,...parsedQuery.data,filter: processedFilter,+ preferredClickhouseService: BATCH_EXPORT_CLICKHOUSE_SERVICE,});// Transform data to desired formatdiff --git a/worker/src/features/database-read-stream/event-stream.ts b/worker/src/features/database-read-stream/event-stream.tsindex 3e31624167..1782a1f4e7 100644--- a/worker/src/features/database-read-stream/event-stream.ts+++ b/worker/src/features/database-read-stream/event-stream.ts@@ -76,11 +76,13 @@ export const getEventsStream = async (props: {const { query, params: queryParams } = queryBuilder.buildWithParams();// Get distinct score names for empty columns+ // Same service as the values read below: a missing name is silently dropped.const distinctScoreNames = await getDistinctScoreNames({projectId,cutoffCreatedAt,startTimeFrom,clickhouseConfigs,+ preferredClickhouseService: "EventsReadOnly",});const emptyScoreColumns = distinctScoreNames.reduce(diff --git a/worker/src/features/database-read-stream/getDatabaseReadStream.ts b/worker/src/features/database-read-stream/getDatabaseReadStream.tsindex a60b747247..9d4dde4992 100644--- a/worker/src/features/database-read-stream/getDatabaseReadStream.ts+++ b/worker/src/features/database-read-stream/getDatabaseReadStream.ts@@ -28,6 +28,7 @@ import {getTracesByIds,getScoresForTraces,getDatasetItems,+ type PreferredClickhouseService,} from "@langfuse/shared/src/server";import Decimal from "decimal.js";import { env } from "../../env";@@ -153,6 +154,7 @@ export const getDatabaseReadStreamPaginated = async ({searchQuery,searchType,useEventsTable,+ preferredClickhouseService,rowLimit = env.BATCH_EXPORT_ROW_LIMIT,}: {projectId: string;@@ -160,6 +162,7 @@ export const getDatabaseReadStreamPaginated = async ({searchQuery?: string;searchType?: TracingSearchType[];rowLimit?: number;+ preferredClickhouseService?: PreferredClickhouseService;} & BatchExportQueryType): Promise<DatabaseReadStream<unknown>> => {// Set createdAt cutoff to prevent exporting data that was created after the job was queuedconst createdAtCutoffFilter: FilterCondition = {@@ -214,6 +217,7 @@ export const getDatabaseReadStreamPaginated = async ({limit: pageSize,offset,clickhouseConfigs,+ preferredClickhouseService,});// Get author user info for scores@@ -293,6 +297,7 @@ export const getDatabaseReadStreamPaginated = async ({limit: pageSize,page: Math.floor(offset / pageSize),clickhouseConfigs,+ preferredClickhouseService,});const prismaSessionInfo = await prisma.traceSession.findMany({@@ -561,6 +566,7 @@ export const getDatabaseReadStreamPaginated = async ({},offset,clickhouseConfigs,+ preferredClickhouseService,});// fetch all project dataset namesdiff --git a/worker/src/features/database-read-stream/observation-stream.ts b/worker/src/features/database-read-stream/observation-stream.tsindex fea0e74492..f25f156c47 100644--- a/worker/src/features/database-read-stream/observation-stream.ts+++ b/worker/src/features/database-read-stream/observation-stream.ts@@ -29,6 +29,7 @@ import {eventsTracesScoresAggregation,toLevelAgnosticScoreFilter,scoreBooleansAggregation,+ type PreferredClickhouseService,} from "@langfuse/shared/src/server";import { Readable } from "stream";import { env } from "../../env";@@ -53,6 +54,7 @@ type ObservationStreamProps = {// BatchExportQuerySchema). When true, read from the ClickHouse events table// instead of the legacy observations/traces tables.useEventsTable?: boolean;+ preferredClickhouseService?: PreferredClickhouseService;};export const getObservationStream = async (@@ -106,6 +108,7 @@ export const getObservationStream = async (return filter.column === "Start Time" && filter.type === "datetime";},clickhouseConfigs,+ preferredClickhouseService: props.preferredClickhouseService,});const scoresFilter = new FilterList([@@ -280,6 +283,7 @@ export const getObservationStream = async (},clickhouseConfigs,tags: { projectId },+ preferredClickhouseService: props.preferredClickhouseService,});// Helper function to process a single observation row@@ -482,6 +486,8 @@ const getObservationStreamFromEvents = async (): filterItem is TimeFilter =>filterItem.column === "Start Time" && filterItem.type === "datetime",clickhouseConfigs,+ // Same service as the values read below: a missing name is silently dropped.+ preferredClickhouseService: "EventsReadOnly",});const emptyScoreColumns = distinctScoreNames.reduce(diff --git a/worker/src/features/database-read-stream/trace-stream.ts b/worker/src/features/database-read-stream/trace-stream.tsindex 011aa3fa13..6b359a65ff 100644--- a/worker/src/features/database-read-stream/trace-stream.ts+++ b/worker/src/features/database-read-stream/trace-stream.ts@@ -72,6 +72,7 @@ export const getTraceStream = async (props: {filter: traceOnlyFilters,isTimestampFilter: isTraceTimestampFilter,clickhouseConfigs,+ preferredClickhouseService: "ReadOnly",});const emptyScoreColumns = distinctScoreNames.reduce(@@ -224,6 +225,7 @@ export const getTraceStream = async (props: {},clickhouseConfigs,tags: { projectId },+ preferredClickhouseService: "ReadOnly",});// Helper function to process a single trace row