🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(worker): add the ClickHouse Native write strategy (#17939)
Valery Meleshkin98b09b9d7043
diff --git a/worker/AGENTS.md b/worker/AGENTS.mdindex f4d81a8998..c4b3db68ef 100644--- a/worker/AGENTS.md+++ b/worker/AGENTS.md@@ -40,6 +40,9 @@stream's partial final trace; completion covers the query window, not future arrivals.- Evaluation terminal-outcome classification: `src/features/evaluation/evalExecutionMetrics.ts`. Keep it aligned with shared code evaluator dispatcher error codes and user-visible error mapping.- Service layer: `src/services/*`+- `ClickhouseWriter.getInstance()` queues JSON rows; `getNativeInstance()` queues+ opaque `PreparedEvent` handles for `events_full`. Use `shutdownAll()` to drain+ both singletons.- Rust addon (`@langfuse/native`): telemetry init and the startup hello call livein `src/initialize.ts`, the health probe call in `src/api/index.ts`. Native coderecords its own metrics and logs; see `../packages/native/AGENTS.md`.diff --git a/worker/src/__tests__/helpers/otelReplayHarness.ts b/worker/src/__tests__/helpers/otelReplayHarness.tsindex 92ce1a40c3..d8b1f03df7 100644--- a/worker/src/__tests__/helpers/otelReplayHarness.ts+++ b/worker/src/__tests__/helpers/otelReplayHarness.ts@@ -77,7 +77,8 @@ export async function runOtelReplay(shouldWriteToEventsTable: true,});- await writer.shutdown();+ // Reset the singletons here so cleanup cannot retry rows retained by this drain.+ await ClickhouseWriter.shutdownAll();writerShutdown = true;const pendingRows = writer.queue[TableName.EventsFull].length;@@ -109,13 +110,10 @@ export async function runOtelReplay(} finally {try {if (writer && !writerShutdown) {- await writer.shutdown();- }- } finally {- if (writer) {// Each corpus replay needs a fresh singleton and interval timer.- ClickhouseWriter["instance"] = null;+ await ClickhouseWriter.shutdownAll();}+ } finally {if (tableCreated) {await client.command({ query: `DROP TABLE IF EXISTS ${tableName}` });}diff --git a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.tsindex c539dfdc71..a2e6e701f5 100644--- a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts+++ b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts@@ -1,6 +1,9 @@import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { Readable } from "node:stream";import * as serverExports from "@langfuse/shared/src/server";+import type { NativeEventBlock } from "@langfuse/native";+import type { PreparedEvent } from "@langfuse/native";import { env } from "../../env";import { logger } from "@langfuse/shared/src/server";@@ -11,6 +14,23 @@ import {truncateOversizedRecord,} from "./jsonRecords";+const { encodeClickhouseEventsMock } = vi.hoisted(() => ({+ encodeClickhouseEventsMock: vi.fn(),+}));++vi.mock("@langfuse/native", () => ({+ encodeClickhouseEvents: encodeClickhouseEventsMock,+}));++const preparedEvent = (id: string): PreparedEvent =>+ ({+ ids: {+ project_id: "project-1",+ trace_id: "trace-1",+ id,+ },+ }) as PreparedEvent;+// Mock recordHistogram, recordDistribution, recordCount, recordGaugevi.mock("@langfuse/shared/src/server", async (importOriginal) => {const original = (await importOriginal()) as {};@@ -26,6 +46,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {debug: vi.fn(),warn: vi.fn(),error: vi.fn(),+ child: vi.fn().mockReturnThis(),},};});@@ -45,13 +66,18 @@ vi.mock("../../env", async (importOriginal) => {describe("ClickhouseWriter", () => {let clickhouseClientMock: {insert: ReturnType<typeof vi.fn>;+ exec: ReturnType<typeof vi.fn>;};let writer: ClickhouseWriter;beforeEach(() => {vi.clearAllMocks();+ encodeClickhouseEventsMock.mockResolvedValue([+ { bytes: Buffer.from("native-block"), rowCount: 1 },+ ] satisfies NativeEventBlock[]);clickhouseClientMock = {insert: vi.fn(),+ exec: vi.fn(),};vi.useFakeTimers();writer = ClickhouseWriter.getInstance(clickhouseClientMock);@@ -60,10 +86,7 @@ describe("ClickhouseWriter", () => {afterEach(async () => {vi.useRealTimers();- // Reset singleton instance- await writer.shutdown();-- ClickhouseWriter.instance = null;+ await ClickhouseWriter.shutdownAll();vi.restoreAllMocks();vi.clearAllMocks();});@@ -75,6 +98,69 @@ describe("ClickhouseWriter", () => {expect(instance1).toBe(instance2);});+ it("uses a replacement client only for the selected singleton", async () => {+ const jsonReplacementClient = {+ insert: vi.fn().mockResolvedValue(),+ exec: vi.fn(),+ };+ const nativeReplacementClient = {+ insert: vi.fn(),+ exec: vi.fn().mockResolvedValue({ stream: Readable.from([]) }),+ };+ const nativeWriter =+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);+ vi.spyOn(clickhouseClientMock, "exec").mockResolvedValue({+ stream: Readable.from([]),+ });++ expect(ClickhouseWriter.getInstance(jsonReplacementClient)).toBe(writer);+ writer.addToQueue(TableName.Traces, { id: "1", name: "test" });+ nativeWriter.addToQueue(TableName.EventsFull, preparedEvent("native-1"));+ await Promise.all([+ writer["flushAll"](true),+ nativeWriter["flushAll"](true),+ ]);++ expect(jsonReplacementClient.insert).toHaveBeenCalledTimes(1);+ expect(jsonReplacementClient.exec).not.toHaveBeenCalled();+ expect(clickhouseClientMock.insert).not.toHaveBeenCalled();+ expect(clickhouseClientMock.exec).toHaveBeenCalledTimes(1);++ expect(ClickhouseWriter.getNativeInstance(nativeReplacementClient)).toBe(+ nativeWriter,+ );+ writer.addToQueue(TableName.Traces, { id: "2", name: "test" });+ nativeWriter.addToQueue(TableName.EventsFull, preparedEvent("native-2"));+ await Promise.all([+ writer["flushAll"](true),+ nativeWriter["flushAll"](true),+ ]);++ expect(jsonReplacementClient.insert).toHaveBeenCalledTimes(2);+ expect(jsonReplacementClient.exec).not.toHaveBeenCalled();+ expect(nativeReplacementClient.exec).toHaveBeenCalledTimes(1);+ expect(nativeReplacementClient.insert).not.toHaveBeenCalled();+ expect(clickhouseClientMock.insert).not.toHaveBeenCalled();+ expect(clickhouseClientMock.exec).toHaveBeenCalledTimes(1);+ expect(writer["queue"][TableName.Traces]).toHaveLength(0);+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(0);+ });++ it("keeps JSON and Native writers as separate singletons", () => {+ const nativeWriter =+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);++ expect(nativeWriter).not.toBe(writer);+ expect(ClickhouseWriter.getInstance()).toBe(writer);+ expect(ClickhouseWriter.getNativeInstance()).toBe(nativeWriter);+ expect(logger.child).toHaveBeenNthCalledWith(1, { format: "json" });+ expect(logger.child).toHaveBeenNthCalledWith(2, { format: "native" });++ expect(() =>+ nativeWriter.addToQueue(TableName.Traces, {} as never),+ ).toThrow("Native ClickHouse writer only accepts events_full");+ });+it("should initialize with correct values", () => {expect(writer.batchSize).toBe(env.LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE,@@ -121,6 +207,236 @@ describe("ClickhouseWriter", () => {expect(mockInsert).toHaveBeenCalledTimes(1);});+ it("writes events_full rows through the JSON singleton", async () => {+ const mockInsert = vi+ .spyOn(clickhouseClientMock, "insert")+ .mockResolvedValue();+ const event = {+ id: "event-1",+ project_id: "project-1",+ trace_id: "trace-1",+ };++ writer.addToQueue(TableName.EventsFull, event as any);+ await writer["flushAll"](true);++ expect(mockInsert).toHaveBeenCalledWith(+ expect.objectContaining({+ table: TableName.EventsFull,+ format: "JSONEachRow",+ values: [event],+ }),+ );+ expect(clickhouseClientMock.exec).not.toHaveBeenCalled();+ });++ it("encodes a selected Native batch once and reuses its bytes on a transport retry", async () => {+ const event = preparedEvent("event-1");+ const bytes = Buffer.from("native-block");+ encodeClickhouseEventsMock.mockResolvedValueOnce([+ { bytes, rowCount: 1 },+ ] satisfies NativeEventBlock[]);+ const responseStream = Readable.from([]);+ const streamedBuffers: Buffer[] = [];+ const mockExec = vi+ .spyOn(clickhouseClientMock, "exec")+ .mockImplementationOnce(async ({ values }) => {+ for await (const chunk of values as Readable) {+ streamedBuffers.push(chunk as Buffer);+ }+ throw new Error("Timeout error.");+ })+ .mockImplementationOnce(async ({ values }) => {+ for await (const chunk of values as Readable) {+ streamedBuffers.push(chunk as Buffer);+ }+ return { stream: responseStream };+ });++ const nativeWriter =+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);+ nativeWriter.addToQueue(TableName.EventsFull, event);+ await vi.advanceTimersByTimeAsync(nativeWriter.writeInterval);+ await vi.advanceTimersByTimeAsync(200);++ expect(encodeClickhouseEventsMock).toHaveBeenCalledTimes(1);+ expect(encodeClickhouseEventsMock).toHaveBeenCalledWith([event], 1);+ expect(mockExec).toHaveBeenCalledTimes(2);+ expect(mockExec.mock.calls[0][0]).toMatchObject({+ query: "INSERT INTO events_full FORMAT Native",+ clickhouse_settings: { wait_end_of_query: 1 },+ });+ expect(+ JSON.parse(mockExec.mock.calls[0][0].clickhouse_settings.log_comment),+ ).toMatchObject({+ surface: "worker",+ route: "clickhouse-writer",+ projectId: "MULTI_PROJECT",+ });+ expect(mockExec.mock.calls[1][0]).toMatchObject({+ query: "INSERT INTO events_full FORMAT Native",+ });+ expect(mockExec.mock.calls[0][0].values).not.toBe(+ mockExec.mock.calls[1][0].values,+ );+ expect(streamedBuffers).toHaveLength(2);+ expect(streamedBuffers[0]).toBe(bytes);+ expect(streamedBuffers[1]).toBe(bytes);+ expect(responseStream.readableEnded).toBe(true);+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(0);+ });++ it("requeues the prepared event after a failed flush and encodes it again later", async () => {+ const event = preparedEvent("event-1");+ const mockExec = vi+ .spyOn(clickhouseClientMock, "exec")+ .mockRejectedValueOnce(new Error("permanent insert error"))+ .mockResolvedValueOnce({ stream: Readable.from([]) });+ const nativeWriter =+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);++ nativeWriter.addToQueue(TableName.EventsFull, event);+ await nativeWriter["flushAll"](true);++ expect(clickhouseClientMock.insert).not.toHaveBeenCalled();+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(1);+ expect(nativeWriter["queue"][TableName.EventsFull][0].data).toBe(event);+ expect(nativeWriter["queue"][TableName.EventsFull][0].attempts).toBe(2);+ expect(encodeClickhouseEventsMock).toHaveBeenCalledTimes(1);++ await nativeWriter["flushAll"](true);++ expect(mockExec).toHaveBeenCalledTimes(2);+ expect(encodeClickhouseEventsMock).toHaveBeenCalledTimes(2);+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(0);+ });++ it("requeues Native rows when the response stream fails while draining", async () => {+ const event = preparedEvent("event-1");+ const streamError = new Error("response stream failed");+ const responseStream = new Readable({+ read() {+ this.destroy(streamError);+ },+ });+ const mockExec = vi+ .spyOn(clickhouseClientMock, "exec")+ .mockResolvedValue({ stream: responseStream });+ const nativeWriter =+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);++ nativeWriter.addToQueue(TableName.EventsFull, event);+ await nativeWriter["flushAll"](true);++ expect(mockExec).toHaveBeenCalledTimes(1);+ expect(nativeWriter["queue"][TableName.EventsFull][0].data).toBe(event);+ expect(nativeWriter["queue"][TableName.EventsFull][0].attempts).toBe(2);+ const flushError = vi+ .mocked(logger.error)+ .mock.calls.find(+ ([message]) =>+ message === `[native] ClickhouseWriter.flush ${TableName.EventsFull}`,+ )?.[1];+ expect(flushError).toBe(streamError);+ });++ it("requeues Native rows when ClickHouse returns exception text", async () => {+ const event = preparedEvent("event-1");+ const exceptionText = "Code: 60. DB::Exception: Table '表🔥' doesn't exist";+ const paddedException = `${exceptionText}${"x".repeat(4095 - exceptionText.length)}🔥`;+ const responseBytes = Buffer.from(+ `${paddedException}${"x".repeat(10_000)}`,+ );+ const splitOffset =+ Buffer.byteLength("Code: 60. DB::Exception: Table '") + 1;+ // Split inside both the three-byte character and the four-byte emoji.+ const responseStream = Readable.from([+ responseBytes.subarray(0, splitOffset),+ responseBytes.subarray(splitOffset, splitOffset + 3),+ responseBytes.subarray(splitOffset + 3),+ ]);+ vi.spyOn(clickhouseClientMock, "exec").mockResolvedValue({+ stream: responseStream,+ });+ const nativeWriter =+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);++ nativeWriter.addToQueue(TableName.EventsFull, event);+ await nativeWriter["flushAll"](true);++ expect(nativeWriter["queue"][TableName.EventsFull][0].data).toBe(event);+ expect(nativeWriter["queue"][TableName.EventsFull][0].attempts).toBe(2);+ expect(responseStream.readableEnded).toBe(true);++ const flushError = vi+ .mocked(logger.error)+ .mock.calls.find(+ ([message]) =>+ message === `[native] ClickhouseWriter.flush ${TableName.EventsFull}`,+ )?.[1] as Error;+ expect(flushError.message).toContain(exceptionText);+ expect(flushError.message.length).toBeLessThanOrEqual(4200);+ expect(flushError.message).not.toMatch(/[\uD800-\uDBFF]$/);+ });++ it("batches prepared Native rows into one encoder invocation", async () => {+ const nativeWriter =+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);+ nativeWriter.batchSize = 3;+ const firstEvent = preparedEvent("event-1");+ const secondEvent = preparedEvent("event-2");+ const bytes = Buffer.from("combined");+ encodeClickhouseEventsMock.mockResolvedValueOnce([+ { bytes, rowCount: 2 },+ ] satisfies NativeEventBlock[]);+ const mockExec = vi.spyOn(clickhouseClientMock, "exec").mockResolvedValue({+ stream: Readable.from([]),+ });++ nativeWriter.addToQueue(TableName.EventsFull, firstEvent);+ nativeWriter.addToQueue(TableName.EventsFull, secondEvent);+ await nativeWriter["flushAll"](true);++ expect(encodeClickhouseEventsMock).toHaveBeenCalledTimes(1);+ expect(encodeClickhouseEventsMock).toHaveBeenCalledWith(+ [firstEvent, secondEvent],+ 2,+ );+ expect(mockExec).toHaveBeenCalledTimes(1);+ const request = mockExec.mock.calls[0][0];+ const streamed: Buffer[] = [];+ for await (const chunk of request.values as Readable) {+ streamed.push(chunk as Buffer);+ }+ expect(streamed).toEqual([bytes]);+ expect(streamed[0]).toBe(bytes);+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(0);+ });++ it("counts prepared rows and logs their IDs when dropping a failed flush", async () => {+ const nativeWriter =+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);+ nativeWriter.maxAttempts = 1;+ const event = preparedEvent("event-1");+ vi.spyOn(clickhouseClientMock, "exec").mockRejectedValue(+ new Error("permanent insert error"),+ );++ nativeWriter.addToQueue(TableName.EventsFull, event);+ await nativeWriter["flushAll"](true);++ expect(serverExports.recordIncrement).toHaveBeenCalledWith(+ "langfuse.queue.clickhouse_writer.rows_dropped",+ 1,+ { entity_type: TableName.EventsFull, format: "native" },+ );+ expect(logger.error).toHaveBeenCalledWith(+ expect.stringContaining("Max attempts reached"),+ expect.objectContaining({ droppedIds: [event.ids] }),+ );+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(0);+ });+it("should mark writer insert log comments as multi-project", async () => {const mockInsert = vi.spyOn(clickhouseClientMock, "insert")@@ -192,7 +508,7 @@ describe("ClickhouseWriter", () => {expect(serverExports.recordIncrement).toHaveBeenCalledWith("langfuse.queue.clickhouse_writer.rows_dropped",1,- { entity_type: TableName.Traces },+ { entity_type: TableName.Traces, format: "json" },);});@@ -217,16 +533,27 @@ describe("ClickhouseWriter", () => {);});- it("should shutdown gracefully", async () => {+ it("should shutdown both writers gracefully", async () => {+ const nativeWriter =+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);writer.addToQueue(TableName.Traces, { id: "1", name: "test" });+ nativeWriter.addToQueue(TableName.EventsFull, preparedEvent("native-1"));const mockInsert = vi.spyOn(clickhouseClientMock, "insert").mockResolvedValue();+ const mockExec = vi.spyOn(clickhouseClientMock, "exec").mockResolvedValue({+ stream: Readable.from([]),+ });- await writer.shutdown();+ await ClickhouseWriter.shutdownAll();expect(mockInsert).toHaveBeenCalledTimes(1);+ expect(mockExec).toHaveBeenCalledTimes(1);+ expect(writer["queue"][TableName.Traces]).toHaveLength(0);+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(0);expect(writer["intervalId"]).toBeNull();+ expect(nativeWriter["intervalId"]).toBeNull();+ expect(vi.getTimerCount()).toBe(0);expect(logger.info).toHaveBeenCalledWith("ClickhouseWriter shutdown complete.",);@@ -322,6 +649,43 @@ describe("ClickhouseWriter", () => {expect(writer["queue"][TableName.Scores]).toHaveLength(0);});+ it("instruments idle ticks for both writers without issuing inserts", async () => {+ const nativeWriter =+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);+ const instrumentAsyncSpy = vi.spyOn(serverExports, "instrumentAsync");+ const mockInsert = vi+ .spyOn(clickhouseClientMock, "insert")+ .mockResolvedValue();+ const mockExec = vi.spyOn(clickhouseClientMock, "exec").mockResolvedValue({+ stream: Readable.from([]),+ });++ await vi.advanceTimersByTimeAsync(writer.writeInterval);++ expect(instrumentAsyncSpy).toHaveBeenCalledTimes(2);+ expect(serverExports.recordIncrement).toHaveBeenCalledTimes(2);+ expect(serverExports.recordIncrement).toHaveBeenCalledWith(+ "langfuse.queue.clickhouse_writer.request",+ 1,+ { format: "json" },+ );+ expect(serverExports.recordIncrement).toHaveBeenCalledWith(+ "langfuse.queue.clickhouse_writer.request",+ 1,+ { format: "native" },+ );+ expect(mockInsert).not.toHaveBeenCalled();+ expect(mockExec).not.toHaveBeenCalled();++ nativeWriter.addToQueue(TableName.EventsFull, preparedEvent("native-1"));+ await vi.advanceTimersByTimeAsync(nativeWriter.writeInterval);++ expect(instrumentAsyncSpy).toHaveBeenCalledTimes(4);+ expect(serverExports.recordIncrement).toHaveBeenCalledTimes(4);+ expect(mockExec).toHaveBeenCalledTimes(1);+ expect(mockInsert).not.toHaveBeenCalled();+ });+it("should handle adding items to queue while flush is in progress", async () => {const mockInsert = vi.spyOn(clickhouseClientMock, "insert")@@ -374,13 +738,13 @@ describe("ClickhouseWriter", () => {expect(histogramSpy).toHaveBeenCalledWith("langfuse.queue.clickhouse_writer.wait_time",expect.any(Number),- { unit: "milliseconds" },+ { format: "json", unit: "milliseconds" },);expect(histogramSpy).toHaveBeenCalledWith("langfuse.queue.clickhouse_writer.processing_time",expect.any(Number),- { unit: "milliseconds" },+ { format: "json", unit: "milliseconds" },);expect(distributionSpy).toHaveBeenCalledWith(@@ -389,6 +753,7 @@ describe("ClickhouseWriter", () => {{entity_type: TableName.Traces,type: "wait",+ format: "json",unit: "milliseconds",},);@@ -399,11 +764,55 @@ describe("ClickhouseWriter", () => {{entity_type: TableName.Traces,type: "processing",+ format: "json",unit: "milliseconds",},);});+ it("tags events_full queue gauges by writer format", async () => {+ const nativeWriter =+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);+ vi.spyOn(clickhouseClientMock, "insert").mockResolvedValue();+ vi.spyOn(clickhouseClientMock, "exec").mockResolvedValue({+ stream: Readable.from([]),+ });++ // Enqueue below the default batch size, then lower it to leave distinct+ // queue lengths after each writer flushes one row.+ writer.addToQueue(TableName.EventsFull, { id: "json-1" } as any);+ writer.addToQueue(TableName.EventsFull, { id: "json-2" } as any);+ nativeWriter.addToQueue(TableName.EventsFull, preparedEvent("native-1"));+ nativeWriter.addToQueue(TableName.EventsFull, preparedEvent("native-2"));+ nativeWriter.addToQueue(TableName.EventsFull, preparedEvent("native-3"));+ writer.batchSize = 1;+ nativeWriter.batchSize = 1;++ await writer["flushAll"]();+ await nativeWriter["flushAll"]();++ expect(writer["queue"][TableName.EventsFull]).toHaveLength(1);+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(2);+ expect(serverExports.recordGauge).toHaveBeenCalledWith(+ "ingestion_clickhouse_insert_queue_length",+ 1,+ {+ unit: "records",+ entityType: TableName.EventsFull,+ format: "json",+ },+ );+ expect(serverExports.recordGauge).toHaveBeenCalledWith(+ "ingestion_clickhouse_insert_queue_length",+ 2,+ {+ unit: "records",+ entityType: TableName.EventsFull,+ format: "native",+ },+ );+ });+it("should handle different types of Clickhouse client errors", async () => {const mockInsert = vi.spyOn(clickhouseClientMock, "insert")diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.tsindex b68d9a6e9e..d745b21357 100644--- a/worker/src/services/ClickhouseWriter/index.ts+++ b/worker/src/services/ClickhouseWriter/index.ts@@ -9,13 +9,17 @@ import {buildClickHouseLogComment,} from "@langfuse/shared/src/server";+import type { PreparedEvent } from "@langfuse/native";+import { env } from "../../env";import { logger } from "@langfuse/shared/src/server";import { instrumentAsync } from "@langfuse/shared/src/server";import { backOff } from "exponential-backoff";import {- jsonWriteStrategy,+ jsonWriteStrategyFactory,+ nativeWriteStrategyFactory,type ClickhouseWriteStrategy,+ type ClickhouseWriteStrategyFactory,} from "./writeStrategies";import { TableName, type RecordInsertType } from "./types";export { TableName } from "./types";@@ -26,10 +30,15 @@ export class ClickhouseWriter<PayloadMap extends WriterPayloadMap = JsonWriterPayloadMap,> {private static instance: ClickhouseWriter<JsonWriterPayloadMap> | null = null;+ private static nativeInstance: ClickhouseWriter<NativeWriterPayloadMap> | null =+ null;private client: ClickhouseClientType | null;- private readonly strategyFactory: () => ClickhouseWriteStrategy<+ private readonly strategyFactory: ClickhouseWriteStrategyFactory<PayloadMap[TableName]>;+ private readonly allowedTable: TableName | undefined;+ private readonly logger: typeof logger;+ private readonly logPrefix: string;private readonly activeFlushes = new Set<Promise<void>>();batchSize: number;writeInterval: number;@@ -41,10 +50,14 @@ export class ClickhouseWriter<private constructor(client: ClickhouseClientType | undefined,- strategyFactory: () => ClickhouseWriteStrategy<PayloadMap[TableName]>,+ strategyFactory: ClickhouseWriteStrategyFactory<PayloadMap[TableName]>,+ allowedTable?: TableName,) {this.client = client ?? null;this.strategyFactory = strategyFactory;+ this.allowedTable = allowedTable;+ this.logger = logger.child({ format: strategyFactory.format });+ this.logPrefix = strategyFactory.format === "native" ? "[native] " : "";this.batchSize = env.LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE;this.writeInterval = env.LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS;this.maxAttempts = env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS;@@ -73,7 +86,7 @@ export class ClickhouseWriter<if (!instance) {instance = new ClickhouseWriter<JsonWriterPayloadMap>(client,- () => jsonWriteStrategy,+ jsonWriteStrategyFactory,);ClickhouseWriter.instance = instance;} else if (client) {@@ -82,9 +95,37 @@ export class ClickhouseWriter<return instance;}+ /** Get the singleton Native writer, which accepts only prepared events for events_full. */+ public static getNativeInstance(+ client?: ClickhouseClientType,+ ): ClickhouseWriter<NativeWriterPayloadMap> {+ let instance = ClickhouseWriter.nativeInstance;+ if (!instance) {+ instance = new ClickhouseWriter<NativeWriterPayloadMap>(+ client,+ nativeWriteStrategyFactory,+ TableName.EventsFull,+ );+ ClickhouseWriter.nativeInstance = instance;+ } else if (client) {+ instance.client = client;+ }+ return instance;+ }++ /** Stop every writer instance created in this process. */+ public static async shutdownAll(): Promise<void> {+ await Promise.all([+ ClickhouseWriter.instance?.shutdown(),+ ClickhouseWriter.nativeInstance?.shutdown(),+ ]);+ ClickhouseWriter.instance = null;+ ClickhouseWriter.nativeInstance = null;+ }+private start() {- logger.info(- `Starting ClickhouseWriter. Max interval: ${this.writeInterval} ms, Max batch size: ${this.batchSize}`,+ this.logger.info(+ `${this.logPrefix}Starting ClickhouseWriter. Max interval: ${this.writeInterval} ms, Max batch size: ${this.batchSize}`,);this.intervalId = setInterval(() => {@@ -99,7 +140,7 @@ export class ClickhouseWriter<}public async shutdown(): Promise<void> {- logger.info("Shutting down ClickhouseWriter...");+ this.logger.info(`${this.logPrefix}Shutting down ClickhouseWriter...`);if (this.intervalId) {clearInterval(this.intervalId);@@ -112,28 +153,39 @@ export class ClickhouseWriter<await this.flushAll(true);- logger.info("ClickhouseWriter shutdown complete.");+ this.logger.info(`${this.logPrefix}ClickhouseWriter shutdown complete.`);}public async flushAll(fullQueue = false) {+ const tables = this.allowedTable+ ? [this.allowedTable]+ : [+ TableName.Traces,+ TableName.TracesNull,+ TableName.Scores,+ TableName.Observations,+ TableName.ObservationsBatchStaging,+ TableName.BlobStorageFileLog,+ TableName.DatasetRunItems,+ TableName.EventsFull,+ ];return this.trackActiveFlush(instrumentAsync({name: "write-to-clickhouse",},- async () => {- recordIncrement("langfuse.queue.clickhouse_writer.request");- await Promise.all([- this.flush(TableName.Traces, fullQueue),- this.flush(TableName.TracesNull, fullQueue),- this.flush(TableName.Scores, fullQueue),- this.flush(TableName.Observations, fullQueue),- this.flush(TableName.ObservationsBatchStaging, fullQueue),- this.flush(TableName.BlobStorageFileLog, fullQueue),- this.flush(TableName.DatasetRunItems, fullQueue),- this.flush(TableName.EventsFull, fullQueue),- ]).catch((err) => {- logger.error("ClickhouseWriter.flushAll", err);+ async (span) => {+ span.setAttribute("format", this.strategyFactory.format);+ recordIncrement("langfuse.queue.clickhouse_writer.request", 1, {+ format: this.strategyFactory.format,+ });+ await Promise.all(+ tables.map((table) => this.flush(table, fullQueue)),+ ).catch((err) => {+ this.logger.error(+ `${this.logPrefix}ClickhouseWriter.flushAll`,+ err,+ );});},),@@ -205,8 +257,8 @@ export class ClickhouseWriter<if (queueItems.length === 1) {const record = queueItems[0].data;const truncatedRecord = truncate(tableName, record);- logger.warn(- `String length error with single record for ${tableName}, falling back to truncation`,+ this.logger.warn(+ `${this.logPrefix}String length error with single record for ${tableName}, falling back to truncation`,{recordId: strategy.droppedId(record).id,},@@ -226,8 +278,8 @@ export class ClickhouseWriter<const retryItems = queueItems.slice(0, splitPoint);const requeueItems = queueItems.slice(splitPoint);- logger.info(- `Splitting batch for ${tableName} due to string length error. Retrying ${retryItems.length}, requeueing ${requeueItems.length}`,+ this.logger.info(+ `${this.logPrefix}Splitting batch for ${tableName} due to string length error. Retrying ${retryItems.length}, requeueing ${requeueItems.length}`,);return { retryItems, requeueItems };@@ -241,13 +293,14 @@ export class ClickhouseWriter<0,fullQueue ? entityQueue.length : this.batchSize,);- const writeStrategy = this.strategyFactory();+ const writeStrategy = this.strategyFactory.create();const { prepare, truncate } = writeStrategy;// Log wait timequeueItems.forEach((item) => {const waitTime = Date.now() - item.createdAt;recordHistogram("langfuse.queue.clickhouse_writer.wait_time", waitTime, {+ format: this.strategyFactory.format,unit: "milliseconds",});recordDistribution(@@ -256,6 +309,7 @@ export class ClickhouseWriter<{entity_type: tableName,type: "wait",+ format: this.strategyFactory.format,unit: "milliseconds",},);@@ -294,8 +348,8 @@ export class ClickhouseWriter<const isStringLengthError = this.isStringLengthError(error);if (isRetryable) {- logger.warn(- `ClickHouse Writer failed with retryable error for ${tableName} (attempt ${attemptNumber}/${env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS}): ${error.message}`,+ this.logger.warn(+ `${this.logPrefix}ClickHouse Writer failed with retryable error for ${tableName} (attempt ${attemptNumber}/${env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS}): ${error.message}`,{error: error.message,attemptNumber,@@ -307,8 +361,8 @@ export class ClickhouseWriter<});return true;} else if (isStringLengthError && truncate) {- logger.warn(- `ClickHouse Writer failed with string length error for ${tableName} (attempt ${attemptNumber}/${env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS}): Splitting batch and retrying`,+ this.logger.warn(+ `${this.logPrefix}ClickHouse Writer failed with string length error for ${tableName} (attempt ${attemptNumber}/${env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS}): Splitting batch and retrying`,{error: error.message,attemptNumber,@@ -340,8 +394,8 @@ export class ClickhouseWriter<});return true;} else if (isSizeError && truncate && !hasBeenTruncated) {- logger.warn(- `ClickHouse Writer failed with size error for ${tableName} (attempt ${attemptNumber}/${env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS}): Truncating oversized records and retrying`,+ this.logger.warn(+ `${this.logPrefix}ClickHouse Writer failed with size error for ${tableName} (attempt ${attemptNumber}/${env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS}): Truncating oversized records and retrying`,{error: error.message,attemptNumber,@@ -362,8 +416,8 @@ export class ClickhouseWriter<return true;}- logger.error(- `ClickHouse query failed with non-retryable error: ${error.message}`,+ this.logger.error(+ `${this.logPrefix}ClickHouse query failed with non-retryable error: ${error.message}`,{error: error.message,},@@ -383,6 +437,7 @@ export class ClickhouseWriter<"langfuse.queue.clickhouse_writer.processing_time",processingTime,{+ format: this.strategyFactory.format,unit: "milliseconds",},);@@ -392,24 +447,29 @@ export class ClickhouseWriter<{entity_type: tableName,type: "processing",+ format: this.strategyFactory.format,unit: "milliseconds",},);- logger.debug(- `Flushed ${queueItems.length} records to Clickhouse ${tableName}. New queue length: ${entityQueue.length}`,+ this.logger.debug(+ `${this.logPrefix}Flushed ${queueItems.length} records to Clickhouse ${tableName}. New queue length: ${entityQueue.length}`,);recordGauge("ingestion_clickhouse_insert_queue_length",entityQueue.length,{+ format: this.strategyFactory.format,unit: "records",entityType: tableName,},);} catch (err) {- logger.error(`ClickhouseWriter.flush ${tableName}`, err);+ this.logger.error(+ `${this.logPrefix}ClickhouseWriter.flush ${tableName}`,+ err,+ );// Re-add the records to the queue with incremented attemptslet droppedCount = 0;@@ -421,7 +481,9 @@ export class ClickhouseWriter<});} else {// TODO - Add to a dead letter queue in Redis rather than dropping- recordIncrement("langfuse.queue.clickhouse_writer.error");+ recordIncrement("langfuse.queue.clickhouse_writer.error", 1, {+ format: this.strategyFactory.format,+ });droppedCount++;}});@@ -430,15 +492,15 @@ export class ClickhouseWriter<recordIncrement("langfuse.queue.clickhouse_writer.rows_dropped",droppedCount,- { entity_type: tableName },+ { entity_type: tableName, format: this.strategyFactory.format },);const droppedIds = queueItems.filter((item) => item.attempts >= this.maxAttempts).map((item) => writeStrategy.droppedId(item.data));- logger.error(- `ClickhouseWriter: Max attempts reached, dropped ${droppedCount} ${tableName} record(s)`,+ this.logger.error(+ `${this.logPrefix}ClickhouseWriter: Max attempts reached, dropped ${droppedCount} ${tableName} record(s)`,{ droppedIds },);}@@ -446,6 +508,12 @@ export class ClickhouseWriter<}public addToQueue<T extends TableName>(tableName: T, data: PayloadMap[T]) {+ if (this.allowedTable && tableName !== this.allowedTable) {+ throw new Error(+ `Native ClickHouse writer only accepts ${this.allowedTable}, got ${tableName}`,+ );+ }+const entityQueue = this.queue[tableName];entityQueue.push({createdAt: Date.now(),@@ -454,10 +522,15 @@ export class ClickhouseWriter<});if (entityQueue.length >= this.batchSize) {- logger.debug(`Queue is full. Flushing ${tableName}...`);+ this.logger.debug(+ `${this.logPrefix}Queue is full. Flushing ${tableName}...`,+ );this.trackActiveFlush(this.flush(tableName)).catch((err) => {- logger.error("ClickhouseWriter.addToQueue flush", err);+ this.logger.error(+ `${this.logPrefix}ClickhouseWriter.addToQueue flush`,+ err,+ );});}}@@ -482,14 +555,18 @@ export class ClickhouseWriter<},}).catch((err) => {- logger.error(`ClickhouseWriter.writeToClickhouse ${err}`);+ this.logger.error(+ `${this.logPrefix}ClickhouseWriter.writeToClickhouse ${err}`,+ );throw err;});- logger.debug(- `ClickhouseWriter.writeToClickhouse: ${Date.now() - startTime} ms`,+ this.logger.debug(+ `${this.logPrefix}ClickhouseWriter.writeToClickhouse: ${Date.now() - startTime} ms`,);- recordGauge("ingestion_clickhouse_insert", params.records.length);+ recordGauge("ingestion_clickhouse_insert", params.records.length, {+ format: this.strategyFactory.format,+ });}}@@ -499,6 +576,10 @@ type JsonWriterPayloadMap = {[T in TableName]: RecordInsertType<T>;};+type NativeWriterPayloadMap = {+ [T in TableName]: T extends TableName.EventsFull ? PreparedEvent : never;+};+type ClickhouseQueue<PayloadMap extends WriterPayloadMap> = {[T in TableName]: ClickhouseWriterQueueItem<PayloadMap[T]>[];};diff --git a/worker/src/services/ClickhouseWriter/writeStrategies.ts b/worker/src/services/ClickhouseWriter/writeStrategies.tsindex e304aeff13..d236fb1963 100644--- a/worker/src/services/ClickhouseWriter/writeStrategies.ts+++ b/worker/src/services/ClickhouseWriter/writeStrategies.ts@@ -1,3 +1,10 @@+import { Readable } from "node:stream";++import {+ encodeClickhouseEvents,+ type NativeEventBlock,+ type PreparedEvent,+} from "@langfuse/native";import type { ClickhouseClientType } from "@langfuse/shared/src/server";import type { RecordInsertType, TableName } from "./types";@@ -21,27 +28,100 @@ export type ClickhouseWriteStrategy<Row> = {};};-export const jsonWriteStrategy: ClickhouseWriteStrategy<+export type ClickhouseWriteStrategyFactory<Row> = {+ readonly format: "json" | "native";+ create(): ClickhouseWriteStrategy<Row>;+};++const jsonWriteStrategy: ClickhouseWriteStrategy<RecordInsertType<TableName>> =+ {+ write(client, { table, records, clickhouse_settings }) {+ return client+ .insert({+ table,+ format: "JSONEachRow",+ values: records,+ clickhouse_settings,+ })+ .then(() => {});+ },+ prepare: clampDecimal64Fields,+ truncate: truncateOversizedRecord,+ droppedId(record) {+ return {+ project_id: record.project_id,+ trace_id:+ ("trace_id" in record ? record.trace_id : undefined) ?? record.id,+ id: record.id,+ };+ },+ };++export const jsonWriteStrategyFactory: ClickhouseWriteStrategyFactory<RecordInsertType<TableName>> = {- write(client, { table, records, clickhouse_settings }) {- return client- .insert({- table,- format: "JSONEachRow",- values: records,- clickhouse_settings,- })- .then(() => {});- },- prepare: clampDecimal64Fields,- truncate: truncateOversizedRecord,- droppedId(record) {- return {- project_id: record.project_id,- trace_id:- ("trace_id" in record ? record.trace_id : undefined) ?? record.id,- id: record.id,- };- },+ format: "json",+ create: () => jsonWriteStrategy,};++const MAX_NATIVE_RESPONSE_EXCERPT_LENGTH = 4096;++/**+ * Encode one selected row batch once, then send the owned buffers on every transport attempt.+ * Requeued rows are deliberately encoded again when they enter a later flush.+ */+function createNativeWriteStrategy(): ClickhouseWriteStrategy<PreparedEvent> {+ let encodedBlocks: Promise<NativeEventBlock[]> | undefined;++ return {+ async write(client, { table, records: events, clickhouse_settings }) {+ if (!encodedBlocks) {+ encodedBlocks = encodeClickhouseEvents(events, events.length);+ }+ const blocks = await encodedBlocks;+ const response = await client.exec({+ query: `INSERT INTO ${table} FORMAT Native`,+ // Each attempt consumes its own stream cursor. The queued buffers remain reusable.+ values: Readable.from(+ blocks.map((block) => block.bytes),+ { objectMode: false },+ ),+ clickhouse_settings: {+ ...clickhouse_settings,+ // Ask ClickHouse to finish the insert before sending response headers.+ wait_end_of_query: 1,+ },+ });++ // Successful Native INSERTs return no body. exec() checks status/headers, but does not+ // parse late exceptions in the raw HTTP 200 response body. Drain it fully and retain+ // only a short error excerpt for the writer's retry/requeue path.+ let responseExcerpt = "";+ // Decode across chunk boundaries so split UTF-8 characters remain intact.+ for await (const chunk of response.stream.setEncoding("utf8")) {+ const remainingLength =+ MAX_NATIVE_RESPONSE_EXCERPT_LENGTH - responseExcerpt.length;+ if (remainingLength <= 0) continue;++ responseExcerpt += chunk.slice(0, remainingLength);+ }++ // The length cap counts UTF-16 units; do not log half of a truncated astral character.+ responseExcerpt = responseExcerpt.replace(/[\uD800-\uDBFF]$/, "");+ if (responseExcerpt.length > 0) {+ throw new Error(+ `ClickHouse Native insert returned a non-empty response body: ${responseExcerpt}`,+ );+ }+ },+ droppedId(payload) {+ return payload.ids;+ },+ };+}++export const nativeWriteStrategyFactory: ClickhouseWriteStrategyFactory<PreparedEvent> =+ {+ format: "native",+ create: createNativeWriteStrategy,+ };diff --git a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts b/worker/src/services/IngestionService/tests/IngestionService.integration.test.tsindex eaf47c8a87..bdebea3325 100644--- a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts+++ b/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts@@ -79,10 +79,7 @@ describe("Ingestion end-to-end tests", () => {vi.restoreAllMocks();vi.useRealTimers();- // Reset singleton instance- await clickhouseWriter.shutdown();-- ClickhouseWriter.instance = null;+ await ClickhouseWriter.shutdownAll();});it("should correctly ingest a trace", async () => {diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.tsindex 381b4a16bb..149ee32c02 100644--- a/worker/src/utils/shutdown.ts+++ b/worker/src/utils/shutdown.ts@@ -111,7 +111,7 @@ const runDrainAndClose = async () => {await BackgroundMigrationManager.close();// Flush all pending writes to Clickhouse AFTER closing ingestion queue worker that is writing to it- await ClickhouseWriter.getInstance().shutdown();+ await ClickhouseWriter.shutdownAll();logger.info("Clickhouse writer has been shut down.");// Closes the shared client and every per-queue client in one pass. Each