Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(worker): add the ClickHouse Native write strategy (#17939)

Valery Meleshkin98b09b9d7043
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index f4d81a8998..c4b3db68ef 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -40,6 +40,9 @@
stream's partial final trace; completion covers the query window, not future arrivals.
- Evaluation terminal-outcome classification: `src/features/evaluation/evalExecutionMetrics.ts`. Keep it aligned with shared code evaluator dispatcher error codes and user-visible error mapping.
- Service layer: `src/services/*`
+- `ClickhouseWriter.getInstance()` queues JSON rows; `getNativeInstance()` queues
+ opaque `PreparedEvent` handles for `events_full`. Use `shutdownAll()` to drain
+ both singletons.
- Rust addon (`@langfuse/native`): telemetry init and the startup hello call live
in `src/initialize.ts`, the health probe call in `src/api/index.ts`. Native code
records its own metrics and logs; see `../packages/native/AGENTS.md`.
diff --git a/worker/src/__tests__/helpers/otelReplayHarness.ts b/worker/src/__tests__/helpers/otelReplayHarness.ts
index 92ce1a40c3..d8b1f03df7 100644
--- a/worker/src/__tests__/helpers/otelReplayHarness.ts
+++ b/worker/src/__tests__/helpers/otelReplayHarness.ts
@@ -77,7 +77,8 @@ export async function runOtelReplay(
shouldWriteToEventsTable: true,
});
- await writer.shutdown();
+ // Reset the singletons here so cleanup cannot retry rows retained by this drain.
+ await ClickhouseWriter.shutdownAll();
writerShutdown = true;
const pendingRows = writer.queue[TableName.EventsFull].length;
@@ -109,13 +110,10 @@ export async function runOtelReplay(
} finally {
try {
if (writer && !writerShutdown) {
- await writer.shutdown();
- }
- } finally {
- if (writer) {
// Each corpus replay needs a fresh singleton and interval timer.
- ClickhouseWriter["instance"] = null;
+ await ClickhouseWriter.shutdownAll();
}
+ } finally {
if (tableCreated) {
await client.command({ query: `DROP TABLE IF EXISTS ${tableName}` });
}
diff --git a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
index c539dfdc71..a2e6e701f5 100644
--- a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
+++ b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
@@ -1,6 +1,9 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { Readable } from "node:stream";
import * as serverExports from "@langfuse/shared/src/server";
+import type { NativeEventBlock } from "@langfuse/native";
+import type { PreparedEvent } from "@langfuse/native";
import { env } from "../../env";
import { logger } from "@langfuse/shared/src/server";
@@ -11,6 +14,23 @@ import {
truncateOversizedRecord,
} from "./jsonRecords";
+const { encodeClickhouseEventsMock } = vi.hoisted(() => ({
+ encodeClickhouseEventsMock: vi.fn(),
+}));
+
+vi.mock("@langfuse/native", () => ({
+ encodeClickhouseEvents: encodeClickhouseEventsMock,
+}));
+
+const preparedEvent = (id: string): PreparedEvent =>
+ ({
+ ids: {
+ project_id: "project-1",
+ trace_id: "trace-1",
+ id,
+ },
+ }) as PreparedEvent;
+
// Mock recordHistogram, recordDistribution, recordCount, recordGauge
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
const original = (await importOriginal()) as {};
@@ -26,6 +46,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
debug: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
+ child: vi.fn().mockReturnThis(),
},
};
});
@@ -45,13 +66,18 @@ vi.mock("../../env", async (importOriginal) => {
describe("ClickhouseWriter", () => {
let clickhouseClientMock: {
insert: ReturnType<typeof vi.fn>;
+ exec: ReturnType<typeof vi.fn>;
};
let writer: ClickhouseWriter;
beforeEach(() => {
vi.clearAllMocks();
+ encodeClickhouseEventsMock.mockResolvedValue([
+ { bytes: Buffer.from("native-block"), rowCount: 1 },
+ ] satisfies NativeEventBlock[]);
clickhouseClientMock = {
insert: vi.fn(),
+ exec: vi.fn(),
};
vi.useFakeTimers();
writer = ClickhouseWriter.getInstance(clickhouseClientMock);
@@ -60,10 +86,7 @@ describe("ClickhouseWriter", () => {
afterEach(async () => {
vi.useRealTimers();
- // Reset singleton instance
- await writer.shutdown();
-
- ClickhouseWriter.instance = null;
+ await ClickhouseWriter.shutdownAll();
vi.restoreAllMocks();
vi.clearAllMocks();
});
@@ -75,6 +98,69 @@ describe("ClickhouseWriter", () => {
expect(instance1).toBe(instance2);
});
+ it("uses a replacement client only for the selected singleton", async () => {
+ const jsonReplacementClient = {
+ insert: vi.fn().mockResolvedValue(),
+ exec: vi.fn(),
+ };
+ const nativeReplacementClient = {
+ insert: vi.fn(),
+ exec: vi.fn().mockResolvedValue({ stream: Readable.from([]) }),
+ };
+ const nativeWriter =
+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);
+ vi.spyOn(clickhouseClientMock, "exec").mockResolvedValue({
+ stream: Readable.from([]),
+ });
+
+ expect(ClickhouseWriter.getInstance(jsonReplacementClient)).toBe(writer);
+ writer.addToQueue(TableName.Traces, { id: "1", name: "test" });
+ nativeWriter.addToQueue(TableName.EventsFull, preparedEvent("native-1"));
+ await Promise.all([
+ writer["flushAll"](true),
+ nativeWriter["flushAll"](true),
+ ]);
+
+ expect(jsonReplacementClient.insert).toHaveBeenCalledTimes(1);
+ expect(jsonReplacementClient.exec).not.toHaveBeenCalled();
+ expect(clickhouseClientMock.insert).not.toHaveBeenCalled();
+ expect(clickhouseClientMock.exec).toHaveBeenCalledTimes(1);
+
+ expect(ClickhouseWriter.getNativeInstance(nativeReplacementClient)).toBe(
+ nativeWriter,
+ );
+ writer.addToQueue(TableName.Traces, { id: "2", name: "test" });
+ nativeWriter.addToQueue(TableName.EventsFull, preparedEvent("native-2"));
+ await Promise.all([
+ writer["flushAll"](true),
+ nativeWriter["flushAll"](true),
+ ]);
+
+ expect(jsonReplacementClient.insert).toHaveBeenCalledTimes(2);
+ expect(jsonReplacementClient.exec).not.toHaveBeenCalled();
+ expect(nativeReplacementClient.exec).toHaveBeenCalledTimes(1);
+ expect(nativeReplacementClient.insert).not.toHaveBeenCalled();
+ expect(clickhouseClientMock.insert).not.toHaveBeenCalled();
+ expect(clickhouseClientMock.exec).toHaveBeenCalledTimes(1);
+ expect(writer["queue"][TableName.Traces]).toHaveLength(0);
+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(0);
+ });
+
+ it("keeps JSON and Native writers as separate singletons", () => {
+ const nativeWriter =
+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);
+
+ expect(nativeWriter).not.toBe(writer);
+ expect(ClickhouseWriter.getInstance()).toBe(writer);
+ expect(ClickhouseWriter.getNativeInstance()).toBe(nativeWriter);
+ expect(logger.child).toHaveBeenNthCalledWith(1, { format: "json" });
+ expect(logger.child).toHaveBeenNthCalledWith(2, { format: "native" });
+
+ expect(() =>
+ nativeWriter.addToQueue(TableName.Traces, {} as never),
+ ).toThrow("Native ClickHouse writer only accepts events_full");
+ });
+
it("should initialize with correct values", () => {
expect(writer.batchSize).toBe(
env.LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE,
@@ -121,6 +207,236 @@ describe("ClickhouseWriter", () => {
expect(mockInsert).toHaveBeenCalledTimes(1);
});
+ it("writes events_full rows through the JSON singleton", async () => {
+ const mockInsert = vi
+ .spyOn(clickhouseClientMock, "insert")
+ .mockResolvedValue();
+ const event = {
+ id: "event-1",
+ project_id: "project-1",
+ trace_id: "trace-1",
+ };
+
+ writer.addToQueue(TableName.EventsFull, event as any);
+ await writer["flushAll"](true);
+
+ expect(mockInsert).toHaveBeenCalledWith(
+ expect.objectContaining({
+ table: TableName.EventsFull,
+ format: "JSONEachRow",
+ values: [event],
+ }),
+ );
+ expect(clickhouseClientMock.exec).not.toHaveBeenCalled();
+ });
+
+ it("encodes a selected Native batch once and reuses its bytes on a transport retry", async () => {
+ const event = preparedEvent("event-1");
+ const bytes = Buffer.from("native-block");
+ encodeClickhouseEventsMock.mockResolvedValueOnce([
+ { bytes, rowCount: 1 },
+ ] satisfies NativeEventBlock[]);
+ const responseStream = Readable.from([]);
+ const streamedBuffers: Buffer[] = [];
+ const mockExec = vi
+ .spyOn(clickhouseClientMock, "exec")
+ .mockImplementationOnce(async ({ values }) => {
+ for await (const chunk of values as Readable) {
+ streamedBuffers.push(chunk as Buffer);
+ }
+ throw new Error("Timeout error.");
+ })
+ .mockImplementationOnce(async ({ values }) => {
+ for await (const chunk of values as Readable) {
+ streamedBuffers.push(chunk as Buffer);
+ }
+ return { stream: responseStream };
+ });
+
+ const nativeWriter =
+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);
+ nativeWriter.addToQueue(TableName.EventsFull, event);
+ await vi.advanceTimersByTimeAsync(nativeWriter.writeInterval);
+ await vi.advanceTimersByTimeAsync(200);
+
+ expect(encodeClickhouseEventsMock).toHaveBeenCalledTimes(1);
+ expect(encodeClickhouseEventsMock).toHaveBeenCalledWith([event], 1);
+ expect(mockExec).toHaveBeenCalledTimes(2);
+ expect(mockExec.mock.calls[0][0]).toMatchObject({
+ query: "INSERT INTO events_full FORMAT Native",
+ clickhouse_settings: { wait_end_of_query: 1 },
+ });
+ expect(
+ JSON.parse(mockExec.mock.calls[0][0].clickhouse_settings.log_comment),
+ ).toMatchObject({
+ surface: "worker",
+ route: "clickhouse-writer",
+ projectId: "MULTI_PROJECT",
+ });
+ expect(mockExec.mock.calls[1][0]).toMatchObject({
+ query: "INSERT INTO events_full FORMAT Native",
+ });
+ expect(mockExec.mock.calls[0][0].values).not.toBe(
+ mockExec.mock.calls[1][0].values,
+ );
+ expect(streamedBuffers).toHaveLength(2);
+ expect(streamedBuffers[0]).toBe(bytes);
+ expect(streamedBuffers[1]).toBe(bytes);
+ expect(responseStream.readableEnded).toBe(true);
+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(0);
+ });
+
+ it("requeues the prepared event after a failed flush and encodes it again later", async () => {
+ const event = preparedEvent("event-1");
+ const mockExec = vi
+ .spyOn(clickhouseClientMock, "exec")
+ .mockRejectedValueOnce(new Error("permanent insert error"))
+ .mockResolvedValueOnce({ stream: Readable.from([]) });
+ const nativeWriter =
+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);
+
+ nativeWriter.addToQueue(TableName.EventsFull, event);
+ await nativeWriter["flushAll"](true);
+
+ expect(clickhouseClientMock.insert).not.toHaveBeenCalled();
+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(1);
+ expect(nativeWriter["queue"][TableName.EventsFull][0].data).toBe(event);
+ expect(nativeWriter["queue"][TableName.EventsFull][0].attempts).toBe(2);
+ expect(encodeClickhouseEventsMock).toHaveBeenCalledTimes(1);
+
+ await nativeWriter["flushAll"](true);
+
+ expect(mockExec).toHaveBeenCalledTimes(2);
+ expect(encodeClickhouseEventsMock).toHaveBeenCalledTimes(2);
+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(0);
+ });
+
+ it("requeues Native rows when the response stream fails while draining", async () => {
+ const event = preparedEvent("event-1");
+ const streamError = new Error("response stream failed");
+ const responseStream = new Readable({
+ read() {
+ this.destroy(streamError);
+ },
+ });
+ const mockExec = vi
+ .spyOn(clickhouseClientMock, "exec")
+ .mockResolvedValue({ stream: responseStream });
+ const nativeWriter =
+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);
+
+ nativeWriter.addToQueue(TableName.EventsFull, event);
+ await nativeWriter["flushAll"](true);
+
+ expect(mockExec).toHaveBeenCalledTimes(1);
+ expect(nativeWriter["queue"][TableName.EventsFull][0].data).toBe(event);
+ expect(nativeWriter["queue"][TableName.EventsFull][0].attempts).toBe(2);
+ const flushError = vi
+ .mocked(logger.error)
+ .mock.calls.find(
+ ([message]) =>
+ message === `[native] ClickhouseWriter.flush ${TableName.EventsFull}`,
+ )?.[1];
+ expect(flushError).toBe(streamError);
+ });
+
+ it("requeues Native rows when ClickHouse returns exception text", async () => {
+ const event = preparedEvent("event-1");
+ const exceptionText = "Code: 60. DB::Exception: Table '表🔥' doesn't exist";
+ const paddedException = `${exceptionText}${"x".repeat(4095 - exceptionText.length)}🔥`;
+ const responseBytes = Buffer.from(
+ `${paddedException}${"x".repeat(10_000)}`,
+ );
+ const splitOffset =
+ Buffer.byteLength("Code: 60. DB::Exception: Table '") + 1;
+ // Split inside both the three-byte character and the four-byte emoji.
+ const responseStream = Readable.from([
+ responseBytes.subarray(0, splitOffset),
+ responseBytes.subarray(splitOffset, splitOffset + 3),
+ responseBytes.subarray(splitOffset + 3),
+ ]);
+ vi.spyOn(clickhouseClientMock, "exec").mockResolvedValue({
+ stream: responseStream,
+ });
+ const nativeWriter =
+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);
+
+ nativeWriter.addToQueue(TableName.EventsFull, event);
+ await nativeWriter["flushAll"](true);
+
+ expect(nativeWriter["queue"][TableName.EventsFull][0].data).toBe(event);
+ expect(nativeWriter["queue"][TableName.EventsFull][0].attempts).toBe(2);
+ expect(responseStream.readableEnded).toBe(true);
+
+ const flushError = vi
+ .mocked(logger.error)
+ .mock.calls.find(
+ ([message]) =>
+ message === `[native] ClickhouseWriter.flush ${TableName.EventsFull}`,
+ )?.[1] as Error;
+ expect(flushError.message).toContain(exceptionText);
+ expect(flushError.message.length).toBeLessThanOrEqual(4200);
+ expect(flushError.message).not.toMatch(/[\uD800-\uDBFF]$/);
+ });
+
+ it("batches prepared Native rows into one encoder invocation", async () => {
+ const nativeWriter =
+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);
+ nativeWriter.batchSize = 3;
+ const firstEvent = preparedEvent("event-1");
+ const secondEvent = preparedEvent("event-2");
+ const bytes = Buffer.from("combined");
+ encodeClickhouseEventsMock.mockResolvedValueOnce([
+ { bytes, rowCount: 2 },
+ ] satisfies NativeEventBlock[]);
+ const mockExec = vi.spyOn(clickhouseClientMock, "exec").mockResolvedValue({
+ stream: Readable.from([]),
+ });
+
+ nativeWriter.addToQueue(TableName.EventsFull, firstEvent);
+ nativeWriter.addToQueue(TableName.EventsFull, secondEvent);
+ await nativeWriter["flushAll"](true);
+
+ expect(encodeClickhouseEventsMock).toHaveBeenCalledTimes(1);
+ expect(encodeClickhouseEventsMock).toHaveBeenCalledWith(
+ [firstEvent, secondEvent],
+ 2,
+ );
+ expect(mockExec).toHaveBeenCalledTimes(1);
+ const request = mockExec.mock.calls[0][0];
+ const streamed: Buffer[] = [];
+ for await (const chunk of request.values as Readable) {
+ streamed.push(chunk as Buffer);
+ }
+ expect(streamed).toEqual([bytes]);
+ expect(streamed[0]).toBe(bytes);
+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(0);
+ });
+
+ it("counts prepared rows and logs their IDs when dropping a failed flush", async () => {
+ const nativeWriter =
+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);
+ nativeWriter.maxAttempts = 1;
+ const event = preparedEvent("event-1");
+ vi.spyOn(clickhouseClientMock, "exec").mockRejectedValue(
+ new Error("permanent insert error"),
+ );
+
+ nativeWriter.addToQueue(TableName.EventsFull, event);
+ await nativeWriter["flushAll"](true);
+
+ expect(serverExports.recordIncrement).toHaveBeenCalledWith(
+ "langfuse.queue.clickhouse_writer.rows_dropped",
+ 1,
+ { entity_type: TableName.EventsFull, format: "native" },
+ );
+ expect(logger.error).toHaveBeenCalledWith(
+ expect.stringContaining("Max attempts reached"),
+ expect.objectContaining({ droppedIds: [event.ids] }),
+ );
+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(0);
+ });
+
it("should mark writer insert log comments as multi-project", async () => {
const mockInsert = vi
.spyOn(clickhouseClientMock, "insert")
@@ -192,7 +508,7 @@ describe("ClickhouseWriter", () => {
expect(serverExports.recordIncrement).toHaveBeenCalledWith(
"langfuse.queue.clickhouse_writer.rows_dropped",
1,
- { entity_type: TableName.Traces },
+ { entity_type: TableName.Traces, format: "json" },
);
});
@@ -217,16 +533,27 @@ describe("ClickhouseWriter", () => {
);
});
- it("should shutdown gracefully", async () => {
+ it("should shutdown both writers gracefully", async () => {
+ const nativeWriter =
+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);
writer.addToQueue(TableName.Traces, { id: "1", name: "test" });
+ nativeWriter.addToQueue(TableName.EventsFull, preparedEvent("native-1"));
const mockInsert = vi
.spyOn(clickhouseClientMock, "insert")
.mockResolvedValue();
+ const mockExec = vi.spyOn(clickhouseClientMock, "exec").mockResolvedValue({
+ stream: Readable.from([]),
+ });
- await writer.shutdown();
+ await ClickhouseWriter.shutdownAll();
expect(mockInsert).toHaveBeenCalledTimes(1);
+ expect(mockExec).toHaveBeenCalledTimes(1);
+ expect(writer["queue"][TableName.Traces]).toHaveLength(0);
+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(0);
expect(writer["intervalId"]).toBeNull();
+ expect(nativeWriter["intervalId"]).toBeNull();
+ expect(vi.getTimerCount()).toBe(0);
expect(logger.info).toHaveBeenCalledWith(
"ClickhouseWriter shutdown complete.",
);
@@ -322,6 +649,43 @@ describe("ClickhouseWriter", () => {
expect(writer["queue"][TableName.Scores]).toHaveLength(0);
});
+ it("instruments idle ticks for both writers without issuing inserts", async () => {
+ const nativeWriter =
+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);
+ const instrumentAsyncSpy = vi.spyOn(serverExports, "instrumentAsync");
+ const mockInsert = vi
+ .spyOn(clickhouseClientMock, "insert")
+ .mockResolvedValue();
+ const mockExec = vi.spyOn(clickhouseClientMock, "exec").mockResolvedValue({
+ stream: Readable.from([]),
+ });
+
+ await vi.advanceTimersByTimeAsync(writer.writeInterval);
+
+ expect(instrumentAsyncSpy).toHaveBeenCalledTimes(2);
+ expect(serverExports.recordIncrement).toHaveBeenCalledTimes(2);
+ expect(serverExports.recordIncrement).toHaveBeenCalledWith(
+ "langfuse.queue.clickhouse_writer.request",
+ 1,
+ { format: "json" },
+ );
+ expect(serverExports.recordIncrement).toHaveBeenCalledWith(
+ "langfuse.queue.clickhouse_writer.request",
+ 1,
+ { format: "native" },
+ );
+ expect(mockInsert).not.toHaveBeenCalled();
+ expect(mockExec).not.toHaveBeenCalled();
+
+ nativeWriter.addToQueue(TableName.EventsFull, preparedEvent("native-1"));
+ await vi.advanceTimersByTimeAsync(nativeWriter.writeInterval);
+
+ expect(instrumentAsyncSpy).toHaveBeenCalledTimes(4);
+ expect(serverExports.recordIncrement).toHaveBeenCalledTimes(4);
+ expect(mockExec).toHaveBeenCalledTimes(1);
+ expect(mockInsert).not.toHaveBeenCalled();
+ });
+
it("should handle adding items to queue while flush is in progress", async () => {
const mockInsert = vi
.spyOn(clickhouseClientMock, "insert")
@@ -374,13 +738,13 @@ describe("ClickhouseWriter", () => {
expect(histogramSpy).toHaveBeenCalledWith(
"langfuse.queue.clickhouse_writer.wait_time",
expect.any(Number),
- { unit: "milliseconds" },
+ { format: "json", unit: "milliseconds" },
);
expect(histogramSpy).toHaveBeenCalledWith(
"langfuse.queue.clickhouse_writer.processing_time",
expect.any(Number),
- { unit: "milliseconds" },
+ { format: "json", unit: "milliseconds" },
);
expect(distributionSpy).toHaveBeenCalledWith(
@@ -389,6 +753,7 @@ describe("ClickhouseWriter", () => {
{
entity_type: TableName.Traces,
type: "wait",
+ format: "json",
unit: "milliseconds",
},
);
@@ -399,11 +764,55 @@ describe("ClickhouseWriter", () => {
{
entity_type: TableName.Traces,
type: "processing",
+ format: "json",
unit: "milliseconds",
},
);
});
+ it("tags events_full queue gauges by writer format", async () => {
+ const nativeWriter =
+ ClickhouseWriter.getNativeInstance(clickhouseClientMock);
+ vi.spyOn(clickhouseClientMock, "insert").mockResolvedValue();
+ vi.spyOn(clickhouseClientMock, "exec").mockResolvedValue({
+ stream: Readable.from([]),
+ });
+
+ // Enqueue below the default batch size, then lower it to leave distinct
+ // queue lengths after each writer flushes one row.
+ writer.addToQueue(TableName.EventsFull, { id: "json-1" } as any);
+ writer.addToQueue(TableName.EventsFull, { id: "json-2" } as any);
+ nativeWriter.addToQueue(TableName.EventsFull, preparedEvent("native-1"));
+ nativeWriter.addToQueue(TableName.EventsFull, preparedEvent("native-2"));
+ nativeWriter.addToQueue(TableName.EventsFull, preparedEvent("native-3"));
+ writer.batchSize = 1;
+ nativeWriter.batchSize = 1;
+
+ await writer["flushAll"]();
+ await nativeWriter["flushAll"]();
+
+ expect(writer["queue"][TableName.EventsFull]).toHaveLength(1);
+ expect(nativeWriter["queue"][TableName.EventsFull]).toHaveLength(2);
+ expect(serverExports.recordGauge).toHaveBeenCalledWith(
+ "ingestion_clickhouse_insert_queue_length",
+ 1,
+ {
+ unit: "records",
+ entityType: TableName.EventsFull,
+ format: "json",
+ },
+ );
+ expect(serverExports.recordGauge).toHaveBeenCalledWith(
+ "ingestion_clickhouse_insert_queue_length",
+ 2,
+ {
+ unit: "records",
+ entityType: TableName.EventsFull,
+ format: "native",
+ },
+ );
+ });
+
it("should handle different types of Clickhouse client errors", async () => {
const mockInsert = vi
.spyOn(clickhouseClientMock, "insert")
diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.ts
index b68d9a6e9e..d745b21357 100644
--- a/worker/src/services/ClickhouseWriter/index.ts
+++ b/worker/src/services/ClickhouseWriter/index.ts
@@ -9,13 +9,17 @@ import {
buildClickHouseLogComment,
} from "@langfuse/shared/src/server";
+import type { PreparedEvent } from "@langfuse/native";
+
import { env } from "../../env";
import { logger } from "@langfuse/shared/src/server";
import { instrumentAsync } from "@langfuse/shared/src/server";
import { backOff } from "exponential-backoff";
import {
- jsonWriteStrategy,
+ jsonWriteStrategyFactory,
+ nativeWriteStrategyFactory,
type ClickhouseWriteStrategy,
+ type ClickhouseWriteStrategyFactory,
} from "./writeStrategies";
import { TableName, type RecordInsertType } from "./types";
export { TableName } from "./types";
@@ -26,10 +30,15 @@ export class ClickhouseWriter<
PayloadMap extends WriterPayloadMap = JsonWriterPayloadMap,
> {
private static instance: ClickhouseWriter<JsonWriterPayloadMap> | null = null;
+ private static nativeInstance: ClickhouseWriter<NativeWriterPayloadMap> | null =
+ null;
private client: ClickhouseClientType | null;
- private readonly strategyFactory: () => ClickhouseWriteStrategy<
+ private readonly strategyFactory: ClickhouseWriteStrategyFactory<
PayloadMap[TableName]
>;
+ private readonly allowedTable: TableName | undefined;
+ private readonly logger: typeof logger;
+ private readonly logPrefix: string;
private readonly activeFlushes = new Set<Promise<void>>();
batchSize: number;
writeInterval: number;
@@ -41,10 +50,14 @@ export class ClickhouseWriter<
private constructor(
client: ClickhouseClientType | undefined,
- strategyFactory: () => ClickhouseWriteStrategy<PayloadMap[TableName]>,
+ strategyFactory: ClickhouseWriteStrategyFactory<PayloadMap[TableName]>,
+ allowedTable?: TableName,
) {
this.client = client ?? null;
this.strategyFactory = strategyFactory;
+ this.allowedTable = allowedTable;
+ this.logger = logger.child({ format: strategyFactory.format });
+ this.logPrefix = strategyFactory.format === "native" ? "[native] " : "";
this.batchSize = env.LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE;
this.writeInterval = env.LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS;
this.maxAttempts = env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS;
@@ -73,7 +86,7 @@ export class ClickhouseWriter<
if (!instance) {
instance = new ClickhouseWriter<JsonWriterPayloadMap>(
client,
- () => jsonWriteStrategy,
+ jsonWriteStrategyFactory,
);
ClickhouseWriter.instance = instance;
} else if (client) {
@@ -82,9 +95,37 @@ export class ClickhouseWriter<
return instance;
}
+ /** Get the singleton Native writer, which accepts only prepared events for events_full. */
+ public static getNativeInstance(
+ client?: ClickhouseClientType,
+ ): ClickhouseWriter<NativeWriterPayloadMap> {
+ let instance = ClickhouseWriter.nativeInstance;
+ if (!instance) {
+ instance = new ClickhouseWriter<NativeWriterPayloadMap>(
+ client,
+ nativeWriteStrategyFactory,
+ TableName.EventsFull,
+ );
+ ClickhouseWriter.nativeInstance = instance;
+ } else if (client) {
+ instance.client = client;
+ }
+ return instance;
+ }
+
+ /** Stop every writer instance created in this process. */
+ public static async shutdownAll(): Promise<void> {
+ await Promise.all([
+ ClickhouseWriter.instance?.shutdown(),
+ ClickhouseWriter.nativeInstance?.shutdown(),
+ ]);
+ ClickhouseWriter.instance = null;
+ ClickhouseWriter.nativeInstance = null;
+ }
+
private start() {
- logger.info(
- `Starting ClickhouseWriter. Max interval: ${this.writeInterval} ms, Max batch size: ${this.batchSize}`,
+ this.logger.info(
+ `${this.logPrefix}Starting ClickhouseWriter. Max interval: ${this.writeInterval} ms, Max batch size: ${this.batchSize}`,
);
this.intervalId = setInterval(() => {
@@ -99,7 +140,7 @@ export class ClickhouseWriter<
}
public async shutdown(): Promise<void> {
- logger.info("Shutting down ClickhouseWriter...");
+ this.logger.info(`${this.logPrefix}Shutting down ClickhouseWriter...`);
if (this.intervalId) {
clearInterval(this.intervalId);
@@ -112,28 +153,39 @@ export class ClickhouseWriter<
await this.flushAll(true);
- logger.info("ClickhouseWriter shutdown complete.");
+ this.logger.info(`${this.logPrefix}ClickhouseWriter shutdown complete.`);
}
public async flushAll(fullQueue = false) {
+ const tables = this.allowedTable
+ ? [this.allowedTable]
+ : [
+ TableName.Traces,
+ TableName.TracesNull,
+ TableName.Scores,
+ TableName.Observations,
+ TableName.ObservationsBatchStaging,
+ TableName.BlobStorageFileLog,
+ TableName.DatasetRunItems,
+ TableName.EventsFull,
+ ];
return this.trackActiveFlush(
instrumentAsync(
{
name: "write-to-clickhouse",
},
- async () => {
- recordIncrement("langfuse.queue.clickhouse_writer.request");
- await Promise.all([
- this.flush(TableName.Traces, fullQueue),
- this.flush(TableName.TracesNull, fullQueue),
- this.flush(TableName.Scores, fullQueue),
- this.flush(TableName.Observations, fullQueue),
- this.flush(TableName.ObservationsBatchStaging, fullQueue),
- this.flush(TableName.BlobStorageFileLog, fullQueue),
- this.flush(TableName.DatasetRunItems, fullQueue),
- this.flush(TableName.EventsFull, fullQueue),
- ]).catch((err) => {
- logger.error("ClickhouseWriter.flushAll", err);
+ async (span) => {
+ span.setAttribute("format", this.strategyFactory.format);
+ recordIncrement("langfuse.queue.clickhouse_writer.request", 1, {
+ format: this.strategyFactory.format,
+ });
+ await Promise.all(
+ tables.map((table) => this.flush(table, fullQueue)),
+ ).catch((err) => {
+ this.logger.error(
+ `${this.logPrefix}ClickhouseWriter.flushAll`,
+ err,
+ );
});
},
),
@@ -205,8 +257,8 @@ export class ClickhouseWriter<
if (queueItems.length === 1) {
const record = queueItems[0].data;
const truncatedRecord = truncate(tableName, record);
- logger.warn(
- `String length error with single record for ${tableName}, falling back to truncation`,
+ this.logger.warn(
+ `${this.logPrefix}String length error with single record for ${tableName}, falling back to truncation`,
{
recordId: strategy.droppedId(record).id,
},
@@ -226,8 +278,8 @@ export class ClickhouseWriter<
const retryItems = queueItems.slice(0, splitPoint);
const requeueItems = queueItems.slice(splitPoint);
- logger.info(
- `Splitting batch for ${tableName} due to string length error. Retrying ${retryItems.length}, requeueing ${requeueItems.length}`,
+ this.logger.info(
+ `${this.logPrefix}Splitting batch for ${tableName} due to string length error. Retrying ${retryItems.length}, requeueing ${requeueItems.length}`,
);
return { retryItems, requeueItems };
@@ -241,13 +293,14 @@ export class ClickhouseWriter<
0,
fullQueue ? entityQueue.length : this.batchSize,
);
- const writeStrategy = this.strategyFactory();
+ const writeStrategy = this.strategyFactory.create();
const { prepare, truncate } = writeStrategy;
// Log wait time
queueItems.forEach((item) => {
const waitTime = Date.now() - item.createdAt;
recordHistogram("langfuse.queue.clickhouse_writer.wait_time", waitTime, {
+ format: this.strategyFactory.format,
unit: "milliseconds",
});
recordDistribution(
@@ -256,6 +309,7 @@ export class ClickhouseWriter<
{
entity_type: tableName,
type: "wait",
+ format: this.strategyFactory.format,
unit: "milliseconds",
},
);
@@ -294,8 +348,8 @@ export class ClickhouseWriter<
const isStringLengthError = this.isStringLengthError(error);
if (isRetryable) {
- logger.warn(
- `ClickHouse Writer failed with retryable error for ${tableName} (attempt ${attemptNumber}/${env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS}): ${error.message}`,
+ this.logger.warn(
+ `${this.logPrefix}ClickHouse Writer failed with retryable error for ${tableName} (attempt ${attemptNumber}/${env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS}): ${error.message}`,
{
error: error.message,
attemptNumber,
@@ -307,8 +361,8 @@ export class ClickhouseWriter<
});
return true;
} else if (isStringLengthError && truncate) {
- logger.warn(
- `ClickHouse Writer failed with string length error for ${tableName} (attempt ${attemptNumber}/${env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS}): Splitting batch and retrying`,
+ this.logger.warn(
+ `${this.logPrefix}ClickHouse Writer failed with string length error for ${tableName} (attempt ${attemptNumber}/${env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS}): Splitting batch and retrying`,
{
error: error.message,
attemptNumber,
@@ -340,8 +394,8 @@ export class ClickhouseWriter<
});
return true;
} else if (isSizeError && truncate && !hasBeenTruncated) {
- logger.warn(
- `ClickHouse Writer failed with size error for ${tableName} (attempt ${attemptNumber}/${env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS}): Truncating oversized records and retrying`,
+ this.logger.warn(
+ `${this.logPrefix}ClickHouse Writer failed with size error for ${tableName} (attempt ${attemptNumber}/${env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS}): Truncating oversized records and retrying`,
{
error: error.message,
attemptNumber,
@@ -362,8 +416,8 @@ export class ClickhouseWriter<
return true;
}
- logger.error(
- `ClickHouse query failed with non-retryable error: ${error.message}`,
+ this.logger.error(
+ `${this.logPrefix}ClickHouse query failed with non-retryable error: ${error.message}`,
{
error: error.message,
},
@@ -383,6 +437,7 @@ export class ClickhouseWriter<
"langfuse.queue.clickhouse_writer.processing_time",
processingTime,
{
+ format: this.strategyFactory.format,
unit: "milliseconds",
},
);
@@ -392,24 +447,29 @@ export class ClickhouseWriter<
{
entity_type: tableName,
type: "processing",
+ format: this.strategyFactory.format,
unit: "milliseconds",
},
);
- logger.debug(
- `Flushed ${queueItems.length} records to Clickhouse ${tableName}. New queue length: ${entityQueue.length}`,
+ this.logger.debug(
+ `${this.logPrefix}Flushed ${queueItems.length} records to Clickhouse ${tableName}. New queue length: ${entityQueue.length}`,
);
recordGauge(
"ingestion_clickhouse_insert_queue_length",
entityQueue.length,
{
+ format: this.strategyFactory.format,
unit: "records",
entityType: tableName,
},
);
} catch (err) {
- logger.error(`ClickhouseWriter.flush ${tableName}`, err);
+ this.logger.error(
+ `${this.logPrefix}ClickhouseWriter.flush ${tableName}`,
+ err,
+ );
// Re-add the records to the queue with incremented attempts
let droppedCount = 0;
@@ -421,7 +481,9 @@ export class ClickhouseWriter<
});
} else {
// TODO - Add to a dead letter queue in Redis rather than dropping
- recordIncrement("langfuse.queue.clickhouse_writer.error");
+ recordIncrement("langfuse.queue.clickhouse_writer.error", 1, {
+ format: this.strategyFactory.format,
+ });
droppedCount++;
}
});
@@ -430,15 +492,15 @@ export class ClickhouseWriter<
recordIncrement(
"langfuse.queue.clickhouse_writer.rows_dropped",
droppedCount,
- { entity_type: tableName },
+ { entity_type: tableName, format: this.strategyFactory.format },
);
const droppedIds = queueItems
.filter((item) => item.attempts >= this.maxAttempts)
.map((item) => writeStrategy.droppedId(item.data));
- logger.error(
- `ClickhouseWriter: Max attempts reached, dropped ${droppedCount} ${tableName} record(s)`,
+ this.logger.error(
+ `${this.logPrefix}ClickhouseWriter: Max attempts reached, dropped ${droppedCount} ${tableName} record(s)`,
{ droppedIds },
);
}
@@ -446,6 +508,12 @@ export class ClickhouseWriter<
}
public addToQueue<T extends TableName>(tableName: T, data: PayloadMap[T]) {
+ if (this.allowedTable && tableName !== this.allowedTable) {
+ throw new Error(
+ `Native ClickHouse writer only accepts ${this.allowedTable}, got ${tableName}`,
+ );
+ }
+
const entityQueue = this.queue[tableName];
entityQueue.push({
createdAt: Date.now(),
@@ -454,10 +522,15 @@ export class ClickhouseWriter<
});
if (entityQueue.length >= this.batchSize) {
- logger.debug(`Queue is full. Flushing ${tableName}...`);
+ this.logger.debug(
+ `${this.logPrefix}Queue is full. Flushing ${tableName}...`,
+ );
this.trackActiveFlush(this.flush(tableName)).catch((err) => {
- logger.error("ClickhouseWriter.addToQueue flush", err);
+ this.logger.error(
+ `${this.logPrefix}ClickhouseWriter.addToQueue flush`,
+ err,
+ );
});
}
}
@@ -482,14 +555,18 @@ export class ClickhouseWriter<
},
})
.catch((err) => {
- logger.error(`ClickhouseWriter.writeToClickhouse ${err}`);
+ this.logger.error(
+ `${this.logPrefix}ClickhouseWriter.writeToClickhouse ${err}`,
+ );
throw err;
});
- logger.debug(
- `ClickhouseWriter.writeToClickhouse: ${Date.now() - startTime} ms`,
+ this.logger.debug(
+ `${this.logPrefix}ClickhouseWriter.writeToClickhouse: ${Date.now() - startTime} ms`,
);
- recordGauge("ingestion_clickhouse_insert", params.records.length);
+ recordGauge("ingestion_clickhouse_insert", params.records.length, {
+ format: this.strategyFactory.format,
+ });
}
}
@@ -499,6 +576,10 @@ type JsonWriterPayloadMap = {
[T in TableName]: RecordInsertType<T>;
};
+type NativeWriterPayloadMap = {
+ [T in TableName]: T extends TableName.EventsFull ? PreparedEvent : never;
+};
+
type ClickhouseQueue<PayloadMap extends WriterPayloadMap> = {
[T in TableName]: ClickhouseWriterQueueItem<PayloadMap[T]>[];
};
diff --git a/worker/src/services/ClickhouseWriter/writeStrategies.ts b/worker/src/services/ClickhouseWriter/writeStrategies.ts
index e304aeff13..d236fb1963 100644
--- a/worker/src/services/ClickhouseWriter/writeStrategies.ts
+++ b/worker/src/services/ClickhouseWriter/writeStrategies.ts
@@ -1,3 +1,10 @@
+import { Readable } from "node:stream";
+
+import {
+ encodeClickhouseEvents,
+ type NativeEventBlock,
+ type PreparedEvent,
+} from "@langfuse/native";
import type { ClickhouseClientType } from "@langfuse/shared/src/server";
import type { RecordInsertType, TableName } from "./types";
@@ -21,27 +28,100 @@ export type ClickhouseWriteStrategy<Row> = {
};
};
-export const jsonWriteStrategy: ClickhouseWriteStrategy<
+export type ClickhouseWriteStrategyFactory<Row> = {
+ readonly format: "json" | "native";
+ create(): ClickhouseWriteStrategy<Row>;
+};
+
+const jsonWriteStrategy: ClickhouseWriteStrategy<RecordInsertType<TableName>> =
+ {
+ write(client, { table, records, clickhouse_settings }) {
+ return client
+ .insert({
+ table,
+ format: "JSONEachRow",
+ values: records,
+ clickhouse_settings,
+ })
+ .then(() => {});
+ },
+ prepare: clampDecimal64Fields,
+ truncate: truncateOversizedRecord,
+ droppedId(record) {
+ return {
+ project_id: record.project_id,
+ trace_id:
+ ("trace_id" in record ? record.trace_id : undefined) ?? record.id,
+ id: record.id,
+ };
+ },
+ };
+
+export const jsonWriteStrategyFactory: ClickhouseWriteStrategyFactory<
RecordInsertType<TableName>
> = {
- write(client, { table, records, clickhouse_settings }) {
- return client
- .insert({
- table,
- format: "JSONEachRow",
- values: records,
- clickhouse_settings,
- })
- .then(() => {});
- },
- prepare: clampDecimal64Fields,
- truncate: truncateOversizedRecord,
- droppedId(record) {
- return {
- project_id: record.project_id,
- trace_id:
- ("trace_id" in record ? record.trace_id : undefined) ?? record.id,
- id: record.id,
- };
- },
+ format: "json",
+ create: () => jsonWriteStrategy,
};
+
+const MAX_NATIVE_RESPONSE_EXCERPT_LENGTH = 4096;
+
+/**
+ * Encode one selected row batch once, then send the owned buffers on every transport attempt.
+ * Requeued rows are deliberately encoded again when they enter a later flush.
+ */
+function createNativeWriteStrategy(): ClickhouseWriteStrategy<PreparedEvent> {
+ let encodedBlocks: Promise<NativeEventBlock[]> | undefined;
+
+ return {
+ async write(client, { table, records: events, clickhouse_settings }) {
+ if (!encodedBlocks) {
+ encodedBlocks = encodeClickhouseEvents(events, events.length);
+ }
+ const blocks = await encodedBlocks;
+ const response = await client.exec({
+ query: `INSERT INTO ${table} FORMAT Native`,
+ // Each attempt consumes its own stream cursor. The queued buffers remain reusable.
+ values: Readable.from(
+ blocks.map((block) => block.bytes),
+ { objectMode: false },
+ ),
+ clickhouse_settings: {
+ ...clickhouse_settings,
+ // Ask ClickHouse to finish the insert before sending response headers.
+ wait_end_of_query: 1,
+ },
+ });
+
+ // Successful Native INSERTs return no body. exec() checks status/headers, but does not
+ // parse late exceptions in the raw HTTP 200 response body. Drain it fully and retain
+ // only a short error excerpt for the writer's retry/requeue path.
+ let responseExcerpt = "";
+ // Decode across chunk boundaries so split UTF-8 characters remain intact.
+ for await (const chunk of response.stream.setEncoding("utf8")) {
+ const remainingLength =
+ MAX_NATIVE_RESPONSE_EXCERPT_LENGTH - responseExcerpt.length;
+ if (remainingLength <= 0) continue;
+
+ responseExcerpt += chunk.slice(0, remainingLength);
+ }
+
+ // The length cap counts UTF-16 units; do not log half of a truncated astral character.
+ responseExcerpt = responseExcerpt.replace(/[\uD800-\uDBFF]$/, "");
+ if (responseExcerpt.length > 0) {
+ throw new Error(
+ `ClickHouse Native insert returned a non-empty response body: ${responseExcerpt}`,
+ );
+ }
+ },
+ droppedId(payload) {
+ return payload.ids;
+ },
+ };
+}
+
+export const nativeWriteStrategyFactory: ClickhouseWriteStrategyFactory<PreparedEvent> =
+ {
+ format: "native",
+ create: createNativeWriteStrategy,
+ };
diff --git a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts b/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts
index eaf47c8a87..bdebea3325 100644
--- a/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts
+++ b/worker/src/services/IngestionService/tests/IngestionService.integration.test.ts
@@ -79,10 +79,7 @@ describe("Ingestion end-to-end tests", () => {
vi.restoreAllMocks();
vi.useRealTimers();
- // Reset singleton instance
- await clickhouseWriter.shutdown();
-
- ClickhouseWriter.instance = null;
+ await ClickhouseWriter.shutdownAll();
});
it("should correctly ingest a trace", async () => {
diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.ts
index 381b4a16bb..149ee32c02 100644
--- a/worker/src/utils/shutdown.ts
+++ b/worker/src/utils/shutdown.ts
@@ -111,7 +111,7 @@ const runDrainAndClose = async () => {
await BackgroundMigrationManager.close();
// Flush all pending writes to Clickhouse AFTER closing ingestion queue worker that is writing to it
- await ClickhouseWriter.getInstance().shutdown();
+ await ClickhouseWriter.shutdownAll();
logger.info("Clickhouse writer has been shut down.");
// Closes the shared client and every per-queue client in one pass. Each