🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(worker): extract the ClickHouse JSON write strategy (#17937)
Valery Meleshkina6f2a8844fbf
diff --git a/worker/src/__tests__/helpers/otelReplayHarness.ts b/worker/src/__tests__/helpers/otelReplayHarness.tsindex 71b35574e1..92ce1a40c3 100644--- a/worker/src/__tests__/helpers/otelReplayHarness.ts+++ b/worker/src/__tests__/helpers/otelReplayHarness.ts@@ -115,7 +115,6 @@ export async function runOtelReplay(if (writer) {// Each corpus replay needs a fresh singleton and interval timer.ClickhouseWriter["instance"] = null;- ClickhouseWriter["client"] = null;}if (tableCreated) {await client.command({ query: `DROP TABLE IF EXISTS ${tableName}` });diff --git a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.tsindex c24cd60dc5..c539dfdc71 100644--- a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts+++ b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts@@ -5,6 +5,11 @@ import * as serverExports from "@langfuse/shared/src/server";import { env } from "../../env";import { logger } from "@langfuse/shared/src/server";import { ClickhouseWriter, TableName } from "../ClickhouseWriter";+import {+ clampDecimal64Map,+ clampDecimal64Value,+ truncateOversizedRecord,+} from "./jsonRecords";// Mock recordHistogram, recordDistribution, recordCount, recordGaugevi.mock("@langfuse/shared/src/server", async (importOriginal) => {@@ -501,15 +506,13 @@ describe("ClickhouseWriter", () => {{ input: Infinity, expected: [0, true], name: "positive Infinity" },{ input: -Infinity, expected: [0, true], name: "negative Infinity" },])("$name ($input)", ({ input, expected }) => {- expect(ClickhouseWriter["clampDecimal64Value"](input)).toEqual(- expected,- );+ expect(clampDecimal64Value(input)).toEqual(expected);});});describe("clampDecimal64Map", () => {it("returns undefined for undefined input", () => {- const result = writer["clampDecimal64Map"](undefined, {+ const result = clampDecimal64Map(undefined, {recordId: "r1",projectId: "p1",fieldName: "cost_details",@@ -519,7 +522,7 @@ describe("ClickhouseWriter", () => {it("returns original map when no values need clamping", () => {const map = { input: 0.001, output: 42.5 };- const result = writer["clampDecimal64Map"](map, {+ const result = clampDecimal64Map(map, {recordId: "r1",projectId: "p1",fieldName: "cost_details",@@ -529,7 +532,7 @@ describe("ClickhouseWriter", () => {});it("clamps multiple overflowing entries correctly", () => {- const result = writer["clampDecimal64Map"](+ const result = clampDecimal64Map({ input: 2_000_000, output: -5_000_000, total: NaN },{ recordId: "r1", projectId: "p1", fieldName: "cost_details" },);@@ -541,7 +544,7 @@ describe("ClickhouseWriter", () => {});it("clamps only overflowing entries and logs once", () => {- const result = writer["clampDecimal64Map"](+ const result = clampDecimal64Map({ input: 0.001, output: 8_859_794 },{ recordId: "r1", projectId: "p1", fieldName: "cost_details" },);@@ -590,10 +593,7 @@ describe("ClickhouseWriter", () => {metadata: { key: "value" },} as any;- const truncatedRecord = writer["truncateOversizedRecord"](- TableName.Traces,- record,- );+ const truncatedRecord = truncateOversizedRecord(TableName.Traces, record);expect(truncatedRecord.id).toBe("1");expect((truncatedRecord as any).output).toBe("normal output");@@ -618,10 +618,7 @@ describe("ClickhouseWriter", () => {metadata: { key: "value" },};- const truncatedRecord = writer["truncateOversizedRecord"](- TableName.Traces,- record,- );+ const truncatedRecord = truncateOversizedRecord(TableName.Traces, record);expect(truncatedRecord.id).toBe("1");expect(truncatedRecord.input).toBe("normal input");@@ -648,10 +645,7 @@ describe("ClickhouseWriter", () => {},};- const truncatedRecord = writer["truncateOversizedRecord"](- TableName.Traces,- record,- );+ const truncatedRecord = truncateOversizedRecord(TableName.Traces, record);expect(truncatedRecord.id).toBe("1");expect(truncatedRecord.input).toBe("normal input");@@ -679,7 +673,7 @@ describe("ClickhouseWriter", () => {metadata: { key: "value" },};- const truncatedRecord = writer["truncateOversizedRecord"](+ const truncatedRecord = truncateOversizedRecord(TableName.Traces,normalRecord,);diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.tsindex 76dffadc85..b68d9a6e9e 100644--- a/worker/src/services/ClickhouseWriter/index.ts+++ b/worker/src/services/ClickhouseWriter/index.ts@@ -1,51 +1,50 @@-/* eslint-disable @repo/no-exotic-operators */import {clickhouseClient,ClickhouseClientType,- BlobStorageFileLogInsertType,getCurrentSpan,- ObservationRecordInsertType,- ObservationBatchStagingRecordInsertType,recordDistribution,recordGauge,recordHistogram,recordIncrement,- ScoreRecordInsertType,- TraceRecordInsertType,- TraceNullRecordInsertType,- DatasetRunItemRecordInsertType,- EventRecordInsertType,buildClickHouseLogComment,} from "@langfuse/shared/src/server";-import { Decimal } from "decimal.js";-import { env } from "../../env";import { logger } from "@langfuse/shared/src/server";import { instrumentAsync } from "@langfuse/shared/src/server";import { backOff } from "exponential-backoff";+import {+ jsonWriteStrategy,+ type ClickhouseWriteStrategy,+} from "./writeStrategies";+import { TableName, type RecordInsertType } from "./types";+export { TableName } from "./types";-// Decimal64(12): valid range is (-10^6, 10^6), i.e. 18 total digits with 12 fractional.-// JS double can't represent 999999.999999999999 exactly (rounds to 1e6), so we use a-// value with enough decimal places to be safe while staying representable as a JS number.-const DECIMAL_64_12_LIMIT = new Decimal("1e6");-const DECIMAL_64_12_MAX_NUM = 999_999.999_999;-const DECIMAL_64_12_MIN_NUM = -DECIMAL_64_12_MAX_NUM;const MULTI_PROJECT_LOG_COMMENT_PROJECT_ID = "MULTI_PROJECT";-export class ClickhouseWriter {- private static instance: ClickhouseWriter | null = null;- private static client: ClickhouseClientType | null = null;+export class ClickhouseWriter<+ PayloadMap extends WriterPayloadMap = JsonWriterPayloadMap,+> {+ private static instance: ClickhouseWriter<JsonWriterPayloadMap> | null = null;+ private client: ClickhouseClientType | null;+ private readonly strategyFactory: () => ClickhouseWriteStrategy<+ PayloadMap[TableName]+ >;private readonly activeFlushes = new Set<Promise<void>>();batchSize: number;writeInterval: number;maxAttempts: number;- queue: ClickhouseQueue;+ queue: ClickhouseQueue<PayloadMap>;isIntervalFlushInProgress: boolean;intervalId: NodeJS.Timeout | null = null;- private constructor() {+ private constructor(+ client: ClickhouseClientType | undefined,+ strategyFactory: () => ClickhouseWriteStrategy<PayloadMap[TableName]>,+ ) {+ this.client = client ?? null;+ this.strategyFactory = strategyFactory;this.batchSize = env.LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE;this.writeInterval = env.LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS;this.maxAttempts = env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS;@@ -66,20 +65,21 @@ export class ClickhouseWriter {this.start();}- /**- * Get the singleton instance of ClickhouseWriter.- * Client parameter is only used for testing.- */- public static getInstance(clickhouseClient?: ClickhouseClientType) {- if (clickhouseClient) {- ClickhouseWriter.client = clickhouseClient;- }-- if (!ClickhouseWriter.instance) {- ClickhouseWriter.instance = new ClickhouseWriter();+ /** Get the singleton JSON writer. A supplied client replaces the current client. */+ public static getInstance(+ client?: ClickhouseClientType,+ ): ClickhouseWriter<JsonWriterPayloadMap> {+ let instance = ClickhouseWriter.instance;+ if (!instance) {+ instance = new ClickhouseWriter<JsonWriterPayloadMap>(+ client,+ () => jsonWriteStrategy,+ );+ ClickhouseWriter.instance = instance;+ } else if (client) {+ instance.client = client;}-- return ClickhouseWriter.instance;+ return instance;}private start() {@@ -194,25 +194,30 @@ export class ClickhouseWriter {*/private handleStringLengthError<T extends TableName>(tableName: T,- queueItems: ClickhouseWriterQueueItem<T>[],+ queueItems: ClickhouseWriterQueueItem<PayloadMap[T]>[],+ truncate: (table: TableName, row: PayloadMap[T]) => PayloadMap[T],+ strategy: ClickhouseWriteStrategy<PayloadMap[TableName]>,): {- retryItems: ClickhouseWriterQueueItem<T>[];- requeueItems: ClickhouseWriterQueueItem<T>[];+ retryItems: ClickhouseWriterQueueItem<PayloadMap[T]>[];+ requeueItems: ClickhouseWriterQueueItem<PayloadMap[T]>[];} {// If batch size is 1, fallback to truncation to prevent infinite loopsif (queueItems.length === 1) {- const truncatedRecord = this.truncateOversizedRecord(- tableName,- queueItems[0].data,- );+ const record = queueItems[0].data;+ const truncatedRecord = truncate(tableName, record);logger.warn(`String length error with single record for ${tableName}, falling back to truncation`,{- recordId: queueItems[0].data.id,+ recordId: strategy.droppedId(record).id,},);return {- retryItems: [{ ...queueItems[0], data: truncatedRecord }],+ retryItems: [+ {+ ...queueItems[0],+ data: truncatedRecord,+ },+ ],requeueItems: [],};}@@ -228,154 +233,6 @@ export class ClickhouseWriter {return { retryItems, requeueItems };}- private truncateOversizedRecord<T extends TableName>(- tableName: T,- record: RecordInsertType<T>,- ): RecordInsertType<T> {- const maxFieldSize = 1024 * 1024; // 1MB per field as safety margin- const truncationMessage = "[TRUNCATED: Field exceeded size limit]";-- // Helper function to safely truncate string fields- const truncateField = (value: string | null | undefined): string | null => {- if (!value) return value || null;- if (value.length > maxFieldSize) {- return (- // Keep the first 500KB and append a truncation message- value.substring(0, 500 * 1024) + truncationMessage- );- }- return value;- };-- // Truncate input field if present- if (- "input" in record &&- record.input &&- record.input.length > maxFieldSize- ) {- record.input = truncateField(record.input);- logger.info(- `Truncated oversized input field for record ${record.id} of type ${tableName}`,- {- projectId: record.project_id,- },- );- }-- // Truncate output field if present- if (- "output" in record &&- record.output &&- record.output.length > maxFieldSize- ) {- record.output = truncateField(record.output);- logger.info(- `Truncated oversized output field for record ${record.id} of type ${tableName}`,- {- projectId: record.project_id,- },- );- }-- // Truncate metadata field if present- if ("metadata" in record && record.metadata) {- const metadata = record.metadata;- const truncatedMetadata: Record<string, string> = {};- for (const [key, value] of Object.entries(metadata)) {- if (value && value.length > maxFieldSize) {- truncatedMetadata[key] = truncateField(value) || "";- logger.info(- `Truncated oversized metadata for record ${record.id} of type ${tableName} and key ${key}`,- {- projectId: record.project_id,- },- );- } else {- truncatedMetadata[key] = value;- }- }- record.metadata = truncatedMetadata;- }-- return record;- }-- private static clampDecimal64Value(value: number): [number, boolean] {- if (!Number.isFinite(value)) return [0, true];- if (new Decimal(value).abs().gte(DECIMAL_64_12_LIMIT)) {- return [value >= 0 ? DECIMAL_64_12_MAX_NUM : DECIMAL_64_12_MIN_NUM, true];- }- return [value, false];- }-- private clampDecimal64Map(- map: Record<string, number> | undefined,- context: { recordId: string; projectId: string; fieldName: string },- ): Record<string, number> | undefined {- if (!map) return map;-- let result: Record<string, number> | undefined;- for (const [key, value] of Object.entries(map)) {- const [cv, wasClamped] = ClickhouseWriter.clampDecimal64Value(value);- if (wasClamped) {- result ??= { ...map };- result[key] = cv;- }- }- if (!result) return map;-- logger.warn("Clamped Decimal64(12) overflow in cost map", {- projectId: context.projectId,- recordId: context.recordId,- fieldName: context.fieldName,- });- recordIncrement("langfuse.clickhouse_writer.decimal64_clamped");- return result;- }-- private clampDecimal64Fields<T extends TableName>(- tableName: T,- record: RecordInsertType<T>,- ): RecordInsertType<T> {- const r = record as Record<string, unknown>;- const ctx = {- recordId: r.id as string,- projectId: r.project_id as string,- };-- switch (tableName) {- case TableName.Observations:- case TableName.ObservationsBatchStaging:- case TableName.EventsFull: {- r.provided_cost_details = this.clampDecimal64Map(- r.provided_cost_details as Record<string, number> | undefined,- { ...ctx, fieldName: "provided_cost_details" },- );- r.cost_details = this.clampDecimal64Map(- r.cost_details as Record<string, number> | undefined,- { ...ctx, fieldName: "cost_details" },- );- if (r.total_cost != null && typeof r.total_cost === "number") {- const [cv, wasClamped] = ClickhouseWriter.clampDecimal64Value(- r.total_cost,- );- if (wasClamped) {- r.total_cost = cv;- logger.warn("Clamped Decimal64(12) overflow in total_cost", {- projectId: ctx.projectId,- recordId: ctx.recordId,- fieldName: "total_cost",- });- recordIncrement("langfuse.clickhouse_writer.decimal64_clamped");- }- }- break;- }- }-- return record;- }-private async flush<T extends TableName>(tableName: T, fullQueue = false) {const entityQueue = this.queue[tableName];if (entityQueue.length === 0) return;@@ -384,6 +241,8 @@ export class ClickhouseWriter {0,fullQueue ? entityQueue.length : this.batchSize,);+ const writeStrategy = this.strategyFactory();+ const { prepare, truncate } = writeStrategy;// Log wait timequeueItems.forEach((item) => {@@ -413,9 +272,11 @@ export class ClickhouseWriter {const processingStartTime = Date.now();let recordsToWrite = queueItems.map((item) => item.data);- recordsToWrite = recordsToWrite.map((r) =>- this.clampDecimal64Fields(tableName, r),- );+ if (prepare) {+ recordsToWrite = recordsToWrite.map((record) =>+ prepare(tableName, record),+ );+ }let hasBeenTruncated = false;await backOff(@@ -423,6 +284,7 @@ export class ClickhouseWriter {this.writeToClickhouse({table: tableName,records: recordsToWrite,+ strategy: writeStrategy,}),{numOfAttempts: env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS,@@ -444,7 +306,7 @@ export class ClickhouseWriter {"retry.error": error.message,});return true;- } else if (isStringLengthError) {+ } else if (isStringLengthError && truncate) {logger.warn(`ClickHouse Writer failed with string length error for ${tableName} (attempt ${attemptNumber}/${env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS}): Splitting batch and retrying`,{@@ -457,6 +319,8 @@ export class ClickhouseWriter {const { retryItems, requeueItems } = this.handleStringLengthError(tableName,queueItems,+ truncate,+ writeStrategy,);// Update records to write with only the retry items@@ -475,7 +339,7 @@ export class ClickhouseWriter {"split.requeue_count": requeueItems.length,});return true;- } else if (isSizeError && !hasBeenTruncated) {+ } else if (isSizeError && truncate && !hasBeenTruncated) {logger.warn(`ClickHouse Writer failed with size error for ${tableName} (attempt ${attemptNumber}/${env.LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS}): Truncating oversized records and retrying`,{@@ -486,7 +350,7 @@ export class ClickhouseWriter {// Truncate oversized recordsrecordsToWrite = recordsToWrite.map((record) =>- this.truncateOversizedRecord(tableName, record),+ truncate(tableName, record),);hasBeenTruncated = true;@@ -571,14 +435,7 @@ export class ClickhouseWriter {const droppedIds = queueItems.filter((item) => item.attempts >= this.maxAttempts)- .map((item) => {- const r = item.data as Record<string, unknown>;- return {- project_id: r.project_id,- trace_id: r.trace_id ?? r.id,- id: r.id,- };- });+ .map((item) => writeStrategy.droppedId(item.data));logger.error(`ClickhouseWriter: Max attempts reached, dropped ${droppedCount} ${tableName} record(s)`,@@ -588,10 +445,7 @@ export class ClickhouseWriter {}}- public addToQueue<T extends TableName>(- tableName: T,- data: RecordInsertType<T>,- ) {+ public addToQueue<T extends TableName>(tableName: T, data: PayloadMap[T]) {const entityQueue = this.queue[tableName];entityQueue.push({createdAt: Date.now(),@@ -610,15 +464,15 @@ export class ClickhouseWriter {private async writeToClickhouse<T extends TableName>(params: {table: T;- records: RecordInsertType<T>[];+ records: PayloadMap[T][];+ strategy: ClickhouseWriteStrategy<PayloadMap[TableName]>;}): Promise<void> {const startTime = Date.now();- await (ClickhouseWriter.client ?? clickhouseClient())- .insert({+ await params.strategy+ .write(this.client ?? clickhouseClient(), {table: params.table,- format: "JSONEachRow",- values: params.records,+ records: params.records,clickhouse_settings: {log_comment: buildClickHouseLogComment({surface: "worker",@@ -629,53 +483,28 @@ export class ClickhouseWriter {}).catch((err) => {logger.error(`ClickhouseWriter.writeToClickhouse ${err}`);-throw err;});logger.debug(`ClickhouseWriter.writeToClickhouse: ${Date.now() - startTime} ms`,);-recordGauge("ingestion_clickhouse_insert", params.records.length);}}-export enum TableName {- Traces = "traces",- TracesNull = "traces_null",- Scores = "scores",- Observations = "observations",- ObservationsBatchStaging = "observations_batch_staging",- BlobStorageFileLog = "blob_storage_file_log",- DatasetRunItems = "dataset_run_items_rmt",- EventsFull = "events_full", // Primary write target - MV auto-populates events_core-}+type WriterPayloadMap = { [T in TableName]: object };++type JsonWriterPayloadMap = {+ [T in TableName]: RecordInsertType<T>;+};-type RecordInsertType<T extends TableName> = T extends TableName.Scores- ? ScoreRecordInsertType- : T extends TableName.Observations- ? ObservationRecordInsertType- : T extends TableName.ObservationsBatchStaging- ? ObservationBatchStagingRecordInsertType- : T extends TableName.Traces- ? TraceRecordInsertType- : T extends TableName.TracesNull- ? TraceNullRecordInsertType- : T extends TableName.BlobStorageFileLog- ? BlobStorageFileLogInsertType- : T extends TableName.DatasetRunItems- ? DatasetRunItemRecordInsertType- : T extends TableName.EventsFull- ? EventRecordInsertType- : never;--type ClickhouseQueue = {- [T in TableName]: ClickhouseWriterQueueItem<T>[];+type ClickhouseQueue<PayloadMap extends WriterPayloadMap> = {+ [T in TableName]: ClickhouseWriterQueueItem<PayloadMap[T]>[];};-type ClickhouseWriterQueueItem<T extends TableName> = {+type ClickhouseWriterQueueItem<Payload> = {createdAt: number;attempts: number;- data: RecordInsertType<T>;+ data: Payload;};diff --git a/worker/src/services/ClickhouseWriter/jsonRecords.ts b/worker/src/services/ClickhouseWriter/jsonRecords.tsnew file mode 100644index 0000000000..2b1368bbd8--- /dev/null+++ b/worker/src/services/ClickhouseWriter/jsonRecords.ts@@ -0,0 +1,165 @@+import { Decimal } from "decimal.js";+import { logger, recordIncrement } from "@langfuse/shared/src/server";+import { TableName, type RecordInsertType } from "./types";++// Decimal64(12): valid range is (-10^6, 10^6), i.e. 18 total digits with 12 fractional.+// JS double can't represent 999999.999999999999 exactly (rounds to 1e6), so we use a+// value with enough decimal places to be safe while staying representable as a JS number.+const DECIMAL_64_12_LIMIT = new Decimal("1e6");+const DECIMAL_64_12_MAX_NUM = 999_999.999_999;+const DECIMAL_64_12_MIN_NUM = -DECIMAL_64_12_MAX_NUM;++export function truncateOversizedRecord<+ Row extends RecordInsertType<TableName>,+>(tableName: TableName, record: Row): Row {+ const maxFieldSize = 1024 * 1024; // 1MB per field as safety margin+ const truncationMessage = "[TRUNCATED: Field exceeded size limit]";++ // Helper function to safely truncate string fields+ const truncateField = (value: string | null | undefined): string | null => {+ if (!value) return value || null;+ if (value.length > maxFieldSize) {+ return (+ // Keep the first 500KB and append a truncation message+ value.substring(0, 500 * 1024) + truncationMessage+ );+ }+ return value;+ };++ // Truncate input field if present+ if ("input" in record && record.input && record.input.length > maxFieldSize) {+ record.input = truncateField(record.input);+ logger.info(+ `Truncated oversized input field for record ${record.id} of type ${tableName}`,+ {+ projectId: record.project_id,+ },+ );+ }++ // Truncate output field if present+ if (+ "output" in record &&+ record.output &&+ record.output.length > maxFieldSize+ ) {+ record.output = truncateField(record.output);+ logger.info(+ `Truncated oversized output field for record ${record.id} of type ${tableName}`,+ {+ projectId: record.project_id,+ },+ );+ }++ // Truncate metadata field if present+ if ("metadata" in record && record.metadata) {+ const metadata = record.metadata;+ const truncatedMetadata: Record<string, string> = {};+ for (const [key, value] of Object.entries(metadata)) {+ if (value && value.length > maxFieldSize) {+ truncatedMetadata[key] = truncateField(value) || "";+ logger.info(+ `Truncated oversized metadata for record ${record.id} of type ${tableName} and key ${key}`,+ {+ projectId: record.project_id,+ },+ );+ } else {+ truncatedMetadata[key] = value;+ }+ }+ record.metadata = truncatedMetadata;+ }++ return record;+}++export function clampDecimal64Value(value: number): [number, boolean] {+ if (!Number.isFinite(value)) return [0, true];+ if (new Decimal(value).abs().gte(DECIMAL_64_12_LIMIT)) {+ return [value >= 0 ? DECIMAL_64_12_MAX_NUM : DECIMAL_64_12_MIN_NUM, true];+ }+ return [value, false];+}++type CostMapContext = {+ recordId: string;+ projectId: string;+ fieldName: string;+};++export function clampDecimal64Map(+ map: Record<string, number>,+ context: CostMapContext,+): Record<string, number>;+export function clampDecimal64Map(+ map: Record<string, number> | undefined,+ context: CostMapContext,+): Record<string, number> | undefined;+export function clampDecimal64Map(+ map: Record<string, number> | undefined,+ context: CostMapContext,+): Record<string, number> | undefined {+ if (!map) return map;++ let result: Record<string, number> | undefined;+ for (const [key, value] of Object.entries(map)) {+ const [cv, wasClamped] = clampDecimal64Value(value);+ if (wasClamped) {+ if (!result) result = { ...map };+ result[key] = cv;+ }+ }+ if (!result) return map;++ logger.warn("Clamped Decimal64(12) overflow in cost map", {+ projectId: context.projectId,+ recordId: context.recordId,+ fieldName: context.fieldName,+ });+ recordIncrement("langfuse.clickhouse_writer.decimal64_clamped");+ return result;+}++export function clampDecimal64Fields<Row extends RecordInsertType<TableName>>(+ tableName: TableName,+ record: Row,+): Row {+ const ctx = { recordId: record.id, projectId: record.project_id };++ switch (tableName) {+ case TableName.Observations:+ case TableName.ObservationsBatchStaging:+ case TableName.EventsFull: {+ if ("provided_cost_details" in record) {+ record.provided_cost_details = clampDecimal64Map(+ record.provided_cost_details,+ { ...ctx, fieldName: "provided_cost_details" },+ );+ }+ if ("cost_details" in record) {+ record.cost_details = clampDecimal64Map(record.cost_details, {+ ...ctx,+ fieldName: "cost_details",+ });+ }+ if ("total_cost" in record && typeof record.total_cost === "number") {+ const [cv, wasClamped] = clampDecimal64Value(record.total_cost);+ if (wasClamped) {+ record.total_cost = cv;+ logger.warn("Clamped Decimal64(12) overflow in total_cost", {+ projectId: ctx.projectId,+ recordId: ctx.recordId,+ fieldName: "total_cost",+ });+ recordIncrement("langfuse.clickhouse_writer.decimal64_clamped");+ }+ }+ break;+ }+ }++ return record;+}diff --git a/worker/src/services/ClickhouseWriter/types.ts b/worker/src/services/ClickhouseWriter/types.tsnew file mode 100644index 0000000000..25633f5370--- /dev/null+++ b/worker/src/services/ClickhouseWriter/types.ts@@ -0,0 +1,39 @@+import type {+ BlobStorageFileLogInsertType,+ ObservationRecordInsertType,+ ObservationBatchStagingRecordInsertType,+ ScoreRecordInsertType,+ TraceRecordInsertType,+ TraceNullRecordInsertType,+ DatasetRunItemRecordInsertType,+ EventRecordInsertType,+} from "@langfuse/shared/src/server";++export enum TableName {+ Traces = "traces",+ TracesNull = "traces_null",+ Scores = "scores",+ Observations = "observations",+ ObservationsBatchStaging = "observations_batch_staging",+ BlobStorageFileLog = "blob_storage_file_log",+ DatasetRunItems = "dataset_run_items_rmt",+ EventsFull = "events_full", // Primary write target - MV auto-populates events_core+}++export type RecordInsertType<T extends TableName> = T extends TableName.Scores+ ? ScoreRecordInsertType+ : T extends TableName.Observations+ ? ObservationRecordInsertType+ : T extends TableName.ObservationsBatchStaging+ ? ObservationBatchStagingRecordInsertType+ : T extends TableName.Traces+ ? TraceRecordInsertType+ : T extends TableName.TracesNull+ ? TraceNullRecordInsertType+ : T extends TableName.BlobStorageFileLog+ ? BlobStorageFileLogInsertType+ : T extends TableName.DatasetRunItems+ ? DatasetRunItemRecordInsertType+ : T extends TableName.EventsFull+ ? EventRecordInsertType+ : never;diff --git a/worker/src/services/ClickhouseWriter/writeStrategies.ts b/worker/src/services/ClickhouseWriter/writeStrategies.tsnew file mode 100644index 0000000000..e304aeff13--- /dev/null+++ b/worker/src/services/ClickhouseWriter/writeStrategies.ts@@ -0,0 +1,47 @@+import type { ClickhouseClientType } from "@langfuse/shared/src/server";++import type { RecordInsertType, TableName } from "./types";+import { clampDecimal64Fields, truncateOversizedRecord } from "./jsonRecords";++export type ClickhouseWriteStrategy<Row> = {+ write(+ client: ClickhouseClientType,+ params: {+ table: string;+ records: Row[];+ clickhouse_settings: { log_comment: string };+ },+ ): Promise<void>;+ prepare?<R extends Row>(table: TableName, row: R): R;+ truncate?<R extends Row>(table: TableName, row: R): R;+ droppedId(row: Row): {+ project_id: string;+ trace_id: string | null | undefined;+ id: string;+ };+};++export const jsonWriteStrategy: ClickhouseWriteStrategy<+ RecordInsertType<TableName>+> = {+ write(client, { table, records, clickhouse_settings }) {+ return client+ .insert({+ table,+ format: "JSONEachRow",+ values: records,+ clickhouse_settings,+ })+ .then(() => {});+ },+ prepare: clampDecimal64Fields,+ truncate: truncateOversizedRecord,+ droppedId(record) {+ return {+ project_id: record.project_id,+ trace_id:+ ("trace_id" in record ? record.trace_id : undefined) ?? record.id,+ id: record.id,+ };+ },+};