🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix: handle malformed Unicode escapes in ClickHouse JSON across v3 and v4 (#16559)
Valery Meleshkin036732d4d996
diff --git a/.env.prod.example b/.env.prod.exampleindex 8051bf8e59..fa4c47b10d 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -242,6 +242,9 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# CLICKHOUSE_USER=# CLICKHOUSE_PASSWORD=# CLICKHOUSE_CLUSTER_ENABLED=true+# `auto` uses `no_throw` on ClickHouse 24.4+ and `sanitize` otherwise.+# Defaults: v4=`no_throw`, v3=`auto`. Explicit `no_throw` requires 24.4+.+# LANGFUSE_JSON_BAD_UNICODE_ESCAPE=auto# LANGFUSE_CLICKHOUSE_DELETED_MASK_CLEANER_ENABLED=false# LANGFUSE_CLICKHOUSE_DELETED_MASK_CLEANER_INTERVAL_MS=3600000# LANGFUSE_CLICKHOUSE_DELETED_MASK_CLEANER_SUBMIT_TIMEOUT_MS=60000diff --git a/docker-compose.build.yml b/docker-compose.build.ymlindex d4a2848459..1b4297c73b 100644--- a/docker-compose.build.yml+++ b/docker-compose.build.yml@@ -41,6 +41,7 @@ services:CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}+ LANGFUSE_JSON_BAD_UNICODE_ESCAPE: ${LANGFUSE_JSON_BAD_UNICODE_ESCAPE:-}LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-us-east-1}LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}diff --git a/docker-compose.yml b/docker-compose.ymlindex 55eafbf73e..b3e2d117dd 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -33,6 +33,7 @@ services:CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEMECLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}+ LANGFUSE_JSON_BAD_UNICODE_ESCAPE: ${LANGFUSE_JSON_BAD_UNICODE_ESCAPE:-}LANGFUSE_USE_AZURE_BLOB: ${LANGFUSE_USE_AZURE_BLOB:-false}LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE: ${LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE:-false}LANGFUSE_OCI_AUTH_TYPE: ${LANGFUSE_OCI_AUTH_TYPE:-workload_identity}diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.tsindex 7d83207ada..f9fb833b2b 100644--- a/packages/shared/src/env.ts+++ b/packages/shared/src/env.ts@@ -127,6 +127,9 @@ const EnvSchema = z.object({CLICKHOUSE_PASSWORD: z.string(),CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL: z.coerce.number().int().default(9000),CLICKHOUSE_MAX_OPEN_CONNECTIONS: z.coerce.number().int().default(25),+ LANGFUSE_JSON_BAD_UNICODE_ESCAPE: z+ .enum(["auto", "no_throw", "sanitize"])+ .optional(),// Optional to allow for server-setting fallbacksCLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: z.string().optional(),CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: z.coerce.number().int().optional(),diff --git a/packages/shared/src/server/clickhouse/client.test.ts b/packages/shared/src/server/clickhouse/client.test.tsindex 71ad47f869..d7332c699d 100644--- a/packages/shared/src/server/clickhouse/client.test.ts+++ b/packages/shared/src/server/clickhouse/client.test.ts@@ -11,6 +11,11 @@ const mocks = vi.hoisted(() => ({CLICKHOUSE_USER: "default",CLICKHOUSE_PASSWORD: "",CLICKHOUSE_DB: "default",+ LANGFUSE_JSON_BAD_UNICODE_ESCAPE: undefined as+ | "auto"+ | "no_throw"+ | "sanitize"+ | undefined,CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL: 9000,CLICKHOUSE_MAX_OPEN_CONNECTIONS: 25,CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: undefined,@@ -41,6 +46,7 @@ import { setClickHouseCompatibilityVersionForTests } from "./compatibility";describe("ClickHouseClientManager compatibility settings", () => {beforeEach(async () => {await ClickHouseClientManager.getInstance().closeAllConnections();+ mocks.env.LANGFUSE_JSON_BAD_UNICODE_ESCAPE = undefined;mocks.close.mockClear();mocks.createClient.mockReset();@@ -125,4 +131,44 @@ describe("ClickHouseClientManager compatibility settings", () => {max_execution_time: 60,});});++ it.each([+ ["no_throw", null, 0, "undefined"],+ ["auto", "24.3.0.0", undefined, "function"],+ ] as const)(+ "configures %s JSON handling",+ (+ mode,+ clickHouseVersion,+ expectedClickHouseSetting,+ expectedStringifyType,+ ) => {+ mocks.env.LANGFUSE_JSON_BAD_UNICODE_ESCAPE = mode;+ setClickHouseCompatibilityVersionForTests(clickHouseVersion);++ clickhouseClient();++ const config = mocks.createClient.mock.calls[0][0];+ expect(+ config.clickhouse_settings+ .input_format_json_throw_on_bad_escape_sequence,+ ).toBe(expectedClickHouseSetting);+ expect(typeof config.json?.stringify).toBe(expectedStringifyType);+ },+ );++ it("uses a new cached client when automatic JSON handling changes", () => {+ mocks.env.LANGFUSE_JSON_BAD_UNICODE_ESCAPE = "auto";+ const opts = {+ clickhouse_settings: {+ input_format_json_throw_on_bad_escape_sequence: 0,+ } as ClickHouseSettings,+ };++ clickhouseClient(opts);+ setClickHouseCompatibilityVersionForTests("24.4.0.0");+ clickhouseClient(opts);++ expect(mocks.createClient).toHaveBeenCalledTimes(2);+ });});diff --git a/packages/shared/src/server/clickhouse/client.ts b/packages/shared/src/server/clickhouse/client.tsindex efc87278a6..ad81a02918 100644--- a/packages/shared/src/server/clickhouse/client.ts+++ b/packages/shared/src/server/clickhouse/client.ts@@ -5,7 +5,11 @@ import { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/confiimport { getCurrentSpan } from "../instrumentation";import { propagation, context } from "@opentelemetry/api";import { ClickHouseLogger, mapLogLevel } from "./clickhouse-logger";-import { getClickHouseCompatibilitySettings } from "./compatibility";+import {+ getClickHouseCompatibilitySettings,+ getClickHouseJsonBadUnicodeEscapeMode,+} from "./compatibility";+import { stringifyJsonWithSanitizedSurrogates } from "./json";export { EXCEPTION_TAG_HEADER_NAME } from "@clickhouse/client";@@ -64,15 +68,31 @@ export class ClickHouseClientManager {settings: NodeClickHouseClientConfigOptions;serviceClickhouseSettings: ServiceClickhouseSettings;} {- const serviceClickhouseSettings = this.getServiceClickhouseSettings(- preferredClickhouseService,- );+ const jsonBadUnicodeEscapeMode = getClickHouseJsonBadUnicodeEscapeMode();+ const jsonBadUnicodeEscapeClickhouseSettings: ServiceClickhouseSettings =+ jsonBadUnicodeEscapeMode === "no_throw"+ ? { input_format_json_throw_on_bad_escape_sequence: 0 }+ : {};+ const jsonBadUnicodeEscapeClientSettings =+ jsonBadUnicodeEscapeMode === "sanitize"+ ? {+ json: {+ ...opts.json,+ stringify: stringifyJsonWithSanitizedSurrogates,+ },+ }+ : {};+ const serviceClickhouseSettings: ServiceClickhouseSettings = {+ ...this.getServiceClickhouseSettings(preferredClickhouseService),+ ...jsonBadUnicodeEscapeClickhouseSettings,+ };const keyParams = {url: this.getClickhouseUrl(preferredClickhouseService),username: env.CLICKHOUSE_USER,password: env.CLICKHOUSE_PASSWORD,database: env.CLICKHOUSE_DB,http_headers: opts?.http_headers ?? {},+ ...jsonBadUnicodeEscapeClientSettings,settings: {...serviceClickhouseSettings,...opts?.clickhouse_settings,@@ -116,12 +136,6 @@ export class ClickHouseClientManager {};}- private generateClientSettingsKey(- settings: NodeClickHouseClientConfigOptions,- ): string {- return JSON.stringify(settings);- }-private getClickhouseUrl = (preferredClickhouseService: PreferredClickhouseService,) => {@@ -153,22 +167,13 @@ export class ClickHouseClientManager {opts,preferredClickhouseService,);- const key = this.generateClientSettingsKey(settings);+ const key = JSON.stringify(settings);if (!this.clientMap.has(key)) {const activeSpan = getCurrentSpan();if (activeSpan) {propagation.inject(context.active(), settings.http_headers);}- const cloudOptions: Record<string, unknown> = {};- if (- ["STAGING", "EU", "US", "HIPAA", "JP"].includes(- env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ?? "",- )- ) {- cloudOptions.input_format_json_throw_on_bad_escape_sequence = 0;- }-const clickHouseRequestTimeout =opts.request_timeout ?? CLICKHOUSE_CLIENT_DEFAULT_REQUEST_TIMEOUT_MS;const shouldSendProgressInHttpHeaders =@@ -213,7 +218,6 @@ export class ClickHouseClientManager {update_parallel_mode: env.CLICKHOUSE_UPDATE_PARALLEL_MODE,}: {}),- ...cloudOptions,...serviceClickhouseSettings,...this.getRequestTimeoutClickHouseSettings(clickHouseRequestTimeout),...opts.clickhouse_settings,diff --git a/packages/shared/src/server/clickhouse/compatibility.test.ts b/packages/shared/src/server/clickhouse/compatibility.test.tsindex 2a836b93a4..b1a667b566 100644--- a/packages/shared/src/server/clickhouse/compatibility.test.ts+++ b/packages/shared/src/server/clickhouse/compatibility.test.ts@@ -10,6 +10,7 @@ const envMock = vi.hoisted(() => ({vi.mock("../../env", () => envMock);import {+ resolveClickHouseJsonBadUnicodeEscapeMode,isClickHouseVersionInBand,parseClickHouseVersion,resolveClickHouseCompatibility,@@ -109,3 +110,25 @@ describe("resolveClickHouseCompatibility", () => {).toEqual(disableLazyMaterialization);});});++describe("resolveClickHouseJsonBadUnicodeEscapeMode", () => {+ it.each([+ ["24.3.0.0", undefined, "v3.225.4", "sanitize"],+ ["24.4.0.0", "auto", "v3.225.4", "no_throw"],+ [null, "auto", "v3.225.4", "sanitize"],+ ["24.3.0.0", "no_throw", "v3.225.4", "no_throw"],+ ["25.12.0.0", "sanitize", "v4.17.0", "sanitize"],+ ["24.3.0.0", undefined, "v4.17.0", "no_throw"],+ ] as const)(+ "resolves %s / %s / %s to %s",+ (version, configuredMode, applicationVersion, expected) => {+ expect(+ resolveClickHouseJsonBadUnicodeEscapeMode({+ version,+ configuredMode,+ applicationVersion,+ }),+ ).toBe(expected);+ },+ );+});diff --git a/packages/shared/src/server/clickhouse/compatibility.ts b/packages/shared/src/server/clickhouse/compatibility.tsindex 295ee7eba5..b6101817ac 100644--- a/packages/shared/src/server/clickhouse/compatibility.ts+++ b/packages/shared/src/server/clickhouse/compatibility.ts@@ -154,6 +154,47 @@ export const isClickHouseVersionInBand = (return true;};+// The setting was introduced in ClickHouse 24.4.+const CLICKHOUSE_JSON_BAD_UNICODE_ESCAPE_MIN_VERSION = "24.4.0.0";++type ClickHouseJsonBadUnicodeEscapeMode = NonNullable<+ typeof env.LANGFUSE_JSON_BAD_UNICODE_ESCAPE+>;++type ResolvedClickHouseJsonBadUnicodeEscapeMode = Exclude<+ ClickHouseJsonBadUnicodeEscapeMode,+ "auto"+>;++export const resolveClickHouseJsonBadUnicodeEscapeMode = ({+ version,+ configuredMode,+ applicationVersion = VERSION,+}: {+ version?: string | null;+ configuredMode?: ClickHouseJsonBadUnicodeEscapeMode;+ applicationVersion?: string;+} = {}): ResolvedClickHouseJsonBadUnicodeEscapeMode => {+ const mode =+ configuredMode ??+ (applicationVersion.startsWith("v4.") ? "no_throw" : "auto");++ if (mode !== "auto") return mode;++ return version &&+ isClickHouseVersionInBand(version, {+ minInclusive: CLICKHOUSE_JSON_BAD_UNICODE_ESCAPE_MIN_VERSION,+ })+ ? "no_throw"+ : "sanitize";+};++export const getClickHouseJsonBadUnicodeEscapeMode = () =>+ resolveClickHouseJsonBadUnicodeEscapeMode({+ version: detectedClickHouseVersion,+ configuredMode: env.LANGFUSE_JSON_BAD_UNICODE_ESCAPE,+ });+export const resolveClickHouseCompatibility = ({version,overrides,diff --git a/packages/shared/src/server/clickhouse/json.test.ts b/packages/shared/src/server/clickhouse/json.test.tsnew file mode 100644index 0000000000..e6566f5bd7--- /dev/null+++ b/packages/shared/src/server/clickhouse/json.test.ts@@ -0,0 +1,23 @@+import { describe, expect, it } from "vitest";++import { stringifyJsonWithSanitizedSurrogates } from "./json";++describe("stringifyJsonWithSanitizedSurrogates", () => {+ it("replaces lone surrogate code units", () => {+ const serialized = stringifyJsonWithSanitizedSurrogates({+ high: String.fromCharCode(0xd800),+ low: String.fromCharCode(0xdfff),+ });++ expect(JSON.parse(serialized)).toEqual({ high: "�", low: "�" });+ });++ it("preserves literal unicode escape text", () => {+ const literalEscape = String.raw`\ud800`;+ const serialized = stringifyJsonWithSanitizedSurrogates({+ value: literalEscape,+ });++ expect(JSON.parse(serialized)).toEqual({ value: literalEscape });+ });+});diff --git a/packages/shared/src/server/clickhouse/json.ts b/packages/shared/src/server/clickhouse/json.tsnew file mode 100644index 0000000000..7cc7650401--- /dev/null+++ b/packages/shared/src/server/clickhouse/json.ts@@ -0,0 +1,15 @@+// JSON.stringify emits lone UTF-16 surrogates as \\uD800-\\uDFFF escapes. Keep+// escaped backslashes intact so literal text such as `\ud800` is not mistaken+// for a malformed code unit.+const LONE_SURROGATE_ESCAPE = /(?<!\\)((?:\\\\)*)\\u(d[89a-f][0-9a-f]{2})/gi;++export const stringifyJsonWithSanitizedSurrogates = (+ value: unknown,+): string => {+ const serialized = JSON.stringify(value);+ if (serialized === undefined) {+ throw new TypeError("Cannot serialize undefined as JSON");+ }++ return serialized.replace(LONE_SURROGATE_ESCAPE, "$1\uFFFD");+};