Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix: handle malformed Unicode escapes in ClickHouse JSON across v3 and v4 (#16559)

Valery Meleshkin036732d4d996
diff --git a/.env.prod.example b/.env.prod.example
index 8051bf8e59..fa4c47b10d 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -242,6 +242,9 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# CLICKHOUSE_USER=
# CLICKHOUSE_PASSWORD=
# CLICKHOUSE_CLUSTER_ENABLED=true
+# `auto` uses `no_throw` on ClickHouse 24.4+ and `sanitize` otherwise.
+# Defaults: v4=`no_throw`, v3=`auto`. Explicit `no_throw` requires 24.4+.
+# LANGFUSE_JSON_BAD_UNICODE_ESCAPE=auto
# LANGFUSE_CLICKHOUSE_DELETED_MASK_CLEANER_ENABLED=false
# LANGFUSE_CLICKHOUSE_DELETED_MASK_CLEANER_INTERVAL_MS=3600000
# LANGFUSE_CLICKHOUSE_DELETED_MASK_CLEANER_SUBMIT_TIMEOUT_MS=60000
diff --git a/docker-compose.build.yml b/docker-compose.build.yml
index d4a2848459..1b4297c73b 100644
--- a/docker-compose.build.yml
+++ b/docker-compose.build.yml
@@ -41,6 +41,7 @@ services:
CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
+ LANGFUSE_JSON_BAD_UNICODE_ESCAPE: ${LANGFUSE_JSON_BAD_UNICODE_ESCAPE:-}
LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-us-east-1}
LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
diff --git a/docker-compose.yml b/docker-compose.yml
index 55eafbf73e..b3e2d117dd 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -33,6 +33,7 @@ services:
CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME
CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
+ LANGFUSE_JSON_BAD_UNICODE_ESCAPE: ${LANGFUSE_JSON_BAD_UNICODE_ESCAPE:-}
LANGFUSE_USE_AZURE_BLOB: ${LANGFUSE_USE_AZURE_BLOB:-false}
LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE: ${LANGFUSE_USE_OCI_NATIVE_OBJECT_STORAGE:-false}
LANGFUSE_OCI_AUTH_TYPE: ${LANGFUSE_OCI_AUTH_TYPE:-workload_identity}
diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts
index 7d83207ada..f9fb833b2b 100644
--- a/packages/shared/src/env.ts
+++ b/packages/shared/src/env.ts
@@ -127,6 +127,9 @@ const EnvSchema = z.object({
CLICKHOUSE_PASSWORD: z.string(),
CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL: z.coerce.number().int().default(9000),
CLICKHOUSE_MAX_OPEN_CONNECTIONS: z.coerce.number().int().default(25),
+ LANGFUSE_JSON_BAD_UNICODE_ESCAPE: z
+ .enum(["auto", "no_throw", "sanitize"])
+ .optional(),
// Optional to allow for server-setting fallbacks
CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: z.string().optional(),
CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: z.coerce.number().int().optional(),
diff --git a/packages/shared/src/server/clickhouse/client.test.ts b/packages/shared/src/server/clickhouse/client.test.ts
index 71ad47f869..d7332c699d 100644
--- a/packages/shared/src/server/clickhouse/client.test.ts
+++ b/packages/shared/src/server/clickhouse/client.test.ts
@@ -11,6 +11,11 @@ const mocks = vi.hoisted(() => ({
CLICKHOUSE_USER: "default",
CLICKHOUSE_PASSWORD: "",
CLICKHOUSE_DB: "default",
+ LANGFUSE_JSON_BAD_UNICODE_ESCAPE: undefined as
+ | "auto"
+ | "no_throw"
+ | "sanitize"
+ | undefined,
CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL: 9000,
CLICKHOUSE_MAX_OPEN_CONNECTIONS: 25,
CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: undefined,
@@ -41,6 +46,7 @@ import { setClickHouseCompatibilityVersionForTests } from "./compatibility";
describe("ClickHouseClientManager compatibility settings", () => {
beforeEach(async () => {
await ClickHouseClientManager.getInstance().closeAllConnections();
+ mocks.env.LANGFUSE_JSON_BAD_UNICODE_ESCAPE = undefined;
mocks.close.mockClear();
mocks.createClient.mockReset();
@@ -125,4 +131,44 @@ describe("ClickHouseClientManager compatibility settings", () => {
max_execution_time: 60,
});
});
+
+ it.each([
+ ["no_throw", null, 0, "undefined"],
+ ["auto", "24.3.0.0", undefined, "function"],
+ ] as const)(
+ "configures %s JSON handling",
+ (
+ mode,
+ clickHouseVersion,
+ expectedClickHouseSetting,
+ expectedStringifyType,
+ ) => {
+ mocks.env.LANGFUSE_JSON_BAD_UNICODE_ESCAPE = mode;
+ setClickHouseCompatibilityVersionForTests(clickHouseVersion);
+
+ clickhouseClient();
+
+ const config = mocks.createClient.mock.calls[0][0];
+ expect(
+ config.clickhouse_settings
+ .input_format_json_throw_on_bad_escape_sequence,
+ ).toBe(expectedClickHouseSetting);
+ expect(typeof config.json?.stringify).toBe(expectedStringifyType);
+ },
+ );
+
+ it("uses a new cached client when automatic JSON handling changes", () => {
+ mocks.env.LANGFUSE_JSON_BAD_UNICODE_ESCAPE = "auto";
+ const opts = {
+ clickhouse_settings: {
+ input_format_json_throw_on_bad_escape_sequence: 0,
+ } as ClickHouseSettings,
+ };
+
+ clickhouseClient(opts);
+ setClickHouseCompatibilityVersionForTests("24.4.0.0");
+ clickhouseClient(opts);
+
+ expect(mocks.createClient).toHaveBeenCalledTimes(2);
+ });
});
diff --git a/packages/shared/src/server/clickhouse/client.ts b/packages/shared/src/server/clickhouse/client.ts
index efc87278a6..ad81a02918 100644
--- a/packages/shared/src/server/clickhouse/client.ts
+++ b/packages/shared/src/server/clickhouse/client.ts
@@ -5,7 +5,11 @@ import { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/confi
import { getCurrentSpan } from "../instrumentation";
import { propagation, context } from "@opentelemetry/api";
import { ClickHouseLogger, mapLogLevel } from "./clickhouse-logger";
-import { getClickHouseCompatibilitySettings } from "./compatibility";
+import {
+ getClickHouseCompatibilitySettings,
+ getClickHouseJsonBadUnicodeEscapeMode,
+} from "./compatibility";
+import { stringifyJsonWithSanitizedSurrogates } from "./json";
export { EXCEPTION_TAG_HEADER_NAME } from "@clickhouse/client";
@@ -64,15 +68,31 @@ export class ClickHouseClientManager {
settings: NodeClickHouseClientConfigOptions;
serviceClickhouseSettings: ServiceClickhouseSettings;
} {
- const serviceClickhouseSettings = this.getServiceClickhouseSettings(
- preferredClickhouseService,
- );
+ const jsonBadUnicodeEscapeMode = getClickHouseJsonBadUnicodeEscapeMode();
+ const jsonBadUnicodeEscapeClickhouseSettings: ServiceClickhouseSettings =
+ jsonBadUnicodeEscapeMode === "no_throw"
+ ? { input_format_json_throw_on_bad_escape_sequence: 0 }
+ : {};
+ const jsonBadUnicodeEscapeClientSettings =
+ jsonBadUnicodeEscapeMode === "sanitize"
+ ? {
+ json: {
+ ...opts.json,
+ stringify: stringifyJsonWithSanitizedSurrogates,
+ },
+ }
+ : {};
+ const serviceClickhouseSettings: ServiceClickhouseSettings = {
+ ...this.getServiceClickhouseSettings(preferredClickhouseService),
+ ...jsonBadUnicodeEscapeClickhouseSettings,
+ };
const keyParams = {
url: this.getClickhouseUrl(preferredClickhouseService),
username: env.CLICKHOUSE_USER,
password: env.CLICKHOUSE_PASSWORD,
database: env.CLICKHOUSE_DB,
http_headers: opts?.http_headers ?? {},
+ ...jsonBadUnicodeEscapeClientSettings,
settings: {
...serviceClickhouseSettings,
...opts?.clickhouse_settings,
@@ -116,12 +136,6 @@ export class ClickHouseClientManager {
};
}
- private generateClientSettingsKey(
- settings: NodeClickHouseClientConfigOptions,
- ): string {
- return JSON.stringify(settings);
- }
-
private getClickhouseUrl = (
preferredClickhouseService: PreferredClickhouseService,
) => {
@@ -153,22 +167,13 @@ export class ClickHouseClientManager {
opts,
preferredClickhouseService,
);
- const key = this.generateClientSettingsKey(settings);
+ const key = JSON.stringify(settings);
if (!this.clientMap.has(key)) {
const activeSpan = getCurrentSpan();
if (activeSpan) {
propagation.inject(context.active(), settings.http_headers);
}
- const cloudOptions: Record<string, unknown> = {};
- if (
- ["STAGING", "EU", "US", "HIPAA", "JP"].includes(
- env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION ?? "",
- )
- ) {
- cloudOptions.input_format_json_throw_on_bad_escape_sequence = 0;
- }
-
const clickHouseRequestTimeout =
opts.request_timeout ?? CLICKHOUSE_CLIENT_DEFAULT_REQUEST_TIMEOUT_MS;
const shouldSendProgressInHttpHeaders =
@@ -213,7 +218,6 @@ export class ClickHouseClientManager {
update_parallel_mode: env.CLICKHOUSE_UPDATE_PARALLEL_MODE,
}
: {}),
- ...cloudOptions,
...serviceClickhouseSettings,
...this.getRequestTimeoutClickHouseSettings(clickHouseRequestTimeout),
...opts.clickhouse_settings,
diff --git a/packages/shared/src/server/clickhouse/compatibility.test.ts b/packages/shared/src/server/clickhouse/compatibility.test.ts
index 2a836b93a4..b1a667b566 100644
--- a/packages/shared/src/server/clickhouse/compatibility.test.ts
+++ b/packages/shared/src/server/clickhouse/compatibility.test.ts
@@ -10,6 +10,7 @@ const envMock = vi.hoisted(() => ({
vi.mock("../../env", () => envMock);
import {
+ resolveClickHouseJsonBadUnicodeEscapeMode,
isClickHouseVersionInBand,
parseClickHouseVersion,
resolveClickHouseCompatibility,
@@ -109,3 +110,25 @@ describe("resolveClickHouseCompatibility", () => {
).toEqual(disableLazyMaterialization);
});
});
+
+describe("resolveClickHouseJsonBadUnicodeEscapeMode", () => {
+ it.each([
+ ["24.3.0.0", undefined, "v3.225.4", "sanitize"],
+ ["24.4.0.0", "auto", "v3.225.4", "no_throw"],
+ [null, "auto", "v3.225.4", "sanitize"],
+ ["24.3.0.0", "no_throw", "v3.225.4", "no_throw"],
+ ["25.12.0.0", "sanitize", "v4.17.0", "sanitize"],
+ ["24.3.0.0", undefined, "v4.17.0", "no_throw"],
+ ] as const)(
+ "resolves %s / %s / %s to %s",
+ (version, configuredMode, applicationVersion, expected) => {
+ expect(
+ resolveClickHouseJsonBadUnicodeEscapeMode({
+ version,
+ configuredMode,
+ applicationVersion,
+ }),
+ ).toBe(expected);
+ },
+ );
+});
diff --git a/packages/shared/src/server/clickhouse/compatibility.ts b/packages/shared/src/server/clickhouse/compatibility.ts
index 295ee7eba5..b6101817ac 100644
--- a/packages/shared/src/server/clickhouse/compatibility.ts
+++ b/packages/shared/src/server/clickhouse/compatibility.ts
@@ -154,6 +154,47 @@ export const isClickHouseVersionInBand = (
return true;
};
+// The setting was introduced in ClickHouse 24.4.
+const CLICKHOUSE_JSON_BAD_UNICODE_ESCAPE_MIN_VERSION = "24.4.0.0";
+
+type ClickHouseJsonBadUnicodeEscapeMode = NonNullable<
+ typeof env.LANGFUSE_JSON_BAD_UNICODE_ESCAPE
+>;
+
+type ResolvedClickHouseJsonBadUnicodeEscapeMode = Exclude<
+ ClickHouseJsonBadUnicodeEscapeMode,
+ "auto"
+>;
+
+export const resolveClickHouseJsonBadUnicodeEscapeMode = ({
+ version,
+ configuredMode,
+ applicationVersion = VERSION,
+}: {
+ version?: string | null;
+ configuredMode?: ClickHouseJsonBadUnicodeEscapeMode;
+ applicationVersion?: string;
+} = {}): ResolvedClickHouseJsonBadUnicodeEscapeMode => {
+ const mode =
+ configuredMode ??
+ (applicationVersion.startsWith("v4.") ? "no_throw" : "auto");
+
+ if (mode !== "auto") return mode;
+
+ return version &&
+ isClickHouseVersionInBand(version, {
+ minInclusive: CLICKHOUSE_JSON_BAD_UNICODE_ESCAPE_MIN_VERSION,
+ })
+ ? "no_throw"
+ : "sanitize";
+};
+
+export const getClickHouseJsonBadUnicodeEscapeMode = () =>
+ resolveClickHouseJsonBadUnicodeEscapeMode({
+ version: detectedClickHouseVersion,
+ configuredMode: env.LANGFUSE_JSON_BAD_UNICODE_ESCAPE,
+ });
+
export const resolveClickHouseCompatibility = ({
version,
overrides,
diff --git a/packages/shared/src/server/clickhouse/json.test.ts b/packages/shared/src/server/clickhouse/json.test.ts
new file mode 100644
index 0000000000..e6566f5bd7
--- /dev/null
+++ b/packages/shared/src/server/clickhouse/json.test.ts
@@ -0,0 +1,23 @@
+import { describe, expect, it } from "vitest";
+
+import { stringifyJsonWithSanitizedSurrogates } from "./json";
+
+describe("stringifyJsonWithSanitizedSurrogates", () => {
+ it("replaces lone surrogate code units", () => {
+ const serialized = stringifyJsonWithSanitizedSurrogates({
+ high: String.fromCharCode(0xd800),
+ low: String.fromCharCode(0xdfff),
+ });
+
+ expect(JSON.parse(serialized)).toEqual({ high: "�", low: "�" });
+ });
+
+ it("preserves literal unicode escape text", () => {
+ const literalEscape = String.raw`\ud800`;
+ const serialized = stringifyJsonWithSanitizedSurrogates({
+ value: literalEscape,
+ });
+
+ expect(JSON.parse(serialized)).toEqual({ value: literalEscape });
+ });
+});
diff --git a/packages/shared/src/server/clickhouse/json.ts b/packages/shared/src/server/clickhouse/json.ts
new file mode 100644
index 0000000000..7cc7650401
--- /dev/null
+++ b/packages/shared/src/server/clickhouse/json.ts
@@ -0,0 +1,15 @@
+// JSON.stringify emits lone UTF-16 surrogates as \\uD800-\\uDFFF escapes. Keep
+// escaped backslashes intact so literal text such as `\ud800` is not mistaken
+// for a malformed code unit.
+const LONE_SURROGATE_ESCAPE = /(?<!\\)((?:\\\\)*)\\u(d[89a-f][0-9a-f]{2})/gi;
+
+export const stringifyJsonWithSanitizedSurrogates = (
+ value: unknown,
+): string => {
+ const serialized = JSON.stringify(value);
+ if (serialized === undefined) {
+ throw new TypeError("Cannot serialize undefined as JSON");
+ }
+
+ return serialized.replace(LONE_SURROGATE_ESCAPE, "$1\uFFFD");
+};