Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(clickhouse): add CLICKHOUSE_EXTRA_SETTINGS passthrough for query settings (#18007)

Niklas Semmler7643ceac199d
diff --git a/.env.dev.example b/.env.dev.example
index 88b4444908..8616ea1466 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -339,6 +339,12 @@ CLICKHOUSE_USE_LIGHTWEIGHT_UPDATE="true"
CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION=auto
CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN=auto
CLICKHOUSE_ENABLE_SKIP_INDEXES_ON_DATA_READ=auto
+# Extra ClickHouse settings as a JSON object, sent with every query. The
+# `_READ_ONLY` / `_EVENTS_READ_ONLY` variants apply only to those clients.
+# Settings Langfuse sets itself take precedence. Invalid JSON fails startup.
+# CLICKHOUSE_EXTRA_SETTINGS={"use_skip_indexes_for_disjunctions":"1"}
+# CLICKHOUSE_EXTRA_SETTINGS_READ_ONLY=
+# CLICKHOUSE_EXTRA_SETTINGS_EVENTS_READ_ONLY=
# ClickHouse Billing (CHB) integration (Langfuse Cloud only). First-time
# upgrades on/after this UTC date (YYYY-MM-DD) route to CHB; unset = off.
diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts
index 13f52ae40b..d73d197c90 100644
--- a/packages/shared/src/env.ts
+++ b/packages/shared/src/env.ts
@@ -27,6 +27,35 @@ export const redisSocketTimeoutMsSchema = z.coerce
})
.default(30_000);
+// ClickHouse rejects unknown settings on every query, so a malformed value
+// must fail the process at startup instead.
+const clickhouseExtraSettingsSchema = (name: string) =>
+ z
+ .string()
+ .transform((value, ctx) => {
+ try {
+ return JSON.parse(value) as unknown;
+ } catch {
+ ctx.addIssue({ code: "custom", message: `${name} must be valid JSON` });
+ return z.NEVER;
+ }
+ })
+ .pipe(
+ z.record(
+ z
+ .string()
+ .regex(
+ /^[A-Za-z_][A-Za-z0-9_]*$/,
+ `${name} keys must be ClickHouse setting names`,
+ ),
+ z.union([z.string(), z.number(), z.boolean()]),
+ {
+ error: `${name} must be a JSON object of setting names to string, number, or boolean values`,
+ },
+ ),
+ )
+ .optional();
+
const DEFAULT_LLM_COMPLETION_TIMEOUT_MS = 120_000;
const EnvSchema = z.object({
@@ -165,6 +194,18 @@ const EnvSchema = z.object({
CLICKHOUSE_USE_QUERY_CONDITION_CACHE: z
.enum(["true", "false"])
.default("false"),
+ // JSON objects of ClickHouse setting name to value. Every client gets
+ // CLICKHOUSE_EXTRA_SETTINGS; the service variants add to it for clients of
+ // that service. Settings set by Langfuse itself take precedence.
+ CLICKHOUSE_EXTRA_SETTINGS: clickhouseExtraSettingsSchema(
+ "CLICKHOUSE_EXTRA_SETTINGS",
+ ),
+ CLICKHOUSE_EXTRA_SETTINGS_READ_ONLY: clickhouseExtraSettingsSchema(
+ "CLICKHOUSE_EXTRA_SETTINGS_READ_ONLY",
+ ),
+ CLICKHOUSE_EXTRA_SETTINGS_EVENTS_READ_ONLY: clickhouseExtraSettingsSchema(
+ "CLICKHOUSE_EXTRA_SETTINGS_EVENTS_READ_ONLY",
+ ),
LANGFUSE_ENABLE_SINGLE_LEVEL_QUERY_OPTIMIZATION: z
.enum(["true", "false"])
.default("false"),
diff --git a/packages/shared/src/server/clickhouse/client.test.ts b/packages/shared/src/server/clickhouse/client.test.ts
index f395d123d6..84775365e8 100644
--- a/packages/shared/src/server/clickhouse/client.test.ts
+++ b/packages/shared/src/server/clickhouse/client.test.ts
@@ -1,6 +1,8 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { type ClickHouseSettings } from "@clickhouse/client";
+type ExtraSettings = Record<string, string | number | boolean> | undefined;
+
const mocks = vi.hoisted(() => ({
createClient: vi.fn(),
close: vi.fn(async () => undefined),
@@ -18,13 +20,16 @@ const mocks = vi.hoisted(() => ({
| undefined,
CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL: 9000,
CLICKHOUSE_MAX_OPEN_CONNECTIONS: 25,
- CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: undefined,
+ CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: undefined as string | undefined,
CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: undefined,
CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MIN_MS: undefined,
CLICKHOUSE_LIGHTWEIGHT_DELETE_MODE: "alter_update",
CLICKHOUSE_UPDATE_PARALLEL_MODE: "auto",
CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "auto",
CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN: "auto",
+ CLICKHOUSE_EXTRA_SETTINGS: undefined as ExtraSettings,
+ CLICKHOUSE_EXTRA_SETTINGS_READ_ONLY: undefined as ExtraSettings,
+ CLICKHOUSE_EXTRA_SETTINGS_EVENTS_READ_ONLY: undefined as ExtraSettings,
LANGFUSE_LOG_LEVEL: "error",
NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
},
@@ -58,6 +63,9 @@ describe("ClickHouseClientManager compatibility settings", () => {
mocks.createClient.mockReturnValue({ close: mocks.close });
mocks.env.CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION = "auto";
mocks.env.CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN = "auto";
+ mocks.env.CLICKHOUSE_EXTRA_SETTINGS = undefined;
+ mocks.env.CLICKHOUSE_EXTRA_SETTINGS_READ_ONLY = undefined;
+ mocks.env.CLICKHOUSE_EXTRA_SETTINGS_EVENTS_READ_ONLY = undefined;
setClickHouseCompatibilityVersionForTests(null);
});
@@ -90,6 +98,94 @@ describe("ClickHouseClientManager compatibility settings", () => {
).toBe(1);
});
+ it("sends no extra settings when unset", () => {
+ clickhouseClient();
+
+ expect(
+ mocks.createClient.mock.calls[0][0].clickhouse_settings,
+ ).not.toHaveProperty("use_skip_indexes_for_disjunctions");
+ });
+
+ it("applies extra settings to every client", () => {
+ mocks.env.CLICKHOUSE_EXTRA_SETTINGS = {
+ use_skip_indexes_for_disjunctions: "1",
+ };
+
+ clickhouseClient();
+ clickhouseClient({}, "EventsReadOnly");
+
+ expect(mocks.createClient).toHaveBeenCalledTimes(2);
+ for (const [config] of mocks.createClient.mock.calls) {
+ expect(config.clickhouse_settings).toMatchObject({
+ use_skip_indexes_for_disjunctions: "1",
+ });
+ }
+ });
+
+ it.each([
+ ["ReadWrite", undefined],
+ ["ReadOnly", "read_only"],
+ ["EventsReadOnly", "events_read_only"],
+ ] as const)(
+ "applies only the matching service extra settings to %s",
+ (preferred, expectedMarker) => {
+ mocks.env.CLICKHOUSE_EXTRA_SETTINGS_READ_ONLY = {
+ log_comment: "read_only",
+ };
+ mocks.env.CLICKHOUSE_EXTRA_SETTINGS_EVENTS_READ_ONLY = {
+ log_comment: "events_read_only",
+ };
+
+ clickhouseClient({}, preferred);
+
+ expect(
+ mocks.createClient.mock.calls[0][0].clickhouse_settings.log_comment,
+ ).toBe(expectedMarker);
+ },
+ );
+
+ it("keeps read-only extra settings off the write client on a shared URL", () => {
+ mocks.env.CLICKHOUSE_EXTRA_SETTINGS_READ_ONLY = {
+ log_comment: "read_only",
+ };
+
+ clickhouseClient({}, "ReadOnly");
+ clickhouseClient({}, "ReadWrite");
+
+ expect(mocks.createClient).toHaveBeenCalledTimes(2);
+ expect(
+ mocks.createClient.mock.calls[1][0].clickhouse_settings,
+ ).not.toHaveProperty("log_comment");
+ });
+
+ it("lets derived and per-query settings override extra settings", () => {
+ setClickHouseCompatibilityVersionForTests("26.5.5.8");
+ mocks.env.CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE = "10485760";
+ mocks.env.CLICKHOUSE_EXTRA_SETTINGS = {
+ async_insert_max_data_size: "1",
+ query_plan_top_k_through_join: 1,
+ max_execution_time: 1,
+ date_time_output_format: "simple",
+ };
+
+ try {
+ clickhouseClient({
+ clickhouse_settings: { date_time_output_format: "iso" },
+ });
+ } finally {
+ mocks.env.CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE = undefined;
+ }
+
+ expect(
+ mocks.createClient.mock.calls[0][0].clickhouse_settings,
+ ).toMatchObject({
+ async_insert_max_data_size: "10485760",
+ query_plan_top_k_through_join: 0,
+ max_execution_time: 35,
+ date_time_output_format: "iso",
+ });
+ });
+
it("uses a new cached client key after compatibility settings change", () => {
clickhouseClient();
setClickHouseCompatibilityVersionForTests("26.5.5.8");
@@ -256,3 +352,35 @@ describe("resolveClickhouseService", () => {
},
);
});
+
+describe("CLICKHOUSE_EXTRA_SETTINGS validation", () => {
+ const loadEnv = async (value: string) => {
+ vi.stubEnv("CLICKHOUSE_EXTRA_SETTINGS", value);
+ vi.resetModules();
+ try {
+ return await vi.importActual<typeof import("../../env")>("../../env");
+ } finally {
+ vi.unstubAllEnvs();
+ }
+ };
+
+ it("parses a JSON object of settings", async () => {
+ const { env } = await loadEnv(
+ '{"use_skip_indexes_for_disjunctions":"1","max_threads":4}',
+ );
+
+ expect(env.CLICKHOUSE_EXTRA_SETTINGS).toEqual({
+ use_skip_indexes_for_disjunctions: "1",
+ max_threads: 4,
+ });
+ });
+
+ it.each([
+ ["invalid JSON", "{use_skip_indexes_for_disjunctions:1}"],
+ ["a non-object", '["max_threads"]'],
+ ["a nested value", '{"max_threads":{"value":4}}'],
+ ["an invalid setting name", '{"max threads":4}'],
+ ])("rejects %s at startup", async (_, value) => {
+ await expect(loadEnv(value)).rejects.toThrow("CLICKHOUSE_EXTRA_SETTINGS");
+ });
+});
diff --git a/packages/shared/src/server/clickhouse/client.ts b/packages/shared/src/server/clickhouse/client.ts
index a3d77f8f52..0427ba4bfb 100644
--- a/packages/shared/src/server/clickhouse/client.ts
+++ b/packages/shared/src/server/clickhouse/client.ts
@@ -84,6 +84,7 @@ export class ClickHouseClientManager {
preferredClickhouseService: PreferredClickhouseService = "ReadWrite",
): {
settings: NodeClickHouseClientConfigOptions;
+ extraClickhouseSettings: ClickHouseSettings;
serviceClickhouseSettings: ServiceClickhouseSettings;
} {
const jsonBadUnicodeEscapeMode = getClickHouseJsonBadUnicodeEscapeMode();
@@ -100,6 +101,9 @@ export class ClickHouseClientManager {
},
}
: {};
+ const extraClickhouseSettings = this.getExtraClickhouseSettings(
+ preferredClickhouseService,
+ );
const serviceClickhouseSettings: ServiceClickhouseSettings = {
...this.getServiceClickhouseSettings(preferredClickhouseService),
...jsonBadUnicodeEscapeClickhouseSettings,
@@ -112,6 +116,7 @@ export class ClickHouseClientManager {
http_headers: opts?.http_headers ?? {},
...jsonBadUnicodeEscapeClientSettings,
settings: {
+ ...extraClickhouseSettings,
...serviceClickhouseSettings,
...opts?.clickhouse_settings,
},
@@ -124,6 +129,7 @@ export class ClickHouseClientManager {
};
return {
settings: keyParams,
+ extraClickhouseSettings,
serviceClickhouseSettings,
};
}
@@ -151,6 +157,21 @@ export class ClickHouseClientManager {
};
}
+ private getExtraClickhouseSettings(
+ preferredClickhouseService: PreferredClickhouseService,
+ ): ClickHouseSettings {
+ const serviceExtraSettings = {
+ ReadWrite: undefined,
+ ReadOnly: env.CLICKHOUSE_EXTRA_SETTINGS_READ_ONLY,
+ EventsReadOnly: env.CLICKHOUSE_EXTRA_SETTINGS_EVENTS_READ_ONLY,
+ }[preferredClickhouseService];
+
+ return {
+ ...env.CLICKHOUSE_EXTRA_SETTINGS,
+ ...serviceExtraSettings,
+ } as ClickHouseSettings;
+ }
+
private getRequestTimeoutClickHouseSettings(
requestTimeout?: number,
): RequestTimeoutClickHouseSettings {
@@ -186,10 +207,8 @@ export class ClickHouseClientManager {
opts: NodeClickHouseClientConfigOptions,
preferredClickhouseService: PreferredClickhouseService = "ReadWrite",
): ClickhouseClientType {
- const { settings, serviceClickhouseSettings } = this.generateClientSettings(
- opts,
- preferredClickhouseService,
- );
+ const { settings, extraClickhouseSettings, serviceClickhouseSettings } =
+ this.generateClientSettings(opts, preferredClickhouseService);
const key = JSON.stringify(settings);
if (!this.clientMap.has(key)) {
const activeSpan = getCurrentSpan();
@@ -216,6 +235,8 @@ export class ClickHouseClientManager {
level: mapLogLevel(env.LANGFUSE_LOG_LEVEL ?? "info"),
},
clickhouse_settings: {
+ // Operator-supplied extras first, so every setting below wins.
+ ...extraClickhouseSettings,
// Overwrite async insert settings to tune throughput
...(env.CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE
? {