Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix: add ClickHouse Top-K-through-Join compatibility workaround (#16019)

Valery Meleshkincabbc2fa8f79
diff --git a/.env.dev.example b/.env.dev.example
index cf1eadefac..411d571e89 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -240,9 +240,10 @@ LANGFUSE_BACKGROUND_MIGRATION_V4_ENABLE_HISTORIC_BACKFILL=false
LANGFUSE_DISABLE_LEGACY_TRACING_IO_SEARCH=false
CLICKHOUSE_USE_LIGHTWEIGHT_UPDATE="true"
-# `auto` detects affected ClickHouse versions on startup. Set `true` to force
-# the workaround or `false` to force-disable it.
+# Each `auto` detects its affected ClickHouse versions on startup. Set `true`
+# to force a workaround or `false` to force-disable it.
CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION=auto
+CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN=auto
# ClickHouse Billing (CHB) integration (Langfuse Cloud only). First-time
# upgrades on/after this UTC date (YYYY-MM-DD) route to CHB; unset = off.
diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts
index f91c94d08b..5a98d60249 100644
--- a/packages/shared/src/env.ts
+++ b/packages/shared/src/env.ts
@@ -147,6 +147,9 @@ const EnvSchema = z.object({
CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: z
.enum(["auto", "true", "false"])
.default("auto"),
+ CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN: z
+ .enum(["auto", "true", "false"])
+ .default("auto"),
CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY: z.coerce
.number()
.default(32_000_000_000), // ~32GB
diff --git a/packages/shared/src/server/clickhouse/client.test.ts b/packages/shared/src/server/clickhouse/client.test.ts
index f71c32ae1a..71ad47f869 100644
--- a/packages/shared/src/server/clickhouse/client.test.ts
+++ b/packages/shared/src/server/clickhouse/client.test.ts
@@ -19,6 +19,7 @@ const mocks = vi.hoisted(() => ({
CLICKHOUSE_LIGHTWEIGHT_DELETE_MODE: "alter_update",
CLICKHOUSE_UPDATE_PARALLEL_MODE: "auto",
CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "auto",
+ CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN: "auto",
LANGFUSE_LOG_LEVEL: "error",
NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
},
@@ -45,11 +46,12 @@ describe("ClickHouseClientManager compatibility settings", () => {
mocks.createClient.mockReset();
mocks.createClient.mockReturnValue({ close: mocks.close });
mocks.env.CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION = "auto";
+ mocks.env.CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN = "auto";
setClickHouseCompatibilityVersionForTests(null);
});
it("applies resolved compatibility settings globally", () => {
- setClickHouseCompatibilityVersionForTests("26.5.1.882");
+ setClickHouseCompatibilityVersionForTests("26.5.5.8");
clickhouseClient();
@@ -57,29 +59,29 @@ describe("ClickHouseClientManager compatibility settings", () => {
expect(
mocks.createClient.mock.calls[0][0].clickhouse_settings,
).toMatchObject({
- query_plan_optimize_lazy_materialization: 0,
+ query_plan_top_k_through_join: 0,
});
});
it("lets explicit client settings override compatibility settings", () => {
- setClickHouseCompatibilityVersionForTests("26.5.1.882");
+ setClickHouseCompatibilityVersionForTests("26.5.5.8");
clickhouseClient({
clickhouse_settings: {
- query_plan_optimize_lazy_materialization: 1,
+ query_plan_top_k_through_join: 1,
} as ClickHouseSettings,
});
expect(mocks.createClient).toHaveBeenCalledTimes(1);
expect(
mocks.createClient.mock.calls[0][0].clickhouse_settings
- .query_plan_optimize_lazy_materialization,
+ .query_plan_top_k_through_join,
).toBe(1);
});
it("uses a new cached client key after compatibility settings change", () => {
clickhouseClient();
- setClickHouseCompatibilityVersionForTests("26.5.1.882");
+ setClickHouseCompatibilityVersionForTests("26.5.5.8");
clickhouseClient();
expect(mocks.createClient).toHaveBeenCalledTimes(2);
diff --git a/packages/shared/src/server/clickhouse/compatibility.test.ts b/packages/shared/src/server/clickhouse/compatibility.test.ts
index 0f86707e4d..2a836b93a4 100644
--- a/packages/shared/src/server/clickhouse/compatibility.test.ts
+++ b/packages/shared/src/server/clickhouse/compatibility.test.ts
@@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest";
const envMock = vi.hoisted(() => ({
env: {
CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "auto",
+ CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN: "auto",
},
}));
@@ -14,105 +15,97 @@ import {
resolveClickHouseCompatibility,
} from "./compatibility";
+const noCompatibilitySettings = {};
+const disableLazyMaterialization = {
+ query_plan_optimize_lazy_materialization: 0,
+};
+const disableTopKThroughJoin = { query_plan_top_k_through_join: 0 };
+
describe("ClickHouse compatibility version parsing", () => {
- it("parses ClickHouse versions with build components", () => {
+ it("parses and compares ClickHouse build components", () => {
expect(parseClickHouseVersion("26.5.1.882")).toMatchObject({
major: 26,
minor: 5,
patch: 1,
- tuple: [26, 5, 1],
+ build: 882,
+ tuple: [26, 5, 1, 882],
});
+
+ const band = {
+ minInclusive: "26.5.1.651",
+ maxExclusive: "26.5.6.70",
+ };
+
+ expect(isClickHouseVersionInBand("26.5.1.650", band)).toBe(false);
+ expect(isClickHouseVersionInBand("26.5.1.651", band)).toBe(true);
+ expect(isClickHouseVersionInBand("26.5.6.69", band)).toBe(true);
+ expect(isClickHouseVersionInBand("26.5.6.70", band)).toBe(false);
});
- it("parses ClickHouse versions with dotted vendor suffixes", () => {
+ it("parses dotted vendor suffixes after the build component", () => {
expect(parseClickHouseVersion("25.4.1.1.altinitystable")).toMatchObject({
major: 25,
minor: 4,
patch: 1,
- tuple: [25, 4, 1],
+ build: 1,
+ tuple: [25, 4, 1, 1],
});
});
-
- it("matches inclusive lower bounds", () => {
- const band = { minInclusive: "25.4.0" };
-
- expect(isClickHouseVersionInBand("25.3.99", band)).toBe(false);
- expect(isClickHouseVersionInBand("25.4.0", band)).toBe(true);
- expect(isClickHouseVersionInBand("25.4.1.1234", band)).toBe(true);
- expect(isClickHouseVersionInBand("25.4.1.1.altinitystable", band)).toBe(
- true,
- );
- expect(isClickHouseVersionInBand("26.5.1.882", band)).toBe(true);
- });
-
- it("matches exclusive upper bounds", () => {
- const band = { minInclusive: "25.4.0", maxExclusive: "26.4.0" };
-
- expect(isClickHouseVersionInBand("26.3.99", band)).toBe(true);
- expect(isClickHouseVersionInBand("26.4.0", band)).toBe(false);
- });
});
describe("resolveClickHouseCompatibility", () => {
- it("applies lazy materialization workaround in auto mode for affected versions", () => {
- const resolved = resolveClickHouseCompatibility({
- version: "26.5.1.882",
- overrides: { CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "auto" },
- });
-
- expect(resolved.settings).toEqual({
- query_plan_optimize_lazy_materialization: 0,
- });
- expect(resolved.parsedVersion).toMatchObject({
- major: 26,
- minor: 5,
- patch: 1,
- tuple: [26, 5, 1],
- });
- expect(resolved.flags).toEqual([
- expect.objectContaining({
- id: "disable-lazy-materialization",
- setting: "query_plan_optimize_lazy_materialization",
- value: 0,
- override: "auto",
- matchesVersionBand: true,
- applied: true,
- }),
- ]);
+ it.each([
+ // Patch-parts lower bound and upstream fix.
+ ["25.3.99.9999", noCompatibilitySettings],
+ ["25.4.0.0", disableLazyMaterialization],
+ ["26.4.1.1005", noCompatibilitySettings],
+ // Top-K introduction and the exact fix on each release line.
+ ["26.5.1.650", noCompatibilitySettings],
+ ["26.5.1.651", disableTopKThroughJoin],
+ ["26.5.6.70", noCompatibilitySettings],
+ ["26.6.0.0", disableTopKThroughJoin],
+ ["26.6.2.108", noCompatibilitySettings],
+ ["26.7.0.0", disableTopKThroughJoin],
+ ["26.7.1.1334", noCompatibilitySettings],
+ ["26.7.2.0", disableTopKThroughJoin],
+ ["26.7.2.11", noCompatibilitySettings],
+ ])("resolves automatic settings for %s", (version, expectedSettings) => {
+ expect(resolveClickHouseCompatibility({ version }).settings).toEqual(
+ expectedSettings,
+ );
});
- it("does not apply lazy materialization workaround in auto mode before the affected band", () => {
+ it("reports both independently computed compatibility flags", () => {
const resolved = resolveClickHouseCompatibility({
- version: "25.3.99",
- overrides: { CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "auto" },
+ version: "26.5.5.8",
});
- expect(resolved.settings).toEqual({});
+ expect(resolved.settings).toEqual(disableTopKThroughJoin);
expect(resolved.flags).toEqual([
expect.objectContaining({
- id: "disable-lazy-materialization",
- override: "auto",
+ id: "disable-lazy-materialization-for-patch-parts",
+ setting: "query_plan_optimize_lazy_materialization",
matchesVersionBand: false,
applied: false,
}),
+ expect.objectContaining({
+ id: "disable-top-k-through-join",
+ setting: "query_plan_top_k_through_join",
+ matchesVersionBand: true,
+ applied: true,
+ }),
]);
});
- it("allows forcing lazy materialization workaround on", () => {
- expect(
- resolveClickHouseCompatibility({
- version: null,
- overrides: { CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "true" },
- }).settings,
- ).toEqual({ query_plan_optimize_lazy_materialization: 0 });
- });
-
- it("allows forcing lazy materialization workaround off", () => {
+ it("applies compatibility overrides independently", () => {
expect(
resolveClickHouseCompatibility({
- version: "26.5.1.882",
- overrides: { CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "false" },
+ version: "26.5.5.8",
+ overrides: {
+ CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "true",
+ CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN: "false",
+ },
}).settings,
- ).toEqual({});
+ ).toEqual(disableLazyMaterialization);
});
});
diff --git a/packages/shared/src/server/clickhouse/compatibility.ts b/packages/shared/src/server/clickhouse/compatibility.ts
index 57e850d18d..295ee7eba5 100644
--- a/packages/shared/src/server/clickhouse/compatibility.ts
+++ b/packages/shared/src/server/clickhouse/compatibility.ts
@@ -4,21 +4,28 @@ import type { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/
import { VERSION } from "../../constants/VERSION";
import { env } from "../../env";
import { logger } from "../logger";
-import {
- compareParsedVersions,
- parseVersionString,
- type ParsedVersion,
-} from "../utils/compareVersions";
+import { compareParsedVersions } from "../utils/compareVersions";
import { ClickHouseLogger, mapLogLevel } from "./clickhouse-logger";
-export type ClickHouseVersion = ParsedVersion;
+type ClickHouseVersionTuple = readonly [number, number, number, number];
+
+export type ClickHouseVersion = {
+ raw: string;
+ major: number;
+ minor: number;
+ patch: number;
+ build: number;
+ tuple: ClickHouseVersionTuple;
+};
export type ClickHouseVersionBand = {
minInclusive: string;
maxExclusive?: string;
};
-type ClickHouseCompatibilityEnvKey = "CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION";
+type ClickHouseCompatibilityEnvKey =
+ | "CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION"
+ | "CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN";
type ClickHouseCompatibilityEnvValue = "auto" | "true" | "false";
@@ -58,14 +65,28 @@ type ResolvedClickHouseCompatibility = {
const CLICKHOUSE_COMPATIBILITY_RULES: ClickHouseCompatibilityRule[] = [
{
- id: "disable-lazy-materialization",
+ id: "disable-lazy-materialization-for-patch-parts",
setting: "query_plan_optimize_lazy_materialization",
value: 0,
reason:
- "Work around ClickHouse analyzer failures that can surface as `Not found column and(...)` on compound predicates.",
- versionBands: [{ minInclusive: "25.4.0" }],
+ "Work around ClickHouse #102904, where lazy materialization can lose `_block_number` while reading lightweight-update patch parts.",
+ versionBands: [{ minInclusive: "25.4.0.0", maxExclusive: "26.4.1.1005" }],
overrideEnvKey: "CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION",
},
+ {
+ id: "disable-top-k-through-join",
+ setting: "query_plan_top_k_through_join",
+ value: 0,
+ reason:
+ "Work around ClickHouse #109210, where top-K-through-join can leave lazy materialization with a dangling filter input.",
+ versionBands: [
+ { minInclusive: "26.5.1.651", maxExclusive: "26.5.6.70" },
+ { minInclusive: "26.6.0.0", maxExclusive: "26.6.2.108" },
+ { minInclusive: "26.7.0.0", maxExclusive: "26.7.1.1334" },
+ { minInclusive: "26.7.2.0", maxExclusive: "26.7.2.11" },
+ ],
+ overrideEnvKey: "CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN",
+ },
];
let detectedClickHouseVersion: string | null = null;
@@ -73,15 +94,42 @@ let initializationPromise: Promise<void> | null = null;
export const parseClickHouseVersion = (
rawVersion: string,
-): ClickHouseVersion | null => parseVersionString(rawVersion);
+): ClickHouseVersion | null => {
+ const match = rawVersion
+ .trim()
+ .match(/^v?(\d+)\.(\d+)\.(\d+)(?:\.(\d+))?(?:[.+-].+)?$/);
+ if (!match) return null;
+
+ const major = Number(match[1]);
+ const minor = Number(match[2]);
+ const patch = Number(match[3]);
+ const build = Number(match[4] ?? 0);
+
+ if (![major, minor, patch, build].every(Number.isSafeInteger)) return null;
+
+ return {
+ raw: rawVersion,
+ major,
+ minor,
+ patch,
+ build,
+ tuple: [major, minor, patch, build],
+ };
+};
+const parsedVersionBoundCache = new Map<string, ClickHouseVersion>();
const parseVersionBound = (version: string): ClickHouseVersion => {
+ const cached = parsedVersionBoundCache.get(version);
+ if (cached) return cached;
+
const parsed = parseClickHouseVersion(version);
if (!parsed) {
throw new Error(
`Invalid ClickHouse compatibility version bound: ${version}`,
);
}
+
+ parsedVersionBoundCache.set(version, parsed);
return parsed;
};
diff --git a/packages/shared/src/server/utils/compareVersions.ts b/packages/shared/src/server/utils/compareVersions.ts
index 8b1c0e8faa..e8c3fd4254 100644
--- a/packages/shared/src/server/utils/compareVersions.ts
+++ b/packages/shared/src/server/utils/compareVersions.ts
@@ -4,6 +4,10 @@ export const versionSchema = z.string().regex(/^v?\d+\.\d+\.\d+(?:[-+].+)?$/); /
type VersionTuple = readonly [number, number, number];
+type VersionTupleComparable = {
+ tuple: readonly number[];
+};
+
export type ParsedVersion = {
raw: string;
major: number;
@@ -36,8 +40,8 @@ export const parseVersionString = (
};
export const compareParsedVersions = (
- left: ParsedVersion,
- right: ParsedVersion,
+ left: VersionTupleComparable,
+ right: VersionTupleComparable,
): number => {
for (let i = 0; i < left.tuple.length; i++) {
if (left.tuple[i] > right.tuple[i]) return 1;