🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix: add ClickHouse Top-K-through-Join compatibility workaround (#16019)
Valery Meleshkincabbc2fa8f79
diff --git a/.env.dev.example b/.env.dev.exampleindex cf1eadefac..411d571e89 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -240,9 +240,10 @@ LANGFUSE_BACKGROUND_MIGRATION_V4_ENABLE_HISTORIC_BACKFILL=falseLANGFUSE_DISABLE_LEGACY_TRACING_IO_SEARCH=falseCLICKHOUSE_USE_LIGHTWEIGHT_UPDATE="true"-# `auto` detects affected ClickHouse versions on startup. Set `true` to force-# the workaround or `false` to force-disable it.+# Each `auto` detects its affected ClickHouse versions on startup. Set `true`+# to force a workaround or `false` to force-disable it.CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION=auto+CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN=auto# ClickHouse Billing (CHB) integration (Langfuse Cloud only). First-time# upgrades on/after this UTC date (YYYY-MM-DD) route to CHB; unset = off.diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.tsindex f91c94d08b..5a98d60249 100644--- a/packages/shared/src/env.ts+++ b/packages/shared/src/env.ts@@ -147,6 +147,9 @@ const EnvSchema = z.object({CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: z.enum(["auto", "true", "false"]).default("auto"),+ CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN: z+ .enum(["auto", "true", "false"])+ .default("auto"),CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY: z.coerce.number().default(32_000_000_000), // ~32GBdiff --git a/packages/shared/src/server/clickhouse/client.test.ts b/packages/shared/src/server/clickhouse/client.test.tsindex f71c32ae1a..71ad47f869 100644--- a/packages/shared/src/server/clickhouse/client.test.ts+++ b/packages/shared/src/server/clickhouse/client.test.ts@@ -19,6 +19,7 @@ const mocks = vi.hoisted(() => ({CLICKHOUSE_LIGHTWEIGHT_DELETE_MODE: "alter_update",CLICKHOUSE_UPDATE_PARALLEL_MODE: "auto",CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "auto",+ CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN: "auto",LANGFUSE_LOG_LEVEL: "error",NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,},@@ -45,11 +46,12 @@ describe("ClickHouseClientManager compatibility settings", () => {mocks.createClient.mockReset();mocks.createClient.mockReturnValue({ close: mocks.close });mocks.env.CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION = "auto";+ mocks.env.CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN = "auto";setClickHouseCompatibilityVersionForTests(null);});it("applies resolved compatibility settings globally", () => {- setClickHouseCompatibilityVersionForTests("26.5.1.882");+ setClickHouseCompatibilityVersionForTests("26.5.5.8");clickhouseClient();@@ -57,29 +59,29 @@ describe("ClickHouseClientManager compatibility settings", () => {expect(mocks.createClient.mock.calls[0][0].clickhouse_settings,).toMatchObject({- query_plan_optimize_lazy_materialization: 0,+ query_plan_top_k_through_join: 0,});});it("lets explicit client settings override compatibility settings", () => {- setClickHouseCompatibilityVersionForTests("26.5.1.882");+ setClickHouseCompatibilityVersionForTests("26.5.5.8");clickhouseClient({clickhouse_settings: {- query_plan_optimize_lazy_materialization: 1,+ query_plan_top_k_through_join: 1,} as ClickHouseSettings,});expect(mocks.createClient).toHaveBeenCalledTimes(1);expect(mocks.createClient.mock.calls[0][0].clickhouse_settings- .query_plan_optimize_lazy_materialization,+ .query_plan_top_k_through_join,).toBe(1);});it("uses a new cached client key after compatibility settings change", () => {clickhouseClient();- setClickHouseCompatibilityVersionForTests("26.5.1.882");+ setClickHouseCompatibilityVersionForTests("26.5.5.8");clickhouseClient();expect(mocks.createClient).toHaveBeenCalledTimes(2);diff --git a/packages/shared/src/server/clickhouse/compatibility.test.ts b/packages/shared/src/server/clickhouse/compatibility.test.tsindex 0f86707e4d..2a836b93a4 100644--- a/packages/shared/src/server/clickhouse/compatibility.test.ts+++ b/packages/shared/src/server/clickhouse/compatibility.test.ts@@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest";const envMock = vi.hoisted(() => ({env: {CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "auto",+ CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN: "auto",},}));@@ -14,105 +15,97 @@ import {resolveClickHouseCompatibility,} from "./compatibility";+const noCompatibilitySettings = {};+const disableLazyMaterialization = {+ query_plan_optimize_lazy_materialization: 0,+};+const disableTopKThroughJoin = { query_plan_top_k_through_join: 0 };+describe("ClickHouse compatibility version parsing", () => {- it("parses ClickHouse versions with build components", () => {+ it("parses and compares ClickHouse build components", () => {expect(parseClickHouseVersion("26.5.1.882")).toMatchObject({major: 26,minor: 5,patch: 1,- tuple: [26, 5, 1],+ build: 882,+ tuple: [26, 5, 1, 882],});++ const band = {+ minInclusive: "26.5.1.651",+ maxExclusive: "26.5.6.70",+ };++ expect(isClickHouseVersionInBand("26.5.1.650", band)).toBe(false);+ expect(isClickHouseVersionInBand("26.5.1.651", band)).toBe(true);+ expect(isClickHouseVersionInBand("26.5.6.69", band)).toBe(true);+ expect(isClickHouseVersionInBand("26.5.6.70", band)).toBe(false);});- it("parses ClickHouse versions with dotted vendor suffixes", () => {+ it("parses dotted vendor suffixes after the build component", () => {expect(parseClickHouseVersion("25.4.1.1.altinitystable")).toMatchObject({major: 25,minor: 4,patch: 1,- tuple: [25, 4, 1],+ build: 1,+ tuple: [25, 4, 1, 1],});});-- it("matches inclusive lower bounds", () => {- const band = { minInclusive: "25.4.0" };-- expect(isClickHouseVersionInBand("25.3.99", band)).toBe(false);- expect(isClickHouseVersionInBand("25.4.0", band)).toBe(true);- expect(isClickHouseVersionInBand("25.4.1.1234", band)).toBe(true);- expect(isClickHouseVersionInBand("25.4.1.1.altinitystable", band)).toBe(- true,- );- expect(isClickHouseVersionInBand("26.5.1.882", band)).toBe(true);- });-- it("matches exclusive upper bounds", () => {- const band = { minInclusive: "25.4.0", maxExclusive: "26.4.0" };-- expect(isClickHouseVersionInBand("26.3.99", band)).toBe(true);- expect(isClickHouseVersionInBand("26.4.0", band)).toBe(false);- });});describe("resolveClickHouseCompatibility", () => {- it("applies lazy materialization workaround in auto mode for affected versions", () => {- const resolved = resolveClickHouseCompatibility({- version: "26.5.1.882",- overrides: { CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "auto" },- });-- expect(resolved.settings).toEqual({- query_plan_optimize_lazy_materialization: 0,- });- expect(resolved.parsedVersion).toMatchObject({- major: 26,- minor: 5,- patch: 1,- tuple: [26, 5, 1],- });- expect(resolved.flags).toEqual([- expect.objectContaining({- id: "disable-lazy-materialization",- setting: "query_plan_optimize_lazy_materialization",- value: 0,- override: "auto",- matchesVersionBand: true,- applied: true,- }),- ]);+ it.each([+ // Patch-parts lower bound and upstream fix.+ ["25.3.99.9999", noCompatibilitySettings],+ ["25.4.0.0", disableLazyMaterialization],+ ["26.4.1.1005", noCompatibilitySettings],+ // Top-K introduction and the exact fix on each release line.+ ["26.5.1.650", noCompatibilitySettings],+ ["26.5.1.651", disableTopKThroughJoin],+ ["26.5.6.70", noCompatibilitySettings],+ ["26.6.0.0", disableTopKThroughJoin],+ ["26.6.2.108", noCompatibilitySettings],+ ["26.7.0.0", disableTopKThroughJoin],+ ["26.7.1.1334", noCompatibilitySettings],+ ["26.7.2.0", disableTopKThroughJoin],+ ["26.7.2.11", noCompatibilitySettings],+ ])("resolves automatic settings for %s", (version, expectedSettings) => {+ expect(resolveClickHouseCompatibility({ version }).settings).toEqual(+ expectedSettings,+ );});- it("does not apply lazy materialization workaround in auto mode before the affected band", () => {+ it("reports both independently computed compatibility flags", () => {const resolved = resolveClickHouseCompatibility({- version: "25.3.99",- overrides: { CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "auto" },+ version: "26.5.5.8",});- expect(resolved.settings).toEqual({});+ expect(resolved.settings).toEqual(disableTopKThroughJoin);expect(resolved.flags).toEqual([expect.objectContaining({- id: "disable-lazy-materialization",- override: "auto",+ id: "disable-lazy-materialization-for-patch-parts",+ setting: "query_plan_optimize_lazy_materialization",matchesVersionBand: false,applied: false,}),+ expect.objectContaining({+ id: "disable-top-k-through-join",+ setting: "query_plan_top_k_through_join",+ matchesVersionBand: true,+ applied: true,+ }),]);});- it("allows forcing lazy materialization workaround on", () => {- expect(- resolveClickHouseCompatibility({- version: null,- overrides: { CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "true" },- }).settings,- ).toEqual({ query_plan_optimize_lazy_materialization: 0 });- });-- it("allows forcing lazy materialization workaround off", () => {+ it("applies compatibility overrides independently", () => {expect(resolveClickHouseCompatibility({- version: "26.5.1.882",- overrides: { CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "false" },+ version: "26.5.5.8",+ overrides: {+ CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "true",+ CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN: "false",+ },}).settings,- ).toEqual({});+ ).toEqual(disableLazyMaterialization);});});diff --git a/packages/shared/src/server/clickhouse/compatibility.ts b/packages/shared/src/server/clickhouse/compatibility.tsindex 57e850d18d..295ee7eba5 100644--- a/packages/shared/src/server/clickhouse/compatibility.ts+++ b/packages/shared/src/server/clickhouse/compatibility.ts@@ -4,21 +4,28 @@ import type { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/import { VERSION } from "../../constants/VERSION";import { env } from "../../env";import { logger } from "../logger";-import {- compareParsedVersions,- parseVersionString,- type ParsedVersion,-} from "../utils/compareVersions";+import { compareParsedVersions } from "../utils/compareVersions";import { ClickHouseLogger, mapLogLevel } from "./clickhouse-logger";-export type ClickHouseVersion = ParsedVersion;+type ClickHouseVersionTuple = readonly [number, number, number, number];++export type ClickHouseVersion = {+ raw: string;+ major: number;+ minor: number;+ patch: number;+ build: number;+ tuple: ClickHouseVersionTuple;+};export type ClickHouseVersionBand = {minInclusive: string;maxExclusive?: string;};-type ClickHouseCompatibilityEnvKey = "CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION";+type ClickHouseCompatibilityEnvKey =+ | "CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION"+ | "CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN";type ClickHouseCompatibilityEnvValue = "auto" | "true" | "false";@@ -58,14 +65,28 @@ type ResolvedClickHouseCompatibility = {const CLICKHOUSE_COMPATIBILITY_RULES: ClickHouseCompatibilityRule[] = [{- id: "disable-lazy-materialization",+ id: "disable-lazy-materialization-for-patch-parts",setting: "query_plan_optimize_lazy_materialization",value: 0,reason:- "Work around ClickHouse analyzer failures that can surface as `Not found column and(...)` on compound predicates.",- versionBands: [{ minInclusive: "25.4.0" }],+ "Work around ClickHouse #102904, where lazy materialization can lose `_block_number` while reading lightweight-update patch parts.",+ versionBands: [{ minInclusive: "25.4.0.0", maxExclusive: "26.4.1.1005" }],overrideEnvKey: "CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION",},+ {+ id: "disable-top-k-through-join",+ setting: "query_plan_top_k_through_join",+ value: 0,+ reason:+ "Work around ClickHouse #109210, where top-K-through-join can leave lazy materialization with a dangling filter input.",+ versionBands: [+ { minInclusive: "26.5.1.651", maxExclusive: "26.5.6.70" },+ { minInclusive: "26.6.0.0", maxExclusive: "26.6.2.108" },+ { minInclusive: "26.7.0.0", maxExclusive: "26.7.1.1334" },+ { minInclusive: "26.7.2.0", maxExclusive: "26.7.2.11" },+ ],+ overrideEnvKey: "CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN",+ },];let detectedClickHouseVersion: string | null = null;@@ -73,15 +94,42 @@ let initializationPromise: Promise<void> | null = null;export const parseClickHouseVersion = (rawVersion: string,-): ClickHouseVersion | null => parseVersionString(rawVersion);+): ClickHouseVersion | null => {+ const match = rawVersion+ .trim()+ .match(/^v?(\d+)\.(\d+)\.(\d+)(?:\.(\d+))?(?:[.+-].+)?$/);+ if (!match) return null;++ const major = Number(match[1]);+ const minor = Number(match[2]);+ const patch = Number(match[3]);+ const build = Number(match[4] ?? 0);++ if (![major, minor, patch, build].every(Number.isSafeInteger)) return null;++ return {+ raw: rawVersion,+ major,+ minor,+ patch,+ build,+ tuple: [major, minor, patch, build],+ };+};+const parsedVersionBoundCache = new Map<string, ClickHouseVersion>();const parseVersionBound = (version: string): ClickHouseVersion => {+ const cached = parsedVersionBoundCache.get(version);+ if (cached) return cached;+const parsed = parseClickHouseVersion(version);if (!parsed) {throw new Error(`Invalid ClickHouse compatibility version bound: ${version}`,);}++ parsedVersionBoundCache.set(version, parsed);return parsed;};diff --git a/packages/shared/src/server/utils/compareVersions.ts b/packages/shared/src/server/utils/compareVersions.tsindex 8b1c0e8faa..e8c3fd4254 100644--- a/packages/shared/src/server/utils/compareVersions.ts+++ b/packages/shared/src/server/utils/compareVersions.ts@@ -4,6 +4,10 @@ export const versionSchema = z.string().regex(/^v?\d+\.\d+\.\d+(?:[-+].+)?$/); /type VersionTuple = readonly [number, number, number];+type VersionTupleComparable = {+ tuple: readonly number[];+};+export type ParsedVersion = {raw: string;major: number;@@ -36,8 +40,8 @@ export const parseVersionString = (};export const compareParsedVersions = (- left: ParsedVersion,- right: ParsedVersion,+ left: VersionTupleComparable,+ right: VersionTupleComparable,): number => {for (let i = 0; i < left.tuple.length; i++) {if (left.tuple[i] > right.tuple[i]) return 1;