🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(clickhouse): enable text index caches on the events read path (#17023)
Valery Meleshkin3c11bd91d8f0
diff --git a/.env.dev.example b/.env.dev.exampleindex 446a5f59d1..1384e1a3b9 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -290,9 +290,10 @@ LANGFUSE_DISABLE_LEGACY_TRACING_IO_SEARCH=falseCLICKHOUSE_USE_LIGHTWEIGHT_UPDATE="true"# Each `auto` detects its affected ClickHouse versions on startup. Set `true`-# to force a workaround or `false` to force-disable it.+# to force the setting on or `false` to force it off.CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION=autoCLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN=auto+CLICKHOUSE_ENABLE_SKIP_INDEXES_ON_DATA_READ=auto# ClickHouse Billing (CHB) integration (Langfuse Cloud only). First-time# upgrades on/after this UTC date (YYYY-MM-DD) route to CHB; unset = off.diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.tsindex 61f1b6bc67..430a75130d 100644--- a/packages/shared/src/env.ts+++ b/packages/shared/src/env.ts@@ -154,6 +154,11 @@ const EnvSchema = z.object({CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN: z.enum(["auto", "true", "false"]).default("auto"),+ // Read-time skip-index evaluation, applied only from the version that fixes+ // the patch-part read bug. See the compatibility rule for details.+ CLICKHOUSE_ENABLE_SKIP_INDEXES_ON_DATA_READ: z+ .enum(["auto", "true", "false"])+ .default("auto"),CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY: z.coerce.number().default(32_000_000_000), // ~32GBdiff --git a/packages/shared/src/server/clickhouse/client.ts b/packages/shared/src/server/clickhouse/client.tsindex e3d00b3c35..43b4a6e112 100644--- a/packages/shared/src/server/clickhouse/client.ts+++ b/packages/shared/src/server/clickhouse/client.ts@@ -114,7 +114,16 @@ export class ClickHouseClientManager {): ServiceClickhouseSettings {const eventROSettings: ServiceClickhouseSettings =preferredClickhouseService === "EventsReadOnly"- ? { enable_full_text_index: 1 }+ ? {+ enable_full_text_index: 1,+ // Text index caches, all default off on ClickHouse 25.12 (the+ // Langfuse v4 minimum). use_text_index_header_cache defaults on+ // from 26.7, and use_text_index_dictionary_cache is accepted but+ // ignored from 26.3.+ use_text_index_dictionary_cache: 1,+ use_text_index_header_cache: 1,+ use_text_index_postings_cache: 1,+ }: {};return {diff --git a/packages/shared/src/server/clickhouse/compatibility.test.ts b/packages/shared/src/server/clickhouse/compatibility.test.tsindex 31744173d3..a38400dfd4 100644--- a/packages/shared/src/server/clickhouse/compatibility.test.ts+++ b/packages/shared/src/server/clickhouse/compatibility.test.ts@@ -4,6 +4,7 @@ const envMock = vi.hoisted(() => ({env: {CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION: "auto",CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN: "auto",+ CLICKHOUSE_ENABLE_SKIP_INDEXES_ON_DATA_READ: "auto",},}));@@ -21,6 +22,7 @@ const disableLazyMaterialization = {query_plan_optimize_lazy_materialization: 0,};const disableTopKThroughJoin = { query_plan_top_k_through_join: 0 };+const enableSkipIndexesOnDataRead = { use_skip_indexes_on_data_read: 1 };describe("ClickHouse compatibility version parsing", () => {it("parses and compares ClickHouse build components", () => {@@ -59,32 +61,47 @@ describe("resolveClickHouseCompatibility", () => {// Patch-parts lower bound and upstream fix.["25.3.99.9999", noCompatibilitySettings],["25.4.0.0", disableLazyMaterialization],- ["26.4.1.1005", noCompatibilitySettings],+ ["26.4.1.1005", enableSkipIndexesOnDataRead],+ // Read-time skip-index evaluation, from the release that fixes the+ // patch-part read. It must stay off on 25.12, the Langfuse v4 minimum,+ // where it breaks reads of lightweight-update patch parts.+ ["25.12.11.4", disableLazyMaterialization],+ ["26.1.99.9999", disableLazyMaterialization],+ [+ "26.2.0.0",+ { ...disableLazyMaterialization, ...enableSkipIndexesOnDataRead },+ ],// Top-K introduction and the exact fix on each release line.- ["26.5.1.650", noCompatibilitySettings],- ["26.5.1.651", disableTopKThroughJoin],- ["26.5.6.70", noCompatibilitySettings],- ["26.6.0.0", disableTopKThroughJoin],- ["26.6.2.108", noCompatibilitySettings],- ["26.7.0.0", disableTopKThroughJoin],- ["26.7.1.1334", noCompatibilitySettings],- ["26.7.2.0", disableTopKThroughJoin],- ["26.7.2.11", noCompatibilitySettings],- ["26.7.99.9999", noCompatibilitySettings],- ["26.8.0.0", noCompatibilitySettings],- ["26.8.1.2041", noCompatibilitySettings],+ ["26.5.1.650", enableSkipIndexesOnDataRead],+ [+ "26.5.1.651",+ { ...disableTopKThroughJoin, ...enableSkipIndexesOnDataRead },+ ],+ ["26.5.6.70", enableSkipIndexesOnDataRead],+ ["26.6.0.0", { ...disableTopKThroughJoin, ...enableSkipIndexesOnDataRead }],+ ["26.6.2.108", enableSkipIndexesOnDataRead],+ ["26.7.0.0", { ...disableTopKThroughJoin, ...enableSkipIndexesOnDataRead }],+ ["26.7.1.1334", enableSkipIndexesOnDataRead],+ ["26.7.2.0", { ...disableTopKThroughJoin, ...enableSkipIndexesOnDataRead }],+ ["26.7.2.11", enableSkipIndexesOnDataRead],+ ["26.7.99.9999", enableSkipIndexesOnDataRead],+ ["26.8.0.0", enableSkipIndexesOnDataRead],+ ["26.8.1.2041", enableSkipIndexesOnDataRead],])("resolves automatic settings for %s", (version, expectedSettings) => {expect(resolveClickHouseCompatibility({ version }).settings).toEqual(expectedSettings,);});- it("reports both independently computed compatibility flags", () => {+ it("reports each independently computed compatibility flag", () => {const resolved = resolveClickHouseCompatibility({version: "26.5.5.8",});- expect(resolved.settings).toEqual(disableTopKThroughJoin);+ expect(resolved.settings).toEqual({+ ...disableTopKThroughJoin,+ ...enableSkipIndexesOnDataRead,+ });expect(resolved.flags).toEqual([expect.objectContaining({id: "disable-lazy-materialization-for-patch-parts",@@ -98,6 +115,12 @@ describe("resolveClickHouseCompatibility", () => {matchesVersionBand: true,applied: true,}),+ expect.objectContaining({+ id: "enable-skip-indexes-on-data-read",+ setting: "use_skip_indexes_on_data_read",+ matchesVersionBand: true,+ applied: true,+ }),]);});@@ -112,6 +135,7 @@ describe("resolveClickHouseCompatibility", () => {}).settings,).toEqual({...disableLazyMaterialization,+ ...enableSkipIndexesOnDataRead,});});});diff --git a/packages/shared/src/server/clickhouse/compatibility.ts b/packages/shared/src/server/clickhouse/compatibility.tsindex b6101817ac..4ffb022307 100644--- a/packages/shared/src/server/clickhouse/compatibility.ts+++ b/packages/shared/src/server/clickhouse/compatibility.ts@@ -25,7 +25,8 @@ export type ClickHouseVersionBand = {type ClickHouseCompatibilityEnvKey =| "CLICKHOUSE_DISABLE_LAZY_MATERIALIZATION"- | "CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN";+ | "CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN"+ | "CLICKHOUSE_ENABLE_SKIP_INDEXES_ON_DATA_READ";type ClickHouseCompatibilityEnvValue = "auto" | "true" | "false";@@ -87,6 +88,15 @@ const CLICKHOUSE_COMPATIBILITY_RULES: ClickHouseCompatibilityRule[] = [],overrideEnvKey: "CLICKHOUSE_DISABLE_TOP_K_THROUGH_JOIN",},+ {+ id: "enable-skip-indexes-on-data-read",+ setting: "use_skip_indexes_on_data_read",+ value: 1,+ reason:+ "Evaluate skip indexes at granule read time to cut query startup latency. Held back below 26.2, where reading a column patched by a lightweight update fails with `Not found column _part_offset in block`.",+ versionBands: [{ minInclusive: "26.2.0.0" }],+ overrideEnvKey: "CLICKHOUSE_ENABLE_SKIP_INDEXES_ON_DATA_READ",+ },];let detectedClickHouseVersion: string | null = null;