🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(worker): make event propagation block sizes configurable (#17395)
Max Deichmann2ee19085513d
diff --git a/.env.dev.example b/.env.dev.exampleindex fdfed9a79f..101ee270b1 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -52,6 +52,17 @@ CLICKHOUSE_USER="clickhouse"CLICKHOUSE_PASSWORD="clickhouse"CLICKHOUSE_CLUSTER_ENABLED="false"+# Optional worker overrides for the event-propagation INSERT SELECT only.+# Leave unset to inherit ClickHouse profile settings. MAX_BLOCK_SIZE is a source+# row target; MIN_INSERT thresholds accumulate chunks and are not memory caps.+# Zero disables that minimum threshold. Smaller blocks may increase part/merge load.+# Change only between successfully completed partitions; changed block boundaries+# can alter deduplication of retries after a partially completed INSERT.+# LANGFUSE_EVENT_PROPAGATION_MAX_BLOCK_SIZE=+# LANGFUSE_EVENT_PROPAGATION_MIN_INSERT_BLOCK_SIZE_ROWS=+# LANGFUSE_EVENT_PROPAGATION_MIN_INSERT_BLOCK_SIZE_BYTES=++# Next Auth# You can generate a new secret on the command line with:# openssl rand -base64 32diff --git a/.env.prod.example b/.env.prod.exampleindex 548b531e7c..826b246598 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -258,6 +258,17 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# CLICKHOUSE_USER=# CLICKHOUSE_PASSWORD=# CLICKHOUSE_CLUSTER_ENABLED=true++# Optional worker overrides for the event-propagation INSERT SELECT only.+# Leave unset to inherit ClickHouse profile settings. MAX_BLOCK_SIZE is a source+# row target; MIN_INSERT thresholds accumulate chunks and are not memory caps.+# Zero disables that minimum threshold. Smaller blocks may increase part/merge load.+# Change only between successfully completed partitions; changed block boundaries+# can alter deduplication of retries after a partially completed INSERT.+# LANGFUSE_EVENT_PROPAGATION_MAX_BLOCK_SIZE=+# LANGFUSE_EVENT_PROPAGATION_MIN_INSERT_BLOCK_SIZE_ROWS=+# LANGFUSE_EVENT_PROPAGATION_MIN_INSERT_BLOCK_SIZE_BYTES=+# `auto` uses `no_throw` on ClickHouse 24.4+ and `sanitize` otherwise.# Defaults: v4=`no_throw`, v3=`auto`. Explicit `no_throw` requires 24.4+.# LANGFUSE_JSON_BAD_UNICODE_ESCAPE=autodiff --git a/worker/src/__tests__/eventPropagationSettings.test.ts b/worker/src/__tests__/eventPropagationSettings.test.tsnew file mode 100644index 0000000000..2e36dad52f--- /dev/null+++ b/worker/src/__tests__/eventPropagationSettings.test.ts@@ -0,0 +1,121 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ queryClickhouse: vi.fn(),+ commandClickhouse: vi.fn(),+ redisGet: vi.fn(),+ redisSet: vi.fn(),+ maxBlockSize: undefined as number | undefined,+ minInsertBlockSizeRows: undefined as number | undefined,+ minInsertBlockSizeBytes: undefined as number | undefined,+}));++vi.mock("@langfuse/shared/src/server", () => ({+ queryClickhouse: mocks.queryClickhouse,+ commandClickhouse: mocks.commandClickhouse,+ getCurrentSpan: vi.fn(() => undefined),+ logger: { info: vi.fn(), error: vi.fn() },+ traceException: vi.fn(),+ recordGauge: vi.fn(),+ redis: { get: mocks.redisGet, set: mocks.redisSet },+}));++vi.mock("../env", async (importOriginal) => {+ const mod = await importOriginal<typeof import("../env")>();+ return {+ ...mod,+ env: {+ ...mod.env,+ get LANGFUSE_EVENT_PROPAGATION_MAX_BLOCK_SIZE() {+ return mocks.maxBlockSize;+ },+ get LANGFUSE_EVENT_PROPAGATION_MIN_INSERT_BLOCK_SIZE_ROWS() {+ return mocks.minInsertBlockSizeRows;+ },+ get LANGFUSE_EVENT_PROPAGATION_MIN_INSERT_BLOCK_SIZE_BYTES() {+ return mocks.minInsertBlockSizeBytes;+ },+ },+ };+});++import { handleEventPropagationJob } from "../features/eventPropagation/handleEventPropagationJob";++const cursorKey = "langfuse:event-propagation:last-processed-partition";+const previousPartition = "2026-09-13 23:15:00";+const nextPartition = "2026-09-13 23:18:00";+const job = {+ data: { id: "event-propagation-settings-test" },+} as Parameters<typeof handleEventPropagationJob>[0];++describe("event propagation query settings", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ mocks.maxBlockSize = undefined;+ mocks.minInsertBlockSizeRows = undefined;+ mocks.minInsertBlockSizeBytes = undefined;+ mocks.queryClickhouse.mockResolvedValue([{ partition: nextPartition }]);+ mocks.commandClickhouse.mockResolvedValue(undefined);+ mocks.redisGet.mockResolvedValue(previousPartition);+ mocks.redisSet.mockResolvedValue("OK");+ });++ it("preserves ClickHouse block defaults when no overrides are configured", async () => {+ await handleEventPropagationJob(job);++ const settings =+ mocks.commandClickhouse.mock.calls[0]![0].clickhouseSettings;+ expect(settings).not.toHaveProperty("max_block_size");+ expect(settings).not.toHaveProperty("min_insert_block_size_rows");+ expect(settings).not.toHaveProperty("min_insert_block_size_bytes");+ expect(mocks.redisSet).toHaveBeenCalledWith(cursorKey, nextPartition);+ });++ it("applies block overrides only to the insert and preserves a zero row threshold", async () => {+ mocks.maxBlockSize = 256;+ mocks.minInsertBlockSizeRows = 0;+ mocks.minInsertBlockSizeBytes = 67_108_864;++ await handleEventPropagationJob(job);++ expect(mocks.commandClickhouse).toHaveBeenCalledWith(+ expect.objectContaining({+ clickhouseSettings: expect.objectContaining({+ max_block_size: "256",+ min_insert_block_size_rows: "0",+ min_insert_block_size_bytes: "67108864",+ }),+ }),+ );+ expect(mocks.queryClickhouse).toHaveBeenCalledTimes(1);+ expect(mocks.queryClickhouse.mock.calls[0]![0]).not.toHaveProperty(+ "clickhouseSettings",+ );+ expect(mocks.redisSet).toHaveBeenCalledWith(cursorKey, nextPartition);+ });++ it("preserves an explicit zero byte threshold independently of the other overrides", async () => {+ mocks.minInsertBlockSizeBytes = 0;++ await handleEventPropagationJob(job);++ const settings =+ mocks.commandClickhouse.mock.calls[0]![0].clickhouseSettings;+ expect(settings).toHaveProperty("min_insert_block_size_bytes", "0");+ expect(settings).not.toHaveProperty("max_block_size");+ expect(settings).not.toHaveProperty("min_insert_block_size_rows");+ });++ it("keeps the partition cursor unchanged when the configured insert fails", async () => {+ mocks.maxBlockSize = 256;+ mocks.minInsertBlockSizeRows = 0;+ mocks.minInsertBlockSizeBytes = 67_108_864;+ const failure = new Error("ClickHouse insert failed");+ mocks.commandClickhouse.mockRejectedValueOnce(failure);++ await expect(handleEventPropagationJob(job)).rejects.toBe(failure);++ expect(mocks.redisSet).not.toHaveBeenCalledWith(cursorKey, nextPartition);+ expect(mocks.redisSet).toHaveBeenCalledTimes(1);+ });+});diff --git a/worker/src/env.ts b/worker/src/env.tsindex ef41a6834c..4cae45be30 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -386,6 +386,23 @@ const EnvSchema = z.object({.positive().default(8),+ // Optional propagation-only overrides; unset values use ClickHouse profile settings.+ LANGFUSE_EVENT_PROPAGATION_MAX_BLOCK_SIZE: z.coerce+ .number()+ .int()+ .positive()+ .optional(),+ LANGFUSE_EVENT_PROPAGATION_MIN_INSERT_BLOCK_SIZE_ROWS: z.coerce+ .number()+ .int()+ .nonnegative()+ .optional(),+ LANGFUSE_EVENT_PROPAGATION_MIN_INSERT_BLOCK_SIZE_BYTES: z.coerce+ .number()+ .int()+ .nonnegative()+ .optional(),+// Core data S3 upload - Langfuse CloudLANGFUSE_S3_CORE_DATA_EXPORT_IS_ENABLED: z.enum(["true", "false"])diff --git a/worker/src/features/eventPropagation/handleEventPropagationJob.ts b/worker/src/features/eventPropagation/handleEventPropagationJob.tsindex aa1f717a4a..3fd1d5bc37 100644--- a/worker/src/features/eventPropagation/handleEventPropagationJob.ts+++ b/worker/src/features/eventPropagation/handleEventPropagationJob.ts@@ -345,6 +345,21 @@ export const handleEventPropagationJob = async (max_insert_threads: String(env.LANGFUSE_EVENT_PROPAGATION_MAX_INSERT_THREADS,),+ ...(env.LANGFUSE_EVENT_PROPAGATION_MAX_BLOCK_SIZE !== undefined && {+ max_block_size: String(env.LANGFUSE_EVENT_PROPAGATION_MAX_BLOCK_SIZE),+ }),+ ...(env.LANGFUSE_EVENT_PROPAGATION_MIN_INSERT_BLOCK_SIZE_ROWS !==+ undefined && {+ min_insert_block_size_rows: String(+ env.LANGFUSE_EVENT_PROPAGATION_MIN_INSERT_BLOCK_SIZE_ROWS,+ ),+ }),+ ...(env.LANGFUSE_EVENT_PROPAGATION_MIN_INSERT_BLOCK_SIZE_BYTES !==+ undefined && {+ min_insert_block_size_bytes: String(+ env.LANGFUSE_EVENT_PROPAGATION_MIN_INSERT_BLOCK_SIZE_BYTES,+ ),+ }),type_json_skip_duplicated_paths: true,},});