Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix: add round robin for clickhouse client

Carl-Gerhard Lindesvärd6c9eb05a810b
diff --git a/packages/db/package.json b/packages/db/package.json
index fb4cd393e..634a264b1 100644
--- a/packages/db/package.json
+++ b/packages/db/package.json
@@ -13,7 +13,7 @@
"with-env": "dotenv -e ../../.env -c --"
},
"dependencies": {
- "@clickhouse/client": "^1.12.1",
+ "@clickhouse/client": "^1.18.5",
"@openpanel/common": "workspace:*",
"@openpanel/constants": "workspace:*",
"@openpanel/json": "workspace:*",
@@ -42,4 +42,4 @@
"prisma": "^6.14.0",
"typescript": "catalog:"
}
-}
+}
\ No newline at end of file
diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/db/src/buffers/base-buffer.ts
index a50ecf26a..0d092cf4e 100644
--- a/packages/db/src/buffers/base-buffer.ts
+++ b/packages/db/src/buffers/base-buffer.ts
@@ -1,5 +1,6 @@
+import type { ClickHouseSettings } from '@clickhouse/client';
import { generateSecureId } from '@openpanel/common/server';
-import { type ILogger, createLogger } from '@openpanel/logger';
+import { createLogger, type ILogger } from '@openpanel/logger';
import { cronQueue } from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
@@ -78,6 +79,18 @@ export class BaseBuffer {
skipReason?: string;
} = {};
+ protected getClickhouseSettings(): ClickHouseSettings {
+ if (process.env.BUFFER_ASYNC_INSERTS) {
+ return {
+ async_insert: 1,
+ wait_for_async_insert: 0,
+ parallel_view_processing: 1,
+ };
+ }
+
+ return {};
+ }
+
/** Optional hook used by the worker to bridge flushes into Prometheus. */
public flushObserver: FlushObserver | null = null;
@@ -102,9 +115,7 @@ export class BaseBuffer {
* observability. Subclasses MUST override.
*/
protected getRedisListKey(): string {
- throw new Error(
- `${this.name}: subclass must override getRedisListKey()`,
- );
+ throw new Error(`${this.name}: subclass must override getRedisListKey()`);
}
protected chunks<T>(items: T[], size: number) {
@@ -135,9 +146,11 @@ export class BaseBuffer {
protected async parallelLimit<T, R>(
items: T[],
fn: (item: T, index: number) => Promise<R>,
- concurrency: number = this.chInsertConcurrency,
+ concurrency: number = this.chInsertConcurrency
): Promise<R[]> {
- if (items.length === 0) return [];
+ if (items.length === 0) {
+ return [];
+ }
if (concurrency <= 1 || items.length === 1) {
const out: R[] = [];
for (let i = 0; i < items.length; i++) {
@@ -150,13 +163,15 @@ export class BaseBuffer {
const worker = async (): Promise<void> => {
while (true) {
const idx = nextIndex++;
- if (idx >= items.length) return;
+ if (idx >= items.length) {
+ return;
+ }
results[idx] = await fn(items[idx] as T, idx);
}
};
const workers = Array.from(
{ length: Math.min(concurrency, items.length) },
- () => worker(),
+ () => worker()
);
await Promise.all(workers);
return results;
@@ -247,6 +262,7 @@ export class BaseBuffer {
}
const logPayload = {
+ buffer: obs.buffer,
result: obs.result,
trigger: obs.trigger,
totalMs: Math.round(obs.totalMs),
@@ -265,9 +281,7 @@ export class BaseBuffer {
? Math.round(obs.phases.chInsertMs)
: undefined,
trimMs:
- obs.phases?.trimMs != null
- ? Math.round(obs.phases.trimMs)
- : undefined,
+ obs.phases?.trimMs != null ? Math.round(obs.phases.trimMs) : undefined,
onFlushMs:
obs.phases?.onFlushMs != null
? Math.round(obs.phases.onFlushMs)
@@ -275,12 +289,20 @@ export class BaseBuffer {
};
if (obs.result === 'error') {
- this.logger.error({ ...logPayload, err: obs.err }, 'Buffer flush');
+ this.logger.error(
+ { ...logPayload, err: obs.err },
+ `Flush failed for ${this.name}`
+ );
} else if (obs.result === 'paused') {
// Demoted to debug — happens every 10s when intentionally paused
- this.logger.debug(logPayload, 'Buffer flush skipped (cron paused)');
+ this.logger.debug(
+ logPayload,
+ `Flush skipped for ${this.name} (cron paused)`
+ );
+ } else if (obs.result === 'locked') {
+ this.logger.debug(logPayload, `Flush skipped for ${this.name} (locked)`);
} else {
- this.logger.info(logPayload, 'Buffer flush');
+ this.logger.info(logPayload, `Flush completed for ${this.name}`);
}
}
@@ -348,7 +370,7 @@ export class BaseBuffer {
lockId,
'EX',
this.lockTimeout,
- 'NX',
+ 'NX'
);
if (acquired !== 'OK') {
diff --git a/packages/db/src/buffers/bot-buffer.ts b/packages/db/src/buffers/bot-buffer.ts
index c54092cd6..7c645a9aa 100644
--- a/packages/db/src/buffers/bot-buffer.ts
+++ b/packages/db/src/buffers/bot-buffer.ts
@@ -65,11 +65,7 @@ export class BotBuffer extends BaseBuffer {
table: TABLE_NAMES.events_bots,
values: parsedEvents,
format: 'JSONEachRow',
- clickhouse_settings: {
- async_insert: 1,
- wait_for_async_insert: 0,
- parallel_view_processing: 1,
- },
+ clickhouse_settings: this.getClickhouseSettings(),
});
const chInsertMs = performance.now() - chStart;
diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.ts
index 517d937d7..9b7e4b30a 100644
--- a/packages/db/src/buffers/event-buffer.ts
+++ b/packages/db/src/buffers/event-buffer.ts
@@ -176,11 +176,7 @@ export class EventBuffer extends BaseBuffer {
table: 'events',
values: chunk,
format: 'JSONEachRow',
- clickhouse_settings: {
- async_insert: 1,
- wait_for_async_insert: 0,
- parallel_view_processing: 1,
- },
+ clickhouse_settings: this.getClickhouseSettings(),
})
);
const chInsertMs = performance.now() - chStart;
diff --git a/packages/db/src/buffers/group-buffer.ts b/packages/db/src/buffers/group-buffer.ts
index 5b8138efe..d42e08f79 100644
--- a/packages/db/src/buffers/group-buffer.ts
+++ b/packages/db/src/buffers/group-buffer.ts
@@ -184,11 +184,7 @@ export class GroupBuffer extends BaseBuffer {
table: TABLE_NAMES.groups,
values: chunk,
format: 'JSONEachRow',
- clickhouse_settings: {
- async_insert: 1,
- wait_for_async_insert: 0,
- parallel_view_processing: 1,
- },
+ clickhouse_settings: this.getClickhouseSettings(),
})
);
const chInsertMs = performance.now() - chStart;
diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.ts
index 2d7b25cde..e90d3fb66 100644
--- a/packages/db/src/buffers/profile-buffer.ts
+++ b/packages/db/src/buffers/profile-buffer.ts
@@ -341,11 +341,7 @@ export class ProfileBuffer extends BaseBuffer {
table: TABLE_NAMES.profiles,
values: chunk,
format: 'JSONEachRow',
- clickhouse_settings: {
- async_insert: 1,
- wait_for_async_insert: 0,
- parallel_view_processing: 1,
- },
+ clickhouse_settings: this.getClickhouseSettings(),
})
);
const chInsertMs = performance.now() - chStart;
diff --git a/packages/db/src/buffers/replay-buffer.ts b/packages/db/src/buffers/replay-buffer.ts
index a3e33f2c2..7c3aecc56 100644
--- a/packages/db/src/buffers/replay-buffer.ts
+++ b/packages/db/src/buffers/replay-buffer.ts
@@ -82,11 +82,7 @@ export class ReplayBuffer extends BaseBuffer {
table: TABLE_NAMES.session_replay_chunks,
values: chunk,
format: 'JSONEachRow',
- clickhouse_settings: {
- async_insert: 1,
- wait_for_async_insert: 0,
- parallel_view_processing: 1,
- },
+ clickhouse_settings: this.getClickhouseSettings(),
})
);
const chInsertMs = performance.now() - chStart;
diff --git a/packages/db/src/buffers/session-buffer.ts b/packages/db/src/buffers/session-buffer.ts
index d2a6e890d..7a7240783 100644
--- a/packages/db/src/buffers/session-buffer.ts
+++ b/packages/db/src/buffers/session-buffer.ts
@@ -390,11 +390,7 @@ export class SessionBuffer extends BaseBuffer {
table: TABLE_NAMES.sessions,
values: chunk,
format: 'JSONEachRow',
- clickhouse_settings: {
- async_insert: 1,
- wait_for_async_insert: 0,
- parallel_view_processing: 1,
- },
+ clickhouse_settings: this.getClickhouseSettings(),
})
);
const chInsertMs = performance.now() - chStart;
diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.ts
index 0f78035f3..1277f6d61 100644
--- a/packages/db/src/clickhouse/client.ts
+++ b/packages/db/src/clickhouse/client.ts
@@ -1,9 +1,14 @@
-import type { ClickHouseSettings, ResponseJSON } from '@clickhouse/client';
+import type {
+ ClickHouseClient,
+ ClickHouseSettings,
+ ResponseJSON,
+} from '@clickhouse/client';
import { ClickHouseLogLevel, createClient } from '@clickhouse/client';
import type { NodeClickHouseClientConfigOptions } from '@clickhouse/client/dist/config';
import { createLogger } from '@openpanel/logger';
import type { IInterval } from '@openpanel/validation';
import sqlstring from 'sqlstring';
+import { RoundRobinPicker, withRoundRobinRetry } from './round-robin';
export { createClient };
@@ -128,13 +133,30 @@ const requestCompressionEnabled =
const maxOpenConnections = process.env.CLICKHOUSE_MAX_OPEN_CONNECTIONS
? Math.max(
1,
- Number.parseInt(process.env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, 10),
+ Number.parseInt(process.env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, 10)
)
: 50;
+// Per-request timeout. Was 300_000 (5 min) — way too long for fast failover
+// against a dead node, and after the Hetzner LB story we want stuck inserts
+// to fail and retry on a different node well before 5 min. 30s is enough for
+// any legitimate batch insert in this codebase. Bumpable via env if needed.
+const requestTimeoutMs = process.env.CLICKHOUSE_REQUEST_TIMEOUT_MS
+ ? Math.max(
+ 1000,
+ Number.parseInt(process.env.CLICKHOUSE_REQUEST_TIMEOUT_MS, 10)
+ )
+ : 30_000;
+
+// How long to skip a node after a connection failure, before trying it
+// again. Short by default — we want fast recovery once a node comes back.
+const unhealthyMarkMs = process.env.CLICKHOUSE_UNHEALTHY_MARK_MS
+ ? Math.max(0, Number.parseInt(process.env.CLICKHOUSE_UNHEALTHY_MARK_MS, 10))
+ : 5000;
+
export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {
max_open_connections: maxOpenConnections,
- request_timeout: 300_000,
+ request_timeout: requestTimeoutMs,
keep_alive: {
enabled: true,
// Must be lower than server-side keep_alive_timeout (default 10s) so we
@@ -151,100 +173,150 @@ export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {
},
};
-logger.info({ options: CLICKHOUSE_OPTIONS }, 'Clickhouse options');
+// CLICKHOUSE_URL accepts a single URL or comma-separated list. With a list,
+// we round-robin across nodes ourselves and fail over on connection errors —
+// avoids putting an L4 LB on the hot path.
+//
+// Tests sometimes import this module without CLICKHOUSE_URL set; in that
+// case we fall back to a single client created the same way the old code
+// did (URL = undefined, delegated to the client library's own handling).
+const rawClickhouseUrls = (process.env.CLICKHOUSE_URL ?? '')
+ .split(',')
+ .map((s) => s.trim())
+ .filter(Boolean);
+
+const clickhouseUrls = rawClickhouseUrls.length > 0 ? rawClickhouseUrls : [''];
+
+const clients: ClickHouseClient[] = clickhouseUrls.map((url) =>
+ createClient({
+ url: url || process.env.CLICKHOUSE_URL,
+ ...CLICKHOUSE_OPTIONS,
+ })
+);
+
+const picker = new RoundRobinPicker(clients, clickhouseUrls, unhealthyMarkMs);
+
+function maskUrlCredentials(url: string): string {
+ if (!url) return '<unset>';
+ try {
+ const u = new URL(url);
+ if (u.username) {
+ u.username = u.username.slice(0, 2) + '*'.repeat(u.username.length - 2);
+ }
+ if (u.password) {
+ u.password = '***';
+ }
+ return u.toString();
+ } catch {
+ // Malformed URL — fall back to the original; better than crashing module
+ // load in test envs where CLICKHOUSE_URL may be empty/garbage.
+ return url;
+ }
+}
+
+logger.info(
+ {
+ nodeCount: clients.length,
+ urls: clickhouseUrls.map(maskUrlCredentials),
+ requestTimeoutMs,
+ unhealthyMarkMs,
+ options: { ...CLICKHOUSE_OPTIONS, log: undefined },
+ },
+ 'ClickHouse clients initialized'
+);
-export const originalCh = createClient({
- url: process.env.CLICKHOUSE_URL,
- ...CLICKHOUSE_OPTIONS,
-});
+// Backwards-compat export. Some callers (notably gsc.ts) use `originalCh`
+// directly to bypass the retry/round-robin layer for one-off DDL or for
+// places where retry semantics aren't wanted. Points at the first node.
+export const originalCh = clients[0]!;
const cleanQuery = (query?: string) =>
typeof query === 'string'
? query.replace(/\n/g, '').replace(/\s+/g, ' ').trim()
: undefined;
+/**
+ * Run `operation` against the next round-robin node, retrying on the next
+ * node when a connection error is hit. Non-connection errors (SQL, auth,
+ * 4xx) propagate immediately.
+ *
+ * `operation` receives the picked client and a context with the node URL
+ * (useful for adding `host` to log lines / metrics).
+ */
export async function withRetry<T>(
- operation: () => Promise<T>,
- maxRetries = 3,
- baseDelay = 500
+ operation: (
+ client: ClickHouseClient,
+ ctx: { url: string; index: number },
+ ) => Promise<T>,
): Promise<T> {
- let lastError: Error | undefined;
-
- for (let attempt = 0; attempt < maxRetries; attempt++) {
- try {
- const res = await operation();
- if (attempt > 0) {
- logger.info({ attempt }, 'Retry operation succeeded');
- }
- return res;
- } catch (error: any) {
- lastError = error;
-
- if (
- error.message.includes('Connect') ||
- error.message.includes('socket hang up') ||
- error.message.includes('Timeout error')
- ) {
- const delay = baseDelay * 2 ** attempt;
- logger.warn(
- { err: error },
- `Attempt ${attempt + 1}/${maxRetries} failed, retrying in ${delay}ms`
- );
- await new Promise((resolve) => setTimeout(resolve, delay));
- continue;
- }
-
- throw error; // Non-retriable error
- }
- }
+ return withRoundRobinRetry(picker, operation, logger);
+}
- throw lastError;
+/** Best-effort URL → hostname extraction for log labels. */
+function urlHostname(url: string | undefined): string | undefined {
+ if (!url) return undefined;
+ try {
+ return new URL(url).hostname;
+ } catch {
+ return undefined;
+ }
}
-export const ch = new Proxy(originalCh, {
- get(target, property, receiver) {
- const value = Reflect.get(target, property, receiver);
-
- if (property === 'insert') {
- return (...args: any[]) =>
- withRetry(() => {
- args[0].clickhouse_settings = {
- // Increase insert timeouts and buffer sizes for large batches
- max_execution_time: 300,
- max_insert_block_size: '500000',
- max_http_get_redirects: '0',
- // Ensure JSONEachRow stays efficient
- input_format_parallel_parsing: 1,
- // Keep long-running inserts/queries from idling out at proxies by sending progress headers
- send_progress_in_http_headers: 1,
- http_headers_progress_interval_ms: '50000',
- // Ensure server holds the connection until the query is finished
- wait_end_of_query: 1,
- ...args[0].clickhouse_settings,
- };
- return value.apply(target, args);
- });
- }
+const INSERT_DEFAULT_SETTINGS: ClickHouseSettings = {
+ // Increase insert timeouts and buffer sizes for large batches
+ max_execution_time: 300,
+ max_insert_block_size: '500000',
+ max_http_get_redirects: '0',
+ // Ensure JSONEachRow stays efficient
+ input_format_parallel_parsing: 1,
+ // Keep long-running inserts/queries from idling out at proxies by sending progress headers
+ send_progress_in_http_headers: 1,
+ http_headers_progress_interval_ms: '50000',
+ // Ensure server holds the connection until the query is finished
+ wait_end_of_query: 1,
+};
- if (property === 'command') {
- return (...args: any[]) =>
- withRetry(() => {
- return value.apply(target, args);
- });
- }
+/**
+ * High-level CH client. For insert/command/query, picks a node round-robin
+ * and retries on a *different* node if the connection fails. The methods
+ * live as own-properties on the proxy target so `vi.spyOn(ch, 'insert')`
+ * works in tests. Other method accesses (rare — config readers, instance
+ * checks) fall through to the first client.
+ */
+const chTarget = {
+ insert: (params: any) =>
+ withRetry((client) => {
+ params.clickhouse_settings = {
+ ...INSERT_DEFAULT_SETTINGS,
+ ...params.clickhouse_settings,
+ };
+ return client.insert(params);
+ }),
+ command: (params: any) => withRetry((client) => client.command(params)),
+ query: (params: any) => withRetry((client) => client.query(params)),
+};
- return value;
+export const ch = new Proxy(chTarget as unknown as ClickHouseClient, {
+ get(target, property, receiver) {
+ if (property in target) {
+ return Reflect.get(target, property, receiver);
+ }
+ return (originalCh as any)[property];
},
-});
+}) as ClickHouseClient;
export async function chQueryWithMeta<T extends Record<string, any>>(
query: string,
clickhouseSettings?: ClickHouseSettings
): Promise<ResponseJSON<T>> {
const start = Date.now();
- const res = await ch.query({
- query,
- clickhouse_settings: clickhouseSettings,
+ let host: string | undefined;
+ const res = await withRetry((client, ctx) => {
+ host = urlHostname(ctx.url);
+ return client.query({
+ query,
+ clickhouse_settings: clickhouseSettings,
+ });
});
const json = await res.json<T>();
const keys = Object.keys(json.data[0] || {});
@@ -266,6 +338,7 @@ export async function chQueryWithMeta<T extends Record<string, any>>(
logger.info(
{
+ host,
query: cleanQuery(query),
rows: json.rows,
stats: response.statistics,
diff --git a/packages/db/src/clickhouse/round-robin.test.ts b/packages/db/src/clickhouse/round-robin.test.ts
new file mode 100644
index 000000000..d9e2fad16
--- /dev/null
+++ b/packages/db/src/clickhouse/round-robin.test.ts
@@ -0,0 +1,322 @@
+import { describe, expect, it } from 'vitest';
+import {
+ type ErrorClass,
+ classifyError,
+ isRetriableConnectionError,
+} from './round-robin';
+
+/**
+ * The contract: anything from the ClickHouse server (errors that traveled
+ * back over a working connection) must NOT mark a node unhealthy. Only true
+ * transport failures should.
+ *
+ * If you ever see a node getting incorrectly sin-binned in prod, add the
+ * exact error message here as a `not retriable` case and update
+ * isRetriableConnectionError to handle it.
+ */
+describe('isRetriableConnectionError', () => {
+ describe('ClickHouse server errors — must NOT be retried', () => {
+ const cases: Array<{ label: string; err: unknown }> = [
+ {
+ label: 'SQL syntax error',
+ err: new Error(
+ 'Code: 62, DB::Exception: Syntax error: failed at position 42',
+ ),
+ },
+ {
+ label: 'Unknown identifier (e.g. wrong column)',
+ err: new Error(
+ "Code: 47, DB::Exception: Unknown identifier 'fooooo'",
+ ),
+ },
+ {
+ label: 'ILLEGAL_AGGREGATION',
+ err: new Error(
+ 'Code: 184, DB::Exception: Aggregate function max(...) is found in WHERE',
+ ),
+ },
+ {
+ label: 'Missing columns',
+ err: new Error(
+ 'Code: 47, DB::Exception: Missing columns: a, b while processing query',
+ ),
+ },
+ {
+ label: 'TOO_MANY_PARTS — back-pressure, not connectivity',
+ err: new Error('Code: 252, DB::Exception: Too many parts in partition'),
+ },
+ {
+ label: 'MEMORY_LIMIT_EXCEEDED',
+ err: new Error(
+ 'Code: 241, DB::Exception: Memory limit (for query) exceeded',
+ ),
+ },
+ {
+ label: 'TIMEOUT_EXCEEDED — server-side query timeout',
+ err: new Error(
+ 'Code: 159, DB::Exception: Timeout exceeded: elapsed 60.001 sec',
+ ),
+ },
+ {
+ label: 'Authentication denied',
+ err: new Error('Code: 516, DB::Exception: Authentication failed'),
+ },
+ {
+ label: 'CH error with DB::Exception type marker (no Code prefix)',
+ err: Object.assign(new Error('some message'), {
+ type: 'DB::Exception',
+ }),
+ },
+ {
+ label: 'CH error whose body happens to mention "connect"',
+ err: new Error(
+ 'Code: 999, DB::Exception: Cannot connect to ZooKeeper',
+ ),
+ },
+ ];
+
+ for (const { label, err } of cases) {
+ it(`is NOT retriable: ${label}`, () => {
+ expect(isRetriableConnectionError(err)).toBe(false);
+ });
+ }
+ });
+
+ describe('Transport errors — must be retried on next node', () => {
+ const cases: Array<{ label: string; err: unknown }> = [
+ {
+ label: 'ECONNREFUSED via err.code',
+ err: Object.assign(new Error('connect ECONNREFUSED 10.1.0.18:8123'), {
+ code: 'ECONNREFUSED',
+ }),
+ },
+ {
+ label: 'ECONNRESET via err.code',
+ err: Object.assign(new Error('read ECONNRESET'), {
+ code: 'ECONNRESET',
+ }),
+ },
+ {
+ label: 'ETIMEDOUT via err.code',
+ err: Object.assign(new Error('connect ETIMEDOUT 10.1.0.20:8123'), {
+ code: 'ETIMEDOUT',
+ }),
+ },
+ {
+ label: 'EHOSTUNREACH via err.code',
+ err: Object.assign(new Error('No route to host'), {
+ code: 'EHOSTUNREACH',
+ }),
+ },
+ {
+ label: 'EAI_AGAIN (transient DNS)',
+ err: Object.assign(
+ new Error('getaddrinfo EAI_AGAIN ch.internal'),
+ { code: 'EAI_AGAIN' },
+ ),
+ },
+ {
+ label: 'EPIPE on broken socket',
+ err: Object.assign(new Error('write EPIPE'), { code: 'EPIPE' }),
+ },
+ {
+ label: 'undici connect timeout',
+ err: Object.assign(new Error('Connect Timeout Error'), {
+ code: 'UND_ERR_CONNECT_TIMEOUT',
+ }),
+ },
+ {
+ label: 'undici headers timeout',
+ err: Object.assign(new Error('Headers Timeout Error'), {
+ code: 'UND_ERR_HEADERS_TIMEOUT',
+ }),
+ },
+ {
+ label: 'undici body timeout',
+ err: Object.assign(new Error('Body Timeout Error'), {
+ code: 'UND_ERR_BODY_TIMEOUT',
+ }),
+ },
+ {
+ label: 'plain socket hang up (no code)',
+ err: new Error('socket hang up'),
+ },
+ {
+ label: 'undici premature close',
+ err: new Error('Premature close'),
+ },
+ {
+ label: 'cause-chained ECONNREFUSED',
+ err: Object.assign(new Error('Failed to fetch'), {
+ cause: Object.assign(new Error('connect ECONNREFUSED'), {
+ code: 'ECONNREFUSED',
+ }),
+ }),
+ },
+ {
+ label: 'deeply nested cause chain',
+ err: Object.assign(new Error('outer wrapper'), {
+ cause: Object.assign(new Error('middle wrapper'), {
+ cause: Object.assign(new Error('inner socket hang up'), {
+ code: 'ECONNRESET',
+ }),
+ }),
+ }),
+ },
+ ];
+
+ for (const { label, err } of cases) {
+ it(`is retriable: ${label}`, () => {
+ expect(isRetriableConnectionError(err)).toBe(true);
+ });
+ }
+ });
+
+ describe('classifyError — bucket split (node-down vs transient)', () => {
+ const cases: Array<{
+ label: string;
+ err: unknown;
+ expected: ErrorClass;
+ }> = [
+ // ── transient: don't sin-bin the node ────────────────────────────
+ {
+ label: 'ECONNRESET (stale keep-alive race — the common case)',
+ err: Object.assign(new Error('read ECONNRESET'), {
+ code: 'ECONNRESET',
+ }),
+ expected: 'transient',
+ },
+ {
+ label: 'EPIPE (write to closed socket)',
+ err: Object.assign(new Error('write EPIPE'), { code: 'EPIPE' }),
+ expected: 'transient',
+ },
+ {
+ label: 'socket hang up (no code)',
+ err: new Error('socket hang up'),
+ expected: 'transient',
+ },
+ {
+ label: 'undici Premature close',
+ err: new Error('Premature close'),
+ expected: 'transient',
+ },
+ {
+ label: 'undici body timeout (server slow mid-stream, not down)',
+ err: Object.assign(new Error('Body Timeout Error'), {
+ code: 'UND_ERR_BODY_TIMEOUT',
+ }),
+ expected: 'transient',
+ },
+ {
+ label: 'undici headers timeout',
+ err: Object.assign(new Error('Headers Timeout Error'), {
+ code: 'UND_ERR_HEADERS_TIMEOUT',
+ }),
+ expected: 'transient',
+ },
+ // ── node-down: sin-bin the node ──────────────────────────────────
+ {
+ label: 'ECONNREFUSED (node not listening)',
+ err: Object.assign(new Error('connect ECONNREFUSED 10.1.0.18:8123'), {
+ code: 'ECONNREFUSED',
+ }),
+ expected: 'node-down',
+ },
+ {
+ label: 'EHOSTUNREACH (no route)',
+ err: Object.assign(new Error('No route to host'), {
+ code: 'EHOSTUNREACH',
+ }),
+ expected: 'node-down',
+ },
+ {
+ label: 'connect ETIMEDOUT (TCP-connect timeout)',
+ err: Object.assign(new Error('connect ETIMEDOUT 10.1.0.20:8123'), {
+ code: 'ETIMEDOUT',
+ }),
+ expected: 'node-down',
+ },
+ {
+ label: 'EAI_AGAIN (DNS down)',
+ err: Object.assign(new Error('getaddrinfo EAI_AGAIN'), {
+ code: 'EAI_AGAIN',
+ }),
+ expected: 'node-down',
+ },
+ {
+ label: 'undici connect timeout',
+ err: Object.assign(new Error('Connect Timeout Error'), {
+ code: 'UND_ERR_CONNECT_TIMEOUT',
+ }),
+ expected: 'node-down',
+ },
+ // ── ch-server: propagate ────────────────────────────────────────
+ {
+ label: 'CH server SQL error',
+ err: new Error('Code: 47, DB::Exception: Unknown identifier'),
+ expected: 'ch-server',
+ },
+ {
+ label: 'CH TIMEOUT_EXCEEDED (server-side query timeout)',
+ err: new Error('Code: 159, DB::Exception: Timeout exceeded'),
+ expected: 'ch-server',
+ },
+ // ── cause-chain preserves classification ─────────────────────────
+ {
+ label: 'wrapped error: outer is opaque, cause is ECONNRESET',
+ err: Object.assign(new Error('Request failed'), {
+ cause: Object.assign(new Error('read ECONNRESET'), {
+ code: 'ECONNRESET',
+ }),
+ }),
+ expected: 'transient',
+ },
+ {
+ label: 'wrapped error: outer is opaque, cause is ECONNREFUSED',
+ err: Object.assign(new Error('Request failed'), {
+ cause: Object.assign(new Error('connect ECONNREFUSED'), {
+ code: 'ECONNREFUSED',
+ }),
+ }),
+ expected: 'node-down',
+ },
+ ];
+
+ for (const { label, err, expected } of cases) {
+ it(`classifies as ${expected}: ${label}`, () => {
+ expect(classifyError(err)).toBe(expected);
+ });
+ }
+ });
+
+ describe('Unknown errors — must NOT be retried (safety default)', () => {
+ const cases: Array<{ label: string; err: unknown }> = [
+ { label: 'null', err: null },
+ { label: 'undefined', err: undefined },
+ { label: 'string', err: 'something went wrong' as unknown },
+ { label: 'number', err: 42 as unknown },
+ {
+ label: 'plain Error with no recognizable shape',
+ err: new Error('whatever happened was weird'),
+ },
+ {
+ label: 'Error with unknown code',
+ err: Object.assign(new Error('strange'), { code: 'EUNKNOWN' }),
+ },
+ {
+ label: 'Error mentioning "connect" in a non-network context',
+ // This is the kind of false-positive the old "Connect" substring
+ // match was at risk of. Ensure the new matcher doesn't take the
+ // bait — there's no system code and no precise transport keyword.
+ err: new Error('Cannot connect the dots in this query plan'),
+ },
+ ];
+
+ for (const { label, err } of cases) {
+ it(`is NOT retriable: ${label}`, () => {
+ expect(isRetriableConnectionError(err)).toBe(false);
+ });
+ }
+ });
+});
diff --git a/packages/db/src/clickhouse/round-robin.ts b/packages/db/src/clickhouse/round-robin.ts
new file mode 100644
index 000000000..20cc05a38
--- /dev/null
+++ b/packages/db/src/clickhouse/round-robin.ts
@@ -0,0 +1,318 @@
+import type { ClickHouseClient } from '@clickhouse/client';
+import type { ILogger } from '@openpanel/logger';
+
+/**
+ * Round-robin selection across multiple ClickHouse clients with simple
+ * health tracking and a withRetry wrapper that moves to a different node
+ * on connection failures.
+ *
+ * The Hetzner LB experience taught us that L4 TCP forwarders can add
+ * unbounded tail latency. Instead of going through an LB, we now hold a
+ * direct connection to each CH node and pick between them ourselves.
+ */
+
+export interface ClientSlot {
+ client: ClickHouseClient;
+ url: string;
+ index: number;
+}
+
+/**
+ * Three buckets of error:
+ *
+ * - `node-down`: TCP/DNS/CONNECT-level failure. The node is genuinely
+ * unreachable from this worker right now. **Retry on next node AND
+ * sin-bin the failing node** so we don't keep trying it.
+ *
+ * - `transient`: A live socket got reset, hung up, or timed out mid-stream.
+ * The node itself is almost certainly healthy — it's a stale keep-alive,
+ * a one-off race, or a slow query. **Retry on next node but DO NOT
+ * sin-bin** — penalising the node for a flaky socket throws away a
+ * perfectly good replica.
+ *
+ * - `ch-server`: The server received and rejected the request (SQL error,
+ * permission, TOO_MANY_PARTS, TIMEOUT_EXCEEDED, MEMORY_LIMIT_EXCEEDED,
+ * etc.). **Propagate immediately, no retry.** The query is wrong or the
+ * cluster is overloaded — retrying on another node wouldn't help.
+ *
+ * - `other`: Unknown shape. **Propagate, no retry.** Safer to surface than
+ * to risk wrongly handling something we don't recognise.
+ */
+export type ErrorClass = 'node-down' | 'transient' | 'ch-server' | 'other';
+
+/** TCP/DNS failures — the node is unreachable. */
+const NODE_DOWN_CODES = new Set([
+ 'ECONNREFUSED', // node not listening
+ 'EHOSTUNREACH', // routing failure
+ 'EHOSTDOWN', // host explicitly down
+ 'ENETUNREACH', // network unreachable
+ 'ETIMEDOUT', // typically a TCP-connect timeout from Node
+ 'EAI_AGAIN', // transient DNS failure (treat as node unreachable)
+ 'UND_ERR_CONNECT_TIMEOUT', // undici connect timeout
+]);
+
+/** Mid-stream socket issues — the connection was alive but failed. The node
+ * is almost certainly fine; we just lost this particular socket. */
+const TRANSIENT_CODES = new Set([
+ 'ECONNRESET', // peer reset (often a stale keep-alive socket)
+ 'EPIPE', // write to half-closed socket
+ 'UND_ERR_SOCKET', // generic undici socket error
+ 'UND_ERR_HEADERS_TIMEOUT', // server accepted conn but didn't respond in time
+ 'UND_ERR_BODY_TIMEOUT', // body streaming timed out
+ 'UND_ERR_RESPONSE_TIMEOUT',
+]);
+
+/** Connect-time message keywords — node down. */
+const NODE_DOWN_MESSAGE_KEYWORDS = [
+ 'connect ETIMEDOUT',
+ 'connect ECONNREFUSED',
+ 'Connect Timeout Error', // undici
+];
+
+/** Mid-stream message keywords — transient socket. */
+const TRANSIENT_MESSAGE_KEYWORDS = [
+ 'socket hang up', // Node http
+ 'Premature close', // undici stream
+ 'Headers Timeout Error', // undici
+ 'Body Timeout Error', // undici
+ 'Response Timeout Error', // undici
+];
+
+/** Bare-code substrings used as a fallback when err.code wasn't propagated
+ * (sometimes happens when libraries re-wrap errors and only keep the
+ * message). Listed separately so the order of classification is
+ * predictable: explicit code first, then message keywords, then bare-code
+ * substring as last resort. */
+const NODE_DOWN_BARE_CODE_REGEX =
+ /\b(ECONNREFUSED|EHOSTUNREACH|EHOSTDOWN|ENETUNREACH|ETIMEDOUT|EAI_AGAIN)\b/;
+const TRANSIENT_BARE_CODE_REGEX = /\b(ECONNRESET|EPIPE)\b/;
+
+/** Match a ClickHouse server error. CH formats every exception as
+ * `Code: NNN. DB::Exception: <text>. (NAMED_CONSTANT)` over the HTTP
+ * interface. SQL errors, TOO_MANY_PARTS, TIMEOUT_EXCEEDED, auth errors —
+ * all of them. */
+const CH_SERVER_ERROR_PREFIX = /^Code:\s*\d+/;
+
+export function classifyError(err: unknown): ErrorClass {
+ if (!err || typeof err !== 'object') {
+ return 'other';
+ }
+ const e = err as {
+ message?: string;
+ code?: unknown;
+ cause?: unknown;
+ type?: unknown;
+ };
+ const msg = String(e.message ?? '');
+
+ // ── Deny: ClickHouse server errors ────────────────────────────────────
+ if (CH_SERVER_ERROR_PREFIX.test(msg)) {
+ return 'ch-server';
+ }
+ if (typeof e.type === 'string' && e.type.startsWith('DB::')) {
+ return 'ch-server';
+ }
+
+ // ── Explicit error.code wins ──────────────────────────────────────────
+ if (typeof e.code === 'string') {
+ if (NODE_DOWN_CODES.has(e.code)) return 'node-down';
+ if (TRANSIENT_CODES.has(e.code)) return 'transient';
+ }
+
+ // ── Message keyword fallback ──────────────────────────────────────────
+ if (NODE_DOWN_MESSAGE_KEYWORDS.some((k) => msg.includes(k))) {
+ return 'node-down';
+ }
+ if (TRANSIENT_MESSAGE_KEYWORDS.some((k) => msg.includes(k))) {
+ return 'transient';
+ }
+
+ // ── Bare-code substring fallback ──────────────────────────────────────
+ if (NODE_DOWN_BARE_CODE_REGEX.test(msg)) return 'node-down';
+ if (TRANSIENT_BARE_CODE_REGEX.test(msg)) return 'transient';
+
+ // ── Wrapped causes (Node 18+ AggregateError, undici wrapping) ─────────
+ if (e.cause) {
+ return classifyError(e.cause);
+ }
+
+ return 'other';
+}
+
+/** Convenience: should we retry this error at all (regardless of whether we
+ * sin-bin the node)? */
+export function isRetriableConnectionError(err: unknown): boolean {
+ const c = classifyError(err);
+ return c === 'node-down' || c === 'transient';
+}
+
+export class RoundRobinPicker {
+ private rrIndex = 0;
+ private unhealthyUntil = new Map<number, number>();
+
+ constructor(
+ private readonly clients: ClickHouseClient[],
+ private readonly urls: string[],
+ /** How long to skip a node after a connection failure, in ms. */
+ private readonly unhealthyMarkMs: number
+ ) {
+ if (clients.length === 0) {
+ throw new Error('RoundRobinPicker: at least one client required');
+ }
+ if (clients.length !== urls.length) {
+ throw new Error('RoundRobinPicker: clients and urls must be same length');
+ }
+ }
+
+ get size(): number {
+ return this.clients.length;
+ }
+
+ get allUrls(): readonly string[] {
+ return this.urls;
+ }
+
+ /**
+ * Pick the next client in round-robin order. Skips nodes that were
+ * recently marked unhealthy. If all are marked unhealthy, returns the
+ * next one anyway (the marks might be stale and we'd rather try than
+ * fail).
+ */
+ next(): ClientSlot {
+ if (this.clients.length === 1) {
+ return { client: this.clients[0]!, url: this.urls[0]!, index: 0 };
+ }
+ const now = Date.now();
+ for (let i = 0; i < this.clients.length; i++) {
+ const idx = this.rrIndex;
+ this.rrIndex = (this.rrIndex + 1) % this.clients.length;
+ const until = this.unhealthyUntil.get(idx);
+ if (!until || until <= now) {
+ return {
+ client: this.clients[idx]!,
+ url: this.urls[idx]!,
+ index: idx,
+ };
+ }
+ }
+ // All nodes are sin-binned — try the next one anyway.
+ const idx = this.rrIndex;
+ this.rrIndex = (this.rrIndex + 1) % this.clients.length;
+ this.unhealthyUntil.delete(idx);
+ return { client: this.clients[idx]!, url: this.urls[idx]!, index: idx };
+ }
+
+ markUnhealthy(slot: ClientSlot): void {
+ if (this.clients.length > 1) {
+ this.unhealthyUntil.set(slot.index, Date.now() + this.unhealthyMarkMs);
+ }
+ }
+
+ markHealthy(slot: ClientSlot): void {
+ this.unhealthyUntil.delete(slot.index);
+ }
+
+ /** For debugging / metrics — which nodes are currently in the sin bin. */
+ getUnhealthySnapshot(): { url: string; until: number }[] {
+ const now = Date.now();
+ const out: { url: string; until: number }[] = [];
+ for (const [idx, until] of this.unhealthyUntil) {
+ if (until > now) {
+ out.push({ url: this.urls[idx] ?? '?', until });
+ }
+ }
+ return out;
+ }
+}
+
+/** Context passed to the operation callback so callers can record which
+ * node was actually used (e.g. for logs / metrics). */
+export interface OperationContext {
+ url: string;
+ index: number;
+}
+
+/**
+ * Run `operation` against the round-robin picker. On a connection error,
+ * mark that node unhealthy and try the next one. Non-connection errors
+ * (e.g. SQL syntax) propagate immediately — retrying wouldn't help.
+ */
+export async function withRoundRobinRetry<T>(
+ picker: RoundRobinPicker,
+ operation: (client: ClickHouseClient, ctx: OperationContext) => Promise<T>,
+ logger: ILogger,
+ opts: {
+ /** Max total attempts (default = picker.size + 1, min 3). */
+ maxAttempts?: number;
+ /** Base delay for exponential backoff in ms. */
+ baseDelayMs?: number;
+ /** Cap on backoff delay between attempts. */
+ maxDelayMs?: number;
+ } = {}
+): Promise<T> {
+ const maxAttempts = opts.maxAttempts ?? Math.max(3, picker.size + 1);
+ const baseDelayMs = opts.baseDelayMs ?? 200;
+ const maxDelayMs = opts.maxDelayMs ?? 2000;
+
+ let lastError: unknown;
+
+ for (let attempt = 0; attempt < maxAttempts; attempt++) {
+ const slot = picker.next();
+ try {
+ const res = await operation(slot.client, {
+ url: slot.url,
+ index: slot.index,
+ });
+ if (attempt > 0) {
+ picker.markHealthy(slot);
+ logger.info(
+ { url: slot.url, attempt: attempt + 1 },
+ 'CH operation succeeded after retry'
+ );
+ }
+ return res;
+ } catch (error) {
+ lastError = error;
+
+ const errClass = classifyError(error);
+ if (errClass === 'ch-server' || errClass === 'other') {
+ // SQL/auth/business error or unknown shape — propagate immediately.
+ // Retrying wouldn't help and we don't want to wrongly handle
+ // unfamiliar errors as transport failures.
+ throw error;
+ }
+
+ // Only sin-bin nodes that are actually unreachable. A `transient`
+ // socket-level error (ECONNRESET, EPIPE, stream timeout) doesn't
+ // mean the node is sick — it's a stale keep-alive race or one-off
+ // glitch. Penalising the node would throw away a healthy replica.
+ if (errClass === 'node-down') {
+ picker.markUnhealthy(slot);
+ }
+
+ if (attempt === maxAttempts - 1) {
+ break;
+ }
+
+ const delay = Math.min(baseDelayMs * 2 ** attempt, maxDelayMs);
+ logger.warn(
+ {
+ err: (error as { message?: string })?.message,
+ code: (error as { code?: string })?.code,
+ errClass,
+ markedUnhealthy: errClass === 'node-down',
+ url: slot.url,
+ attempt: attempt + 1,
+ maxAttempts,
+ nextDelayMs: delay,
+ },
+ errClass === 'node-down'
+ ? 'CH node unreachable; sin-binning and retrying on next'
+ : 'CH transient socket error; retrying on next'
+ );
+ await new Promise((r) => setTimeout(r, delay));
+ }
+ }
+
+ throw lastError;
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index ef7a87ac1..7241166c6 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1179,8 +1179,8 @@ importers:
packages/db:
dependencies:
'@clickhouse/client':
- specifier: ^1.12.1
- version: 1.12.1
+ specifier: ^1.18.5
+ version: 1.18.5
'@openpanel/common':
specifier: workspace:*
version: link:../common
@@ -3454,11 +3454,11 @@ packages:
resolution: {integrity: sha512-/42G73JkuYdyWZ6m8d/CJtBrGl1Hegyc7Fy78m5Ob+jF85TOUmLR5XLce/U3LxYAw0kJ8CT5aI99RIvPHcGp/Q==}
- '@clickhouse/[email protected]':
- resolution: {integrity: sha512-ccw1N6hB4+MyaAHIaWBwGZ6O2GgMlO99FlMj0B0UEGfjxM9v5dYVYql6FpP19rMwrVAroYs/IgX2vyZEBvzQLg==}
+ '@clickhouse/[email protected]':
+ resolution: {integrity: sha512-g9LwcS1dvkatKDsIjT1PwUHldsiYzwdKAB0nXfd9APLd+t4PrNJa+my+dXcqJdmcWyhWjKLP/2/ztBwgxp+sbQ==}
- '@clickhouse/[email protected]':
- resolution: {integrity: sha512-7ORY85rphRazqHzImNXMrh4vsaPrpetFoTWpZYueCO2bbO6PXYDXp/GQ4DgxnGIqbWB/Di1Ai+Xuwq2o7DJ36A==}
+ '@clickhouse/[email protected]':
+ resolution: {integrity: sha512-4FfoyMkFWhsdNMuXsoEL6l3c12svA63BBJBtDo9SrxRZ14RdmN6jLr/rF3f84BK8cFoxETZCSeKlsbk6NNYebw==}
engines: {node: '>=16'}
'@cloudflare/[email protected]':
@@ -23305,11 +23305,11 @@ snapshots:
picocolors: 1.1.1
sisteransi: 1.0.5
- '@clickhouse/[email protected]': {}
+ '@clickhouse/[email protected]': {}
- '@clickhouse/[email protected]':
+ '@clickhouse/[email protected]':
dependencies:
- '@clickhouse/client-common': 1.12.1
+ '@clickhouse/client-common': 1.18.5
'@cloudflare/[email protected]':
dependencies:
@@ -37640,7 +37640,7 @@ snapshots:
compatx: 0.2.0
confbox: 0.2.2
consola: 3.4.2
- cookie-es: 2.0.0
+ cookie-es: 2.0.1
croner: 9.1.0
crossws: 0.3.5
db0: 0.3.4
@@ -42744,7 +42744,7 @@ snapshots:
'@poppinss/colors': 4.1.5
'@poppinss/dumper': 0.6.5
'@speed-highlight/core': 1.2.12
- cookie-es: 2.0.0
+ cookie-es: 2.0.1
youch-core: 0.3.3