OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: add round robin for clickhouse client
Carl-Gerhard Lindesvärd6c9eb05a810b
diff --git a/packages/db/package.json b/packages/db/package.jsonindex fb4cd393e..634a264b1 100644--- a/packages/db/package.json+++ b/packages/db/package.json@@ -13,7 +13,7 @@"with-env": "dotenv -e ../../.env -c --"},"dependencies": {- "@clickhouse/client": "^1.12.1",+ "@clickhouse/client": "^1.18.5","@openpanel/common": "workspace:*","@openpanel/constants": "workspace:*","@openpanel/json": "workspace:*",@@ -42,4 +42,4 @@"prisma": "^6.14.0","typescript": "catalog:"}-}+}\ No newline at end of filediff --git a/packages/db/src/buffers/base-buffer.ts b/packages/db/src/buffers/base-buffer.tsindex a50ecf26a..0d092cf4e 100644--- a/packages/db/src/buffers/base-buffer.ts+++ b/packages/db/src/buffers/base-buffer.ts@@ -1,5 +1,6 @@+import type { ClickHouseSettings } from '@clickhouse/client';import { generateSecureId } from '@openpanel/common/server';-import { type ILogger, createLogger } from '@openpanel/logger';+import { createLogger, type ILogger } from '@openpanel/logger';import { cronQueue } from '@openpanel/queue';import { getRedisCache } from '@openpanel/redis';@@ -78,6 +79,18 @@ export class BaseBuffer {skipReason?: string;} = {};+ protected getClickhouseSettings(): ClickHouseSettings {+ if (process.env.BUFFER_ASYNC_INSERTS) {+ return {+ async_insert: 1,+ wait_for_async_insert: 0,+ parallel_view_processing: 1,+ };+ }++ return {};+ }+/** Optional hook used by the worker to bridge flushes into Prometheus. */public flushObserver: FlushObserver | null = null;@@ -102,9 +115,7 @@ export class BaseBuffer {* observability. Subclasses MUST override.*/protected getRedisListKey(): string {- throw new Error(- `${this.name}: subclass must override getRedisListKey()`,- );+ throw new Error(`${this.name}: subclass must override getRedisListKey()`);}protected chunks<T>(items: T[], size: number) {@@ -135,9 +146,11 @@ export class BaseBuffer {protected async parallelLimit<T, R>(items: T[],fn: (item: T, index: number) => Promise<R>,- concurrency: number = this.chInsertConcurrency,+ concurrency: number = this.chInsertConcurrency): Promise<R[]> {- if (items.length === 0) return [];+ if (items.length === 0) {+ return [];+ }if (concurrency <= 1 || items.length === 1) {const out: R[] = [];for (let i = 0; i < items.length; i++) {@@ -150,13 +163,15 @@ export class BaseBuffer {const worker = async (): Promise<void> => {while (true) {const idx = nextIndex++;- if (idx >= items.length) return;+ if (idx >= items.length) {+ return;+ }results[idx] = await fn(items[idx] as T, idx);}};const workers = Array.from({ length: Math.min(concurrency, items.length) },- () => worker(),+ () => worker());await Promise.all(workers);return results;@@ -247,6 +262,7 @@ export class BaseBuffer {}const logPayload = {+ buffer: obs.buffer,result: obs.result,trigger: obs.trigger,totalMs: Math.round(obs.totalMs),@@ -265,9 +281,7 @@ export class BaseBuffer {? Math.round(obs.phases.chInsertMs): undefined,trimMs:- obs.phases?.trimMs != null- ? Math.round(obs.phases.trimMs)- : undefined,+ obs.phases?.trimMs != null ? Math.round(obs.phases.trimMs) : undefined,onFlushMs:obs.phases?.onFlushMs != null? Math.round(obs.phases.onFlushMs)@@ -275,12 +289,20 @@ export class BaseBuffer {};if (obs.result === 'error') {- this.logger.error({ ...logPayload, err: obs.err }, 'Buffer flush');+ this.logger.error(+ { ...logPayload, err: obs.err },+ `Flush failed for ${this.name}`+ );} else if (obs.result === 'paused') {// Demoted to debug — happens every 10s when intentionally paused- this.logger.debug(logPayload, 'Buffer flush skipped (cron paused)');+ this.logger.debug(+ logPayload,+ `Flush skipped for ${this.name} (cron paused)`+ );+ } else if (obs.result === 'locked') {+ this.logger.debug(logPayload, `Flush skipped for ${this.name} (locked)`);} else {- this.logger.info(logPayload, 'Buffer flush');+ this.logger.info(logPayload, `Flush completed for ${this.name}`);}}@@ -348,7 +370,7 @@ export class BaseBuffer {lockId,'EX',this.lockTimeout,- 'NX',+ 'NX');if (acquired !== 'OK') {diff --git a/packages/db/src/buffers/bot-buffer.ts b/packages/db/src/buffers/bot-buffer.tsindex c54092cd6..7c645a9aa 100644--- a/packages/db/src/buffers/bot-buffer.ts+++ b/packages/db/src/buffers/bot-buffer.ts@@ -65,11 +65,7 @@ export class BotBuffer extends BaseBuffer {table: TABLE_NAMES.events_bots,values: parsedEvents,format: 'JSONEachRow',- clickhouse_settings: {- async_insert: 1,- wait_for_async_insert: 0,- parallel_view_processing: 1,- },+ clickhouse_settings: this.getClickhouseSettings(),});const chInsertMs = performance.now() - chStart;diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.tsindex 517d937d7..9b7e4b30a 100644--- a/packages/db/src/buffers/event-buffer.ts+++ b/packages/db/src/buffers/event-buffer.ts@@ -176,11 +176,7 @@ export class EventBuffer extends BaseBuffer {table: 'events',values: chunk,format: 'JSONEachRow',- clickhouse_settings: {- async_insert: 1,- wait_for_async_insert: 0,- parallel_view_processing: 1,- },+ clickhouse_settings: this.getClickhouseSettings(),}));const chInsertMs = performance.now() - chStart;diff --git a/packages/db/src/buffers/group-buffer.ts b/packages/db/src/buffers/group-buffer.tsindex 5b8138efe..d42e08f79 100644--- a/packages/db/src/buffers/group-buffer.ts+++ b/packages/db/src/buffers/group-buffer.ts@@ -184,11 +184,7 @@ export class GroupBuffer extends BaseBuffer {table: TABLE_NAMES.groups,values: chunk,format: 'JSONEachRow',- clickhouse_settings: {- async_insert: 1,- wait_for_async_insert: 0,- parallel_view_processing: 1,- },+ clickhouse_settings: this.getClickhouseSettings(),}));const chInsertMs = performance.now() - chStart;diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.tsindex 2d7b25cde..e90d3fb66 100644--- a/packages/db/src/buffers/profile-buffer.ts+++ b/packages/db/src/buffers/profile-buffer.ts@@ -341,11 +341,7 @@ export class ProfileBuffer extends BaseBuffer {table: TABLE_NAMES.profiles,values: chunk,format: 'JSONEachRow',- clickhouse_settings: {- async_insert: 1,- wait_for_async_insert: 0,- parallel_view_processing: 1,- },+ clickhouse_settings: this.getClickhouseSettings(),}));const chInsertMs = performance.now() - chStart;diff --git a/packages/db/src/buffers/replay-buffer.ts b/packages/db/src/buffers/replay-buffer.tsindex a3e33f2c2..7c3aecc56 100644--- a/packages/db/src/buffers/replay-buffer.ts+++ b/packages/db/src/buffers/replay-buffer.ts@@ -82,11 +82,7 @@ export class ReplayBuffer extends BaseBuffer {table: TABLE_NAMES.session_replay_chunks,values: chunk,format: 'JSONEachRow',- clickhouse_settings: {- async_insert: 1,- wait_for_async_insert: 0,- parallel_view_processing: 1,- },+ clickhouse_settings: this.getClickhouseSettings(),}));const chInsertMs = performance.now() - chStart;diff --git a/packages/db/src/buffers/session-buffer.ts b/packages/db/src/buffers/session-buffer.tsindex d2a6e890d..7a7240783 100644--- a/packages/db/src/buffers/session-buffer.ts+++ b/packages/db/src/buffers/session-buffer.ts@@ -390,11 +390,7 @@ export class SessionBuffer extends BaseBuffer {table: TABLE_NAMES.sessions,values: chunk,format: 'JSONEachRow',- clickhouse_settings: {- async_insert: 1,- wait_for_async_insert: 0,- parallel_view_processing: 1,- },+ clickhouse_settings: this.getClickhouseSettings(),}));const chInsertMs = performance.now() - chStart;diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.tsindex 0f78035f3..1277f6d61 100644--- a/packages/db/src/clickhouse/client.ts+++ b/packages/db/src/clickhouse/client.ts@@ -1,9 +1,14 @@-import type { ClickHouseSettings, ResponseJSON } from '@clickhouse/client';+import type {+ ClickHouseClient,+ ClickHouseSettings,+ ResponseJSON,+} from '@clickhouse/client';import { ClickHouseLogLevel, createClient } from '@clickhouse/client';import type { NodeClickHouseClientConfigOptions } from '@clickhouse/client/dist/config';import { createLogger } from '@openpanel/logger';import type { IInterval } from '@openpanel/validation';import sqlstring from 'sqlstring';+import { RoundRobinPicker, withRoundRobinRetry } from './round-robin';export { createClient };@@ -128,13 +133,30 @@ const requestCompressionEnabled =const maxOpenConnections = process.env.CLICKHOUSE_MAX_OPEN_CONNECTIONS? Math.max(1,- Number.parseInt(process.env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, 10),+ Number.parseInt(process.env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, 10)): 50;+// Per-request timeout. Was 300_000 (5 min) — way too long for fast failover+// against a dead node, and after the Hetzner LB story we want stuck inserts+// to fail and retry on a different node well before 5 min. 30s is enough for+// any legitimate batch insert in this codebase. Bumpable via env if needed.+const requestTimeoutMs = process.env.CLICKHOUSE_REQUEST_TIMEOUT_MS+ ? Math.max(+ 1000,+ Number.parseInt(process.env.CLICKHOUSE_REQUEST_TIMEOUT_MS, 10)+ )+ : 30_000;++// How long to skip a node after a connection failure, before trying it+// again. Short by default — we want fast recovery once a node comes back.+const unhealthyMarkMs = process.env.CLICKHOUSE_UNHEALTHY_MARK_MS+ ? Math.max(0, Number.parseInt(process.env.CLICKHOUSE_UNHEALTHY_MARK_MS, 10))+ : 5000;+export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {max_open_connections: maxOpenConnections,- request_timeout: 300_000,+ request_timeout: requestTimeoutMs,keep_alive: {enabled: true,// Must be lower than server-side keep_alive_timeout (default 10s) so we@@ -151,100 +173,150 @@ export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {},};-logger.info({ options: CLICKHOUSE_OPTIONS }, 'Clickhouse options');+// CLICKHOUSE_URL accepts a single URL or comma-separated list. With a list,+// we round-robin across nodes ourselves and fail over on connection errors —+// avoids putting an L4 LB on the hot path.+//+// Tests sometimes import this module without CLICKHOUSE_URL set; in that+// case we fall back to a single client created the same way the old code+// did (URL = undefined, delegated to the client library's own handling).+const rawClickhouseUrls = (process.env.CLICKHOUSE_URL ?? '')+ .split(',')+ .map((s) => s.trim())+ .filter(Boolean);++const clickhouseUrls = rawClickhouseUrls.length > 0 ? rawClickhouseUrls : [''];++const clients: ClickHouseClient[] = clickhouseUrls.map((url) =>+ createClient({+ url: url || process.env.CLICKHOUSE_URL,+ ...CLICKHOUSE_OPTIONS,+ })+);++const picker = new RoundRobinPicker(clients, clickhouseUrls, unhealthyMarkMs);++function maskUrlCredentials(url: string): string {+ if (!url) return '<unset>';+ try {+ const u = new URL(url);+ if (u.username) {+ u.username = u.username.slice(0, 2) + '*'.repeat(u.username.length - 2);+ }+ if (u.password) {+ u.password = '***';+ }+ return u.toString();+ } catch {+ // Malformed URL — fall back to the original; better than crashing module+ // load in test envs where CLICKHOUSE_URL may be empty/garbage.+ return url;+ }+}++logger.info(+ {+ nodeCount: clients.length,+ urls: clickhouseUrls.map(maskUrlCredentials),+ requestTimeoutMs,+ unhealthyMarkMs,+ options: { ...CLICKHOUSE_OPTIONS, log: undefined },+ },+ 'ClickHouse clients initialized'+);-export const originalCh = createClient({- url: process.env.CLICKHOUSE_URL,- ...CLICKHOUSE_OPTIONS,-});+// Backwards-compat export. Some callers (notably gsc.ts) use `originalCh`+// directly to bypass the retry/round-robin layer for one-off DDL or for+// places where retry semantics aren't wanted. Points at the first node.+export const originalCh = clients[0]!;const cleanQuery = (query?: string) =>typeof query === 'string'? query.replace(/\n/g, '').replace(/\s+/g, ' ').trim(): undefined;+/**+ * Run `operation` against the next round-robin node, retrying on the next+ * node when a connection error is hit. Non-connection errors (SQL, auth,+ * 4xx) propagate immediately.+ *+ * `operation` receives the picked client and a context with the node URL+ * (useful for adding `host` to log lines / metrics).+ */export async function withRetry<T>(- operation: () => Promise<T>,- maxRetries = 3,- baseDelay = 500+ operation: (+ client: ClickHouseClient,+ ctx: { url: string; index: number },+ ) => Promise<T>,): Promise<T> {- let lastError: Error | undefined;-- for (let attempt = 0; attempt < maxRetries; attempt++) {- try {- const res = await operation();- if (attempt > 0) {- logger.info({ attempt }, 'Retry operation succeeded');- }- return res;- } catch (error: any) {- lastError = error;-- if (- error.message.includes('Connect') ||- error.message.includes('socket hang up') ||- error.message.includes('Timeout error')- ) {- const delay = baseDelay * 2 ** attempt;- logger.warn(- { err: error },- `Attempt ${attempt + 1}/${maxRetries} failed, retrying in ${delay}ms`- );- await new Promise((resolve) => setTimeout(resolve, delay));- continue;- }-- throw error; // Non-retriable error- }- }+ return withRoundRobinRetry(picker, operation, logger);+}- throw lastError;+/** Best-effort URL → hostname extraction for log labels. */+function urlHostname(url: string | undefined): string | undefined {+ if (!url) return undefined;+ try {+ return new URL(url).hostname;+ } catch {+ return undefined;+ }}-export const ch = new Proxy(originalCh, {- get(target, property, receiver) {- const value = Reflect.get(target, property, receiver);-- if (property === 'insert') {- return (...args: any[]) =>- withRetry(() => {- args[0].clickhouse_settings = {- // Increase insert timeouts and buffer sizes for large batches- max_execution_time: 300,- max_insert_block_size: '500000',- max_http_get_redirects: '0',- // Ensure JSONEachRow stays efficient- input_format_parallel_parsing: 1,- // Keep long-running inserts/queries from idling out at proxies by sending progress headers- send_progress_in_http_headers: 1,- http_headers_progress_interval_ms: '50000',- // Ensure server holds the connection until the query is finished- wait_end_of_query: 1,- ...args[0].clickhouse_settings,- };- return value.apply(target, args);- });- }+const INSERT_DEFAULT_SETTINGS: ClickHouseSettings = {+ // Increase insert timeouts and buffer sizes for large batches+ max_execution_time: 300,+ max_insert_block_size: '500000',+ max_http_get_redirects: '0',+ // Ensure JSONEachRow stays efficient+ input_format_parallel_parsing: 1,+ // Keep long-running inserts/queries from idling out at proxies by sending progress headers+ send_progress_in_http_headers: 1,+ http_headers_progress_interval_ms: '50000',+ // Ensure server holds the connection until the query is finished+ wait_end_of_query: 1,+};- if (property === 'command') {- return (...args: any[]) =>- withRetry(() => {- return value.apply(target, args);- });- }+/**+ * High-level CH client. For insert/command/query, picks a node round-robin+ * and retries on a *different* node if the connection fails. The methods+ * live as own-properties on the proxy target so `vi.spyOn(ch, 'insert')`+ * works in tests. Other method accesses (rare — config readers, instance+ * checks) fall through to the first client.+ */+const chTarget = {+ insert: (params: any) =>+ withRetry((client) => {+ params.clickhouse_settings = {+ ...INSERT_DEFAULT_SETTINGS,+ ...params.clickhouse_settings,+ };+ return client.insert(params);+ }),+ command: (params: any) => withRetry((client) => client.command(params)),+ query: (params: any) => withRetry((client) => client.query(params)),+};- return value;+export const ch = new Proxy(chTarget as unknown as ClickHouseClient, {+ get(target, property, receiver) {+ if (property in target) {+ return Reflect.get(target, property, receiver);+ }+ return (originalCh as any)[property];},-});+}) as ClickHouseClient;export async function chQueryWithMeta<T extends Record<string, any>>(query: string,clickhouseSettings?: ClickHouseSettings): Promise<ResponseJSON<T>> {const start = Date.now();- const res = await ch.query({- query,- clickhouse_settings: clickhouseSettings,+ let host: string | undefined;+ const res = await withRetry((client, ctx) => {+ host = urlHostname(ctx.url);+ return client.query({+ query,+ clickhouse_settings: clickhouseSettings,+ });});const json = await res.json<T>();const keys = Object.keys(json.data[0] || {});@@ -266,6 +338,7 @@ export async function chQueryWithMeta<T extends Record<string, any>>(logger.info({+ host,query: cleanQuery(query),rows: json.rows,stats: response.statistics,diff --git a/packages/db/src/clickhouse/round-robin.test.ts b/packages/db/src/clickhouse/round-robin.test.tsnew file mode 100644index 000000000..d9e2fad16--- /dev/null+++ b/packages/db/src/clickhouse/round-robin.test.ts@@ -0,0 +1,322 @@+import { describe, expect, it } from 'vitest';+import {+ type ErrorClass,+ classifyError,+ isRetriableConnectionError,+} from './round-robin';++/**+ * The contract: anything from the ClickHouse server (errors that traveled+ * back over a working connection) must NOT mark a node unhealthy. Only true+ * transport failures should.+ *+ * If you ever see a node getting incorrectly sin-binned in prod, add the+ * exact error message here as a `not retriable` case and update+ * isRetriableConnectionError to handle it.+ */+describe('isRetriableConnectionError', () => {+ describe('ClickHouse server errors — must NOT be retried', () => {+ const cases: Array<{ label: string; err: unknown }> = [+ {+ label: 'SQL syntax error',+ err: new Error(+ 'Code: 62, DB::Exception: Syntax error: failed at position 42',+ ),+ },+ {+ label: 'Unknown identifier (e.g. wrong column)',+ err: new Error(+ "Code: 47, DB::Exception: Unknown identifier 'fooooo'",+ ),+ },+ {+ label: 'ILLEGAL_AGGREGATION',+ err: new Error(+ 'Code: 184, DB::Exception: Aggregate function max(...) is found in WHERE',+ ),+ },+ {+ label: 'Missing columns',+ err: new Error(+ 'Code: 47, DB::Exception: Missing columns: a, b while processing query',+ ),+ },+ {+ label: 'TOO_MANY_PARTS — back-pressure, not connectivity',+ err: new Error('Code: 252, DB::Exception: Too many parts in partition'),+ },+ {+ label: 'MEMORY_LIMIT_EXCEEDED',+ err: new Error(+ 'Code: 241, DB::Exception: Memory limit (for query) exceeded',+ ),+ },+ {+ label: 'TIMEOUT_EXCEEDED — server-side query timeout',+ err: new Error(+ 'Code: 159, DB::Exception: Timeout exceeded: elapsed 60.001 sec',+ ),+ },+ {+ label: 'Authentication denied',+ err: new Error('Code: 516, DB::Exception: Authentication failed'),+ },+ {+ label: 'CH error with DB::Exception type marker (no Code prefix)',+ err: Object.assign(new Error('some message'), {+ type: 'DB::Exception',+ }),+ },+ {+ label: 'CH error whose body happens to mention "connect"',+ err: new Error(+ 'Code: 999, DB::Exception: Cannot connect to ZooKeeper',+ ),+ },+ ];++ for (const { label, err } of cases) {+ it(`is NOT retriable: ${label}`, () => {+ expect(isRetriableConnectionError(err)).toBe(false);+ });+ }+ });++ describe('Transport errors — must be retried on next node', () => {+ const cases: Array<{ label: string; err: unknown }> = [+ {+ label: 'ECONNREFUSED via err.code',+ err: Object.assign(new Error('connect ECONNREFUSED 10.1.0.18:8123'), {+ code: 'ECONNREFUSED',+ }),+ },+ {+ label: 'ECONNRESET via err.code',+ err: Object.assign(new Error('read ECONNRESET'), {+ code: 'ECONNRESET',+ }),+ },+ {+ label: 'ETIMEDOUT via err.code',+ err: Object.assign(new Error('connect ETIMEDOUT 10.1.0.20:8123'), {+ code: 'ETIMEDOUT',+ }),+ },+ {+ label: 'EHOSTUNREACH via err.code',+ err: Object.assign(new Error('No route to host'), {+ code: 'EHOSTUNREACH',+ }),+ },+ {+ label: 'EAI_AGAIN (transient DNS)',+ err: Object.assign(+ new Error('getaddrinfo EAI_AGAIN ch.internal'),+ { code: 'EAI_AGAIN' },+ ),+ },+ {+ label: 'EPIPE on broken socket',+ err: Object.assign(new Error('write EPIPE'), { code: 'EPIPE' }),+ },+ {+ label: 'undici connect timeout',+ err: Object.assign(new Error('Connect Timeout Error'), {+ code: 'UND_ERR_CONNECT_TIMEOUT',+ }),+ },+ {+ label: 'undici headers timeout',+ err: Object.assign(new Error('Headers Timeout Error'), {+ code: 'UND_ERR_HEADERS_TIMEOUT',+ }),+ },+ {+ label: 'undici body timeout',+ err: Object.assign(new Error('Body Timeout Error'), {+ code: 'UND_ERR_BODY_TIMEOUT',+ }),+ },+ {+ label: 'plain socket hang up (no code)',+ err: new Error('socket hang up'),+ },+ {+ label: 'undici premature close',+ err: new Error('Premature close'),+ },+ {+ label: 'cause-chained ECONNREFUSED',+ err: Object.assign(new Error('Failed to fetch'), {+ cause: Object.assign(new Error('connect ECONNREFUSED'), {+ code: 'ECONNREFUSED',+ }),+ }),+ },+ {+ label: 'deeply nested cause chain',+ err: Object.assign(new Error('outer wrapper'), {+ cause: Object.assign(new Error('middle wrapper'), {+ cause: Object.assign(new Error('inner socket hang up'), {+ code: 'ECONNRESET',+ }),+ }),+ }),+ },+ ];++ for (const { label, err } of cases) {+ it(`is retriable: ${label}`, () => {+ expect(isRetriableConnectionError(err)).toBe(true);+ });+ }+ });++ describe('classifyError — bucket split (node-down vs transient)', () => {+ const cases: Array<{+ label: string;+ err: unknown;+ expected: ErrorClass;+ }> = [+ // ── transient: don't sin-bin the node ────────────────────────────+ {+ label: 'ECONNRESET (stale keep-alive race — the common case)',+ err: Object.assign(new Error('read ECONNRESET'), {+ code: 'ECONNRESET',+ }),+ expected: 'transient',+ },+ {+ label: 'EPIPE (write to closed socket)',+ err: Object.assign(new Error('write EPIPE'), { code: 'EPIPE' }),+ expected: 'transient',+ },+ {+ label: 'socket hang up (no code)',+ err: new Error('socket hang up'),+ expected: 'transient',+ },+ {+ label: 'undici Premature close',+ err: new Error('Premature close'),+ expected: 'transient',+ },+ {+ label: 'undici body timeout (server slow mid-stream, not down)',+ err: Object.assign(new Error('Body Timeout Error'), {+ code: 'UND_ERR_BODY_TIMEOUT',+ }),+ expected: 'transient',+ },+ {+ label: 'undici headers timeout',+ err: Object.assign(new Error('Headers Timeout Error'), {+ code: 'UND_ERR_HEADERS_TIMEOUT',+ }),+ expected: 'transient',+ },+ // ── node-down: sin-bin the node ──────────────────────────────────+ {+ label: 'ECONNREFUSED (node not listening)',+ err: Object.assign(new Error('connect ECONNREFUSED 10.1.0.18:8123'), {+ code: 'ECONNREFUSED',+ }),+ expected: 'node-down',+ },+ {+ label: 'EHOSTUNREACH (no route)',+ err: Object.assign(new Error('No route to host'), {+ code: 'EHOSTUNREACH',+ }),+ expected: 'node-down',+ },+ {+ label: 'connect ETIMEDOUT (TCP-connect timeout)',+ err: Object.assign(new Error('connect ETIMEDOUT 10.1.0.20:8123'), {+ code: 'ETIMEDOUT',+ }),+ expected: 'node-down',+ },+ {+ label: 'EAI_AGAIN (DNS down)',+ err: Object.assign(new Error('getaddrinfo EAI_AGAIN'), {+ code: 'EAI_AGAIN',+ }),+ expected: 'node-down',+ },+ {+ label: 'undici connect timeout',+ err: Object.assign(new Error('Connect Timeout Error'), {+ code: 'UND_ERR_CONNECT_TIMEOUT',+ }),+ expected: 'node-down',+ },+ // ── ch-server: propagate ────────────────────────────────────────+ {+ label: 'CH server SQL error',+ err: new Error('Code: 47, DB::Exception: Unknown identifier'),+ expected: 'ch-server',+ },+ {+ label: 'CH TIMEOUT_EXCEEDED (server-side query timeout)',+ err: new Error('Code: 159, DB::Exception: Timeout exceeded'),+ expected: 'ch-server',+ },+ // ── cause-chain preserves classification ─────────────────────────+ {+ label: 'wrapped error: outer is opaque, cause is ECONNRESET',+ err: Object.assign(new Error('Request failed'), {+ cause: Object.assign(new Error('read ECONNRESET'), {+ code: 'ECONNRESET',+ }),+ }),+ expected: 'transient',+ },+ {+ label: 'wrapped error: outer is opaque, cause is ECONNREFUSED',+ err: Object.assign(new Error('Request failed'), {+ cause: Object.assign(new Error('connect ECONNREFUSED'), {+ code: 'ECONNREFUSED',+ }),+ }),+ expected: 'node-down',+ },+ ];++ for (const { label, err, expected } of cases) {+ it(`classifies as ${expected}: ${label}`, () => {+ expect(classifyError(err)).toBe(expected);+ });+ }+ });++ describe('Unknown errors — must NOT be retried (safety default)', () => {+ const cases: Array<{ label: string; err: unknown }> = [+ { label: 'null', err: null },+ { label: 'undefined', err: undefined },+ { label: 'string', err: 'something went wrong' as unknown },+ { label: 'number', err: 42 as unknown },+ {+ label: 'plain Error with no recognizable shape',+ err: new Error('whatever happened was weird'),+ },+ {+ label: 'Error with unknown code',+ err: Object.assign(new Error('strange'), { code: 'EUNKNOWN' }),+ },+ {+ label: 'Error mentioning "connect" in a non-network context',+ // This is the kind of false-positive the old "Connect" substring+ // match was at risk of. Ensure the new matcher doesn't take the+ // bait — there's no system code and no precise transport keyword.+ err: new Error('Cannot connect the dots in this query plan'),+ },+ ];++ for (const { label, err } of cases) {+ it(`is NOT retriable: ${label}`, () => {+ expect(isRetriableConnectionError(err)).toBe(false);+ });+ }+ });+});diff --git a/packages/db/src/clickhouse/round-robin.ts b/packages/db/src/clickhouse/round-robin.tsnew file mode 100644index 000000000..20cc05a38--- /dev/null+++ b/packages/db/src/clickhouse/round-robin.ts@@ -0,0 +1,318 @@+import type { ClickHouseClient } from '@clickhouse/client';+import type { ILogger } from '@openpanel/logger';++/**+ * Round-robin selection across multiple ClickHouse clients with simple+ * health tracking and a withRetry wrapper that moves to a different node+ * on connection failures.+ *+ * The Hetzner LB experience taught us that L4 TCP forwarders can add+ * unbounded tail latency. Instead of going through an LB, we now hold a+ * direct connection to each CH node and pick between them ourselves.+ */++export interface ClientSlot {+ client: ClickHouseClient;+ url: string;+ index: number;+}++/**+ * Three buckets of error:+ *+ * - `node-down`: TCP/DNS/CONNECT-level failure. The node is genuinely+ * unreachable from this worker right now. **Retry on next node AND+ * sin-bin the failing node** so we don't keep trying it.+ *+ * - `transient`: A live socket got reset, hung up, or timed out mid-stream.+ * The node itself is almost certainly healthy — it's a stale keep-alive,+ * a one-off race, or a slow query. **Retry on next node but DO NOT+ * sin-bin** — penalising the node for a flaky socket throws away a+ * perfectly good replica.+ *+ * - `ch-server`: The server received and rejected the request (SQL error,+ * permission, TOO_MANY_PARTS, TIMEOUT_EXCEEDED, MEMORY_LIMIT_EXCEEDED,+ * etc.). **Propagate immediately, no retry.** The query is wrong or the+ * cluster is overloaded — retrying on another node wouldn't help.+ *+ * - `other`: Unknown shape. **Propagate, no retry.** Safer to surface than+ * to risk wrongly handling something we don't recognise.+ */+export type ErrorClass = 'node-down' | 'transient' | 'ch-server' | 'other';++/** TCP/DNS failures — the node is unreachable. */+const NODE_DOWN_CODES = new Set([+ 'ECONNREFUSED', // node not listening+ 'EHOSTUNREACH', // routing failure+ 'EHOSTDOWN', // host explicitly down+ 'ENETUNREACH', // network unreachable+ 'ETIMEDOUT', // typically a TCP-connect timeout from Node+ 'EAI_AGAIN', // transient DNS failure (treat as node unreachable)+ 'UND_ERR_CONNECT_TIMEOUT', // undici connect timeout+]);++/** Mid-stream socket issues — the connection was alive but failed. The node+ * is almost certainly fine; we just lost this particular socket. */+const TRANSIENT_CODES = new Set([+ 'ECONNRESET', // peer reset (often a stale keep-alive socket)+ 'EPIPE', // write to half-closed socket+ 'UND_ERR_SOCKET', // generic undici socket error+ 'UND_ERR_HEADERS_TIMEOUT', // server accepted conn but didn't respond in time+ 'UND_ERR_BODY_TIMEOUT', // body streaming timed out+ 'UND_ERR_RESPONSE_TIMEOUT',+]);++/** Connect-time message keywords — node down. */+const NODE_DOWN_MESSAGE_KEYWORDS = [+ 'connect ETIMEDOUT',+ 'connect ECONNREFUSED',+ 'Connect Timeout Error', // undici+];++/** Mid-stream message keywords — transient socket. */+const TRANSIENT_MESSAGE_KEYWORDS = [+ 'socket hang up', // Node http+ 'Premature close', // undici stream+ 'Headers Timeout Error', // undici+ 'Body Timeout Error', // undici+ 'Response Timeout Error', // undici+];++/** Bare-code substrings used as a fallback when err.code wasn't propagated+ * (sometimes happens when libraries re-wrap errors and only keep the+ * message). Listed separately so the order of classification is+ * predictable: explicit code first, then message keywords, then bare-code+ * substring as last resort. */+const NODE_DOWN_BARE_CODE_REGEX =+ /\b(ECONNREFUSED|EHOSTUNREACH|EHOSTDOWN|ENETUNREACH|ETIMEDOUT|EAI_AGAIN)\b/;+const TRANSIENT_BARE_CODE_REGEX = /\b(ECONNRESET|EPIPE)\b/;++/** Match a ClickHouse server error. CH formats every exception as+ * `Code: NNN. DB::Exception: <text>. (NAMED_CONSTANT)` over the HTTP+ * interface. SQL errors, TOO_MANY_PARTS, TIMEOUT_EXCEEDED, auth errors —+ * all of them. */+const CH_SERVER_ERROR_PREFIX = /^Code:\s*\d+/;++export function classifyError(err: unknown): ErrorClass {+ if (!err || typeof err !== 'object') {+ return 'other';+ }+ const e = err as {+ message?: string;+ code?: unknown;+ cause?: unknown;+ type?: unknown;+ };+ const msg = String(e.message ?? '');++ // ── Deny: ClickHouse server errors ────────────────────────────────────+ if (CH_SERVER_ERROR_PREFIX.test(msg)) {+ return 'ch-server';+ }+ if (typeof e.type === 'string' && e.type.startsWith('DB::')) {+ return 'ch-server';+ }++ // ── Explicit error.code wins ──────────────────────────────────────────+ if (typeof e.code === 'string') {+ if (NODE_DOWN_CODES.has(e.code)) return 'node-down';+ if (TRANSIENT_CODES.has(e.code)) return 'transient';+ }++ // ── Message keyword fallback ──────────────────────────────────────────+ if (NODE_DOWN_MESSAGE_KEYWORDS.some((k) => msg.includes(k))) {+ return 'node-down';+ }+ if (TRANSIENT_MESSAGE_KEYWORDS.some((k) => msg.includes(k))) {+ return 'transient';+ }++ // ── Bare-code substring fallback ──────────────────────────────────────+ if (NODE_DOWN_BARE_CODE_REGEX.test(msg)) return 'node-down';+ if (TRANSIENT_BARE_CODE_REGEX.test(msg)) return 'transient';++ // ── Wrapped causes (Node 18+ AggregateError, undici wrapping) ─────────+ if (e.cause) {+ return classifyError(e.cause);+ }++ return 'other';+}++/** Convenience: should we retry this error at all (regardless of whether we+ * sin-bin the node)? */+export function isRetriableConnectionError(err: unknown): boolean {+ const c = classifyError(err);+ return c === 'node-down' || c === 'transient';+}++export class RoundRobinPicker {+ private rrIndex = 0;+ private unhealthyUntil = new Map<number, number>();++ constructor(+ private readonly clients: ClickHouseClient[],+ private readonly urls: string[],+ /** How long to skip a node after a connection failure, in ms. */+ private readonly unhealthyMarkMs: number+ ) {+ if (clients.length === 0) {+ throw new Error('RoundRobinPicker: at least one client required');+ }+ if (clients.length !== urls.length) {+ throw new Error('RoundRobinPicker: clients and urls must be same length');+ }+ }++ get size(): number {+ return this.clients.length;+ }++ get allUrls(): readonly string[] {+ return this.urls;+ }++ /**+ * Pick the next client in round-robin order. Skips nodes that were+ * recently marked unhealthy. If all are marked unhealthy, returns the+ * next one anyway (the marks might be stale and we'd rather try than+ * fail).+ */+ next(): ClientSlot {+ if (this.clients.length === 1) {+ return { client: this.clients[0]!, url: this.urls[0]!, index: 0 };+ }+ const now = Date.now();+ for (let i = 0; i < this.clients.length; i++) {+ const idx = this.rrIndex;+ this.rrIndex = (this.rrIndex + 1) % this.clients.length;+ const until = this.unhealthyUntil.get(idx);+ if (!until || until <= now) {+ return {+ client: this.clients[idx]!,+ url: this.urls[idx]!,+ index: idx,+ };+ }+ }+ // All nodes are sin-binned — try the next one anyway.+ const idx = this.rrIndex;+ this.rrIndex = (this.rrIndex + 1) % this.clients.length;+ this.unhealthyUntil.delete(idx);+ return { client: this.clients[idx]!, url: this.urls[idx]!, index: idx };+ }++ markUnhealthy(slot: ClientSlot): void {+ if (this.clients.length > 1) {+ this.unhealthyUntil.set(slot.index, Date.now() + this.unhealthyMarkMs);+ }+ }++ markHealthy(slot: ClientSlot): void {+ this.unhealthyUntil.delete(slot.index);+ }++ /** For debugging / metrics — which nodes are currently in the sin bin. */+ getUnhealthySnapshot(): { url: string; until: number }[] {+ const now = Date.now();+ const out: { url: string; until: number }[] = [];+ for (const [idx, until] of this.unhealthyUntil) {+ if (until > now) {+ out.push({ url: this.urls[idx] ?? '?', until });+ }+ }+ return out;+ }+}++/** Context passed to the operation callback so callers can record which+ * node was actually used (e.g. for logs / metrics). */+export interface OperationContext {+ url: string;+ index: number;+}++/**+ * Run `operation` against the round-robin picker. On a connection error,+ * mark that node unhealthy and try the next one. Non-connection errors+ * (e.g. SQL syntax) propagate immediately — retrying wouldn't help.+ */+export async function withRoundRobinRetry<T>(+ picker: RoundRobinPicker,+ operation: (client: ClickHouseClient, ctx: OperationContext) => Promise<T>,+ logger: ILogger,+ opts: {+ /** Max total attempts (default = picker.size + 1, min 3). */+ maxAttempts?: number;+ /** Base delay for exponential backoff in ms. */+ baseDelayMs?: number;+ /** Cap on backoff delay between attempts. */+ maxDelayMs?: number;+ } = {}+): Promise<T> {+ const maxAttempts = opts.maxAttempts ?? Math.max(3, picker.size + 1);+ const baseDelayMs = opts.baseDelayMs ?? 200;+ const maxDelayMs = opts.maxDelayMs ?? 2000;++ let lastError: unknown;++ for (let attempt = 0; attempt < maxAttempts; attempt++) {+ const slot = picker.next();+ try {+ const res = await operation(slot.client, {+ url: slot.url,+ index: slot.index,+ });+ if (attempt > 0) {+ picker.markHealthy(slot);+ logger.info(+ { url: slot.url, attempt: attempt + 1 },+ 'CH operation succeeded after retry'+ );+ }+ return res;+ } catch (error) {+ lastError = error;++ const errClass = classifyError(error);+ if (errClass === 'ch-server' || errClass === 'other') {+ // SQL/auth/business error or unknown shape — propagate immediately.+ // Retrying wouldn't help and we don't want to wrongly handle+ // unfamiliar errors as transport failures.+ throw error;+ }++ // Only sin-bin nodes that are actually unreachable. A `transient`+ // socket-level error (ECONNRESET, EPIPE, stream timeout) doesn't+ // mean the node is sick — it's a stale keep-alive race or one-off+ // glitch. Penalising the node would throw away a healthy replica.+ if (errClass === 'node-down') {+ picker.markUnhealthy(slot);+ }++ if (attempt === maxAttempts - 1) {+ break;+ }++ const delay = Math.min(baseDelayMs * 2 ** attempt, maxDelayMs);+ logger.warn(+ {+ err: (error as { message?: string })?.message,+ code: (error as { code?: string })?.code,+ errClass,+ markedUnhealthy: errClass === 'node-down',+ url: slot.url,+ attempt: attempt + 1,+ maxAttempts,+ nextDelayMs: delay,+ },+ errClass === 'node-down'+ ? 'CH node unreachable; sin-binning and retrying on next'+ : 'CH transient socket error; retrying on next'+ );+ await new Promise((r) => setTimeout(r, delay));+ }+ }++ throw lastError;+}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex ef7a87ac1..7241166c6 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1179,8 +1179,8 @@ importers:packages/db:dependencies:'@clickhouse/client':- specifier: ^1.12.1- version: 1.12.1+ specifier: ^1.18.5+ version: 1.18.5'@openpanel/common':specifier: workspace:*version: link:../common@@ -3454,11 +3454,11 @@ packages:'@clack/[email protected]':resolution: {integrity: sha512-/42G73JkuYdyWZ6m8d/CJtBrGl1Hegyc7Fy78m5Ob+jF85TOUmLR5XLce/U3LxYAw0kJ8CT5aI99RIvPHcGp/Q==}- '@clickhouse/[email protected]':- resolution: {integrity: sha512-ccw1N6hB4+MyaAHIaWBwGZ6O2GgMlO99FlMj0B0UEGfjxM9v5dYVYql6FpP19rMwrVAroYs/IgX2vyZEBvzQLg==}+ '@clickhouse/[email protected]':+ resolution: {integrity: sha512-g9LwcS1dvkatKDsIjT1PwUHldsiYzwdKAB0nXfd9APLd+t4PrNJa+my+dXcqJdmcWyhWjKLP/2/ztBwgxp+sbQ==}- '@clickhouse/[email protected]':- resolution: {integrity: sha512-7ORY85rphRazqHzImNXMrh4vsaPrpetFoTWpZYueCO2bbO6PXYDXp/GQ4DgxnGIqbWB/Di1Ai+Xuwq2o7DJ36A==}+ '@clickhouse/[email protected]':+ resolution: {integrity: sha512-4FfoyMkFWhsdNMuXsoEL6l3c12svA63BBJBtDo9SrxRZ14RdmN6jLr/rF3f84BK8cFoxETZCSeKlsbk6NNYebw==}engines: {node: '>=16'}'@cloudflare/[email protected]':@@ -23305,11 +23305,11 @@ snapshots:picocolors: 1.1.1sisteransi: 1.0.5- '@clickhouse/[email protected]': {}+ '@clickhouse/[email protected]': {}- '@clickhouse/[email protected]':+ '@clickhouse/[email protected]':dependencies:- '@clickhouse/client-common': 1.12.1+ '@clickhouse/client-common': 1.18.5'@cloudflare/[email protected]':dependencies:@@ -37640,7 +37640,7 @@ snapshots:compatx: 0.2.0confbox: 0.2.2consola: 3.4.2- cookie-es: 2.0.0+ cookie-es: 2.0.1croner: 9.1.0crossws: 0.3.5db0: 0.3.4@@ -42744,7 +42744,7 @@ snapshots:'@poppinss/colors': 4.1.5'@poppinss/dumper': 0.6.5'@speed-highlight/core': 1.2.12- cookie-es: 2.0.0+ cookie-es: 2.0.1youch-core: 0.3.3