A modern and elegant dashboard for network traffic visualization and analysis.
fix(clickhouse): track runtime health in writer to prevent data loss when CH is down
luoleieee093fa3b48
diff --git a/apps/collector/src/modules/clickhouse/clickhouse.writer.ts b/apps/collector/src/modules/clickhouse/clickhouse.writer.tsindex acb078c..60690b8 100644--- a/apps/collector/src/modules/clickhouse/clickhouse.writer.ts+++ b/apps/collector/src/modules/clickhouse/clickhouse.writer.ts@@ -46,6 +46,11 @@ export class ClickHouseWriter {private writeChain: Promise<void> = Promise.resolve();private pendingBatches = 0;private pendingRows = 0;+ private consecutiveFailures = 0;+ private readonly maxConsecutiveFailures = Math.max(+ 1,+ Number.parseInt(process.env.CH_UNHEALTHY_THRESHOLD || '5', 10) || 5,+ );private metricsWindowStartedAt = Date.now();private metrics = {trafficBatches: 0,@@ -59,6 +64,15 @@ export class ClickHouseWriter {return this.writeEnabled;}+ /**+ * Returns true when the writer is configured AND has not exceeded the+ * consecutive-failure threshold. Use this (not isEnabled) to decide+ * whether it is safe to skip SQLite writes.+ */+ isHealthy(): boolean {+ return this.writeEnabled && this.consecutiveFailures < this.maxConsecutiveFailures;+ }+writeTrafficBatch(backendId: number,updates: TrafficUpdate[],@@ -248,6 +262,10 @@ export class ClickHouseWriter {throw new Error(`status=${response.status} body=${text.slice(0, 200)}`);}+ if (this.consecutiveFailures > 0) {+ console.info('[ClickHouse Writer] Recovered from failure, marking healthy again.');+ this.consecutiveFailures = 0;+ }if (metricType === 'traffic') {this.metrics.trafficBatches += 1;this.metrics.trafficRows += rows.length;@@ -258,6 +276,12 @@ export class ClickHouseWriter {this.maybeLogMetrics();} catch (error) {this.metrics.failures += 1;+ this.consecutiveFailures += 1;+ if (this.consecutiveFailures === this.maxConsecutiveFailures) {+ console.warn(+ `[ClickHouse Writer] Marked unhealthy after ${this.maxConsecutiveFailures} consecutive failures. SQLite writes will resume as fallback.`,+ );+ }this.maybeLogMetrics();const message = error instanceof Error ? error.message : String(error);console.warn(`[ClickHouse Writer] Failed to insert ${metricType} batch rows=${rows.length}: ${message}`);diff --git a/apps/collector/src/modules/collector/batch-buffer.ts b/apps/collector/src/modules/collector/batch-buffer.tsindex 59cc3fc..a8ed0a8 100644--- a/apps/collector/src/modules/collector/batch-buffer.ts+++ b/apps/collector/src/modules/collector/batch-buffer.ts@@ -109,7 +109,7 @@ export class BatchBuffer {): FlushResult {const clickHouseWriter = getClickHouseWriter();const skipSqliteStatsWrites = shouldSkipSqliteStatsWrites(- clickHouseWriter.isEnabled(),+ clickHouseWriter.isHealthy(),);const updates = Array.from(this.buffer.values());const geoResults = [...this.geoQueue];@@ -136,7 +136,7 @@ export class BatchBuffer {if (hasTrafficUpdates) {try {- const reduceSQLiteWrites = clickHouseWriter.isEnabled() && process.env.CH_DISABLE_SQLITE_REDUCTION !== '1';+ const reduceSQLiteWrites = clickHouseWriter.isHealthy() && process.env.CH_DISABLE_SQLITE_REDUCTION !== '1';if (!skipSqliteStatsWrites) {db.batchUpdateTrafficStats(backendId, updates, reduceSQLiteWrites);}diff --git a/apps/collector/src/modules/stats/stats-write-mode.test.ts b/apps/collector/src/modules/stats/stats-write-mode.test.tsindex 9d05bc5..94c754b 100644--- a/apps/collector/src/modules/stats/stats-write-mode.test.ts+++ b/apps/collector/src/modules/stats/stats-write-mode.test.ts@@ -23,7 +23,7 @@ describe('stats-write-mode', () => {expect(mode.shouldSkipSqliteStatsWrites(false)).toBe(false);});- it('should keep sqlite writes and warn when writer is disabled', async () => {+ it('should keep sqlite writes and warn when writer is unhealthy', async () => {process.env.CH_ONLY_MODE = '1';const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});const mode = await import('./stats-write-mode.js');@@ -32,7 +32,7 @@ describe('stats-write-mode', () => {expect(warn).toHaveBeenCalledTimes(1);});- it('should skip sqlite writes when writer is enabled', async () => {+ it('should skip sqlite writes when writer is healthy', async () => {process.env.CH_ONLY_MODE = '1';const info = vi.spyOn(console, 'info').mockImplementation(() => {});const mode = await import('./stats-write-mode.js');