Skip to content

A modern and elegant dashboard for network traffic visualization and analysis.

fix(clickhouse): track runtime health in writer to prevent data loss when CH is down

luoleieee093fa3b48
diff --git a/apps/collector/src/modules/clickhouse/clickhouse.writer.ts b/apps/collector/src/modules/clickhouse/clickhouse.writer.ts
index acb078c..60690b8 100644
--- a/apps/collector/src/modules/clickhouse/clickhouse.writer.ts
+++ b/apps/collector/src/modules/clickhouse/clickhouse.writer.ts
@@ -46,6 +46,11 @@ export class ClickHouseWriter {
private writeChain: Promise<void> = Promise.resolve();
private pendingBatches = 0;
private pendingRows = 0;
+ private consecutiveFailures = 0;
+ private readonly maxConsecutiveFailures = Math.max(
+ 1,
+ Number.parseInt(process.env.CH_UNHEALTHY_THRESHOLD || '5', 10) || 5,
+ );
private metricsWindowStartedAt = Date.now();
private metrics = {
trafficBatches: 0,
@@ -59,6 +64,15 @@ export class ClickHouseWriter {
return this.writeEnabled;
}
+ /**
+ * Returns true when the writer is configured AND has not exceeded the
+ * consecutive-failure threshold. Use this (not isEnabled) to decide
+ * whether it is safe to skip SQLite writes.
+ */
+ isHealthy(): boolean {
+ return this.writeEnabled && this.consecutiveFailures < this.maxConsecutiveFailures;
+ }
+
writeTrafficBatch(
backendId: number,
updates: TrafficUpdate[],
@@ -248,6 +262,10 @@ export class ClickHouseWriter {
throw new Error(`status=${response.status} body=${text.slice(0, 200)}`);
}
+ if (this.consecutiveFailures > 0) {
+ console.info('[ClickHouse Writer] Recovered from failure, marking healthy again.');
+ this.consecutiveFailures = 0;
+ }
if (metricType === 'traffic') {
this.metrics.trafficBatches += 1;
this.metrics.trafficRows += rows.length;
@@ -258,6 +276,12 @@ export class ClickHouseWriter {
this.maybeLogMetrics();
} catch (error) {
this.metrics.failures += 1;
+ this.consecutiveFailures += 1;
+ if (this.consecutiveFailures === this.maxConsecutiveFailures) {
+ console.warn(
+ `[ClickHouse Writer] Marked unhealthy after ${this.maxConsecutiveFailures} consecutive failures. SQLite writes will resume as fallback.`,
+ );
+ }
this.maybeLogMetrics();
const message = error instanceof Error ? error.message : String(error);
console.warn(`[ClickHouse Writer] Failed to insert ${metricType} batch rows=${rows.length}: ${message}`);
diff --git a/apps/collector/src/modules/collector/batch-buffer.ts b/apps/collector/src/modules/collector/batch-buffer.ts
index 59cc3fc..a8ed0a8 100644
--- a/apps/collector/src/modules/collector/batch-buffer.ts
+++ b/apps/collector/src/modules/collector/batch-buffer.ts
@@ -109,7 +109,7 @@ export class BatchBuffer {
): FlushResult {
const clickHouseWriter = getClickHouseWriter();
const skipSqliteStatsWrites = shouldSkipSqliteStatsWrites(
- clickHouseWriter.isEnabled(),
+ clickHouseWriter.isHealthy(),
);
const updates = Array.from(this.buffer.values());
const geoResults = [...this.geoQueue];
@@ -136,7 +136,7 @@ export class BatchBuffer {
if (hasTrafficUpdates) {
try {
- const reduceSQLiteWrites = clickHouseWriter.isEnabled() && process.env.CH_DISABLE_SQLITE_REDUCTION !== '1';
+ const reduceSQLiteWrites = clickHouseWriter.isHealthy() && process.env.CH_DISABLE_SQLITE_REDUCTION !== '1';
if (!skipSqliteStatsWrites) {
db.batchUpdateTrafficStats(backendId, updates, reduceSQLiteWrites);
}
diff --git a/apps/collector/src/modules/stats/stats-write-mode.test.ts b/apps/collector/src/modules/stats/stats-write-mode.test.ts
index 9d05bc5..94c754b 100644
--- a/apps/collector/src/modules/stats/stats-write-mode.test.ts
+++ b/apps/collector/src/modules/stats/stats-write-mode.test.ts
@@ -23,7 +23,7 @@ describe('stats-write-mode', () => {
expect(mode.shouldSkipSqliteStatsWrites(false)).toBe(false);
});
- it('should keep sqlite writes and warn when writer is disabled', async () => {
+ it('should keep sqlite writes and warn when writer is unhealthy', async () => {
process.env.CH_ONLY_MODE = '1';
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
const mode = await import('./stats-write-mode.js');
@@ -32,7 +32,7 @@ describe('stats-write-mode', () => {
expect(warn).toHaveBeenCalledTimes(1);
});
- it('should skip sqlite writes when writer is enabled', async () => {
+ it('should skip sqlite writes when writer is healthy', async () => {
process.env.CH_ONLY_MODE = '1';
const info = vi.spyOn(console, 'info').mockImplementation(() => {});
const mode = await import('./stats-write-mode.js');