Skip to content

A modern and elegant dashboard for network traffic visualization and analysis.

feat: Implement asynchronous ClickHouse writes with granular outcome handling and refined realtime store clearing.

luolei335178ed5c4b
diff --git a/apps/collector/src/modules/clickhouse/clickhouse.writer.ts b/apps/collector/src/modules/clickhouse/clickhouse.writer.ts
index 59571b6..acb078c 100644
--- a/apps/collector/src/modules/clickhouse/clickhouse.writer.ts
+++ b/apps/collector/src/modules/clickhouse/clickhouse.writer.ts
@@ -10,6 +10,23 @@ interface CountryMinuteUpdate {
timestampMs?: number;
}
+export interface TrafficWriteOutcome {
+ detailOk: boolean;
+ aggOk: boolean;
+}
+
+export class TrafficWriteError extends Error {
+ detailOk: boolean;
+ aggOk: boolean;
+
+ constructor(message: string, detailOk: boolean, aggOk: boolean) {
+ super(message);
+ this.name = 'TrafficWriteError';
+ this.detailOk = detailOk;
+ this.aggOk = aggOk;
+ }
+}
+
export class ClickHouseWriter {
private readonly config = loadClickHouseConfig();
private readonly writeEnabled =
@@ -42,8 +59,13 @@ export class ClickHouseWriter {
return this.writeEnabled;
}
- writeTrafficBatch(backendId: number, updates: TrafficUpdate[]): void {
- if (!this.writeEnabled || updates.length === 0) return;
+ writeTrafficBatch(
+ backendId: number,
+ updates: TrafficUpdate[],
+ ): Promise<TrafficWriteOutcome> {
+ if (!this.writeEnabled || updates.length === 0) {
+ return Promise.resolve({ detailOk: true, aggOk: true });
+ }
const detailRows = updates.map((item) => ({
backend_id: backendId,
@@ -78,13 +100,14 @@ export class ClickHouseWriter {
}
const aggRows = Array.from(aggMap.values());
- // Write to buffer tables — data goes to memory first, auto-flushed to disk every ~60s
- this.enqueue(() => this.insertRows('traffic_detail_buffer', detailRows, 'traffic'), detailRows.length);
- this.enqueue(() => this.insertRows('traffic_agg_buffer', aggRows, 'traffic'), aggRows.length);
+ return this.enqueue(
+ () => this.writeTrafficTables(detailRows, aggRows),
+ detailRows.length + aggRows.length,
+ );
}
- writeCountryBatch(backendId: number, updates: CountryMinuteUpdate[]): void {
- if (!this.writeEnabled || updates.length === 0) return;
+ writeCountryBatch(backendId: number, updates: CountryMinuteUpdate[]): Promise<void> {
+ if (!this.writeEnabled || updates.length === 0) return Promise.resolve();
const rows = updates.map((item) => ({
backend_id: backendId,
@@ -97,10 +120,16 @@ export class ClickHouseWriter {
connections: 1,
}));
- this.enqueue(() => this.insertRows('country_buffer', rows, 'country'), rows.length);
+ return this.enqueue(() => this.insertRows('country_buffer', rows, 'country'), rows.length);
}
- private enqueue(task: () => Promise<void>, rowCount: number): void {
+ /**
+ * Enqueue a write task onto the serial write chain.
+ * Returns a promise that **rejects** if the task fails or the batch is dropped,
+ * so callers can avoid clearing the realtime store on failure.
+ * The internal write chain itself always recovers so subsequent writes proceed.
+ */
+ private enqueue<T>(task: () => Promise<T>, rowCount: number): Promise<T> {
const safeRowCount = Math.max(0, rowCount);
if (
this.pendingBatches >= this.maxPendingBatches ||
@@ -111,21 +140,84 @@ export class ClickHouseWriter {
console.warn(
`[ClickHouse Writer] Dropped batch because pending queue is full (batches=${this.pendingBatches}/${this.maxPendingBatches}, rows=${this.pendingRows}/${this.maxPendingRows})`,
);
- return;
+ return Promise.reject(
+ new Error('ClickHouse write queue full, batch dropped'),
+ );
}
this.pendingBatches += 1;
this.pendingRows += safeRowCount;
+
+ // Deferred promise so failure surfaces to caller while chain continues.
+ let resolve!: (value: T) => void;
+ let reject!: (err: unknown) => void;
+ const result = new Promise<T>((res, rej) => {
+ resolve = res;
+ reject = rej;
+ });
+
this.writeChain = this.writeChain
.then(() => task())
+ .then((value) => {
+ resolve(value);
+ })
.catch((error) => {
const message = error instanceof Error ? error.message : String(error);
console.warn(`[ClickHouse Writer] Queue task failed: ${message}`);
+ reject(error);
})
.finally(() => {
this.pendingBatches = Math.max(0, this.pendingBatches - 1);
this.pendingRows = Math.max(0, this.pendingRows - safeRowCount);
});
+
+ return result;
+ }
+
+ private async writeTrafficTables(
+ detailRows: Array<Record<string, unknown>>,
+ aggRows: Array<Record<string, unknown>>,
+ ): Promise<TrafficWriteOutcome> {
+ let detailOk = false;
+ let aggOk = false;
+ let detailError: unknown;
+ let aggError: unknown;
+
+ try {
+ await this.insertRows('traffic_detail_buffer', detailRows, 'traffic');
+ detailOk = true;
+ } catch (error) {
+ detailError = error;
+ }
+
+ try {
+ await this.insertRows('traffic_agg_buffer', aggRows, 'traffic');
+ aggOk = true;
+ } catch (error) {
+ aggError = error;
+ }
+
+ if (!detailOk || !aggOk) {
+ const detailMsg =
+ detailError instanceof Error
+ ? detailError.message
+ : detailError
+ ? String(detailError)
+ : 'none';
+ const aggMsg =
+ aggError instanceof Error
+ ? aggError.message
+ : aggError
+ ? String(aggError)
+ : 'none';
+ throw new TrafficWriteError(
+ `traffic write failed detail_ok=${detailOk} agg_ok=${aggOk} detail_err=${detailMsg} agg_err=${aggMsg}`,
+ detailOk,
+ aggOk,
+ );
+ }
+
+ return { detailOk: true, aggOk: true };
}
private async insertRows(
@@ -169,6 +261,8 @@ export class ClickHouseWriter {
this.maybeLogMetrics();
const message = error instanceof Error ? error.message : String(error);
console.warn(`[ClickHouse Writer] Failed to insert ${metricType} batch rows=${rows.length}: ${message}`);
+ // Re-throw so enqueue's deferred promise rejects and caller retains realtime data
+ throw error;
}
}
diff --git a/apps/collector/src/modules/collector/batch-buffer.ts b/apps/collector/src/modules/collector/batch-buffer.ts
index 0740e85..7ef0ac9 100644
--- a/apps/collector/src/modules/collector/batch-buffer.ts
+++ b/apps/collector/src/modules/collector/batch-buffer.ts
@@ -6,7 +6,10 @@
*/
import type { StatsDatabase } from "../db/db.js";
import type { GeoIPService } from "../geo/geo.service.js";
-import { getClickHouseWriter } from "../clickhouse/clickhouse.writer.js";
+import {
+ getClickHouseWriter,
+ type TrafficWriteOutcome,
+} from "../clickhouse/clickhouse.writer.js";
import { shouldSkipSqliteStatsWrites } from "../stats/stats-write-mode.js";
export interface TrafficUpdate {
@@ -44,7 +47,13 @@ export interface FlushResult {
rules: number;
trafficOk: boolean;
countryOk: boolean;
+ hasTrafficUpdates: boolean;
+ hasCountryUpdates: boolean;
hasUpdates: boolean;
+ /** Pending traffic write, including detail+agg table status */
+ pendingTrafficWrite?: Promise<TrafficWriteOutcome>;
+ /** Pending country write */
+ pendingCountryWrite?: Promise<void>;
}
export class BatchBuffer {
@@ -120,15 +129,22 @@ export class BatchBuffer {
let trafficOk = true;
let countryOk = true;
+ const hasTrafficUpdates = updates.length > 0;
+ let hasCountryUpdates = false;
+ let pendingTrafficWrite: Promise<TrafficWriteOutcome> | undefined;
+ let pendingCountryWrite: Promise<void> | undefined;
- if (updates.length > 0) {
+ if (hasTrafficUpdates) {
try {
const reduceSQLiteWrites = clickHouseWriter.isEnabled() && process.env.CH_DISABLE_SQLITE_REDUCTION !== '1';
if (!skipSqliteStatsWrites) {
db.batchUpdateTrafficStats(backendId, updates, reduceSQLiteWrites);
}
if (clickHouseWriter.isEnabled()) {
- clickHouseWriter.writeTrafficBatch(backendId, updates);
+ pendingTrafficWrite = clickHouseWriter.writeTrafficBatch(
+ backendId,
+ updates,
+ );
}
} catch (err) {
trafficOk = false;
@@ -155,11 +171,17 @@ export class BatchBuffer {
download: r.download,
timestampMs: r.timestampMs,
}));
- if (!skipSqliteStatsWrites) {
- db.batchUpdateCountryStats(backendId, countryUpdates);
- }
- if (clickHouseWriter.isEnabled()) {
- clickHouseWriter.writeCountryBatch(backendId, countryUpdates);
+ hasCountryUpdates = countryUpdates.length > 0;
+ if (hasCountryUpdates) {
+ if (!skipSqliteStatsWrites) {
+ db.batchUpdateCountryStats(backendId, countryUpdates);
+ }
+ if (clickHouseWriter.isEnabled()) {
+ pendingCountryWrite = clickHouseWriter.writeCountryBatch(
+ backendId,
+ countryUpdates,
+ );
+ }
}
} catch (err) {
countryOk = false;
@@ -179,7 +201,11 @@ export class BatchBuffer {
rules: rules.size,
trafficOk,
countryOk,
- hasUpdates: updates.length > 0 || geoResults.length > 0,
+ hasTrafficUpdates,
+ hasCountryUpdates,
+ hasUpdates: hasTrafficUpdates || hasCountryUpdates,
+ pendingTrafficWrite,
+ pendingCountryWrite,
};
}
diff --git a/apps/collector/src/modules/collector/gateway.collector.ts b/apps/collector/src/modules/collector/gateway.collector.ts
index ae3094f..f67c234 100644
--- a/apps/collector/src/modules/collector/gateway.collector.ts
+++ b/apps/collector/src/modules/collector/gateway.collector.ts
@@ -2,6 +2,7 @@ import WebSocket from "ws";
import type { ConnectionsData } from "@neko-master/shared";
import { StatsDatabase } from "../db/db.js";
import { GeoIPService } from "../geo/geo.service.js";
+import { TrafficWriteError } from "../clickhouse/clickhouse.writer.js";
import { realtimeStore } from "../realtime/realtime.store.js";
import { BatchBuffer } from "./batch-buffer.js";
@@ -170,7 +171,7 @@ export function createCollector(
}
};
- const flushBatch = () => {
+ const flushBatch = async () => {
if (isFlushing || !batchBuffer.hasPending()) {
return;
}
@@ -179,10 +180,54 @@ export function createCollector(
try {
const stats = batchBuffer.flush(db, geoService, id, "Collector");
- if (stats.trafficOk) {
- realtimeStore.clearTraffic(id);
+ let trafficDetailOk = true;
+ let trafficAggOk = true;
+ if (stats.pendingTrafficWrite) {
+ try {
+ const outcome = await stats.pendingTrafficWrite;
+ trafficDetailOk = outcome.detailOk;
+ trafficAggOk = outcome.aggOk;
+ } catch (err) {
+ if (err instanceof TrafficWriteError) {
+ trafficDetailOk = err.detailOk;
+ trafficAggOk = err.aggOk;
+ } else {
+ trafficDetailOk = false;
+ trafficAggOk = false;
+ }
+ console.warn(
+ `[Collector:${id}] ClickHouse traffic write failed detail_ok=${trafficDetailOk} agg_ok=${trafficAggOk}`,
+ err,
+ );
+ }
+ }
+
+ if (stats.hasTrafficUpdates && stats.trafficOk) {
+ if (trafficDetailOk && trafficAggOk) {
+ realtimeStore.clearTraffic(id);
+ } else if (trafficDetailOk && !trafficAggOk) {
+ // Detail committed, agg failed: clear detail-side realtime only.
+ realtimeStore.clearTrafficDimensions(id);
+ } else if (!trafficDetailOk && trafficAggOk) {
+ // Agg committed, detail failed: clear summary-side realtime only.
+ realtimeStore.clearTrafficSummary(id);
+ }
+ }
+
+ let countryWriteOk = true;
+ if (stats.pendingCountryWrite) {
+ try {
+ await stats.pendingCountryWrite;
+ } catch (err) {
+ countryWriteOk = false;
+ console.warn(
+ `[Collector:${id}] ClickHouse country write failed, keeping realtime country store`,
+ err,
+ );
+ }
}
- if (stats.countryOk) {
+
+ if (stats.hasCountryUpdates && stats.countryOk && countryWriteOk) {
realtimeStore.clearCountries(id);
}
@@ -452,18 +497,24 @@ export function createCollector(
// Override disconnect to clear intervals
const originalDisconnect = collector.disconnect.bind(collector);
+ const waitForFlushThenDisconnect = async () => {
+ while (isFlushing) {
+ await new Promise((resolve) => setTimeout(resolve, 25));
+ }
+ await flushBatch();
+ };
collector.disconnect = () => {
if (flushInterval) {
clearInterval(flushInterval);
flushInterval = null;
- // Final flush
- flushBatch();
}
if (cleanupInterval) {
clearInterval(cleanupInterval);
cleanupInterval = null;
}
- originalDisconnect();
+ void waitForFlushThenDisconnect().finally(() => {
+ originalDisconnect();
+ });
};
return collector;
diff --git a/apps/collector/src/modules/collector/surge.collector.ts b/apps/collector/src/modules/collector/surge.collector.ts
index 0b59c56..0522f29 100644
--- a/apps/collector/src/modules/collector/surge.collector.ts
+++ b/apps/collector/src/modules/collector/surge.collector.ts
@@ -1,6 +1,7 @@
import { isIPv4, isIPv6 } from "net";
import type { StatsDatabase } from "../db/db.js";
import { GeoIPService } from "../geo/geo.service.js";
+import { TrafficWriteError } from "../clickhouse/clickhouse.writer.js";
import { realtimeStore } from "../realtime/realtime.store.js";
import type { SurgeRequest, SurgeRequestsData } from "@neko-master/shared";
import { calculateBackoffDelay } from "../../shared/utils/backoff.js";
@@ -292,7 +293,7 @@ export function createSurgeCollector(
}
};
- const flushBatch = () => {
+ const flushBatch = async () => {
if (isFlushing || !batchBuffer.hasPending()) {
return;
}
@@ -301,10 +302,52 @@ export function createSurgeCollector(
try {
const stats = batchBuffer.flush(db, geoService, id, "SurgeCollector");
- if (stats.trafficOk) {
- realtimeStore.clearTraffic(id);
+ let trafficDetailOk = true;
+ let trafficAggOk = true;
+ if (stats.pendingTrafficWrite) {
+ try {
+ const outcome = await stats.pendingTrafficWrite;
+ trafficDetailOk = outcome.detailOk;
+ trafficAggOk = outcome.aggOk;
+ } catch (err) {
+ if (err instanceof TrafficWriteError) {
+ trafficDetailOk = err.detailOk;
+ trafficAggOk = err.aggOk;
+ } else {
+ trafficDetailOk = false;
+ trafficAggOk = false;
+ }
+ console.warn(
+ `[SurgeCollector:${id}] ClickHouse traffic write failed detail_ok=${trafficDetailOk} agg_ok=${trafficAggOk}`,
+ err,
+ );
+ }
}
- if (stats.countryOk) {
+
+ if (stats.hasTrafficUpdates && stats.trafficOk) {
+ if (trafficDetailOk && trafficAggOk) {
+ realtimeStore.clearTraffic(id);
+ } else if (trafficDetailOk && !trafficAggOk) {
+ realtimeStore.clearTrafficDimensions(id);
+ } else if (!trafficDetailOk && trafficAggOk) {
+ realtimeStore.clearTrafficSummary(id);
+ }
+ }
+
+ let countryWriteOk = true;
+ if (stats.pendingCountryWrite) {
+ try {
+ await stats.pendingCountryWrite;
+ } catch (err) {
+ countryWriteOk = false;
+ console.warn(
+ `[SurgeCollector:${id}] ClickHouse country write failed, keeping realtime country store`,
+ err,
+ );
+ }
+ }
+
+ if (stats.hasCountryUpdates && stats.countryOk && countryWriteOk) {
realtimeStore.clearCountries(id);
}
@@ -697,17 +740,24 @@ export function createSurgeCollector(
});
const originalStop = collector.stop.bind(collector);
+ const waitForFlushThenStop = async () => {
+ while (isFlushing) {
+ await new Promise((resolve) => setTimeout(resolve, 25));
+ }
+ await flushBatch();
+ };
collector.stop = () => {
if (flushInterval) {
clearInterval(flushInterval);
flushInterval = null;
- flushBatch();
}
if (cleanupInterval) {
clearInterval(cleanupInterval);
cleanupInterval = null;
}
- originalStop();
+ void waitForFlushThenStop().finally(() => {
+ originalStop();
+ });
};
// Add debug stats method
diff --git a/apps/collector/src/modules/realtime/realtime.store.ts b/apps/collector/src/modules/realtime/realtime.store.ts
index 1d20b07..e32d4ad 100644
--- a/apps/collector/src/modules/realtime/realtime.store.ts
+++ b/apps/collector/src/modules/realtime/realtime.store.ts
@@ -1312,9 +1312,12 @@ export class RealtimeStore {
);
}
- clearTraffic(backendId: number): void {
+ clearTrafficSummary(backendId: number): void {
this.summaryByBackend.delete(backendId);
this.minuteByBackend.delete(backendId);
+ }
+
+ clearTrafficDimensions(backendId: number): void {
this.domainByBackend.delete(backendId);
this.ipByBackend.delete(backendId);
this.proxyByBackend.delete(backendId);
@@ -1325,6 +1328,11 @@ export class RealtimeStore {
this.ruleChainByBackend.delete(backendId);
}
+ clearTraffic(backendId: number): void {
+ this.clearTrafficSummary(backendId);
+ this.clearTrafficDimensions(backendId);
+ }
+
clearCountries(backendId: number): void {
this.countryByBackend.delete(backendId);
}