A modern and elegant dashboard for network traffic visualization and analysis.
feat: Implement asynchronous ClickHouse writes with granular outcome handling and refined realtime store clearing.
luolei335178ed5c4b
diff --git a/apps/collector/src/modules/clickhouse/clickhouse.writer.ts b/apps/collector/src/modules/clickhouse/clickhouse.writer.tsindex 59571b6..acb078c 100644--- a/apps/collector/src/modules/clickhouse/clickhouse.writer.ts+++ b/apps/collector/src/modules/clickhouse/clickhouse.writer.ts@@ -10,6 +10,23 @@ interface CountryMinuteUpdate {timestampMs?: number;}+export interface TrafficWriteOutcome {+ detailOk: boolean;+ aggOk: boolean;+}++export class TrafficWriteError extends Error {+ detailOk: boolean;+ aggOk: boolean;++ constructor(message: string, detailOk: boolean, aggOk: boolean) {+ super(message);+ this.name = 'TrafficWriteError';+ this.detailOk = detailOk;+ this.aggOk = aggOk;+ }+}+export class ClickHouseWriter {private readonly config = loadClickHouseConfig();private readonly writeEnabled =@@ -42,8 +59,13 @@ export class ClickHouseWriter {return this.writeEnabled;}- writeTrafficBatch(backendId: number, updates: TrafficUpdate[]): void {- if (!this.writeEnabled || updates.length === 0) return;+ writeTrafficBatch(+ backendId: number,+ updates: TrafficUpdate[],+ ): Promise<TrafficWriteOutcome> {+ if (!this.writeEnabled || updates.length === 0) {+ return Promise.resolve({ detailOk: true, aggOk: true });+ }const detailRows = updates.map((item) => ({backend_id: backendId,@@ -78,13 +100,14 @@ export class ClickHouseWriter {}const aggRows = Array.from(aggMap.values());- // Write to buffer tables — data goes to memory first, auto-flushed to disk every ~60s- this.enqueue(() => this.insertRows('traffic_detail_buffer', detailRows, 'traffic'), detailRows.length);- this.enqueue(() => this.insertRows('traffic_agg_buffer', aggRows, 'traffic'), aggRows.length);+ return this.enqueue(+ () => this.writeTrafficTables(detailRows, aggRows),+ detailRows.length + aggRows.length,+ );}- writeCountryBatch(backendId: number, updates: CountryMinuteUpdate[]): void {- if (!this.writeEnabled || updates.length === 0) return;+ writeCountryBatch(backendId: number, updates: CountryMinuteUpdate[]): Promise<void> {+ if (!this.writeEnabled || updates.length === 0) return Promise.resolve();const rows = updates.map((item) => ({backend_id: backendId,@@ -97,10 +120,16 @@ export class ClickHouseWriter {connections: 1,}));- this.enqueue(() => this.insertRows('country_buffer', rows, 'country'), rows.length);+ return this.enqueue(() => this.insertRows('country_buffer', rows, 'country'), rows.length);}- private enqueue(task: () => Promise<void>, rowCount: number): void {+ /**+ * Enqueue a write task onto the serial write chain.+ * Returns a promise that **rejects** if the task fails or the batch is dropped,+ * so callers can avoid clearing the realtime store on failure.+ * The internal write chain itself always recovers so subsequent writes proceed.+ */+ private enqueue<T>(task: () => Promise<T>, rowCount: number): Promise<T> {const safeRowCount = Math.max(0, rowCount);if (this.pendingBatches >= this.maxPendingBatches ||@@ -111,21 +140,84 @@ export class ClickHouseWriter {console.warn(`[ClickHouse Writer] Dropped batch because pending queue is full (batches=${this.pendingBatches}/${this.maxPendingBatches}, rows=${this.pendingRows}/${this.maxPendingRows})`,);- return;+ return Promise.reject(+ new Error('ClickHouse write queue full, batch dropped'),+ );}this.pendingBatches += 1;this.pendingRows += safeRowCount;++ // Deferred promise so failure surfaces to caller while chain continues.+ let resolve!: (value: T) => void;+ let reject!: (err: unknown) => void;+ const result = new Promise<T>((res, rej) => {+ resolve = res;+ reject = rej;+ });+this.writeChain = this.writeChain.then(() => task())+ .then((value) => {+ resolve(value);+ }).catch((error) => {const message = error instanceof Error ? error.message : String(error);console.warn(`[ClickHouse Writer] Queue task failed: ${message}`);+ reject(error);}).finally(() => {this.pendingBatches = Math.max(0, this.pendingBatches - 1);this.pendingRows = Math.max(0, this.pendingRows - safeRowCount);});++ return result;+ }++ private async writeTrafficTables(+ detailRows: Array<Record<string, unknown>>,+ aggRows: Array<Record<string, unknown>>,+ ): Promise<TrafficWriteOutcome> {+ let detailOk = false;+ let aggOk = false;+ let detailError: unknown;+ let aggError: unknown;++ try {+ await this.insertRows('traffic_detail_buffer', detailRows, 'traffic');+ detailOk = true;+ } catch (error) {+ detailError = error;+ }++ try {+ await this.insertRows('traffic_agg_buffer', aggRows, 'traffic');+ aggOk = true;+ } catch (error) {+ aggError = error;+ }++ if (!detailOk || !aggOk) {+ const detailMsg =+ detailError instanceof Error+ ? detailError.message+ : detailError+ ? String(detailError)+ : 'none';+ const aggMsg =+ aggError instanceof Error+ ? aggError.message+ : aggError+ ? String(aggError)+ : 'none';+ throw new TrafficWriteError(+ `traffic write failed detail_ok=${detailOk} agg_ok=${aggOk} detail_err=${detailMsg} agg_err=${aggMsg}`,+ detailOk,+ aggOk,+ );+ }++ return { detailOk: true, aggOk: true };}private async insertRows(@@ -169,6 +261,8 @@ export class ClickHouseWriter {this.maybeLogMetrics();const message = error instanceof Error ? error.message : String(error);console.warn(`[ClickHouse Writer] Failed to insert ${metricType} batch rows=${rows.length}: ${message}`);+ // Re-throw so enqueue's deferred promise rejects and caller retains realtime data+ throw error;}}diff --git a/apps/collector/src/modules/collector/batch-buffer.ts b/apps/collector/src/modules/collector/batch-buffer.tsindex 0740e85..7ef0ac9 100644--- a/apps/collector/src/modules/collector/batch-buffer.ts+++ b/apps/collector/src/modules/collector/batch-buffer.ts@@ -6,7 +6,10 @@*/import type { StatsDatabase } from "../db/db.js";import type { GeoIPService } from "../geo/geo.service.js";-import { getClickHouseWriter } from "../clickhouse/clickhouse.writer.js";+import {+ getClickHouseWriter,+ type TrafficWriteOutcome,+} from "../clickhouse/clickhouse.writer.js";import { shouldSkipSqliteStatsWrites } from "../stats/stats-write-mode.js";export interface TrafficUpdate {@@ -44,7 +47,13 @@ export interface FlushResult {rules: number;trafficOk: boolean;countryOk: boolean;+ hasTrafficUpdates: boolean;+ hasCountryUpdates: boolean;hasUpdates: boolean;+ /** Pending traffic write, including detail+agg table status */+ pendingTrafficWrite?: Promise<TrafficWriteOutcome>;+ /** Pending country write */+ pendingCountryWrite?: Promise<void>;}export class BatchBuffer {@@ -120,15 +129,22 @@ export class BatchBuffer {let trafficOk = true;let countryOk = true;+ const hasTrafficUpdates = updates.length > 0;+ let hasCountryUpdates = false;+ let pendingTrafficWrite: Promise<TrafficWriteOutcome> | undefined;+ let pendingCountryWrite: Promise<void> | undefined;- if (updates.length > 0) {+ if (hasTrafficUpdates) {try {const reduceSQLiteWrites = clickHouseWriter.isEnabled() && process.env.CH_DISABLE_SQLITE_REDUCTION !== '1';if (!skipSqliteStatsWrites) {db.batchUpdateTrafficStats(backendId, updates, reduceSQLiteWrites);}if (clickHouseWriter.isEnabled()) {- clickHouseWriter.writeTrafficBatch(backendId, updates);+ pendingTrafficWrite = clickHouseWriter.writeTrafficBatch(+ backendId,+ updates,+ );}} catch (err) {trafficOk = false;@@ -155,11 +171,17 @@ export class BatchBuffer {download: r.download,timestampMs: r.timestampMs,}));- if (!skipSqliteStatsWrites) {- db.batchUpdateCountryStats(backendId, countryUpdates);- }- if (clickHouseWriter.isEnabled()) {- clickHouseWriter.writeCountryBatch(backendId, countryUpdates);+ hasCountryUpdates = countryUpdates.length > 0;+ if (hasCountryUpdates) {+ if (!skipSqliteStatsWrites) {+ db.batchUpdateCountryStats(backendId, countryUpdates);+ }+ if (clickHouseWriter.isEnabled()) {+ pendingCountryWrite = clickHouseWriter.writeCountryBatch(+ backendId,+ countryUpdates,+ );+ }}} catch (err) {countryOk = false;@@ -179,7 +201,11 @@ export class BatchBuffer {rules: rules.size,trafficOk,countryOk,- hasUpdates: updates.length > 0 || geoResults.length > 0,+ hasTrafficUpdates,+ hasCountryUpdates,+ hasUpdates: hasTrafficUpdates || hasCountryUpdates,+ pendingTrafficWrite,+ pendingCountryWrite,};}diff --git a/apps/collector/src/modules/collector/gateway.collector.ts b/apps/collector/src/modules/collector/gateway.collector.tsindex ae3094f..f67c234 100644--- a/apps/collector/src/modules/collector/gateway.collector.ts+++ b/apps/collector/src/modules/collector/gateway.collector.ts@@ -2,6 +2,7 @@ import WebSocket from "ws";import type { ConnectionsData } from "@neko-master/shared";import { StatsDatabase } from "../db/db.js";import { GeoIPService } from "../geo/geo.service.js";+import { TrafficWriteError } from "../clickhouse/clickhouse.writer.js";import { realtimeStore } from "../realtime/realtime.store.js";import { BatchBuffer } from "./batch-buffer.js";@@ -170,7 +171,7 @@ export function createCollector(}};- const flushBatch = () => {+ const flushBatch = async () => {if (isFlushing || !batchBuffer.hasPending()) {return;}@@ -179,10 +180,54 @@ export function createCollector(try {const stats = batchBuffer.flush(db, geoService, id, "Collector");- if (stats.trafficOk) {- realtimeStore.clearTraffic(id);+ let trafficDetailOk = true;+ let trafficAggOk = true;+ if (stats.pendingTrafficWrite) {+ try {+ const outcome = await stats.pendingTrafficWrite;+ trafficDetailOk = outcome.detailOk;+ trafficAggOk = outcome.aggOk;+ } catch (err) {+ if (err instanceof TrafficWriteError) {+ trafficDetailOk = err.detailOk;+ trafficAggOk = err.aggOk;+ } else {+ trafficDetailOk = false;+ trafficAggOk = false;+ }+ console.warn(+ `[Collector:${id}] ClickHouse traffic write failed detail_ok=${trafficDetailOk} agg_ok=${trafficAggOk}`,+ err,+ );+ }+ }++ if (stats.hasTrafficUpdates && stats.trafficOk) {+ if (trafficDetailOk && trafficAggOk) {+ realtimeStore.clearTraffic(id);+ } else if (trafficDetailOk && !trafficAggOk) {+ // Detail committed, agg failed: clear detail-side realtime only.+ realtimeStore.clearTrafficDimensions(id);+ } else if (!trafficDetailOk && trafficAggOk) {+ // Agg committed, detail failed: clear summary-side realtime only.+ realtimeStore.clearTrafficSummary(id);+ }+ }++ let countryWriteOk = true;+ if (stats.pendingCountryWrite) {+ try {+ await stats.pendingCountryWrite;+ } catch (err) {+ countryWriteOk = false;+ console.warn(+ `[Collector:${id}] ClickHouse country write failed, keeping realtime country store`,+ err,+ );+ }}- if (stats.countryOk) {++ if (stats.hasCountryUpdates && stats.countryOk && countryWriteOk) {realtimeStore.clearCountries(id);}@@ -452,18 +497,24 @@ export function createCollector(// Override disconnect to clear intervalsconst originalDisconnect = collector.disconnect.bind(collector);+ const waitForFlushThenDisconnect = async () => {+ while (isFlushing) {+ await new Promise((resolve) => setTimeout(resolve, 25));+ }+ await flushBatch();+ };collector.disconnect = () => {if (flushInterval) {clearInterval(flushInterval);flushInterval = null;- // Final flush- flushBatch();}if (cleanupInterval) {clearInterval(cleanupInterval);cleanupInterval = null;}- originalDisconnect();+ void waitForFlushThenDisconnect().finally(() => {+ originalDisconnect();+ });};return collector;diff --git a/apps/collector/src/modules/collector/surge.collector.ts b/apps/collector/src/modules/collector/surge.collector.tsindex 0b59c56..0522f29 100644--- a/apps/collector/src/modules/collector/surge.collector.ts+++ b/apps/collector/src/modules/collector/surge.collector.ts@@ -1,6 +1,7 @@import { isIPv4, isIPv6 } from "net";import type { StatsDatabase } from "../db/db.js";import { GeoIPService } from "../geo/geo.service.js";+import { TrafficWriteError } from "../clickhouse/clickhouse.writer.js";import { realtimeStore } from "../realtime/realtime.store.js";import type { SurgeRequest, SurgeRequestsData } from "@neko-master/shared";import { calculateBackoffDelay } from "../../shared/utils/backoff.js";@@ -292,7 +293,7 @@ export function createSurgeCollector(}};- const flushBatch = () => {+ const flushBatch = async () => {if (isFlushing || !batchBuffer.hasPending()) {return;}@@ -301,10 +302,52 @@ export function createSurgeCollector(try {const stats = batchBuffer.flush(db, geoService, id, "SurgeCollector");- if (stats.trafficOk) {- realtimeStore.clearTraffic(id);+ let trafficDetailOk = true;+ let trafficAggOk = true;+ if (stats.pendingTrafficWrite) {+ try {+ const outcome = await stats.pendingTrafficWrite;+ trafficDetailOk = outcome.detailOk;+ trafficAggOk = outcome.aggOk;+ } catch (err) {+ if (err instanceof TrafficWriteError) {+ trafficDetailOk = err.detailOk;+ trafficAggOk = err.aggOk;+ } else {+ trafficDetailOk = false;+ trafficAggOk = false;+ }+ console.warn(+ `[SurgeCollector:${id}] ClickHouse traffic write failed detail_ok=${trafficDetailOk} agg_ok=${trafficAggOk}`,+ err,+ );+ }}- if (stats.countryOk) {++ if (stats.hasTrafficUpdates && stats.trafficOk) {+ if (trafficDetailOk && trafficAggOk) {+ realtimeStore.clearTraffic(id);+ } else if (trafficDetailOk && !trafficAggOk) {+ realtimeStore.clearTrafficDimensions(id);+ } else if (!trafficDetailOk && trafficAggOk) {+ realtimeStore.clearTrafficSummary(id);+ }+ }++ let countryWriteOk = true;+ if (stats.pendingCountryWrite) {+ try {+ await stats.pendingCountryWrite;+ } catch (err) {+ countryWriteOk = false;+ console.warn(+ `[SurgeCollector:${id}] ClickHouse country write failed, keeping realtime country store`,+ err,+ );+ }+ }++ if (stats.hasCountryUpdates && stats.countryOk && countryWriteOk) {realtimeStore.clearCountries(id);}@@ -697,17 +740,24 @@ export function createSurgeCollector(});const originalStop = collector.stop.bind(collector);+ const waitForFlushThenStop = async () => {+ while (isFlushing) {+ await new Promise((resolve) => setTimeout(resolve, 25));+ }+ await flushBatch();+ };collector.stop = () => {if (flushInterval) {clearInterval(flushInterval);flushInterval = null;- flushBatch();}if (cleanupInterval) {clearInterval(cleanupInterval);cleanupInterval = null;}- originalStop();+ void waitForFlushThenStop().finally(() => {+ originalStop();+ });};// Add debug stats methoddiff --git a/apps/collector/src/modules/realtime/realtime.store.ts b/apps/collector/src/modules/realtime/realtime.store.tsindex 1d20b07..e32d4ad 100644--- a/apps/collector/src/modules/realtime/realtime.store.ts+++ b/apps/collector/src/modules/realtime/realtime.store.ts@@ -1312,9 +1312,12 @@ export class RealtimeStore {);}- clearTraffic(backendId: number): void {+ clearTrafficSummary(backendId: number): void {this.summaryByBackend.delete(backendId);this.minuteByBackend.delete(backendId);+ }++ clearTrafficDimensions(backendId: number): void {this.domainByBackend.delete(backendId);this.ipByBackend.delete(backendId);this.proxyByBackend.delete(backendId);@@ -1325,6 +1328,11 @@ export class RealtimeStore {this.ruleChainByBackend.delete(backendId);}+ clearTraffic(backendId: number): void {+ this.clearTrafficSummary(backendId);+ this.clearTrafficDimensions(backendId);+ }+clearCountries(backendId: number): void {this.countryByBackend.delete(backendId);}