A modern and elegant dashboard for network traffic visualization and analysis.
feat: Introduce ClickHouse Buffer tables for traffic and country data, updating all relevant queries to use them and generalizing `.dockerignore` patterns.
luoleiea50c8b68fdd
diff --git a/.dockerignore b/.dockerignoreindex 537f098..4e10934 100644--- a/.dockerignore+++ b/.dockerignore@@ -1,10 +1,11 @@# Dependencies-node_modules+**/node_modules.pnpm-store-# Build outputs-.next-dist+# Build outputs & caches+**/.next+**/dist+**/.turbo*.tsbuildinfo# Developmentdiff --git a/apps/collector/src/clickhouse-compare.ts b/apps/collector/src/clickhouse-compare.tsindex e77692a..b64769a 100644--- a/apps/collector/src/clickhouse-compare.ts+++ b/apps/collector/src/clickhouse-compare.ts@@ -95,7 +95,7 @@ export class ClickHouseCompareService {SELECTtoUInt64(COALESCE(SUM(upload), 0)) as upload,toUInt64(COALESCE(SUM(download), 0)) as download-FROM ${this.config.database}.traffic_agg+FROM ${this.config.database}.traffic_agg_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toClickHouseDateTime(start)}')FORMAT JSONdiff --git a/apps/collector/src/clickhouse-reader.ts b/apps/collector/src/clickhouse-reader.tsindex fdf7f94..8cc0976 100644--- a/apps/collector/src/clickhouse-reader.ts+++ b/apps/collector/src/clickhouse-reader.ts@@ -80,7 +80,7 @@ SELECTtoUInt64(COALESCE(SUM(connections), 0)) AS totalConnections,toUInt64(COALESCE(SUM(upload), 0)) AS totalUpload,toUInt64(COALESCE(SUM(download), 0)) AS totalDownload-FROM ${this.config.database}.traffic_agg+FROM ${this.config.database}.traffic_agg_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -89,7 +89,7 @@ WHERE backend_id = ${backendId}SELECTtoUInt64(uniqIf(domain, domain != '')) AS uniqueDomains,toUInt64(uniqIf(ip, ip != '')) AS uniqueIPs-FROM ${this.config.database}.traffic_detail+FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -121,7 +121,7 @@ SELECTtoUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections,toString(max(minute)) AS lastSeen-FROM ${this.config.database}.traffic_detail+FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -156,7 +156,7 @@ SELECTtoUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections,toString(max(minute)) AS lastSeen-FROM ${this.config.database}.traffic_detail+FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -189,7 +189,7 @@ SELECTtoUInt64(SUM(upload)) AS upload,toUInt64(SUM(download)) AS download,toUInt64(SUM(connections)) AS connections-FROM ${this.config.database}.traffic_agg+FROM ${this.config.database}.traffic_agg_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -229,7 +229,7 @@ SELECTtoUInt64(SUM(upload)) AS totalUpload,toUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections-FROM ${this.config.database}.country_minute+FROM ${this.config.database}.country_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -257,7 +257,7 @@ LIMIT ${Math.max(1, limit)}SELECTtoUInt64(COALESCE(SUM(upload), 0)) AS upload,toUInt64(COALESCE(SUM(download), 0)) AS download-FROM ${this.config.database}.traffic_agg+FROM ${this.config.database}.traffic_agg_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -280,7 +280,7 @@ SELECTtoString(minute) AS time,toUInt64(SUM(upload)) AS upload,toUInt64(SUM(download)) AS download-FROM ${this.config.database}.traffic_agg+FROM ${this.config.database}.traffic_agg_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -312,7 +312,7 @@ SELECTtoString(${bucketExpr}) AS time,toUInt64(SUM(upload)) AS upload,toUInt64(SUM(download)) AS download-FROM ${this.config.database}.traffic_agg+FROM ${this.config.database}.traffic_agg_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -359,7 +359,7 @@ ORDER BY time ASCSELECT toUInt64(COUNT()) AS totalFROM (SELECT domain- FROM ${this.config.database}.traffic_detail+ FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -378,7 +378,7 @@ SELECTtoString(max(minute)) AS lastSeen,arrayDistinct(groupArrayIf(rule, rule != '')) AS rules,arrayDistinct(groupArrayIf(chain, chain != '')) AS chains-FROM ${this.config.database}.traffic_detail+FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -436,7 +436,7 @@ LIMIT ${limit} OFFSET ${offset}SELECT toUInt64(COUNT()) AS totalFROM (SELECT ip- FROM ${this.config.database}.traffic_detail+ FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -454,7 +454,7 @@ SELECTtoUInt64(SUM(connections)) AS totalConnections,toString(max(minute)) AS lastSeen,arrayDistinct(groupArrayIf(chain, chain != '')) AS chains-FROM ${this.config.database}.traffic_detail+FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -491,7 +491,7 @@ SELECTtoUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections,toString(max(minute)) AS lastSeen-FROM ${this.config.database}.traffic_detail+FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -521,7 +521,7 @@ SELECTtoUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections,toString(max(minute)) AS lastSeen-FROM ${this.config.database}.traffic_detail+FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -554,14 +554,14 @@ SELECTtoUInt64(COALESCE(SUM(connections), 0)) AS totalConnections,toUInt64(COALESCE(SUM(upload), 0)) AS totalUpload,toUInt64(COALESCE(SUM(download), 0)) AS totalDownload-FROM ${this.config.database}.traffic_agg+FROM ${this.config.database}.traffic_agg_bufferWHERE minute >= now() - INTERVAL 90 DAY`),this.query<Record<string, unknown>>(`SELECTtoUInt64(uniqIf(domain, domain != '')) AS uniqueDomains,toUInt64(uniqIf(ip, ip != '')) AS uniqueIPs-FROM ${this.config.database}.traffic_detail+FROM ${this.config.database}.traffic_detail_bufferWHERE minute >= now() - INTERVAL 90 DAY`),]);@@ -594,7 +594,7 @@ WHERE minute >= now() - INTERVAL 90 DAYSELECTrule,groupUniqArray(arrayElement(splitByString(' > ', chain), 1)) AS proxies-FROM ${this.config.database}.traffic_detail+FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND rule != ''AND chain != ''${timeWhere}@@ -879,7 +879,7 @@ SELECTtoUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections,toString(max(minute)) AS lastSeen-FROM ${this.config.database}.traffic_detail+FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -923,7 +923,7 @@ SELECTtoString(max(minute)) AS lastSeen,arrayDistinct(groupArrayIf(rule, rule != '')) AS rules,arrayDistinct(groupArrayIf(chain, chain != '')) AS chains-FROM ${this.config.database}.traffic_detail+FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -969,7 +969,7 @@ SELECTtoUInt64(SUM(connections)) AS totalConnections,toString(max(minute)) AS lastSeen,arrayDistinct(groupArrayIf(chain, chain != '')) AS chains-FROM ${this.config.database}.traffic_detail+FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -1010,7 +1010,7 @@ SELECTtoUInt64(SUM(upload)) AS totalUpload,toUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections-FROM ${this.config.database}.traffic_detail+FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -1042,7 +1042,7 @@ SELECTtoUInt64(SUM(upload)) AS totalUpload,toUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections-FROM ${this.config.database}.traffic_detail+FROM ${this.config.database}.traffic_detail_bufferWHERE backend_id = ${backendId}AND rule != ''AND chain != ''${timeWhere}${ruleWhere}diff --git a/apps/collector/src/clickhouse-writer.ts b/apps/collector/src/clickhouse-writer.tsindex 59fdb84..a6dc044 100644--- a/apps/collector/src/clickhouse-writer.ts+++ b/apps/collector/src/clickhouse-writer.ts@@ -78,10 +78,9 @@ class ClickHouseWriter {}const aggRows = Array.from(aggMap.values());- // Write to all three tables: legacy compat + optimised pair- this.enqueue(() => this.insertRows('traffic_minute', detailRows, 'traffic'), detailRows.length);- this.enqueue(() => this.insertRows('traffic_detail', detailRows, 'traffic'), detailRows.length);- this.enqueue(() => this.insertRows('traffic_agg', aggRows, 'traffic'), aggRows.length);+ // Write to buffer tables — data goes to memory first, auto-flushed to disk every ~60s+ this.enqueue(() => this.insertRows('traffic_detail_buffer', detailRows, 'traffic'), detailRows.length);+ this.enqueue(() => this.insertRows('traffic_agg_buffer', aggRows, 'traffic'), aggRows.length);}writeCountryBatch(backendId: number, updates: CountryMinuteUpdate[]): void {@@ -98,7 +97,7 @@ class ClickHouseWriter {connections: 1,}));- this.enqueue(() => this.insertRows('country_minute', rows, 'country'), rows.length);+ this.enqueue(() => this.insertRows('country_buffer', rows, 'country'), rows.length);}private enqueue(task: () => Promise<void>, rowCount: number): void {@@ -130,7 +129,7 @@ class ClickHouseWriter {}private async insertRows(- table: 'traffic_minute' | 'traffic_agg' | 'traffic_detail' | 'country_minute',+ table: 'traffic_agg_buffer' | 'traffic_detail_buffer' | 'country_buffer' | 'traffic_minute' | 'traffic_agg' | 'traffic_detail' | 'country_minute',rows: Array<Record<string, unknown>>,metricType: 'traffic' | 'country',): Promise<void> {diff --git a/apps/collector/src/clickhouse.ts b/apps/collector/src/clickhouse.tsindex 5c37abd..d31f520 100644--- a/apps/collector/src/clickhouse.ts+++ b/apps/collector/src/clickhouse.ts@@ -252,8 +252,54 @@ PARTITION BY toYYYYMM(minute)ORDER BY (backend_id, minute, domain, ip)TTL minute + INTERVAL 90 DAYSETTINGS index_granularity = 8192+`,+ );+ // Buffer tables — data is written to memory first, then auto-flushed to the+ // underlying MergeTree tables. This reduces INSERT frequency from ~40/min to+ // ~2/min and dramatically reduces ClickHouse background merge I/O.+ //+ // Buffer(database, destination_table, num_layers,+ // min_time, max_time,+ // min_rows, max_rows,+ // min_bytes, max_bytes)+ await runClickHouseQuery(+ config,+ `+CREATE TABLE IF NOT EXISTS ${config.database}.traffic_agg_buffer AS ${config.database}.traffic_agg+ENGINE = Buffer(+ '${config.database}', 'traffic_agg', 4,+ 10, 60,+ 100, 10000,+ 10000, 1000000+)+`,+ );++ await runClickHouseQuery(+ config,+ `+CREATE TABLE IF NOT EXISTS ${config.database}.traffic_detail_buffer AS ${config.database}.traffic_detail+ENGINE = Buffer(+ '${config.database}', 'traffic_detail', 4,+ 10, 60,+ 100, 10000,+ 10000, 1000000+)+`,+ );++ await runClickHouseQuery(+ config,+ `+CREATE TABLE IF NOT EXISTS ${config.database}.country_buffer AS ${config.database}.country_minute+ENGINE = Buffer(+ '${config.database}', 'country_minute', 4,+ 10, 60,+ 100, 10000,+ 10000, 1000000+)`,);- console.info('[ClickHouse] Schema ensured (traffic_minute, country_minute, traffic_agg, traffic_detail)');+ console.info('[ClickHouse] Schema ensured (traffic_minute, country_minute, traffic_agg, traffic_detail, buffer tables)');}