A modern and elegant dashboard for network traffic visualization and analysis.
refactor: update ClickHouse queries to use `traffic_agg` for aggregated data and `traffic_detail` for granular data.
luoleif4bd5c9ee974
diff --git a/apps/collector/src/clickhouse-compare.ts b/apps/collector/src/clickhouse-compare.tsindex 0889003..e77692a 100644--- a/apps/collector/src/clickhouse-compare.ts+++ b/apps/collector/src/clickhouse-compare.ts@@ -95,7 +95,7 @@ export class ClickHouseCompareService {SELECTtoUInt64(COALESCE(SUM(upload), 0)) as upload,toUInt64(COALESCE(SUM(download), 0)) as download-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_aggWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toClickHouseDateTime(start)}')FORMAT JSONdiff --git a/apps/collector/src/clickhouse-reader.ts b/apps/collector/src/clickhouse-reader.tsindex 18d1ebf..fdf7f94 100644--- a/apps/collector/src/clickhouse-reader.ts+++ b/apps/collector/src/clickhouse-reader.ts@@ -73,18 +73,38 @@ export class ClickHouseReader {start: string,end: string,): Promise<ClickHouseSummary | null> {- const rows = await this.query<ClickHouseSummary>(`+ // SUM from lightweight agg table; unique counts from detail table+ const [aggRows, detailRows] = await Promise.all([+ this.query<Record<string, unknown>>(`SELECTtoUInt64(COALESCE(SUM(connections), 0)) AS totalConnections,toUInt64(COALESCE(SUM(upload), 0)) AS totalUpload,- toUInt64(COALESCE(SUM(download), 0)) AS totalDownload,- toUInt64(uniqExactIf(domain, domain != '')) AS uniqueDomains,- toUInt64(uniqExactIf(ip, ip != '')) AS uniqueIPs-FROM ${this.config.database}.traffic_minute+ toUInt64(COALESCE(SUM(download), 0)) AS totalDownload+FROM ${this.config.database}.traffic_aggWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')-`);+`),+ this.query<Record<string, unknown>>(`+SELECT+ toUInt64(uniqIf(domain, domain != '')) AS uniqueDomains,+ toUInt64(uniqIf(ip, ip != '')) AS uniqueIPs+FROM ${this.config.database}.traffic_detail+WHERE backend_id = ${backendId}+ AND minute >= toDateTime('${this.toDateTime(start)}')+ AND minute <= toDateTime('${this.toDateTime(end)}')+`),+ ]);+ if (!aggRows || !detailRows) return null;+ const agg = aggRows[0] || {};+ const detail = detailRows[0] || {};+ const rows: ClickHouseSummary[] = [{+ totalConnections: Number(agg.totalConnections || 0),+ totalUpload: Number(agg.totalUpload || 0),+ totalDownload: Number(agg.totalDownload || 0),+ uniqueDomains: Number(detail.uniqueDomains || 0),+ uniqueIPs: Number(detail.uniqueIPs || 0),+ }];return rows?.[0] || null;}@@ -101,7 +121,7 @@ SELECTtoUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections,toString(max(minute)) AS lastSeen-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_detailWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -136,7 +156,7 @@ SELECTtoUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections,toString(max(minute)) AS lastSeen-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_detailWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -169,7 +189,7 @@ SELECTtoUInt64(SUM(upload)) AS upload,toUInt64(SUM(download)) AS download,toUInt64(SUM(connections)) AS connections-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_aggWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -237,7 +257,7 @@ LIMIT ${Math.max(1, limit)}SELECTtoUInt64(COALESCE(SUM(upload), 0)) AS upload,toUInt64(COALESCE(SUM(download), 0)) AS download-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_aggWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -260,7 +280,7 @@ SELECTtoString(minute) AS time,toUInt64(SUM(upload)) AS upload,toUInt64(SUM(download)) AS download-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_aggWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -292,7 +312,7 @@ SELECTtoString(${bucketExpr}) AS time,toUInt64(SUM(upload)) AS upload,toUInt64(SUM(download)) AS download-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_aggWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -339,7 +359,7 @@ ORDER BY time ASCSELECT toUInt64(COUNT()) AS totalFROM (SELECT domain- FROM ${this.config.database}.traffic_minute+ FROM ${this.config.database}.traffic_detailWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -358,7 +378,7 @@ SELECTtoString(max(minute)) AS lastSeen,arrayDistinct(groupArrayIf(rule, rule != '')) AS rules,arrayDistinct(groupArrayIf(chain, chain != '')) AS chains-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_detailWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -416,7 +436,7 @@ LIMIT ${limit} OFFSET ${offset}SELECT toUInt64(COUNT()) AS totalFROM (SELECT ip- FROM ${this.config.database}.traffic_minute+ FROM ${this.config.database}.traffic_detailWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -434,7 +454,7 @@ SELECTtoUInt64(SUM(connections)) AS totalConnections,toString(max(minute)) AS lastSeen,arrayDistinct(groupArrayIf(chain, chain != '')) AS chains-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_detailWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -466,21 +486,21 @@ LIMIT ${limit} OFFSET ${offset}): Promise<ProxyStats[] | null> {const rows = await this.query<Record<string, unknown>>(`SELECT- arrayElement(splitByString(' > ', chain), 1) AS chain,+ arrayElement(splitByString(' > ', chain), 1) AS proxy_name,toUInt64(SUM(upload)) AS totalUpload,toUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections,toString(max(minute)) AS lastSeen-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_detailWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')-GROUP BY chain+GROUP BY proxy_nameORDER BY (SUM(upload) + SUM(download)) DESC`);if (!rows) return null;return rows.map((row) => ({- chain: String(row.chain || ''),+ chain: String(row.proxy_name || ''),totalUpload: Number(row.totalUpload || 0),totalDownload: Number(row.totalDownload || 0),totalConnections: Number(row.totalConnections || 0),@@ -501,7 +521,7 @@ SELECTtoUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections,toString(max(minute)) AS lastSeen-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_detailWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -528,15 +548,30 @@ ORDER BY (SUM(upload) + SUM(download)) DESCuniqueIPs: number;backendCount: number;} | null> {- const rows = await this.query<Record<string, unknown>>(`+ const [aggRows, detailRows] = await Promise.all([+ this.query<Record<string, unknown>>(`SELECTtoUInt64(COALESCE(SUM(connections), 0)) AS totalConnections,toUInt64(COALESCE(SUM(upload), 0)) AS totalUpload,- toUInt64(COALESCE(SUM(download), 0)) AS totalDownload,- toUInt64(uniqExactIf(domain, domain != '')) AS uniqueDomains,- toUInt64(uniqExactIf(ip, ip != '')) AS uniqueIPs-FROM ${this.config.database}.traffic_minute-`);+ toUInt64(COALESCE(SUM(download), 0)) AS totalDownload+FROM ${this.config.database}.traffic_agg+WHERE minute >= now() - INTERVAL 90 DAY+`),+ this.query<Record<string, unknown>>(`+SELECT+ toUInt64(uniqIf(domain, domain != '')) AS uniqueDomains,+ toUInt64(uniqIf(ip, ip != '')) AS uniqueIPs+FROM ${this.config.database}.traffic_detail+WHERE minute >= now() - INTERVAL 90 DAY+`),+ ]);+ const rows = aggRows && detailRows ? [{+ totalConnections: (aggRows[0] as any)?.totalConnections || 0,+ totalUpload: (aggRows[0] as any)?.totalUpload || 0,+ totalDownload: (aggRows[0] as any)?.totalDownload || 0,+ uniqueDomains: (detailRows[0] as any)?.uniqueDomains || 0,+ uniqueIPs: (detailRows[0] as any)?.uniqueIPs || 0,+ }] : null;if (!rows || rows.length === 0) return null;const row = rows[0];return {@@ -559,7 +594,7 @@ FROM ${this.config.database}.traffic_minuteSELECTrule,groupUniqArray(arrayElement(splitByString(' > ', chain), 1)) AS proxies-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_detailWHERE backend_id = ${backendId}AND rule != ''AND chain != ''${timeWhere}@@ -844,7 +879,7 @@ SELECTtoUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections,toString(max(minute)) AS lastSeen-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_detailWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -888,7 +923,7 @@ SELECTtoString(max(minute)) AS lastSeen,arrayDistinct(groupArrayIf(rule, rule != '')) AS rules,arrayDistinct(groupArrayIf(chain, chain != '')) AS chains-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_detailWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -934,7 +969,7 @@ SELECTtoUInt64(SUM(connections)) AS totalConnections,toString(max(minute)) AS lastSeen,arrayDistinct(groupArrayIf(chain, chain != '')) AS chains-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_detailWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -975,7 +1010,7 @@ SELECTtoUInt64(SUM(upload)) AS totalUpload,toUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_detailWHERE backend_id = ${backendId}AND minute >= toDateTime('${this.toDateTime(start)}')AND minute <= toDateTime('${this.toDateTime(end)}')@@ -1007,7 +1042,7 @@ SELECTtoUInt64(SUM(upload)) AS totalUpload,toUInt64(SUM(download)) AS totalDownload,toUInt64(SUM(connections)) AS totalConnections-FROM ${this.config.database}.traffic_minute+FROM ${this.config.database}.traffic_detailWHERE backend_id = ${backendId}AND rule != ''AND chain != ''${timeWhere}${ruleWhere}@@ -1166,7 +1201,7 @@ ORDER BY rule, chain}private esc(value: string): string {- return value.replace(/'/g, "''");+ return value.replace(/\\/g, '\\\\').replace(/'/g, "''");}private parseSource(value: string | undefined): StatsQuerySource {diff --git a/apps/collector/src/clickhouse-writer.ts b/apps/collector/src/clickhouse-writer.tsindex e650e42..59fdb84 100644--- a/apps/collector/src/clickhouse-writer.ts+++ b/apps/collector/src/clickhouse-writer.ts@@ -45,7 +45,7 @@ class ClickHouseWriter {writeTrafficBatch(backendId: number, updates: TrafficUpdate[]): void {if (!this.writeEnabled || updates.length === 0) return;- const rows = updates.map((item) => ({+ const detailRows = updates.map((item) => ({backend_id: backendId,minute: this.toMinuteDateTime(item.timestampMs),domain: item.domain || '',@@ -63,7 +63,25 @@ class ClickHouseWriter {connections: 1,}));- this.enqueue(() => this.insertRows('traffic_minute', rows, 'traffic'), rows.length);+ // Pre-aggregate by (backend_id, minute) for the lightweight agg table+ const aggMap = new Map<string, { backend_id: number; minute: string; upload: number; download: number; connections: number }>();+ for (const row of detailRows) {+ const key = `${row.backend_id}:${row.minute}`;+ const existing = aggMap.get(key);+ if (existing) {+ existing.upload += row.upload;+ existing.download += row.download;+ existing.connections += row.connections;+ } else {+ aggMap.set(key, { backend_id: row.backend_id, minute: row.minute, upload: row.upload, download: row.download, connections: row.connections });+ }+ }+ const aggRows = Array.from(aggMap.values());++ // Write to all three tables: legacy compat + optimised pair+ this.enqueue(() => this.insertRows('traffic_minute', detailRows, 'traffic'), detailRows.length);+ this.enqueue(() => this.insertRows('traffic_detail', detailRows, 'traffic'), detailRows.length);+ this.enqueue(() => this.insertRows('traffic_agg', aggRows, 'traffic'), aggRows.length);}writeCountryBatch(backendId: number, updates: CountryMinuteUpdate[]): void {@@ -112,7 +130,7 @@ class ClickHouseWriter {}private async insertRows(- table: 'traffic_minute' | 'country_minute',+ table: 'traffic_minute' | 'traffic_agg' | 'traffic_detail' | 'country_minute',rows: Array<Record<string, unknown>>,metricType: 'traffic' | 'country',): Promise<void> {diff --git a/apps/collector/src/clickhouse.ts b/apps/collector/src/clickhouse.tsindex 9cde3b0..5c37abd 100644--- a/apps/collector/src/clickhouse.ts+++ b/apps/collector/src/clickhouse.ts@@ -213,5 +213,47 @@ SETTINGS index_granularity = 8192`,);- console.info('[ClickHouse] Schema ensured (traffic_minute, country_minute)');+ // Optimised tables — split by query pattern to reduce I/O+ // traffic_agg: lightweight SummingMergeTree for summary / trend / hourly queries+ await runClickHouseQuery(+ config,+ `+CREATE TABLE IF NOT EXISTS ${config.database}.traffic_agg (+ backend_id UInt32,+ minute DateTime,+ upload UInt64,+ download UInt64,+ connections UInt32+) ENGINE = SummingMergeTree+PARTITION BY toYYYYMM(minute)+ORDER BY (backend_id, minute)+TTL minute + INTERVAL 90 DAY+SETTINGS index_granularity = 8192+`,+ );++ // traffic_detail: full-dimension MergeTree for top-N / filter / rule-chain queries+ await runClickHouseQuery(+ config,+ `+CREATE TABLE IF NOT EXISTS ${config.database}.traffic_detail (+ backend_id UInt32,+ minute DateTime,+ domain String,+ ip String,+ source_ip String,+ chain String,+ rule String,+ upload UInt64,+ download UInt64,+ connections UInt32+) ENGINE = MergeTree+PARTITION BY toYYYYMM(minute)+ORDER BY (backend_id, minute, domain, ip)+TTL minute + INTERVAL 90 DAY+SETTINGS index_granularity = 8192+`,+ );++ console.info('[ClickHouse] Schema ensured (traffic_minute, country_minute, traffic_agg, traffic_detail)');}