Skip to content

A modern and elegant dashboard for network traffic visualization and analysis.

refactor: update ClickHouse queries to use `traffic_agg` for aggregated data and `traffic_detail` for granular data.

luoleif4bd5c9ee974
diff --git a/apps/collector/src/clickhouse-compare.ts b/apps/collector/src/clickhouse-compare.ts
index 0889003..e77692a 100644
--- a/apps/collector/src/clickhouse-compare.ts
+++ b/apps/collector/src/clickhouse-compare.ts
@@ -95,7 +95,7 @@ export class ClickHouseCompareService {
SELECT
toUInt64(COALESCE(SUM(upload), 0)) as upload,
toUInt64(COALESCE(SUM(download), 0)) as download
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_agg
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toClickHouseDateTime(start)}')
FORMAT JSON
diff --git a/apps/collector/src/clickhouse-reader.ts b/apps/collector/src/clickhouse-reader.ts
index 18d1ebf..fdf7f94 100644
--- a/apps/collector/src/clickhouse-reader.ts
+++ b/apps/collector/src/clickhouse-reader.ts
@@ -73,18 +73,38 @@ export class ClickHouseReader {
start: string,
end: string,
): Promise<ClickHouseSummary | null> {
- const rows = await this.query<ClickHouseSummary>(`
+ // SUM from lightweight agg table; unique counts from detail table
+ const [aggRows, detailRows] = await Promise.all([
+ this.query<Record<string, unknown>>(`
SELECT
toUInt64(COALESCE(SUM(connections), 0)) AS totalConnections,
toUInt64(COALESCE(SUM(upload), 0)) AS totalUpload,
- toUInt64(COALESCE(SUM(download), 0)) AS totalDownload,
- toUInt64(uniqExactIf(domain, domain != '')) AS uniqueDomains,
- toUInt64(uniqExactIf(ip, ip != '')) AS uniqueIPs
-FROM ${this.config.database}.traffic_minute
+ toUInt64(COALESCE(SUM(download), 0)) AS totalDownload
+FROM ${this.config.database}.traffic_agg
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
-`);
+`),
+ this.query<Record<string, unknown>>(`
+SELECT
+ toUInt64(uniqIf(domain, domain != '')) AS uniqueDomains,
+ toUInt64(uniqIf(ip, ip != '')) AS uniqueIPs
+FROM ${this.config.database}.traffic_detail
+WHERE backend_id = ${backendId}
+ AND minute >= toDateTime('${this.toDateTime(start)}')
+ AND minute <= toDateTime('${this.toDateTime(end)}')
+`),
+ ]);
+ if (!aggRows || !detailRows) return null;
+ const agg = aggRows[0] || {};
+ const detail = detailRows[0] || {};
+ const rows: ClickHouseSummary[] = [{
+ totalConnections: Number(agg.totalConnections || 0),
+ totalUpload: Number(agg.totalUpload || 0),
+ totalDownload: Number(agg.totalDownload || 0),
+ uniqueDomains: Number(detail.uniqueDomains || 0),
+ uniqueIPs: Number(detail.uniqueIPs || 0),
+ }];
return rows?.[0] || null;
}
@@ -101,7 +121,7 @@ SELECT
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections,
toString(max(minute)) AS lastSeen
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_detail
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -136,7 +156,7 @@ SELECT
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections,
toString(max(minute)) AS lastSeen
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_detail
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -169,7 +189,7 @@ SELECT
toUInt64(SUM(upload)) AS upload,
toUInt64(SUM(download)) AS download,
toUInt64(SUM(connections)) AS connections
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_agg
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -237,7 +257,7 @@ LIMIT ${Math.max(1, limit)}
SELECT
toUInt64(COALESCE(SUM(upload), 0)) AS upload,
toUInt64(COALESCE(SUM(download), 0)) AS download
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_agg
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -260,7 +280,7 @@ SELECT
toString(minute) AS time,
toUInt64(SUM(upload)) AS upload,
toUInt64(SUM(download)) AS download
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_agg
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -292,7 +312,7 @@ SELECT
toString(${bucketExpr}) AS time,
toUInt64(SUM(upload)) AS upload,
toUInt64(SUM(download)) AS download
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_agg
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -339,7 +359,7 @@ ORDER BY time ASC
SELECT toUInt64(COUNT()) AS total
FROM (
SELECT domain
- FROM ${this.config.database}.traffic_minute
+ FROM ${this.config.database}.traffic_detail
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -358,7 +378,7 @@ SELECT
toString(max(minute)) AS lastSeen,
arrayDistinct(groupArrayIf(rule, rule != '')) AS rules,
arrayDistinct(groupArrayIf(chain, chain != '')) AS chains
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_detail
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -416,7 +436,7 @@ LIMIT ${limit} OFFSET ${offset}
SELECT toUInt64(COUNT()) AS total
FROM (
SELECT ip
- FROM ${this.config.database}.traffic_minute
+ FROM ${this.config.database}.traffic_detail
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -434,7 +454,7 @@ SELECT
toUInt64(SUM(connections)) AS totalConnections,
toString(max(minute)) AS lastSeen,
arrayDistinct(groupArrayIf(chain, chain != '')) AS chains
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_detail
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -466,21 +486,21 @@ LIMIT ${limit} OFFSET ${offset}
): Promise<ProxyStats[] | null> {
const rows = await this.query<Record<string, unknown>>(`
SELECT
- arrayElement(splitByString(' > ', chain), 1) AS chain,
+ arrayElement(splitByString(' > ', chain), 1) AS proxy_name,
toUInt64(SUM(upload)) AS totalUpload,
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections,
toString(max(minute)) AS lastSeen
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_detail
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
-GROUP BY chain
+GROUP BY proxy_name
ORDER BY (SUM(upload) + SUM(download)) DESC
`);
if (!rows) return null;
return rows.map((row) => ({
- chain: String(row.chain || ''),
+ chain: String(row.proxy_name || ''),
totalUpload: Number(row.totalUpload || 0),
totalDownload: Number(row.totalDownload || 0),
totalConnections: Number(row.totalConnections || 0),
@@ -501,7 +521,7 @@ SELECT
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections,
toString(max(minute)) AS lastSeen
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_detail
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -528,15 +548,30 @@ ORDER BY (SUM(upload) + SUM(download)) DESC
uniqueIPs: number;
backendCount: number;
} | null> {
- const rows = await this.query<Record<string, unknown>>(`
+ const [aggRows, detailRows] = await Promise.all([
+ this.query<Record<string, unknown>>(`
SELECT
toUInt64(COALESCE(SUM(connections), 0)) AS totalConnections,
toUInt64(COALESCE(SUM(upload), 0)) AS totalUpload,
- toUInt64(COALESCE(SUM(download), 0)) AS totalDownload,
- toUInt64(uniqExactIf(domain, domain != '')) AS uniqueDomains,
- toUInt64(uniqExactIf(ip, ip != '')) AS uniqueIPs
-FROM ${this.config.database}.traffic_minute
-`);
+ toUInt64(COALESCE(SUM(download), 0)) AS totalDownload
+FROM ${this.config.database}.traffic_agg
+WHERE minute >= now() - INTERVAL 90 DAY
+`),
+ this.query<Record<string, unknown>>(`
+SELECT
+ toUInt64(uniqIf(domain, domain != '')) AS uniqueDomains,
+ toUInt64(uniqIf(ip, ip != '')) AS uniqueIPs
+FROM ${this.config.database}.traffic_detail
+WHERE minute >= now() - INTERVAL 90 DAY
+`),
+ ]);
+ const rows = aggRows && detailRows ? [{
+ totalConnections: (aggRows[0] as any)?.totalConnections || 0,
+ totalUpload: (aggRows[0] as any)?.totalUpload || 0,
+ totalDownload: (aggRows[0] as any)?.totalDownload || 0,
+ uniqueDomains: (detailRows[0] as any)?.uniqueDomains || 0,
+ uniqueIPs: (detailRows[0] as any)?.uniqueIPs || 0,
+ }] : null;
if (!rows || rows.length === 0) return null;
const row = rows[0];
return {
@@ -559,7 +594,7 @@ FROM ${this.config.database}.traffic_minute
SELECT
rule,
groupUniqArray(arrayElement(splitByString(' > ', chain), 1)) AS proxies
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_detail
WHERE backend_id = ${backendId}
AND rule != ''
AND chain != ''${timeWhere}
@@ -844,7 +879,7 @@ SELECT
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections,
toString(max(minute)) AS lastSeen
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_detail
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -888,7 +923,7 @@ SELECT
toString(max(minute)) AS lastSeen,
arrayDistinct(groupArrayIf(rule, rule != '')) AS rules,
arrayDistinct(groupArrayIf(chain, chain != '')) AS chains
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_detail
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -934,7 +969,7 @@ SELECT
toUInt64(SUM(connections)) AS totalConnections,
toString(max(minute)) AS lastSeen,
arrayDistinct(groupArrayIf(chain, chain != '')) AS chains
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_detail
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -975,7 +1010,7 @@ SELECT
toUInt64(SUM(upload)) AS totalUpload,
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_detail
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -1007,7 +1042,7 @@ SELECT
toUInt64(SUM(upload)) AS totalUpload,
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections
-FROM ${this.config.database}.traffic_minute
+FROM ${this.config.database}.traffic_detail
WHERE backend_id = ${backendId}
AND rule != ''
AND chain != ''${timeWhere}${ruleWhere}
@@ -1166,7 +1201,7 @@ ORDER BY rule, chain
}
private esc(value: string): string {
- return value.replace(/'/g, "''");
+ return value.replace(/\\/g, '\\\\').replace(/'/g, "''");
}
private parseSource(value: string | undefined): StatsQuerySource {
diff --git a/apps/collector/src/clickhouse-writer.ts b/apps/collector/src/clickhouse-writer.ts
index e650e42..59fdb84 100644
--- a/apps/collector/src/clickhouse-writer.ts
+++ b/apps/collector/src/clickhouse-writer.ts
@@ -45,7 +45,7 @@ class ClickHouseWriter {
writeTrafficBatch(backendId: number, updates: TrafficUpdate[]): void {
if (!this.writeEnabled || updates.length === 0) return;
- const rows = updates.map((item) => ({
+ const detailRows = updates.map((item) => ({
backend_id: backendId,
minute: this.toMinuteDateTime(item.timestampMs),
domain: item.domain || '',
@@ -63,7 +63,25 @@ class ClickHouseWriter {
connections: 1,
}));
- this.enqueue(() => this.insertRows('traffic_minute', rows, 'traffic'), rows.length);
+ // Pre-aggregate by (backend_id, minute) for the lightweight agg table
+ const aggMap = new Map<string, { backend_id: number; minute: string; upload: number; download: number; connections: number }>();
+ for (const row of detailRows) {
+ const key = `${row.backend_id}:${row.minute}`;
+ const existing = aggMap.get(key);
+ if (existing) {
+ existing.upload += row.upload;
+ existing.download += row.download;
+ existing.connections += row.connections;
+ } else {
+ aggMap.set(key, { backend_id: row.backend_id, minute: row.minute, upload: row.upload, download: row.download, connections: row.connections });
+ }
+ }
+ const aggRows = Array.from(aggMap.values());
+
+ // Write to all three tables: legacy compat + optimised pair
+ this.enqueue(() => this.insertRows('traffic_minute', detailRows, 'traffic'), detailRows.length);
+ this.enqueue(() => this.insertRows('traffic_detail', detailRows, 'traffic'), detailRows.length);
+ this.enqueue(() => this.insertRows('traffic_agg', aggRows, 'traffic'), aggRows.length);
}
writeCountryBatch(backendId: number, updates: CountryMinuteUpdate[]): void {
@@ -112,7 +130,7 @@ class ClickHouseWriter {
}
private async insertRows(
- table: 'traffic_minute' | 'country_minute',
+ table: 'traffic_minute' | 'traffic_agg' | 'traffic_detail' | 'country_minute',
rows: Array<Record<string, unknown>>,
metricType: 'traffic' | 'country',
): Promise<void> {
diff --git a/apps/collector/src/clickhouse.ts b/apps/collector/src/clickhouse.ts
index 9cde3b0..5c37abd 100644
--- a/apps/collector/src/clickhouse.ts
+++ b/apps/collector/src/clickhouse.ts
@@ -213,5 +213,47 @@ SETTINGS index_granularity = 8192
`,
);
- console.info('[ClickHouse] Schema ensured (traffic_minute, country_minute)');
+ // Optimised tables — split by query pattern to reduce I/O
+ // traffic_agg: lightweight SummingMergeTree for summary / trend / hourly queries
+ await runClickHouseQuery(
+ config,
+ `
+CREATE TABLE IF NOT EXISTS ${config.database}.traffic_agg (
+ backend_id UInt32,
+ minute DateTime,
+ upload UInt64,
+ download UInt64,
+ connections UInt32
+) ENGINE = SummingMergeTree
+PARTITION BY toYYYYMM(minute)
+ORDER BY (backend_id, minute)
+TTL minute + INTERVAL 90 DAY
+SETTINGS index_granularity = 8192
+`,
+ );
+
+ // traffic_detail: full-dimension MergeTree for top-N / filter / rule-chain queries
+ await runClickHouseQuery(
+ config,
+ `
+CREATE TABLE IF NOT EXISTS ${config.database}.traffic_detail (
+ backend_id UInt32,
+ minute DateTime,
+ domain String,
+ ip String,
+ source_ip String,
+ chain String,
+ rule String,
+ upload UInt64,
+ download UInt64,
+ connections UInt32
+) ENGINE = MergeTree
+PARTITION BY toYYYYMM(minute)
+ORDER BY (backend_id, minute, domain, ip)
+TTL minute + INTERVAL 90 DAY
+SETTINGS index_granularity = 8192
+`,
+ );
+
+ console.info('[ClickHouse] Schema ensured (traffic_minute, country_minute, traffic_agg, traffic_detail)');
}