Skip to content

A modern and elegant dashboard for network traffic visualization and analysis.

feat: Introduce ClickHouse Buffer tables for traffic and country data, updating all relevant queries to use them and generalizing `.dockerignore` patterns.

luoleiea50c8b68fdd
diff --git a/.dockerignore b/.dockerignore
index 537f098..4e10934 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,10 +1,11 @@
# Dependencies
-node_modules
+**/node_modules
.pnpm-store
-# Build outputs
-.next
-dist
+# Build outputs & caches
+**/.next
+**/dist
+**/.turbo
*.tsbuildinfo
# Development
diff --git a/apps/collector/src/clickhouse-compare.ts b/apps/collector/src/clickhouse-compare.ts
index e77692a..b64769a 100644
--- a/apps/collector/src/clickhouse-compare.ts
+++ b/apps/collector/src/clickhouse-compare.ts
@@ -95,7 +95,7 @@ export class ClickHouseCompareService {
SELECT
toUInt64(COALESCE(SUM(upload), 0)) as upload,
toUInt64(COALESCE(SUM(download), 0)) as download
-FROM ${this.config.database}.traffic_agg
+FROM ${this.config.database}.traffic_agg_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toClickHouseDateTime(start)}')
FORMAT JSON
diff --git a/apps/collector/src/clickhouse-reader.ts b/apps/collector/src/clickhouse-reader.ts
index fdf7f94..8cc0976 100644
--- a/apps/collector/src/clickhouse-reader.ts
+++ b/apps/collector/src/clickhouse-reader.ts
@@ -80,7 +80,7 @@ SELECT
toUInt64(COALESCE(SUM(connections), 0)) AS totalConnections,
toUInt64(COALESCE(SUM(upload), 0)) AS totalUpload,
toUInt64(COALESCE(SUM(download), 0)) AS totalDownload
-FROM ${this.config.database}.traffic_agg
+FROM ${this.config.database}.traffic_agg_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -89,7 +89,7 @@ WHERE backend_id = ${backendId}
SELECT
toUInt64(uniqIf(domain, domain != '')) AS uniqueDomains,
toUInt64(uniqIf(ip, ip != '')) AS uniqueIPs
-FROM ${this.config.database}.traffic_detail
+FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -121,7 +121,7 @@ SELECT
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections,
toString(max(minute)) AS lastSeen
-FROM ${this.config.database}.traffic_detail
+FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -156,7 +156,7 @@ SELECT
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections,
toString(max(minute)) AS lastSeen
-FROM ${this.config.database}.traffic_detail
+FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -189,7 +189,7 @@ SELECT
toUInt64(SUM(upload)) AS upload,
toUInt64(SUM(download)) AS download,
toUInt64(SUM(connections)) AS connections
-FROM ${this.config.database}.traffic_agg
+FROM ${this.config.database}.traffic_agg_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -229,7 +229,7 @@ SELECT
toUInt64(SUM(upload)) AS totalUpload,
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections
-FROM ${this.config.database}.country_minute
+FROM ${this.config.database}.country_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -257,7 +257,7 @@ LIMIT ${Math.max(1, limit)}
SELECT
toUInt64(COALESCE(SUM(upload), 0)) AS upload,
toUInt64(COALESCE(SUM(download), 0)) AS download
-FROM ${this.config.database}.traffic_agg
+FROM ${this.config.database}.traffic_agg_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -280,7 +280,7 @@ SELECT
toString(minute) AS time,
toUInt64(SUM(upload)) AS upload,
toUInt64(SUM(download)) AS download
-FROM ${this.config.database}.traffic_agg
+FROM ${this.config.database}.traffic_agg_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -312,7 +312,7 @@ SELECT
toString(${bucketExpr}) AS time,
toUInt64(SUM(upload)) AS upload,
toUInt64(SUM(download)) AS download
-FROM ${this.config.database}.traffic_agg
+FROM ${this.config.database}.traffic_agg_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -359,7 +359,7 @@ ORDER BY time ASC
SELECT toUInt64(COUNT()) AS total
FROM (
SELECT domain
- FROM ${this.config.database}.traffic_detail
+ FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -378,7 +378,7 @@ SELECT
toString(max(minute)) AS lastSeen,
arrayDistinct(groupArrayIf(rule, rule != '')) AS rules,
arrayDistinct(groupArrayIf(chain, chain != '')) AS chains
-FROM ${this.config.database}.traffic_detail
+FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -436,7 +436,7 @@ LIMIT ${limit} OFFSET ${offset}
SELECT toUInt64(COUNT()) AS total
FROM (
SELECT ip
- FROM ${this.config.database}.traffic_detail
+ FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -454,7 +454,7 @@ SELECT
toUInt64(SUM(connections)) AS totalConnections,
toString(max(minute)) AS lastSeen,
arrayDistinct(groupArrayIf(chain, chain != '')) AS chains
-FROM ${this.config.database}.traffic_detail
+FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -491,7 +491,7 @@ SELECT
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections,
toString(max(minute)) AS lastSeen
-FROM ${this.config.database}.traffic_detail
+FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -521,7 +521,7 @@ SELECT
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections,
toString(max(minute)) AS lastSeen
-FROM ${this.config.database}.traffic_detail
+FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -554,14 +554,14 @@ SELECT
toUInt64(COALESCE(SUM(connections), 0)) AS totalConnections,
toUInt64(COALESCE(SUM(upload), 0)) AS totalUpload,
toUInt64(COALESCE(SUM(download), 0)) AS totalDownload
-FROM ${this.config.database}.traffic_agg
+FROM ${this.config.database}.traffic_agg_buffer
WHERE minute >= now() - INTERVAL 90 DAY
`),
this.query<Record<string, unknown>>(`
SELECT
toUInt64(uniqIf(domain, domain != '')) AS uniqueDomains,
toUInt64(uniqIf(ip, ip != '')) AS uniqueIPs
-FROM ${this.config.database}.traffic_detail
+FROM ${this.config.database}.traffic_detail_buffer
WHERE minute >= now() - INTERVAL 90 DAY
`),
]);
@@ -594,7 +594,7 @@ WHERE minute >= now() - INTERVAL 90 DAY
SELECT
rule,
groupUniqArray(arrayElement(splitByString(' > ', chain), 1)) AS proxies
-FROM ${this.config.database}.traffic_detail
+FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND rule != ''
AND chain != ''${timeWhere}
@@ -879,7 +879,7 @@ SELECT
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections,
toString(max(minute)) AS lastSeen
-FROM ${this.config.database}.traffic_detail
+FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -923,7 +923,7 @@ SELECT
toString(max(minute)) AS lastSeen,
arrayDistinct(groupArrayIf(rule, rule != '')) AS rules,
arrayDistinct(groupArrayIf(chain, chain != '')) AS chains
-FROM ${this.config.database}.traffic_detail
+FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -969,7 +969,7 @@ SELECT
toUInt64(SUM(connections)) AS totalConnections,
toString(max(minute)) AS lastSeen,
arrayDistinct(groupArrayIf(chain, chain != '')) AS chains
-FROM ${this.config.database}.traffic_detail
+FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -1010,7 +1010,7 @@ SELECT
toUInt64(SUM(upload)) AS totalUpload,
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections
-FROM ${this.config.database}.traffic_detail
+FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND minute >= toDateTime('${this.toDateTime(start)}')
AND minute <= toDateTime('${this.toDateTime(end)}')
@@ -1042,7 +1042,7 @@ SELECT
toUInt64(SUM(upload)) AS totalUpload,
toUInt64(SUM(download)) AS totalDownload,
toUInt64(SUM(connections)) AS totalConnections
-FROM ${this.config.database}.traffic_detail
+FROM ${this.config.database}.traffic_detail_buffer
WHERE backend_id = ${backendId}
AND rule != ''
AND chain != ''${timeWhere}${ruleWhere}
diff --git a/apps/collector/src/clickhouse-writer.ts b/apps/collector/src/clickhouse-writer.ts
index 59fdb84..a6dc044 100644
--- a/apps/collector/src/clickhouse-writer.ts
+++ b/apps/collector/src/clickhouse-writer.ts
@@ -78,10 +78,9 @@ class ClickHouseWriter {
}
const aggRows = Array.from(aggMap.values());
- // Write to all three tables: legacy compat + optimised pair
- this.enqueue(() => this.insertRows('traffic_minute', detailRows, 'traffic'), detailRows.length);
- this.enqueue(() => this.insertRows('traffic_detail', detailRows, 'traffic'), detailRows.length);
- this.enqueue(() => this.insertRows('traffic_agg', aggRows, 'traffic'), aggRows.length);
+ // Write to buffer tables — data goes to memory first, auto-flushed to disk every ~60s
+ this.enqueue(() => this.insertRows('traffic_detail_buffer', detailRows, 'traffic'), detailRows.length);
+ this.enqueue(() => this.insertRows('traffic_agg_buffer', aggRows, 'traffic'), aggRows.length);
}
writeCountryBatch(backendId: number, updates: CountryMinuteUpdate[]): void {
@@ -98,7 +97,7 @@ class ClickHouseWriter {
connections: 1,
}));
- this.enqueue(() => this.insertRows('country_minute', rows, 'country'), rows.length);
+ this.enqueue(() => this.insertRows('country_buffer', rows, 'country'), rows.length);
}
private enqueue(task: () => Promise<void>, rowCount: number): void {
@@ -130,7 +129,7 @@ class ClickHouseWriter {
}
private async insertRows(
- table: 'traffic_minute' | 'traffic_agg' | 'traffic_detail' | 'country_minute',
+ table: 'traffic_agg_buffer' | 'traffic_detail_buffer' | 'country_buffer' | 'traffic_minute' | 'traffic_agg' | 'traffic_detail' | 'country_minute',
rows: Array<Record<string, unknown>>,
metricType: 'traffic' | 'country',
): Promise<void> {
diff --git a/apps/collector/src/clickhouse.ts b/apps/collector/src/clickhouse.ts
index 5c37abd..d31f520 100644
--- a/apps/collector/src/clickhouse.ts
+++ b/apps/collector/src/clickhouse.ts
@@ -252,8 +252,54 @@ PARTITION BY toYYYYMM(minute)
ORDER BY (backend_id, minute, domain, ip)
TTL minute + INTERVAL 90 DAY
SETTINGS index_granularity = 8192
+`,
+ );
+ // Buffer tables — data is written to memory first, then auto-flushed to the
+ // underlying MergeTree tables. This reduces INSERT frequency from ~40/min to
+ // ~2/min and dramatically reduces ClickHouse background merge I/O.
+ //
+ // Buffer(database, destination_table, num_layers,
+ // min_time, max_time,
+ // min_rows, max_rows,
+ // min_bytes, max_bytes)
+ await runClickHouseQuery(
+ config,
+ `
+CREATE TABLE IF NOT EXISTS ${config.database}.traffic_agg_buffer AS ${config.database}.traffic_agg
+ENGINE = Buffer(
+ '${config.database}', 'traffic_agg', 4,
+ 10, 60,
+ 100, 10000,
+ 10000, 1000000
+)
+`,
+ );
+
+ await runClickHouseQuery(
+ config,
+ `
+CREATE TABLE IF NOT EXISTS ${config.database}.traffic_detail_buffer AS ${config.database}.traffic_detail
+ENGINE = Buffer(
+ '${config.database}', 'traffic_detail', 4,
+ 10, 60,
+ 100, 10000,
+ 10000, 1000000
+)
+`,
+ );
+
+ await runClickHouseQuery(
+ config,
+ `
+CREATE TABLE IF NOT EXISTS ${config.database}.country_buffer AS ${config.database}.country_minute
+ENGINE = Buffer(
+ '${config.database}', 'country_minute', 4,
+ 10, 60,
+ 100, 10000,
+ 10000, 1000000
+)
`,
);
- console.info('[ClickHouse] Schema ensured (traffic_minute, country_minute, traffic_agg, traffic_detail)');
+ console.info('[ClickHouse] Schema ensured (traffic_minute, country_minute, traffic_agg, traffic_detail, buffer tables)');
}