Skip to content

A modern and elegant dashboard for network traffic visualization and analysis.

fix(retention): stop minute-level cutoff from deleting hourly dim tables

luolei56bcc61f84ca
diff --git a/apps/collector/src/database/repositories/config.repository.ts b/apps/collector/src/database/repositories/config.repository.ts
index dbf19be..5dc3a03 100644
--- a/apps/collector/src/database/repositories/config.repository.ts
+++ b/apps/collector/src/database/repositories/config.repository.ts
@@ -336,10 +336,6 @@ export class ConfigRepository extends BaseRepository {
this.db.prepare(`DELETE FROM minute_dim_stats WHERE minute < ?`).run(minuteCutoff);
this.db.prepare(`DELETE FROM minute_country_stats WHERE minute < ?`).run(minuteCutoff);
this.db.prepare(`DELETE FROM connection_logs WHERE timestamp < ?`).run(cutoff);
- // Also clean hourly_dim_stats and hourly_country_stats using the same cutoff (hour granularity)
- const hourCutoff = cutoff.slice(0, 13) + ':00:00';
- this.db.prepare(`DELETE FROM hourly_dim_stats WHERE hour < ?`).run(hourCutoff);
- this.db.prepare(`DELETE FROM hourly_country_stats WHERE hour < ?`).run(hourCutoff);
return this.db.prepare(`DELETE FROM minute_stats WHERE minute < ?`).run(minuteCutoff).changes;
}
@@ -348,6 +344,10 @@ export class ConfigRepository extends BaseRepository {
}
deleteOldHourlyStats(cutoff: string): number {
+ // hourly_dim/hourly_country back all >6h range queries (resolveFactTable),
+ // so they must follow the hourly retention window, not the minute one.
+ this.db.prepare(`DELETE FROM hourly_dim_stats WHERE hour < ?`).run(cutoff);
+ this.db.prepare(`DELETE FROM hourly_country_stats WHERE hour < ?`).run(cutoff);
return this.db.prepare(`DELETE FROM hourly_stats WHERE hour < ?`).run(cutoff).changes;
}
diff --git a/apps/collector/src/database/repositories/retention.test.ts b/apps/collector/src/database/repositories/retention.test.ts
new file mode 100644
index 0000000..f78c3d4
--- /dev/null
+++ b/apps/collector/src/database/repositories/retention.test.ts
@@ -0,0 +1,78 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import Database from 'better-sqlite3';
+import { getAllSchemaStatements } from '../schema.js';
+import { ConfigRepository } from './config.repository.js';
+
+const BACKEND_ID = 1;
+
+function createRawDb(): Database.Database {
+ const db = new Database(':memory:');
+ for (const stmt of getAllSchemaStatements()) {
+ db.exec(stmt);
+ }
+ db.prepare(`
+ INSERT INTO backend_configs (name, url, token, enabled, is_active, listening)
+ VALUES ('test', 'http://127.0.0.1:9090', '', 1, 1, 1)
+ `).run();
+ return db;
+}
+
+function seedDimRows(db: Database.Database, isoMinute: string) {
+ const hour = isoMinute.slice(0, 13) + ':00:00';
+ db.prepare(`
+ INSERT INTO minute_dim_stats (backend_id, minute, domain, ip, source_ip, chain, rule, upload, download, connections)
+ VALUES (?, ?, 'a.com', '1.1.1.1', '', 'ProxyA', 'RuleA', 1, 1, 1)
+ `).run(BACKEND_ID, isoMinute);
+ db.prepare(`
+ INSERT INTO hourly_dim_stats (backend_id, hour, domain, ip, source_ip, chain, rule, upload, download, connections)
+ VALUES (?, ?, 'a.com', '1.1.1.1', '', 'ProxyA', 'RuleA', 1, 1, 1)
+ `).run(BACKEND_ID, hour);
+ db.prepare(`
+ INSERT INTO hourly_country_stats (backend_id, hour, country, country_name, continent, upload, download, connections)
+ VALUES (?, ?, 'US', 'United States', 'NA', 1, 1, 1)
+ `).run(BACKEND_ID, hour);
+}
+
+function count(db: Database.Database, table: string): number {
+ return (db.prepare(`SELECT COUNT(*) as c FROM ${table}`).get() as { c: number }).c;
+}
+
+describe('tiered retention', () => {
+ let db: Database.Database;
+ let repo: ConfigRepository;
+
+ beforeEach(() => {
+ db = createRawDb();
+ repo = new ConfigRepository(db, ':memory:');
+ });
+
+ it('minute cutoff must not touch hourly dim/country tables', () => {
+ seedDimRows(db, '2026-06-01T00:00:00');
+
+ // 7-day style cutoff far after the seeded rows
+ repo.deleteOldMinuteStats('2026-06-20T00:00:00.000Z');
+
+ expect(count(db, 'minute_dim_stats')).toBe(0);
+ expect(count(db, 'hourly_dim_stats')).toBe(1);
+ expect(count(db, 'hourly_country_stats')).toBe(1);
+ });
+
+ it('hourly cutoff cleans hourly_stats plus hourly dim/country tables', () => {
+ seedDimRows(db, '2026-06-01T00:00:00');
+ db.prepare(`INSERT INTO hourly_stats (backend_id, hour, upload, download, connections) VALUES (?, '2026-06-01T00:00:00', 1, 1, 1)`).run(BACKEND_ID);
+
+ repo.deleteOldHourlyStats('2026-06-20T00:00:00');
+
+ expect(count(db, 'hourly_stats')).toBe(0);
+ expect(count(db, 'hourly_dim_stats')).toBe(0);
+ expect(count(db, 'hourly_country_stats')).toBe(0);
+ });
+
+ it('hourly cutoff keeps rows newer than the window', () => {
+ seedDimRows(db, '2026-06-25T00:00:00');
+
+ repo.deleteOldHourlyStats('2026-06-20T00:00:00');
+
+ expect(count(db, 'hourly_dim_stats')).toBe(1);
+ });
+});