A modern and elegant dashboard for network traffic visualization and analysis.
fix(retention,web): enable auto-cleanup by default and preserve backend URL on edit
luolei3322da1dd66f
diff --git a/CHANGELOG.en.md b/CHANGELOG.en.mdindex ef644d3..f06c98c 100644--- a/CHANGELOG.en.md+++ b/CHANGELOG.en.md@@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file.The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).+## [1.3.7] - 2026-04-12++### Fixed++- **SQLite retention policy silently disabled** 🐛 ([#66](https://github.com/foru17/neko-master/issues/66))+ - Fixed `autoCleanup` defaulting to `false` on fresh installs: when no row exists in `app_config`, the DB read returned `false` instead of the declared default `true`, so minute-level and hourly stats tables were never pruned+ - Fixed `backend_health_logs` never being cleaned up: the startup flow used an inline `runAutoCleanup` that only covered `minute_*` / `hourly_*` tables; the real `CleanupService` (which handles health logs) was never instantiated+ - Fixed UI autoCleanup toggle not taking effect: `CleanupService.start()` used to early-return when `autoCleanup=false`, so the timer was never scheduled and subsequent toggles had no effect+ - Added env-var overrides: `SQLITE_RETENTION_MINUTE_DAYS` / `SQLITE_RETENTION_HOURLY_DAYS` / `SQLITE_RETENTION_HEALTH_LOG_DAYS` for set-and-forget Docker Compose deployments+ - Thanks to @airobot-bot for the detailed row-level measurements++- **Backend URL silently rewritten on edit** 🐛 ([#65](https://github.com/foru17/neko-master/issues/65))+ - Fixed an issue where editing any field of a backend (including token-only changes) would rebuild the URL from `host` / `port` / `ssl`, silently dropping any path, query string, or embedded credentials present in the original URL+ - Fix: when `host` / `port` / `ssl` haven't been touched, the original URL is preserved as-is+ - Collector now logs the specific fields that changed when restarting a collector (e.g. `changed: token`), so users can self-verify from logs that the reconcile loop picked up their change+ - Thanks to @yuhongwei380 for the report+## [1.3.6] - 2026-03-26### Fixeddiff --git a/CHANGELOG.md b/CHANGELOG.mdindex b8721a1..18a8bc8 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -5,6 +5,23 @@格式基于 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.0.0/),并且本项目遵循 [语义化版本](https://semver.org/lang/zh-CN/spec/v2.0.0.html)。+## [1.3.7] - 2026-04-12++### 修复++- **SQLite 数据保留策略静默失效** 🐛 ([#66](https://github.com/foru17/neko-master/issues/66))+ - 修复全新安装的实例默认 `autoCleanup` 为 `false` 的 bug:`app_config` 表无配置行时读取返回 `false`,与代码声明的默认值 `true` 不一致,导致 minute 级和 hourly 级统计表从未被清理+ - 修复 `backend_health_logs` 从未被清理:启动流程使用的是内联 `runAutoCleanup`,只清 `minute_*` / `hourly_*`,真正包含健康日志清理的 `CleanupService` 从未被实例化+ - 修复 UI 切换 `autoCleanup` 开关不生效:`CleanupService.start()` 在 `autoCleanup=false` 时早退,定时器未挂起,后续切换无效+ - 新增环境变量覆盖:`SQLITE_RETENTION_MINUTE_DAYS` / `SQLITE_RETENTION_HOURLY_DAYS` / `SQLITE_RETENTION_HEALTH_LOG_DAYS`,便于 Docker Compose 用户 set-and-forget+ - 感谢 @airobot-bot 提供的详细行级统计数据++- **编辑后端时 URL 被静默重建** 🐛 ([#65](https://github.com/foru17/neko-master/issues/65))+ - 修复更新后端任何字段(包括只改密钥)时 URL 被强制重建的 bug:前端编辑表单只保留 `host` / `port` / `ssl`,保存时用 `buildDirectUrl` 重建,丢掉原 URL 中的 path / query / embedded credentials+ - 修复方式:当 `host` / `port` / `ssl` 未被修改时,保留原始 URL 原样提交+ - Collector 后端日志增强:重启时打印具体变更字段(如 `changed: token`),便于用户自证重启逻辑触发+ - 感谢 @yuhongwei380 的报告+## [1.3.6] - 2026-03-26### 修复diff --git a/apps/collector/src/database/repositories/config.repository.ts b/apps/collector/src/database/repositories/config.repository.tsindex be4e710..307a77d 100644--- a/apps/collector/src/database/repositories/config.repository.ts+++ b/apps/collector/src/database/repositories/config.repository.ts@@ -63,7 +63,8 @@ export class ConfigRepository extends BaseRepository {return {connectionLogsDays: parseInt(connectionLogsDays?.value || '7', 10),hourlyStatsDays: parseInt(hourlyStatsDays?.value || '30', 10),- autoCleanup: autoCleanup?.value === '1',+ // Default ON when no row exists; an explicit '0' disables it.+ autoCleanup: autoCleanup ? autoCleanup.value !== '0' : true,};}diff --git a/apps/collector/src/index.ts b/apps/collector/src/index.tsindex 7b61223..c548b09 100644--- a/apps/collector/src/index.ts+++ b/apps/collector/src/index.ts@@ -31,11 +31,35 @@ import {loadClickHouseConfig,} from './modules/clickhouse/clickhouse.config.js';import { ClickHouseCompareService } from './modules/clickhouse/clickhouse.compare.js';+import { CleanupService, type CleanupOverrides } from './modules/cleanup/index.js';const COLLECTOR_WS_PORT = parseInt(process.env.COLLECTOR_WS_PORT || '3002');const API_PORT = parseInt(process.env.API_PORT || '3001');const DB_PATH = process.env.DB_PATH || path.join(process.cwd(), 'stats.db');+/**+ * Parse an integer retention env var. Returns undefined for unset/invalid values+ * so the service falls back to the DB-stored config or built-in defaults.+ */+function parseRetentionEnvDays(name: string): number | undefined {+ const raw = process.env[name];+ if (raw === undefined || raw === '') return undefined;+ const parsed = Number.parseInt(raw, 10);+ if (!Number.isFinite(parsed) || parsed < 1) {+ console.warn(`[Cleanup] Ignoring invalid ${name}=${raw} (expected positive integer)`);+ return undefined;+ }+ return parsed;+}++function loadCleanupOverridesFromEnv(): CleanupOverrides {+ return {+ connectionLogsDays: parseRetentionEnvDays('SQLITE_RETENTION_MINUTE_DAYS'),+ hourlyStatsDays: parseRetentionEnvDays('SQLITE_RETENTION_HOURLY_DAYS'),+ healthLogDays: parseRetentionEnvDays('SQLITE_RETENTION_HEALTH_LOG_DAYS'),+ };+}+// Map of backend connections: backendId -> GatewayCollector | SurgeCollectorconst collectors = new Map<number, GatewayCollector | SurgeCollector>();let db: StatsDatabase;@@ -44,6 +68,7 @@ let apiServer: APIServer;let geoService: GeoIPService;let policySyncService: SurgePolicySyncService;let clickHouseCompareService: ClickHouseCompareService;+let cleanupService: CleanupService | undefined;// Track last known backend configs to detect changeslet lastBackendConfigs: Map<number, BackendConfig> = new Map();@@ -108,29 +133,10 @@ async function main() {// Check for backend config changes every 5 secondssetInterval(manageBackends, 5000);- // Auto-cleanup: enforce data retention policy- function runAutoCleanup() {- try {- const config = db.getRetentionConfig();- if (!config.autoCleanup) return;-- const connCutoff = new Date(Date.now() - config.connectionLogsDays * 86400000).toISOString();- const hourlyCutoff = new Date(Date.now() - config.hourlyStatsDays * 86400000).toISOString();-- const deletedLogs = db.deleteOldMinuteStats(connCutoff);- const deletedHourly = db.deleteOldHourlyStats(hourlyCutoff);-- if (deletedLogs > 0 || deletedHourly > 0) {- console.log(`[Cleanup] Deleted ${deletedLogs} connection logs, ${deletedHourly} hourly stats`);- }- } catch (err) {- console.error('[Cleanup] Auto-cleanup failed:', err);- }- }-- // First cleanup 30 seconds after startup, then every 6 hours- setTimeout(runAutoCleanup, 30000);- setInterval(runAutoCleanup, 6 * 60 * 60 * 1000);+ // Start the retention cleanup service. Env vars (SQLITE_RETENTION_*) override+ // the DB-stored config for operators who prefer set-and-forget Docker deploys.+ cleanupService = new CleanupService(db, loadCleanupOverridesFromEnv());+ cleanupService.start();// Handle graceful shutdownprocess.on('SIGINT', shutdown);@@ -152,16 +158,20 @@ async function manageBackends() {// Check if we need to start or restart this backend connectionconst needsStart = backend.listening && backend.enabled && !existingCollector && !isAgentBackend;- const needsRestart = existingCollector && lastConfig && (- lastConfig.url !== backend.url ||- lastConfig.token !== backend.token ||- lastConfig.type !== backend.type ||- lastConfig.listening !== backend.listening ||- lastConfig.enabled !== backend.enabled- );+ const changedFields: string[] = [];+ if (existingCollector && lastConfig) {+ if (lastConfig.url !== backend.url) changedFields.push('url');+ if (lastConfig.token !== backend.token) changedFields.push('token');+ if (lastConfig.type !== backend.type) changedFields.push('type');+ if (lastConfig.listening !== backend.listening) changedFields.push('listening');+ if (lastConfig.enabled !== backend.enabled) changedFields.push('enabled');+ }+ const needsRestart = changedFields.length > 0;if (needsRestart) {- console.log(`[Backends] Restarting collector for backend "${backend.name}" (ID: ${backend.id}) due to config change`);+ console.log(+ `[Backends] Restarting collector for backend "${backend.name}" (ID: ${backend.id}) — changed: ${changedFields.join(', ')}`,+ );stopCollector(backend.id);}@@ -290,6 +300,9 @@ function shutdown() {clickHouseCompareService?.stop();geoService?.destroy();+ // Stop retention cleanup+ cleanupService?.stop();+// Close databasedb?.close();diff --git a/apps/collector/src/modules/cleanup/cleanup.service.ts b/apps/collector/src/modules/cleanup/cleanup.service.tsindex be33500..e6f0593 100644--- a/apps/collector/src/modules/cleanup/cleanup.service.ts+++ b/apps/collector/src/modules/cleanup/cleanup.service.ts@@ -1,63 +1,78 @@import { StatsDatabase } from '../db/db.js';import { RetentionConfig, DEFAULT_RETENTION } from './cleanup.types.js';+export interface CleanupOverrides {+ cleanupInterval?: number;+ connectionLogsDays?: number;+ hourlyStatsDays?: number;+ healthLogDays?: number;+ autoCleanup?: boolean;+}+/*** Automatic data cleanup service- *+ ** Implements tiered data retention:- * - Raw connection logs: Short term (configurable, default 7 days)- * - Hourly stats: Medium term (configurable, default 30 days)+ * - Minute-level stats: Short term (configurable, default 7 days)+ * - Hourly stats: Medium term (configurable, default 30 days)+ * - Backend health logs: Medium term (defaults to hourlyStatsDays, independently overridable)* - Daily/domain stats: Long term (permanent, continuously updated)*/export class CleanupService {private db: StatsDatabase;private cleanupTimer: NodeJS.Timeout | null = null;private isRunning = false;- private customInterval: number | undefined;+ private overrides: CleanupOverrides;- constructor(db: StatsDatabase, config: Partial<RetentionConfig> = {}) {+ constructor(db: StatsDatabase, overrides: CleanupOverrides = {}) {this.db = db;- this.customInterval = config.cleanupInterval;+ this.overrides = overrides;}/**- * Get current config from database+ * Get effective config by merging (in precedence order):+ * env/constructor overrides > DB config > defaults.*/- private getConfig(): RetentionConfig {+ private getConfig(): RetentionConfig & { healthLogDays: number } {const dbConfig = this.db.getRetentionConfig();- return {+ const merged: RetentionConfig = {...DEFAULT_RETENTION,...dbConfig,- // Use custom interval if provided in constructor- cleanupInterval: this.customInterval ?? DEFAULT_RETENTION.cleanupInterval,+ cleanupInterval: this.overrides.cleanupInterval ?? DEFAULT_RETENTION.cleanupInterval,+ };+ if (this.overrides.connectionLogsDays !== undefined) merged.connectionLogsDays = this.overrides.connectionLogsDays;+ if (this.overrides.hourlyStatsDays !== undefined) merged.hourlyStatsDays = this.overrides.hourlyStatsDays;+ if (this.overrides.autoCleanup !== undefined) merged.autoCleanup = this.overrides.autoCleanup;+ return {+ ...merged,+ healthLogDays: this.overrides.healthLogDays ?? merged.hourlyStatsDays,};}/**- * Start automatic cleanup scheduling+ * Start automatic cleanup scheduling.+ *+ * The timer is always scheduled so that toggling autoCleanup via the UI+ * takes effect on the next tick without needing to restart the service.+ * Each tick re-reads the config and short-circuits if autoCleanup is off.*/start(): void {- const config = this.getConfig();-- if (!config.autoCleanup) {- console.log('[Cleanup] Auto-cleanup disabled');- return;- }-if (this.cleanupTimer) {return; // Already running}+ const config = this.getConfig();console.log(`[Cleanup] Starting with retention policy:`, {- connectionLogs: `${config.connectionLogsDays} days`,+ autoCleanup: config.autoCleanup,+ minuteStats: `${config.connectionLogsDays} days`,hourlyStats: `${config.hourlyStatsDays} days`,+ healthLogs: `${config.healthLogDays} days`,interval: `${config.cleanupInterval} hours`,});- // Run initial cleanup+ // Run initial cleanup immediately so upgrading users see the effect at boot.this.runCleanup();- // Schedule periodic cleanupconst intervalMs = config.cleanupInterval * 60 * 60 * 1000;this.cleanupTimer = setInterval(() => {this.runCleanup();@@ -79,6 +94,11 @@ export class CleanupService {* Run cleanup manually*/async runCleanup(): Promise<void> {+ const config = this.getConfig();+ if (!config.autoCleanup) {+ return;+ }+if (this.isRunning) {console.log('[Cleanup] Previous cleanup still running, skipping');return;@@ -88,13 +108,13 @@ export class CleanupService {const startTime = Date.now();try {- // Clean up old connection logs+ // Clean up old minute-level statsconst logsDeleted = this.cleanupConnectionLogs();// Clean up old hourly statsconst hourlyDeleted = this.cleanupHourlyStats();- // Clean up old health logs (same retention as hourly stats)+ // Clean up old health logs (independently overridable, defaults to hourlyStatsDays)this.cleanupHealthLogs();// Vacuum database to reclaim space (only if significant data deleted)@@ -126,11 +146,11 @@ export class CleanupService {}/**- * Clean up old health logs (retention matches hourly stats)+ * Clean up old health logs (uses healthLogDays override, falling back to hourlyStatsDays)*/private cleanupHealthLogs(): void {const config = this.getConfig();- this.db.repos.health.pruneOldLogs(config.hourlyStatsDays);+ this.db.repos.health.pruneOldLogs(config.healthLogDays);}/**@@ -167,14 +187,15 @@ export class CleanupService {hourlyStatsDays: config.hourlyStatsDays,autoCleanup: config.autoCleanup,});-+// Handle interval changeif (config.cleanupInterval !== undefined) {- this.customInterval = config.cleanupInterval;+ this.overrides.cleanupInterval = config.cleanupInterval;}-- // Restart if interval changed or autoCleanup was toggled- if ((config.cleanupInterval !== undefined || config.autoCleanup !== undefined) && this.cleanupTimer) {++ // Restart only when the tick interval changed — autoCleanup is re-evaluated+ // on every tick now, so toggling it doesn't require a restart.+ if (config.cleanupInterval !== undefined && this.cleanupTimer) {this.stop();this.start();}diff --git a/apps/collector/src/modules/cleanup/index.ts b/apps/collector/src/modules/cleanup/index.tsindex c1c13e0..22ee2d3 100644--- a/apps/collector/src/modules/cleanup/index.ts+++ b/apps/collector/src/modules/cleanup/index.ts@@ -1,3 +1,4 @@export { CleanupService } from './cleanup.service.js';+export type { CleanupOverrides } from './cleanup.service.js';export { DEFAULT_RETENTION } from './cleanup.types.js';export type { RetentionConfig } from './cleanup.types.js';diff --git a/apps/web/components/features/backend/backend-config-dialog.tsx b/apps/web/components/features/backend/backend-config-dialog.tsxindex e788905..f069922 100644--- a/apps/web/components/features/backend/backend-config-dialog.tsx+++ b/apps/web/components/features/backend/backend-config-dialog.tsx@@ -1078,11 +1078,27 @@ export function BackendConfigDialog({setLoading(true);try {- const url = isAgentMode- ? current && isAgentBackendUrl(current.url)- ? current.url- : buildAgentUrl(generateAgentMarker(name))- : buildDirectUrl(editFormData.host, editFormData.port, editFormData.ssl);+ // Preserve the original URL when host/port/ssl haven't been touched.+ // Rebuilding via buildDirectUrl would silently drop any path, query, or+ // embedded credentials present in the original URL — see issue #65.+ let url: string;+ if (isAgentMode) {+ url =+ current && isAgentBackendUrl(current.url)+ ? current.url+ : buildAgentUrl(generateAgentMarker(name));+ } else if (current && !isAgentBackendUrl(current.url)) {+ const parsed = parseBackendUrl(current.url);+ const hostUnchanged = parsed.host === editFormData.host.trim();+ const portUnchanged = parsed.port === editFormData.port.trim();+ const sslUnchanged = parsed.ssl === editFormData.ssl;+ url =+ hostUnchanged && portUnchanged && sslUnchanged+ ? current.url+ : buildDirectUrl(editFormData.host, editFormData.port, editFormData.ssl);+ } else {+ url = buildDirectUrl(editFormData.host, editFormData.port, editFormData.ssl);+ }await api.updateBackend(id, {name,url,diff --git a/package.json b/package.jsonindex 81042c2..faecae8 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "neko-master",- "version": "1.3.6",+ "version": "1.3.7","description": "Modern traffic analytics for edge gateways","private": true,"workspaces": [