Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(jobs): bound telemetry growth with a retention handler

Juan Denis46fdf5fc13a1
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.py
index 6d24febb..46ae0378 100644
--- a/backend/app/jobs/builtin_handlers.py
+++ b/backend/app/jobs/builtin_handlers.py
@@ -315,6 +315,42 @@ def run_job_retention():
return None
+def run_telemetry_retention():
+ """Prune the telemetry stream so the database cannot grow without bound.
+
+ ``jobs`` has had retention since day one, but the three tables that grow
+ alongside it never did: ``queue_messages`` and ``system_events`` gain a row
+ per scheduler tick and ``api_usage_logs`` one per request. Measured on a
+ single-app box that is ~25k rows/day, or ~11 MB/day forever — which is how
+ a 25 GB host filled from nothing but routine updates (each update copies
+ the database twice as its pre-upgrade safety net).
+
+ Kept ``telemetry.retention_days`` (default 30); ``0`` disables it. Rows are
+ only deleted once terminal, and a queue message still referenced by a
+ surviving job is always kept.
+ """
+ from app.services.settings_service import SettingsService
+ from app.services import disk_reclaim_service
+ days = SettingsService.get('telemetry.retention_days', 30)
+ try:
+ days = int(days)
+ except (TypeError, ValueError):
+ days = 30
+ if days <= 0:
+ return None
+ # No VACUUM here: it needs an exclusive lock and free space equal to the
+ # database. Steady-state pruning keeps the freed pages on the freelist for
+ # reuse, so the file stops growing without ever blocking the panel. Use
+ # `serverkit disk` to actually shrink the file after a big backlog.
+ report = disk_reclaim_service.prune_telemetry(days=days, vacuum=False)
+ deleted = report.get('deleted_rows') or 0
+ if deleted:
+ logger.info('Telemetry retention pruned %s row(s): %s',
+ deleted, report.get('deleted'))
+ return {'deleted': deleted, 'by_table': report.get('deleted')}
+ return None
+
+
def run_security_feed_check():
"""Daily security-advisory feed check.
@@ -398,6 +434,7 @@ _BUILTINS = [
('builtin.extension_updates', run_extension_update_check, 'extension-updates', 86400, 600),
('builtin.security_feed', run_security_feed_check, 'security-feed', 86400, 600),
('builtin.job_retention', run_job_retention, 'job-retention', 21600, 1500),
+ ('builtin.telemetry_retention', run_telemetry_retention, 'telemetry-retention', 21600, 1800),
]
diff --git a/backend/app/services/settings_service.py b/backend/app/services/settings_service.py
index e4871aa7..7d027fec 100644
--- a/backend/app/services/settings_service.py
+++ b/backend/app/services/settings_service.py
@@ -54,6 +54,11 @@ class SettingsService:
'type': 'integer',
'description': 'Days to retain succeeded/cancelled background jobs before pruning (failed jobs are kept 3x as long). 0 disables pruning.'
},
+ 'telemetry.retention_days': {
+ 'value': 30,
+ 'type': 'integer',
+ 'description': 'Days to retain telemetry rows (queue messages, system events, API usage logs) before pruning. Without this the database grows ~11 MB/day forever. 0 disables pruning.'
+ },
'onboarding_use_cases': {
'value': [],
'type': 'json',
diff --git a/backend/tests/test_disk_reclaim.py b/backend/tests/test_disk_reclaim.py
index abca9b41..77ef28c6 100644
--- a/backend/tests/test_disk_reclaim.py
+++ b/backend/tests/test_disk_reclaim.py
@@ -325,6 +325,67 @@ def test_scan_telemetry_reports_counts_without_deleting(app):
assert still_there == 5, 'scan must not delete anything'
+# ── the scheduled handler that keeps this from recurring ────────────────────
+
+def test_telemetry_retention_handler_is_scheduled():
+ """Without a registered schedule the tables grow forever — which is the
+ whole reason a 25 GB host filled from nothing but updates."""
+ from app.jobs.builtin_handlers import _BUILTINS
+
+ entry = next((b for b in _BUILTINS if b[0] == 'builtin.telemetry_retention'), None)
+ assert entry is not None, 'telemetry retention must be a registered builtin'
+ kind, handler, name, interval, delay = entry
+ assert name == 'telemetry-retention'
+ assert 0 < interval <= 86400, 'must run at least daily'
+
+
+def test_telemetry_retention_handler_prunes(app):
+ from datetime import datetime, timedelta
+ from sqlalchemy import text
+ from app import db
+ from app.jobs.builtin_handlers import run_telemetry_retention
+ from app.services.settings_service import SettingsService
+
+ old = datetime.utcnow() - timedelta(days=90)
+ for i in range(3):
+ db.session.execute(text(
+ 'INSERT INTO api_usage_logs (method, endpoint, status_code, created_at) '
+ 'VALUES (:m, :e, 200, :c)'), {'m': 'GET', 'e': f'/x/{i}', 'c': old})
+ db.session.commit()
+
+ result = run_telemetry_retention()
+ assert result and result['deleted'] == 3
+ assert db.session.execute(text('SELECT COUNT(*) FROM api_usage_logs')).scalar() == 0
+
+
+def test_telemetry_retention_zero_disables(app):
+ from datetime import datetime, timedelta
+ from sqlalchemy import text
+ from app import db
+ from app.jobs.builtin_handlers import run_telemetry_retention
+ from app.services.settings_service import SettingsService
+
+ old = datetime.utcnow() - timedelta(days=90)
+ db.session.execute(text(
+ 'INSERT INTO api_usage_logs (method, endpoint, status_code, created_at) '
+ 'VALUES (:m, :e, 200, :c)'), {'m': 'GET', 'e': '/x', 'c': old})
+ db.session.commit()
+
+ SettingsService.set('telemetry.retention_days', 0)
+ assert run_telemetry_retention() is None
+ assert db.session.execute(text('SELECT COUNT(*) FROM api_usage_logs')).scalar() == 1
+
+
+def test_telemetry_retention_never_vacuums_inline(app, monkeypatch):
+ """VACUUM takes an exclusive lock and needs free space equal to the whole
+ database — never acceptable on a background tick."""
+ from app.jobs.builtin_handlers import run_telemetry_retention
+
+ monkeypatch.setattr(svc, '_vacuum', lambda *a, **k: pytest.fail(
+ 'the scheduled handler must not VACUUM'))
+ run_telemetry_retention()
+
+
# ── formatting ──────────────────────────────────────────────────────────────
@pytest.mark.parametrize('value,expected', [
diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.py
index ee96b869..a9c174c8 100644
--- a/backend/tests/test_jobs.py
+++ b/backend/tests/test_jobs.py
@@ -182,18 +182,23 @@ class TestBuiltins:
# tombstone holds its data volumes and source tree until purge, and
# nothing else calls purge_expired on a schedule.
assert 'builtin.recycle_retention' in kinds
- assert len([k for k in kinds if k.startswith('builtin.')]) == 14
+ # Bounds the telemetry stream. queue_messages/system_events grow a row
+ # per scheduler tick and api_usage_logs one per request; unpruned that
+ # is ~11 MB/day of database forever, which is what filled a 25 GB host
+ # from routine updates alone (each update copies the DB twice).
+ assert 'builtin.telemetry_retention' in kinds
+ assert len([k for k in kinds if k.startswith('builtin.')]) == 15
builtin_handlers.seed_builtin_schedules()
- # 14 builtin.* schedules (incl. job-retention, the monitor sweep, the
- # security-feed check and the recycle-bin retention sweep) +
- # login-link/SSO reapers + drift/FIM/bandwidth sweeps + the host doctor
- # sweep AND the fleet doctor sweep (plan 26) + the setup-health nag
- # (plan 22).
- assert ScheduledJob.query.count() == 22
+ # 15 builtin.* schedules (incl. job-retention, telemetry-retention, the
+ # monitor sweep, the security-feed check and the recycle-bin retention
+ # sweep) + login-link/SSO reapers + drift/FIM/bandwidth sweeps + the
+ # host doctor sweep AND the fleet doctor sweep (plan 26) + the
+ # setup-health nag (plan 22).
+ assert ScheduledJob.query.count() == 23
# Seeding twice doesn't duplicate.
builtin_handlers.seed_builtin_schedules()
- assert ScheduledJob.query.count() == 22
+ assert ScheduledJob.query.count() == 23
class TestApi: