Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Age out history tables that nothing pruned

Juan Denis80eb84f14a82
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.py
index e775dd5d..78f88099 100644
--- a/backend/app/jobs/builtin_handlers.py
+++ b/backend/app/jobs/builtin_handlers.py
@@ -358,18 +358,22 @@ def run_telemetry_retention():
days = int(days)
except (TypeError, ValueError):
days = 30
- if days <= 0:
- return None
- # No VACUUM here: it needs an exclusive lock and free space equal to the
- # database. Steady-state pruning keeps the freed pages on the freelist for
- # reuse, so the file stops growing without ever blocking the panel. Use
- # `serverkit disk` to actually shrink the file after a big backlog.
- report = disk_reclaim_service.prune_telemetry(days=days, vacuum=False)
- deleted = report.get('deleted_rows') or 0
+ by_table = {}
+ if days > 0:
+ # No VACUUM here: it needs an exclusive lock and free space equal to
+ # the database. Steady-state pruning keeps the freed pages on the
+ # freelist for reuse, so the file stops growing without ever blocking
+ # the panel. Use `serverkit disk` to actually shrink the file.
+ report = disk_reclaim_service.prune_telemetry(days=days, vacuum=False)
+ by_table.update(report.get('deleted') or {})
+ # History tables (audit, notifications, cron runs, ...) ride the same tick
+ # under their own settings, so disabling telemetry retention leaves them on.
+ from app.services import history_retention_service
+ by_table.update(history_retention_service.prune())
+ deleted = sum(by_table.values())
if deleted:
- logger.info('Telemetry retention pruned %s row(s): %s',
- deleted, report.get('deleted'))
- return {'deleted': deleted, 'by_table': report.get('deleted')}
+ logger.info('Retention pruned %s row(s): %s', deleted, by_table)
+ return {'deleted': deleted, 'by_table': by_table}
return None
diff --git a/backend/app/services/history_retention_service.py b/backend/app/services/history_retention_service.py
new file mode 100644
index 00000000..cbdb2111
--- /dev/null
+++ b/backend/app/services/history_retention_service.py
@@ -0,0 +1,127 @@
+"""Age out history tables that nothing else prunes (plan 85 A4).
+
+Every table here records something that already happened — an audit entry,
+a delivered notification, a finished cron run — and grew forever: no handler
+deleted from any of them. They are pruned by age from the same 6-hour tick
+as telemetry (``builtin.telemetry_retention``).
+
+Rules each spec encodes:
+
+- Only finished rows go: a running cron run, a pending delivery or a live
+ deployment job is never touched, whatever its age.
+- The newest row of a series that the UI reads as "current" is kept (an
+ app's latest deployment job, its latest image update check).
+- Children before parents, and a parent only once it has no children left.
+
+``audit_log_retention_days`` (default 90) covers the audit log; everything
+else follows ``history.retention_days`` (default 90). ``0`` disables either.
+"""
+import logging
+from datetime import datetime, timedelta
+
+logger = logging.getLogger(__name__)
+
+AUDIT_SETTING = 'audit_log_retention_days'
+HISTORY_SETTING = 'history.retention_days'
+DEFAULT_DAYS = 90
+
+_DEPLOY_DONE = "('succeeded', 'failed', 'cancelled', 'rolled_back')"
+# A deployment job is kept while it is its app's newest one — that row is the
+# app's "last deploy" — and while it still has log lines (they go first).
+_DEPLOY_JOB_PRUNABLE = (
+ f"status IN {_DEPLOY_DONE} AND created_at < "
+ "(SELECT MAX(d2.created_at) FROM deployment_jobs d2 "
+ "WHERE d2.app_id = deployment_jobs.app_id)"
+)
+
+# (table, time column or None, extra WHERE clause or '', setting key), in
+# deletion order.
+SPECS = (
+ ('audit_logs', 'created_at', '', AUDIT_SETTING),
+ ('error_logs', 'last_seen', '', HISTORY_SETTING),
+ ('notification_deliveries', 'created_at', '', HISTORY_SETTING),
+ ('notifications', 'created_at',
+ 'NOT EXISTS (SELECT 1 FROM notification_deliveries nd '
+ 'WHERE nd.notification_id = notifications.id)', HISTORY_SETTING),
+ # Lines follow their job, not their own age: a long job's last lines can
+ # be newer than the cutoff, and would then pin the job forever.
+ ('deployment_job_logs', None,
+ 'job_id IN (SELECT id FROM deployment_jobs WHERE '
+ + _DEPLOY_JOB_PRUNABLE + ' AND created_at < :cutoff)', HISTORY_SETTING),
+ ('deployment_jobs', 'created_at',
+ _DEPLOY_JOB_PRUNABLE + ' AND NOT EXISTS (SELECT 1 FROM deployment_job_logs dl '
+ 'WHERE dl.job_id = deployment_jobs.id)', HISTORY_SETTING),
+ ('cron_runs', 'created_at', "status <> 'running'", HISTORY_SETTING),
+ ('webhook_logs', 'received_at', '', HISTORY_SETTING),
+ ('webhook_deliveries', 'received_at',
+ "status IN ('filtered', 'forwarded', 'failed')", HISTORY_SETTING),
+ ('event_deliveries', 'created_at', "status IN ('success', 'failed')", HISTORY_SETTING),
+ ('image_update_checks', 'checked_at',
+ 'checked_at < (SELECT MAX(c2.checked_at) FROM image_update_checks c2 '
+ 'WHERE c2.application_id = image_update_checks.application_id)', HISTORY_SETTING),
+ ('workflow_logs', 'timestamp', '', HISTORY_SETTING),
+ ('server_onboarding_logs', 'created_at', '', HISTORY_SETTING),
+ ('sandbox_runs', 'created_at', "status <> 'running'", HISTORY_SETTING),
+)
+
+
+def _days(key):
+ from app.services.settings_service import SettingsService
+ try:
+ return int(SettingsService.get(key, DEFAULT_DAYS))
+ except (TypeError, ValueError):
+ return DEFAULT_DAYS
+
+
+def _delete(table, clause, params, sqlite, batch_size):
+ from sqlalchemy import text
+ from app import db
+ if not sqlite:
+ # Other engines have no portable rowid; one statement, one commit.
+ result = db.session.execute(text(f'DELETE FROM {table} WHERE {clause}'), params)
+ db.session.commit()
+ return result.rowcount or 0
+ removed = 0
+ while True:
+ # Batch on rowid so a large backlog never locks the table in one go.
+ result = db.session.execute(text(
+ f'DELETE FROM {table} WHERE rowid IN '
+ f'(SELECT rowid FROM {table} WHERE {clause} LIMIT {int(batch_size)})'
+ ), params)
+ db.session.commit()
+ if not result.rowcount:
+ break
+ removed += result.rowcount
+ if result.rowcount < batch_size:
+ break
+ return removed
+
+
+def prune(now=None, batch_size=5000):
+ """Delete aged history rows. Returns ``{table: rows_deleted}`` for every
+ table that lost rows. A table missing from this database is skipped."""
+ from sqlalchemy import inspect
+ from app import db
+
+ now = now or datetime.utcnow()
+ present = set(inspect(db.engine).get_table_names())
+ sqlite = db.engine.dialect.name == 'sqlite'
+ days = {key: _days(key) for key in (AUDIT_SETTING, HISTORY_SETTING)}
+
+ deleted = {}
+ for table, time_col, extra, key in SPECS:
+ if table not in present or days[key] <= 0:
+ continue
+ cutoff = now - timedelta(days=days[key])
+ clause = ' AND '.join(part for part in (
+ f'{time_col} < :cutoff' if time_col else '', extra) if part)
+ try:
+ removed = _delete(table, clause, {'cutoff': cutoff}, sqlite, batch_size)
+ except Exception as exc:
+ # One table's schema drift must not stop the rest from pruning.
+ db.session.rollback()
+ logger.warning('History retention skipped %s: %s', table, exc)
+ continue
+ if removed:
+ deleted[table] = removed
+ return deleted
diff --git a/backend/tests/test_history_retention.py b/backend/tests/test_history_retention.py
new file mode 100644
index 00000000..fdb76a6c
--- /dev/null
+++ b/backend/tests/test_history_retention.py
@@ -0,0 +1,131 @@
+"""Plan 85 A4: history tables that nothing pruned are aged out, finished rows
+only, and the rows the UI reads as "current" survive."""
+from datetime import datetime, timedelta
+
+from sqlalchemy import text
+
+from app import db
+from app.services import history_retention_service as svc
+from app.services.settings_service import SettingsService
+
+NOW = datetime.utcnow()
+OLD = NOW - timedelta(days=120)
+NEW = NOW - timedelta(days=5)
+
+
+def _sql(statement, **params):
+ return db.session.execute(text(statement), params)
+
+
+def _count(table, where='1=1', **params):
+ return _sql(f'SELECT COUNT(*) FROM {table} WHERE {where}', **params).scalar()
+
+
+def _app_id(name):
+ user_id = _sql('SELECT id FROM users ORDER BY id LIMIT 1').scalar()
+ if user_id is None:
+ _sql("INSERT INTO users (email, username) VALUES ('[email protected]', 'retention')")
+ user_id = _sql('SELECT id FROM users ORDER BY id LIMIT 1').scalar()
+ from app.models import Application
+ application = Application(name=name, app_type='docker', user_id=user_id)
+ db.session.add(application)
+ db.session.commit()
+ return application.id
+
+
+def test_every_spec_runs_against_the_real_schema(app):
+ # A typo in a table or column name would only surface as a warning on a
+ # live box; on an empty database every spec must simply delete nothing.
+ tables = {spec[0] for spec in svc.SPECS}
+ from sqlalchemy import inspect
+ assert tables <= set(inspect(db.engine).get_table_names())
+ assert svc.prune() == {}
+
+
+def test_old_audit_rows_go_and_recent_ones_stay(app):
+ _sql("INSERT INTO audit_logs (action, created_at) VALUES ('old', :t)", t=OLD)
+ _sql("INSERT INTO audit_logs (action, created_at) VALUES ('new', :t)", t=NEW)
+ db.session.commit()
+
+ assert svc.prune(now=NOW) == {'audit_logs': 1}
+ assert [r[0] for r in _sql('SELECT action FROM audit_logs')] == ['new']
+
+
+def test_zero_keeps_everything(app):
+ _sql("INSERT INTO audit_logs (action, created_at) VALUES ('old', :t)", t=OLD)
+ db.session.commit()
+ SettingsService.set(svc.AUDIT_SETTING, 0)
+
+ assert svc.prune(now=NOW) == {}
+ assert _count('audit_logs') == 1
+
+
+def test_a_notification_goes_only_after_its_deliveries(app):
+ for key, created in (('old', OLD), ('new', NEW)):
+ _sql("INSERT INTO notifications (event_key, title, created_at) VALUES (:k, :k, :t)",
+ k=key, t=created)
+ old_id = _sql("SELECT id FROM notifications WHERE event_key = 'old'").scalar()
+ new_id = _sql("SELECT id FROM notifications WHERE event_key = 'new'").scalar()
+ # The old notification still has a recent delivery: it must stay.
+ _sql("INSERT INTO notification_deliveries (notification_id, channel, created_at) "
+ "VALUES (:n, 'email', :t)", n=old_id, t=NEW)
+ _sql("INSERT INTO notification_deliveries (notification_id, channel, created_at) "
+ "VALUES (:n, 'email', :t)", n=new_id, t=OLD)
+ db.session.commit()
+
+ svc.prune(now=NOW)
+ assert _count('notifications') == 2
+ assert _count('notification_deliveries') == 1
+
+
+def test_deployment_jobs_keep_the_newest_per_app_and_anything_live(app):
+ app_id = _app_id('retention-app')
+ rows = (
+ ('ancient-done', 'succeeded', OLD - timedelta(days=10)),
+ ('old-running', 'running', OLD - timedelta(days=5)),
+ ('newest-but-old', 'failed', OLD),
+ )
+ for job_id, status, created in rows:
+ _sql("INSERT INTO deployment_jobs (id, kind, status, app_id, created_at) "
+ "VALUES (:i, 'deploy', :s, :a, :t)", i=job_id, s=status, a=app_id, t=created)
+ _sql("INSERT INTO deployment_job_logs (job_id, message, created_at) "
+ "VALUES (:i, 'line', :t)", i=job_id, t=NEW) # lines newer than the job
+ db.session.commit()
+
+ svc.prune(now=NOW)
+ left = {r[0] for r in _sql('SELECT id FROM deployment_jobs')}
+ assert left == {'old-running', 'newest-but-old'}
+ assert _count('deployment_job_logs', "job_id = 'ancient-done'") == 0
+ assert _count('deployment_job_logs') == 2
+
+
+def test_a_running_cron_run_is_never_pruned(app):
+ _sql("INSERT INTO cron_runs (job_id, status, created_at) VALUES ('a', 'running', :t)", t=OLD)
+ _sql("INSERT INTO cron_runs (job_id, status, created_at) VALUES ('b', 'success', :t)", t=OLD)
+ db.session.commit()
+
+ svc.prune(now=NOW)
+ assert [r[0] for r in _sql('SELECT job_id FROM cron_runs')] == ['a']
+
+
+def test_the_latest_image_check_per_app_survives(app):
+ app_id = _app_id('image-app')
+ for checked in (OLD - timedelta(days=1), OLD):
+ _sql("INSERT INTO image_update_checks (application_id, image_ref, update_available, checked_at) "
+ "VALUES (:a, 'nginx:latest', 0, :t)", a=app_id, t=checked)
+ db.session.commit()
+
+ svc.prune(now=NOW)
+ assert _count('image_update_checks') == 1
+ assert _sql('SELECT checked_at FROM image_update_checks').scalar().startswith(str(OLD.date()))
+
+
+def test_the_retention_tick_runs_history_even_with_telemetry_off(app):
+ from app.jobs.builtin_handlers import run_telemetry_retention
+
+ _sql("INSERT INTO audit_logs (action, created_at) VALUES ('old', :t)", t=OLD)
+ db.session.commit()
+ SettingsService.set('telemetry.retention_days', 0)
+
+ result = run_telemetry_retention()
+ assert result == {'deleted': 1, 'by_table': {'audit_logs': 1}}