ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
Age out history tables that nothing pruned
Juan Denis80eb84f14a82
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.pyindex e775dd5d..78f88099 100644--- a/backend/app/jobs/builtin_handlers.py+++ b/backend/app/jobs/builtin_handlers.py@@ -358,18 +358,22 @@ def run_telemetry_retention():days = int(days)except (TypeError, ValueError):days = 30- if days <= 0:- return None- # No VACUUM here: it needs an exclusive lock and free space equal to the- # database. Steady-state pruning keeps the freed pages on the freelist for- # reuse, so the file stops growing without ever blocking the panel. Use- # `serverkit disk` to actually shrink the file after a big backlog.- report = disk_reclaim_service.prune_telemetry(days=days, vacuum=False)- deleted = report.get('deleted_rows') or 0+ by_table = {}+ if days > 0:+ # No VACUUM here: it needs an exclusive lock and free space equal to+ # the database. Steady-state pruning keeps the freed pages on the+ # freelist for reuse, so the file stops growing without ever blocking+ # the panel. Use `serverkit disk` to actually shrink the file.+ report = disk_reclaim_service.prune_telemetry(days=days, vacuum=False)+ by_table.update(report.get('deleted') or {})+ # History tables (audit, notifications, cron runs, ...) ride the same tick+ # under their own settings, so disabling telemetry retention leaves them on.+ from app.services import history_retention_service+ by_table.update(history_retention_service.prune())+ deleted = sum(by_table.values())if deleted:- logger.info('Telemetry retention pruned %s row(s): %s',- deleted, report.get('deleted'))- return {'deleted': deleted, 'by_table': report.get('deleted')}+ logger.info('Retention pruned %s row(s): %s', deleted, by_table)+ return {'deleted': deleted, 'by_table': by_table}return Nonediff --git a/backend/app/services/history_retention_service.py b/backend/app/services/history_retention_service.pynew file mode 100644index 00000000..cbdb2111--- /dev/null+++ b/backend/app/services/history_retention_service.py@@ -0,0 +1,127 @@+"""Age out history tables that nothing else prunes (plan 85 A4).++Every table here records something that already happened — an audit entry,+a delivered notification, a finished cron run — and grew forever: no handler+deleted from any of them. They are pruned by age from the same 6-hour tick+as telemetry (``builtin.telemetry_retention``).++Rules each spec encodes:++- Only finished rows go: a running cron run, a pending delivery or a live+ deployment job is never touched, whatever its age.+- The newest row of a series that the UI reads as "current" is kept (an+ app's latest deployment job, its latest image update check).+- Children before parents, and a parent only once it has no children left.++``audit_log_retention_days`` (default 90) covers the audit log; everything+else follows ``history.retention_days`` (default 90). ``0`` disables either.+"""+import logging+from datetime import datetime, timedelta++logger = logging.getLogger(__name__)++AUDIT_SETTING = 'audit_log_retention_days'+HISTORY_SETTING = 'history.retention_days'+DEFAULT_DAYS = 90++_DEPLOY_DONE = "('succeeded', 'failed', 'cancelled', 'rolled_back')"+# A deployment job is kept while it is its app's newest one — that row is the+# app's "last deploy" — and while it still has log lines (they go first).+_DEPLOY_JOB_PRUNABLE = (+ f"status IN {_DEPLOY_DONE} AND created_at < "+ "(SELECT MAX(d2.created_at) FROM deployment_jobs d2 "+ "WHERE d2.app_id = deployment_jobs.app_id)"+)++# (table, time column or None, extra WHERE clause or '', setting key), in+# deletion order.+SPECS = (+ ('audit_logs', 'created_at', '', AUDIT_SETTING),+ ('error_logs', 'last_seen', '', HISTORY_SETTING),+ ('notification_deliveries', 'created_at', '', HISTORY_SETTING),+ ('notifications', 'created_at',+ 'NOT EXISTS (SELECT 1 FROM notification_deliveries nd '+ 'WHERE nd.notification_id = notifications.id)', HISTORY_SETTING),+ # Lines follow their job, not their own age: a long job's last lines can+ # be newer than the cutoff, and would then pin the job forever.+ ('deployment_job_logs', None,+ 'job_id IN (SELECT id FROM deployment_jobs WHERE '+ + _DEPLOY_JOB_PRUNABLE + ' AND created_at < :cutoff)', HISTORY_SETTING),+ ('deployment_jobs', 'created_at',+ _DEPLOY_JOB_PRUNABLE + ' AND NOT EXISTS (SELECT 1 FROM deployment_job_logs dl '+ 'WHERE dl.job_id = deployment_jobs.id)', HISTORY_SETTING),+ ('cron_runs', 'created_at', "status <> 'running'", HISTORY_SETTING),+ ('webhook_logs', 'received_at', '', HISTORY_SETTING),+ ('webhook_deliveries', 'received_at',+ "status IN ('filtered', 'forwarded', 'failed')", HISTORY_SETTING),+ ('event_deliveries', 'created_at', "status IN ('success', 'failed')", HISTORY_SETTING),+ ('image_update_checks', 'checked_at',+ 'checked_at < (SELECT MAX(c2.checked_at) FROM image_update_checks c2 '+ 'WHERE c2.application_id = image_update_checks.application_id)', HISTORY_SETTING),+ ('workflow_logs', 'timestamp', '', HISTORY_SETTING),+ ('server_onboarding_logs', 'created_at', '', HISTORY_SETTING),+ ('sandbox_runs', 'created_at', "status <> 'running'", HISTORY_SETTING),+)+++def _days(key):+ from app.services.settings_service import SettingsService+ try:+ return int(SettingsService.get(key, DEFAULT_DAYS))+ except (TypeError, ValueError):+ return DEFAULT_DAYS+++def _delete(table, clause, params, sqlite, batch_size):+ from sqlalchemy import text+ from app import db+ if not sqlite:+ # Other engines have no portable rowid; one statement, one commit.+ result = db.session.execute(text(f'DELETE FROM {table} WHERE {clause}'), params)+ db.session.commit()+ return result.rowcount or 0+ removed = 0+ while True:+ # Batch on rowid so a large backlog never locks the table in one go.+ result = db.session.execute(text(+ f'DELETE FROM {table} WHERE rowid IN '+ f'(SELECT rowid FROM {table} WHERE {clause} LIMIT {int(batch_size)})'+ ), params)+ db.session.commit()+ if not result.rowcount:+ break+ removed += result.rowcount+ if result.rowcount < batch_size:+ break+ return removed+++def prune(now=None, batch_size=5000):+ """Delete aged history rows. Returns ``{table: rows_deleted}`` for every+ table that lost rows. A table missing from this database is skipped."""+ from sqlalchemy import inspect+ from app import db++ now = now or datetime.utcnow()+ present = set(inspect(db.engine).get_table_names())+ sqlite = db.engine.dialect.name == 'sqlite'+ days = {key: _days(key) for key in (AUDIT_SETTING, HISTORY_SETTING)}++ deleted = {}+ for table, time_col, extra, key in SPECS:+ if table not in present or days[key] <= 0:+ continue+ cutoff = now - timedelta(days=days[key])+ clause = ' AND '.join(part for part in (+ f'{time_col} < :cutoff' if time_col else '', extra) if part)+ try:+ removed = _delete(table, clause, {'cutoff': cutoff}, sqlite, batch_size)+ except Exception as exc:+ # One table's schema drift must not stop the rest from pruning.+ db.session.rollback()+ logger.warning('History retention skipped %s: %s', table, exc)+ continue+ if removed:+ deleted[table] = removed+ return deleteddiff --git a/backend/tests/test_history_retention.py b/backend/tests/test_history_retention.pynew file mode 100644index 00000000..fdb76a6c--- /dev/null+++ b/backend/tests/test_history_retention.py@@ -0,0 +1,131 @@+"""Plan 85 A4: history tables that nothing pruned are aged out, finished rows+only, and the rows the UI reads as "current" survive."""+from datetime import datetime, timedelta++from sqlalchemy import text++from app import db+from app.services import history_retention_service as svc+from app.services.settings_service import SettingsService++NOW = datetime.utcnow()+OLD = NOW - timedelta(days=120)+NEW = NOW - timedelta(days=5)+++def _sql(statement, **params):+ return db.session.execute(text(statement), params)+++def _count(table, where='1=1', **params):+ return _sql(f'SELECT COUNT(*) FROM {table} WHERE {where}', **params).scalar()+++def _app_id(name):+ user_id = _sql('SELECT id FROM users ORDER BY id LIMIT 1').scalar()+ if user_id is None:+ _sql("INSERT INTO users (email, username) VALUES ('[email protected]', 'retention')")+ user_id = _sql('SELECT id FROM users ORDER BY id LIMIT 1').scalar()+ from app.models import Application+ application = Application(name=name, app_type='docker', user_id=user_id)+ db.session.add(application)+ db.session.commit()+ return application.id+++def test_every_spec_runs_against_the_real_schema(app):+ # A typo in a table or column name would only surface as a warning on a+ # live box; on an empty database every spec must simply delete nothing.+ tables = {spec[0] for spec in svc.SPECS}+ from sqlalchemy import inspect+ assert tables <= set(inspect(db.engine).get_table_names())+ assert svc.prune() == {}+++def test_old_audit_rows_go_and_recent_ones_stay(app):+ _sql("INSERT INTO audit_logs (action, created_at) VALUES ('old', :t)", t=OLD)+ _sql("INSERT INTO audit_logs (action, created_at) VALUES ('new', :t)", t=NEW)+ db.session.commit()++ assert svc.prune(now=NOW) == {'audit_logs': 1}+ assert [r[0] for r in _sql('SELECT action FROM audit_logs')] == ['new']+++def test_zero_keeps_everything(app):+ _sql("INSERT INTO audit_logs (action, created_at) VALUES ('old', :t)", t=OLD)+ db.session.commit()+ SettingsService.set(svc.AUDIT_SETTING, 0)++ assert svc.prune(now=NOW) == {}+ assert _count('audit_logs') == 1+++def test_a_notification_goes_only_after_its_deliveries(app):+ for key, created in (('old', OLD), ('new', NEW)):+ _sql("INSERT INTO notifications (event_key, title, created_at) VALUES (:k, :k, :t)",+ k=key, t=created)+ old_id = _sql("SELECT id FROM notifications WHERE event_key = 'old'").scalar()+ new_id = _sql("SELECT id FROM notifications WHERE event_key = 'new'").scalar()+ # The old notification still has a recent delivery: it must stay.+ _sql("INSERT INTO notification_deliveries (notification_id, channel, created_at) "+ "VALUES (:n, 'email', :t)", n=old_id, t=NEW)+ _sql("INSERT INTO notification_deliveries (notification_id, channel, created_at) "+ "VALUES (:n, 'email', :t)", n=new_id, t=OLD)+ db.session.commit()++ svc.prune(now=NOW)+ assert _count('notifications') == 2+ assert _count('notification_deliveries') == 1+++def test_deployment_jobs_keep_the_newest_per_app_and_anything_live(app):+ app_id = _app_id('retention-app')+ rows = (+ ('ancient-done', 'succeeded', OLD - timedelta(days=10)),+ ('old-running', 'running', OLD - timedelta(days=5)),+ ('newest-but-old', 'failed', OLD),+ )+ for job_id, status, created in rows:+ _sql("INSERT INTO deployment_jobs (id, kind, status, app_id, created_at) "+ "VALUES (:i, 'deploy', :s, :a, :t)", i=job_id, s=status, a=app_id, t=created)+ _sql("INSERT INTO deployment_job_logs (job_id, message, created_at) "+ "VALUES (:i, 'line', :t)", i=job_id, t=NEW) # lines newer than the job+ db.session.commit()++ svc.prune(now=NOW)+ left = {r[0] for r in _sql('SELECT id FROM deployment_jobs')}+ assert left == {'old-running', 'newest-but-old'}+ assert _count('deployment_job_logs', "job_id = 'ancient-done'") == 0+ assert _count('deployment_job_logs') == 2+++def test_a_running_cron_run_is_never_pruned(app):+ _sql("INSERT INTO cron_runs (job_id, status, created_at) VALUES ('a', 'running', :t)", t=OLD)+ _sql("INSERT INTO cron_runs (job_id, status, created_at) VALUES ('b', 'success', :t)", t=OLD)+ db.session.commit()++ svc.prune(now=NOW)+ assert [r[0] for r in _sql('SELECT job_id FROM cron_runs')] == ['a']+++def test_the_latest_image_check_per_app_survives(app):+ app_id = _app_id('image-app')+ for checked in (OLD - timedelta(days=1), OLD):+ _sql("INSERT INTO image_update_checks (application_id, image_ref, update_available, checked_at) "+ "VALUES (:a, 'nginx:latest', 0, :t)", a=app_id, t=checked)+ db.session.commit()++ svc.prune(now=NOW)+ assert _count('image_update_checks') == 1+ assert _sql('SELECT checked_at FROM image_update_checks').scalar().startswith(str(OLD.date()))+++def test_the_retention_tick_runs_history_even_with_telemetry_off(app):+ from app.jobs.builtin_handlers import run_telemetry_retention++ _sql("INSERT INTO audit_logs (action, created_at) VALUES ('old', :t)", t=OLD)+ db.session.commit()+ SettingsService.set('telemetry.retention_days', 0)++ result = run_telemetry_retention()+ assert result == {'deleted': 1, 'by_table': {'audit_logs': 1}}