Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(cli): add `serverkit disk` to report and reclaim disk space

Juan Denise10ecc87d025
diff --git a/backend/app/services/disk_reclaim_service.py b/backend/app/services/disk_reclaim_service.py
new file mode 100644
index 00000000..ee4c0eea
--- /dev/null
+++ b/backend/app/services/disk_reclaim_service.py
@@ -0,0 +1,758 @@
+"""Disk reclamation — find and free space when the panel host fills up.
+
+Built for the failure mode where routine ``serverkit update`` runs fill the
+disk on their own:
+
+* ``queue_messages``, ``system_events`` and ``api_usage_logs`` have no
+ retention handler, so the database grows without bound (~11 MB/day on a
+ single-app box).
+* Every update copies that database **twice** — a ``serverkit-pre-upgrade-*.db``
+ and a full ``serverkit-tree-*/`` backup that contains it — and nothing caps
+ how many of those snapshots accumulate.
+
+Six updates against an 800 MB database is ~9.6 GB, which is enough to fill a
+25 GB VPS from a standing start.
+
+Two ordering rules are baked in because they are the difference between this
+working and not working on a disk that is already at 100%:
+
+1. Filesystem candidates always run **before** the database VACUUM. VACUUM
+ rewrites the database into a temporary file alongside it, so it needs free
+ space roughly equal to the database itself — it is the one step that cannot
+ go first on a full disk.
+2. Rows are deleted from ``jobs`` before ``queue_messages``, and any queue
+ message still referenced by a surviving job is kept, so the
+ ``jobs.queue_message_id`` link never dangles.
+
+Everything is measured before anything is deleted, and every candidate can be
+previewed with ``dry_run=True``.
+"""
+import os
+import re
+import shutil
+import subprocess
+from datetime import datetime, timedelta
+
+# ── Defaults (overridable per call so tests never touch real paths) ──────────
+
+INSTALL_DIR = os.environ.get('SERVERKIT_DIR', '/opt/serverkit')
+# Must match BACKUP_DIR in scripts/update.sh — that is what writes the
+# snapshots this module reclaims.
+BACKUP_DIR = os.environ.get('SERVERKIT_BACKUP_DIR', '/var/backups/serverkit')
+TMP_DIR = os.environ.get('SERVERKIT_TMP_DIR', '/tmp')
+
+DEFAULT_RETENTION_DAYS = 7
+DEFAULT_KEEP_SNAPSHOTS = 1
+DEFAULT_JOURNAL_CAP = '100M'
+
+# mktemp -d names: "tmp." + exactly 10 alphanumerics. Anything else in /tmp is
+# not ours to touch.
+_MKTEMP_RE = re.compile(r'^tmp\.[A-Za-z0-9]{10}$')
+_SNAPSHOT_RES = (
+ re.compile(r'^serverkit-tree-(\d{8}-\d{6})$'),
+ re.compile(r'^serverkit-pre-upgrade-(\d{8}-\d{6})\.db$'),
+ # PostgreSQL installs snapshot with pg_dump instead of a file copy.
+ re.compile(r'^serverkit-pre-upgrade-(\d{8}-\d{6})\.dump$'),
+ re.compile(r'^serverkit-pre-prune-(\d{8}(?:-\d{6})?)\.db$'),
+)
+
+# Telemetry tables, in deletion order. ``keep_status`` rows are never removed
+# regardless of age.
+TELEMETRY_SPECS = (
+ # (key, table, time_column, status_column, deletable_statuses)
+ ('jobs', 'jobs', 'created_at', 'status', ('succeeded', 'cancelled')),
+ ('queue_messages', 'queue_messages', 'created_at', 'status', ('completed',)),
+ ('system_events', 'system_events', 'created_at', None, None),
+ ('api_usage_logs', 'api_usage_logs', 'created_at', None, None),
+)
+
+
+def human_bytes(n):
+ """Format a byte count the way ``df -h`` does."""
+ if n is None:
+ return '-'
+ n = float(n)
+ for unit in ('B', 'KB', 'MB', 'GB', 'TB'):
+ if abs(n) < 1024.0 or unit == 'TB':
+ return f'{n:.0f} {unit}' if unit == 'B' else f'{n:.1f} {unit}'
+ n /= 1024.0
+
+
+def _run(cmd, timeout=300):
+ """Run a command, returning (ok, stdout, stderr). Never raises."""
+ try:
+ proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
+ return proc.returncode == 0, proc.stdout or '', proc.stderr or ''
+ except (OSError, subprocess.SubprocessError) as exc:
+ # A missing binary (docker/journalctl not installed) is a normal
+ # outcome here, not an error worth aborting the whole sweep for.
+ return False, '', str(exc)
+
+
+def _path_size(path):
+ """Bytes used by a file or directory tree. Symlinks are counted, not followed."""
+ try:
+ st = os.lstat(path)
+ except OSError:
+ return 0
+ if not os.path.isdir(path) or os.path.islink(path):
+ return st.st_size
+ total = 0
+ for root, dirs, files in os.walk(path, onerror=lambda e: None):
+ dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(root, d))]
+ for name in files + dirs:
+ try:
+ total += os.lstat(os.path.join(root, name)).st_size
+ except OSError:
+ continue
+ return total
+
+
+def disk_usage(path='/'):
+ """Free/total bytes for the filesystem holding ``path``."""
+ try:
+ usage = shutil.disk_usage(path)
+ except OSError:
+ return None
+ return {
+ 'path': path,
+ 'total': usage.total,
+ 'used': usage.used,
+ 'free': usage.free,
+ 'percent_used': round(usage.used * 100.0 / usage.total, 1) if usage.total else 0.0,
+ }
+
+
+# ── Filesystem candidates ───────────────────────────────────────────────────
+
+def _parse_stamp(stamp):
+ """YYYYmmdd-HHMMSS (or bare YYYYmmdd) to a datetime, or None."""
+ for fmt in ('%Y%m%d-%H%M%S', '%Y%m%d'):
+ try:
+ return datetime.strptime(stamp, fmt)
+ except ValueError:
+ continue
+ return None
+
+
+def list_snapshots(backup_dir=BACKUP_DIR):
+ """Every upgrade snapshot, newest first, grouped by timestamp.
+
+ One update writes several files under the same stamp — a database copy
+ (``.db`` or a PostgreSQL ``.dump``) and a full tree backup — so they are
+ grouped and offered as a single unit. Deleting half a snapshot helps nobody.
+ """
+ try:
+ entries = os.listdir(backup_dir)
+ except OSError:
+ return []
+
+ groups = {}
+ for name in entries:
+ for pattern in _SNAPSHOT_RES:
+ match = pattern.match(name)
+ if match:
+ groups.setdefault(match.group(1), []).append(os.path.join(backup_dir, name))
+ break
+
+ now = datetime.utcnow()
+ snapshots = []
+ for stamp, paths in groups.items():
+ when = _parse_stamp(stamp)
+ snapshots.append({
+ 'stamp': stamp,
+ 'paths': sorted(paths),
+ 'bytes': sum(_path_size(p) for p in paths),
+ 'taken_at': when.isoformat() if when else None,
+ 'age_days': (now - when).days if when else None,
+ })
+ # Stamps are zero-padded, so lexical order is chronological order.
+ snapshots.sort(key=lambda s: s['stamp'], reverse=True)
+ return snapshots
+
+
+def select_snapshots(snapshots, keep=DEFAULT_KEEP_SNAPSHOTS, older_than_days=None):
+ """Which snapshots to drop: everything past ``keep``, and — when
+ ``older_than_days`` is given — only those at least that old.
+
+ ``keep`` always wins, so an age filter can never empty the directory.
+ """
+ doomed = snapshots[keep:] if keep >= 0 else []
+ if older_than_days is not None:
+ doomed = [s for s in doomed
+ if s['age_days'] is not None and s['age_days'] >= older_than_days]
+ return doomed
+
+
+def _scan_upgrade_snapshots(backup_dir=BACKUP_DIR, keep=DEFAULT_KEEP_SNAPSHOTS,
+ older_than_days=None):
+ """Pre-upgrade database copies and tree backups beyond the newest ``keep``."""
+ doomed = select_snapshots(list_snapshots(backup_dir), keep, older_than_days)
+ paths = sorted(path for snap in doomed for path in snap['paths'])
+ return paths, sum(snap['bytes'] for snap in doomed)
+
+
+def _looks_like_serverkit_staging(path):
+ """True when a /tmp scratch dir holds an unpacked ServerKit release.
+
+ Update staging dirs contain the extracted tree (``opt/serverkit`` and
+ friends). Anything else under /tmp belongs to someone else.
+ """
+ try:
+ top = os.listdir(path)
+ except OSError:
+ return False
+ for name in top:
+ if 'serverkit' in name.lower():
+ return True
+ if name in ('opt', 'usr', 'srv'):
+ try:
+ if any('serverkit' in child.lower()
+ for child in os.listdir(os.path.join(path, name))):
+ return True
+ except OSError:
+ continue
+ return False
+
+
+def _scan_tmp_staging(tmp_dir=TMP_DIR, min_age_hours=24):
+ """Abandoned ``mktemp -d`` staging dirs left behind by past updates."""
+ cutoff = datetime.utcnow() - timedelta(hours=min_age_hours)
+ # Only ever reap scratch dirs belonging to whoever is running the reclaim
+ # (root in production). Another user's /tmp is never ours to clear.
+ euid = os.geteuid() if hasattr(os, 'geteuid') else None
+ paths = []
+ try:
+ entries = os.listdir(tmp_dir)
+ except OSError:
+ return [], 0
+
+ for name in entries:
+ if not _MKTEMP_RE.match(name):
+ continue
+ path = os.path.join(tmp_dir, name)
+ try:
+ st = os.lstat(path)
+ except OSError:
+ continue
+ if euid is not None and st.st_uid != euid:
+ continue
+ if datetime.utcfromtimestamp(st.st_mtime) > cutoff:
+ continue
+ if os.path.isdir(path) and not _looks_like_serverkit_staging(path):
+ continue
+ paths.append(path)
+ paths.sort()
+ return paths, sum(_path_size(p) for p in paths)
+
+
+def _scan_oversized_logs(min_bytes=10 * 1024 * 1024):
+ """Failed/successful-login binary logs that have grown past a sane size.
+
+ ``btmp`` in particular balloons under sustained SSH brute-force. These are
+ truncated rather than deleted so the files keep their ownership and mode.
+ """
+ paths = []
+ for path in ('/var/log/btmp', '/var/log/btmp.1', '/var/log/wtmp', '/var/log/wtmp.1'):
+ try:
+ size = os.lstat(path).st_size
+ except OSError:
+ continue
+ if size >= min_bytes:
+ paths.append(path)
+ return paths, sum(_path_size(p) for p in paths)
+
+
+_SCALE = {'': 1, 'K': 1024, 'M': 1024 ** 2, 'G': 1024 ** 3, 'T': 1024 ** 4}
+
+
+def _to_bytes(number, suffix):
+ return int(float(number) * _SCALE.get(suffix.upper().rstrip('B'), 1))
+
+
+def _parse_reclaimed(output):
+ """Bytes from docker's 'Total reclaimed space: 1.234GB' trailer."""
+ match = re.search(r'Total reclaimed space:\s*([\d.]+)\s*([KMGT]?B)', output or '')
+ return _to_bytes(match.group(1), match.group(2)) if match else 0
+
+
+def _journal_bytes():
+ """Bytes currently held by the systemd journal, or 0 if journalctl is absent."""
+ ok, out, _ = _run(['journalctl', '--disk-usage'], timeout=30)
+ if not ok:
+ return 0
+ match = re.search(r'take up ([\d.]+)([KMGT]?)B?', out)
+ return _to_bytes(match.group(1), match.group(2)) if match else 0
+
+
+def _scan_journal(cap=DEFAULT_JOURNAL_CAP):
+ """Systemd journal bytes above ``cap``."""
+ current = _journal_bytes()
+ cap_match = re.match(r'^([\d.]+)([KMGT]?)$', str(cap).upper().rstrip('B'))
+ cap_bytes = _to_bytes(cap_match.group(1), cap_match.group(2)) if cap_match else 0
+ return max(0, current - cap_bytes)
+
+
+def _scan_package_caches():
+ """Package-manager and language caches — all re-downloadable."""
+ paths = []
+ for path in ('/var/cache/apt/archives', '/var/cache/dnf', '/var/cache/yum',
+ '/root/.cache/pip', '/root/.npm/_cacache'):
+ if os.path.isdir(path) and _path_size(path) > 0:
+ paths.append(path)
+ return paths, sum(_path_size(p) for p in paths)
+
+
+def _scan_docker():
+ """Dangling images and build cache. Tagged images and volumes are never touched."""
+ ok, out, _ = _run(['docker', 'system', 'df', '--format', '{{.Type}}|{{.Reclaimable}}'],
+ timeout=60)
+ if not ok:
+ return 0
+ total = 0
+ for line in out.splitlines():
+ if '|' not in line:
+ continue
+ kind, reclaimable = line.split('|', 1)
+ if kind.strip() != 'Build Cache':
+ # Docker's image/volume "reclaimable" counts tagged-but-unused
+ # images and named volumes, which we deliberately never remove —
+ # so only the build cache can be estimated up front. Dangling
+ # images are pruned too, and reported from docker's own tally.
+ continue
+ match = re.match(r'\s*([\d.]+)\s*([KMGT]?B)', reclaimable)
+ if match:
+ total += _to_bytes(match.group(1), match.group(2))
+ return total
+
+
+# ── Database candidate ──────────────────────────────────────────────────────
+
+def _sqlite_path():
+ """Filesystem path of the SQLite database, or None for other engines."""
+ from flask import current_app
+ uri = current_app.config.get('SQLALCHEMY_DATABASE_URI', '')
+ if not uri.startswith('sqlite:'):
+ return None
+ path = uri.split('///')[-1].split('?')[0]
+ return path or None
+
+
+def _table_shares(db_path):
+ """Bytes held by each table *including its indexes*, via the dbstat vtable.
+
+ Returns {} when dbstat is unavailable — it is a compile-time option, so its
+ absence is a missing estimate, not an error.
+ """
+ import sqlite3
+ try:
+ conn = sqlite3.connect(f'file:{db_path}?mode=ro', uri=True)
+ except Exception:
+ return {}
+ try:
+ # Each dbstat row names one btree — a table, or one of its indexes.
+ # Map every index back to its table before summing, otherwise the
+ # indexes (which here outweigh the tables) go uncounted.
+ owner = {
+ name: tbl_name for name, tbl_name in conn.execute(
+ "SELECT name, tbl_name FROM sqlite_master WHERE type IN ('table', 'index')")
+ }
+ shares = {}
+ for name, size in conn.execute('SELECT name, SUM(pgsize) FROM dbstat GROUP BY name'):
+ table = owner.get(name, name)
+ shares[table] = shares.get(table, 0) + (size or 0)
+ return shares
+ except Exception:
+ return {}
+ finally:
+ conn.close()
+
+
+def scan_telemetry(days=DEFAULT_RETENTION_DAYS):
+ """Count prunable telemetry rows and estimate the bytes they hold.
+
+ Must run inside a Flask app context.
+ """
+ from sqlalchemy import text
+ from app import db
+
+ cutoff = datetime.utcnow() - timedelta(days=days)
+ db_path = _sqlite_path()
+ shares = _table_shares(db_path) if db_path and os.path.exists(db_path) else {}
+
+ tables = []
+ estimated = 0
+ for key, table, time_col, status_col, statuses in TELEMETRY_SPECS:
+ where = [f'{time_col} < :cutoff']
+ params = {'cutoff': cutoff}
+ if status_col and statuses:
+ placeholders = ', '.join(f':s{i}' for i in range(len(statuses)))
+ where.append(f'{status_col} IN ({placeholders})')
+ params.update({f's{i}': value for i, value in enumerate(statuses)})
+ clause = ' AND '.join(where)
+ try:
+ deletable = db.session.execute(
+ text(f'SELECT COUNT(*) FROM {table} WHERE {clause}'), params).scalar() or 0
+ total = db.session.execute(
+ text(f'SELECT COUNT(*) FROM {table}')).scalar() or 0
+ except Exception:
+ continue
+
+ own = shares.get(table, 0)
+ share = int(own * deletable / total) if (own and total) else 0
+ estimated += share
+ tables.append({
+ 'key': key, 'table': table, 'rows': total,
+ 'deletable': deletable, 'bytes': share or None,
+ })
+
+ return {
+ 'days': days,
+ 'cutoff': cutoff.isoformat(),
+ 'tables': tables,
+ 'deletable_rows': sum(t['deletable'] for t in tables),
+ 'bytes': estimated or None,
+ 'db_path': db_path,
+ 'db_bytes': os.path.getsize(db_path) if db_path and os.path.exists(db_path) else None,
+ }
+
+
+def prune_telemetry(days=DEFAULT_RETENTION_DAYS, batch_size=5000, vacuum=True,
+ allow_restart=False, dry_run=False):
+ """Delete aged telemetry rows, then VACUUM to hand the pages back to the OS.
+
+ Deleting alone does not shrink the file — the pages land on SQLite's
+ freelist and get reused. VACUUM is what actually returns the space, and it
+ needs free disk roughly equal to the current database size, so run the
+ filesystem reclaimers first.
+ """
+ from sqlalchemy import text
+ from app import db
+
+ report = scan_telemetry(days=days)
+ if dry_run:
+ report['dry_run'] = True
+ return report
+
+ cutoff = datetime.utcnow() - timedelta(days=days)
+ deleted = {}
+ for key, table, time_col, status_col, statuses in TELEMETRY_SPECS:
+ where = [f'{time_col} < :cutoff']
+ params = {'cutoff': cutoff}
+ if status_col and statuses:
+ placeholders = ', '.join(f':s{i}' for i in range(len(statuses)))
+ where.append(f'{status_col} IN ({placeholders})')
+ params.update({f's{i}': value for i, value in enumerate(statuses)})
+ if table == 'queue_messages':
+ # Keep anything a surviving job still points at, or
+ # jobs.queue_message_id starts dangling.
+ where.append('id NOT IN (SELECT queue_message_id FROM jobs '
+ 'WHERE queue_message_id IS NOT NULL)')
+ clause = ' AND '.join(where)
+
+ removed = 0
+ if report.get('db_path'):
+ # SQLite: batch on rowid so a large backlog never locks the table
+ # in one statement.
+ while True:
+ result = db.session.execute(text(
+ f'DELETE FROM {table} WHERE rowid IN '
+ f'(SELECT rowid FROM {table} WHERE {clause} LIMIT {int(batch_size)})'
+ ), params)
+ db.session.commit()
+ if not result.rowcount:
+ break
+ removed += result.rowcount
+ if result.rowcount < batch_size:
+ break
+ else:
+ # Other engines have no portable rowid; one statement, one commit.
+ result = db.session.execute(
+ text(f'DELETE FROM {table} WHERE {clause}'), params)
+ db.session.commit()
+ removed = result.rowcount or 0
+ deleted[key] = removed
+
+ report['deleted'] = deleted
+ report['deleted_rows'] = sum(deleted.values())
+
+ if vacuum:
+ report['vacuum'] = _vacuum(report.get('db_path'), allow_restart=allow_restart)
+ if report.get('db_path') and os.path.exists(report['db_path']):
+ report['db_bytes_after'] = os.path.getsize(report['db_path'])
+ return report
+
+
+def _vacuum(db_path, allow_restart=False, service='serverkit'):
+ """VACUUM the database, restarting the panel only if a lock demands it."""
+ from sqlalchemy import text
+ from app import db
+
+ if not db_path or not os.path.exists(db_path):
+ return {'ok': False, 'error': 'not a SQLite database; skipped'}
+
+ needed = os.path.getsize(db_path)
+ free = disk_usage(os.path.dirname(db_path) or '/')
+ if free and free['free'] < needed:
+ # VACUUM rewrites the whole database beside itself before swapping.
+ return {
+ 'ok': False,
+ 'error': (f'needs ~{human_bytes(needed)} free to rewrite the database, '
+ f'only {human_bytes(free["free"])} available — '
+ f'reclaim filesystem space first'),
+ }
+
+ try:
+ # VACUUM cannot run inside a transaction, and SQLAlchemy opens one for
+ # every session statement — so it needs its own autocommit connection.
+ db.session.commit()
+ with db.engine.connect().execution_options(isolation_level='AUTOCOMMIT') as conn:
+ conn.execute(text('VACUUM'))
+ return {'ok': True, 'restarted': False}
+ except Exception as exc:
+ if not allow_restart:
+ return {'ok': False, 'error': f'{exc} (retry with --allow-restart)'}
+ locked_by = exc
+
+ # Something else holds a write lock. Stop the panel, vacuum, start it back.
+ db.session.remove()
+ stopped, _, err = _run(['systemctl', 'stop', service], timeout=120)
+ if not stopped:
+ return {'ok': False,
+ 'error': f'{locked_by}; could not stop {service}: {err.strip()}'}
+ try:
+ import sqlite3
+ conn = sqlite3.connect(db_path, timeout=60)
+ try:
+ conn.execute('PRAGMA wal_checkpoint(TRUNCATE)')
+ conn.execute('VACUUM')
+ finally:
+ conn.close()
+ return {'ok': True, 'restarted': True}
+ except Exception as exc:
+ return {'ok': False, 'error': str(exc), 'restarted': True}
+ finally:
+ _run(['systemctl', 'start', service], timeout=120)
+
+
+# ── Candidate registry ──────────────────────────────────────────────────────
+
+def scan(days=DEFAULT_RETENTION_DAYS, keep=DEFAULT_KEEP_SNAPSHOTS,
+ backup_dir=BACKUP_DIR, tmp_dir=TMP_DIR, with_db=True,
+ older_than_days=None):
+ """Measure every reclaim candidate. Reads only — deletes nothing."""
+ candidates = []
+
+ all_snapshots = list_snapshots(backup_dir)
+ doomed = select_snapshots(all_snapshots, keep, older_than_days)
+ kept = len(all_snapshots) - len(doomed)
+ if older_than_days is not None:
+ title = f'Upgrade snapshots older than {older_than_days} days'
+ else:
+ title = f'Old upgrade snapshots (keeping newest {keep})'
+ candidates.append({
+ 'key': 'upgrade-snapshots', 'safety': 'safe',
+ 'title': title,
+ 'detail': (f'{len(doomed)} of {len(all_snapshots)} snapshot(s) in {backup_dir}'
+ f', {kept} kept' if all_snapshots else f'none in {backup_dir}'),
+ 'bytes': sum(s['bytes'] for s in doomed),
+ 'paths': sorted(p for s in doomed for p in s['paths']),
+ # The whole inventory, so the CLI can offer a per-snapshot choice.
+ 'snapshots': all_snapshots,
+ 'doomed_stamps': [s['stamp'] for s in doomed],
+ })
+
+ staging, staging_bytes = _scan_tmp_staging(tmp_dir)
+ candidates.append({
+ 'key': 'tmp-staging', 'safety': 'safe',
+ 'title': 'Abandoned update staging dirs',
+ 'detail': f'{len(staging)} item(s) in {tmp_dir}, older than 24h',
+ 'bytes': staging_bytes, 'paths': staging,
+ })
+
+ logs, log_bytes = _scan_oversized_logs()
+ candidates.append({
+ 'key': 'login-logs', 'safety': 'safe',
+ 'title': 'Oversized btmp/wtmp login logs',
+ 'detail': ', '.join(os.path.basename(p) for p in logs) or 'none oversized',
+ 'bytes': log_bytes, 'paths': logs,
+ })
+
+ caches, cache_bytes = _scan_package_caches()
+ candidates.append({
+ 'key': 'package-caches', 'safety': 'safe',
+ 'title': 'Package and build caches (apt/dnf/pip/npm)',
+ 'detail': ', '.join(caches) or 'none found',
+ 'bytes': cache_bytes, 'paths': caches,
+ })
+
+ candidates.append({
+ 'key': 'journal', 'safety': 'safe',
+ 'title': f'Systemd journal above {DEFAULT_JOURNAL_CAP}',
+ 'detail': 'rotates then vacuums archived journals',
+ 'bytes': _scan_journal(), 'paths': [],
+ })
+
+ candidates.append({
+ 'key': 'docker-build-cache', 'safety': 'safe',
+ 'title': 'Docker build cache and dangling images',
+ 'detail': 'build cache measured; dangling images extra. '
+ 'Tagged images and named volumes are left alone',
+ 'bytes': _scan_docker(), 'paths': [],
+ })
+
+ if with_db:
+ telemetry = scan_telemetry(days=days)
+ rows = telemetry['deletable_rows']
+ if rows:
+ hit = ', '.join(t['table'] for t in telemetry['tables'] if t['deletable'])
+ detail = f'{rows:,} rows across {hit} — then VACUUM'
+ else:
+ detail = 'nothing to prune'
+ candidates.append({
+ 'key': 'telemetry', 'safety': 'review',
+ 'title': f'Telemetry rows older than {days} days',
+ 'detail': detail,
+ 'bytes': telemetry['bytes'] or 0, 'paths': [],
+ 'telemetry': telemetry,
+ })
+
+ return {
+ 'disk': disk_usage('/'),
+ 'candidates': candidates,
+ 'total_bytes': sum(c['bytes'] or 0 for c in candidates),
+ }
+
+
+def _reclaim_paths(paths, truncate=False):
+ """Delete (or truncate) a list of paths, returning bytes actually freed."""
+ freed = 0
+ for path in paths:
+ size = _path_size(path)
+ try:
+ if truncate:
+ with open(path, 'wb'):
+ pass
+ elif os.path.isdir(path) and not os.path.islink(path):
+ shutil.rmtree(path, ignore_errors=True)
+ else:
+ os.remove(path)
+ freed += size
+ except OSError:
+ continue
+ return freed
+
+
+def reclaim(keys, days=DEFAULT_RETENTION_DAYS, keep=DEFAULT_KEEP_SNAPSHOTS,
+ backup_dir=BACKUP_DIR, tmp_dir=TMP_DIR, dry_run=False,
+ allow_restart=False, older_than_days=None, snapshot_stamps=None):
+ """Run the selected reclaimers.
+
+ Filesystem work is forced ahead of the database VACUUM regardless of the
+ order ``keys`` arrives in — see the module docstring.
+
+ ``snapshot_stamps`` deletes exactly those upgrade snapshots, for when the
+ operator hand-picked them; otherwise ``keep``/``older_than_days`` decide.
+ """
+ keys = list(keys)
+ before = disk_usage('/')
+ results = []
+
+ def record(key, freed, note=''):
+ results.append({'key': key, 'bytes': freed, 'note': note})
+
+ if 'upgrade-snapshots' in keys:
+ if snapshot_stamps is not None:
+ wanted = set(snapshot_stamps)
+ chosen = [s for s in list_snapshots(backup_dir) if s['stamp'] in wanted]
+ paths = sorted(p for s in chosen for p in s['paths'])
+ size = sum(s['bytes'] for s in chosen)
+ count = len(chosen)
+ else:
+ doomed = select_snapshots(list_snapshots(backup_dir), keep, older_than_days)
+ paths = sorted(p for s in doomed for p in s['paths'])
+ size = sum(s['bytes'] for s in doomed)
+ count = len(doomed)
+ record('upgrade-snapshots', size if dry_run else _reclaim_paths(paths),
+ f'{count} snapshot(s), {len(paths)} file(s)')
+
+ if 'tmp-staging' in keys:
+ paths, size = _scan_tmp_staging(tmp_dir)
+ record('tmp-staging', size if dry_run else _reclaim_paths(paths),
+ f'{len(paths)} staging dir(s)')
+
+ if 'login-logs' in keys:
+ paths, size = _scan_oversized_logs()
+ record('login-logs', size if dry_run else _reclaim_paths(paths, truncate=True),
+ f'{len(paths)} log(s) truncated')
+
+ if 'package-caches' in keys:
+ paths, size = _scan_package_caches()
+ if dry_run:
+ record('package-caches', size)
+ else:
+ # Measure the delta: apt-get clean empties the archives itself, so
+ # anything counted afterwards would read as zero.
+ _run(['apt-get', 'clean'], timeout=120)
+ _reclaim_paths(paths)
+ after_size = sum(_path_size(p) for p in paths)
+ record('package-caches', max(0, size - after_size))
+
+ if 'journal' in keys:
+ size = _scan_journal()
+ if dry_run:
+ record('journal', size)
+ else:
+ before_journal = _journal_bytes()
+ # --rotate first: --vacuum-size only ever removes *archived*
+ # journals, so without a rotation an over-cap active journal is
+ # reported as "freed 0B".
+ _run(['journalctl', '--rotate'], timeout=60)
+ ok, _, err = _run(['journalctl', f'--vacuum-size={DEFAULT_JOURNAL_CAP}'], timeout=120)
+ freed = max(0, before_journal - _journal_bytes()) if ok else 0
+ record('journal', freed, '' if ok else err.strip()[:120])
+
+ if 'docker-build-cache' in keys:
+ size = _scan_docker()
+ if dry_run:
+ record('docker-build-cache', size)
+ else:
+ freed = 0
+ notes = []
+ for cmd in (['docker', 'image', 'prune', '-f'],
+ ['docker', 'builder', 'prune', '-f']):
+ ok, out, err = _run(cmd, timeout=600)
+ if ok:
+ # docker reports the truth; our scan can only estimate,
+ # because shared layers make dangling-image sizes overlap.
+ freed += _parse_reclaimed(out)
+ else:
+ notes.append(err.strip()[:80])
+ record('docker-build-cache', freed, '; '.join(notes))
+
+ # Database last: VACUUM needs the headroom the steps above just freed.
+ if 'telemetry' in keys:
+ report = prune_telemetry(days=days, dry_run=dry_run, allow_restart=allow_restart)
+ if dry_run:
+ record('telemetry', report.get('bytes') or 0,
+ f"{report['deletable_rows']:,} row(s)")
+ else:
+ freed = 0
+ if report.get('db_bytes') and report.get('db_bytes_after'):
+ freed = max(0, report['db_bytes'] - report['db_bytes_after'])
+ vacuum = report.get('vacuum') or {}
+ note = f"{report['deleted_rows']:,} row(s) deleted"
+ if not vacuum.get('ok'):
+ note += f"; VACUUM skipped: {vacuum.get('error', 'unknown')}"
+ elif vacuum.get('restarted'):
+ note += '; panel restarted for VACUUM'
+ record('telemetry', freed, note)
+
+ after = disk_usage('/')
+ return {
+ 'dry_run': dry_run,
+ 'results': results,
+ 'freed_bytes': sum(r['bytes'] or 0 for r in results),
+ 'disk_before': before,
+ 'disk_after': after,
+ }
diff --git a/backend/cli.py b/backend/cli.py
index a1320330..eae2db9d 100644
--- a/backend/cli.py
+++ b/backend/cli.py
@@ -1270,5 +1270,266 @@ def support_bundle(out_path, passphrase):
click.echo(click.style(f'Support bundle written: {path}', fg='green'))
+# ── disk (reclaim space) ─────────────────────────────────────────────────────
+
[email protected]('--days', default=None, type=int,
+ help='Prune telemetry rows older than this many days (default 7)')
[email protected]('--keep', default=None, type=int,
+ help='Upgrade snapshots to keep, newest first (default 1)')
[email protected]('--older-than', 'older_than', default=None, type=int,
+ help='Only offer upgrade snapshots at least this many days old')
[email protected]('--only', default=None,
+ help='Comma-separated candidate keys to reclaim, skipping the menu')
[email protected]('--safe', is_flag=True, help='Reclaim every candidate marked safe, no prompt')
[email protected]('--all', 'do_all', is_flag=True, help='Reclaim every candidate, no prompt')
[email protected]('--dry-run', is_flag=True, help='Report what would be freed, change nothing')
[email protected]('--allow-restart', is_flag=True,
+ help='Let the panel restart if the database is locked during VACUUM')
[email protected]('--yes', is_flag=True, help='Skip the confirmation prompt')
[email protected]('--json', 'as_json', is_flag=True, help='Output machine-readable JSON')
+def disk(days, keep, older_than, only, safe, do_all, dry_run, allow_restart, yes, as_json):
+ """Show what is using disk space and reclaim it.
+
+ With no options this lists every reclaimable item with its measured size and
+ asks which to free — pick by number ("1,3"), by "safe", or "all".
+ """
+ from app.services import disk_reclaim_service as svc
+
+ days = svc.DEFAULT_RETENTION_DAYS if days is None else days
+ keep = svc.DEFAULT_KEEP_SNAPSHOTS if keep is None else keep
+ if keep < 0:
+ _fail('--keep cannot be negative.')
+ if days < 0:
+ _fail('--days cannot be negative.')
+
+ app = create_app()
+ with app.app_context():
+ report = svc.scan(days=days, keep=keep, older_than_days=older_than)
+ candidates = report['candidates']
+ usable = [c for c in candidates if (c['bytes'] or 0) > 0]
+
+ if as_json and not (only or safe or do_all):
+ _echo_json(report)
+ return
+
+ disk_info = report['disk'] or {}
+ if not as_json:
+ click.echo()
+ if disk_info:
+ bar_used = disk_info['percent_used']
+ colour = 'red' if bar_used >= 90 else ('yellow' if bar_used >= 75 else 'green')
+ click.echo(click.style(
+ f"Disk {disk_info['path']}: {svc.human_bytes(disk_info['used'])} used "
+ f"of {svc.human_bytes(disk_info['total'])} "
+ f"({bar_used}%), {svc.human_bytes(disk_info['free'])} free",
+ fg=colour))
+ click.echo()
+ # Number only the items that can actually free something, and
+ # number them exactly the way the prompt reads them back.
+ numbers = {cand['key']: n for n, cand in enumerate(usable, start=1)}
+ rows = []
+ for cand in candidates:
+ size = cand['bytes'] or 0
+ rows.append((
+ str(numbers.get(cand['key'], '-')),
+ svc.human_bytes(size) if size else '—',
+ cand['safety'],
+ cand['title'],
+ cand['detail'],
+ ))
+ _echo_table(['#', 'Reclaims', 'Safety', 'Item', 'Detail'], rows)
+ click.echo()
+ click.echo(f"Total reclaimable: "
+ f"{click.style(svc.human_bytes(report['total_bytes']), bold=True)}")
+
+ # Work out the selection.
+ if do_all:
+ chosen = [c['key'] for c in usable]
+ elif safe:
+ chosen = [c['key'] for c in usable if c['safety'] == 'safe']
+ elif only:
+ wanted = {k.strip() for k in only.split(',') if k.strip()}
+ known = {c['key'] for c in candidates}
+ unknown = wanted - known
+ if unknown:
+ _fail('unknown candidate(s): ' + ', '.join(sorted(unknown))
+ + '. Known: ' + ', '.join(sorted(known)))
+ chosen = [c['key'] for c in candidates if c['key'] in wanted]
+ elif not usable:
+ click.echo('\nNothing to reclaim.')
+ return
+ else:
+ chosen = _prompt_disk_selection(usable)
+ if chosen is None:
+ click.echo('Aborted.')
+ return
+
+ if not chosen:
+ click.echo('Nothing selected.')
+ return
+
+ # When snapshots were chosen interactively, let the operator say which
+ # ones rather than assuming the keep/age rule.
+ stamps = None
+ interactive = not (do_all or safe or only)
+ if 'upgrade-snapshots' in chosen and interactive and not as_json:
+ snapshot_cand = next(c for c in candidates if c['key'] == 'upgrade-snapshots')
+ stamps = _prompt_snapshot_choice(snapshot_cand)
+ if not stamps:
+ chosen = [k for k in chosen if k != 'upgrade-snapshots']
+ if not chosen:
+ click.echo('Nothing selected.')
+ return
+
+ picked = [c for c in candidates if c['key'] in chosen]
+ planned = sum(
+ (sum(s['bytes'] for s in c['snapshots'] if s['stamp'] in stamps)
+ if (stamps is not None and c['key'] == 'upgrade-snapshots')
+ else (c['bytes'] or 0))
+ for c in picked)
+ if not as_json:
+ click.echo()
+ for cand in picked:
+ size = cand['bytes'] or 0
+ label = cand['title']
+ if stamps is not None and cand['key'] == 'upgrade-snapshots':
+ size = sum(s['bytes'] for s in cand['snapshots'] if s['stamp'] in stamps)
+ label = f'{len(stamps)} upgrade snapshot(s)'
+ click.echo(f' • {label} ({svc.human_bytes(size)})')
+ verb = 'Would free' if dry_run else 'Free'
+ click.echo(f"\n{verb} about {click.style(svc.human_bytes(planned), bold=True)}.")
+
+ if not dry_run and not yes and not as_json:
+ if not click.confirm('Proceed?', default=False):
+ click.echo('Aborted.')
+ return
+
+ result = svc.reclaim(chosen, days=days, keep=keep, dry_run=dry_run,
+ allow_restart=allow_restart, older_than_days=older_than,
+ snapshot_stamps=stamps)
+
+ if as_json:
+ _echo_json(result)
+ return
+
+ click.echo()
+ _echo_table(['Item', 'Freed', 'Note'],
+ [(r['key'], svc.human_bytes(r['bytes'] or 0), r['note']) for r in result['results']])
+ after = result['disk_after'] or {}
+ click.echo()
+ click.echo(click.style(
+ f"{'Would free' if dry_run else 'Freed'} "
+ f"{svc.human_bytes(result['freed_bytes'])}.", fg='green', bold=True))
+ if after and not dry_run:
+ click.echo(f"Disk now {after['percent_used']}% used, "
+ f"{svc.human_bytes(after['free'])} free.")
+
+
+def _prompt_snapshot_choice(candidate):
+ """Offer the upgrade snapshots one by one. Returns the stamps to delete.
+
+ One update writes a database copy and a whole tree backup under the same
+ timestamp, so snapshots are picked as units — a half-deleted snapshot is
+ not a restore point.
+ """
+ from app.services import disk_reclaim_service as svc
+
+ snapshots = candidate.get('snapshots') or []
+ if len(snapshots) <= 1:
+ return candidate.get('doomed_stamps') or []
+
+ default_stamps = candidate.get('doomed_stamps') or []
+ click.echo(f'\n{len(snapshots)} upgrade snapshot(s), newest first:')
+ rows = []
+ for index, snap in enumerate(snapshots, start=1):
+ age = snap['age_days']
+ if age is None:
+ when = '-'
+ elif age == 0:
+ when = 'today'
+ else:
+ when = f'{age} day{"s" if age != 1 else ""} ago'
+ rows.append((
+ str(index),
+ snap['stamp'],
+ when,
+ svc.human_bytes(snap['bytes']),
+ 'newest — kept by default' if index == 1 else
+ ('would delete' if snap['stamp'] in default_stamps else 'kept'),
+ ))
+ _echo_table(['#', 'Snapshot', 'Taken', 'Size', ''], rows)
+
+ click.echo('\nDelete which? numbers like "1,3", "older-than 14" (days), '
+ '"all-but-newest", or "none".')
+ by_index = {index: snap for index, snap in enumerate(snapshots, start=1)}
+ while True:
+ answer = click.prompt('Snapshots', default='all-but-newest',
+ show_default=True).strip().lower()
+ if answer in ('none', 'n', 'q'):
+ return []
+ if answer == 'all-but-newest':
+ return [s['stamp'] for s in snapshots[1:]]
+ if answer.startswith('older-than'):
+ raw = answer.replace('older-than', '').strip()
+ if not raw.isdigit():
+ click.echo(click.style('Give a number of days, e.g. "older-than 14".',
+ fg='yellow'))
+ continue
+ chosen = svc.select_snapshots(snapshots, keep=1, older_than_days=int(raw))
+ if not chosen:
+ click.echo(click.style(
+ f'No snapshot besides the newest is {raw}+ days old.', fg='yellow'))
+ continue
+ return [s['stamp'] for s in chosen]
+ picked, bad = [], []
+ for token in answer.replace(' ', ',').split(','):
+ if not token:
+ continue
+ if token.isdigit() and int(token) in by_index:
+ picked.append(by_index[int(token)]['stamp'])
+ else:
+ bad.append(token)
+ if bad:
+ click.echo(click.style('Not a listed number: ' + ', '.join(bad), fg='yellow'))
+ continue
+ if picked:
+ if len(picked) == len(snapshots) and not click.confirm(
+ 'That deletes every snapshot, leaving no restore point. Sure?',
+ default=False):
+ continue
+ return picked
+ click.echo(click.style('Nothing selected — pick at least one number.', fg='yellow'))
+
+
+def _prompt_disk_selection(usable):
+ """Ask which candidates to reclaim. Returns keys, or None to abort."""
+ by_index = {index: cand for index, cand in enumerate(usable, start=1)}
+ click.echo('\nSelect what to free: numbers like "1,3", "safe" for everything '
+ 'marked safe, "all", or "q" to quit.')
+ while True:
+ answer = click.prompt('Selection', default='safe', show_default=True).strip().lower()
+ if answer in ('q', 'quit', 'n', 'no'):
+ return None
+ if answer == 'all':
+ return [c['key'] for c in usable]
+ if answer == 'safe':
+ return [c['key'] for c in usable if c['safety'] == 'safe']
+ picked, bad = [], []
+ for token in answer.replace(' ', ',').split(','):
+ if not token:
+ continue
+ if token.isdigit() and int(token) in by_index:
+ picked.append(by_index[int(token)]['key'])
+ else:
+ bad.append(token)
+ if bad:
+ click.echo(click.style('Not a listed number: ' + ', '.join(bad), fg='yellow'))
+ continue
+ if picked:
+ return picked
+ click.echo(click.style('Nothing selected — pick at least one number.', fg='yellow'))
+
+
if __name__ == '__main__':
cli()
diff --git a/backend/tests/test_cli_json.py b/backend/tests/test_cli_json.py
index 31cc9e13..2092548c 100644
--- a/backend/tests/test_cli_json.py
+++ b/backend/tests/test_cli_json.py
@@ -182,3 +182,19 @@ def test_deployment_status_json(app, monkeypatch):
serverkit_cli.cli, ['deployment-status', 'j-1', '--json'])
data = _json_out(result)
assert data['job']['status'] == 'succeeded'
+
+
+def test_disk_json_is_a_read_only_report(app, monkeypatch):
+ """`disk --json` with no selection flag reports and changes nothing."""
+ from app.services import disk_reclaim_service as svc
+
+ monkeypatch.setattr(svc, 'reclaim', lambda *a, **k: pytest.fail(
+ 'disk --json without a selection flag must not reclaim anything'))
+
+ result = CliRunner().invoke(serverkit_cli.cli, ['disk', '--json'])
+ data = _json_out(result)
+ assert 'candidates' in data and 'disk' in data
+ keys = {c['key'] for c in data['candidates']}
+ assert {'upgrade-snapshots', 'telemetry'} <= keys
+ for cand in data['candidates']:
+ assert cand['safety'] in ('safe', 'review')
diff --git a/backend/tests/test_disk_reclaim.py b/backend/tests/test_disk_reclaim.py
new file mode 100644
index 00000000..abca9b41
--- /dev/null
+++ b/backend/tests/test_disk_reclaim.py
@@ -0,0 +1,473 @@
+"""Disk reclamation: the scan must measure honestly and the reclaimers must
+only ever touch what they claim to.
+
+The guards that matter here are the ones that were learned the hard way when a
+25 GB panel host filled to 100% purely from `serverkit update` runs:
+
+* filesystem reclaimers run BEFORE the database VACUUM (VACUUM needs free space
+ equal to the database it rewrites, so it cannot go first on a full disk);
+* the newest upgrade snapshot is never deleted;
+* /tmp entries that aren't ours are never deleted;
+* a queue message still referenced by a surviving job is never deleted.
+"""
+import os
+import time
+
+import pytest
+
+from app.services import disk_reclaim_service as svc
+
+
+# ── upgrade snapshots ───────────────────────────────────────────────────────
+
+def _make_snapshots(root):
+ """Three upgrade snapshot sets, oldest to newest."""
+ for stamp in ('20260817-025332', '20260819-212804', '20260821-045737'):
+ tree = root / f'serverkit-tree-{stamp}'
+ (tree / 'backend').mkdir(parents=True)
+ (tree / 'backend' / 'serverkit.db').write_bytes(b'x' * 1000)
+ (root / f'serverkit-pre-upgrade-{stamp}.db').write_bytes(b'y' * 1000)
+ return root
+
+
+def test_snapshot_scan_keeps_the_newest_set(tmp_path):
+ _make_snapshots(tmp_path)
+ paths, size = svc._scan_upgrade_snapshots(str(tmp_path), keep=1)
+
+ names = {os.path.basename(p) for p in paths}
+ assert names == {
+ 'serverkit-tree-20260817-025332', 'serverkit-pre-upgrade-20260817-025332.db',
+ 'serverkit-tree-20260819-212804', 'serverkit-pre-upgrade-20260819-212804.db',
+ }
+ # The newest stamp survives in both forms.
+ assert not any('20260821-045737' in n for n in names)
+ assert size > 0
+
+
+def test_snapshot_scan_keep_zero_takes_everything(tmp_path):
+ _make_snapshots(tmp_path)
+ paths, _ = svc._scan_upgrade_snapshots(str(tmp_path), keep=0)
+ assert len(paths) == 6
+
+
+def test_snapshot_scan_ignores_unrelated_files(tmp_path):
+ _make_snapshots(tmp_path)
+ (tmp_path / 'important-customer-backup.db').write_bytes(b'z' * 100)
+ (tmp_path / 'scheduled').mkdir()
+ paths, _ = svc._scan_upgrade_snapshots(str(tmp_path), keep=0)
+ assert all('serverkit-' in os.path.basename(p) for p in paths)
+ assert (tmp_path / 'important-customer-backup.db').exists()
+
+
+def test_snapshot_reclaim_deletes_only_the_selected(tmp_path):
+ _make_snapshots(tmp_path)
+ result = svc.reclaim(['upgrade-snapshots'], backup_dir=str(tmp_path), keep=1)
+
+ assert result['freed_bytes'] > 0
+ remaining = sorted(p.name for p in tmp_path.iterdir())
+ assert remaining == [
+ 'serverkit-pre-upgrade-20260821-045737.db',
+ 'serverkit-tree-20260821-045737',
+ ]
+
+
+def test_snapshots_are_grouped_by_stamp_with_ages(tmp_path):
+ """One update writes several files under one stamp; they are offered as a
+ unit, because half a snapshot is not a restore point."""
+ from datetime import datetime, timedelta
+
+ now = datetime.utcnow()
+ stamps = [(now - timedelta(days=n)).strftime('%Y%m%d-%H%M%S') for n in (10, 3, 0)]
+ for stamp in stamps:
+ tree = tmp_path / f'serverkit-tree-{stamp}'
+ tree.mkdir()
+ (tree / 'db').write_bytes(b'x' * 100)
+ (tmp_path / f'serverkit-pre-upgrade-{stamp}.db').write_bytes(b'y' * 100)
+
+ snaps = svc.list_snapshots(str(tmp_path))
+ assert [s['stamp'] for s in snaps] == list(reversed(stamps)), 'newest first'
+ assert [s['age_days'] for s in snaps] == [0, 3, 10]
+ assert all(len(s['paths']) == 2 for s in snaps), 'tree + db grouped together'
+ assert all(s['bytes'] > 0 for s in snaps)
+
+
+def test_postgres_dump_snapshots_are_seen(tmp_path):
+ """PostgreSQL installs snapshot with pg_dump, not a file copy."""
+ (tmp_path / 'serverkit-pre-upgrade-20260101-000000.dump').write_bytes(b'x' * 500)
+ (tmp_path / 'serverkit-pre-upgrade-20260102-000000.dump').write_bytes(b'x' * 500)
+ snaps = svc.list_snapshots(str(tmp_path))
+ assert len(snaps) == 2
+ paths, size = svc._scan_upgrade_snapshots(str(tmp_path), keep=1)
+ assert [os.path.basename(p) for p in paths] == \
+ ['serverkit-pre-upgrade-20260101-000000.dump']
+ assert size == 500
+
+
+def test_select_snapshots_by_age_never_beats_keep(tmp_path):
+ from datetime import datetime, timedelta
+
+ now = datetime.utcnow()
+ snaps = [{'stamp': f's{n}', 'age_days': n, 'paths': [], 'bytes': 1}
+ for n in (0, 5, 20, 40)]
+
+ # Age filter applies only to what `keep` already released.
+ assert [s['stamp'] for s in svc.select_snapshots(snaps, keep=1, older_than_days=10)] == \
+ ['s20', 's40']
+ # Even a 0-day filter cannot take the newest when keep=1.
+ assert 's0' not in [s['stamp'] for s in svc.select_snapshots(snaps, keep=1,
+ older_than_days=0)]
+ # No age filter: everything past keep.
+ assert len(svc.select_snapshots(snaps, keep=2)) == 2
+
+
+def test_reclaim_honours_hand_picked_snapshot_stamps(tmp_path):
+ _make_snapshots(tmp_path)
+ # Deliberately pick the OLDEST and NEWEST, skipping the middle one.
+ result = svc.reclaim(['upgrade-snapshots'], backup_dir=str(tmp_path),
+ snapshot_stamps=['20260817-025332', '20260821-045737'])
+ assert result['freed_bytes'] > 0
+ remaining = sorted(p.name for p in tmp_path.iterdir())
+ assert remaining == [
+ 'serverkit-pre-upgrade-20260819-212804.db',
+ 'serverkit-tree-20260819-212804',
+ ]
+
+
+# ── /tmp staging ────────────────────────────────────────────────────────────
+
+def _age(path, hours):
+ old = time.time() - hours * 3600
+ os.utime(path, (old, old))
+
+
[email protected](os.name == 'nt', reason='uid ownership check is POSIX-only')
+def test_tmp_staging_only_matches_serverkit_update_dirs(tmp_path):
+ ours = tmp_path / 'tmp.AbCdEf1234'
+ (ours / 'opt' / 'serverkit').mkdir(parents=True)
+ _age(ours, 48)
+
+ # Right name shape, but nothing of ours inside.
+ theirs = tmp_path / 'tmp.ZzZzZz9999'
+ (theirs / 'postgres-data').mkdir(parents=True)
+ _age(theirs, 48)
+
+ # Our shape and content, but still fresh — an update may be running now.
+ fresh = tmp_path / 'tmp.FrEsH12345'
+ (fresh / 'opt' / 'serverkit').mkdir(parents=True)
+
+ # Not a mktemp name at all.
+ other = tmp_path / 'tmp.short'
+ other.mkdir()
+ _age(other, 48)
+
+ paths, _ = svc._scan_tmp_staging(str(tmp_path))
+ assert [os.path.basename(p) for p in paths] == ['tmp.AbCdEf1234']
+
+
+def test_tmp_staging_name_pattern_is_exact():
+ assert svc._MKTEMP_RE.match('tmp.AbCdEf1234')
+ assert not svc._MKTEMP_RE.match('tmp.short')
+ assert not svc._MKTEMP_RE.match('tmp.WayTooLongName99')
+ assert not svc._MKTEMP_RE.match('systemd-private-abc')
+
+
+# ── oversized login logs ────────────────────────────────────────────────────
+
+def test_login_logs_are_truncated_not_deleted(tmp_path):
+ log = tmp_path / 'btmp'
+ log.write_bytes(b'\0' * 2048)
+ freed = svc._reclaim_paths([str(log)], truncate=True)
+ assert freed == 2048
+ assert log.exists(), 'btmp must survive so it keeps its ownership and mode'
+ assert log.stat().st_size == 0
+
+
+# ── ordering: filesystem before VACUUM ──────────────────────────────────────
+
+def test_reclaim_runs_filesystem_before_the_database(tmp_path, monkeypatch):
+ """VACUUM needs headroom the filesystem steps free, so it must go last."""
+ order = []
+
+ monkeypatch.setattr(svc, '_scan_upgrade_snapshots', lambda *a, **k: ([], 0))
+ monkeypatch.setattr(svc, '_scan_tmp_staging', lambda *a, **k: ([], 0))
+ monkeypatch.setattr(svc, '_scan_journal', lambda *a, **k: (order.append('journal'), 0)[1])
+ monkeypatch.setattr(svc, '_scan_docker', lambda *a, **k: (order.append('docker'), 0)[1])
+ monkeypatch.setattr(svc, '_scan_package_caches',
+ lambda *a, **k: (order.append('caches'), ([], 0))[1])
+ monkeypatch.setattr(svc, '_scan_oversized_logs',
+ lambda *a, **k: (order.append('logs'), ([], 0))[1])
+ monkeypatch.setattr(svc, 'prune_telemetry',
+ lambda **k: (order.append('telemetry'),
+ {'bytes': 0, 'deletable_rows': 0, 'deleted_rows': 0})[1])
+ monkeypatch.setattr(svc, 'disk_usage', lambda *a, **k: None)
+
+ keys = ['telemetry', 'journal', 'docker-build-cache', 'package-caches', 'login-logs']
+ svc.reclaim(keys, dry_run=True)
+
+ assert order[-1] == 'telemetry', f'telemetry must run last, got {order}'
+
+
+def test_reclaim_ignores_keys_that_were_not_selected(tmp_path, monkeypatch):
+ called = []
+
+ def fake_list(*args, **kwargs):
+ called.append('snap')
+ return []
+
+ def fake_prune(**kwargs):
+ called.append('telemetry')
+ return {'bytes': 0, 'deletable_rows': 0, 'deleted_rows': 0}
+
+ monkeypatch.setattr(svc, 'list_snapshots', fake_list)
+ monkeypatch.setattr(svc, 'prune_telemetry', fake_prune)
+ monkeypatch.setattr(svc, 'disk_usage', lambda *a, **k: None)
+
+ svc.reclaim(['upgrade-snapshots'], dry_run=True)
+ assert called == ['snap'], 'only the selected reclaimer may run'
+
+
+# ── VACUUM preconditions ────────────────────────────────────────────────────
+
+def test_vacuum_refuses_when_the_disk_cannot_hold_the_rewrite(tmp_path, monkeypatch):
+ db_file = tmp_path / 'serverkit.db'
+ db_file.write_bytes(b'x' * 4096)
+ monkeypatch.setattr(svc, 'disk_usage',
+ lambda *a, **k: {'free': 100, 'total': 1, 'used': 1,
+ 'percent_used': 100.0, 'path': '/'})
+
+ result = svc._vacuum(str(db_file))
+ assert result['ok'] is False
+ assert 'reclaim filesystem space first' in result['error']
+
+
+def test_vacuum_skips_cleanly_on_a_non_sqlite_database():
+ result = svc._vacuum(None)
+ assert result['ok'] is False
+ assert 'not a SQLite database' in result['error']
+
+
+# ── telemetry pruning against a real database ───────────────────────────────
+
+def test_prune_keeps_fresh_rows_running_work_and_referenced_messages(app):
+ """The three things the prune must never take."""
+ from datetime import datetime, timedelta
+ from sqlalchemy import text
+ from app import db
+
+ old = datetime.utcnow() - timedelta(days=30)
+ fresh = datetime.utcnow()
+
+ def seed_queue(msg_id, status, created):
+ db.session.execute(text(
+ 'INSERT INTO queue_messages (id, queue_id, group_id, status, priority, '
+ 'payload_json, attempts, max_attempts, visible_after, created_at, updated_at) '
+ 'VALUES (:id, :q, :g, :s, 0, :p, 0, 3, :c, :c, :c)'),
+ {'id': msg_id, 'q': 'q1', 'g': 'g1', 's': status, 'p': '{}', 'c': created})
+
+ def seed_job(job_id, status, created, queue_message_id=None):
+ db.session.execute(text(
+ 'INSERT INTO jobs (id, kind, status, attempts, max_attempts, priority, '
+ 'created_at, updated_at, queue_message_id) '
+ 'VALUES (:id, :k, :s, 0, 1, 0, :c, :c, :qm)'),
+ {'id': job_id, 'k': 'builtin.test', 's': status, 'c': created,
+ 'qm': queue_message_id})
+
+ seed_queue('m-old-orphan', 'completed', old) # goes
+ seed_queue('m-old-referenced', 'completed', old) # stays: a job points at it
+ seed_queue('m-old-inflight', 'in_flight', old) # stays: still running
+ seed_queue('m-fresh', 'completed', fresh) # stays: inside the window
+
+ seed_job('j-old-succeeded', 'succeeded', old) # goes
+ seed_job('j-old-pending', 'pending', old) # stays: not terminal
+ seed_job('j-old-failed', 'failed', old) # stays: kept for diagnosis
+ seed_job('j-fresh', 'succeeded', fresh, 'm-old-referenced')
+ db.session.commit()
+
+ svc.prune_telemetry(days=7, vacuum=False)
+
+ surviving_msgs = {
+ row[0] for row in db.session.execute(text('SELECT id FROM queue_messages'))}
+ surviving_jobs = {
+ row[0] for row in db.session.execute(text('SELECT id FROM jobs'))}
+
+ assert 'm-old-orphan' not in surviving_msgs
+ assert {'m-old-referenced', 'm-old-inflight', 'm-fresh'} <= surviving_msgs
+ assert 'j-old-succeeded' not in surviving_jobs
+ assert {'j-old-pending', 'j-old-failed', 'j-fresh'} <= surviving_jobs
+
+ # No job may point at a message that is gone.
+ dangling = db.session.execute(text(
+ 'SELECT COUNT(*) FROM jobs j WHERE j.queue_message_id IS NOT NULL '
+ 'AND NOT EXISTS (SELECT 1 FROM queue_messages m WHERE m.id = j.queue_message_id)'
+ )).scalar()
+ assert dangling == 0
+
+
+def test_scan_telemetry_reports_counts_without_deleting(app):
+ from datetime import datetime, timedelta
+ from sqlalchemy import text
+ from app import db
+
+ old = datetime.utcnow() - timedelta(days=30)
+ for i in range(5):
+ db.session.execute(text(
+ 'INSERT INTO api_usage_logs (method, endpoint, status_code, created_at) '
+ 'VALUES (:m, :e, 200, :c)'),
+ {'m': 'GET', 'e': f'/api/v1/thing/{i}', 'c': old})
+ db.session.commit()
+
+ report = svc.scan_telemetry(days=7)
+ usage = next(t for t in report['tables'] if t['table'] == 'api_usage_logs')
+ assert usage['deletable'] == 5
+
+ still_there = db.session.execute(
+ text('SELECT COUNT(*) FROM api_usage_logs')).scalar()
+ assert still_there == 5, 'scan must not delete anything'
+
+
+# ── formatting ──────────────────────────────────────────────────────────────
+
[email protected]('value,expected', [
+ (0, '0 B'), (512, '512 B'), (1024, '1.0 KB'),
+ (1536, '1.5 KB'), (1024 ** 3, '1.0 GB'), (None, '-'),
+])
+def test_human_bytes(value, expected):
+ assert svc.human_bytes(value) == expected
+
+
+# ── the menu's numbers must mean what they show ─────────────────────────────
+
+def _fake_scan_with_empties():
+ """Two candidates that can free nothing, then two that can."""
+ return {
+ 'disk': {'path': '/', 'total': 100, 'used': 90, 'free': 10, 'percent_used': 90.0},
+ 'total_bytes': 3000,
+ 'candidates': [
+ {'key': 'upgrade-snapshots', 'safety': 'safe', 'title': 'Snapshots',
+ 'detail': 'none', 'bytes': 0, 'paths': []},
+ {'key': 'tmp-staging', 'safety': 'safe', 'title': 'Staging',
+ 'detail': 'none', 'bytes': 0, 'paths': []},
+ {'key': 'journal', 'safety': 'safe', 'title': 'Journal',
+ 'detail': 'over cap', 'bytes': 1000, 'paths': []},
+ {'key': 'telemetry', 'safety': 'review', 'title': 'Telemetry',
+ 'detail': 'old rows', 'bytes': 2000, 'paths': []},
+ ],
+ }
+
+
+def test_menu_number_selects_the_item_it_is_printed_next_to(monkeypatch):
+ """Regression: numbering the full list while reading numbers back off the
+ non-empty list silently selects a different item than the one shown."""
+ from click.testing import CliRunner
+ import cli as serverkit_cli
+
+ monkeypatch.setattr(svc, 'scan', lambda **k: _fake_scan_with_empties())
+ monkeypatch.setattr(serverkit_cli, 'create_app', lambda: _NullApp())
+ captured = {}
+
+ def fake_reclaim(keys, **kwargs):
+ captured['keys'] = list(keys)
+ return {'dry_run': True, 'results': [], 'freed_bytes': 0,
+ 'disk_before': None, 'disk_after': None}
+
+ monkeypatch.setattr(svc, 'reclaim', fake_reclaim)
+
+ result = CliRunner().invoke(serverkit_cli.disk, ['--dry-run'], input='2\n')
+ assert result.exit_code == 0, result.output
+
+ # 'Telemetry' is printed against a number; typing it must select telemetry.
+ line = next(ln for ln in result.output.splitlines() if 'Telemetry' in ln and 'old rows' in ln)
+ shown = line.split()[0]
+ assert shown == '2', f'expected Telemetry to be listed as #2, got {shown!r}'
+ assert captured['keys'] == ['telemetry']
+
+
+def test_menu_hides_numbers_for_items_that_free_nothing(monkeypatch):
+ from click.testing import CliRunner
+ import cli as serverkit_cli
+
+ monkeypatch.setattr(svc, 'scan', lambda **k: _fake_scan_with_empties())
+ monkeypatch.setattr(serverkit_cli, 'create_app', lambda: _NullApp())
+ monkeypatch.setattr(svc, 'reclaim', lambda keys, **k: {
+ 'dry_run': True, 'results': [], 'freed_bytes': 0,
+ 'disk_before': None, 'disk_after': None})
+
+ result = CliRunner().invoke(serverkit_cli.disk, ['--dry-run'], input='q\n')
+ snapshots = next(ln for ln in result.output.splitlines() if 'Snapshots' in ln)
+ assert snapshots.split()[0] == '-'
+
+
+class _NullApp:
+ """Stand-in for create_app() — the disk command only needs a context."""
+
+ def app_context(self):
+ return self
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+
+def test_parse_docker_reclaimed():
+ assert svc._parse_reclaimed('deleted: sha256:abc\nTotal reclaimed space: 1.234GB\n') == \
+ int(1.234 * 1024 ** 3)
+ assert svc._parse_reclaimed('Total reclaimed space: 0B') == 0
+ assert svc._parse_reclaimed('') == 0
+ assert svc._parse_reclaimed('nothing useful here') == 0
+
+
+def test_journal_scan_subtracts_the_cap(monkeypatch):
+ monkeypatch.setattr(svc, '_run',
+ lambda *a, **k: (True, 'Archived and active journals take up 260.0M '
+ 'in the file system.', ''))
+ # 260M held, 100M cap -> 160M over.
+ assert svc._scan_journal('100M') == int(160.0 * 1024 ** 2)
+ # A journal already under the cap is not a candidate.
+ assert svc._scan_journal('1G') == 0
+
+
+def test_journal_scan_is_zero_without_journalctl(monkeypatch):
+ monkeypatch.setattr(svc, '_run', lambda *a, **k: (False, '', 'not found'))
+ assert svc._scan_journal() == 0
+ assert svc._journal_bytes() == 0
+
+
+def test_package_cache_reclaim_reports_the_delta_not_zero(tmp_path, monkeypatch):
+ """apt-get clean empties the archives itself; a naive count afterwards
+ reports 0 freed even though the space came back."""
+ cache = tmp_path / 'archives'
+ cache.mkdir()
+ (cache / 'a.deb').write_bytes(b'x' * 5000)
+
+ def fake_run(cmd, timeout=300):
+ if cmd[:2] == ['apt-get', 'clean']:
+ for child in cache.iterdir():
+ child.unlink()
+ return True, '', ''
+ return False, '', 'n/a'
+
+ monkeypatch.setattr(svc, '_run', fake_run)
+ monkeypatch.setattr(svc, '_scan_package_caches', lambda: ([str(cache)], 5000))
+ monkeypatch.setattr(svc, 'disk_usage', lambda *a, **k: None)
+
+ result = svc.reclaim(['package-caches'])
+ assert result['results'][0]['bytes'] == 5000
+
+
+def test_docker_reclaim_reports_dockers_own_tally(monkeypatch):
+ monkeypatch.setattr(svc, '_scan_docker', lambda: 1024)
+ monkeypatch.setattr(svc, 'disk_usage', lambda *a, **k: None)
+
+ def fake_run(cmd, timeout=300):
+ if cmd[:3] == ['docker', 'image', 'prune']:
+ return True, 'Total reclaimed space: 2GB', ''
+ if cmd[:3] == ['docker', 'builder', 'prune']:
+ return True, 'Total reclaimed space: 1GB', ''
+ return False, '', ''
+
+ monkeypatch.setattr(svc, '_run', fake_run)
+ result = svc.reclaim(['docker-build-cache'])
+ # 3 GB actually freed, not the 1 KB the scan could estimate up front.
+ assert result['results'][0]['bytes'] == 3 * 1024 ** 3
diff --git a/serverkit b/serverkit
index fe786a1c..d3860740 100644
--- a/serverkit
+++ b/serverkit
@@ -245,6 +245,15 @@ cmd_help() {
echo " restore-db Restore database from backup"
echo ""
echo "Cleanup Commands:"
+ echo " disk Show what is using disk space and reclaim it (interactive)"
+ echo " --safe free every item marked safe, no prompt"
+ echo " --all free every item, no prompt"
+ echo " --only <keys> comma-separated items, e.g. telemetry"
+ echo " --days <n> telemetry retention window (default 7)"
+ echo " --keep <n> upgrade snapshots to keep (default 1)"
+ echo " --older-than <n> only offer snapshots at least n days old"
+ echo " --dry-run report what would be freed, change nothing"
+ echo " --allow-restart permit a panel restart if VACUUM is blocked"
echo " cleanup-apps Delete all apps and containers"
echo " factory-reset Complete reset (delete everything)"
echo ""
@@ -306,7 +315,7 @@ cmd_completion() {
_serverkit() {
local cur commands
cur="${COMP_WORDS[COMP_CWORD]}"
- commands="help start stop restart status logs update branch version check-update uninstall create-admin reset-password unlock-user list-users make-admin deactivate-user activate-user init-db migrate-db db-migrate db-status db-history backup-db restore-db cleanup-apps factory-reset generate-keys config setup-proxy create-app deploy-template list-servers deployment-status list-apps start-app stop-app app-logs add-site list-sites remove-site doctor manifest services apps repair login-url support-bundle panel-status panel-doctor completion"
+ commands="help start stop restart status logs update branch version check-update uninstall create-admin reset-password unlock-user list-users make-admin deactivate-user activate-user init-db migrate-db db-migrate db-status db-history backup-db restore-db disk cleanup-apps factory-reset generate-keys config setup-proxy create-app deploy-template list-servers deployment-status list-apps start-app stop-app app-logs add-site list-sites remove-site doctor manifest services apps repair login-url support-bundle panel-status panel-doctor completion"
if [ "$COMP_CWORD" -eq 1 ]; then
COMPREPLY=( $(compgen -W "$commands" -- "$cur") )
@@ -323,6 +332,7 @@ _serverkit() {
doctor) COMPREPLY=( $(compgen -W "--fix" -- "$cur") ) ;;
panel-doctor) COMPREPLY=( $(compgen -W "--repair --yes --json" -- "$cur") ) ;;
panel-status|list-users|list-servers|db-status) COMPREPLY=( $(compgen -W "--json" -- "$cur") ) ;;
+ disk) COMPREPLY=( $(compgen -W "--safe --all --only --days --keep --older-than --dry-run --allow-restart --yes --json" -- "$cur") ) ;;
cleanup-apps) COMPREPLY=( $(compgen -W "--delete-volumes --keep-db" -- "$cur") ) ;;
list-apps) COMPREPLY=( $(compgen -W "--all --json" -- "$cur") ) ;;
deployment-status) COMPREPLY=( $(compgen -W "--logs --json" -- "$cur") ) ;;
@@ -706,6 +716,14 @@ cmd_cleanup_apps() {
run_cli cleanup-apps $DELETE_VOLUMES $KEEP_DB
}
+cmd_disk() {
+ # Reclaim disk space. Root because the reclaimers touch /var/backups,
+ # /tmp staging, the journal and the package caches.
+ check_root
+ check_installed
+ run_cli disk "$@"
+}
+
cmd_factory_reset() {
check_root
check_installed
@@ -1481,6 +1499,10 @@ case "${1:-help}" in
completion)
cmd_completion
;;
+ disk)
+ shift
+ cmd_disk "$@"
+ ;;
cleanup-apps)
shift
cmd_cleanup_apps "$@"