ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
feat(cli): add `serverkit disk` to report and reclaim disk space
Juan Denise10ecc87d025
diff --git a/backend/app/services/disk_reclaim_service.py b/backend/app/services/disk_reclaim_service.pynew file mode 100644index 00000000..ee4c0eea--- /dev/null+++ b/backend/app/services/disk_reclaim_service.py@@ -0,0 +1,758 @@+"""Disk reclamation — find and free space when the panel host fills up.++Built for the failure mode where routine ``serverkit update`` runs fill the+disk on their own:++* ``queue_messages``, ``system_events`` and ``api_usage_logs`` have no+ retention handler, so the database grows without bound (~11 MB/day on a+ single-app box).+* Every update copies that database **twice** — a ``serverkit-pre-upgrade-*.db``+ and a full ``serverkit-tree-*/`` backup that contains it — and nothing caps+ how many of those snapshots accumulate.++Six updates against an 800 MB database is ~9.6 GB, which is enough to fill a+25 GB VPS from a standing start.++Two ordering rules are baked in because they are the difference between this+working and not working on a disk that is already at 100%:++1. Filesystem candidates always run **before** the database VACUUM. VACUUM+ rewrites the database into a temporary file alongside it, so it needs free+ space roughly equal to the database itself — it is the one step that cannot+ go first on a full disk.+2. Rows are deleted from ``jobs`` before ``queue_messages``, and any queue+ message still referenced by a surviving job is kept, so the+ ``jobs.queue_message_id`` link never dangles.++Everything is measured before anything is deleted, and every candidate can be+previewed with ``dry_run=True``.+"""+import os+import re+import shutil+import subprocess+from datetime import datetime, timedelta++# ── Defaults (overridable per call so tests never touch real paths) ──────────++INSTALL_DIR = os.environ.get('SERVERKIT_DIR', '/opt/serverkit')+# Must match BACKUP_DIR in scripts/update.sh — that is what writes the+# snapshots this module reclaims.+BACKUP_DIR = os.environ.get('SERVERKIT_BACKUP_DIR', '/var/backups/serverkit')+TMP_DIR = os.environ.get('SERVERKIT_TMP_DIR', '/tmp')++DEFAULT_RETENTION_DAYS = 7+DEFAULT_KEEP_SNAPSHOTS = 1+DEFAULT_JOURNAL_CAP = '100M'++# mktemp -d names: "tmp." + exactly 10 alphanumerics. Anything else in /tmp is+# not ours to touch.+_MKTEMP_RE = re.compile(r'^tmp\.[A-Za-z0-9]{10}$')+_SNAPSHOT_RES = (+ re.compile(r'^serverkit-tree-(\d{8}-\d{6})$'),+ re.compile(r'^serverkit-pre-upgrade-(\d{8}-\d{6})\.db$'),+ # PostgreSQL installs snapshot with pg_dump instead of a file copy.+ re.compile(r'^serverkit-pre-upgrade-(\d{8}-\d{6})\.dump$'),+ re.compile(r'^serverkit-pre-prune-(\d{8}(?:-\d{6})?)\.db$'),+)++# Telemetry tables, in deletion order. ``keep_status`` rows are never removed+# regardless of age.+TELEMETRY_SPECS = (+ # (key, table, time_column, status_column, deletable_statuses)+ ('jobs', 'jobs', 'created_at', 'status', ('succeeded', 'cancelled')),+ ('queue_messages', 'queue_messages', 'created_at', 'status', ('completed',)),+ ('system_events', 'system_events', 'created_at', None, None),+ ('api_usage_logs', 'api_usage_logs', 'created_at', None, None),+)+++def human_bytes(n):+ """Format a byte count the way ``df -h`` does."""+ if n is None:+ return '-'+ n = float(n)+ for unit in ('B', 'KB', 'MB', 'GB', 'TB'):+ if abs(n) < 1024.0 or unit == 'TB':+ return f'{n:.0f} {unit}' if unit == 'B' else f'{n:.1f} {unit}'+ n /= 1024.0+++def _run(cmd, timeout=300):+ """Run a command, returning (ok, stdout, stderr). Never raises."""+ try:+ proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)+ return proc.returncode == 0, proc.stdout or '', proc.stderr or ''+ except (OSError, subprocess.SubprocessError) as exc:+ # A missing binary (docker/journalctl not installed) is a normal+ # outcome here, not an error worth aborting the whole sweep for.+ return False, '', str(exc)+++def _path_size(path):+ """Bytes used by a file or directory tree. Symlinks are counted, not followed."""+ try:+ st = os.lstat(path)+ except OSError:+ return 0+ if not os.path.isdir(path) or os.path.islink(path):+ return st.st_size+ total = 0+ for root, dirs, files in os.walk(path, onerror=lambda e: None):+ dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(root, d))]+ for name in files + dirs:+ try:+ total += os.lstat(os.path.join(root, name)).st_size+ except OSError:+ continue+ return total+++def disk_usage(path='/'):+ """Free/total bytes for the filesystem holding ``path``."""+ try:+ usage = shutil.disk_usage(path)+ except OSError:+ return None+ return {+ 'path': path,+ 'total': usage.total,+ 'used': usage.used,+ 'free': usage.free,+ 'percent_used': round(usage.used * 100.0 / usage.total, 1) if usage.total else 0.0,+ }+++# ── Filesystem candidates ───────────────────────────────────────────────────++def _parse_stamp(stamp):+ """YYYYmmdd-HHMMSS (or bare YYYYmmdd) to a datetime, or None."""+ for fmt in ('%Y%m%d-%H%M%S', '%Y%m%d'):+ try:+ return datetime.strptime(stamp, fmt)+ except ValueError:+ continue+ return None+++def list_snapshots(backup_dir=BACKUP_DIR):+ """Every upgrade snapshot, newest first, grouped by timestamp.++ One update writes several files under the same stamp — a database copy+ (``.db`` or a PostgreSQL ``.dump``) and a full tree backup — so they are+ grouped and offered as a single unit. Deleting half a snapshot helps nobody.+ """+ try:+ entries = os.listdir(backup_dir)+ except OSError:+ return []++ groups = {}+ for name in entries:+ for pattern in _SNAPSHOT_RES:+ match = pattern.match(name)+ if match:+ groups.setdefault(match.group(1), []).append(os.path.join(backup_dir, name))+ break++ now = datetime.utcnow()+ snapshots = []+ for stamp, paths in groups.items():+ when = _parse_stamp(stamp)+ snapshots.append({+ 'stamp': stamp,+ 'paths': sorted(paths),+ 'bytes': sum(_path_size(p) for p in paths),+ 'taken_at': when.isoformat() if when else None,+ 'age_days': (now - when).days if when else None,+ })+ # Stamps are zero-padded, so lexical order is chronological order.+ snapshots.sort(key=lambda s: s['stamp'], reverse=True)+ return snapshots+++def select_snapshots(snapshots, keep=DEFAULT_KEEP_SNAPSHOTS, older_than_days=None):+ """Which snapshots to drop: everything past ``keep``, and — when+ ``older_than_days`` is given — only those at least that old.++ ``keep`` always wins, so an age filter can never empty the directory.+ """+ doomed = snapshots[keep:] if keep >= 0 else []+ if older_than_days is not None:+ doomed = [s for s in doomed+ if s['age_days'] is not None and s['age_days'] >= older_than_days]+ return doomed+++def _scan_upgrade_snapshots(backup_dir=BACKUP_DIR, keep=DEFAULT_KEEP_SNAPSHOTS,+ older_than_days=None):+ """Pre-upgrade database copies and tree backups beyond the newest ``keep``."""+ doomed = select_snapshots(list_snapshots(backup_dir), keep, older_than_days)+ paths = sorted(path for snap in doomed for path in snap['paths'])+ return paths, sum(snap['bytes'] for snap in doomed)+++def _looks_like_serverkit_staging(path):+ """True when a /tmp scratch dir holds an unpacked ServerKit release.++ Update staging dirs contain the extracted tree (``opt/serverkit`` and+ friends). Anything else under /tmp belongs to someone else.+ """+ try:+ top = os.listdir(path)+ except OSError:+ return False+ for name in top:+ if 'serverkit' in name.lower():+ return True+ if name in ('opt', 'usr', 'srv'):+ try:+ if any('serverkit' in child.lower()+ for child in os.listdir(os.path.join(path, name))):+ return True+ except OSError:+ continue+ return False+++def _scan_tmp_staging(tmp_dir=TMP_DIR, min_age_hours=24):+ """Abandoned ``mktemp -d`` staging dirs left behind by past updates."""+ cutoff = datetime.utcnow() - timedelta(hours=min_age_hours)+ # Only ever reap scratch dirs belonging to whoever is running the reclaim+ # (root in production). Another user's /tmp is never ours to clear.+ euid = os.geteuid() if hasattr(os, 'geteuid') else None+ paths = []+ try:+ entries = os.listdir(tmp_dir)+ except OSError:+ return [], 0++ for name in entries:+ if not _MKTEMP_RE.match(name):+ continue+ path = os.path.join(tmp_dir, name)+ try:+ st = os.lstat(path)+ except OSError:+ continue+ if euid is not None and st.st_uid != euid:+ continue+ if datetime.utcfromtimestamp(st.st_mtime) > cutoff:+ continue+ if os.path.isdir(path) and not _looks_like_serverkit_staging(path):+ continue+ paths.append(path)+ paths.sort()+ return paths, sum(_path_size(p) for p in paths)+++def _scan_oversized_logs(min_bytes=10 * 1024 * 1024):+ """Failed/successful-login binary logs that have grown past a sane size.++ ``btmp`` in particular balloons under sustained SSH brute-force. These are+ truncated rather than deleted so the files keep their ownership and mode.+ """+ paths = []+ for path in ('/var/log/btmp', '/var/log/btmp.1', '/var/log/wtmp', '/var/log/wtmp.1'):+ try:+ size = os.lstat(path).st_size+ except OSError:+ continue+ if size >= min_bytes:+ paths.append(path)+ return paths, sum(_path_size(p) for p in paths)+++_SCALE = {'': 1, 'K': 1024, 'M': 1024 ** 2, 'G': 1024 ** 3, 'T': 1024 ** 4}+++def _to_bytes(number, suffix):+ return int(float(number) * _SCALE.get(suffix.upper().rstrip('B'), 1))+++def _parse_reclaimed(output):+ """Bytes from docker's 'Total reclaimed space: 1.234GB' trailer."""+ match = re.search(r'Total reclaimed space:\s*([\d.]+)\s*([KMGT]?B)', output or '')+ return _to_bytes(match.group(1), match.group(2)) if match else 0+++def _journal_bytes():+ """Bytes currently held by the systemd journal, or 0 if journalctl is absent."""+ ok, out, _ = _run(['journalctl', '--disk-usage'], timeout=30)+ if not ok:+ return 0+ match = re.search(r'take up ([\d.]+)([KMGT]?)B?', out)+ return _to_bytes(match.group(1), match.group(2)) if match else 0+++def _scan_journal(cap=DEFAULT_JOURNAL_CAP):+ """Systemd journal bytes above ``cap``."""+ current = _journal_bytes()+ cap_match = re.match(r'^([\d.]+)([KMGT]?)$', str(cap).upper().rstrip('B'))+ cap_bytes = _to_bytes(cap_match.group(1), cap_match.group(2)) if cap_match else 0+ return max(0, current - cap_bytes)+++def _scan_package_caches():+ """Package-manager and language caches — all re-downloadable."""+ paths = []+ for path in ('/var/cache/apt/archives', '/var/cache/dnf', '/var/cache/yum',+ '/root/.cache/pip', '/root/.npm/_cacache'):+ if os.path.isdir(path) and _path_size(path) > 0:+ paths.append(path)+ return paths, sum(_path_size(p) for p in paths)+++def _scan_docker():+ """Dangling images and build cache. Tagged images and volumes are never touched."""+ ok, out, _ = _run(['docker', 'system', 'df', '--format', '{{.Type}}|{{.Reclaimable}}'],+ timeout=60)+ if not ok:+ return 0+ total = 0+ for line in out.splitlines():+ if '|' not in line:+ continue+ kind, reclaimable = line.split('|', 1)+ if kind.strip() != 'Build Cache':+ # Docker's image/volume "reclaimable" counts tagged-but-unused+ # images and named volumes, which we deliberately never remove —+ # so only the build cache can be estimated up front. Dangling+ # images are pruned too, and reported from docker's own tally.+ continue+ match = re.match(r'\s*([\d.]+)\s*([KMGT]?B)', reclaimable)+ if match:+ total += _to_bytes(match.group(1), match.group(2))+ return total+++# ── Database candidate ──────────────────────────────────────────────────────++def _sqlite_path():+ """Filesystem path of the SQLite database, or None for other engines."""+ from flask import current_app+ uri = current_app.config.get('SQLALCHEMY_DATABASE_URI', '')+ if not uri.startswith('sqlite:'):+ return None+ path = uri.split('///')[-1].split('?')[0]+ return path or None+++def _table_shares(db_path):+ """Bytes held by each table *including its indexes*, via the dbstat vtable.++ Returns {} when dbstat is unavailable — it is a compile-time option, so its+ absence is a missing estimate, not an error.+ """+ import sqlite3+ try:+ conn = sqlite3.connect(f'file:{db_path}?mode=ro', uri=True)+ except Exception:+ return {}+ try:+ # Each dbstat row names one btree — a table, or one of its indexes.+ # Map every index back to its table before summing, otherwise the+ # indexes (which here outweigh the tables) go uncounted.+ owner = {+ name: tbl_name for name, tbl_name in conn.execute(+ "SELECT name, tbl_name FROM sqlite_master WHERE type IN ('table', 'index')")+ }+ shares = {}+ for name, size in conn.execute('SELECT name, SUM(pgsize) FROM dbstat GROUP BY name'):+ table = owner.get(name, name)+ shares[table] = shares.get(table, 0) + (size or 0)+ return shares+ except Exception:+ return {}+ finally:+ conn.close()+++def scan_telemetry(days=DEFAULT_RETENTION_DAYS):+ """Count prunable telemetry rows and estimate the bytes they hold.++ Must run inside a Flask app context.+ """+ from sqlalchemy import text+ from app import db++ cutoff = datetime.utcnow() - timedelta(days=days)+ db_path = _sqlite_path()+ shares = _table_shares(db_path) if db_path and os.path.exists(db_path) else {}++ tables = []+ estimated = 0+ for key, table, time_col, status_col, statuses in TELEMETRY_SPECS:+ where = [f'{time_col} < :cutoff']+ params = {'cutoff': cutoff}+ if status_col and statuses:+ placeholders = ', '.join(f':s{i}' for i in range(len(statuses)))+ where.append(f'{status_col} IN ({placeholders})')+ params.update({f's{i}': value for i, value in enumerate(statuses)})+ clause = ' AND '.join(where)+ try:+ deletable = db.session.execute(+ text(f'SELECT COUNT(*) FROM {table} WHERE {clause}'), params).scalar() or 0+ total = db.session.execute(+ text(f'SELECT COUNT(*) FROM {table}')).scalar() or 0+ except Exception:+ continue++ own = shares.get(table, 0)+ share = int(own * deletable / total) if (own and total) else 0+ estimated += share+ tables.append({+ 'key': key, 'table': table, 'rows': total,+ 'deletable': deletable, 'bytes': share or None,+ })++ return {+ 'days': days,+ 'cutoff': cutoff.isoformat(),+ 'tables': tables,+ 'deletable_rows': sum(t['deletable'] for t in tables),+ 'bytes': estimated or None,+ 'db_path': db_path,+ 'db_bytes': os.path.getsize(db_path) if db_path and os.path.exists(db_path) else None,+ }+++def prune_telemetry(days=DEFAULT_RETENTION_DAYS, batch_size=5000, vacuum=True,+ allow_restart=False, dry_run=False):+ """Delete aged telemetry rows, then VACUUM to hand the pages back to the OS.++ Deleting alone does not shrink the file — the pages land on SQLite's+ freelist and get reused. VACUUM is what actually returns the space, and it+ needs free disk roughly equal to the current database size, so run the+ filesystem reclaimers first.+ """+ from sqlalchemy import text+ from app import db++ report = scan_telemetry(days=days)+ if dry_run:+ report['dry_run'] = True+ return report++ cutoff = datetime.utcnow() - timedelta(days=days)+ deleted = {}+ for key, table, time_col, status_col, statuses in TELEMETRY_SPECS:+ where = [f'{time_col} < :cutoff']+ params = {'cutoff': cutoff}+ if status_col and statuses:+ placeholders = ', '.join(f':s{i}' for i in range(len(statuses)))+ where.append(f'{status_col} IN ({placeholders})')+ params.update({f's{i}': value for i, value in enumerate(statuses)})+ if table == 'queue_messages':+ # Keep anything a surviving job still points at, or+ # jobs.queue_message_id starts dangling.+ where.append('id NOT IN (SELECT queue_message_id FROM jobs '+ 'WHERE queue_message_id IS NOT NULL)')+ clause = ' AND '.join(where)++ removed = 0+ if report.get('db_path'):+ # SQLite: batch on rowid so a large backlog never locks the table+ # in one statement.+ while True:+ result = db.session.execute(text(+ f'DELETE FROM {table} WHERE rowid IN '+ f'(SELECT rowid FROM {table} WHERE {clause} LIMIT {int(batch_size)})'+ ), params)+ db.session.commit()+ if not result.rowcount:+ break+ removed += result.rowcount+ if result.rowcount < batch_size:+ break+ else:+ # Other engines have no portable rowid; one statement, one commit.+ result = db.session.execute(+ text(f'DELETE FROM {table} WHERE {clause}'), params)+ db.session.commit()+ removed = result.rowcount or 0+ deleted[key] = removed++ report['deleted'] = deleted+ report['deleted_rows'] = sum(deleted.values())++ if vacuum:+ report['vacuum'] = _vacuum(report.get('db_path'), allow_restart=allow_restart)+ if report.get('db_path') and os.path.exists(report['db_path']):+ report['db_bytes_after'] = os.path.getsize(report['db_path'])+ return report+++def _vacuum(db_path, allow_restart=False, service='serverkit'):+ """VACUUM the database, restarting the panel only if a lock demands it."""+ from sqlalchemy import text+ from app import db++ if not db_path or not os.path.exists(db_path):+ return {'ok': False, 'error': 'not a SQLite database; skipped'}++ needed = os.path.getsize(db_path)+ free = disk_usage(os.path.dirname(db_path) or '/')+ if free and free['free'] < needed:+ # VACUUM rewrites the whole database beside itself before swapping.+ return {+ 'ok': False,+ 'error': (f'needs ~{human_bytes(needed)} free to rewrite the database, '+ f'only {human_bytes(free["free"])} available — '+ f'reclaim filesystem space first'),+ }++ try:+ # VACUUM cannot run inside a transaction, and SQLAlchemy opens one for+ # every session statement — so it needs its own autocommit connection.+ db.session.commit()+ with db.engine.connect().execution_options(isolation_level='AUTOCOMMIT') as conn:+ conn.execute(text('VACUUM'))+ return {'ok': True, 'restarted': False}+ except Exception as exc:+ if not allow_restart:+ return {'ok': False, 'error': f'{exc} (retry with --allow-restart)'}+ locked_by = exc++ # Something else holds a write lock. Stop the panel, vacuum, start it back.+ db.session.remove()+ stopped, _, err = _run(['systemctl', 'stop', service], timeout=120)+ if not stopped:+ return {'ok': False,+ 'error': f'{locked_by}; could not stop {service}: {err.strip()}'}+ try:+ import sqlite3+ conn = sqlite3.connect(db_path, timeout=60)+ try:+ conn.execute('PRAGMA wal_checkpoint(TRUNCATE)')+ conn.execute('VACUUM')+ finally:+ conn.close()+ return {'ok': True, 'restarted': True}+ except Exception as exc:+ return {'ok': False, 'error': str(exc), 'restarted': True}+ finally:+ _run(['systemctl', 'start', service], timeout=120)+++# ── Candidate registry ──────────────────────────────────────────────────────++def scan(days=DEFAULT_RETENTION_DAYS, keep=DEFAULT_KEEP_SNAPSHOTS,+ backup_dir=BACKUP_DIR, tmp_dir=TMP_DIR, with_db=True,+ older_than_days=None):+ """Measure every reclaim candidate. Reads only — deletes nothing."""+ candidates = []++ all_snapshots = list_snapshots(backup_dir)+ doomed = select_snapshots(all_snapshots, keep, older_than_days)+ kept = len(all_snapshots) - len(doomed)+ if older_than_days is not None:+ title = f'Upgrade snapshots older than {older_than_days} days'+ else:+ title = f'Old upgrade snapshots (keeping newest {keep})'+ candidates.append({+ 'key': 'upgrade-snapshots', 'safety': 'safe',+ 'title': title,+ 'detail': (f'{len(doomed)} of {len(all_snapshots)} snapshot(s) in {backup_dir}'+ f', {kept} kept' if all_snapshots else f'none in {backup_dir}'),+ 'bytes': sum(s['bytes'] for s in doomed),+ 'paths': sorted(p for s in doomed for p in s['paths']),+ # The whole inventory, so the CLI can offer a per-snapshot choice.+ 'snapshots': all_snapshots,+ 'doomed_stamps': [s['stamp'] for s in doomed],+ })++ staging, staging_bytes = _scan_tmp_staging(tmp_dir)+ candidates.append({+ 'key': 'tmp-staging', 'safety': 'safe',+ 'title': 'Abandoned update staging dirs',+ 'detail': f'{len(staging)} item(s) in {tmp_dir}, older than 24h',+ 'bytes': staging_bytes, 'paths': staging,+ })++ logs, log_bytes = _scan_oversized_logs()+ candidates.append({+ 'key': 'login-logs', 'safety': 'safe',+ 'title': 'Oversized btmp/wtmp login logs',+ 'detail': ', '.join(os.path.basename(p) for p in logs) or 'none oversized',+ 'bytes': log_bytes, 'paths': logs,+ })++ caches, cache_bytes = _scan_package_caches()+ candidates.append({+ 'key': 'package-caches', 'safety': 'safe',+ 'title': 'Package and build caches (apt/dnf/pip/npm)',+ 'detail': ', '.join(caches) or 'none found',+ 'bytes': cache_bytes, 'paths': caches,+ })++ candidates.append({+ 'key': 'journal', 'safety': 'safe',+ 'title': f'Systemd journal above {DEFAULT_JOURNAL_CAP}',+ 'detail': 'rotates then vacuums archived journals',+ 'bytes': _scan_journal(), 'paths': [],+ })++ candidates.append({+ 'key': 'docker-build-cache', 'safety': 'safe',+ 'title': 'Docker build cache and dangling images',+ 'detail': 'build cache measured; dangling images extra. '+ 'Tagged images and named volumes are left alone',+ 'bytes': _scan_docker(), 'paths': [],+ })++ if with_db:+ telemetry = scan_telemetry(days=days)+ rows = telemetry['deletable_rows']+ if rows:+ hit = ', '.join(t['table'] for t in telemetry['tables'] if t['deletable'])+ detail = f'{rows:,} rows across {hit} — then VACUUM'+ else:+ detail = 'nothing to prune'+ candidates.append({+ 'key': 'telemetry', 'safety': 'review',+ 'title': f'Telemetry rows older than {days} days',+ 'detail': detail,+ 'bytes': telemetry['bytes'] or 0, 'paths': [],+ 'telemetry': telemetry,+ })++ return {+ 'disk': disk_usage('/'),+ 'candidates': candidates,+ 'total_bytes': sum(c['bytes'] or 0 for c in candidates),+ }+++def _reclaim_paths(paths, truncate=False):+ """Delete (or truncate) a list of paths, returning bytes actually freed."""+ freed = 0+ for path in paths:+ size = _path_size(path)+ try:+ if truncate:+ with open(path, 'wb'):+ pass+ elif os.path.isdir(path) and not os.path.islink(path):+ shutil.rmtree(path, ignore_errors=True)+ else:+ os.remove(path)+ freed += size+ except OSError:+ continue+ return freed+++def reclaim(keys, days=DEFAULT_RETENTION_DAYS, keep=DEFAULT_KEEP_SNAPSHOTS,+ backup_dir=BACKUP_DIR, tmp_dir=TMP_DIR, dry_run=False,+ allow_restart=False, older_than_days=None, snapshot_stamps=None):+ """Run the selected reclaimers.++ Filesystem work is forced ahead of the database VACUUM regardless of the+ order ``keys`` arrives in — see the module docstring.++ ``snapshot_stamps`` deletes exactly those upgrade snapshots, for when the+ operator hand-picked them; otherwise ``keep``/``older_than_days`` decide.+ """+ keys = list(keys)+ before = disk_usage('/')+ results = []++ def record(key, freed, note=''):+ results.append({'key': key, 'bytes': freed, 'note': note})++ if 'upgrade-snapshots' in keys:+ if snapshot_stamps is not None:+ wanted = set(snapshot_stamps)+ chosen = [s for s in list_snapshots(backup_dir) if s['stamp'] in wanted]+ paths = sorted(p for s in chosen for p in s['paths'])+ size = sum(s['bytes'] for s in chosen)+ count = len(chosen)+ else:+ doomed = select_snapshots(list_snapshots(backup_dir), keep, older_than_days)+ paths = sorted(p for s in doomed for p in s['paths'])+ size = sum(s['bytes'] for s in doomed)+ count = len(doomed)+ record('upgrade-snapshots', size if dry_run else _reclaim_paths(paths),+ f'{count} snapshot(s), {len(paths)} file(s)')++ if 'tmp-staging' in keys:+ paths, size = _scan_tmp_staging(tmp_dir)+ record('tmp-staging', size if dry_run else _reclaim_paths(paths),+ f'{len(paths)} staging dir(s)')++ if 'login-logs' in keys:+ paths, size = _scan_oversized_logs()+ record('login-logs', size if dry_run else _reclaim_paths(paths, truncate=True),+ f'{len(paths)} log(s) truncated')++ if 'package-caches' in keys:+ paths, size = _scan_package_caches()+ if dry_run:+ record('package-caches', size)+ else:+ # Measure the delta: apt-get clean empties the archives itself, so+ # anything counted afterwards would read as zero.+ _run(['apt-get', 'clean'], timeout=120)+ _reclaim_paths(paths)+ after_size = sum(_path_size(p) for p in paths)+ record('package-caches', max(0, size - after_size))++ if 'journal' in keys:+ size = _scan_journal()+ if dry_run:+ record('journal', size)+ else:+ before_journal = _journal_bytes()+ # --rotate first: --vacuum-size only ever removes *archived*+ # journals, so without a rotation an over-cap active journal is+ # reported as "freed 0B".+ _run(['journalctl', '--rotate'], timeout=60)+ ok, _, err = _run(['journalctl', f'--vacuum-size={DEFAULT_JOURNAL_CAP}'], timeout=120)+ freed = max(0, before_journal - _journal_bytes()) if ok else 0+ record('journal', freed, '' if ok else err.strip()[:120])++ if 'docker-build-cache' in keys:+ size = _scan_docker()+ if dry_run:+ record('docker-build-cache', size)+ else:+ freed = 0+ notes = []+ for cmd in (['docker', 'image', 'prune', '-f'],+ ['docker', 'builder', 'prune', '-f']):+ ok, out, err = _run(cmd, timeout=600)+ if ok:+ # docker reports the truth; our scan can only estimate,+ # because shared layers make dangling-image sizes overlap.+ freed += _parse_reclaimed(out)+ else:+ notes.append(err.strip()[:80])+ record('docker-build-cache', freed, '; '.join(notes))++ # Database last: VACUUM needs the headroom the steps above just freed.+ if 'telemetry' in keys:+ report = prune_telemetry(days=days, dry_run=dry_run, allow_restart=allow_restart)+ if dry_run:+ record('telemetry', report.get('bytes') or 0,+ f"{report['deletable_rows']:,} row(s)")+ else:+ freed = 0+ if report.get('db_bytes') and report.get('db_bytes_after'):+ freed = max(0, report['db_bytes'] - report['db_bytes_after'])+ vacuum = report.get('vacuum') or {}+ note = f"{report['deleted_rows']:,} row(s) deleted"+ if not vacuum.get('ok'):+ note += f"; VACUUM skipped: {vacuum.get('error', 'unknown')}"+ elif vacuum.get('restarted'):+ note += '; panel restarted for VACUUM'+ record('telemetry', freed, note)++ after = disk_usage('/')+ return {+ 'dry_run': dry_run,+ 'results': results,+ 'freed_bytes': sum(r['bytes'] or 0 for r in results),+ 'disk_before': before,+ 'disk_after': after,+ }diff --git a/backend/cli.py b/backend/cli.pyindex a1320330..eae2db9d 100644--- a/backend/cli.py+++ b/backend/cli.py@@ -1270,5 +1270,266 @@ def support_bundle(out_path, passphrase):click.echo(click.style(f'Support bundle written: {path}', fg='green'))+# ── disk (reclaim space) ─────────────────────────────────────────────────────+[email protected]('--days', default=None, type=int,+ help='Prune telemetry rows older than this many days (default 7)')[email protected]('--keep', default=None, type=int,+ help='Upgrade snapshots to keep, newest first (default 1)')[email protected]('--older-than', 'older_than', default=None, type=int,+ help='Only offer upgrade snapshots at least this many days old')[email protected]('--only', default=None,+ help='Comma-separated candidate keys to reclaim, skipping the menu')[email protected]('--safe', is_flag=True, help='Reclaim every candidate marked safe, no prompt')[email protected]('--all', 'do_all', is_flag=True, help='Reclaim every candidate, no prompt')[email protected]('--dry-run', is_flag=True, help='Report what would be freed, change nothing')[email protected]('--allow-restart', is_flag=True,+ help='Let the panel restart if the database is locked during VACUUM')[email protected]('--yes', is_flag=True, help='Skip the confirmation prompt')[email protected]('--json', 'as_json', is_flag=True, help='Output machine-readable JSON')+def disk(days, keep, older_than, only, safe, do_all, dry_run, allow_restart, yes, as_json):+ """Show what is using disk space and reclaim it.++ With no options this lists every reclaimable item with its measured size and+ asks which to free — pick by number ("1,3"), by "safe", or "all".+ """+ from app.services import disk_reclaim_service as svc++ days = svc.DEFAULT_RETENTION_DAYS if days is None else days+ keep = svc.DEFAULT_KEEP_SNAPSHOTS if keep is None else keep+ if keep < 0:+ _fail('--keep cannot be negative.')+ if days < 0:+ _fail('--days cannot be negative.')++ app = create_app()+ with app.app_context():+ report = svc.scan(days=days, keep=keep, older_than_days=older_than)+ candidates = report['candidates']+ usable = [c for c in candidates if (c['bytes'] or 0) > 0]++ if as_json and not (only or safe or do_all):+ _echo_json(report)+ return++ disk_info = report['disk'] or {}+ if not as_json:+ click.echo()+ if disk_info:+ bar_used = disk_info['percent_used']+ colour = 'red' if bar_used >= 90 else ('yellow' if bar_used >= 75 else 'green')+ click.echo(click.style(+ f"Disk {disk_info['path']}: {svc.human_bytes(disk_info['used'])} used "+ f"of {svc.human_bytes(disk_info['total'])} "+ f"({bar_used}%), {svc.human_bytes(disk_info['free'])} free",+ fg=colour))+ click.echo()+ # Number only the items that can actually free something, and+ # number them exactly the way the prompt reads them back.+ numbers = {cand['key']: n for n, cand in enumerate(usable, start=1)}+ rows = []+ for cand in candidates:+ size = cand['bytes'] or 0+ rows.append((+ str(numbers.get(cand['key'], '-')),+ svc.human_bytes(size) if size else '—',+ cand['safety'],+ cand['title'],+ cand['detail'],+ ))+ _echo_table(['#', 'Reclaims', 'Safety', 'Item', 'Detail'], rows)+ click.echo()+ click.echo(f"Total reclaimable: "+ f"{click.style(svc.human_bytes(report['total_bytes']), bold=True)}")++ # Work out the selection.+ if do_all:+ chosen = [c['key'] for c in usable]+ elif safe:+ chosen = [c['key'] for c in usable if c['safety'] == 'safe']+ elif only:+ wanted = {k.strip() for k in only.split(',') if k.strip()}+ known = {c['key'] for c in candidates}+ unknown = wanted - known+ if unknown:+ _fail('unknown candidate(s): ' + ', '.join(sorted(unknown))+ + '. Known: ' + ', '.join(sorted(known)))+ chosen = [c['key'] for c in candidates if c['key'] in wanted]+ elif not usable:+ click.echo('\nNothing to reclaim.')+ return+ else:+ chosen = _prompt_disk_selection(usable)+ if chosen is None:+ click.echo('Aborted.')+ return++ if not chosen:+ click.echo('Nothing selected.')+ return++ # When snapshots were chosen interactively, let the operator say which+ # ones rather than assuming the keep/age rule.+ stamps = None+ interactive = not (do_all or safe or only)+ if 'upgrade-snapshots' in chosen and interactive and not as_json:+ snapshot_cand = next(c for c in candidates if c['key'] == 'upgrade-snapshots')+ stamps = _prompt_snapshot_choice(snapshot_cand)+ if not stamps:+ chosen = [k for k in chosen if k != 'upgrade-snapshots']+ if not chosen:+ click.echo('Nothing selected.')+ return++ picked = [c for c in candidates if c['key'] in chosen]+ planned = sum(+ (sum(s['bytes'] for s in c['snapshots'] if s['stamp'] in stamps)+ if (stamps is not None and c['key'] == 'upgrade-snapshots')+ else (c['bytes'] or 0))+ for c in picked)+ if not as_json:+ click.echo()+ for cand in picked:+ size = cand['bytes'] or 0+ label = cand['title']+ if stamps is not None and cand['key'] == 'upgrade-snapshots':+ size = sum(s['bytes'] for s in cand['snapshots'] if s['stamp'] in stamps)+ label = f'{len(stamps)} upgrade snapshot(s)'+ click.echo(f' • {label} ({svc.human_bytes(size)})')+ verb = 'Would free' if dry_run else 'Free'+ click.echo(f"\n{verb} about {click.style(svc.human_bytes(planned), bold=True)}.")++ if not dry_run and not yes and not as_json:+ if not click.confirm('Proceed?', default=False):+ click.echo('Aborted.')+ return++ result = svc.reclaim(chosen, days=days, keep=keep, dry_run=dry_run,+ allow_restart=allow_restart, older_than_days=older_than,+ snapshot_stamps=stamps)++ if as_json:+ _echo_json(result)+ return++ click.echo()+ _echo_table(['Item', 'Freed', 'Note'],+ [(r['key'], svc.human_bytes(r['bytes'] or 0), r['note']) for r in result['results']])+ after = result['disk_after'] or {}+ click.echo()+ click.echo(click.style(+ f"{'Would free' if dry_run else 'Freed'} "+ f"{svc.human_bytes(result['freed_bytes'])}.", fg='green', bold=True))+ if after and not dry_run:+ click.echo(f"Disk now {after['percent_used']}% used, "+ f"{svc.human_bytes(after['free'])} free.")+++def _prompt_snapshot_choice(candidate):+ """Offer the upgrade snapshots one by one. Returns the stamps to delete.++ One update writes a database copy and a whole tree backup under the same+ timestamp, so snapshots are picked as units — a half-deleted snapshot is+ not a restore point.+ """+ from app.services import disk_reclaim_service as svc++ snapshots = candidate.get('snapshots') or []+ if len(snapshots) <= 1:+ return candidate.get('doomed_stamps') or []++ default_stamps = candidate.get('doomed_stamps') or []+ click.echo(f'\n{len(snapshots)} upgrade snapshot(s), newest first:')+ rows = []+ for index, snap in enumerate(snapshots, start=1):+ age = snap['age_days']+ if age is None:+ when = '-'+ elif age == 0:+ when = 'today'+ else:+ when = f'{age} day{"s" if age != 1 else ""} ago'+ rows.append((+ str(index),+ snap['stamp'],+ when,+ svc.human_bytes(snap['bytes']),+ 'newest — kept by default' if index == 1 else+ ('would delete' if snap['stamp'] in default_stamps else 'kept'),+ ))+ _echo_table(['#', 'Snapshot', 'Taken', 'Size', ''], rows)++ click.echo('\nDelete which? numbers like "1,3", "older-than 14" (days), '+ '"all-but-newest", or "none".')+ by_index = {index: snap for index, snap in enumerate(snapshots, start=1)}+ while True:+ answer = click.prompt('Snapshots', default='all-but-newest',+ show_default=True).strip().lower()+ if answer in ('none', 'n', 'q'):+ return []+ if answer == 'all-but-newest':+ return [s['stamp'] for s in snapshots[1:]]+ if answer.startswith('older-than'):+ raw = answer.replace('older-than', '').strip()+ if not raw.isdigit():+ click.echo(click.style('Give a number of days, e.g. "older-than 14".',+ fg='yellow'))+ continue+ chosen = svc.select_snapshots(snapshots, keep=1, older_than_days=int(raw))+ if not chosen:+ click.echo(click.style(+ f'No snapshot besides the newest is {raw}+ days old.', fg='yellow'))+ continue+ return [s['stamp'] for s in chosen]+ picked, bad = [], []+ for token in answer.replace(' ', ',').split(','):+ if not token:+ continue+ if token.isdigit() and int(token) in by_index:+ picked.append(by_index[int(token)]['stamp'])+ else:+ bad.append(token)+ if bad:+ click.echo(click.style('Not a listed number: ' + ', '.join(bad), fg='yellow'))+ continue+ if picked:+ if len(picked) == len(snapshots) and not click.confirm(+ 'That deletes every snapshot, leaving no restore point. Sure?',+ default=False):+ continue+ return picked+ click.echo(click.style('Nothing selected — pick at least one number.', fg='yellow'))+++def _prompt_disk_selection(usable):+ """Ask which candidates to reclaim. Returns keys, or None to abort."""+ by_index = {index: cand for index, cand in enumerate(usable, start=1)}+ click.echo('\nSelect what to free: numbers like "1,3", "safe" for everything '+ 'marked safe, "all", or "q" to quit.')+ while True:+ answer = click.prompt('Selection', default='safe', show_default=True).strip().lower()+ if answer in ('q', 'quit', 'n', 'no'):+ return None+ if answer == 'all':+ return [c['key'] for c in usable]+ if answer == 'safe':+ return [c['key'] for c in usable if c['safety'] == 'safe']+ picked, bad = [], []+ for token in answer.replace(' ', ',').split(','):+ if not token:+ continue+ if token.isdigit() and int(token) in by_index:+ picked.append(by_index[int(token)]['key'])+ else:+ bad.append(token)+ if bad:+ click.echo(click.style('Not a listed number: ' + ', '.join(bad), fg='yellow'))+ continue+ if picked:+ return picked+ click.echo(click.style('Nothing selected — pick at least one number.', fg='yellow'))++if __name__ == '__main__':cli()diff --git a/backend/tests/test_cli_json.py b/backend/tests/test_cli_json.pyindex 31cc9e13..2092548c 100644--- a/backend/tests/test_cli_json.py+++ b/backend/tests/test_cli_json.py@@ -182,3 +182,19 @@ def test_deployment_status_json(app, monkeypatch):serverkit_cli.cli, ['deployment-status', 'j-1', '--json'])data = _json_out(result)assert data['job']['status'] == 'succeeded'+++def test_disk_json_is_a_read_only_report(app, monkeypatch):+ """`disk --json` with no selection flag reports and changes nothing."""+ from app.services import disk_reclaim_service as svc++ monkeypatch.setattr(svc, 'reclaim', lambda *a, **k: pytest.fail(+ 'disk --json without a selection flag must not reclaim anything'))++ result = CliRunner().invoke(serverkit_cli.cli, ['disk', '--json'])+ data = _json_out(result)+ assert 'candidates' in data and 'disk' in data+ keys = {c['key'] for c in data['candidates']}+ assert {'upgrade-snapshots', 'telemetry'} <= keys+ for cand in data['candidates']:+ assert cand['safety'] in ('safe', 'review')diff --git a/backend/tests/test_disk_reclaim.py b/backend/tests/test_disk_reclaim.pynew file mode 100644index 00000000..abca9b41--- /dev/null+++ b/backend/tests/test_disk_reclaim.py@@ -0,0 +1,473 @@+"""Disk reclamation: the scan must measure honestly and the reclaimers must+only ever touch what they claim to.++The guards that matter here are the ones that were learned the hard way when a+25 GB panel host filled to 100% purely from `serverkit update` runs:++* filesystem reclaimers run BEFORE the database VACUUM (VACUUM needs free space+ equal to the database it rewrites, so it cannot go first on a full disk);+* the newest upgrade snapshot is never deleted;+* /tmp entries that aren't ours are never deleted;+* a queue message still referenced by a surviving job is never deleted.+"""+import os+import time++import pytest++from app.services import disk_reclaim_service as svc+++# ── upgrade snapshots ───────────────────────────────────────────────────────++def _make_snapshots(root):+ """Three upgrade snapshot sets, oldest to newest."""+ for stamp in ('20260817-025332', '20260819-212804', '20260821-045737'):+ tree = root / f'serverkit-tree-{stamp}'+ (tree / 'backend').mkdir(parents=True)+ (tree / 'backend' / 'serverkit.db').write_bytes(b'x' * 1000)+ (root / f'serverkit-pre-upgrade-{stamp}.db').write_bytes(b'y' * 1000)+ return root+++def test_snapshot_scan_keeps_the_newest_set(tmp_path):+ _make_snapshots(tmp_path)+ paths, size = svc._scan_upgrade_snapshots(str(tmp_path), keep=1)++ names = {os.path.basename(p) for p in paths}+ assert names == {+ 'serverkit-tree-20260817-025332', 'serverkit-pre-upgrade-20260817-025332.db',+ 'serverkit-tree-20260819-212804', 'serverkit-pre-upgrade-20260819-212804.db',+ }+ # The newest stamp survives in both forms.+ assert not any('20260821-045737' in n for n in names)+ assert size > 0+++def test_snapshot_scan_keep_zero_takes_everything(tmp_path):+ _make_snapshots(tmp_path)+ paths, _ = svc._scan_upgrade_snapshots(str(tmp_path), keep=0)+ assert len(paths) == 6+++def test_snapshot_scan_ignores_unrelated_files(tmp_path):+ _make_snapshots(tmp_path)+ (tmp_path / 'important-customer-backup.db').write_bytes(b'z' * 100)+ (tmp_path / 'scheduled').mkdir()+ paths, _ = svc._scan_upgrade_snapshots(str(tmp_path), keep=0)+ assert all('serverkit-' in os.path.basename(p) for p in paths)+ assert (tmp_path / 'important-customer-backup.db').exists()+++def test_snapshot_reclaim_deletes_only_the_selected(tmp_path):+ _make_snapshots(tmp_path)+ result = svc.reclaim(['upgrade-snapshots'], backup_dir=str(tmp_path), keep=1)++ assert result['freed_bytes'] > 0+ remaining = sorted(p.name for p in tmp_path.iterdir())+ assert remaining == [+ 'serverkit-pre-upgrade-20260821-045737.db',+ 'serverkit-tree-20260821-045737',+ ]+++def test_snapshots_are_grouped_by_stamp_with_ages(tmp_path):+ """One update writes several files under one stamp; they are offered as a+ unit, because half a snapshot is not a restore point."""+ from datetime import datetime, timedelta++ now = datetime.utcnow()+ stamps = [(now - timedelta(days=n)).strftime('%Y%m%d-%H%M%S') for n in (10, 3, 0)]+ for stamp in stamps:+ tree = tmp_path / f'serverkit-tree-{stamp}'+ tree.mkdir()+ (tree / 'db').write_bytes(b'x' * 100)+ (tmp_path / f'serverkit-pre-upgrade-{stamp}.db').write_bytes(b'y' * 100)++ snaps = svc.list_snapshots(str(tmp_path))+ assert [s['stamp'] for s in snaps] == list(reversed(stamps)), 'newest first'+ assert [s['age_days'] for s in snaps] == [0, 3, 10]+ assert all(len(s['paths']) == 2 for s in snaps), 'tree + db grouped together'+ assert all(s['bytes'] > 0 for s in snaps)+++def test_postgres_dump_snapshots_are_seen(tmp_path):+ """PostgreSQL installs snapshot with pg_dump, not a file copy."""+ (tmp_path / 'serverkit-pre-upgrade-20260101-000000.dump').write_bytes(b'x' * 500)+ (tmp_path / 'serverkit-pre-upgrade-20260102-000000.dump').write_bytes(b'x' * 500)+ snaps = svc.list_snapshots(str(tmp_path))+ assert len(snaps) == 2+ paths, size = svc._scan_upgrade_snapshots(str(tmp_path), keep=1)+ assert [os.path.basename(p) for p in paths] == \+ ['serverkit-pre-upgrade-20260101-000000.dump']+ assert size == 500+++def test_select_snapshots_by_age_never_beats_keep(tmp_path):+ from datetime import datetime, timedelta++ now = datetime.utcnow()+ snaps = [{'stamp': f's{n}', 'age_days': n, 'paths': [], 'bytes': 1}+ for n in (0, 5, 20, 40)]++ # Age filter applies only to what `keep` already released.+ assert [s['stamp'] for s in svc.select_snapshots(snaps, keep=1, older_than_days=10)] == \+ ['s20', 's40']+ # Even a 0-day filter cannot take the newest when keep=1.+ assert 's0' not in [s['stamp'] for s in svc.select_snapshots(snaps, keep=1,+ older_than_days=0)]+ # No age filter: everything past keep.+ assert len(svc.select_snapshots(snaps, keep=2)) == 2+++def test_reclaim_honours_hand_picked_snapshot_stamps(tmp_path):+ _make_snapshots(tmp_path)+ # Deliberately pick the OLDEST and NEWEST, skipping the middle one.+ result = svc.reclaim(['upgrade-snapshots'], backup_dir=str(tmp_path),+ snapshot_stamps=['20260817-025332', '20260821-045737'])+ assert result['freed_bytes'] > 0+ remaining = sorted(p.name for p in tmp_path.iterdir())+ assert remaining == [+ 'serverkit-pre-upgrade-20260819-212804.db',+ 'serverkit-tree-20260819-212804',+ ]+++# ── /tmp staging ────────────────────────────────────────────────────────────++def _age(path, hours):+ old = time.time() - hours * 3600+ os.utime(path, (old, old))++[email protected](os.name == 'nt', reason='uid ownership check is POSIX-only')+def test_tmp_staging_only_matches_serverkit_update_dirs(tmp_path):+ ours = tmp_path / 'tmp.AbCdEf1234'+ (ours / 'opt' / 'serverkit').mkdir(parents=True)+ _age(ours, 48)++ # Right name shape, but nothing of ours inside.+ theirs = tmp_path / 'tmp.ZzZzZz9999'+ (theirs / 'postgres-data').mkdir(parents=True)+ _age(theirs, 48)++ # Our shape and content, but still fresh — an update may be running now.+ fresh = tmp_path / 'tmp.FrEsH12345'+ (fresh / 'opt' / 'serverkit').mkdir(parents=True)++ # Not a mktemp name at all.+ other = tmp_path / 'tmp.short'+ other.mkdir()+ _age(other, 48)++ paths, _ = svc._scan_tmp_staging(str(tmp_path))+ assert [os.path.basename(p) for p in paths] == ['tmp.AbCdEf1234']+++def test_tmp_staging_name_pattern_is_exact():+ assert svc._MKTEMP_RE.match('tmp.AbCdEf1234')+ assert not svc._MKTEMP_RE.match('tmp.short')+ assert not svc._MKTEMP_RE.match('tmp.WayTooLongName99')+ assert not svc._MKTEMP_RE.match('systemd-private-abc')+++# ── oversized login logs ────────────────────────────────────────────────────++def test_login_logs_are_truncated_not_deleted(tmp_path):+ log = tmp_path / 'btmp'+ log.write_bytes(b'\0' * 2048)+ freed = svc._reclaim_paths([str(log)], truncate=True)+ assert freed == 2048+ assert log.exists(), 'btmp must survive so it keeps its ownership and mode'+ assert log.stat().st_size == 0+++# ── ordering: filesystem before VACUUM ──────────────────────────────────────++def test_reclaim_runs_filesystem_before_the_database(tmp_path, monkeypatch):+ """VACUUM needs headroom the filesystem steps free, so it must go last."""+ order = []++ monkeypatch.setattr(svc, '_scan_upgrade_snapshots', lambda *a, **k: ([], 0))+ monkeypatch.setattr(svc, '_scan_tmp_staging', lambda *a, **k: ([], 0))+ monkeypatch.setattr(svc, '_scan_journal', lambda *a, **k: (order.append('journal'), 0)[1])+ monkeypatch.setattr(svc, '_scan_docker', lambda *a, **k: (order.append('docker'), 0)[1])+ monkeypatch.setattr(svc, '_scan_package_caches',+ lambda *a, **k: (order.append('caches'), ([], 0))[1])+ monkeypatch.setattr(svc, '_scan_oversized_logs',+ lambda *a, **k: (order.append('logs'), ([], 0))[1])+ monkeypatch.setattr(svc, 'prune_telemetry',+ lambda **k: (order.append('telemetry'),+ {'bytes': 0, 'deletable_rows': 0, 'deleted_rows': 0})[1])+ monkeypatch.setattr(svc, 'disk_usage', lambda *a, **k: None)++ keys = ['telemetry', 'journal', 'docker-build-cache', 'package-caches', 'login-logs']+ svc.reclaim(keys, dry_run=True)++ assert order[-1] == 'telemetry', f'telemetry must run last, got {order}'+++def test_reclaim_ignores_keys_that_were_not_selected(tmp_path, monkeypatch):+ called = []++ def fake_list(*args, **kwargs):+ called.append('snap')+ return []++ def fake_prune(**kwargs):+ called.append('telemetry')+ return {'bytes': 0, 'deletable_rows': 0, 'deleted_rows': 0}++ monkeypatch.setattr(svc, 'list_snapshots', fake_list)+ monkeypatch.setattr(svc, 'prune_telemetry', fake_prune)+ monkeypatch.setattr(svc, 'disk_usage', lambda *a, **k: None)++ svc.reclaim(['upgrade-snapshots'], dry_run=True)+ assert called == ['snap'], 'only the selected reclaimer may run'+++# ── VACUUM preconditions ────────────────────────────────────────────────────++def test_vacuum_refuses_when_the_disk_cannot_hold_the_rewrite(tmp_path, monkeypatch):+ db_file = tmp_path / 'serverkit.db'+ db_file.write_bytes(b'x' * 4096)+ monkeypatch.setattr(svc, 'disk_usage',+ lambda *a, **k: {'free': 100, 'total': 1, 'used': 1,+ 'percent_used': 100.0, 'path': '/'})++ result = svc._vacuum(str(db_file))+ assert result['ok'] is False+ assert 'reclaim filesystem space first' in result['error']+++def test_vacuum_skips_cleanly_on_a_non_sqlite_database():+ result = svc._vacuum(None)+ assert result['ok'] is False+ assert 'not a SQLite database' in result['error']+++# ── telemetry pruning against a real database ───────────────────────────────++def test_prune_keeps_fresh_rows_running_work_and_referenced_messages(app):+ """The three things the prune must never take."""+ from datetime import datetime, timedelta+ from sqlalchemy import text+ from app import db++ old = datetime.utcnow() - timedelta(days=30)+ fresh = datetime.utcnow()++ def seed_queue(msg_id, status, created):+ db.session.execute(text(+ 'INSERT INTO queue_messages (id, queue_id, group_id, status, priority, '+ 'payload_json, attempts, max_attempts, visible_after, created_at, updated_at) '+ 'VALUES (:id, :q, :g, :s, 0, :p, 0, 3, :c, :c, :c)'),+ {'id': msg_id, 'q': 'q1', 'g': 'g1', 's': status, 'p': '{}', 'c': created})++ def seed_job(job_id, status, created, queue_message_id=None):+ db.session.execute(text(+ 'INSERT INTO jobs (id, kind, status, attempts, max_attempts, priority, '+ 'created_at, updated_at, queue_message_id) '+ 'VALUES (:id, :k, :s, 0, 1, 0, :c, :c, :qm)'),+ {'id': job_id, 'k': 'builtin.test', 's': status, 'c': created,+ 'qm': queue_message_id})++ seed_queue('m-old-orphan', 'completed', old) # goes+ seed_queue('m-old-referenced', 'completed', old) # stays: a job points at it+ seed_queue('m-old-inflight', 'in_flight', old) # stays: still running+ seed_queue('m-fresh', 'completed', fresh) # stays: inside the window++ seed_job('j-old-succeeded', 'succeeded', old) # goes+ seed_job('j-old-pending', 'pending', old) # stays: not terminal+ seed_job('j-old-failed', 'failed', old) # stays: kept for diagnosis+ seed_job('j-fresh', 'succeeded', fresh, 'm-old-referenced')+ db.session.commit()++ svc.prune_telemetry(days=7, vacuum=False)++ surviving_msgs = {+ row[0] for row in db.session.execute(text('SELECT id FROM queue_messages'))}+ surviving_jobs = {+ row[0] for row in db.session.execute(text('SELECT id FROM jobs'))}++ assert 'm-old-orphan' not in surviving_msgs+ assert {'m-old-referenced', 'm-old-inflight', 'm-fresh'} <= surviving_msgs+ assert 'j-old-succeeded' not in surviving_jobs+ assert {'j-old-pending', 'j-old-failed', 'j-fresh'} <= surviving_jobs++ # No job may point at a message that is gone.+ dangling = db.session.execute(text(+ 'SELECT COUNT(*) FROM jobs j WHERE j.queue_message_id IS NOT NULL '+ 'AND NOT EXISTS (SELECT 1 FROM queue_messages m WHERE m.id = j.queue_message_id)'+ )).scalar()+ assert dangling == 0+++def test_scan_telemetry_reports_counts_without_deleting(app):+ from datetime import datetime, timedelta+ from sqlalchemy import text+ from app import db++ old = datetime.utcnow() - timedelta(days=30)+ for i in range(5):+ db.session.execute(text(+ 'INSERT INTO api_usage_logs (method, endpoint, status_code, created_at) '+ 'VALUES (:m, :e, 200, :c)'),+ {'m': 'GET', 'e': f'/api/v1/thing/{i}', 'c': old})+ db.session.commit()++ report = svc.scan_telemetry(days=7)+ usage = next(t for t in report['tables'] if t['table'] == 'api_usage_logs')+ assert usage['deletable'] == 5++ still_there = db.session.execute(+ text('SELECT COUNT(*) FROM api_usage_logs')).scalar()+ assert still_there == 5, 'scan must not delete anything'+++# ── formatting ──────────────────────────────────────────────────────────────+[email protected]('value,expected', [+ (0, '0 B'), (512, '512 B'), (1024, '1.0 KB'),+ (1536, '1.5 KB'), (1024 ** 3, '1.0 GB'), (None, '-'),+])+def test_human_bytes(value, expected):+ assert svc.human_bytes(value) == expected+++# ── the menu's numbers must mean what they show ─────────────────────────────++def _fake_scan_with_empties():+ """Two candidates that can free nothing, then two that can."""+ return {+ 'disk': {'path': '/', 'total': 100, 'used': 90, 'free': 10, 'percent_used': 90.0},+ 'total_bytes': 3000,+ 'candidates': [+ {'key': 'upgrade-snapshots', 'safety': 'safe', 'title': 'Snapshots',+ 'detail': 'none', 'bytes': 0, 'paths': []},+ {'key': 'tmp-staging', 'safety': 'safe', 'title': 'Staging',+ 'detail': 'none', 'bytes': 0, 'paths': []},+ {'key': 'journal', 'safety': 'safe', 'title': 'Journal',+ 'detail': 'over cap', 'bytes': 1000, 'paths': []},+ {'key': 'telemetry', 'safety': 'review', 'title': 'Telemetry',+ 'detail': 'old rows', 'bytes': 2000, 'paths': []},+ ],+ }+++def test_menu_number_selects_the_item_it_is_printed_next_to(monkeypatch):+ """Regression: numbering the full list while reading numbers back off the+ non-empty list silently selects a different item than the one shown."""+ from click.testing import CliRunner+ import cli as serverkit_cli++ monkeypatch.setattr(svc, 'scan', lambda **k: _fake_scan_with_empties())+ monkeypatch.setattr(serverkit_cli, 'create_app', lambda: _NullApp())+ captured = {}++ def fake_reclaim(keys, **kwargs):+ captured['keys'] = list(keys)+ return {'dry_run': True, 'results': [], 'freed_bytes': 0,+ 'disk_before': None, 'disk_after': None}++ monkeypatch.setattr(svc, 'reclaim', fake_reclaim)++ result = CliRunner().invoke(serverkit_cli.disk, ['--dry-run'], input='2\n')+ assert result.exit_code == 0, result.output++ # 'Telemetry' is printed against a number; typing it must select telemetry.+ line = next(ln for ln in result.output.splitlines() if 'Telemetry' in ln and 'old rows' in ln)+ shown = line.split()[0]+ assert shown == '2', f'expected Telemetry to be listed as #2, got {shown!r}'+ assert captured['keys'] == ['telemetry']+++def test_menu_hides_numbers_for_items_that_free_nothing(monkeypatch):+ from click.testing import CliRunner+ import cli as serverkit_cli++ monkeypatch.setattr(svc, 'scan', lambda **k: _fake_scan_with_empties())+ monkeypatch.setattr(serverkit_cli, 'create_app', lambda: _NullApp())+ monkeypatch.setattr(svc, 'reclaim', lambda keys, **k: {+ 'dry_run': True, 'results': [], 'freed_bytes': 0,+ 'disk_before': None, 'disk_after': None})++ result = CliRunner().invoke(serverkit_cli.disk, ['--dry-run'], input='q\n')+ snapshots = next(ln for ln in result.output.splitlines() if 'Snapshots' in ln)+ assert snapshots.split()[0] == '-'+++class _NullApp:+ """Stand-in for create_app() — the disk command only needs a context."""++ def app_context(self):+ return self++ def __enter__(self):+ return self++ def __exit__(self, *exc):+ return False+++def test_parse_docker_reclaimed():+ assert svc._parse_reclaimed('deleted: sha256:abc\nTotal reclaimed space: 1.234GB\n') == \+ int(1.234 * 1024 ** 3)+ assert svc._parse_reclaimed('Total reclaimed space: 0B') == 0+ assert svc._parse_reclaimed('') == 0+ assert svc._parse_reclaimed('nothing useful here') == 0+++def test_journal_scan_subtracts_the_cap(monkeypatch):+ monkeypatch.setattr(svc, '_run',+ lambda *a, **k: (True, 'Archived and active journals take up 260.0M '+ 'in the file system.', ''))+ # 260M held, 100M cap -> 160M over.+ assert svc._scan_journal('100M') == int(160.0 * 1024 ** 2)+ # A journal already under the cap is not a candidate.+ assert svc._scan_journal('1G') == 0+++def test_journal_scan_is_zero_without_journalctl(monkeypatch):+ monkeypatch.setattr(svc, '_run', lambda *a, **k: (False, '', 'not found'))+ assert svc._scan_journal() == 0+ assert svc._journal_bytes() == 0+++def test_package_cache_reclaim_reports_the_delta_not_zero(tmp_path, monkeypatch):+ """apt-get clean empties the archives itself; a naive count afterwards+ reports 0 freed even though the space came back."""+ cache = tmp_path / 'archives'+ cache.mkdir()+ (cache / 'a.deb').write_bytes(b'x' * 5000)++ def fake_run(cmd, timeout=300):+ if cmd[:2] == ['apt-get', 'clean']:+ for child in cache.iterdir():+ child.unlink()+ return True, '', ''+ return False, '', 'n/a'++ monkeypatch.setattr(svc, '_run', fake_run)+ monkeypatch.setattr(svc, '_scan_package_caches', lambda: ([str(cache)], 5000))+ monkeypatch.setattr(svc, 'disk_usage', lambda *a, **k: None)++ result = svc.reclaim(['package-caches'])+ assert result['results'][0]['bytes'] == 5000+++def test_docker_reclaim_reports_dockers_own_tally(monkeypatch):+ monkeypatch.setattr(svc, '_scan_docker', lambda: 1024)+ monkeypatch.setattr(svc, 'disk_usage', lambda *a, **k: None)++ def fake_run(cmd, timeout=300):+ if cmd[:3] == ['docker', 'image', 'prune']:+ return True, 'Total reclaimed space: 2GB', ''+ if cmd[:3] == ['docker', 'builder', 'prune']:+ return True, 'Total reclaimed space: 1GB', ''+ return False, '', ''++ monkeypatch.setattr(svc, '_run', fake_run)+ result = svc.reclaim(['docker-build-cache'])+ # 3 GB actually freed, not the 1 KB the scan could estimate up front.+ assert result['results'][0]['bytes'] == 3 * 1024 ** 3diff --git a/serverkit b/serverkitindex fe786a1c..d3860740 100644--- a/serverkit+++ b/serverkit@@ -245,6 +245,15 @@ cmd_help() {echo " restore-db Restore database from backup"echo ""echo "Cleanup Commands:"+ echo " disk Show what is using disk space and reclaim it (interactive)"+ echo " --safe free every item marked safe, no prompt"+ echo " --all free every item, no prompt"+ echo " --only <keys> comma-separated items, e.g. telemetry"+ echo " --days <n> telemetry retention window (default 7)"+ echo " --keep <n> upgrade snapshots to keep (default 1)"+ echo " --older-than <n> only offer snapshots at least n days old"+ echo " --dry-run report what would be freed, change nothing"+ echo " --allow-restart permit a panel restart if VACUUM is blocked"echo " cleanup-apps Delete all apps and containers"echo " factory-reset Complete reset (delete everything)"echo ""@@ -306,7 +315,7 @@ cmd_completion() {_serverkit() {local cur commandscur="${COMP_WORDS[COMP_CWORD]}"- commands="help start stop restart status logs update branch version check-update uninstall create-admin reset-password unlock-user list-users make-admin deactivate-user activate-user init-db migrate-db db-migrate db-status db-history backup-db restore-db cleanup-apps factory-reset generate-keys config setup-proxy create-app deploy-template list-servers deployment-status list-apps start-app stop-app app-logs add-site list-sites remove-site doctor manifest services apps repair login-url support-bundle panel-status panel-doctor completion"+ commands="help start stop restart status logs update branch version check-update uninstall create-admin reset-password unlock-user list-users make-admin deactivate-user activate-user init-db migrate-db db-migrate db-status db-history backup-db restore-db disk cleanup-apps factory-reset generate-keys config setup-proxy create-app deploy-template list-servers deployment-status list-apps start-app stop-app app-logs add-site list-sites remove-site doctor manifest services apps repair login-url support-bundle panel-status panel-doctor completion"if [ "$COMP_CWORD" -eq 1 ]; thenCOMPREPLY=( $(compgen -W "$commands" -- "$cur") )@@ -323,6 +332,7 @@ _serverkit() {doctor) COMPREPLY=( $(compgen -W "--fix" -- "$cur") ) ;;panel-doctor) COMPREPLY=( $(compgen -W "--repair --yes --json" -- "$cur") ) ;;panel-status|list-users|list-servers|db-status) COMPREPLY=( $(compgen -W "--json" -- "$cur") ) ;;+ disk) COMPREPLY=( $(compgen -W "--safe --all --only --days --keep --older-than --dry-run --allow-restart --yes --json" -- "$cur") ) ;;cleanup-apps) COMPREPLY=( $(compgen -W "--delete-volumes --keep-db" -- "$cur") ) ;;list-apps) COMPREPLY=( $(compgen -W "--all --json" -- "$cur") ) ;;deployment-status) COMPREPLY=( $(compgen -W "--logs --json" -- "$cur") ) ;;@@ -706,6 +716,14 @@ cmd_cleanup_apps() {run_cli cleanup-apps $DELETE_VOLUMES $KEEP_DB}+cmd_disk() {+ # Reclaim disk space. Root because the reclaimers touch /var/backups,+ # /tmp staging, the journal and the package caches.+ check_root+ check_installed+ run_cli disk "$@"+}+cmd_factory_reset() {check_rootcheck_installed@@ -1481,6 +1499,10 @@ case "${1:-help}" incompletion)cmd_completion;;+ disk)+ shift+ cmd_disk "$@"+ ;;cleanup-apps)shiftcmd_cleanup_apps "$@"