Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(monitoring): detect host spec changes and inventory filesystems

Juan Denis33be47b6052a
diff --git a/backend/app/__init__.py b/backend/app/__init__.py
index e38c9eee..29254ee4 100644
--- a/backend/app/__init__.py
+++ b/backend/app/__init__.py
@@ -708,6 +708,20 @@ def create_app(config_name=None):
import logging as _logging
_logging.getLogger(__name__).warning(f'Legacy secret encryption skipped: {e}')
+ # Record this boot's hardware and compare it to the last one (plan 74).
+ # Boot is the right hook: a VPS resize requires a power-off, so the
+ # change and the restart are the same event — and the in-memory spec
+ # cache that used to be the only "before" value dies with the process.
+ # Skipped under TESTING: create_app() runs per test session, and a
+ # capture there would write rows and emit notifications into fixtures.
+ if not app.config.get('TESTING'):
+ try:
+ from app.services import host_snapshot_service
+ host_snapshot_service.record_snapshot()
+ except Exception as e:
+ import logging
+ logging.getLogger(__name__).warning(f'Host snapshot skipped: {e}')
+
# Seed bundled flagship extensions (D4) — WordPress ships installed by
# default on every panel (fresh and upgrade) unless the user uninstalled
# it. Done BEFORE load_all_plugins so the loader registers the seeded
diff --git a/backend/app/api/system.py b/backend/app/api/system.py
index 16f8dd2e..204eb295 100644
--- a/backend/app/api/system.py
+++ b/backend/app/api/system.py
@@ -284,6 +284,49 @@ def get_resource_tier():
return jsonify(tier_info), 200
+@system_bp.route('/host-snapshot', methods=['GET'])
+@jwt_required()
+def get_host_snapshot():
+ """Per-filesystem inventory, storage advisories, and the last spec delta.
+
+ Unlike /disk — which lists partitions and stops — this says which
+ filesystem the panel actually writes to, which mounts are missing from
+ /etc/fstab, and what changed since the previous boot.
+ """
+ current_user_id = get_jwt_identity()
+ user = User.query.get(current_user_id)
+
+ if not user or user.role != 'admin':
+ return jsonify({'error': 'Admin access required'}), 403
+
+ from app.services import host_snapshot_service
+ return jsonify(host_snapshot_service.current_state()), 200
+
+
+@system_bp.route('/host-snapshot/history', methods=['GET'])
+@jwt_required()
+def get_host_snapshot_history():
+ """Recent host snapshots, newest first."""
+ current_user_id = get_jwt_identity()
+ user = User.query.get(current_user_id)
+
+ if not user or user.role != 'admin':
+ return jsonify({'error': 'Admin access required'}), 403
+
+ from app.models.host_snapshot import HostSnapshot
+
+ try:
+ limit = min(max(int(request.args.get('limit', 20)), 1), 100)
+ except (TypeError, ValueError):
+ limit = 20
+
+ rows = (HostSnapshot.query
+ .order_by(HostSnapshot.captured_at.desc(), HostSnapshot.id.desc())
+ .limit(limit)
+ .all())
+ return jsonify({'snapshots': [row.to_dict() for row in rows]}), 200
+
+
@system_bp.route('/capacity', methods=['GET'])
@jwt_required()
def get_capacity():
diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py
index ef78f06a..6b3b31b1 100644
--- a/backend/app/models/__init__.py
+++ b/backend/app/models/__init__.py
@@ -16,6 +16,7 @@ from app.models.linked_panel import LinkedPanelConfig
from app.models.system_settings import SystemSettings
from app.models.audit_log import AuditLog
from app.models.error_log import ErrorLog
+from app.models.host_snapshot import HostSnapshot
from app.models.metrics_history import MetricsHistory
from app.models.workflow import Workflow, WorkflowExecution, WorkflowLog
from app.models.webhook import GitWebhook, WebhookLog, GitDeployment
@@ -88,7 +89,7 @@ __all__ = [
'User', 'Application', 'Domain', 'EnvironmentVariable', 'EnvironmentVariableHistory',
'NotificationPreferences', 'Deployment', 'DeploymentDiff', 'DeploymentSnapshot', 'DeploymentJob', 'DeploymentJobLog', 'SystemSettings', 'AuditLog',
'MetricsHistory', 'Workflow', 'WorkflowExecution', 'WorkflowLog', 'GitWebhook', 'WebhookLog', 'GitDeployment',
- 'ErrorLog',
+ 'ErrorLog', 'HostSnapshot',
'Server', 'ServerGroup', 'ServerMetrics', 'ServerCommand', 'AgentSession', 'AgentVersion', 'AgentRollout', 'ServerOnboardingLog', 'SecurityAlert',
'WordPressSite', 'DatabaseSnapshot', 'SyncJob', 'WordPressVulnerability',
'WordPressCustomPlugin', 'WordPressSitePlugin',
diff --git a/backend/app/models/host_snapshot.py b/backend/app/models/host_snapshot.py
new file mode 100644
index 00000000..6cfe3d29
--- /dev/null
+++ b/backend/app/models/host_snapshot.py
@@ -0,0 +1,89 @@
+"""The panel host's own hardware, recorded so a change can be noticed.
+
+Every other spec reader in the codebase answers "what is it now?" — psutil is
+consulted live and the answer is thrown away. That is fine until the operator
+resizes the box, at which point the panel reads the new numbers, has nothing to
+compare them against, and says nothing.
+
+The subtle part is that a resize requires a power-off, so the process (and any
+in-memory cache) dies with it. The baseline has to outlive a reboot or it is
+not a baseline, which is why this is a table and not a module-level dict.
+"""
+import json
+from datetime import datetime
+
+from app import db
+
+
+class HostSnapshot(db.Model):
+ """One reading of the panel host's specs and filesystems."""
+ __tablename__ = 'host_snapshots'
+
+ id = db.Column(db.Integer, primary_key=True)
+ captured_at = db.Column(db.DateTime, default=datetime.utcnow, index=True)
+
+ # /proc/sys/kernel/random/boot_id — a fresh value every boot. Lets a change
+ # that happened while the box was off (a resize) be told apart from one that
+ # happened underneath a running panel (a volume attached hot).
+ boot_id = db.Column(db.String(64), nullable=True, index=True)
+
+ cpu_cores = db.Column(db.Integer, nullable=True)
+ ram_bytes = db.Column(db.BigInteger, nullable=True)
+ swap_bytes = db.Column(db.BigInteger, nullable=True)
+ container = db.Column(db.String(32), nullable=True) # lxc | openvz | docker | None
+
+ # [{device, mountpoint, fstype, total, used, free, percent,
+ # in_fstab, is_data_volume}, ...]
+ filesystems_json = db.Column(db.Text, nullable=True)
+
+ # Deltas against the previous snapshot: [{field, kind, from, to, summary}].
+ # NULL on the first ever capture — there was nothing to compare against,
+ # which is different from "compared and found nothing".
+ changes_json = db.Column(db.Text, nullable=True)
+
+ # Advisories active at capture time, so the next capture can notify only on
+ # a state transition instead of nagging every boot.
+ advisories_json = db.Column(db.Text, nullable=True)
+
+ def _load(self, raw, default):
+ if not raw:
+ return default
+ try:
+ return json.loads(raw)
+ except (json.JSONDecodeError, TypeError):
+ return default
+
+ def get_filesystems(self):
+ return self._load(self.filesystems_json, [])
+
+ def set_filesystems(self, value):
+ self.filesystems_json = json.dumps(value) if value else None
+
+ def get_changes(self):
+ """Deltas vs the previous snapshot, or None if this is the first."""
+ if self.changes_json is None:
+ return None
+ return self._load(self.changes_json, [])
+
+ def set_changes(self, value):
+ self.changes_json = json.dumps(value) if value is not None else None
+
+ def get_advisories(self):
+ return self._load(self.advisories_json, [])
+
+ def set_advisories(self, value):
+ self.advisories_json = json.dumps(value) if value else None
+
+ def to_dict(self):
+ return {
+ 'id': self.id,
+ 'captured_at': self.captured_at.isoformat() if self.captured_at else None,
+ 'boot_id': self.boot_id,
+ 'cpu_cores': self.cpu_cores,
+ 'ram_bytes': self.ram_bytes,
+ 'swap_bytes': self.swap_bytes,
+ 'container': self.container,
+ 'filesystems': self.get_filesystems(),
+ 'changes': self.get_changes(),
+ 'advisories': self.get_advisories(),
+ }
diff --git a/backend/app/notifications/catalog.py b/backend/app/notifications/catalog.py
index 191bf3be..cc816294 100644
--- a/backend/app/notifications/catalog.py
+++ b/backend/app/notifications/catalog.py
@@ -93,6 +93,23 @@ _CATALOG = {
'severity': 'warning',
'category': 'system',
},
+ # The panel host's own hardware moved between boots — a VPS resize, a
+ # volume attached or detached (plan 74). Informational: the operator
+ # usually caused it, but until now nothing confirmed it landed.
+ 'host.specs_changed': {
+ 'title': "This server's hardware changed",
+ 'template': 'generic',
+ 'severity': 'info',
+ 'category': 'system',
+ },
+ # A storage correlation worth acting on: a mount missing from /etc/fstab,
+ # or a filesystem running out while an attached volume sits idle.
+ 'host.storage_advisory': {
+ 'title': 'Storage needs attention on this server',
+ 'template': 'generic',
+ 'severity': 'warning',
+ 'category': 'system',
+ },
# Daily doctor sweep found managed site domains that no longer resolve
# (see app/services/doctor_service.py:_dns_checks).
'dns.unresolved': {
@@ -194,6 +211,8 @@ _LINKS = {
# system / monitoring
'system.alert': '/monitoring',
'monitoring.alert': '/monitoring',
+ 'host.specs_changed': '/monitoring',
+ 'host.storage_advisory': '/monitoring',
'drift.detected': '/monitoring/doctor',
'dns.sync_failed': '/domains',
'dns.unresolved': '/domains',
diff --git a/backend/app/services/capacity_service.py b/backend/app/services/capacity_service.py
index 997d5743..6b172cc7 100644
--- a/backend/app/services/capacity_service.py
+++ b/backend/app/services/capacity_service.py
@@ -231,15 +231,22 @@ def _local_headroom():
except ImportError:
return result
try:
+ from app.services import host_inventory_service
+
memory = psutil.virtual_memory()
- disk = psutil.disk_usage('/')
+ # Not hardcoded `/` any more: images, volumes and backups may live on an
+ # attached volume, and measuring root would then answer a question
+ # nobody asked. `mountpoint` records which filesystem this actually is,
+ # so an unlabelled number can never be misread again (plan 74).
+ mountpoint, disk = host_inventory_service.data_path_usage()
result.update({
'memory_total': memory.total,
# `available`, not `free`: cache and buffers are reclaimable, and
# counting them as used would call every healthy Linux box full.
'memory_free': memory.available,
- 'disk_total': disk.total,
- 'disk_free': disk.free,
+ 'disk_total': disk.total if disk else None,
+ 'disk_free': disk.free if disk else None,
+ 'disk_mountpoint': mountpoint,
'measured_at': datetime.utcnow().isoformat() + 'Z',
})
except Exception: # noqa: BLE001 — a probe failure is "unknown", not fatal
diff --git a/backend/app/services/host_inventory_service.py b/backend/app/services/host_inventory_service.py
new file mode 100644
index 00000000..43c68558
--- /dev/null
+++ b/backend/app/services/host_inventory_service.py
@@ -0,0 +1,437 @@
+"""What this host is made of, and whether that changed since last time.
+
+The panel already reads CPU, RAM and disk live in three places. What it never
+did was *remember*, so a VPS resize or a newly attached volume produced a
+different number that nobody was told about. This module is the memory: it
+captures a snapshot at boot, diffs it against the last one, and turns the
+difference into something worth interrupting an operator for.
+
+It also owns the single definition of "the filesystem that matters"
+(``data_path``). Two services used to answer that question privately and
+disagree — ``resource_tier_service`` measured the first of
+``/var/lib/serverkit`` / ``/var/lib/docker`` / ``/`` while ``capacity_service``
+hardcoded ``/``. On a default box those coincide; move Docker onto a volume and
+they silently diverge, and neither number said which disk it came from.
+
+Every probe here is best-effort. Windows dev boxes have no ``/proc`` and no
+``/etc/fstab``, and a restricted container may refuse any of it — the honest
+answer is None/False, never an exception.
+"""
+import logging
+import os
+
+import psutil
+
+logger = logging.getLogger(__name__)
+
+#: Below this, image pulls and backups start failing before RAM is ever the
+#: binding constraint. Shared with resource_tier_service so the panel has one
+#: idea of "low".
+LOW_FREE_GB = 5
+
+#: A data volume with at least this much free space is worth pointing at when
+#: something else is short. Deliberately about free space and not emptiness: a
+#: volume already holding 20 GB with 40 GB spare is still exactly where backups
+#: or Docker data should be relocated to.
+SPARE_VOLUME_MIN_FREE_GB = 10
+
+#: Mounts that are never interesting as storage: kernel/pseudo filesystems and
+#: the per-container overlays Docker stacks on top of a real disk (counting
+#: those would report the same bytes a dozen times).
+_EXCLUDED_FSTYPES = {
+ 'autofs', 'binfmt_misc', 'bpf', 'cgroup', 'cgroup2', 'configfs', 'debugfs',
+ 'devpts', 'devtmpfs', 'fuse.snapfuse', 'fusectl', 'hugetlbfs', 'iso9660',
+ 'mqueue', 'nsfs', 'overlay', 'overlayfs', 'proc', 'pstore', 'ramfs',
+ 'securityfs', 'squashfs', 'sysfs', 'tmpfs', 'tracefs', 'udf',
+}
+
+_EXCLUDED_PREFIXES = (
+ '/proc', '/sys', '/dev', '/run', '/snap',
+ '/var/lib/docker/', '/var/lib/containerd/',
+ # WSL's own plumbing on a Windows dev box — not storage anyone manages.
+ '/mnt/wsl', '/mnt/wslg',
+)
+
+
+# --------------------------------------------------------------------------- #
+# The one definition of "which disk matters"
+# --------------------------------------------------------------------------- #
+
+def data_path():
+ """The directory whose filesystem holds images, volumes and backups.
+
+ Ordered most-specific-first: an operator who mounted a volume at
+ ``/var/lib/serverkit`` means that one, and someone who only moved Docker
+ means ``/var/lib/docker``. Falls back to the root filesystem.
+ """
+ for path in ('/var/lib/serverkit', '/var/lib/docker', '/'):
+ if os.path.isdir(path):
+ return path
+ return os.getcwd()
+
+
+def data_path_usage():
+ """``(mountpoint, usage)`` for :func:`data_path`, or ``(path, None)``.
+
+ Returns the mountpoint alongside the numbers so a caller can say *which*
+ filesystem it measured. An unlabelled free-space figure is how the two
+ previous implementations managed to disagree without anyone noticing.
+ """
+ path = data_path()
+ try:
+ import shutil
+ usage = shutil.disk_usage(path)
+ except Exception as e: # noqa: BLE001 — a probe failure is "unknown"
+ logger.debug(f'Could not read disk usage for {path}: {e}')
+ return path, None
+ return mountpoint_for(path) or path, usage
+
+
+def mountpoint_for(path):
+ """Which mounted filesystem *path* actually lives on, or None."""
+ try:
+ best = None
+ real = os.path.realpath(path)
+ for part in psutil.disk_partitions(all=False):
+ mp = part.mountpoint
+ if real == mp or real.startswith(mp.rstrip('/') + '/'):
+ if best is None or len(mp) > len(best):
+ best = mp
+ return best
+ except Exception: # noqa: BLE001
+ return None
+
+
+# --------------------------------------------------------------------------- #
+# Probes
+# --------------------------------------------------------------------------- #
+
+def boot_id():
+ """This boot's kernel-assigned id, or None off Linux.
+
+ Changes on every reboot, which is what separates "the box was resized while
+ it was off" from "a volume was attached under a running panel".
+ """
+ try:
+ with open('/proc/sys/kernel/random/boot_id', 'r') as fh:
+ return fh.read().strip() or None
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _fstab_entries():
+ """``(device_specs, mountpoints)`` named in /etc/fstab.
+
+ Both are returned because a mount can be named four ways — by device, by
+ ``UUID=``, by ``LABEL=`` or simply by where it lands — and matching only one
+ of them produces false "not persisted" warnings. On DigitalOcean's Ubuntu
+ image the root filesystem is written as ``LABEL=cloudimg-rootfs``, so a
+ device-only check would flag ``/`` itself as unpersisted.
+ """
+ specs, mounts = set(), set()
+ try:
+ with open('/etc/fstab', 'r') as fh:
+ for line in fh:
+ line = line.strip()
+ if not line or line.startswith('#'):
+ continue
+ fields = line.split()
+ if len(fields) < 2:
+ continue
+ specs.add(fields[0])
+ mounts.add(os.path.normpath(fields[1]))
+ except Exception: # noqa: BLE001 — absent on Windows, unreadable in some sandboxes
+ return None, None
+ return specs, mounts
+
+
+def _resolve_spec(spec):
+ """Turn a ``UUID=``/``LABEL=``/``PARTUUID=`` fstab spec into a device path."""
+ by_dir = {
+ 'UUID': '/dev/disk/by-uuid',
+ 'LABEL': '/dev/disk/by-label',
+ 'PARTUUID': '/dev/disk/by-partuuid',
+ 'PARTLABEL': '/dev/disk/by-partlabel',
+ }
+ if '=' not in spec:
+ return spec
+ kind, _, value = spec.partition('=')
+ directory = by_dir.get(kind.upper())
+ if not directory:
+ return spec
+ try:
+ return os.path.realpath(os.path.join(directory, value))
+ except Exception: # noqa: BLE001
+ return spec
+
+
+def enumerate_filesystems():
+ """Every real mounted filesystem, with usage and persistence facts."""
+ specs, fstab_mounts = _fstab_entries()
+ resolved = {_resolve_spec(s) for s in specs} if specs else set()
+ fstab_known = specs is not None
+
+ root_device = None
+ filesystems = []
+ try:
+ partitions = psutil.disk_partitions(all=False)
+ except Exception as e: # noqa: BLE001
+ logger.debug(f'Could not enumerate partitions: {e}')
+ return []
+
+ # A device mounted more than once (bind mounts, btrfs subvolumes) is still
+ # one disk. Counting each mountpoint separately would report the same free
+ # space several times and invent storage that does not exist — measured on
+ # a dev box where one disk surfaced as six mounts of 936 GB each.
+ seen_devices = set()
+
+ for part in sorted(partitions, key=lambda p: len(p.mountpoint)):
+ if part.fstype in _EXCLUDED_FSTYPES:
+ continue
+ mount = os.path.normpath(part.mountpoint)
+ if any(mount.startswith(prefix) for prefix in _EXCLUDED_PREFIXES):
+ continue
+ # Read-only media is not storage anything can be relocated onto.
+ opts = (part.opts or '').split(',')
+ if 'ro' in opts:
+ continue
+ if part.device in seen_devices:
+ continue
+ try:
+ usage = psutil.disk_usage(part.mountpoint)
+ except (PermissionError, OSError):
+ continue
+ seen_devices.add(part.device)
+
+ if mount == os.sep or mount == '/':
+ root_device = part.device
+
+ # `in_fstab` is None — not False — when /etc/fstab could not be read at
+ # all. "We don't know" must not render as "it will vanish on reboot".
+ in_fstab = None
+ if fstab_known:
+ in_fstab = (part.device in specs
+ or part.device in resolved
+ or mount in fstab_mounts)
+
+ filesystems.append({
+ 'device': part.device,
+ 'mountpoint': mount,
+ 'fstype': part.fstype,
+ 'total': usage.total,
+ 'used': usage.used,
+ 'free': usage.free,
+ 'percent': usage.percent,
+ 'in_fstab': in_fstab,
+ 'is_data_volume': False, # filled in below, once root is known
+ })
+
+ for entry in filesystems:
+ entry['is_data_volume'] = (
+ entry['mountpoint'] not in ('/', '/boot', '/boot/efi')
+ and entry['device'] != root_device
+ )
+ # Stable order so a snapshot diff and the UI both read predictably.
+ return sorted(filesystems, key=lambda f: f['mountpoint'])
+
+
+def capture():
+ """A full reading of this host, ready to persist or diff."""
+ try:
+ cpu_cores = psutil.cpu_count(logical=False) or psutil.cpu_count(logical=True) or 1
+ except Exception: # noqa: BLE001
+ cpu_cores = None
+ try:
+ ram_bytes = psutil.virtual_memory().total
+ except Exception: # noqa: BLE001
+ ram_bytes = None
+ try:
+ swap_bytes = psutil.swap_memory().total
+ except Exception: # noqa: BLE001
+ swap_bytes = None
+
+ return {
+ 'boot_id': boot_id(),
+ 'cpu_cores': cpu_cores,
+ 'ram_bytes': ram_bytes,
+ 'swap_bytes': swap_bytes,
+ 'container': _detect_container(),
+ 'filesystems': enumerate_filesystems(),
+ }
+
+
+def _detect_container():
+ """Container virtualisation, or None on bare metal / a full VM."""
+ try:
+ if os.path.exists('/.dockerenv'):
+ return 'docker'
+ if os.path.isdir('/proc/vz') and not os.path.isdir('/proc/bc'):
+ return 'openvz'
+ with open('/proc/1/environ', 'rb') as fh:
+ environ = fh.read().decode('utf-8', 'replace')
+ for entry in environ.split('\0'):
+ if entry.startswith('container='):
+ return entry.split('=', 1)[1] or 'container'
+ except Exception: # noqa: BLE001
+ return None
+ return None
+
+
+# --------------------------------------------------------------------------- #
+# Diffing
+# --------------------------------------------------------------------------- #
+
+def _format_bytes(num_bytes):
+ if num_bytes is None:
+ return 'unknown'
+ step = float(num_bytes)
+ for unit in ('B', 'KB', 'MB', 'GB', 'TB'):
+ if step < 1024 or unit == 'TB':
+ return f'{step:.0f} {unit}' if step >= 10 or unit == 'B' else f'{step:.1f} {unit}'
+ step /= 1024
+ return f'{step:.1f} TB'
+
+
+#: Ignore total-size jitter below this. A filesystem's reported total can move
+#: by a few blocks across remounts without anything having actually changed.
+_SIZE_NOISE_FLOOR = 64 * 1024 ** 2 # 64 MB
+
+
+def diff(previous, current):
+ """Human-readable deltas between two captures.
+
+ ``previous`` is a dict in :func:`capture` shape (or a HostSnapshot's
+ equivalent). Returns ``[]`` when nothing moved — which is a real answer,
+ distinct from the ``None`` a first-ever capture records.
+ """
+ changes = []
+ if not previous:
+ return changes
+
+ for field, label, fmt in (
+ ('cpu_cores', 'CPU cores', str),
+ ('ram_bytes', 'RAM', _format_bytes),
+ ('swap_bytes', 'Swap', _format_bytes),
+ ):
+ before, after = previous.get(field), current.get(field)
+ # A probe that failed this time must not be reported as a loss.
+ if before is None or after is None or before == after:
+ continue
+ changes.append({
+ 'field': field,
+ 'kind': 'increased' if after > before else 'decreased',
+ 'from': before,
+ 'to': after,
+ 'summary': f'{label} {fmt(before)} → {fmt(after)}',
+ })
+
+ before_fs = {f['mountpoint']: f for f in (previous.get('filesystems') or [])}
+ after_fs = {f['mountpoint']: f for f in (current.get('filesystems') or [])}
+
+ for mount in sorted(after_fs.keys() - before_fs.keys()):
+ entry = after_fs[mount]
+ changes.append({
+ 'field': 'filesystem',
+ 'kind': 'mounted',
+ 'from': None,
+ 'to': mount,
+ 'summary': (f'New filesystem {mount} '
+ f'({_format_bytes(entry.get("total"))}, {entry.get("device")})'),
+ })
+
+ for mount in sorted(before_fs.keys() - after_fs.keys()):
+ changes.append({
+ 'field': 'filesystem',
+ 'kind': 'unmounted',
+ 'from': mount,
+ 'to': None,
+ 'summary': f'Filesystem {mount} is no longer mounted',
+ })
+
+ for mount in sorted(after_fs.keys() & before_fs.keys()):
+ before_total = before_fs[mount].get('total')
+ after_total = after_fs[mount].get('total')
+ if before_total is None or after_total is None:
+ continue
+ if abs(after_total - before_total) < _SIZE_NOISE_FLOOR:
+ continue
+ changes.append({
+ 'field': 'filesystem',
+ 'kind': 'resized',
+ 'from': before_total,
+ 'to': after_total,
+ 'summary': (f'{mount} {_format_bytes(before_total)} → '
+ f'{_format_bytes(after_total)}'),
+ })
+
+ return changes
+
+
+# --------------------------------------------------------------------------- #
+# Advisories
+# --------------------------------------------------------------------------- #
+
+def advisories(filesystems=None):
+ """Correlations worth telling an operator about.
+
+ Each has a stable ``key`` so the caller can notify on a state *transition*
+ rather than on every boot — an fstab entry nobody has got round to adding
+ should not produce a notification every time the panel restarts.
+ """
+ if filesystems is None:
+ filesystems = enumerate_filesystems()
+
+ found = []
+ gb = 1024 ** 3
+
+ # 1. Mounted, in use, and absent from /etc/fstab: it disappears on the next
+ # reboot, taking whatever was written to it out of the panel's reach.
+ for entry in filesystems:
+ if entry.get('in_fstab') is False and entry.get('is_data_volume'):
+ found.append({
+ 'key': f'fstab_missing:{entry["mountpoint"]}',
+ 'severity': 'warning',
+ 'mountpoint': entry['mountpoint'],
+ 'summary': (
+ f'{entry["mountpoint"]} ({entry["device"]}) is mounted but has no '
+ f'/etc/fstab entry — it will not come back after a reboot.'
+ ),
+ })
+
+ # 2. Somewhere is short while a data volume has room to spare. This is the
+ # sentence that turns "disk full" into a next step.
+ low = [f for f in filesystems
+ if f.get('free') is not None and f['free'] < LOW_FREE_GB * gb]
+ spare_volumes = [f for f in filesystems
+ if f.get('is_data_volume')
+ and f.get('free') is not None
+ and f['free'] >= SPARE_VOLUME_MIN_FREE_GB * gb]
+
+ for short in low:
+ for spare in spare_volumes:
+ if spare['mountpoint'] == short['mountpoint']:
+ continue
+ found.append({
+ 'key': f'spare_volume:{short["mountpoint"]}:{spare["mountpoint"]}',
+ 'severity': 'warning',
+ 'mountpoint': short['mountpoint'],
+ 'summary': (
+ f'{short["mountpoint"]} has only {_format_bytes(short["free"])} free '
+ f'while {spare["mountpoint"]} has {_format_bytes(spare["free"])} '
+ f'spare. Consider relocating Docker data or backups onto it.'
+ ),
+ })
+
+ # 3. Plain low space, reported once per filesystem regardless of #2.
+ for short in low:
+ found.append({
+ 'key': f'low_space:{short["mountpoint"]}',
+ 'severity': 'warning',
+ 'mountpoint': short['mountpoint'],
+ 'summary': (
+ f'{short["mountpoint"]} is at {short.get("percent")}% — '
+ f'{_format_bytes(short["free"])} free.'
+ ),
+ })
+
+ return found
diff --git a/backend/app/services/host_snapshot_service.py b/backend/app/services/host_snapshot_service.py
new file mode 100644
index 00000000..ee939193
--- /dev/null
+++ b/backend/app/services/host_snapshot_service.py
@@ -0,0 +1,171 @@
+"""Persist a host reading, compare it to the last one, and speak up.
+
+Split from ``host_inventory_service`` on purpose: that module is pure probing
+and stays importable during early boot, before the database is ready. This one
+owns the parts that need a session and the notification bus.
+
+The notification is the whole point. The panel already *knew* it had four cores
+the moment it booted; what it never did was tell anyone the number used to be
+one.
+"""
+import logging
+
+from app import db
+from app.models.host_snapshot import HostSnapshot
+from app.services import host_inventory_service
+
+logger = logging.getLogger(__name__)
+
+#: Snapshots are small and only written at boot, but an appliance that reboots
+#: often should not accumulate them forever.
+RETAIN_SNAPSHOTS = 50
+
+
+def latest():
+ """The most recent snapshot, or None on a panel that has never captured."""
+ return (HostSnapshot.query
+ .order_by(HostSnapshot.captured_at.desc(), HostSnapshot.id.desc())
+ .first())
+
+
+def _as_capture(snapshot):
+ """A stored row in the shape :func:`host_inventory_service.diff` expects."""
+ return {
+ 'cpu_cores': snapshot.cpu_cores,
+ 'ram_bytes': snapshot.ram_bytes,
+ 'swap_bytes': snapshot.swap_bytes,
+ 'container': snapshot.container,
+ 'filesystems': snapshot.get_filesystems(),
+ }
+
+
+def record_snapshot(notify=True):
+ """Capture, diff against the previous row, persist, and notify on a change.
+
+ Returns the new :class:`HostSnapshot`. Raises nothing the caller must
+ handle — boot callers wrap it anyway, but a failure here must never be the
+ reason a panel does not start.
+ """
+ current = host_inventory_service.capture()
+ current_advisories = host_inventory_service.advisories(current['filesystems'])
+
+ previous = latest()
+ # None (not []) on the first ever capture: there was no baseline, which is a
+ # different statement from "compared and found nothing".
+ changes = None
+ if previous is not None:
+ changes = host_inventory_service.diff(_as_capture(previous), current)
+
+ snapshot = HostSnapshot(
+ boot_id=current.get('boot_id'),
+ cpu_cores=current.get('cpu_cores'),
+ ram_bytes=current.get('ram_bytes'),
+ swap_bytes=current.get('swap_bytes'),
+ container=current.get('container'),
+ )
+ snapshot.set_filesystems(current['filesystems'])
+ snapshot.set_changes(changes)
+ snapshot.set_advisories(current_advisories)
+ db.session.add(snapshot)
+ db.session.commit()
+
+ if notify:
+ try:
+ _notify(previous, snapshot, changes or [], current_advisories)
+ except Exception: # noqa: BLE001 — a notification failure is not a capture failure
+ logger.exception('Host snapshot recorded but notification failed')
+
+ _prune()
+ return snapshot
+
+
+def _prune():
+ """Drop all but the newest RETAIN_SNAPSHOTS rows."""
+ try:
+ stale = (HostSnapshot.query
+ .order_by(HostSnapshot.captured_at.desc(), HostSnapshot.id.desc())
+ .offset(RETAIN_SNAPSHOTS)
+ .all())
+ if not stale:
+ return
+ for row in stale:
+ db.session.delete(row)
+ db.session.commit()
+ except Exception: # noqa: BLE001
+ db.session.rollback()
+ logger.debug('Host snapshot prune skipped', exc_info=True)
+
+
+def _notify(previous, snapshot, changes, current_advisories):
+ from app.notifications.service import NotificationBusService
+
+ if changes:
+ summary = '; '.join(c['summary'] for c in changes)
+ data = {
+ 'summary': summary,
+ 'count': len(changes),
+ 'changes': changes,
+ 'boot_changed': bool(
+ previous is not None
+ and previous.boot_id
+ and snapshot.boot_id
+ and previous.boot_id != snapshot.boot_id
+ ),
+ }
+ data.update(_profile_advice(snapshot))
+ NotificationBusService.send('host.specs_changed', to='admins', data=data)
+
+ # Only transitions. An fstab entry nobody has got round to adding should
+ # not produce a fresh notification on every restart.
+ previous_keys = {a.get('key') for a in (previous.get_advisories() if previous else [])}
+ fresh = [a for a in current_advisories if a.get('key') not in previous_keys]
+ if fresh:
+ NotificationBusService.send('host.storage_advisory', to='admins', data={
+ 'summary': ' '.join(a['summary'] for a in fresh),
+ 'count': len(fresh),
+ 'advisories': fresh,
+ })
+
+
+def _profile_advice(snapshot):
+ """Whether the install profile still suits the hardware.
+
+ Advice only. ``recommend_profile`` is re-run and reported; the profile is
+ never silently rewritten, matching the "reports, never refuses" stance the
+ capacity services already take.
+ """
+ try:
+ from app.services import install_profile_service
+
+ _, usage = host_inventory_service.data_path_usage()
+ specs = {
+ 'ram_gb': round((snapshot.ram_bytes or 0) / (1024 ** 3), 2),
+ 'cpu_cores': snapshot.cpu_cores or 1,
+ 'disk_free_gb': round(usage.free / (1024 ** 3), 2) if usage else None,
+ 'container': snapshot.container,
+ }
+ current = install_profile_service.get_profile()
+ recommended = install_profile_service.recommend_profile(specs)
+ return {
+ 'profile': current,
+ 'recommended_profile': recommended,
+ 'profile_outdated': current != recommended,
+ }
+ except Exception: # noqa: BLE001 — advice is optional; the delta is not
+ logger.debug('Could not re-evaluate install profile', exc_info=True)
+ return {}
+
+
+def current_state():
+ """Live inventory plus the stored delta — what the API and UI render."""
+ filesystems = host_inventory_service.enumerate_filesystems()
+ mountpoint, usage = host_inventory_service.data_path_usage()
+ snapshot = latest()
+ return {
+ 'filesystems': filesystems,
+ 'advisories': host_inventory_service.advisories(filesystems),
+ 'data_path': host_inventory_service.data_path(),
+ 'data_mountpoint': mountpoint,
+ 'data_free_bytes': usage.free if usage else None,
+ 'last_snapshot': snapshot.to_dict() if snapshot else None,
+ }
diff --git a/backend/app/services/resource_tier_service.py b/backend/app/services/resource_tier_service.py
index e5acc453..cf44c88c 100644
--- a/backend/app/services/resource_tier_service.py
+++ b/backend/app/services/resource_tier_service.py
@@ -59,8 +59,9 @@ WORKLOAD_FOOTPRINTS_MB = {
}
# Disk below this on the data volume means image pulls and backups will fail
-# before RAM ever becomes the binding constraint.
-LOW_DISK_THRESHOLD_GB = 5
+# before RAM ever becomes the binding constraint. Defined once in
+# host_inventory_service so the panel has a single idea of "low".
+from app.services.host_inventory_service import LOW_FREE_GB as LOW_DISK_THRESHOLD_GB # noqa: E402
class ResourceTierService:
@@ -166,11 +167,13 @@ class ResourceTierService:
@staticmethod
def _data_path():
- """Volume to measure free space on — where images and backups land."""
- for path in ('/var/lib/serverkit', '/var/lib/docker', '/'):
- if os.path.isdir(path):
- return path
- return os.getcwd()
+ """Volume to measure free space on — where images and backups land.
+
+ Delegates to host_inventory_service so this and capacity_service can no
+ longer answer the same question differently (plan 74).
+ """
+ from app.services import host_inventory_service
+ return host_inventory_service.data_path()
@staticmethod
def _detect_container():
diff --git a/backend/migrations/versions/086_host_snapshots.py b/backend/migrations/versions/086_host_snapshots.py
new file mode 100644
index 00000000..6451bbd9
--- /dev/null
+++ b/backend/migrations/versions/086_host_snapshots.py
@@ -0,0 +1,57 @@
+"""Panel-host spec baseline: new `host_snapshots` table.
+
+Persists the panel host's CPU/RAM/swap and per-filesystem inventory so a change
+across a reboot (a VPS resize, a newly attached volume) can be detected at all.
+An in-memory cache cannot serve as the baseline: the resize requires a
+power-off, so the process holding it dies at exactly the moment the before-value
+is needed.
+
+Indexed on captured_at (the diff reads the newest previous row) and boot_id
+(distinguishes an across-reboot change from a hot one).
+
+Revision ID: 086_host_snapshots
+Revises: 085_error_logs
+"""
+import sqlalchemy as sa
+from alembic import op
+
+revision = '086_host_snapshots'
+down_revision = '085_error_logs'
+branch_labels = None
+depends_on = None
+
+
+def _has_table(name):
+ bind = op.get_bind()
+ return name in sa.inspect(bind).get_table_names()
+
+
+def upgrade():
+ if _has_table('host_snapshots'):
+ return
+ op.create_table(
+ 'host_snapshots',
+ sa.Column('id', sa.Integer(), primary_key=True),
+ sa.Column('captured_at', sa.DateTime(), nullable=True),
+ sa.Column('boot_id', sa.String(length=64), nullable=True),
+ sa.Column('cpu_cores', sa.Integer(), nullable=True),
+ sa.Column('ram_bytes', sa.BigInteger(), nullable=True),
+ sa.Column('swap_bytes', sa.BigInteger(), nullable=True),
+ sa.Column('container', sa.String(length=32), nullable=True),
+ sa.Column('filesystems_json', sa.Text(), nullable=True),
+ sa.Column('changes_json', sa.Text(), nullable=True),
+ sa.Column('advisories_json', sa.Text(), nullable=True),
+ )
+ op.create_index('ix_host_snapshots_captured_at', 'host_snapshots', ['captured_at'])
+ op.create_index('ix_host_snapshots_boot_id', 'host_snapshots', ['boot_id'])
+
+
+def downgrade():
+ if not _has_table('host_snapshots'):
+ return
+ for name in ('ix_host_snapshots_boot_id', 'ix_host_snapshots_captured_at'):
+ try:
+ op.drop_index(name, table_name='host_snapshots')
+ except Exception: # noqa: BLE001
+ pass
+ op.drop_table('host_snapshots')
diff --git a/backend/tests/test_host_inventory.py b/backend/tests/test_host_inventory.py
new file mode 100644
index 00000000..90bc8e53
--- /dev/null
+++ b/backend/tests/test_host_inventory.py
@@ -0,0 +1,395 @@
+"""Host inventory and spec-change detection (plan 74).
+
+The bug this pins down: an operator resized a droplet from 1 vCPU / 1 GB to
+4 vCPU / 8 GB, attached a 60 GB volume, and the panel never mentioned any of it.
+Not because it read stale numbers — a resize forces a power-off, so the process
+and its in-memory cache die and the next boot reads the truth — but because
+nothing ever persisted the *previous* numbers to compare against.
+
+So the rules worth holding: the baseline survives a reboot, the first capture
+does not invent a change, an unchanged reboot stays quiet, and an advisory
+already reported does not nag again. Plus the two capacity services must answer
+"which disk?" identically, which they did not.
+"""
+
+import json
+
+import pytest
+
+from app import db
+from app.models.host_snapshot import HostSnapshot
+from app.services import capacity_service, host_inventory_service, host_snapshot_service
+
+GB = 1024 ** 3
+
+
+def _fs(mountpoint, device, total, free, in_fstab=True, is_data_volume=False,
+ fstype='ext4'):
+ used = total - free
+ return {
+ 'device': device,
+ 'mountpoint': mountpoint,
+ 'fstype': fstype,
+ 'total': total,
+ 'used': used,
+ 'free': free,
+ 'percent': round(used / total * 100, 1) if total else 0,
+ 'in_fstab': in_fstab,
+ 'is_data_volume': is_data_volume,
+ }
+
+
+def _capture(cpu=1, ram=1 * GB, swap=0, filesystems=None, boot='boot-a'):
+ return {
+ 'boot_id': boot,
+ 'cpu_cores': cpu,
+ 'ram_bytes': ram,
+ 'swap_bytes': swap,
+ 'container': None,
+ 'filesystems': filesystems if filesystems is not None else [
+ _fs('/', '/dev/vda1', 25 * GB, 20 * GB),
+ ],
+ }
+
+
[email protected](autouse=True)
+def _clean_snapshots(app):
+ HostSnapshot.query.delete()
+ db.session.commit()
+ yield
+ HostSnapshot.query.delete()
+ db.session.commit()
+
+
+def sent(monkeypatch):
+ """Capture notification-bus sends instead of delivering them."""
+ events = []
+
+ class _Bus:
+ @classmethod
+ def send(cls, event, to=None, data=None, **kwargs):
+ events.append({'event': event, 'to': to, 'data': data or {}})
+ return {'notification_id': len(events), 'deliveries': []}
+
+ import app.notifications.service as bus_module
+ monkeypatch.setattr(bus_module, 'NotificationBusService', _Bus)
+ return events
+
+
+def _stub_capture(monkeypatch, capture):
+ monkeypatch.setattr(host_inventory_service, 'capture', lambda: capture)
+
+
+# --------------------------------------------------------------------------- #
+# Diffing
+# --------------------------------------------------------------------------- #
+
+def test_first_capture_records_no_changes(monkeypatch, sent):
+ """No baseline is not the same as "compared and found nothing"."""
+ _stub_capture(monkeypatch, _capture())
+
+ snapshot = host_snapshot_service.record_snapshot()
+
+ # None, not [] — and nothing to announce.
+ assert snapshot.get_changes() is None
+ assert HostSnapshot.query.count() == 1
+ assert [e['event'] for e in sent] == []
+
+
+def test_resize_across_reboot_is_detected_and_notified(monkeypatch, sent):
+ """The actual bug: 1 vCPU/1 GB -> 4 vCPU/8 GB must produce a delta."""
+ _stub_capture(monkeypatch, _capture(cpu=1, ram=1 * GB, boot='boot-a'))
+ host_snapshot_service.record_snapshot()
+ sent.clear()
+
+ _stub_capture(monkeypatch, _capture(cpu=4, ram=8 * GB, boot='boot-b'))
+ snapshot = host_snapshot_service.record_snapshot()
+
+ changes = snapshot.get_changes()
+ summaries = ' | '.join(c['summary'] for c in changes)
+ assert 'CPU cores 1 → 4' in summaries
+ assert any(c['field'] == 'ram_bytes' and c['kind'] == 'increased' for c in changes)
+
+ events = [e for e in sent if e['event'] == 'host.specs_changed']
+ assert len(events) == 1
+ # boot_id moved, so this was a change made while the box was powered off.
+ assert events[0]['data']['boot_changed'] is True
+
+
+def test_unchanged_reboot_is_silent(monkeypatch, sent):
+ """A restart that changed nothing must not notify."""
+ _stub_capture(monkeypatch, _capture(boot='boot-a'))
+ host_snapshot_service.record_snapshot()
+ sent.clear()
+
+ _stub_capture(monkeypatch, _capture(boot='boot-b'))
+ snapshot = host_snapshot_service.record_snapshot()
+
+ assert snapshot.get_changes() == []
+ assert [e['event'] for e in sent] == []
+
+
+def test_failed_probe_is_not_reported_as_a_loss(monkeypatch, sent):
+ """psutil returning None must not read as "the RAM went away"."""
+ _stub_capture(monkeypatch, _capture(ram=8 * GB))
+ host_snapshot_service.record_snapshot()
+ sent.clear()
+
+ _stub_capture(monkeypatch, _capture(ram=None))
+ snapshot = host_snapshot_service.record_snapshot()
+
+ assert not any(c['field'] == 'ram_bytes' for c in snapshot.get_changes())
+ assert [e['event'] for e in sent] == []
+
+
+def test_new_volume_appears_as_a_change(monkeypatch, sent):
+ _stub_capture(monkeypatch, _capture())
+ host_snapshot_service.record_snapshot()
+ sent.clear()
+
+ _stub_capture(monkeypatch, _capture(filesystems=[
+ _fs('/', '/dev/vda1', 25 * GB, 20 * GB),
+ _fs('/mnt/vol', '/dev/sda', 60 * GB, 57 * GB, is_data_volume=True),
+ ]))
+ snapshot = host_snapshot_service.record_snapshot()
+
+ mounted = [c for c in snapshot.get_changes() if c['kind'] == 'mounted']
+ assert len(mounted) == 1
+ assert mounted[0]['to'] == '/mnt/vol'
+
+
+def test_total_size_jitter_below_the_noise_floor_is_ignored(monkeypatch):
+ """A few blocks' drift across a remount is not a resize."""
+ before = _capture(filesystems=[_fs('/', '/dev/vda1', 25 * GB, 20 * GB)])
+ after = _capture(filesystems=[_fs('/', '/dev/vda1', 25 * GB + 1024 ** 2, 20 * GB)])
+
+ assert host_inventory_service.diff(before, after) == []
+
+
+# --------------------------------------------------------------------------- #
+# fstab detection
+# --------------------------------------------------------------------------- #
+
[email protected]('spec_line', [
+ '/dev/sda /mnt/vol ext4 defaults 0 2', # by device
+ 'UUID=abc-123 /mnt/vol ext4 defaults 0 2', # by UUID
+ 'LABEL=data /mnt/vol ext4 defaults 0 2', # by label
+ '/dev/disk/by-id/scsi-x /mnt/vol ext4 defaults 0 2', # by mountpoint only
+])
+def test_fstab_entry_recognised_in_every_spelling(monkeypatch, tmp_path, spec_line):
+ """A mount can be named four ways; matching one produces false warnings.
+
+ DigitalOcean's Ubuntu image writes root as ``LABEL=cloudimg-rootfs``, so a
+ device-only check flags ``/`` itself as unpersisted.
+ """
+ fstab = tmp_path / 'fstab'
+ fstab.write_text(spec_line + '\n')
+
+ real_open = open
+
+ def fake_open(path, *args, **kwargs):
+ if str(path) == '/etc/fstab':
+ return real_open(fstab, *args, **kwargs)
+ return real_open(path, *args, **kwargs)
+
+ monkeypatch.setattr('builtins.open', fake_open)
+
+ specs, mounts = host_inventory_service._fstab_entries()
+ assert specs is not None
+ assert '/mnt/vol' in mounts
+
+
+def test_unreadable_fstab_yields_unknown_not_false(monkeypatch):
+ """No /etc/fstab (Windows dev box) must not render as "will vanish"."""
+ def boom(path, *args, **kwargs):
+ raise FileNotFoundError(path)
+
+ monkeypatch.setattr('builtins.open', boom)
+ specs, mounts = host_inventory_service._fstab_entries()
+ assert specs is None and mounts is None
+
+
+# --------------------------------------------------------------------------- #
+# Enumeration
+# --------------------------------------------------------------------------- #
+
+class _Part:
+ def __init__(self, device, mountpoint, fstype='ext4', opts='rw,relatime'):
+ self.device = device
+ self.mountpoint = mountpoint
+ self.fstype = fstype
+ self.opts = opts
+
+
+def _stub_partitions(monkeypatch, parts, usage_free=10 * GB):
+ monkeypatch.setattr(host_inventory_service.psutil, 'disk_partitions',
+ lambda all=False: parts)
+
+ class _Usage:
+ total, used, free, percent = 100 * GB, 90 * GB, usage_free, 90.0
+
+ monkeypatch.setattr(host_inventory_service.psutil, 'disk_usage',
+ lambda path: _Usage())
+ monkeypatch.setattr(host_inventory_service, '_fstab_entries',
+ lambda: ({'/dev/sdc'}, {'/'}))
+
+
+def test_one_device_mounted_many_times_is_counted_once(monkeypatch):
+ """Bind mounts and subvolumes are one disk, not several.
+
+ Found by running the real probe: a dev box reported one 936 GB disk as six
+ separate filesystems, which would have invented storage that isn't there.
+ """
+ _stub_partitions(monkeypatch, [
+ _Part('/dev/sdc', '/'),
+ _Part('/dev/sdc', '/home/user/bind-a'),
+ _Part('/dev/sdc', '/home/user/bind-b'),
+ _Part('/dev/sda', '/mnt/vol'),
+ ])
+
+ filesystems = host_inventory_service.enumerate_filesystems()
+
+ assert [f['mountpoint'] for f in filesystems] == ['/', '/mnt/vol']
+
+
+def test_read_only_media_is_not_storage(monkeypatch):
+ """A read-only loop mount raised a bogus "missing from fstab" warning."""
+ _stub_partitions(monkeypatch, [
+ _Part('/dev/sdc', '/'),
+ _Part('/dev/loop0', '/mnt/cli-tools', fstype='iso9660', opts='ro'),
+ _Part('/dev/loop1', '/mnt/readonly', opts='ro,relatime'),
+ ])
+
+ filesystems = host_inventory_service.enumerate_filesystems()
+
+ assert [f['mountpoint'] for f in filesystems] == ['/']
+
+
+def test_the_root_device_is_never_a_data_volume(monkeypatch):
+ _stub_partitions(monkeypatch, [
+ _Part('/dev/sdc', '/'),
+ _Part('/dev/sda', '/mnt/vol'),
+ ])
+
+ by_mount = {f['mountpoint']: f for f in host_inventory_service.enumerate_filesystems()}
+
+ assert by_mount['/']['is_data_volume'] is False
+ assert by_mount['/mnt/vol']['is_data_volume'] is True
+
+
+# --------------------------------------------------------------------------- #
+# Advisories
+# --------------------------------------------------------------------------- #
+
+def test_mounted_but_not_in_fstab_raises_an_advisory():
+ found = host_inventory_service.advisories([
+ _fs('/', '/dev/vda1', 25 * GB, 20 * GB),
+ _fs('/mnt/vol', '/dev/sda', 60 * GB, 57 * GB,
+ in_fstab=False, is_data_volume=True),
+ ])
+ keys = [a['key'] for a in found]
+ assert 'fstab_missing:/mnt/vol' in keys
+
+
+def test_unknown_fstab_state_does_not_raise_an_advisory():
+ """in_fstab=None means we could not read it — not that it is missing."""
+ found = host_inventory_service.advisories([
+ _fs('/mnt/vol', '/dev/sda', 60 * GB, 57 * GB,
+ in_fstab=None, is_data_volume=True),
+ ])
+ assert not any(a['key'].startswith('fstab_missing') for a in found)
+
+
+def test_spare_volume_beside_a_full_disk_is_correlated():
+ """The sentence that turns "disk full" into a next step.
+
+ Mirrors the box that prompted this: / at 1 GiB free with a 60 GB volume
+ attached and essentially untouched.
+ """
+ found = host_inventory_service.advisories([
+ _fs('/', '/dev/vda1', 25 * GB, 1 * GB),
+ _fs('/mnt/vol', '/dev/sda', 60 * GB, 59 * GB, is_data_volume=True),
+ ])
+ spare = [a for a in found if a['key'].startswith('spare_volume:')]
+ assert len(spare) == 1
+ assert '/mnt/vol' in spare[0]['summary']
+ assert spare[0]['mountpoint'] == '/'
+
+
+def test_a_volume_in_use_still_counts_as_somewhere_to_relocate():
+ """Spare room is the signal, not emptiness — 20 GB used, 40 GB free."""
+ found = host_inventory_service.advisories([
+ _fs('/', '/dev/vda1', 25 * GB, 1 * GB),
+ _fs('/mnt/vol', '/dev/sda', 60 * GB, 40 * GB, is_data_volume=True),
+ ])
+ assert any(a['key'].startswith('spare_volume:') for a in found)
+
+
+def test_healthy_host_raises_nothing():
+ found = host_inventory_service.advisories([
+ _fs('/', '/dev/vda1', 25 * GB, 20 * GB),
+ ])
+ assert found == []
+
+
+def test_advisory_already_reported_does_not_notify_again(monkeypatch, sent):
+ """An fstab entry nobody has fixed must not nag on every restart."""
+ filesystems = [
+ _fs('/', '/dev/vda1', 25 * GB, 20 * GB),
+ _fs('/mnt/vol', '/dev/sda', 60 * GB, 57 * GB,
+ in_fstab=False, is_data_volume=True),
+ ]
+ _stub_capture(monkeypatch, _capture(filesystems=filesystems))
+
+ host_snapshot_service.record_snapshot()
+ first = [e for e in sent if e['event'] == 'host.storage_advisory']
+ assert len(first) == 1
+
+ sent.clear()
+ host_snapshot_service.record_snapshot() # same state, next boot
+ assert [e for e in sent if e['event'] == 'host.storage_advisory'] == []
+
+
+# --------------------------------------------------------------------------- #
+# The divergence this plan repaired
+# --------------------------------------------------------------------------- #
+
+def test_capacity_and_tier_measure_the_same_filesystem():
+ """Two services used to answer "which disk?" privately, and disagree."""
+ from app.services.resource_tier_service import ResourceTierService
+
+ assert ResourceTierService._data_path() == host_inventory_service.data_path()
+
+
+def test_local_headroom_reports_which_filesystem_it_measured():
+ """An unlabelled free-space number is how the two managed to diverge."""
+ headroom = capacity_service.server_headroom(None)
+
+ assert 'disk_mountpoint' in headroom
+
+
+# --------------------------------------------------------------------------- #
+# Retention
+# --------------------------------------------------------------------------- #
+
+def test_snapshots_are_pruned_to_the_retention_limit(monkeypatch, sent):
+ _stub_capture(monkeypatch, _capture())
+ monkeypatch.setattr(host_snapshot_service, 'RETAIN_SNAPSHOTS', 3)
+
+ for _ in range(6):
+ host_snapshot_service.record_snapshot()
+
+ assert HostSnapshot.query.count() == 3
+
+
+def test_snapshot_round_trips_through_json_columns(monkeypatch, sent):
+ filesystems = [_fs('/mnt/vol', '/dev/sda', 60 * GB, 57 * GB, is_data_volume=True)]
+ _stub_capture(monkeypatch, _capture(filesystems=filesystems))
+
+ snapshot = host_snapshot_service.record_snapshot()
+ db.session.expire(snapshot)
+
+ stored = HostSnapshot.query.get(snapshot.id)
+ assert stored.get_filesystems()[0]['mountpoint'] == '/mnt/vol'
+ assert json.loads(stored.filesystems_json)[0]['device'] == '/dev/sda'