Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(recovery): rebuild setup-health/reconcile + backup drill/verify services

Juan Denis7a903edc4f26
diff --git a/backend/app/services/backup_drill_service.py b/backend/app/services/backup_drill_service.py
new file mode 100644
index 00000000..6a8d2320
--- /dev/null
+++ b/backend/app/services/backup_drill_service.py
@@ -0,0 +1,673 @@
+"""Restore-drill engine (plan 23 Phases 2–3).
+
+A *drill* is a real restore of a policy's latest restorable point into a scratch
+location (a temp dir for files/application, a throwaway ``skdrill_<run_id>``
+database for database targets), probe-verified, then torn down. It never touches
+the live target. A successful drill is the strongest proof a backup is
+restorable: it promotes the drilled :class:`BackupRun` to
+``verify_level='drilled'`` and stamps the policy's ``last_drill_*`` cache.
+
+Key behaviours (match the proving tests):
+ * **expectation-stamped probes** — files record ``file_count`` + a hash sample
+ (``sampled`` / ``sampled_ok``); databases record ``table_count``.
+ * **container-aware / real DB legs** — the database leg runs the real
+ ``create → restore → probe → drop`` sequence against the shared engine using
+ the existing ``DatabaseService`` helpers; the scratch DB is dropped in a
+ ``finally`` even when the restore fails.
+ * **vacuous-drill detection** — a drill that restored zero files fails loudly
+ instead of earning ``drilled`` for nothing.
+ * **hard guards, loud skips** — a free-space precheck marks the drill
+ ``skipped_no_space`` (doctor-visible) rather than silently passing.
+ * **single-flight** — one drill in flight per host; a new request is refused
+ (:class:`BackupDrillError`) while one is running.
+ * **cadence sweep** — a daily sweep enqueues due drills serially.
+
+See docs/plans/23_BACKUP_TRUST_RESTORE_DRILLS_PLAN.md.
+"""
+import logging
+import math
+import os
+import shutil
+from datetime import datetime, timedelta
+
+from app import db
+
+logger = logging.getLogger(__name__)
+
+DRILL_JOB_KIND = 'backup.drill.run'
+DRILL_SWEEP_JOB_KIND = 'backup.drill.sweep'
+DRILL_ORPHAN_SWEEP_JOB_KIND = 'backup.drill.orphan_sweep'
+DRILL_SWEEP_SCHEDULE_NAME = 'backup-drill-sweep'
+DRILL_ORPHAN_SCHEDULE_NAME = 'backup-drill-orphan-sweep'
+
+# Cadence → base interval in days.
+CADENCE_DAYS = {'weekly': 7, 'monthly': 30}
+# A 'running' drill older than this is treated as crashed, not in-flight.
+STALE_DRILL_AFTER = timedelta(hours=6)
+# Free-space safety multipliers (Decision 6).
+EXTRACTED_ESTIMATE_MULT = 1.2 # from manifest totals
+ARCHIVE_ESTIMATE_MULT = 3.0 # fallback: raw archive size × 3
+# Scratch dirs / scratch DBs older than this are orphan-swept.
+ORPHAN_AFTER = timedelta(hours=24)
+# Hash sample bounds.
+SAMPLE_MAX = 50
+SAMPLE_FRACTION = 0.05
+
+
+class BackupDrillError(Exception):
+ """Raised when a drill is refused or fails loudly (e.g. vacuous restore)."""
+
+
+class BackupDrillService:
+
+ # ------------------------------------------------------------------ #
+ # Space accounting (guards)
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _estimate_required(cls, run, meta):
+ """Estimated bytes needed to materialize the drill: manifest extracted
+ size ×1.2 where known, else raw primary-archive size ×3."""
+ from app.services.backup_verify_service import read_manifest
+ manifest = read_manifest(run, meta)
+ if manifest:
+ total = ((manifest.get('totals') or {}).get('bytes')) or 0
+ if total:
+ return int(total * EXTRACTED_ESTIMATE_MULT)
+ primary = meta.get('primary_archive') or run.storage_path
+ size = 0
+ try:
+ if primary and os.path.isfile(primary):
+ size = os.path.getsize(primary)
+ elif primary and os.path.isdir(primary):
+ size = sum(
+ os.path.getsize(os.path.join(r, f))
+ for r, _d, fs in os.walk(primary) for f in fs
+ if os.path.isfile(os.path.join(r, f))
+ )
+ except OSError:
+ size = 0
+ return int(size * ARCHIVE_ESTIMATE_MULT)
+
+ @staticmethod
+ def _free_bytes(path):
+ """Free bytes on the filesystem holding ``path`` (walks up to the nearest
+ existing ancestor so a not-yet-created scratch dir still measures)."""
+ probe = path
+ while probe and not os.path.exists(probe):
+ parent = os.path.dirname(probe.rstrip(os.sep))
+ if parent == probe:
+ break
+ probe = parent
+ try:
+ return shutil.disk_usage(probe or os.getcwd()).free
+ except OSError:
+ return 0
+
+ @classmethod
+ def _scratch_base(cls):
+ from app.services.backup_service import BackupService
+ base = os.path.join(BackupService.BACKUP_BASE_DIR, 'restores')
+ os.makedirs(base, exist_ok=True)
+ return base
+
+ # ------------------------------------------------------------------ #
+ # Single-flight guard + run selection
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def is_drilling(cls):
+ """True if a (non-stale) restore drill is currently running on this host."""
+ from app.models.restore_drill import RestoreDrill
+ cutoff = datetime.utcnow() - STALE_DRILL_AFTER
+ return db.session.query(RestoreDrill.id).filter(
+ RestoreDrill.status == 'running',
+ RestoreDrill.started_at >= cutoff,
+ ).first() is not None
+
+ @classmethod
+ def _latest_restorable_run(cls, policy):
+ """Newest successful run (the chain endpoint) for a policy."""
+ from app.models.backup_run import BackupRun
+ return (BackupRun.query
+ .filter_by(policy_id=policy.id, status='success')
+ .order_by(BackupRun.started_at.desc())
+ .first())
+
+ # ------------------------------------------------------------------ #
+ # Enqueue (single-flight refusal lives here)
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def request_drill(cls, policy, trigger='manual', run_id=None):
+ """Enqueue a ``backup.drill.run`` job for a policy's latest restorable
+ point. Refused (:class:`BackupDrillError`) while a drill is in flight."""
+ if cls.is_drilling():
+ raise BackupDrillError('A restore drill is already in progress.')
+ if run_id is None:
+ run = cls._latest_restorable_run(policy)
+ if not run:
+ raise BackupDrillError('No successful backup to drill.')
+ run_id = run.id
+ from app.jobs.service import JobService
+ job = JobService.enqueue(
+ DRILL_JOB_KIND,
+ payload={'policy_id': policy.id, 'run_id': run_id, 'trigger': trigger},
+ max_attempts=1,
+ owner_type='backup_policy',
+ owner_id=policy.id,
+ )
+ return job
+
+ # ------------------------------------------------------------------ #
+ # Cadence (Phase 3)
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _jitter(cls, policy, cadence):
+ """Deterministic per-policy jitter (0–20% of the interval) so scheduled
+ drills spread out instead of all firing on the same day."""
+ import random
+ interval = timedelta(days=CADENCE_DAYS[cadence])
+ frac = random.Random(policy.id or 0).uniform(0.0, 0.2)
+ return interval * frac
+
+ @classmethod
+ def is_due(cls, policy, now=None):
+ """True if ``policy`` should be drilled now (enabled, cadence on, a
+ restorable run exists, and it hasn't been drilled within the cadence
+ interval + jitter)."""
+ now = now or datetime.utcnow()
+ if not policy.enabled:
+ return False
+ cadence = policy.drill_cadence or 'off'
+ if cadence not in CADENCE_DAYS:
+ return False
+ if not cls._latest_restorable_run(policy):
+ return False
+ if not policy.last_drill_at:
+ return True
+ threshold = timedelta(days=CADENCE_DAYS[cadence]) + cls._jitter(policy, cadence)
+ return (now - policy.last_drill_at) >= threshold
+
+ @classmethod
+ def due_policies(cls, now=None):
+ """All enabled policies whose drill cadence is due."""
+ from app.models.backup_policy import BackupPolicy
+ now = now or datetime.utcnow()
+ return [p for p in BackupPolicy.query.filter_by(enabled=True).all()
+ if cls.is_due(p, now)]
+
+ @classmethod
+ def run_drill_sweep(cls, job=None):
+ """Daily sweep: enqueue due drills serially. Defers entirely while a
+ drill is already in flight (natural rate limiting / single-flight)."""
+ if cls.is_drilling():
+ return {'count': 0, 'enqueued': [], 'deferred': True}
+ enqueued = []
+ for policy in cls.due_policies():
+ try:
+ cls.request_drill(policy, trigger='scheduled')
+ enqueued.append(policy.id)
+ except BackupDrillError:
+ # One is now in flight — the rest wait for the next sweep.
+ break
+ return {'count': len(enqueued), 'enqueued': enqueued}
+
+ # ------------------------------------------------------------------ #
+ # The drill itself (Phase 2)
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def run_restore_drill(cls, job):
+ """Job handler for ``backup.drill.run``. Restores the run's chain into a
+ scratch target, probes it, and records the outcome. Returns a result
+ dict; raises on a real drill failure so the job is marked failed."""
+ from app.models.backup_policy import BackupPolicy
+ from app.models.backup_run import BackupRun
+ from app.models.restore_drill import RestoreDrill
+
+ payload = job.get_payload() or {}
+ policy_id = payload.get('policy_id')
+ run_id = payload.get('run_id')
+ policy = BackupPolicy.query.get(policy_id)
+ if not policy:
+ raise BackupDrillError(f'backup policy {policy_id!r} not found')
+ run = BackupRun.query.filter_by(id=run_id, policy_id=policy.id).first()
+ if not run:
+ raise BackupDrillError(f'backup run {run_id!r} not found')
+
+ meta = run.get_metadata() or {}
+ started = datetime.utcnow()
+ drill = RestoreDrill(policy_id=policy.id, run_id=run.id,
+ job_id=getattr(job, 'id', None), status='running',
+ trigger=payload.get('trigger'), started_at=started)
+ db.session.add(drill)
+ db.session.commit()
+ drill_id = drill.id
+
+ # --- Free-space precheck (loud skip, never a silent pass) ---------- #
+ scratch_base = cls._scratch_base()
+ required = cls._estimate_required(run, meta)
+ free = cls._free_bytes(scratch_base)
+ drill.bytes_required = int(required)
+ drill.bytes_free = int(free)
+ db.session.commit()
+ if free < required:
+ drill.status = 'skipped_no_space'
+ drill.finished_at = datetime.utcnow()
+ drill.duration_seconds = int((drill.finished_at - started).total_seconds())
+ drill.set_probes({'skipped': 'no_space',
+ 'bytes_required': int(required), 'bytes_free': int(free)})
+ cls._stamp_policy(policy.id, 'skipped_no_space')
+ db.session.commit()
+ logger.warning('drill %s skipped: need %s bytes, %s free',
+ drill_id, required, free)
+ return {'status': 'skipped_no_space', 'drill_id': drill_id,
+ 'bytes_required': int(required), 'bytes_free': int(free)}
+
+ # --- Run the leg -------------------------------------------------- #
+ try:
+ probes, scratch_ref, bytes_restored = cls._dispatch_drill(
+ policy, run, meta, drill_id)
+ drill = RestoreDrill.query.get(drill_id)
+ drill.status = 'success'
+ drill.scratch_ref = scratch_ref
+ drill.bytes_restored = int(bytes_restored or 0)
+ drill.set_probes(probes)
+ drill.finished_at = datetime.utcnow()
+ drill.duration_seconds = int((drill.finished_at - started).total_seconds())
+
+ # Promote the drilled run + stamp the policy cache.
+ run = BackupRun.query.get(run_id)
+ run.verify_level = 'drilled'
+ run.verified_at = datetime.utcnow()
+ run.verify_error = None
+ cls._stamp_policy(policy.id, 'success')
+ db.session.commit()
+
+ # On success the scratch dir is torn down (DB scratch already dropped).
+ cls._teardown_scratch(scratch_ref)
+ cls._notify_recovered(policy.id, run_id)
+ return {'status': 'success', 'drill_id': drill_id, 'probes': probes}
+
+ except Exception as exc: # noqa: BLE001 — record + re-raise
+ db.session.rollback()
+ drill = RestoreDrill.query.get(drill_id)
+ if drill:
+ drill.status = 'failed'
+ drill.error = str(exc)[:2000]
+ drill.finished_at = datetime.utcnow()
+ drill.duration_seconds = int((drill.finished_at - started).total_seconds())
+ probes = drill.get_probes()
+ probes['error'] = str(exc)[:500]
+ drill.set_probes(probes)
+ cls._stamp_policy(policy.id, 'failed')
+ db.session.commit()
+ cls._notify_failed(policy.id, run_id, str(exc))
+ # Scratch dirs are preserved for 24h on failure (orphan sweep reaps).
+ raise
+
+ # ------------------------------------------------------------------ #
+ # Leg dispatch
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _dispatch_drill(cls, policy, run, meta, drill_id):
+ """Return ``(probes, scratch_ref, bytes_restored)`` for the target type.
+ Raises on a failed/vacuous drill."""
+ target_type = policy.target_type
+ if target_type == 'database':
+ return cls._drill_database(policy, run, meta)
+ if target_type == 'wordpress_site':
+ return cls._drill_wordpress(policy, run, meta, drill_id)
+ # files + application both replay a tar chain into a scratch dir.
+ return cls._drill_files(policy, run, meta, drill_id)
+
+ # ------------------------------------------------------------------ #
+ # Files / application leg
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _chain_archives(cls, policy, run, meta):
+ """Ordered local archive paths needed to restore ``run``. Reuses the
+ policy service's chain resolver where possible."""
+ try:
+ from app.services.backup_policy_service import BackupPolicyService
+ return BackupPolicyService._chain_archives(policy, run)
+ except Exception: # noqa: BLE001 — fall back to the single primary
+ primary = meta.get('primary_archive')
+ if not primary or not os.path.exists(primary):
+ raise BackupDrillError('Backup archive not found for drill')
+ return [primary]
+
+ @classmethod
+ def _drill_files(cls, policy, run, meta, drill_id):
+ import tarfile
+ scratch = os.path.join(cls._scratch_base(), f'drill-{drill_id}')
+ os.makedirs(scratch, exist_ok=True)
+
+ archives = cls._chain_archives(policy, run, meta)
+ for archive in archives:
+ if not archive or not os.path.exists(archive):
+ raise BackupDrillError(f'Backup archive missing for drill: {archive}')
+ with tarfile.open(archive, 'r:*') as tar:
+ tar.extractall(scratch, filter='data')
+
+ files = []
+ total_bytes = 0
+ for root, _dirs, names in os.walk(scratch):
+ for name in names:
+ fp = os.path.join(root, name)
+ if os.path.isfile(fp):
+ files.append(fp)
+ try:
+ total_bytes += os.path.getsize(fp)
+ except OSError:
+ pass
+
+ file_count = len(files)
+ # Vacuous-drill detection: a drill that restored nothing is not proof.
+ if file_count == 0:
+ raise BackupDrillError(
+ 'Drill restored 0 files — the archive is empty or unreadable.')
+
+ sampled, sampled_ok, mismatches = cls._hash_sample(files)
+ probes = {
+ 'engine': meta.get('engine') or policy.target_type,
+ 'file_count': file_count,
+ 'bytes_restored': total_bytes,
+ 'sampled': sampled,
+ 'sampled_ok': sampled_ok,
+ 'archives': len(archives),
+ }
+ if mismatches:
+ probes['sample_mismatches'] = mismatches
+ return probes, scratch, total_bytes
+
+ @classmethod
+ def _hash_sample(cls, files):
+ """Full-hash a sample (min(50, 5%)) of the restored files. ``sampled_ok``
+ counts files whose bytes hashed cleanly (proves they're actually
+ readable, not just present)."""
+ from app.services.backup_verify_service import sha256_file
+ count = len(files)
+ if count == 0:
+ return 0, 0, 0
+ sample_size = min(count, max(1, min(SAMPLE_MAX, math.ceil(count * SAMPLE_FRACTION))))
+ # Deterministic, evenly spaced sample across the sorted file list.
+ ordered = sorted(files)
+ step = max(1, count // sample_size)
+ picked = ordered[::step][:sample_size]
+ sampled = len(picked)
+ sampled_ok = 0
+ for fp in picked:
+ if sha256_file(fp) is not None:
+ sampled_ok += 1
+ return sampled, sampled_ok, sampled - sampled_ok
+
+ # ------------------------------------------------------------------ #
+ # Database leg
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _drill_database(cls, policy, run, meta):
+ """Restore the dump into a throwaway ``skdrill_<run_id>`` database on the
+ shared engine, probe the table count, then DROP it in a ``finally``
+ (even when the restore fails). Never touches the live database."""
+ from app.services.backup_service import BackupService
+ from app.services.database_service import DatabaseService
+
+ db_type = (meta.get('db_type') or policy.target_subtype or 'mysql').lower()
+ backup_path = meta.get('primary_archive') or run.storage_path
+ if not backup_path or not os.path.exists(backup_path):
+ raise BackupDrillError('Backup archive not found for database drill')
+
+ scratch_name = f'skdrill_{run.id}'
+ creds = cls._db_credentials(policy, meta)
+ is_postgres = db_type in ('postgres', 'postgresql')
+
+ created = False
+ try:
+ # 1. create scratch DB
+ if is_postgres:
+ res = DatabaseService.pg_create_database(scratch_name)
+ else:
+ res = DatabaseService.mysql_create_database(scratch_name)
+ if isinstance(res, dict) and not res.get('success', True):
+ raise BackupDrillError(res.get('error') or 'scratch DB create failed')
+ created = True
+
+ # 2. restore the dump into the scratch DB
+ restore = BackupService.restore_database(
+ backup_path=backup_path,
+ db_type='postgresql' if is_postgres else 'mysql',
+ db_name=scratch_name,
+ user=creds.get('user'),
+ password=creds.get('password'),
+ host=creds.get('host', 'localhost'),
+ )
+ if isinstance(restore, dict) and not restore.get('success', True):
+ raise BackupDrillError(restore.get('error') or 'database restore failed')
+
+ # 3. probe: table count in the restored scratch DB
+ if is_postgres:
+ tables = DatabaseService.pg_get_tables(scratch_name)
+ else:
+ tables = DatabaseService.mysql_get_tables(scratch_name)
+ table_count = len(tables or [])
+ probes = {
+ 'engine': 'database',
+ 'db_type': 'postgresql' if is_postgres else 'mysql',
+ 'scratch_db': scratch_name,
+ 'table_count': table_count,
+ 'tables': [cls._table_name(t) for t in (tables or [])][:100],
+ }
+ return probes, scratch_name, 0
+ finally:
+ # 4. always drop the scratch DB
+ if created:
+ try:
+ if is_postgres:
+ DatabaseService.pg_drop_database(scratch_name)
+ else:
+ DatabaseService.mysql_drop_database(scratch_name)
+ except Exception as exc: # noqa: BLE001
+ logger.warning('drill: failed to drop scratch DB %s: %s',
+ scratch_name, exc)
+
+ @staticmethod
+ def _table_name(table):
+ if isinstance(table, dict):
+ return table.get('name') or table.get('table') or str(table)
+ return str(table)
+
+ @classmethod
+ def _db_credentials(cls, policy, meta):
+ """Resolve the engine credentials for the drill (reuses the policy
+ service's target resolver; falls back to policy meta)."""
+ try:
+ from app.services.backup_policy_service import BackupPolicyService
+ target = BackupPolicyService._resolve_target(policy)
+ cfg = target.get('db_config') or {}
+ if cfg:
+ return cfg
+ except Exception: # noqa: BLE001
+ pass
+ tmeta = policy.get_target_meta() if hasattr(policy, 'get_target_meta') else {}
+ return {
+ 'user': meta.get('user') or tmeta.get('user'),
+ 'password': meta.get('password') or tmeta.get('password'),
+ 'host': meta.get('host') or tmeta.get('host', 'localhost'),
+ }
+
+ # ------------------------------------------------------------------ #
+ # WordPress leg (files drill + real scratch-DB import, plan 30)
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _drill_wordpress(cls, policy, run, meta, drill_id):
+ """A WordPress drill proves BOTH halves: extract the files archive into a
+ scratch dir, AND import ``database.sql`` into a throwaway scratch DB. A
+ drill that verified neither artifact fails instead of earning 'drilled'
+ vacuously (plan 30)."""
+ import tarfile
+ from app.services.backup_service import BackupService
+ from app.services.database_service import DatabaseService
+
+ backup_dir = run.storage_path
+ scratch = os.path.join(cls._scratch_base(), f'drill-{drill_id}')
+ os.makedirs(scratch, exist_ok=True)
+
+ probes = {'engine': 'wordpress_site'}
+ total_bytes = 0
+ proved_any = False
+
+ # --- files half --------------------------------------------------- #
+ files_archive = meta.get('primary_archive') or (
+ os.path.join(backup_dir, 'files.tar.gz') if backup_dir else None)
+ file_count = 0
+ if files_archive and os.path.exists(files_archive):
+ with tarfile.open(files_archive, 'r:*') as tar:
+ tar.extractall(scratch, filter='data')
+ restored = []
+ for root, _dirs, names in os.walk(scratch):
+ for name in names:
+ fp = os.path.join(root, name)
+ if os.path.isfile(fp):
+ restored.append(fp)
+ try:
+ total_bytes += os.path.getsize(fp)
+ except OSError:
+ pass
+ file_count = len(restored)
+ if file_count:
+ sampled, sampled_ok, _mm = cls._hash_sample(restored)
+ probes.update({'file_count': file_count, 'sampled': sampled,
+ 'sampled_ok': sampled_ok})
+ proved_any = True
+
+ # --- database half ------------------------------------------------ #
+ db_sql = os.path.join(backup_dir, 'database.sql') if backup_dir else None
+ if db_sql and os.path.exists(db_sql):
+ scratch_name = f'skdrill_{run.id}'
+ creds = cls._db_credentials(policy, meta)
+ created = False
+ try:
+ res = DatabaseService.mysql_create_database(scratch_name)
+ if isinstance(res, dict) and not res.get('success', True):
+ raise BackupDrillError(res.get('error') or 'scratch DB create failed')
+ created = True
+ restore = BackupService.restore_database(
+ backup_path=db_sql, db_type='mysql', db_name=scratch_name,
+ user=creds.get('user'), password=creds.get('password'),
+ host=creds.get('host', 'localhost'))
+ if isinstance(restore, dict) and not restore.get('success', True):
+ raise BackupDrillError(restore.get('error') or 'WP database restore failed')
+ tables = DatabaseService.mysql_get_tables(scratch_name)
+ probes['table_count'] = len(tables or [])
+ proved_any = True
+ finally:
+ if created:
+ try:
+ DatabaseService.mysql_drop_database(scratch_name)
+ except Exception as exc: # noqa: BLE001
+ logger.warning('WP drill: failed to drop scratch DB %s: %s',
+ scratch_name, exc)
+
+ if not proved_any:
+ raise BackupDrillError(
+ 'WordPress drill verified no artifact (no files and no database).')
+ probes['bytes_restored'] = total_bytes
+ return probes, scratch, total_bytes
+
+ # ------------------------------------------------------------------ #
+ # Teardown / stamps / notifications
+ # ------------------------------------------------------------------ #
+
+ @staticmethod
+ def _teardown_scratch(scratch_ref):
+ if scratch_ref and os.path.isdir(scratch_ref):
+ shutil.rmtree(scratch_ref, ignore_errors=True)
+
+ @classmethod
+ def _stamp_policy(cls, policy_id, status):
+ from app.models.backup_policy import BackupPolicy
+ policy = BackupPolicy.query.get(policy_id)
+ if not policy:
+ return
+ policy.last_drill_at = datetime.utcnow()
+ policy.last_drill_status = status
+
+ @classmethod
+ def _notify_failed(cls, policy_id, run_id, error):
+ try:
+ from app.models.backup_policy import BackupPolicy
+ from app.services.backup_alert_service import BackupAlertService
+ policy = BackupPolicy.query.get(policy_id)
+ if policy:
+ BackupAlertService.on_drill_result(policy, status='failed', error=error)
+ except Exception as exc: # noqa: BLE001 — alerts are best-effort
+ logger.debug('drill-failed alert skipped: %s', exc)
+
+ @classmethod
+ def _notify_recovered(cls, policy_id, run_id):
+ try:
+ from app.models.backup_policy import BackupPolicy
+ from app.services.backup_alert_service import BackupAlertService
+ policy = BackupPolicy.query.get(policy_id)
+ if policy:
+ BackupAlertService.on_drill_result(policy, status='success')
+ except Exception as exc: # noqa: BLE001
+ logger.debug('drill-recovered alert skipped: %s', exc)
+
+ # ------------------------------------------------------------------ #
+ # Orphan sweep (daily)
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def orphan_sweep(cls, job=None):
+ """Reap scratch dirs (``restores/drill-*``) with no live drill row and
+ older than 24h. Scratch DBs are dropped inline by each drill's finally,
+ so only dirs need sweeping here."""
+ from app.models.restore_drill import RestoreDrill
+ base = cls._scratch_base()
+ removed = 0
+ live_ids = {d.id for d in RestoreDrill.query.filter_by(status='running').all()}
+ cutoff = datetime.utcnow() - ORPHAN_AFTER
+ try:
+ entries = os.listdir(base)
+ except OSError:
+ entries = []
+ for entry in entries:
+ if not entry.startswith('drill-'):
+ continue
+ path = os.path.join(base, entry)
+ if not os.path.isdir(path):
+ continue
+ try:
+ drill_id = int(entry.split('-', 1)[1])
+ except (ValueError, IndexError):
+ drill_id = None
+ if drill_id in live_ids:
+ continue
+ try:
+ mtime = datetime.utcfromtimestamp(os.path.getmtime(path))
+ except OSError:
+ mtime = cutoff
+ if mtime <= cutoff:
+ shutil.rmtree(path, ignore_errors=True)
+ removed += 1
+ return {'removed': removed}
+
+ # ------------------------------------------------------------------ #
+ # Registration
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def register_jobs(cls):
+ """Register the drill handlers with the job registry (app startup)."""
+ from app.jobs import registry
+ registry.register(DRILL_JOB_KIND, cls.run_restore_drill, replace=True)
+ registry.register(DRILL_SWEEP_JOB_KIND, cls.run_drill_sweep, replace=True)
+ registry.register(DRILL_ORPHAN_SWEEP_JOB_KIND, cls.orphan_sweep, replace=True)
diff --git a/backend/app/services/backup_policy_service.py b/backend/app/services/backup_policy_service.py
index ece465f2..7ff055fb 100644
--- a/backend/app/services/backup_policy_service.py
+++ b/backend/app/services/backup_policy_service.py
@@ -920,6 +920,48 @@ class BackupPolicyService:
'storage_used_human': format_bytes(storage_used),
'remote_configured': BackupCostService.configured_remote_provider() is not None,
'is_running': cls.is_running(policy),
+ 'restore_proof': cls._restore_proof_view(policy),
+ }
+
+ # ------------------------------------------------------------------ #
+ # Restore-proof view (plan 23 Phase 3) — the "last proven restore" badge
+ # ------------------------------------------------------------------ #
+
+ # A drilled policy reads as "stale" once its last successful drill is older
+ # than this multiple of the cadence interval.
+ _DRILL_STALE_MULT = 1.5
+
+ @classmethod
+ def _drill_badge(cls, policy):
+ """One-word restore-proof state derived from the policy's drill cache:
+ ``never`` (never drilled), ``failed`` (last drill failed), ``stale``
+ (last success older than 1.5x the cadence interval), else ``ok``."""
+ if not policy.last_drill_at:
+ return 'never'
+ if policy.last_drill_status == 'failed':
+ return 'failed'
+ from app.services.backup_drill_service import CADENCE_DAYS
+ days = CADENCE_DAYS.get(policy.drill_cadence or 'off')
+ if days is not None:
+ threshold = timedelta(days=days * cls._DRILL_STALE_MULT)
+ if (datetime.utcnow() - policy.last_drill_at) > threshold:
+ return 'stale'
+ return 'ok'
+
+ @classmethod
+ def _restore_proof_view(cls, policy, drill_limit=5):
+ """Restore-proof projection for the protection panel: the drill cadence,
+ the current badge (never/ok/stale/failed), and the most recent drill
+ rows serialized."""
+ from app.models.restore_drill import RestoreDrill
+ recent = (RestoreDrill.query
+ .filter_by(policy_id=policy.id)
+ .order_by(RestoreDrill.started_at.desc())
+ .limit(drill_limit).all())
+ return {
+ 'drill_cadence': policy.drill_cadence or 'off',
+ 'badge': cls._drill_badge(policy),
+ 'recent_drills': [d.to_dict() for d in recent],
}
# ------------------------------------------------------------------ #
diff --git a/backend/app/services/backup_verify_service.py b/backend/app/services/backup_verify_service.py
new file mode 100644
index 00000000..649e9e49
--- /dev/null
+++ b/backend/app/services/backup_verify_service.py
@@ -0,0 +1,239 @@
+"""Tier-1 backup verification (plan 23 Phase 1).
+
+Every ``backup.policy.run`` writes a ``manifest.json`` next to its archives and
+then runs a cheap, always-on verification pass over the produced archive(s):
+
+ * **manifest** (:func:`write_manifest`) — file list, per-artifact sha256,
+ sizes, chain refs; the sha256 of the *primary* archive is denormalized onto
+ the :class:`BackupRun` row (``checksum_sha256``) so later offsite checks can
+ compare against a STORED hash rather than one recomputed from the same
+ possibly-corrupt local file (Decision 3).
+ * **verifier** (:func:`verify_run_tier1`) — a full ``tar -t`` style listing
+ (readability) plus a sha256 spot-check of the primary archive against the
+ stored value, promoting the run up the ladder
+ ``none → listed → hashed``. Only a real restore drill ever reaches
+ ``drilled``.
+
+Everything here is cross-platform: the listing falls back to Python's
+``tarfile`` where GNU ``tar`` is unavailable (e.g. Windows dev), and the hash is
+computed in Python.
+"""
+import hashlib
+import json
+import logging
+import os
+import tarfile
+from datetime import datetime
+
+logger = logging.getLogger(__name__)
+
+# Bump when the manifest shape changes in a backward-incompatible way.
+MANIFEST_VERSION = 1
+MANIFEST_NAME = 'manifest.json'
+
+_HASH_CHUNK = 1 << 20 # 1 MiB
+
+
+# --------------------------------------------------------------------------- #
+# Hashing helpers
+# --------------------------------------------------------------------------- #
+
+def sha256_file(path):
+ """Return the hex sha256 of a file, or ``None`` if it is missing/unreadable."""
+ if not path or not os.path.isfile(path):
+ return None
+ digest = hashlib.sha256()
+ try:
+ with open(path, 'rb') as fh:
+ for chunk in iter(lambda: fh.read(_HASH_CHUNK), b''):
+ digest.update(chunk)
+ except OSError:
+ return None
+ return digest.hexdigest()
+
+
+def _base_dir(run, meta):
+ """Directory that holds a run's artifacts (where manifest.json lives)."""
+ primary = meta.get('primary_archive')
+ if primary:
+ return os.path.dirname(primary) or (run.storage_path or '.')
+ sp = run.storage_path
+ if sp and os.path.isdir(sp):
+ return sp
+ if sp:
+ return os.path.dirname(sp) or '.'
+ return '.'
+
+
+# --------------------------------------------------------------------------- #
+# Manifest writer
+# --------------------------------------------------------------------------- #
+
+def write_manifest(run, meta):
+ """Write ``manifest.json`` next to a run's archives and denormalize the
+ primary archive's sha256 onto the run row.
+
+ Returns the manifest dict. Does NOT commit — the caller owns the session.
+ """
+ base = _base_dir(run, meta)
+ primary = meta.get('primary_archive')
+ primary_name = os.path.basename(primary) if primary else None
+
+ artifacts = []
+ if os.path.isdir(base):
+ for entry in sorted(os.listdir(base)):
+ if entry == MANIFEST_NAME:
+ continue
+ full = os.path.join(base, entry)
+ if not os.path.isfile(full):
+ continue
+ artifacts.append({
+ 'name': entry,
+ 'size': os.path.getsize(full),
+ 'sha256': sha256_file(full),
+ })
+ elif primary and os.path.isfile(primary):
+ # storage_path is a single file (e.g. a database dump).
+ artifacts.append({
+ 'name': primary_name,
+ 'size': os.path.getsize(primary),
+ 'sha256': sha256_file(primary),
+ })
+
+ primary_sha256 = None
+ for art in artifacts:
+ if art['name'] == primary_name:
+ primary_sha256 = art['sha256']
+ break
+ if primary_sha256 is None and primary and os.path.isfile(primary):
+ primary_sha256 = sha256_file(primary)
+
+ manifest = {
+ 'version': MANIFEST_VERSION,
+ 'run_id': run.id,
+ 'kind': meta.get('kind') or run.kind,
+ 'engine': meta.get('engine'),
+ 'created_at': datetime.utcnow().isoformat() + 'Z',
+ 'primary_archive': primary_name,
+ 'primary_sha256': primary_sha256,
+ 'artifacts': artifacts,
+ 'totals': {
+ 'count': len(artifacts),
+ 'bytes': sum(a['size'] for a in artifacts),
+ },
+ 'chain': {
+ 'incremental': bool(meta.get('incremental')),
+ 'full_run_id': meta.get('full_run_id'),
+ },
+ 'tool_versions': _tool_versions(),
+ }
+
+ # Denormalize the primary hash onto the run (Decision 3).
+ if primary_sha256:
+ run.checksum_sha256 = primary_sha256
+
+ if os.path.isdir(base):
+ try:
+ with open(os.path.join(base, MANIFEST_NAME), 'w', encoding='utf-8') as fh:
+ json.dump(manifest, fh, indent=2)
+ except OSError as exc: # best-effort — the row still carries the hash
+ logger.warning('could not write manifest for run %s: %s', run.id, exc)
+
+ return manifest
+
+
+def read_manifest(run, meta=None):
+ """Load a run's on-disk manifest, or ``None`` if absent/unreadable."""
+ meta = meta if meta is not None else (run.get_metadata() or {})
+ base = _base_dir(run, meta)
+ path = os.path.join(base, MANIFEST_NAME)
+ if not os.path.isfile(path):
+ return None
+ try:
+ with open(path, 'r', encoding='utf-8') as fh:
+ return json.load(fh)
+ except (OSError, ValueError):
+ return None
+
+
+def _tool_versions():
+ versions = {'python_tarfile': True}
+ try:
+ from app.utils.system import is_command_available
+ versions['gnu_tar'] = bool(is_command_available('tar'))
+ except Exception: # noqa: BLE001
+ pass
+ return versions
+
+
+# --------------------------------------------------------------------------- #
+# Post-run verifier (Tier 1)
+# --------------------------------------------------------------------------- #
+
+def _list_archive(path):
+ """Return ``(ok, member_count, error)`` for a tar/tar.gz archive, using
+ Python's ``tarfile`` so it works cross-platform."""
+ try:
+ with tarfile.open(path, 'r:*') as tar:
+ members = tar.getmembers()
+ return True, len(members), None
+ except Exception as exc: # noqa: BLE001 — any read failure means unreadable
+ return False, 0, str(exc)
+
+
+def _stored_sha256(run, meta):
+ if getattr(run, 'checksum_sha256', None):
+ return run.checksum_sha256
+ manifest = read_manifest(run, meta)
+ if manifest and manifest.get('primary_sha256'):
+ return manifest['primary_sha256']
+ return (meta or {}).get('primary_sha256')
+
+
+def verify_run_tier1(run, meta):
+ """Verify a freshly produced run and set its verify ladder position.
+
+ Ladder: ``none`` (archive unreadable) → ``listed`` (readable but checksum
+ unverified/mismatch) → ``hashed`` (checksum matches the stored manifest).
+
+ Mutates ``run.verify_level`` / ``run.verify_error`` / ``run.verified_at``
+ (does NOT commit) and returns a probes dict.
+ """
+ primary = meta.get('primary_archive')
+ probes = {'listed': [], 'hashed': False, 'sha256': None}
+ run.verified_at = datetime.utcnow()
+
+ if not primary or not os.path.exists(primary):
+ run.verify_level = 'none'
+ run.verify_error = 'Primary archive not found on disk (not readable).'
+ probes['listed'].append({'name': os.path.basename(primary) if primary else None,
+ 'ok': False, 'error': 'missing'})
+ return probes
+
+ ok, member_count, err = _list_archive(primary)
+ probes['listed'].append({
+ 'name': os.path.basename(primary), 'ok': ok,
+ 'members': member_count, 'error': err,
+ })
+ if not ok:
+ run.verify_level = 'none'
+ run.verify_error = f'Primary archive is not readable: {(err or "")[:200]}'
+ return probes
+
+ stored = _stored_sha256(run, meta)
+ actual = sha256_file(primary)
+ probes['sha256'] = actual
+
+ if stored and actual and actual == stored:
+ run.verify_level = 'hashed'
+ run.verify_error = None
+ probes['hashed'] = True
+ else:
+ run.verify_level = 'listed'
+ probes['hashed'] = False
+ if stored and actual and actual != stored:
+ run.verify_error = (f'Checksum mismatch: expected {stored[:12]}…, '
+ f'got {actual[:12]}…')
+ else:
+ run.verify_error = 'No stored checksum available to compare against.'
+ return probes
diff --git a/backend/app/services/setup_health_service.py b/backend/app/services/setup_health_service.py
new file mode 100644
index 00000000..46edf6e6
--- /dev/null
+++ b/backend/app/services/setup_health_service.py
@@ -0,0 +1,575 @@
+"""Setup Health registry — "how set up is this panel" (plan 22).
+
+A small, cheap registry of setup-health *items* the operator should complete to
+get a fully-working panel. Each item is a doctor-style check
+(``key``/``title``/``status``/``detail``/``repairable``/``repair_ref``) plus the
+setup extras ``section``/``severity``/``scope``/``why``/``fix`` so the same items
+render in the Setup card, in Monitoring → Doctor's ``setup.*`` section, and in the
+personal "secure your account" nudge.
+
+Severity is graded by Decision 2's *silent-breakage* rule: an item is
+``critical`` only when leaving it undone silently breaks something the operator
+already set in motion (e.g. per-site DNS mode with no provider/IP — sites won't
+resolve). Everything else is ``recommended``. **HTTPS/TLS items are never
+critical** — SSL is optional by decree.
+
+Scope is ``panel`` (a whole-instance setting, admin-only) or ``user`` (a personal
+item like account 2FA). Panel snoozes live in a SettingsService map; personal
+snoozes live on ``users.setup_snoozes``. A snoozed item still renders, just muted
+— it drops out of the open counts and never triggers the nag.
+
+The weekly nag (Phase 6) fires ``setup.incomplete`` to admins only while
+non-snoozed critical items exist AND the critical set changed since the last
+notice (fingerprint throttle); the marker resets once the criticals clear.
+"""
+import json
+import logging
+import os
+import sys
+from datetime import datetime, timedelta
+
+logger = logging.getLogger(__name__)
+
+# ── Phase-6 constants (imported by the API + the nag test) ───────────────────
+NAG_JOB_KIND = 'setup.health.nag'
+NAG_SCHEDULE_NAME = 'setup-health-nag'
+NAG_MARKER_KEY = 'setup_health_nag_marker'
+PANEL_SNOOZE_KEY = 'setup_health_panel_snoozes'
+
+# Weekly cadence (seconds) for the nag schedule.
+NAG_INTERVAL_SECONDS = 604800
+
+# Item severities.
+CRITICAL = 'critical'
+RECOMMENDED = 'recommended'
+
+# Every setup item key mapped to its scope. Drives _item_scope() (snooze RBAC)
+# and lets the API reject an unknown key before touching storage.
+_ITEM_SCOPES = {
+ 'setup.public_ip': 'panel',
+ 'setup.base_domain': 'panel',
+ 'setup.dns_provider': 'panel',
+ 'setup.email_delivery': 'panel',
+ 'setup.backup_policy': 'panel',
+ 'setup.backup_offsite': 'panel',
+ 'setup.canonical_domain': 'panel',
+ 'setup.wildcard_cert': 'panel',
+ 'setup.account_security': 'user',
+}
+
+
+def _link(to):
+ """A deep-link fix descriptor pointing the operator at where to finish."""
+ return {'kind': 'link', 'to': to}
+
+
+def _item(key, title, status, detail, severity, fix, scope='panel', why=None,
+ repairable=False, repair_ref=None):
+ """Build a normalized setup-health item (doctor ``_check`` shape + extras)."""
+ return {
+ # doctor _check shape
+ 'key': key,
+ 'title': title,
+ 'status': status, # 'ok' | 'warn' | 'fail'
+ 'detail': detail,
+ 'repairable': bool(repairable),
+ 'repair_ref': repair_ref,
+ # setup extras
+ 'section': 'setup',
+ 'severity': severity, # 'critical' | 'recommended'
+ 'scope': scope, # 'panel' | 'user'
+ 'why': why or detail,
+ 'fix': fix,
+ }
+
+
+class SetupHealthService:
+ """Evaluate the setup-health registry, snooze/unsnooze items, run the nag."""
+
+ # ------------------------------------------------------------------ #
+ # Small settings helpers (read raw settings, no dev fallbacks — an item
+ # like base_domain must reflect what the operator actually set, not the
+ # SITES_BASE_DOMAIN dev default).
+ # ------------------------------------------------------------------ #
+
+ @staticmethod
+ def _get(key, default=None):
+ from app.models.system_settings import SystemSettings
+ val = SystemSettings.get(key, default)
+ return val if val is not None else default
+
+ @classmethod
+ def _base_domain(cls):
+ return (cls._get('sites_base_domain', '') or '').strip().lower()
+
+ @classmethod
+ def _dns_mode(cls):
+ return (cls._get('sites_dns_mode', '') or '').strip().lower()
+
+ @classmethod
+ def _server_ip(cls):
+ return (cls._get('server_public_ip', '') or '').strip()
+
+ @classmethod
+ def _https_enabled(cls):
+ return bool(cls._get('sites_https_enabled', False))
+
+ # ------------------------------------------------------------------ #
+ # Item scope + snooze storage
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _item_scope(cls, key):
+ """'panel' | 'user' for a known item key, else None."""
+ return _ITEM_SCOPES.get(key)
+
+ @staticmethod
+ def _panel_snoozes():
+ """The panel snooze map ``{key: iso_expiry}`` from the settings map."""
+ from app.services.settings_service import SettingsService
+ raw = SettingsService.get(PANEL_SNOOZE_KEY)
+ if not raw:
+ return {}
+ try:
+ data = json.loads(raw)
+ return data if isinstance(data, dict) else {}
+ except (ValueError, TypeError):
+ return {}
+
+ @staticmethod
+ def _user_snoozes(user):
+ if not user or not getattr(user, 'setup_snoozes', None):
+ return {}
+ try:
+ data = json.loads(user.setup_snoozes)
+ return data if isinstance(data, dict) else {}
+ except (ValueError, TypeError):
+ return {}
+
+ @staticmethod
+ def _active_until(snooze_map, key, now):
+ """The (non-expired) snooze expiry for ``key``, or None."""
+ iso = (snooze_map or {}).get(key)
+ if not iso:
+ return None
+ try:
+ expiry = datetime.fromisoformat(iso)
+ except (ValueError, TypeError):
+ return None
+ return iso if expiry > now else None
+
+ # ------------------------------------------------------------------ #
+ # The probe matrix — one builder per item
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _panel_items(cls):
+ base = cls._base_domain()
+ mode = cls._dns_mode()
+ ip = cls._server_ip()
+ per_site = bool(base) and mode == 'per-site'
+ items = []
+
+ # --- server public IP ---
+ if ip:
+ items.append(_item(
+ 'setup.public_ip', 'Server public IP', 'ok',
+ f'Server public IP is set ({ip}).', RECOMMENDED,
+ _link('/settings')))
+ elif per_site:
+ items.append(_item(
+ 'setup.public_ip', 'Server public IP', 'fail',
+ 'DNS mode is per-site but no server public IP is set, so each '
+ "managed site's A record can't be auto-created — new sites won't "
+ 'resolve.', CRITICAL, _link('/settings')))
+ else:
+ items.append(_item(
+ 'setup.public_ip', 'Server public IP', 'warn',
+ 'No server public IP set. It is used to auto-create A records for '
+ 'managed domains.', RECOMMENDED, _link('/settings')))
+
+ # --- managed-sites base domain ---
+ if base:
+ items.append(_item(
+ 'setup.base_domain', 'Managed-sites base domain', 'ok',
+ f'Managed sites are published under {base}.', RECOMMENDED,
+ _link('/settings')))
+ else:
+ items.append(_item(
+ 'setup.base_domain', 'Managed-sites base domain', 'warn',
+ 'No base domain set — new sites are only reachable at '
+ 'localhost:<port>. Set one so each site publishes at '
+ '<name>.<domain>.', RECOMMENDED, _link('/settings')))
+
+ # --- DNS provider ---
+ has_provider = cls._has_dns_provider()
+ if has_provider:
+ items.append(_item(
+ 'setup.dns_provider', 'DNS provider connected', 'ok',
+ 'A DNS provider is connected for auto-managing records.',
+ RECOMMENDED, _link('/settings/connections')))
+ elif per_site:
+ items.append(_item(
+ 'setup.dns_provider', 'DNS provider connected', 'fail',
+ 'DNS mode is per-site but no DNS provider is connected, so each '
+ "site's A record can't be auto-created — new sites won't resolve.",
+ CRITICAL, _link('/settings/connections')))
+ else:
+ items.append(_item(
+ 'setup.dns_provider', 'DNS provider connected', 'warn',
+ 'No DNS provider connected. Connect one to auto-create and heal '
+ 'DNS records for managed domains.', RECOMMENDED,
+ _link('/settings/connections')))
+
+ # --- email delivery ---
+ items.append(cls._email_delivery_item())
+
+ # --- backup policy ---
+ items.append(cls._backup_policy_item())
+
+ # --- offsite backup ---
+ items.append(cls._backup_offsite_item())
+
+ # --- canonical panel domain ---
+ canonical = (cls._get('canonical_domain', '') or '').strip()
+ if canonical:
+ items.append(_item(
+ 'setup.canonical_domain', 'Canonical panel domain', 'ok',
+ f'Panel canonical domain is set ({canonical}).', RECOMMENDED,
+ _link('/settings')))
+ else:
+ items.append(_item(
+ 'setup.canonical_domain', 'Canonical panel domain', 'warn',
+ 'No canonical panel domain set. Setting one gives the panel a '
+ 'stable public URL for agent installs and CORS.', RECOMMENDED,
+ _link('/settings')))
+
+ # --- wildcard certificate (only once wildcard HTTPS is switched on;
+ # never critical — SSL is optional by decree) ---
+ cert_item = cls._wildcard_cert_item(base, mode)
+ if cert_item is not None:
+ items.append(cert_item)
+
+ return items
+
+ @classmethod
+ def _has_dns_provider(cls):
+ try:
+ from app.models.email import DNSProviderConfig
+ return DNSProviderConfig.query.count() > 0
+ except Exception: # noqa: BLE001
+ return False
+
+ @classmethod
+ def _email_delivery_item(cls):
+ try:
+ from app.models.email_provider import EmailProviderConnection
+ providers = EmailProviderConnection.query.filter_by(
+ is_active=True, uses_notifications=True).all()
+ except Exception: # noqa: BLE001
+ providers = []
+ tested = [p for p in providers if p.last_test_ok]
+ if tested:
+ return _item(
+ 'setup.email_delivery', 'Email delivery', 'ok',
+ 'A verified email delivery path is configured.', RECOMMENDED,
+ _link('/settings/connections'))
+ if providers:
+ return _item(
+ 'setup.email_delivery', 'Email delivery', 'warn',
+ 'An email provider is configured but its test send has not '
+ 'passed — notifications may not be delivered.', RECOMMENDED,
+ _link('/settings/connections'))
+ return _item(
+ 'setup.email_delivery', 'Email delivery', 'warn',
+ 'No email delivery path configured — the panel cannot send '
+ 'notifications or invitations.', RECOMMENDED,
+ _link('/settings/connections'))
+
+ @classmethod
+ def _backup_policy_item(cls):
+ try:
+ from app.models.backup_policy import BackupPolicy
+ enabled = BackupPolicy.query.filter_by(enabled=True).count() > 0
+ except Exception: # noqa: BLE001
+ enabled = False
+ if enabled:
+ return _item(
+ 'setup.backup_policy', 'Backups scheduled', 'ok',
+ 'At least one enabled backup policy is protecting a target.',
+ RECOMMENDED, _link('/backups'))
+ return _item(
+ 'setup.backup_policy', 'Backups scheduled', 'warn',
+ 'No enabled backup policy — nothing is being backed up on a '
+ 'schedule.', RECOMMENDED, _link('/backups'))
+
+ @classmethod
+ def _backup_offsite_item(cls):
+ provider = None
+ try:
+ from app.services.storage_provider_service import StorageProviderService
+ cfg = StorageProviderService.get_config() or {}
+ provider = cfg.get('provider')
+ except Exception: # noqa: BLE001
+ provider = None
+ if provider and provider != 'local':
+ return _item(
+ 'setup.backup_offsite', 'Offsite backup storage', 'ok',
+ f'Offsite backup storage is configured ({provider}).',
+ RECOMMENDED, _link('/settings/storage'))
+ return _item(
+ 'setup.backup_offsite', 'Offsite backup storage', 'warn',
+ 'Backups are kept on this server only. Configure S3/B2 offsite '
+ 'storage so a backup survives losing the box.', RECOMMENDED,
+ _link('/settings/storage'))
+
+ @classmethod
+ def _wildcard_cert_item(cls, base, mode):
+ """The wildcard-cert item — only applicable once wildcard HTTPS is on for
+ a wildcard base. Never critical (HTTPS is optional)."""
+ if not base or mode != 'wildcard' or not cls._https_enabled():
+ return None
+ exists = True
+ if sys.platform.startswith('linux'):
+ try:
+ from app.services.site_domain_service import SiteDomainService
+ cert, _key = SiteDomainService.wildcard_cert_paths(base)
+ exists = bool(cert and os.path.exists(cert))
+ except Exception: # noqa: BLE001
+ exists = True
+ if exists:
+ return _item(
+ 'setup.wildcard_cert', 'Wildcard HTTPS certificate', 'ok',
+ f'A wildcard certificate for *.{base} is present.', RECOMMENDED,
+ _link('/settings'))
+ return _item(
+ 'setup.wildcard_cert', 'Wildcard HTTPS certificate', 'warn',
+ f'Wildcard HTTPS is enabled for *.{base} but its certificate is not '
+ 'present yet — managed subdomains fall back to HTTP until it issues.',
+ RECOMMENDED, _link('/settings'))
+
+ @classmethod
+ def _user_items(cls, user):
+ if not user:
+ return []
+ has_factor = bool(getattr(user, 'totp_enabled', False))
+ if not has_factor:
+ try:
+ has_factor = user.passkeys.filter_by(is_active=True).count() > 0
+ except Exception: # noqa: BLE001
+ has_factor = False
+ if has_factor:
+ item = _item(
+ 'setup.account_security', 'Secure your account', 'ok',
+ 'Your account has a second factor (passkey or authenticator).',
+ RECOMMENDED, _link('/settings/security'), scope='user')
+ else:
+ item = _item(
+ 'setup.account_security', 'Secure your account', 'warn',
+ 'Add a passkey or authenticator app so a stolen password alone '
+ "can't sign in as you.", RECOMMENDED,
+ _link('/settings/security'), scope='user')
+ return [item]
+
+ # ------------------------------------------------------------------ #
+ # Evaluate
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def evaluate(cls, scope='panel', user=None):
+ """Evaluate the registry for ``scope`` ('panel' | 'user'), applying
+ snoozes, and return ``{'items': [...], 'summary': {...}}``."""
+ now = datetime.utcnow()
+ if scope == 'user':
+ items = cls._user_items(user)
+ snooze_map = cls._user_snoozes(user)
+ else:
+ items = cls._panel_items()
+ snooze_map = cls._panel_snoozes()
+
+ for c in items:
+ until = cls._active_until(snooze_map, c['key'], now)
+ if until:
+ c['snoozed'] = True
+ c['snoozed_until'] = until
+ else:
+ c['snoozed'] = False
+ c['snoozed_until'] = None
+
+ return {'items': items, 'summary': cls._summary(items)}
+
+ @staticmethod
+ def _summary(items):
+ weights = {CRITICAL: 3, RECOMMENDED: 1}
+ ok = critical_open = recommended_open = snoozed = 0
+ earned = total_weight = 0
+ for c in items:
+ w = weights.get(c['severity'], 1)
+ total_weight += w
+ is_ok = c['status'] == 'ok'
+ is_snoozed = bool(c.get('snoozed'))
+ if is_ok or is_snoozed:
+ earned += w
+ if is_ok:
+ ok += 1
+ elif is_snoozed:
+ snoozed += 1
+ elif c['severity'] == CRITICAL:
+ critical_open += 1
+ else:
+ recommended_open += 1
+ score = round(100 * earned / total_weight) if total_weight else 100
+ return {
+ 'total': len(items),
+ 'ok': ok,
+ 'critical_open': critical_open,
+ 'recommended_open': recommended_open,
+ 'snoozed': snoozed,
+ 'score': max(0, min(100, score)),
+ }
+
+ # ------------------------------------------------------------------ #
+ # Snooze / unsnooze
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def snooze(cls, key, days=30, user=None):
+ """Snooze a setup item for ``days`` (mutes it). Panel items store into the
+ settings map; personal items store on the user row. Returns a descriptor
+ dict, or ``{'error': ...}`` for an unknown item."""
+ scope = cls._item_scope(key)
+ if scope is None:
+ return {'error': f'Unknown setup item: {key}'}
+ try:
+ days = int(days)
+ except (TypeError, ValueError):
+ days = 30
+ until = (datetime.utcnow() + timedelta(days=days)).isoformat()
+
+ if scope == 'user':
+ if user is None:
+ return {'error': 'A user is required to snooze a personal item.'}
+ from app import db
+ snoozes = cls._user_snoozes(user)
+ snoozes[key] = until
+ user.setup_snoozes = json.dumps(snoozes)
+ db.session.commit()
+ else:
+ from app.services.settings_service import SettingsService
+ snoozes = cls._panel_snoozes()
+ snoozes[key] = until
+ SettingsService.set(PANEL_SNOOZE_KEY, json.dumps(snoozes))
+ return {'success': True, 'key': key, 'scope': scope, 'snoozed_until': until}
+
+ @classmethod
+ def unsnooze(cls, key, user=None):
+ """Clear a snooze so the item is active again."""
+ scope = cls._item_scope(key)
+ if scope is None:
+ return {'error': f'Unknown setup item: {key}'}
+ if scope == 'user':
+ if user is None:
+ return {'error': 'A user is required to unsnooze a personal item.'}
+ from app import db
+ snoozes = cls._user_snoozes(user)
+ snoozes.pop(key, None)
+ user.setup_snoozes = json.dumps(snoozes)
+ db.session.commit()
+ else:
+ from app.services.settings_service import SettingsService
+ snoozes = cls._panel_snoozes()
+ snoozes.pop(key, None)
+ SettingsService.set(PANEL_SNOOZE_KEY, json.dumps(snoozes))
+ return {'success': True, 'key': key, 'scope': scope}
+
+ # ------------------------------------------------------------------ #
+ # Fingerprint + nag
+ # ------------------------------------------------------------------ #
+
+ @staticmethod
+ def _critical_fingerprint(result):
+ """A stable, human-readable fingerprint of the non-snoozed critical open
+ items (the comma-joined sorted keys). Empty string when none."""
+ keys = sorted(
+ c['key'] for c in result['items']
+ if c['severity'] == CRITICAL and c['status'] != 'ok'
+ and not c.get('snoozed'))
+ return ','.join(keys)
+
+ @classmethod
+ def fingerprint(cls):
+ """The current non-snoozed-critical fingerprint (panel scope)."""
+ return cls._critical_fingerprint(cls.evaluate(scope='panel'))
+
+ @classmethod
+ def run_nag_job(cls, job):
+ """Weekly nag handler: notify admins once per critical-set change while
+ non-snoozed critical items exist; reset the marker once they clear."""
+ from app.services.settings_service import SettingsService
+ result = cls.evaluate(scope='panel')
+ critical_open = result['summary']['critical_open']
+ marker = SettingsService.get(NAG_MARKER_KEY) or ''
+
+ if critical_open == 0:
+ if marker:
+ SettingsService.set(NAG_MARKER_KEY, '')
+ return {'notified': False, 'critical_open': 0}
+
+ fingerprint = cls._critical_fingerprint(result)
+ if fingerprint == marker:
+ return {'notified': False, 'reason': 'unchanged',
+ 'critical_open': critical_open}
+
+ try:
+ from app.plugins_sdk import notify
+ notify.send(
+ 'setup.incomplete', to='admins',
+ data={
+ 'count': critical_open,
+ 'summary': f'{critical_open} critical setup item(s) need '
+ 'attention.',
+ 'message': f'{critical_open} critical setup item(s) are '
+ 'incomplete. Finish them in Monitoring → Doctor '
+ '(Setup section).',
+ })
+ except Exception as e: # noqa: BLE001 — a nag must never break the job
+ logger.warning('setup-health nag notify failed: %s', e)
+
+ SettingsService.set(NAG_MARKER_KEY, fingerprint)
+ return {'notified': True, 'critical_open': critical_open}
+
+ # ------------------------------------------------------------------ #
+ # Doctor integration + job plumbing
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def doctor_checks(cls):
+ """The panel setup items as doctor ``setup.*`` checks, for inclusion in
+ the doctor report's Setup section (Monitoring → Doctor)."""
+ return cls.evaluate(scope='panel')['items']
+
+ @classmethod
+ def register_jobs(cls):
+ """Register the nag handler with the job registry. Called once at app
+ startup (see app/__init__.py)."""
+ from app.jobs import registry
+ registry.register(NAG_JOB_KIND, cls.run_nag_job, replace=True)
+
+
+def _register_catalog_events():
+ """Register the ``setup.incomplete`` event so the nag renders through the same
+ pipeline as any core notification. Safe + idempotent at import time."""
+ try:
+ from app.notifications import catalog
+ catalog.register(
+ 'setup.incomplete',
+ title='Finish setting up ServerKit',
+ template='generic',
+ severity='warning',
+ category='system',
+ )
+ except Exception as e: # noqa: BLE001
+ logger.debug('could not register setup.incomplete catalog event: %s', e)
+
+
+_register_catalog_events()
diff --git a/backend/app/services/setup_reconcile_service.py b/backend/app/services/setup_reconcile_service.py
new file mode 100644
index 00000000..d4d11ec5
--- /dev/null
+++ b/backend/app/services/setup_reconcile_service.py
@@ -0,0 +1,371 @@
+"""Reconcile-on-connect (plan 22 Phase 3).
+
+When the operator finally connects a DNS provider / sets a server IP / picks a
+base domain, the sites they already created are still pointing at nothing. This
+service reconciles that backlog explicitly (never auto-applied — Decision 3):
+
+* **DNS backfill** — enumerate the managed hosts that need an A record given the
+ current base/mode, preview which a connected provider would cover, then loop
+ ``DNSProviderService.ensure_a_record`` over them. Idempotent: an existing
+ record returns ``created: False`` and is left untouched.
+* **WordPress URL-fix** — detect WordPress sites whose live siteurl/home still
+ points at ``localhost``/a stale IP and swap it to the site's real managed URL
+ via the URL-swap rails (keeping the old-host redirect). A site already on its
+ target URL is not detected, so a second run is a no-op.
+
+Both apply loops are exposed as job handlers so a large backfill/heal runs off
+the request thread (the API enqueues; see ``app/api/setup_health.py``).
+"""
+import ipaddress
+import logging
+from urllib.parse import urlparse
+
+logger = logging.getLogger(__name__)
+
+DNS_BACKFILL_JOB_KIND = 'setup.reconcile.dns_backfill'
+URL_FIX_JOB_KIND = 'setup.reconcile.url_fix'
+
+# Loopback / local host labels that a live site URL must be swapped away from.
+_LOCAL_HOSTS = {'localhost', '127.0.0.1', '0.0.0.0', '::1'}
+
+# Reserved / dev suffixes whose hosts never get a public DNS record (the doctor's
+# own filter — bare IPs and no-dot single labels are handled separately).
+_DEV_SUFFIXES = ('.lvh.me', '.local', '.test', '.internal', '.localhost')
+
+
+class SetupReconcileService:
+ """DNS backfill + WordPress URL-fix for apps adopted before setup finished."""
+
+ # ------------------------------------------------------------------ #
+ # Host classification
+ # ------------------------------------------------------------------ #
+
+ @staticmethod
+ def _is_ip(host):
+ try:
+ ipaddress.ip_address(host)
+ return True
+ except ValueError:
+ return False
+
+ @classmethod
+ def _skip_dns_host(cls, host):
+ """Dev/loopback names and bare IPs are skipped (the doctor's own filter)."""
+ h = (host or '').strip().lower().rstrip('.')
+ if not h or '.' not in h:
+ return True
+ if cls._is_ip(h):
+ return True
+ if h == 'localhost' or any(h.endswith(suf) for suf in _DEV_SUFFIXES):
+ return True
+ return False
+
+ # ------------------------------------------------------------------ #
+ # DNS backfill — candidate enumeration
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _dns_candidates(cls):
+ """The managed hosts that need a DNS record, by mode:
+
+ * a base in ``wildcard`` mode → a single ``*.<base>`` candidate (the
+ per-site subdomains are covered by it, so they aren't separate),
+ * a base in ``per-site`` mode → each subdomain as its own candidate,
+ * a domain outside every base → a ``custom`` candidate.
+
+ Dev/loopback names and bare IPs are skipped.
+ """
+ from app.models.domain import Domain
+ from app.services.site_domain_service import SiteDomainService
+
+ candidates = []
+ seen = set()
+ for domain in Domain.query.order_by(Domain.name.asc()).all():
+ host = (domain.name or '').strip().lower()
+ if cls._skip_dns_host(host):
+ continue
+ base = SiteDomainService.covering_base(host)
+ if base:
+ mode = SiteDomainService.dns_mode(base)
+ if mode == 'wildcard':
+ wildcard = f'*.{base}'
+ if wildcard in seen:
+ continue
+ seen.add(wildcard)
+ candidates.append({'host': wildcard, 'mode': 'wildcard',
+ 'base': base})
+ else: # per-site
+ if host in seen:
+ continue
+ seen.add(host)
+ candidates.append({'host': host, 'mode': 'per-site',
+ 'base': base})
+ else:
+ if host in seen:
+ continue
+ seen.add(host)
+ candidates.append({'host': host, 'mode': 'custom', 'base': None})
+ return candidates
+
+ @staticmethod
+ def _provider_covers_zone(host):
+ """Whether a connected provider authoritatively covers ``host``'s zone.
+ Never raises (provider listing hits the network)."""
+ try:
+ from app.services.dns_provider_service import DNSProviderService
+ config, _zone = DNSProviderService.find_zone_for_domain(host)
+ return bool(config)
+ except Exception: # noqa: BLE001
+ return False
+
+ @classmethod
+ def dns_backfill_preview(cls):
+ """Dry-run: the managed hosts a backfill would ensure an A record for, and
+ whether a connected provider covers each host's zone. Never writes."""
+ from app.models.email import DNSProviderConfig
+ from app.services.site_domain_service import SiteDomainService
+
+ server_ip = SiteDomainService.server_ip()
+ has_provider = DNSProviderConfig.query.count() > 0
+ ready = bool(server_ip)
+
+ items = []
+ for cand in cls._dns_candidates():
+ # Only probe provider coverage when the backfill could actually run —
+ # avoids a network round-trip when there's nothing to apply.
+ covered = (cls._provider_covers_zone(cand['host'])
+ if (has_provider and ready) else False)
+ items.append({
+ 'host': cand['host'],
+ 'mode': cand['mode'],
+ 'target_ip': server_ip,
+ 'provider_covers_zone': covered,
+ })
+
+ return {
+ 'ready': ready,
+ 'has_provider': has_provider,
+ 'server_ip': server_ip,
+ 'items': items,
+ 'count': len(items),
+ 'total': len(items),
+ }
+
+ @classmethod
+ def dns_backfill_apply(cls):
+ """Loop ``ensure_a_record`` over every candidate. Idempotent — an existing
+ record returns ``created: False`` and is left untouched. Returns per-host
+ results + a created count."""
+ from app.services.dns_provider_service import DNSProviderService
+ from app.services.site_domain_service import SiteDomainService
+
+ server_ip = SiteDomainService.server_ip()
+ if not server_ip:
+ return {'applied': 0, 'results': [], 'total': 0,
+ 'error': 'No server public IP set.'}
+
+ candidates = cls._dns_candidates()
+ results = []
+ applied = 0
+ for cand in candidates:
+ host = cand['host']
+ try:
+ res = DNSProviderService.ensure_a_record(host, server_ip)
+ except Exception as e: # noqa: BLE001
+ res = {'created': False, 'reason': 'error', 'error': str(e)}
+ created = bool(res.get('created'))
+ if created:
+ applied += 1
+ row = {'host': host, 'mode': cand['mode']}
+ row.update(res)
+ row['created'] = created
+ results.append(row)
+
+ return {'applied': applied, 'results': results, 'total': len(candidates)}
+
+ @classmethod
+ def run_dns_backfill_job(cls, job):
+ """Job handler for ``setup.reconcile.dns_backfill``."""
+ return cls.dns_backfill_apply()
+
+ # ------------------------------------------------------------------ #
+ # WordPress url-fix — classifier
+ # ------------------------------------------------------------------ #
+
+ @staticmethod
+ def _host_of(url):
+ try:
+ return (urlparse(url).hostname or '').strip().lower()
+ except Exception: # noqa: BLE001
+ return ''
+
+ @classmethod
+ def _is_local_host(cls, host):
+ """A loopback/local label or a bare IP — not a real public host."""
+ if not host:
+ return False
+ if host in _LOCAL_HOSTS:
+ return True
+ if cls._is_ip(host):
+ return True
+ if '.' not in host: # a bare single label (e.g. 'localhost')
+ return True
+ return False
+
+ @classmethod
+ def _is_real_domain(cls, host):
+ return bool(host) and '.' in host and not cls._is_ip(host) \
+ and host not in _LOCAL_HOSTS
+
+ @classmethod
+ def url_fix_needed(cls, current_url, target_url):
+ """Pure classifier (testable without wp-cli): does a site at
+ ``current_url`` need swapping to ``target_url``?
+
+ Needs a fix when the current host is a local/loopback name or a bare IP
+ AND there is a real domain target that differs. A current host that is
+ already a proper domain (== target or otherwise) is left alone.
+ """
+ if not current_url or not target_url:
+ return False
+ current = cls._host_of(current_url)
+ target = cls._host_of(target_url)
+ if not current or not target:
+ return False
+ if not cls._is_real_domain(target):
+ return False
+ if not cls._is_local_host(current):
+ return False
+ return current != target
+
+ # ------------------------------------------------------------------ #
+ # WordPress url-fix — site enumeration + target resolution
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _wp_sites(cls):
+ """Production WordPress sites with a root path, or [] when the WordPress
+ extension isn't loadable (dev without it, minimal test env)."""
+ try:
+ from app.models.wordpress_site import WordPressSite
+ except Exception: # noqa: BLE001
+ return []
+ try:
+ sites = WordPressSite.query.all()
+ except Exception: # noqa: BLE001
+ return []
+ return [s for s in sites
+ if s.application and getattr(s.application, 'root_path', None)]
+
+ @classmethod
+ def _target_url_for(cls, app):
+ """The real managed URL a site should serve under — its primary domain,
+ else None (nothing to point at)."""
+ from app.models.domain import Domain
+ from app.services.site_domain_service import SiteDomainService
+
+ domain = Domain.query.filter_by(
+ application_id=app.id, is_primary=True).first()
+ if not domain or not domain.name:
+ return None
+ host = domain.name.strip().lower()
+
+ ssl = bool(getattr(domain, 'ssl_enabled', False))
+ if not ssl:
+ try:
+ base = SiteDomainService.covering_base(host)
+ ssl = bool(base and SiteDomainService.https_enabled(base)
+ and SiteDomainService.covers(host))
+ except Exception: # noqa: BLE001
+ ssl = False
+ scheme = 'https' if ssl else 'http'
+ return f'{scheme}://{host}'
+
+ @classmethod
+ def url_fix_preview(cls):
+ """Detect WordPress sites whose live URL is still localhost/a stale IP and
+ show the per-site swap (current → target) via the URL-swap preview rails.
+ Never mutates."""
+ import app.services.wordpress_bridge as bridge
+ try:
+ wp = bridge.wordpress_service()
+ except Exception as e: # noqa: BLE001
+ return {'count': 0, 'items': [], 'warning': str(e)}
+
+ items = []
+ for site in cls._wp_sites():
+ app = site.application
+ target = cls._target_url_for(app)
+ if not target:
+ continue
+ try:
+ current = wp._site_current_url(app)
+ except Exception: # noqa: BLE001
+ continue
+ if not cls.url_fix_needed(current, target):
+ continue
+ entry = {'site_id': site.id, 'current_url': current, 'new_url': target}
+ try:
+ preview = wp.preview_url_change(app, target)
+ entry['pairs'] = preview.get('pairs')
+ entry['total'] = preview.get('total')
+ entry['success'] = preview.get('success', True)
+ except Exception as e: # noqa: BLE001
+ entry['warning'] = str(e)
+ items.append(entry)
+
+ return {'count': len(items), 'items': items}
+
+ @classmethod
+ def url_fix_apply(cls):
+ """Apply the URL swap to every detected site via ``change_site_url``
+ (keeps the old-host redirect). Idempotent — a site already on its target
+ URL is not detected, so a second run is a no-op."""
+ import app.services.wordpress_bridge as bridge
+ try:
+ wp = bridge.wordpress_service()
+ except Exception as e: # noqa: BLE001
+ return {'fixed': 0, 'results': [], 'error': str(e)}
+
+ results = []
+ fixed = 0
+ for site in cls._wp_sites():
+ app = site.application
+ target = cls._target_url_for(app)
+ if not target:
+ continue
+ try:
+ current = wp._site_current_url(app)
+ except Exception: # noqa: BLE001
+ continue
+ if not cls.url_fix_needed(current, target):
+ continue
+ try:
+ res = wp.change_site_url(app, target, keep_old_redirect=True)
+ except Exception as e: # noqa: BLE001
+ res = {'success': False, 'error': str(e)}
+ if res.get('success'):
+ fixed += 1
+ results.append({'site_id': site.id, 'new_url': target,
+ 'old_url': res.get('old_url', current),
+ 'success': bool(res.get('success'))})
+
+ return {'fixed': fixed, 'results': results}
+
+ @classmethod
+ def run_url_fix_job(cls, job):
+ """Job handler for ``setup.reconcile.url_fix``."""
+ return cls.url_fix_apply()
+
+ # ------------------------------------------------------------------ #
+ # Job registration
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def register_jobs(cls):
+ """Register both reconcile handlers with the job registry (called once at
+ app startup, see app/__init__.py)."""
+ from app.jobs import registry
+ registry.register(DNS_BACKFILL_JOB_KIND, cls.run_dns_backfill_job,
+ replace=True)
+ registry.register(URL_FIX_JOB_KIND, cls.run_url_fix_job, replace=True)