ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
fix(recovery): rebuild setup-health/reconcile + backup drill/verify services
Juan Denis7a903edc4f26
diff --git a/backend/app/services/backup_drill_service.py b/backend/app/services/backup_drill_service.pynew file mode 100644index 00000000..6a8d2320--- /dev/null+++ b/backend/app/services/backup_drill_service.py@@ -0,0 +1,673 @@+"""Restore-drill engine (plan 23 Phases 2–3).++A *drill* is a real restore of a policy's latest restorable point into a scratch+location (a temp dir for files/application, a throwaway ``skdrill_<run_id>``+database for database targets), probe-verified, then torn down. It never touches+the live target. A successful drill is the strongest proof a backup is+restorable: it promotes the drilled :class:`BackupRun` to+``verify_level='drilled'`` and stamps the policy's ``last_drill_*`` cache.++Key behaviours (match the proving tests):+ * **expectation-stamped probes** — files record ``file_count`` + a hash sample+ (``sampled`` / ``sampled_ok``); databases record ``table_count``.+ * **container-aware / real DB legs** — the database leg runs the real+ ``create → restore → probe → drop`` sequence against the shared engine using+ the existing ``DatabaseService`` helpers; the scratch DB is dropped in a+ ``finally`` even when the restore fails.+ * **vacuous-drill detection** — a drill that restored zero files fails loudly+ instead of earning ``drilled`` for nothing.+ * **hard guards, loud skips** — a free-space precheck marks the drill+ ``skipped_no_space`` (doctor-visible) rather than silently passing.+ * **single-flight** — one drill in flight per host; a new request is refused+ (:class:`BackupDrillError`) while one is running.+ * **cadence sweep** — a daily sweep enqueues due drills serially.++See docs/plans/23_BACKUP_TRUST_RESTORE_DRILLS_PLAN.md.+"""+import logging+import math+import os+import shutil+from datetime import datetime, timedelta++from app import db++logger = logging.getLogger(__name__)++DRILL_JOB_KIND = 'backup.drill.run'+DRILL_SWEEP_JOB_KIND = 'backup.drill.sweep'+DRILL_ORPHAN_SWEEP_JOB_KIND = 'backup.drill.orphan_sweep'+DRILL_SWEEP_SCHEDULE_NAME = 'backup-drill-sweep'+DRILL_ORPHAN_SCHEDULE_NAME = 'backup-drill-orphan-sweep'++# Cadence → base interval in days.+CADENCE_DAYS = {'weekly': 7, 'monthly': 30}+# A 'running' drill older than this is treated as crashed, not in-flight.+STALE_DRILL_AFTER = timedelta(hours=6)+# Free-space safety multipliers (Decision 6).+EXTRACTED_ESTIMATE_MULT = 1.2 # from manifest totals+ARCHIVE_ESTIMATE_MULT = 3.0 # fallback: raw archive size × 3+# Scratch dirs / scratch DBs older than this are orphan-swept.+ORPHAN_AFTER = timedelta(hours=24)+# Hash sample bounds.+SAMPLE_MAX = 50+SAMPLE_FRACTION = 0.05+++class BackupDrillError(Exception):+ """Raised when a drill is refused or fails loudly (e.g. vacuous restore)."""+++class BackupDrillService:++ # ------------------------------------------------------------------ #+ # Space accounting (guards)+ # ------------------------------------------------------------------ #++ @classmethod+ def _estimate_required(cls, run, meta):+ """Estimated bytes needed to materialize the drill: manifest extracted+ size ×1.2 where known, else raw primary-archive size ×3."""+ from app.services.backup_verify_service import read_manifest+ manifest = read_manifest(run, meta)+ if manifest:+ total = ((manifest.get('totals') or {}).get('bytes')) or 0+ if total:+ return int(total * EXTRACTED_ESTIMATE_MULT)+ primary = meta.get('primary_archive') or run.storage_path+ size = 0+ try:+ if primary and os.path.isfile(primary):+ size = os.path.getsize(primary)+ elif primary and os.path.isdir(primary):+ size = sum(+ os.path.getsize(os.path.join(r, f))+ for r, _d, fs in os.walk(primary) for f in fs+ if os.path.isfile(os.path.join(r, f))+ )+ except OSError:+ size = 0+ return int(size * ARCHIVE_ESTIMATE_MULT)++ @staticmethod+ def _free_bytes(path):+ """Free bytes on the filesystem holding ``path`` (walks up to the nearest+ existing ancestor so a not-yet-created scratch dir still measures)."""+ probe = path+ while probe and not os.path.exists(probe):+ parent = os.path.dirname(probe.rstrip(os.sep))+ if parent == probe:+ break+ probe = parent+ try:+ return shutil.disk_usage(probe or os.getcwd()).free+ except OSError:+ return 0++ @classmethod+ def _scratch_base(cls):+ from app.services.backup_service import BackupService+ base = os.path.join(BackupService.BACKUP_BASE_DIR, 'restores')+ os.makedirs(base, exist_ok=True)+ return base++ # ------------------------------------------------------------------ #+ # Single-flight guard + run selection+ # ------------------------------------------------------------------ #++ @classmethod+ def is_drilling(cls):+ """True if a (non-stale) restore drill is currently running on this host."""+ from app.models.restore_drill import RestoreDrill+ cutoff = datetime.utcnow() - STALE_DRILL_AFTER+ return db.session.query(RestoreDrill.id).filter(+ RestoreDrill.status == 'running',+ RestoreDrill.started_at >= cutoff,+ ).first() is not None++ @classmethod+ def _latest_restorable_run(cls, policy):+ """Newest successful run (the chain endpoint) for a policy."""+ from app.models.backup_run import BackupRun+ return (BackupRun.query+ .filter_by(policy_id=policy.id, status='success')+ .order_by(BackupRun.started_at.desc())+ .first())++ # ------------------------------------------------------------------ #+ # Enqueue (single-flight refusal lives here)+ # ------------------------------------------------------------------ #++ @classmethod+ def request_drill(cls, policy, trigger='manual', run_id=None):+ """Enqueue a ``backup.drill.run`` job for a policy's latest restorable+ point. Refused (:class:`BackupDrillError`) while a drill is in flight."""+ if cls.is_drilling():+ raise BackupDrillError('A restore drill is already in progress.')+ if run_id is None:+ run = cls._latest_restorable_run(policy)+ if not run:+ raise BackupDrillError('No successful backup to drill.')+ run_id = run.id+ from app.jobs.service import JobService+ job = JobService.enqueue(+ DRILL_JOB_KIND,+ payload={'policy_id': policy.id, 'run_id': run_id, 'trigger': trigger},+ max_attempts=1,+ owner_type='backup_policy',+ owner_id=policy.id,+ )+ return job++ # ------------------------------------------------------------------ #+ # Cadence (Phase 3)+ # ------------------------------------------------------------------ #++ @classmethod+ def _jitter(cls, policy, cadence):+ """Deterministic per-policy jitter (0–20% of the interval) so scheduled+ drills spread out instead of all firing on the same day."""+ import random+ interval = timedelta(days=CADENCE_DAYS[cadence])+ frac = random.Random(policy.id or 0).uniform(0.0, 0.2)+ return interval * frac++ @classmethod+ def is_due(cls, policy, now=None):+ """True if ``policy`` should be drilled now (enabled, cadence on, a+ restorable run exists, and it hasn't been drilled within the cadence+ interval + jitter)."""+ now = now or datetime.utcnow()+ if not policy.enabled:+ return False+ cadence = policy.drill_cadence or 'off'+ if cadence not in CADENCE_DAYS:+ return False+ if not cls._latest_restorable_run(policy):+ return False+ if not policy.last_drill_at:+ return True+ threshold = timedelta(days=CADENCE_DAYS[cadence]) + cls._jitter(policy, cadence)+ return (now - policy.last_drill_at) >= threshold++ @classmethod+ def due_policies(cls, now=None):+ """All enabled policies whose drill cadence is due."""+ from app.models.backup_policy import BackupPolicy+ now = now or datetime.utcnow()+ return [p for p in BackupPolicy.query.filter_by(enabled=True).all()+ if cls.is_due(p, now)]++ @classmethod+ def run_drill_sweep(cls, job=None):+ """Daily sweep: enqueue due drills serially. Defers entirely while a+ drill is already in flight (natural rate limiting / single-flight)."""+ if cls.is_drilling():+ return {'count': 0, 'enqueued': [], 'deferred': True}+ enqueued = []+ for policy in cls.due_policies():+ try:+ cls.request_drill(policy, trigger='scheduled')+ enqueued.append(policy.id)+ except BackupDrillError:+ # One is now in flight — the rest wait for the next sweep.+ break+ return {'count': len(enqueued), 'enqueued': enqueued}++ # ------------------------------------------------------------------ #+ # The drill itself (Phase 2)+ # ------------------------------------------------------------------ #++ @classmethod+ def run_restore_drill(cls, job):+ """Job handler for ``backup.drill.run``. Restores the run's chain into a+ scratch target, probes it, and records the outcome. Returns a result+ dict; raises on a real drill failure so the job is marked failed."""+ from app.models.backup_policy import BackupPolicy+ from app.models.backup_run import BackupRun+ from app.models.restore_drill import RestoreDrill++ payload = job.get_payload() or {}+ policy_id = payload.get('policy_id')+ run_id = payload.get('run_id')+ policy = BackupPolicy.query.get(policy_id)+ if not policy:+ raise BackupDrillError(f'backup policy {policy_id!r} not found')+ run = BackupRun.query.filter_by(id=run_id, policy_id=policy.id).first()+ if not run:+ raise BackupDrillError(f'backup run {run_id!r} not found')++ meta = run.get_metadata() or {}+ started = datetime.utcnow()+ drill = RestoreDrill(policy_id=policy.id, run_id=run.id,+ job_id=getattr(job, 'id', None), status='running',+ trigger=payload.get('trigger'), started_at=started)+ db.session.add(drill)+ db.session.commit()+ drill_id = drill.id++ # --- Free-space precheck (loud skip, never a silent pass) ---------- #+ scratch_base = cls._scratch_base()+ required = cls._estimate_required(run, meta)+ free = cls._free_bytes(scratch_base)+ drill.bytes_required = int(required)+ drill.bytes_free = int(free)+ db.session.commit()+ if free < required:+ drill.status = 'skipped_no_space'+ drill.finished_at = datetime.utcnow()+ drill.duration_seconds = int((drill.finished_at - started).total_seconds())+ drill.set_probes({'skipped': 'no_space',+ 'bytes_required': int(required), 'bytes_free': int(free)})+ cls._stamp_policy(policy.id, 'skipped_no_space')+ db.session.commit()+ logger.warning('drill %s skipped: need %s bytes, %s free',+ drill_id, required, free)+ return {'status': 'skipped_no_space', 'drill_id': drill_id,+ 'bytes_required': int(required), 'bytes_free': int(free)}++ # --- Run the leg -------------------------------------------------- #+ try:+ probes, scratch_ref, bytes_restored = cls._dispatch_drill(+ policy, run, meta, drill_id)+ drill = RestoreDrill.query.get(drill_id)+ drill.status = 'success'+ drill.scratch_ref = scratch_ref+ drill.bytes_restored = int(bytes_restored or 0)+ drill.set_probes(probes)+ drill.finished_at = datetime.utcnow()+ drill.duration_seconds = int((drill.finished_at - started).total_seconds())++ # Promote the drilled run + stamp the policy cache.+ run = BackupRun.query.get(run_id)+ run.verify_level = 'drilled'+ run.verified_at = datetime.utcnow()+ run.verify_error = None+ cls._stamp_policy(policy.id, 'success')+ db.session.commit()++ # On success the scratch dir is torn down (DB scratch already dropped).+ cls._teardown_scratch(scratch_ref)+ cls._notify_recovered(policy.id, run_id)+ return {'status': 'success', 'drill_id': drill_id, 'probes': probes}++ except Exception as exc: # noqa: BLE001 — record + re-raise+ db.session.rollback()+ drill = RestoreDrill.query.get(drill_id)+ if drill:+ drill.status = 'failed'+ drill.error = str(exc)[:2000]+ drill.finished_at = datetime.utcnow()+ drill.duration_seconds = int((drill.finished_at - started).total_seconds())+ probes = drill.get_probes()+ probes['error'] = str(exc)[:500]+ drill.set_probes(probes)+ cls._stamp_policy(policy.id, 'failed')+ db.session.commit()+ cls._notify_failed(policy.id, run_id, str(exc))+ # Scratch dirs are preserved for 24h on failure (orphan sweep reaps).+ raise++ # ------------------------------------------------------------------ #+ # Leg dispatch+ # ------------------------------------------------------------------ #++ @classmethod+ def _dispatch_drill(cls, policy, run, meta, drill_id):+ """Return ``(probes, scratch_ref, bytes_restored)`` for the target type.+ Raises on a failed/vacuous drill."""+ target_type = policy.target_type+ if target_type == 'database':+ return cls._drill_database(policy, run, meta)+ if target_type == 'wordpress_site':+ return cls._drill_wordpress(policy, run, meta, drill_id)+ # files + application both replay a tar chain into a scratch dir.+ return cls._drill_files(policy, run, meta, drill_id)++ # ------------------------------------------------------------------ #+ # Files / application leg+ # ------------------------------------------------------------------ #++ @classmethod+ def _chain_archives(cls, policy, run, meta):+ """Ordered local archive paths needed to restore ``run``. Reuses the+ policy service's chain resolver where possible."""+ try:+ from app.services.backup_policy_service import BackupPolicyService+ return BackupPolicyService._chain_archives(policy, run)+ except Exception: # noqa: BLE001 — fall back to the single primary+ primary = meta.get('primary_archive')+ if not primary or not os.path.exists(primary):+ raise BackupDrillError('Backup archive not found for drill')+ return [primary]++ @classmethod+ def _drill_files(cls, policy, run, meta, drill_id):+ import tarfile+ scratch = os.path.join(cls._scratch_base(), f'drill-{drill_id}')+ os.makedirs(scratch, exist_ok=True)++ archives = cls._chain_archives(policy, run, meta)+ for archive in archives:+ if not archive or not os.path.exists(archive):+ raise BackupDrillError(f'Backup archive missing for drill: {archive}')+ with tarfile.open(archive, 'r:*') as tar:+ tar.extractall(scratch, filter='data')++ files = []+ total_bytes = 0+ for root, _dirs, names in os.walk(scratch):+ for name in names:+ fp = os.path.join(root, name)+ if os.path.isfile(fp):+ files.append(fp)+ try:+ total_bytes += os.path.getsize(fp)+ except OSError:+ pass++ file_count = len(files)+ # Vacuous-drill detection: a drill that restored nothing is not proof.+ if file_count == 0:+ raise BackupDrillError(+ 'Drill restored 0 files — the archive is empty or unreadable.')++ sampled, sampled_ok, mismatches = cls._hash_sample(files)+ probes = {+ 'engine': meta.get('engine') or policy.target_type,+ 'file_count': file_count,+ 'bytes_restored': total_bytes,+ 'sampled': sampled,+ 'sampled_ok': sampled_ok,+ 'archives': len(archives),+ }+ if mismatches:+ probes['sample_mismatches'] = mismatches+ return probes, scratch, total_bytes++ @classmethod+ def _hash_sample(cls, files):+ """Full-hash a sample (min(50, 5%)) of the restored files. ``sampled_ok``+ counts files whose bytes hashed cleanly (proves they're actually+ readable, not just present)."""+ from app.services.backup_verify_service import sha256_file+ count = len(files)+ if count == 0:+ return 0, 0, 0+ sample_size = min(count, max(1, min(SAMPLE_MAX, math.ceil(count * SAMPLE_FRACTION))))+ # Deterministic, evenly spaced sample across the sorted file list.+ ordered = sorted(files)+ step = max(1, count // sample_size)+ picked = ordered[::step][:sample_size]+ sampled = len(picked)+ sampled_ok = 0+ for fp in picked:+ if sha256_file(fp) is not None:+ sampled_ok += 1+ return sampled, sampled_ok, sampled - sampled_ok++ # ------------------------------------------------------------------ #+ # Database leg+ # ------------------------------------------------------------------ #++ @classmethod+ def _drill_database(cls, policy, run, meta):+ """Restore the dump into a throwaway ``skdrill_<run_id>`` database on the+ shared engine, probe the table count, then DROP it in a ``finally``+ (even when the restore fails). Never touches the live database."""+ from app.services.backup_service import BackupService+ from app.services.database_service import DatabaseService++ db_type = (meta.get('db_type') or policy.target_subtype or 'mysql').lower()+ backup_path = meta.get('primary_archive') or run.storage_path+ if not backup_path or not os.path.exists(backup_path):+ raise BackupDrillError('Backup archive not found for database drill')++ scratch_name = f'skdrill_{run.id}'+ creds = cls._db_credentials(policy, meta)+ is_postgres = db_type in ('postgres', 'postgresql')++ created = False+ try:+ # 1. create scratch DB+ if is_postgres:+ res = DatabaseService.pg_create_database(scratch_name)+ else:+ res = DatabaseService.mysql_create_database(scratch_name)+ if isinstance(res, dict) and not res.get('success', True):+ raise BackupDrillError(res.get('error') or 'scratch DB create failed')+ created = True++ # 2. restore the dump into the scratch DB+ restore = BackupService.restore_database(+ backup_path=backup_path,+ db_type='postgresql' if is_postgres else 'mysql',+ db_name=scratch_name,+ user=creds.get('user'),+ password=creds.get('password'),+ host=creds.get('host', 'localhost'),+ )+ if isinstance(restore, dict) and not restore.get('success', True):+ raise BackupDrillError(restore.get('error') or 'database restore failed')++ # 3. probe: table count in the restored scratch DB+ if is_postgres:+ tables = DatabaseService.pg_get_tables(scratch_name)+ else:+ tables = DatabaseService.mysql_get_tables(scratch_name)+ table_count = len(tables or [])+ probes = {+ 'engine': 'database',+ 'db_type': 'postgresql' if is_postgres else 'mysql',+ 'scratch_db': scratch_name,+ 'table_count': table_count,+ 'tables': [cls._table_name(t) for t in (tables or [])][:100],+ }+ return probes, scratch_name, 0+ finally:+ # 4. always drop the scratch DB+ if created:+ try:+ if is_postgres:+ DatabaseService.pg_drop_database(scratch_name)+ else:+ DatabaseService.mysql_drop_database(scratch_name)+ except Exception as exc: # noqa: BLE001+ logger.warning('drill: failed to drop scratch DB %s: %s',+ scratch_name, exc)++ @staticmethod+ def _table_name(table):+ if isinstance(table, dict):+ return table.get('name') or table.get('table') or str(table)+ return str(table)++ @classmethod+ def _db_credentials(cls, policy, meta):+ """Resolve the engine credentials for the drill (reuses the policy+ service's target resolver; falls back to policy meta)."""+ try:+ from app.services.backup_policy_service import BackupPolicyService+ target = BackupPolicyService._resolve_target(policy)+ cfg = target.get('db_config') or {}+ if cfg:+ return cfg+ except Exception: # noqa: BLE001+ pass+ tmeta = policy.get_target_meta() if hasattr(policy, 'get_target_meta') else {}+ return {+ 'user': meta.get('user') or tmeta.get('user'),+ 'password': meta.get('password') or tmeta.get('password'),+ 'host': meta.get('host') or tmeta.get('host', 'localhost'),+ }++ # ------------------------------------------------------------------ #+ # WordPress leg (files drill + real scratch-DB import, plan 30)+ # ------------------------------------------------------------------ #++ @classmethod+ def _drill_wordpress(cls, policy, run, meta, drill_id):+ """A WordPress drill proves BOTH halves: extract the files archive into a+ scratch dir, AND import ``database.sql`` into a throwaway scratch DB. A+ drill that verified neither artifact fails instead of earning 'drilled'+ vacuously (plan 30)."""+ import tarfile+ from app.services.backup_service import BackupService+ from app.services.database_service import DatabaseService++ backup_dir = run.storage_path+ scratch = os.path.join(cls._scratch_base(), f'drill-{drill_id}')+ os.makedirs(scratch, exist_ok=True)++ probes = {'engine': 'wordpress_site'}+ total_bytes = 0+ proved_any = False++ # --- files half --------------------------------------------------- #+ files_archive = meta.get('primary_archive') or (+ os.path.join(backup_dir, 'files.tar.gz') if backup_dir else None)+ file_count = 0+ if files_archive and os.path.exists(files_archive):+ with tarfile.open(files_archive, 'r:*') as tar:+ tar.extractall(scratch, filter='data')+ restored = []+ for root, _dirs, names in os.walk(scratch):+ for name in names:+ fp = os.path.join(root, name)+ if os.path.isfile(fp):+ restored.append(fp)+ try:+ total_bytes += os.path.getsize(fp)+ except OSError:+ pass+ file_count = len(restored)+ if file_count:+ sampled, sampled_ok, _mm = cls._hash_sample(restored)+ probes.update({'file_count': file_count, 'sampled': sampled,+ 'sampled_ok': sampled_ok})+ proved_any = True++ # --- database half ------------------------------------------------ #+ db_sql = os.path.join(backup_dir, 'database.sql') if backup_dir else None+ if db_sql and os.path.exists(db_sql):+ scratch_name = f'skdrill_{run.id}'+ creds = cls._db_credentials(policy, meta)+ created = False+ try:+ res = DatabaseService.mysql_create_database(scratch_name)+ if isinstance(res, dict) and not res.get('success', True):+ raise BackupDrillError(res.get('error') or 'scratch DB create failed')+ created = True+ restore = BackupService.restore_database(+ backup_path=db_sql, db_type='mysql', db_name=scratch_name,+ user=creds.get('user'), password=creds.get('password'),+ host=creds.get('host', 'localhost'))+ if isinstance(restore, dict) and not restore.get('success', True):+ raise BackupDrillError(restore.get('error') or 'WP database restore failed')+ tables = DatabaseService.mysql_get_tables(scratch_name)+ probes['table_count'] = len(tables or [])+ proved_any = True+ finally:+ if created:+ try:+ DatabaseService.mysql_drop_database(scratch_name)+ except Exception as exc: # noqa: BLE001+ logger.warning('WP drill: failed to drop scratch DB %s: %s',+ scratch_name, exc)++ if not proved_any:+ raise BackupDrillError(+ 'WordPress drill verified no artifact (no files and no database).')+ probes['bytes_restored'] = total_bytes+ return probes, scratch, total_bytes++ # ------------------------------------------------------------------ #+ # Teardown / stamps / notifications+ # ------------------------------------------------------------------ #++ @staticmethod+ def _teardown_scratch(scratch_ref):+ if scratch_ref and os.path.isdir(scratch_ref):+ shutil.rmtree(scratch_ref, ignore_errors=True)++ @classmethod+ def _stamp_policy(cls, policy_id, status):+ from app.models.backup_policy import BackupPolicy+ policy = BackupPolicy.query.get(policy_id)+ if not policy:+ return+ policy.last_drill_at = datetime.utcnow()+ policy.last_drill_status = status++ @classmethod+ def _notify_failed(cls, policy_id, run_id, error):+ try:+ from app.models.backup_policy import BackupPolicy+ from app.services.backup_alert_service import BackupAlertService+ policy = BackupPolicy.query.get(policy_id)+ if policy:+ BackupAlertService.on_drill_result(policy, status='failed', error=error)+ except Exception as exc: # noqa: BLE001 — alerts are best-effort+ logger.debug('drill-failed alert skipped: %s', exc)++ @classmethod+ def _notify_recovered(cls, policy_id, run_id):+ try:+ from app.models.backup_policy import BackupPolicy+ from app.services.backup_alert_service import BackupAlertService+ policy = BackupPolicy.query.get(policy_id)+ if policy:+ BackupAlertService.on_drill_result(policy, status='success')+ except Exception as exc: # noqa: BLE001+ logger.debug('drill-recovered alert skipped: %s', exc)++ # ------------------------------------------------------------------ #+ # Orphan sweep (daily)+ # ------------------------------------------------------------------ #++ @classmethod+ def orphan_sweep(cls, job=None):+ """Reap scratch dirs (``restores/drill-*``) with no live drill row and+ older than 24h. Scratch DBs are dropped inline by each drill's finally,+ so only dirs need sweeping here."""+ from app.models.restore_drill import RestoreDrill+ base = cls._scratch_base()+ removed = 0+ live_ids = {d.id for d in RestoreDrill.query.filter_by(status='running').all()}+ cutoff = datetime.utcnow() - ORPHAN_AFTER+ try:+ entries = os.listdir(base)+ except OSError:+ entries = []+ for entry in entries:+ if not entry.startswith('drill-'):+ continue+ path = os.path.join(base, entry)+ if not os.path.isdir(path):+ continue+ try:+ drill_id = int(entry.split('-', 1)[1])+ except (ValueError, IndexError):+ drill_id = None+ if drill_id in live_ids:+ continue+ try:+ mtime = datetime.utcfromtimestamp(os.path.getmtime(path))+ except OSError:+ mtime = cutoff+ if mtime <= cutoff:+ shutil.rmtree(path, ignore_errors=True)+ removed += 1+ return {'removed': removed}++ # ------------------------------------------------------------------ #+ # Registration+ # ------------------------------------------------------------------ #++ @classmethod+ def register_jobs(cls):+ """Register the drill handlers with the job registry (app startup)."""+ from app.jobs import registry+ registry.register(DRILL_JOB_KIND, cls.run_restore_drill, replace=True)+ registry.register(DRILL_SWEEP_JOB_KIND, cls.run_drill_sweep, replace=True)+ registry.register(DRILL_ORPHAN_SWEEP_JOB_KIND, cls.orphan_sweep, replace=True)diff --git a/backend/app/services/backup_policy_service.py b/backend/app/services/backup_policy_service.pyindex ece465f2..7ff055fb 100644--- a/backend/app/services/backup_policy_service.py+++ b/backend/app/services/backup_policy_service.py@@ -920,6 +920,48 @@ class BackupPolicyService:'storage_used_human': format_bytes(storage_used),'remote_configured': BackupCostService.configured_remote_provider() is not None,'is_running': cls.is_running(policy),+ 'restore_proof': cls._restore_proof_view(policy),+ }++ # ------------------------------------------------------------------ #+ # Restore-proof view (plan 23 Phase 3) — the "last proven restore" badge+ # ------------------------------------------------------------------ #++ # A drilled policy reads as "stale" once its last successful drill is older+ # than this multiple of the cadence interval.+ _DRILL_STALE_MULT = 1.5++ @classmethod+ def _drill_badge(cls, policy):+ """One-word restore-proof state derived from the policy's drill cache:+ ``never`` (never drilled), ``failed`` (last drill failed), ``stale``+ (last success older than 1.5x the cadence interval), else ``ok``."""+ if not policy.last_drill_at:+ return 'never'+ if policy.last_drill_status == 'failed':+ return 'failed'+ from app.services.backup_drill_service import CADENCE_DAYS+ days = CADENCE_DAYS.get(policy.drill_cadence or 'off')+ if days is not None:+ threshold = timedelta(days=days * cls._DRILL_STALE_MULT)+ if (datetime.utcnow() - policy.last_drill_at) > threshold:+ return 'stale'+ return 'ok'++ @classmethod+ def _restore_proof_view(cls, policy, drill_limit=5):+ """Restore-proof projection for the protection panel: the drill cadence,+ the current badge (never/ok/stale/failed), and the most recent drill+ rows serialized."""+ from app.models.restore_drill import RestoreDrill+ recent = (RestoreDrill.query+ .filter_by(policy_id=policy.id)+ .order_by(RestoreDrill.started_at.desc())+ .limit(drill_limit).all())+ return {+ 'drill_cadence': policy.drill_cadence or 'off',+ 'badge': cls._drill_badge(policy),+ 'recent_drills': [d.to_dict() for d in recent],}# ------------------------------------------------------------------ #diff --git a/backend/app/services/backup_verify_service.py b/backend/app/services/backup_verify_service.pynew file mode 100644index 00000000..649e9e49--- /dev/null+++ b/backend/app/services/backup_verify_service.py@@ -0,0 +1,239 @@+"""Tier-1 backup verification (plan 23 Phase 1).++Every ``backup.policy.run`` writes a ``manifest.json`` next to its archives and+then runs a cheap, always-on verification pass over the produced archive(s):++ * **manifest** (:func:`write_manifest`) — file list, per-artifact sha256,+ sizes, chain refs; the sha256 of the *primary* archive is denormalized onto+ the :class:`BackupRun` row (``checksum_sha256``) so later offsite checks can+ compare against a STORED hash rather than one recomputed from the same+ possibly-corrupt local file (Decision 3).+ * **verifier** (:func:`verify_run_tier1`) — a full ``tar -t`` style listing+ (readability) plus a sha256 spot-check of the primary archive against the+ stored value, promoting the run up the ladder+ ``none → listed → hashed``. Only a real restore drill ever reaches+ ``drilled``.++Everything here is cross-platform: the listing falls back to Python's+``tarfile`` where GNU ``tar`` is unavailable (e.g. Windows dev), and the hash is+computed in Python.+"""+import hashlib+import json+import logging+import os+import tarfile+from datetime import datetime++logger = logging.getLogger(__name__)++# Bump when the manifest shape changes in a backward-incompatible way.+MANIFEST_VERSION = 1+MANIFEST_NAME = 'manifest.json'++_HASH_CHUNK = 1 << 20 # 1 MiB+++# --------------------------------------------------------------------------- #+# Hashing helpers+# --------------------------------------------------------------------------- #++def sha256_file(path):+ """Return the hex sha256 of a file, or ``None`` if it is missing/unreadable."""+ if not path or not os.path.isfile(path):+ return None+ digest = hashlib.sha256()+ try:+ with open(path, 'rb') as fh:+ for chunk in iter(lambda: fh.read(_HASH_CHUNK), b''):+ digest.update(chunk)+ except OSError:+ return None+ return digest.hexdigest()+++def _base_dir(run, meta):+ """Directory that holds a run's artifacts (where manifest.json lives)."""+ primary = meta.get('primary_archive')+ if primary:+ return os.path.dirname(primary) or (run.storage_path or '.')+ sp = run.storage_path+ if sp and os.path.isdir(sp):+ return sp+ if sp:+ return os.path.dirname(sp) or '.'+ return '.'+++# --------------------------------------------------------------------------- #+# Manifest writer+# --------------------------------------------------------------------------- #++def write_manifest(run, meta):+ """Write ``manifest.json`` next to a run's archives and denormalize the+ primary archive's sha256 onto the run row.++ Returns the manifest dict. Does NOT commit — the caller owns the session.+ """+ base = _base_dir(run, meta)+ primary = meta.get('primary_archive')+ primary_name = os.path.basename(primary) if primary else None++ artifacts = []+ if os.path.isdir(base):+ for entry in sorted(os.listdir(base)):+ if entry == MANIFEST_NAME:+ continue+ full = os.path.join(base, entry)+ if not os.path.isfile(full):+ continue+ artifacts.append({+ 'name': entry,+ 'size': os.path.getsize(full),+ 'sha256': sha256_file(full),+ })+ elif primary and os.path.isfile(primary):+ # storage_path is a single file (e.g. a database dump).+ artifacts.append({+ 'name': primary_name,+ 'size': os.path.getsize(primary),+ 'sha256': sha256_file(primary),+ })++ primary_sha256 = None+ for art in artifacts:+ if art['name'] == primary_name:+ primary_sha256 = art['sha256']+ break+ if primary_sha256 is None and primary and os.path.isfile(primary):+ primary_sha256 = sha256_file(primary)++ manifest = {+ 'version': MANIFEST_VERSION,+ 'run_id': run.id,+ 'kind': meta.get('kind') or run.kind,+ 'engine': meta.get('engine'),+ 'created_at': datetime.utcnow().isoformat() + 'Z',+ 'primary_archive': primary_name,+ 'primary_sha256': primary_sha256,+ 'artifacts': artifacts,+ 'totals': {+ 'count': len(artifacts),+ 'bytes': sum(a['size'] for a in artifacts),+ },+ 'chain': {+ 'incremental': bool(meta.get('incremental')),+ 'full_run_id': meta.get('full_run_id'),+ },+ 'tool_versions': _tool_versions(),+ }++ # Denormalize the primary hash onto the run (Decision 3).+ if primary_sha256:+ run.checksum_sha256 = primary_sha256++ if os.path.isdir(base):+ try:+ with open(os.path.join(base, MANIFEST_NAME), 'w', encoding='utf-8') as fh:+ json.dump(manifest, fh, indent=2)+ except OSError as exc: # best-effort — the row still carries the hash+ logger.warning('could not write manifest for run %s: %s', run.id, exc)++ return manifest+++def read_manifest(run, meta=None):+ """Load a run's on-disk manifest, or ``None`` if absent/unreadable."""+ meta = meta if meta is not None else (run.get_metadata() or {})+ base = _base_dir(run, meta)+ path = os.path.join(base, MANIFEST_NAME)+ if not os.path.isfile(path):+ return None+ try:+ with open(path, 'r', encoding='utf-8') as fh:+ return json.load(fh)+ except (OSError, ValueError):+ return None+++def _tool_versions():+ versions = {'python_tarfile': True}+ try:+ from app.utils.system import is_command_available+ versions['gnu_tar'] = bool(is_command_available('tar'))+ except Exception: # noqa: BLE001+ pass+ return versions+++# --------------------------------------------------------------------------- #+# Post-run verifier (Tier 1)+# --------------------------------------------------------------------------- #++def _list_archive(path):+ """Return ``(ok, member_count, error)`` for a tar/tar.gz archive, using+ Python's ``tarfile`` so it works cross-platform."""+ try:+ with tarfile.open(path, 'r:*') as tar:+ members = tar.getmembers()+ return True, len(members), None+ except Exception as exc: # noqa: BLE001 — any read failure means unreadable+ return False, 0, str(exc)+++def _stored_sha256(run, meta):+ if getattr(run, 'checksum_sha256', None):+ return run.checksum_sha256+ manifest = read_manifest(run, meta)+ if manifest and manifest.get('primary_sha256'):+ return manifest['primary_sha256']+ return (meta or {}).get('primary_sha256')+++def verify_run_tier1(run, meta):+ """Verify a freshly produced run and set its verify ladder position.++ Ladder: ``none`` (archive unreadable) → ``listed`` (readable but checksum+ unverified/mismatch) → ``hashed`` (checksum matches the stored manifest).++ Mutates ``run.verify_level`` / ``run.verify_error`` / ``run.verified_at``+ (does NOT commit) and returns a probes dict.+ """+ primary = meta.get('primary_archive')+ probes = {'listed': [], 'hashed': False, 'sha256': None}+ run.verified_at = datetime.utcnow()++ if not primary or not os.path.exists(primary):+ run.verify_level = 'none'+ run.verify_error = 'Primary archive not found on disk (not readable).'+ probes['listed'].append({'name': os.path.basename(primary) if primary else None,+ 'ok': False, 'error': 'missing'})+ return probes++ ok, member_count, err = _list_archive(primary)+ probes['listed'].append({+ 'name': os.path.basename(primary), 'ok': ok,+ 'members': member_count, 'error': err,+ })+ if not ok:+ run.verify_level = 'none'+ run.verify_error = f'Primary archive is not readable: {(err or "")[:200]}'+ return probes++ stored = _stored_sha256(run, meta)+ actual = sha256_file(primary)+ probes['sha256'] = actual++ if stored and actual and actual == stored:+ run.verify_level = 'hashed'+ run.verify_error = None+ probes['hashed'] = True+ else:+ run.verify_level = 'listed'+ probes['hashed'] = False+ if stored and actual and actual != stored:+ run.verify_error = (f'Checksum mismatch: expected {stored[:12]}…, '+ f'got {actual[:12]}…')+ else:+ run.verify_error = 'No stored checksum available to compare against.'+ return probesdiff --git a/backend/app/services/setup_health_service.py b/backend/app/services/setup_health_service.pynew file mode 100644index 00000000..46edf6e6--- /dev/null+++ b/backend/app/services/setup_health_service.py@@ -0,0 +1,575 @@+"""Setup Health registry — "how set up is this panel" (plan 22).++A small, cheap registry of setup-health *items* the operator should complete to+get a fully-working panel. Each item is a doctor-style check+(``key``/``title``/``status``/``detail``/``repairable``/``repair_ref``) plus the+setup extras ``section``/``severity``/``scope``/``why``/``fix`` so the same items+render in the Setup card, in Monitoring → Doctor's ``setup.*`` section, and in the+personal "secure your account" nudge.++Severity is graded by Decision 2's *silent-breakage* rule: an item is+``critical`` only when leaving it undone silently breaks something the operator+already set in motion (e.g. per-site DNS mode with no provider/IP — sites won't+resolve). Everything else is ``recommended``. **HTTPS/TLS items are never+critical** — SSL is optional by decree.++Scope is ``panel`` (a whole-instance setting, admin-only) or ``user`` (a personal+item like account 2FA). Panel snoozes live in a SettingsService map; personal+snoozes live on ``users.setup_snoozes``. A snoozed item still renders, just muted+— it drops out of the open counts and never triggers the nag.++The weekly nag (Phase 6) fires ``setup.incomplete`` to admins only while+non-snoozed critical items exist AND the critical set changed since the last+notice (fingerprint throttle); the marker resets once the criticals clear.+"""+import json+import logging+import os+import sys+from datetime import datetime, timedelta++logger = logging.getLogger(__name__)++# ── Phase-6 constants (imported by the API + the nag test) ───────────────────+NAG_JOB_KIND = 'setup.health.nag'+NAG_SCHEDULE_NAME = 'setup-health-nag'+NAG_MARKER_KEY = 'setup_health_nag_marker'+PANEL_SNOOZE_KEY = 'setup_health_panel_snoozes'++# Weekly cadence (seconds) for the nag schedule.+NAG_INTERVAL_SECONDS = 604800++# Item severities.+CRITICAL = 'critical'+RECOMMENDED = 'recommended'++# Every setup item key mapped to its scope. Drives _item_scope() (snooze RBAC)+# and lets the API reject an unknown key before touching storage.+_ITEM_SCOPES = {+ 'setup.public_ip': 'panel',+ 'setup.base_domain': 'panel',+ 'setup.dns_provider': 'panel',+ 'setup.email_delivery': 'panel',+ 'setup.backup_policy': 'panel',+ 'setup.backup_offsite': 'panel',+ 'setup.canonical_domain': 'panel',+ 'setup.wildcard_cert': 'panel',+ 'setup.account_security': 'user',+}+++def _link(to):+ """A deep-link fix descriptor pointing the operator at where to finish."""+ return {'kind': 'link', 'to': to}+++def _item(key, title, status, detail, severity, fix, scope='panel', why=None,+ repairable=False, repair_ref=None):+ """Build a normalized setup-health item (doctor ``_check`` shape + extras)."""+ return {+ # doctor _check shape+ 'key': key,+ 'title': title,+ 'status': status, # 'ok' | 'warn' | 'fail'+ 'detail': detail,+ 'repairable': bool(repairable),+ 'repair_ref': repair_ref,+ # setup extras+ 'section': 'setup',+ 'severity': severity, # 'critical' | 'recommended'+ 'scope': scope, # 'panel' | 'user'+ 'why': why or detail,+ 'fix': fix,+ }+++class SetupHealthService:+ """Evaluate the setup-health registry, snooze/unsnooze items, run the nag."""++ # ------------------------------------------------------------------ #+ # Small settings helpers (read raw settings, no dev fallbacks — an item+ # like base_domain must reflect what the operator actually set, not the+ # SITES_BASE_DOMAIN dev default).+ # ------------------------------------------------------------------ #++ @staticmethod+ def _get(key, default=None):+ from app.models.system_settings import SystemSettings+ val = SystemSettings.get(key, default)+ return val if val is not None else default++ @classmethod+ def _base_domain(cls):+ return (cls._get('sites_base_domain', '') or '').strip().lower()++ @classmethod+ def _dns_mode(cls):+ return (cls._get('sites_dns_mode', '') or '').strip().lower()++ @classmethod+ def _server_ip(cls):+ return (cls._get('server_public_ip', '') or '').strip()++ @classmethod+ def _https_enabled(cls):+ return bool(cls._get('sites_https_enabled', False))++ # ------------------------------------------------------------------ #+ # Item scope + snooze storage+ # ------------------------------------------------------------------ #++ @classmethod+ def _item_scope(cls, key):+ """'panel' | 'user' for a known item key, else None."""+ return _ITEM_SCOPES.get(key)++ @staticmethod+ def _panel_snoozes():+ """The panel snooze map ``{key: iso_expiry}`` from the settings map."""+ from app.services.settings_service import SettingsService+ raw = SettingsService.get(PANEL_SNOOZE_KEY)+ if not raw:+ return {}+ try:+ data = json.loads(raw)+ return data if isinstance(data, dict) else {}+ except (ValueError, TypeError):+ return {}++ @staticmethod+ def _user_snoozes(user):+ if not user or not getattr(user, 'setup_snoozes', None):+ return {}+ try:+ data = json.loads(user.setup_snoozes)+ return data if isinstance(data, dict) else {}+ except (ValueError, TypeError):+ return {}++ @staticmethod+ def _active_until(snooze_map, key, now):+ """The (non-expired) snooze expiry for ``key``, or None."""+ iso = (snooze_map or {}).get(key)+ if not iso:+ return None+ try:+ expiry = datetime.fromisoformat(iso)+ except (ValueError, TypeError):+ return None+ return iso if expiry > now else None++ # ------------------------------------------------------------------ #+ # The probe matrix — one builder per item+ # ------------------------------------------------------------------ #++ @classmethod+ def _panel_items(cls):+ base = cls._base_domain()+ mode = cls._dns_mode()+ ip = cls._server_ip()+ per_site = bool(base) and mode == 'per-site'+ items = []++ # --- server public IP ---+ if ip:+ items.append(_item(+ 'setup.public_ip', 'Server public IP', 'ok',+ f'Server public IP is set ({ip}).', RECOMMENDED,+ _link('/settings')))+ elif per_site:+ items.append(_item(+ 'setup.public_ip', 'Server public IP', 'fail',+ 'DNS mode is per-site but no server public IP is set, so each '+ "managed site's A record can't be auto-created — new sites won't "+ 'resolve.', CRITICAL, _link('/settings')))+ else:+ items.append(_item(+ 'setup.public_ip', 'Server public IP', 'warn',+ 'No server public IP set. It is used to auto-create A records for '+ 'managed domains.', RECOMMENDED, _link('/settings')))++ # --- managed-sites base domain ---+ if base:+ items.append(_item(+ 'setup.base_domain', 'Managed-sites base domain', 'ok',+ f'Managed sites are published under {base}.', RECOMMENDED,+ _link('/settings')))+ else:+ items.append(_item(+ 'setup.base_domain', 'Managed-sites base domain', 'warn',+ 'No base domain set — new sites are only reachable at '+ 'localhost:<port>. Set one so each site publishes at '+ '<name>.<domain>.', RECOMMENDED, _link('/settings')))++ # --- DNS provider ---+ has_provider = cls._has_dns_provider()+ if has_provider:+ items.append(_item(+ 'setup.dns_provider', 'DNS provider connected', 'ok',+ 'A DNS provider is connected for auto-managing records.',+ RECOMMENDED, _link('/settings/connections')))+ elif per_site:+ items.append(_item(+ 'setup.dns_provider', 'DNS provider connected', 'fail',+ 'DNS mode is per-site but no DNS provider is connected, so each '+ "site's A record can't be auto-created — new sites won't resolve.",+ CRITICAL, _link('/settings/connections')))+ else:+ items.append(_item(+ 'setup.dns_provider', 'DNS provider connected', 'warn',+ 'No DNS provider connected. Connect one to auto-create and heal '+ 'DNS records for managed domains.', RECOMMENDED,+ _link('/settings/connections')))++ # --- email delivery ---+ items.append(cls._email_delivery_item())++ # --- backup policy ---+ items.append(cls._backup_policy_item())++ # --- offsite backup ---+ items.append(cls._backup_offsite_item())++ # --- canonical panel domain ---+ canonical = (cls._get('canonical_domain', '') or '').strip()+ if canonical:+ items.append(_item(+ 'setup.canonical_domain', 'Canonical panel domain', 'ok',+ f'Panel canonical domain is set ({canonical}).', RECOMMENDED,+ _link('/settings')))+ else:+ items.append(_item(+ 'setup.canonical_domain', 'Canonical panel domain', 'warn',+ 'No canonical panel domain set. Setting one gives the panel a '+ 'stable public URL for agent installs and CORS.', RECOMMENDED,+ _link('/settings')))++ # --- wildcard certificate (only once wildcard HTTPS is switched on;+ # never critical — SSL is optional by decree) ---+ cert_item = cls._wildcard_cert_item(base, mode)+ if cert_item is not None:+ items.append(cert_item)++ return items++ @classmethod+ def _has_dns_provider(cls):+ try:+ from app.models.email import DNSProviderConfig+ return DNSProviderConfig.query.count() > 0+ except Exception: # noqa: BLE001+ return False++ @classmethod+ def _email_delivery_item(cls):+ try:+ from app.models.email_provider import EmailProviderConnection+ providers = EmailProviderConnection.query.filter_by(+ is_active=True, uses_notifications=True).all()+ except Exception: # noqa: BLE001+ providers = []+ tested = [p for p in providers if p.last_test_ok]+ if tested:+ return _item(+ 'setup.email_delivery', 'Email delivery', 'ok',+ 'A verified email delivery path is configured.', RECOMMENDED,+ _link('/settings/connections'))+ if providers:+ return _item(+ 'setup.email_delivery', 'Email delivery', 'warn',+ 'An email provider is configured but its test send has not '+ 'passed — notifications may not be delivered.', RECOMMENDED,+ _link('/settings/connections'))+ return _item(+ 'setup.email_delivery', 'Email delivery', 'warn',+ 'No email delivery path configured — the panel cannot send '+ 'notifications or invitations.', RECOMMENDED,+ _link('/settings/connections'))++ @classmethod+ def _backup_policy_item(cls):+ try:+ from app.models.backup_policy import BackupPolicy+ enabled = BackupPolicy.query.filter_by(enabled=True).count() > 0+ except Exception: # noqa: BLE001+ enabled = False+ if enabled:+ return _item(+ 'setup.backup_policy', 'Backups scheduled', 'ok',+ 'At least one enabled backup policy is protecting a target.',+ RECOMMENDED, _link('/backups'))+ return _item(+ 'setup.backup_policy', 'Backups scheduled', 'warn',+ 'No enabled backup policy — nothing is being backed up on a '+ 'schedule.', RECOMMENDED, _link('/backups'))++ @classmethod+ def _backup_offsite_item(cls):+ provider = None+ try:+ from app.services.storage_provider_service import StorageProviderService+ cfg = StorageProviderService.get_config() or {}+ provider = cfg.get('provider')+ except Exception: # noqa: BLE001+ provider = None+ if provider and provider != 'local':+ return _item(+ 'setup.backup_offsite', 'Offsite backup storage', 'ok',+ f'Offsite backup storage is configured ({provider}).',+ RECOMMENDED, _link('/settings/storage'))+ return _item(+ 'setup.backup_offsite', 'Offsite backup storage', 'warn',+ 'Backups are kept on this server only. Configure S3/B2 offsite '+ 'storage so a backup survives losing the box.', RECOMMENDED,+ _link('/settings/storage'))++ @classmethod+ def _wildcard_cert_item(cls, base, mode):+ """The wildcard-cert item — only applicable once wildcard HTTPS is on for+ a wildcard base. Never critical (HTTPS is optional)."""+ if not base or mode != 'wildcard' or not cls._https_enabled():+ return None+ exists = True+ if sys.platform.startswith('linux'):+ try:+ from app.services.site_domain_service import SiteDomainService+ cert, _key = SiteDomainService.wildcard_cert_paths(base)+ exists = bool(cert and os.path.exists(cert))+ except Exception: # noqa: BLE001+ exists = True+ if exists:+ return _item(+ 'setup.wildcard_cert', 'Wildcard HTTPS certificate', 'ok',+ f'A wildcard certificate for *.{base} is present.', RECOMMENDED,+ _link('/settings'))+ return _item(+ 'setup.wildcard_cert', 'Wildcard HTTPS certificate', 'warn',+ f'Wildcard HTTPS is enabled for *.{base} but its certificate is not '+ 'present yet — managed subdomains fall back to HTTP until it issues.',+ RECOMMENDED, _link('/settings'))++ @classmethod+ def _user_items(cls, user):+ if not user:+ return []+ has_factor = bool(getattr(user, 'totp_enabled', False))+ if not has_factor:+ try:+ has_factor = user.passkeys.filter_by(is_active=True).count() > 0+ except Exception: # noqa: BLE001+ has_factor = False+ if has_factor:+ item = _item(+ 'setup.account_security', 'Secure your account', 'ok',+ 'Your account has a second factor (passkey or authenticator).',+ RECOMMENDED, _link('/settings/security'), scope='user')+ else:+ item = _item(+ 'setup.account_security', 'Secure your account', 'warn',+ 'Add a passkey or authenticator app so a stolen password alone '+ "can't sign in as you.", RECOMMENDED,+ _link('/settings/security'), scope='user')+ return [item]++ # ------------------------------------------------------------------ #+ # Evaluate+ # ------------------------------------------------------------------ #++ @classmethod+ def evaluate(cls, scope='panel', user=None):+ """Evaluate the registry for ``scope`` ('panel' | 'user'), applying+ snoozes, and return ``{'items': [...], 'summary': {...}}``."""+ now = datetime.utcnow()+ if scope == 'user':+ items = cls._user_items(user)+ snooze_map = cls._user_snoozes(user)+ else:+ items = cls._panel_items()+ snooze_map = cls._panel_snoozes()++ for c in items:+ until = cls._active_until(snooze_map, c['key'], now)+ if until:+ c['snoozed'] = True+ c['snoozed_until'] = until+ else:+ c['snoozed'] = False+ c['snoozed_until'] = None++ return {'items': items, 'summary': cls._summary(items)}++ @staticmethod+ def _summary(items):+ weights = {CRITICAL: 3, RECOMMENDED: 1}+ ok = critical_open = recommended_open = snoozed = 0+ earned = total_weight = 0+ for c in items:+ w = weights.get(c['severity'], 1)+ total_weight += w+ is_ok = c['status'] == 'ok'+ is_snoozed = bool(c.get('snoozed'))+ if is_ok or is_snoozed:+ earned += w+ if is_ok:+ ok += 1+ elif is_snoozed:+ snoozed += 1+ elif c['severity'] == CRITICAL:+ critical_open += 1+ else:+ recommended_open += 1+ score = round(100 * earned / total_weight) if total_weight else 100+ return {+ 'total': len(items),+ 'ok': ok,+ 'critical_open': critical_open,+ 'recommended_open': recommended_open,+ 'snoozed': snoozed,+ 'score': max(0, min(100, score)),+ }++ # ------------------------------------------------------------------ #+ # Snooze / unsnooze+ # ------------------------------------------------------------------ #++ @classmethod+ def snooze(cls, key, days=30, user=None):+ """Snooze a setup item for ``days`` (mutes it). Panel items store into the+ settings map; personal items store on the user row. Returns a descriptor+ dict, or ``{'error': ...}`` for an unknown item."""+ scope = cls._item_scope(key)+ if scope is None:+ return {'error': f'Unknown setup item: {key}'}+ try:+ days = int(days)+ except (TypeError, ValueError):+ days = 30+ until = (datetime.utcnow() + timedelta(days=days)).isoformat()++ if scope == 'user':+ if user is None:+ return {'error': 'A user is required to snooze a personal item.'}+ from app import db+ snoozes = cls._user_snoozes(user)+ snoozes[key] = until+ user.setup_snoozes = json.dumps(snoozes)+ db.session.commit()+ else:+ from app.services.settings_service import SettingsService+ snoozes = cls._panel_snoozes()+ snoozes[key] = until+ SettingsService.set(PANEL_SNOOZE_KEY, json.dumps(snoozes))+ return {'success': True, 'key': key, 'scope': scope, 'snoozed_until': until}++ @classmethod+ def unsnooze(cls, key, user=None):+ """Clear a snooze so the item is active again."""+ scope = cls._item_scope(key)+ if scope is None:+ return {'error': f'Unknown setup item: {key}'}+ if scope == 'user':+ if user is None:+ return {'error': 'A user is required to unsnooze a personal item.'}+ from app import db+ snoozes = cls._user_snoozes(user)+ snoozes.pop(key, None)+ user.setup_snoozes = json.dumps(snoozes)+ db.session.commit()+ else:+ from app.services.settings_service import SettingsService+ snoozes = cls._panel_snoozes()+ snoozes.pop(key, None)+ SettingsService.set(PANEL_SNOOZE_KEY, json.dumps(snoozes))+ return {'success': True, 'key': key, 'scope': scope}++ # ------------------------------------------------------------------ #+ # Fingerprint + nag+ # ------------------------------------------------------------------ #++ @staticmethod+ def _critical_fingerprint(result):+ """A stable, human-readable fingerprint of the non-snoozed critical open+ items (the comma-joined sorted keys). Empty string when none."""+ keys = sorted(+ c['key'] for c in result['items']+ if c['severity'] == CRITICAL and c['status'] != 'ok'+ and not c.get('snoozed'))+ return ','.join(keys)++ @classmethod+ def fingerprint(cls):+ """The current non-snoozed-critical fingerprint (panel scope)."""+ return cls._critical_fingerprint(cls.evaluate(scope='panel'))++ @classmethod+ def run_nag_job(cls, job):+ """Weekly nag handler: notify admins once per critical-set change while+ non-snoozed critical items exist; reset the marker once they clear."""+ from app.services.settings_service import SettingsService+ result = cls.evaluate(scope='panel')+ critical_open = result['summary']['critical_open']+ marker = SettingsService.get(NAG_MARKER_KEY) or ''++ if critical_open == 0:+ if marker:+ SettingsService.set(NAG_MARKER_KEY, '')+ return {'notified': False, 'critical_open': 0}++ fingerprint = cls._critical_fingerprint(result)+ if fingerprint == marker:+ return {'notified': False, 'reason': 'unchanged',+ 'critical_open': critical_open}++ try:+ from app.plugins_sdk import notify+ notify.send(+ 'setup.incomplete', to='admins',+ data={+ 'count': critical_open,+ 'summary': f'{critical_open} critical setup item(s) need '+ 'attention.',+ 'message': f'{critical_open} critical setup item(s) are '+ 'incomplete. Finish them in Monitoring → Doctor '+ '(Setup section).',+ })+ except Exception as e: # noqa: BLE001 — a nag must never break the job+ logger.warning('setup-health nag notify failed: %s', e)++ SettingsService.set(NAG_MARKER_KEY, fingerprint)+ return {'notified': True, 'critical_open': critical_open}++ # ------------------------------------------------------------------ #+ # Doctor integration + job plumbing+ # ------------------------------------------------------------------ #++ @classmethod+ def doctor_checks(cls):+ """The panel setup items as doctor ``setup.*`` checks, for inclusion in+ the doctor report's Setup section (Monitoring → Doctor)."""+ return cls.evaluate(scope='panel')['items']++ @classmethod+ def register_jobs(cls):+ """Register the nag handler with the job registry. Called once at app+ startup (see app/__init__.py)."""+ from app.jobs import registry+ registry.register(NAG_JOB_KIND, cls.run_nag_job, replace=True)+++def _register_catalog_events():+ """Register the ``setup.incomplete`` event so the nag renders through the same+ pipeline as any core notification. Safe + idempotent at import time."""+ try:+ from app.notifications import catalog+ catalog.register(+ 'setup.incomplete',+ title='Finish setting up ServerKit',+ template='generic',+ severity='warning',+ category='system',+ )+ except Exception as e: # noqa: BLE001+ logger.debug('could not register setup.incomplete catalog event: %s', e)+++_register_catalog_events()diff --git a/backend/app/services/setup_reconcile_service.py b/backend/app/services/setup_reconcile_service.pynew file mode 100644index 00000000..d4d11ec5--- /dev/null+++ b/backend/app/services/setup_reconcile_service.py@@ -0,0 +1,371 @@+"""Reconcile-on-connect (plan 22 Phase 3).++When the operator finally connects a DNS provider / sets a server IP / picks a+base domain, the sites they already created are still pointing at nothing. This+service reconciles that backlog explicitly (never auto-applied — Decision 3):++* **DNS backfill** — enumerate the managed hosts that need an A record given the+ current base/mode, preview which a connected provider would cover, then loop+ ``DNSProviderService.ensure_a_record`` over them. Idempotent: an existing+ record returns ``created: False`` and is left untouched.+* **WordPress URL-fix** — detect WordPress sites whose live siteurl/home still+ points at ``localhost``/a stale IP and swap it to the site's real managed URL+ via the URL-swap rails (keeping the old-host redirect). A site already on its+ target URL is not detected, so a second run is a no-op.++Both apply loops are exposed as job handlers so a large backfill/heal runs off+the request thread (the API enqueues; see ``app/api/setup_health.py``).+"""+import ipaddress+import logging+from urllib.parse import urlparse++logger = logging.getLogger(__name__)++DNS_BACKFILL_JOB_KIND = 'setup.reconcile.dns_backfill'+URL_FIX_JOB_KIND = 'setup.reconcile.url_fix'++# Loopback / local host labels that a live site URL must be swapped away from.+_LOCAL_HOSTS = {'localhost', '127.0.0.1', '0.0.0.0', '::1'}++# Reserved / dev suffixes whose hosts never get a public DNS record (the doctor's+# own filter — bare IPs and no-dot single labels are handled separately).+_DEV_SUFFIXES = ('.lvh.me', '.local', '.test', '.internal', '.localhost')+++class SetupReconcileService:+ """DNS backfill + WordPress URL-fix for apps adopted before setup finished."""++ # ------------------------------------------------------------------ #+ # Host classification+ # ------------------------------------------------------------------ #++ @staticmethod+ def _is_ip(host):+ try:+ ipaddress.ip_address(host)+ return True+ except ValueError:+ return False++ @classmethod+ def _skip_dns_host(cls, host):+ """Dev/loopback names and bare IPs are skipped (the doctor's own filter)."""+ h = (host or '').strip().lower().rstrip('.')+ if not h or '.' not in h:+ return True+ if cls._is_ip(h):+ return True+ if h == 'localhost' or any(h.endswith(suf) for suf in _DEV_SUFFIXES):+ return True+ return False++ # ------------------------------------------------------------------ #+ # DNS backfill — candidate enumeration+ # ------------------------------------------------------------------ #++ @classmethod+ def _dns_candidates(cls):+ """The managed hosts that need a DNS record, by mode:++ * a base in ``wildcard`` mode → a single ``*.<base>`` candidate (the+ per-site subdomains are covered by it, so they aren't separate),+ * a base in ``per-site`` mode → each subdomain as its own candidate,+ * a domain outside every base → a ``custom`` candidate.++ Dev/loopback names and bare IPs are skipped.+ """+ from app.models.domain import Domain+ from app.services.site_domain_service import SiteDomainService++ candidates = []+ seen = set()+ for domain in Domain.query.order_by(Domain.name.asc()).all():+ host = (domain.name or '').strip().lower()+ if cls._skip_dns_host(host):+ continue+ base = SiteDomainService.covering_base(host)+ if base:+ mode = SiteDomainService.dns_mode(base)+ if mode == 'wildcard':+ wildcard = f'*.{base}'+ if wildcard in seen:+ continue+ seen.add(wildcard)+ candidates.append({'host': wildcard, 'mode': 'wildcard',+ 'base': base})+ else: # per-site+ if host in seen:+ continue+ seen.add(host)+ candidates.append({'host': host, 'mode': 'per-site',+ 'base': base})+ else:+ if host in seen:+ continue+ seen.add(host)+ candidates.append({'host': host, 'mode': 'custom', 'base': None})+ return candidates++ @staticmethod+ def _provider_covers_zone(host):+ """Whether a connected provider authoritatively covers ``host``'s zone.+ Never raises (provider listing hits the network)."""+ try:+ from app.services.dns_provider_service import DNSProviderService+ config, _zone = DNSProviderService.find_zone_for_domain(host)+ return bool(config)+ except Exception: # noqa: BLE001+ return False++ @classmethod+ def dns_backfill_preview(cls):+ """Dry-run: the managed hosts a backfill would ensure an A record for, and+ whether a connected provider covers each host's zone. Never writes."""+ from app.models.email import DNSProviderConfig+ from app.services.site_domain_service import SiteDomainService++ server_ip = SiteDomainService.server_ip()+ has_provider = DNSProviderConfig.query.count() > 0+ ready = bool(server_ip)++ items = []+ for cand in cls._dns_candidates():+ # Only probe provider coverage when the backfill could actually run —+ # avoids a network round-trip when there's nothing to apply.+ covered = (cls._provider_covers_zone(cand['host'])+ if (has_provider and ready) else False)+ items.append({+ 'host': cand['host'],+ 'mode': cand['mode'],+ 'target_ip': server_ip,+ 'provider_covers_zone': covered,+ })++ return {+ 'ready': ready,+ 'has_provider': has_provider,+ 'server_ip': server_ip,+ 'items': items,+ 'count': len(items),+ 'total': len(items),+ }++ @classmethod+ def dns_backfill_apply(cls):+ """Loop ``ensure_a_record`` over every candidate. Idempotent — an existing+ record returns ``created: False`` and is left untouched. Returns per-host+ results + a created count."""+ from app.services.dns_provider_service import DNSProviderService+ from app.services.site_domain_service import SiteDomainService++ server_ip = SiteDomainService.server_ip()+ if not server_ip:+ return {'applied': 0, 'results': [], 'total': 0,+ 'error': 'No server public IP set.'}++ candidates = cls._dns_candidates()+ results = []+ applied = 0+ for cand in candidates:+ host = cand['host']+ try:+ res = DNSProviderService.ensure_a_record(host, server_ip)+ except Exception as e: # noqa: BLE001+ res = {'created': False, 'reason': 'error', 'error': str(e)}+ created = bool(res.get('created'))+ if created:+ applied += 1+ row = {'host': host, 'mode': cand['mode']}+ row.update(res)+ row['created'] = created+ results.append(row)++ return {'applied': applied, 'results': results, 'total': len(candidates)}++ @classmethod+ def run_dns_backfill_job(cls, job):+ """Job handler for ``setup.reconcile.dns_backfill``."""+ return cls.dns_backfill_apply()++ # ------------------------------------------------------------------ #+ # WordPress url-fix — classifier+ # ------------------------------------------------------------------ #++ @staticmethod+ def _host_of(url):+ try:+ return (urlparse(url).hostname or '').strip().lower()+ except Exception: # noqa: BLE001+ return ''++ @classmethod+ def _is_local_host(cls, host):+ """A loopback/local label or a bare IP — not a real public host."""+ if not host:+ return False+ if host in _LOCAL_HOSTS:+ return True+ if cls._is_ip(host):+ return True+ if '.' not in host: # a bare single label (e.g. 'localhost')+ return True+ return False++ @classmethod+ def _is_real_domain(cls, host):+ return bool(host) and '.' in host and not cls._is_ip(host) \+ and host not in _LOCAL_HOSTS++ @classmethod+ def url_fix_needed(cls, current_url, target_url):+ """Pure classifier (testable without wp-cli): does a site at+ ``current_url`` need swapping to ``target_url``?++ Needs a fix when the current host is a local/loopback name or a bare IP+ AND there is a real domain target that differs. A current host that is+ already a proper domain (== target or otherwise) is left alone.+ """+ if not current_url or not target_url:+ return False+ current = cls._host_of(current_url)+ target = cls._host_of(target_url)+ if not current or not target:+ return False+ if not cls._is_real_domain(target):+ return False+ if not cls._is_local_host(current):+ return False+ return current != target++ # ------------------------------------------------------------------ #+ # WordPress url-fix — site enumeration + target resolution+ # ------------------------------------------------------------------ #++ @classmethod+ def _wp_sites(cls):+ """Production WordPress sites with a root path, or [] when the WordPress+ extension isn't loadable (dev without it, minimal test env)."""+ try:+ from app.models.wordpress_site import WordPressSite+ except Exception: # noqa: BLE001+ return []+ try:+ sites = WordPressSite.query.all()+ except Exception: # noqa: BLE001+ return []+ return [s for s in sites+ if s.application and getattr(s.application, 'root_path', None)]++ @classmethod+ def _target_url_for(cls, app):+ """The real managed URL a site should serve under — its primary domain,+ else None (nothing to point at)."""+ from app.models.domain import Domain+ from app.services.site_domain_service import SiteDomainService++ domain = Domain.query.filter_by(+ application_id=app.id, is_primary=True).first()+ if not domain or not domain.name:+ return None+ host = domain.name.strip().lower()++ ssl = bool(getattr(domain, 'ssl_enabled', False))+ if not ssl:+ try:+ base = SiteDomainService.covering_base(host)+ ssl = bool(base and SiteDomainService.https_enabled(base)+ and SiteDomainService.covers(host))+ except Exception: # noqa: BLE001+ ssl = False+ scheme = 'https' if ssl else 'http'+ return f'{scheme}://{host}'++ @classmethod+ def url_fix_preview(cls):+ """Detect WordPress sites whose live URL is still localhost/a stale IP and+ show the per-site swap (current → target) via the URL-swap preview rails.+ Never mutates."""+ import app.services.wordpress_bridge as bridge+ try:+ wp = bridge.wordpress_service()+ except Exception as e: # noqa: BLE001+ return {'count': 0, 'items': [], 'warning': str(e)}++ items = []+ for site in cls._wp_sites():+ app = site.application+ target = cls._target_url_for(app)+ if not target:+ continue+ try:+ current = wp._site_current_url(app)+ except Exception: # noqa: BLE001+ continue+ if not cls.url_fix_needed(current, target):+ continue+ entry = {'site_id': site.id, 'current_url': current, 'new_url': target}+ try:+ preview = wp.preview_url_change(app, target)+ entry['pairs'] = preview.get('pairs')+ entry['total'] = preview.get('total')+ entry['success'] = preview.get('success', True)+ except Exception as e: # noqa: BLE001+ entry['warning'] = str(e)+ items.append(entry)++ return {'count': len(items), 'items': items}++ @classmethod+ def url_fix_apply(cls):+ """Apply the URL swap to every detected site via ``change_site_url``+ (keeps the old-host redirect). Idempotent — a site already on its target+ URL is not detected, so a second run is a no-op."""+ import app.services.wordpress_bridge as bridge+ try:+ wp = bridge.wordpress_service()+ except Exception as e: # noqa: BLE001+ return {'fixed': 0, 'results': [], 'error': str(e)}++ results = []+ fixed = 0+ for site in cls._wp_sites():+ app = site.application+ target = cls._target_url_for(app)+ if not target:+ continue+ try:+ current = wp._site_current_url(app)+ except Exception: # noqa: BLE001+ continue+ if not cls.url_fix_needed(current, target):+ continue+ try:+ res = wp.change_site_url(app, target, keep_old_redirect=True)+ except Exception as e: # noqa: BLE001+ res = {'success': False, 'error': str(e)}+ if res.get('success'):+ fixed += 1+ results.append({'site_id': site.id, 'new_url': target,+ 'old_url': res.get('old_url', current),+ 'success': bool(res.get('success'))})++ return {'fixed': fixed, 'results': results}++ @classmethod+ def run_url_fix_job(cls, job):+ """Job handler for ``setup.reconcile.url_fix``."""+ return cls.url_fix_apply()++ # ------------------------------------------------------------------ #+ # Job registration+ # ------------------------------------------------------------------ #++ @classmethod+ def register_jobs(cls):+ """Register both reconcile handlers with the job registry (called once at+ app startup, see app/__init__.py)."""+ from app.jobs import registry+ registry.register(DNS_BACKFILL_JOB_KIND, cls.run_dns_backfill_job,+ replace=True)+ registry.register(URL_FIX_JOB_KIND, cls.run_url_fix_job, replace=True)