ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
feat(backup): trust round 2 — restore drills, offsite/verify (plan 30)
Juan Denis7d045f32af9a
diff --git a/backend/app/models/backup_policy.py b/backend/app/models/backup_policy.pyindex 6ba08979..4fd54e76 100644--- a/backend/app/models/backup_policy.py+++ b/backend/app/models/backup_policy.py@@ -60,6 +60,13 @@ class BackupPolicy(db.Model):# Remoteremote_copy = db.Column(db.Boolean, default=False, nullable=False)+ # Restore-drill cadence (plan 23). 'off' | 'weekly' | 'monthly'. A single+ # daily sweep enqueues due drills; last_drill_* is the denormalized cache+ # that powers the "last proven restore" badge + doctor staleness check.+ drill_cadence = db.Column(db.String(12), default='off', nullable=False)+ last_drill_at = db.Column(db.DateTime, nullable=True)+ last_drill_status = db.Column(db.String(24), nullable=True) # success|failed|skipped_no_space+# Hooks (optional shell snippets run before/after a backup)pre_backup_hook = db.Column(db.Text, nullable=True)post_backup_hook = db.Column(db.Text, nullable=True)@@ -112,6 +119,9 @@ class BackupPolicy(db.Model):'full_every_n_days': self.full_every_n_days,'compression': self.compression,'remote_copy': self.remote_copy,+ 'drill_cadence': self.drill_cadence or 'off',+ 'last_drill_at': self.last_drill_at.isoformat() if self.last_drill_at else None,+ 'last_drill_status': self.last_drill_status,'pre_backup_hook': self.pre_backup_hook,'post_backup_hook': self.post_backup_hook,'last_run_at': self.last_run_at.isoformat() if self.last_run_at else None,diff --git a/backend/app/models/backup_run.py b/backend/app/models/backup_run.pyindex d302b50a..a39012d8 100644--- a/backend/app/models/backup_run.py+++ b/backend/app/models/backup_run.py@@ -42,11 +42,39 @@ class BackupRun(db.Model):storage_path = db.Column(db.Text, nullable=True) # local archive directory/pathremote_key = db.Column(db.Text, nullable=True)- verified = db.Column(db.Boolean, default=False)+ verified = db.Column(db.Boolean, default=False) # remote-side evidence (legacy S3 HEAD)++ # Verification ladder (plan 23). 'none' → 'listed' (tar -t readable) → 'hashed'+ # (manifest sha256 spot-check passed) → 'drilled' (a real restore drill of a+ # chain ending at this run succeeded). Only a drill ever sets 'drilled'.+ verify_level = db.Column(db.String(12), nullable=False, default='none')+ verified_at = db.Column(db.DateTime, nullable=True)+ verify_error = db.Column(db.Text, nullable=True)+ # sha256 of the primary archive, computed at build time (Decision 3).+ checksum_sha256 = db.Column(db.String(64), nullable=True)error_message = db.Column(db.Text, nullable=True)metadata_json = db.Column(db.Text, default='{}') # tables, files, compression, etc.+ # Ordered verification levels, weakest → strongest.+ VERIFY_LEVELS = ('none', 'listed', 'hashed', 'drilled')++ @classmethod+ def _verify_rank(cls, level):+ try:+ return cls.VERIFY_LEVELS.index(level or 'none')+ except ValueError:+ return 0++ def effective_verify_level(self):+ """Verify level mapping the legacy ``verified`` boolean forward: a run+ with no explicit level but a remote-verified copy reads as 'listed' (the+ remote HEAD proves it exists + size-matches) so old rows aren't 'none'."""+ level = self.verify_level or 'none'+ if level == 'none' and self.verified:+ return 'listed'+ return level+def get_metadata(self):if not self.metadata_json:return {}@@ -80,6 +108,10 @@ class BackupRun(db.Model):'storage_path': self.storage_path,'remote_key': self.remote_key,'verified': self.verified,+ 'verify_level': self.effective_verify_level(),+ 'verified_at': self.verified_at.isoformat() if self.verified_at else None,+ 'verify_error': self.verify_error,+ 'checksum_sha256': self.checksum_sha256,'error_message': self.error_message,'compression': meta.get('compression'),'tables': meta.get('tables'),diff --git a/backend/app/models/restore_drill.py b/backend/app/models/restore_drill.pynew file mode 100644index 00000000..709c2a89--- /dev/null+++ b/backend/app/models/restore_drill.py@@ -0,0 +1,76 @@+import json+from datetime import datetime++from app import db+++class RestoreDrill(db.Model):+ """A single restore *drill* — a real restore of a policy's latest restorable+ point into a scratch location (temp dir / scratch DB), probe-verified, then+ torn down. Never touches the live target.++ A successful drill is the strongest proof a backup is restorable: it promotes+ the drilled :class:`BackupRun` to ``verify_level='drilled'`` and stamps the+ policy's ``last_drill_*`` cache. Drilling the latest point implicitly proves+ the whole chain (incrementals are useless without their base).++ See docs/plans/23_BACKUP_TRUST_RESTORE_DRILLS_PLAN.md (Decision 4).+ """++ __tablename__ = 'restore_drills'++ id = db.Column(db.Integer, primary_key=True)+ policy_id = db.Column(db.Integer, db.ForeignKey('backup_policies.id'), nullable=False, index=True)+ # The run whose restore-point (chain endpoint) was drilled.+ run_id = db.Column(db.Integer, db.ForeignKey('backup_runs.id'), nullable=True, index=True)+ job_id = db.Column(db.String(36), nullable=True, index=True)++ # 'running' | 'success' | 'failed' | 'skipped_no_space'+ status = db.Column(db.String(24), nullable=False, default='running')+ trigger = db.Column(db.String(20), nullable=True) # 'manual' | 'scheduled'++ started_at = db.Column(db.DateTime, default=datetime.utcnow)+ finished_at = db.Column(db.DateTime, nullable=True)+ duration_seconds = db.Column(db.Integer, nullable=True)++ # Space accounting (operator's own disk — raw bytes, never a fee).+ bytes_required = db.Column(db.BigInteger, nullable=True) # estimated extracted size+ bytes_free = db.Column(db.BigInteger, nullable=True) # free space at precheck+ bytes_restored = db.Column(db.BigInteger, nullable=True) # actually written to scratch++ scratch_ref = db.Column(db.Text, nullable=True) # scratch dir path / scratch DB name+ probes_json = db.Column(db.Text, default='{}') # probe results + error tail+ error = db.Column(db.Text, nullable=True)++ def get_probes(self):+ if not self.probes_json:+ return {}+ try:+ return json.loads(self.probes_json)+ except (json.JSONDecodeError, TypeError):+ return {}++ def set_probes(self, value):+ self.probes_json = json.dumps(value or {})++ def to_dict(self):+ return {+ 'id': self.id,+ 'policy_id': self.policy_id,+ 'run_id': self.run_id,+ 'job_id': self.job_id,+ 'status': self.status,+ 'trigger': self.trigger,+ 'started_at': self.started_at.isoformat() if self.started_at else None,+ 'finished_at': self.finished_at.isoformat() if self.finished_at else None,+ 'duration_seconds': self.duration_seconds,+ 'bytes_required': self.bytes_required,+ 'bytes_free': self.bytes_free,+ 'bytes_restored': self.bytes_restored,+ 'scratch_ref': self.scratch_ref,+ 'probes': self.get_probes(),+ 'error': self.error,+ }++ def __repr__(self):+ return f'<RestoreDrill {self.id} policy={self.policy_id} {self.status}>'diff --git a/backend/app/services/backup_alert_service.py b/backend/app/services/backup_alert_service.pynew file mode 100644index 00000000..4c56c885--- /dev/null+++ b/backend/app/services/backup_alert_service.py@@ -0,0 +1,114 @@+"""Edge-triggered restore-proof alerts (plan 23 Phase 5).++Drills and verification run on a schedule, so a naive "notify on failure" would+re-alert every sweep. These alerts are STATE-TRANSITION only: a failure fires+once (the first sweep that flips healthy→failed for a policy) and a recovery+fires once (the first success after a failure). The previous outcome per policy+is remembered in SettingsService, keyed by policy id.++Events (category ``backups``, admin-targeted):+ * ``backup.drill_failed`` / ``backup.drill_recovered``+ * ``backup.verify_failed`` / ``backup.verify_recovered``+"""+import logging++logger = logging.getLogger(__name__)++_DRILL_KEY = 'backup_alert_drill_{policy_id}'+_VERIFY_KEY = 'backup_alert_verify_{policy_id}'+++class BackupAlertService:++ # ------------------------------------------------------------------ #+ # State store (best-effort)+ # ------------------------------------------------------------------ #++ @staticmethod+ def _get_state(key):+ try:+ from app.services.settings_service import SettingsService+ return SettingsService.get(key)+ except Exception: # noqa: BLE001+ return None++ @staticmethod+ def _set_state(key, value):+ try:+ from app.services.settings_service import SettingsService+ SettingsService.set(key, value)+ except Exception as exc: # noqa: BLE001+ logger.debug('could not persist backup alert state %s: %s', key, exc)++ @staticmethod+ def _policy_label(policy):+ return f'{policy.target_type}:{policy.target_id}'++ @staticmethod+ def _notify(event, data, policy=None):+ # Carry the policy target so the notification deep-links straight to its+ # protection panel (focus=policy:<target_type>:<target_id>, plan 33).+ if policy is not None:+ data = {**data, 'target_type': policy.target_type, 'target_id': policy.target_id}+ try:+ from app.plugins_sdk import notify+ notify.send(event, to='admins', data=data, category='backups')+ except Exception as exc: # noqa: BLE001 — a notification must never break a sweep+ logger.debug('notify %s failed: %s', event, exc)++ # ------------------------------------------------------------------ #+ # Drill outcome+ # ------------------------------------------------------------------ #++ @classmethod+ def on_drill_result(cls, policy, status, error=None):+ """Record a drill's outcome and fire a failed/recovered notice only on a+ state transition. ``status`` is the RestoreDrill status."""+ outcome = 'failed' if status in ('failed', 'skipped_no_space') else 'success'+ key = _DRILL_KEY.format(policy_id=policy.id)+ prev = cls._get_state(key)+ label = cls._policy_label(policy)+ if outcome == 'failed' and prev != 'failed':+ cls._notify('backup.drill_failed', {+ 'policy': label, 'target': label, 'status': status,+ 'error_message': (error or status or 'drill failed')[:300],+ 'message': (f'A restore drill for {label} {status}. The latest backup '+ 'may not be restorable — open Backups → the policy to '+ 'inspect the drill, or Monitoring → Doctor to re-run it.'),+ }, policy=policy)+ elif outcome == 'success' and prev == 'failed':+ cls._notify('backup.drill_recovered', {+ 'policy': label, 'target': label,+ 'message': f'Restore drills for {label} are passing again.',+ }, policy=policy)+ cls._set_state(key, outcome)++ # ------------------------------------------------------------------ #+ # Verification outcome+ # ------------------------------------------------------------------ #++ @classmethod+ def on_verify_result(cls, policy, run):+ """Record a run's verification outcome and fire on transition. A run is a+ verification failure if it has a ``verify_error`` or its offsite deep+ check reported a mismatch."""+ offsite = (run.get_metadata() or {}).get('offsite') or {}+ failed = bool(run.verify_error) or (offsite.get('deep_match') is False)+ outcome = 'failed' if failed else 'success'+ key = _VERIFY_KEY.format(policy_id=policy.id)+ prev = cls._get_state(key)+ label = cls._policy_label(policy)+ if outcome == 'failed' and prev != 'failed':+ detail = run.verify_error or 'The offsite copy did not match its stored checksum.'+ cls._notify('backup.verify_failed', {+ 'policy': label, 'target': label,+ 'error_message': detail[:300],+ 'message': (f'The latest backup for {label} failed verification: {detail} '+ 'Open Backups → the policy to inspect it.'),+ }, policy=policy)+ elif outcome == 'success' and prev == 'failed':+ cls._notify('backup.verify_recovered', {+ 'policy': label, 'target': label,+ 'message': f'Backups for {label} are verifying cleanly again.',+ }, policy=policy)+ cls._set_state(key, outcome)diff --git a/backend/app/services/backup_offsite_service.py b/backend/app/services/backup_offsite_service.pynew file mode 100644index 00000000..15f19e2e--- /dev/null+++ b/backend/app/services/backup_offsite_service.py@@ -0,0 +1,235 @@+"""Offsite verification hardening (plan 23 Phase 4).++For remote-copy policies, prove the offsite object actually matches — not just+"the upload call returned":++ * **weekly shallow** (`backup.offsite.verify`) — HEAD + size + single-part+ ETag/MD5 compare for the newest run of every remote-copy policy, recording+ an honest evidence label (``checksum`` where a single-part ETag confirms+ content, ``size_only`` where a multipart ETag can only confirm size).+ * **monthly deep** — ONE deterministically sampled run per destination gets a+ full download-and-hash against the STORED sha256 (Decision 3 — never a hash+ recomputed from the same possibly-corrupt local file). Egress-bounded to one+ object/month and the downloaded bytes are reported raw (Decision 8).++Everything here is best-effort: a verification failure is recorded on the run's+offsite evidence + surfaced to the drawer, never raised out of the sweep.+"""+import logging+import os+import random+import shutil+from datetime import datetime++from app import db++logger = logging.getLogger(__name__)++OFFSITE_VERIFY_JOB_KIND = 'backup.offsite.verify'+OFFSITE_VERIFY_SCHEDULE_NAME = 'backup-offsite-verify'+++class BackupOffsiteService:++ # ------------------------------------------------------------------ #+ # Candidate selection+ # ------------------------------------------------------------------ #++ @classmethod+ def _remote_copy_policies(cls):+ from app.models.backup_policy import BackupPolicy+ return BackupPolicy.query.filter_by(remote_copy=True).all()++ @classmethod+ def _newest_remote_run(cls, policy):+ from app.models.backup_run import BackupRun+ return (BackupRun.query+ .filter(BackupRun.policy_id == policy.id,+ BackupRun.status == 'success',+ BackupRun.remote_key.isnot(None))+ .order_by(BackupRun.started_at.desc()).first())++ @staticmethod+ def _is_deep_week(now):+ """Monthly cadence on a weekly schedule: deep-check on ~1 week in 4."""+ return (now.isocalendar()[1] % 4) == 0++ @classmethod+ def _sample_one(cls, runs, seed):+ """Deterministically pick one run (seeded — reproducible for a given+ seed, so the monthly egress is bounded AND testable)."""+ if not runs:+ return None+ ordered = sorted(runs, key=lambda r: r.id)+ return ordered[random.Random(seed).randrange(len(ordered))]++ @staticmethod+ def _destination_of(run):+ """A stable key for the destination a run's offsite copy lives in. Today+ every run shares one configured provider (one group), but the day a+ second bucket/provider lands (plan 36) each destination samples on its+ own — so a full backup is never left unproven because a different+ destination happened to win the single-sample lottery."""+ meta = run.get_metadata() or {}+ dest = (meta.get('offsite') or {}).get('destination')+ if dest:+ return str(dest)+ try:+ from app.services.backup_cost_service import BackupCostService+ prov = BackupCostService.configured_remote_provider()+ if prov is not None:+ return str(getattr(prov, 'name', None) or getattr(prov, 'id', None) or prov)+ except Exception: # noqa: BLE001+ pass+ return 'default'++ @classmethod+ def _sample_per_destination(cls, runs, seed):+ """One deterministically sampled run per distinct destination (plan 30+ #10 / Decision 7). A no-op vs the old one-run-total behaviour while+ storage is single-provider; correct the day a second destination exists."""+ groups = {}+ for r in runs:+ groups.setdefault(cls._destination_of(r), []).append(r)+ picks = []+ for dest in sorted(groups.keys()):+ pick = cls._sample_one(groups[dest], f'{seed}:{dest}')+ if pick is not None:+ picks.append(pick)+ return picks++ # ------------------------------------------------------------------ #+ # Verify handlers+ # ------------------------------------------------------------------ #++ @classmethod+ def run_offsite_verify(cls, job=None):+ payload = (job.get_payload() if job else None) or {}+ now = datetime.utcnow()+ deep = payload.get('deep')+ if deep is None:+ deep = cls._is_deep_week(now)+ seed = payload.get('seed', now.isocalendar()[1])++ results = {'shallow': [], 'deep': []}+ candidates = []+ for policy in cls._remote_copy_policies():+ run = cls._newest_remote_run(policy)+ if not run:+ continue+ candidates.append(run)+ ev = cls._shallow_verify(policy, run)+ results['shallow'].append({'policy_id': policy.id, 'run_id': run.id,+ 'evidence': ev.get('evidence'),+ 'verified': ev.get('verified')})++ if deep and candidates:+ for pick in cls._sample_per_destination(candidates, seed):+ deep_res = cls._deep_verify(pick)+ results['deep'].append({'run_id': pick.id, **deep_res})+ # A deep mismatch is a verification failure — alert on transition.+ try:+ from app.models.backup_policy import BackupPolicy+ from app.services.backup_alert_service import BackupAlertService+ pol = BackupPolicy.query.get(pick.policy_id)+ if pol:+ BackupAlertService.on_verify_result(pol, pick)+ except Exception as exc: # noqa: BLE001+ logger.debug('offsite verify alert skipped: %s', exc)++ db.session.commit()+ logger.info('offsite verify: %s shallow, %s deep',+ len(results['shallow']), len(results['deep']))+ return results++ @classmethod+ def _shallow_verify(cls, policy, run):+ """HEAD + size + single-part content compare via verify_run, which records+ the honest evidence label ('checksum' | 'size_only') on the run."""+ from app.services.backup_policy_service import BackupPolicyService, BackupPolicyError+ try:+ res = BackupPolicyService.verify_run(policy, run.id)+ return {'verified': res.get('verified'), 'evidence': res.get('evidence')}+ except BackupPolicyError as exc:+ return {'verified': False, 'evidence': 'error', 'error': str(exc)}+ except Exception as exc: # noqa: BLE001+ logger.warning('shallow offsite verify failed for run %s: %s', run.id, exc)+ return {'verified': False, 'evidence': 'error', 'error': str(exc)}++ @classmethod+ def _deep_verify(cls, run):+ """Download the remote object and hash it against the STORED sha256 — the+ only check that can promote a multipart ``size_only`` run to real content+ proof. Egress-bounded (one object) and the bytes are reported raw."""+ from app.services.storage_provider_service import StorageProviderService+ from app.services.backup_service import BackupService+ from app.services.backup_verify_service import sha256_file++ meta = run.get_metadata() or {}+ offsite = dict(meta.get('offsite') or {})+ stored = offsite.get('our_sha256') or run.checksum_sha256+ remote_key = run.remote_key+ if not remote_key:+ return {'ok': False, 'reason': 'no remote key'}+ if not stored:+ offsite['deep_evidence'] = 'no_stored_hash'+ meta['offsite'] = offsite+ run.set_metadata(meta)+ return {'ok': False, 'reason': 'no stored sha256 to compare against'}++ tmp_dir = os.path.join(BackupService.BACKUP_BASE_DIR, 'restores', 'offsite-verify')+ os.makedirs(tmp_dir, exist_ok=True)+ tmp_path = os.path.join(tmp_dir, f'deep-{run.id}.bin')+ bytes_downloaded = 0+ try:+ dl = StorageProviderService.download_file(remote_key, tmp_path)+ if not dl.get('success'):+ offsite['deep_evidence'] = 'download_failed'+ offsite['deep_error'] = dl.get('error')+ meta['offsite'] = offsite+ run.set_metadata(meta)+ return {'ok': False, 'reason': dl.get('error')}+ bytes_downloaded = dl.get('size') or (os.path.getsize(tmp_path)+ if os.path.exists(tmp_path) else 0)+ actual = sha256_file(tmp_path)+ match = bool(actual and actual == stored)+ offsite['deep_evidence'] = 'deep_sha256'+ offsite['deep_match'] = match+ offsite['deep_bytes'] = bytes_downloaded+ offsite['deep_checked_at'] = datetime.utcnow().isoformat() + 'Z'+ if match:+ # A confirmed offsite content match is at least as strong as a+ # single-part checksum — the run's remote copy is proven.+ run.verified = True+ offsite['evidence'] = 'checksum'+ meta['offsite'] = offsite+ run.set_metadata(meta)+ return {'ok': match, 'bytes': bytes_downloaded, 'evidence': 'deep_sha256'}+ except Exception as exc: # noqa: BLE001+ logger.warning('deep offsite verify failed for run %s: %s', run.id, exc)+ offsite['deep_evidence'] = 'error'+ offsite['deep_error'] = str(exc)[:300]+ meta['offsite'] = offsite+ run.set_metadata(meta)+ return {'ok': False, 'reason': str(exc)}+ finally:+ try:+ if os.path.exists(tmp_path):+ os.remove(tmp_path)+ except OSError:+ pass+ # keep the dir tidy+ try:+ if os.path.isdir(tmp_dir) and not os.listdir(tmp_dir):+ shutil.rmtree(tmp_dir, ignore_errors=True)+ except OSError:+ pass++ # ------------------------------------------------------------------ #+ # Registration+ # ------------------------------------------------------------------ #++ @classmethod+ def register_jobs(cls):+ from app.jobs import registry+ registry.register(OFFSITE_VERIFY_JOB_KIND, cls.run_offsite_verify, replace=True)diff --git a/backend/migrations/versions/058_backup_restore_drills.py b/backend/migrations/versions/058_backup_restore_drills.pynew file mode 100644index 00000000..b7cca066--- /dev/null+++ b/backend/migrations/versions/058_backup_restore_drills.py@@ -0,0 +1,104 @@+"""Backup Trust — verification ladder, checksums, drill cadence + restore_drills.++Adds the "restore proof" columns (plan 23):+ * backup_runs: verify_level, verified_at, verify_error, checksum_sha256+ * backup_policies: drill_cadence, last_drill_at, last_drill_status+ * new restore_drills table (a scratch-mode restore, probe-verified, torn down)++Backfill: existing runs keep verify_level='none' (the column default); the+legacy `verified` boolean is preserved as-is and mapped forward in serialization+(BackupRun.effective_verify_level), so old rows aren't silently downgraded.++Idempotent: MigrationService runs _fix_missing_columns() + db.create_all() on+boot before Alembic, so guard on the live schema.++Revision ID: 058_backup_restore_drills+Revises: 057_user_setup_snoozes+Create Date: 2026-07-05+"""+from alembic import op+import sqlalchemy as sa++revision = '058_backup_restore_drills'+down_revision = '057_user_setup_snoozes'+branch_labels = None+depends_on = None+++def _cols(inspector, table):+ return {c['name'] for c in inspector.get_columns(table)}+++def upgrade():+ conn = op.get_bind()+ inspector = sa.inspect(conn)+ tables = set(inspector.get_table_names())++ if 'backup_runs' in tables:+ cols = _cols(inspector, 'backup_runs')+ if 'verify_level' not in cols:+ op.add_column('backup_runs', sa.Column(+ 'verify_level', sa.String(length=12), nullable=False, server_default='none'))+ if 'verified_at' not in cols:+ op.add_column('backup_runs', sa.Column('verified_at', sa.DateTime(), nullable=True))+ if 'verify_error' not in cols:+ op.add_column('backup_runs', sa.Column('verify_error', sa.Text(), nullable=True))+ if 'checksum_sha256' not in cols:+ op.add_column('backup_runs', sa.Column('checksum_sha256', sa.String(length=64), nullable=True))++ if 'backup_policies' in tables:+ cols = _cols(inspector, 'backup_policies')+ if 'drill_cadence' not in cols:+ op.add_column('backup_policies', sa.Column(+ 'drill_cadence', sa.String(length=12), nullable=False, server_default='off'))+ if 'last_drill_at' not in cols:+ op.add_column('backup_policies', sa.Column('last_drill_at', sa.DateTime(), nullable=True))+ if 'last_drill_status' not in cols:+ op.add_column('backup_policies', sa.Column('last_drill_status', sa.String(length=24), nullable=True))++ if 'restore_drills' not in tables:+ op.create_table(+ 'restore_drills',+ sa.Column('id', sa.Integer(), nullable=False),+ sa.Column('policy_id', sa.Integer(), nullable=False),+ sa.Column('run_id', sa.Integer(), nullable=True),+ sa.Column('job_id', sa.String(length=36), nullable=True),+ sa.Column('status', sa.String(length=24), nullable=False, server_default='running'),+ sa.Column('trigger', sa.String(length=20), nullable=True),+ sa.Column('started_at', sa.DateTime(), nullable=True),+ sa.Column('finished_at', sa.DateTime(), nullable=True),+ sa.Column('duration_seconds', sa.Integer(), nullable=True),+ sa.Column('bytes_required', sa.BigInteger(), nullable=True),+ sa.Column('bytes_free', sa.BigInteger(), nullable=True),+ sa.Column('bytes_restored', sa.BigInteger(), nullable=True),+ sa.Column('scratch_ref', sa.Text(), nullable=True),+ sa.Column('probes_json', sa.Text(), nullable=True),+ sa.Column('error', sa.Text(), nullable=True),+ sa.ForeignKeyConstraint(['policy_id'], ['backup_policies.id']),+ sa.ForeignKeyConstraint(['run_id'], ['backup_runs.id']),+ sa.PrimaryKeyConstraint('id'),+ )+ op.create_index('ix_restore_drills_policy_id', 'restore_drills', ['policy_id'])+ op.create_index('ix_restore_drills_run_id', 'restore_drills', ['run_id'])+ op.create_index('ix_restore_drills_job_id', 'restore_drills', ['job_id'])+++def downgrade():+ conn = op.get_bind()+ inspector = sa.inspect(conn)+ tables = set(inspector.get_table_names())++ if 'restore_drills' in tables:+ op.drop_table('restore_drills')++ if 'backup_policies' in tables:+ cols = _cols(inspector, 'backup_policies')+ for col in ('drill_cadence', 'last_drill_at', 'last_drill_status'):+ if col in cols:+ op.drop_column('backup_policies', col)++ if 'backup_runs' in tables:+ cols = _cols(inspector, 'backup_runs')+ for col in ('verify_level', 'verified_at', 'verify_error', 'checksum_sha256'):+ if col in cols:+ op.drop_column('backup_runs', col)diff --git a/backend/tests/test_backup_alerts.py b/backend/tests/test_backup_alerts.pynew file mode 100644index 00000000..4b131fe3--- /dev/null+++ b/backend/tests/test_backup_alerts.py@@ -0,0 +1,144 @@+"""Plan 23 Phase 5 — edge-triggered restore-proof alerts + support-bundle section."""+from datetime import datetime++from app import db+from app.services.backup_policy_service import BackupPolicyService+from app.services.backup_alert_service import BackupAlertService+++def _capture_notify(monkeypatch):+ import app.plugins_sdk as sdk+ sent = []+ monkeypatch.setattr(sdk.notify, 'send',+ lambda event, to=None, data=None, **kw: sent.append((event, data)))+ return sent+++def _policy(target_id):+ return BackupPolicyService.get_or_create_policy('files', target_id,+ target_meta={'paths': ['/x']})+++# --------------------------------------------------------------------------- #+# Drill alerts — fire once on transition (#17)+# --------------------------------------------------------------------------- #++def test_drill_failed_fires_once_then_throttles(app, monkeypatch):+ with app.app_context():+ sent = _capture_notify(monkeypatch)+ policy = _policy(5501)+ BackupAlertService.on_drill_result(policy, 'failed', 'boom')+ assert [e for e, _ in sent] == ['backup.drill_failed']+ # A second consecutive failure does NOT re-alert.+ BackupAlertService.on_drill_result(policy, 'failed', 'boom again')+ assert [e for e, _ in sent] == ['backup.drill_failed']+++def test_drill_recovery_fires_after_failure(app, monkeypatch):+ with app.app_context():+ sent = _capture_notify(monkeypatch)+ policy = _policy(5502)+ BackupAlertService.on_drill_result(policy, 'failed', 'boom')+ BackupAlertService.on_drill_result(policy, 'success', None)+ assert [e for e, _ in sent] == ['backup.drill_failed', 'backup.drill_recovered']+ # Staying healthy doesn't repeat the recovery notice.+ BackupAlertService.on_drill_result(policy, 'success', None)+ assert [e for e, _ in sent] == ['backup.drill_failed', 'backup.drill_recovered']+++def test_first_success_is_silent(app, monkeypatch):+ with app.app_context():+ sent = _capture_notify(monkeypatch)+ policy = _policy(5503)+ # A healthy-from-the-start policy never had a failure -> no recovery spam.+ BackupAlertService.on_drill_result(policy, 'success', None)+ assert sent == []+++def test_skipped_no_space_counts_as_failure(app, monkeypatch):+ with app.app_context():+ sent = _capture_notify(monkeypatch)+ policy = _policy(5504)+ BackupAlertService.on_drill_result(policy, 'skipped_no_space', 'no space')+ assert [e for e, _ in sent] == ['backup.drill_failed']+++# --------------------------------------------------------------------------- #+# Verify alerts (#17)+# --------------------------------------------------------------------------- #++def test_verify_failed_and_recovered(app, monkeypatch):+ from app.models.backup_run import BackupRun+ with app.app_context():+ sent = _capture_notify(monkeypatch)+ policy = _policy(5505)+ bad = BackupRun(policy_id=policy.id, kind='full', status='success',+ verify_level='listed', verify_error='Checksum mismatch')+ db.session.add(bad)+ db.session.commit()+ BackupAlertService.on_verify_result(policy, bad)+ assert [e for e, _ in sent] == ['backup.verify_failed']++ good = BackupRun(policy_id=policy.id, kind='full', status='success',+ verify_level='hashed')+ db.session.add(good)+ db.session.commit()+ BackupAlertService.on_verify_result(policy, good)+ assert [e for e, _ in sent] == ['backup.verify_failed', 'backup.verify_recovered']+++def test_verify_failed_on_deep_offsite_mismatch(app, monkeypatch):+ from app.models.backup_run import BackupRun+ with app.app_context():+ sent = _capture_notify(monkeypatch)+ policy = _policy(5506)+ run = BackupRun(policy_id=policy.id, kind='full', status='success',+ verify_level='hashed')+ run.set_metadata({'offsite': {'evidence': 'size_only', 'deep_match': False}})+ db.session.add(run)+ db.session.commit()+ BackupAlertService.on_verify_result(policy, run)+ assert [e for e, _ in sent] == ['backup.verify_failed']+++# --------------------------------------------------------------------------- #+# Catalog registration (#17)+# --------------------------------------------------------------------------- #++def test_catalog_has_restore_proof_events():+ from app.notifications import catalog+ for key in ('backup.drill_failed', 'backup.drill_recovered',+ 'backup.verify_failed', 'backup.verify_recovered'):+ entry = catalog.get(key)+ assert entry is not None+ assert entry['category'] == 'backups'+++# --------------------------------------------------------------------------- #+# Support bundle restore_confidence section (#18)+# --------------------------------------------------------------------------- #++def test_support_bundle_restore_confidence(app):+ from app.services.support_bundle_service import _collect_restore_confidence+ from app.models.backup_run import BackupRun+ with app.app_context():+ policy = _policy(5507)+ policy.drill_cadence = 'weekly'+ policy.last_drill_status = 'success'+ policy.last_drill_at = datetime.utcnow()+ run = BackupRun(policy_id=policy.id, kind='full', status='success',+ verify_level='drilled')+ run.set_metadata({'offsite': {'evidence': 'checksum', 'deep_evidence': 'deep_sha256'}})+ db.session.add(run)+ db.session.commit()++ section = _collect_restore_confidence()+ assert section['total'] >= 1+ entry = next(p for p in section['policies']+ if p['target_type'] == 'files' and p['target_id'] == 5507)+ assert entry['drill_cadence'] == 'weekly'+ assert entry['latest_verify_level'] == 'drilled'+ assert entry['last_drill_status'] == 'success'+ assert entry['offsite_evidence'] == 'checksum'+ # No paths / secrets leak into the section.+ assert 'storage_path' not in entry and 'remote_key' not in entrydiff --git a/backend/tests/test_backup_drill.py b/backend/tests/test_backup_drill.pynew file mode 100644index 00000000..2a0ffcdb--- /dev/null+++ b/backend/tests/test_backup_drill.py@@ -0,0 +1,251 @@+"""Plan 23 Phase 2 — restore-drill engine proving tests.++End-to-end files drill on tmp dirs (real archive, cross-platform), a database+drill against a stubbed engine (asserts create -> restore -> probe -> drop call+order), the no-space skip path, teardown/record-on-exception, and single-flight+refusal."""+import io+import os+import tarfile+from datetime import datetime++import pytest++from app import db+from app.services.backup_policy_service import BackupPolicyService+from app.services.backup_drill_service import BackupDrillService, BackupDrillError+from app.services import backup_verify_service as bv+++class _FakeJob:+ def __init__(self, payload, jid='job-drill-1'):+ self.id = jid+ self._payload = payload++ def get_payload(self):+ return self._payload+++def _write_tar_gz_tree(dest, files):+ with tarfile.open(dest, 'w:gz') as tar:+ for name, content in files.items():+ data = io.BytesIO(content)+ info = tarfile.TarInfo(name)+ info.size = len(content)+ tar.addfile(info, data)+++def _make_files_run(policy, tmp_path, name='run', files=None):+ """A successful files BackupRun with a real .tar.gz + manifest on disk."""+ from app.models.backup_run import BackupRun+ run_dir = tmp_path / name+ run_dir.mkdir()+ primary = str(run_dir / 'files.tar.gz')+ _write_tar_gz_tree(primary, files if files is not None else {'a.txt': b'alpha', 'b.txt': b'beta'})+ run = BackupRun(policy_id=policy.id, kind='full', status='success',+ storage_path=str(run_dir), started_at=datetime.utcnow())+ meta = {'engine': 'files', 'kind': 'full', 'primary_archive': primary,+ 'incremental': False}+ run.set_metadata(meta)+ db.session.add(run)+ db.session.commit()+ bv.write_manifest(run, meta)+ run.set_metadata(meta)+ db.session.commit()+ return run+++# --------------------------------------------------------------------------- #+# End-to-end files drill (#9)+# --------------------------------------------------------------------------- #++def test_files_drill_success_promotes_run_and_stamps_policy(app, tmp_path, monkeypatch):+ from app.services.backup_service import BackupService+ from app.models.backup_run import BackupRun+ from app.models.backup_policy import BackupPolicy+ monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))+ os.makedirs(BackupService.BACKUP_BASE_DIR, exist_ok=True)++ with app.app_context():+ policy = BackupPolicyService.get_or_create_policy(+ 'files', 8201, target_meta={'paths': ['/etc/x'], 'label': 'configs'})+ run = _make_files_run(policy, tmp_path)++ BackupDrillService.run_restore_drill(_FakeJob(+ {'policy_id': policy.id, 'run_id': run.id, 'trigger': 'manual'}))++ from app.models.restore_drill import RestoreDrill+ drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()+ assert drill.status == 'success'+ assert drill.get_probes()['file_count'] == 2+ assert drill.get_probes()['sampled_ok'] == drill.get_probes()['sampled']++ # The drilled run is promoted; the policy cache is stamped.+ assert BackupRun.query.get(run.id).verify_level == 'drilled'+ pol = BackupPolicy.query.get(policy.id)+ assert pol.last_drill_status == 'success'+ assert pol.last_drill_at is not None++ # Scratch is torn down on success.+ scratch = os.path.join(BackupService.BACKUP_BASE_DIR, 'restores', f'drill-{drill.id}')+ assert not os.path.exists(scratch)+++# --------------------------------------------------------------------------- #+# Database drill call order against a stubbed engine (#9)+# --------------------------------------------------------------------------- #++def test_database_drill_create_restore_probe_drop_order(app, tmp_path, monkeypatch):+ from app.services.backup_service import BackupService+ from app.services import database_service as dbsvc+ from app.models.backup_run import BackupRun+ from app.models.backup_policy import BackupPolicy+ monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))++ calls = []+ monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_create_database',+ staticmethod(lambda name, **kw: calls.append(('create', name)) or {'success': True}))+ monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_get_tables',+ staticmethod(lambda name, **kw: calls.append(('tables', name)) or ['wp_posts', 'wp_users']))+ monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_drop_database',+ staticmethod(lambda name, **kw: calls.append(('drop', name)) or {'success': True}))+ monkeypatch.setattr(BackupService, 'restore_database',+ staticmethod(lambda **kw: calls.append(('restore', kw['db_name'])) or {'success': True}))++ with app.app_context():+ policy = BackupPolicyService.get_or_create_policy(+ 'database', 8202, target_subtype='mysql', target_meta={'db_name': 'shop'})+ dump = tmp_path / 'shop.sql.gz'+ dump.write_bytes(b'dummy')+ run = BackupRun(policy_id=policy.id, kind='full', status='success',+ storage_path=str(dump))+ run.set_metadata({'engine': 'database', 'primary_archive': str(dump),+ 'db_type': 'mysql', 'db_name': 'shop'})+ db.session.add(run)+ db.session.commit()++ scratch_name = f'skdrill_{run.id}'+ BackupDrillService.run_restore_drill(_FakeJob(+ {'policy_id': policy.id, 'run_id': run.id, 'trigger': 'scheduled'}))++ kinds = [c[0] for c in calls]+ assert kinds == ['create', 'restore', 'tables', 'drop']+ # All operate on the scratch DB, never the live 'shop'.+ assert all(c[1] == scratch_name for c in calls)+ from app.models.restore_drill import RestoreDrill+ drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()+ assert drill.status == 'success'+ assert drill.get_probes()['table_count'] == 2+++def test_database_drill_drops_scratch_even_on_restore_failure(app, tmp_path, monkeypatch):+ from app.services.backup_service import BackupService+ from app.services import database_service as dbsvc+ from app.models.backup_run import BackupRun+ monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))++ calls = []+ monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_create_database',+ staticmethod(lambda name, **kw: calls.append(('create', name)) or {'success': True}))+ monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_drop_database',+ staticmethod(lambda name, **kw: calls.append(('drop', name)) or {'success': True}))+ monkeypatch.setattr(BackupService, 'restore_database',+ staticmethod(lambda **kw: {'success': False, 'error': 'boom'}))++ with app.app_context():+ policy = BackupPolicyService.get_or_create_policy(+ 'database', 8203, target_subtype='mysql', target_meta={'db_name': 'shop'})+ dump = tmp_path / 'shop2.sql.gz'+ dump.write_bytes(b'dummy')+ run = BackupRun(policy_id=policy.id, kind='full', status='success',+ storage_path=str(dump))+ run.set_metadata({'engine': 'database', 'primary_archive': str(dump),+ 'db_type': 'mysql', 'db_name': 'shop'})+ db.session.add(run)+ db.session.commit()++ scratch_name = f'skdrill_{run.id}'+ with pytest.raises(Exception):+ BackupDrillService.run_restore_drill(_FakeJob(+ {'policy_id': policy.id, 'run_id': run.id}))++ # Scratch DB was created then dropped despite the restore failing.+ assert ('create', scratch_name) in calls+ assert ('drop', scratch_name) in calls+ from app.models.restore_drill import RestoreDrill+ drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()+ assert drill.status == 'failed'+ assert 'boom' in (drill.error or '')+++# --------------------------------------------------------------------------- #+# No-space skip (#9)+# --------------------------------------------------------------------------- #++def test_drill_skipped_no_space(app, tmp_path, monkeypatch):+ from app.services.backup_service import BackupService+ from app.models.backup_policy import BackupPolicy+ monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))+ os.makedirs(BackupService.BACKUP_BASE_DIR, exist_ok=True)+ # Huge requirement, tiny free space -> loud skip, never a silent pass.+ monkeypatch.setattr(BackupDrillService, '_estimate_required', classmethod(lambda cls, r, m: 10 ** 12))+ monkeypatch.setattr(BackupDrillService, '_free_bytes', staticmethod(lambda p: 1024))++ with app.app_context():+ policy = BackupPolicyService.get_or_create_policy('files', 8204,+ target_meta={'paths': ['/x']})+ run = _make_files_run(policy, tmp_path)+ result = BackupDrillService.run_restore_drill(_FakeJob(+ {'policy_id': policy.id, 'run_id': run.id}))++ assert result['status'] == 'skipped_no_space'+ from app.models.restore_drill import RestoreDrill+ drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()+ assert drill.status == 'skipped_no_space'+ assert BackupPolicy.query.get(policy.id).last_drill_status == 'skipped_no_space'+++# --------------------------------------------------------------------------- #+# Teardown / record on exception (#9)+# --------------------------------------------------------------------------- #++def test_drill_records_failure_on_empty_extract(app, tmp_path, monkeypatch):+ from app.services.backup_service import BackupService+ from app.models.backup_run import BackupRun+ monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))+ os.makedirs(BackupService.BACKUP_BASE_DIR, exist_ok=True)++ with app.app_context():+ policy = BackupPolicyService.get_or_create_policy('files', 8205,+ target_meta={'paths': ['/x']})+ # An empty archive -> the drill must fail loudly (0 files restored).+ run = _make_files_run(policy, tmp_path, files={})+ with pytest.raises(Exception):+ BackupDrillService.run_restore_drill(_FakeJob(+ {'policy_id': policy.id, 'run_id': run.id}))++ from app.models.restore_drill import RestoreDrill+ drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()+ assert drill.status == 'failed'+ assert drill.error+ # The run is NOT promoted to drilled on a failed drill.+ assert BackupRun.query.get(run.id).verify_level != 'drilled'+++# --------------------------------------------------------------------------- #+# Single-flight refusal (#9)+# --------------------------------------------------------------------------- #++def test_single_flight_refusal(app, tmp_path):+ from app.models.restore_drill import RestoreDrill+ with app.app_context():+ policy = BackupPolicyService.get_or_create_policy('files', 8206,+ target_meta={'paths': ['/x']})+ _make_files_run(policy, tmp_path)+ # A running drill already exists -> request is refused.+ db.session.add(RestoreDrill(policy_id=policy.id, status='running',+ started_at=datetime.utcnow()))+ db.session.commit()+ assert BackupDrillService.is_drilling() is True+ with pytest.raises(BackupDrillError):+ BackupDrillService.request_drill(policy)diff --git a/backend/tests/test_backup_drill_cadence.py b/backend/tests/test_backup_drill_cadence.pynew file mode 100644index 00000000..7f0dd842--- /dev/null+++ b/backend/tests/test_backup_drill_cadence.py@@ -0,0 +1,154 @@+"""Plan 23 Phase 3 — cadence sweep, badge serialization, doctor check matrix."""+from datetime import datetime, timedelta++from app import db+from app.services.backup_policy_service import BackupPolicyService+from app.services.backup_drill_service import BackupDrillService+++def _policy_with_run(target_id, cadence='weekly', enabled=True, last_drill_at=None,+ last_drill_status=None):+ from app.models.backup_run import BackupRun+ policy = BackupPolicyService.get_or_create_policy('files', target_id,+ target_meta={'paths': ['/x']})+ policy.enabled = enabled+ policy.drill_cadence = cadence+ policy.last_drill_at = last_drill_at+ policy.last_drill_status = last_drill_status+ db.session.add(BackupRun(policy_id=policy.id, kind='full', status='success',+ started_at=datetime.utcnow()))+ db.session.commit()+ return policy+++# --------------------------------------------------------------------------- #+# Cadence due-selection (#13)+# --------------------------------------------------------------------------- #++def test_is_due_matrix(app):+ with app.app_context():+ now = datetime.utcnow()+ # never drilled + cadenced + enabled -> due+ p_never = _policy_with_run(7301, cadence='weekly', last_drill_at=None)+ assert BackupDrillService.is_due(p_never, now) is True++ # drilled recently -> not due+ p_fresh = _policy_with_run(7302, cadence='weekly',+ last_drill_at=now - timedelta(days=1))+ assert BackupDrillService.is_due(p_fresh, now) is False++ # drilled long ago (> weekly interval incl. jitter) -> due+ p_old = _policy_with_run(7303, cadence='weekly',+ last_drill_at=now - timedelta(days=30))+ assert BackupDrillService.is_due(p_old, now) is True++ # cadence off -> never due+ p_off = _policy_with_run(7304, cadence='off', last_drill_at=None)+ assert BackupDrillService.is_due(p_off, now) is False++ # disabled -> never due+ p_dis = _policy_with_run(7305, cadence='weekly', enabled=False, last_drill_at=None)+ assert BackupDrillService.is_due(p_dis, now) is False+++def test_due_policies_and_sweep_enqueues(app, monkeypatch):+ enqueued = []+ monkeypatch.setattr(BackupDrillService, 'request_drill',+ classmethod(lambda cls, policy, trigger='scheduled':+ enqueued.append(policy.id) or type('J', (), {'id': f'j{policy.id}'})()))+ monkeypatch.setattr(BackupDrillService, 'is_drilling', classmethod(lambda cls: False))+ with app.app_context():+ due_policy = _policy_with_run(7311, cadence='weekly', last_drill_at=None) # due+ fresh_policy = _policy_with_run(7312, cadence='weekly',+ last_drill_at=datetime.utcnow() - timedelta(hours=1)) # fresh+ off_policy = _policy_with_run(7313, cadence='off', last_drill_at=None) # off+ due = {p.id for p in BackupDrillService.due_policies()}+ assert due_policy.id in due+ assert fresh_policy.id not in due and off_policy.id not in due++ result = BackupDrillService.run_drill_sweep()+ assert due_policy.id in enqueued+ assert result['count'] == len(enqueued)+++def test_sweep_defers_when_a_drill_is_in_flight(app, monkeypatch):+ monkeypatch.setattr(BackupDrillService, 'is_drilling', classmethod(lambda cls: True))+ called = []+ monkeypatch.setattr(BackupDrillService, 'request_drill',+ classmethod(lambda cls, policy, trigger='scheduled': called.append(policy.id)))+ with app.app_context():+ _policy_with_run(7321, cadence='weekly', last_drill_at=None)+ result = BackupDrillService.run_drill_sweep()+ assert result['count'] == 0+ assert called == [] # deferred — nothing enqueued while one is running+++# --------------------------------------------------------------------------- #+# Badge serialization states (#13)+# --------------------------------------------------------------------------- #++def test_badge_states(app):+ with app.app_context():+ now = datetime.utcnow()+ never = _policy_with_run(7331, cadence='weekly', last_drill_at=None)+ assert BackupPolicyService._restore_proof_view(never)['badge'] == 'never'++ ok = _policy_with_run(7332, cadence='weekly', last_drill_status='success',+ last_drill_at=now - timedelta(days=1))+ assert BackupPolicyService._restore_proof_view(ok)['badge'] == 'ok'++ stale = _policy_with_run(7333, cadence='weekly', last_drill_status='success',+ last_drill_at=now - timedelta(days=60))+ assert BackupPolicyService._restore_proof_view(stale)['badge'] == 'stale'++ failed = _policy_with_run(7334, cadence='weekly', last_drill_status='failed',+ last_drill_at=now - timedelta(days=1))+ assert BackupPolicyService._restore_proof_view(failed)['badge'] == 'failed'+++def test_serialize_policy_view_includes_restore_proof(app):+ with app.app_context():+ policy = _policy_with_run(7341, cadence='monthly', last_drill_status='success',+ last_drill_at=datetime.utcnow())+ view = BackupPolicyService.serialize_policy_view(policy)+ assert 'restore_proof' in view+ proof = view['restore_proof']+ assert proof['drill_cadence'] == 'monthly'+ assert proof['badge'] == 'ok'+ assert 'recent_drills' in proof+++# --------------------------------------------------------------------------- #+# Doctor check matrix (#13)+# --------------------------------------------------------------------------- #++def test_doctor_backup_proof_matrix(app):+ from app.services.doctor_service import DoctorService+ with app.app_context():+ now = datetime.utcnow()+ _policy_with_run(7351, cadence='weekly', last_drill_status='success',+ last_drill_at=now - timedelta(days=1)) # fresh -> ok+ _policy_with_run(7352, cadence='weekly', last_drill_status='success',+ last_drill_at=now - timedelta(days=60)) # stale -> warn+ _policy_with_run(7353, cadence='weekly', last_drill_status='failed',+ last_drill_at=now - timedelta(days=1)) # failed -> fail+ _policy_with_run(7354, cadence='off', last_drill_at=None) # no cadence -> no drill check++ checks = DoctorService._backup_proof_checks()+ by_key = {c['key']: c for c in checks}++ # Locate the drill checks by policy.+ from app.models.backup_policy import BackupPolicy+ pid = {tid: BackupPolicy.query.filter_by(target_type='files', target_id=tid).first().id+ for tid in (7351, 7352, 7353, 7354)}+ assert by_key[f'backup_drill_stale.{pid[7351]}']['status'] == 'ok'+ assert by_key[f'backup_drill_stale.{pid[7352]}']['status'] == 'warn'+ assert by_key[f'backup_drill_stale.{pid[7353]}']['status'] == 'fail'+ assert by_key[f'backup_drill_stale.{pid[7353]}']['repairable'] is True+ # A no-cadence policy has no drill-stale check.+ assert f'backup_drill_stale.{pid[7354]}' not in by_key++ # Every policy with a successful run gets an unverified check; the runs+ # here were created verify_level='none' -> warn + repairable.+ assert by_key[f'backup_unverified.{pid[7351]}']['status'] == 'warn'+ assert by_key[f'backup_unverified.{pid[7351]}']['repair_ref']['kind'] == 'backup_verify'diff --git a/backend/tests/test_backup_verify.py b/backend/tests/test_backup_verify.pynew file mode 100644index 00000000..126cec23--- /dev/null+++ b/backend/tests/test_backup_verify.py@@ -0,0 +1,203 @@+"""Plan 23 Phase 1 — Tier-1 backup verification proving tests.++Covers the manifest writer + post-run verifier (backup_verify_service): manifest+shape, corrupted-archive detection (flip a byte -> verify_error), incremental+chain metadata intact, and legacy-run serialization compatibility. All tests use+real (small) tar archives so they run cross-platform (the verifier falls back to+Python's tarfile where GNU tar is unavailable, e.g. Windows dev)."""+import os+import tarfile++from app import db+from app.services.backup_policy_service import BackupPolicyService+from app.services import backup_verify_service as bv+++def _mk_run(policy_id, storage_path, primary, kind='full', meta_extra=None):+ from app.models.backup_run import BackupRun+ run = BackupRun(policy_id=policy_id, kind=kind, status='verifying',+ storage_path=storage_path)+ meta = {'engine': 'files', 'kind': kind, 'primary_archive': primary,+ 'incremental': False}+ if meta_extra:+ meta.update(meta_extra)+ run.set_metadata(meta)+ db.session.add(run)+ db.session.commit()+ return run, meta+++def _write_tar_gz(dest, payload=b'hello world'):+ """A tiny valid .tar.gz containing one file."""+ import io+ with tarfile.open(dest, 'w:gz') as tar:+ data = io.BytesIO(payload)+ info = tarfile.TarInfo('data.txt')+ info.size = len(payload)+ tar.addfile(info, data)+++def _write_tar_plain(dest, payload=b'A' * 1024):+ """An uncompressed .tar — flipping a content byte stays listable but changes+ the sha256, so it isolates the checksum spot-check from the readability one."""+ import io+ with tarfile.open(dest, 'w') as tar:+ data = io.BytesIO(payload)+ info = tarfile.TarInfo('data.bin')+ info.size = len(payload)+ tar.addfile(info, data)+++# --------------------------------------------------------------------------- #+# Manifest shape (#5)+# --------------------------------------------------------------------------- #++def test_manifest_shape(app, tmp_path):+ with app.app_context():+ policy = BackupPolicyService.get_or_create_policy('files', 9101,+ target_meta={'paths': ['/x']})+ run_dir = tmp_path / 'run1'+ run_dir.mkdir()+ primary = str(run_dir / 'files.tar.gz')+ _write_tar_gz(primary)+ # a second artifact to prove the manifest lists everything+ (run_dir / 'notes.txt').write_text('side file')++ run, meta = _mk_run(policy.id, str(run_dir), primary)+ manifest = bv.write_manifest(run, meta)++ assert manifest['version'] == bv.MANIFEST_VERSION+ assert manifest['primary_archive'] == 'files.tar.gz'+ assert manifest['primary_sha256'] and len(manifest['primary_sha256']) == 64+ names = {a['name'] for a in manifest['artifacts']}+ assert names == {'files.tar.gz', 'notes.txt'}+ for art in manifest['artifacts']:+ assert art['sha256'] and len(art['sha256']) == 64+ assert art['size'] >= 0+ assert manifest['totals']['count'] == 2+ assert manifest['chain'] == {'incremental': False, 'full_run_id': None}+ # sha256 is denormalized onto the run row.+ assert run.checksum_sha256 == manifest['primary_sha256']+ # manifest.json actually lands next to the archives.+ assert os.path.exists(run_dir / bv.MANIFEST_NAME)+++# --------------------------------------------------------------------------- #+# Good archive -> hashed (#5)+# --------------------------------------------------------------------------- #++def test_verify_promotes_good_archive_to_hashed(app, tmp_path):+ with app.app_context():+ policy = BackupPolicyService.get_or_create_policy('files', 9102,+ target_meta={'paths': ['/x']})+ run_dir = tmp_path / 'run2'+ run_dir.mkdir()+ primary = str(run_dir / 'files.tar.gz')+ _write_tar_gz(primary)+ run, meta = _mk_run(policy.id, str(run_dir), primary)+ bv.write_manifest(run, meta)++ probes = bv.verify_run_tier1(run, meta)+ assert run.verify_level == 'hashed'+ assert run.verify_error is None+ assert run.verified_at is not None+ assert probes['hashed'] is True+ assert probes['listed'][0]['ok'] is True+++# --------------------------------------------------------------------------- #+# Corrupted archive -> verify_error, not 'hashed' (#5)+# --------------------------------------------------------------------------- #++def test_corrupted_archive_flags_checksum_mismatch(app, tmp_path):+ with app.app_context():+ policy = BackupPolicyService.get_or_create_policy('files', 9103,+ target_meta={'paths': ['/x']})+ run_dir = tmp_path / 'run3'+ run_dir.mkdir()+ primary = str(run_dir / 'files.tar') # uncompressed: stays listable+ _write_tar_plain(primary)+ run, meta = _mk_run(policy.id, str(run_dir), primary)+ bv.write_manifest(run, meta) # records the good sha256++ # Flip a byte inside the file-content region (offset well past the header).+ with open(primary, 'r+b') as f:+ f.seek(512 + 256)+ b = f.read(1)+ f.seek(512 + 256)+ f.write(bytes([b[0] ^ 0xFF]))++ probes = bv.verify_run_tier1(run, meta)+ # The archive is still readable (listing ok) but the hash no longer matches.+ assert run.verify_level == 'listed'+ assert run.verify_error and 'mismatch' in run.verify_error.lower()+ assert probes['hashed'] is False+++def test_unreadable_archive_flags_verify_error(app, tmp_path):+ with app.app_context():+ policy = BackupPolicyService.get_or_create_policy('files', 9104,+ target_meta={'paths': ['/x']})+ run_dir = tmp_path / 'run4'+ run_dir.mkdir()+ primary = str(run_dir / 'files.tar.gz')+ # Not a real gzip/tar at all -> listing must fail.+ with open(primary, 'wb') as f:+ f.write(b'this is not a tar archive at all')+ run, meta = _mk_run(policy.id, str(run_dir), primary)+ bv.write_manifest(run, meta)++ bv.verify_run_tier1(run, meta)+ assert run.verify_level == 'none'+ assert run.verify_error and 'readable' in run.verify_error.lower()+++# --------------------------------------------------------------------------- #+# Incremental chain metadata intact (#5)+# --------------------------------------------------------------------------- #++def test_incremental_chain_metadata_intact(app, tmp_path):+ with app.app_context():+ policy = BackupPolicyService.get_or_create_policy('application', 9105)+ run_dir = tmp_path / 'incr'+ run_dir.mkdir()+ primary = str(run_dir / 'files.tar.gz')+ _write_tar_gz(primary)+ run, meta = _mk_run(policy.id, str(run_dir), primary, kind='incremental',+ meta_extra={'incremental': True, 'full_run_id': 4242})+ manifest = bv.write_manifest(run, meta)+ assert manifest['chain'] == {'incremental': True, 'full_run_id': 4242}+ assert manifest['kind'] == 'incremental'+++# --------------------------------------------------------------------------- #+# Legacy-run serialization compatibility (#5)+# --------------------------------------------------------------------------- #++def test_legacy_run_serialization_maps_forward(app):+ from app.models.backup_run import BackupRun+ with app.app_context():+ policy = BackupPolicyService.get_or_create_policy('files', 9106,+ target_meta={'paths': ['/x']})+ # A pre-plan-23 run: verify_level defaulted to 'none' but it had a+ # remote-verified copy — must read forward as 'listed', not 'none'.+ legacy = BackupRun(policy_id=policy.id, kind='full', status='success',+ verified=True)+ db.session.add(legacy)+ db.session.commit()+ assert legacy.verify_level == 'none' # stored value untouched+ assert legacy.effective_verify_level() == 'listed'+ assert legacy.to_dict()['verify_level'] == 'listed'++ # A run explicitly hashed serializes as hashed.+ fresh = BackupRun(policy_id=policy.id, kind='full', status='success',+ verify_level='hashed')+ db.session.add(fresh)+ db.session.commit()+ assert fresh.to_dict()['verify_level'] == 'hashed'++ # A never-verified local-only run stays 'none'.+ plain = BackupRun(policy_id=policy.id, kind='full', status='success')+ db.session.add(plain)+ db.session.commit()+ assert plain.to_dict()['verify_level'] == 'none'diff --git a/docs/BACKUP_PROTECTION.md b/docs/BACKUP_PROTECTION.mdindex 655992cd..54d1a5bc 100644--- a/docs/BACKUP_PROTECTION.md+++ b/docs/BACKUP_PROTECTION.md@@ -81,6 +81,50 @@ Open a backup from the history → **Restore** → choose the scope and confirm:Restore runs as a `restore.run` job; you're notified on completion.+## How restore proof works++A backup you have never restored is an assumption, not a safety net. ServerKit+records how strongly each backup has been *proven*, on a ladder:++| Level | What it means |+|-------|---------------|+| `none` | Produced, nothing checked yet. |+| `listed` | Every archive is readable end to end (`tar -t`). |+| `hashed` | The primary archive still matches the sha256 recorded in its `manifest.json` — catches silent corruption. |+| `drilled` | A real restore of this point actually succeeded (strongest proof). |++`listed` and `hashed` run automatically at the end of **every** backup (inside+the existing `verifying` window). Each run writes a `manifest.json` next to its+archives — file list, per-artifact sha256, chain refs, tool versions — and the+primary archive's hash is stored on the run.++A **restore drill** earns `drilled`. On a per-policy cadence (**Off / Weekly /+Monthly**, set on the Schedule card) — or on demand with **Run drill now** — the+latest restorable point is restored into a throwaway scratch location (a+`restores/drill-<id>/` dir for files/apps/WordPress, a `skdrill_<run_id>`+scratch database for databases), probe-verified (file count + bytes + a hashed+sample; table listing for databases), then torn down. It never touches the live+target. Drilling the *latest* point proves the whole incremental chain by+construction. A free-space precheck marks an under-provisioned drill+`skipped_no_space` rather than failing mid-extract, and only one drill runs at a+time per host.++The Protection panel shows a **Last proven restore** badge (never / ok / stale /+failed) and the Doctor raises a repairable warning when a cadenced policy is+overdue (> 1.5× its cadence) or its latest backup is unverified.++For **remote copies**, a weekly check compares the offsite object's size and —+for single-part uploads — its ETag/MD5; multipart uploads are honestly labelled+`size_only` until a **monthly deep check** downloads one sampled object and+hashes it against the *stored* sha256 (never a hash recomputed from the local+file, which could share the same corruption). Egress stays bounded to one object+per month.++Drill/verify failures and recoveries notify admins **once per transition**+(`backup.drill_failed` / `backup.verify_failed` and their recovery notices), not+on every sweep. The diagnostic support bundle includes a scrubbed+`restore_confidence` summary (levels + states only, no paths or secrets).+## Under the hood- **Models**: `BackupPolicy` (one per target) and `BackupRun` (one per run, the@@ -89,6 +133,10 @@ Restore runs as a `restore.run` job; you're notified on completion.- **Services**: `backup_policy_service.py` (policy CRUD, schedule wiring, the`backup.policy.run` / `restore.run` job handlers, retention),`backup_cost_service.py` (cost), and the smart-backup helpers in- `backup_service.py`.+ `backup_service.py`. Restore proof lives in `backup_verify_service.py`+ (manifest + tier-1 verifier), `backup_drill_service.py` (`backup.drill.run` ++ cadence/orphan sweeps, `RestoreDrill` model), `backup_offsite_service.py`+ (`backup.offsite.verify`), and `backup_alert_service.py` (edge-triggered+ drill/verify notices).- **Scheduling**: the policy's cron is mirrored into a `ScheduledJob` on theunified job bus, so firing is observable and retryable like any other job.diff --git a/frontend/src/components/backups/BackupHistoryList.jsx b/frontend/src/components/backups/BackupHistoryList.jsxindex 9694f6e9..8f17b1c2 100644--- a/frontend/src/components/backups/BackupHistoryList.jsx+++ b/frontend/src/components/backups/BackupHistoryList.jsx@@ -1,7 +1,11 @@+import { useState } from 'react';import { Pill } from '@/components/ds';import { Button } from '@/components/ui/button';import EmptyState from '@/components/EmptyState';-import { Archive, RotateCcw, ShieldCheck, Trash2, HardDrive, Cloud, Layers } from 'lucide-react';+import {+ Archive, RotateCcw, ShieldCheck, Trash2, HardDrive, Cloud, Layers,+ FlaskConical, ChevronRight, ChevronDown,+} from 'lucide-react';import { humanSize, formatMoney, formatWhen, statusKind, storageLabel } from './format';// Card 3 of the backup "Protection" panel: a data-table of backup runs.@@ -29,72 +33,167 @@ function storageIcon(run) {);}+// Pill kind for a restore-drill outcome. skipped_no_space is a soft warning+// (no disk headroom to drill), never a hard failure.+function drillStatusKind(status) {+ switch (status) {+ case 'success': return 'green';+ case 'failed': return 'red';+ case 'skipped_no_space': return 'amber';+ default: return 'gray';+ }+}++function drillStatusLabel(status) {+ if (status === 'skipped_no_space') return 'skipped (no space)';+ return status || 'unknown';+}++// Render whatever shape `probes` arrives in (array of checks, keyed object, or+// a plain string) as compact key/value rows without assuming a schema.+function renderProbes(probes) {+ if (!probes) return <span className="backup-drills__probe-empty">No probe details recorded.</span>;+ const entries = Array.isArray(probes)+ ? probes.map((p, i) => [p?.name || `Probe ${i + 1}`, p?.detail ?? p?.result ?? p?.ok ?? p])+ : (typeof probes === 'object' ? Object.entries(probes) : [['result', probes]]);+ return (+ <dl className="backup-drills__probes">+ {entries.map(([key, value]) => (+ <div key={key} className="backup-drills__probe">+ <dt>{key}</dt>+ <dd>{typeof value === 'object' ? JSON.stringify(value) : String(value)}</dd>+ </div>+ ))}+ </dl>+ );+}++function RestoreDrills({ drills }) {+ const [expanded, setExpanded] = useState(null);+ if (!drills || drills.length === 0) return null;++ return (+ <div className="backup-drills">+ <div className="backup-drills__head">+ <FlaskConical size={14} />+ <span>Restore drills</span>+ <span className="backup-drills__sub">Proof that these backups can actually be recovered.</span>+ </div>+ <ul className="backup-drills__list">+ {drills.map((drill) => {+ const isOpen = expanded === drill.id;+ return (+ <li key={drill.id} className="backup-drills__item">+ <button+ type="button"+ className="backup-drills__row"+ onClick={() => setExpanded(isOpen ? null : drill.id)}+ aria-expanded={isOpen}+ >+ <span className="backup-drills__chev">+ {isOpen ? <ChevronDown size={14} /> : <ChevronRight size={14} />}+ </span>+ <Pill kind={drillStatusKind(drill.status)}>{drillStatusLabel(drill.status)}</Pill>+ <span className="backup-drills__trigger">{drill.trigger || 'manual'}</span>+ <span className="backup-drills__when">{formatWhen(drill.started_at)}</span>+ {drill.duration_seconds != null && (+ <span className="backup-drills__meta">{drill.duration_seconds}s</span>+ )}+ {drill.bytes_restored != null && (+ <span className="backup-drills__meta">{humanSize(drill.bytes_restored)}</span>+ )}+ </button>+ {isOpen && (+ <div className="backup-drills__detail">+ {drill.error && <p className="backup-drills__error">{drill.error}</p>}+ {renderProbes(drill.probes)}+ </div>+ )}+ </li>+ );+ })}+ </ul>+ </div>+ );+}+export default function BackupHistoryList({runs,+ drills,loading,onRestore,onVerify,onDelete,onRowClick,}) {+ const hasDrills = drills && drills.length > 0;+if (loading && (!runs || runs.length === 0)) {return <EmptyState icon={Archive} title="Loading backups…" loading />;}if (!loading && (!runs || runs.length === 0)) {return (- <EmptyState- icon={Archive}- title="No backups yet"- description="Turn on protection or click Back up now."- />+ <>+ <RestoreDrills drills={drills} />+ {!hasDrills && (+ <EmptyState+ icon={Archive}+ title="No backups yet"+ description="Turn on protection or click Back up now."+ />+ )}+ </>);}return (- <table className="sk-dtable backup-history-list">- <thead>- <tr>- <th>Backup</th>- <th>Date</th>- <th>Size</th>- <th>Cost</th>- <th>Status</th>- <th>Storage</th>- <th aria-label="Actions" />- </tr>- </thead>- <tbody>- {runs.map((run) => (- <tr- key={run.id}- className={`backup-history-list__row ${onRowClick ? 'is-clickable' : ''}`}- onClick={onRowClick ? () => onRowClick(run) : undefined}- >- <td>- <div className="sk-cell-name">- <span className="backup-history-list__ico"><Archive size={14} /></span>- <span>{run.metadata?.backup_name || `Backup #${run.id}`}</span>- <Pill kind={run.kind === 'full' ? 'violet' : 'gray'} dot={false}>{run.kind}</Pill>- </div>- </td>- <td className="backup-history-list__when">{formatWhen(run.started_at)}</td>- <td className="sk-cell-mono">{humanSize(run.size_total)}</td>- <td className="sk-cell-mono">{formatMoney(run.cost_total)}</td>- <td><Pill kind={statusKind(run.status)}>{run.status}</Pill></td>- <td>{storageIcon(run)}</td>- <td>- <div className="backup-history-list__actions" onClick={(e) => e.stopPropagation()}>- <Button size="icon" variant="outline" title="Restore" disabled={run.status !== 'success'} onClick={() => onRestore(run)}><RotateCcw size={14} /></Button>- {run.remote_key && (- <Button size="icon" variant="outline" title="Verify remote copy" onClick={() => onVerify(run)}><ShieldCheck size={14} /></Button>- )}- <Button size="icon" variant="destructive" title="Delete" onClick={() => onDelete(run)}><Trash2 size={14} /></Button>- </div>- </td>+ <>+ <RestoreDrills drills={drills} />+ <table className="sk-dtable backup-history-list">+ <thead>+ <tr>+ <th>Backup</th>+ <th>Date</th>+ <th>Size</th>+ <th>Cost</th>+ <th>Status</th>+ <th>Storage</th>+ <th aria-label="Actions" /></tr>- ))}- </tbody>- </table>+ </thead>+ <tbody>+ {runs.map((run) => (+ <tr+ key={run.id}+ className={`backup-history-list__row ${onRowClick ? 'is-clickable' : ''}`}+ onClick={onRowClick ? () => onRowClick(run) : undefined}+ >+ <td>+ <div className="sk-cell-name">+ <span className="backup-history-list__ico"><Archive size={14} /></span>+ <span>{run.metadata?.backup_name || `Backup #${run.id}`}</span>+ <Pill kind={run.kind === 'full' ? 'violet' : 'gray'} dot={false}>{run.kind}</Pill>+ </div>+ </td>+ <td className="backup-history-list__when">{formatWhen(run.started_at)}</td>+ <td className="sk-cell-mono">{humanSize(run.size_total)}</td>+ <td className="sk-cell-mono">{formatMoney(run.cost_total)}</td>+ <td><Pill kind={statusKind(run.status)}>{run.status}</Pill></td>+ <td>{storageIcon(run)}</td>+ <td>+ <div className="backup-history-list__actions" onClick={(e) => e.stopPropagation()}>+ <Button size="icon" variant="outline" title="Restore" disabled={run.status !== 'success'} onClick={() => onRestore(run)}><RotateCcw size={14} /></Button>+ {run.remote_key && (+ <Button size="icon" variant="outline" title="Verify remote copy" onClick={() => onVerify(run)}><ShieldCheck size={14} /></Button>+ )}+ <Button size="icon" variant="destructive" title="Delete" onClick={() => onDelete(run)}><Trash2 size={14} /></Button>+ </div>+ </td>+ </tr>+ ))}+ </tbody>+ </table>+ </>);}diff --git a/frontend/src/components/backups/ProtectionPanel.jsx b/frontend/src/components/backups/ProtectionPanel.jsxindex 6ebb0fd6..cc659dab 100644--- a/frontend/src/components/backups/ProtectionPanel.jsx+++ b/frontend/src/components/backups/ProtectionPanel.jsx@@ -36,6 +36,7 @@ export default function ProtectionPanel({ targetType, targetId, targetName, showconst [loading, setLoading] = useState(true);const [saving, setSaving] = useState(false);const [backingUp, setBackingUp] = useState(false);+ const [drilling, setDrilling] = useState(false);const [historyView, setHistoryView] = useState('list'); // 'list' | 'calendar'const [dayFilter, setDayFilter] = useState(null); // Date | nullconst [detailRun, setDetailRun] = useState(null);@@ -117,6 +118,19 @@ export default function ProtectionPanel({ targetType, targetId, targetName, show}}, [targetType, targetId, toast, scheduleReloads]);+ const handleRunDrill = useCallback(async () => {+ setDrilling(true);+ try {+ await api.runBackupDrill(targetType, targetId);+ toast.success('Restore drill started');+ scheduleReloads();+ } catch (err) {+ toast.error(err.message || 'Failed to start restore drill');+ } finally {+ setDrilling(false);+ }+ }, [targetType, targetId, toast, scheduleReloads]);+// Restore is always initiated from a run — opening the restore drawer.const openRestore = useCallback((run) => setRestoreRun(run), []);@@ -177,10 +191,12 @@ export default function ProtectionPanel({ targetType, targetId, targetName, showpolicyView={view}onToggle={handleToggle}onBackupNow={handleBackupNow}+ onRunDrill={handleRunDrill}onViewGlobal={() => navigate('/backups')}onViewJobs={() => navigate('/jobs')}busy={saving}backingUp={backingUp}+ drilling={drilling || !!view?.restore_proof?.is_drilling}/><ScheduleCard@@ -236,6 +252,7 @@ export default function ProtectionPanel({ targetType, targetId, targetName, show) : (<BackupHistoryListruns={visibleRuns}+ drills={dayFilter ? [] : view?.restore_proof?.recent_drills}loading={loading}onRestore={openRestore}onVerify={handleVerify}diff --git a/frontend/src/components/backups/ScheduleCard.jsx b/frontend/src/components/backups/ScheduleCard.jsxindex 3cb557bd..c9a451f2 100644--- a/frontend/src/components/backups/ScheduleCard.jsx+++ b/frontend/src/components/backups/ScheduleCard.jsx@@ -1,32 +1,22 @@-import { useState, useEffect, useMemo } from 'react';+import { useState, useEffect } from 'react';import { SegControl } from '@/components/ds';import { Switch } from '@/components/ui/switch';import { Button } from '@/components/ui/button';import { Input } from '@/components/ui/input';import { Calendar, Save, Loader2 } from 'lucide-react';+import SchedulePicker from '../SchedulePicker';// Card 2 of the backup "Protection" panel: the editable schedule form.-// The saved policy carries a raw cron string; this card translates it into a-// friendly Daily / Weekly / Custom UI and back. Local state mirrors the form-// while a useEffect re-seeds it whenever the parent reloads the policy, so an-// external save stays in sync. Save is disabled until something actually+// The saved policy carries a raw cron string; this card now mounts the shared+// SchedulePicker (compact) so backups get the same Presets/Builder/Advanced UI+// and server-side preview as every other cron surface. Local state mirrors the+// form while a useEffect re-seeds it whenever the parent reloads the policy, so+// an external save stays in sync. Save is disabled until something actually// changes (dirty tracking) and shows a spinner while the request is in flight.-// Canonical cron strings for the two preset frequencies (02:00 server time).-const DAILY_CRON = '0 2 * * *';-const WEEKLY_CRON = '0 2 * * 0'; // Sunday-// Day-of-week chip labels; index 0 = Sunday … 6 = Saturday (cron dow order).-const DOW_LABELS = ['S', 'M', 'T', 'W', 'T', 'F', 'S'];--const DEFAULT_TIME = '02:00';+const DEFAULT_CRON = '0 2 * * *'; // Daily at 02:00 server time.const MIN_COUNT = 1; // floor shared by every numeric field (count/days/full-every)-// Two-digit zero-pad for assembling/displaying HH:MM from cron integers.-const pad = (n) => String(n).padStart(2, '0');--// True when a cron token is a plain non-negative integer (e.g. "2", not "*").-const isInt = (token) => /^\d+$/.test(token);-// Coerce a numeric form field (kept as a string) to a number, treating empty// or sub-floor values as the minimum so we never persist 0 / NaN.const toNum = (value) => {@@ -34,81 +24,30 @@ const toNum = (value) => {return Number.isFinite(n) && n >= MIN_COUNT ? n : MIN_COUNT;};-// Expand a cron day-of-week field into a sorted int array (0..6).-// "*"/"?" -> [] (every day); supports comma lists and "a-b" ranges.-function parseDow(field) {- if (!field || field === '*' || field === '?') return [];- const out = new Set();- for (const part of field.split(',')) {- const range = part.split('-');- if (range.length === 2 && isInt(range[0]) && isInt(range[1])) {- const lo = Number(range[0]);- const hi = Number(range[1]);- for (let d = lo; d <= hi; d += 1) {- if (d >= 0 && d <= 6) out.add(d);- }- } else if (isInt(part)) {- const d = Number(part);- if (d >= 0 && d <= 6) out.add(d);- }- }- return Array.from(out).sort((a, b) => a - b);-}--// Translate a saved cron string into the form's { frequency, time, days }.-function deriveFromCron(cron) {- if (cron === DAILY_CRON) return { frequency: 'daily', time: DEFAULT_TIME, days: [] };- if (cron === WEEKLY_CRON) return { frequency: 'weekly', time: DEFAULT_TIME, days: [0] };-- const fields = (cron || '').trim().split(/\s+/);- const min = fields[0];- const hour = fields[1];- const time = isInt(hour) && isInt(min) ? `${pad(Number(hour))}:${pad(Number(min))}` : DEFAULT_TIME;- return { frequency: 'custom', time, days: parseDow(fields[4]) };-}--// Assemble a cron string from the form state.-function buildCron(frequency, time, days) {- if (frequency === 'daily') return DAILY_CRON;- if (frequency === 'weekly') return WEEKLY_CRON;-- const [hour = '02', min = '00'] = (time || DEFAULT_TIME).split(':');- const dowField =- days.length > 0 && days.length < 7- ? days.slice().sort((a, b) => a - b).join(',')- : '*';- return `${Number(min)} ${Number(hour)} * * ${dowField}`;-}-const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {// All hooks run unconditionally, before any early return (Rules of Hooks).// The form is only shown once a policy exists, so the seed values used when// `policy` is null are throwaway and get re-seeded by the effect on load.- const initial = deriveFromCron(policy?.schedule_cron);-- const [frequency, setFrequency] = useState(initial.frequency);- const [time, setTime] = useState(initial.time);- const [days, setDays] = useState(initial.days);+ const [cron, setCron] = useState(policy?.schedule_cron || DEFAULT_CRON);const [retentionCount, setRetentionCount] = useState(String(policy?.retention_count ?? ''));const [retentionDays, setRetentionDays] = useState(String(policy?.retention_days ?? ''));const [fullEvery, setFullEvery] = useState(String(policy?.full_every_n_days ?? ''));const [compression, setCompression] = useState(policy?.compression ?? 'balanced');const [remoteCopy, setRemoteCopy] = useState(!!policy?.remote_copy);+ const [drillCadence, setDrillCadence] = useState(policy?.drill_cadence ?? 'off');// Re-seed the form when the saved policy changes (e.g. external reload after// a successful save). Keyed on the persisted values so it only fires when// the source of truth actually moves, not on every render.useEffect(() => {if (!policy) return;- const next = deriveFromCron(policy.schedule_cron);- setFrequency(next.frequency);- setTime(next.time);- setDays(next.days);+ setCron(policy.schedule_cron || DEFAULT_CRON);setRetentionCount(String(policy.retention_count));setRetentionDays(String(policy.retention_days));setFullEvery(String(policy.full_every_n_days));setCompression(policy.compression);setRemoteCopy(!!policy.remote_copy);+ setDrillCadence(policy.drill_cadence || 'off');}, [policy,policy?.schedule_cron,@@ -117,9 +56,10 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {policy?.full_every_n_days,policy?.compression,policy?.remote_copy,+ policy?.drill_cadence,]);- const currentCron = useMemo(() => buildCron(frequency, time, days), [frequency, time, days]);+ const currentCron = cron;// Conditional render AFTER all hooks have run.if (!policy) {@@ -136,12 +76,6 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {);}- const toggleDay = (i) => {- setDays((prev) =>- prev.includes(i) ? prev.filter((d) => d !== i) : [...prev, i].sort((a, b) => a - b)- );- };-// The form is dirty when any field diverges from the saved policy. Numeric// fields are compared via toNum so an empty box reads as the floor (= what// would actually be persisted), avoiding a phantom-dirty Save button.@@ -151,7 +85,8 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {toNum(retentionDays) !== policy.retention_days ||toNum(fullEvery) !== policy.full_every_n_days ||compression !== policy.compression ||- remoteCopy !== !!policy.remote_copy;+ remoteCopy !== !!policy.remote_copy ||+ drillCadence !== (policy.drill_cadence || 'off');const handleSave = () => {onSave({@@ -161,6 +96,7 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {full_every_n_days: toNum(fullEvery),compression,remote_copy: remoteCopy,+ drill_cadence: drillCadence,});};@@ -176,44 +112,7 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {<div className="app-panel-body"><div className="schedule-card__field"><label>Frequency</label>- <SegControl- options={[- { value: 'daily', label: 'Daily' },- { value: 'weekly', label: 'Weekly' },- { value: 'custom', label: 'Custom' },- ]}- value={frequency}- onChange={setFrequency}- />- </div>-- {frequency === 'custom' && (- <>- <div className="schedule-card__field">- <label>Time</label>- <Input type="time" value={time} onChange={(e) => setTime(e.target.value)} />- </div>- <div className="schedule-card__field">- <label>Days</label>- <div className="schedule-card__days">- {DOW_LABELS.map((d, i) => (- <button- type="button"- key={i}- className={`schedule-card__day ${days.includes(i) ? 'is-active' : ''}`}- onClick={() => toggleDay(i)}- >- {d}- </button>- ))}- </div>- </div>- </>- )}-- <div className="schedule-card__preview">- <span className="schedule-card__preview-label">Cron</span>- <code>{currentCron}</code>+ <SchedulePicker value={cron} onChange={setCron} compact /></div><div className="schedule-card__field schedule-card__retention">@@ -273,6 +172,25 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {/></div>+ <div className="schedule-card__field">+ <label>Restore drill cadence</label>+ <SegControl+ options={[+ { value: 'off', label: 'Off' },+ { value: 'weekly', label: 'Weekly' },+ { value: 'monthly', label: 'Monthly' },+ ]}+ value={drillCadence}+ onChange={setDrillCadence}+ />+ <p className="app-panel-hint">+ Periodically test-restore the latest backup to prove it can actually be+ recovered. Drills run into a throwaway location and never touch this site.+ Replaying an incremental backup chain during a drill needs GNU tar, so+ those drills run on Linux hosts only.+ </p>+ </div>+<div className="schedule-card__field schedule-card__remote"><Switchid="remote-copy"diff --git a/frontend/src/services/api/backupProtection.js b/frontend/src/services/api/backupProtection.jsindex a2489801..b2e30cbd 100644--- a/frontend/src/services/api/backupProtection.js+++ b/frontend/src/services/api/backupProtection.js@@ -43,6 +43,17 @@ export async function verifyBackupRun(targetType, targetId, runId) {return this.request(`${runsBase(targetType, targetId)}/${runId}/verify`, { method: 'POST' });}+// Trigger a restore drill (a proving test-restore into a throwaway location).+// Applications and WordPress sites use their own edit-gated, target-scoped+// endpoint (the same owner who can back up a target can drill it); any other+// target routes through the admin-only generic policy-by-target endpoint.+export async function runBackupDrill(targetType, targetId) {+ if (targetType === 'application' || targetType === 'wordpress_site') {+ return this.request(`${runsBase(targetType, targetId)}/drill`, { method: 'POST' });+ }+ return this.request(`/backups/policies/${targetType}/${targetId}/drill`, { method: 'POST' });+}+export async function deleteBackupRun(targetType, targetId, runId) {return this.request(`${runsBase(targetType, targetId)}/${runId}`, { method: 'DELETE' });}diff --git a/frontend/src/styles/pages/_backups.scss b/frontend/src/styles/pages/_backups.scssindex aa8e04cb..af48f954 100644--- a/frontend/src/styles/pages/_backups.scss+++ b/frontend/src/styles/pages/_backups.scss@@ -1,732 +1,10 @@-// ============================================-// BACKUPS PAGE (redesign — infra console look)-// ============================================--.backups-page {-- // ── KPI strip (4 tiles, +1 when a remote provider is set) ──- .bk-kpis {- display: grid;- grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));- gap: $space-4;- margin-bottom: $space-5;-- @media (max-width: $breakpoint-sm) { grid-template-columns: 1fr; }- }-- // ── list head: mono eyebrow + segmented filter + actions ─- .bk-listhead {- display: flex;- align-items: center;- gap: $space-3;- margin-bottom: $space-3;-- h2 {- flex: 1;- min-width: 0;- margin: 0;- font-family: var(--mono);- font-size: 10.5px;- font-weight: $font-weight-semibold;- letter-spacing: 0.08em;- text-transform: uppercase;- color: var(--text-faint);- }-- @media (max-width: $breakpoint-md) {- flex-wrap: wrap;-- h2 { flex-basis: 100%; }- }- }-- // ── dense tables (backup archive + schedules) ───────────- .bk-card {- background: var(--surface);- border: 1px solid var(--border-soft);- border-radius: var(--radius);- overflow: hidden;- }-- .bk-empty {- padding: $space-10;- text-align: center;- color: var(--text-faint);- font-size: 13px;- background: var(--surface);- border: 1px solid var(--border-soft);- border-radius: var(--radius);- }-- .bk-table {- // actions column shrinks to fit, buttons never wrap- th:last-child { width: 1%; }- td:last-child { white-space: nowrap; }-- tbody tr.is-disabled {- opacity: 0.55;-- .bk-ico {- background: var(--surface-3);- color: var(--text-faint);- }- }- }-- // backup kind tile (tinted per type)- .bk-ico {- width: 30px;- height: 30px;- border-radius: 8px;- display: grid;- place-items: center;- flex-shrink: 0;- background: var(--surface-2);- color: var(--text-dim);-- &--application { background: var(--accent-bg); color: var(--accent-bright); }- &--database { background: var(--violet-bg); color: var(--violet); }- &--files { background: var(--amber-bg); color: var(--amber); }- }-- // mono type chip (application / database / files) — %sk-state shape- // (_design-system.scss) with local sizing + bordered per-type tints kept- .bk-type {- @extend %sk-state;- padding: 2px 7px;- border-radius: 5px;- font-size: 10.5px;- background: var(--surface-2);- border: 1px solid var(--border);- color: var(--text-dim);-- &--application {- background: var(--accent-bg);- border-color: fade($accent-primary-raw, 30%);- color: var(--accent-bright);- }-- &--database {- background: var(--violet-bg);- border-color: rgba(176, 123, 245, 0.3);- color: var(--violet);- }-- &--files {- background: var(--amber-bg);- border-color: rgba(245, 185, 69, 0.3);- color: var(--amber);- }- }-- .bk-when {- font-family: var(--mono);- font-size: 12px;- color: var(--text-faint);- white-space: nowrap;- }-- .bk-actions {- display: flex;- justify-content: flex-end;- gap: $space-1;- }-- // schedule time chip (matches the cron expression chip)- .bk-sched {- display: inline-flex;- align-items: center;- gap: 5px;- padding: 2px 8px;- font-family: var(--mono);- font-size: 11.5px;- color: var(--text-dim);- background: var(--surface-2);- border: 1px solid var(--border);- border-radius: 6px;- white-space: nowrap;-- svg {- color: var(--text-faint);- flex-shrink: 0;- }- }-- // remote-upload indicator on schedule rows- .bk-remote-on { color: var(--green); }- .bk-remote-off { color: var(--text-ghost); }-- // storage pills carry a small lucide icon instead of the dot- .sk-pill svg { flex-shrink: 0; }-- @media (max-width: $breakpoint-md) {- .bk-card { overflow-x: auto; }- .bk-table { min-width: 720px; }- .bk-table--schedules { min-width: 880px; }- }-- // ── storage tab: real-data spec strip ───────────────────- .bk-specs {- display: grid;- grid-template-columns: repeat(3, 1fr);- gap: $space-3;- margin-bottom: $space-4;-- @media (max-width: $breakpoint-sm) { grid-template-columns: 1fr; }- }-- // storage/settings cards → surface chrome + eyebrow headers- .card {- background: var(--surface);- border: 1px solid var(--border-soft);- border-radius: var(--radius);- }-- .card-header h3 {- margin: 0;- font-family: var(--mono);- font-size: 10.5px;- font-weight: $font-weight-semibold;- letter-spacing: 0.08em;- text-transform: uppercase;- color: var(--text-faint);- }-- .storage-provider-config {- margin: $space-4 0;- padding: $space-4;- background: var(--surface-2);- border: 1px solid var(--border-soft);- border-radius: var(--radius-sm);-- h4 {- margin: 0 0 $space-4;- font-family: var(--mono);- font-size: 10.5px;- font-weight: $font-weight-semibold;- letter-spacing: 0.08em;- text-transform: uppercase;- color: var(--text-faint);- }- }-- // ── forms (cards + modals): mono eyebrow labels ─────────- .form-group > label:not(.checkbox-label) {- font-family: var(--mono);- font-size: 10.5px;- font-weight: $font-weight-semibold;- letter-spacing: 0.08em;- text-transform: uppercase;- color: var(--text-faint);- }-- .form-row {- display: grid;- grid-template-columns: 1fr 1fr;- gap: $space-4;-- @media (max-width: $breakpoint-sm) { grid-template-columns: 1fr; }- }-- .form-help {- display: block;- margin-top: $space-2;- font-family: var(--mono);- font-size: 11px;- color: var(--text-faint);- }-- // inline hint inside an eyebrow label — undo the uppercase- .form-help-inline {- font-family: var(--mono);- font-size: 10px;- font-weight: $font-weight-normal;- letter-spacing: normal;- text-transform: none;- color: var(--text-ghost);- }-- .form-actions {- display: flex;- gap: $space-2;- margin-top: $space-5;- }-- .checkbox-label {- display: flex;- align-items: center;- gap: $space-2;- cursor: pointer;-- input[type="checkbox"] {- width: 16px;- height: 16px;- cursor: pointer;- accent-color: var(--accent);- }-- span {- font-size: 13px;- color: var(--text);- }- }-- // ── restore modal ───────────────────────────────────────- .bk-restore-warn {- display: flex;- gap: 11px;- padding: 13px 15px;- margin-bottom: $space-5;- border-radius: 11px;- background: var(--amber-bg);- border: 1px solid rgba(245, 185, 69, 0.28);- font-size: 12.5px;- line-height: 1.5;- color: var(--text-dim);-- svg {- color: var(--amber);- flex-shrink: 0;- margin-top: 1px;- }-- b { color: var(--text); }- }-- .bk-restore-details {- padding: $space-1 $space-4;- background: var(--surface-2);- border: 1px solid var(--border-soft);- border-radius: var(--radius-sm);- }-- // ── spinners (upload / test-connection buttons) ─────────- .spinning {- animation: spin 1s linear infinite;- }-}--@keyframes spin {- from { transform: rotate(0deg); }- to { transform: rotate(360deg); }-}--// Reusable inline spinner for the protection components (lucide <Loader2 className="spin" />).-.spin { animation: spin 1s linear infinite; }--// ============================================-// PROTECTION PANEL (backup protection — WordPress + Services settings)-// Rendered inside .app-overview-grid > .app-overview-left, so these styles are-// top-level (not nested under .backups-page).-// ============================================-.protection-panel {- // Single column inside a settings pane (no right rail).- grid-template-columns: 1fr;-- .app-overview-left {- display: flex;- flex-direction: column;- gap: $space-4;- }-}--// ── Card 1: status ──────────────────────────────────────────-.protection-status-card {- &__toggle {- display: flex;- align-items: flex-start;- gap: $space-3;- padding-bottom: $space-4;- border-bottom: 1px solid var(--border-soft);-- label {- display: flex;- flex-direction: column;- gap: 2px;- cursor: pointer;- }- }-- &__toggle-title { font-weight: $font-weight-semibold; color: var(--text); }- &__toggle-sub { font-size: 12px; color: var(--text-faint); }-- &__status {- display: flex;- align-items: center;- flex-wrap: wrap;- gap: $space-2;- margin: $space-4 0;- font-size: 13px;- color: var(--text-dim);- }-- &__kpis {- display: grid;- grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));- gap: $space-3;- margin-bottom: $space-4;-- @media (max-width: $breakpoint-sm) { grid-template-columns: 1fr; }- }-- &__actions {- display: flex;- align-items: center;- flex-wrap: wrap;- gap: $space-2;- }-}--// ── Card 2: schedule ────────────────────────────────────────-.schedule-card {- .app-panel-body {- display: flex;- flex-direction: column;- gap: $space-4;- }-- &__field {- display: flex;- flex-direction: column;- gap: $space-2;-- > label {- font-size: 12px;- font-weight: $font-weight-semibold;- color: var(--text);- }- }-- &__days {- display: flex;- gap: $space-2;- }-- &__day {- width: 32px;- height: 32px;- border: 1px solid var(--border-soft);- background: var(--surface-2);- color: var(--text-dim);- border-radius: $radius-sm;- font-size: 12px;- font-weight: $font-weight-semibold;- cursor: pointer;- transition: border-color .12s ease, background .12s ease, color .12s ease;-- &:hover { border-color: var(--accent-bright); }-- &.is-active {- background: var(--accent-bg);- color: var(--accent-bright);- border-color: var(--accent-bright);- }- }-- &__preview {- display: flex;- align-items: center;- gap: $space-2;-- code {- font-family: var(--mono);- font-size: 12px;- color: var(--text);- background: var(--surface-2);- border: 1px solid var(--border-soft);- border-radius: $radius-sm;- padding: 2px 8px;- }- }-- &__preview-label {- font-family: var(--mono);- font-size: 10.5px;- text-transform: uppercase;- letter-spacing: 0.04em;- color: var(--text-faint);- }-- &__retention-row {- display: flex;- align-items: center;- gap: $space-2;- font-size: 13px;- color: var(--text-dim);-- input { width: 80px; }- }-- &__remote {- flex-direction: row;- align-items: flex-start;- gap: $space-3;-- label {- display: flex;- flex-direction: column;- gap: 2px;- cursor: pointer;- }- }-- &__actions { margin-top: $space-1; }-}--// ── Card 3: history list ────────────────────────────────────-.backup-history-list {- width: 100%;-- &__row.is-clickable { cursor: pointer; }-- &__ico {- display: inline-flex;- color: var(--text-faint);- }-- &__when {- color: var(--text-dim);- font-size: 13px;- white-space: nowrap;- }-- &__actions {- display: flex;- justify-content: flex-end;- gap: $space-1;- }-- &__storage {- display: inline-flex;- align-items: center;- gap: 4px;- font-size: 12px;- color: var(--text-dim);- }-}--// ── History card chrome (list/calendar toggle + day-filter chip) ────-.backup-history-card {- &__tools {- display: flex;- align-items: center;- gap: $space-2;- }-- &__toggle {- display: inline-flex;- border: 1px solid var(--border-soft);- border-radius: $radius-sm;- overflow: hidden;-- button {- display: inline-flex;- align-items: center;- justify-content: center;- width: 28px;- height: 26px;- background: var(--surface-2);- color: var(--text-dim);- border: none;- cursor: pointer;-- &.is-active { background: var(--accent-bg); color: var(--accent-bright); }- & + button { border-left: 1px solid var(--border-soft); }- }- }-- &__chip {- display: inline-flex;- align-items: center;- gap: 4px;- padding: 2px 8px;- font-size: 12px;- color: var(--text-dim);- background: var(--surface-2);- border: 1px solid var(--border-soft);- border-radius: $radius-full;- cursor: pointer;-- &:hover { color: var(--text); }- }-}--// ── Calendar view ───────────────────────────────────────────-.backup-calendar {- &__head {- display: flex;- align-items: center;- gap: $space-2;- margin-bottom: $space-3;- }-- &__month {- flex: 1;- font-weight: $font-weight-semibold;- color: var(--text);- }-- &__nav,- &__today {- display: inline-flex;- align-items: center;- justify-content: center;- height: 28px;- padding: 0 $space-2;- background: var(--surface-2);- border: 1px solid var(--border-soft);- border-radius: $radius-sm;- color: var(--text-dim);- cursor: pointer;-- &:hover { border-color: var(--accent-bright); color: var(--text); }- }-- &__grid {- display: grid;- grid-template-columns: repeat(7, 1fr);- gap: $space-1;- }-- &__dow {- text-align: center;- font-family: var(--mono);- font-size: 10px;- text-transform: uppercase;- letter-spacing: 0.04em;- color: var(--text-faint);- padding-bottom: $space-1;- }-- &__cell {- position: relative;- aspect-ratio: 1 / 1;- display: flex;- flex-direction: column;- align-items: center;- justify-content: center;- border: 1px solid var(--border-soft);- border-radius: $radius-sm;- background: var(--surface-2);- color: var(--text-dim);- font-size: 12px;-- &--empty { background: transparent; border-color: transparent; }-- &--has {- cursor: pointer;-- &:hover { border-color: var(--accent-bright); }- &.is-success { border-color: var(--green); background: var(--green-bg); }- &.is-failed { border-color: var(--red); background: var(--red-bg); }- &.is-running { border-color: var(--cyan); background: var(--cyan-bg); }- }- }-- &__badge {- position: absolute;- top: 2px;- right: 3px;- font-size: 9px;- font-weight: $font-weight-semibold;- color: var(--text-faint);- }-}--// ── Detail drawer ───────────────────────────────────────────-.backup-detail-drawer {- display: flex;- flex-direction: column;- gap: $space-4;-- &__error {- padding: $space-2 $space-3;- font-size: 13px;- color: var(--red);- background: var(--red-bg);- border: 1px solid var(--border-soft);- border-radius: $radius-sm;- }-- &__meta {- display: grid;- gap: $space-1;- }-- &__row {- display: flex;- justify-content: space-between;- gap: $space-3;- padding: 6px 0;- border-bottom: 1px solid var(--border-soft);- font-size: 13px;-- span:first-child { color: var(--text-faint); }- span:last-child { color: var(--text); }- }-- &__cost {- font-size: 13px;- color: var(--text-dim);- font-family: var(--mono);- }-- &__actions {- display: flex;- flex-wrap: wrap;- gap: $space-2;- }-- &__joblink {- display: inline-flex;- align-items: center;- gap: 4px;- font-size: 13px;- color: var(--accent-bright);- }-}--// ── Restore drawer ──────────────────────────────────────────-.restore-drawer {- display: flex;- flex-direction: column;- gap: $space-4;-- &__section { display: flex; flex-direction: column; gap: $space-2; }- &__label { font-size: 12px; font-weight: $font-weight-semibold; color: var(--text); }-- &__tables {- display: flex;- flex-direction: column;- gap: $space-1;- max-height: 180px;- overflow-y: auto;- padding: $space-2;- border: 1px solid var(--border-soft);- border-radius: $radius-sm;-- label { display: flex; align-items: center; gap: $space-2; font-size: 13px; color: var(--text-dim); }- }-- &__options { display: flex; flex-direction: column; gap: $space-3; }-- &__option {- display: flex;- align-items: flex-start;- gap: $space-3;-- label { display: flex; flex-direction: column; font-size: 13px; color: var(--text); cursor: pointer; }- }-- &__warning {- display: flex;- align-items: flex-start;- gap: $space-2;- padding: $space-2 $space-3;- font-size: 13px;- color: var(--amber);- background: var(--amber-bg);- border: 1px solid var(--border-soft);- border-radius: $radius-sm;- }-}+LdbCls2cOX529YMxMz86/EK7TsxOpe4t6kpI7BC641eG9mHH7NVye8FM7vi+tL9v0qu9bYJ6pzb0CwUHAy0ZKWSdJx7Gq29zY1TnVmlBarE5P3L/w/HFv7EXeXsAr0VfhTb85V3tfcHlMR/QLbzcVKIbHfPua1b7b8+cqRWAdFXvv7agazNpvEoUB5VKONznN17CxjvmNZfykjN1aWBsLP6CUTaNXpwuvakQdiMSJTS5pGOVKAcXtcST+63/46nJ2Dmc+Vcvd1bTggna7Svr0/fuA7r6lHaLcGnK6DO1qwGGy3t7lHve4KniN4YcLKI7TYpMjMxV85FMMyRobTU69k71xSyJT0YwaHbidW068FnQZ+L3DzoMlGTu1qHwN4iZH9yL4p2lcgN5EEED8dGLLW5uLeeeAb0wJbrSg9sJQH8qKkGO5DuhjcmnigNZH0oJIb7/W+hRg4p4cviVksEKOgjRALuucGqXuq+R9g5mdimKauZXzaDPx5aeGzSgDcBihc36BS5MrIeYh7G/k7mllUUebh6g52CSgjj0gKxn4SkrzCyvM7yXod9MicpxG5r81n8HAgbZo1ACDVVyslkv2YgbkKNz39Aio521nZHXgTwtRGHjd/hSWBFAHpxW1VPpbsNvteNza+Qd0gKuaFeNyheF/R0kPQop6+SEDXk1bpJdXjDn3QrzvBCUM+JdZcTcRkDVQyI4Kg8mpIOp46JixyKXCh1bULPiD93t6w93/bxbKfOcGudX/d2IX2chD0yT7gSfV9B4/lZnshkEQFH71G4I9TMJJ2d/MfTLv9dQL4WU7UjQXOMTpwDQS4LfQSU5oin7Pw4dNyJ3RhWvMzi57Kx8UXrr6iou3/ZI4O8iWbq1Do3Ev1HInHFjBnRosDzwMnN/5LQNzVEhnerPmpW/GFQ0uEadQ3n3ahBHMuhrap2qUELMP28uwHXoYP61twxle2IL69Y+itfAGNKk0j/TkOXSyrpKvMkI3vyyWe9+GV8EZDkF6RhNrjgH9c2R4lUMOimFc59spWVyQrnANcBZ1TJExSQnrXQ2gYBw1i0bjiCf88Xt82eQ4XJYirDwOpac163g+R2b+uXFqGkvGeb7gVpZpP8Q65jwcRwShs+Do/N5yI0DtzNZVXrbJHyDuehfpPYgqGgemOg/rrcWNyER33TgYLwbIDDPkQZsl4xHBtFrbj3ISoLuACKbr7pZXXF8D/K1t6jWeSIPoebs7WjS5POT5dkbYhTx01dlTYrvTkzwXhieqmh9Oz/qdzCcSIR6beiL9x5wCeEtnxYaW2faEiCOATUJOuRJP2CrVYoWrcsG0AFj0/InNf2DDJmG1xEAWHBI05vPIRnNCuFi7SM0DRI6aIoa61/ysJ8kdkSJtbOSAv+cdOiiw2rOrpXiQkPDN3x94psoa4opVSSqifOrU4byOcbWXOJ7I1Nq/cd0Jim8H4Ptz6m0tP836TULUiOsk6UR/0/b/CWXQ4y7XLdm8mpIouWlFObBsGLrBwYCVttdnd8sWb3Be9eJJ2W8H3gb0C+Ai9kH4GZC2D4GWmoW1wHTS09wUqYZMkqhGf0RJXaPAi5sRnmPzVLl2c5P0QQWq0XYatcnm4l0grE4WnF8VMzqBN15rhb0KRWMRjS/WjPL4OKMR6xlWNuUmSjLh8lA769QqXaUQarIAivoHL/+cHkw9oRPwGGGICts1CQhNJ7GvfFaLn+ZOVHLuZ4sV9Sq+MGs7nwPFHCiX7LXHt/3xkOS+IhxUlTBrMQXsO75SkSltyqtZuNliegA1g/vIWYlsw0Qz2IjDRDTdSVoJUY7TOxBQ/GeNEjzBNsur7dZqxxwpSbus6B8uFcMg6L+NkqCUOZDZyOhn0MX6A6Mj4ogJpdIedZrHIG6MlRaibbqUUqXOH69k5riaoGtVxeSvXitaDkCiVE0ds9tPuwRpSRWgdiK9ow9V73vP0tGED8SjdyOjLNZ283EXEFaArM4A2bpl62DO9vGYxogQWjlIRnIghmJ+Skj0bPD5LueD1xG+dx/Zl9OIFK/6EphQAz+3DH249uXTja0w0tqeEOc4eNd6SbuZZh1wDcjwhJUIlJHEYEuxB+yL2Joj9Pw7sVl2or13I6QFzUKDucSUfjktAqYdUWeRrnewG2cuz3ptm/ng4hutM7OkL74k0WEhVXDrDVv0rrqDYcNByLb0lzgOhtaBHsyR/Nel/z4EZPEY471bT7islXgsewHjWg5aSzmKlHO9O0mrtRIX1PwlwXSNTRbhMMIluBxydlOq2oWXQEIqhoKc/JhFKge6czzPTXOVi8sNrghvmEpzGeYcG6hbKzhzIaF3HTWh/57VyD2N2VLML7365/Jtyz5u7W7lCbyPueH+8Eq1voBLUK3+W6HbuKPagr39M0nXFazmUXIeLtB5oPawaejea5LLVIng2fAYVFf1CnbpzF61H5tSDZps/KmlF9+S7pSceACd4EaxSDuTYcWUaf6R+5WCJIlHv0AnMzN/A+eu/7xTUhIlFV4LeSqhBAzr+HZjsfMustVFDkxHmZU8syK+7ydIRpGbAm1jk+ACxAyhP2OUJWPX36Ul1gWBzJN4cpAdSdzabfUFpkBV2tgHjGj5sWTnjC/wyoyEyTEo2ZENJkxSXLb4DKFudvRtogIsvmFTfIKtaMe7efHW7frTp3b5y7AWkZXZqXGrxXufm8Wi/cYck2UxeqMnVrTQrTimynACFfr3ByMYTsp0MZQGgQ2OL2qdPJVePlEosk3rYrKB4DXUJ0DS1A+r/+/upvWRZfqLY4YHz//fd5BF+86FCAnm57TJV9yggF438GKiGXz0oWxyDO/AdJnPn+N7KKb9thLp0b4WcNf8RjgdCjMT+zJGTjBuwz6WwvISD3Pi0QahIkpAKtu38JHbKo5S2u2QMDmGAVQHuxcB2s8QKuBOQ7u8uJailqWcBUNQB8/pqBKSAjUcZueRdNJ8/76yttgFJaLhnaWXTEe2joA55qX71f0Hn+VUmY7GX+KRr3ZJp+TuJItEXhxlOMwvIcALyCco+OkZ9IN58A7IqGFqJBNdkLUjPDpBEODvjkBSIQbu1U67tARfnCq2LZijGt8AJlDEG130gPac28wPVkUobkZ7QTOOiIDFlWQ+8oUGB3MaOCGn34rxc1t0yOiL0rreI14vfDxv8Idgj+yP2C82WUkpojU8i/+oTrDLM42hIgy5x5fcjTdLWYoyYsxJog2U55nKeUmeJu3lvduwV/CXTL69dfRhLEZPn7MfSt7U33bp/QdwDy9HfCjj9A7rCTs4PXiGrJjZ3HfNvVdF7xHQu262yXP0xcOBceK5+RiF0Xc/SDctNHh5zvIQIFbgHJFBSxaA+a7Sko+Nt49TU/aOWV0SdhAtO4/BreOv9/fvR6aiwlmd9Qunw4I40u4I1QSaZr7gG7IB9nt6BRaJBg548DFiVufmlWwIqHg4wX6yaFDGg6BieSquZourJ7w4Z0GsH8m3tTMq2B3fUBDpev4IXN3QyVHdQVCPU5pmcbVxoxs1O/ZX++Wr1DJHrZsvkB/KvMZEpB0AMnhC8eLbsszefUQk9xcJCFR5XKssAjs1IbBDyYMjLZqI50UxMF8QAaoGktFXPLD9AUKgAdqFP2TlxhmaBixEN1rXk6UNQikK6pWFIFclsGj0SZo/oSYxaIe9O785JzXfpjJzlH0V8gbeAlRTwz8BTFJPexJMVJv0DZA8hjxgR4t8dgYxLwg+XP68vb6M03VhLug6hcep4yRoB3QsJI+FQMKVaQRS3vQcsyUmtZi6x9zWf/OhohTwQ6bZRRzl0EwBpivxoxdCnifLaxAqS9RV9gCGnWxzhpfwv2gNhj5I1uY+Wp6MyGFdno2sVIRWChMODmsR2k0hkJZ4yfdJgFK9extMH8R96sa+SOlPYMQAPLL4r8W6fyagyfZFwjFMVYy7c66XJIyoNsy7QCw1bxZvkFj+MvsqcCYcp1H1hH0gSLxL8vsgtDKYrzm4m32rxBKyYdqx7U9udwlaiHPS3n02CRwXnMr4PvoQ4OfVPovU1dG6SLPuBLt0kqpyZInhdBBQgDr+4vlhyve/WgNrGvwGgYEcSIkdldlpGHau4OhI3dAm/v3yYYbHLzULH3Sxdpkd8x4rEIJpRrAKprgxCEhO3kw3T5Ob4vXxdXgrRVHPGd40z4SmFkIfVj0a5EdGxe7vlt7NR7ABTLrdvamJ44nht9T/qqyhfbicpj2bNUkEtomqnzusyLcBTyJhBclIxMKnpYSITx0fMEPosCc3ghVx3yvUYAQ=="}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":3328,"cache_read_input_tokens":258148,"output_tokens":1984,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":3328,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":1984,"cache_read_input_tokens":258148,"cache_creation_input_tokens":3328,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":3328},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CcnxA9cThuB62hk1nQa9x","type":"assistant","uuid":"dd93c7f8-826a-4eb8-8674-31eae9f9214b","timestamp":"2026-07-07T15:42:37.720Z","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}+{"parentUuid":"dd93c7f8-826a-4eb8-8674-31eae9f9214b","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxAC2ZhePYuahgxeMmH","type":"message","role":"assistant","content":[{"type":"text","text":"Critical findings: Documents is **not** OneDrive-backed, Recycle Bin bypassed. The unpushed commits only survive if there's a Volume Shadow Copy / restore point, or the GitHub remote. Let me check both recovery avenues:"}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":3328,"cache_read_input_tokens":258148,"output_tokens":1984,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":3328,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":1984,"cache_read_input_tokens":258148,"cache_creation_input_tokens":3328,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":3328},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CcnxA9cThuB62hk1nQa9x","type":"assistant","uuid":"0310623c-db69-4a0d-8002-967d7527480d","timestamp":"2026-07-07T15:42:38.597Z","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}+{"parentUuid":"0310623c-db69-4a0d-8002-967d7527480d","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxAC2ZhePYuahgxeMmH","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01N4eBeWoctiXCTxxptNQgm7","name":"PowerShell","input":{"command":"Write-Output \"=== Restore points ===\"; try { Get-ComputerRestorePoint -ErrorAction Stop | Format-Table SequenceNumber, CreationTime, Description -Auto } catch { \"restore-point query: $($_.Exception.Message)\" }\nWrite-Output \"=== vssadmin shadows ===\"; & vssadmin list shadows 2>&1 | Select-Object -First 30\nWrite-Output \"=== Previous Versions dir listing via wmic ===\"; & cmd /c \"vssadmin list shadowstorage\" 2>&1 | Select-Object -First 15\nWrite-Output \"=== git remote reachable? ===\"; & git ls-remote https://github.com/jhd3197/ServerKit.git 2>&1 | Select-Object -First 15","description":"Check restore points, shadow copies, remote"},"caller":{"type":"direct"}}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":3328,"cache_read_input_tokens":258148,"output_tokens":1984,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":3328,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":1984,"cache_read_input_tokens":258148,"cache_creation_input_tokens":3328,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":3328},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CcnxA9cThuB62hk1nQa9x","type":"assistant","uuid":"e1c3e155-b961-4b17-8951-00a5b7570c2c","timestamp":"2026-07-07T15:42:42.234Z","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}+{"type":"last-prompt","lastPrompt":"# ServerKit Plan Executor > Set the plan path below, paste this whole file as your message. Claude executes every > phase, one by one, with subagents — no questions, no stopping early. ## PLAN ```…","leafUuid":"e1c3e155-b961-4b17-8951-00a5b7570c2c","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498"}+{"type":"ai-title","aiTitle":"Execute ServerKit staging testbed plan","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498"}+{"type":"mode","mode":"normal","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498"}+{"parentUuid":"e1c3e155-b961-4b17-8951-00a5b7570c2c","isSidechain":false,"promptId":"487ba573-d3df-4194-aced-415c39485354","type":"user","message":{"role":"user","content":[{"type":"tool_result","content":"Exit code 2\n=== Restore points ===\r\nrestore-point query: Access denied \r\n=== vssadmin shadows ===\r\nvssadmin 1.1 - Volume Shadow Copy Service administrative command-line tool\r\n(C) Copyright 2001-2013 Microsoft Corp.\r\n\r\nError: You don't have the correct permissions to run this command. Please run this utility from a command\r\nwindow that has elevated administrator privileges.\r\n \r\n=== Previous Versions dir listing via wmic ===\r\nvssadmin 1.1 - Volume Shadow Copy Service administrative command-line tool\r\n(C) Copyright 2001-2013 Microsoft Corp.\r\n\r\nError: You don't have the correct permissions to run this command. Please run this utility from a command\r\nwindow that has elevated administrator privileges.\r\n \r\n=== git remote reachable? ===\r\n578b034448c492f905628217c90c6dd73da80937\tHEAD\r\n07baefb53f12e070f600746443856ea7fa059c32\trefs/heads/dev\r\n578b034448c492f905628217c90c6dd73da80937\trefs/heads/main\r\n739a106d0763c57e267e83924c6a68f011e7b55e\trefs/heads/new-ui\r\n976ba5ad372b8477bfe476786a6060ef4542d0c8\trefs/pull/1/head\r\n1e8afb7a438aa318b678dbf7f05dcce8229462b9\trefs/pull/10/head\r\n62fda1697626f1c67cd7168a60193b8c97fb524e\trefs/pull/11/head\r\ne1fe2a290ef230c158839c396917c942f063f770\trefs/pull/13/head\r\nbefcdb9e9e1e413b848a9e14a9d76c8079a30c42\trefs/pull/14/head\r\n401fd71402b2dddd7196e6ba117081160003c9a5\trefs/pull/15/head\r\n34ca8dee8fc5b1d03379144f1484be9227eaab35\trefs/pull/16/head\r\nd9e12d1eb14844ad1ad33ebf3b07fa270c7661ed\trefs/pull/17/head\r\n5f0b70b06c4d029328635a7115208783056502be\trefs/pull/18/head\r\nbbf24a4704ea7a7f29b87539bd87956b2d6e99e7\trefs/pull/2/head\r\n43004416f9f74e8346c6b31d96ce3ff4769ce0df\trefs/pull/20/head","is_error":true,"tool_use_id":"toolu_01N4eBeWoctiXCTxxptNQgm7"}]},"uuid":"5ac719ea-4bb5-47e7-a9ae-eeaf451b415a","timestamp":"2026-07-07T15:42:46.010Z","toolUseResult":"Error: Exit code 2\n=== Restore points ===\r\nrestore-point query: Access denied \r\n=== vssadmin shadows ===\r\nvssadmin 1.1 - Volume Shadow Copy Service administrative command-line tool\r\n(C) Copyright 2001-2013 Microsoft Corp.\r\n\r\nError: You don't have the correct permissions to run this command. Please run this utility from a command\r\nwindow that has elevated administrator privileges.\r\n \r\n=== Previous Versions dir listing via wmic ===\r\nvssadmin 1.1 - Volume Shadow Copy Service administrative command-line tool\r\n(C) Copyright 2001-2013 Microsoft Corp.\r\n\r\nError: You don't have the correct permissions to run this command. Please run this utility from a command\r\nwindow that has elevated administrator privileges.\r\n \r\n=== git remote reachable? ===\r\n578b034448c492f905628217c90c6dd73da80937\tHEAD\r\n07baefb53f12e070f600746443856ea7fa059c32\trefs/heads/dev\r\n578b034448c492f905628217c90c6dd73da80937\trefs/heads/main\r\n739a106d0763c57e267e83924c6a68f011e7b55e\trefs/heads/new-ui\r\n976ba5ad372b8477bfe476786a6060ef4542d0c8\trefs/pull/1/head\r\n1e8afb7a438aa318b678dbf7f05dcce8229462b9\trefs/pull/10/head\r\n62fda1697626f1c67cd7168a60193b8c97fb524e\trefs/pull/11/head\r\ne1fe2a290ef230c158839c396917c942f063f770\trefs/pull/13/head\r\nbefcdb9e9e1e413b848a9e14a9d76c8079a30c42\trefs/pull/14/head\r\n401fd71402b2dddd7196e6ba117081160003c9a5\trefs/pull/15/head\r\n34ca8dee8fc5b1d03379144f1484be9227eaab35\trefs/pull/16/head\r\nd9e12d1eb14844ad1ad33ebf3b07fa270c7661ed\trefs/pull/17/head\r\n5f0b70b06c4d029328635a7115208783056502be\trefs/pull/18/head\r\nbbf24a4704ea7a7f29b87539bd87956b2d6e99e7\trefs/pull/2/head\r\n43004416f9f74e8346c6b31d96ce3ff4769ce0df\trefs/pull/20/head","sourceToolAssistantUUID":"e1c3e155-b961-4b17-8951-00a5b7570c2c","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}+{"parentUuid":"5ac719ea-4bb5-47e7-a9ae-eeaf451b415a","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxCgqyAWrW7sHmVtEk6","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"","signature":"EpIiCmMIDxgCKkCM83hkUpKsTAdmjQp4vaBEBrTs/q6o49h7HnmFvPTNFVbiGnNcjzlGZQKnB3yabCxxeN+4KfpbPbWdBrwpeEHHMg9jbGF1ZGUtb3B1cy00LTg4AEIIdGhpbmtpbmcSDNo1GA/83N9NkvkQ9RoMo5aU3Kqe56UyV/CkIjBX8QcqeyjcOI2tVS8kHUG9BhsEUc0iJlFz5c8n7vjSP4MFScy6QFD/jjsSpTeZjUYq3CCUKlpZbUpsH4poqn9OY4X5Xeh4vFU+JCmMgedHIBx3xMjzEu2mBmaC0wOPH21TF5Mk4piWemSm8oVOEniiOAf2FC4tCsvOAWK3hdXKmxeBiKKI2G8z8j6lGf4etRGWbYiA6VdwTBiUPAB7bEJfRLbC1Yl+vsqTaCtx/VSj9bEGtjS9UaJdnU629rJmTZauDkfylfLbBaZKzDeaa4UVxQZt1FKhHFLlEvaZ9A31kxalIGnSWA36z4/vL0gBi0bpWecj+9Sct/zPYcSVD4edSUq1za9TTKjB57Sy5Ycv1xhvntTYoFxvKPSvSKqRycLYsKQkx0+BL5mVJvloE6mL1aOR4IcqJco3/uyOzxwITjBy3T9Hp8yqwNT2/2kYr1x3aKxU+XelNTjb5bTRwf3Q2iqin9rlggdEgwO7MpPqpHSxXIDL0Cq5PU4x/7cM1mCF0FX5P2kk6jD7ZhO9sZFOCp3UntusycVrQYgthciBedqnHT1UcvAWMJ7sLIqxmtqax50oUF++7plG69QXNEUbzzahJ5uLJmx7oOUuSX/CmD4H4j0yqjmUSR48m5EtdTv6hZJS5lmapQms93Nf4Bva0aSBPNVJ5spKSzYPY8Jm3ksOof/mnQl3t0iFRTIP5JxKFl4waT3aCVslD2IpcrnEjkDxyHs+c+OXViU2C2P+dPodnpKHEdFShZnWahR4FxB3TkhcGePnPVb0+t5/2jT+8ZflgvDMfSOp79vzypgcPBwxaHHiYGQx/xRBJm+W/COvYnub9U0JeBq5Q6N/dg8ss7ieWCf1hohTHMjeY74NDb8VBBYU6iliD9xMOh+yZSXOv+NIdjLCXZ3ievPFvxHUT4q5Yh46XEYfcfBj1iiSCPvf1xMM+oKjoBMGDhm/c9VhBHEitSKzcmdQ380tPho5F0vSx1sfhVbjU2Hx8QTlCDzKwvDvg8FOW26dbjCr182M4Rqs7y9r/82uZMBBi2tfnx0A+VVr+d4tA2tcoexWBKfgEt80//yRMtScgMXpsQibkxf/rlPjUCO8gGJql5dKB0BS+CTqQGhaL9q9kDoL1fjFdfzpyGt3rhTxjynRrgJsQBVsVymuafrhsehl6+bQQYiQp0NYW+PtoadbPYcErSZoPJBDMezG0zLwf1pj566vtd896tHP7VJduEC2Vq376M0N/nE/VQXzVnEikeR/v5XZH/y/ZcB+JLrs63Ng3Kht/vFUncJp71OAmhKZD8slEs78i0XYOzty1OxgaGQJzZ1Qy2y/4bvW9k+xvQllgm5WzmFtLYspP63WEFSlBNu8SNUKw8iC9M/sFg3G2xu0q56WbiM6bYQ4i63tzLVH7JqEG87CbdejCdv6VXco1QUNH7BW8x+IX9toaPYsd2jbyHqv3vThsue3mK0U+KDibDbzKc27SHmhR1re8CTLrINaKb+u+U7A1Cp0nvPyIZxVA+d34E3FCG3ctqMN49tAXybRp+3FWOMx9iQlR9Hf9CUxf6Syfox0mvUG2qBnog+bprOh1fZgIdj+imEYfXcyXedEljl7ZQmmC1nmfiHFcN8T6CGZRrTrxLYKUG3Oj9bo9YwG3TQRpWy91HzdZZ1S974eB8cplHDfAjjRVjhmuAeB+t0zr5FBokxODUjHpVism43WHH+tknTtqHUkk6HA5sFyOhfHi0AODbiBGAIkKUz2v4yrLKXSQWuTgbxRU+9ejCge4F1PlFwDB8pmUc83y+06LIZH+AEpxAWcWwYQ1m980bGnpXwgU19zMqMgDXP02pM1/G4zOeK9I197OP+6GT0kkduBcVDOS1Nrd4c2LcvWUBYCFhfsJUXIO13tON9vwTCom1dJTFYvZUHj7op5iwYcn1StqbhglXDro+ZQf0utu7vkRrCaM7ZBZWrzXcK8XMLyoChS4bwUk1p5teyQbBRypp8L/sV1vuYYShmfoxewPzlVYvCD1O3nWtP63DwR1K6MHboojE1cue/6qucVViLamob76T3QRe5RJ1/o2Ec9ZFq1BBlSnDMNVqapEhSQJURfVesxsM+UUuGYGkj+278wT79/D9YFo+cKlUn2gk1S8xZ+cNIFSYDuIb9Fypby533cVXk+mVwzTzpNzzOER/Da4zHWNDkmSSUhQktUNP3hs8KaYuNkXdZzeIVkDxU5sXtD0yfv33ir2zRYDGoDveEs9KJ4IUhueqBD/PYj++Ur9MhZ3W+sZG7W94jC3hwNrldc1FxgP8GnkEyUUjYscLzmD5Lc1Ii2ahQKpGS4n5CaTd5+pC1a5k5HbG+O38rrlzzY4ocuaAht0MH5Ya0babz2TKS673sjPtX2OrIguY67x7PE9ffG17qhC8lDK4UJXYYHK5Vs27Qx/AcKXKj070N6GfqV+k79yPRor239fBG3qn0xQKb1WuWbDfrsNeWL8j0Dh/5s/A9VJxuue2ExlusdBkFCsuM8eFdDbn79fMqTDFoWles1Rb3cHWnJZ4DxqzA5X8B3VKmTGi24CL9KnP9NWMgYjKFtzhVSR8eaa3zJNpcRktMDz5nRw8CV7jx+yuL0QK4YsnOwSOSjR0kqNKMvaaHX+59F875aOjJc8S9rk+0hsQaBgCOmCAjRqp2iAR6Pp3q2qdetNVGWRD9Y+rqq5v9i2tWpuV7v8bsLF8kjGOxyo+r8zwDtYaKuHxCrwisviaXrSd42/XGsbpdMG52wN4oSqd9FjEs+xVui9LNxMGy7glmASKDeyf/eQBrFFHMe82SAYzP+Z41RgW/pXwofoYxs5+aFodL18lpoJp+TLQoVV/Q1gPEJ+u0j9C34nzhbspF2q14DZLKJa9tULjxVaa0wLsKjKWiSE0pusL78zny/02LmFXlYm+Y4Em6v02nzusvYC56lP6l6bKe317D+J1YN8dyQO1LM5E55wUXkZIfF2pw4sdbupNFvnZC3YxgoR/7+4JPdkMe6KLkxSMZtrsMr44HmVm/F/xJQWC38c0EmWMzczX8YlXTxR4/72SAEeO6uNdvo+jzc2Wlh6qOZBlQN/O4LuDfdPu/bpcF9V7VHLsAq74L8uWRwqpMdp4KY5SUyX6YH8kLm818vdo5fkDyiEmy5VujBVlwrZmb3TslcY2zKag3Ga+aEtF18efaHVseoQphsH4mV/O8l7GH8simO0TysDXTCYwEkboZpXGFkvAmKaQRhwSWMXS6XF0CZkdfwHWo9sLQTrLq7CSeblwbpAvmrjSCb5cIe0Ngr5jtoJA/c0eCLh7ZS2OG2QKTgyQeRSOaXWxQsVGzMHyF/Xv1UnajStSSktgJYOKX0QM+Oxy6egaJuS4Mb5IRssYLKSvQ2w4hydViXEop/wUyI3v7NOz/js7YzXZ7zPvFjYwM+4flijh7KH5A0QMCp4BQQ7YOOOzN/snLEqLKSOOy32WKEyCnNL698dWhNfZOT3SzRhtinYJU3aL9HPV36apg3TgNWfBfM8IfSbq2Q8hGa0B6VjT4wmjIhmt5Sl6yKxxNho4XBok9gjTi8aNdse+ZRSUBet0uDlObrm+ZqX58ppfHLlF6LFp3+zpefiYGxlhZQu78ykIdBXAeU1e1sxi55yHSkYfsaE6Dt5MmBZu31WABjRC/hYn1nM7D/jvXqq+6W3TP9r6ES8mCE/JgiNS+bBTmu7VfIRhDKUZKvAxT7Sq5kdY9Sqa7rck4ICK6roEv6dfkjCqfURlwseGzb7EVni8oHVBp7qMoUT3hqt0XiIPttcKjOVqiAXZ9ArWW91A1gpin2DQRKoQDmPpWa5SMdjxmhxPzdp6MtjFsCCXOEf/wU1r94y8V84U+hlKxzZQqWnrIumpFTUdyqa0oa9zdMxlBGGy4AZIgHgKAlNA9CD30CVD29Cr0oYw3ZML3a5+HAeMGjwJ+e5Lpjx7S7tGldYc/TD4s3uZi6vbw8SlHSwfxskYdXdFq6DffMP5dKep2QlvKSUClODXtKnj7ETM6rg+4nvv1DtJsPHse2tRodJ2vtNlrvTZlCku15Hizi8GlZwX+NmFVI/y+bHvVlylnsAMKSH68IHkqy2YY5a0LWzihQp0dbMPGzSPXLH+zC8TmbRNJGRI/N725YgNkSm0fjE1AEmW7tKRSg1pOauJ8Ld/vUASy1nKsaBMvAjQvRSgrdN+djfSt8eM3J6srvpbyUGMhmNNXCmcVfRr2jCKnr1uxTRBRpn96LvMa4k6RJ2wKL+/rI9N9Mrpsj2IrgKJmOC/zUo5ijtuBjuelV9lxaB+iWpXIiVKz5wsPKcrcmC4Q7mNPQDtlctQFmlpiDNhwIyuaNEOSXmVseuMrGBoUpgPp90p7BmEikcggYLObwaQ2C3fYUJ0aWXJrTgKwYoIGn4JFYpl/5VOQ7cRk30942wUtdevm3WvieyS2CynQ6To74noZ7bO3qLeFRrA97nadPuyRmo1vaHXTb8bxO+ljXPgaWJD6QBCXFTTkQZGctOzIYnlHDIiYXl1D/SzTDdD8ZGJdxiEll3OaNb8qNdsCefBdaE+y6eut6ATHHZssDTUBGCiX3yV8+lydxQbdpaOKXCIgbG54JOUEu6+IkR3VWrMRlQcwqUsChorI6+4tQOAdzZQRZA0OWSqW1RkLaf0FafVTauQpsocHknpNnCGsq2WaQneEa8BOik/xzoNzma3JxZ/A8qHbAJRzoU+P/0hUEMJA4ALbcp6W6t8fBN7GkVwvhXUIvXc/H/CU99XXBV859HefOy83aFyKLpBC6MEYYT+PuMHD0VzT4BpEjXdqO3/82qSJ5FRX0y5KlRBlLh795XUp437dT1iC+Ac9sUEwO2VxhZDqXsUG39Sw6VvVykLf3qfVbTlNBEKeKgQkz/2GNJ9/tbbT3Jhv/6oNI+fPxZdeYAErH8GkeLxXU2U/mqxsnoPJrpjuL1Ywhs3iaBOqCDH+0LI2hY4Kt1dWQ5muLx/q24ouFk0l0TP4HUv3P9GODra4Iwnu7fdyrfZWNTZS/zRlZA6YwZ2zZR84TCk6dXgpZDTpDCAxnXiWUq9QSElJW6jCOPrtRcNm7R+0vL4YSGEQTF4dWkUKZVVduRgElFYkGZwTxBzfPi6zAoLXzscEa38O6xCvuqEXtMsKejieXwoa2JraV3aSlR7+5pNdUysu84k/ujpOZxy3tCb6Zv4/rcDxOrTan2mvTrtIZaRQq+m3eqoR1vl7hnw6S2tMaqST4JcLsmd/R5PDvAhIg6uixGJBe7PdCIqzdrlI9qCPW6etXIDc1gOIZnUA90Ft2Jo4neAt9WmSe3CKeMLfoi1Bgl46Ws67eAwLijeuElHrGoofZp82Y2y4MKhkeMNRUEWFzk5BK7e5aHYb6v9hvkphKUlpJpS1OlVJdLyJz+Uf0xPJIWBBcI0wie/NzvY0copgFaxIcn8Ze2pWEnxqaPtzhOwuX1QWVRcmsATRKiT2ZpMXoZfzxjRAzrGuYdsIBzsUfk477KzD1lAejtEN+hMv/Q8Y716+aYxwa2rt48M6f4CSW7wpFKM8lRc2fQS1lw3LL7+pgTuE1VJfFXdL+YoS4hbCousvzMTu6I/AZ1l3wr3kA1czX82aAboXPsglHAEO1CrtkED1yaBR/F1II8uUMQOGCQg6KEHxoXkAYAQ=="}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":2797,"cache_read_input_tokens":261476,"output_tokens":2090,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":2797,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":2090,"cache_read_input_tokens":261476,"cache_creation_input_tokens":2797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":2797},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CcnxCdtcwMwH94on8fJ6v","type":"assistant","uuid":"1a34a991-1644-4726-8618-96c08a2cba2a","timestamp":"2026-07-07T15:43:13.995Z","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}+{"parentUuid":"1a34a991-1644-4726-8618-96c08a2cba2a","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxCgqyAWrW7sHmVtEk6","type":"message","role":"assistant","content":[{"type":"text","text":"The situation is now clear and serious. Let me do one more **non-elevated** recovery check (File History), then I must stop and report to you — this needs your decision and elevated access:"}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":2797,"cache_read_input_tokens":261476,"output_tokens":2090,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":2797,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":2090,"cache_read_input_tokens":261476,"cache_creation_input_tokens":2797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":2797},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CcnxCdtcwMwH94on8fJ6v","type":"assistant","uuid":"f8e04adb-fafb-4b12-855a-7d82123acc09","timestamp":"2026-07-07T15:43:14.826Z","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}+{"parentUuid":"f8e04adb-fafb-4b12-855a-7d82123acc09","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxCgqyAWrW7sHmVtEk6","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01MWz2Ja4hPnZG7qF1PHtNT2","name":"PowerShell","input":{"command":"Write-Output \"=== File History configured? ===\"\n$fh = Get-ItemProperty 'HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\FileHistory' -ErrorAction SilentlyCon\ No newline at end of file