Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(backup): trust round 2 — restore drills, offsite/verify (plan 30)

Juan Denis7d045f32af9a
diff --git a/backend/app/models/backup_policy.py b/backend/app/models/backup_policy.py
index 6ba08979..4fd54e76 100644
--- a/backend/app/models/backup_policy.py
+++ b/backend/app/models/backup_policy.py
@@ -60,6 +60,13 @@ class BackupPolicy(db.Model):
# Remote
remote_copy = db.Column(db.Boolean, default=False, nullable=False)
+ # Restore-drill cadence (plan 23). 'off' | 'weekly' | 'monthly'. A single
+ # daily sweep enqueues due drills; last_drill_* is the denormalized cache
+ # that powers the "last proven restore" badge + doctor staleness check.
+ drill_cadence = db.Column(db.String(12), default='off', nullable=False)
+ last_drill_at = db.Column(db.DateTime, nullable=True)
+ last_drill_status = db.Column(db.String(24), nullable=True) # success|failed|skipped_no_space
+
# Hooks (optional shell snippets run before/after a backup)
pre_backup_hook = db.Column(db.Text, nullable=True)
post_backup_hook = db.Column(db.Text, nullable=True)
@@ -112,6 +119,9 @@ class BackupPolicy(db.Model):
'full_every_n_days': self.full_every_n_days,
'compression': self.compression,
'remote_copy': self.remote_copy,
+ 'drill_cadence': self.drill_cadence or 'off',
+ 'last_drill_at': self.last_drill_at.isoformat() if self.last_drill_at else None,
+ 'last_drill_status': self.last_drill_status,
'pre_backup_hook': self.pre_backup_hook,
'post_backup_hook': self.post_backup_hook,
'last_run_at': self.last_run_at.isoformat() if self.last_run_at else None,
diff --git a/backend/app/models/backup_run.py b/backend/app/models/backup_run.py
index d302b50a..a39012d8 100644
--- a/backend/app/models/backup_run.py
+++ b/backend/app/models/backup_run.py
@@ -42,11 +42,39 @@ class BackupRun(db.Model):
storage_path = db.Column(db.Text, nullable=True) # local archive directory/path
remote_key = db.Column(db.Text, nullable=True)
- verified = db.Column(db.Boolean, default=False)
+ verified = db.Column(db.Boolean, default=False) # remote-side evidence (legacy S3 HEAD)
+
+ # Verification ladder (plan 23). 'none' → 'listed' (tar -t readable) → 'hashed'
+ # (manifest sha256 spot-check passed) → 'drilled' (a real restore drill of a
+ # chain ending at this run succeeded). Only a drill ever sets 'drilled'.
+ verify_level = db.Column(db.String(12), nullable=False, default='none')
+ verified_at = db.Column(db.DateTime, nullable=True)
+ verify_error = db.Column(db.Text, nullable=True)
+ # sha256 of the primary archive, computed at build time (Decision 3).
+ checksum_sha256 = db.Column(db.String(64), nullable=True)
error_message = db.Column(db.Text, nullable=True)
metadata_json = db.Column(db.Text, default='{}') # tables, files, compression, etc.
+ # Ordered verification levels, weakest → strongest.
+ VERIFY_LEVELS = ('none', 'listed', 'hashed', 'drilled')
+
+ @classmethod
+ def _verify_rank(cls, level):
+ try:
+ return cls.VERIFY_LEVELS.index(level or 'none')
+ except ValueError:
+ return 0
+
+ def effective_verify_level(self):
+ """Verify level mapping the legacy ``verified`` boolean forward: a run
+ with no explicit level but a remote-verified copy reads as 'listed' (the
+ remote HEAD proves it exists + size-matches) so old rows aren't 'none'."""
+ level = self.verify_level or 'none'
+ if level == 'none' and self.verified:
+ return 'listed'
+ return level
+
def get_metadata(self):
if not self.metadata_json:
return {}
@@ -80,6 +108,10 @@ class BackupRun(db.Model):
'storage_path': self.storage_path,
'remote_key': self.remote_key,
'verified': self.verified,
+ 'verify_level': self.effective_verify_level(),
+ 'verified_at': self.verified_at.isoformat() if self.verified_at else None,
+ 'verify_error': self.verify_error,
+ 'checksum_sha256': self.checksum_sha256,
'error_message': self.error_message,
'compression': meta.get('compression'),
'tables': meta.get('tables'),
diff --git a/backend/app/models/restore_drill.py b/backend/app/models/restore_drill.py
new file mode 100644
index 00000000..709c2a89
--- /dev/null
+++ b/backend/app/models/restore_drill.py
@@ -0,0 +1,76 @@
+import json
+from datetime import datetime
+
+from app import db
+
+
+class RestoreDrill(db.Model):
+ """A single restore *drill* — a real restore of a policy's latest restorable
+ point into a scratch location (temp dir / scratch DB), probe-verified, then
+ torn down. Never touches the live target.
+
+ A successful drill is the strongest proof a backup is restorable: it promotes
+ the drilled :class:`BackupRun` to ``verify_level='drilled'`` and stamps the
+ policy's ``last_drill_*`` cache. Drilling the latest point implicitly proves
+ the whole chain (incrementals are useless without their base).
+
+ See docs/plans/23_BACKUP_TRUST_RESTORE_DRILLS_PLAN.md (Decision 4).
+ """
+
+ __tablename__ = 'restore_drills'
+
+ id = db.Column(db.Integer, primary_key=True)
+ policy_id = db.Column(db.Integer, db.ForeignKey('backup_policies.id'), nullable=False, index=True)
+ # The run whose restore-point (chain endpoint) was drilled.
+ run_id = db.Column(db.Integer, db.ForeignKey('backup_runs.id'), nullable=True, index=True)
+ job_id = db.Column(db.String(36), nullable=True, index=True)
+
+ # 'running' | 'success' | 'failed' | 'skipped_no_space'
+ status = db.Column(db.String(24), nullable=False, default='running')
+ trigger = db.Column(db.String(20), nullable=True) # 'manual' | 'scheduled'
+
+ started_at = db.Column(db.DateTime, default=datetime.utcnow)
+ finished_at = db.Column(db.DateTime, nullable=True)
+ duration_seconds = db.Column(db.Integer, nullable=True)
+
+ # Space accounting (operator's own disk — raw bytes, never a fee).
+ bytes_required = db.Column(db.BigInteger, nullable=True) # estimated extracted size
+ bytes_free = db.Column(db.BigInteger, nullable=True) # free space at precheck
+ bytes_restored = db.Column(db.BigInteger, nullable=True) # actually written to scratch
+
+ scratch_ref = db.Column(db.Text, nullable=True) # scratch dir path / scratch DB name
+ probes_json = db.Column(db.Text, default='{}') # probe results + error tail
+ error = db.Column(db.Text, nullable=True)
+
+ def get_probes(self):
+ if not self.probes_json:
+ return {}
+ try:
+ return json.loads(self.probes_json)
+ except (json.JSONDecodeError, TypeError):
+ return {}
+
+ def set_probes(self, value):
+ self.probes_json = json.dumps(value or {})
+
+ def to_dict(self):
+ return {
+ 'id': self.id,
+ 'policy_id': self.policy_id,
+ 'run_id': self.run_id,
+ 'job_id': self.job_id,
+ 'status': self.status,
+ 'trigger': self.trigger,
+ 'started_at': self.started_at.isoformat() if self.started_at else None,
+ 'finished_at': self.finished_at.isoformat() if self.finished_at else None,
+ 'duration_seconds': self.duration_seconds,
+ 'bytes_required': self.bytes_required,
+ 'bytes_free': self.bytes_free,
+ 'bytes_restored': self.bytes_restored,
+ 'scratch_ref': self.scratch_ref,
+ 'probes': self.get_probes(),
+ 'error': self.error,
+ }
+
+ def __repr__(self):
+ return f'<RestoreDrill {self.id} policy={self.policy_id} {self.status}>'
diff --git a/backend/app/services/backup_alert_service.py b/backend/app/services/backup_alert_service.py
new file mode 100644
index 00000000..4c56c885
--- /dev/null
+++ b/backend/app/services/backup_alert_service.py
@@ -0,0 +1,114 @@
+"""Edge-triggered restore-proof alerts (plan 23 Phase 5).
+
+Drills and verification run on a schedule, so a naive "notify on failure" would
+re-alert every sweep. These alerts are STATE-TRANSITION only: a failure fires
+once (the first sweep that flips healthy→failed for a policy) and a recovery
+fires once (the first success after a failure). The previous outcome per policy
+is remembered in SettingsService, keyed by policy id.
+
+Events (category ``backups``, admin-targeted):
+ * ``backup.drill_failed`` / ``backup.drill_recovered``
+ * ``backup.verify_failed`` / ``backup.verify_recovered``
+"""
+import logging
+
+logger = logging.getLogger(__name__)
+
+_DRILL_KEY = 'backup_alert_drill_{policy_id}'
+_VERIFY_KEY = 'backup_alert_verify_{policy_id}'
+
+
+class BackupAlertService:
+
+ # ------------------------------------------------------------------ #
+ # State store (best-effort)
+ # ------------------------------------------------------------------ #
+
+ @staticmethod
+ def _get_state(key):
+ try:
+ from app.services.settings_service import SettingsService
+ return SettingsService.get(key)
+ except Exception: # noqa: BLE001
+ return None
+
+ @staticmethod
+ def _set_state(key, value):
+ try:
+ from app.services.settings_service import SettingsService
+ SettingsService.set(key, value)
+ except Exception as exc: # noqa: BLE001
+ logger.debug('could not persist backup alert state %s: %s', key, exc)
+
+ @staticmethod
+ def _policy_label(policy):
+ return f'{policy.target_type}:{policy.target_id}'
+
+ @staticmethod
+ def _notify(event, data, policy=None):
+ # Carry the policy target so the notification deep-links straight to its
+ # protection panel (focus=policy:<target_type>:<target_id>, plan 33).
+ if policy is not None:
+ data = {**data, 'target_type': policy.target_type, 'target_id': policy.target_id}
+ try:
+ from app.plugins_sdk import notify
+ notify.send(event, to='admins', data=data, category='backups')
+ except Exception as exc: # noqa: BLE001 — a notification must never break a sweep
+ logger.debug('notify %s failed: %s', event, exc)
+
+ # ------------------------------------------------------------------ #
+ # Drill outcome
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def on_drill_result(cls, policy, status, error=None):
+ """Record a drill's outcome and fire a failed/recovered notice only on a
+ state transition. ``status`` is the RestoreDrill status."""
+ outcome = 'failed' if status in ('failed', 'skipped_no_space') else 'success'
+ key = _DRILL_KEY.format(policy_id=policy.id)
+ prev = cls._get_state(key)
+ label = cls._policy_label(policy)
+ if outcome == 'failed' and prev != 'failed':
+ cls._notify('backup.drill_failed', {
+ 'policy': label, 'target': label, 'status': status,
+ 'error_message': (error or status or 'drill failed')[:300],
+ 'message': (f'A restore drill for {label} {status}. The latest backup '
+ 'may not be restorable — open Backups → the policy to '
+ 'inspect the drill, or Monitoring → Doctor to re-run it.'),
+ }, policy=policy)
+ elif outcome == 'success' and prev == 'failed':
+ cls._notify('backup.drill_recovered', {
+ 'policy': label, 'target': label,
+ 'message': f'Restore drills for {label} are passing again.',
+ }, policy=policy)
+ cls._set_state(key, outcome)
+
+ # ------------------------------------------------------------------ #
+ # Verification outcome
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def on_verify_result(cls, policy, run):
+ """Record a run's verification outcome and fire on transition. A run is a
+ verification failure if it has a ``verify_error`` or its offsite deep
+ check reported a mismatch."""
+ offsite = (run.get_metadata() or {}).get('offsite') or {}
+ failed = bool(run.verify_error) or (offsite.get('deep_match') is False)
+ outcome = 'failed' if failed else 'success'
+ key = _VERIFY_KEY.format(policy_id=policy.id)
+ prev = cls._get_state(key)
+ label = cls._policy_label(policy)
+ if outcome == 'failed' and prev != 'failed':
+ detail = run.verify_error or 'The offsite copy did not match its stored checksum.'
+ cls._notify('backup.verify_failed', {
+ 'policy': label, 'target': label,
+ 'error_message': detail[:300],
+ 'message': (f'The latest backup for {label} failed verification: {detail} '
+ 'Open Backups → the policy to inspect it.'),
+ }, policy=policy)
+ elif outcome == 'success' and prev == 'failed':
+ cls._notify('backup.verify_recovered', {
+ 'policy': label, 'target': label,
+ 'message': f'Backups for {label} are verifying cleanly again.',
+ }, policy=policy)
+ cls._set_state(key, outcome)
diff --git a/backend/app/services/backup_offsite_service.py b/backend/app/services/backup_offsite_service.py
new file mode 100644
index 00000000..15f19e2e
--- /dev/null
+++ b/backend/app/services/backup_offsite_service.py
@@ -0,0 +1,235 @@
+"""Offsite verification hardening (plan 23 Phase 4).
+
+For remote-copy policies, prove the offsite object actually matches — not just
+"the upload call returned":
+
+ * **weekly shallow** (`backup.offsite.verify`) — HEAD + size + single-part
+ ETag/MD5 compare for the newest run of every remote-copy policy, recording
+ an honest evidence label (``checksum`` where a single-part ETag confirms
+ content, ``size_only`` where a multipart ETag can only confirm size).
+ * **monthly deep** — ONE deterministically sampled run per destination gets a
+ full download-and-hash against the STORED sha256 (Decision 3 — never a hash
+ recomputed from the same possibly-corrupt local file). Egress-bounded to one
+ object/month and the downloaded bytes are reported raw (Decision 8).
+
+Everything here is best-effort: a verification failure is recorded on the run's
+offsite evidence + surfaced to the drawer, never raised out of the sweep.
+"""
+import logging
+import os
+import random
+import shutil
+from datetime import datetime
+
+from app import db
+
+logger = logging.getLogger(__name__)
+
+OFFSITE_VERIFY_JOB_KIND = 'backup.offsite.verify'
+OFFSITE_VERIFY_SCHEDULE_NAME = 'backup-offsite-verify'
+
+
+class BackupOffsiteService:
+
+ # ------------------------------------------------------------------ #
+ # Candidate selection
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def _remote_copy_policies(cls):
+ from app.models.backup_policy import BackupPolicy
+ return BackupPolicy.query.filter_by(remote_copy=True).all()
+
+ @classmethod
+ def _newest_remote_run(cls, policy):
+ from app.models.backup_run import BackupRun
+ return (BackupRun.query
+ .filter(BackupRun.policy_id == policy.id,
+ BackupRun.status == 'success',
+ BackupRun.remote_key.isnot(None))
+ .order_by(BackupRun.started_at.desc()).first())
+
+ @staticmethod
+ def _is_deep_week(now):
+ """Monthly cadence on a weekly schedule: deep-check on ~1 week in 4."""
+ return (now.isocalendar()[1] % 4) == 0
+
+ @classmethod
+ def _sample_one(cls, runs, seed):
+ """Deterministically pick one run (seeded — reproducible for a given
+ seed, so the monthly egress is bounded AND testable)."""
+ if not runs:
+ return None
+ ordered = sorted(runs, key=lambda r: r.id)
+ return ordered[random.Random(seed).randrange(len(ordered))]
+
+ @staticmethod
+ def _destination_of(run):
+ """A stable key for the destination a run's offsite copy lives in. Today
+ every run shares one configured provider (one group), but the day a
+ second bucket/provider lands (plan 36) each destination samples on its
+ own — so a full backup is never left unproven because a different
+ destination happened to win the single-sample lottery."""
+ meta = run.get_metadata() or {}
+ dest = (meta.get('offsite') or {}).get('destination')
+ if dest:
+ return str(dest)
+ try:
+ from app.services.backup_cost_service import BackupCostService
+ prov = BackupCostService.configured_remote_provider()
+ if prov is not None:
+ return str(getattr(prov, 'name', None) or getattr(prov, 'id', None) or prov)
+ except Exception: # noqa: BLE001
+ pass
+ return 'default'
+
+ @classmethod
+ def _sample_per_destination(cls, runs, seed):
+ """One deterministically sampled run per distinct destination (plan 30
+ #10 / Decision 7). A no-op vs the old one-run-total behaviour while
+ storage is single-provider; correct the day a second destination exists."""
+ groups = {}
+ for r in runs:
+ groups.setdefault(cls._destination_of(r), []).append(r)
+ picks = []
+ for dest in sorted(groups.keys()):
+ pick = cls._sample_one(groups[dest], f'{seed}:{dest}')
+ if pick is not None:
+ picks.append(pick)
+ return picks
+
+ # ------------------------------------------------------------------ #
+ # Verify handlers
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def run_offsite_verify(cls, job=None):
+ payload = (job.get_payload() if job else None) or {}
+ now = datetime.utcnow()
+ deep = payload.get('deep')
+ if deep is None:
+ deep = cls._is_deep_week(now)
+ seed = payload.get('seed', now.isocalendar()[1])
+
+ results = {'shallow': [], 'deep': []}
+ candidates = []
+ for policy in cls._remote_copy_policies():
+ run = cls._newest_remote_run(policy)
+ if not run:
+ continue
+ candidates.append(run)
+ ev = cls._shallow_verify(policy, run)
+ results['shallow'].append({'policy_id': policy.id, 'run_id': run.id,
+ 'evidence': ev.get('evidence'),
+ 'verified': ev.get('verified')})
+
+ if deep and candidates:
+ for pick in cls._sample_per_destination(candidates, seed):
+ deep_res = cls._deep_verify(pick)
+ results['deep'].append({'run_id': pick.id, **deep_res})
+ # A deep mismatch is a verification failure — alert on transition.
+ try:
+ from app.models.backup_policy import BackupPolicy
+ from app.services.backup_alert_service import BackupAlertService
+ pol = BackupPolicy.query.get(pick.policy_id)
+ if pol:
+ BackupAlertService.on_verify_result(pol, pick)
+ except Exception as exc: # noqa: BLE001
+ logger.debug('offsite verify alert skipped: %s', exc)
+
+ db.session.commit()
+ logger.info('offsite verify: %s shallow, %s deep',
+ len(results['shallow']), len(results['deep']))
+ return results
+
+ @classmethod
+ def _shallow_verify(cls, policy, run):
+ """HEAD + size + single-part content compare via verify_run, which records
+ the honest evidence label ('checksum' | 'size_only') on the run."""
+ from app.services.backup_policy_service import BackupPolicyService, BackupPolicyError
+ try:
+ res = BackupPolicyService.verify_run(policy, run.id)
+ return {'verified': res.get('verified'), 'evidence': res.get('evidence')}
+ except BackupPolicyError as exc:
+ return {'verified': False, 'evidence': 'error', 'error': str(exc)}
+ except Exception as exc: # noqa: BLE001
+ logger.warning('shallow offsite verify failed for run %s: %s', run.id, exc)
+ return {'verified': False, 'evidence': 'error', 'error': str(exc)}
+
+ @classmethod
+ def _deep_verify(cls, run):
+ """Download the remote object and hash it against the STORED sha256 — the
+ only check that can promote a multipart ``size_only`` run to real content
+ proof. Egress-bounded (one object) and the bytes are reported raw."""
+ from app.services.storage_provider_service import StorageProviderService
+ from app.services.backup_service import BackupService
+ from app.services.backup_verify_service import sha256_file
+
+ meta = run.get_metadata() or {}
+ offsite = dict(meta.get('offsite') or {})
+ stored = offsite.get('our_sha256') or run.checksum_sha256
+ remote_key = run.remote_key
+ if not remote_key:
+ return {'ok': False, 'reason': 'no remote key'}
+ if not stored:
+ offsite['deep_evidence'] = 'no_stored_hash'
+ meta['offsite'] = offsite
+ run.set_metadata(meta)
+ return {'ok': False, 'reason': 'no stored sha256 to compare against'}
+
+ tmp_dir = os.path.join(BackupService.BACKUP_BASE_DIR, 'restores', 'offsite-verify')
+ os.makedirs(tmp_dir, exist_ok=True)
+ tmp_path = os.path.join(tmp_dir, f'deep-{run.id}.bin')
+ bytes_downloaded = 0
+ try:
+ dl = StorageProviderService.download_file(remote_key, tmp_path)
+ if not dl.get('success'):
+ offsite['deep_evidence'] = 'download_failed'
+ offsite['deep_error'] = dl.get('error')
+ meta['offsite'] = offsite
+ run.set_metadata(meta)
+ return {'ok': False, 'reason': dl.get('error')}
+ bytes_downloaded = dl.get('size') or (os.path.getsize(tmp_path)
+ if os.path.exists(tmp_path) else 0)
+ actual = sha256_file(tmp_path)
+ match = bool(actual and actual == stored)
+ offsite['deep_evidence'] = 'deep_sha256'
+ offsite['deep_match'] = match
+ offsite['deep_bytes'] = bytes_downloaded
+ offsite['deep_checked_at'] = datetime.utcnow().isoformat() + 'Z'
+ if match:
+ # A confirmed offsite content match is at least as strong as a
+ # single-part checksum — the run's remote copy is proven.
+ run.verified = True
+ offsite['evidence'] = 'checksum'
+ meta['offsite'] = offsite
+ run.set_metadata(meta)
+ return {'ok': match, 'bytes': bytes_downloaded, 'evidence': 'deep_sha256'}
+ except Exception as exc: # noqa: BLE001
+ logger.warning('deep offsite verify failed for run %s: %s', run.id, exc)
+ offsite['deep_evidence'] = 'error'
+ offsite['deep_error'] = str(exc)[:300]
+ meta['offsite'] = offsite
+ run.set_metadata(meta)
+ return {'ok': False, 'reason': str(exc)}
+ finally:
+ try:
+ if os.path.exists(tmp_path):
+ os.remove(tmp_path)
+ except OSError:
+ pass
+ # keep the dir tidy
+ try:
+ if os.path.isdir(tmp_dir) and not os.listdir(tmp_dir):
+ shutil.rmtree(tmp_dir, ignore_errors=True)
+ except OSError:
+ pass
+
+ # ------------------------------------------------------------------ #
+ # Registration
+ # ------------------------------------------------------------------ #
+
+ @classmethod
+ def register_jobs(cls):
+ from app.jobs import registry
+ registry.register(OFFSITE_VERIFY_JOB_KIND, cls.run_offsite_verify, replace=True)
diff --git a/backend/migrations/versions/058_backup_restore_drills.py b/backend/migrations/versions/058_backup_restore_drills.py
new file mode 100644
index 00000000..b7cca066
--- /dev/null
+++ b/backend/migrations/versions/058_backup_restore_drills.py
@@ -0,0 +1,104 @@
+"""Backup Trust — verification ladder, checksums, drill cadence + restore_drills.
+
+Adds the "restore proof" columns (plan 23):
+ * backup_runs: verify_level, verified_at, verify_error, checksum_sha256
+ * backup_policies: drill_cadence, last_drill_at, last_drill_status
+ * new restore_drills table (a scratch-mode restore, probe-verified, torn down)
+
+Backfill: existing runs keep verify_level='none' (the column default); the
+legacy `verified` boolean is preserved as-is and mapped forward in serialization
+(BackupRun.effective_verify_level), so old rows aren't silently downgraded.
+
+Idempotent: MigrationService runs _fix_missing_columns() + db.create_all() on
+boot before Alembic, so guard on the live schema.
+
+Revision ID: 058_backup_restore_drills
+Revises: 057_user_setup_snoozes
+Create Date: 2026-07-05
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = '058_backup_restore_drills'
+down_revision = '057_user_setup_snoozes'
+branch_labels = None
+depends_on = None
+
+
+def _cols(inspector, table):
+ return {c['name'] for c in inspector.get_columns(table)}
+
+
+def upgrade():
+ conn = op.get_bind()
+ inspector = sa.inspect(conn)
+ tables = set(inspector.get_table_names())
+
+ if 'backup_runs' in tables:
+ cols = _cols(inspector, 'backup_runs')
+ if 'verify_level' not in cols:
+ op.add_column('backup_runs', sa.Column(
+ 'verify_level', sa.String(length=12), nullable=False, server_default='none'))
+ if 'verified_at' not in cols:
+ op.add_column('backup_runs', sa.Column('verified_at', sa.DateTime(), nullable=True))
+ if 'verify_error' not in cols:
+ op.add_column('backup_runs', sa.Column('verify_error', sa.Text(), nullable=True))
+ if 'checksum_sha256' not in cols:
+ op.add_column('backup_runs', sa.Column('checksum_sha256', sa.String(length=64), nullable=True))
+
+ if 'backup_policies' in tables:
+ cols = _cols(inspector, 'backup_policies')
+ if 'drill_cadence' not in cols:
+ op.add_column('backup_policies', sa.Column(
+ 'drill_cadence', sa.String(length=12), nullable=False, server_default='off'))
+ if 'last_drill_at' not in cols:
+ op.add_column('backup_policies', sa.Column('last_drill_at', sa.DateTime(), nullable=True))
+ if 'last_drill_status' not in cols:
+ op.add_column('backup_policies', sa.Column('last_drill_status', sa.String(length=24), nullable=True))
+
+ if 'restore_drills' not in tables:
+ op.create_table(
+ 'restore_drills',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('policy_id', sa.Integer(), nullable=False),
+ sa.Column('run_id', sa.Integer(), nullable=True),
+ sa.Column('job_id', sa.String(length=36), nullable=True),
+ sa.Column('status', sa.String(length=24), nullable=False, server_default='running'),
+ sa.Column('trigger', sa.String(length=20), nullable=True),
+ sa.Column('started_at', sa.DateTime(), nullable=True),
+ sa.Column('finished_at', sa.DateTime(), nullable=True),
+ sa.Column('duration_seconds', sa.Integer(), nullable=True),
+ sa.Column('bytes_required', sa.BigInteger(), nullable=True),
+ sa.Column('bytes_free', sa.BigInteger(), nullable=True),
+ sa.Column('bytes_restored', sa.BigInteger(), nullable=True),
+ sa.Column('scratch_ref', sa.Text(), nullable=True),
+ sa.Column('probes_json', sa.Text(), nullable=True),
+ sa.Column('error', sa.Text(), nullable=True),
+ sa.ForeignKeyConstraint(['policy_id'], ['backup_policies.id']),
+ sa.ForeignKeyConstraint(['run_id'], ['backup_runs.id']),
+ sa.PrimaryKeyConstraint('id'),
+ )
+ op.create_index('ix_restore_drills_policy_id', 'restore_drills', ['policy_id'])
+ op.create_index('ix_restore_drills_run_id', 'restore_drills', ['run_id'])
+ op.create_index('ix_restore_drills_job_id', 'restore_drills', ['job_id'])
+
+
+def downgrade():
+ conn = op.get_bind()
+ inspector = sa.inspect(conn)
+ tables = set(inspector.get_table_names())
+
+ if 'restore_drills' in tables:
+ op.drop_table('restore_drills')
+
+ if 'backup_policies' in tables:
+ cols = _cols(inspector, 'backup_policies')
+ for col in ('drill_cadence', 'last_drill_at', 'last_drill_status'):
+ if col in cols:
+ op.drop_column('backup_policies', col)
+
+ if 'backup_runs' in tables:
+ cols = _cols(inspector, 'backup_runs')
+ for col in ('verify_level', 'verified_at', 'verify_error', 'checksum_sha256'):
+ if col in cols:
+ op.drop_column('backup_runs', col)
diff --git a/backend/tests/test_backup_alerts.py b/backend/tests/test_backup_alerts.py
new file mode 100644
index 00000000..4b131fe3
--- /dev/null
+++ b/backend/tests/test_backup_alerts.py
@@ -0,0 +1,144 @@
+"""Plan 23 Phase 5 — edge-triggered restore-proof alerts + support-bundle section."""
+from datetime import datetime
+
+from app import db
+from app.services.backup_policy_service import BackupPolicyService
+from app.services.backup_alert_service import BackupAlertService
+
+
+def _capture_notify(monkeypatch):
+ import app.plugins_sdk as sdk
+ sent = []
+ monkeypatch.setattr(sdk.notify, 'send',
+ lambda event, to=None, data=None, **kw: sent.append((event, data)))
+ return sent
+
+
+def _policy(target_id):
+ return BackupPolicyService.get_or_create_policy('files', target_id,
+ target_meta={'paths': ['/x']})
+
+
+# --------------------------------------------------------------------------- #
+# Drill alerts — fire once on transition (#17)
+# --------------------------------------------------------------------------- #
+
+def test_drill_failed_fires_once_then_throttles(app, monkeypatch):
+ with app.app_context():
+ sent = _capture_notify(monkeypatch)
+ policy = _policy(5501)
+ BackupAlertService.on_drill_result(policy, 'failed', 'boom')
+ assert [e for e, _ in sent] == ['backup.drill_failed']
+ # A second consecutive failure does NOT re-alert.
+ BackupAlertService.on_drill_result(policy, 'failed', 'boom again')
+ assert [e for e, _ in sent] == ['backup.drill_failed']
+
+
+def test_drill_recovery_fires_after_failure(app, monkeypatch):
+ with app.app_context():
+ sent = _capture_notify(monkeypatch)
+ policy = _policy(5502)
+ BackupAlertService.on_drill_result(policy, 'failed', 'boom')
+ BackupAlertService.on_drill_result(policy, 'success', None)
+ assert [e for e, _ in sent] == ['backup.drill_failed', 'backup.drill_recovered']
+ # Staying healthy doesn't repeat the recovery notice.
+ BackupAlertService.on_drill_result(policy, 'success', None)
+ assert [e for e, _ in sent] == ['backup.drill_failed', 'backup.drill_recovered']
+
+
+def test_first_success_is_silent(app, monkeypatch):
+ with app.app_context():
+ sent = _capture_notify(monkeypatch)
+ policy = _policy(5503)
+ # A healthy-from-the-start policy never had a failure -> no recovery spam.
+ BackupAlertService.on_drill_result(policy, 'success', None)
+ assert sent == []
+
+
+def test_skipped_no_space_counts_as_failure(app, monkeypatch):
+ with app.app_context():
+ sent = _capture_notify(monkeypatch)
+ policy = _policy(5504)
+ BackupAlertService.on_drill_result(policy, 'skipped_no_space', 'no space')
+ assert [e for e, _ in sent] == ['backup.drill_failed']
+
+
+# --------------------------------------------------------------------------- #
+# Verify alerts (#17)
+# --------------------------------------------------------------------------- #
+
+def test_verify_failed_and_recovered(app, monkeypatch):
+ from app.models.backup_run import BackupRun
+ with app.app_context():
+ sent = _capture_notify(monkeypatch)
+ policy = _policy(5505)
+ bad = BackupRun(policy_id=policy.id, kind='full', status='success',
+ verify_level='listed', verify_error='Checksum mismatch')
+ db.session.add(bad)
+ db.session.commit()
+ BackupAlertService.on_verify_result(policy, bad)
+ assert [e for e, _ in sent] == ['backup.verify_failed']
+
+ good = BackupRun(policy_id=policy.id, kind='full', status='success',
+ verify_level='hashed')
+ db.session.add(good)
+ db.session.commit()
+ BackupAlertService.on_verify_result(policy, good)
+ assert [e for e, _ in sent] == ['backup.verify_failed', 'backup.verify_recovered']
+
+
+def test_verify_failed_on_deep_offsite_mismatch(app, monkeypatch):
+ from app.models.backup_run import BackupRun
+ with app.app_context():
+ sent = _capture_notify(monkeypatch)
+ policy = _policy(5506)
+ run = BackupRun(policy_id=policy.id, kind='full', status='success',
+ verify_level='hashed')
+ run.set_metadata({'offsite': {'evidence': 'size_only', 'deep_match': False}})
+ db.session.add(run)
+ db.session.commit()
+ BackupAlertService.on_verify_result(policy, run)
+ assert [e for e, _ in sent] == ['backup.verify_failed']
+
+
+# --------------------------------------------------------------------------- #
+# Catalog registration (#17)
+# --------------------------------------------------------------------------- #
+
+def test_catalog_has_restore_proof_events():
+ from app.notifications import catalog
+ for key in ('backup.drill_failed', 'backup.drill_recovered',
+ 'backup.verify_failed', 'backup.verify_recovered'):
+ entry = catalog.get(key)
+ assert entry is not None
+ assert entry['category'] == 'backups'
+
+
+# --------------------------------------------------------------------------- #
+# Support bundle restore_confidence section (#18)
+# --------------------------------------------------------------------------- #
+
+def test_support_bundle_restore_confidence(app):
+ from app.services.support_bundle_service import _collect_restore_confidence
+ from app.models.backup_run import BackupRun
+ with app.app_context():
+ policy = _policy(5507)
+ policy.drill_cadence = 'weekly'
+ policy.last_drill_status = 'success'
+ policy.last_drill_at = datetime.utcnow()
+ run = BackupRun(policy_id=policy.id, kind='full', status='success',
+ verify_level='drilled')
+ run.set_metadata({'offsite': {'evidence': 'checksum', 'deep_evidence': 'deep_sha256'}})
+ db.session.add(run)
+ db.session.commit()
+
+ section = _collect_restore_confidence()
+ assert section['total'] >= 1
+ entry = next(p for p in section['policies']
+ if p['target_type'] == 'files' and p['target_id'] == 5507)
+ assert entry['drill_cadence'] == 'weekly'
+ assert entry['latest_verify_level'] == 'drilled'
+ assert entry['last_drill_status'] == 'success'
+ assert entry['offsite_evidence'] == 'checksum'
+ # No paths / secrets leak into the section.
+ assert 'storage_path' not in entry and 'remote_key' not in entry
diff --git a/backend/tests/test_backup_drill.py b/backend/tests/test_backup_drill.py
new file mode 100644
index 00000000..2a0ffcdb
--- /dev/null
+++ b/backend/tests/test_backup_drill.py
@@ -0,0 +1,251 @@
+"""Plan 23 Phase 2 — restore-drill engine proving tests.
+
+End-to-end files drill on tmp dirs (real archive, cross-platform), a database
+drill against a stubbed engine (asserts create -> restore -> probe -> drop call
+order), the no-space skip path, teardown/record-on-exception, and single-flight
+refusal."""
+import io
+import os
+import tarfile
+from datetime import datetime
+
+import pytest
+
+from app import db
+from app.services.backup_policy_service import BackupPolicyService
+from app.services.backup_drill_service import BackupDrillService, BackupDrillError
+from app.services import backup_verify_service as bv
+
+
+class _FakeJob:
+ def __init__(self, payload, jid='job-drill-1'):
+ self.id = jid
+ self._payload = payload
+
+ def get_payload(self):
+ return self._payload
+
+
+def _write_tar_gz_tree(dest, files):
+ with tarfile.open(dest, 'w:gz') as tar:
+ for name, content in files.items():
+ data = io.BytesIO(content)
+ info = tarfile.TarInfo(name)
+ info.size = len(content)
+ tar.addfile(info, data)
+
+
+def _make_files_run(policy, tmp_path, name='run', files=None):
+ """A successful files BackupRun with a real .tar.gz + manifest on disk."""
+ from app.models.backup_run import BackupRun
+ run_dir = tmp_path / name
+ run_dir.mkdir()
+ primary = str(run_dir / 'files.tar.gz')
+ _write_tar_gz_tree(primary, files if files is not None else {'a.txt': b'alpha', 'b.txt': b'beta'})
+ run = BackupRun(policy_id=policy.id, kind='full', status='success',
+ storage_path=str(run_dir), started_at=datetime.utcnow())
+ meta = {'engine': 'files', 'kind': 'full', 'primary_archive': primary,
+ 'incremental': False}
+ run.set_metadata(meta)
+ db.session.add(run)
+ db.session.commit()
+ bv.write_manifest(run, meta)
+ run.set_metadata(meta)
+ db.session.commit()
+ return run
+
+
+# --------------------------------------------------------------------------- #
+# End-to-end files drill (#9)
+# --------------------------------------------------------------------------- #
+
+def test_files_drill_success_promotes_run_and_stamps_policy(app, tmp_path, monkeypatch):
+ from app.services.backup_service import BackupService
+ from app.models.backup_run import BackupRun
+ from app.models.backup_policy import BackupPolicy
+ monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))
+ os.makedirs(BackupService.BACKUP_BASE_DIR, exist_ok=True)
+
+ with app.app_context():
+ policy = BackupPolicyService.get_or_create_policy(
+ 'files', 8201, target_meta={'paths': ['/etc/x'], 'label': 'configs'})
+ run = _make_files_run(policy, tmp_path)
+
+ BackupDrillService.run_restore_drill(_FakeJob(
+ {'policy_id': policy.id, 'run_id': run.id, 'trigger': 'manual'}))
+
+ from app.models.restore_drill import RestoreDrill
+ drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()
+ assert drill.status == 'success'
+ assert drill.get_probes()['file_count'] == 2
+ assert drill.get_probes()['sampled_ok'] == drill.get_probes()['sampled']
+
+ # The drilled run is promoted; the policy cache is stamped.
+ assert BackupRun.query.get(run.id).verify_level == 'drilled'
+ pol = BackupPolicy.query.get(policy.id)
+ assert pol.last_drill_status == 'success'
+ assert pol.last_drill_at is not None
+
+ # Scratch is torn down on success.
+ scratch = os.path.join(BackupService.BACKUP_BASE_DIR, 'restores', f'drill-{drill.id}')
+ assert not os.path.exists(scratch)
+
+
+# --------------------------------------------------------------------------- #
+# Database drill call order against a stubbed engine (#9)
+# --------------------------------------------------------------------------- #
+
+def test_database_drill_create_restore_probe_drop_order(app, tmp_path, monkeypatch):
+ from app.services.backup_service import BackupService
+ from app.services import database_service as dbsvc
+ from app.models.backup_run import BackupRun
+ from app.models.backup_policy import BackupPolicy
+ monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))
+
+ calls = []
+ monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_create_database',
+ staticmethod(lambda name, **kw: calls.append(('create', name)) or {'success': True}))
+ monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_get_tables',
+ staticmethod(lambda name, **kw: calls.append(('tables', name)) or ['wp_posts', 'wp_users']))
+ monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_drop_database',
+ staticmethod(lambda name, **kw: calls.append(('drop', name)) or {'success': True}))
+ monkeypatch.setattr(BackupService, 'restore_database',
+ staticmethod(lambda **kw: calls.append(('restore', kw['db_name'])) or {'success': True}))
+
+ with app.app_context():
+ policy = BackupPolicyService.get_or_create_policy(
+ 'database', 8202, target_subtype='mysql', target_meta={'db_name': 'shop'})
+ dump = tmp_path / 'shop.sql.gz'
+ dump.write_bytes(b'dummy')
+ run = BackupRun(policy_id=policy.id, kind='full', status='success',
+ storage_path=str(dump))
+ run.set_metadata({'engine': 'database', 'primary_archive': str(dump),
+ 'db_type': 'mysql', 'db_name': 'shop'})
+ db.session.add(run)
+ db.session.commit()
+
+ scratch_name = f'skdrill_{run.id}'
+ BackupDrillService.run_restore_drill(_FakeJob(
+ {'policy_id': policy.id, 'run_id': run.id, 'trigger': 'scheduled'}))
+
+ kinds = [c[0] for c in calls]
+ assert kinds == ['create', 'restore', 'tables', 'drop']
+ # All operate on the scratch DB, never the live 'shop'.
+ assert all(c[1] == scratch_name for c in calls)
+ from app.models.restore_drill import RestoreDrill
+ drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()
+ assert drill.status == 'success'
+ assert drill.get_probes()['table_count'] == 2
+
+
+def test_database_drill_drops_scratch_even_on_restore_failure(app, tmp_path, monkeypatch):
+ from app.services.backup_service import BackupService
+ from app.services import database_service as dbsvc
+ from app.models.backup_run import BackupRun
+ monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))
+
+ calls = []
+ monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_create_database',
+ staticmethod(lambda name, **kw: calls.append(('create', name)) or {'success': True}))
+ monkeypatch.setattr(dbsvc.DatabaseService, 'mysql_drop_database',
+ staticmethod(lambda name, **kw: calls.append(('drop', name)) or {'success': True}))
+ monkeypatch.setattr(BackupService, 'restore_database',
+ staticmethod(lambda **kw: {'success': False, 'error': 'boom'}))
+
+ with app.app_context():
+ policy = BackupPolicyService.get_or_create_policy(
+ 'database', 8203, target_subtype='mysql', target_meta={'db_name': 'shop'})
+ dump = tmp_path / 'shop2.sql.gz'
+ dump.write_bytes(b'dummy')
+ run = BackupRun(policy_id=policy.id, kind='full', status='success',
+ storage_path=str(dump))
+ run.set_metadata({'engine': 'database', 'primary_archive': str(dump),
+ 'db_type': 'mysql', 'db_name': 'shop'})
+ db.session.add(run)
+ db.session.commit()
+
+ scratch_name = f'skdrill_{run.id}'
+ with pytest.raises(Exception):
+ BackupDrillService.run_restore_drill(_FakeJob(
+ {'policy_id': policy.id, 'run_id': run.id}))
+
+ # Scratch DB was created then dropped despite the restore failing.
+ assert ('create', scratch_name) in calls
+ assert ('drop', scratch_name) in calls
+ from app.models.restore_drill import RestoreDrill
+ drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()
+ assert drill.status == 'failed'
+ assert 'boom' in (drill.error or '')
+
+
+# --------------------------------------------------------------------------- #
+# No-space skip (#9)
+# --------------------------------------------------------------------------- #
+
+def test_drill_skipped_no_space(app, tmp_path, monkeypatch):
+ from app.services.backup_service import BackupService
+ from app.models.backup_policy import BackupPolicy
+ monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))
+ os.makedirs(BackupService.BACKUP_BASE_DIR, exist_ok=True)
+ # Huge requirement, tiny free space -> loud skip, never a silent pass.
+ monkeypatch.setattr(BackupDrillService, '_estimate_required', classmethod(lambda cls, r, m: 10 ** 12))
+ monkeypatch.setattr(BackupDrillService, '_free_bytes', staticmethod(lambda p: 1024))
+
+ with app.app_context():
+ policy = BackupPolicyService.get_or_create_policy('files', 8204,
+ target_meta={'paths': ['/x']})
+ run = _make_files_run(policy, tmp_path)
+ result = BackupDrillService.run_restore_drill(_FakeJob(
+ {'policy_id': policy.id, 'run_id': run.id}))
+
+ assert result['status'] == 'skipped_no_space'
+ from app.models.restore_drill import RestoreDrill
+ drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()
+ assert drill.status == 'skipped_no_space'
+ assert BackupPolicy.query.get(policy.id).last_drill_status == 'skipped_no_space'
+
+
+# --------------------------------------------------------------------------- #
+# Teardown / record on exception (#9)
+# --------------------------------------------------------------------------- #
+
+def test_drill_records_failure_on_empty_extract(app, tmp_path, monkeypatch):
+ from app.services.backup_service import BackupService
+ from app.models.backup_run import BackupRun
+ monkeypatch.setattr(BackupService, 'BACKUP_BASE_DIR', str(tmp_path / 'backups'))
+ os.makedirs(BackupService.BACKUP_BASE_DIR, exist_ok=True)
+
+ with app.app_context():
+ policy = BackupPolicyService.get_or_create_policy('files', 8205,
+ target_meta={'paths': ['/x']})
+ # An empty archive -> the drill must fail loudly (0 files restored).
+ run = _make_files_run(policy, tmp_path, files={})
+ with pytest.raises(Exception):
+ BackupDrillService.run_restore_drill(_FakeJob(
+ {'policy_id': policy.id, 'run_id': run.id}))
+
+ from app.models.restore_drill import RestoreDrill
+ drill = RestoreDrill.query.filter_by(policy_id=policy.id).first()
+ assert drill.status == 'failed'
+ assert drill.error
+ # The run is NOT promoted to drilled on a failed drill.
+ assert BackupRun.query.get(run.id).verify_level != 'drilled'
+
+
+# --------------------------------------------------------------------------- #
+# Single-flight refusal (#9)
+# --------------------------------------------------------------------------- #
+
+def test_single_flight_refusal(app, tmp_path):
+ from app.models.restore_drill import RestoreDrill
+ with app.app_context():
+ policy = BackupPolicyService.get_or_create_policy('files', 8206,
+ target_meta={'paths': ['/x']})
+ _make_files_run(policy, tmp_path)
+ # A running drill already exists -> request is refused.
+ db.session.add(RestoreDrill(policy_id=policy.id, status='running',
+ started_at=datetime.utcnow()))
+ db.session.commit()
+ assert BackupDrillService.is_drilling() is True
+ with pytest.raises(BackupDrillError):
+ BackupDrillService.request_drill(policy)
diff --git a/backend/tests/test_backup_drill_cadence.py b/backend/tests/test_backup_drill_cadence.py
new file mode 100644
index 00000000..7f0dd842
--- /dev/null
+++ b/backend/tests/test_backup_drill_cadence.py
@@ -0,0 +1,154 @@
+"""Plan 23 Phase 3 — cadence sweep, badge serialization, doctor check matrix."""
+from datetime import datetime, timedelta
+
+from app import db
+from app.services.backup_policy_service import BackupPolicyService
+from app.services.backup_drill_service import BackupDrillService
+
+
+def _policy_with_run(target_id, cadence='weekly', enabled=True, last_drill_at=None,
+ last_drill_status=None):
+ from app.models.backup_run import BackupRun
+ policy = BackupPolicyService.get_or_create_policy('files', target_id,
+ target_meta={'paths': ['/x']})
+ policy.enabled = enabled
+ policy.drill_cadence = cadence
+ policy.last_drill_at = last_drill_at
+ policy.last_drill_status = last_drill_status
+ db.session.add(BackupRun(policy_id=policy.id, kind='full', status='success',
+ started_at=datetime.utcnow()))
+ db.session.commit()
+ return policy
+
+
+# --------------------------------------------------------------------------- #
+# Cadence due-selection (#13)
+# --------------------------------------------------------------------------- #
+
+def test_is_due_matrix(app):
+ with app.app_context():
+ now = datetime.utcnow()
+ # never drilled + cadenced + enabled -> due
+ p_never = _policy_with_run(7301, cadence='weekly', last_drill_at=None)
+ assert BackupDrillService.is_due(p_never, now) is True
+
+ # drilled recently -> not due
+ p_fresh = _policy_with_run(7302, cadence='weekly',
+ last_drill_at=now - timedelta(days=1))
+ assert BackupDrillService.is_due(p_fresh, now) is False
+
+ # drilled long ago (> weekly interval incl. jitter) -> due
+ p_old = _policy_with_run(7303, cadence='weekly',
+ last_drill_at=now - timedelta(days=30))
+ assert BackupDrillService.is_due(p_old, now) is True
+
+ # cadence off -> never due
+ p_off = _policy_with_run(7304, cadence='off', last_drill_at=None)
+ assert BackupDrillService.is_due(p_off, now) is False
+
+ # disabled -> never due
+ p_dis = _policy_with_run(7305, cadence='weekly', enabled=False, last_drill_at=None)
+ assert BackupDrillService.is_due(p_dis, now) is False
+
+
+def test_due_policies_and_sweep_enqueues(app, monkeypatch):
+ enqueued = []
+ monkeypatch.setattr(BackupDrillService, 'request_drill',
+ classmethod(lambda cls, policy, trigger='scheduled':
+ enqueued.append(policy.id) or type('J', (), {'id': f'j{policy.id}'})()))
+ monkeypatch.setattr(BackupDrillService, 'is_drilling', classmethod(lambda cls: False))
+ with app.app_context():
+ due_policy = _policy_with_run(7311, cadence='weekly', last_drill_at=None) # due
+ fresh_policy = _policy_with_run(7312, cadence='weekly',
+ last_drill_at=datetime.utcnow() - timedelta(hours=1)) # fresh
+ off_policy = _policy_with_run(7313, cadence='off', last_drill_at=None) # off
+ due = {p.id for p in BackupDrillService.due_policies()}
+ assert due_policy.id in due
+ assert fresh_policy.id not in due and off_policy.id not in due
+
+ result = BackupDrillService.run_drill_sweep()
+ assert due_policy.id in enqueued
+ assert result['count'] == len(enqueued)
+
+
+def test_sweep_defers_when_a_drill_is_in_flight(app, monkeypatch):
+ monkeypatch.setattr(BackupDrillService, 'is_drilling', classmethod(lambda cls: True))
+ called = []
+ monkeypatch.setattr(BackupDrillService, 'request_drill',
+ classmethod(lambda cls, policy, trigger='scheduled': called.append(policy.id)))
+ with app.app_context():
+ _policy_with_run(7321, cadence='weekly', last_drill_at=None)
+ result = BackupDrillService.run_drill_sweep()
+ assert result['count'] == 0
+ assert called == [] # deferred — nothing enqueued while one is running
+
+
+# --------------------------------------------------------------------------- #
+# Badge serialization states (#13)
+# --------------------------------------------------------------------------- #
+
+def test_badge_states(app):
+ with app.app_context():
+ now = datetime.utcnow()
+ never = _policy_with_run(7331, cadence='weekly', last_drill_at=None)
+ assert BackupPolicyService._restore_proof_view(never)['badge'] == 'never'
+
+ ok = _policy_with_run(7332, cadence='weekly', last_drill_status='success',
+ last_drill_at=now - timedelta(days=1))
+ assert BackupPolicyService._restore_proof_view(ok)['badge'] == 'ok'
+
+ stale = _policy_with_run(7333, cadence='weekly', last_drill_status='success',
+ last_drill_at=now - timedelta(days=60))
+ assert BackupPolicyService._restore_proof_view(stale)['badge'] == 'stale'
+
+ failed = _policy_with_run(7334, cadence='weekly', last_drill_status='failed',
+ last_drill_at=now - timedelta(days=1))
+ assert BackupPolicyService._restore_proof_view(failed)['badge'] == 'failed'
+
+
+def test_serialize_policy_view_includes_restore_proof(app):
+ with app.app_context():
+ policy = _policy_with_run(7341, cadence='monthly', last_drill_status='success',
+ last_drill_at=datetime.utcnow())
+ view = BackupPolicyService.serialize_policy_view(policy)
+ assert 'restore_proof' in view
+ proof = view['restore_proof']
+ assert proof['drill_cadence'] == 'monthly'
+ assert proof['badge'] == 'ok'
+ assert 'recent_drills' in proof
+
+
+# --------------------------------------------------------------------------- #
+# Doctor check matrix (#13)
+# --------------------------------------------------------------------------- #
+
+def test_doctor_backup_proof_matrix(app):
+ from app.services.doctor_service import DoctorService
+ with app.app_context():
+ now = datetime.utcnow()
+ _policy_with_run(7351, cadence='weekly', last_drill_status='success',
+ last_drill_at=now - timedelta(days=1)) # fresh -> ok
+ _policy_with_run(7352, cadence='weekly', last_drill_status='success',
+ last_drill_at=now - timedelta(days=60)) # stale -> warn
+ _policy_with_run(7353, cadence='weekly', last_drill_status='failed',
+ last_drill_at=now - timedelta(days=1)) # failed -> fail
+ _policy_with_run(7354, cadence='off', last_drill_at=None) # no cadence -> no drill check
+
+ checks = DoctorService._backup_proof_checks()
+ by_key = {c['key']: c for c in checks}
+
+ # Locate the drill checks by policy.
+ from app.models.backup_policy import BackupPolicy
+ pid = {tid: BackupPolicy.query.filter_by(target_type='files', target_id=tid).first().id
+ for tid in (7351, 7352, 7353, 7354)}
+ assert by_key[f'backup_drill_stale.{pid[7351]}']['status'] == 'ok'
+ assert by_key[f'backup_drill_stale.{pid[7352]}']['status'] == 'warn'
+ assert by_key[f'backup_drill_stale.{pid[7353]}']['status'] == 'fail'
+ assert by_key[f'backup_drill_stale.{pid[7353]}']['repairable'] is True
+ # A no-cadence policy has no drill-stale check.
+ assert f'backup_drill_stale.{pid[7354]}' not in by_key
+
+ # Every policy with a successful run gets an unverified check; the runs
+ # here were created verify_level='none' -> warn + repairable.
+ assert by_key[f'backup_unverified.{pid[7351]}']['status'] == 'warn'
+ assert by_key[f'backup_unverified.{pid[7351]}']['repair_ref']['kind'] == 'backup_verify'
diff --git a/backend/tests/test_backup_verify.py b/backend/tests/test_backup_verify.py
new file mode 100644
index 00000000..126cec23
--- /dev/null
+++ b/backend/tests/test_backup_verify.py
@@ -0,0 +1,203 @@
+"""Plan 23 Phase 1 — Tier-1 backup verification proving tests.
+
+Covers the manifest writer + post-run verifier (backup_verify_service): manifest
+shape, corrupted-archive detection (flip a byte -> verify_error), incremental
+chain metadata intact, and legacy-run serialization compatibility. All tests use
+real (small) tar archives so they run cross-platform (the verifier falls back to
+Python's tarfile where GNU tar is unavailable, e.g. Windows dev)."""
+import os
+import tarfile
+
+from app import db
+from app.services.backup_policy_service import BackupPolicyService
+from app.services import backup_verify_service as bv
+
+
+def _mk_run(policy_id, storage_path, primary, kind='full', meta_extra=None):
+ from app.models.backup_run import BackupRun
+ run = BackupRun(policy_id=policy_id, kind=kind, status='verifying',
+ storage_path=storage_path)
+ meta = {'engine': 'files', 'kind': kind, 'primary_archive': primary,
+ 'incremental': False}
+ if meta_extra:
+ meta.update(meta_extra)
+ run.set_metadata(meta)
+ db.session.add(run)
+ db.session.commit()
+ return run, meta
+
+
+def _write_tar_gz(dest, payload=b'hello world'):
+ """A tiny valid .tar.gz containing one file."""
+ import io
+ with tarfile.open(dest, 'w:gz') as tar:
+ data = io.BytesIO(payload)
+ info = tarfile.TarInfo('data.txt')
+ info.size = len(payload)
+ tar.addfile(info, data)
+
+
+def _write_tar_plain(dest, payload=b'A' * 1024):
+ """An uncompressed .tar — flipping a content byte stays listable but changes
+ the sha256, so it isolates the checksum spot-check from the readability one."""
+ import io
+ with tarfile.open(dest, 'w') as tar:
+ data = io.BytesIO(payload)
+ info = tarfile.TarInfo('data.bin')
+ info.size = len(payload)
+ tar.addfile(info, data)
+
+
+# --------------------------------------------------------------------------- #
+# Manifest shape (#5)
+# --------------------------------------------------------------------------- #
+
+def test_manifest_shape(app, tmp_path):
+ with app.app_context():
+ policy = BackupPolicyService.get_or_create_policy('files', 9101,
+ target_meta={'paths': ['/x']})
+ run_dir = tmp_path / 'run1'
+ run_dir.mkdir()
+ primary = str(run_dir / 'files.tar.gz')
+ _write_tar_gz(primary)
+ # a second artifact to prove the manifest lists everything
+ (run_dir / 'notes.txt').write_text('side file')
+
+ run, meta = _mk_run(policy.id, str(run_dir), primary)
+ manifest = bv.write_manifest(run, meta)
+
+ assert manifest['version'] == bv.MANIFEST_VERSION
+ assert manifest['primary_archive'] == 'files.tar.gz'
+ assert manifest['primary_sha256'] and len(manifest['primary_sha256']) == 64
+ names = {a['name'] for a in manifest['artifacts']}
+ assert names == {'files.tar.gz', 'notes.txt'}
+ for art in manifest['artifacts']:
+ assert art['sha256'] and len(art['sha256']) == 64
+ assert art['size'] >= 0
+ assert manifest['totals']['count'] == 2
+ assert manifest['chain'] == {'incremental': False, 'full_run_id': None}
+ # sha256 is denormalized onto the run row.
+ assert run.checksum_sha256 == manifest['primary_sha256']
+ # manifest.json actually lands next to the archives.
+ assert os.path.exists(run_dir / bv.MANIFEST_NAME)
+
+
+# --------------------------------------------------------------------------- #
+# Good archive -> hashed (#5)
+# --------------------------------------------------------------------------- #
+
+def test_verify_promotes_good_archive_to_hashed(app, tmp_path):
+ with app.app_context():
+ policy = BackupPolicyService.get_or_create_policy('files', 9102,
+ target_meta={'paths': ['/x']})
+ run_dir = tmp_path / 'run2'
+ run_dir.mkdir()
+ primary = str(run_dir / 'files.tar.gz')
+ _write_tar_gz(primary)
+ run, meta = _mk_run(policy.id, str(run_dir), primary)
+ bv.write_manifest(run, meta)
+
+ probes = bv.verify_run_tier1(run, meta)
+ assert run.verify_level == 'hashed'
+ assert run.verify_error is None
+ assert run.verified_at is not None
+ assert probes['hashed'] is True
+ assert probes['listed'][0]['ok'] is True
+
+
+# --------------------------------------------------------------------------- #
+# Corrupted archive -> verify_error, not 'hashed' (#5)
+# --------------------------------------------------------------------------- #
+
+def test_corrupted_archive_flags_checksum_mismatch(app, tmp_path):
+ with app.app_context():
+ policy = BackupPolicyService.get_or_create_policy('files', 9103,
+ target_meta={'paths': ['/x']})
+ run_dir = tmp_path / 'run3'
+ run_dir.mkdir()
+ primary = str(run_dir / 'files.tar') # uncompressed: stays listable
+ _write_tar_plain(primary)
+ run, meta = _mk_run(policy.id, str(run_dir), primary)
+ bv.write_manifest(run, meta) # records the good sha256
+
+ # Flip a byte inside the file-content region (offset well past the header).
+ with open(primary, 'r+b') as f:
+ f.seek(512 + 256)
+ b = f.read(1)
+ f.seek(512 + 256)
+ f.write(bytes([b[0] ^ 0xFF]))
+
+ probes = bv.verify_run_tier1(run, meta)
+ # The archive is still readable (listing ok) but the hash no longer matches.
+ assert run.verify_level == 'listed'
+ assert run.verify_error and 'mismatch' in run.verify_error.lower()
+ assert probes['hashed'] is False
+
+
+def test_unreadable_archive_flags_verify_error(app, tmp_path):
+ with app.app_context():
+ policy = BackupPolicyService.get_or_create_policy('files', 9104,
+ target_meta={'paths': ['/x']})
+ run_dir = tmp_path / 'run4'
+ run_dir.mkdir()
+ primary = str(run_dir / 'files.tar.gz')
+ # Not a real gzip/tar at all -> listing must fail.
+ with open(primary, 'wb') as f:
+ f.write(b'this is not a tar archive at all')
+ run, meta = _mk_run(policy.id, str(run_dir), primary)
+ bv.write_manifest(run, meta)
+
+ bv.verify_run_tier1(run, meta)
+ assert run.verify_level == 'none'
+ assert run.verify_error and 'readable' in run.verify_error.lower()
+
+
+# --------------------------------------------------------------------------- #
+# Incremental chain metadata intact (#5)
+# --------------------------------------------------------------------------- #
+
+def test_incremental_chain_metadata_intact(app, tmp_path):
+ with app.app_context():
+ policy = BackupPolicyService.get_or_create_policy('application', 9105)
+ run_dir = tmp_path / 'incr'
+ run_dir.mkdir()
+ primary = str(run_dir / 'files.tar.gz')
+ _write_tar_gz(primary)
+ run, meta = _mk_run(policy.id, str(run_dir), primary, kind='incremental',
+ meta_extra={'incremental': True, 'full_run_id': 4242})
+ manifest = bv.write_manifest(run, meta)
+ assert manifest['chain'] == {'incremental': True, 'full_run_id': 4242}
+ assert manifest['kind'] == 'incremental'
+
+
+# --------------------------------------------------------------------------- #
+# Legacy-run serialization compatibility (#5)
+# --------------------------------------------------------------------------- #
+
+def test_legacy_run_serialization_maps_forward(app):
+ from app.models.backup_run import BackupRun
+ with app.app_context():
+ policy = BackupPolicyService.get_or_create_policy('files', 9106,
+ target_meta={'paths': ['/x']})
+ # A pre-plan-23 run: verify_level defaulted to 'none' but it had a
+ # remote-verified copy — must read forward as 'listed', not 'none'.
+ legacy = BackupRun(policy_id=policy.id, kind='full', status='success',
+ verified=True)
+ db.session.add(legacy)
+ db.session.commit()
+ assert legacy.verify_level == 'none' # stored value untouched
+ assert legacy.effective_verify_level() == 'listed'
+ assert legacy.to_dict()['verify_level'] == 'listed'
+
+ # A run explicitly hashed serializes as hashed.
+ fresh = BackupRun(policy_id=policy.id, kind='full', status='success',
+ verify_level='hashed')
+ db.session.add(fresh)
+ db.session.commit()
+ assert fresh.to_dict()['verify_level'] == 'hashed'
+
+ # A never-verified local-only run stays 'none'.
+ plain = BackupRun(policy_id=policy.id, kind='full', status='success')
+ db.session.add(plain)
+ db.session.commit()
+ assert plain.to_dict()['verify_level'] == 'none'
diff --git a/docs/BACKUP_PROTECTION.md b/docs/BACKUP_PROTECTION.md
index 655992cd..54d1a5bc 100644
--- a/docs/BACKUP_PROTECTION.md
+++ b/docs/BACKUP_PROTECTION.md
@@ -81,6 +81,50 @@ Open a backup from the history → **Restore** → choose the scope and confirm:
Restore runs as a `restore.run` job; you're notified on completion.
+## How restore proof works
+
+A backup you have never restored is an assumption, not a safety net. ServerKit
+records how strongly each backup has been *proven*, on a ladder:
+
+| Level | What it means |
+|-------|---------------|
+| `none` | Produced, nothing checked yet. |
+| `listed` | Every archive is readable end to end (`tar -t`). |
+| `hashed` | The primary archive still matches the sha256 recorded in its `manifest.json` — catches silent corruption. |
+| `drilled` | A real restore of this point actually succeeded (strongest proof). |
+
+`listed` and `hashed` run automatically at the end of **every** backup (inside
+the existing `verifying` window). Each run writes a `manifest.json` next to its
+archives — file list, per-artifact sha256, chain refs, tool versions — and the
+primary archive's hash is stored on the run.
+
+A **restore drill** earns `drilled`. On a per-policy cadence (**Off / Weekly /
+Monthly**, set on the Schedule card) — or on demand with **Run drill now** — the
+latest restorable point is restored into a throwaway scratch location (a
+`restores/drill-<id>/` dir for files/apps/WordPress, a `skdrill_<run_id>`
+scratch database for databases), probe-verified (file count + bytes + a hashed
+sample; table listing for databases), then torn down. It never touches the live
+target. Drilling the *latest* point proves the whole incremental chain by
+construction. A free-space precheck marks an under-provisioned drill
+`skipped_no_space` rather than failing mid-extract, and only one drill runs at a
+time per host.
+
+The Protection panel shows a **Last proven restore** badge (never / ok / stale /
+failed) and the Doctor raises a repairable warning when a cadenced policy is
+overdue (> 1.5× its cadence) or its latest backup is unverified.
+
+For **remote copies**, a weekly check compares the offsite object's size and —
+for single-part uploads — its ETag/MD5; multipart uploads are honestly labelled
+`size_only` until a **monthly deep check** downloads one sampled object and
+hashes it against the *stored* sha256 (never a hash recomputed from the local
+file, which could share the same corruption). Egress stays bounded to one object
+per month.
+
+Drill/verify failures and recoveries notify admins **once per transition**
+(`backup.drill_failed` / `backup.verify_failed` and their recovery notices), not
+on every sweep. The diagnostic support bundle includes a scrubbed
+`restore_confidence` summary (levels + states only, no paths or secrets).
+
## Under the hood
- **Models**: `BackupPolicy` (one per target) and `BackupRun` (one per run, the
@@ -89,6 +133,10 @@ Restore runs as a `restore.run` job; you're notified on completion.
- **Services**: `backup_policy_service.py` (policy CRUD, schedule wiring, the
`backup.policy.run` / `restore.run` job handlers, retention),
`backup_cost_service.py` (cost), and the smart-backup helpers in
- `backup_service.py`.
+ `backup_service.py`. Restore proof lives in `backup_verify_service.py`
+ (manifest + tier-1 verifier), `backup_drill_service.py` (`backup.drill.run` +
+ cadence/orphan sweeps, `RestoreDrill` model), `backup_offsite_service.py`
+ (`backup.offsite.verify`), and `backup_alert_service.py` (edge-triggered
+ drill/verify notices).
- **Scheduling**: the policy's cron is mirrored into a `ScheduledJob` on the
unified job bus, so firing is observable and retryable like any other job.
diff --git a/frontend/src/components/backups/BackupHistoryList.jsx b/frontend/src/components/backups/BackupHistoryList.jsx
index 9694f6e9..8f17b1c2 100644
--- a/frontend/src/components/backups/BackupHistoryList.jsx
+++ b/frontend/src/components/backups/BackupHistoryList.jsx
@@ -1,7 +1,11 @@
+import { useState } from 'react';
import { Pill } from '@/components/ds';
import { Button } from '@/components/ui/button';
import EmptyState from '@/components/EmptyState';
-import { Archive, RotateCcw, ShieldCheck, Trash2, HardDrive, Cloud, Layers } from 'lucide-react';
+import {
+ Archive, RotateCcw, ShieldCheck, Trash2, HardDrive, Cloud, Layers,
+ FlaskConical, ChevronRight, ChevronDown,
+} from 'lucide-react';
import { humanSize, formatMoney, formatWhen, statusKind, storageLabel } from './format';
// Card 3 of the backup "Protection" panel: a data-table of backup runs.
@@ -29,72 +33,167 @@ function storageIcon(run) {
);
}
+// Pill kind for a restore-drill outcome. skipped_no_space is a soft warning
+// (no disk headroom to drill), never a hard failure.
+function drillStatusKind(status) {
+ switch (status) {
+ case 'success': return 'green';
+ case 'failed': return 'red';
+ case 'skipped_no_space': return 'amber';
+ default: return 'gray';
+ }
+}
+
+function drillStatusLabel(status) {
+ if (status === 'skipped_no_space') return 'skipped (no space)';
+ return status || 'unknown';
+}
+
+// Render whatever shape `probes` arrives in (array of checks, keyed object, or
+// a plain string) as compact key/value rows without assuming a schema.
+function renderProbes(probes) {
+ if (!probes) return <span className="backup-drills__probe-empty">No probe details recorded.</span>;
+ const entries = Array.isArray(probes)
+ ? probes.map((p, i) => [p?.name || `Probe ${i + 1}`, p?.detail ?? p?.result ?? p?.ok ?? p])
+ : (typeof probes === 'object' ? Object.entries(probes) : [['result', probes]]);
+ return (
+ <dl className="backup-drills__probes">
+ {entries.map(([key, value]) => (
+ <div key={key} className="backup-drills__probe">
+ <dt>{key}</dt>
+ <dd>{typeof value === 'object' ? JSON.stringify(value) : String(value)}</dd>
+ </div>
+ ))}
+ </dl>
+ );
+}
+
+function RestoreDrills({ drills }) {
+ const [expanded, setExpanded] = useState(null);
+ if (!drills || drills.length === 0) return null;
+
+ return (
+ <div className="backup-drills">
+ <div className="backup-drills__head">
+ <FlaskConical size={14} />
+ <span>Restore drills</span>
+ <span className="backup-drills__sub">Proof that these backups can actually be recovered.</span>
+ </div>
+ <ul className="backup-drills__list">
+ {drills.map((drill) => {
+ const isOpen = expanded === drill.id;
+ return (
+ <li key={drill.id} className="backup-drills__item">
+ <button
+ type="button"
+ className="backup-drills__row"
+ onClick={() => setExpanded(isOpen ? null : drill.id)}
+ aria-expanded={isOpen}
+ >
+ <span className="backup-drills__chev">
+ {isOpen ? <ChevronDown size={14} /> : <ChevronRight size={14} />}
+ </span>
+ <Pill kind={drillStatusKind(drill.status)}>{drillStatusLabel(drill.status)}</Pill>
+ <span className="backup-drills__trigger">{drill.trigger || 'manual'}</span>
+ <span className="backup-drills__when">{formatWhen(drill.started_at)}</span>
+ {drill.duration_seconds != null && (
+ <span className="backup-drills__meta">{drill.duration_seconds}s</span>
+ )}
+ {drill.bytes_restored != null && (
+ <span className="backup-drills__meta">{humanSize(drill.bytes_restored)}</span>
+ )}
+ </button>
+ {isOpen && (
+ <div className="backup-drills__detail">
+ {drill.error && <p className="backup-drills__error">{drill.error}</p>}
+ {renderProbes(drill.probes)}
+ </div>
+ )}
+ </li>
+ );
+ })}
+ </ul>
+ </div>
+ );
+}
+
export default function BackupHistoryList({
runs,
+ drills,
loading,
onRestore,
onVerify,
onDelete,
onRowClick,
}) {
+ const hasDrills = drills && drills.length > 0;
+
if (loading && (!runs || runs.length === 0)) {
return <EmptyState icon={Archive} title="Loading backups…" loading />;
}
if (!loading && (!runs || runs.length === 0)) {
return (
- <EmptyState
- icon={Archive}
- title="No backups yet"
- description="Turn on protection or click Back up now."
- />
+ <>
+ <RestoreDrills drills={drills} />
+ {!hasDrills && (
+ <EmptyState
+ icon={Archive}
+ title="No backups yet"
+ description="Turn on protection or click Back up now."
+ />
+ )}
+ </>
);
}
return (
- <table className="sk-dtable backup-history-list">
- <thead>
- <tr>
- <th>Backup</th>
- <th>Date</th>
- <th>Size</th>
- <th>Cost</th>
- <th>Status</th>
- <th>Storage</th>
- <th aria-label="Actions" />
- </tr>
- </thead>
- <tbody>
- {runs.map((run) => (
- <tr
- key={run.id}
- className={`backup-history-list__row ${onRowClick ? 'is-clickable' : ''}`}
- onClick={onRowClick ? () => onRowClick(run) : undefined}
- >
- <td>
- <div className="sk-cell-name">
- <span className="backup-history-list__ico"><Archive size={14} /></span>
- <span>{run.metadata?.backup_name || `Backup #${run.id}`}</span>
- <Pill kind={run.kind === 'full' ? 'violet' : 'gray'} dot={false}>{run.kind}</Pill>
- </div>
- </td>
- <td className="backup-history-list__when">{formatWhen(run.started_at)}</td>
- <td className="sk-cell-mono">{humanSize(run.size_total)}</td>
- <td className="sk-cell-mono">{formatMoney(run.cost_total)}</td>
- <td><Pill kind={statusKind(run.status)}>{run.status}</Pill></td>
- <td>{storageIcon(run)}</td>
- <td>
- <div className="backup-history-list__actions" onClick={(e) => e.stopPropagation()}>
- <Button size="icon" variant="outline" title="Restore" disabled={run.status !== 'success'} onClick={() => onRestore(run)}><RotateCcw size={14} /></Button>
- {run.remote_key && (
- <Button size="icon" variant="outline" title="Verify remote copy" onClick={() => onVerify(run)}><ShieldCheck size={14} /></Button>
- )}
- <Button size="icon" variant="destructive" title="Delete" onClick={() => onDelete(run)}><Trash2 size={14} /></Button>
- </div>
- </td>
+ <>
+ <RestoreDrills drills={drills} />
+ <table className="sk-dtable backup-history-list">
+ <thead>
+ <tr>
+ <th>Backup</th>
+ <th>Date</th>
+ <th>Size</th>
+ <th>Cost</th>
+ <th>Status</th>
+ <th>Storage</th>
+ <th aria-label="Actions" />
</tr>
- ))}
- </tbody>
- </table>
+ </thead>
+ <tbody>
+ {runs.map((run) => (
+ <tr
+ key={run.id}
+ className={`backup-history-list__row ${onRowClick ? 'is-clickable' : ''}`}
+ onClick={onRowClick ? () => onRowClick(run) : undefined}
+ >
+ <td>
+ <div className="sk-cell-name">
+ <span className="backup-history-list__ico"><Archive size={14} /></span>
+ <span>{run.metadata?.backup_name || `Backup #${run.id}`}</span>
+ <Pill kind={run.kind === 'full' ? 'violet' : 'gray'} dot={false}>{run.kind}</Pill>
+ </div>
+ </td>
+ <td className="backup-history-list__when">{formatWhen(run.started_at)}</td>
+ <td className="sk-cell-mono">{humanSize(run.size_total)}</td>
+ <td className="sk-cell-mono">{formatMoney(run.cost_total)}</td>
+ <td><Pill kind={statusKind(run.status)}>{run.status}</Pill></td>
+ <td>{storageIcon(run)}</td>
+ <td>
+ <div className="backup-history-list__actions" onClick={(e) => e.stopPropagation()}>
+ <Button size="icon" variant="outline" title="Restore" disabled={run.status !== 'success'} onClick={() => onRestore(run)}><RotateCcw size={14} /></Button>
+ {run.remote_key && (
+ <Button size="icon" variant="outline" title="Verify remote copy" onClick={() => onVerify(run)}><ShieldCheck size={14} /></Button>
+ )}
+ <Button size="icon" variant="destructive" title="Delete" onClick={() => onDelete(run)}><Trash2 size={14} /></Button>
+ </div>
+ </td>
+ </tr>
+ ))}
+ </tbody>
+ </table>
+ </>
);
}
diff --git a/frontend/src/components/backups/ProtectionPanel.jsx b/frontend/src/components/backups/ProtectionPanel.jsx
index 6ebb0fd6..cc659dab 100644
--- a/frontend/src/components/backups/ProtectionPanel.jsx
+++ b/frontend/src/components/backups/ProtectionPanel.jsx
@@ -36,6 +36,7 @@ export default function ProtectionPanel({ targetType, targetId, targetName, show
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
const [backingUp, setBackingUp] = useState(false);
+ const [drilling, setDrilling] = useState(false);
const [historyView, setHistoryView] = useState('list'); // 'list' | 'calendar'
const [dayFilter, setDayFilter] = useState(null); // Date | null
const [detailRun, setDetailRun] = useState(null);
@@ -117,6 +118,19 @@ export default function ProtectionPanel({ targetType, targetId, targetName, show
}
}, [targetType, targetId, toast, scheduleReloads]);
+ const handleRunDrill = useCallback(async () => {
+ setDrilling(true);
+ try {
+ await api.runBackupDrill(targetType, targetId);
+ toast.success('Restore drill started');
+ scheduleReloads();
+ } catch (err) {
+ toast.error(err.message || 'Failed to start restore drill');
+ } finally {
+ setDrilling(false);
+ }
+ }, [targetType, targetId, toast, scheduleReloads]);
+
// Restore is always initiated from a run — opening the restore drawer.
const openRestore = useCallback((run) => setRestoreRun(run), []);
@@ -177,10 +191,12 @@ export default function ProtectionPanel({ targetType, targetId, targetName, show
policyView={view}
onToggle={handleToggle}
onBackupNow={handleBackupNow}
+ onRunDrill={handleRunDrill}
onViewGlobal={() => navigate('/backups')}
onViewJobs={() => navigate('/jobs')}
busy={saving}
backingUp={backingUp}
+ drilling={drilling || !!view?.restore_proof?.is_drilling}
/>
<ScheduleCard
@@ -236,6 +252,7 @@ export default function ProtectionPanel({ targetType, targetId, targetName, show
) : (
<BackupHistoryList
runs={visibleRuns}
+ drills={dayFilter ? [] : view?.restore_proof?.recent_drills}
loading={loading}
onRestore={openRestore}
onVerify={handleVerify}
diff --git a/frontend/src/components/backups/ScheduleCard.jsx b/frontend/src/components/backups/ScheduleCard.jsx
index 3cb557bd..c9a451f2 100644
--- a/frontend/src/components/backups/ScheduleCard.jsx
+++ b/frontend/src/components/backups/ScheduleCard.jsx
@@ -1,32 +1,22 @@
-import { useState, useEffect, useMemo } from 'react';
+import { useState, useEffect } from 'react';
import { SegControl } from '@/components/ds';
import { Switch } from '@/components/ui/switch';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
import { Calendar, Save, Loader2 } from 'lucide-react';
+import SchedulePicker from '../SchedulePicker';
// Card 2 of the backup "Protection" panel: the editable schedule form.
-// The saved policy carries a raw cron string; this card translates it into a
-// friendly Daily / Weekly / Custom UI and back. Local state mirrors the form
-// while a useEffect re-seeds it whenever the parent reloads the policy, so an
-// external save stays in sync. Save is disabled until something actually
+// The saved policy carries a raw cron string; this card now mounts the shared
+// SchedulePicker (compact) so backups get the same Presets/Builder/Advanced UI
+// and server-side preview as every other cron surface. Local state mirrors the
+// form while a useEffect re-seeds it whenever the parent reloads the policy, so
+// an external save stays in sync. Save is disabled until something actually
// changes (dirty tracking) and shows a spinner while the request is in flight.
-// Canonical cron strings for the two preset frequencies (02:00 server time).
-const DAILY_CRON = '0 2 * * *';
-const WEEKLY_CRON = '0 2 * * 0'; // Sunday
-// Day-of-week chip labels; index 0 = Sunday … 6 = Saturday (cron dow order).
-const DOW_LABELS = ['S', 'M', 'T', 'W', 'T', 'F', 'S'];
-
-const DEFAULT_TIME = '02:00';
+const DEFAULT_CRON = '0 2 * * *'; // Daily at 02:00 server time.
const MIN_COUNT = 1; // floor shared by every numeric field (count/days/full-every)
-// Two-digit zero-pad for assembling/displaying HH:MM from cron integers.
-const pad = (n) => String(n).padStart(2, '0');
-
-// True when a cron token is a plain non-negative integer (e.g. "2", not "*").
-const isInt = (token) => /^\d+$/.test(token);
-
// Coerce a numeric form field (kept as a string) to a number, treating empty
// or sub-floor values as the minimum so we never persist 0 / NaN.
const toNum = (value) => {
@@ -34,81 +24,30 @@ const toNum = (value) => {
return Number.isFinite(n) && n >= MIN_COUNT ? n : MIN_COUNT;
};
-// Expand a cron day-of-week field into a sorted int array (0..6).
-// "*"/"?" -> [] (every day); supports comma lists and "a-b" ranges.
-function parseDow(field) {
- if (!field || field === '*' || field === '?') return [];
- const out = new Set();
- for (const part of field.split(',')) {
- const range = part.split('-');
- if (range.length === 2 && isInt(range[0]) && isInt(range[1])) {
- const lo = Number(range[0]);
- const hi = Number(range[1]);
- for (let d = lo; d <= hi; d += 1) {
- if (d >= 0 && d <= 6) out.add(d);
- }
- } else if (isInt(part)) {
- const d = Number(part);
- if (d >= 0 && d <= 6) out.add(d);
- }
- }
- return Array.from(out).sort((a, b) => a - b);
-}
-
-// Translate a saved cron string into the form's { frequency, time, days }.
-function deriveFromCron(cron) {
- if (cron === DAILY_CRON) return { frequency: 'daily', time: DEFAULT_TIME, days: [] };
- if (cron === WEEKLY_CRON) return { frequency: 'weekly', time: DEFAULT_TIME, days: [0] };
-
- const fields = (cron || '').trim().split(/\s+/);
- const min = fields[0];
- const hour = fields[1];
- const time = isInt(hour) && isInt(min) ? `${pad(Number(hour))}:${pad(Number(min))}` : DEFAULT_TIME;
- return { frequency: 'custom', time, days: parseDow(fields[4]) };
-}
-
-// Assemble a cron string from the form state.
-function buildCron(frequency, time, days) {
- if (frequency === 'daily') return DAILY_CRON;
- if (frequency === 'weekly') return WEEKLY_CRON;
-
- const [hour = '02', min = '00'] = (time || DEFAULT_TIME).split(':');
- const dowField =
- days.length > 0 && days.length < 7
- ? days.slice().sort((a, b) => a - b).join(',')
- : '*';
- return `${Number(min)} ${Number(hour)} * * ${dowField}`;
-}
-
const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {
// All hooks run unconditionally, before any early return (Rules of Hooks).
// The form is only shown once a policy exists, so the seed values used when
// `policy` is null are throwaway and get re-seeded by the effect on load.
- const initial = deriveFromCron(policy?.schedule_cron);
-
- const [frequency, setFrequency] = useState(initial.frequency);
- const [time, setTime] = useState(initial.time);
- const [days, setDays] = useState(initial.days);
+ const [cron, setCron] = useState(policy?.schedule_cron || DEFAULT_CRON);
const [retentionCount, setRetentionCount] = useState(String(policy?.retention_count ?? ''));
const [retentionDays, setRetentionDays] = useState(String(policy?.retention_days ?? ''));
const [fullEvery, setFullEvery] = useState(String(policy?.full_every_n_days ?? ''));
const [compression, setCompression] = useState(policy?.compression ?? 'balanced');
const [remoteCopy, setRemoteCopy] = useState(!!policy?.remote_copy);
+ const [drillCadence, setDrillCadence] = useState(policy?.drill_cadence ?? 'off');
// Re-seed the form when the saved policy changes (e.g. external reload after
// a successful save). Keyed on the persisted values so it only fires when
// the source of truth actually moves, not on every render.
useEffect(() => {
if (!policy) return;
- const next = deriveFromCron(policy.schedule_cron);
- setFrequency(next.frequency);
- setTime(next.time);
- setDays(next.days);
+ setCron(policy.schedule_cron || DEFAULT_CRON);
setRetentionCount(String(policy.retention_count));
setRetentionDays(String(policy.retention_days));
setFullEvery(String(policy.full_every_n_days));
setCompression(policy.compression);
setRemoteCopy(!!policy.remote_copy);
+ setDrillCadence(policy.drill_cadence || 'off');
}, [
policy,
policy?.schedule_cron,
@@ -117,9 +56,10 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {
policy?.full_every_n_days,
policy?.compression,
policy?.remote_copy,
+ policy?.drill_cadence,
]);
- const currentCron = useMemo(() => buildCron(frequency, time, days), [frequency, time, days]);
+ const currentCron = cron;
// Conditional render AFTER all hooks have run.
if (!policy) {
@@ -136,12 +76,6 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {
);
}
- const toggleDay = (i) => {
- setDays((prev) =>
- prev.includes(i) ? prev.filter((d) => d !== i) : [...prev, i].sort((a, b) => a - b)
- );
- };
-
// The form is dirty when any field diverges from the saved policy. Numeric
// fields are compared via toNum so an empty box reads as the floor (= what
// would actually be persisted), avoiding a phantom-dirty Save button.
@@ -151,7 +85,8 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {
toNum(retentionDays) !== policy.retention_days ||
toNum(fullEvery) !== policy.full_every_n_days ||
compression !== policy.compression ||
- remoteCopy !== !!policy.remote_copy;
+ remoteCopy !== !!policy.remote_copy ||
+ drillCadence !== (policy.drill_cadence || 'off');
const handleSave = () => {
onSave({
@@ -161,6 +96,7 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {
full_every_n_days: toNum(fullEvery),
compression,
remote_copy: remoteCopy,
+ drill_cadence: drillCadence,
});
};
@@ -176,44 +112,7 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {
<div className="app-panel-body">
<div className="schedule-card__field">
<label>Frequency</label>
- <SegControl
- options={[
- { value: 'daily', label: 'Daily' },
- { value: 'weekly', label: 'Weekly' },
- { value: 'custom', label: 'Custom' },
- ]}
- value={frequency}
- onChange={setFrequency}
- />
- </div>
-
- {frequency === 'custom' && (
- <>
- <div className="schedule-card__field">
- <label>Time</label>
- <Input type="time" value={time} onChange={(e) => setTime(e.target.value)} />
- </div>
- <div className="schedule-card__field">
- <label>Days</label>
- <div className="schedule-card__days">
- {DOW_LABELS.map((d, i) => (
- <button
- type="button"
- key={i}
- className={`schedule-card__day ${days.includes(i) ? 'is-active' : ''}`}
- onClick={() => toggleDay(i)}
- >
- {d}
- </button>
- ))}
- </div>
- </div>
- </>
- )}
-
- <div className="schedule-card__preview">
- <span className="schedule-card__preview-label">Cron</span>
- <code>{currentCron}</code>
+ <SchedulePicker value={cron} onChange={setCron} compact />
</div>
<div className="schedule-card__field schedule-card__retention">
@@ -273,6 +172,25 @@ const ScheduleCard = ({ policy, remoteConfigured, onSave, saving }) => {
/>
</div>
+ <div className="schedule-card__field">
+ <label>Restore drill cadence</label>
+ <SegControl
+ options={[
+ { value: 'off', label: 'Off' },
+ { value: 'weekly', label: 'Weekly' },
+ { value: 'monthly', label: 'Monthly' },
+ ]}
+ value={drillCadence}
+ onChange={setDrillCadence}
+ />
+ <p className="app-panel-hint">
+ Periodically test-restore the latest backup to prove it can actually be
+ recovered. Drills run into a throwaway location and never touch this site.
+ Replaying an incremental backup chain during a drill needs GNU tar, so
+ those drills run on Linux hosts only.
+ </p>
+ </div>
+
<div className="schedule-card__field schedule-card__remote">
<Switch
id="remote-copy"
diff --git a/frontend/src/services/api/backupProtection.js b/frontend/src/services/api/backupProtection.js
index a2489801..b2e30cbd 100644
--- a/frontend/src/services/api/backupProtection.js
+++ b/frontend/src/services/api/backupProtection.js
@@ -43,6 +43,17 @@ export async function verifyBackupRun(targetType, targetId, runId) {
return this.request(`${runsBase(targetType, targetId)}/${runId}/verify`, { method: 'POST' });
}
+// Trigger a restore drill (a proving test-restore into a throwaway location).
+// Applications and WordPress sites use their own edit-gated, target-scoped
+// endpoint (the same owner who can back up a target can drill it); any other
+// target routes through the admin-only generic policy-by-target endpoint.
+export async function runBackupDrill(targetType, targetId) {
+ if (targetType === 'application' || targetType === 'wordpress_site') {
+ return this.request(`${runsBase(targetType, targetId)}/drill`, { method: 'POST' });
+ }
+ return this.request(`/backups/policies/${targetType}/${targetId}/drill`, { method: 'POST' });
+}
+
export async function deleteBackupRun(targetType, targetId, runId) {
return this.request(`${runsBase(targetType, targetId)}/${runId}`, { method: 'DELETE' });
}
diff --git a/frontend/src/styles/pages/_backups.scss b/frontend/src/styles/pages/_backups.scss
index aa8e04cb..af48f954 100644
--- a/frontend/src/styles/pages/_backups.scss
+++ b/frontend/src/styles/pages/_backups.scss
@@ -1,732 +1,10 @@
-// ============================================
-// BACKUPS PAGE (redesign — infra console look)
-// ============================================
-
-.backups-page {
-
- // ── KPI strip (4 tiles, +1 when a remote provider is set) ──
- .bk-kpis {
- display: grid;
- grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
- gap: $space-4;
- margin-bottom: $space-5;
-
- @media (max-width: $breakpoint-sm) { grid-template-columns: 1fr; }
- }
-
- // ── list head: mono eyebrow + segmented filter + actions ─
- .bk-listhead {
- display: flex;
- align-items: center;
- gap: $space-3;
- margin-bottom: $space-3;
-
- h2 {
- flex: 1;
- min-width: 0;
- margin: 0;
- font-family: var(--mono);
- font-size: 10.5px;
- font-weight: $font-weight-semibold;
- letter-spacing: 0.08em;
- text-transform: uppercase;
- color: var(--text-faint);
- }
-
- @media (max-width: $breakpoint-md) {
- flex-wrap: wrap;
-
- h2 { flex-basis: 100%; }
- }
- }
-
- // ── dense tables (backup archive + schedules) ───────────
- .bk-card {
- background: var(--surface);
- border: 1px solid var(--border-soft);
- border-radius: var(--radius);
- overflow: hidden;
- }
-
- .bk-empty {
- padding: $space-10;
- text-align: center;
- color: var(--text-faint);
- font-size: 13px;
- background: var(--surface);
- border: 1px solid var(--border-soft);
- border-radius: var(--radius);
- }
-
- .bk-table {
- // actions column shrinks to fit, buttons never wrap
- th:last-child { width: 1%; }
- td:last-child { white-space: nowrap; }
-
- tbody tr.is-disabled {
- opacity: 0.55;
-
- .bk-ico {
- background: var(--surface-3);
- color: var(--text-faint);
- }
- }
- }
-
- // backup kind tile (tinted per type)
- .bk-ico {
- width: 30px;
- height: 30px;
- border-radius: 8px;
- display: grid;
- place-items: center;
- flex-shrink: 0;
- background: var(--surface-2);
- color: var(--text-dim);
-
- &--application { background: var(--accent-bg); color: var(--accent-bright); }
- &--database { background: var(--violet-bg); color: var(--violet); }
- &--files { background: var(--amber-bg); color: var(--amber); }
- }
-
- // mono type chip (application / database / files) — %sk-state shape
- // (_design-system.scss) with local sizing + bordered per-type tints kept
- .bk-type {
- @extend %sk-state;
- padding: 2px 7px;
- border-radius: 5px;
- font-size: 10.5px;
- background: var(--surface-2);
- border: 1px solid var(--border);
- color: var(--text-dim);
-
- &--application {
- background: var(--accent-bg);
- border-color: fade($accent-primary-raw, 30%);
- color: var(--accent-bright);
- }
-
- &--database {
- background: var(--violet-bg);
- border-color: rgba(176, 123, 245, 0.3);
- color: var(--violet);
- }
-
- &--files {
- background: var(--amber-bg);
- border-color: rgba(245, 185, 69, 0.3);
- color: var(--amber);
- }
- }
-
- .bk-when {
- font-family: var(--mono);
- font-size: 12px;
- color: var(--text-faint);
- white-space: nowrap;
- }
-
- .bk-actions {
- display: flex;
- justify-content: flex-end;
- gap: $space-1;
- }
-
- // schedule time chip (matches the cron expression chip)
- .bk-sched {
- display: inline-flex;
- align-items: center;
- gap: 5px;
- padding: 2px 8px;
- font-family: var(--mono);
- font-size: 11.5px;
- color: var(--text-dim);
- background: var(--surface-2);
- border: 1px solid var(--border);
- border-radius: 6px;
- white-space: nowrap;
-
- svg {
- color: var(--text-faint);
- flex-shrink: 0;
- }
- }
-
- // remote-upload indicator on schedule rows
- .bk-remote-on { color: var(--green); }
- .bk-remote-off { color: var(--text-ghost); }
-
- // storage pills carry a small lucide icon instead of the dot
- .sk-pill svg { flex-shrink: 0; }
-
- @media (max-width: $breakpoint-md) {
- .bk-card { overflow-x: auto; }
- .bk-table { min-width: 720px; }
- .bk-table--schedules { min-width: 880px; }
- }
-
- // ── storage tab: real-data spec strip ───────────────────
- .bk-specs {
- display: grid;
- grid-template-columns: repeat(3, 1fr);
- gap: $space-3;
- margin-bottom: $space-4;
-
- @media (max-width: $breakpoint-sm) { grid-template-columns: 1fr; }
- }
-
- // storage/settings cards → surface chrome + eyebrow headers
- .card {
- background: var(--surface);
- border: 1px solid var(--border-soft);
- border-radius: var(--radius);
- }
-
- .card-header h3 {
- margin: 0;
- font-family: var(--mono);
- font-size: 10.5px;
- font-weight: $font-weight-semibold;
- letter-spacing: 0.08em;
- text-transform: uppercase;
- color: var(--text-faint);
- }
-
- .storage-provider-config {
- margin: $space-4 0;
- padding: $space-4;
- background: var(--surface-2);
- border: 1px solid var(--border-soft);
- border-radius: var(--radius-sm);
-
- h4 {
- margin: 0 0 $space-4;
- font-family: var(--mono);
- font-size: 10.5px;
- font-weight: $font-weight-semibold;
- letter-spacing: 0.08em;
- text-transform: uppercase;
- color: var(--text-faint);
- }
- }
-
- // ── forms (cards + modals): mono eyebrow labels ─────────
- .form-group > label:not(.checkbox-label) {
- font-family: var(--mono);
- font-size: 10.5px;
- font-weight: $font-weight-semibold;
- letter-spacing: 0.08em;
- text-transform: uppercase;
- color: var(--text-faint);
- }
-
- .form-row {
- display: grid;
- grid-template-columns: 1fr 1fr;
- gap: $space-4;
-
- @media (max-width: $breakpoint-sm) { grid-template-columns: 1fr; }
- }
-
- .form-help {
- display: block;
- margin-top: $space-2;
- font-family: var(--mono);
- font-size: 11px;
- color: var(--text-faint);
- }
-
- // inline hint inside an eyebrow label — undo the uppercase
- .form-help-inline {
- font-family: var(--mono);
- font-size: 10px;
- font-weight: $font-weight-normal;
- letter-spacing: normal;
- text-transform: none;
- color: var(--text-ghost);
- }
-
- .form-actions {
- display: flex;
- gap: $space-2;
- margin-top: $space-5;
- }
-
- .checkbox-label {
- display: flex;
- align-items: center;
- gap: $space-2;
- cursor: pointer;
-
- input[type="checkbox"] {
- width: 16px;
- height: 16px;
- cursor: pointer;
- accent-color: var(--accent);
- }
-
- span {
- font-size: 13px;
- color: var(--text);
- }
- }
-
- // ── restore modal ───────────────────────────────────────
- .bk-restore-warn {
- display: flex;
- gap: 11px;
- padding: 13px 15px;
- margin-bottom: $space-5;
- border-radius: 11px;
- background: var(--amber-bg);
- border: 1px solid rgba(245, 185, 69, 0.28);
- font-size: 12.5px;
- line-height: 1.5;
- color: var(--text-dim);
-
- svg {
- color: var(--amber);
- flex-shrink: 0;
- margin-top: 1px;
- }
-
- b { color: var(--text); }
- }
-
- .bk-restore-details {
- padding: $space-1 $space-4;
- background: var(--surface-2);
- border: 1px solid var(--border-soft);
- border-radius: var(--radius-sm);
- }
-
- // ── spinners (upload / test-connection buttons) ─────────
- .spinning {
- animation: spin 1s linear infinite;
- }
-}
-
-@keyframes spin {
- from { transform: rotate(0deg); }
- to { transform: rotate(360deg); }
-}
-
-// Reusable inline spinner for the protection components (lucide <Loader2 className="spin" />).
-.spin { animation: spin 1s linear infinite; }
-
-// ============================================
-// PROTECTION PANEL (backup protection — WordPress + Services settings)
-// Rendered inside .app-overview-grid > .app-overview-left, so these styles are
-// top-level (not nested under .backups-page).
-// ============================================
-.protection-panel {
- // Single column inside a settings pane (no right rail).
- grid-template-columns: 1fr;
-
- .app-overview-left {
- display: flex;
- flex-direction: column;
- gap: $space-4;
- }
-}
-
-// ── Card 1: status ──────────────────────────────────────────
-.protection-status-card {
- &__toggle {
- display: flex;
- align-items: flex-start;
- gap: $space-3;
- padding-bottom: $space-4;
- border-bottom: 1px solid var(--border-soft);
-
- label {
- display: flex;
- flex-direction: column;
- gap: 2px;
- cursor: pointer;
- }
- }
-
- &__toggle-title { font-weight: $font-weight-semibold; color: var(--text); }
- &__toggle-sub { font-size: 12px; color: var(--text-faint); }
-
- &__status {
- display: flex;
- align-items: center;
- flex-wrap: wrap;
- gap: $space-2;
- margin: $space-4 0;
- font-size: 13px;
- color: var(--text-dim);
- }
-
- &__kpis {
- display: grid;
- grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
- gap: $space-3;
- margin-bottom: $space-4;
-
- @media (max-width: $breakpoint-sm) { grid-template-columns: 1fr; }
- }
-
- &__actions {
- display: flex;
- align-items: center;
- flex-wrap: wrap;
- gap: $space-2;
- }
-}
-
-// ── Card 2: schedule ────────────────────────────────────────
-.schedule-card {
- .app-panel-body {
- display: flex;
- flex-direction: column;
- gap: $space-4;
- }
-
- &__field {
- display: flex;
- flex-direction: column;
- gap: $space-2;
-
- > label {
- font-size: 12px;
- font-weight: $font-weight-semibold;
- color: var(--text);
- }
- }
-
- &__days {
- display: flex;
- gap: $space-2;
- }
-
- &__day {
- width: 32px;
- height: 32px;
- border: 1px solid var(--border-soft);
- background: var(--surface-2);
- color: var(--text-dim);
- border-radius: $radius-sm;
- font-size: 12px;
- font-weight: $font-weight-semibold;
- cursor: pointer;
- transition: border-color .12s ease, background .12s ease, color .12s ease;
-
- &:hover { border-color: var(--accent-bright); }
-
- &.is-active {
- background: var(--accent-bg);
- color: var(--accent-bright);
- border-color: var(--accent-bright);
- }
- }
-
- &__preview {
- display: flex;
- align-items: center;
- gap: $space-2;
-
- code {
- font-family: var(--mono);
- font-size: 12px;
- color: var(--text);
- background: var(--surface-2);
- border: 1px solid var(--border-soft);
- border-radius: $radius-sm;
- padding: 2px 8px;
- }
- }
-
- &__preview-label {
- font-family: var(--mono);
- font-size: 10.5px;
- text-transform: uppercase;
- letter-spacing: 0.04em;
- color: var(--text-faint);
- }
-
- &__retention-row {
- display: flex;
- align-items: center;
- gap: $space-2;
- font-size: 13px;
- color: var(--text-dim);
-
- input { width: 80px; }
- }
-
- &__remote {
- flex-direction: row;
- align-items: flex-start;
- gap: $space-3;
-
- label {
- display: flex;
- flex-direction: column;
- gap: 2px;
- cursor: pointer;
- }
- }
-
- &__actions { margin-top: $space-1; }
-}
-
-// ── Card 3: history list ────────────────────────────────────
-.backup-history-list {
- width: 100%;
-
- &__row.is-clickable { cursor: pointer; }
-
- &__ico {
- display: inline-flex;
- color: var(--text-faint);
- }
-
- &__when {
- color: var(--text-dim);
- font-size: 13px;
- white-space: nowrap;
- }
-
- &__actions {
- display: flex;
- justify-content: flex-end;
- gap: $space-1;
- }
-
- &__storage {
- display: inline-flex;
- align-items: center;
- gap: 4px;
- font-size: 12px;
- color: var(--text-dim);
- }
-}
-
-// ── History card chrome (list/calendar toggle + day-filter chip) ────
-.backup-history-card {
- &__tools {
- display: flex;
- align-items: center;
- gap: $space-2;
- }
-
- &__toggle {
- display: inline-flex;
- border: 1px solid var(--border-soft);
- border-radius: $radius-sm;
- overflow: hidden;
-
- button {
- display: inline-flex;
- align-items: center;
- justify-content: center;
- width: 28px;
- height: 26px;
- background: var(--surface-2);
- color: var(--text-dim);
- border: none;
- cursor: pointer;
-
- &.is-active { background: var(--accent-bg); color: var(--accent-bright); }
- & + button { border-left: 1px solid var(--border-soft); }
- }
- }
-
- &__chip {
- display: inline-flex;
- align-items: center;
- gap: 4px;
- padding: 2px 8px;
- font-size: 12px;
- color: var(--text-dim);
- background: var(--surface-2);
- border: 1px solid var(--border-soft);
- border-radius: $radius-full;
- cursor: pointer;
-
- &:hover { color: var(--text); }
- }
-}
-
-// ── Calendar view ───────────────────────────────────────────
-.backup-calendar {
- &__head {
- display: flex;
- align-items: center;
- gap: $space-2;
- margin-bottom: $space-3;
- }
-
- &__month {
- flex: 1;
- font-weight: $font-weight-semibold;
- color: var(--text);
- }
-
- &__nav,
- &__today {
- display: inline-flex;
- align-items: center;
- justify-content: center;
- height: 28px;
- padding: 0 $space-2;
- background: var(--surface-2);
- border: 1px solid var(--border-soft);
- border-radius: $radius-sm;
- color: var(--text-dim);
- cursor: pointer;
-
- &:hover { border-color: var(--accent-bright); color: var(--text); }
- }
-
- &__grid {
- display: grid;
- grid-template-columns: repeat(7, 1fr);
- gap: $space-1;
- }
-
- &__dow {
- text-align: center;
- font-family: var(--mono);
- font-size: 10px;
- text-transform: uppercase;
- letter-spacing: 0.04em;
- color: var(--text-faint);
- padding-bottom: $space-1;
- }
-
- &__cell {
- position: relative;
- aspect-ratio: 1 / 1;
- display: flex;
- flex-direction: column;
- align-items: center;
- justify-content: center;
- border: 1px solid var(--border-soft);
- border-radius: $radius-sm;
- background: var(--surface-2);
- color: var(--text-dim);
- font-size: 12px;
-
- &--empty { background: transparent; border-color: transparent; }
-
- &--has {
- cursor: pointer;
-
- &:hover { border-color: var(--accent-bright); }
- &.is-success { border-color: var(--green); background: var(--green-bg); }
- &.is-failed { border-color: var(--red); background: var(--red-bg); }
- &.is-running { border-color: var(--cyan); background: var(--cyan-bg); }
- }
- }
-
- &__badge {
- position: absolute;
- top: 2px;
- right: 3px;
- font-size: 9px;
- font-weight: $font-weight-semibold;
- color: var(--text-faint);
- }
-}
-
-// ── Detail drawer ───────────────────────────────────────────
-.backup-detail-drawer {
- display: flex;
- flex-direction: column;
- gap: $space-4;
-
- &__error {
- padding: $space-2 $space-3;
- font-size: 13px;
- color: var(--red);
- background: var(--red-bg);
- border: 1px solid var(--border-soft);
- border-radius: $radius-sm;
- }
-
- &__meta {
- display: grid;
- gap: $space-1;
- }
-
- &__row {
- display: flex;
- justify-content: space-between;
- gap: $space-3;
- padding: 6px 0;
- border-bottom: 1px solid var(--border-soft);
- font-size: 13px;
-
- span:first-child { color: var(--text-faint); }
- span:last-child { color: var(--text); }
- }
-
- &__cost {
- font-size: 13px;
- color: var(--text-dim);
- font-family: var(--mono);
- }
-
- &__actions {
- display: flex;
- flex-wrap: wrap;
- gap: $space-2;
- }
-
- &__joblink {
- display: inline-flex;
- align-items: center;
- gap: 4px;
- font-size: 13px;
- color: var(--accent-bright);
- }
-}
-
-// ── Restore drawer ──────────────────────────────────────────
-.restore-drawer {
- display: flex;
- flex-direction: column;
- gap: $space-4;
-
- &__section { display: flex; flex-direction: column; gap: $space-2; }
- &__label { font-size: 12px; font-weight: $font-weight-semibold; color: var(--text); }
-
- &__tables {
- display: flex;
- flex-direction: column;
- gap: $space-1;
- max-height: 180px;
- overflow-y: auto;
- padding: $space-2;
- border: 1px solid var(--border-soft);
- border-radius: $radius-sm;
-
- label { display: flex; align-items: center; gap: $space-2; font-size: 13px; color: var(--text-dim); }
- }
-
- &__options { display: flex; flex-direction: column; gap: $space-3; }
-
- &__option {
- display: flex;
- align-items: flex-start;
- gap: $space-3;
-
- label { display: flex; flex-direction: column; font-size: 13px; color: var(--text); cursor: pointer; }
- }
-
- &__warning {
- display: flex;
- align-items: flex-start;
- gap: $space-2;
- padding: $space-2 $space-3;
- font-size: 13px;
- color: var(--amber);
- background: var(--amber-bg);
- border: 1px solid var(--border-soft);
- border-radius: $radius-sm;
- }
-}
+LdbCls2cOX529YMxMz86/EK7TsxOpe4t6kpI7BC641eG9mHH7NVye8FM7vi+tL9v0qu9bYJ6pzb0CwUHAy0ZKWSdJx7Gq29zY1TnVmlBarE5P3L/w/HFv7EXeXsAr0VfhTb85V3tfcHlMR/QLbzcVKIbHfPua1b7b8+cqRWAdFXvv7agazNpvEoUB5VKONznN17CxjvmNZfykjN1aWBsLP6CUTaNXpwuvakQdiMSJTS5pGOVKAcXtcST+63/46nJ2Dmc+Vcvd1bTggna7Svr0/fuA7r6lHaLcGnK6DO1qwGGy3t7lHve4KniN4YcLKI7TYpMjMxV85FMMyRobTU69k71xSyJT0YwaHbidW068FnQZ+L3DzoMlGTu1qHwN4iZH9yL4p2lcgN5EEED8dGLLW5uLeeeAb0wJbrSg9sJQH8qKkGO5DuhjcmnigNZH0oJIb7/W+hRg4p4cviVksEKOgjRALuucGqXuq+R9g5mdimKauZXzaDPx5aeGzSgDcBihc36BS5MrIeYh7G/k7mllUUebh6g52CSgjj0gKxn4SkrzCyvM7yXod9MicpxG5r81n8HAgbZo1ACDVVyslkv2YgbkKNz39Aio521nZHXgTwtRGHjd/hSWBFAHpxW1VPpbsNvteNza+Qd0gKuaFeNyheF/R0kPQop6+SEDXk1bpJdXjDn3QrzvBCUM+JdZcTcRkDVQyI4Kg8mpIOp46JixyKXCh1bULPiD93t6w93/bxbKfOcGudX/d2IX2chD0yT7gSfV9B4/lZnshkEQFH71G4I9TMJJ2d/MfTLv9dQL4WU7UjQXOMTpwDQS4LfQSU5oin7Pw4dNyJ3RhWvMzi57Kx8UXrr6iou3/ZI4O8iWbq1Do3Ev1HInHFjBnRosDzwMnN/5LQNzVEhnerPmpW/GFQ0uEadQ3n3ahBHMuhrap2qUELMP28uwHXoYP61twxle2IL69Y+itfAGNKk0j/TkOXSyrpKvMkI3vyyWe9+GV8EZDkF6RhNrjgH9c2R4lUMOimFc59spWVyQrnANcBZ1TJExSQnrXQ2gYBw1i0bjiCf88Xt82eQ4XJYirDwOpac163g+R2b+uXFqGkvGeb7gVpZpP8Q65jwcRwShs+Do/N5yI0DtzNZVXrbJHyDuehfpPYgqGgemOg/rrcWNyER33TgYLwbIDDPkQZsl4xHBtFrbj3ISoLuACKbr7pZXXF8D/K1t6jWeSIPoebs7WjS5POT5dkbYhTx01dlTYrvTkzwXhieqmh9Oz/qdzCcSIR6beiL9x5wCeEtnxYaW2faEiCOATUJOuRJP2CrVYoWrcsG0AFj0/InNf2DDJmG1xEAWHBI05vPIRnNCuFi7SM0DRI6aIoa61/ysJ8kdkSJtbOSAv+cdOiiw2rOrpXiQkPDN3x94psoa4opVSSqifOrU4byOcbWXOJ7I1Nq/cd0Jim8H4Ptz6m0tP836TULUiOsk6UR/0/b/CWXQ4y7XLdm8mpIouWlFObBsGLrBwYCVttdnd8sWb3Be9eJJ2W8H3gb0C+Ai9kH4GZC2D4GWmoW1wHTS09wUqYZMkqhGf0RJXaPAi5sRnmPzVLl2c5P0QQWq0XYatcnm4l0grE4WnF8VMzqBN15rhb0KRWMRjS/WjPL4OKMR6xlWNuUmSjLh8lA769QqXaUQarIAivoHL/+cHkw9oRPwGGGICts1CQhNJ7GvfFaLn+ZOVHLuZ4sV9Sq+MGs7nwPFHCiX7LXHt/3xkOS+IhxUlTBrMQXsO75SkSltyqtZuNliegA1g/vIWYlsw0Qz2IjDRDTdSVoJUY7TOxBQ/GeNEjzBNsur7dZqxxwpSbus6B8uFcMg6L+NkqCUOZDZyOhn0MX6A6Mj4ogJpdIedZrHIG6MlRaibbqUUqXOH69k5riaoGtVxeSvXitaDkCiVE0ds9tPuwRpSRWgdiK9ow9V73vP0tGED8SjdyOjLNZ283EXEFaArM4A2bpl62DO9vGYxogQWjlIRnIghmJ+Skj0bPD5LueD1xG+dx/Zl9OIFK/6EphQAz+3DH249uXTja0w0tqeEOc4eNd6SbuZZh1wDcjwhJUIlJHEYEuxB+yL2Joj9Pw7sVl2or13I6QFzUKDucSUfjktAqYdUWeRrnewG2cuz3ptm/ng4hutM7OkL74k0WEhVXDrDVv0rrqDYcNByLb0lzgOhtaBHsyR/Nel/z4EZPEY471bT7islXgsewHjWg5aSzmKlHO9O0mrtRIX1PwlwXSNTRbhMMIluBxydlOq2oWXQEIqhoKc/JhFKge6czzPTXOVi8sNrghvmEpzGeYcG6hbKzhzIaF3HTWh/57VyD2N2VLML7365/Jtyz5u7W7lCbyPueH+8Eq1voBLUK3+W6HbuKPagr39M0nXFazmUXIeLtB5oPawaejea5LLVIng2fAYVFf1CnbpzF61H5tSDZps/KmlF9+S7pSceACd4EaxSDuTYcWUaf6R+5WCJIlHv0AnMzN/A+eu/7xTUhIlFV4LeSqhBAzr+HZjsfMustVFDkxHmZU8syK+7ydIRpGbAm1jk+ACxAyhP2OUJWPX36Ul1gWBzJN4cpAdSdzabfUFpkBV2tgHjGj5sWTnjC/wyoyEyTEo2ZENJkxSXLb4DKFudvRtogIsvmFTfIKtaMe7efHW7frTp3b5y7AWkZXZqXGrxXufm8Wi/cYck2UxeqMnVrTQrTimynACFfr3ByMYTsp0MZQGgQ2OL2qdPJVePlEosk3rYrKB4DXUJ0DS1A+r/+/upvWRZfqLY4YHz//fd5BF+86FCAnm57TJV9yggF438GKiGXz0oWxyDO/AdJnPn+N7KKb9thLp0b4WcNf8RjgdCjMT+zJGTjBuwz6WwvISD3Pi0QahIkpAKtu38JHbKo5S2u2QMDmGAVQHuxcB2s8QKuBOQ7u8uJailqWcBUNQB8/pqBKSAjUcZueRdNJ8/76yttgFJaLhnaWXTEe2joA55qX71f0Hn+VUmY7GX+KRr3ZJp+TuJItEXhxlOMwvIcALyCco+OkZ9IN58A7IqGFqJBNdkLUjPDpBEODvjkBSIQbu1U67tARfnCq2LZijGt8AJlDEG130gPac28wPVkUobkZ7QTOOiIDFlWQ+8oUGB3MaOCGn34rxc1t0yOiL0rreI14vfDxv8Idgj+yP2C82WUkpojU8i/+oTrDLM42hIgy5x5fcjTdLWYoyYsxJog2U55nKeUmeJu3lvduwV/CXTL69dfRhLEZPn7MfSt7U33bp/QdwDy9HfCjj9A7rCTs4PXiGrJjZ3HfNvVdF7xHQu262yXP0xcOBceK5+RiF0Xc/SDctNHh5zvIQIFbgHJFBSxaA+a7Sko+Nt49TU/aOWV0SdhAtO4/BreOv9/fvR6aiwlmd9Qunw4I40u4I1QSaZr7gG7IB9nt6BRaJBg548DFiVufmlWwIqHg4wX6yaFDGg6BieSquZourJ7w4Z0GsH8m3tTMq2B3fUBDpev4IXN3QyVHdQVCPU5pmcbVxoxs1O/ZX++Wr1DJHrZsvkB/KvMZEpB0AMnhC8eLbsszefUQk9xcJCFR5XKssAjs1IbBDyYMjLZqI50UxMF8QAaoGktFXPLD9AUKgAdqFP2TlxhmaBixEN1rXk6UNQikK6pWFIFclsGj0SZo/oSYxaIe9O785JzXfpjJzlH0V8gbeAlRTwz8BTFJPexJMVJv0DZA8hjxgR4t8dgYxLwg+XP68vb6M03VhLug6hcep4yRoB3QsJI+FQMKVaQRS3vQcsyUmtZi6x9zWf/OhohTwQ6bZRRzl0EwBpivxoxdCnifLaxAqS9RV9gCGnWxzhpfwv2gNhj5I1uY+Wp6MyGFdno2sVIRWChMODmsR2k0hkJZ4yfdJgFK9extMH8R96sa+SOlPYMQAPLL4r8W6fyagyfZFwjFMVYy7c66XJIyoNsy7QCw1bxZvkFj+MvsqcCYcp1H1hH0gSLxL8vsgtDKYrzm4m32rxBKyYdqx7U9udwlaiHPS3n02CRwXnMr4PvoQ4OfVPovU1dG6SLPuBLt0kqpyZInhdBBQgDr+4vlhyve/WgNrGvwGgYEcSIkdldlpGHau4OhI3dAm/v3yYYbHLzULH3Sxdpkd8x4rEIJpRrAKprgxCEhO3kw3T5Ob4vXxdXgrRVHPGd40z4SmFkIfVj0a5EdGxe7vlt7NR7ABTLrdvamJ44nht9T/qqyhfbicpj2bNUkEtomqnzusyLcBTyJhBclIxMKnpYSITx0fMEPosCc3ghVx3yvUYAQ=="}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":3328,"cache_read_input_tokens":258148,"output_tokens":1984,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":3328,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":1984,"cache_read_input_tokens":258148,"cache_creation_input_tokens":3328,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":3328},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CcnxA9cThuB62hk1nQa9x","type":"assistant","uuid":"dd93c7f8-826a-4eb8-8674-31eae9f9214b","timestamp":"2026-07-07T15:42:37.720Z","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}
+{"parentUuid":"dd93c7f8-826a-4eb8-8674-31eae9f9214b","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxAC2ZhePYuahgxeMmH","type":"message","role":"assistant","content":[{"type":"text","text":"Critical findings: Documents is **not** OneDrive-backed, Recycle Bin bypassed. The unpushed commits only survive if there's a Volume Shadow Copy / restore point, or the GitHub remote. Let me check both recovery avenues:"}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":3328,"cache_read_input_tokens":258148,"output_tokens":1984,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":3328,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":1984,"cache_read_input_tokens":258148,"cache_creation_input_tokens":3328,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":3328},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CcnxA9cThuB62hk1nQa9x","type":"assistant","uuid":"0310623c-db69-4a0d-8002-967d7527480d","timestamp":"2026-07-07T15:42:38.597Z","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}
+{"parentUuid":"0310623c-db69-4a0d-8002-967d7527480d","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxAC2ZhePYuahgxeMmH","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01N4eBeWoctiXCTxxptNQgm7","name":"PowerShell","input":{"command":"Write-Output \"=== Restore points ===\"; try { Get-ComputerRestorePoint -ErrorAction Stop | Format-Table SequenceNumber, CreationTime, Description -Auto } catch { \"restore-point query: $($_.Exception.Message)\" }\nWrite-Output \"=== vssadmin shadows ===\"; & vssadmin list shadows 2>&1 | Select-Object -First 30\nWrite-Output \"=== Previous Versions dir listing via wmic ===\"; & cmd /c \"vssadmin list shadowstorage\" 2>&1 | Select-Object -First 15\nWrite-Output \"=== git remote reachable? ===\"; & git ls-remote https://github.com/jhd3197/ServerKit.git 2>&1 | Select-Object -First 15","description":"Check restore points, shadow copies, remote"},"caller":{"type":"direct"}}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":3328,"cache_read_input_tokens":258148,"output_tokens":1984,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":3328,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":1984,"cache_read_input_tokens":258148,"cache_creation_input_tokens":3328,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":3328},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CcnxA9cThuB62hk1nQa9x","type":"assistant","uuid":"e1c3e155-b961-4b17-8951-00a5b7570c2c","timestamp":"2026-07-07T15:42:42.234Z","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}
+{"type":"last-prompt","lastPrompt":"# ServerKit Plan Executor > Set the plan path below, paste this whole file as your message. Claude executes every > phase, one by one, with subagents — no questions, no stopping early. ## PLAN ```…","leafUuid":"e1c3e155-b961-4b17-8951-00a5b7570c2c","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498"}
+{"type":"ai-title","aiTitle":"Execute ServerKit staging testbed plan","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498"}
+{"type":"mode","mode":"normal","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498"}
+{"parentUuid":"e1c3e155-b961-4b17-8951-00a5b7570c2c","isSidechain":false,"promptId":"487ba573-d3df-4194-aced-415c39485354","type":"user","message":{"role":"user","content":[{"type":"tool_result","content":"Exit code 2\n=== Restore points ===\r\nrestore-point query: Access denied \r\n=== vssadmin shadows ===\r\nvssadmin 1.1 - Volume Shadow Copy Service administrative command-line tool\r\n(C) Copyright 2001-2013 Microsoft Corp.\r\n\r\nError: You don't have the correct permissions to run this command. Please run this utility from a command\r\nwindow that has elevated administrator privileges.\r\n \r\n=== Previous Versions dir listing via wmic ===\r\nvssadmin 1.1 - Volume Shadow Copy Service administrative command-line tool\r\n(C) Copyright 2001-2013 Microsoft Corp.\r\n\r\nError: You don't have the correct permissions to run this command. Please run this utility from a command\r\nwindow that has elevated administrator privileges.\r\n \r\n=== git remote reachable? ===\r\n578b034448c492f905628217c90c6dd73da80937\tHEAD\r\n07baefb53f12e070f600746443856ea7fa059c32\trefs/heads/dev\r\n578b034448c492f905628217c90c6dd73da80937\trefs/heads/main\r\n739a106d0763c57e267e83924c6a68f011e7b55e\trefs/heads/new-ui\r\n976ba5ad372b8477bfe476786a6060ef4542d0c8\trefs/pull/1/head\r\n1e8afb7a438aa318b678dbf7f05dcce8229462b9\trefs/pull/10/head\r\n62fda1697626f1c67cd7168a60193b8c97fb524e\trefs/pull/11/head\r\ne1fe2a290ef230c158839c396917c942f063f770\trefs/pull/13/head\r\nbefcdb9e9e1e413b848a9e14a9d76c8079a30c42\trefs/pull/14/head\r\n401fd71402b2dddd7196e6ba117081160003c9a5\trefs/pull/15/head\r\n34ca8dee8fc5b1d03379144f1484be9227eaab35\trefs/pull/16/head\r\nd9e12d1eb14844ad1ad33ebf3b07fa270c7661ed\trefs/pull/17/head\r\n5f0b70b06c4d029328635a7115208783056502be\trefs/pull/18/head\r\nbbf24a4704ea7a7f29b87539bd87956b2d6e99e7\trefs/pull/2/head\r\n43004416f9f74e8346c6b31d96ce3ff4769ce0df\trefs/pull/20/head","is_error":true,"tool_use_id":"toolu_01N4eBeWoctiXCTxxptNQgm7"}]},"uuid":"5ac719ea-4bb5-47e7-a9ae-eeaf451b415a","timestamp":"2026-07-07T15:42:46.010Z","toolUseResult":"Error: Exit code 2\n=== Restore points ===\r\nrestore-point query: Access denied \r\n=== vssadmin shadows ===\r\nvssadmin 1.1 - Volume Shadow Copy Service administrative command-line tool\r\n(C) Copyright 2001-2013 Microsoft Corp.\r\n\r\nError: You don't have the correct permissions to run this command. Please run this utility from a command\r\nwindow that has elevated administrator privileges.\r\n \r\n=== Previous Versions dir listing via wmic ===\r\nvssadmin 1.1 - Volume Shadow Copy Service administrative command-line tool\r\n(C) Copyright 2001-2013 Microsoft Corp.\r\n\r\nError: You don't have the correct permissions to run this command. Please run this utility from a command\r\nwindow that has elevated administrator privileges.\r\n \r\n=== git remote reachable? ===\r\n578b034448c492f905628217c90c6dd73da80937\tHEAD\r\n07baefb53f12e070f600746443856ea7fa059c32\trefs/heads/dev\r\n578b034448c492f905628217c90c6dd73da80937\trefs/heads/main\r\n739a106d0763c57e267e83924c6a68f011e7b55e\trefs/heads/new-ui\r\n976ba5ad372b8477bfe476786a6060ef4542d0c8\trefs/pull/1/head\r\n1e8afb7a438aa318b678dbf7f05dcce8229462b9\trefs/pull/10/head\r\n62fda1697626f1c67cd7168a60193b8c97fb524e\trefs/pull/11/head\r\ne1fe2a290ef230c158839c396917c942f063f770\trefs/pull/13/head\r\nbefcdb9e9e1e413b848a9e14a9d76c8079a30c42\trefs/pull/14/head\r\n401fd71402b2dddd7196e6ba117081160003c9a5\trefs/pull/15/head\r\n34ca8dee8fc5b1d03379144f1484be9227eaab35\trefs/pull/16/head\r\nd9e12d1eb14844ad1ad33ebf3b07fa270c7661ed\trefs/pull/17/head\r\n5f0b70b06c4d029328635a7115208783056502be\trefs/pull/18/head\r\nbbf24a4704ea7a7f29b87539bd87956b2d6e99e7\trefs/pull/2/head\r\n43004416f9f74e8346c6b31d96ce3ff4769ce0df\trefs/pull/20/head","sourceToolAssistantUUID":"e1c3e155-b961-4b17-8951-00a5b7570c2c","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}
+{"parentUuid":"5ac719ea-4bb5-47e7-a9ae-eeaf451b415a","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxCgqyAWrW7sHmVtEk6","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"","signature":"EpIiCmMIDxgCKkCM83hkUpKsTAdmjQp4vaBEBrTs/q6o49h7HnmFvPTNFVbiGnNcjzlGZQKnB3yabCxxeN+4KfpbPbWdBrwpeEHHMg9jbGF1ZGUtb3B1cy00LTg4AEIIdGhpbmtpbmcSDNo1GA/83N9NkvkQ9RoMo5aU3Kqe56UyV/CkIjBX8QcqeyjcOI2tVS8kHUG9BhsEUc0iJlFz5c8n7vjSP4MFScy6QFD/jjsSpTeZjUYq3CCUKlpZbUpsH4poqn9OY4X5Xeh4vFU+JCmMgedHIBx3xMjzEu2mBmaC0wOPH21TF5Mk4piWemSm8oVOEniiOAf2FC4tCsvOAWK3hdXKmxeBiKKI2G8z8j6lGf4etRGWbYiA6VdwTBiUPAB7bEJfRLbC1Yl+vsqTaCtx/VSj9bEGtjS9UaJdnU629rJmTZauDkfylfLbBaZKzDeaa4UVxQZt1FKhHFLlEvaZ9A31kxalIGnSWA36z4/vL0gBi0bpWecj+9Sct/zPYcSVD4edSUq1za9TTKjB57Sy5Ycv1xhvntTYoFxvKPSvSKqRycLYsKQkx0+BL5mVJvloE6mL1aOR4IcqJco3/uyOzxwITjBy3T9Hp8yqwNT2/2kYr1x3aKxU+XelNTjb5bTRwf3Q2iqin9rlggdEgwO7MpPqpHSxXIDL0Cq5PU4x/7cM1mCF0FX5P2kk6jD7ZhO9sZFOCp3UntusycVrQYgthciBedqnHT1UcvAWMJ7sLIqxmtqax50oUF++7plG69QXNEUbzzahJ5uLJmx7oOUuSX/CmD4H4j0yqjmUSR48m5EtdTv6hZJS5lmapQms93Nf4Bva0aSBPNVJ5spKSzYPY8Jm3ksOof/mnQl3t0iFRTIP5JxKFl4waT3aCVslD2IpcrnEjkDxyHs+c+OXViU2C2P+dPodnpKHEdFShZnWahR4FxB3TkhcGePnPVb0+t5/2jT+8ZflgvDMfSOp79vzypgcPBwxaHHiYGQx/xRBJm+W/COvYnub9U0JeBq5Q6N/dg8ss7ieWCf1hohTHMjeY74NDb8VBBYU6iliD9xMOh+yZSXOv+NIdjLCXZ3ievPFvxHUT4q5Yh46XEYfcfBj1iiSCPvf1xMM+oKjoBMGDhm/c9VhBHEitSKzcmdQ380tPho5F0vSx1sfhVbjU2Hx8QTlCDzKwvDvg8FOW26dbjCr182M4Rqs7y9r/82uZMBBi2tfnx0A+VVr+d4tA2tcoexWBKfgEt80//yRMtScgMXpsQibkxf/rlPjUCO8gGJql5dKB0BS+CTqQGhaL9q9kDoL1fjFdfzpyGt3rhTxjynRrgJsQBVsVymuafrhsehl6+bQQYiQp0NYW+PtoadbPYcErSZoPJBDMezG0zLwf1pj566vtd896tHP7VJduEC2Vq376M0N/nE/VQXzVnEikeR/v5XZH/y/ZcB+JLrs63Ng3Kht/vFUncJp71OAmhKZD8slEs78i0XYOzty1OxgaGQJzZ1Qy2y/4bvW9k+xvQllgm5WzmFtLYspP63WEFSlBNu8SNUKw8iC9M/sFg3G2xu0q56WbiM6bYQ4i63tzLVH7JqEG87CbdejCdv6VXco1QUNH7BW8x+IX9toaPYsd2jbyHqv3vThsue3mK0U+KDibDbzKc27SHmhR1re8CTLrINaKb+u+U7A1Cp0nvPyIZxVA+d34E3FCG3ctqMN49tAXybRp+3FWOMx9iQlR9Hf9CUxf6Syfox0mvUG2qBnog+bprOh1fZgIdj+imEYfXcyXedEljl7ZQmmC1nmfiHFcN8T6CGZRrTrxLYKUG3Oj9bo9YwG3TQRpWy91HzdZZ1S974eB8cplHDfAjjRVjhmuAeB+t0zr5FBokxODUjHpVism43WHH+tknTtqHUkk6HA5sFyOhfHi0AODbiBGAIkKUz2v4yrLKXSQWuTgbxRU+9ejCge4F1PlFwDB8pmUc83y+06LIZH+AEpxAWcWwYQ1m980bGnpXwgU19zMqMgDXP02pM1/G4zOeK9I197OP+6GT0kkduBcVDOS1Nrd4c2LcvWUBYCFhfsJUXIO13tON9vwTCom1dJTFYvZUHj7op5iwYcn1StqbhglXDro+ZQf0utu7vkRrCaM7ZBZWrzXcK8XMLyoChS4bwUk1p5teyQbBRypp8L/sV1vuYYShmfoxewPzlVYvCD1O3nWtP63DwR1K6MHboojE1cue/6qucVViLamob76T3QRe5RJ1/o2Ec9ZFq1BBlSnDMNVqapEhSQJURfVesxsM+UUuGYGkj+278wT79/D9YFo+cKlUn2gk1S8xZ+cNIFSYDuIb9Fypby533cVXk+mVwzTzpNzzOER/Da4zHWNDkmSSUhQktUNP3hs8KaYuNkXdZzeIVkDxU5sXtD0yfv33ir2zRYDGoDveEs9KJ4IUhueqBD/PYj++Ur9MhZ3W+sZG7W94jC3hwNrldc1FxgP8GnkEyUUjYscLzmD5Lc1Ii2ahQKpGS4n5CaTd5+pC1a5k5HbG+O38rrlzzY4ocuaAht0MH5Ya0babz2TKS673sjPtX2OrIguY67x7PE9ffG17qhC8lDK4UJXYYHK5Vs27Qx/AcKXKj070N6GfqV+k79yPRor239fBG3qn0xQKb1WuWbDfrsNeWL8j0Dh/5s/A9VJxuue2ExlusdBkFCsuM8eFdDbn79fMqTDFoWles1Rb3cHWnJZ4DxqzA5X8B3VKmTGi24CL9KnP9NWMgYjKFtzhVSR8eaa3zJNpcRktMDz5nRw8CV7jx+yuL0QK4YsnOwSOSjR0kqNKMvaaHX+59F875aOjJc8S9rk+0hsQaBgCOmCAjRqp2iAR6Pp3q2qdetNVGWRD9Y+rqq5v9i2tWpuV7v8bsLF8kjGOxyo+r8zwDtYaKuHxCrwisviaXrSd42/XGsbpdMG52wN4oSqd9FjEs+xVui9LNxMGy7glmASKDeyf/eQBrFFHMe82SAYzP+Z41RgW/pXwofoYxs5+aFodL18lpoJp+TLQoVV/Q1gPEJ+u0j9C34nzhbspF2q14DZLKJa9tULjxVaa0wLsKjKWiSE0pusL78zny/02LmFXlYm+Y4Em6v02nzusvYC56lP6l6bKe317D+J1YN8dyQO1LM5E55wUXkZIfF2pw4sdbupNFvnZC3YxgoR/7+4JPdkMe6KLkxSMZtrsMr44HmVm/F/xJQWC38c0EmWMzczX8YlXTxR4/72SAEeO6uNdvo+jzc2Wlh6qOZBlQN/O4LuDfdPu/bpcF9V7VHLsAq74L8uWRwqpMdp4KY5SUyX6YH8kLm818vdo5fkDyiEmy5VujBVlwrZmb3TslcY2zKag3Ga+aEtF18efaHVseoQphsH4mV/O8l7GH8simO0TysDXTCYwEkboZpXGFkvAmKaQRhwSWMXS6XF0CZkdfwHWo9sLQTrLq7CSeblwbpAvmrjSCb5cIe0Ngr5jtoJA/c0eCLh7ZS2OG2QKTgyQeRSOaXWxQsVGzMHyF/Xv1UnajStSSktgJYOKX0QM+Oxy6egaJuS4Mb5IRssYLKSvQ2w4hydViXEop/wUyI3v7NOz/js7YzXZ7zPvFjYwM+4flijh7KH5A0QMCp4BQQ7YOOOzN/snLEqLKSOOy32WKEyCnNL698dWhNfZOT3SzRhtinYJU3aL9HPV36apg3TgNWfBfM8IfSbq2Q8hGa0B6VjT4wmjIhmt5Sl6yKxxNho4XBok9gjTi8aNdse+ZRSUBet0uDlObrm+ZqX58ppfHLlF6LFp3+zpefiYGxlhZQu78ykIdBXAeU1e1sxi55yHSkYfsaE6Dt5MmBZu31WABjRC/hYn1nM7D/jvXqq+6W3TP9r6ES8mCE/JgiNS+bBTmu7VfIRhDKUZKvAxT7Sq5kdY9Sqa7rck4ICK6roEv6dfkjCqfURlwseGzb7EVni8oHVBp7qMoUT3hqt0XiIPttcKjOVqiAXZ9ArWW91A1gpin2DQRKoQDmPpWa5SMdjxmhxPzdp6MtjFsCCXOEf/wU1r94y8V84U+hlKxzZQqWnrIumpFTUdyqa0oa9zdMxlBGGy4AZIgHgKAlNA9CD30CVD29Cr0oYw3ZML3a5+HAeMGjwJ+e5Lpjx7S7tGldYc/TD4s3uZi6vbw8SlHSwfxskYdXdFq6DffMP5dKep2QlvKSUClODXtKnj7ETM6rg+4nvv1DtJsPHse2tRodJ2vtNlrvTZlCku15Hizi8GlZwX+NmFVI/y+bHvVlylnsAMKSH68IHkqy2YY5a0LWzihQp0dbMPGzSPXLH+zC8TmbRNJGRI/N725YgNkSm0fjE1AEmW7tKRSg1pOauJ8Ld/vUASy1nKsaBMvAjQvRSgrdN+djfSt8eM3J6srvpbyUGMhmNNXCmcVfRr2jCKnr1uxTRBRpn96LvMa4k6RJ2wKL+/rI9N9Mrpsj2IrgKJmOC/zUo5ijtuBjuelV9lxaB+iWpXIiVKz5wsPKcrcmC4Q7mNPQDtlctQFmlpiDNhwIyuaNEOSXmVseuMrGBoUpgPp90p7BmEikcggYLObwaQ2C3fYUJ0aWXJrTgKwYoIGn4JFYpl/5VOQ7cRk30942wUtdevm3WvieyS2CynQ6To74noZ7bO3qLeFRrA97nadPuyRmo1vaHXTb8bxO+ljXPgaWJD6QBCXFTTkQZGctOzIYnlHDIiYXl1D/SzTDdD8ZGJdxiEll3OaNb8qNdsCefBdaE+y6eut6ATHHZssDTUBGCiX3yV8+lydxQbdpaOKXCIgbG54JOUEu6+IkR3VWrMRlQcwqUsChorI6+4tQOAdzZQRZA0OWSqW1RkLaf0FafVTauQpsocHknpNnCGsq2WaQneEa8BOik/xzoNzma3JxZ/A8qHbAJRzoU+P/0hUEMJA4ALbcp6W6t8fBN7GkVwvhXUIvXc/H/CU99XXBV859HefOy83aFyKLpBC6MEYYT+PuMHD0VzT4BpEjXdqO3/82qSJ5FRX0y5KlRBlLh795XUp437dT1iC+Ac9sUEwO2VxhZDqXsUG39Sw6VvVykLf3qfVbTlNBEKeKgQkz/2GNJ9/tbbT3Jhv/6oNI+fPxZdeYAErH8GkeLxXU2U/mqxsnoPJrpjuL1Ywhs3iaBOqCDH+0LI2hY4Kt1dWQ5muLx/q24ouFk0l0TP4HUv3P9GODra4Iwnu7fdyrfZWNTZS/zRlZA6YwZ2zZR84TCk6dXgpZDTpDCAxnXiWUq9QSElJW6jCOPrtRcNm7R+0vL4YSGEQTF4dWkUKZVVduRgElFYkGZwTxBzfPi6zAoLXzscEa38O6xCvuqEXtMsKejieXwoa2JraV3aSlR7+5pNdUysu84k/ujpOZxy3tCb6Zv4/rcDxOrTan2mvTrtIZaRQq+m3eqoR1vl7hnw6S2tMaqST4JcLsmd/R5PDvAhIg6uixGJBe7PdCIqzdrlI9qCPW6etXIDc1gOIZnUA90Ft2Jo4neAt9WmSe3CKeMLfoi1Bgl46Ws67eAwLijeuElHrGoofZp82Y2y4MKhkeMNRUEWFzk5BK7e5aHYb6v9hvkphKUlpJpS1OlVJdLyJz+Uf0xPJIWBBcI0wie/NzvY0copgFaxIcn8Ze2pWEnxqaPtzhOwuX1QWVRcmsATRKiT2ZpMXoZfzxjRAzrGuYdsIBzsUfk477KzD1lAejtEN+hMv/Q8Y716+aYxwa2rt48M6f4CSW7wpFKM8lRc2fQS1lw3LL7+pgTuE1VJfFXdL+YoS4hbCousvzMTu6I/AZ1l3wr3kA1czX82aAboXPsglHAEO1CrtkED1yaBR/F1II8uUMQOGCQg6KEHxoXkAYAQ=="}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":2797,"cache_read_input_tokens":261476,"output_tokens":2090,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":2797,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":2090,"cache_read_input_tokens":261476,"cache_creation_input_tokens":2797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":2797},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CcnxCdtcwMwH94on8fJ6v","type":"assistant","uuid":"1a34a991-1644-4726-8618-96c08a2cba2a","timestamp":"2026-07-07T15:43:13.995Z","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}
+{"parentUuid":"1a34a991-1644-4726-8618-96c08a2cba2a","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxCgqyAWrW7sHmVtEk6","type":"message","role":"assistant","content":[{"type":"text","text":"The situation is now clear and serious. Let me do one more **non-elevated** recovery check (File History), then I must stop and report to you — this needs your decision and elevated access:"}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":2797,"cache_read_input_tokens":261476,"output_tokens":2090,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":2797,"ephemeral_5m_input_tokens":0},"inference_geo":"not_available","iterations":[{"input_tokens":2,"output_tokens":2090,"cache_read_input_tokens":261476,"cache_creation_input_tokens":2797,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":2797},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req_011CcnxCdtcwMwH94on8fJ6v","type":"assistant","uuid":"f8e04adb-fafb-4b12-855a-7d82123acc09","timestamp":"2026-07-07T15:43:14.826Z","userType":"external","entrypoint":"cli","cwd":"C:\\Users\\Juan\\Documents\\GitHub\\ServerKit","sessionId":"cdfd365e-6138-447e-ae3d-1d17eaa16498","version":"2.1.197","gitBranch":"HEAD"}
+{"parentUuid":"f8e04adb-fafb-4b12-855a-7d82123acc09","isSidechain":false,"message":{"model":"claude-opus-4-8","id":"msg_011CcnxCgqyAWrW7sHmVtEk6","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_01MWz2Ja4hPnZG7qF1PHtNT2","name":"PowerShell","input":{"command":"Write-Output \"=== File History configured? ===\"\n$fh = Get-ItemProperty 'HKCU:\\Software\\Microsoft\\Windows\\CurrentVersion\\FileHistory' -ErrorAction SilentlyCon
\ No newline at end of file