ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
feat(restore-points): add generic checkpoint core
Juan Deniscd956d1cb05b
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.pyindex 46ae0378..1a84524e 100644--- a/backend/app/jobs/builtin_handlers.py+++ b/backend/app/jobs/builtin_handlers.py@@ -244,6 +244,28 @@ def run_snapshot_retention():return result if isinstance(result, dict) else None+def run_restore_point_retention():+ """Prune expired and over-cap generic restore points."""+ from app.services.restore_point_service import (+ DEFAULT_RETENTION_DAYS,+ prune,+ )+ from app.services.settings_service import SettingsService++ days = SettingsService.get(+ 'restore_point_retention_days', DEFAULT_RETENTION_DAYS,+ )+ try:+ days = int(days)+ except (TypeError, ValueError):+ days = DEFAULT_RETENTION_DAYS+ result = prune(retention_days=days)+ if not isinstance(result, dict) or result.get('success') is False:+ detail = result.get('error') if isinstance(result, dict) else None+ raise RuntimeError(detail or 'Restore-point retention failed')+ return result++def run_pairing_prune():"""Prune expired pending agent pairings."""from app.services import pairing_service@@ -420,6 +442,7 @@ def run_extension_update_check():_BUILTINS = [('builtin.auto_sync', check_auto_sync_schedules, 'auto-sync', 60, 60),('builtin.snapshot_retention', run_snapshot_retention, 'snapshot-retention', 3600, 120),+ ('builtin.restore_point_retention', run_restore_point_retention, 'restore-point-retention', 3600, 180),('builtin.workflow_schedules', check_workflow_schedules, 'workflow-schedules', 60, 60),('builtin.health_check', run_health_checks, 'health-check', 300, 30),# 30s tick, not the monitor interval: the sweep only polls monitors whose owndiff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.pyindex e7c919a2..1da3b759 100644--- a/backend/app/models/__init__.py+++ b/backend/app/models/__init__.py@@ -85,6 +85,7 @@ from app.models.dns_cutover_snapshot import DnsCutoverSnapshotfrom app.models.theme import Themefrom app.models.plugin_store import PluginStorefrom app.models.saved_view import SavedView+from app.models.restore_point import RestorePoint__all__ = ['User', 'Application', 'Domain', 'EnvironmentVariable', 'EnvironmentVariableHistory',@@ -139,4 +140,5 @@ __all__ = ['Theme','PluginStore','SavedView',+ 'RestorePoint',]diff --git a/backend/app/models/audit_log.py b/backend/app/models/audit_log.pyindex b945637a..f2c7d481 100644--- a/backend/app/models/audit_log.py+++ b/backend/app/models/audit_log.py@@ -49,6 +49,8 @@ class AuditLog(JsonColumnMixin, db.Model):ACTION_DEPLOY = 'app.deploy'ACTION_BACKUP_CREATE = 'backup.create'ACTION_BACKUP_RESTORE = 'backup.restore'+ ACTION_RESTORE_POINT_CREATE = 'restore_point.create'+ ACTION_RESTORE_POINT_RESTORE = 'restore_point.restore'ACTION_SSO_LOGIN = 'sso.login'ACTION_SSO_LOGIN_FAILED = 'sso.login_failed'ACTION_SSO_PROVISION = 'sso.provision'diff --git a/backend/app/models/restore_point.py b/backend/app/models/restore_point.pynew file mode 100644index 00000000..56074dae--- /dev/null+++ b/backend/app/models/restore_point.py@@ -0,0 +1,70 @@+"""Immutable checkpoints for panel-managed configuration surfaces."""++import json++from app import db+from app.models.mixins import SerializableMixin, TimestampMixin, uuid_pk+++class RestorePoint(TimestampMixin, SerializableMixin, db.Model):+ """One point on a scope's linear configuration timeline."""++ __tablename__ = 'restore_points'++ id = uuid_pk()+ server_id = db.Column(+ db.String(36), db.ForeignKey('servers.id'), nullable=True, index=True,+ )+ scope_type = db.Column(db.String(32), nullable=False)+ scope_id = db.Column(db.String(255), nullable=False)+ trigger = db.Column(db.String(32), nullable=False)+ label = db.Column(db.String(255), nullable=True)+ payload_hash = db.Column(db.String(64), nullable=False)+ payload_json = db.Column(db.Text, nullable=False)+ coverage_json = db.Column(db.Text, nullable=False, default='[]')+ actor_user_id = db.Column(+ db.Integer, db.ForeignKey('users.id'), nullable=True, index=True,+ )+ expires_at = db.Column(db.DateTime, nullable=True, index=True)+ keep = db.Column(db.Boolean, nullable=False, default=False)++ server = db.relationship('Server')+ actor = db.relationship('User', foreign_keys=[actor_user_id])++ __table_args__ = (+ db.Index('ix_restore_points_scope', 'scope_type', 'scope_id'),+ db.Index(+ 'ix_restore_points_scope_timeline',+ 'server_id', 'scope_type', 'scope_id', 'created_at',+ ),+ )++ __serialize_exclude__ = ('payload_json', 'coverage_json')++ def get_payload(self):+ try:+ return json.loads(self.payload_json)+ except (TypeError, ValueError):+ return {}++ def get_coverage(self):+ try:+ value = json.loads(self.coverage_json)+ except (TypeError, ValueError):+ return []+ return value if isinstance(value, list) else []++ def serialize_extra(self):+ return {+ 'payload': self.get_payload(),+ 'coverage': self.get_coverage(),+ }++ @classmethod+ def latest_for_scope(cls, scope_type, scope_id, server_id=None):+ query = cls.query.filter_by(+ scope_type=str(scope_type),+ scope_id=str(scope_id),+ server_id=server_id,+ )+ return query.order_by(cls.created_at.desc(), cls.id.desc()).first()diff --git a/backend/app/services/restore_point_service.py b/backend/app/services/restore_point_service.pynew file mode 100644index 00000000..1687f55d--- /dev/null+++ b/backend/app/services/restore_point_service.py@@ -0,0 +1,660 @@+"""Generic restore-point lifecycle and adapter registry.++Adapters keep OS/provider knowledge out of this module. Each adapter may be+an object or mapping with these members::++ capture(scope_id, server_id=None) -> JSON-serializable payload+ restore(scope_id, payload, actor=None, server_id=None) -> result+ diff(old_payload, new_payload) -> structured diff # optional+ validate_restore(scope_id, payload, current_payload,+ actor=None, server_id=None) -> refusals # optional+ coverage -> list[str] or callable(scope_id, server_id=None) # optional++Capture is deliberately best-effort: it returns ``None`` and logs when the+adapter or database cannot produce a checkpoint. This preserves the existing+capture-before-mutation contract -- a checkpoint outage must not turn into a+configuration outage.+"""++from contextlib import contextmanager+from contextvars import ContextVar+from datetime import datetime, timedelta+import hashlib+import json+import logging++from sqlalchemy import and_, or_++from app import db+from app.exceptions import (+ ApplicationError,+ ConflictError,+ DependencyUnavailableError,+ NotFoundError,+ ValidationError,+)++logger = logging.getLogger(__name__)++DEFAULT_RETENTION_DAYS = 30+DEFAULT_SCOPE_CAP = 50++BASE_COVERAGE = (+ 'Secrets are stored masked and are never overwritten by a restore. '+ 'State outside the panel (files on disk, provider-side records created '+ 'elsewhere, container runtime state) is outside the checkpoint.'+)++SCOPE_COVERAGE = {+ 'firewall': (+ "Rules added outside the panel (fail2ban runtime bans, Docker's "+ 'iptables chains) are not captured.',+ ),+ 'dns': (+ 'Records that exist only at the provider and were never managed by '+ 'the panel are not captured.',+ ),+ 'nginx_vhost': (+ 'Only panel-rendered vhosts are captured. Hand-edited files in '+ 'sites-enabled are drift, not checkpoint content — see the Drift report.',+ ),+}+++class RestorePointError(ValidationError):+ """Invalid restore-point operation (capture remains best-effort)."""++ code = 'restore_point_invalid'+++class RestorePointNotFoundError(NotFoundError):+ code = 'restore_point_not_found'+++class RestorePointAdapterError(DependencyUnavailableError):+ code = 'restore_point_adapter_unavailable'+++class RestorePointCorruptError(ConflictError):+ code = 'restore_point_corrupt'+++class RestorePointRefusedError(ConflictError):+ code = 'restore_point_refused'++ def __init__(self, refusals):+ self.refusals = list(refusals)+ super().__init__(+ 'Restore refused by the surface safety check',+ details={'refusals': self.refusals},+ )+++# scope_type -> adapter. The alias is intentionally public for extensions and+# for registry introspection without coupling callers to the implementation.+ADAPTERS = {}+RESTORE_POINT_ADAPTERS = ADAPTERS++# Restore adapters deliberately replay through the ordinary mutation doors.+# Hooks at those doors call auto_capture(), so suppress only those automatic+# calls while a restore is replaying. Direct capture() remains available for+# the one deliberate pre-restore checkpoint.+_AUTO_CAPTURE_SUPPRESSED = ContextVar(+ 'restore_point_auto_capture_suppressed', default=False,+)+++def _member(adapter, name):+ if isinstance(adapter, dict):+ return adapter.get(name)+ return getattr(adapter, name, None)+++def register_adapter(scope_type, adapter=None):+ """Register (or replace) an adapter; also supports decorator use."""+ if adapter is None:+ return lambda value: register_adapter(scope_type, value)+ if not scope_type:+ raise ValueError('scope_type is required')+ if not callable(_member(adapter, 'capture')):+ raise ValueError('restore-point adapter requires capture()')+ if not callable(_member(adapter, 'restore')):+ raise ValueError('restore-point adapter requires restore()')+ ADAPTERS[str(scope_type)] = adapter+ return adapter+++def get_adapter(scope_type):+ return ADAPTERS.get(str(scope_type))+++def clear_adapters():+ """Test/plugin-reload helper."""+ ADAPTERS.clear()+++def auto_capture(scope_type, scope_id, action, label=None, actor=None,+ server_id=None):+ """Best-effort pre-mutation capture for use at M2 mutation doors."""+ if _AUTO_CAPTURE_SUPPRESSED.get():+ return None+ action = str(action or 'mutation')+ if '.' not in action:+ action = f'{scope_type}.{action}'+ return capture(+ scope_type,+ scope_id,+ 'pre_mutation',+ label=label or f'before {action}',+ actor=actor,+ server_id=server_id,+ )+++@contextmanager+def suppress_auto_capture():+ """Suppress nested door hooks in this context, resetting even on error."""+ token = _AUTO_CAPTURE_SUPPRESSED.set(True)+ try:+ yield+ finally:+ _AUTO_CAPTURE_SUPPRESSED.reset(token)+++def canonical_json(payload):+ """Match the proven DeploymentSnapshot canonical JSON construction."""+ return json.dumps(+ payload, sort_keys=True, separators=(',', ':'), default=str,+ )+++def payload_hash(payload):+ return hashlib.sha256(canonical_json(payload).encode('utf-8')).hexdigest()+++def _actor_id(actor):+ if actor is None:+ return None+ return getattr(actor, 'id', actor)+++def _adapter_coverage(adapter, scope_type, scope_id, server_id):+ coverage = [BASE_COVERAGE]+ coverage.extend(SCOPE_COVERAGE.get(scope_type, ()))+ extra = _member(adapter, 'coverage')+ if callable(extra):+ extra = extra(scope_id, server_id=server_id)+ if isinstance(extra, str):+ extra = [extra]+ for statement in extra or ():+ if statement and statement not in coverage:+ coverage.append(statement)+ return coverage+++def _retention_days():+ from app.services.settings_service import SettingsService++ value = SettingsService.get(+ 'restore_point_retention_days', DEFAULT_RETENTION_DAYS,+ )+ try:+ return max(0, int(value))+ except (TypeError, ValueError):+ return DEFAULT_RETENTION_DAYS+++def _capture_payload(adapter, scope_id, server_id):+ return _member(adapter, 'capture')(scope_id, server_id=server_id)+++def _session_has_pending_changes():+ """Whether capture would accidentally own another service's unit of work."""+ session = db.session()+ return bool(session.new or session.dirty or session.deleted)+++def _audit_capture(point):+ from app.models.audit_log import AuditLog+ from app.services.audit_service import AuditService++ AuditService.log(+ AuditLog.ACTION_RESTORE_POINT_CREATE,+ user_id=point.actor_user_id,+ target_type='restore_point',+ target_id=None,+ details={+ 'restore_point_id': point.id,+ 'server_id': point.server_id,+ 'scope_type': point.scope_type,+ 'scope_id': point.scope_id,+ 'trigger': point.trigger,+ 'label': point.label,+ },+ commit=False,+ )+++def capture(scope_type, scope_id, trigger, label=None, actor=None,+ server_id=None):+ """Capture and dedupe one scope, returning a row or ``None`` on failure."""+ from app.models.restore_point import RestorePoint++ scope_type = str(scope_type or '')+ scope_id = str(scope_id or '')+ adapter = get_adapter(scope_type)+ if not scope_type or not scope_id or adapter is None:+ logger.warning(+ 'Restore-point capture skipped for %s/%s: adapter not registered',+ scope_type or '?', scope_id or '?',+ )+ return None++ # capture() commits its point + audit atomically. Never let that commit (or+ # its failure rollback) absorb a caller's pending unit of work.+ if _session_has_pending_changes():+ logger.warning(+ 'Restore-point capture skipped for %s/%s: caller transaction '+ 'has pending changes', scope_type, scope_id,+ )+ return None++ owns_transaction = False+ try:+ payload = _capture_payload(adapter, scope_id, server_id)+ # Adapters are read-only during capture. If one dirtied the session,+ # leave its objects untouched for the caller and refuse the checkpoint.+ if _session_has_pending_changes():+ logger.warning(+ 'Restore-point capture skipped for %s/%s: adapter left '+ 'pending database changes', scope_type, scope_id,+ )+ return None+ owns_transaction = True++ if not isinstance(payload, dict):+ raise ValueError('restore-point adapter capture must return a mapping')++ canonical = canonical_json(payload)+ digest = hashlib.sha256(canonical.encode('utf-8')).hexdigest()++ latest = RestorePoint.latest_for_scope(+ scope_type, scope_id, server_id=server_id,+ )+ if latest and latest.payload_hash == digest:+ # A manual quicksave doubles as a retained tag. Dedupe must not+ # throw away that operator intent just because an automatic point+ # already captured the same bytes.+ if str(trigger) == 'manual':+ now = datetime.utcnow()+ latest.keep = True+ latest.trigger = 'manual'+ latest.actor_user_id = _actor_id(actor)+ if label and latest.label != str(label)[:255]:+ latest.label = str(label)[:255]+ latest.created_at = now+ latest.updated_at = now+ _audit_capture(latest)+ db.session.commit()+ return latest++ now = datetime.utcnow()+ point = RestorePoint(+ server_id=server_id,+ scope_type=scope_type,+ scope_id=scope_id,+ trigger=str(trigger),+ label=str(label)[:255] if label else None,+ payload_hash=digest,+ payload_json=canonical,+ coverage_json=canonical_json(_adapter_coverage(+ adapter, scope_type, scope_id, server_id,+ )),+ actor_user_id=_actor_id(actor),+ expires_at=now + timedelta(days=_retention_days()),+ keep=str(trigger) == 'manual',+ )+ db.session.add(point)+ db.session.flush() # uuid/defaults are available to the audit details+ _audit_capture(point)+ db.session.commit()+ return point+ except Exception as exc: # noqa: BLE001 -- must never block the mutation+ # Before the clean post-adapter boundary, rollback could destroy work+ # the caller/adapter still owns. Once the boundary passes, every write+ # in this transaction belongs to restore-point capture.+ if owns_transaction:+ try:+ db.session.rollback()+ except Exception: # pragma: no cover - defensive+ pass+ logger.warning(+ 'Restore-point capture failed for %s/%s: %s',+ scope_type, scope_id, exc, exc_info=True,+ )+ return None+++def _point_or_raise(point_id):+ from app.models.restore_point import RestorePoint++ point = db.session.get(RestorePoint, point_id)+ if point is None:+ raise RestorePointNotFoundError(f'Restore point {point_id} not found')+ return point+++def _strict_payload(point):+ """Parse a stored payload without JsonColumnMixin's lossy fallback."""+ try:+ payload = json.loads(point.payload_json)+ except (TypeError, ValueError) as exc:+ raise RestorePointCorruptError(+ f'Restore point {point.id} has a corrupt payload',+ details={'restore_point_id': point.id},+ ) from exc+ if not isinstance(payload, dict):+ raise RestorePointCorruptError(+ f'Restore point {point.id} has a corrupt payload',+ details={'restore_point_id': point.id},+ )+ return payload+++def _previous_point(point):+ from app.models.restore_point import RestorePoint++ earlier = or_(+ RestorePoint.created_at < point.created_at,+ and_(+ RestorePoint.created_at == point.created_at,+ RestorePoint.id < point.id,+ ),+ )+ return (RestorePoint.query.filter_by(+ server_id=point.server_id,+ scope_type=point.scope_type,+ scope_id=point.scope_id,+ ).filter(earlier)+ .order_by(RestorePoint.created_at.desc(), RestorePoint.id.desc())+ .first())+++def diff_payloads(old, new):+ """Generic secret-safe diff over already-masked checkpoint payloads."""+ if isinstance(old, dict) and isinstance(new, dict):+ old_keys = set(old)+ new_keys = set(new)+ return {+ 'added': {key: new[key] for key in sorted(new_keys - old_keys)},+ 'removed': {key: old[key] for key in sorted(old_keys - new_keys)},+ 'changed': {+ key: {'old': old[key], 'new': new[key]}+ for key in sorted(old_keys & new_keys)+ if old[key] != new[key]+ },+ }+ if isinstance(old, list) and isinstance(new, list):+ old_by_json = {canonical_json(value): value for value in old}+ new_by_json = {canonical_json(value): value for value in new}+ return {+ 'added': [new_by_json[key] for key in sorted(new_by_json.keys() - old_by_json.keys())],+ 'removed': [old_by_json[key] for key in sorted(old_by_json.keys() - new_by_json.keys())],+ 'changed': [],+ }+ return {+ 'added': [],+ 'removed': [],+ 'changed': {'old': old, 'new': new} if old != new else {},+ }+++def _build_diff(scope_type, old, new):+ adapter = get_adapter(scope_type)+ adapter_diff = _member(adapter, 'diff') if adapter is not None else None+ if not callable(adapter_diff):+ return diff_payloads(old, new)+ try:+ return adapter_diff(old, new)+ except RestorePointAdapterError:+ raise+ except Exception as exc:+ raise RestorePointAdapterError(+ f'{scope_type} restore-point diff failed: {exc}',+ ) from exc+++def diff(point_id, against='previous'):+ """Diff a point against the previous point or an explicit point id."""+ point = _point_or_raise(point_id)+ if against in (None, '', 'previous'):+ against_point = _previous_point(point)+ else:+ against_point = _point_or_raise(against)+ if (+ against_point.server_id != point.server_id+ or against_point.scope_type != point.scope_type+ or against_point.scope_id != point.scope_id+ ):+ raise RestorePointError('Restore points belong to different scopes')++ old = _strict_payload(against_point) if against_point else {}+ new = _strict_payload(point)+ return {+ 'point_id': point.id,+ 'against_point_id': against_point.id if against_point else None,+ 'diff': _build_diff(point.scope_type, old, new),+ 'has_changes': canonical_json(old) != canonical_json(new),+ }+++def _normalize_refusals(value):+ if value in (None, True):+ return []+ if value is False:+ return ['The surface safety check refused this restore.']+ if isinstance(value, dict):+ if value.get('allowed', True) is False:+ value = value.get('refusals') or value.get('error') or value+ else:+ value = value.get('refusals', [])+ if isinstance(value, (str, bytes)):+ return [value.decode() if isinstance(value, bytes) else value]+ if isinstance(value, (list, tuple)):+ return list(value)+ return [value]+++def _restore_refusals(adapter, point, target_payload, current_payload, actor):+ validator = _member(adapter, 'validate_restore')+ if not callable(validator):+ return []+ return _normalize_refusals(validator(+ point.scope_id,+ target_payload,+ current_payload,+ actor=actor,+ server_id=point.server_id,+ ))+++def preview(point_id, actor=None):+ """Read-only current → checkpoint diff, coverage, and safety refusals."""+ point = _point_or_raise(point_id)+ adapter = get_adapter(point.scope_type)+ if adapter is None:+ raise RestorePointAdapterError(+ f'No restore-point adapter registered for {point.scope_type}',+ )+ # Validate stored bytes before invoking any adapter code. A corrupt point+ # must be inert: no current-state reads, safety probes, diffs, or restores.+ target = _strict_payload(point)+ try:+ current = _capture_payload(adapter, point.scope_id, point.server_id)+ refusals = _restore_refusals(adapter, point, target, current, actor)+ except ApplicationError:+ raise+ except Exception as exc:+ raise RestorePointAdapterError(str(exc)) from exc+ return {+ 'point_id': point.id,+ 'scope_type': point.scope_type,+ 'scope_id': point.scope_id,+ 'diff': _build_diff(point.scope_type, current, target),+ 'has_changes': canonical_json(current) != canonical_json(target),+ 'outside_checkpoint': point.get_coverage(),+ 'refusals': refusals,+ 'can_restore': not refusals,+ }+++def restore(point_id, actor=None):+ """Re-converge through the adapter after a recoverable pre-checkpoint."""+ point = _point_or_raise(point_id)+ adapter = get_adapter(point.scope_type)+ if adapter is None:+ raise RestorePointAdapterError(+ f'No restore-point adapter registered for {point.scope_type}',+ )+ target = _strict_payload(point)++ restore_preview = preview(point.id, actor=actor)+ if restore_preview['refusals']:+ raise RestorePointRefusedError(restore_preview['refusals'])++ before = capture(+ point.scope_type,+ point.scope_id,+ 'pre_mutation',+ label=f'before restore {point.id}',+ actor=actor,+ server_id=point.server_id,+ )+ if before is None:+ raise RestorePointAdapterError(+ 'Restore aborted because the pre-restore checkpoint could not be captured',+ )++ try:+ with suppress_auto_capture():+ result = _member(adapter, 'restore')(+ point.scope_id,+ target,+ actor=actor,+ server_id=point.server_id,+ )+ except ApplicationError:+ raise+ except Exception as exc:+ raise RestorePointAdapterError(str(exc)) from exc++ if isinstance(result, dict) and result.get('success') is False:+ return result++ from app.models.audit_log import AuditLog+ from app.services.audit_service import AuditService++ AuditService.log(+ AuditLog.ACTION_RESTORE_POINT_RESTORE,+ user_id=_actor_id(actor),+ target_type='restore_point',+ target_id=None,+ details={+ 'restore_point_id': point.id,+ 'pre_restore_point_id': before.id if before else None,+ 'server_id': point.server_id,+ 'scope_type': point.scope_type,+ 'scope_id': point.scope_id,+ },+ )+ return result if result is not None else {'success': True}+++def prune(retention_days=DEFAULT_RETENTION_DAYS, per_scope_cap=DEFAULT_SCOPE_CAP,+ now=None):+ """Expire old unkept rows and cap each server/scope timeline at 50."""+ from app.models.restore_point import RestorePoint++ try:+ retention_days = max(0, int(retention_days))+ except (TypeError, ValueError):+ retention_days = DEFAULT_RETENTION_DAYS+ try:+ per_scope_cap = max(0, int(per_scope_cap))+ except (TypeError, ValueError):+ per_scope_cap = DEFAULT_SCOPE_CAP+ now = now or datetime.utcnow()++ backfilled = 0+ expired_deleted = 0+ cap_deleted = 0+ try:+ pending = RestorePoint.query.filter(+ RestorePoint.keep.is_(False),+ RestorePoint.expires_at.is_(None),+ ).all()+ for point in pending:+ point.expires_at = (point.created_at or now) + timedelta(+ days=retention_days,+ )+ backfilled += 1++ expired = RestorePoint.query.filter(+ RestorePoint.keep.is_(False),+ RestorePoint.expires_at.isnot(None),+ RestorePoint.expires_at <= now,+ ).all()+ for point in expired:+ db.session.delete(point)+ expired_deleted += 1+ db.session.flush()++ scopes = db.session.query(+ RestorePoint.server_id,+ RestorePoint.scope_type,+ RestorePoint.scope_id,+ ).filter(RestorePoint.keep.is_(False)).distinct().all()+ for server_id, scope_type, scope_id in scopes:+ rows = (RestorePoint.query.filter_by(+ server_id=server_id,+ scope_type=scope_type,+ scope_id=scope_id,+ keep=False,+ ).order_by(RestorePoint.created_at.desc(), RestorePoint.id.desc())+ .all())+ for point in rows[per_scope_cap:]:+ db.session.delete(point)+ cap_deleted += 1++ db.session.commit()+ return {+ 'success': True,+ 'backfilled': backfilled,+ 'expired_deleted': expired_deleted,+ 'cap_deleted': cap_deleted,+ 'deleted': expired_deleted + cap_deleted,+ }+ except Exception as exc: # noqa: BLE001 -- scheduled job reports cleanly+ db.session.rollback()+ logger.error('Restore-point retention failed: %s', exc, exc_info=True)+ return {+ 'success': False,+ 'error': str(exc),+ 'backfilled': backfilled,+ 'expired_deleted': expired_deleted,+ 'cap_deleted': cap_deleted,+ 'deleted': expired_deleted + cap_deleted,+ }+++class RestorePointService:+ """Class facade matching other ServerKit service modules."""++ capture = staticmethod(capture)+ diff = staticmethod(diff)+ preview = staticmethod(preview)+ restore = staticmethod(restore)+ prune = staticmethod(prune)+ register_adapter = staticmethod(register_adapter)+ auto_capture = staticmethod(auto_capture)+ suppress_auto_capture = staticmethod(suppress_auto_capture)diff --git a/backend/app/services/settings_service.py b/backend/app/services/settings_service.pyindex 7d027fec..dd369ff8 100644--- a/backend/app/services/settings_service.py+++ b/backend/app/services/settings_service.py@@ -49,6 +49,11 @@ class SettingsService:'type': 'integer','description': 'Number of days to retain audit logs'},+ 'restore_point_retention_days': {+ 'value': 30,+ 'type': 'integer',+ 'description': 'Number of days to retain untagged restore points'+ },'jobs.retention_days': {'value': 14,'type': 'integer',diff --git a/backend/migrations/versions/091_restore_points.py b/backend/migrations/versions/091_restore_points.pynew file mode 100644index 00000000..d37f897a--- /dev/null+++ b/backend/migrations/versions/091_restore_points.py@@ -0,0 +1,65 @@+"""Add the generic restore-point timeline.++Revision ID: 091_restore_points+Revises: 090_ai_message_attachments+"""++import sqlalchemy as sa+from alembic import op++revision = '091_restore_points'+down_revision = '090_ai_message_attachments'+branch_labels = None+depends_on = None+++def _has_table(name):+ return name in sa.inspect(op.get_bind()).get_table_names()+++def upgrade():+ if _has_table('restore_points'):+ return++ op.create_table(+ 'restore_points',+ sa.Column('id', sa.String(length=36), nullable=False),+ sa.Column('server_id', sa.String(length=36), nullable=True),+ sa.Column('scope_type', sa.String(length=32), nullable=False),+ sa.Column('scope_id', sa.String(length=255), nullable=False),+ sa.Column('trigger', sa.String(length=32), nullable=False),+ sa.Column('label', sa.String(length=255), nullable=True),+ sa.Column('payload_hash', sa.String(length=64), nullable=False),+ sa.Column('payload_json', sa.Text(), nullable=False),+ sa.Column('coverage_json', sa.Text(), nullable=False),+ sa.Column('actor_user_id', sa.Integer(), nullable=True),+ sa.Column('expires_at', sa.DateTime(), nullable=True),+ sa.Column('keep', sa.Boolean(), nullable=False),+ sa.Column('created_at', sa.DateTime(), nullable=True),+ sa.Column('updated_at', sa.DateTime(), nullable=True),+ sa.ForeignKeyConstraint(['actor_user_id'], ['users.id']),+ sa.ForeignKeyConstraint(['server_id'], ['servers.id']),+ sa.PrimaryKeyConstraint('id'),+ )+ op.create_index(+ 'ix_restore_points_actor_user_id', 'restore_points', ['actor_user_id'],+ )+ op.create_index(+ 'ix_restore_points_expires_at', 'restore_points', ['expires_at'],+ )+ op.create_index(+ 'ix_restore_points_server_id', 'restore_points', ['server_id'],+ )+ op.create_index(+ 'ix_restore_points_scope', 'restore_points', ['scope_type', 'scope_id'],+ )+ op.create_index(+ 'ix_restore_points_scope_timeline', 'restore_points',+ ['server_id', 'scope_type', 'scope_id', 'created_at'],+ )+++def downgrade():+ if not _has_table('restore_points'):+ return+ op.drop_table('restore_points')diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.pyindex a9c174c8..bf67ef3d 100644--- a/backend/tests/test_jobs.py+++ b/backend/tests/test_jobs.py@@ -178,6 +178,7 @@ class TestBuiltins:assert 'builtin.security_feed' in kindsassert 'builtin.job_retention' in kindsassert 'builtin.monitor_check' in kinds+ assert 'builtin.restore_point_retention' in kinds# Reclaims the disk a soft delete no longer frees: an Application# tombstone holds its data volumes and source tree until purge, and# nothing else calls purge_expired on a schedule.@@ -187,18 +188,18 @@ class TestBuiltins:# is ~11 MB/day of database forever, which is what filled a 25 GB host# from routine updates alone (each update copies the DB twice).assert 'builtin.telemetry_retention' in kinds- assert len([k for k in kinds if k.startswith('builtin.')]) == 15+ assert len([k for k in kinds if k.startswith('builtin.')]) == 16builtin_handlers.seed_builtin_schedules()- # 15 builtin.* schedules (incl. job-retention, telemetry-retention, the- # monitor sweep, the security-feed check and the recycle-bin retention- # sweep) + login-link/SSO reapers + drift/FIM/bandwidth sweeps + the- # host doctor sweep AND the fleet doctor sweep (plan 26) + the- # setup-health nag (plan 22).- assert ScheduledJob.query.count() == 23+ # 16 builtin.* schedules (incl. restore-point/job/telemetry retention,+ # the monitor sweep, security-feed check and recycle-bin retention)+ # + login-link/SSO reapers + drift/FIM/bandwidth sweeps + the host+ # doctor sweep AND the fleet doctor sweep (plan 26) + the setup-health+ # nag (plan 22).+ assert ScheduledJob.query.count() == 24# Seeding twice doesn't duplicate.builtin_handlers.seed_builtin_schedules()- assert ScheduledJob.query.count() == 23+ assert ScheduledJob.query.count() == 24class TestApi:diff --git a/backend/tests/test_restore_point_service.py b/backend/tests/test_restore_point_service.pynew file mode 100644index 00000000..9d417455--- /dev/null+++ b/backend/tests/test_restore_point_service.py@@ -0,0 +1,473 @@+"""Plan 81 M1: generic restore-point store and lifecycle."""++from copy import deepcopy+from datetime import datetime, timedelta++import pytest+++class MemoryAdapter:+ def __init__(self, state=None, coverage=None):+ self.state = state or {}+ self.restore_calls = 0+ self.capture_calls = 0+ self.diff_calls = 0+ self.coverage = coverage or []+ self.refusals = []+ self.nested_auto_capture = False++ def capture(self, scope_id, server_id=None):+ self.capture_calls += 1+ return deepcopy(self.state)++ def diff(self, old, new):+ from app.services.restore_point_service import diff_payloads+ self.diff_calls += 1+ return diff_payloads(old, new)++ def restore(self, scope_id, payload, actor=None, server_id=None):+ if self.nested_auto_capture:+ from app.services import restore_point_service+ self.state = {'mode': 'partial replay'}+ restore_point_service.auto_capture(+ 'cron', scope_id, 'update', server_id=server_id,+ )+ self.restore_calls += 1+ self.state = deepcopy(payload)+ return {'success': True, 'scope_id': scope_id}++ def validate_restore(self, scope_id, payload, current_payload,+ actor=None, server_id=None):+ return self.refusals++[email protected](autouse=True)+def clean_registry(app):+ from app.services import restore_point_service++ existing = dict(restore_point_service.ADAPTERS)+ restore_point_service.clear_adapters()+ yield+ restore_point_service.clear_adapters()+ restore_point_service.ADAPTERS.update(existing)+++def test_registry_registers_replaces_and_validates(app):+ from app.services import restore_point_service as service++ first = MemoryAdapter()+ second = MemoryAdapter()+ assert service.register_adapter('cron', first) is first+ assert service.get_adapter('cron') is first+ service.register_adapter('cron', second)+ assert service.get_adapter('cron') is second++ with pytest.raises(ValueError, match=r'capture\(\)'):+ service.register_adapter('broken', {'restore': lambda *_a, **_k: None})+ with pytest.raises(ValueError, match=r'restore\(\)'):+ service.register_adapter('broken', {'capture': lambda *_a, **_k: {}})+++def test_capture_dedupes_canonical_payload_and_audits(app):+ from app.models import AuditLog, RestorePoint+ from app.services import restore_point_service as service++ adapter = MemoryAdapter({'beta': 2, 'alpha': 1}, ['Only test state is covered.'])+ service.register_adapter('cron', adapter)++ first = service.capture('cron', 'cron', 'pre_mutation', label='before update')+ assert first is not None+ assert first.payload_json == '{"alpha":1,"beta":2}'+ assert first.keep is False+ assert first.expires_at is not None+ assert first.get_coverage()[0] == service.BASE_COVERAGE+ assert first.get_coverage()[-1] == 'Only test state is covered.'++ # Logically equal with the insertion order reversed: hash/capture dedupes.+ adapter.state = {'alpha': 1, 'beta': 2}+ again = service.capture('cron', 'cron', 'scheduled')+ assert again.id == first.id+ assert RestorePoint.query.count() == 1+ assert AuditLog.query.filter_by(+ action=AuditLog.ACTION_RESTORE_POINT_CREATE,+ ).count() == 1+++def test_manual_capture_is_kept_and_setting_defaults_to_30(app):+ from app.services import restore_point_service as service+ from app.services.settings_service import SettingsService++ assert SettingsService.DEFAULT_SETTINGS['restore_point_retention_days']['value'] == 30+ service.register_adapter('env', MemoryAdapter({'PUBLIC': 'value'}))+ before = datetime.utcnow()+ point = service.capture('env', '42', 'manual', label='before maintenance')+ assert point.keep is True+ assert point.expires_at >= before + timedelta(days=29)+++def test_manual_dedupe_promotes_existing_point_to_kept_tag(app):+ from app import db+ from app.models import AuditLog, RestorePoint+ from app.services import restore_point_service as service+ from factories import make_user++ service.register_adapter('env', MemoryAdapter({'PUBLIC': 'value'}))+ automatic = service.capture(+ 'env', '42', 'pre_mutation', label='before env.update',+ )+ assert automatic.keep is False+ original_created_at = automatic.created_at+ actor = make_user(db, username='quicksave_actor')++ manual = service.capture(+ 'env', '42', 'manual', label='known-good configuration', actor=actor,+ )+ assert manual.id == automatic.id+ assert manual.keep is True+ assert manual.trigger == 'manual'+ assert manual.actor_user_id == actor.id+ assert manual.label == 'known-good configuration'+ assert manual.created_at >= original_created_at+ assert manual.updated_at == manual.created_at+ assert RestorePoint.query.count() == 1+ audits = AuditLog.query.filter_by(+ action=AuditLog.ACTION_RESTORE_POINT_CREATE,+ ).order_by(AuditLog.id).all()+ assert len(audits) == 2+ assert audits[-1].get_details()['trigger'] == 'manual'+++def test_capture_refuses_caller_owned_pending_transaction_without_touching_it(app):+ from app import db+ from app.models import SystemSettings+ from app.services import restore_point_service as service++ adapter = MemoryAdapter({'jobs': 1})+ service.register_adapter('cron', adapter)++ caller_row = SystemSettings(key='capture-boundary', value='new')+ db.session.add(caller_row)+ assert service.capture('cron', 'cron', 'pre_mutation') is None+ assert caller_row in db.session.new+ assert adapter.capture_calls == 0+ db.session.commit()++ caller_row.value = 'dirty'+ assert service.capture('cron', 'cron', 'pre_mutation') is None+ assert caller_row in db.session.dirty+ assert caller_row.value == 'dirty'+ assert adapter.capture_calls == 0+ db.session.commit()++ db.session.delete(caller_row)+ assert service.capture('cron', 'cron', 'pre_mutation') is None+ assert caller_row in db.session.deleted+ assert adapter.capture_calls == 0+ db.session.rollback()+ assert db.session.get(SystemSettings, caller_row.id) is not None+++def test_capture_leaves_adapter_pending_data_for_caller(app):+ from app import db+ from app.models import RestorePoint, SystemSettings+ from app.services import restore_point_service as service++ adapter = MemoryAdapter({'jobs': 1})+ pending = SystemSettings(key='adapter-pending', value='owned-by-caller')++ def capture_with_pending_data(_scope_id, server_id=None):+ adapter.capture_calls += 1+ db.session.add(pending)+ return {'jobs': 1}++ adapter.capture = capture_with_pending_data+ service.register_adapter('cron', adapter)+ assert service.capture('cron', 'cron', 'pre_mutation') is None+ assert pending in db.session.new+ assert adapter.capture_calls == 1+ assert not any(isinstance(row, RestorePoint) for row in db.session.new)++ db.session.commit()+ assert SystemSettings.query.filter_by(key='adapter-pending').one().value == (+ 'owned-by-caller'+ )+++def test_capture_failure_is_best_effort(app):+ from app.models import RestorePoint+ from app.services import restore_point_service as service++ assert service.capture('missing', 'scope', 'pre_mutation') is None++ adapter = MemoryAdapter()+ adapter.capture = lambda *_a, **_k: (_ for _ in ()).throw(RuntimeError('read failed'))+ service.register_adapter('cron', adapter)+ assert service.capture('cron', 'cron', 'pre_mutation') is None+ assert RestorePoint.query.count() == 0+++def test_diff_uses_previous_point_in_same_scope(app):+ from app.services import restore_point_service as service++ adapter = MemoryAdapter({'mode': 'one', 'jobs': 1})+ service.register_adapter('cron', adapter)+ first = service.capture('cron', 'cron', 'pre_mutation')+ adapter.state = {'mode': 'two', 'jobs': 1}+ second = service.capture('cron', 'cron', 'pre_mutation')++ result = service.diff(second.id)+ assert result['against_point_id'] == first.id+ assert result['has_changes'] is True+ assert result['diff']['changed']['mode'] == {'old': 'one', 'new': 'two'}+++def test_preview_and_restore_reconverge_with_pre_restore_point(app):+ from app.models import AuditLog, RestorePoint+ from app.services import restore_point_service as service++ adapter = MemoryAdapter({'mode': 'saved'})+ adapter.nested_auto_capture = True+ service.register_adapter('cron', adapter)+ saved = service.capture('cron', 'cron', 'manual', label='known good')+ adapter.state = {'mode': 'current'}++ dry_run = service.preview(saved.id)+ assert dry_run['can_restore'] is True+ assert dry_run['has_changes'] is True+ assert dry_run['diff']['changed']['mode'] == {+ 'old': 'current', 'new': 'saved',+ }+ assert dry_run['outside_checkpoint'] == saved.get_coverage()++ result = service.restore(saved.id)+ assert result['success'] is True+ assert adapter.state == {'mode': 'saved'}+ assert adapter.restore_calls == 1++ before = RestorePoint.query.filter_by(trigger='pre_mutation').one()+ assert before.label == f'before restore {saved.id}'+ assert before.get_payload() == {'mode': 'current'}+ restore_audit = AuditLog.query.filter_by(+ action=AuditLog.ACTION_RESTORE_POINT_RESTORE,+ ).one()+ assert restore_audit.get_details()['pre_restore_point_id'] == before.id+++def test_restore_refusal_is_typed_and_never_partially_applies(app):+ from app.models import RestorePoint+ from app.services import restore_point_service as service++ adapter = MemoryAdapter({'allow_ssh': True})+ service.register_adapter('firewall', adapter)+ saved = service.capture('firewall', 'firewall', 'manual')+ adapter.state = {'allow_ssh': False}+ adapter.refusals = ['Current SSH admission would be dropped.']++ with pytest.raises(service.RestorePointRefusedError) as caught:+ service.restore(saved.id)+ assert caught.value.status_code == 409+ assert caught.value.code == 'restore_point_refused'+ assert caught.value.details == {+ 'refusals': ['Current SSH admission would be dropped.'],+ }+ assert adapter.restore_calls == 0+ assert RestorePoint.query.count() == 1+++def test_restore_aborts_when_pre_restore_capture_fails(app, monkeypatch):+ from app.services import restore_point_service as service++ adapter = MemoryAdapter({'mode': 'saved'})+ service.register_adapter('cron', adapter)+ saved = service.capture('cron', 'cron', 'manual')+ adapter.state = {'mode': 'current'}+ monkeypatch.setattr(service, 'capture', lambda *_args, **_kwargs: None)++ with pytest.raises(service.RestorePointAdapterError) as caught:+ service.restore(saved.id)+ assert caught.value.status_code == 503+ assert adapter.restore_calls == 0++[email protected]('payload_json', ['{broken', '[]'])+def test_corrupt_payload_is_typed_and_never_invokes_adapter(app, payload_json):+ from app import db+ from app.models import RestorePoint+ from app.services import restore_point_service as service++ adapter = MemoryAdapter({'mode': 'current'})+ service.register_adapter('cron', adapter)+ point = RestorePoint(+ scope_type='cron', scope_id='cron', trigger='manual',+ payload_hash='c' * 64, payload_json=payload_json,+ coverage_json='[]', keep=True,+ )+ db.session.add(point)+ db.session.commit()++ for operation in (service.diff, service.preview, service.restore):+ with pytest.raises(service.RestorePointCorruptError) as caught:+ operation(point.id)+ assert (caught.value.status_code, caught.value.code) == (+ 409, 'restore_point_corrupt',+ )+ assert adapter.capture_calls == 0+ assert adapter.diff_calls == 0+ assert adapter.restore_calls == 0+++def test_adapter_diff_errors_are_typed_in_diff_and_preview(app):+ from app.services import restore_point_service as service++ adapter = MemoryAdapter({'mode': 'saved'})+ service.register_adapter('cron', adapter)+ point = service.capture('cron', 'cron', 'manual')++ def broken_diff(_old, _new):+ raise RuntimeError('diff engine failed')++ adapter.diff = broken_diff+ for operation in (service.diff, service.preview):+ with pytest.raises(service.RestorePointAdapterError) as caught:+ operation(point.id)+ assert (caught.value.status_code, caught.value.code) == (+ 503, 'restore_point_adapter_unavailable',+ )+++def test_typed_not_found_and_adapter_unavailable_errors(app):+ from app.models import RestorePoint+ from app.services import restore_point_service as service++ with pytest.raises(service.RestorePointNotFoundError) as missing:+ service.preview('00000000-0000-0000-0000-000000000000')+ assert (missing.value.status_code, missing.value.code) == (+ 404, 'restore_point_not_found',+ )++ point = RestorePoint(+ scope_type='cron', scope_id='cron', trigger='manual',+ payload_hash='a' * 64, payload_json='{}', coverage_json='[]', keep=True,+ )+ from app import db+ db.session.add(point)+ db.session.commit()+ with pytest.raises(service.RestorePointAdapterError) as unavailable:+ service.preview(point.id)+ assert (unavailable.value.status_code, unavailable.value.code) == (+ 503, 'restore_point_adapter_unavailable',+ )+++def test_restore_preserves_typed_adapter_errors(app):+ from app.exceptions import ConflictError+ from app.services import restore_point_service as service++ adapter = MemoryAdapter({'mode': 'saved'})+ service.register_adapter('cron', adapter)+ point = service.capture('cron', 'cron', 'manual')++ def conflict(*_args, **_kwargs):+ raise ConflictError('Surface is busy', code='surface_busy')++ adapter.restore = conflict+ with pytest.raises(ConflictError) as caught:+ service.restore(point.id)+ assert (caught.value.status_code, caught.value.code) == (409, 'surface_busy')+++def test_prune_honors_expiry_keep_and_per_scope_cap(app):+ from app import db+ from app.models import RestorePoint+ from app.services import restore_point_service as service++ now = datetime.utcnow()+ for index in range(52):+ db.session.add(RestorePoint(+ scope_type='cron', scope_id='cron', trigger='pre_mutation',+ payload_hash=f'{index:064x}', payload_json=f'{{"n":{index}}}',+ coverage_json='[]', keep=False,+ created_at=now - timedelta(minutes=index),+ expires_at=now + timedelta(days=30),+ ))+ expired = RestorePoint(+ scope_type='env', scope_id='7', trigger='pre_mutation',+ payload_hash='e' * 64, payload_json='{}', coverage_json='[]', keep=False,+ created_at=now - timedelta(days=60),+ expires_at=now - timedelta(seconds=1),+ )+ kept = RestorePoint(+ scope_type='env', scope_id='7', trigger='manual',+ payload_hash='f' * 64, payload_json='{}', coverage_json='[]', keep=True,+ created_at=now - timedelta(days=60),+ expires_at=now - timedelta(days=30),+ )+ db.session.add_all([expired, kept])+ db.session.commit()++ result = service.prune(now=now)+ assert result == {+ 'success': True,+ 'backfilled': 0,+ 'expired_deleted': 1,+ 'cap_deleted': 2,+ 'deleted': 3,+ }+ assert RestorePoint.query.filter_by(+ scope_type='cron', scope_id='cron', keep=False,+ ).count() == 50+ assert db.session.get(RestorePoint, kept.id) is not None+ assert db.session.get(RestorePoint, expired.id) is None+++def test_auto_capture_suppression_is_nested_and_resets_after_exception(app):+ from app.models import RestorePoint+ from app.services import restore_point_service as service++ service.register_adapter('cron', MemoryAdapter({'jobs': 1}))+ with pytest.raises(RuntimeError, match='restore failed'):+ with service.suppress_auto_capture():+ assert service.auto_capture('cron', 'cron', 'update') is None+ with service.suppress_auto_capture():+ assert service.auto_capture('cron', 'cron', 'delete') is None+ raise RuntimeError('restore failed')++ point = service.auto_capture('cron', 'cron', 'update')+ assert point is not None+ assert point.label == 'before cron.update'+ assert RestorePoint.query.count() == 1+++def test_restore_point_prune_job_is_registered_and_uses_setting(app, monkeypatch):+ from app.jobs import builtin_handlers+ from app.services import restore_point_service+ from app.services.settings_service import SettingsService++ entry = next(+ item for item in builtin_handlers._BUILTINS+ if item[0] == 'builtin.restore_point_retention'+ )+ assert entry[2:] == ('restore-point-retention', 3600, 180)++ monkeypatch.setattr(+ SettingsService, 'get',+ staticmethod(lambda key, default=None: '7' if key == 'restore_point_retention_days' else default),+ )+ seen = {}++ def fake_prune(**kwargs):+ seen.update(kwargs)+ return {'success': True, 'deleted': 2}++ monkeypatch.setattr(restore_point_service, 'prune', fake_prune)+ assert builtin_handlers.run_restore_point_retention() == {+ 'success': True, 'deleted': 2,+ }+ assert seen == {'retention_days': 7}++ monkeypatch.setattr(+ restore_point_service, 'prune',+ lambda **_kwargs: {'success': False, 'error': 'database busy'},+ )+ with pytest.raises(RuntimeError, match='database busy'):+ builtin_handlers.run_restore_point_retention()