Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(backups): a skipped restore drill no longer reads as proof

Juan Denisff2aef1d447f
diff --git a/backend/app/services/backup_drill_service.py b/backend/app/services/backup_drill_service.py
index c7436a8a..0b4a4a3f 100644
--- a/backend/app/services/backup_drill_service.py
+++ b/backend/app/services/backup_drill_service.py
@@ -263,6 +263,12 @@ class BackupDrillService:
db.session.commit()
logger.warning('drill %s skipped: need %s bytes, %s free',
drill_id, required, free)
+ # A log line is not a notification. Without this the operator learns
+ # nothing, and the policy still reads as recently drilled.
+ cls._notify_failed(
+ policy.id, run_id,
+ f'needs {required} bytes of scratch space, {free} free',
+ status='skipped_no_space')
return {'status': 'skipped_no_space', 'drill_id': drill_id,
'bytes_required': int(required), 'bytes_free': int(free)}
@@ -612,15 +618,22 @@ class BackupDrillService:
policy.last_drill_status = status
@classmethod
- def _notify_failed(cls, policy_id, run_id, error):
+ def _notify_failed(cls, policy_id, run_id, error, status='failed'):
+ """Alert on a drill that did not prove the backup.
+
+ ``status`` is passed through because ``BackupAlertService`` already
+ treats ``skipped_no_space`` as a failed outcome — that branch was simply
+ never reached, since only the exception path called this. A drill that
+ could not run left no trace beyond a log line.
+ """
try:
from app.models.backup_policy import BackupPolicy
from app.services.backup_alert_service import BackupAlertService
policy = BackupPolicy.query.get(policy_id)
if policy:
- BackupAlertService.on_drill_result(policy, status='failed', error=error)
+ BackupAlertService.on_drill_result(policy, status=status, error=error)
except Exception as exc: # noqa: BLE001 — alerts are best-effort
- logger.debug('drill-failed alert skipped: %s', exc)
+ logger.debug('drill-%s alert skipped: %s', status, exc)
@classmethod
def _notify_recovered(cls, policy_id, run_id):
diff --git a/backend/app/services/backup_policy_service.py b/backend/app/services/backup_policy_service.py
index 474dbbaa..060097ad 100644
--- a/backend/app/services/backup_policy_service.py
+++ b/backend/app/services/backup_policy_service.py
@@ -946,13 +946,36 @@ class BackupPolicyService:
@classmethod
def _drill_badge(cls, policy):
- """One-word restore-proof state derived from the policy's drill cache:
- ``never`` (never drilled), ``failed`` (last drill failed), ``stale``
- (last success older than 1.5x the cadence interval), else ``ok``."""
+ """One-word restore-proof state from the policy's drill cache.
+
+ ``never`` | ``failed`` | ``skipped`` | ``unknown`` | ``stale`` | ``ok``.
+
+ **``ok`` must be positively earned.** This used to return ``ok`` for any
+ status that was not literally ``'failed'``, so a drill skipped for lack
+ of scratch space — which records ``skipped_no_space`` and stamps
+ ``last_drill_at`` — fell through every branch and the doctor reported
+ "A recent restore drill proved this backup restores." A drill that never
+ ran had become proof that the backup restores.
+
+ It also self-perpetuated: each skip refreshed ``last_drill_at``, so a
+ policy permanently short of scratch space could never even go ``stale``.
+
+ Staleness is only meaningful for a drill that actually succeeded, so it
+ is evaluated after the status is known good.
+ """
if not policy.last_drill_at:
return 'never'
- if policy.last_drill_status == 'failed':
+
+ status = policy.last_drill_status
+ if status == 'failed':
return 'failed'
+ if status == 'skipped_no_space':
+ return 'skipped'
+ if status != 'success':
+ # An unrecognised status is not evidence of anything. Reporting it
+ # as ok is how this went wrong the first time.
+ return 'unknown'
+
from app.services.backup_drill_service import CADENCE_DAYS
days = CADENCE_DAYS.get(policy.drill_cadence or 'off')
if days is not None:
diff --git a/backend/app/services/doctor_service.py b/backend/app/services/doctor_service.py
index 8da78df0..ba9635ee 100644
--- a/backend/app/services/doctor_service.py
+++ b/backend/app/services/doctor_service.py
@@ -568,6 +568,20 @@ class DoctorService:
"The last restore drill is older than this policy's "
'cadence — run a fresh drill.',
repairable=True, repair_ref=repair_ref)
+ if badge == 'skipped':
+ # "Could not run" is its own answer. It used to render as ok, which
+ # told the operator a drill that never happened had proved the
+ # backup restores.
+ return _check(key, title, 'warn',
+ 'The last restore drill could not run — not enough '
+ 'scratch space to restore into. Nothing has been '
+ 'proven about this backup; free space and re-run.',
+ repairable=True, repair_ref=repair_ref)
+ if badge == 'unknown':
+ return _check(key, title, 'warn',
+ 'The last restore drill finished in a state the panel '
+ 'does not recognise, so this backup is unproven.',
+ repairable=True, repair_ref=repair_ref)
detail = ('This backup has never been restore-drilled.'
if badge == 'never'
else 'The last restore drill failed — the backup may not restore.')
diff --git a/backend/tests/test_backup_drill_honesty.py b/backend/tests/test_backup_drill_honesty.py
new file mode 100644
index 00000000..6741f0bd
--- /dev/null
+++ b/backend/tests/test_backup_drill_honesty.py
@@ -0,0 +1,191 @@
+"""A restore drill that never ran must not read as proof that it did.
+
+Plan 75 §A1. The live bug this pins down:
+
+`_drill_badge` returned `ok` for any status that was not literally `'failed'`.
+A drill skipped for lack of scratch space records `skipped_no_space` AND stamps
+`last_drill_at`, so it fell through every branch — and Doctor reported
+
+ "A recent restore drill proved this backup restores."
+
+about a drill that never executed. It also self-perpetuated: each skip
+refreshed `last_drill_at`, so a policy permanently short of scratch space could
+never even go `stale`. It would report proven, forever, having proven nothing.
+
+The drill service itself was careful — its own comment reads "loud skip, never a
+silent pass" — and the frontend renders an amber pill. The honesty was lost in
+the one place that summarises it, which is the place Doctor reads.
+
+Second defect: `BackupAlertService.on_drill_result` already maps
+`skipped_no_space` to a failed outcome, but only the exception path ever called
+it, so that branch was unreachable and a skipped drill notified nobody.
+
+The rule: **`ok` must be positively earned.**
+"""
+
+from datetime import datetime, timedelta
+from unittest.mock import patch
+
+import pytest
+
+from app import db
+from app.models.backup_policy import BackupPolicy
+from app.services.backup_policy_service import BackupPolicyService
+from app.services.doctor_service import DoctorService
+
+
+def _policy(app, status, drilled_ago=timedelta(hours=1), cadence='weekly'):
+ policy = BackupPolicy(target_type='application', target_id=1,
+ drill_cadence=cadence)
+ policy.last_drill_status = status
+ policy.last_drill_at = datetime.utcnow() - drilled_ago
+ db.session.add(policy)
+ db.session.commit()
+ return policy
+
+
+# --------------------------------------------------------------------------- #
+# The badge
+# --------------------------------------------------------------------------- #
+
+def test_skipped_drill_is_not_ok(app):
+ """The bug, stated directly."""
+ policy = _policy(app, 'skipped_no_space')
+
+ assert BackupPolicyService._drill_badge(policy) == 'skipped'
+
+
+def test_successful_drill_is_ok(app):
+ policy = _policy(app, 'success')
+
+ assert BackupPolicyService._drill_badge(policy) == 'ok'
+
+
+def test_failed_drill_is_failed(app):
+ policy = _policy(app, 'failed')
+
+ assert BackupPolicyService._drill_badge(policy) == 'failed'
+
+
+def test_never_drilled_is_never(app):
+ policy = BackupPolicy(target_type='application', target_id=1,
+ drill_cadence='weekly')
+ db.session.add(policy)
+ db.session.commit()
+
+ assert BackupPolicyService._drill_badge(policy) == 'never'
+
+
+def test_unrecognised_status_is_not_ok(app):
+ """A status the panel does not know is not evidence of anything.
+
+ Reporting an unhandled state as ok is precisely how the skip bug happened;
+ a future drill outcome must not be able to reintroduce it.
+ """
+ policy = _policy(app, 'partially_restored_maybe')
+
+ assert BackupPolicyService._drill_badge(policy) == 'unknown'
+
+
+def test_stale_only_applies_to_a_successful_drill(app):
+ """Staleness is a property of proof, not of attempts."""
+ old = _policy(app, 'success', drilled_ago=timedelta(days=90))
+
+ assert BackupPolicyService._drill_badge(old) == 'stale'
+
+
+def test_a_recent_skip_does_not_mask_an_old_success(app):
+ """The self-perpetuating half: a skip refreshes last_drill_at.
+
+ Before the fix this returned 'ok' — the skip reset the staleness clock and
+ the policy looked freshly proven. It must now report the skip.
+ """
+ policy = _policy(app, 'skipped_no_space', drilled_ago=timedelta(minutes=1))
+
+ assert BackupPolicyService._drill_badge(policy) == 'skipped'
+
+
+# --------------------------------------------------------------------------- #
+# What Doctor tells the operator
+# --------------------------------------------------------------------------- #
+
+def test_doctor_does_not_claim_a_skipped_backup_is_proven(app):
+ """The sentence that was wrong: "proved this backup restores"."""
+ policy = _policy(app, 'skipped_no_space')
+
+ check = DoctorService._backup_drill_stale_check(policy)
+
+ assert check['status'] == 'warn'
+ assert 'proved' not in check['detail']
+ assert 'could not run' in check['detail']
+ assert 'Nothing has been proven' in check['detail']
+
+
+def test_doctor_reports_a_successful_drill_as_ok(app):
+ policy = _policy(app, 'success')
+
+ check = DoctorService._backup_drill_stale_check(policy)
+
+ assert check['status'] == 'ok'
+
+
+def test_doctor_warns_on_an_unrecognised_state(app):
+ policy = _policy(app, 'something_new')
+
+ check = DoctorService._backup_drill_stale_check(policy)
+
+ assert check['status'] == 'warn'
+ assert 'unproven' in check['detail']
+
+
+def test_drill_check_is_skipped_when_cadence_is_off(app):
+ """No cadence means no expectation — not a warning."""
+ policy = _policy(app, 'skipped_no_space', cadence='off')
+
+ assert DoctorService._backup_drill_stale_check(policy) is None
+
+
+# --------------------------------------------------------------------------- #
+# The operator actually gets told
+# --------------------------------------------------------------------------- #
+
+def test_a_skipped_drill_notifies(app):
+ """on_drill_result already handled skipped_no_space; nothing called it."""
+ policy = _policy(app, 'skipped_no_space')
+
+ with patch('app.services.backup_alert_service.BackupAlertService'
+ '.on_drill_result') as alert:
+ from app.services.backup_drill_service import BackupDrillService
+ BackupDrillService._notify_failed(
+ policy.id, 'run-1', 'needs 10 bytes, 1 free',
+ status='skipped_no_space')
+
+ alert.assert_called_once()
+ assert alert.call_args.kwargs['status'] == 'skipped_no_space'
+
+
+def test_a_failed_drill_still_notifies_as_failed(app):
+ """The pre-existing path must keep its behaviour."""
+ policy = _policy(app, 'failed')
+
+ with patch('app.services.backup_alert_service.BackupAlertService'
+ '.on_drill_result') as alert:
+ from app.services.backup_drill_service import BackupDrillService
+ BackupDrillService._notify_failed(policy.id, 'run-1', 'boom')
+
+ assert alert.call_args.kwargs['status'] == 'failed'
+
+
+def test_alert_service_treats_a_skip_as_a_failed_outcome(app):
+ """Pins the mapping the skip path now depends on."""
+ from app.services.backup_alert_service import BackupAlertService
+
+ policy = _policy(app, 'skipped_no_space')
+
+ with patch.object(BackupAlertService, '_get_state', return_value=None), \
+ patch.object(BackupAlertService, '_set_state'), \
+ patch.object(BackupAlertService, '_notify') as notify:
+ BackupAlertService.on_drill_result(policy, status='skipped_no_space',
+ error='no space')
+
+ assert notify.call_args[0][0] == 'backup.drill_failed'