ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
fix(doctor): verify local-only backups on disk instead of hiding the button
Juan Denisbff2c1fa5953
diff --git a/backend/app/services/doctor_service.py b/backend/app/services/doctor_service.pyindex 0a8a6c4b..8da78df0 100644--- a/backend/app/services/doctor_service.py+++ b/backend/app/services/doctor_service.py@@ -590,27 +590,15 @@ class DoctorService:key = f'backup_unverified.{policy.id}'title = f'Backup verification: {cls._policy_label(policy)}'level = run.effective_verify_level()- if level != 'none':- return _check(key, title, 'ok', f'Latest backup verified ({level}).')-- # The repair verifies the REMOTE copy (size + checksum against the- # provider). With no remote copy there is nothing for it to compare, so- # offering the button would be offering a guaranteed failure — the- # exact thing this check family got wrong. Say what would fix it- # instead: an offsite copy, or a restore drill.- if not getattr(run, 'remote_key', None):+ if level == 'none':return _check(key, title, 'warn',- 'The latest backup has not been verified, and it has no offsite '- 'copy to verify against. Send this policy to remote storage, or '- 'run a restore drill to prove the backup restores.')- return _check(- key, title, 'warn',- 'The latest backup has not been verified (no integrity check or '- 'restore drill has run against it yet).',- repairable=True,- repair_ref={'kind': 'backup_verify', 'policy_id': policy.id,- 'run_id': run.id})+ 'The latest backup has not been verified (no integrity check or '+ 'restore drill has run against it yet).',+ repairable=True,+ repair_ref={'kind': 'backup_verify', 'policy_id': policy.id,+ 'run_id': run.id})+ return _check(key, title, 'ok', f'Latest backup verified ({level}).')@classmethoddef _setup_checks(cls):@@ -765,11 +753,18 @@ class DoctorService:@classmethoddef _repair_backup_verify(cls, policy_id, run_id):- """Verify a backup run's remote copy.-- Unlike the drill this is synchronous — verify_run does the size and- checksum comparison inline and returns the verdict, so there is no job- id to hand back.+ """Verify a backup run by whatever means that run allows.++ Two ladders exist and the check cannot tell them apart from outside. A+ run with an offsite copy is verified against the provider (size ++ checksum via verify_run). A local-only run is verified in place: read+ the archive and checksum it against the stored manifest. Routing every+ run down the remote path would make this button useless for anyone who+ has not configured offsite storage — which is most installs, and which+ is the same "button that always fails" defect this item exists to+ remove.++ Synchronous either way, unlike the drill, so there is no job id."""policy = cls._load_policy(policy_id)if policy is None:@@ -778,6 +773,16 @@ class DoctorService:from app.services.backup_policy_service import (BackupPolicyError, BackupPolicyService,)+ try:+ run = BackupPolicyService.get_run(policy, run_id)+ except Exception: # noqa: BLE001+ run = None+ if run is None:+ return {'success': False, 'error': 'Backup not found.'}++ if not getattr(run, 'remote_key', None):+ return cls._verify_run_locally(run)+try:result = BackupPolicyService.verify_run(policy, run_id)except BackupPolicyError as e:@@ -794,6 +799,37 @@ class DoctorService:return {'success': True, 'verified': True,'detail': 'Remote copy verified.'}+ @staticmethod+ def _verify_run_locally(run):+ """Tier-1 (on-disk) verification: is the archive readable, and does it+ still checksum to what the manifest recorded?++ verify_run_tier1 mutates the run's verify ladder without committing,+ so the commit is ours.+ """+ from app import db+ from app.services import backup_verify_service+ try:+ backup_verify_service.verify_run_tier1(run, run.get_metadata() or {})+ db.session.commit()+ except Exception as e: # noqa: BLE001+ db.session.rollback()+ return {'success': False, 'error': str(e)}++ level = run.verify_level or 'none'+ if level == 'none':+ # Unreadable archive — the worst answer, and the one most worth+ # knowing before a restore depends on it.+ return {'success': False,+ 'error': run.verify_error+ or 'The backup archive could not be read.'}+ if run.verify_error:+ # Readable but the checksum disagrees with the manifest.+ return {'success': False, 'error': run.verify_error,+ 'verify_level': level}+ return {'success': True, 'verify_level': level,+ 'detail': f'Backup verified on disk ({level}).'}+@classmethoddef _restart_service(cls, name):if name not in CORE_SERVICES:diff --git a/backend/tests/test_doctor_backup_repairs.py b/backend/tests/test_doctor_backup_repairs.pyindex a64c69dc..d28c146d 100644--- a/backend/tests/test_doctor_backup_repairs.py+++ b/backend/tests/test_doctor_backup_repairs.py@@ -3,11 +3,14 @@``backup_drill_stale.*`` and ``backup_unverified.*`` both emitted``repairable: True``, but ``DoctorService.repair()`` only knew drift, service,dns and extension. Every Repair button those checks rendered came back-"Unknown repair kind: backup_drill" — a button that failed 100% of the time.+"Unknown repair kind: backup_drill" — a button that failed 100% of the time,+on the check family whose entire job is telling you whether your backups can+be trusted.The dispatcher now handles both. They are deliberately asymmetric: a drill isenqueued as a job (so the answer carries a job id), while verification runs-inline and returns a verdict.+inline and returns a verdict — and verification itself picks its ladder from+what the run actually has, on disk or offsite."""from unittest.mock import patch@@ -32,18 +35,42 @@ def policy(app):return row+def _run_for(policy, remote_key=None):+ """A successful backup run, offsite or local-only."""+ from app import db+ from app.models.backup_run import BackupRun++ run = BackupRun(policy_id=policy.id, kind='full', status='success')+ run.remote_key = remote_key+ db.session.add(run)+ db.session.commit()+ return run+++def _tier1_setting(level, error=None):+ """Stand-in for verify_run_tier1, which mutates the run's verify ladder in+ place and does not commit."""+ def _apply(run, meta):+ run.verify_level = level+ run.verify_error = error+ return {}+ return _apply++# --------------------------------------------------------------------------- ## The regression itself# --------------------------------------------------------------------------- #def test_both_kinds_are_known_to_the_dispatcher(app, policy):"""The whole bug: these two came back 'Unknown repair kind'."""+ run = _run_for(policy, remote_key='s3://bucket/key')+with patch('app.services.backup_drill_service.BackupDrillService.request_drill',return_value=_Job()), \patch('app.services.backup_policy_service.BackupPolicyService.verify_run',return_value={'success': True, 'verified': True}):results = DoctorService.repair([{'kind': 'backup_drill', 'policy_id': policy.id},- {'kind': 'backup_verify', 'policy_id': policy.id, 'run_id': 7},+ {'kind': 'backup_verify', 'policy_id': policy.id, 'run_id': run.id},])assert [r['success'] for r in results] == [True, True]@@ -124,40 +151,54 @@ class TestBackupDrillRepair:# --------------------------------------------------------------------------- #-# backup_verify — synchronous, returns a verdict+# backup_verify — synchronous, and routed by what the run actually has+#+# verify_run compares against the REMOTE copy and raises when there is none, so+# routing every run through it would leave the button broken for every install+# without offsite storage — the same defect this item exists to remove.# --------------------------------------------------------------------------- #class TestBackupVerifyRepair:- def _verify(self, policy, **patch_kwargs):- with patch('app.services.backup_policy_service.BackupPolicyService.verify_run',- **patch_kwargs) as verify:- result = DoctorService.repair([{'kind': 'backup_verify',- 'policy_id': policy.id,- 'run_id': 7}])[0]- return result, verify+ def _repair(self, policy, run):+ return DoctorService.repair([{'kind': 'backup_verify',+ 'policy_id': policy.id,+ 'run_id': run.id}])[0]- def test_a_verified_copy_succeeds(self, app, policy):- result, verify = self._verify(- policy, return_value={'success': True, 'verified': True})+ # -- offsite runs ------------------------------------------------------ #++ def test_a_verified_remote_copy_succeeds(self, app, policy):+ run = _run_for(policy, remote_key='s3://bucket/key')++ with patch('app.services.backup_policy_service.BackupPolicyService.verify_run',+ return_value={'success': True, 'verified': True}) as remote, \+ patch('app.services.backup_verify_service.verify_run_tier1') as tier1:+ result = self._repair(policy, run)assert result['success'] is Trueassert result['verified'] is True- assert verify.call_args.args[1] == 7+ assert remote.call_args.args[1] == run.id+ tier1.assert_not_called()def test_no_job_id_because_it_is_synchronous(self, app, policy):"""Unlike the drill, nothing is queued — claiming a job id would sendthe console looking for a job that does not exist."""- result, _ = self._verify(- policy, return_value={'success': True, 'verified': True})+ run = _run_for(policy, remote_key='s3://bucket/key')++ with patch('app.services.backup_policy_service.BackupPolicyService.verify_run',+ return_value={'success': True, 'verified': True}):+ result = self._repair(policy, run)assert 'job_id' not in result- def test_a_mismatching_copy_is_a_failure_not_a_fix(self, app, policy):+ def test_a_mismatching_remote_copy_is_a_failure_not_a_fix(self, app, policy):"""Verification running and finding a BAD copy is a worse outcome than'unverified' — reporting it as a successful repair would be a lie."""- result, _ = self._verify(- policy, return_value={'success': True, 'verified': False,- 'detail': {'size': 'mismatch'}})+ run = _run_for(policy, remote_key='s3://bucket/key')++ with patch('app.services.backup_policy_service.BackupPolicyService.verify_run',+ return_value={'success': True, 'verified': False,+ 'detail': {'size': 'mismatch'}}):+ result = self._repair(policy, run)assert result['success'] is Falseassert 'did not match' in result['error']@@ -165,51 +206,95 @@ class TestBackupVerifyRepair:def test_a_policy_error_is_reported_not_raised(self, app, policy):from app.services.backup_policy_service import BackupPolicyError- result, _ = self._verify(- policy, side_effect=BackupPolicyError('Backup not found'))+ run = _run_for(policy, remote_key='s3://bucket/key')+ with patch('app.services.backup_policy_service.BackupPolicyService.verify_run',+ side_effect=BackupPolicyError('Backup not found')):+ result = self._repair(policy, run)assert result['success'] is Falseassert 'Backup not found' in result['error']- def test_a_deleted_policy_reads_as_gone(self, app):- result = DoctorService.repair(- [{'kind': 'backup_verify', 'policy_id': 999999, 'run_id': 7}])[0]+ # -- local-only runs --------------------------------------------------- #++ def test_local_only_run_is_verified_on_disk_not_remotely(self, app, policy):+ run = _run_for(policy, remote_key=None)++ with patch('app.services.backup_verify_service.verify_run_tier1',+ side_effect=_tier1_setting('hashed')) as tier1, \+ patch('app.services.backup_policy_service.BackupPolicyService.verify_run') as remote:+ result = self._repair(policy, run)++ assert result['success'] is True+ assert result['verify_level'] == 'hashed'+ tier1.assert_called_once()+ remote.assert_not_called() # there is nothing to compare against++ def test_an_unreadable_archive_is_a_failure(self, app, policy):+ """Tier 1 leaves the level at 'none' when it cannot read the archive —+ the worst answer, and the one most worth knowing before a restore+ depends on it."""+ run = _run_for(policy)++ with patch('app.services.backup_verify_service.verify_run_tier1',+ side_effect=_tier1_setting(+ 'none', 'Primary archive not found on disk (not readable).')):+ result = self._repair(policy, run)assert result['success'] is False- assert 'no longer exists' in result['error']+ assert 'not readable' in result['error']+ def test_a_checksum_mismatch_is_a_failure_even_though_it_is_readable(self, app, policy):+ """'listed' clears the doctor check, but a mismatch means the archive+ is not the one the manifest describes — not a fix."""+ run = _run_for(policy)-# --------------------------------------------------------------------------- #-# The other half: don't render a button that cannot work-# --------------------------------------------------------------------------- #-class TestVerifyButtonIsOnlyOfferedWhenItCanWork:- """verify_run compares against the REMOTE copy and raises when there is- none. A local-only backup would therefore render a Repair button that- fails every time — the same defect A.1 exists to remove."""+ with patch('app.services.backup_verify_service.verify_run_tier1',+ side_effect=_tier1_setting(+ 'listed', 'Checksum mismatch: expected abc123…')):+ result = self._repair(policy, run)- def _check_for(self, app, policy, remote_key):+ assert result['success'] is False+ assert 'Checksum mismatch' in result['error']++ def test_the_verified_level_is_persisted(self, app, policy):+ """verify_run_tier1 mutates without committing, so the commit is the+ repair's job — otherwise the doctor reports 'unverified' again on the+ very next sweep."""from app import dbfrom app.models.backup_run import BackupRun- run = BackupRun(policy_id=policy.id, kind='full', status='success')- run.remote_key = remote_key- db.session.add(run)- db.session.commit()- return DoctorService._backup_unverified_check(policy)+ run = _run_for(policy)+ with patch('app.services.backup_verify_service.verify_run_tier1',+ side_effect=_tier1_setting('hashed')):+ self._repair(policy, run)+ db.session.expire_all()++ assert BackupRun.query.filter_by(id=run.id).first().verify_level == 'hashed'- def test_local_only_backup_is_not_repairable(self, app, policy):- check = self._check_for(app, policy, None)+ # -- lookup failures --------------------------------------------------- #++ def test_a_missing_run_reads_as_not_found(self, app, policy):+ result = DoctorService.repair([{'kind': 'backup_verify',+ 'policy_id': policy.id,+ 'run_id': 987654}])[0]++ assert result['success'] is False+ assert 'not found' in result['error'].lower()++ def test_a_deleted_policy_reads_as_gone(self, app):+ result = DoctorService.repair(+ [{'kind': 'backup_verify', 'policy_id': 999999, 'run_id': 7}])[0]++ assert result['success'] is False+ assert 'no longer exists' in result['error']- assert check['status'] == 'warn'- assert check['repairable'] is False- assert check['repair_ref'] is None- # and it says what WOULD fix it- assert 'offsite' in check['detail'] or 'restore drill' in check['detail']- def test_backup_with_a_remote_copy_is_repairable(self, app, policy):- check = self._check_for(app, policy, 's3://bucket/key')+def test_every_unverified_backup_still_gets_a_button(app, policy):+ """The check must keep offering the repair regardless of offsite storage —+ now that both ladders work, gating it would hide a working fix."""+ _run_for(policy, remote_key=None)+ check = DoctorService._backup_unverified_check(policy)- assert check['status'] == 'warn'- assert check['repairable'] is True- assert check['repair_ref']['kind'] == 'backup_verify'- assert check['repair_ref']['policy_id'] == policy.id+ assert check['status'] == 'warn'+ assert check['repairable'] is True+ assert check['repair_ref']['kind'] == 'backup_verify'