ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
feat(jobs): scheduled backups as backup.run jobs; fix dormant scheduler (Phase 7)
Juan Denisfe7415842fdc
diff --git a/backend/app/__init__.py b/backend/app/__init__.pyindex d96414f8..abf6d105 100644--- a/backend/app/__init__.py+++ b/backend/app/__init__.py@@ -437,11 +437,14 @@ def create_app(config_name=None):from app.jobs import start_job_systemfrom app.jobs.builtin_handlers import register_builtin_handlers, seed_builtin_schedulesregister_builtin_handlers()- # Register event-driven job handlers (deployment installs, workflow runs).+ # Register event-driven job handlers (deployment installs, workflow runs,+ # scheduled backups).from app.services.deployment_job_service import DeploymentJobServiceDeploymentJobService.register_jobs()from app.services.workflow_engine import WorkflowEngineWorkflowEngine.register_jobs()+ from app.services.backup_service import BackupService+ BackupService.register_jobs()start_job_system(app, seed=seed_builtin_schedules)# Request body size limitdiff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.pyindex 51eae503..ba1cee05 100644--- a/backend/app/jobs/builtin_handlers.py+++ b/backend/app/jobs/builtin_handlers.py@@ -256,6 +256,12 @@ def run_api_background():EventService.retry_failed()+def run_backup_scheduler():+ """Enqueue any scheduled backups that are due (gated by backup config)."""+ from app.services.backup_service import BackupService+ BackupService.check_backup_schedules()++# ---------------------------------------------------------------------------# Handler registration + schedule seeding# ---------------------------------------------------------------------------@@ -271,6 +277,7 @@ _BUILTINS = [('builtin.api_background', run_api_background, 'api-background', 3600, 3600),('builtin.pairing_prune', run_pairing_prune, 'pairing-prune', 3600, 60),('builtin.registrar_expiry', run_registrar_expiry, 'registrar-expiry', 86400, 300),+ ('builtin.backup_scheduler', run_backup_scheduler, 'backup-scheduler', 30, 30),]diff --git a/backend/app/services/backup_service.py b/backend/app/services/backup_service.pyindex 8f6094aa..2b7741c1 100644--- a/backend/app/services/backup_service.py+++ b/backend/app/services/backup_service.py@@ -19,6 +19,9 @@ from app.utils.formatting import format_bytesfrom app.utils.system import is_command_availablefrom app.services.telemetry_service import TelemetryService, generate_correlation_id+# Unified job kind for asynchronous scheduled backups (see register_jobs).+BACKUP_JOB_KIND = 'backup.run'+class BackupService:"""Service for automated backups of applications and databases."""@@ -33,9 +36,6 @@ class BackupService:TYPE_FULL = 'full'TYPE_FILES = 'files'- _scheduler_thread = None- _stop_scheduler = False-@classmethoddef get_backup_dir(cls, backup_type: str = None) -> str:"""Get backup directory for type."""@@ -804,79 +804,89 @@ class BackupService:except Exception:pass- # --- Scheduler ---+ # --- Scheduler (unified job system) ---++ @classmethod+ def register_jobs(cls):+ """Register the backup.run handler with the unified job registry. Called+ once at app startup (see app/__init__.py)."""+ from app.jobs import registry+ registry.register(BACKUP_JOB_KIND, cls.run_backup_job, replace=True)@classmethod- def start_scheduler(cls) -> None:- """Start the backup scheduler background thread."""- if cls._scheduler_thread and cls._scheduler_thread.is_alive():+ def check_backup_schedules(cls) -> None:+ """Enqueue a backup.run job for each schedule that is due now.++ Runs on the unified job scheduler (builtin.backup_scheduler) instead of a+ dedicated daemon thread. Gated by the existing backup config, so it stays+ inert unless backups are configured + enabled.+ """+ from app.jobs.service import JobService+ config = cls.get_config()+ if not config.get('enabled', False):return- cls._stop_scheduler = False- cls._scheduler_thread = threading.Thread(- target=cls._scheduler_loop,- daemon=True,- name='backup-scheduler'- )- cls._scheduler_thread.start()+ now = datetime.now()+ current_time = now.strftime('%H:%M')+ current_day = now.strftime('%A').lower()+ dirty = False- @classmethod- def stop_scheduler(cls) -> None:- """Stop the backup scheduler."""- cls._stop_scheduler = True- if cls._scheduler_thread:- cls._scheduler_thread.join(timeout=5)- cls._scheduler_thread = None+ for sched in config.get('schedules', []):+ if not sched.get('enabled', False):+ continue+ if sched.get('schedule_time') != current_time:+ continue+ days = sched.get('days', ['daily'])+ if 'daily' not in days and current_day not in days:+ continue+ # Skip if a run was enqueued / ran within the last ~2 minutes.+ last_run = sched.get('last_run')+ if last_run:+ try:+ if (now - datetime.fromisoformat(last_run)).total_seconds() < 120:+ continue+ except Exception:+ pass- @classmethod- def _scheduler_loop(cls) -> None:- """Background loop that checks and runs scheduled backups."""- while not cls._stop_scheduler:- try:- config = cls.get_config()- if config.get('enabled', False):- now = datetime.now()- current_time = now.strftime('%H:%M')- current_day = now.strftime('%A').lower()-- for sched in config.get('schedules', []):- if not sched.get('enabled', False):- continue-- # Check if it's time to run- if sched.get('schedule_time') != current_time:- continue-- # Check day- days = sched.get('days', ['daily'])- if 'daily' not in days and current_day not in days:- continue-- # Check if already ran this minute- last_run = sched.get('last_run')- if last_run:- try:- last_run_time = datetime.fromisoformat(last_run)- if (now - last_run_time).total_seconds() < 120:- continue- except Exception:- pass-- # Run the backup- cls._run_scheduled_backup(sched)-- # Run retention cleanup once daily at midnight- if current_time == '00:00':- cls.cleanup_old_backups()+ JobService.enqueue(+ BACKUP_JOB_KIND,+ payload={'schedule_id': sched.get('id')},+ max_attempts=1, # backups aren't idempotent — no auto-retry+ owner_type='backup_schedule',+ owner_id=sched.get('id'),+ )+ # Optimistic dedup so the next tick in the same minute doesn't+ # double-enqueue; the run itself rewrites last_run/last_status.+ sched['last_run'] = now.isoformat()+ dirty = True- except Exception:- pass+ if dirty:+ cls.save_config(config)++ if current_time == '00:00':+ cls.cleanup_old_backups()- # Check every 30 seconds- for _ in range(30):- if cls._stop_scheduler:- return- time.sleep(1)+ @staticmethod+ def run_backup_job(job):+ """Unified-job handler for ``backup.run`` — execute one scheduled backup.+ Raises if the backup failed so the unified job is marked failed too (the+ schedule's last_status carries the detail)."""+ schedule_id = (job.get_payload() or {}).get('schedule_id')+ config = BackupService.get_config()+ sched = next((s for s in config.get('schedules', []) if s.get('id') == schedule_id), None)+ if not sched:+ raise ValueError(f'backup schedule {schedule_id!r} not found')++ BackupService._run_scheduled_backup(sched)++ updated = next(+ (s for s in BackupService.get_config().get('schedules', []) if s.get('id') == schedule_id),+ None,+ )+ if updated and updated.get('last_status') == 'failed':+ raise RuntimeError(f"Backup '{sched.get('name', schedule_id)}' failed")+ return {'schedule_id': schedule_id, 'name': sched.get('name'),+ 'status': updated.get('last_status') if updated else None}@classmethoddef _run_scheduled_backup(cls, sched: Dict) -> None:@@ -989,11 +999,13 @@ class BackupService:severity = 'success' if success else 'critical'status = 'completed successfully' if success else 'failed'- NotificationService.send_all(- title=f'Backup {status}: {backup_name}',- message=message,- severity=severity- )+ # send_all takes a list of alert dicts (it fans out to the configured+ # system channels through the queue-backed Notification Bus).+ NotificationService.send_all([{+ 'type': 'backup',+ 'severity': 'critical' if not success else 'info',+ 'message': f'Backup {status}: {backup_name}' + (f' — {message}' if message else ''),+ }])except Exception:passdiff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.pyindex a852c14a..4019d0f1 100644--- a/backend/tests/test_jobs.py+++ b/backend/tests/test_jobs.py@@ -173,13 +173,14 @@ class TestBuiltins:kinds = registry.registered_kinds()assert 'builtin.auto_sync' in kindsassert 'builtin.health_check' in kinds- assert len([k for k in kinds if k.startswith('builtin.')]) == 8+ assert 'builtin.backup_scheduler' in kinds+ assert len([k for k in kinds if k.startswith('builtin.')]) == 9builtin_handlers.seed_builtin_schedules()- assert ScheduledJob.query.count() == 8+ assert ScheduledJob.query.count() == 9# Seeding twice doesn't duplicate.builtin_handlers.seed_builtin_schedules()- assert ScheduledJob.query.count() == 8+ assert ScheduledJob.query.count() == 9class TestApi:@@ -364,3 +365,68 @@ class TestWorkflowExecutionHandler:assert WorkflowExecution.query.get(execution_id).status == 'failed'unified = Job.query.filter_by(kind='workflow.execute').first()assert unified.status == Job.STATUS_FAILED+++class TestBackupScheduleHandler:+ """Phase 7 — scheduled backups run as 'backup.run' unified jobs, enqueued by+ the builtin.backup_scheduler tick (replacing the orphaned daemon loop).+ BackupService internals are patched so these exercise the wiring."""++ def test_register_jobs_adds_handler(self, app):+ from app.services.backup_service import BackupService, BACKUP_JOB_KIND+ BackupService.register_jobs()+ assert BACKUP_JOB_KIND == 'backup.run'+ assert registry.is_registered('backup.run')++ def test_check_schedules_enqueues_only_due_enabled(self, app, monkeypatch):+ from datetime import datetime+ from app.services.backup_service import BackupService++ now_hm = datetime.now().strftime('%H:%M')+ cfg = {'enabled': True, 'schedules': [+ {'id': 'b1', 'enabled': True, 'schedule_time': now_hm, 'days': ['daily'], 'name': 'nightly'},+ {'id': 'b2', 'enabled': False, 'schedule_time': now_hm, 'days': ['daily'], 'name': 'off'},+ ]}+ monkeypatch.setattr(BackupService, 'get_config', staticmethod(lambda: cfg))+ monkeypatch.setattr(BackupService, 'save_config', staticmethod(lambda c: None))+ monkeypatch.setattr(BackupService, 'cleanup_old_backups', staticmethod(lambda *a, **k: None))++ BackupService.check_backup_schedules()++ due = Job.query.filter_by(kind='backup.run').all()+ assert len(due) == 1+ assert due[0].owner_id == 'b1'+ assert due[0].get_payload() == {'schedule_id': 'b1'}+ assert due[0].max_attempts == 1++ def test_check_schedules_inert_when_disabled(self, app, monkeypatch):+ from datetime import datetime+ from app.services.backup_service import BackupService++ now_hm = datetime.now().strftime('%H:%M')+ cfg = {'enabled': False, 'schedules': [+ {'id': 'b1', 'enabled': True, 'schedule_time': now_hm, 'days': ['daily'], 'name': 'nightly'},+ ]}+ monkeypatch.setattr(BackupService, 'get_config', staticmethod(lambda: cfg))+ BackupService.check_backup_schedules()+ assert Job.query.filter_by(kind='backup.run').count() == 0++ def test_handler_success_and_failure(self, app, monkeypatch):+ from app.services.backup_service import BackupService++ monkeypatch.setattr(BackupService, '_run_scheduled_backup', staticmethod(lambda sched: None))+ BackupService.register_jobs()++ # Success: schedule ends up 'success'.+ monkeypatch.setattr(BackupService, 'get_config',+ staticmethod(lambda: {'schedules': [{'id': 'b1', 'name': 'n', 'last_status': 'success'}]}))+ ok = JobService.enqueue('backup.run', {'schedule_id': 'b1'}, max_attempts=1)+ _drain_once()+ assert Job.query.get(ok.id).status == Job.STATUS_SUCCEEDED++ # Failure: schedule ends up 'failed' -> handler raises -> job failed.+ monkeypatch.setattr(BackupService, 'get_config',+ staticmethod(lambda: {'schedules': [{'id': 'b2', 'name': 'n', 'last_status': 'failed'}]}))+ bad = JobService.enqueue('backup.run', {'schedule_id': 'b2'}, max_attempts=1)+ _drain_once()+ assert Job.query.get(bad.id).status == Job.STATUS_FAILED