Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(jobs): scheduled backups as backup.run jobs; fix dormant scheduler (Phase 7)

Juan Denisfe7415842fdc
diff --git a/backend/app/__init__.py b/backend/app/__init__.py
index d96414f8..abf6d105 100644
--- a/backend/app/__init__.py
+++ b/backend/app/__init__.py
@@ -437,11 +437,14 @@ def create_app(config_name=None):
from app.jobs import start_job_system
from app.jobs.builtin_handlers import register_builtin_handlers, seed_builtin_schedules
register_builtin_handlers()
- # Register event-driven job handlers (deployment installs, workflow runs).
+ # Register event-driven job handlers (deployment installs, workflow runs,
+ # scheduled backups).
from app.services.deployment_job_service import DeploymentJobService
DeploymentJobService.register_jobs()
from app.services.workflow_engine import WorkflowEngine
WorkflowEngine.register_jobs()
+ from app.services.backup_service import BackupService
+ BackupService.register_jobs()
start_job_system(app, seed=seed_builtin_schedules)
# Request body size limit
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.py
index 51eae503..ba1cee05 100644
--- a/backend/app/jobs/builtin_handlers.py
+++ b/backend/app/jobs/builtin_handlers.py
@@ -256,6 +256,12 @@ def run_api_background():
EventService.retry_failed()
+def run_backup_scheduler():
+ """Enqueue any scheduled backups that are due (gated by backup config)."""
+ from app.services.backup_service import BackupService
+ BackupService.check_backup_schedules()
+
+
# ---------------------------------------------------------------------------
# Handler registration + schedule seeding
# ---------------------------------------------------------------------------
@@ -271,6 +277,7 @@ _BUILTINS = [
('builtin.api_background', run_api_background, 'api-background', 3600, 3600),
('builtin.pairing_prune', run_pairing_prune, 'pairing-prune', 3600, 60),
('builtin.registrar_expiry', run_registrar_expiry, 'registrar-expiry', 86400, 300),
+ ('builtin.backup_scheduler', run_backup_scheduler, 'backup-scheduler', 30, 30),
]
diff --git a/backend/app/services/backup_service.py b/backend/app/services/backup_service.py
index 8f6094aa..2b7741c1 100644
--- a/backend/app/services/backup_service.py
+++ b/backend/app/services/backup_service.py
@@ -19,6 +19,9 @@ from app.utils.formatting import format_bytes
from app.utils.system import is_command_available
from app.services.telemetry_service import TelemetryService, generate_correlation_id
+# Unified job kind for asynchronous scheduled backups (see register_jobs).
+BACKUP_JOB_KIND = 'backup.run'
+
class BackupService:
"""Service for automated backups of applications and databases."""
@@ -33,9 +36,6 @@ class BackupService:
TYPE_FULL = 'full'
TYPE_FILES = 'files'
- _scheduler_thread = None
- _stop_scheduler = False
-
@classmethod
def get_backup_dir(cls, backup_type: str = None) -> str:
"""Get backup directory for type."""
@@ -804,79 +804,89 @@ class BackupService:
except Exception:
pass
- # --- Scheduler ---
+ # --- Scheduler (unified job system) ---
+
+ @classmethod
+ def register_jobs(cls):
+ """Register the backup.run handler with the unified job registry. Called
+ once at app startup (see app/__init__.py)."""
+ from app.jobs import registry
+ registry.register(BACKUP_JOB_KIND, cls.run_backup_job, replace=True)
@classmethod
- def start_scheduler(cls) -> None:
- """Start the backup scheduler background thread."""
- if cls._scheduler_thread and cls._scheduler_thread.is_alive():
+ def check_backup_schedules(cls) -> None:
+ """Enqueue a backup.run job for each schedule that is due now.
+
+ Runs on the unified job scheduler (builtin.backup_scheduler) instead of a
+ dedicated daemon thread. Gated by the existing backup config, so it stays
+ inert unless backups are configured + enabled.
+ """
+ from app.jobs.service import JobService
+ config = cls.get_config()
+ if not config.get('enabled', False):
return
- cls._stop_scheduler = False
- cls._scheduler_thread = threading.Thread(
- target=cls._scheduler_loop,
- daemon=True,
- name='backup-scheduler'
- )
- cls._scheduler_thread.start()
+ now = datetime.now()
+ current_time = now.strftime('%H:%M')
+ current_day = now.strftime('%A').lower()
+ dirty = False
- @classmethod
- def stop_scheduler(cls) -> None:
- """Stop the backup scheduler."""
- cls._stop_scheduler = True
- if cls._scheduler_thread:
- cls._scheduler_thread.join(timeout=5)
- cls._scheduler_thread = None
+ for sched in config.get('schedules', []):
+ if not sched.get('enabled', False):
+ continue
+ if sched.get('schedule_time') != current_time:
+ continue
+ days = sched.get('days', ['daily'])
+ if 'daily' not in days and current_day not in days:
+ continue
+ # Skip if a run was enqueued / ran within the last ~2 minutes.
+ last_run = sched.get('last_run')
+ if last_run:
+ try:
+ if (now - datetime.fromisoformat(last_run)).total_seconds() < 120:
+ continue
+ except Exception:
+ pass
- @classmethod
- def _scheduler_loop(cls) -> None:
- """Background loop that checks and runs scheduled backups."""
- while not cls._stop_scheduler:
- try:
- config = cls.get_config()
- if config.get('enabled', False):
- now = datetime.now()
- current_time = now.strftime('%H:%M')
- current_day = now.strftime('%A').lower()
-
- for sched in config.get('schedules', []):
- if not sched.get('enabled', False):
- continue
-
- # Check if it's time to run
- if sched.get('schedule_time') != current_time:
- continue
-
- # Check day
- days = sched.get('days', ['daily'])
- if 'daily' not in days and current_day not in days:
- continue
-
- # Check if already ran this minute
- last_run = sched.get('last_run')
- if last_run:
- try:
- last_run_time = datetime.fromisoformat(last_run)
- if (now - last_run_time).total_seconds() < 120:
- continue
- except Exception:
- pass
-
- # Run the backup
- cls._run_scheduled_backup(sched)
-
- # Run retention cleanup once daily at midnight
- if current_time == '00:00':
- cls.cleanup_old_backups()
+ JobService.enqueue(
+ BACKUP_JOB_KIND,
+ payload={'schedule_id': sched.get('id')},
+ max_attempts=1, # backups aren't idempotent — no auto-retry
+ owner_type='backup_schedule',
+ owner_id=sched.get('id'),
+ )
+ # Optimistic dedup so the next tick in the same minute doesn't
+ # double-enqueue; the run itself rewrites last_run/last_status.
+ sched['last_run'] = now.isoformat()
+ dirty = True
- except Exception:
- pass
+ if dirty:
+ cls.save_config(config)
+
+ if current_time == '00:00':
+ cls.cleanup_old_backups()
- # Check every 30 seconds
- for _ in range(30):
- if cls._stop_scheduler:
- return
- time.sleep(1)
+ @staticmethod
+ def run_backup_job(job):
+ """Unified-job handler for ``backup.run`` — execute one scheduled backup.
+ Raises if the backup failed so the unified job is marked failed too (the
+ schedule's last_status carries the detail)."""
+ schedule_id = (job.get_payload() or {}).get('schedule_id')
+ config = BackupService.get_config()
+ sched = next((s for s in config.get('schedules', []) if s.get('id') == schedule_id), None)
+ if not sched:
+ raise ValueError(f'backup schedule {schedule_id!r} not found')
+
+ BackupService._run_scheduled_backup(sched)
+
+ updated = next(
+ (s for s in BackupService.get_config().get('schedules', []) if s.get('id') == schedule_id),
+ None,
+ )
+ if updated and updated.get('last_status') == 'failed':
+ raise RuntimeError(f"Backup '{sched.get('name', schedule_id)}' failed")
+ return {'schedule_id': schedule_id, 'name': sched.get('name'),
+ 'status': updated.get('last_status') if updated else None}
@classmethod
def _run_scheduled_backup(cls, sched: Dict) -> None:
@@ -989,11 +999,13 @@ class BackupService:
severity = 'success' if success else 'critical'
status = 'completed successfully' if success else 'failed'
- NotificationService.send_all(
- title=f'Backup {status}: {backup_name}',
- message=message,
- severity=severity
- )
+ # send_all takes a list of alert dicts (it fans out to the configured
+ # system channels through the queue-backed Notification Bus).
+ NotificationService.send_all([{
+ 'type': 'backup',
+ 'severity': 'critical' if not success else 'info',
+ 'message': f'Backup {status}: {backup_name}' + (f' — {message}' if message else ''),
+ }])
except Exception:
pass
diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.py
index a852c14a..4019d0f1 100644
--- a/backend/tests/test_jobs.py
+++ b/backend/tests/test_jobs.py
@@ -173,13 +173,14 @@ class TestBuiltins:
kinds = registry.registered_kinds()
assert 'builtin.auto_sync' in kinds
assert 'builtin.health_check' in kinds
- assert len([k for k in kinds if k.startswith('builtin.')]) == 8
+ assert 'builtin.backup_scheduler' in kinds
+ assert len([k for k in kinds if k.startswith('builtin.')]) == 9
builtin_handlers.seed_builtin_schedules()
- assert ScheduledJob.query.count() == 8
+ assert ScheduledJob.query.count() == 9
# Seeding twice doesn't duplicate.
builtin_handlers.seed_builtin_schedules()
- assert ScheduledJob.query.count() == 8
+ assert ScheduledJob.query.count() == 9
class TestApi:
@@ -364,3 +365,68 @@ class TestWorkflowExecutionHandler:
assert WorkflowExecution.query.get(execution_id).status == 'failed'
unified = Job.query.filter_by(kind='workflow.execute').first()
assert unified.status == Job.STATUS_FAILED
+
+
+class TestBackupScheduleHandler:
+ """Phase 7 — scheduled backups run as 'backup.run' unified jobs, enqueued by
+ the builtin.backup_scheduler tick (replacing the orphaned daemon loop).
+ BackupService internals are patched so these exercise the wiring."""
+
+ def test_register_jobs_adds_handler(self, app):
+ from app.services.backup_service import BackupService, BACKUP_JOB_KIND
+ BackupService.register_jobs()
+ assert BACKUP_JOB_KIND == 'backup.run'
+ assert registry.is_registered('backup.run')
+
+ def test_check_schedules_enqueues_only_due_enabled(self, app, monkeypatch):
+ from datetime import datetime
+ from app.services.backup_service import BackupService
+
+ now_hm = datetime.now().strftime('%H:%M')
+ cfg = {'enabled': True, 'schedules': [
+ {'id': 'b1', 'enabled': True, 'schedule_time': now_hm, 'days': ['daily'], 'name': 'nightly'},
+ {'id': 'b2', 'enabled': False, 'schedule_time': now_hm, 'days': ['daily'], 'name': 'off'},
+ ]}
+ monkeypatch.setattr(BackupService, 'get_config', staticmethod(lambda: cfg))
+ monkeypatch.setattr(BackupService, 'save_config', staticmethod(lambda c: None))
+ monkeypatch.setattr(BackupService, 'cleanup_old_backups', staticmethod(lambda *a, **k: None))
+
+ BackupService.check_backup_schedules()
+
+ due = Job.query.filter_by(kind='backup.run').all()
+ assert len(due) == 1
+ assert due[0].owner_id == 'b1'
+ assert due[0].get_payload() == {'schedule_id': 'b1'}
+ assert due[0].max_attempts == 1
+
+ def test_check_schedules_inert_when_disabled(self, app, monkeypatch):
+ from datetime import datetime
+ from app.services.backup_service import BackupService
+
+ now_hm = datetime.now().strftime('%H:%M')
+ cfg = {'enabled': False, 'schedules': [
+ {'id': 'b1', 'enabled': True, 'schedule_time': now_hm, 'days': ['daily'], 'name': 'nightly'},
+ ]}
+ monkeypatch.setattr(BackupService, 'get_config', staticmethod(lambda: cfg))
+ BackupService.check_backup_schedules()
+ assert Job.query.filter_by(kind='backup.run').count() == 0
+
+ def test_handler_success_and_failure(self, app, monkeypatch):
+ from app.services.backup_service import BackupService
+
+ monkeypatch.setattr(BackupService, '_run_scheduled_backup', staticmethod(lambda sched: None))
+ BackupService.register_jobs()
+
+ # Success: schedule ends up 'success'.
+ monkeypatch.setattr(BackupService, 'get_config',
+ staticmethod(lambda: {'schedules': [{'id': 'b1', 'name': 'n', 'last_status': 'success'}]}))
+ ok = JobService.enqueue('backup.run', {'schedule_id': 'b1'}, max_attempts=1)
+ _drain_once()
+ assert Job.query.get(ok.id).status == Job.STATUS_SUCCEEDED
+
+ # Failure: schedule ends up 'failed' -> handler raises -> job failed.
+ monkeypatch.setattr(BackupService, 'get_config',
+ staticmethod(lambda: {'schedules': [{'id': 'b2', 'name': 'n', 'last_status': 'failed'}]}))
+ bad = JobService.enqueue('backup.run', {'schedule_id': 'b2'}, max_attempts=1)
+ _drain_once()
+ assert Job.query.get(bad.id).status == Job.STATUS_FAILED