Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(jobs): unified background-job orchestration on the Queue Bus

Juan Denis8428326a262a
diff --git a/backend/app/__init__.py b/backend/app/__init__.py
index 44cf6dd0..5166802a 100644
--- a/backend/app/__init__.py
+++ b/backend/app/__init__.py
@@ -354,6 +354,10 @@ def create_app(config_name=None):
from app.api.queue_bus import queue_bus_bp
app.register_blueprint(queue_bus_bp, url_prefix='/api/v1/queue')
+ # Register blueprints - Unified Jobs (work orchestration on the Queue Bus)
+ from app.api.jobs import jobs_bp
+ app.register_blueprint(jobs_bp, url_prefix='/api/v1/jobs')
+
# Register blueprints - Telemetry / System Event Stream
from app.api.telemetry import telemetry_bp
app.register_blueprint(telemetry_bp, url_prefix='/api/v1/telemetry')
@@ -419,33 +423,21 @@ def create_app(config_name=None):
from app.notifications.consumer import start_notification_consumer
start_notification_consumer(app)
- # Start auto-sync scheduler for WordPress environments
- _start_auto_sync_scheduler(app)
-
- # Start snapshot-retention scheduler (sets expires_at + prunes expired)
- _start_snapshot_retention_scheduler(app)
-
- # Start workflow scheduler
- _start_workflow_scheduler(app)
-
- # Start per-site WordPress health poller (uptime % + auto-incidents + alerts)
- _start_health_check_scheduler(app)
-
- # Start per-site WordPress safe-update scheduler (#29)
- _start_update_scheduler(app)
-
- # Start API analytics flush thread
+ # Start the API analytics flush thread (a 5s buffer flush — a real-time
+ # stream, deliberately NOT modeled as a job).
from app.middleware.api_analytics import start_analytics_flush_thread
start_analytics_flush_thread(app)
- # Start hourly analytics aggregation and event retry threads
- _start_api_background_threads(app)
-
- # Start hourly pruner for expired pending agent pairings
- _start_pairing_pruner(app)
-
- # Start daily registrar domain-expiry notifier
- _start_registrar_expiry_scheduler(app)
+ # Start the unified job system: ONE consumer runs every enqueued Job and
+ # ONE scheduler ticks all periodic work. This supersedes the former set
+ # of per-domain daemon scheduler threads (auto-sync, snapshot-retention,
+ # workflow, health-check, wp-update, api-background, pairing-prune,
+ # registrar-expiry) — they are now ScheduledJob rows backed by the
+ # built-in handlers in app/jobs/builtin_handlers.py.
+ from app.jobs import start_job_system
+ from app.jobs.builtin_handlers import register_builtin_handlers, seed_builtin_schedules
+ register_builtin_handlers()
+ start_job_system(app, seed=seed_builtin_schedules)
# Request body size limit
app.config['MAX_CONTENT_LENGTH'] = 100 * 1024 * 1024 # 100MB limit
@@ -496,504 +488,3 @@ def create_app(config_name=None):
def get_socketio():
"""Get the SocketIO instance."""
return socketio
-
-
-_auto_sync_thread = None
-
-
-def _start_auto_sync_scheduler(app):
- """Start a background thread that checks for auto-sync schedules."""
- global _auto_sync_thread
- if _auto_sync_thread is not None:
- return
-
- import threading
- import time
- import logging
-
- logger = logging.getLogger(__name__)
-
- def auto_sync_loop():
- while True:
- try:
- time.sleep(60) # Check every 60 seconds
- with app.app_context():
- _check_auto_sync_schedules(logger)
- except Exception as e:
- logger.error(f'Auto-sync scheduler error: {e}')
-
- _auto_sync_thread = threading.Thread(
- target=auto_sync_loop,
- daemon=True,
- name='auto-sync-scheduler'
- )
- _auto_sync_thread.start()
-
-
-def _check_auto_sync_schedules(logger):
- """Check all auto-sync enabled sites and run syncs that are due."""
- from app.models.wordpress_site import WordPressSite
- from datetime import datetime
-
- sites = WordPressSite.query.filter_by(auto_sync_enabled=True).all()
- if not sites:
- return
-
- try:
- from croniter import croniter
- except ImportError:
- logger.debug('croniter not installed, skipping auto-sync check')
- return
-
- now = datetime.utcnow()
-
- for site in sites:
- if not site.auto_sync_schedule:
- continue
-
- try:
- if not croniter.is_valid(site.auto_sync_schedule):
- continue
-
- cron = croniter(site.auto_sync_schedule, now)
- prev_run = cron.get_prev(datetime)
-
- # Check if a run was due in the last 90 seconds (to account for check interval)
- seconds_since_due = (now - prev_run).total_seconds()
- if seconds_since_due <= 90:
- logger.info(f'Auto-sync triggered for site {site.id} ({site.name})')
- from app.services.environment_pipeline_service import EnvironmentPipelineService
- EnvironmentPipelineService.sync_from_production(
- env_site_id=site.id,
- sync_type='full',
- user_id=None
- )
- except Exception as e:
- logger.error(f'Auto-sync check failed for site {site.id}: {e}')
-
-
-_api_bg_thread = None
-
-
-def _start_pairing_pruner(app):
- """Start a background thread that prunes expired PendingAgent rows hourly."""
- global _pairing_prune_thread
- if _pairing_prune_thread is not None:
- return
-
- import threading
- import time
- import logging
-
- logger = logging.getLogger(__name__)
-
- def prune_loop():
- # Wait a bit before first run so app is fully initialized.
- time.sleep(60)
- while True:
- try:
- with app.app_context():
- from app.services import pairing_service
- pairing_service.prune_expired()
- except Exception as e:
- logger.error(f'Pairing pruner error: {e}')
- time.sleep(3600)
-
- _pairing_prune_thread = threading.Thread(
- target=prune_loop,
- daemon=True,
- name='pairing-pruner'
- )
- _pairing_prune_thread.start()
-
-
-_pairing_prune_thread = None
-
-
-_registrar_expiry_thread = None
-
-
-def _start_registrar_expiry_scheduler(app):
- """Daily check that notifies when a registrar domain crosses an expiry
- threshold (30/14/7/1 days, or expired). Single-worker only (module-global
- guard), like the other in-process schedulers."""
- global _registrar_expiry_thread
- if _registrar_expiry_thread is not None:
- return
-
- import threading
- import time
- import logging
-
- logger = logging.getLogger(__name__)
-
- def loop():
- time.sleep(300) # let startup settle
- while True:
- try:
- with app.app_context():
- from app.services.registrar_service import RegistrarService
- n = RegistrarService.notify_expiring()
- if n:
- logger.info(f'Registrar expiry: sent {n} notification(s)')
- except Exception as e:
- logger.error(f'Registrar expiry scheduler error: {e}')
- time.sleep(86400) # daily
-
- _registrar_expiry_thread = threading.Thread(
- target=loop, daemon=True, name='registrar-expiry')
- _registrar_expiry_thread.start()
-
-
-_snapshot_retention_thread = None
-
-
-def _start_snapshot_retention_scheduler(app):
- """Start a background thread that sets DatabaseSnapshot.expires_at per the
- retention policy and prunes expired snapshots (file + DB row) hourly."""
- global _snapshot_retention_thread
- if _snapshot_retention_thread is not None:
- return
-
- import threading
- import time
- import logging
-
- logger = logging.getLogger(__name__)
-
- def retention_loop():
- # Delay first run so the app is fully initialized.
- time.sleep(120)
- while True:
- try:
- with app.app_context():
- from app.services.db_sync_service import DatabaseSyncService
- from app.services.settings_service import SettingsService
- # Honor an admin-set override if present; otherwise use the
- # code default (no settings seed / migration required).
- days = SettingsService.get(
- 'snapshot_retention_days',
- DatabaseSyncService.DEFAULT_SNAPSHOT_RETENTION_DAYS,
- )
- try:
- days = int(days)
- except (TypeError, ValueError):
- days = DatabaseSyncService.DEFAULT_SNAPSHOT_RETENTION_DAYS
- result = DatabaseSyncService.prune_expired_snapshots(retention_days=days)
- if result.get('deleted') or result.get('backfilled'):
- logger.info(
- f"Snapshot retention: backfilled={result.get('backfilled', 0)} "
- f"deleted={result.get('deleted', 0)}"
- )
- except Exception as e:
- logger.error(f'Snapshot retention scheduler error: {e}')
- time.sleep(3600) # hourly
-
- _snapshot_retention_thread = threading.Thread(
- target=retention_loop,
- daemon=True,
- name='snapshot-retention',
- )
- _snapshot_retention_thread.start()
-
-
-def _start_api_background_threads(app):
- """Start background threads for API analytics aggregation and event delivery retry."""
- global _api_bg_thread
- if _api_bg_thread is not None:
- return
-
- import threading
- import time
- import logging
-
- logger = logging.getLogger(__name__)
-
- def api_bg_loop():
- while True:
- try:
- time.sleep(3600) # Run hourly
- with app.app_context():
- from app.services.api_analytics_service import ApiAnalyticsService
- ApiAnalyticsService.aggregate_hourly()
-
- from app.services.event_service import EventService
- EventService.retry_failed()
- except Exception as e:
- logger.error(f'API background thread error: {e}')
-
- _api_bg_thread = threading.Thread(
- target=api_bg_loop,
- daemon=True,
- name='api-background'
- )
- _api_bg_thread.start()
-
-
-_workflow_thread = None
-
-
-def _start_workflow_scheduler(app):
- """Start a background thread that checks for scheduled workflows."""
- global _workflow_thread
- if _workflow_thread is not None:
- return
-
- import threading
- import time
- import logging
-
- logger = logging.getLogger(__name__)
-
- def workflow_loop():
- while True:
- try:
- time.sleep(60) # Check every 60 seconds
- with app.app_context():
- _check_workflow_schedules(logger)
- except Exception as e:
- logger.error(f'Workflow scheduler error: {e}')
-
- _workflow_thread = threading.Thread(
- target=workflow_loop,
- daemon=True,
- name='workflow-scheduler'
- )
- _workflow_thread.start()
-
-
-def _check_workflow_schedules(logger):
- """Check all active workflows with cron triggers and run those that are due."""
- from app.models.workflow import Workflow
- from app.services.workflow_engine import WorkflowEngine
- from datetime import datetime
- import json
-
- try:
- from croniter import croniter
- except ImportError:
- logger.debug('croniter not installed, skipping workflow schedule check')
- return
-
- # Find active workflows with cron triggers
- workflows = Workflow.query.filter_by(is_active=True, trigger_type='cron').all()
- if not workflows:
- return
-
- now = datetime.utcnow()
-
- for workflow in workflows:
- try:
- config = json.loads(workflow.trigger_config) if workflow.trigger_config else {}
- cron_expr = config.get('cron')
-
- if not cron_expr or not croniter.is_valid(cron_expr):
- continue
-
- cron = croniter(cron_expr, now)
- prev_run = cron.get_prev(datetime)
-
- # Check if a run was due in the last 90 seconds
- seconds_since_due = (now - prev_run).total_seconds()
-
- # Also ensure we don't run it multiple times for the same slot
- if 0 < seconds_since_due <= 90:
- # Check if it already ran in the last 2 minutes
- if workflow.last_run_at:
- seconds_since_last_run = (now - workflow.last_run_at).total_seconds()
- if seconds_since_last_run < 110:
- continue
-
- logger.info(f'Scheduled workflow triggered: {workflow.name} (ID: {workflow.id})')
- WorkflowEngine.execute_workflow(
- workflow_id=workflow.id,
- trigger_type='cron',
- context={'scheduled_at': prev_run.isoformat()}
- )
- except Exception as e:
- logger.error(f'Workflow schedule check failed for workflow {workflow.id}: {e}')
-
-
-_health_check_thread = None
-
-# How often the per-site WordPress health poller runs (seconds).
-HEALTH_CHECK_INTERVAL = 300
-
-# Retention for recorded health-check samples (days) — bounds unbounded growth
-# from the continuous poller; matches the longest uptime window (uptime_90d).
-# Pruned at most once per day.
-HEALTH_CHECK_RETENTION_DAYS = 90
-_last_health_prune = None
-
-
-def _start_health_check_scheduler(app):
- """Start a background thread that polls every managed WordPress site's
- health on an interval. This keeps health_status fresh (so #27 transition
- alerts fire autonomously, not only while the health card is open) and drives
- any bound status-page components (#26): a real uptime % and auto-incidents.
-
- Single-worker only — like the other in-process schedulers, this must not be
- multiplied across Gunicorn workers (see CLAUDE.md). The module-global guard
- ensures one thread per process.
- """
- global _health_check_thread
- if _health_check_thread is not None:
- return
-
- import threading
- import time
- import logging
-
- logger = logging.getLogger(__name__)
-
- def health_loop():
- # Small initial delay so startup isn't slowed by blocking probes.
- time.sleep(30)
- while True:
- try:
- with app.app_context():
- _run_health_checks(logger)
- except Exception as e:
- logger.error(f'Health-check scheduler error: {e}')
- time.sleep(HEALTH_CHECK_INTERVAL)
-
- _health_check_thread = threading.Thread(
- target=health_loop,
- daemon=True,
- name='health-check-scheduler'
- )
- _health_check_thread.start()
-
-
-def _run_health_checks(logger):
- """Run a health check for every managed (production) WordPress site and sync
- any status-page components bound to it. Per-site try/except so one hung site
- never stalls the whole sweep."""
- from app import db
- from app.models.wordpress_site import WordPressSite
- from app.models.status_page import StatusComponent
- from app.services.environment_health_service import EnvironmentHealthService
- from app.services.status_page_service import StatusPageService
-
- _prune_old_health_checks(logger)
-
- sites = WordPressSite.query.filter_by(is_production=True).all()
- for site in sites:
- try:
- # Only poll sites the operator expects to be up — skip archived/stopped
- # stacks so an intentional stop never looks like an outage.
- if not site.application or site.application.status != 'running':
- continue
- result = EnvironmentHealthService.check_health(site.id)
- overall = result.get('overall_status')
- if not overall:
- continue
- # Drive any status-page components bound to this site.
- components = StatusComponent.query.filter_by(wordpress_site_id=site.id).all()
- for comp in components:
- StatusPageService.sync_component_from_health(comp, overall)
- except Exception as e:
- logger.error(f'Health check failed for site {site.id}: {e}')
- try:
- db.session.rollback()
- except Exception:
- pass
-
-
-def _prune_old_health_checks(logger):
- """Delete health-check samples older than the retention window, at most once
- per day, so the continuous poller doesn't grow the health_checks table without
- bound. Best-effort — failure never stalls the health sweep."""
- global _last_health_prune
- from datetime import datetime, timedelta
- now = datetime.utcnow()
- if _last_health_prune is not None and (now - _last_health_prune).total_seconds() < 86400:
- return
- from app import db
- from app.models.status_page import HealthCheck
- cutoff = now - timedelta(days=HEALTH_CHECK_RETENTION_DAYS)
- try:
- deleted = HealthCheck.query.filter(HealthCheck.checked_at < cutoff).delete(synchronize_session=False)
- db.session.commit()
- _last_health_prune = now
- if deleted:
- logger.info(f'Pruned {deleted} health-check row(s) older than {HEALTH_CHECK_RETENTION_DAYS}d')
- except Exception as e:
- logger.error(f'Health-check prune failed: {e}')
- try:
- db.session.rollback()
- except Exception:
- pass
-
-
-_update_scheduler_thread = None
-
-
-def _start_update_scheduler(app):
- """Background thread that runs due per-site WordPress auto-updates (safe-update
- with snapshot + health-check + auto-rollback). Single-worker only (module-global
- guard); each run itself runs in its own thread, so this loop only triggers them."""
- global _update_scheduler_thread
- if _update_scheduler_thread is not None:
- return
-
- import threading
- import time
- import logging
-
- logger = logging.getLogger(__name__)
-
- def loop():
- time.sleep(45) # let startup settle
- while True:
- try:
- time.sleep(60)
- with app.app_context():
- _check_update_schedules(logger)
- except Exception as e:
- logger.error(f'Update scheduler error: {e}')
-
- _update_scheduler_thread = threading.Thread(target=loop, daemon=True, name='wp-update-scheduler')
- _update_scheduler_thread.start()
-
-
-def _check_update_schedules(logger):
- from app.models.wordpress_site import WordPressSite, WordPressUpdateRun
- from app.services.wp_update_service import WpUpdateService
- from datetime import datetime
- import json as _json
-
- try:
- from croniter import croniter
- except ImportError:
- return
-
- sites = WordPressSite.query.filter(WordPressSite.auto_update_schedule.isnot(None)).all()
- if not sites:
- return
- now = datetime.utcnow()
- for site in sites:
- try:
- expr = (site.auto_update_schedule or '').strip()
- if not expr or not croniter.is_valid(expr):
- continue
- if not site.application or site.application.status != 'running':
- continue
- prev = croniter(expr, now).get_prev(datetime)
- if not (0 < (now - prev).total_seconds() <= 90):
- continue
- # de-dup: skip if a run already started in the last ~10 minutes
- last = (WordPressUpdateRun.query.filter_by(site_id=site.id)
- .order_by(WordPressUpdateRun.started_at.desc()).first())
- if last and last.started_at and (now - last.started_at).total_seconds() < 600:
- continue
- exclude = []
- if site.auto_update_exclude:
- try:
- exclude = _json.loads(site.auto_update_exclude)
- except Exception:
- exclude = []
- logger.info(f'Scheduled WordPress safe-update: site {site.id}')
- WpUpdateService.start_update(site, exclude=exclude, trigger='scheduled')
- except Exception as e:
- logger.error(f'Update schedule check failed for site {site.id}: {e}')
diff --git a/backend/app/api/jobs.py b/backend/app/api/jobs.py
new file mode 100644
index 00000000..7e3cd8bf
--- /dev/null
+++ b/backend/app/api/jobs.py
@@ -0,0 +1,120 @@
+"""Read/ops API for the unified job system — the single pane to observe all
+background work. Admin-gated, mirroring app/api/queue_bus.py conventions."""
+from flask import Blueprint, request, jsonify
+from flask_jwt_extended import jwt_required, get_jwt_identity
+
+from app.jobs import registry
+from app.jobs.service import JobService, ScheduledJobService
+
+jobs_bp = Blueprint('jobs', __name__)
+
+
+def _require_admin():
+ from app.models.user import User
+ user = User.query.get(get_jwt_identity())
+ if not user or not user.is_admin:
+ return None
+ return user
+
+
+# --- Static routes first; Werkzeug ranks these above the dynamic /<job_id>. ---
+@jobs_bp.route('', methods=['GET'])
+@jobs_bp.route('/', methods=['GET'])
+@jwt_required()
+def list_jobs():
+ if not _require_admin():
+ return jsonify({'error': 'Admin access required'}), 403
+ try:
+ limit = min(int(request.args.get('limit', 50)), 200)
+ offset = int(request.args.get('offset', 0))
+ except (TypeError, ValueError):
+ limit, offset = 50, 0
+ jobs = JobService.list(
+ status=request.args.get('status'),
+ kind=request.args.get('kind'),
+ owner_type=request.args.get('owner_type'),
+ owner_id=request.args.get('owner_id'),
+ limit=limit,
+ offset=offset,
+ )
+ return jsonify({'jobs': [j.to_dict() for j in jobs]})
+
+
+@jobs_bp.route('/stats', methods=['GET'])
+@jwt_required()
+def job_stats():
+ if not _require_admin():
+ return jsonify({'error': 'Admin access required'}), 403
+ return jsonify(JobService.stats())
+
+
+@jobs_bp.route('/kinds', methods=['GET'])
+@jwt_required()
+def job_kinds():
+ if not _require_admin():
+ return jsonify({'error': 'Admin access required'}), 403
+ return jsonify({'kinds': registry.registered_kinds()})
+
+
+@jobs_bp.route('/scheduled', methods=['GET'])
+@jwt_required()
+def list_scheduled():
+ if not _require_admin():
+ return jsonify({'error': 'Admin access required'}), 403
+ return jsonify({'scheduled': [s.to_dict() for s in ScheduledJobService.list()]})
+
+
+@jobs_bp.route('/scheduled/<int:scheduled_id>/run', methods=['POST'])
+@jwt_required()
+def run_scheduled(scheduled_id):
+ if not _require_admin():
+ return jsonify({'error': 'Admin access required'}), 403
+ job = ScheduledJobService.run_now(scheduled_id)
+ if not job:
+ return jsonify({'error': 'Scheduled job not found'}), 404
+ return jsonify({'job': job.to_dict()})
+
+
+@jobs_bp.route('/scheduled/<int:scheduled_id>/enabled', methods=['POST'])
+@jwt_required()
+def toggle_scheduled(scheduled_id):
+ if not _require_admin():
+ return jsonify({'error': 'Admin access required'}), 403
+ body = request.get_json(silent=True) or {}
+ scheduled = ScheduledJobService.set_enabled(scheduled_id, bool(body.get('enabled', True)))
+ if not scheduled:
+ return jsonify({'error': 'Scheduled job not found'}), 404
+ return jsonify({'scheduled': scheduled.to_dict()})
+
+
+@jobs_bp.route('/<job_id>', methods=['GET'])
+@jwt_required()
+def get_job(job_id):
+ if not _require_admin():
+ return jsonify({'error': 'Admin access required'}), 403
+ job = JobService.get(job_id)
+ if not job:
+ return jsonify({'error': 'Job not found'}), 404
+ return jsonify({'job': job.to_dict(include_payload=True)})
+
+
+@jobs_bp.route('/<job_id>/cancel', methods=['POST'])
+@jwt_required()
+def cancel_job(job_id):
+ if not _require_admin():
+ return jsonify({'error': 'Admin access required'}), 403
+ job = JobService.cancel(job_id)
+ if not job:
+ return jsonify({'error': 'Job not found'}), 404
+ return jsonify({'job': job.to_dict()})
+
+
+@jobs_bp.route('/<job_id>/retry', methods=['POST'])
+@jwt_required()
+def retry_job(job_id):
+ if not _require_admin():
+ return jsonify({'error': 'Admin access required'}), 403
+ job = JobService.retry(job_id)
+ if not job:
+ return jsonify({'error': 'Job not found'}), 404
+ return jsonify({'job': job.to_dict()})
diff --git a/backend/app/api/performance.py b/backend/app/api/performance.py
index e41eeeff..623f6b5b 100644
--- a/backend/app/api/performance.py
+++ b/backend/app/api/performance.py
@@ -1,7 +1,7 @@
from flask import Blueprint, request, jsonify
from flask_jwt_extended import jwt_required
from app.services.cache_service import CacheService
-from app.services.background_job_service import BackgroundJobService
+from app.jobs.service import JobService
performance_bp = Blueprint('performance', __name__)
@@ -37,16 +37,17 @@ def list_jobs():
user = get_current_user()
if not user or not user.is_admin:
return jsonify({'error': 'Admin access required'}), 403
- return jsonify({'jobs': BackgroundJobService.list_jobs()})
+ # Keyed by id to preserve the prior {job_id: info} response shape.
+ return jsonify({'jobs': {j.id: j.to_dict() for j in JobService.list(limit=200)}})
@performance_bp.route('/jobs/<job_id>', methods=['GET'])
@jwt_required()
def get_job(job_id):
- status = BackgroundJobService.get_job_status(job_id)
- if not status:
+ job = JobService.get(job_id)
+ if not job:
return jsonify({'error': 'Job not found'}), 404
- return jsonify({'job_id': job_id, **status})
+ return jsonify({'job_id': job_id, **job.to_dict(include_payload=True)})
@performance_bp.route('/jobs/stats', methods=['GET'])
@@ -55,7 +56,14 @@ def job_stats():
user = get_current_user()
if not user or not user.is_admin:
return jsonify({'error': 'Admin access required'}), 403
- return jsonify(BackgroundJobService.get_queue_stats())
+ stats = JobService.stats()
+ return jsonify({
+ 'total_jobs': stats['total'],
+ 'by_status': stats['by_status'],
+ 'by_kind': stats['by_kind'],
+ 'running': stats['by_status'].get('running', 0),
+ 'queue_size': stats['by_status'].get('pending', 0),
+ })
@performance_bp.route('/jobs/cleanup', methods=['POST'])
@@ -64,5 +72,5 @@ def cleanup_jobs():
user = get_current_user()
if not user or not user.is_admin:
return jsonify({'error': 'Admin access required'}), 403
- BackgroundJobService.cleanup_old()
- return jsonify({'message': 'Old jobs cleaned up'})
+ deleted = JobService.cleanup_old()
+ return jsonify({'message': f'Cleaned up {deleted} old job(s)'})
diff --git a/backend/app/jobs/__init__.py b/backend/app/jobs/__init__.py
new file mode 100644
index 00000000..201f67f3
--- /dev/null
+++ b/backend/app/jobs/__init__.py
@@ -0,0 +1,48 @@
+"""ServerKit unified job system.
+
+A thin Job / WorkUnit + scheduler layer on top of the Queue Bus. A producer
+calls ``JobService.enqueue(kind, payload)``; that persists a ``Job`` row and
+publishes a thin ``{'job_id': ...}`` message onto the ``serverkit-system/jobs``
+queue. One ``JobConsumer`` runs every job by dispatching its ``kind`` to a
+registered handler, and one ``JobScheduler`` ticks all periodic work. This is
+the same pattern the Notification Bus uses, generalized.
+
+See ``docs/ORCHESTRATION_ROADMAP.md`` (local, git-ignored) for the plan.
+"""
+# Import the models at package-import time so they register on ``db.metadata``
+# and ``MigrationService._fix_missing_columns`` -> ``db.create_all()`` picks them
+# up on boot — mirrors how ``app/queue_bus`` registers its models. No Alembic
+# migration is required for new tables.
+from app.jobs.models import Job, ScheduledJob # noqa: F401
+from app.jobs import registry # noqa: F401
+
+__all__ = ['Job', 'ScheduledJob', 'registry', 'start_job_system']
+
+
+def start_job_system(app, seed=None):
+ """Ensure the jobs queue exists, seed built-in schedules, and start the
+ consumer + scheduler daemon threads.
+
+ No-op under testing config (mirrors the queue-bus / notification consumers),
+ so the suite drives ``JobConsumer.process_message`` / ``JobScheduler.tick``
+ directly instead of racing background threads against per-test databases.
+ """
+ if app.config.get('ENV') == 'testing' or app.config.get('TESTING'):
+ return
+
+ import logging
+ from app.queue_bus.service import QueueBusService
+ from app.jobs.service import GROUP_SLUG, QUEUE_SLUG, QUEUE_CONFIG
+ from app.jobs.consumer import start_job_consumer
+ from app.jobs.scheduler import start_job_scheduler
+
+ with app.app_context():
+ QueueBusService.ensure_queue(GROUP_SLUG, QUEUE_SLUG, config=QUEUE_CONFIG)
+ if seed is not None:
+ try:
+ seed()
+ except Exception:
+ logging.getLogger(__name__).exception('Built-in schedule seeding failed')
+
+ start_job_consumer(app)
+ start_job_scheduler(app)
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.py
new file mode 100644
index 00000000..f4ae1406
--- /dev/null
+++ b/backend/app/jobs/builtin_handlers.py
@@ -0,0 +1,290 @@
+"""Built-in periodic job handlers + their schedules.
+
+These are the platform's own recurring tasks. They used to each run in a
+dedicated daemon thread spawned from ``app/__init__.py`` (auto-sync,
+snapshot-retention, workflow scheduler, health-check poller, WP safe-update,
+hourly API background, pairing pruner, registrar expiry). They now run as
+``ScheduledJob`` rows enqueued by the single ``JobScheduler`` and executed by the
+single ``JobConsumer``.
+
+The check/run functions below are relocated verbatim from ``app/__init__.py`` so
+behavior — cadence handling, per-task de-dup, idempotency — is unchanged; only
+the trigger moved from a bare thread to the unified job system.
+"""
+import logging
+
+from app.jobs.registry import register
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Relocated check/run functions (formerly in app/__init__.py)
+# ---------------------------------------------------------------------------
+def check_auto_sync_schedules():
+ """Check all auto-sync enabled sites and run syncs that are due."""
+ from app.models.wordpress_site import WordPressSite
+ from datetime import datetime
+
+ sites = WordPressSite.query.filter_by(auto_sync_enabled=True).all()
+ if not sites:
+ return
+
+ try:
+ from croniter import croniter
+ except ImportError:
+ logger.debug('croniter not installed, skipping auto-sync check')
+ return
+
+ now = datetime.utcnow()
+
+ for site in sites:
+ if not site.auto_sync_schedule:
+ continue
+ try:
+ if not croniter.is_valid(site.auto_sync_schedule):
+ continue
+ cron = croniter(site.auto_sync_schedule, now)
+ prev_run = cron.get_prev(datetime)
+ # Was a run due in the last 90 seconds (accounts for the tick interval)?
+ seconds_since_due = (now - prev_run).total_seconds()
+ if seconds_since_due <= 90:
+ logger.info(f'Auto-sync triggered for site {site.id} ({site.name})')
+ from app.services.environment_pipeline_service import EnvironmentPipelineService
+ EnvironmentPipelineService.sync_from_production(
+ env_site_id=site.id,
+ sync_type='full',
+ user_id=None,
+ )
+ except Exception as e:
+ logger.error(f'Auto-sync check failed for site {site.id}: {e}')
+
+
+def check_workflow_schedules():
+ """Check all active workflows with cron triggers and run those that are due."""
+ from app.models.workflow import Workflow
+ from app.services.workflow_engine import WorkflowEngine
+ from datetime import datetime
+ import json
+
+ try:
+ from croniter import croniter
+ except ImportError:
+ logger.debug('croniter not installed, skipping workflow schedule check')
+ return
+
+ workflows = Workflow.query.filter_by(is_active=True, trigger_type='cron').all()
+ if not workflows:
+ return
+
+ now = datetime.utcnow()
+
+ for workflow in workflows:
+ try:
+ config = json.loads(workflow.trigger_config) if workflow.trigger_config else {}
+ cron_expr = config.get('cron')
+ if not cron_expr or not croniter.is_valid(cron_expr):
+ continue
+ cron = croniter(cron_expr, now)
+ prev_run = cron.get_prev(datetime)
+ seconds_since_due = (now - prev_run).total_seconds()
+ # Don't run multiple times for the same slot.
+ if 0 < seconds_since_due <= 90:
+ if workflow.last_run_at:
+ seconds_since_last_run = (now - workflow.last_run_at).total_seconds()
+ if seconds_since_last_run < 110:
+ continue
+ logger.info(f'Scheduled workflow triggered: {workflow.name} (ID: {workflow.id})')
+ WorkflowEngine.execute_workflow(
+ workflow_id=workflow.id,
+ trigger_type='cron',
+ context={'scheduled_at': prev_run.isoformat()},
+ )
+ except Exception as e:
+ logger.error(f'Workflow schedule check failed for workflow {workflow.id}: {e}')
+
+
+# How often the per-site WordPress health poller runs (seconds).
+HEALTH_CHECK_INTERVAL = 300
+# Retention for recorded health-check samples (days) — bounds unbounded growth
+# from the continuous poller; matches the longest uptime window (uptime_90d).
+# Pruned at most once per day.
+HEALTH_CHECK_RETENTION_DAYS = 90
+_last_health_prune = None
+
+
+def run_health_checks():
+ """Run a health check for every managed (production) WordPress site and sync
+ any status-page components bound to it. Per-site try/except so one hung site
+ never stalls the whole sweep."""
+ from app import db
+ from app.models.wordpress_site import WordPressSite
+ from app.models.status_page import StatusComponent
+ from app.services.environment_health_service import EnvironmentHealthService
+ from app.services.status_page_service import StatusPageService
+
+ _prune_old_health_checks()
+
+ sites = WordPressSite.query.filter_by(is_production=True).all()
+ for site in sites:
+ try:
+ # Only poll sites the operator expects to be up — skip archived/stopped
+ # stacks so an intentional stop never looks like an outage.
+ if not site.application or site.application.status != 'running':
+ continue
+ result = EnvironmentHealthService.check_health(site.id)
+ overall = result.get('overall_status')
+ if not overall:
+ continue
+ components = StatusComponent.query.filter_by(wordpress_site_id=site.id).all()
+ for comp in components:
+ StatusPageService.sync_component_from_health(comp, overall)
+ except Exception as e:
+ logger.error(f'Health check failed for site {site.id}: {e}')
+ try:
+ db.session.rollback()
+ except Exception:
+ pass
+
+
+def _prune_old_health_checks():
+ """Delete health-check samples older than the retention window, at most once
+ per day, so the continuous poller doesn't grow the health_checks table without
+ bound. Best-effort — failure never stalls the health sweep."""
+ global _last_health_prune
+ from datetime import datetime, timedelta
+ now = datetime.utcnow()
+ if _last_health_prune is not None and (now - _last_health_prune).total_seconds() < 86400:
+ return
+ from app import db
+ from app.models.status_page import HealthCheck
+ cutoff = now - timedelta(days=HEALTH_CHECK_RETENTION_DAYS)
+ try:
+ deleted = HealthCheck.query.filter(HealthCheck.checked_at < cutoff).delete(synchronize_session=False)
+ db.session.commit()
+ _last_health_prune = now
+ if deleted:
+ logger.info(f'Pruned {deleted} health-check row(s) older than {HEALTH_CHECK_RETENTION_DAYS}d')
+ except Exception as e:
+ logger.error(f'Health-check prune failed: {e}')
+ try:
+ db.session.rollback()
+ except Exception:
+ pass
+
+
+def check_update_schedules():
+ from app.models.wordpress_site import WordPressSite, WordPressUpdateRun
+ from app.services.wp_update_service import WpUpdateService
+ from datetime import datetime
+ import json as _json
+
+ try:
+ from croniter import croniter
+ except ImportError:
+ return
+
+ sites = WordPressSite.query.filter(WordPressSite.auto_update_schedule.isnot(None)).all()
+ if not sites:
+ return
+ now = datetime.utcnow()
+ for site in sites:
+ try:
+ expr = (site.auto_update_schedule or '').strip()
+ if not expr or not croniter.is_valid(expr):
+ continue
+ if not site.application or site.application.status != 'running':
+ continue
+ prev = croniter(expr, now).get_prev(datetime)
+ if not (0 < (now - prev).total_seconds() <= 90):
+ continue
+ # de-dup: skip if a run already started in the last ~10 minutes
+ last = (WordPressUpdateRun.query.filter_by(site_id=site.id)
+ .order_by(WordPressUpdateRun.started_at.desc()).first())
+ if last and last.started_at and (now - last.started_at).total_seconds() < 600:
+ continue
+ exclude = []
+ if site.auto_update_exclude:
+ try:
+ exclude = _json.loads(site.auto_update_exclude)
+ except Exception:
+ exclude = []
+ logger.info(f'Scheduled WordPress safe-update: site {site.id}')
+ WpUpdateService.start_update(site, exclude=exclude, trigger='scheduled')
+ except Exception as e:
+ logger.error(f'Update schedule check failed for site {site.id}: {e}')
+
+
+def run_snapshot_retention():
+ """Set DatabaseSnapshot.expires_at per the retention policy and prune expired
+ snapshots (file + DB row)."""
+ from app.services.db_sync_service import DatabaseSyncService
+ from app.services.settings_service import SettingsService
+ days = SettingsService.get(
+ 'snapshot_retention_days',
+ DatabaseSyncService.DEFAULT_SNAPSHOT_RETENTION_DAYS,
+ )
+ try:
+ days = int(days)
+ except (TypeError, ValueError):
+ days = DatabaseSyncService.DEFAULT_SNAPSHOT_RETENTION_DAYS
+ result = DatabaseSyncService.prune_expired_snapshots(retention_days=days)
+ return result if isinstance(result, dict) else None
+
+
+def run_pairing_prune():
+ """Prune expired pending agent pairings."""
+ from app.services import pairing_service
+ pairing_service.prune_expired()
+
+
+def run_registrar_expiry():
+ """Notify when a registrar domain crosses an expiry threshold."""
+ from app.services.registrar_service import RegistrarService
+ n = RegistrarService.notify_expiring()
+ if n:
+ logger.info(f'Registrar expiry: sent {n} notification(s)')
+ return {'notified': n}
+ return None
+
+
+def run_api_background():
+ """Hourly API analytics aggregation + event delivery retry."""
+ from app.services.api_analytics_service import ApiAnalyticsService
+ from app.services.event_service import EventService
+ ApiAnalyticsService.aggregate_hourly()
+ EventService.retry_failed()
+
+
+# ---------------------------------------------------------------------------
+# Handler registration + schedule seeding
+# ---------------------------------------------------------------------------
+# (kind, handler, schedule-name, interval seconds, startup-delay seconds)
+# The interval/delay pairs reproduce the original per-thread cadence: each
+# former loop's sleep(settle) + sleep(interval) maps to startup_delay + interval.
+_BUILTINS = [
+ ('builtin.auto_sync', check_auto_sync_schedules, 'auto-sync', 60, 60),
+ ('builtin.snapshot_retention', run_snapshot_retention, 'snapshot-retention', 3600, 120),
+ ('builtin.workflow_schedules', check_workflow_schedules, 'workflow-schedules', 60, 60),
+ ('builtin.health_check', run_health_checks, 'health-check', 300, 30),
+ ('builtin.wp_update', check_update_schedules, 'wp-update', 60, 105),
+ ('builtin.api_background', run_api_background, 'api-background', 3600, 3600),
+ ('builtin.pairing_prune', run_pairing_prune, 'pairing-prune', 3600, 60),
+ ('builtin.registrar_expiry', run_registrar_expiry, 'registrar-expiry', 86400, 300),
+]
+
+
+def register_builtin_handlers():
+ """Register all built-in periodic handlers. Pure in-memory; idempotent."""
+ for kind, fn, _name, _interval, _delay in _BUILTINS:
+ # Wrap so handlers match the fn(job) signature and ignore the job arg.
+ register(kind, (lambda f: (lambda job: f()))(fn), replace=True)
+
+
+def seed_builtin_schedules():
+ """Idempotently create the ScheduledJob rows for the built-in tasks."""
+ from app.jobs.service import ScheduledJobService
+ for kind, _fn, name, interval, delay in _BUILTINS:
+ ScheduledJobService.ensure(
+ name, kind, interval_seconds=interval, startup_delay_seconds=delay,
+ )
diff --git a/backend/app/jobs/consumer.py b/backend/app/jobs/consumer.py
new file mode 100644
index 00000000..70061482
--- /dev/null
+++ b/backend/app/jobs/consumer.py
@@ -0,0 +1,182 @@
+"""The generic job consumer — the single worker that runs every enqueued Job.
+
+Mirrors the webhook / notification consumers: a daemon thread polls the
+``serverkit-system/jobs`` queue, but the per-message work is delegated to the
+handler registered for the job's ``kind``. Retry / backoff / dead-lettering come
+from the Queue Bus; this consumer maps the queue outcome back onto the Job row.
+"""
+import logging
+import threading
+import time
+from datetime import datetime
+
+from app import db
+from app.jobs import registry
+from app.jobs.models import Job
+from app.jobs.service import GROUP_SLUG, QUEUE_SLUG, QUEUE_CONFIG
+from app.queue_bus.service import QueueBusService
+
+logger = logging.getLogger(__name__)
+
+_job_consumer = None
+
+
+class JobConsumer:
+ """Polls the queue bus for job messages and runs each via its handler."""
+
+ def __init__(self, app=None, poll_interval_seconds=1, batch_size=5):
+ self.app = app
+ self.running = False
+ self.poll_interval_seconds = poll_interval_seconds
+ self.batch_size = batch_size
+
+ def start(self):
+ if self.running:
+ return
+ self.running = True
+ self._ensure_queue()
+ thread = threading.Thread(target=self._run, daemon=True, name='job-consumer')
+ thread.start()
+ logger.info('Job consumer started')
+
+ def stop(self):
+ self.running = False
+
+ def _ensure_queue(self):
+ with self.app.app_context():
+ QueueBusService.ensure_queue(GROUP_SLUG, QUEUE_SLUG, config=QUEUE_CONFIG)
+
+ def _run(self):
+ while self.running:
+ try:
+ with self.app.app_context():
+ self.process_batch()
+ except Exception as e:
+ logger.error(f'Job consumer error: {e}')
+ time.sleep(self.poll_interval_seconds)
+
+ def process_batch(self):
+ messages = QueueBusService.receive(
+ GROUP_SLUG, QUEUE_SLUG,
+ visibility_timeout_ms=QUEUE_CONFIG['visibility_timeout_ms'],
+ max_messages=self.batch_size,
+ )
+ for message in messages:
+ try:
+ self.process_message(message)
+ except Exception as e: # pragma: no cover - defensive, per-message isolation
+ logger.error(f'Job message {message.get("id")} crashed the consumer: {e}')
+ return len(messages)
+
+ def process_message(self, message):
+ """Run one job. Safe to call directly (the test-suite does)."""
+ payload = message.get('payload') or {}
+ job_id = payload.get('job_id')
+ if not job_id:
+ QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, message['id'])
+ return
+
+ job = Job.query.get(job_id)
+ if not job:
+ QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, message['id'])
+ return
+
+ # Honor a cancellation requested before the job was picked up.
+ if job.status == Job.STATUS_CANCELLED:
+ QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, message['id'])
+ return
+
+ handler = registry.get(job.kind)
+ if handler is None:
+ # An unroutable job can never succeed — fail it without burning
+ # retries, and complete the message so it doesn't loop.
+ job.status = Job.STATUS_FAILED
+ job.error_message = f'No handler registered for kind {job.kind!r}'
+ job.completed_at = datetime.utcnow()
+ db.session.commit()
+ QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, message['id'])
+ self._emit(job, 'job.failed')
+ return
+
+ job.status = Job.STATUS_RUNNING
+ job.started_at = job.started_at or datetime.utcnow()
+ job.attempts = message.get('attempts') or job.attempts or 0
+ job.queue_message_id = message['id']
+ db.session.commit()
+
+ try:
+ result = handler(job)
+ job = Job.query.get(job_id) # handler may have committed/expired the row
+ job.set_result(result)
+ job.status = Job.STATUS_SUCCEEDED
+ job.completed_at = datetime.utcnow()
+ job.error_message = None
+ db.session.commit()
+ QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, message['id'])
+ self._emit(job, 'job.succeeded')
+ except Exception as e:
+ logger.error(f'Job {job_id} ({job.kind}) failed: {e}')
+ try:
+ db.session.rollback()
+ except Exception:
+ pass
+ self._fail(job_id, message, str(e))
+
+ def _fail(self, job_id, message, error):
+ outcome = None
+ try:
+ outcome = QueueBusService.fail(
+ GROUP_SLUG, QUEUE_SLUG, message['id'], error_message=(error or '')[:500],
+ )
+ except Exception as e:
+ logger.error(f'Failed to mark job message failed: {e}')
+
+ job = Job.query.get(job_id)
+ if job is None:
+ return
+ job.error_message = (error or '')[:2000]
+ if outcome and outcome.get('status') == 'dead_letter':
+ job.status = Job.STATUS_FAILED
+ job.completed_at = datetime.utcnow()
+ db.session.commit()
+ self._emit(job, 'job.failed')
+ else:
+ # The queue will redeliver after a backoff; reflect that as pending.
+ job.status = Job.STATUS_PENDING
+ db.session.commit()
+
+ @staticmethod
+ def _emit(job, event_type):
+ """Best-effort telemetry — never let observability break a job."""
+ try:
+ from app.services.telemetry_service import TelemetryService
+ TelemetryService.emit(
+ source='jobs',
+ event_type=event_type,
+ message=f'Job {event_type.split(".")[-1]}: {job.kind}',
+ severity='error' if event_type == 'job.failed' else 'info',
+ correlation_id=job.correlation_id,
+ payload={'job_id': job.id, 'kind': job.kind, 'attempts': job.attempts},
+ commit=True,
+ )
+ except Exception:
+ pass
+
+
+def start_job_consumer(app):
+ """Start the singleton job consumer thread (skipped under testing config)."""
+ global _job_consumer
+ if _job_consumer is not None:
+ return
+ if app.config.get('ENV') == 'testing' or app.config.get('TESTING'):
+ return
+ consumer = JobConsumer(app)
+ consumer.start()
+ _job_consumer = consumer
+
+
+def stop_job_consumer():
+ global _job_consumer
+ if _job_consumer is not None:
+ _job_consumer.stop()
+ _job_consumer = None
diff --git a/backend/app/jobs/models.py b/backend/app/jobs/models.py
new file mode 100644
index 00000000..d310a2e5
--- /dev/null
+++ b/backend/app/jobs/models.py
@@ -0,0 +1,178 @@
+"""Models for the unified background-job system.
+
+``Job`` is a persisted unit of work with a stable lifecycle
+(``pending -> running -> succeeded | failed | cancelled``). The Queue Bus stays
+the transport: enqueuing a Job publishes a thin ``{'job_id': ...}`` message; the
+``JobConsumer`` loads the row and dispatches by ``kind``. Retries / backoff /
+dead-lettering are inherited from the queue — the Job row mirrors the outcome so
+there is a single place to observe all background work.
+
+``ScheduledJob`` is the periodic side: one ticker enqueues a Job for each due
+schedule, replacing the per-domain daemon threads that each ran their own
+``while True: sleep`` loop.
+"""
+from datetime import datetime, timedelta
+import json
+import uuid
+
+from app import db
+
+
+class Job(db.Model):
+ __tablename__ = 'jobs'
+
+ STATUS_PENDING = 'pending'
+ STATUS_RUNNING = 'running'
+ STATUS_SUCCEEDED = 'succeeded'
+ STATUS_FAILED = 'failed'
+ STATUS_CANCELLED = 'cancelled'
+
+ TERMINAL_STATUSES = (STATUS_SUCCEEDED, STATUS_FAILED, STATUS_CANCELLED)
+
+ id = db.Column(db.String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+ kind = db.Column(db.String(80), nullable=False, index=True)
+ status = db.Column(db.String(20), default=STATUS_PENDING, nullable=False, index=True)
+
+ payload = db.Column(db.Text, default='{}')
+ result = db.Column(db.Text, nullable=True)
+ error_message = db.Column(db.Text, nullable=True)
+
+ attempts = db.Column(db.Integer, default=0, nullable=False)
+ max_attempts = db.Column(db.Integer, default=3, nullable=False)
+ priority = db.Column(db.Integer, default=0)
+
+ # Loose ownership for filtering ("jobs for this server / app / user / schedule").
+ owner_type = db.Column(db.String(40), nullable=True, index=True)
+ owner_id = db.Column(db.String(64), nullable=True, index=True)
+
+ # Backreference to the schedule that spawned this job, if periodic (no FK so
+ # table create-order never matters).
+ scheduled_job_id = db.Column(db.Integer, nullable=True, index=True)
+
+ correlation_id = db.Column(db.String(64), nullable=True, index=True)
+ # The queue message currently carrying this job (for ops / debugging).
+ queue_message_id = db.Column(db.String(36), nullable=True)
+
+ scheduled_at = db.Column(db.DateTime, nullable=True) # requested delay target
+ created_at = db.Column(db.DateTime, default=datetime.utcnow, index=True)
+ started_at = db.Column(db.DateTime, nullable=True)
+ completed_at = db.Column(db.DateTime, nullable=True)
+ updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+ def get_payload(self):
+ try:
+ return json.loads(self.payload) if self.payload else {}
+ except (TypeError, json.JSONDecodeError):
+ return {}
+
+ def set_payload(self, payload):
+ self.payload = json.dumps(payload or {})
+
+ def get_result(self):
+ try:
+ return json.loads(self.result) if self.result else None
+ except (TypeError, json.JSONDecodeError):
+ return None
+
+ def set_result(self, result):
+ """Store a JSON-serializable result; fall back to a repr so a weird
+ return value never blocks marking a job done."""
+ if result is None:
+ self.result = None
+ return
+ try:
+ self.result = json.dumps(result)
+ except (TypeError, ValueError):
+ self.result = json.dumps({'repr': str(result)[:2000]})
+
+ @property
+ def duration(self):
+ if self.started_at and self.completed_at:
+ return (self.completed_at - self.started_at).total_seconds()
+ return None
+
+ def to_dict(self, include_payload=False):
+ data = {
+ 'id': self.id,
+ 'kind': self.kind,
+ 'status': self.status,
+ 'attempts': self.attempts,
+ 'max_attempts': self.max_attempts,
+ 'priority': self.priority,
+ 'owner_type': self.owner_type,
+ 'owner_id': self.owner_id,
+ 'scheduled_job_id': self.scheduled_job_id,
+ 'correlation_id': self.correlation_id,
+ 'result': self.get_result(),
+ 'error_message': self.error_message,
+ 'created_at': self.created_at.isoformat() if self.created_at else None,
+ 'started_at': self.started_at.isoformat() if self.started_at else None,
+ 'completed_at': self.completed_at.isoformat() if self.completed_at else None,
+ 'updated_at': self.updated_at.isoformat() if self.updated_at else None,
+ 'duration': self.duration,
+ }
+ if include_payload:
+ data['payload'] = self.get_payload()
+ return data
+
+
+class ScheduledJob(db.Model):
+ __tablename__ = 'scheduled_jobs'
+
+ SCHEDULE_INTERVAL = 'interval'
+ SCHEDULE_CRON = 'cron'
+
+ id = db.Column(db.Integer, primary_key=True, autoincrement=True)
+ name = db.Column(db.String(80), unique=True, nullable=False) # stable upsert key
+ kind = db.Column(db.String(80), nullable=False) # job kind to enqueue
+ schedule_kind = db.Column(db.String(20), default=SCHEDULE_INTERVAL, nullable=False)
+ interval_seconds = db.Column(db.Integer, nullable=True)
+ cron = db.Column(db.String(120), nullable=True)
+ payload = db.Column(db.Text, default='{}')
+ max_attempts = db.Column(db.Integer, default=1, nullable=False)
+ enabled = db.Column(db.Boolean, default=True, nullable=False)
+
+ next_run_at = db.Column(db.DateTime, nullable=True, index=True)
+ last_run_at = db.Column(db.DateTime, nullable=True)
+ last_job_id = db.Column(db.String(36), nullable=True)
+
+ created_at = db.Column(db.DateTime, default=datetime.utcnow)
+ updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+ def get_payload(self):
+ try:
+ return json.loads(self.payload) if self.payload else {}
+ except (TypeError, json.JSONDecodeError):
+ return {}
+
+ def set_payload(self, payload):
+ self.payload = json.dumps(payload or {})
+
+ def compute_next_run(self, base=None):
+ base = base or datetime.utcnow()
+ if self.schedule_kind == self.SCHEDULE_CRON and self.cron:
+ try:
+ from croniter import croniter
+ if croniter.is_valid(self.cron):
+ return croniter(self.cron, base).get_next(datetime)
+ except ImportError:
+ pass
+ # Cron invalid / croniter unavailable — back off an hour rather than
+ # hot-looping.
+ return base + timedelta(hours=1)
+ return base + timedelta(seconds=self.interval_seconds or 3600)
+
+ def to_dict(self):
+ return {
+ 'id': self.id,
+ 'name': self.name,
+ 'kind': self.kind,
+ 'schedule_kind': self.schedule_kind,
+ 'interval_seconds': self.interval_seconds,
+ 'cron': self.cron,
+ 'enabled': self.enabled,
+ 'max_attempts': self.max_attempts,
+ 'next_run_at': self.next_run_at.isoformat() if self.next_run_at else None,
+ 'last_run_at': self.last_run_at.isoformat() if self.last_run_at else None,
+ 'last_job_id': self.last_job_id,
+ }
diff --git a/backend/app/jobs/registry.py b/backend/app/jobs/registry.py
new file mode 100644
index 00000000..bcc9279c
--- /dev/null
+++ b/backend/app/jobs/registry.py
@@ -0,0 +1,48 @@
+"""In-process registry mapping a job ``kind`` to its handler.
+
+A handler is a callable ``fn(job) -> result`` where ``job`` is the ``Job`` ORM
+row. The return value (JSON-serializable, or ``None``) is stored as the job
+result. Raising propagates to the consumer, which records the error and lets the
+Queue Bus retry / dead-letter.
+"""
+import logging
+
+logger = logging.getLogger(__name__)
+
+_HANDLERS = {}
+
+
+def register(kind, handler, replace=False):
+ """Register ``handler`` for ``kind``. Existing kinds are kept unless
+ ``replace=True`` (so a plugin reload re-registers cleanly)."""
+ if not kind or not callable(handler):
+ raise ValueError('register(kind, handler): kind required and handler must be callable')
+ if kind in _HANDLERS and not replace:
+ logger.debug('Job handler for %r already registered; keeping existing', kind)
+ return
+ _HANDLERS[kind] = handler
+
+
+def handler(kind, replace=False):
+ """Decorator form of :func:`register`."""
+ def _decorator(fn):
+ register(kind, fn, replace=replace)
+ return fn
+ return _decorator
+
+
+def get(kind):
+ return _HANDLERS.get(kind)
+
+
+def is_registered(kind):
+ return kind in _HANDLERS
+
+
+def registered_kinds():
+ return sorted(_HANDLERS.keys())
+
+
+def clear():
+ """Test helper — drop all registered handlers."""
+ _HANDLERS.clear()
diff --git a/backend/app/jobs/scheduler.py b/backend/app/jobs/scheduler.py
new file mode 100644
index 00000000..295d03da
--- /dev/null
+++ b/backend/app/jobs/scheduler.py
@@ -0,0 +1,79 @@
+"""The single periodic ticker that replaces the per-domain scheduler threads.
+
+It wakes on a short interval, finds due ``ScheduledJob`` rows, and enqueues a Job
+for each (advancing ``next_run_at``). All cadence lives in the DB, so adding or
+disabling a periodic task is a row change, not a new daemon thread.
+"""
+import logging
+import threading
+import time
+
+from app.jobs.service import ScheduledJobService
+
+logger = logging.getLogger(__name__)
+
+_job_scheduler = None
+
+
+class JobScheduler:
+ def __init__(self, app=None, tick_seconds=15):
+ self.app = app
+ self.running = False
+ self.tick_seconds = tick_seconds
+
+ def start(self):
+ if self.running:
+ return
+ self.running = True
+ thread = threading.Thread(target=self._run, daemon=True, name='job-scheduler')
+ thread.start()
+ logger.info('Job scheduler started')
+
+ def stop(self):
+ self.running = False
+
+ def _run(self):
+ while self.running:
+ try:
+ with self.app.app_context():
+ self.tick()
+ except Exception as e:
+ logger.error(f'Job scheduler error: {e}')
+ time.sleep(self.tick_seconds)
+
+ def tick(self):
+ """Enqueue all due schedules. Returns the number fired. Test-callable."""
+ fired = 0
+ for scheduled in ScheduledJobService.due():
+ try:
+ ScheduledJobService.fire(scheduled)
+ fired += 1
+ except Exception as e:
+ logger.error(f'Failed to fire scheduled job {scheduled.name}: {e}')
+ # Advance anyway so a poison schedule can't hot-loop the ticker.
+ try:
+ from app import db
+ scheduled.next_run_at = scheduled.compute_next_run()
+ db.session.commit()
+ except Exception:
+ db.session.rollback()
+ return fired
+
+
+def start_job_scheduler(app):
+ """Start the singleton scheduler thread (skipped under testing config)."""
+ global _job_scheduler
+ if _job_scheduler is not None:
+ return
+ if app.config.get('ENV') == 'testing' or app.config.get('TESTING'):
+ return
+ scheduler = JobScheduler(app)
+ scheduler.start()
+ _job_scheduler = scheduler
+
+
+def stop_job_scheduler():
+ global _job_scheduler
+ if _job_scheduler is not None:
+ _job_scheduler.stop()
+ _job_scheduler = None
diff --git a/backend/app/jobs/sdk.py b/backend/app/jobs/sdk.py
new file mode 100644
index 00000000..0374101a
--- /dev/null
+++ b/backend/app/jobs/sdk.py
@@ -0,0 +1,42 @@
+"""Plugin/consumer-facing SDK for the unified job system.
+
+ from app.plugins_sdk import jobs
+
+ jobs.register('myplugin.reindex', handler_fn)
+ jobs.enqueue('myplugin.reindex', {'site_id': 5})
+ jobs.schedule('myplugin-nightly', 'myplugin.reindex', cron='0 3 * * *')
+"""
+from app.jobs import registry
+from app.jobs.service import JobService, ScheduledJobService
+
+
+class JobsSdk:
+ """Stable jobs surface for plugins and internal callers."""
+
+ def register(self, kind, handler, replace=False):
+ """Register a handler ``fn(job) -> result`` for a job ``kind``."""
+ return registry.register(kind, handler, replace=replace)
+
+ def enqueue(self, kind, payload=None, max_attempts=3, priority=0, delay_ms=0,
+ owner_type=None, owner_id=None):
+ job = JobService.enqueue(
+ kind, payload=payload, max_attempts=max_attempts, priority=priority,
+ delay_ms=delay_ms, owner_type=owner_type, owner_id=owner_id,
+ )
+ return job.to_dict()
+
+ def get(self, job_id):
+ job = JobService.get(job_id)
+ return job.to_dict(include_payload=True) if job else None
+
+ def cancel(self, job_id):
+ job = JobService.cancel(job_id)
+ return job.to_dict() if job else None
+
+ def schedule(self, name, kind, interval_seconds=None, cron=None, payload=None,
+ max_attempts=1, startup_delay_seconds=0):
+ scheduled = ScheduledJobService.ensure(
+ name, kind, interval_seconds=interval_seconds, cron=cron, payload=payload,
+ max_attempts=max_attempts, startup_delay_seconds=startup_delay_seconds,
+ )
+ return scheduled.to_dict()
diff --git a/backend/app/jobs/service.py b/backend/app/jobs/service.py
new file mode 100644
index 00000000..d6ab1790
--- /dev/null
+++ b/backend/app/jobs/service.py
@@ -0,0 +1,216 @@
+"""Producer + management API for the unified job system.
+
+``JobService`` is the one entry point to enqueue work; ``ScheduledJobService``
+manages periodic schedules. Both are non-blocking: they persist a row and (for
+jobs) publish a thin pointer onto the Queue Bus.
+"""
+import logging
+from datetime import datetime, timedelta
+
+from app import db
+from app.jobs.models import Job, ScheduledJob
+from app.queue_bus.service import QueueBusService
+
+logger = logging.getLogger(__name__)
+
+GROUP_SLUG = 'serverkit-system'
+QUEUE_SLUG = 'jobs'
+# Jobs can run longer than a webhook POST, so give them a wider visibility
+# window before the queue considers a message abandoned.
+QUEUE_CONFIG = {'max_attempts': 3, 'visibility_timeout_ms': 120000}
+
+
+def _new_correlation_id():
+ try:
+ from app.services.telemetry_service import generate_correlation_id
+ return generate_correlation_id()
+ except Exception:
+ import uuid
+ return uuid.uuid4().hex
+
+
+class JobService:
+
+ @classmethod
+ def enqueue(cls, kind, payload=None, max_attempts=3, priority=0, delay_ms=0,
+ owner_type=None, owner_id=None, correlation_id=None):
+ """Persist a Job and publish it onto the Queue Bus. Returns the Job."""
+ job = Job(
+ kind=kind,
+ status=Job.STATUS_PENDING,
+ max_attempts=max_attempts,
+ priority=priority,
+ owner_type=owner_type,
+ owner_id=str(owner_id) if owner_id is not None else None,
+ correlation_id=correlation_id or _new_correlation_id(),
+ )
+ job.set_payload(payload or {})
+ if delay_ms:
+ job.scheduled_at = datetime.utcnow() + timedelta(milliseconds=delay_ms)
+ db.session.add(job)
+ db.session.commit()
+ cls._publish(job, delay_ms=delay_ms)
+ return job
+
+ @classmethod
+ def _publish(cls, job, delay_ms=0):
+ QueueBusService.ensure_queue(GROUP_SLUG, QUEUE_SLUG, config=QUEUE_CONFIG)
+ msg = QueueBusService.send(
+ GROUP_SLUG, QUEUE_SLUG, {'job_id': job.id},
+ priority=job.priority or 0, delay_ms=delay_ms,
+ max_attempts=job.max_attempts,
+ )
+ if isinstance(msg, dict):
+ job.queue_message_id = msg.get('id')
+ db.session.commit()
+ return msg
+
+ @classmethod
+ def get(cls, job_id):
+ return Job.query.get(job_id)
+
+ @classmethod
+ def list(cls, status=None, kind=None, owner_type=None, owner_id=None, limit=50, offset=0):
+ query = Job.query
+ if status:
+ query = query.filter_by(status=status)
+ if kind:
+ query = query.filter_by(kind=kind)
+ if owner_type:
+ query = query.filter_by(owner_type=owner_type)
+ if owner_id is not None:
+ query = query.filter_by(owner_id=str(owner_id))
+ return query.order_by(Job.created_at.desc()).limit(limit).offset(offset).all()
+
+ @classmethod
+ def cancel(cls, job_id):
+ """Mark a pending/running job cancelled. Cannot interrupt a handler that
+ is already executing — it stops future attempts and is skipped at the
+ next queue pickup."""
+ job = Job.query.get(job_id)
+ if not job:
+ return None
+ if job.status in (Job.STATUS_PENDING, Job.STATUS_RUNNING):
+ job.status = Job.STATUS_CANCELLED
+ job.completed_at = datetime.utcnow()
+ db.session.commit()
+ return job
+
+ @classmethod
+ def retry(cls, job_id):
+ """Re-enqueue a failed/cancelled job as a fresh attempt."""
+ job = Job.query.get(job_id)
+ if not job:
+ return None
+ if job.status not in (Job.STATUS_FAILED, Job.STATUS_CANCELLED):
+ return job
+ job.status = Job.STATUS_PENDING
+ job.error_message = None
+ job.started_at = None
+ job.completed_at = None
+ db.session.commit()
+ cls._publish(job)
+ return job
+
+ @classmethod
+ def stats(cls):
+ from sqlalchemy import func
+ by_status = dict(
+ db.session.query(Job.status, func.count(Job.id)).group_by(Job.status).all()
+ )
+ by_kind = dict(
+ db.session.query(Job.kind, func.count(Job.id)).group_by(Job.kind).all()
+ )
+ return {'by_status': by_status, 'by_kind': by_kind, 'total': sum(by_status.values())}
+
+ @classmethod
+ def cleanup_old(cls, max_age_seconds=86400):
+ """Delete terminal jobs whose completion is older than the window."""
+ cutoff = datetime.utcnow() - timedelta(seconds=max_age_seconds)
+ deleted = (Job.query
+ .filter(Job.status.in_(Job.TERMINAL_STATUSES))
+ .filter(Job.completed_at.isnot(None))
+ .filter(Job.completed_at < cutoff)
+ .delete(synchronize_session=False))
+ db.session.commit()
+ return deleted
+
+
+class ScheduledJobService:
+
+ @classmethod
+ def ensure(cls, name, kind, interval_seconds=None, cron=None, payload=None,
+ max_attempts=1, startup_delay_seconds=0, enabled=True):
+ """Idempotently create/update a periodic schedule keyed by ``name``.
+
+ Updates cadence/kind on an existing row but PRESERVES its
+ ``next_run_at`` / ``last_run_at`` and the admin's ``enabled`` toggle so a
+ restart doesn't reset the clock. On first creation, seeds ``next_run_at``
+ honoring ``startup_delay_seconds``.
+ """
+ scheduled = ScheduledJob.query.filter_by(name=name).first()
+ created = scheduled is None
+ if created:
+ scheduled = ScheduledJob(name=name)
+ db.session.add(scheduled)
+ scheduled.kind = kind
+ scheduled.schedule_kind = ScheduledJob.SCHEDULE_CRON if cron else ScheduledJob.SCHEDULE_INTERVAL
+ scheduled.interval_seconds = interval_seconds
+ scheduled.cron = cron
+ scheduled.max_attempts = max_attempts
+ if payload is not None:
+ scheduled.set_payload(payload)
+ if created:
+ scheduled.enabled = enabled
+ scheduled.next_run_at = datetime.utcnow() + timedelta(seconds=startup_delay_seconds or 0)
+ db.session.commit()
+ return scheduled
+
+ @classmethod
+ def due(cls, now=None):
+ now = now or datetime.utcnow()
+ return (ScheduledJob.query
+ .filter(ScheduledJob.enabled.is_(True))
+ .filter(ScheduledJob.next_run_at.isnot(None))
+ .filter(ScheduledJob.next_run_at <= now)
+ .all())
+
+ @classmethod
+ def fire(cls, scheduled, now=None):
+ """Enqueue a Job for one schedule and advance its ``next_run_at``."""
+ now = now or datetime.utcnow()
+ job = JobService.enqueue(
+ scheduled.kind,
+ payload=scheduled.get_payload(),
+ max_attempts=scheduled.max_attempts,
+ owner_type='schedule',
+ owner_id=scheduled.name,
+ )
+ job.scheduled_job_id = scheduled.id
+ scheduled.last_run_at = now
+ scheduled.last_job_id = job.id
+ scheduled.next_run_at = scheduled.compute_next_run(now)
+ db.session.commit()
+ return job
+
+ @classmethod
+ def list(cls):
+ return ScheduledJob.query.order_by(ScheduledJob.name.asc()).all()
+
+ @classmethod
+ def run_now(cls, scheduled_job_id):
+ scheduled = ScheduledJob.query.get(scheduled_job_id)
+ if not scheduled:
+ return None
+ return cls.fire(scheduled)
+
+ @classmethod
+ def set_enabled(cls, scheduled_job_id, enabled):
+ scheduled = ScheduledJob.query.get(scheduled_job_id)
+ if not scheduled:
+ return None
+ scheduled.enabled = bool(enabled)
+ if enabled and not scheduled.next_run_at:
+ scheduled.next_run_at = datetime.utcnow()
+ db.session.commit()
+ return scheduled
diff --git a/backend/app/plugins_sdk/__init__.py b/backend/app/plugins_sdk/__init__.py
index 017cbfbb..e82aa5c7 100644
--- a/backend/app/plugins_sdk/__init__.py
+++ b/backend/app/plugins_sdk/__init__.py
@@ -90,9 +90,11 @@ def audit(action, target_type, target_id=None, details=None, user_id=None):
from app.plugins_sdk import ai
from app.queue_bus.sdk import QueueBusSdk
from app.notifications.sdk import NotifySdk
+from app.jobs.sdk import JobsSdk
queue = QueueBusSdk()
notify = NotifySdk()
+jobs = JobsSdk()
__all__ = [
'db',
@@ -106,4 +108,5 @@ __all__ = [
'ai',
'queue',
'notify',
+ 'jobs',
]
diff --git a/backend/app/services/background_job_service.py b/backend/app/services/background_job_service.py
deleted file mode 100644
index c5e932ae..00000000
--- a/backend/app/services/background_job_service.py
+++ /dev/null
@@ -1,123 +0,0 @@
-import logging
-import threading
-import queue
-import time
-from datetime import datetime
-from functools import wraps
-
-logger = logging.getLogger(__name__)
-
-
-class BackgroundJobService:
- """Simple background job queue for long-running tasks."""
-
- _queue = queue.Queue()
- _workers = []
- _results = {}
- _lock = threading.Lock()
- _running = False
-
- @classmethod
- def start(cls, app, num_workers=3):
- """Start background worker threads."""
- if cls._running:
- return
-
- cls._running = True
-
- for i in range(num_workers):
- t = threading.Thread(
- target=cls._worker_loop,
- args=(app,),
- daemon=True,
- name=f'bg-worker-{i}',
- )
- t.start()
- cls._workers.append(t)
-
- logger.info(f'Background job service started with {num_workers} workers')
-
- @classmethod
- def _worker_loop(cls, app):
- while cls._running:
- try:
- job = cls._queue.get(timeout=1)
- except queue.Empty:
- continue
-
- job_id = job['id']
- with cls._lock:
- cls._results[job_id] = {'status': 'running', 'started_at': datetime.utcnow().isoformat()}
-
- try:
- with app.app_context():
- result = job['func'](*job.get('args', ()), **job.get('kwargs', {}))
- with cls._lock:
- cls._results[job_id] = {
- 'status': 'completed',
- 'result': result,
- 'completed_at': datetime.utcnow().isoformat(),
- }
- except Exception as e:
- logger.error(f'Background job {job_id} failed: {e}')
- with cls._lock:
- cls._results[job_id] = {
- 'status': 'failed',
- 'error': str(e),
- 'completed_at': datetime.utcnow().isoformat(),
- }
- finally:
- cls._queue.task_done()
-
- @classmethod
- def enqueue(cls, func, *args, job_id=None, **kwargs):
- """Add a job to the queue. Returns job_id."""
- import uuid
- job_id = job_id or str(uuid.uuid4())[:8]
-
- with cls._lock:
- cls._results[job_id] = {'status': 'queued', 'queued_at': datetime.utcnow().isoformat()}
-
- cls._queue.put({
- 'id': job_id,
- 'func': func,
- 'args': args,
- 'kwargs': kwargs,
- })
-
- return job_id
-
- @classmethod
- def get_job_status(cls, job_id):
- with cls._lock:
- return cls._results.get(job_id)
-
- @classmethod
- def list_jobs(cls):
- with cls._lock:
- return dict(cls._results)
-
- @classmethod
- def cleanup_old(cls, max_age_seconds=3600):
- """Remove completed/failed jobs older than max_age."""
- now = datetime.utcnow()
- with cls._lock:
- to_remove = []
- for jid, info in cls._results.items():
- if info['status'] in ('completed', 'failed'):
- completed_at = info.get('completed_at')
- if completed_at:
- dt = datetime.fromisoformat(completed_at)
- if (now - dt).total_seconds() > max_age_seconds:
- to_remove.append(jid)
- for jid in to_remove:
- del cls._results[jid]
-
- @classmethod
- def get_queue_stats(cls):
- return {
- 'queue_size': cls._queue.qsize(),
- 'workers': len(cls._workers),
- 'total_jobs': len(cls._results),
- 'running': cls._running,
- }
diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.py
new file mode 100644
index 00000000..89b0317e
--- /dev/null
+++ b/backend/tests/test_jobs.py
@@ -0,0 +1,213 @@
+"""Tests for the unified job system (app/jobs)."""
+import pytest
+
+from app import db
+from app.jobs import registry
+from app.jobs.models import Job, ScheduledJob
+from app.jobs.service import JobService, ScheduledJobService, GROUP_SLUG, QUEUE_SLUG
+from app.jobs.consumer import JobConsumer
+from app.jobs.scheduler import JobScheduler
+from app.queue_bus.service import QueueBusService
+from app.queue_bus.models import QueueMessage
+
+
[email protected](autouse=True)
+def reset_jobs(app):
+ """Clean broker + handler registry before each test (after create_app has
+ registered the built-in handlers)."""
+ QueueBusService.reset_broker()
+ registry.clear()
+ yield
+ registry.clear()
+
+
+def _drain_once(consumer=None):
+ """Receive and process exactly one job message; return it processed."""
+ consumer = consumer or JobConsumer()
+ messages = QueueBusService.receive(GROUP_SLUG, QUEUE_SLUG, max_messages=1)
+ assert messages, 'expected a queued job message'
+ consumer.process_message(messages[0])
+ return messages[0]
+
+
+class TestEnqueueAndRun:
+ def test_enqueue_creates_job_and_message(self, app):
+ job = JobService.enqueue('test.noop', {'x': 1})
+ assert job.id is not None
+ assert job.status == Job.STATUS_PENDING
+ assert Job.query.count() == 1
+ # A thin {job_id} pointer is on the bus.
+ msgs = QueueBusService.list_messages(GROUP_SLUG, QUEUE_SLUG)
+ assert len(msgs) == 1
+ assert msgs[0]['payload'] == {'job_id': job.id}
+
+ def test_handler_runs_and_succeeds(self, app):
+ seen = {}
+
+ @registry.handler('test.ok')
+ def _ok(job):
+ seen['payload'] = job.get_payload()
+ return {'done': True}
+
+ job = JobService.enqueue('test.ok', {'x': 42})
+ _drain_once()
+
+ refreshed = Job.query.get(job.id)
+ assert refreshed.status == Job.STATUS_SUCCEEDED
+ assert refreshed.get_result() == {'done': True}
+ assert refreshed.completed_at is not None
+ assert seen['payload'] == {'x': 42}
+
+ def test_unregistered_kind_fails_without_retrying(self, app):
+ job = JobService.enqueue('test.no_handler', {}, max_attempts=3)
+ msg = _drain_once()
+
+ refreshed = Job.query.get(job.id)
+ assert refreshed.status == Job.STATUS_FAILED
+ assert 'No handler' in (refreshed.error_message or '')
+ # Message completed (not left to retry an unroutable job).
+ qm = QueueBusService.get_message(GROUP_SLUG, QUEUE_SLUG, msg['id'])
+ assert qm['status'] == QueueMessage.STATUS_COMPLETED
+
+ def test_failure_retries_then_dead_letters_to_failed(self, app):
+ @registry.handler('test.boom')
+ def _boom(job):
+ raise RuntimeError('kaboom')
+
+ # max_attempts=1 → the first failure exhausts attempts → dead-letter.
+ job = JobService.enqueue('test.boom', {}, max_attempts=1)
+ msg = _drain_once()
+
+ refreshed = Job.query.get(job.id)
+ assert refreshed.status == Job.STATUS_FAILED
+ assert 'kaboom' in (refreshed.error_message or '')
+ qm = QueueBusService.get_message(GROUP_SLUG, QUEUE_SLUG, msg['id'])
+ assert qm['status'] == QueueMessage.STATUS_DEAD_LETTER
+
+ def test_failure_with_attempts_left_stays_pending(self, app):
+ @registry.handler('test.flaky')
+ def _flaky(job):
+ raise RuntimeError('again')
+
+ job = JobService.enqueue('test.flaky', {}, max_attempts=3)
+ _drain_once()
+
+ refreshed = Job.query.get(job.id)
+ # Not terminal yet — the queue will redeliver after backoff.
+ assert refreshed.status == Job.STATUS_PENDING
+ assert refreshed.status not in Job.TERMINAL_STATUSES or refreshed.status == Job.STATUS_PENDING
+
+
+class TestCancelAndRetry:
+ def test_cancel_prevents_execution(self, app):
+ ran = {'count': 0}
+
+ @registry.handler('test.cancelme')
+ def _h(job):
+ ran['count'] += 1
+
+ job = JobService.enqueue('test.cancelme', {})
+ JobService.cancel(job.id)
+ assert Job.query.get(job.id).status == Job.STATUS_CANCELLED
+
+ _drain_once() # consumer should skip the cancelled job
+ assert ran['count'] == 0
+ assert Job.query.get(job.id).status == Job.STATUS_CANCELLED
+
+ def test_retry_requeues_failed_job(self, app):
+ job = JobService.enqueue('test.no_handler', {}, max_attempts=1)
+ _drain_once()
+ assert Job.query.get(job.id).status == Job.STATUS_FAILED
+
+ # Now register a handler and retry.
+ @registry.handler('test.no_handler')
+ def _now_ok(job):
+ return 'recovered'
+
+ JobService.retry(job.id)
+ assert Job.query.get(job.id).status == Job.STATUS_PENDING
+ _drain_once()
+ assert Job.query.get(job.id).status == Job.STATUS_SUCCEEDED
+ assert Job.query.get(job.id).get_result() == 'recovered'
+
+
+class TestScheduler:
+ def test_ensure_is_idempotent(self, app):
+ a = ScheduledJobService.ensure('nightly', 'test.x', interval_seconds=60)
+ b = ScheduledJobService.ensure('nightly', 'test.x', interval_seconds=120)
+ assert a.id == b.id
+ assert ScheduledJob.query.count() == 1
+ assert ScheduledJob.query.get(a.id).interval_seconds == 120
+
+ def test_tick_fires_due_schedule_and_advances(self, app):
+ # startup_delay 0 → next_run_at = now → immediately due.
+ scheduled = ScheduledJobService.ensure(
+ 'due-now', 'test.tick', interval_seconds=3600, startup_delay_seconds=0)
+ before = scheduled.next_run_at
+
+ fired = JobScheduler().tick()
+ assert fired == 1
+
+ # A job was enqueued for the schedule, and next_run advanced past now.
+ job = Job.query.filter_by(kind='test.tick').first()
+ assert job is not None
+ assert job.owner_type == 'schedule'
+ assert job.owner_id == 'due-now'
+ refreshed = ScheduledJob.query.get(scheduled.id)
+ assert refreshed.next_run_at > before
+ assert refreshed.last_job_id == job.id
+ # No longer due.
+ assert JobScheduler().tick() == 0
+
+ def test_disabled_schedule_does_not_fire(self, app):
+ scheduled = ScheduledJobService.ensure(
+ 'off', 'test.tick', interval_seconds=60, startup_delay_seconds=0)
+ ScheduledJobService.set_enabled(scheduled.id, False)
+ assert JobScheduler().tick() == 0
+
+
+class TestBuiltins:
+ def test_register_and_seed(self, app):
+ from app.jobs import builtin_handlers
+ builtin_handlers.register_builtin_handlers()
+ kinds = registry.registered_kinds()
+ assert 'builtin.auto_sync' in kinds
+ assert 'builtin.health_check' in kinds
+ assert len([k for k in kinds if k.startswith('builtin.')]) == 8
+
+ builtin_handlers.seed_builtin_schedules()
+ assert ScheduledJob.query.count() == 8
+ # Seeding twice doesn't duplicate.
+ builtin_handlers.seed_builtin_schedules()
+ assert ScheduledJob.query.count() == 8
+
+
+class TestApi:
+ def test_list_and_get_via_api(self, client, auth_headers, app):
+ job = JobService.enqueue('test.api', {'hello': 'world'})
+
+ resp = client.get('/api/v1/jobs', headers=auth_headers)
+ assert resp.status_code == 200
+ assert any(j['id'] == job.id for j in resp.get_json()['jobs'])
+
+ resp = client.get(f'/api/v1/jobs/{job.id}', headers=auth_headers)
+ assert resp.status_code == 200
+ assert resp.get_json()['job']['payload'] == {'hello': 'world'}
+
+ def test_stats_endpoint(self, client, auth_headers, app):
+ JobService.enqueue('test.api', {})
+ resp = client.get('/api/v1/jobs/stats', headers=auth_headers)
+ assert resp.status_code == 200
+ assert resp.get_json()['total'] >= 1
+
+ def test_cancel_via_api(self, client, auth_headers, app):
+ job = JobService.enqueue('test.api', {})
+ resp = client.post(f'/api/v1/jobs/{job.id}/cancel', headers=auth_headers)
+ assert resp.status_code == 200
+ assert resp.get_json()['job']['status'] == Job.STATUS_CANCELLED
+
+ def test_scheduled_listing(self, client, auth_headers, app):
+ ScheduledJobService.ensure('api-sched', 'test.x', interval_seconds=60)
+ resp = client.get('/api/v1/jobs/scheduled', headers=auth_headers)
+ assert resp.status_code == 200
+ assert any(s['name'] == 'api-sched' for s in resp.get_json()['scheduled'])