ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
feat(jobs): unified background-job orchestration on the Queue Bus
Juan Denis8428326a262a
diff --git a/backend/app/__init__.py b/backend/app/__init__.pyindex 44cf6dd0..5166802a 100644--- a/backend/app/__init__.py+++ b/backend/app/__init__.py@@ -354,6 +354,10 @@ def create_app(config_name=None):from app.api.queue_bus import queue_bus_bpapp.register_blueprint(queue_bus_bp, url_prefix='/api/v1/queue')+ # Register blueprints - Unified Jobs (work orchestration on the Queue Bus)+ from app.api.jobs import jobs_bp+ app.register_blueprint(jobs_bp, url_prefix='/api/v1/jobs')+# Register blueprints - Telemetry / System Event Streamfrom app.api.telemetry import telemetry_bpapp.register_blueprint(telemetry_bp, url_prefix='/api/v1/telemetry')@@ -419,33 +423,21 @@ def create_app(config_name=None):from app.notifications.consumer import start_notification_consumerstart_notification_consumer(app)- # Start auto-sync scheduler for WordPress environments- _start_auto_sync_scheduler(app)-- # Start snapshot-retention scheduler (sets expires_at + prunes expired)- _start_snapshot_retention_scheduler(app)-- # Start workflow scheduler- _start_workflow_scheduler(app)-- # Start per-site WordPress health poller (uptime % + auto-incidents + alerts)- _start_health_check_scheduler(app)-- # Start per-site WordPress safe-update scheduler (#29)- _start_update_scheduler(app)-- # Start API analytics flush thread+ # Start the API analytics flush thread (a 5s buffer flush — a real-time+ # stream, deliberately NOT modeled as a job).from app.middleware.api_analytics import start_analytics_flush_threadstart_analytics_flush_thread(app)- # Start hourly analytics aggregation and event retry threads- _start_api_background_threads(app)-- # Start hourly pruner for expired pending agent pairings- _start_pairing_pruner(app)-- # Start daily registrar domain-expiry notifier- _start_registrar_expiry_scheduler(app)+ # Start the unified job system: ONE consumer runs every enqueued Job and+ # ONE scheduler ticks all periodic work. This supersedes the former set+ # of per-domain daemon scheduler threads (auto-sync, snapshot-retention,+ # workflow, health-check, wp-update, api-background, pairing-prune,+ # registrar-expiry) — they are now ScheduledJob rows backed by the+ # built-in handlers in app/jobs/builtin_handlers.py.+ from app.jobs import start_job_system+ from app.jobs.builtin_handlers import register_builtin_handlers, seed_builtin_schedules+ register_builtin_handlers()+ start_job_system(app, seed=seed_builtin_schedules)# Request body size limitapp.config['MAX_CONTENT_LENGTH'] = 100 * 1024 * 1024 # 100MB limit@@ -496,504 +488,3 @@ def create_app(config_name=None):def get_socketio():"""Get the SocketIO instance."""return socketio---_auto_sync_thread = None---def _start_auto_sync_scheduler(app):- """Start a background thread that checks for auto-sync schedules."""- global _auto_sync_thread- if _auto_sync_thread is not None:- return-- import threading- import time- import logging-- logger = logging.getLogger(__name__)-- def auto_sync_loop():- while True:- try:- time.sleep(60) # Check every 60 seconds- with app.app_context():- _check_auto_sync_schedules(logger)- except Exception as e:- logger.error(f'Auto-sync scheduler error: {e}')-- _auto_sync_thread = threading.Thread(- target=auto_sync_loop,- daemon=True,- name='auto-sync-scheduler'- )- _auto_sync_thread.start()---def _check_auto_sync_schedules(logger):- """Check all auto-sync enabled sites and run syncs that are due."""- from app.models.wordpress_site import WordPressSite- from datetime import datetime-- sites = WordPressSite.query.filter_by(auto_sync_enabled=True).all()- if not sites:- return-- try:- from croniter import croniter- except ImportError:- logger.debug('croniter not installed, skipping auto-sync check')- return-- now = datetime.utcnow()-- for site in sites:- if not site.auto_sync_schedule:- continue-- try:- if not croniter.is_valid(site.auto_sync_schedule):- continue-- cron = croniter(site.auto_sync_schedule, now)- prev_run = cron.get_prev(datetime)-- # Check if a run was due in the last 90 seconds (to account for check interval)- seconds_since_due = (now - prev_run).total_seconds()- if seconds_since_due <= 90:- logger.info(f'Auto-sync triggered for site {site.id} ({site.name})')- from app.services.environment_pipeline_service import EnvironmentPipelineService- EnvironmentPipelineService.sync_from_production(- env_site_id=site.id,- sync_type='full',- user_id=None- )- except Exception as e:- logger.error(f'Auto-sync check failed for site {site.id}: {e}')---_api_bg_thread = None---def _start_pairing_pruner(app):- """Start a background thread that prunes expired PendingAgent rows hourly."""- global _pairing_prune_thread- if _pairing_prune_thread is not None:- return-- import threading- import time- import logging-- logger = logging.getLogger(__name__)-- def prune_loop():- # Wait a bit before first run so app is fully initialized.- time.sleep(60)- while True:- try:- with app.app_context():- from app.services import pairing_service- pairing_service.prune_expired()- except Exception as e:- logger.error(f'Pairing pruner error: {e}')- time.sleep(3600)-- _pairing_prune_thread = threading.Thread(- target=prune_loop,- daemon=True,- name='pairing-pruner'- )- _pairing_prune_thread.start()---_pairing_prune_thread = None---_registrar_expiry_thread = None---def _start_registrar_expiry_scheduler(app):- """Daily check that notifies when a registrar domain crosses an expiry- threshold (30/14/7/1 days, or expired). Single-worker only (module-global- guard), like the other in-process schedulers."""- global _registrar_expiry_thread- if _registrar_expiry_thread is not None:- return-- import threading- import time- import logging-- logger = logging.getLogger(__name__)-- def loop():- time.sleep(300) # let startup settle- while True:- try:- with app.app_context():- from app.services.registrar_service import RegistrarService- n = RegistrarService.notify_expiring()- if n:- logger.info(f'Registrar expiry: sent {n} notification(s)')- except Exception as e:- logger.error(f'Registrar expiry scheduler error: {e}')- time.sleep(86400) # daily-- _registrar_expiry_thread = threading.Thread(- target=loop, daemon=True, name='registrar-expiry')- _registrar_expiry_thread.start()---_snapshot_retention_thread = None---def _start_snapshot_retention_scheduler(app):- """Start a background thread that sets DatabaseSnapshot.expires_at per the- retention policy and prunes expired snapshots (file + DB row) hourly."""- global _snapshot_retention_thread- if _snapshot_retention_thread is not None:- return-- import threading- import time- import logging-- logger = logging.getLogger(__name__)-- def retention_loop():- # Delay first run so the app is fully initialized.- time.sleep(120)- while True:- try:- with app.app_context():- from app.services.db_sync_service import DatabaseSyncService- from app.services.settings_service import SettingsService- # Honor an admin-set override if present; otherwise use the- # code default (no settings seed / migration required).- days = SettingsService.get(- 'snapshot_retention_days',- DatabaseSyncService.DEFAULT_SNAPSHOT_RETENTION_DAYS,- )- try:- days = int(days)- except (TypeError, ValueError):- days = DatabaseSyncService.DEFAULT_SNAPSHOT_RETENTION_DAYS- result = DatabaseSyncService.prune_expired_snapshots(retention_days=days)- if result.get('deleted') or result.get('backfilled'):- logger.info(- f"Snapshot retention: backfilled={result.get('backfilled', 0)} "- f"deleted={result.get('deleted', 0)}"- )- except Exception as e:- logger.error(f'Snapshot retention scheduler error: {e}')- time.sleep(3600) # hourly-- _snapshot_retention_thread = threading.Thread(- target=retention_loop,- daemon=True,- name='snapshot-retention',- )- _snapshot_retention_thread.start()---def _start_api_background_threads(app):- """Start background threads for API analytics aggregation and event delivery retry."""- global _api_bg_thread- if _api_bg_thread is not None:- return-- import threading- import time- import logging-- logger = logging.getLogger(__name__)-- def api_bg_loop():- while True:- try:- time.sleep(3600) # Run hourly- with app.app_context():- from app.services.api_analytics_service import ApiAnalyticsService- ApiAnalyticsService.aggregate_hourly()-- from app.services.event_service import EventService- EventService.retry_failed()- except Exception as e:- logger.error(f'API background thread error: {e}')-- _api_bg_thread = threading.Thread(- target=api_bg_loop,- daemon=True,- name='api-background'- )- _api_bg_thread.start()---_workflow_thread = None---def _start_workflow_scheduler(app):- """Start a background thread that checks for scheduled workflows."""- global _workflow_thread- if _workflow_thread is not None:- return-- import threading- import time- import logging-- logger = logging.getLogger(__name__)-- def workflow_loop():- while True:- try:- time.sleep(60) # Check every 60 seconds- with app.app_context():- _check_workflow_schedules(logger)- except Exception as e:- logger.error(f'Workflow scheduler error: {e}')-- _workflow_thread = threading.Thread(- target=workflow_loop,- daemon=True,- name='workflow-scheduler'- )- _workflow_thread.start()---def _check_workflow_schedules(logger):- """Check all active workflows with cron triggers and run those that are due."""- from app.models.workflow import Workflow- from app.services.workflow_engine import WorkflowEngine- from datetime import datetime- import json-- try:- from croniter import croniter- except ImportError:- logger.debug('croniter not installed, skipping workflow schedule check')- return-- # Find active workflows with cron triggers- workflows = Workflow.query.filter_by(is_active=True, trigger_type='cron').all()- if not workflows:- return-- now = datetime.utcnow()-- for workflow in workflows:- try:- config = json.loads(workflow.trigger_config) if workflow.trigger_config else {}- cron_expr = config.get('cron')-- if not cron_expr or not croniter.is_valid(cron_expr):- continue-- cron = croniter(cron_expr, now)- prev_run = cron.get_prev(datetime)-- # Check if a run was due in the last 90 seconds- seconds_since_due = (now - prev_run).total_seconds()-- # Also ensure we don't run it multiple times for the same slot- if 0 < seconds_since_due <= 90:- # Check if it already ran in the last 2 minutes- if workflow.last_run_at:- seconds_since_last_run = (now - workflow.last_run_at).total_seconds()- if seconds_since_last_run < 110:- continue-- logger.info(f'Scheduled workflow triggered: {workflow.name} (ID: {workflow.id})')- WorkflowEngine.execute_workflow(- workflow_id=workflow.id,- trigger_type='cron',- context={'scheduled_at': prev_run.isoformat()}- )- except Exception as e:- logger.error(f'Workflow schedule check failed for workflow {workflow.id}: {e}')---_health_check_thread = None--# How often the per-site WordPress health poller runs (seconds).-HEALTH_CHECK_INTERVAL = 300--# Retention for recorded health-check samples (days) — bounds unbounded growth-# from the continuous poller; matches the longest uptime window (uptime_90d).-# Pruned at most once per day.-HEALTH_CHECK_RETENTION_DAYS = 90-_last_health_prune = None---def _start_health_check_scheduler(app):- """Start a background thread that polls every managed WordPress site's- health on an interval. This keeps health_status fresh (so #27 transition- alerts fire autonomously, not only while the health card is open) and drives- any bound status-page components (#26): a real uptime % and auto-incidents.-- Single-worker only — like the other in-process schedulers, this must not be- multiplied across Gunicorn workers (see CLAUDE.md). The module-global guard- ensures one thread per process.- """- global _health_check_thread- if _health_check_thread is not None:- return-- import threading- import time- import logging-- logger = logging.getLogger(__name__)-- def health_loop():- # Small initial delay so startup isn't slowed by blocking probes.- time.sleep(30)- while True:- try:- with app.app_context():- _run_health_checks(logger)- except Exception as e:- logger.error(f'Health-check scheduler error: {e}')- time.sleep(HEALTH_CHECK_INTERVAL)-- _health_check_thread = threading.Thread(- target=health_loop,- daemon=True,- name='health-check-scheduler'- )- _health_check_thread.start()---def _run_health_checks(logger):- """Run a health check for every managed (production) WordPress site and sync- any status-page components bound to it. Per-site try/except so one hung site- never stalls the whole sweep."""- from app import db- from app.models.wordpress_site import WordPressSite- from app.models.status_page import StatusComponent- from app.services.environment_health_service import EnvironmentHealthService- from app.services.status_page_service import StatusPageService-- _prune_old_health_checks(logger)-- sites = WordPressSite.query.filter_by(is_production=True).all()- for site in sites:- try:- # Only poll sites the operator expects to be up — skip archived/stopped- # stacks so an intentional stop never looks like an outage.- if not site.application or site.application.status != 'running':- continue- result = EnvironmentHealthService.check_health(site.id)- overall = result.get('overall_status')- if not overall:- continue- # Drive any status-page components bound to this site.- components = StatusComponent.query.filter_by(wordpress_site_id=site.id).all()- for comp in components:- StatusPageService.sync_component_from_health(comp, overall)- except Exception as e:- logger.error(f'Health check failed for site {site.id}: {e}')- try:- db.session.rollback()- except Exception:- pass---def _prune_old_health_checks(logger):- """Delete health-check samples older than the retention window, at most once- per day, so the continuous poller doesn't grow the health_checks table without- bound. Best-effort — failure never stalls the health sweep."""- global _last_health_prune- from datetime import datetime, timedelta- now = datetime.utcnow()- if _last_health_prune is not None and (now - _last_health_prune).total_seconds() < 86400:- return- from app import db- from app.models.status_page import HealthCheck- cutoff = now - timedelta(days=HEALTH_CHECK_RETENTION_DAYS)- try:- deleted = HealthCheck.query.filter(HealthCheck.checked_at < cutoff).delete(synchronize_session=False)- db.session.commit()- _last_health_prune = now- if deleted:- logger.info(f'Pruned {deleted} health-check row(s) older than {HEALTH_CHECK_RETENTION_DAYS}d')- except Exception as e:- logger.error(f'Health-check prune failed: {e}')- try:- db.session.rollback()- except Exception:- pass---_update_scheduler_thread = None---def _start_update_scheduler(app):- """Background thread that runs due per-site WordPress auto-updates (safe-update- with snapshot + health-check + auto-rollback). Single-worker only (module-global- guard); each run itself runs in its own thread, so this loop only triggers them."""- global _update_scheduler_thread- if _update_scheduler_thread is not None:- return-- import threading- import time- import logging-- logger = logging.getLogger(__name__)-- def loop():- time.sleep(45) # let startup settle- while True:- try:- time.sleep(60)- with app.app_context():- _check_update_schedules(logger)- except Exception as e:- logger.error(f'Update scheduler error: {e}')-- _update_scheduler_thread = threading.Thread(target=loop, daemon=True, name='wp-update-scheduler')- _update_scheduler_thread.start()---def _check_update_schedules(logger):- from app.models.wordpress_site import WordPressSite, WordPressUpdateRun- from app.services.wp_update_service import WpUpdateService- from datetime import datetime- import json as _json-- try:- from croniter import croniter- except ImportError:- return-- sites = WordPressSite.query.filter(WordPressSite.auto_update_schedule.isnot(None)).all()- if not sites:- return- now = datetime.utcnow()- for site in sites:- try:- expr = (site.auto_update_schedule or '').strip()- if not expr or not croniter.is_valid(expr):- continue- if not site.application or site.application.status != 'running':- continue- prev = croniter(expr, now).get_prev(datetime)- if not (0 < (now - prev).total_seconds() <= 90):- continue- # de-dup: skip if a run already started in the last ~10 minutes- last = (WordPressUpdateRun.query.filter_by(site_id=site.id)- .order_by(WordPressUpdateRun.started_at.desc()).first())- if last and last.started_at and (now - last.started_at).total_seconds() < 600:- continue- exclude = []- if site.auto_update_exclude:- try:- exclude = _json.loads(site.auto_update_exclude)- except Exception:- exclude = []- logger.info(f'Scheduled WordPress safe-update: site {site.id}')- WpUpdateService.start_update(site, exclude=exclude, trigger='scheduled')- except Exception as e:- logger.error(f'Update schedule check failed for site {site.id}: {e}')diff --git a/backend/app/api/jobs.py b/backend/app/api/jobs.pynew file mode 100644index 00000000..7e3cd8bf--- /dev/null+++ b/backend/app/api/jobs.py@@ -0,0 +1,120 @@+"""Read/ops API for the unified job system — the single pane to observe all+background work. Admin-gated, mirroring app/api/queue_bus.py conventions."""+from flask import Blueprint, request, jsonify+from flask_jwt_extended import jwt_required, get_jwt_identity++from app.jobs import registry+from app.jobs.service import JobService, ScheduledJobService++jobs_bp = Blueprint('jobs', __name__)+++def _require_admin():+ from app.models.user import User+ user = User.query.get(get_jwt_identity())+ if not user or not user.is_admin:+ return None+ return user+++# --- Static routes first; Werkzeug ranks these above the dynamic /<job_id>. ---+@jobs_bp.route('', methods=['GET'])+@jobs_bp.route('/', methods=['GET'])+@jwt_required()+def list_jobs():+ if not _require_admin():+ return jsonify({'error': 'Admin access required'}), 403+ try:+ limit = min(int(request.args.get('limit', 50)), 200)+ offset = int(request.args.get('offset', 0))+ except (TypeError, ValueError):+ limit, offset = 50, 0+ jobs = JobService.list(+ status=request.args.get('status'),+ kind=request.args.get('kind'),+ owner_type=request.args.get('owner_type'),+ owner_id=request.args.get('owner_id'),+ limit=limit,+ offset=offset,+ )+ return jsonify({'jobs': [j.to_dict() for j in jobs]})+++@jobs_bp.route('/stats', methods=['GET'])+@jwt_required()+def job_stats():+ if not _require_admin():+ return jsonify({'error': 'Admin access required'}), 403+ return jsonify(JobService.stats())+++@jobs_bp.route('/kinds', methods=['GET'])+@jwt_required()+def job_kinds():+ if not _require_admin():+ return jsonify({'error': 'Admin access required'}), 403+ return jsonify({'kinds': registry.registered_kinds()})+++@jobs_bp.route('/scheduled', methods=['GET'])+@jwt_required()+def list_scheduled():+ if not _require_admin():+ return jsonify({'error': 'Admin access required'}), 403+ return jsonify({'scheduled': [s.to_dict() for s in ScheduledJobService.list()]})+++@jobs_bp.route('/scheduled/<int:scheduled_id>/run', methods=['POST'])+@jwt_required()+def run_scheduled(scheduled_id):+ if not _require_admin():+ return jsonify({'error': 'Admin access required'}), 403+ job = ScheduledJobService.run_now(scheduled_id)+ if not job:+ return jsonify({'error': 'Scheduled job not found'}), 404+ return jsonify({'job': job.to_dict()})+++@jobs_bp.route('/scheduled/<int:scheduled_id>/enabled', methods=['POST'])+@jwt_required()+def toggle_scheduled(scheduled_id):+ if not _require_admin():+ return jsonify({'error': 'Admin access required'}), 403+ body = request.get_json(silent=True) or {}+ scheduled = ScheduledJobService.set_enabled(scheduled_id, bool(body.get('enabled', True)))+ if not scheduled:+ return jsonify({'error': 'Scheduled job not found'}), 404+ return jsonify({'scheduled': scheduled.to_dict()})+++@jobs_bp.route('/<job_id>', methods=['GET'])+@jwt_required()+def get_job(job_id):+ if not _require_admin():+ return jsonify({'error': 'Admin access required'}), 403+ job = JobService.get(job_id)+ if not job:+ return jsonify({'error': 'Job not found'}), 404+ return jsonify({'job': job.to_dict(include_payload=True)})+++@jobs_bp.route('/<job_id>/cancel', methods=['POST'])+@jwt_required()+def cancel_job(job_id):+ if not _require_admin():+ return jsonify({'error': 'Admin access required'}), 403+ job = JobService.cancel(job_id)+ if not job:+ return jsonify({'error': 'Job not found'}), 404+ return jsonify({'job': job.to_dict()})+++@jobs_bp.route('/<job_id>/retry', methods=['POST'])+@jwt_required()+def retry_job(job_id):+ if not _require_admin():+ return jsonify({'error': 'Admin access required'}), 403+ job = JobService.retry(job_id)+ if not job:+ return jsonify({'error': 'Job not found'}), 404+ return jsonify({'job': job.to_dict()})diff --git a/backend/app/api/performance.py b/backend/app/api/performance.pyindex e41eeeff..623f6b5b 100644--- a/backend/app/api/performance.py+++ b/backend/app/api/performance.py@@ -1,7 +1,7 @@from flask import Blueprint, request, jsonifyfrom flask_jwt_extended import jwt_requiredfrom app.services.cache_service import CacheService-from app.services.background_job_service import BackgroundJobService+from app.jobs.service import JobServiceperformance_bp = Blueprint('performance', __name__)@@ -37,16 +37,17 @@ def list_jobs():user = get_current_user()if not user or not user.is_admin:return jsonify({'error': 'Admin access required'}), 403- return jsonify({'jobs': BackgroundJobService.list_jobs()})+ # Keyed by id to preserve the prior {job_id: info} response shape.+ return jsonify({'jobs': {j.id: j.to_dict() for j in JobService.list(limit=200)}})@performance_bp.route('/jobs/<job_id>', methods=['GET'])@jwt_required()def get_job(job_id):- status = BackgroundJobService.get_job_status(job_id)- if not status:+ job = JobService.get(job_id)+ if not job:return jsonify({'error': 'Job not found'}), 404- return jsonify({'job_id': job_id, **status})+ return jsonify({'job_id': job_id, **job.to_dict(include_payload=True)})@performance_bp.route('/jobs/stats', methods=['GET'])@@ -55,7 +56,14 @@ def job_stats():user = get_current_user()if not user or not user.is_admin:return jsonify({'error': 'Admin access required'}), 403- return jsonify(BackgroundJobService.get_queue_stats())+ stats = JobService.stats()+ return jsonify({+ 'total_jobs': stats['total'],+ 'by_status': stats['by_status'],+ 'by_kind': stats['by_kind'],+ 'running': stats['by_status'].get('running', 0),+ 'queue_size': stats['by_status'].get('pending', 0),+ })@performance_bp.route('/jobs/cleanup', methods=['POST'])@@ -64,5 +72,5 @@ def cleanup_jobs():user = get_current_user()if not user or not user.is_admin:return jsonify({'error': 'Admin access required'}), 403- BackgroundJobService.cleanup_old()- return jsonify({'message': 'Old jobs cleaned up'})+ deleted = JobService.cleanup_old()+ return jsonify({'message': f'Cleaned up {deleted} old job(s)'})diff --git a/backend/app/jobs/__init__.py b/backend/app/jobs/__init__.pynew file mode 100644index 00000000..201f67f3--- /dev/null+++ b/backend/app/jobs/__init__.py@@ -0,0 +1,48 @@+"""ServerKit unified job system.++A thin Job / WorkUnit + scheduler layer on top of the Queue Bus. A producer+calls ``JobService.enqueue(kind, payload)``; that persists a ``Job`` row and+publishes a thin ``{'job_id': ...}`` message onto the ``serverkit-system/jobs``+queue. One ``JobConsumer`` runs every job by dispatching its ``kind`` to a+registered handler, and one ``JobScheduler`` ticks all periodic work. This is+the same pattern the Notification Bus uses, generalized.++See ``docs/ORCHESTRATION_ROADMAP.md`` (local, git-ignored) for the plan.+"""+# Import the models at package-import time so they register on ``db.metadata``+# and ``MigrationService._fix_missing_columns`` -> ``db.create_all()`` picks them+# up on boot — mirrors how ``app/queue_bus`` registers its models. No Alembic+# migration is required for new tables.+from app.jobs.models import Job, ScheduledJob # noqa: F401+from app.jobs import registry # noqa: F401++__all__ = ['Job', 'ScheduledJob', 'registry', 'start_job_system']+++def start_job_system(app, seed=None):+ """Ensure the jobs queue exists, seed built-in schedules, and start the+ consumer + scheduler daemon threads.++ No-op under testing config (mirrors the queue-bus / notification consumers),+ so the suite drives ``JobConsumer.process_message`` / ``JobScheduler.tick``+ directly instead of racing background threads against per-test databases.+ """+ if app.config.get('ENV') == 'testing' or app.config.get('TESTING'):+ return++ import logging+ from app.queue_bus.service import QueueBusService+ from app.jobs.service import GROUP_SLUG, QUEUE_SLUG, QUEUE_CONFIG+ from app.jobs.consumer import start_job_consumer+ from app.jobs.scheduler import start_job_scheduler++ with app.app_context():+ QueueBusService.ensure_queue(GROUP_SLUG, QUEUE_SLUG, config=QUEUE_CONFIG)+ if seed is not None:+ try:+ seed()+ except Exception:+ logging.getLogger(__name__).exception('Built-in schedule seeding failed')++ start_job_consumer(app)+ start_job_scheduler(app)diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.pynew file mode 100644index 00000000..f4ae1406--- /dev/null+++ b/backend/app/jobs/builtin_handlers.py@@ -0,0 +1,290 @@+"""Built-in periodic job handlers + their schedules.++These are the platform's own recurring tasks. They used to each run in a+dedicated daemon thread spawned from ``app/__init__.py`` (auto-sync,+snapshot-retention, workflow scheduler, health-check poller, WP safe-update,+hourly API background, pairing pruner, registrar expiry). They now run as+``ScheduledJob`` rows enqueued by the single ``JobScheduler`` and executed by the+single ``JobConsumer``.++The check/run functions below are relocated verbatim from ``app/__init__.py`` so+behavior — cadence handling, per-task de-dup, idempotency — is unchanged; only+the trigger moved from a bare thread to the unified job system.+"""+import logging++from app.jobs.registry import register++logger = logging.getLogger(__name__)+++# ---------------------------------------------------------------------------+# Relocated check/run functions (formerly in app/__init__.py)+# ---------------------------------------------------------------------------+def check_auto_sync_schedules():+ """Check all auto-sync enabled sites and run syncs that are due."""+ from app.models.wordpress_site import WordPressSite+ from datetime import datetime++ sites = WordPressSite.query.filter_by(auto_sync_enabled=True).all()+ if not sites:+ return++ try:+ from croniter import croniter+ except ImportError:+ logger.debug('croniter not installed, skipping auto-sync check')+ return++ now = datetime.utcnow()++ for site in sites:+ if not site.auto_sync_schedule:+ continue+ try:+ if not croniter.is_valid(site.auto_sync_schedule):+ continue+ cron = croniter(site.auto_sync_schedule, now)+ prev_run = cron.get_prev(datetime)+ # Was a run due in the last 90 seconds (accounts for the tick interval)?+ seconds_since_due = (now - prev_run).total_seconds()+ if seconds_since_due <= 90:+ logger.info(f'Auto-sync triggered for site {site.id} ({site.name})')+ from app.services.environment_pipeline_service import EnvironmentPipelineService+ EnvironmentPipelineService.sync_from_production(+ env_site_id=site.id,+ sync_type='full',+ user_id=None,+ )+ except Exception as e:+ logger.error(f'Auto-sync check failed for site {site.id}: {e}')+++def check_workflow_schedules():+ """Check all active workflows with cron triggers and run those that are due."""+ from app.models.workflow import Workflow+ from app.services.workflow_engine import WorkflowEngine+ from datetime import datetime+ import json++ try:+ from croniter import croniter+ except ImportError:+ logger.debug('croniter not installed, skipping workflow schedule check')+ return++ workflows = Workflow.query.filter_by(is_active=True, trigger_type='cron').all()+ if not workflows:+ return++ now = datetime.utcnow()++ for workflow in workflows:+ try:+ config = json.loads(workflow.trigger_config) if workflow.trigger_config else {}+ cron_expr = config.get('cron')+ if not cron_expr or not croniter.is_valid(cron_expr):+ continue+ cron = croniter(cron_expr, now)+ prev_run = cron.get_prev(datetime)+ seconds_since_due = (now - prev_run).total_seconds()+ # Don't run multiple times for the same slot.+ if 0 < seconds_since_due <= 90:+ if workflow.last_run_at:+ seconds_since_last_run = (now - workflow.last_run_at).total_seconds()+ if seconds_since_last_run < 110:+ continue+ logger.info(f'Scheduled workflow triggered: {workflow.name} (ID: {workflow.id})')+ WorkflowEngine.execute_workflow(+ workflow_id=workflow.id,+ trigger_type='cron',+ context={'scheduled_at': prev_run.isoformat()},+ )+ except Exception as e:+ logger.error(f'Workflow schedule check failed for workflow {workflow.id}: {e}')+++# How often the per-site WordPress health poller runs (seconds).+HEALTH_CHECK_INTERVAL = 300+# Retention for recorded health-check samples (days) — bounds unbounded growth+# from the continuous poller; matches the longest uptime window (uptime_90d).+# Pruned at most once per day.+HEALTH_CHECK_RETENTION_DAYS = 90+_last_health_prune = None+++def run_health_checks():+ """Run a health check for every managed (production) WordPress site and sync+ any status-page components bound to it. Per-site try/except so one hung site+ never stalls the whole sweep."""+ from app import db+ from app.models.wordpress_site import WordPressSite+ from app.models.status_page import StatusComponent+ from app.services.environment_health_service import EnvironmentHealthService+ from app.services.status_page_service import StatusPageService++ _prune_old_health_checks()++ sites = WordPressSite.query.filter_by(is_production=True).all()+ for site in sites:+ try:+ # Only poll sites the operator expects to be up — skip archived/stopped+ # stacks so an intentional stop never looks like an outage.+ if not site.application or site.application.status != 'running':+ continue+ result = EnvironmentHealthService.check_health(site.id)+ overall = result.get('overall_status')+ if not overall:+ continue+ components = StatusComponent.query.filter_by(wordpress_site_id=site.id).all()+ for comp in components:+ StatusPageService.sync_component_from_health(comp, overall)+ except Exception as e:+ logger.error(f'Health check failed for site {site.id}: {e}')+ try:+ db.session.rollback()+ except Exception:+ pass+++def _prune_old_health_checks():+ """Delete health-check samples older than the retention window, at most once+ per day, so the continuous poller doesn't grow the health_checks table without+ bound. Best-effort — failure never stalls the health sweep."""+ global _last_health_prune+ from datetime import datetime, timedelta+ now = datetime.utcnow()+ if _last_health_prune is not None and (now - _last_health_prune).total_seconds() < 86400:+ return+ from app import db+ from app.models.status_page import HealthCheck+ cutoff = now - timedelta(days=HEALTH_CHECK_RETENTION_DAYS)+ try:+ deleted = HealthCheck.query.filter(HealthCheck.checked_at < cutoff).delete(synchronize_session=False)+ db.session.commit()+ _last_health_prune = now+ if deleted:+ logger.info(f'Pruned {deleted} health-check row(s) older than {HEALTH_CHECK_RETENTION_DAYS}d')+ except Exception as e:+ logger.error(f'Health-check prune failed: {e}')+ try:+ db.session.rollback()+ except Exception:+ pass+++def check_update_schedules():+ from app.models.wordpress_site import WordPressSite, WordPressUpdateRun+ from app.services.wp_update_service import WpUpdateService+ from datetime import datetime+ import json as _json++ try:+ from croniter import croniter+ except ImportError:+ return++ sites = WordPressSite.query.filter(WordPressSite.auto_update_schedule.isnot(None)).all()+ if not sites:+ return+ now = datetime.utcnow()+ for site in sites:+ try:+ expr = (site.auto_update_schedule or '').strip()+ if not expr or not croniter.is_valid(expr):+ continue+ if not site.application or site.application.status != 'running':+ continue+ prev = croniter(expr, now).get_prev(datetime)+ if not (0 < (now - prev).total_seconds() <= 90):+ continue+ # de-dup: skip if a run already started in the last ~10 minutes+ last = (WordPressUpdateRun.query.filter_by(site_id=site.id)+ .order_by(WordPressUpdateRun.started_at.desc()).first())+ if last and last.started_at and (now - last.started_at).total_seconds() < 600:+ continue+ exclude = []+ if site.auto_update_exclude:+ try:+ exclude = _json.loads(site.auto_update_exclude)+ except Exception:+ exclude = []+ logger.info(f'Scheduled WordPress safe-update: site {site.id}')+ WpUpdateService.start_update(site, exclude=exclude, trigger='scheduled')+ except Exception as e:+ logger.error(f'Update schedule check failed for site {site.id}: {e}')+++def run_snapshot_retention():+ """Set DatabaseSnapshot.expires_at per the retention policy and prune expired+ snapshots (file + DB row)."""+ from app.services.db_sync_service import DatabaseSyncService+ from app.services.settings_service import SettingsService+ days = SettingsService.get(+ 'snapshot_retention_days',+ DatabaseSyncService.DEFAULT_SNAPSHOT_RETENTION_DAYS,+ )+ try:+ days = int(days)+ except (TypeError, ValueError):+ days = DatabaseSyncService.DEFAULT_SNAPSHOT_RETENTION_DAYS+ result = DatabaseSyncService.prune_expired_snapshots(retention_days=days)+ return result if isinstance(result, dict) else None+++def run_pairing_prune():+ """Prune expired pending agent pairings."""+ from app.services import pairing_service+ pairing_service.prune_expired()+++def run_registrar_expiry():+ """Notify when a registrar domain crosses an expiry threshold."""+ from app.services.registrar_service import RegistrarService+ n = RegistrarService.notify_expiring()+ if n:+ logger.info(f'Registrar expiry: sent {n} notification(s)')+ return {'notified': n}+ return None+++def run_api_background():+ """Hourly API analytics aggregation + event delivery retry."""+ from app.services.api_analytics_service import ApiAnalyticsService+ from app.services.event_service import EventService+ ApiAnalyticsService.aggregate_hourly()+ EventService.retry_failed()+++# ---------------------------------------------------------------------------+# Handler registration + schedule seeding+# ---------------------------------------------------------------------------+# (kind, handler, schedule-name, interval seconds, startup-delay seconds)+# The interval/delay pairs reproduce the original per-thread cadence: each+# former loop's sleep(settle) + sleep(interval) maps to startup_delay + interval.+_BUILTINS = [+ ('builtin.auto_sync', check_auto_sync_schedules, 'auto-sync', 60, 60),+ ('builtin.snapshot_retention', run_snapshot_retention, 'snapshot-retention', 3600, 120),+ ('builtin.workflow_schedules', check_workflow_schedules, 'workflow-schedules', 60, 60),+ ('builtin.health_check', run_health_checks, 'health-check', 300, 30),+ ('builtin.wp_update', check_update_schedules, 'wp-update', 60, 105),+ ('builtin.api_background', run_api_background, 'api-background', 3600, 3600),+ ('builtin.pairing_prune', run_pairing_prune, 'pairing-prune', 3600, 60),+ ('builtin.registrar_expiry', run_registrar_expiry, 'registrar-expiry', 86400, 300),+]+++def register_builtin_handlers():+ """Register all built-in periodic handlers. Pure in-memory; idempotent."""+ for kind, fn, _name, _interval, _delay in _BUILTINS:+ # Wrap so handlers match the fn(job) signature and ignore the job arg.+ register(kind, (lambda f: (lambda job: f()))(fn), replace=True)+++def seed_builtin_schedules():+ """Idempotently create the ScheduledJob rows for the built-in tasks."""+ from app.jobs.service import ScheduledJobService+ for kind, _fn, name, interval, delay in _BUILTINS:+ ScheduledJobService.ensure(+ name, kind, interval_seconds=interval, startup_delay_seconds=delay,+ )diff --git a/backend/app/jobs/consumer.py b/backend/app/jobs/consumer.pynew file mode 100644index 00000000..70061482--- /dev/null+++ b/backend/app/jobs/consumer.py@@ -0,0 +1,182 @@+"""The generic job consumer — the single worker that runs every enqueued Job.++Mirrors the webhook / notification consumers: a daemon thread polls the+``serverkit-system/jobs`` queue, but the per-message work is delegated to the+handler registered for the job's ``kind``. Retry / backoff / dead-lettering come+from the Queue Bus; this consumer maps the queue outcome back onto the Job row.+"""+import logging+import threading+import time+from datetime import datetime++from app import db+from app.jobs import registry+from app.jobs.models import Job+from app.jobs.service import GROUP_SLUG, QUEUE_SLUG, QUEUE_CONFIG+from app.queue_bus.service import QueueBusService++logger = logging.getLogger(__name__)++_job_consumer = None+++class JobConsumer:+ """Polls the queue bus for job messages and runs each via its handler."""++ def __init__(self, app=None, poll_interval_seconds=1, batch_size=5):+ self.app = app+ self.running = False+ self.poll_interval_seconds = poll_interval_seconds+ self.batch_size = batch_size++ def start(self):+ if self.running:+ return+ self.running = True+ self._ensure_queue()+ thread = threading.Thread(target=self._run, daemon=True, name='job-consumer')+ thread.start()+ logger.info('Job consumer started')++ def stop(self):+ self.running = False++ def _ensure_queue(self):+ with self.app.app_context():+ QueueBusService.ensure_queue(GROUP_SLUG, QUEUE_SLUG, config=QUEUE_CONFIG)++ def _run(self):+ while self.running:+ try:+ with self.app.app_context():+ self.process_batch()+ except Exception as e:+ logger.error(f'Job consumer error: {e}')+ time.sleep(self.poll_interval_seconds)++ def process_batch(self):+ messages = QueueBusService.receive(+ GROUP_SLUG, QUEUE_SLUG,+ visibility_timeout_ms=QUEUE_CONFIG['visibility_timeout_ms'],+ max_messages=self.batch_size,+ )+ for message in messages:+ try:+ self.process_message(message)+ except Exception as e: # pragma: no cover - defensive, per-message isolation+ logger.error(f'Job message {message.get("id")} crashed the consumer: {e}')+ return len(messages)++ def process_message(self, message):+ """Run one job. Safe to call directly (the test-suite does)."""+ payload = message.get('payload') or {}+ job_id = payload.get('job_id')+ if not job_id:+ QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, message['id'])+ return++ job = Job.query.get(job_id)+ if not job:+ QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, message['id'])+ return++ # Honor a cancellation requested before the job was picked up.+ if job.status == Job.STATUS_CANCELLED:+ QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, message['id'])+ return++ handler = registry.get(job.kind)+ if handler is None:+ # An unroutable job can never succeed — fail it without burning+ # retries, and complete the message so it doesn't loop.+ job.status = Job.STATUS_FAILED+ job.error_message = f'No handler registered for kind {job.kind!r}'+ job.completed_at = datetime.utcnow()+ db.session.commit()+ QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, message['id'])+ self._emit(job, 'job.failed')+ return++ job.status = Job.STATUS_RUNNING+ job.started_at = job.started_at or datetime.utcnow()+ job.attempts = message.get('attempts') or job.attempts or 0+ job.queue_message_id = message['id']+ db.session.commit()++ try:+ result = handler(job)+ job = Job.query.get(job_id) # handler may have committed/expired the row+ job.set_result(result)+ job.status = Job.STATUS_SUCCEEDED+ job.completed_at = datetime.utcnow()+ job.error_message = None+ db.session.commit()+ QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, message['id'])+ self._emit(job, 'job.succeeded')+ except Exception as e:+ logger.error(f'Job {job_id} ({job.kind}) failed: {e}')+ try:+ db.session.rollback()+ except Exception:+ pass+ self._fail(job_id, message, str(e))++ def _fail(self, job_id, message, error):+ outcome = None+ try:+ outcome = QueueBusService.fail(+ GROUP_SLUG, QUEUE_SLUG, message['id'], error_message=(error or '')[:500],+ )+ except Exception as e:+ logger.error(f'Failed to mark job message failed: {e}')++ job = Job.query.get(job_id)+ if job is None:+ return+ job.error_message = (error or '')[:2000]+ if outcome and outcome.get('status') == 'dead_letter':+ job.status = Job.STATUS_FAILED+ job.completed_at = datetime.utcnow()+ db.session.commit()+ self._emit(job, 'job.failed')+ else:+ # The queue will redeliver after a backoff; reflect that as pending.+ job.status = Job.STATUS_PENDING+ db.session.commit()++ @staticmethod+ def _emit(job, event_type):+ """Best-effort telemetry — never let observability break a job."""+ try:+ from app.services.telemetry_service import TelemetryService+ TelemetryService.emit(+ source='jobs',+ event_type=event_type,+ message=f'Job {event_type.split(".")[-1]}: {job.kind}',+ severity='error' if event_type == 'job.failed' else 'info',+ correlation_id=job.correlation_id,+ payload={'job_id': job.id, 'kind': job.kind, 'attempts': job.attempts},+ commit=True,+ )+ except Exception:+ pass+++def start_job_consumer(app):+ """Start the singleton job consumer thread (skipped under testing config)."""+ global _job_consumer+ if _job_consumer is not None:+ return+ if app.config.get('ENV') == 'testing' or app.config.get('TESTING'):+ return+ consumer = JobConsumer(app)+ consumer.start()+ _job_consumer = consumer+++def stop_job_consumer():+ global _job_consumer+ if _job_consumer is not None:+ _job_consumer.stop()+ _job_consumer = Nonediff --git a/backend/app/jobs/models.py b/backend/app/jobs/models.pynew file mode 100644index 00000000..d310a2e5--- /dev/null+++ b/backend/app/jobs/models.py@@ -0,0 +1,178 @@+"""Models for the unified background-job system.++``Job`` is a persisted unit of work with a stable lifecycle+(``pending -> running -> succeeded | failed | cancelled``). The Queue Bus stays+the transport: enqueuing a Job publishes a thin ``{'job_id': ...}`` message; the+``JobConsumer`` loads the row and dispatches by ``kind``. Retries / backoff /+dead-lettering are inherited from the queue — the Job row mirrors the outcome so+there is a single place to observe all background work.++``ScheduledJob`` is the periodic side: one ticker enqueues a Job for each due+schedule, replacing the per-domain daemon threads that each ran their own+``while True: sleep`` loop.+"""+from datetime import datetime, timedelta+import json+import uuid++from app import db+++class Job(db.Model):+ __tablename__ = 'jobs'++ STATUS_PENDING = 'pending'+ STATUS_RUNNING = 'running'+ STATUS_SUCCEEDED = 'succeeded'+ STATUS_FAILED = 'failed'+ STATUS_CANCELLED = 'cancelled'++ TERMINAL_STATUSES = (STATUS_SUCCEEDED, STATUS_FAILED, STATUS_CANCELLED)++ id = db.Column(db.String(36), primary_key=True, default=lambda: str(uuid.uuid4()))+ kind = db.Column(db.String(80), nullable=False, index=True)+ status = db.Column(db.String(20), default=STATUS_PENDING, nullable=False, index=True)++ payload = db.Column(db.Text, default='{}')+ result = db.Column(db.Text, nullable=True)+ error_message = db.Column(db.Text, nullable=True)++ attempts = db.Column(db.Integer, default=0, nullable=False)+ max_attempts = db.Column(db.Integer, default=3, nullable=False)+ priority = db.Column(db.Integer, default=0)++ # Loose ownership for filtering ("jobs for this server / app / user / schedule").+ owner_type = db.Column(db.String(40), nullable=True, index=True)+ owner_id = db.Column(db.String(64), nullable=True, index=True)++ # Backreference to the schedule that spawned this job, if periodic (no FK so+ # table create-order never matters).+ scheduled_job_id = db.Column(db.Integer, nullable=True, index=True)++ correlation_id = db.Column(db.String(64), nullable=True, index=True)+ # The queue message currently carrying this job (for ops / debugging).+ queue_message_id = db.Column(db.String(36), nullable=True)++ scheduled_at = db.Column(db.DateTime, nullable=True) # requested delay target+ created_at = db.Column(db.DateTime, default=datetime.utcnow, index=True)+ started_at = db.Column(db.DateTime, nullable=True)+ completed_at = db.Column(db.DateTime, nullable=True)+ updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)++ def get_payload(self):+ try:+ return json.loads(self.payload) if self.payload else {}+ except (TypeError, json.JSONDecodeError):+ return {}++ def set_payload(self, payload):+ self.payload = json.dumps(payload or {})++ def get_result(self):+ try:+ return json.loads(self.result) if self.result else None+ except (TypeError, json.JSONDecodeError):+ return None++ def set_result(self, result):+ """Store a JSON-serializable result; fall back to a repr so a weird+ return value never blocks marking a job done."""+ if result is None:+ self.result = None+ return+ try:+ self.result = json.dumps(result)+ except (TypeError, ValueError):+ self.result = json.dumps({'repr': str(result)[:2000]})++ @property+ def duration(self):+ if self.started_at and self.completed_at:+ return (self.completed_at - self.started_at).total_seconds()+ return None++ def to_dict(self, include_payload=False):+ data = {+ 'id': self.id,+ 'kind': self.kind,+ 'status': self.status,+ 'attempts': self.attempts,+ 'max_attempts': self.max_attempts,+ 'priority': self.priority,+ 'owner_type': self.owner_type,+ 'owner_id': self.owner_id,+ 'scheduled_job_id': self.scheduled_job_id,+ 'correlation_id': self.correlation_id,+ 'result': self.get_result(),+ 'error_message': self.error_message,+ 'created_at': self.created_at.isoformat() if self.created_at else None,+ 'started_at': self.started_at.isoformat() if self.started_at else None,+ 'completed_at': self.completed_at.isoformat() if self.completed_at else None,+ 'updated_at': self.updated_at.isoformat() if self.updated_at else None,+ 'duration': self.duration,+ }+ if include_payload:+ data['payload'] = self.get_payload()+ return data+++class ScheduledJob(db.Model):+ __tablename__ = 'scheduled_jobs'++ SCHEDULE_INTERVAL = 'interval'+ SCHEDULE_CRON = 'cron'++ id = db.Column(db.Integer, primary_key=True, autoincrement=True)+ name = db.Column(db.String(80), unique=True, nullable=False) # stable upsert key+ kind = db.Column(db.String(80), nullable=False) # job kind to enqueue+ schedule_kind = db.Column(db.String(20), default=SCHEDULE_INTERVAL, nullable=False)+ interval_seconds = db.Column(db.Integer, nullable=True)+ cron = db.Column(db.String(120), nullable=True)+ payload = db.Column(db.Text, default='{}')+ max_attempts = db.Column(db.Integer, default=1, nullable=False)+ enabled = db.Column(db.Boolean, default=True, nullable=False)++ next_run_at = db.Column(db.DateTime, nullable=True, index=True)+ last_run_at = db.Column(db.DateTime, nullable=True)+ last_job_id = db.Column(db.String(36), nullable=True)++ created_at = db.Column(db.DateTime, default=datetime.utcnow)+ updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)++ def get_payload(self):+ try:+ return json.loads(self.payload) if self.payload else {}+ except (TypeError, json.JSONDecodeError):+ return {}++ def set_payload(self, payload):+ self.payload = json.dumps(payload or {})++ def compute_next_run(self, base=None):+ base = base or datetime.utcnow()+ if self.schedule_kind == self.SCHEDULE_CRON and self.cron:+ try:+ from croniter import croniter+ if croniter.is_valid(self.cron):+ return croniter(self.cron, base).get_next(datetime)+ except ImportError:+ pass+ # Cron invalid / croniter unavailable — back off an hour rather than+ # hot-looping.+ return base + timedelta(hours=1)+ return base + timedelta(seconds=self.interval_seconds or 3600)++ def to_dict(self):+ return {+ 'id': self.id,+ 'name': self.name,+ 'kind': self.kind,+ 'schedule_kind': self.schedule_kind,+ 'interval_seconds': self.interval_seconds,+ 'cron': self.cron,+ 'enabled': self.enabled,+ 'max_attempts': self.max_attempts,+ 'next_run_at': self.next_run_at.isoformat() if self.next_run_at else None,+ 'last_run_at': self.last_run_at.isoformat() if self.last_run_at else None,+ 'last_job_id': self.last_job_id,+ }diff --git a/backend/app/jobs/registry.py b/backend/app/jobs/registry.pynew file mode 100644index 00000000..bcc9279c--- /dev/null+++ b/backend/app/jobs/registry.py@@ -0,0 +1,48 @@+"""In-process registry mapping a job ``kind`` to its handler.++A handler is a callable ``fn(job) -> result`` where ``job`` is the ``Job`` ORM+row. The return value (JSON-serializable, or ``None``) is stored as the job+result. Raising propagates to the consumer, which records the error and lets the+Queue Bus retry / dead-letter.+"""+import logging++logger = logging.getLogger(__name__)++_HANDLERS = {}+++def register(kind, handler, replace=False):+ """Register ``handler`` for ``kind``. Existing kinds are kept unless+ ``replace=True`` (so a plugin reload re-registers cleanly)."""+ if not kind or not callable(handler):+ raise ValueError('register(kind, handler): kind required and handler must be callable')+ if kind in _HANDLERS and not replace:+ logger.debug('Job handler for %r already registered; keeping existing', kind)+ return+ _HANDLERS[kind] = handler+++def handler(kind, replace=False):+ """Decorator form of :func:`register`."""+ def _decorator(fn):+ register(kind, fn, replace=replace)+ return fn+ return _decorator+++def get(kind):+ return _HANDLERS.get(kind)+++def is_registered(kind):+ return kind in _HANDLERS+++def registered_kinds():+ return sorted(_HANDLERS.keys())+++def clear():+ """Test helper — drop all registered handlers."""+ _HANDLERS.clear()diff --git a/backend/app/jobs/scheduler.py b/backend/app/jobs/scheduler.pynew file mode 100644index 00000000..295d03da--- /dev/null+++ b/backend/app/jobs/scheduler.py@@ -0,0 +1,79 @@+"""The single periodic ticker that replaces the per-domain scheduler threads.++It wakes on a short interval, finds due ``ScheduledJob`` rows, and enqueues a Job+for each (advancing ``next_run_at``). All cadence lives in the DB, so adding or+disabling a periodic task is a row change, not a new daemon thread.+"""+import logging+import threading+import time++from app.jobs.service import ScheduledJobService++logger = logging.getLogger(__name__)++_job_scheduler = None+++class JobScheduler:+ def __init__(self, app=None, tick_seconds=15):+ self.app = app+ self.running = False+ self.tick_seconds = tick_seconds++ def start(self):+ if self.running:+ return+ self.running = True+ thread = threading.Thread(target=self._run, daemon=True, name='job-scheduler')+ thread.start()+ logger.info('Job scheduler started')++ def stop(self):+ self.running = False++ def _run(self):+ while self.running:+ try:+ with self.app.app_context():+ self.tick()+ except Exception as e:+ logger.error(f'Job scheduler error: {e}')+ time.sleep(self.tick_seconds)++ def tick(self):+ """Enqueue all due schedules. Returns the number fired. Test-callable."""+ fired = 0+ for scheduled in ScheduledJobService.due():+ try:+ ScheduledJobService.fire(scheduled)+ fired += 1+ except Exception as e:+ logger.error(f'Failed to fire scheduled job {scheduled.name}: {e}')+ # Advance anyway so a poison schedule can't hot-loop the ticker.+ try:+ from app import db+ scheduled.next_run_at = scheduled.compute_next_run()+ db.session.commit()+ except Exception:+ db.session.rollback()+ return fired+++def start_job_scheduler(app):+ """Start the singleton scheduler thread (skipped under testing config)."""+ global _job_scheduler+ if _job_scheduler is not None:+ return+ if app.config.get('ENV') == 'testing' or app.config.get('TESTING'):+ return+ scheduler = JobScheduler(app)+ scheduler.start()+ _job_scheduler = scheduler+++def stop_job_scheduler():+ global _job_scheduler+ if _job_scheduler is not None:+ _job_scheduler.stop()+ _job_scheduler = Nonediff --git a/backend/app/jobs/sdk.py b/backend/app/jobs/sdk.pynew file mode 100644index 00000000..0374101a--- /dev/null+++ b/backend/app/jobs/sdk.py@@ -0,0 +1,42 @@+"""Plugin/consumer-facing SDK for the unified job system.++ from app.plugins_sdk import jobs++ jobs.register('myplugin.reindex', handler_fn)+ jobs.enqueue('myplugin.reindex', {'site_id': 5})+ jobs.schedule('myplugin-nightly', 'myplugin.reindex', cron='0 3 * * *')+"""+from app.jobs import registry+from app.jobs.service import JobService, ScheduledJobService+++class JobsSdk:+ """Stable jobs surface for plugins and internal callers."""++ def register(self, kind, handler, replace=False):+ """Register a handler ``fn(job) -> result`` for a job ``kind``."""+ return registry.register(kind, handler, replace=replace)++ def enqueue(self, kind, payload=None, max_attempts=3, priority=0, delay_ms=0,+ owner_type=None, owner_id=None):+ job = JobService.enqueue(+ kind, payload=payload, max_attempts=max_attempts, priority=priority,+ delay_ms=delay_ms, owner_type=owner_type, owner_id=owner_id,+ )+ return job.to_dict()++ def get(self, job_id):+ job = JobService.get(job_id)+ return job.to_dict(include_payload=True) if job else None++ def cancel(self, job_id):+ job = JobService.cancel(job_id)+ return job.to_dict() if job else None++ def schedule(self, name, kind, interval_seconds=None, cron=None, payload=None,+ max_attempts=1, startup_delay_seconds=0):+ scheduled = ScheduledJobService.ensure(+ name, kind, interval_seconds=interval_seconds, cron=cron, payload=payload,+ max_attempts=max_attempts, startup_delay_seconds=startup_delay_seconds,+ )+ return scheduled.to_dict()diff --git a/backend/app/jobs/service.py b/backend/app/jobs/service.pynew file mode 100644index 00000000..d6ab1790--- /dev/null+++ b/backend/app/jobs/service.py@@ -0,0 +1,216 @@+"""Producer + management API for the unified job system.++``JobService`` is the one entry point to enqueue work; ``ScheduledJobService``+manages periodic schedules. Both are non-blocking: they persist a row and (for+jobs) publish a thin pointer onto the Queue Bus.+"""+import logging+from datetime import datetime, timedelta++from app import db+from app.jobs.models import Job, ScheduledJob+from app.queue_bus.service import QueueBusService++logger = logging.getLogger(__name__)++GROUP_SLUG = 'serverkit-system'+QUEUE_SLUG = 'jobs'+# Jobs can run longer than a webhook POST, so give them a wider visibility+# window before the queue considers a message abandoned.+QUEUE_CONFIG = {'max_attempts': 3, 'visibility_timeout_ms': 120000}+++def _new_correlation_id():+ try:+ from app.services.telemetry_service import generate_correlation_id+ return generate_correlation_id()+ except Exception:+ import uuid+ return uuid.uuid4().hex+++class JobService:++ @classmethod+ def enqueue(cls, kind, payload=None, max_attempts=3, priority=0, delay_ms=0,+ owner_type=None, owner_id=None, correlation_id=None):+ """Persist a Job and publish it onto the Queue Bus. Returns the Job."""+ job = Job(+ kind=kind,+ status=Job.STATUS_PENDING,+ max_attempts=max_attempts,+ priority=priority,+ owner_type=owner_type,+ owner_id=str(owner_id) if owner_id is not None else None,+ correlation_id=correlation_id or _new_correlation_id(),+ )+ job.set_payload(payload or {})+ if delay_ms:+ job.scheduled_at = datetime.utcnow() + timedelta(milliseconds=delay_ms)+ db.session.add(job)+ db.session.commit()+ cls._publish(job, delay_ms=delay_ms)+ return job++ @classmethod+ def _publish(cls, job, delay_ms=0):+ QueueBusService.ensure_queue(GROUP_SLUG, QUEUE_SLUG, config=QUEUE_CONFIG)+ msg = QueueBusService.send(+ GROUP_SLUG, QUEUE_SLUG, {'job_id': job.id},+ priority=job.priority or 0, delay_ms=delay_ms,+ max_attempts=job.max_attempts,+ )+ if isinstance(msg, dict):+ job.queue_message_id = msg.get('id')+ db.session.commit()+ return msg++ @classmethod+ def get(cls, job_id):+ return Job.query.get(job_id)++ @classmethod+ def list(cls, status=None, kind=None, owner_type=None, owner_id=None, limit=50, offset=0):+ query = Job.query+ if status:+ query = query.filter_by(status=status)+ if kind:+ query = query.filter_by(kind=kind)+ if owner_type:+ query = query.filter_by(owner_type=owner_type)+ if owner_id is not None:+ query = query.filter_by(owner_id=str(owner_id))+ return query.order_by(Job.created_at.desc()).limit(limit).offset(offset).all()++ @classmethod+ def cancel(cls, job_id):+ """Mark a pending/running job cancelled. Cannot interrupt a handler that+ is already executing — it stops future attempts and is skipped at the+ next queue pickup."""+ job = Job.query.get(job_id)+ if not job:+ return None+ if job.status in (Job.STATUS_PENDING, Job.STATUS_RUNNING):+ job.status = Job.STATUS_CANCELLED+ job.completed_at = datetime.utcnow()+ db.session.commit()+ return job++ @classmethod+ def retry(cls, job_id):+ """Re-enqueue a failed/cancelled job as a fresh attempt."""+ job = Job.query.get(job_id)+ if not job:+ return None+ if job.status not in (Job.STATUS_FAILED, Job.STATUS_CANCELLED):+ return job+ job.status = Job.STATUS_PENDING+ job.error_message = None+ job.started_at = None+ job.completed_at = None+ db.session.commit()+ cls._publish(job)+ return job++ @classmethod+ def stats(cls):+ from sqlalchemy import func+ by_status = dict(+ db.session.query(Job.status, func.count(Job.id)).group_by(Job.status).all()+ )+ by_kind = dict(+ db.session.query(Job.kind, func.count(Job.id)).group_by(Job.kind).all()+ )+ return {'by_status': by_status, 'by_kind': by_kind, 'total': sum(by_status.values())}++ @classmethod+ def cleanup_old(cls, max_age_seconds=86400):+ """Delete terminal jobs whose completion is older than the window."""+ cutoff = datetime.utcnow() - timedelta(seconds=max_age_seconds)+ deleted = (Job.query+ .filter(Job.status.in_(Job.TERMINAL_STATUSES))+ .filter(Job.completed_at.isnot(None))+ .filter(Job.completed_at < cutoff)+ .delete(synchronize_session=False))+ db.session.commit()+ return deleted+++class ScheduledJobService:++ @classmethod+ def ensure(cls, name, kind, interval_seconds=None, cron=None, payload=None,+ max_attempts=1, startup_delay_seconds=0, enabled=True):+ """Idempotently create/update a periodic schedule keyed by ``name``.++ Updates cadence/kind on an existing row but PRESERVES its+ ``next_run_at`` / ``last_run_at`` and the admin's ``enabled`` toggle so a+ restart doesn't reset the clock. On first creation, seeds ``next_run_at``+ honoring ``startup_delay_seconds``.+ """+ scheduled = ScheduledJob.query.filter_by(name=name).first()+ created = scheduled is None+ if created:+ scheduled = ScheduledJob(name=name)+ db.session.add(scheduled)+ scheduled.kind = kind+ scheduled.schedule_kind = ScheduledJob.SCHEDULE_CRON if cron else ScheduledJob.SCHEDULE_INTERVAL+ scheduled.interval_seconds = interval_seconds+ scheduled.cron = cron+ scheduled.max_attempts = max_attempts+ if payload is not None:+ scheduled.set_payload(payload)+ if created:+ scheduled.enabled = enabled+ scheduled.next_run_at = datetime.utcnow() + timedelta(seconds=startup_delay_seconds or 0)+ db.session.commit()+ return scheduled++ @classmethod+ def due(cls, now=None):+ now = now or datetime.utcnow()+ return (ScheduledJob.query+ .filter(ScheduledJob.enabled.is_(True))+ .filter(ScheduledJob.next_run_at.isnot(None))+ .filter(ScheduledJob.next_run_at <= now)+ .all())++ @classmethod+ def fire(cls, scheduled, now=None):+ """Enqueue a Job for one schedule and advance its ``next_run_at``."""+ now = now or datetime.utcnow()+ job = JobService.enqueue(+ scheduled.kind,+ payload=scheduled.get_payload(),+ max_attempts=scheduled.max_attempts,+ owner_type='schedule',+ owner_id=scheduled.name,+ )+ job.scheduled_job_id = scheduled.id+ scheduled.last_run_at = now+ scheduled.last_job_id = job.id+ scheduled.next_run_at = scheduled.compute_next_run(now)+ db.session.commit()+ return job++ @classmethod+ def list(cls):+ return ScheduledJob.query.order_by(ScheduledJob.name.asc()).all()++ @classmethod+ def run_now(cls, scheduled_job_id):+ scheduled = ScheduledJob.query.get(scheduled_job_id)+ if not scheduled:+ return None+ return cls.fire(scheduled)++ @classmethod+ def set_enabled(cls, scheduled_job_id, enabled):+ scheduled = ScheduledJob.query.get(scheduled_job_id)+ if not scheduled:+ return None+ scheduled.enabled = bool(enabled)+ if enabled and not scheduled.next_run_at:+ scheduled.next_run_at = datetime.utcnow()+ db.session.commit()+ return scheduleddiff --git a/backend/app/plugins_sdk/__init__.py b/backend/app/plugins_sdk/__init__.pyindex 017cbfbb..e82aa5c7 100644--- a/backend/app/plugins_sdk/__init__.py+++ b/backend/app/plugins_sdk/__init__.py@@ -90,9 +90,11 @@ def audit(action, target_type, target_id=None, details=None, user_id=None):from app.plugins_sdk import aifrom app.queue_bus.sdk import QueueBusSdkfrom app.notifications.sdk import NotifySdk+from app.jobs.sdk import JobsSdkqueue = QueueBusSdk()notify = NotifySdk()+jobs = JobsSdk()__all__ = ['db',@@ -106,4 +108,5 @@ __all__ = ['ai','queue','notify',+ 'jobs',]diff --git a/backend/app/services/background_job_service.py b/backend/app/services/background_job_service.pydeleted file mode 100644index c5e932ae..00000000--- a/backend/app/services/background_job_service.py+++ /dev/null@@ -1,123 +0,0 @@-import logging-import threading-import queue-import time-from datetime import datetime-from functools import wraps--logger = logging.getLogger(__name__)---class BackgroundJobService:- """Simple background job queue for long-running tasks."""-- _queue = queue.Queue()- _workers = []- _results = {}- _lock = threading.Lock()- _running = False-- @classmethod- def start(cls, app, num_workers=3):- """Start background worker threads."""- if cls._running:- return-- cls._running = True-- for i in range(num_workers):- t = threading.Thread(- target=cls._worker_loop,- args=(app,),- daemon=True,- name=f'bg-worker-{i}',- )- t.start()- cls._workers.append(t)-- logger.info(f'Background job service started with {num_workers} workers')-- @classmethod- def _worker_loop(cls, app):- while cls._running:- try:- job = cls._queue.get(timeout=1)- except queue.Empty:- continue-- job_id = job['id']- with cls._lock:- cls._results[job_id] = {'status': 'running', 'started_at': datetime.utcnow().isoformat()}-- try:- with app.app_context():- result = job['func'](*job.get('args', ()), **job.get('kwargs', {}))- with cls._lock:- cls._results[job_id] = {- 'status': 'completed',- 'result': result,- 'completed_at': datetime.utcnow().isoformat(),- }- except Exception as e:- logger.error(f'Background job {job_id} failed: {e}')- with cls._lock:- cls._results[job_id] = {- 'status': 'failed',- 'error': str(e),- 'completed_at': datetime.utcnow().isoformat(),- }- finally:- cls._queue.task_done()-- @classmethod- def enqueue(cls, func, *args, job_id=None, **kwargs):- """Add a job to the queue. Returns job_id."""- import uuid- job_id = job_id or str(uuid.uuid4())[:8]-- with cls._lock:- cls._results[job_id] = {'status': 'queued', 'queued_at': datetime.utcnow().isoformat()}-- cls._queue.put({- 'id': job_id,- 'func': func,- 'args': args,- 'kwargs': kwargs,- })-- return job_id-- @classmethod- def get_job_status(cls, job_id):- with cls._lock:- return cls._results.get(job_id)-- @classmethod- def list_jobs(cls):- with cls._lock:- return dict(cls._results)-- @classmethod- def cleanup_old(cls, max_age_seconds=3600):- """Remove completed/failed jobs older than max_age."""- now = datetime.utcnow()- with cls._lock:- to_remove = []- for jid, info in cls._results.items():- if info['status'] in ('completed', 'failed'):- completed_at = info.get('completed_at')- if completed_at:- dt = datetime.fromisoformat(completed_at)- if (now - dt).total_seconds() > max_age_seconds:- to_remove.append(jid)- for jid in to_remove:- del cls._results[jid]-- @classmethod- def get_queue_stats(cls):- return {- 'queue_size': cls._queue.qsize(),- 'workers': len(cls._workers),- 'total_jobs': len(cls._results),- 'running': cls._running,- }diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.pynew file mode 100644index 00000000..89b0317e--- /dev/null+++ b/backend/tests/test_jobs.py@@ -0,0 +1,213 @@+"""Tests for the unified job system (app/jobs)."""+import pytest++from app import db+from app.jobs import registry+from app.jobs.models import Job, ScheduledJob+from app.jobs.service import JobService, ScheduledJobService, GROUP_SLUG, QUEUE_SLUG+from app.jobs.consumer import JobConsumer+from app.jobs.scheduler import JobScheduler+from app.queue_bus.service import QueueBusService+from app.queue_bus.models import QueueMessage++[email protected](autouse=True)+def reset_jobs(app):+ """Clean broker + handler registry before each test (after create_app has+ registered the built-in handlers)."""+ QueueBusService.reset_broker()+ registry.clear()+ yield+ registry.clear()+++def _drain_once(consumer=None):+ """Receive and process exactly one job message; return it processed."""+ consumer = consumer or JobConsumer()+ messages = QueueBusService.receive(GROUP_SLUG, QUEUE_SLUG, max_messages=1)+ assert messages, 'expected a queued job message'+ consumer.process_message(messages[0])+ return messages[0]+++class TestEnqueueAndRun:+ def test_enqueue_creates_job_and_message(self, app):+ job = JobService.enqueue('test.noop', {'x': 1})+ assert job.id is not None+ assert job.status == Job.STATUS_PENDING+ assert Job.query.count() == 1+ # A thin {job_id} pointer is on the bus.+ msgs = QueueBusService.list_messages(GROUP_SLUG, QUEUE_SLUG)+ assert len(msgs) == 1+ assert msgs[0]['payload'] == {'job_id': job.id}++ def test_handler_runs_and_succeeds(self, app):+ seen = {}++ @registry.handler('test.ok')+ def _ok(job):+ seen['payload'] = job.get_payload()+ return {'done': True}++ job = JobService.enqueue('test.ok', {'x': 42})+ _drain_once()++ refreshed = Job.query.get(job.id)+ assert refreshed.status == Job.STATUS_SUCCEEDED+ assert refreshed.get_result() == {'done': True}+ assert refreshed.completed_at is not None+ assert seen['payload'] == {'x': 42}++ def test_unregistered_kind_fails_without_retrying(self, app):+ job = JobService.enqueue('test.no_handler', {}, max_attempts=3)+ msg = _drain_once()++ refreshed = Job.query.get(job.id)+ assert refreshed.status == Job.STATUS_FAILED+ assert 'No handler' in (refreshed.error_message or '')+ # Message completed (not left to retry an unroutable job).+ qm = QueueBusService.get_message(GROUP_SLUG, QUEUE_SLUG, msg['id'])+ assert qm['status'] == QueueMessage.STATUS_COMPLETED++ def test_failure_retries_then_dead_letters_to_failed(self, app):+ @registry.handler('test.boom')+ def _boom(job):+ raise RuntimeError('kaboom')++ # max_attempts=1 → the first failure exhausts attempts → dead-letter.+ job = JobService.enqueue('test.boom', {}, max_attempts=1)+ msg = _drain_once()++ refreshed = Job.query.get(job.id)+ assert refreshed.status == Job.STATUS_FAILED+ assert 'kaboom' in (refreshed.error_message or '')+ qm = QueueBusService.get_message(GROUP_SLUG, QUEUE_SLUG, msg['id'])+ assert qm['status'] == QueueMessage.STATUS_DEAD_LETTER++ def test_failure_with_attempts_left_stays_pending(self, app):+ @registry.handler('test.flaky')+ def _flaky(job):+ raise RuntimeError('again')++ job = JobService.enqueue('test.flaky', {}, max_attempts=3)+ _drain_once()++ refreshed = Job.query.get(job.id)+ # Not terminal yet — the queue will redeliver after backoff.+ assert refreshed.status == Job.STATUS_PENDING+ assert refreshed.status not in Job.TERMINAL_STATUSES or refreshed.status == Job.STATUS_PENDING+++class TestCancelAndRetry:+ def test_cancel_prevents_execution(self, app):+ ran = {'count': 0}++ @registry.handler('test.cancelme')+ def _h(job):+ ran['count'] += 1++ job = JobService.enqueue('test.cancelme', {})+ JobService.cancel(job.id)+ assert Job.query.get(job.id).status == Job.STATUS_CANCELLED++ _drain_once() # consumer should skip the cancelled job+ assert ran['count'] == 0+ assert Job.query.get(job.id).status == Job.STATUS_CANCELLED++ def test_retry_requeues_failed_job(self, app):+ job = JobService.enqueue('test.no_handler', {}, max_attempts=1)+ _drain_once()+ assert Job.query.get(job.id).status == Job.STATUS_FAILED++ # Now register a handler and retry.+ @registry.handler('test.no_handler')+ def _now_ok(job):+ return 'recovered'++ JobService.retry(job.id)+ assert Job.query.get(job.id).status == Job.STATUS_PENDING+ _drain_once()+ assert Job.query.get(job.id).status == Job.STATUS_SUCCEEDED+ assert Job.query.get(job.id).get_result() == 'recovered'+++class TestScheduler:+ def test_ensure_is_idempotent(self, app):+ a = ScheduledJobService.ensure('nightly', 'test.x', interval_seconds=60)+ b = ScheduledJobService.ensure('nightly', 'test.x', interval_seconds=120)+ assert a.id == b.id+ assert ScheduledJob.query.count() == 1+ assert ScheduledJob.query.get(a.id).interval_seconds == 120++ def test_tick_fires_due_schedule_and_advances(self, app):+ # startup_delay 0 → next_run_at = now → immediately due.+ scheduled = ScheduledJobService.ensure(+ 'due-now', 'test.tick', interval_seconds=3600, startup_delay_seconds=0)+ before = scheduled.next_run_at++ fired = JobScheduler().tick()+ assert fired == 1++ # A job was enqueued for the schedule, and next_run advanced past now.+ job = Job.query.filter_by(kind='test.tick').first()+ assert job is not None+ assert job.owner_type == 'schedule'+ assert job.owner_id == 'due-now'+ refreshed = ScheduledJob.query.get(scheduled.id)+ assert refreshed.next_run_at > before+ assert refreshed.last_job_id == job.id+ # No longer due.+ assert JobScheduler().tick() == 0++ def test_disabled_schedule_does_not_fire(self, app):+ scheduled = ScheduledJobService.ensure(+ 'off', 'test.tick', interval_seconds=60, startup_delay_seconds=0)+ ScheduledJobService.set_enabled(scheduled.id, False)+ assert JobScheduler().tick() == 0+++class TestBuiltins:+ def test_register_and_seed(self, app):+ from app.jobs import builtin_handlers+ builtin_handlers.register_builtin_handlers()+ kinds = registry.registered_kinds()+ assert 'builtin.auto_sync' in kinds+ assert 'builtin.health_check' in kinds+ assert len([k for k in kinds if k.startswith('builtin.')]) == 8++ builtin_handlers.seed_builtin_schedules()+ assert ScheduledJob.query.count() == 8+ # Seeding twice doesn't duplicate.+ builtin_handlers.seed_builtin_schedules()+ assert ScheduledJob.query.count() == 8+++class TestApi:+ def test_list_and_get_via_api(self, client, auth_headers, app):+ job = JobService.enqueue('test.api', {'hello': 'world'})++ resp = client.get('/api/v1/jobs', headers=auth_headers)+ assert resp.status_code == 200+ assert any(j['id'] == job.id for j in resp.get_json()['jobs'])++ resp = client.get(f'/api/v1/jobs/{job.id}', headers=auth_headers)+ assert resp.status_code == 200+ assert resp.get_json()['job']['payload'] == {'hello': 'world'}++ def test_stats_endpoint(self, client, auth_headers, app):+ JobService.enqueue('test.api', {})+ resp = client.get('/api/v1/jobs/stats', headers=auth_headers)+ assert resp.status_code == 200+ assert resp.get_json()['total'] >= 1++ def test_cancel_via_api(self, client, auth_headers, app):+ job = JobService.enqueue('test.api', {})+ resp = client.post(f'/api/v1/jobs/{job.id}/cancel', headers=auth_headers)+ assert resp.status_code == 200+ assert resp.get_json()['job']['status'] == Job.STATUS_CANCELLED++ def test_scheduled_listing(self, client, auth_headers, app):+ ScheduledJobService.ensure('api-sched', 'test.x', interval_seconds=60)+ resp = client.get('/api/v1/jobs/scheduled', headers=auth_headers)+ assert resp.status_code == 200+ assert any(s['name'] == 'api-sched' for s in resp.get_json()['scheduled'])