ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
Recover jobs stranded by a panel restart
Juan Denis47df5167cb6f
diff --git a/backend/app/__init__.py b/backend/app/__init__.pyindex b5a41b65..843b473a 100644--- a/backend/app/__init__.py+++ b/backend/app/__init__.py@@ -407,6 +407,16 @@ def create_app(config_name=None):from app.utils.env import env_bool_skip_background = env_bool('SERVERKIT_SKIP_BACKGROUND') or _cli_one_shotif not _skip_background:+ # Before any consumer polls: release what the previous process+ # left in flight, or it stays stranded forever.+ if not app.config.get('TESTING'):+ from app.jobs.service import JobService+ try:+ JobService.recover_interrupted()+ except Exception:+ import logging+ logging.getLogger(__name__).exception('Interrupted-job recovery failed')+# Start metrics history collection in backgroundfrom app.services.metrics_history_service import MetricsHistoryServiceif not MetricsHistoryService.is_running():diff --git a/backend/app/jobs/service.py b/backend/app/jobs/service.pyindex 3f3c66d3..6d52a02e 100644--- a/backend/app/jobs/service.py+++ b/backend/app/jobs/service.py@@ -52,6 +52,31 @@ class JobService:cls._publish(job, delay_ms=delay_ms)return job+ @classmethod+ def recover_interrupted(cls):+ """Boot-time cleanup after a restart; call before any consumer starts.++ A job still 'running' lost its runner with the old process. It is+ failed, not re-run: a half-finished deploy or restore is not safe to+ repeat blind. Every other stranded queue message is redelivered.+ """+ now = datetime.utcnow()+ interrupted = Job.query.filter_by(status=Job.STATUS_RUNNING).all()+ for job in interrupted:+ job.status = Job.STATUS_FAILED+ job.completed_at = now+ job.error_message = ('Interrupted: the panel restarted while this '+ 'job was running. Run it again if needed.')+ if job.queue_message_id:+ try:+ QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, job.queue_message_id)+ except Exception:+ logger.warning('No queue message %s for job %s',+ job.queue_message_id, job.id)+ db.session.commit()+ released = QueueBusService.recover_in_flight()+ return {'failed_jobs': len(interrupted), 'released_messages': released}+@classmethoddef _publish(cls, job, delay_ms=0):QueueBusService.ensure_queue(GROUP_SLUG, QUEUE_SLUG, config=QUEUE_CONFIG)diff --git a/backend/app/queue_bus/broker.py b/backend/app/queue_bus/broker.pyindex 0a8ac9fd..1e6da3ec 100644--- a/backend/app/queue_bus/broker.py+++ b/backend/app/queue_bus/broker.py@@ -65,6 +65,10 @@ class AbstractBroker(ABC):def fail(self, group_slug, queue_slug, message_id, error_message=None, requeue=False):"""Mark a message failed. If requeue is False, retry or dead-letter based on attempts."""+ @abstractmethod+ def recover_in_flight(self):+ """Release every in-flight message at boot. Returns how many moved."""+@abstractmethoddef requeue(self, group_slug, queue_slug, message_id):"""Move a failed/dead-letter message back to pending."""diff --git a/backend/app/queue_bus/service.py b/backend/app/queue_bus/service.pyindex aec04969..995eeb79 100644--- a/backend/app/queue_bus/service.py+++ b/backend/app/queue_bus/service.py@@ -118,6 +118,10 @@ class QueueBusService:max_messages=max_messages,)+ @classmethod+ def recover_in_flight(cls):+ return cls.broker().recover_in_flight()+@classmethoddef complete(cls, group_slug, queue_slug, message_id):return cls.broker().complete(group_slug, queue_slug, message_id)diff --git a/backend/app/queue_bus/sqlalchemy_broker.py b/backend/app/queue_bus/sqlalchemy_broker.pyindex 8f5824fd..fbc60e71 100644--- a/backend/app/queue_bus/sqlalchemy_broker.py+++ b/backend/app/queue_bus/sqlalchemy_broker.py@@ -216,6 +216,24 @@ class SQLAlchemyBroker(AbstractBroker):db.session.commit()return message.to_dict()+ def recover_in_flight(self):+ # Only safe at boot: the panel is one process, so anything still+ # in flight belonged to a process that is gone and will never+ # complete it. Redeliver it, or dead-letter it if out of attempts.+ stranded = QueueMessage.query.filter(+ QueueMessage.status == QueueMessage.STATUS_IN_FLIGHT).all()+ for message in stranded:+ message.invisible_until = None+ if message.attempts >= message.max_attempts:+ message.status = QueueMessage.STATUS_DEAD_LETTER+ message.error_message = (message.error_message or '') + (+ f'[{datetime.utcnow().isoformat()}] interrupted by a restart\n')+ else:+ message.status = QueueMessage.STATUS_PENDING+ message.visible_after = datetime.utcnow()+ db.session.commit()+ return len(stranded)+def requeue(self, group_slug, queue_slug, message_id):message = self._get_message_or_raise(group_slug, queue_slug, message_id)if message.status not in (QueueMessage.STATUS_FAILED, QueueMessage.STATUS_DEAD_LETTER):diff --git a/backend/tests/test_jobs_restart_recovery.py b/backend/tests/test_jobs_restart_recovery.pynew file mode 100644index 00000000..63787f46--- /dev/null+++ b/backend/tests/test_jobs_restart_recovery.py@@ -0,0 +1,74 @@+"""A restart must not strand queued work.++The broker only ever delivers 'pending' messages, so a message left+'in_flight' by a dead process was never delivered again, and its Job row+read 'running' forever. Boot recovery fails interrupted jobs visibly and+puts every other stranded message back on the queue.+"""+import pytest++from app import db+from app.jobs import registry+from app.jobs.consumer import JobConsumer+from app.jobs.models import Job+from app.jobs.service import GROUP_SLUG, QUEUE_SLUG, JobService+from app.queue_bus.models import QueueMessage+from app.queue_bus.service import QueueBusService++[email protected](autouse=True)+def clean_bus(app):+ QueueBusService.reset_broker()+ registry.clear()+ yield+ registry.clear()+++def _status(message_id):+ return db.session.get(QueueMessage, message_id).status+++def test_job_running_at_restart_fails_and_is_not_rerun(app):+ ran = []+ registry.register('test.deploy', lambda job: ran.append(job.id))+ job = JobService.enqueue('test.deploy')+ message = QueueBusService.receive(GROUP_SLUG, QUEUE_SLUG)[0]+ job.status = Job.STATUS_RUNNING # the old process died mid-handler+ db.session.commit()++ result = JobService.recover_interrupted()++ job = db.session.get(Job, job.id)+ assert result['failed_jobs'] == 1+ assert job.status == Job.STATUS_FAILED+ assert 'restarted' in job.error_message+ assert _status(message['id']) == QueueMessage.STATUS_COMPLETED+ assert QueueBusService.receive(GROUP_SLUG, QUEUE_SLUG) == []+ assert ran == []+++def test_message_received_but_not_started_is_redelivered(app):+ ran = []+ registry.register('test.tick', lambda job: ran.append(job.id))+ job = JobService.enqueue('test.tick')+ QueueBusService.receive(GROUP_SLUG, QUEUE_SLUG) # received, then the process died+ assert QueueBusService.receive(GROUP_SLUG, QUEUE_SLUG) == []++ assert JobService.recover_interrupted()['released_messages'] == 1++ redelivered = QueueBusService.receive(GROUP_SLUG, QUEUE_SLUG)+ assert len(redelivered) == 1+ JobConsumer().process_message(redelivered[0])+ assert ran == [job.id]+ assert db.session.get(Job, job.id).status == Job.STATUS_SUCCEEDED+++def test_stranded_message_out_of_attempts_is_dead_lettered(app):+ QueueBusService.ensure_queue('g', 'q')+ sent = QueueBusService.send('g', 'q', {'n': 1}, max_attempts=1)+ QueueBusService.receive('g', 'q')++ QueueBusService.recover_in_flight()++ assert _status(sent['id']) == QueueMessage.STATUS_DEAD_LETTER+ assert QueueBusService.receive('g', 'q') == []