Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Recover jobs stranded by a panel restart

Juan Denis47df5167cb6f
diff --git a/backend/app/__init__.py b/backend/app/__init__.py
index b5a41b65..843b473a 100644
--- a/backend/app/__init__.py
+++ b/backend/app/__init__.py
@@ -407,6 +407,16 @@ def create_app(config_name=None):
from app.utils.env import env_bool
_skip_background = env_bool('SERVERKIT_SKIP_BACKGROUND') or _cli_one_shot
if not _skip_background:
+ # Before any consumer polls: release what the previous process
+ # left in flight, or it stays stranded forever.
+ if not app.config.get('TESTING'):
+ from app.jobs.service import JobService
+ try:
+ JobService.recover_interrupted()
+ except Exception:
+ import logging
+ logging.getLogger(__name__).exception('Interrupted-job recovery failed')
+
# Start metrics history collection in background
from app.services.metrics_history_service import MetricsHistoryService
if not MetricsHistoryService.is_running():
diff --git a/backend/app/jobs/service.py b/backend/app/jobs/service.py
index 3f3c66d3..6d52a02e 100644
--- a/backend/app/jobs/service.py
+++ b/backend/app/jobs/service.py
@@ -52,6 +52,31 @@ class JobService:
cls._publish(job, delay_ms=delay_ms)
return job
+ @classmethod
+ def recover_interrupted(cls):
+ """Boot-time cleanup after a restart; call before any consumer starts.
+
+ A job still 'running' lost its runner with the old process. It is
+ failed, not re-run: a half-finished deploy or restore is not safe to
+ repeat blind. Every other stranded queue message is redelivered.
+ """
+ now = datetime.utcnow()
+ interrupted = Job.query.filter_by(status=Job.STATUS_RUNNING).all()
+ for job in interrupted:
+ job.status = Job.STATUS_FAILED
+ job.completed_at = now
+ job.error_message = ('Interrupted: the panel restarted while this '
+ 'job was running. Run it again if needed.')
+ if job.queue_message_id:
+ try:
+ QueueBusService.complete(GROUP_SLUG, QUEUE_SLUG, job.queue_message_id)
+ except Exception:
+ logger.warning('No queue message %s for job %s',
+ job.queue_message_id, job.id)
+ db.session.commit()
+ released = QueueBusService.recover_in_flight()
+ return {'failed_jobs': len(interrupted), 'released_messages': released}
+
@classmethod
def _publish(cls, job, delay_ms=0):
QueueBusService.ensure_queue(GROUP_SLUG, QUEUE_SLUG, config=QUEUE_CONFIG)
diff --git a/backend/app/queue_bus/broker.py b/backend/app/queue_bus/broker.py
index 0a8ac9fd..1e6da3ec 100644
--- a/backend/app/queue_bus/broker.py
+++ b/backend/app/queue_bus/broker.py
@@ -65,6 +65,10 @@ class AbstractBroker(ABC):
def fail(self, group_slug, queue_slug, message_id, error_message=None, requeue=False):
"""Mark a message failed. If requeue is False, retry or dead-letter based on attempts."""
+ @abstractmethod
+ def recover_in_flight(self):
+ """Release every in-flight message at boot. Returns how many moved."""
+
@abstractmethod
def requeue(self, group_slug, queue_slug, message_id):
"""Move a failed/dead-letter message back to pending."""
diff --git a/backend/app/queue_bus/service.py b/backend/app/queue_bus/service.py
index aec04969..995eeb79 100644
--- a/backend/app/queue_bus/service.py
+++ b/backend/app/queue_bus/service.py
@@ -118,6 +118,10 @@ class QueueBusService:
max_messages=max_messages,
)
+ @classmethod
+ def recover_in_flight(cls):
+ return cls.broker().recover_in_flight()
+
@classmethod
def complete(cls, group_slug, queue_slug, message_id):
return cls.broker().complete(group_slug, queue_slug, message_id)
diff --git a/backend/app/queue_bus/sqlalchemy_broker.py b/backend/app/queue_bus/sqlalchemy_broker.py
index 8f5824fd..fbc60e71 100644
--- a/backend/app/queue_bus/sqlalchemy_broker.py
+++ b/backend/app/queue_bus/sqlalchemy_broker.py
@@ -216,6 +216,24 @@ class SQLAlchemyBroker(AbstractBroker):
db.session.commit()
return message.to_dict()
+ def recover_in_flight(self):
+ # Only safe at boot: the panel is one process, so anything still
+ # in flight belonged to a process that is gone and will never
+ # complete it. Redeliver it, or dead-letter it if out of attempts.
+ stranded = QueueMessage.query.filter(
+ QueueMessage.status == QueueMessage.STATUS_IN_FLIGHT).all()
+ for message in stranded:
+ message.invisible_until = None
+ if message.attempts >= message.max_attempts:
+ message.status = QueueMessage.STATUS_DEAD_LETTER
+ message.error_message = (message.error_message or '') + (
+ f'[{datetime.utcnow().isoformat()}] interrupted by a restart\n')
+ else:
+ message.status = QueueMessage.STATUS_PENDING
+ message.visible_after = datetime.utcnow()
+ db.session.commit()
+ return len(stranded)
+
def requeue(self, group_slug, queue_slug, message_id):
message = self._get_message_or_raise(group_slug, queue_slug, message_id)
if message.status not in (QueueMessage.STATUS_FAILED, QueueMessage.STATUS_DEAD_LETTER):
diff --git a/backend/tests/test_jobs_restart_recovery.py b/backend/tests/test_jobs_restart_recovery.py
new file mode 100644
index 00000000..63787f46
--- /dev/null
+++ b/backend/tests/test_jobs_restart_recovery.py
@@ -0,0 +1,74 @@
+"""A restart must not strand queued work.
+
+The broker only ever delivers 'pending' messages, so a message left
+'in_flight' by a dead process was never delivered again, and its Job row
+read 'running' forever. Boot recovery fails interrupted jobs visibly and
+puts every other stranded message back on the queue.
+"""
+import pytest
+
+from app import db
+from app.jobs import registry
+from app.jobs.consumer import JobConsumer
+from app.jobs.models import Job
+from app.jobs.service import GROUP_SLUG, QUEUE_SLUG, JobService
+from app.queue_bus.models import QueueMessage
+from app.queue_bus.service import QueueBusService
+
+
[email protected](autouse=True)
+def clean_bus(app):
+ QueueBusService.reset_broker()
+ registry.clear()
+ yield
+ registry.clear()
+
+
+def _status(message_id):
+ return db.session.get(QueueMessage, message_id).status
+
+
+def test_job_running_at_restart_fails_and_is_not_rerun(app):
+ ran = []
+ registry.register('test.deploy', lambda job: ran.append(job.id))
+ job = JobService.enqueue('test.deploy')
+ message = QueueBusService.receive(GROUP_SLUG, QUEUE_SLUG)[0]
+ job.status = Job.STATUS_RUNNING # the old process died mid-handler
+ db.session.commit()
+
+ result = JobService.recover_interrupted()
+
+ job = db.session.get(Job, job.id)
+ assert result['failed_jobs'] == 1
+ assert job.status == Job.STATUS_FAILED
+ assert 'restarted' in job.error_message
+ assert _status(message['id']) == QueueMessage.STATUS_COMPLETED
+ assert QueueBusService.receive(GROUP_SLUG, QUEUE_SLUG) == []
+ assert ran == []
+
+
+def test_message_received_but_not_started_is_redelivered(app):
+ ran = []
+ registry.register('test.tick', lambda job: ran.append(job.id))
+ job = JobService.enqueue('test.tick')
+ QueueBusService.receive(GROUP_SLUG, QUEUE_SLUG) # received, then the process died
+ assert QueueBusService.receive(GROUP_SLUG, QUEUE_SLUG) == []
+
+ assert JobService.recover_interrupted()['released_messages'] == 1
+
+ redelivered = QueueBusService.receive(GROUP_SLUG, QUEUE_SLUG)
+ assert len(redelivered) == 1
+ JobConsumer().process_message(redelivered[0])
+ assert ran == [job.id]
+ assert db.session.get(Job, job.id).status == Job.STATUS_SUCCEEDED
+
+
+def test_stranded_message_out_of_attempts_is_dead_lettered(app):
+ QueueBusService.ensure_queue('g', 'q')
+ sent = QueueBusService.send('g', 'q', {'n': 1}, max_attempts=1)
+ QueueBusService.receive('g', 'q')
+
+ QueueBusService.recover_in_flight()
+
+ assert _status(sent['id']) == QueueMessage.STATUS_DEAD_LETTER
+ assert QueueBusService.receive('g', 'q') == []