Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Run fleet threads inside the live app

Juan Denis6901a4ff919a
diff --git a/backend/app/jobs/thread_ownership.py b/backend/app/jobs/thread_ownership.py
index fb1f1b30..85e414a0 100644
--- a/backend/app/jobs/thread_ownership.py
+++ b/backend/app/jobs/thread_ownership.py
@@ -69,7 +69,7 @@ THREAD_OWNERSHIP = {
'lifecycle': LIFECYCLE_BOUNDED_DELIVERY,
'rationale': 'Low-latency delivery attempt is owned and recoverable by its persisted command row.',
},
- 'app/services/agent_fleet_service.py:upgrade_servers:dispatch_agent_command': {
+ 'app/services/agent_fleet_service.py:upgrade_servers:self._dispatch_in_app': {
'owner': 'agent fleet upgrade',
'lifecycle': LIFECYCLE_BOUNDED_FANOUT,
'rationale': 'Parallel delivery fan-out creates persisted agent command state.',
diff --git a/backend/app/services/agent_fleet_service.py b/backend/app/services/agent_fleet_service.py
index 7224ff9e..1c139768 100644
--- a/backend/app/services/agent_fleet_service.py
+++ b/backend/app/services/agent_fleet_service.py
@@ -11,6 +11,8 @@ import threading
import time
import uuid
+from flask import current_app
+
from app import db
from app.models.server import (
Server, ServerGroup, AgentVersion, ServerCommand, AgentSession, AgentRollout
@@ -119,8 +121,8 @@ class AgentFleetService:
continue
threading.Thread(
- target=dispatch_agent_command,
- args=(server_id, 'agent:update', params),
+ target=self._dispatch_in_app,
+ args=(current_app._get_current_object(), server_id, 'agent:update', params),
kwargs={'user_id': user_id, 'timeout': 60.0},
daemon=True
).start()
@@ -182,7 +184,7 @@ class AgentFleetService:
thread = threading.Thread(
target=self._run_staged_rollout,
- args=(rollout_id, ids, version_id, batch_size, delay_minutes, user_id, cancel_event),
+ args=(current_app._get_current_object(), rollout_id, ids, version_id, batch_size, delay_minutes, user_id, cancel_event),
daemon=True
)
self._rollout_threads[rollout_id] = thread
@@ -190,11 +192,8 @@ class AgentFleetService:
return {'success': True, 'rollout_id': rollout_id, 'rollout': rollout.to_dict()}
- def _run_staged_rollout(self, rollout_id, server_ids, version_id, batch_size, delay_minutes, user_id, cancel_event):
+ def _run_staged_rollout(self, app, rollout_id, server_ids, version_id, batch_size, delay_minutes, user_id, cancel_event):
"""Run staged rollout in batches with health checks between waves."""
- from app import create_app
- app = create_app()
-
with app.app_context():
total = len(server_ids)
processed = 0
@@ -370,15 +369,17 @@ class AgentFleetService:
# Send asynchronously
threading.Thread(
target=self._deliver_single_command,
- args=(server_id, cmd.id, cmd.command_type, cmd.command_data, cmd.user_id),
+ args=(current_app._get_current_object(), server_id, cmd.id, cmd.command_type, cmd.command_data, cmd.user_id),
daemon=True
).start()
- def _deliver_single_command(self, server_id, command_id, action, params, user_id):
- """Send a single queued command and update its status."""
- from app import create_app
- app = create_app()
+ @staticmethod
+ def _dispatch_in_app(app, *args, **kwargs):
+ with app.app_context():
+ return dispatch_agent_command(*args, **kwargs)
+ def _deliver_single_command(self, app, server_id, command_id, action, params, user_id):
+ """Send a single queued command and update its status."""
with app.app_context():
result = dispatch_agent_command(
server_id, action, params, user_id=user_id, timeout=60.0
@@ -443,7 +444,7 @@ class AgentFleetService:
threading.Thread(
target=self._deliver_single_command,
- args=(cmd.server_id, cmd.id, cmd.command_type, cmd.command_data, cmd.user_id),
+ args=(current_app._get_current_object(), cmd.server_id, cmd.id, cmd.command_type, cmd.command_data, cmd.user_id),
daemon=True
).start()
@@ -469,7 +470,7 @@ class AgentFleetService:
threading.Thread(
target=self._deliver_single_command,
- args=(cmd.server_id, cmd.id, cmd.command_type, cmd.command_data, cmd.user_id),
+ args=(current_app._get_current_object(), cmd.server_id, cmd.id, cmd.command_type, cmd.command_data, cmd.user_id),
daemon=True
).start()
diff --git a/backend/tests/test_fleet_threads_app_context.py b/backend/tests/test_fleet_threads_app_context.py
new file mode 100644
index 00000000..d4aa27bf
--- /dev/null
+++ b/backend/tests/test_fleet_threads_app_context.py
@@ -0,0 +1,72 @@
+"""Fleet worker threads run inside the live app, never a fresh create_app().
+
+A per-thread create_app() reran migrations and plugin loading, rebound
+Socket.IO and ran the boot reconcile against the live panel. The upgrade
+fan-out had the opposite bug: no app context at all, so its first query
+raised and online agents were never told to upgrade.
+"""
+import threading
+from unittest.mock import patch
+
+import pytest
+from flask import current_app, has_app_context
+
+from factories import make_server
+
+
+def calls(app):
+ """Record each dispatch from whatever thread makes it."""
+ seen, done = [], threading.Event()
+
+ def fake_send(**kwargs):
+ seen.append({'context': has_app_context(),
+ 'app': current_app._get_current_object() if has_app_context() else None,
+ 'action': kwargs['action']})
+ done.set()
+ return {'success': True, 'data': {}}
+
+ def no_new_app(*a, **k):
+ raise AssertionError('fleet thread built a second app')
+
+ with patch('app.services.remote_command_dispatcher.agent_registry.send_command',
+ side_effect=fake_send), patch('app.create_app', side_effect=no_new_app):
+ yield seen, done
+
+
+def test_upgrade_dispatch_runs_in_the_live_app(app, calls):
+ from app import db
+ from app.models.server import AgentVersion
+ from app.services.agent_fleet_service import fleet_service
+ seen, done = calls
+ server = make_server(db, name='ft-online', status='online',
+ os_type='linux', architecture='amd64')
+ version = AgentVersion(version='9.9.9', assets={'linux-amd64': 'https://x/a'})
+ db.session.add(version)
+ db.session.commit()
+
+ result = fleet_service.upgrade_servers([server.id], version.id)
+
+ assert result['results'][0]['success'], result
+ assert done.wait(10)
+ assert seen == [{'context': True, 'app': app, 'action': 'agent:update'}]
+
+
+def test_queued_delivery_runs_in_the_live_app(app, calls):
+ from app import db
+ from app.models.server import ServerCommand
+ from app.services.agent_fleet_service import fleet_service
+ seen, done = calls
+ server = make_server(db, name='ft-queued')
+ cmd = fleet_service.queue_command(server.id, 'agent:update', {})
+
+ fleet_service.deliver_queued_commands(server.id)
+
+ assert done.wait(10)
+ assert seen[0]['context'] and seen[0]['app'] is app
+ for _ in range(50): # the thread commits after dispatch returns
+ db.session.expire_all()
+ if db.session.get(ServerCommand, cmd.id).status == 'completed':
+ break
+ threading.Event().wait(0.1)
+ assert db.session.get(ServerCommand, cmd.id).status == 'completed'