Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(realtime): BackgroundLoop helper; sockets broadcasters adopt it (plan 77 E5)

Juan Denisff501b51add6
diff --git a/backend/app/jobs/thread_ownership.py b/backend/app/jobs/thread_ownership.py
index 003f137a..3c7f27f2 100644
--- a/backend/app/jobs/thread_ownership.py
+++ b/backend/app/jobs/thread_ownership.py
@@ -17,6 +17,16 @@ LIFECYCLE_DURABLE_CANDIDATE = 'durable_candidate'
THREAD_OWNERSHIP = {
+ 'app/utils/background_loop.py:start:self._run': {
+ 'owner': 'BackgroundLoop helper (plan 77 E5)',
+ 'lifecycle': LIFECYCLE_PROCESS_LOOP,
+ 'rationale': (
+ 'The ONE spawn site for Event-stopped process polling loops; '
+ 'individual loops (socket metrics/container-status broadcasters, '
+ 'future adopters) are covered by this entry instead of one row '
+ 'per hand-rolled thread.'
+ ),
+ },
'app/api/ai.py:chat_stream:producer': {
'owner': 'AI SSE response',
'lifecycle': LIFECYCLE_REQUEST_STREAM,
@@ -157,16 +167,6 @@ THREAD_OWNERSHIP = {
'lifecycle': LIFECYCLE_REQUEST_STREAM,
'rationale': 'Emitter lifetime is coupled to one socket subscription.',
},
- 'app/sockets.py:handle_subscribe_metrics:broadcast_metrics': {
- 'owner': 'Socket.IO metrics subscription',
- 'lifecycle': LIFECYCLE_REQUEST_STREAM,
- 'rationale': 'Broadcaster lifetime is coupled to an active socket subscription set.',
- },
- 'app/sockets.py:handle_subscribe_container_status:broadcast_container_status': {
- 'owner': 'Socket.IO container status subscription',
- 'lifecycle': LIFECYCLE_REQUEST_STREAM,
- 'rationale': 'Broadcaster lifetime is coupled to an active socket subscription set.',
- },
}
diff --git a/backend/app/sockets.py b/backend/app/sockets.py
index 936003b5..dcf5c581 100644
--- a/backend/app/sockets.py
+++ b/backend/app/sockets.py
@@ -9,19 +9,16 @@ from app.services.system_service import SystemService
from app.services.log_service import LogService, LogStreamer
from app.services.docker_service import DockerService
from app import sockets_rooms as rooms
+from app.utils.background_loop import BackgroundLoop
socketio = SocketIO()
log_streamer = LogStreamer()
# Store active metric subscriptions
metric_subscribers = set()
-metric_thread = None
-metric_stop_event = threading.Event()
# Store active aggregated container-status subscriptions
container_status_subscribers = set()
-container_status_thread = None
-container_status_stop_event = threading.Event()
# Store active container log streams
container_log_streams = {} # sid -> {'process': Popen, 'app_id': int, 'thread': Thread, 'stop_event': Event}
@@ -141,20 +138,24 @@ def handle_disconnect():
connected_clients.pop(sid, None)
[email protected]('subscribe_metrics')
-def handle_subscribe_metrics():
- """Subscribe to real-time system metrics."""
- global metric_thread, metric_stop_event
+def _metrics_tick():
+ metrics = SystemService.get_all_metrics()
+ socketio.emit('metrics', metrics, room=None) # Broadcast to all
- sid = request.sid
- metric_subscribers.add(sid)
- # Start metric broadcast thread if not running
- if metric_thread is None or not metric_thread.is_alive():
- metric_stop_event.clear()
- metric_thread = threading.Thread(target=broadcast_metrics, daemon=True)
- metric_thread.start()
+# Ends itself when the last subscriber leaves; restarted by the next
+# subscribe. Errors go to the logger and never kill the loop (E5).
+metrics_loop = BackgroundLoop(
+ 'socket-metrics', 2, _metrics_tick,
+ run_while=lambda: bool(metric_subscribers),
+)
+
[email protected]('subscribe_metrics')
+def handle_subscribe_metrics():
+ """Subscribe to real-time system metrics."""
+ metric_subscribers.add(request.sid)
+ metrics_loop.start()
emit('subscribed', {'channel': 'metrics'})
@@ -167,44 +168,39 @@ def handle_unsubscribe_metrics():
emit('unsubscribed', {'channel': 'metrics'})
-def broadcast_metrics():
- """Broadcast system metrics to all subscribers."""
- global metric_stop_event
+# ==================== AGGREGATED CONTAINER STATUS ====================
- while not metric_stop_event.is_set() and metric_subscribers:
- try:
- metrics = SystemService.get_all_metrics()
- socketio.emit('metrics', metrics, room=None) # Broadcast to all
- except Exception as e:
- print(f"Error broadcasting metrics: {e}")
+def _container_status_tick():
+ """Emit only the apps whose aggregated status changed since last tick.
- time.sleep(2) # Update every 2 seconds
+ The aggregator keeps the last-emitted snapshot in memory; needs an app
+ context because it touches the ORM (BackgroundLoop provides it).
+ """
+ from app.services import container_status_service as css
+ changed = css.get_changed_app_statuses()
+ if changed:
+ socketio.emit('container_status', {
+ 'statuses': changed,
+ 'timestamp': time.time(),
+ }, room=None)
-# ==================== AGGREGATED CONTAINER STATUS ====================
+container_status_loop = BackgroundLoop(
+ 'socket-container-status', 5, _container_status_tick,
+ run_while=lambda: bool(container_status_subscribers),
+)
+
@socketio.on('subscribe_container_status')
def handle_subscribe_container_status():
"""Subscribe to aggregated container-status change events.
- Mirrors the metrics pattern: a single background thread polls the
- aggregator and broadcasts ONLY the apps whose status changed since the last
- tick (channel 'container_status'). Clients reconcile by app_id.
+ Mirrors the metrics pattern: one background loop polls the aggregator and
+ broadcasts ONLY the apps whose status changed since the last tick
+ (channel 'container_status'). Clients reconcile by app_id.
"""
- global container_status_thread, container_status_stop_event
-
- sid = request.sid
- container_status_subscribers.add(sid)
-
- if container_status_thread is None or not container_status_thread.is_alive():
- container_status_stop_event.clear()
- container_status_thread = threading.Thread(
- target=broadcast_container_status,
- args=(current_app._get_current_object(),),
- daemon=True,
- )
- container_status_thread.start()
-
+ container_status_subscribers.add(request.sid)
+ container_status_loop.start(app=current_app._get_current_object())
emit('subscribed', {'channel': 'container_status'})
@@ -217,34 +213,6 @@ def handle_unsubscribe_container_status():
emit('unsubscribed', {'channel': 'container_status'})
-def broadcast_container_status(flask_app):
- """Broadcast changed aggregated container statuses to all subscribers.
-
- Runs in a background thread for as long as there are subscribers. Each tick
- asks the aggregator for the deltas (it keeps the last-emitted snapshot in
- memory) and emits only those. Defensive: any Docker/DB error is swallowed so
- the loop survives a transient outage. Needs an app context because the
- aggregator touches the ORM.
- """
- global container_status_stop_event
-
- from app.services import container_status_service as css
-
- while not container_status_stop_event.is_set() and container_status_subscribers:
- try:
- with flask_app.app_context():
- changed = css.get_changed_app_statuses()
- if changed:
- socketio.emit('container_status', {
- 'statuses': changed,
- 'timestamp': time.time(),
- }, room=None)
- except Exception as e:
- print(f"Error broadcasting container status: {e}")
-
- time.sleep(5) # Re-evaluate every 5 seconds
-
-
@socketio.on('subscribe_terminal')
def handle_subscribe_terminal(data):
"""Join the stream room for a remote terminal session (agent PTY output).
diff --git a/backend/app/utils/background_loop.py b/backend/app/utils/background_loop.py
new file mode 100644
index 00000000..7ba76d57
--- /dev/null
+++ b/backend/app/utils/background_loop.py
@@ -0,0 +1,93 @@
+"""BackgroundLoop — the one shape for process-lifetime polling loops (plan 77 E5).
+
+The 13 LIFECYCLE_PROCESS_LOOP threads each hand-rolled the same dance: a
+bool flag (or Event), a daemon Thread, a try/except around the tick (some
+reporting errors via ``print()`` — unreadable in prod), a sleep. This helper
+owns that dance once:
+
+- ``threading.Event`` stop — ``stop()`` interrupts the wait immediately
+ instead of finishing a sleep;
+- idempotent ``start()``/``stop()`` guarded by a lock (safe under
+ async_mode='threading' where two subscribers race to start a broadcaster);
+- optional Flask app context per tick;
+- errors go to a real logger and never kill the loop;
+- ``run_while`` predicate for loops that should end when their audience
+ leaves (e.g. "while there are subscribers").
+
+The single ``threading.Thread`` call site below is registered in
+``app/jobs/thread_ownership.py`` — loops built on this helper are covered by
+that one entry instead of one registry row per loop.
+"""
+
+import logging
+import threading
+
+logger = logging.getLogger(__name__)
+
+
+class BackgroundLoop:
+ def __init__(self, name, interval_s, tick, *, app=None, run_while=None):
+ """
+ Args:
+ name: human-readable loop name (thread name + log prefix).
+ interval_s: seconds between ticks (waited on the stop Event, so
+ stop() is immediate).
+ tick: zero-arg callable executed each cycle.
+ app: optional Flask app; when given, each tick runs inside
+ ``app.app_context()``.
+ run_while: optional zero-arg predicate; the loop exits when it
+ returns falsy (checked before each tick).
+ """
+ self.name = name
+ self.interval_s = interval_s
+ self._tick = tick
+ self._app = app
+ self._run_while = run_while
+ self._thread = None
+ self._stop = threading.Event()
+ self._lock = threading.Lock()
+
+ @property
+ def running(self):
+ return self._thread is not None and self._thread.is_alive()
+
+ def start(self, app=None):
+ """Start the loop thread. Idempotent — a live loop is left alone.
+ Returns True when a new thread was started."""
+ with self._lock:
+ if self.running:
+ return False
+ if app is not None:
+ self._app = app
+ self._stop.clear()
+ self._thread = threading.Thread(
+ target=self._run, name=f'loop:{self.name}', daemon=True)
+ self._thread.start()
+ return True
+
+ def stop(self, join_timeout=None):
+ """Signal the loop to exit. Idempotent. Optionally join."""
+ self._stop.set()
+ thread = self._thread
+ if join_timeout is not None and thread is not None:
+ thread.join(timeout=join_timeout)
+
+ def _run(self):
+ while not self._stop.is_set():
+ if self._run_while is not None:
+ try:
+ if not self._run_while():
+ break
+ except Exception:
+ logger.exception('[%s] run_while predicate raised; stopping loop', self.name)
+ break
+ try:
+ if self._app is not None:
+ with self._app.app_context():
+ self._tick()
+ else:
+ self._tick()
+ except Exception:
+ logger.exception('[%s] tick failed; continuing', self.name)
+ # Event-based wait: stop() interrupts immediately.
+ self._stop.wait(self.interval_s)
diff --git a/backend/tests/test_background_loop.py b/backend/tests/test_background_loop.py
new file mode 100644
index 00000000..e2edf62b
--- /dev/null
+++ b/backend/tests/test_background_loop.py
@@ -0,0 +1,73 @@
+"""Plan 77 E5 — BackgroundLoop: Event-stopped, idempotent, error-proof."""
+import threading
+import time
+
+from app.utils.background_loop import BackgroundLoop
+
+
+def test_ticks_and_stops_promptly():
+ ticks = []
+ loop = BackgroundLoop('t', 0.01, lambda: ticks.append(1))
+ assert loop.start() is True
+ time.sleep(0.15)
+ assert ticks, 'loop never ticked'
+ loop.stop(join_timeout=1)
+ assert not loop.running
+ n = len(ticks)
+ time.sleep(0.05)
+ assert len(ticks) == n, 'loop kept ticking after stop()'
+
+
+def test_start_is_idempotent():
+ started = threading.Event()
+ loop = BackgroundLoop('t2', 0.01, started.set)
+ assert loop.start() is True
+ assert loop.start() is False # already running
+ started.wait(1)
+ loop.stop(join_timeout=1)
+
+
+def test_tick_errors_do_not_kill_the_loop():
+ ticks = []
+
+ def tick():
+ ticks.append(1)
+ raise RuntimeError('boom')
+
+ loop = BackgroundLoop('t3', 0.01, tick)
+ loop.start()
+ time.sleep(0.1)
+ loop.stop(join_timeout=1)
+ assert len(ticks) >= 2, 'loop died on first tick error'
+
+
+def test_run_while_predicate_ends_loop():
+ alive = {'v': True}
+ ticks = []
+ loop = BackgroundLoop('t4', 0.01, lambda: ticks.append(1),
+ run_while=lambda: alive['v'])
+ loop.start()
+ time.sleep(0.05)
+ alive['v'] = False
+ time.sleep(0.1)
+ assert not loop.running
+ loop.stop()
+
+
+def test_restart_after_natural_exit():
+ """A loop whose run_while ended can be started again (the sockets
+ broadcaster pattern: last subscriber leaves, a new one arrives)."""
+ gate = {'v': True}
+ ticks = []
+ loop = BackgroundLoop('t5', 0.01, lambda: ticks.append(1),
+ run_while=lambda: gate['v'])
+ loop.start()
+ time.sleep(0.05)
+ gate['v'] = False
+ time.sleep(0.1)
+ assert not loop.running
+ gate['v'] = True
+ assert loop.start() is True
+ time.sleep(0.05)
+ assert loop.running
+ loop.stop(join_timeout=1)