Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Flag apps that keep crashing

Juan Denise3fb1410cdb2
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.py
index 588435f1..91a2c6fe 100644
--- a/backend/app/jobs/builtin_handlers.py
+++ b/backend/app/jobs/builtin_handlers.py
@@ -406,6 +406,13 @@ def run_request_metrics():
return RequestMetricsService.sample()
+def run_crash_loop_check():
+ """Count container restarts and open/resolve crash-loop incidents
+ (plan 86 §E1). A builtin tick: successes keep the 24 h tick retention."""
+ from app.services.crash_loop_service import CrashLoopService
+ return CrashLoopService.sweep()
+
+
def run_security_feed_check():
"""Daily security-advisory feed check.
@@ -589,6 +596,7 @@ _BUILTINS = [
('builtin.telemetry_retention', run_telemetry_retention, 'telemetry-retention', 21600, 1800),
('builtin.disk_alert', run_disk_alert, 'disk-alert', 900, 240),
('builtin.request_metrics', run_request_metrics, 'request-metrics', 60, 90),
+ ('builtin.crash_loop', run_crash_loop_check, 'crash-loop', 60, 120),
# Fleet alert thresholds. Configurable since the Fleet page shipped, and
# never evaluated until this row existed.
('builtin.fleet_thresholds', run_fleet_threshold_checks, 'fleet-thresholds', FLEET_THRESHOLD_INTERVAL, 45),
diff --git a/backend/app/services/bottleneck_hints_service.py b/backend/app/services/bottleneck_hints_service.py
index 07a752f0..a125c520 100644
--- a/backend/app/services/bottleneck_hints_service.py
+++ b/backend/app/services/bottleneck_hints_service.py
@@ -107,12 +107,31 @@ class BottleneckHintsService:
# ------------------------------------------------------------------
+ @staticmethod
+ def _crash_loop(app) -> Dict:
+ from app.services.crash_loop_service import CrashLoopService
+ return CrashLoopService.state_for(app)
+
@classmethod
def hints(cls, app, now: Optional[datetime] = None) -> List[Dict]:
now = now or datetime.utcnow()
+ hints: List[Dict] = []
+ loop = cls._crash_loop(app)
+ if loop.get('looping'):
+ # Before any traffic rule: a crashing app has no meaningful p95.
+ hints.append({
+ 'id': 'crash_loop',
+ 'params': {'restarts': loop.get('restarts_in_window') or 0},
+ 'signal': f"{loop.get('restarts_in_window') or 0} restarts in the last 10 minutes",
+ 'hint': 'The app keeps crashing and Docker keeps restarting it. Read its '
+ 'logs from just before a restart; no cache or database change helps '
+ 'until it stays up.',
+ 'action': {'label': 'Logs', 'target': 'logs'},
+ 'failure_mode': 'A restart policy hides a crash: the app looks running '
+ 'between crashes.',
+ })
metrics = cls._metrics(app)
summary = metrics.get('summary') or {}
- hints: List[Dict] = []
if (summary.get('requests') or 0) < MIN_REQUESTS:
return hints
diff --git a/backend/app/services/crash_loop_service.py b/backend/app/services/crash_loop_service.py
new file mode 100644
index 00000000..fc7c2c88
--- /dev/null
+++ b/backend/app/services/crash_loop_service.py
@@ -0,0 +1,128 @@
+"""Crash-loop visibility (plan 86 §E1).
+
+A container with a restart policy that keeps dying looked "running" in the
+panel: Docker restarts it each time, and nothing counted. A one-minute
+builtin tick now reads each live app's containers' ``RestartCount`` and keeps
+a short sample window per container. More than ``RESTARTS`` restarts within
+``WINDOW`` marks the app crash-looping and opens an incident through the
+monitor incident path; ``WINDOW`` without a restart resolves it.
+
+State lives in one SystemSettings JSON row, keyed by app id, holding only
+live apps and samples inside the window, so it stays bounded.
+"""
+import logging
+from datetime import datetime, timedelta
+from typing import Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+STATE_KEY = 'crash_loop_state'
+RESTARTS = 3
+WINDOW = timedelta(minutes=10)
+
+
+def _restart_counts(app) -> Dict[str, int]:
+ """``{container name: RestartCount}`` for every container of ``app``."""
+ from app.services.docker_service import DockerService
+ names = [c.get('name') or c.get('id') for c in DockerService.get_all_app_containers(app)]
+ if not names:
+ names = [n for n in (getattr(app, 'container_id', None), app.name,
+ f'serverkit-app-{app.id}') if n]
+ result = DockerService.run(['inspect', '--format', '{{.Name}} {{.RestartCount}}', *names],
+ timeout=30)
+ counts = {}
+ # inspect prints the ones it found even when another name is missing.
+ for line in (result.get('output') or '').splitlines():
+ parts = line.strip().lstrip('/').rsplit(' ', 1)
+ if len(parts) == 2 and parts[1].isdigit():
+ counts[parts[0]] = int(parts[1])
+ return counts
+
+
+class CrashLoopService:
+ restart_counts = staticmethod(_restart_counts)
+
+ @classmethod
+ def sweep(cls, now: Optional[datetime] = None) -> Dict:
+ from app import db
+ from app.models.application import Application
+ from app.models.system_settings import SystemSettings
+
+ now = now or datetime.utcnow()
+ cutoff = (now - WINDOW).isoformat()
+ state = SystemSettings.get(STATE_KEY) or {}
+ if not isinstance(state, dict):
+ state = {}
+ new_state, opened, resolved = {}, [], []
+
+ for app in Application.query_active().filter(Application.app_type == 'docker').all():
+ if app.status not in ('running', 'error'):
+ continue
+ entry = state.get(str(app.id)) or {}
+ samples = {name: [s for s in rows if s[0] >= cutoff]
+ for name, rows in (entry.get('samples') or {}).items()}
+ for name, count in cls.restart_counts(app).items():
+ samples.setdefault(name, []).append([now.isoformat(), count])
+ restarts = max((rows[-1][1] - rows[0][1] for rows in samples.values() if rows),
+ default=0)
+ looping = restarts >= RESTARTS
+ last_restart = entry.get('last_restart')
+ if restarts > 0:
+ changed = any(len(rows) > 1 and rows[-1][1] > rows[-2][1] for rows in samples.values())
+ if changed:
+ last_restart = now.isoformat()
+
+ incident_id = entry.get('incident_id')
+ if looping and not incident_id:
+ incident_id = cls._open_incident(app, restarts)
+ opened.append(app.id)
+ elif incident_id and not looping and (not last_restart or last_restart < cutoff):
+ cls._resolve_incident(incident_id)
+ resolved.append(app.id)
+ incident_id = None
+
+ new_state[str(app.id)] = {
+ 'samples': samples, 'restarts_in_window': restarts,
+ 'looping': bool(incident_id), 'incident_id': incident_id,
+ 'last_restart': last_restart,
+ }
+
+ if new_state != state:
+ SystemSettings.set(STATE_KEY, new_state, value_type='json',
+ description='Crash-loop sample windows (plan 86 §E1)')
+ db.session.commit()
+ return {'apps': len(new_state), 'opened': opened, 'resolved': resolved}
+
+ @staticmethod
+ def _open_incident(app, restarts) -> Optional[int]:
+ from app.services.monitor_service import MonitorService
+ try:
+ incident = MonitorService.create_incident(None, {
+ 'title': f'{app.name} is crash-looping',
+ 'status': 'investigating',
+ 'impact': 'major',
+ 'body': (f'Its containers restarted {restarts} times within '
+ f'{int(WINDOW.total_seconds() // 60)} minutes. Check the app logs; '
+ 'Docker keeps restarting it, so it looks running between crashes.'),
+ })
+ return incident.id
+ except Exception as exc: # noqa: BLE001 - visibility must not break the sweep
+ logger.warning('crash-loop incident for app %s failed: %s', app.id, exc)
+ return None
+
+ @staticmethod
+ def _resolve_incident(incident_id) -> None:
+ from app.services.monitor_service import MonitorService
+ try:
+ MonitorService.update_incident(incident_id, {
+ 'status': 'resolved',
+ 'update_body': f'No restarts for {int(WINDOW.total_seconds() // 60)} minutes.',
+ })
+ except Exception as exc: # noqa: BLE001
+ logger.warning('crash-loop incident %s resolve failed: %s', incident_id, exc)
+
+ @staticmethod
+ def state_for(app) -> Dict:
+ from app.models.system_settings import SystemSettings
+ state = SystemSettings.get(STATE_KEY) or {}
+ return (state.get(str(app.id)) or {}) if isinstance(state, dict) else {}
diff --git a/backend/tests/test_bottleneck_hints.py b/backend/tests/test_bottleneck_hints.py
index 71a48b35..6f690c47 100644
--- a/backend/tests/test_bottleneck_hints.py
+++ b/backend/tests/test_bottleneck_hints.py
@@ -22,7 +22,9 @@ def _metrics(p95=1500, requests=1000, ratio=None, points=()):
@pytest.fixture
def signals(monkeypatch):
- state = {'metrics': _metrics(), 'cpu': {}, 'dbs': [], 'share': None, 'windows': []}
+ state = {'metrics': _metrics(), 'cpu': {}, 'dbs': [], 'share': None, 'windows': [],
+ 'loop': {}}
+ monkeypatch.setattr(H, '_crash_loop', staticmethod(lambda app: state['loop']))
monkeypatch.setattr(H, '_metrics', staticmethod(lambda app: state['metrics']))
monkeypatch.setattr(H, 'cpu_percent', staticmethod(lambda name: state['cpu'].get(name)))
monkeypatch.setattr(H, '_databases', staticmethod(lambda app: state['dbs']))
@@ -142,3 +144,10 @@ def test_every_hint_carries_params_for_a_translated_ui(signals):
signals['cpu'] = {'shop': 95}
(hint,) = H.hints(_app(), now=NOW)
assert hint['params'] == {'p95_ms': 1500, 'cpu': 95}
+
+
+def test_a_crash_loop_comes_first_even_without_traffic(signals):
+ signals['metrics'] = _metrics(requests=0)
+ signals['loop'] = {'looping': True, 'restarts_in_window': 5}
+ (hint,) = H.hints(_app(), now=NOW)
+ assert hint['id'] == 'crash_loop' and hint['params'] == {'restarts': 5}
diff --git a/backend/tests/test_crash_loop.py b/backend/tests/test_crash_loop.py
new file mode 100644
index 00000000..9cfe0986
--- /dev/null
+++ b/backend/tests/test_crash_loop.py
@@ -0,0 +1,86 @@
+"""Crash-loop visibility (plan 86 §E1): restart counts over a window, one
+incident per loop, resolved after a quiet window."""
+from datetime import datetime, timedelta
+
+import pytest
+
+from app import db
+from app.models.status_page import StatusIncident
+from app.services import crash_loop_service as cls_mod
+from app.services.crash_loop_service import CrashLoopService
+from tests.factories import make_application
+
+T0 = datetime(2026, 7, 3, 12, 0)
+
+
+def restarts(app, monkeypatch):
+ counts = {}
+ monkeypatch.setattr(CrashLoopService, 'restart_counts',
+ staticmethod(lambda app_row: dict(counts.get(app_row.id, {}))))
+ return counts
+
+
+def _tick(minutes):
+ return CrashLoopService.sweep(now=T0 + timedelta(minutes=minutes))
+
+
+def test_three_restarts_in_ten_minutes_open_one_incident(restarts):
+ site = make_application(db, name='flappy', status='running')
+ for minute, count in ((0, 0), (1, 1), (2, 2)):
+ restarts[site.id] = {'flappy': count}
+ assert _tick(minute)['opened'] == []
+ restarts[site.id] = {'flappy': 3}
+ assert _tick(3)['opened'] == [site.id]
+ restarts[site.id] = {'flappy': 5}
+ assert _tick(4)['opened'] == [], 'one incident per loop'
+ (incident,) = StatusIncident.query.all()
+ assert incident.title == 'flappy is crash-looping' and incident.status == 'investigating'
+ assert CrashLoopService.state_for(site)['looping'] is True
+
+
+def test_a_quiet_window_resolves_it(restarts):
+ site = make_application(db, name='flappy', status='running')
+ for minute, count in ((0, 0), (1, 3)):
+ restarts[site.id] = {'flappy': count}
+ _tick(minute)
+ assert _tick(6)['resolved'] == [], 'a restart inside the window keeps it open'
+ assert _tick(12)['resolved'] == [site.id]
+ (incident,) = StatusIncident.query.all()
+ assert incident.status == 'resolved'
+ assert CrashLoopService.state_for(site)['looping'] is False
+
+
+def test_slow_restarts_are_not_a_loop(restarts):
+ site = make_application(db, name='steady', status='running')
+ for minute, count in ((0, 0), (15, 1), (30, 2), (45, 3)):
+ restarts[site.id] = {'steady': count}
+ assert _tick(minute)['opened'] == []
+ assert StatusIncident.query.count() == 0
+
+
+def test_workers_count_too(restarts):
+ site = make_application(db, name='web', status='running')
+ restarts[site.id] = {'web': 0, f'serverkit-app-{site.id}-worker': 0}
+ _tick(0)
+ restarts[site.id] = {'web': 0, f'serverkit-app-{site.id}-worker': 4}
+ assert _tick(2)['opened'] == [site.id]
+
+
+def test_state_holds_only_live_apps_and_recent_samples(restarts):
+ site = make_application(db, name='web', status='running')
+ restarts[site.id] = {'web': 0}
+ _tick(0)
+ _tick(30)
+ samples = CrashLoopService.state_for(site)['samples']['web']
+ assert len(samples) == 1, 'samples older than the window are dropped'
+ site.status = 'stopped'
+ db.session.commit()
+ _tick(31)
+ assert CrashLoopService.state_for(site) == {}
+
+
+def test_the_sweep_is_a_builtin_tick():
+ from app.jobs.builtin_handlers import _BUILTINS
+ (row,) = [b for b in _BUILTINS if b[2] == 'crash-loop']
+ assert row[0] == 'builtin.crash_loop' and row[3] == 60
diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.py
index 2869ffbe..ce48c7f4 100644
--- a/backend/tests/test_jobs.py
+++ b/backend/tests/test_jobs.py
@@ -194,18 +194,20 @@ class TestBuiltins:
# Per-app request rollups (plan 86 §A2): a builtin so its once-a-minute
# successes get the 24 h tick retention.
assert 'builtin.request_metrics' in kinds
- assert len([k for k in kinds if k.startswith('builtin.')]) == 19
+ # Crash-loop visibility (plan 86 §E1), also once a minute.
+ assert 'builtin.crash_loop' in kinds
+ assert len([k for k in kinds if k.startswith('builtin.')]) == 20
builtin_handlers.seed_builtin_schedules()
- # 19 builtin.* schedules (incl. the request-metrics tick, restore-point/job/telemetry retention,
+ # 20 builtin.* schedules (incl. the request-metrics and crash-loop ticks, restore-point/job/telemetry retention,
# the monitor sweep, security-feed check, recycle-bin retention and the
# fleet threshold check) + login-link/SSO reapers + drift/FIM/bandwidth
# sweeps + the host doctor sweep AND the fleet doctor sweep (plan 26)
# + the setup-health nag (plan 22).
- assert ScheduledJob.query.count() == 27
+ assert ScheduledJob.query.count() == 28
# Seeding twice doesn't duplicate.
builtin_handlers.seed_builtin_schedules()
- assert ScheduledJob.query.count() == 27
+ assert ScheduledJob.query.count() == 28
class TestApi:
diff --git a/frontend/src/components/service-detail/HintsCard.jsx b/frontend/src/components/service-detail/HintsCard.jsx
index ff6375fb..ea3dabde 100644
--- a/frontend/src/components/service-detail/HintsCard.jsx
+++ b/frontend/src/components/service-detail/HintsCard.jsx
@@ -25,6 +25,12 @@ export default function HintsCard({ app }) {
const text = (hint) => {
const p = hint.params || {};
switch (hint.id) {
+ case 'crash_loop':
+ return {
+ signal: t('app.hints.crashLoopSignal', '{{count}} restarts in the last 10 minutes.', { count: p.restarts }),
+ hint: t('app.hints.crashLoop', 'The app keeps crashing and Docker keeps restarting it. Read its logs from just before a restart; no cache or database change helps until it stays up.'),
+ failure: t('app.hints.crashLoopFailure', 'A restart policy hides a crash: the app looks running between crashes.'),
+ };
case 'app_bound':
return {
signal: t('app.hints.appBoundSignal', 'p95 {{ms}} ms while the app uses {{cpu}}% CPU and its database is idle.', { ms: p.p95_ms, cpu: p.cpu }),
@@ -60,10 +66,11 @@ export default function HintsCard({ app }) {
const actionLink = (action) => {
if (!action) return null;
- const label = action.target === 'settings/cache'
- ? t('app.hints.openMicroCache', 'Open micro-cache settings')
- : t('app.hints.attachCache', 'Attach a cache');
- const to = action.target === 'settings/cache' ? `/services/${app.id}/settings/cache` : null;
+ const targets = {
+ 'settings/cache': { to: `/services/${app.id}/settings/cache`, label: t('app.hints.openMicroCache', 'Open micro-cache settings') },
+ logs: { to: `/services/${app.id}/logs`, label: t('app.hints.openLogs', 'Open logs') },
+ };
+ const { to = null, label = t('app.hints.attachCache', 'Attach a cache') } = targets[action.target] || {};
return to
? <Link to={to} className="hints__action">{label}</Link>
: <span className="hints__action hints__action--here">{label}</span>;
diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json
index fd257985..4f9511e7 100644
--- a/frontend/src/i18n/locales/en.json
+++ b/frontend/src/i18n/locales/en.json
@@ -2691,6 +2691,9 @@
"appBoundFailure": "A cached page can be stale for up to its lifetime.",
"appBoundSignal": "p95 {{ms}} ms while the app uses {{cpu}}% CPU and its database is idle.",
"attachCache": "Attach a cache",
+ "crashLoop": "The app keeps crashing and Docker keeps restarting it. Read its logs from just before a restart; no cache or database change helps until it stays up.",
+ "crashLoopFailure": "A restart policy hides a crash: the app looks running between crashes.",
+ "crashLoopSignal": "{{count}} restarts in the last 10 minutes.",
"dbBound": "The database is the bottleneck. Look at its top queries for a missing index, or cache what is read most.",
"dbBoundFailure": "Cached data can be stale; invalidate on write.",
"dbBoundRepeated": "The same query dominates: cache its result, or add an index for it (see the query in the database Insights tab).",
@@ -2702,6 +2705,7 @@
"lowHit": "Most cacheable requests miss. The pages may vary per visitor, or the cache lifetime is too short for the traffic.",
"lowHitFailure": "A longer lifetime serves older pages.",
"lowHitSignal": "Micro-cache hit ratio {{ratio}}%.",
+ "openLogs": "Open logs",
"openMicroCache": "Open micro-cache settings",
"title": "Where it is slow",
"window": "Last 24 hours"
diff --git a/frontend/src/i18n/locales/es.json b/frontend/src/i18n/locales/es.json
index 084c24f0..8d05fa2a 100644
--- a/frontend/src/i18n/locales/es.json
+++ b/frontend/src/i18n/locales/es.json
@@ -6791,7 +6791,11 @@
"lowHitSignal": "Tasa de aciertos de la microcaché: {{ratio}}%.",
"openMicroCache": "Abrir ajustes de microcaché",
"title": "Dónde va lento",
- "window": "Últimas 24 horas"
+ "window": "Últimas 24 horas",
+ "crashLoop": "La app sigue fallando y Docker la sigue reiniciando. Lee sus registros de justo antes de un reinicio; ningún cambio de caché o de base de datos ayuda hasta que se mantenga en pie.",
+ "crashLoopFailure": "Una política de reinicio oculta un fallo: la app parece en marcha entre fallos.",
+ "crashLoopSignal": "{{count}} reinicios en los últimos 10 minutos.",
+ "openLogs": "Abrir registros"
}
},
"auth": {