ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
Flag apps that keep crashing
Juan Denise3fb1410cdb2
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.pyindex 588435f1..91a2c6fe 100644--- a/backend/app/jobs/builtin_handlers.py+++ b/backend/app/jobs/builtin_handlers.py@@ -406,6 +406,13 @@ def run_request_metrics():return RequestMetricsService.sample()+def run_crash_loop_check():+ """Count container restarts and open/resolve crash-loop incidents+ (plan 86 §E1). A builtin tick: successes keep the 24 h tick retention."""+ from app.services.crash_loop_service import CrashLoopService+ return CrashLoopService.sweep()++def run_security_feed_check():"""Daily security-advisory feed check.@@ -589,6 +596,7 @@ _BUILTINS = [('builtin.telemetry_retention', run_telemetry_retention, 'telemetry-retention', 21600, 1800),('builtin.disk_alert', run_disk_alert, 'disk-alert', 900, 240),('builtin.request_metrics', run_request_metrics, 'request-metrics', 60, 90),+ ('builtin.crash_loop', run_crash_loop_check, 'crash-loop', 60, 120),# Fleet alert thresholds. Configurable since the Fleet page shipped, and# never evaluated until this row existed.('builtin.fleet_thresholds', run_fleet_threshold_checks, 'fleet-thresholds', FLEET_THRESHOLD_INTERVAL, 45),diff --git a/backend/app/services/bottleneck_hints_service.py b/backend/app/services/bottleneck_hints_service.pyindex 07a752f0..a125c520 100644--- a/backend/app/services/bottleneck_hints_service.py+++ b/backend/app/services/bottleneck_hints_service.py@@ -107,12 +107,31 @@ class BottleneckHintsService:# ------------------------------------------------------------------+ @staticmethod+ def _crash_loop(app) -> Dict:+ from app.services.crash_loop_service import CrashLoopService+ return CrashLoopService.state_for(app)+@classmethoddef hints(cls, app, now: Optional[datetime] = None) -> List[Dict]:now = now or datetime.utcnow()+ hints: List[Dict] = []+ loop = cls._crash_loop(app)+ if loop.get('looping'):+ # Before any traffic rule: a crashing app has no meaningful p95.+ hints.append({+ 'id': 'crash_loop',+ 'params': {'restarts': loop.get('restarts_in_window') or 0},+ 'signal': f"{loop.get('restarts_in_window') or 0} restarts in the last 10 minutes",+ 'hint': 'The app keeps crashing and Docker keeps restarting it. Read its '+ 'logs from just before a restart; no cache or database change helps '+ 'until it stays up.',+ 'action': {'label': 'Logs', 'target': 'logs'},+ 'failure_mode': 'A restart policy hides a crash: the app looks running '+ 'between crashes.',+ })metrics = cls._metrics(app)summary = metrics.get('summary') or {}- hints: List[Dict] = []if (summary.get('requests') or 0) < MIN_REQUESTS:return hintsdiff --git a/backend/app/services/crash_loop_service.py b/backend/app/services/crash_loop_service.pynew file mode 100644index 00000000..fc7c2c88--- /dev/null+++ b/backend/app/services/crash_loop_service.py@@ -0,0 +1,128 @@+"""Crash-loop visibility (plan 86 §E1).++A container with a restart policy that keeps dying looked "running" in the+panel: Docker restarts it each time, and nothing counted. A one-minute+builtin tick now reads each live app's containers' ``RestartCount`` and keeps+a short sample window per container. More than ``RESTARTS`` restarts within+``WINDOW`` marks the app crash-looping and opens an incident through the+monitor incident path; ``WINDOW`` without a restart resolves it.++State lives in one SystemSettings JSON row, keyed by app id, holding only+live apps and samples inside the window, so it stays bounded.+"""+import logging+from datetime import datetime, timedelta+from typing import Dict, List, Optional++logger = logging.getLogger(__name__)++STATE_KEY = 'crash_loop_state'+RESTARTS = 3+WINDOW = timedelta(minutes=10)+++def _restart_counts(app) -> Dict[str, int]:+ """``{container name: RestartCount}`` for every container of ``app``."""+ from app.services.docker_service import DockerService+ names = [c.get('name') or c.get('id') for c in DockerService.get_all_app_containers(app)]+ if not names:+ names = [n for n in (getattr(app, 'container_id', None), app.name,+ f'serverkit-app-{app.id}') if n]+ result = DockerService.run(['inspect', '--format', '{{.Name}} {{.RestartCount}}', *names],+ timeout=30)+ counts = {}+ # inspect prints the ones it found even when another name is missing.+ for line in (result.get('output') or '').splitlines():+ parts = line.strip().lstrip('/').rsplit(' ', 1)+ if len(parts) == 2 and parts[1].isdigit():+ counts[parts[0]] = int(parts[1])+ return counts+++class CrashLoopService:+ restart_counts = staticmethod(_restart_counts)++ @classmethod+ def sweep(cls, now: Optional[datetime] = None) -> Dict:+ from app import db+ from app.models.application import Application+ from app.models.system_settings import SystemSettings++ now = now or datetime.utcnow()+ cutoff = (now - WINDOW).isoformat()+ state = SystemSettings.get(STATE_KEY) or {}+ if not isinstance(state, dict):+ state = {}+ new_state, opened, resolved = {}, [], []++ for app in Application.query_active().filter(Application.app_type == 'docker').all():+ if app.status not in ('running', 'error'):+ continue+ entry = state.get(str(app.id)) or {}+ samples = {name: [s for s in rows if s[0] >= cutoff]+ for name, rows in (entry.get('samples') or {}).items()}+ for name, count in cls.restart_counts(app).items():+ samples.setdefault(name, []).append([now.isoformat(), count])+ restarts = max((rows[-1][1] - rows[0][1] for rows in samples.values() if rows),+ default=0)+ looping = restarts >= RESTARTS+ last_restart = entry.get('last_restart')+ if restarts > 0:+ changed = any(len(rows) > 1 and rows[-1][1] > rows[-2][1] for rows in samples.values())+ if changed:+ last_restart = now.isoformat()++ incident_id = entry.get('incident_id')+ if looping and not incident_id:+ incident_id = cls._open_incident(app, restarts)+ opened.append(app.id)+ elif incident_id and not looping and (not last_restart or last_restart < cutoff):+ cls._resolve_incident(incident_id)+ resolved.append(app.id)+ incident_id = None++ new_state[str(app.id)] = {+ 'samples': samples, 'restarts_in_window': restarts,+ 'looping': bool(incident_id), 'incident_id': incident_id,+ 'last_restart': last_restart,+ }++ if new_state != state:+ SystemSettings.set(STATE_KEY, new_state, value_type='json',+ description='Crash-loop sample windows (plan 86 §E1)')+ db.session.commit()+ return {'apps': len(new_state), 'opened': opened, 'resolved': resolved}++ @staticmethod+ def _open_incident(app, restarts) -> Optional[int]:+ from app.services.monitor_service import MonitorService+ try:+ incident = MonitorService.create_incident(None, {+ 'title': f'{app.name} is crash-looping',+ 'status': 'investigating',+ 'impact': 'major',+ 'body': (f'Its containers restarted {restarts} times within '+ f'{int(WINDOW.total_seconds() // 60)} minutes. Check the app logs; '+ 'Docker keeps restarting it, so it looks running between crashes.'),+ })+ return incident.id+ except Exception as exc: # noqa: BLE001 - visibility must not break the sweep+ logger.warning('crash-loop incident for app %s failed: %s', app.id, exc)+ return None++ @staticmethod+ def _resolve_incident(incident_id) -> None:+ from app.services.monitor_service import MonitorService+ try:+ MonitorService.update_incident(incident_id, {+ 'status': 'resolved',+ 'update_body': f'No restarts for {int(WINDOW.total_seconds() // 60)} minutes.',+ })+ except Exception as exc: # noqa: BLE001+ logger.warning('crash-loop incident %s resolve failed: %s', incident_id, exc)++ @staticmethod+ def state_for(app) -> Dict:+ from app.models.system_settings import SystemSettings+ state = SystemSettings.get(STATE_KEY) or {}+ return (state.get(str(app.id)) or {}) if isinstance(state, dict) else {}diff --git a/backend/tests/test_bottleneck_hints.py b/backend/tests/test_bottleneck_hints.pyindex 71a48b35..6f690c47 100644--- a/backend/tests/test_bottleneck_hints.py+++ b/backend/tests/test_bottleneck_hints.py@@ -22,7 +22,9 @@ def _metrics(p95=1500, requests=1000, ratio=None, points=()):@pytest.fixturedef signals(monkeypatch):- state = {'metrics': _metrics(), 'cpu': {}, 'dbs': [], 'share': None, 'windows': []}+ state = {'metrics': _metrics(), 'cpu': {}, 'dbs': [], 'share': None, 'windows': [],+ 'loop': {}}+ monkeypatch.setattr(H, '_crash_loop', staticmethod(lambda app: state['loop']))monkeypatch.setattr(H, '_metrics', staticmethod(lambda app: state['metrics']))monkeypatch.setattr(H, 'cpu_percent', staticmethod(lambda name: state['cpu'].get(name)))monkeypatch.setattr(H, '_databases', staticmethod(lambda app: state['dbs']))@@ -142,3 +144,10 @@ def test_every_hint_carries_params_for_a_translated_ui(signals):signals['cpu'] = {'shop': 95}(hint,) = H.hints(_app(), now=NOW)assert hint['params'] == {'p95_ms': 1500, 'cpu': 95}+++def test_a_crash_loop_comes_first_even_without_traffic(signals):+ signals['metrics'] = _metrics(requests=0)+ signals['loop'] = {'looping': True, 'restarts_in_window': 5}+ (hint,) = H.hints(_app(), now=NOW)+ assert hint['id'] == 'crash_loop' and hint['params'] == {'restarts': 5}diff --git a/backend/tests/test_crash_loop.py b/backend/tests/test_crash_loop.pynew file mode 100644index 00000000..9cfe0986--- /dev/null+++ b/backend/tests/test_crash_loop.py@@ -0,0 +1,86 @@+"""Crash-loop visibility (plan 86 §E1): restart counts over a window, one+incident per loop, resolved after a quiet window."""+from datetime import datetime, timedelta++import pytest++from app import db+from app.models.status_page import StatusIncident+from app.services import crash_loop_service as cls_mod+from app.services.crash_loop_service import CrashLoopService+from tests.factories import make_application++T0 = datetime(2026, 7, 3, 12, 0)+++def restarts(app, monkeypatch):+ counts = {}+ monkeypatch.setattr(CrashLoopService, 'restart_counts',+ staticmethod(lambda app_row: dict(counts.get(app_row.id, {}))))+ return counts+++def _tick(minutes):+ return CrashLoopService.sweep(now=T0 + timedelta(minutes=minutes))+++def test_three_restarts_in_ten_minutes_open_one_incident(restarts):+ site = make_application(db, name='flappy', status='running')+ for minute, count in ((0, 0), (1, 1), (2, 2)):+ restarts[site.id] = {'flappy': count}+ assert _tick(minute)['opened'] == []+ restarts[site.id] = {'flappy': 3}+ assert _tick(3)['opened'] == [site.id]+ restarts[site.id] = {'flappy': 5}+ assert _tick(4)['opened'] == [], 'one incident per loop'+ (incident,) = StatusIncident.query.all()+ assert incident.title == 'flappy is crash-looping' and incident.status == 'investigating'+ assert CrashLoopService.state_for(site)['looping'] is True+++def test_a_quiet_window_resolves_it(restarts):+ site = make_application(db, name='flappy', status='running')+ for minute, count in ((0, 0), (1, 3)):+ restarts[site.id] = {'flappy': count}+ _tick(minute)+ assert _tick(6)['resolved'] == [], 'a restart inside the window keeps it open'+ assert _tick(12)['resolved'] == [site.id]+ (incident,) = StatusIncident.query.all()+ assert incident.status == 'resolved'+ assert CrashLoopService.state_for(site)['looping'] is False+++def test_slow_restarts_are_not_a_loop(restarts):+ site = make_application(db, name='steady', status='running')+ for minute, count in ((0, 0), (15, 1), (30, 2), (45, 3)):+ restarts[site.id] = {'steady': count}+ assert _tick(minute)['opened'] == []+ assert StatusIncident.query.count() == 0+++def test_workers_count_too(restarts):+ site = make_application(db, name='web', status='running')+ restarts[site.id] = {'web': 0, f'serverkit-app-{site.id}-worker': 0}+ _tick(0)+ restarts[site.id] = {'web': 0, f'serverkit-app-{site.id}-worker': 4}+ assert _tick(2)['opened'] == [site.id]+++def test_state_holds_only_live_apps_and_recent_samples(restarts):+ site = make_application(db, name='web', status='running')+ restarts[site.id] = {'web': 0}+ _tick(0)+ _tick(30)+ samples = CrashLoopService.state_for(site)['samples']['web']+ assert len(samples) == 1, 'samples older than the window are dropped'+ site.status = 'stopped'+ db.session.commit()+ _tick(31)+ assert CrashLoopService.state_for(site) == {}+++def test_the_sweep_is_a_builtin_tick():+ from app.jobs.builtin_handlers import _BUILTINS+ (row,) = [b for b in _BUILTINS if b[2] == 'crash-loop']+ assert row[0] == 'builtin.crash_loop' and row[3] == 60diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.pyindex 2869ffbe..ce48c7f4 100644--- a/backend/tests/test_jobs.py+++ b/backend/tests/test_jobs.py@@ -194,18 +194,20 @@ class TestBuiltins:# Per-app request rollups (plan 86 §A2): a builtin so its once-a-minute# successes get the 24 h tick retention.assert 'builtin.request_metrics' in kinds- assert len([k for k in kinds if k.startswith('builtin.')]) == 19+ # Crash-loop visibility (plan 86 §E1), also once a minute.+ assert 'builtin.crash_loop' in kinds+ assert len([k for k in kinds if k.startswith('builtin.')]) == 20builtin_handlers.seed_builtin_schedules()- # 19 builtin.* schedules (incl. the request-metrics tick, restore-point/job/telemetry retention,+ # 20 builtin.* schedules (incl. the request-metrics and crash-loop ticks, restore-point/job/telemetry retention,# the monitor sweep, security-feed check, recycle-bin retention and the# fleet threshold check) + login-link/SSO reapers + drift/FIM/bandwidth# sweeps + the host doctor sweep AND the fleet doctor sweep (plan 26)# + the setup-health nag (plan 22).- assert ScheduledJob.query.count() == 27+ assert ScheduledJob.query.count() == 28# Seeding twice doesn't duplicate.builtin_handlers.seed_builtin_schedules()- assert ScheduledJob.query.count() == 27+ assert ScheduledJob.query.count() == 28class TestApi:diff --git a/frontend/src/components/service-detail/HintsCard.jsx b/frontend/src/components/service-detail/HintsCard.jsxindex ff6375fb..ea3dabde 100644--- a/frontend/src/components/service-detail/HintsCard.jsx+++ b/frontend/src/components/service-detail/HintsCard.jsx@@ -25,6 +25,12 @@ export default function HintsCard({ app }) {const text = (hint) => {const p = hint.params || {};switch (hint.id) {+ case 'crash_loop':+ return {+ signal: t('app.hints.crashLoopSignal', '{{count}} restarts in the last 10 minutes.', { count: p.restarts }),+ hint: t('app.hints.crashLoop', 'The app keeps crashing and Docker keeps restarting it. Read its logs from just before a restart; no cache or database change helps until it stays up.'),+ failure: t('app.hints.crashLoopFailure', 'A restart policy hides a crash: the app looks running between crashes.'),+ };case 'app_bound':return {signal: t('app.hints.appBoundSignal', 'p95 {{ms}} ms while the app uses {{cpu}}% CPU and its database is idle.', { ms: p.p95_ms, cpu: p.cpu }),@@ -60,10 +66,11 @@ export default function HintsCard({ app }) {const actionLink = (action) => {if (!action) return null;- const label = action.target === 'settings/cache'- ? t('app.hints.openMicroCache', 'Open micro-cache settings')- : t('app.hints.attachCache', 'Attach a cache');- const to = action.target === 'settings/cache' ? `/services/${app.id}/settings/cache` : null;+ const targets = {+ 'settings/cache': { to: `/services/${app.id}/settings/cache`, label: t('app.hints.openMicroCache', 'Open micro-cache settings') },+ logs: { to: `/services/${app.id}/logs`, label: t('app.hints.openLogs', 'Open logs') },+ };+ const { to = null, label = t('app.hints.attachCache', 'Attach a cache') } = targets[action.target] || {};return to? <Link to={to} className="hints__action">{label}</Link>: <span className="hints__action hints__action--here">{label}</span>;diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.jsonindex fd257985..4f9511e7 100644--- a/frontend/src/i18n/locales/en.json+++ b/frontend/src/i18n/locales/en.json@@ -2691,6 +2691,9 @@"appBoundFailure": "A cached page can be stale for up to its lifetime.","appBoundSignal": "p95 {{ms}} ms while the app uses {{cpu}}% CPU and its database is idle.","attachCache": "Attach a cache",+ "crashLoop": "The app keeps crashing and Docker keeps restarting it. Read its logs from just before a restart; no cache or database change helps until it stays up.",+ "crashLoopFailure": "A restart policy hides a crash: the app looks running between crashes.",+ "crashLoopSignal": "{{count}} restarts in the last 10 minutes.","dbBound": "The database is the bottleneck. Look at its top queries for a missing index, or cache what is read most.","dbBoundFailure": "Cached data can be stale; invalidate on write.","dbBoundRepeated": "The same query dominates: cache its result, or add an index for it (see the query in the database Insights tab).",@@ -2702,6 +2705,7 @@"lowHit": "Most cacheable requests miss. The pages may vary per visitor, or the cache lifetime is too short for the traffic.","lowHitFailure": "A longer lifetime serves older pages.","lowHitSignal": "Micro-cache hit ratio {{ratio}}%.",+ "openLogs": "Open logs","openMicroCache": "Open micro-cache settings","title": "Where it is slow","window": "Last 24 hours"diff --git a/frontend/src/i18n/locales/es.json b/frontend/src/i18n/locales/es.jsonindex 084c24f0..8d05fa2a 100644--- a/frontend/src/i18n/locales/es.json+++ b/frontend/src/i18n/locales/es.json@@ -6791,7 +6791,11 @@"lowHitSignal": "Tasa de aciertos de la microcaché: {{ratio}}%.","openMicroCache": "Abrir ajustes de microcaché","title": "Dónde va lento",- "window": "Últimas 24 horas"+ "window": "Últimas 24 horas",+ "crashLoop": "La app sigue fallando y Docker la sigue reiniciando. Lee sus registros de justo antes de un reinicio; ningún cambio de caché o de base de datos ayuda hasta que se mantenga en pie.",+ "crashLoopFailure": "Una política de reinicio oculta un fallo: la app parece en marcha entre fallos.",+ "crashLoopSignal": "{{count}} reinicios en los últimos 10 minutos.",+ "openLogs": "Abrir registros"}},"auth": {