Skip to content

Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.

feat: alert engine with severity tiers and dedupe

karloks2005293ecb8ba358
diff --git a/src/backend/alerts.py b/src/backend/alerts.py
new file mode 100644
index 0000000..f651a94
--- /dev/null
+++ b/src/backend/alerts.py
@@ -0,0 +1,60 @@
+"""Alert engine: at most one active alert per server and rule, resolved once the rule clears.
+
+Callers describe *conditions* by a stable ``rule_key``. Raising an alert that is
+already active only refreshes its severity and text, so a condition that persists
+across many check-ins never produces duplicates.
+"""
+from __future__ import annotations
+
+from collections.abc import Iterable
+from datetime import datetime, timezone
+
+from .models import IncidentAlert
+
+INFO, WARNING, CRITICAL = "INFO", "WARNING", "CRITICAL"
+SEVERITIES = (INFO, WARNING, CRITICAL)
+
+SERVER_OFFLINE_RULE = "server_offline"
+CONTAINER_RULE_PREFIX = "container:"
+
+
+def _active(db, server_id: int):
+ return db.query(IncidentAlert).filter(IncidentAlert.server_id == server_id, IncidentAlert.status == "active")
+
+
+def active_rule_keys(db, server_id: int, prefix: str) -> set[str]:
+ return {key for (key,) in _active(db, server_id).filter(IncidentAlert.rule_key.startswith(prefix)).with_entities(IncidentAlert.rule_key)}
+
+
+def raise_alert(db, server_id: int, rule_key: str, severity: str, title: str, description: str) -> bool:
+ """Open an alert for ``rule_key``, or refresh the active one. Returns True if a new alert was opened."""
+ if severity not in SEVERITIES:
+ raise ValueError(f"Unknown severity: {severity}")
+ existing = _active(db, server_id).filter(IncidentAlert.rule_key == rule_key).first()
+ if existing:
+ existing.severity, existing.title, existing.description = severity, title[:255], description
+ return False
+ db.add(IncidentAlert(server_id=server_id, rule_key=rule_key, severity=severity, title=title[:255], description=description))
+ return True
+
+
+def resolve_alerts(db, server_id: int, rule_keys: Iterable[str], now: datetime | None = None) -> int:
+ """Resolve the active alerts for the given rules. Returns how many were resolved."""
+ rule_keys = list(rule_keys)
+ if not rule_keys:
+ return 0
+ return _active(db, server_id).filter(IncidentAlert.rule_key.in_(rule_keys)).update(
+ {"status": "resolved", "resolved_at": now or datetime.now(timezone.utc)}, synchronize_session=False
+ )
+
+
+def sync_alerts(db, server_id: int, prefix: str, firing: dict[str, tuple[str, str, str]], now: datetime | None = None) -> None:
+ """Make the active alerts under ``prefix`` match ``firing`` ({rule_key: (severity, title, description)}).
+
+ Rules that are firing are raised or refreshed; active rules under the prefix that
+ are no longer firing are resolved.
+ """
+ for rule_key, (severity, title, description) in firing.items():
+ raise_alert(db, server_id, rule_key, severity, title, description)
+ resolve_alerts(db, server_id, active_rule_keys(db, server_id, prefix) - firing.keys(), now)
+
diff --git a/src/backend/main.py b/src/backend/main.py
index 76fc99b..b731164 100644
--- a/src/backend/main.py
+++ b/src/backend/main.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import json
import re
import hashlib
import secrets
@@ -15,8 +16,9 @@ from fastapi import Cookie, Depends, FastAPI, Header, HTTPException, Query, Requ
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import PlainTextResponse
from pydantic import BaseModel, EmailStr
-from sqlalchemy import inspect, text
+from sqlalchemy import case, inspect, text
+from .alerts import CONTAINER_RULE_PREFIX, CRITICAL, SERVER_OFFLINE_RULE, SEVERITIES, active_rule_keys, raise_alert, resolve_alerts, sync_alerts
from .auth import authenticate_user, get_recent_logs, get_user_from_session, logout_user, record_audit_log, register_user
from .db import SessionLocal, check_database, get_db
from .models import AppSetting, AuditLog, Base, ChatMessage, ContainerMetric, Conversation, EnrollmentKey, HostMetric, IncidentAlert, PendingCollector, Server, User
@@ -34,6 +36,7 @@ from .config import (
SESSION_COOKIE_SECURE,
SESSION_TTL_SECONDS,
)
+from .rules import DEFAULT_THRESHOLDS, HOST_RULE_PREFIX, container_rules, host_rules, validate_thresholds
from .rate_limit import enforce_ip_limit, enforce_login_limit, enforce_register_limit, reset_limits
from ..agent.service import delete_agent_conversation, send_agent_message
@@ -81,6 +84,7 @@ class AdminSettingsRequest(BaseModel):
# Both optional so the admin page can save one setting at a time.
metrics_retention_days: int | None = None
agent_chat_enabled: bool | None = None
+ alert_thresholds: dict | None = None
class MetricsIngestRequest(BaseModel):
@@ -140,6 +144,13 @@ def startup_event():
("started_at", "VARCHAR(64)"), ("ports", "VARCHAR(512)"),
):
connection.execute(text(f"ALTER TABLE container_metrics ADD COLUMN IF NOT EXISTS {column} {column_type}"))
+ connection.execute(text("ALTER TABLE incident_alerts ADD COLUMN IF NOT EXISTS rule_key VARCHAR(255)"))
+ connection.execute(text("ALTER TABLE incident_alerts ADD COLUMN IF NOT EXISTS resolved_at TIMESTAMP WITH TIME ZONE"))
+ connection.execute(text("CREATE INDEX IF NOT EXISTS ix_incident_alerts_rule_key ON incident_alerts (rule_key)"))
+ # Alerts from before the alert engine: offline alerts keep going under their rule, the rest were
+ # duplicated on every check-in with no way to resolve, so close them; the engine re-raises live ones.
+ connection.execute(text("UPDATE incident_alerts SET rule_key = :rule WHERE rule_key IS NULL AND description = :description"), {"rule": SERVER_OFFLINE_RULE, "description": OFFLINE_ALERT_DESCRIPTION})
+ connection.execute(text("UPDATE incident_alerts SET status = 'resolved', resolved_at = NOW() WHERE rule_key IS NULL AND status = 'active'"))
if "servers" in inspect(engine).get_table_names():
columns = {column["name"] for column in inspect(engine).get_columns("servers")}
if "collector_token_hash" not in columns:
@@ -316,6 +327,7 @@ MAX_METRICS_RETENTION_DAYS = 3650
AGENT_CHAT_ENABLED_KEY = "agent_chat_enabled"
+ALERT_THRESHOLDS_KEY = "alert_thresholds"
def _set_setting(db, key: str, value: str) -> None:
@@ -339,8 +351,18 @@ def is_agent_chat_enabled(db) -> bool:
return setting is None or setting.value == "true"
+def get_alert_thresholds(db) -> dict:
+ """Saved thresholds over the defaults, so thresholds added later get their default value."""
+ setting = db.get(AppSetting, ALERT_THRESHOLDS_KEY)
+ try:
+ saved = json.loads(setting.value) if setting else {}
+ return validate_thresholds({k: v for k, v in saved.items() if k in DEFAULT_THRESHOLDS}, DEFAULT_THRESHOLDS)
+ except (ValueError, AttributeError):
+ return dict(DEFAULT_THRESHOLDS)
+
+
def _admin_settings(db) -> dict:
- return {"metrics_retention_days": get_metrics_retention_days(db), "agent_chat_enabled": is_agent_chat_enabled(db)}
+ return {"metrics_retention_days": get_metrics_retention_days(db), "agent_chat_enabled": is_agent_chat_enabled(db), "alert_thresholds": get_alert_thresholds(db)}
def purge_old_metrics(db, now: datetime | None = None) -> int:
@@ -371,6 +393,16 @@ def update_admin_settings(payload: AdminSettingsRequest, session_id: Annotated[s
if payload.agent_chat_enabled is not None:
_set_setting(db, AGENT_CHAT_ENABLED_KEY, "true" if payload.agent_chat_enabled else "false")
changes.append(f"{'Enabled' if payload.agent_chat_enabled else 'Disabled'} chat with the agent")
+ if payload.alert_thresholds is not None:
+ current = get_alert_thresholds(db)
+ try:
+ thresholds = validate_thresholds(payload.alert_thresholds, current)
+ except ValueError as exc:
+ raise HTTPException(status_code=400, detail=str(exc))
+ changed = {key: value for key, value in thresholds.items() if current[key] != value}
+ _set_setting(db, ALERT_THRESHOLDS_KEY, json.dumps(thresholds))
+ if changed:
+ changes.append("Changed alert thresholds: " + ", ".join(f"{key}={value}" for key, value in changed.items()))
db.commit()
for change in changes:
record_audit_log(db, user_id=user["id"], level="INFO", message=change, source="admin")
@@ -384,6 +416,9 @@ def agent_config(session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_N
return {"chat_enabled": is_agent_chat_enabled(db)}
+OVERVIEW_ALERT_LIMIT = 50
+
+
@app.get("/monitoring/overview")
def monitoring_overview(session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):
# Servers belong to the install: everyone signed in can view them, only admins manage them.
@@ -404,9 +439,16 @@ def monitoring_overview(session_id: Annotated[str | None, Cookie(alias=SESSION_C
"host": ({key: getattr(metric, key) for key in ("time", "cpu_utilization_pct", "load_1m", "load_5m", "load_15m", "ram_used_gb", "ram_available_gb", "ram_pct", "swap_pct", "disk_usage_pct", "disk_inodes_pct", "net_bytes_sent_mb", "net_bytes_recv_mb", "system_uptime_seconds")} if metric else None),
"containers": [{"container_id": c.container_id, "name": c.name, "image": c.image, "status": c.status, "restart_count": c.restart_count, "cpu_pct": c.cpu_pct, "memory_used_mb": c.memory_used_mb, "memory_pct": c.memory_pct, "healthcheck_status": c.healthcheck_status, "time": c.time.isoformat()} for c in latest.values()],
})
- alerts = db.query(IncidentAlert).filter(IncidentAlert.status == "active").order_by(IncidentAlert.created_at.desc()).limit(20).all()
+ active_alerts = db.query(IncidentAlert).filter(IncidentAlert.status == "active")
+ # Most severe first, then newest.
+ severity_rank = case({severity: rank for rank, severity in enumerate(reversed(SEVERITIES))}, value=IncidentAlert.severity, else_=len(SEVERITIES))
+ alerts = active_alerts.order_by(severity_rank, IncidentAlert.created_at.desc()).limit(OVERVIEW_ALERT_LIMIT).all()
+ hostnames = {server.id: server.hostname for server in servers}
pending = visible_pending_collectors(db) if _is_admin(db, user["id"]) else []
- return {"servers": result, "pending_collectors": pending, "alerts": [{"id": a.id, "server_id": a.server_id, "severity": a.severity, "title": a.title, "description": a.description, "created_at": a.created_at.isoformat()} for a in alerts]}
+ return {
+ "servers": result, "pending_collectors": pending, "alert_count": active_alerts.count(),
+ "alerts": [{"id": a.id, "server_id": a.server_id, "hostname": hostnames.get(a.server_id), "severity": a.severity, "title": a.title, "description": a.description, "created_at": a.created_at.isoformat()} for a in alerts],
+ }
CONTAINER_DETAIL_FIELDS = (
@@ -640,9 +682,7 @@ def mark_offline_servers(db, now: datetime | None = None) -> int:
last_seen = server.last_seen_at if server.last_seen_at.tzinfo else server.last_seen_at.replace(tzinfo=timezone.utc)
if now - last_seen < timedelta(seconds=server.check_interval_seconds * OFFLINE_AFTER_MISSED_CHECKS):
continue
- already_open = db.query(IncidentAlert).filter(IncidentAlert.server_id == server.id, IncidentAlert.status == "active", IncidentAlert.description == OFFLINE_ALERT_DESCRIPTION).first()
- if not already_open:
- db.add(IncidentAlert(server_id=server.id, severity="CRITICAL", title=_offline_title(server), description=OFFLINE_ALERT_DESCRIPTION))
+ if raise_alert(db, server.id, SERVER_OFFLINE_RULE, CRITICAL, _offline_title(server), OFFLINE_ALERT_DESCRIPTION):
raised += 1
db.commit()
return raised
@@ -723,15 +763,20 @@ def ingest_metrics(payload: MetricsIngestRequest, authorization: Annotated[str |
for field in ("hostname", "ip_address", "os_info", "cpu_cores", "total_ram_gb"):
if field in payload.server and payload.server[field] is not None:
setattr(server, field, payload.server[field])
- db.add(HostMetric(server_id=server.id, **{k: v for k, v in payload.host.items() if k in HostMetric.__table__.columns.keys() and k not in {"id", "server_id", "time"}}))
- for container in payload.containers:
- if container.get("container_id") and container.get("status"):
- allowed = {k: v for k, v in container.items() if k in ContainerMetric.__table__.columns.keys() and k not in {"id", "server_id", "time"}}
- db.add(ContainerMetric(server_id=server.id, **allowed))
- if str(container.get("status", "")).lower() in {"restarting", "unhealthy"}:
- db.add(IncidentAlert(server_id=server.id, severity="CRITICAL", title=f"Container {container.get('name', container['container_id'])} is {container['status']}", description="Automatski detektirano pri zaprimanju metrika."))
- server.last_seen_at = datetime.now(timezone.utc)
- db.query(IncidentAlert).filter(IncidentAlert.server_id == server.id, IncidentAlert.status == "active", IncidentAlert.description == OFFLINE_ALERT_DESCRIPTION).update({"status": "resolved"}, synchronize_session=False)
+ now = datetime.now(timezone.utc)
+ db.add(HostMetric(server_id=server.id, time=now, **{k: v for k, v in payload.host.items() if k in HostMetric.__table__.columns.keys() and k not in {"id", "server_id", "time"}}))
+ containers = [c for c in payload.containers if c.get("container_id") and c.get("status")]
+ for container in containers:
+ allowed = {k: v for k, v in container.items() if k in ContainerMetric.__table__.columns.keys() and k not in {"id", "server_id", "time"}}
+ db.add(ContainerMetric(server_id=server.id, time=now, **allowed))
+ db.flush() # the rules read this check-in back from the history tables
+ thresholds = get_alert_thresholds(db)
+ sync_alerts(db, server.id, HOST_RULE_PREFIX, host_rules(db, server, thresholds, active_rule_keys(db, server.id, HOST_RULE_PREFIX)))
+ # The collector sends no containers when Docker is unreachable; keep container alerts as they are rather than resolving them all.
+ if containers:
+ sync_alerts(db, server.id, CONTAINER_RULE_PREFIX, container_rules(db, server.id, containers, thresholds, now))
+ server.last_seen_at = now
+ resolve_alerts(db, server.id, [SERVER_OFFLINE_RULE])
db.commit()
return {"status": "accepted", "interval_seconds": server.check_interval_seconds}
diff --git a/src/backend/models.py b/src/backend/models.py
index dd9a342..5ceb4a6 100644
--- a/src/backend/models.py
+++ b/src/backend/models.py
@@ -188,4 +188,7 @@ class IncidentAlert(Base):
title: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[str] = mapped_column(Text, nullable=False)
status: Mapped[str] = mapped_column(String(32), default="active", nullable=False)
+ # Identifies the condition behind the alert (e.g. "server_offline"), so a server has at most one active alert per rule.
+ rule_key: Mapped[Optional[str]] = mapped_column(String(255), index=True, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), index=True, nullable=False)
+ resolved_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True)
diff --git a/src/backend/rules.py b/src/backend/rules.py
new file mode 100644
index 0000000..4215f95
--- /dev/null
+++ b/src/backend/rules.py
@@ -0,0 +1,138 @@
+"""Alert rules evaluated on every collector check-in, and the thresholds they use.
+
+Rules only decide *what is firing*; ``alerts.sync_alerts`` turns that into
+opened, refreshed and resolved alerts.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+from sqlalchemy import func
+
+from .alerts import CONTAINER_RULE_PREFIX, CRITICAL, WARNING
+from .models import ContainerMetric, HostMetric
+
+HOST_RULE_PREFIX = "host:"
+
+DEFAULT_THRESHOLDS: dict[str, float | int] = {
+ "disk_warning_pct": 85, "disk_critical_pct": 95,
+ "inodes_warning_pct": 85, "inodes_critical_pct": 95,
+ "memory_warning_pct": 90, "memory_critical_pct": 95,
+ "swap_warning_pct": 50,
+ "load_warning_per_core": 1.5, "load_critical_per_core": 2.0,
+ # How many check-ins in a row a host metric must breach before an alert opens.
+ "sustained_checks": 3,
+ "restart_loop_count": 3, "restart_loop_window_minutes": 30,
+}
+
+_LIMITS: dict[str, tuple[float, float]] = {
+ **{key: (1, 100) for key in DEFAULT_THRESHOLDS if key.endswith("_pct")},
+ "load_warning_per_core": (0.1, 50), "load_critical_per_core": (0.1, 50),
+ "sustained_checks": (1, 20),
+ "restart_loop_count": (1, 100), "restart_loop_window_minutes": (5, 1440),
+}
+_INTEGER_KEYS = {"sustained_checks", "restart_loop_count", "restart_loop_window_minutes"}
+
+
+def validate_thresholds(values: dict, current: dict) -> dict:
+ """Merge ``values`` over ``current`` and validate the result. Raises ValueError with a readable message."""
+ unknown = set(values) - DEFAULT_THRESHOLDS.keys()
+ if unknown:
+ raise ValueError(f"Unknown threshold: {', '.join(sorted(unknown))}.")
+ merged = {**current, **values}
+ for key, value in merged.items():
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ raise ValueError(f"{key} must be a number.")
+ if key in _INTEGER_KEYS and value != int(value):
+ raise ValueError(f"{key} must be a whole number.")
+ low, high = _LIMITS[key]
+ if not low <= value <= high:
+ raise ValueError(f"{key} must be between {low:g} and {high:g}.")
+ merged[key] = int(value) if key in _INTEGER_KEYS else value
+ for metric in ("disk", "inodes", "memory"):
+ if merged[f"{metric}_warning_pct"] >= merged[f"{metric}_critical_pct"]:
+ raise ValueError(f"The {metric} warning threshold must be below its critical threshold.")
+ if merged["load_warning_per_core"] >= merged["load_critical_per_core"]:
+ raise ValueError("The load warning threshold must be below its critical threshold.")
+ return merged
+
+
+def _level(value: float | None, warning: float, critical: float | None) -> int:
+ """0 = fine, 1 = warning, 2 = critical."""
+ if value is None:
+ return 0
+ if critical is not None and value >= critical:
+ return 2
+ return 1 if value >= warning else 0
+
+
+def host_rules(db, server, thresholds: dict, active_keys: set[str]) -> dict[str, tuple[str, str, str]]:
+ """Evaluate the host rules against the server's most recent check-ins (newest first)."""
+ sustained = thresholds["sustained_checks"]
+ samples = db.query(HostMetric).filter(HostMetric.server_id == server.id).order_by(HostMetric.time.desc(), HostMetric.id.desc()).limit(sustained).all()
+ if not samples:
+ return {}
+ cores = server.cpu_cores or None
+
+ def load_per_core(sample):
+ return sample.load_5m / cores if cores and sample.load_5m is not None else None
+
+ checks = (
+ # (rule, value getter, warning, critical, title, description)
+ ("disk", lambda s: s.disk_usage_pct, thresholds["disk_warning_pct"], thresholds["disk_critical_pct"],
+ "Disk usage on {host} is {value:.0f}%", "The root filesystem is {value:.1f}% full. Free up space before it fills completely."),
+ ("inodes", lambda s: s.disk_inodes_pct, thresholds["inodes_warning_pct"], thresholds["inodes_critical_pct"],
+ "Inode usage on {host} is {value:.0f}%", "{value:.1f}% of inodes on the root filesystem are used. Many small files (caches, logs, mail queues) are the usual cause."),
+ ("memory", lambda s: s.ram_pct, thresholds["memory_warning_pct"], thresholds["memory_critical_pct"],
+ "Memory usage on {host} is {value:.0f}%", "{value:.1f}% of RAM is in use. Check the containers and processes using the most memory."),
+ ("swap", lambda s: s.swap_pct, thresholds["swap_warning_pct"], None,
+ "Swap usage on {host} is {value:.0f}%", "{value:.1f}% of swap has been in use for several check-ins, an early sign of memory pressure."),
+ ("load", load_per_core, thresholds["load_warning_per_core"], thresholds["load_critical_per_core"],
+ "Load on {host} is {value:.1f}× its cores", "The 5-minute load average is {value:.2f} per CPU core across " + f"{cores or '?'} cores. Work is queuing for the CPU."),
+ )
+ firing: dict[str, tuple[str, str, str]] = {}
+ for rule, value_of, warning, critical, title, description in checks:
+ key = f"{HOST_RULE_PREFIX}{rule}"
+ latest_value = value_of(samples[0])
+ levels = [_level(value_of(sample), warning, critical) for sample in samples]
+ # Opening an alert, or escalating it to critical, needs every recent check-in to breach.
+ sustained_level = min(levels) if len(samples) >= sustained else 0
+ level = max(sustained_level, min(levels[0], 1)) if key in active_keys else sustained_level
+ if level:
+ firing[key] = (
+ CRITICAL if level == 2 else WARNING,
+ title.format(host=server.hostname, value=latest_value),
+ description.format(value=latest_value),
+ )
+ return firing
+
+
+def container_rules(db, server_id: int, containers: list[dict], thresholds: dict, now: datetime) -> dict[str, tuple[str, str, str]]:
+ """Evaluate the container rules for one check-in's container records."""
+ window = thresholds["restart_loop_window_minutes"]
+ # Lowest restart count seen per container within the window; the rise since then is the number of restarts.
+ # Keyed by container ID because a recreated container starts counting from zero again.
+ ids = [container["container_id"] for container in containers]
+ earliest = dict(
+ db.query(ContainerMetric.container_id, func.min(ContainerMetric.restart_count))
+ .filter(ContainerMetric.server_id == server_id, ContainerMetric.container_id.in_(ids), ContainerMetric.time >= now - timedelta(minutes=window))
+ .group_by(ContainerMetric.container_id)
+ .all()
+ ) if ids else {}
+
+ firing: dict[str, tuple[str, str, str]] = {}
+ for container in containers:
+ # Compose recreates containers with new IDs but the same name, so key alerts by name.
+ name = container.get("name") or container["container_id"]
+ restarts = int(container.get("restart_count") or 0) - earliest.get(container["container_id"], int(container.get("restart_count") or 0))
+ if restarts >= thresholds["restart_loop_count"]:
+ firing[f"{CONTAINER_RULE_PREFIX}{name}:restart_loop"] = (
+ CRITICAL, f"Container {name} is in a restart loop",
+ f"{name} restarted {restarts} times in the last {window} minutes (restart count {container.get('restart_count', 0)}). Check its logs for the reason it exits.",
+ )
+ if str(container.get("healthcheck_status") or "").lower() == "unhealthy":
+ firing[f"{CONTAINER_RULE_PREFIX}{name}:unhealthy"] = (
+ CRITICAL, f"Container {name} is unhealthy",
+ f"The Docker HEALTHCHECK for {name} is failing. Inspect it with: docker inspect --format '{{{{json .State.Health}}}}' {name}",
+ )
+ return firing
diff --git a/src/frontend/src/App.jsx b/src/frontend/src/App.jsx
index bc42b08..ced5a0d 100644
--- a/src/frontend/src/App.jsx
+++ b/src/frontend/src/App.jsx
@@ -431,6 +431,12 @@ function RegisterPage({ onRegister }) {
// Izvan ili na vrhu komponente postavi guard za izbjegavanje paralelnih poziva
let isFetchingOverview = false
+const ALERT_STYLES = {
+ CRITICAL: { box: 'border-red-400/25 bg-red-500/5', badge: 'bg-red-500/15 text-red-300', title: 'text-red-200' },
+ WARNING: { box: 'border-amber-400/20 bg-amber-400/5', badge: 'bg-amber-400/15 text-amber-300', title: 'text-amber-200' },
+ INFO: { box: 'border-sky-400/20 bg-sky-400/5', badge: 'bg-sky-400/15 text-sky-300', title: 'text-sky-200' },
+}
+
function DashboardPage({ user, onLogout }) {
const [overview, setOverview] = useState({ servers: [], alerts: [] })
const [error, setError] = useState('')
@@ -536,7 +542,7 @@ const refresh = async () => {
['Average memory', latestMetrics.length ? `${average('ram_pct')}%` : '—', 'RAM utilization'],
['Average disk', latestMetrics.length ? `${average('disk_usage_pct')}%` : '—', 'Disk utilization'],
['Containers running', String(runningContainers), 'Across monitored servers'],
- ['Active incidents', String(overview.alerts?.length || 0), 'Needs attention'],
+ ['Active incidents', String(overview.alert_count ?? overview.alerts?.length ?? 0), 'Needs attention'],
].map(([label, value, detail]) => <div key={label} className="glass-panel rounded-[24px] p-5"><div className="text-sm text-[#9aa0a6]">{label}</div><div className="mt-3 text-3xl font-semibold text-white">{value}</div><div className="mt-2 text-xs text-[#7f8589]">{detail}</div></div>)}
</section>
@@ -562,7 +568,7 @@ const refresh = async () => {
</article>
})}</div>}
</section>
- <section className="glass-panel rounded-[28px] p-5 md:p-6"><div className="mb-4"><div className="text-xs uppercase tracking-[0.2em] text-[#9aa0a6]">Alerts</div><h2 className="mt-1 text-xl font-semibold text-white">Active incidents</h2></div>{overview.alerts?.length ? <div className="space-y-2">{overview.alerts.map((alert) => <div key={alert.id} className="rounded-xl border border-amber-400/20 bg-amber-400/5 p-3"><div className="font-medium text-amber-200">{alert.title}</div><div className="mt-1 text-sm text-[#bcbcbc]">{alert.description}</div></div>)}</div> : <p className="text-sm text-[#9aa0a6]">No active incidents recorded.</p>}</section>
+ <section className="glass-panel rounded-[28px] p-5 md:p-6"><div className="mb-4"><div className="text-xs uppercase tracking-[0.2em] text-[#9aa0a6]">Alerts</div><h2 className="mt-1 text-xl font-semibold text-white">Active incidents</h2></div>{overview.alerts?.length ? <div className="space-y-2">{overview.alerts.map((alert) => { const style = ALERT_STYLES[alert.severity] || ALERT_STYLES.WARNING; return <div key={alert.id} className={`rounded-xl border p-3 ${style.box}`}><div className="flex items-center gap-2"><span className={`rounded-full px-2 py-0.5 text-[10px] font-semibold tracking-wider ${style.badge}`}>{alert.severity}</span><div className={`font-medium ${style.title}`}>{alert.title}</div></div><div className="mt-1 text-xs text-[#9aa0a6]">{alert.hostname || `Server ${alert.server_id}`} · since {new Date(alert.created_at).toLocaleString([], { dateStyle: 'medium', timeStyle: 'short' })}</div><div className="mt-1 text-sm text-[#bcbcbc]">{alert.description}</div></div> })}{overview.alert_count > overview.alerts.length ? <p className="text-xs text-[#9aa0a6]">Showing the {overview.alerts.length} most severe of {overview.alert_count} incidents.</p> : null}</div> : <p className="text-sm text-[#9aa0a6]">No active incidents recorded.</p>}</section>
</div>
</AppShell>
)
@@ -1316,14 +1322,97 @@ function AdminAgentTab() {
)
}
-function AdminAlertsTab() {
- const thresholds = [
- ['Disk usage above', 90, '%'],
- ['Memory usage above', 90, '%'],
- ['CPU usage above (for 10 min)', 95, '%'],
- ['Container restarts within 30 min', 3, 'restarts'],
- ]
+// [label, hint, warning key, critical key (null when the rule only warns), unit, step]
+const thresholdRows = [
+ ['Disk usage', 'Root filesystem', 'disk_warning_pct', 'disk_critical_pct', '%', 1],
+ ['Inode usage', 'Root filesystem', 'inodes_warning_pct', 'inodes_critical_pct', '%', 1],
+ ['Memory usage', null, 'memory_warning_pct', 'memory_critical_pct', '%', 1],
+ ['Swap usage', 'Warning only', 'swap_warning_pct', null, '%', 1],
+ ['Load per CPU core', '5-minute average', 'load_warning_per_core', 'load_critical_per_core', '×', 0.1],
+]
+
+const thresholdInputClass = 'w-20 rounded-xl border border-[#1f2326] bg-[#0d1013] px-2 py-1 text-right text-white outline-none focus:border-white/30 disabled:opacity-50'
+
+function ThresholdInput({ label, value, step, onChange, disabled }) {
+ return <input type="number" aria-label={label} value={value ?? ''} step={step} min={0} disabled={disabled} onChange={(event) => onChange(event.target.value === '' ? '' : Number(event.target.value))} className={thresholdInputClass} />
+}
+
+function IncidentThresholdsSection() {
+ const [saved, setSaved] = useState(null)
+ const [values, setValues] = useState(null)
+ const [saving, setSaving] = useState(false)
+ const [message, setMessage] = useState(null)
+
+ useEffect(() => {
+ apiRequest('/admin/settings', { method: 'GET' })
+ .then((data) => {
+ setSaved(data.alert_thresholds)
+ setValues(data.alert_thresholds)
+ })
+ .catch((error) => setMessage({ ok: false, text: error.message || 'Could not load the thresholds.' }))
+ }, [])
+
+ const set = (key) => (value) => setValues((current) => ({ ...current, [key]: value }))
+
+ const save = async () => {
+ setSaving(true)
+ setMessage(null)
+ try {
+ const data = await apiRequest('/admin/settings', { method: 'PUT', body: JSON.stringify({ alert_thresholds: values }) })
+ setSaved(data.alert_thresholds)
+ setValues(data.alert_thresholds)
+ setMessage({ ok: true, text: 'Saved. New thresholds apply from each server\'s next check-in.' })
+ } catch (error) {
+ setMessage({ ok: false, text: error.message || 'Could not save the thresholds.' })
+ } finally {
+ setSaving(false)
+ }
+ }
+
+ const loading = values === null
+ const unchanged = loading || JSON.stringify(values) === JSON.stringify(saved)
+ const incomplete = !loading && Object.values(values).some((value) => value === '')
+
+ return (
+ <AdminSection
+ title="Incident thresholds"
+ description="When monitoring opens an incident. Applies to every server."
+ action={
+ <button type="button" onClick={save} disabled={unchanged || incomplete || saving} className="rounded-2xl bg-linear-to-r from-white to-[#cfcfcf] px-4 py-2.5 text-sm font-medium text-[#0b0b0b] disabled:cursor-not-allowed disabled:opacity-40">
+ {saving ? 'Saving…' : 'Save changes'}
+ </button>
+ }
+ >
+ <div className="space-y-2">
+ <div className="flex justify-end gap-2 px-4 text-[10px] uppercase tracking-wider text-[#7f8589]"><span className="w-20 text-right text-amber-300/80">Warning</span><span className="w-20 text-right text-red-300/80">Critical</span><span className="w-4" /></div>
+ {thresholdRows.map(([label, hint, warningKey, criticalKey, unit, step]) => (
+ <div key={label} className="flex items-center justify-between gap-4 rounded-2xl border border-[#1f2326] bg-[#0d1013]/60 px-4 py-2.5 text-sm">
+ <span><span className="text-[#bcbcbc]">{label}</span>{hint && <span className="block text-xs text-[#7f8589]">{hint}</span>}</span>
+ <span className="flex items-center gap-2">
+ <ThresholdInput label={`${label} warning`} value={values?.[warningKey]} step={step} onChange={set(warningKey)} disabled={loading} />
+ {criticalKey ? <ThresholdInput label={`${label} critical`} value={values?.[criticalKey]} step={step} onChange={set(criticalKey)} disabled={loading} /> : <span className="w-20 text-right text-[#7f8589]">—</span>}
+ <span className="w-4 text-xs text-[#9aa0a6]">{unit}</span>
+ </span>
+ </div>
+ ))}
+ <label className="flex items-center justify-between gap-4 rounded-2xl border border-[#1f2326] bg-[#0d1013]/60 px-4 py-2.5 text-sm">
+ <span><span className="text-[#bcbcbc]">Check-ins in a row</span><span className="block text-xs text-[#7f8589]">How long a host metric must stay over before an incident opens</span></span>
+ <ThresholdInput label="Check-ins in a row" value={values?.sustained_checks} step={1} onChange={set('sustained_checks')} disabled={loading} />
+ </label>
+ <div className="flex items-center justify-between gap-4 rounded-2xl border border-[#1f2326] bg-[#0d1013]/60 px-4 py-2.5 text-sm">
+ <span><span className="text-[#bcbcbc]">Container restart loop</span><span className="block text-xs text-[#7f8589]">Critical incident</span></span>
+ <span className="flex items-center gap-2 text-xs text-[#9aa0a6]">
+ <ThresholdInput label="Restarts" value={values?.restart_loop_count} step={1} onChange={set('restart_loop_count')} disabled={loading} />restarts in
+ <ThresholdInput label="Restart window in minutes" value={values?.restart_loop_window_minutes} step={1} onChange={set('restart_loop_window_minutes')} disabled={loading} />min
+ </span>
+ </div>
+ </div>
+ {message && <p className={`mt-3 text-xs ${message.ok ? 'text-emerald-300' : 'text-amber-200'}`}>{message.text}</p>}
+ </AdminSection>
+ )
+}
+function AdminAlertsTab() {
return (
<div className="grid gap-5 xl:grid-cols-2">
<AdminSection title="Webhook" description="Receives a JSON POST for every critical incident. Replaces ADMIN_ALERT_WEBHOOK_URL in .env." action={<SaveButton />}>
@@ -1333,19 +1422,7 @@ function AdminAlertsTab() {
</div>
</AdminSection>
- <AdminSection title="Incident thresholds" description="When monitoring opens an incident." action={<SaveButton />}>
- <div className="space-y-2">
- {thresholds.map(([label, value, unit]) => (
- <label key={label} className="flex items-center justify-between gap-4 rounded-2xl border border-[#1f2326] bg-[#0d1013]/60 px-4 py-2.5 text-sm">
- <span className="text-[#bcbcbc]">{label}</span>
- <span className="flex items-center gap-2">
- <input type="number" defaultValue={value} min={0} className="w-20 rounded-xl border border-[#1f2326] bg-[#0d1013] px-2 py-1 text-right text-white outline-none focus:border-white/30" />
- <span className="w-16 text-xs text-[#9aa0a6]">{unit}</span>
- </span>
- </label>
- ))}
- </div>
- </AdminSection>
+ <IncidentThresholdsSection />
</div>
)
}
diff --git a/tests/test_admin.py b/tests/test_admin.py
index 4e3055b..a25f3af 100644
--- a/tests/test_admin.py
+++ b/tests/test_admin.py
@@ -9,7 +9,7 @@ from fastapi import HTTPException
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
-from src.backend import main
+from src.backend import main, rules
from src.backend.auth import register_user
from src.backend.models import AppSetting, Base, ContainerMetric, HostMetric, PendingCollector, Server, User
@@ -164,7 +164,7 @@ def test_admin_can_turn_agent_chat_off_and_on(fleet, monkeypatch):
result = main.update_admin_settings(main.AdminSettingsRequest(agent_chat_enabled=False), session_id="s", db=fleet)
- assert result == {"metrics_retention_days": main.DEFAULT_METRICS_RETENTION_DAYS, "agent_chat_enabled": False}
+ assert result == {"metrics_retention_days": main.DEFAULT_METRICS_RETENTION_DAYS, "agent_chat_enabled": False, "alert_thresholds": rules.DEFAULT_THRESHOLDS}
_login_as(monkeypatch, 2)
assert main.agent_config(session_id="s", db=fleet) == {"chat_enabled": False}
diff --git a/tests/test_alerts.py b/tests/test_alerts.py
new file mode 100644
index 0000000..d002870
--- /dev/null
+++ b/tests/test_alerts.py
@@ -0,0 +1,125 @@
+import os
+
+os.environ.setdefault("DATABASE_URL", "sqlite://")
+os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0")
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from src.backend import alerts, main
+from src.backend.models import Base, IncidentAlert, Server, User
+
+TOKEN = "collector-token"
+
+
+def db():
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ session = sessionmaker(bind=engine)()
+ session.add(User(id=1, name="Owner", email="[email protected]", password_hash="x", role="admin"))
+ session.add(Server(id=5, owner_id=1, hostname="nas", check_interval_seconds=60, collector_token_hash=main._hash_secret(TOKEN)))
+ session.commit()
+ return session
+
+
+def _ingest(db, containers):
+ main.ingest_metrics(main.MetricsIngestRequest(server_id=5, containers=containers), authorization=f"Bearer {TOKEN}", session_id=None, db=db)
+
+
+def _container(name, status="running", health=None, restarts=0):
+ return {"container_id": f"id-{name}", "name": name, "status": status, "healthcheck_status": health, "restart_count": restarts}
+
+
+def _active(db):
+ return db.query(IncidentAlert).filter_by(status="active").all()
+
+
+def test_raise_alert_dedupes_and_refreshes(db):
+ assert alerts.raise_alert(db, 5, "disk", alerts.WARNING, "Disk 86%", "first") is True
+ assert alerts.raise_alert(db, 5, "disk", alerts.CRITICAL, "Disk 96%", "second") is False
+
+ [alert] = _active(db)
+ assert (alert.severity, alert.title, alert.description) == (alerts.CRITICAL, "Disk 96%", "second")
+
+
+def test_raise_alert_rejects_unknown_severity(db):
+ with pytest.raises(ValueError):
+ alerts.raise_alert(db, 5, "disk", "SEVERE", "t", "d")
+
+
+def test_resolved_rule_can_fire_again_as_new_alert(db):
+ alerts.raise_alert(db, 5, "disk", alerts.WARNING, "t", "d")
+ assert alerts.resolve_alerts(db, 5, ["disk"]) == 1
+ assert alerts.raise_alert(db, 5, "disk", alerts.WARNING, "t", "d") is True
+
+ assert db.query(IncidentAlert).count() == 2
+ assert db.query(IncidentAlert).filter_by(status="resolved").one().resolved_at is not None
+
+
+def test_unhealthy_container_alerts_once_and_resolves(db):
+ for _ in range(3):
+ _ingest(db, [_container("web", health="unhealthy")])
+
+ [alert] = _active(db)
+ assert alert.rule_key == "container:web:unhealthy"
+ assert alert.severity == alerts.CRITICAL
+
+ _ingest(db, [_container("web", health="healthy")])
+ assert _active(db) == []
+
+
+def test_restart_loop_alerts_once_when_count_climbs(db):
+ for restarts in range(0, 5):
+ _ingest(db, [_container("db", status="restarting", restarts=restarts)])
+
+ [alert] = _active(db)
+ assert alert.rule_key == "container:db:restart_loop"
+ assert "restarted 4 times" in alert.description
+
+
+def test_restart_loop_ignores_old_restarts_and_status_alone(db):
+ """A high restart count from long ago, or one restarting sample, is not a loop."""
+ for _ in range(3):
+ _ingest(db, [_container("db", status="restarting", restarts=40)])
+
+ assert _active(db) == []
+
+
+def test_removed_container_alert_resolves_when_others_report(db):
+ _ingest(db, [_container("web", health="unhealthy"), _container("db")])
+ _ingest(db, [_container("db")])
+
+ assert _active(db) == []
+
+
+def test_empty_container_list_keeps_container_alerts(db):
+ """No containers means Docker was unreachable, not that every problem cleared."""
+ _ingest(db, [_container("web", health="unhealthy")])
+ _ingest(db, [])
+
+ assert len(_active(db)) == 1
+
+
+def test_container_sync_leaves_other_rules_alone(db):
+ alerts.raise_alert(db, 5, alerts.SERVER_OFFLINE_RULE, alerts.CRITICAL, "offline", "d")
+ alerts.sync_alerts(db, 5, alerts.CONTAINER_RULE_PREFIX, {})
+
+ assert len(_active(db)) == 1
+
+
+def test_overview_lists_critical_first_with_hostname_and_true_count(db, monkeypatch):
+ monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1})
+ monkeypatch.setattr(main, "OVERVIEW_ALERT_LIMIT", 2)
+ alerts.raise_alert(db, 5, "a", alerts.CRITICAL, "old critical", "d")
+ db.commit()
+ alerts.raise_alert(db, 5, "b", alerts.INFO, "new info", "d")
+ alerts.raise_alert(db, 5, "c", alerts.WARNING, "new warning", "d")
+ db.commit()
+
+ overview = main.monitoring_overview(session_id="s", db=db)
+
+ assert overview["alert_count"] == 3
+ assert [a["title"] for a in overview["alerts"]] == ["old critical", "new warning"]
+ assert overview["alerts"][0]["hostname"] == "nas"
diff --git a/tests/test_rules.py b/tests/test_rules.py
new file mode 100644
index 0000000..d5ea971
--- /dev/null
+++ b/tests/test_rules.py
@@ -0,0 +1,122 @@
+import os
+
+os.environ.setdefault("DATABASE_URL", "sqlite://")
+os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0")
+
+import pytest
+from fastapi import HTTPException
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from src.backend import main, rules
+from src.backend.alerts import CRITICAL, WARNING
+from src.backend.models import AuditLog, Base, IncidentAlert, Server, User
+
+TOKEN = "collector-token"
+
+
+def db(monkeypatch):
+ engine = create_engine("sqlite://")
+ Base.metadata.create_all(engine)
+ session = sessionmaker(bind=engine)()
+ session.add(User(id=1, name="Owner", email="[email protected]", password_hash="x", role="admin"))
+ session.add(Server(id=5, owner_id=1, hostname="nas", cpu_cores=4, check_interval_seconds=60, collector_token_hash=main._hash_secret(TOKEN)))
+ session.commit()
+ monkeypatch.setattr(main, "_require_admin", lambda session_id, db: {"id": 1})
+ monkeypatch.setattr(main, "record_audit_log", lambda db, **kw: db.add(AuditLog(user_id=kw["user_id"], level=kw["level"], message=kw["message"], source=kw["source"])))
+ return session
+
+
+def _ingest(db, **host):
+ main.ingest_metrics(main.MetricsIngestRequest(server_id=5, host=host), authorization=f"Bearer {TOKEN}", session_id=None, db=db)
+
+
+def _active(db):
+ return {a.rule_key: a for a in db.query(IncidentAlert).filter_by(status="active")}
+
+
+def test_host_alert_needs_sustained_breach(db):
+ _ingest(db, disk_usage_pct=90)
+ _ingest(db, disk_usage_pct=90)
+ assert _active(db) == {}
+
+ _ingest(db, disk_usage_pct=90)
+ alert = _active(db)["host:disk"]
+ assert alert.severity == WARNING
+ assert alert.title == "Disk usage on nas is 90%"
+
+
+def test_single_spike_does_not_alert(db):
+ for value in (40, 99, 40, 99, 40):
+ _ingest(db, ram_pct=value)
+
+ assert _active(db) == {}
+
+
+def test_open_alert_stays_open_until_value_drops_and_escalates_only_when_sustained(db):
+ for _ in range(3):
+ _ingest(db, disk_usage_pct=90)
+ _ingest(db, disk_usage_pct=97)
+ assert _active(db)["host:disk"].severity == WARNING
+
+ _ingest(db, disk_usage_pct=97)
+ _ingest(db, disk_usage_pct=97)
+ assert _active(db)["host:disk"].severity == CRITICAL
+
+ _ingest(db, disk_usage_pct=88)
+ assert _active(db)["host:disk"].severity == WARNING
+
+ _ingest(db, disk_usage_pct=50)
+ assert _active(db) == {}
+
+
+def test_load_is_compared_per_core(db):
+ for _ in range(3):
+ _ingest(db, load_5m=5.0) # 1.25 per core on 4 cores
+ assert _active(db) == {}
+
+ for _ in range(3):
+ _ingest(db, load_5m=9.0) # 2.25 per core
+ assert _active(db)["host:load"].severity == CRITICAL
+
+
+def test_swap_only_warns(db):
+ for _ in range(3):
+ _ingest(db, swap_pct=100)
+
+ assert _active(db)["host:swap"].severity == WARNING
+
+
+def test_missing_metric_does_not_alert(db):
+ for _ in range(3):
+ _ingest(db)
+
+ assert _active(db) == {}
+
+
+def test_saved_thresholds_apply(db):
+ main.update_admin_settings(main.AdminSettingsRequest(alert_thresholds={"sustained_checks": 1, "memory_warning_pct": 50}), session_id="s", db=db)
+
+ _ingest(db, ram_pct=60)
+
+ assert _active(db)["host:memory"].severity == WARNING
+
+
+def test_threshold_update_is_validated_and_audited(db):
+ for bad in ({"disk_warning_pct": 99}, {"disk_warning_pct": 0}, {"sustained_checks": 1.5}, {"nope": 1}, {"memory_warning_pct": "90"}):
+ with pytest.raises(HTTPException) as exc:
+ main.update_admin_settings(main.AdminSettingsRequest(alert_thresholds=bad), session_id="s", db=db)
+ assert exc.value.status_code == 400
+
+ result = main.update_admin_settings(main.AdminSettingsRequest(alert_thresholds={"disk_critical_pct": 98}), session_id="s", db=db)
+
+ assert result["alert_thresholds"] == {**rules.DEFAULT_THRESHOLDS, "disk_critical_pct": 98}
+ assert db.query(AuditLog).one().message == "Changed alert thresholds: disk_critical_pct=98"
+
+
+def test_corrupt_saved_thresholds_fall_back_to_defaults(db):
+ main._set_setting(db, main.ALERT_THRESHOLDS_KEY, "not json")
+ db.commit()
+
+ assert main.get_alert_thresholds(db) == rules.DEFAULT_THRESHOLDS