Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.
Python ai-agentsaiopsdashboarddockerfastapihomelabincident-managementllmmonitoringopenaireactroot-cause-analysisself-hosteduptime-monitoring
feat: alert engine with severity tiers and dedupe
karloks2005293ecb8ba358
diff --git a/src/backend/alerts.py b/src/backend/alerts.pynew file mode 100644index 0000000..f651a94--- /dev/null+++ b/src/backend/alerts.py@@ -0,0 +1,60 @@+"""Alert engine: at most one active alert per server and rule, resolved once the rule clears.++Callers describe *conditions* by a stable ``rule_key``. Raising an alert that is+already active only refreshes its severity and text, so a condition that persists+across many check-ins never produces duplicates.+"""+from __future__ import annotations++from collections.abc import Iterable+from datetime import datetime, timezone++from .models import IncidentAlert++INFO, WARNING, CRITICAL = "INFO", "WARNING", "CRITICAL"+SEVERITIES = (INFO, WARNING, CRITICAL)++SERVER_OFFLINE_RULE = "server_offline"+CONTAINER_RULE_PREFIX = "container:"+++def _active(db, server_id: int):+ return db.query(IncidentAlert).filter(IncidentAlert.server_id == server_id, IncidentAlert.status == "active")+++def active_rule_keys(db, server_id: int, prefix: str) -> set[str]:+ return {key for (key,) in _active(db, server_id).filter(IncidentAlert.rule_key.startswith(prefix)).with_entities(IncidentAlert.rule_key)}+++def raise_alert(db, server_id: int, rule_key: str, severity: str, title: str, description: str) -> bool:+ """Open an alert for ``rule_key``, or refresh the active one. Returns True if a new alert was opened."""+ if severity not in SEVERITIES:+ raise ValueError(f"Unknown severity: {severity}")+ existing = _active(db, server_id).filter(IncidentAlert.rule_key == rule_key).first()+ if existing:+ existing.severity, existing.title, existing.description = severity, title[:255], description+ return False+ db.add(IncidentAlert(server_id=server_id, rule_key=rule_key, severity=severity, title=title[:255], description=description))+ return True+++def resolve_alerts(db, server_id: int, rule_keys: Iterable[str], now: datetime | None = None) -> int:+ """Resolve the active alerts for the given rules. Returns how many were resolved."""+ rule_keys = list(rule_keys)+ if not rule_keys:+ return 0+ return _active(db, server_id).filter(IncidentAlert.rule_key.in_(rule_keys)).update(+ {"status": "resolved", "resolved_at": now or datetime.now(timezone.utc)}, synchronize_session=False+ )+++def sync_alerts(db, server_id: int, prefix: str, firing: dict[str, tuple[str, str, str]], now: datetime | None = None) -> None:+ """Make the active alerts under ``prefix`` match ``firing`` ({rule_key: (severity, title, description)}).++ Rules that are firing are raised or refreshed; active rules under the prefix that+ are no longer firing are resolved.+ """+ for rule_key, (severity, title, description) in firing.items():+ raise_alert(db, server_id, rule_key, severity, title, description)+ resolve_alerts(db, server_id, active_rule_keys(db, server_id, prefix) - firing.keys(), now)+diff --git a/src/backend/main.py b/src/backend/main.pyindex 76fc99b..b731164 100644--- a/src/backend/main.py+++ b/src/backend/main.py@@ -1,5 +1,6 @@from __future__ import annotations+import jsonimport reimport hashlibimport secrets@@ -15,8 +16,9 @@ from fastapi import Cookie, Depends, FastAPI, Header, HTTPException, Query, Requfrom fastapi.middleware.cors import CORSMiddlewarefrom fastapi.responses import PlainTextResponsefrom pydantic import BaseModel, EmailStr-from sqlalchemy import inspect, text+from sqlalchemy import case, inspect, text+from .alerts import CONTAINER_RULE_PREFIX, CRITICAL, SERVER_OFFLINE_RULE, SEVERITIES, active_rule_keys, raise_alert, resolve_alerts, sync_alertsfrom .auth import authenticate_user, get_recent_logs, get_user_from_session, logout_user, record_audit_log, register_userfrom .db import SessionLocal, check_database, get_dbfrom .models import AppSetting, AuditLog, Base, ChatMessage, ContainerMetric, Conversation, EnrollmentKey, HostMetric, IncidentAlert, PendingCollector, Server, User@@ -34,6 +36,7 @@ from .config import (SESSION_COOKIE_SECURE,SESSION_TTL_SECONDS,)+from .rules import DEFAULT_THRESHOLDS, HOST_RULE_PREFIX, container_rules, host_rules, validate_thresholdsfrom .rate_limit import enforce_ip_limit, enforce_login_limit, enforce_register_limit, reset_limitsfrom ..agent.service import delete_agent_conversation, send_agent_message@@ -81,6 +84,7 @@ class AdminSettingsRequest(BaseModel):# Both optional so the admin page can save one setting at a time.metrics_retention_days: int | None = Noneagent_chat_enabled: bool | None = None+ alert_thresholds: dict | None = Noneclass MetricsIngestRequest(BaseModel):@@ -140,6 +144,13 @@ def startup_event():("started_at", "VARCHAR(64)"), ("ports", "VARCHAR(512)"),):connection.execute(text(f"ALTER TABLE container_metrics ADD COLUMN IF NOT EXISTS {column} {column_type}"))+ connection.execute(text("ALTER TABLE incident_alerts ADD COLUMN IF NOT EXISTS rule_key VARCHAR(255)"))+ connection.execute(text("ALTER TABLE incident_alerts ADD COLUMN IF NOT EXISTS resolved_at TIMESTAMP WITH TIME ZONE"))+ connection.execute(text("CREATE INDEX IF NOT EXISTS ix_incident_alerts_rule_key ON incident_alerts (rule_key)"))+ # Alerts from before the alert engine: offline alerts keep going under their rule, the rest were+ # duplicated on every check-in with no way to resolve, so close them; the engine re-raises live ones.+ connection.execute(text("UPDATE incident_alerts SET rule_key = :rule WHERE rule_key IS NULL AND description = :description"), {"rule": SERVER_OFFLINE_RULE, "description": OFFLINE_ALERT_DESCRIPTION})+ connection.execute(text("UPDATE incident_alerts SET status = 'resolved', resolved_at = NOW() WHERE rule_key IS NULL AND status = 'active'"))if "servers" in inspect(engine).get_table_names():columns = {column["name"] for column in inspect(engine).get_columns("servers")}if "collector_token_hash" not in columns:@@ -316,6 +327,7 @@ MAX_METRICS_RETENTION_DAYS = 3650AGENT_CHAT_ENABLED_KEY = "agent_chat_enabled"+ALERT_THRESHOLDS_KEY = "alert_thresholds"def _set_setting(db, key: str, value: str) -> None:@@ -339,8 +351,18 @@ def is_agent_chat_enabled(db) -> bool:return setting is None or setting.value == "true"+def get_alert_thresholds(db) -> dict:+ """Saved thresholds over the defaults, so thresholds added later get their default value."""+ setting = db.get(AppSetting, ALERT_THRESHOLDS_KEY)+ try:+ saved = json.loads(setting.value) if setting else {}+ return validate_thresholds({k: v for k, v in saved.items() if k in DEFAULT_THRESHOLDS}, DEFAULT_THRESHOLDS)+ except (ValueError, AttributeError):+ return dict(DEFAULT_THRESHOLDS)++def _admin_settings(db) -> dict:- return {"metrics_retention_days": get_metrics_retention_days(db), "agent_chat_enabled": is_agent_chat_enabled(db)}+ return {"metrics_retention_days": get_metrics_retention_days(db), "agent_chat_enabled": is_agent_chat_enabled(db), "alert_thresholds": get_alert_thresholds(db)}def purge_old_metrics(db, now: datetime | None = None) -> int:@@ -371,6 +393,16 @@ def update_admin_settings(payload: AdminSettingsRequest, session_id: Annotated[sif payload.agent_chat_enabled is not None:_set_setting(db, AGENT_CHAT_ENABLED_KEY, "true" if payload.agent_chat_enabled else "false")changes.append(f"{'Enabled' if payload.agent_chat_enabled else 'Disabled'} chat with the agent")+ if payload.alert_thresholds is not None:+ current = get_alert_thresholds(db)+ try:+ thresholds = validate_thresholds(payload.alert_thresholds, current)+ except ValueError as exc:+ raise HTTPException(status_code=400, detail=str(exc))+ changed = {key: value for key, value in thresholds.items() if current[key] != value}+ _set_setting(db, ALERT_THRESHOLDS_KEY, json.dumps(thresholds))+ if changed:+ changes.append("Changed alert thresholds: " + ", ".join(f"{key}={value}" for key, value in changed.items()))db.commit()for change in changes:record_audit_log(db, user_id=user["id"], level="INFO", message=change, source="admin")@@ -384,6 +416,9 @@ def agent_config(session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_Nreturn {"chat_enabled": is_agent_chat_enabled(db)}+OVERVIEW_ALERT_LIMIT = 50++@app.get("/monitoring/overview")def monitoring_overview(session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):# Servers belong to the install: everyone signed in can view them, only admins manage them.@@ -404,9 +439,16 @@ def monitoring_overview(session_id: Annotated[str | None, Cookie(alias=SESSION_C"host": ({key: getattr(metric, key) for key in ("time", "cpu_utilization_pct", "load_1m", "load_5m", "load_15m", "ram_used_gb", "ram_available_gb", "ram_pct", "swap_pct", "disk_usage_pct", "disk_inodes_pct", "net_bytes_sent_mb", "net_bytes_recv_mb", "system_uptime_seconds")} if metric else None),"containers": [{"container_id": c.container_id, "name": c.name, "image": c.image, "status": c.status, "restart_count": c.restart_count, "cpu_pct": c.cpu_pct, "memory_used_mb": c.memory_used_mb, "memory_pct": c.memory_pct, "healthcheck_status": c.healthcheck_status, "time": c.time.isoformat()} for c in latest.values()],})- alerts = db.query(IncidentAlert).filter(IncidentAlert.status == "active").order_by(IncidentAlert.created_at.desc()).limit(20).all()+ active_alerts = db.query(IncidentAlert).filter(IncidentAlert.status == "active")+ # Most severe first, then newest.+ severity_rank = case({severity: rank for rank, severity in enumerate(reversed(SEVERITIES))}, value=IncidentAlert.severity, else_=len(SEVERITIES))+ alerts = active_alerts.order_by(severity_rank, IncidentAlert.created_at.desc()).limit(OVERVIEW_ALERT_LIMIT).all()+ hostnames = {server.id: server.hostname for server in servers}pending = visible_pending_collectors(db) if _is_admin(db, user["id"]) else []- return {"servers": result, "pending_collectors": pending, "alerts": [{"id": a.id, "server_id": a.server_id, "severity": a.severity, "title": a.title, "description": a.description, "created_at": a.created_at.isoformat()} for a in alerts]}+ return {+ "servers": result, "pending_collectors": pending, "alert_count": active_alerts.count(),+ "alerts": [{"id": a.id, "server_id": a.server_id, "hostname": hostnames.get(a.server_id), "severity": a.severity, "title": a.title, "description": a.description, "created_at": a.created_at.isoformat()} for a in alerts],+ }CONTAINER_DETAIL_FIELDS = (@@ -640,9 +682,7 @@ def mark_offline_servers(db, now: datetime | None = None) -> int:last_seen = server.last_seen_at if server.last_seen_at.tzinfo else server.last_seen_at.replace(tzinfo=timezone.utc)if now - last_seen < timedelta(seconds=server.check_interval_seconds * OFFLINE_AFTER_MISSED_CHECKS):continue- already_open = db.query(IncidentAlert).filter(IncidentAlert.server_id == server.id, IncidentAlert.status == "active", IncidentAlert.description == OFFLINE_ALERT_DESCRIPTION).first()- if not already_open:- db.add(IncidentAlert(server_id=server.id, severity="CRITICAL", title=_offline_title(server), description=OFFLINE_ALERT_DESCRIPTION))+ if raise_alert(db, server.id, SERVER_OFFLINE_RULE, CRITICAL, _offline_title(server), OFFLINE_ALERT_DESCRIPTION):raised += 1db.commit()return raised@@ -723,15 +763,20 @@ def ingest_metrics(payload: MetricsIngestRequest, authorization: Annotated[str |for field in ("hostname", "ip_address", "os_info", "cpu_cores", "total_ram_gb"):if field in payload.server and payload.server[field] is not None:setattr(server, field, payload.server[field])- db.add(HostMetric(server_id=server.id, **{k: v for k, v in payload.host.items() if k in HostMetric.__table__.columns.keys() and k not in {"id", "server_id", "time"}}))- for container in payload.containers:- if container.get("container_id") and container.get("status"):- allowed = {k: v for k, v in container.items() if k in ContainerMetric.__table__.columns.keys() and k not in {"id", "server_id", "time"}}- db.add(ContainerMetric(server_id=server.id, **allowed))- if str(container.get("status", "")).lower() in {"restarting", "unhealthy"}:- db.add(IncidentAlert(server_id=server.id, severity="CRITICAL", title=f"Container {container.get('name', container['container_id'])} is {container['status']}", description="Automatski detektirano pri zaprimanju metrika."))- server.last_seen_at = datetime.now(timezone.utc)- db.query(IncidentAlert).filter(IncidentAlert.server_id == server.id, IncidentAlert.status == "active", IncidentAlert.description == OFFLINE_ALERT_DESCRIPTION).update({"status": "resolved"}, synchronize_session=False)+ now = datetime.now(timezone.utc)+ db.add(HostMetric(server_id=server.id, time=now, **{k: v for k, v in payload.host.items() if k in HostMetric.__table__.columns.keys() and k not in {"id", "server_id", "time"}}))+ containers = [c for c in payload.containers if c.get("container_id") and c.get("status")]+ for container in containers:+ allowed = {k: v for k, v in container.items() if k in ContainerMetric.__table__.columns.keys() and k not in {"id", "server_id", "time"}}+ db.add(ContainerMetric(server_id=server.id, time=now, **allowed))+ db.flush() # the rules read this check-in back from the history tables+ thresholds = get_alert_thresholds(db)+ sync_alerts(db, server.id, HOST_RULE_PREFIX, host_rules(db, server, thresholds, active_rule_keys(db, server.id, HOST_RULE_PREFIX)))+ # The collector sends no containers when Docker is unreachable; keep container alerts as they are rather than resolving them all.+ if containers:+ sync_alerts(db, server.id, CONTAINER_RULE_PREFIX, container_rules(db, server.id, containers, thresholds, now))+ server.last_seen_at = now+ resolve_alerts(db, server.id, [SERVER_OFFLINE_RULE])db.commit()return {"status": "accepted", "interval_seconds": server.check_interval_seconds}diff --git a/src/backend/models.py b/src/backend/models.pyindex dd9a342..5ceb4a6 100644--- a/src/backend/models.py+++ b/src/backend/models.py@@ -188,4 +188,7 @@ class IncidentAlert(Base):title: Mapped[str] = mapped_column(String(255), nullable=False)description: Mapped[str] = mapped_column(Text, nullable=False)status: Mapped[str] = mapped_column(String(32), default="active", nullable=False)+ # Identifies the condition behind the alert (e.g. "server_offline"), so a server has at most one active alert per rule.+ rule_key: Mapped[Optional[str]] = mapped_column(String(255), index=True, nullable=True)created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), index=True, nullable=False)+ resolved_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True)diff --git a/src/backend/rules.py b/src/backend/rules.pynew file mode 100644index 0000000..4215f95--- /dev/null+++ b/src/backend/rules.py@@ -0,0 +1,138 @@+"""Alert rules evaluated on every collector check-in, and the thresholds they use.++Rules only decide *what is firing*; ``alerts.sync_alerts`` turns that into+opened, refreshed and resolved alerts.+"""+from __future__ import annotations++from datetime import datetime, timedelta++from sqlalchemy import func++from .alerts import CONTAINER_RULE_PREFIX, CRITICAL, WARNING+from .models import ContainerMetric, HostMetric++HOST_RULE_PREFIX = "host:"++DEFAULT_THRESHOLDS: dict[str, float | int] = {+ "disk_warning_pct": 85, "disk_critical_pct": 95,+ "inodes_warning_pct": 85, "inodes_critical_pct": 95,+ "memory_warning_pct": 90, "memory_critical_pct": 95,+ "swap_warning_pct": 50,+ "load_warning_per_core": 1.5, "load_critical_per_core": 2.0,+ # How many check-ins in a row a host metric must breach before an alert opens.+ "sustained_checks": 3,+ "restart_loop_count": 3, "restart_loop_window_minutes": 30,+}++_LIMITS: dict[str, tuple[float, float]] = {+ **{key: (1, 100) for key in DEFAULT_THRESHOLDS if key.endswith("_pct")},+ "load_warning_per_core": (0.1, 50), "load_critical_per_core": (0.1, 50),+ "sustained_checks": (1, 20),+ "restart_loop_count": (1, 100), "restart_loop_window_minutes": (5, 1440),+}+_INTEGER_KEYS = {"sustained_checks", "restart_loop_count", "restart_loop_window_minutes"}+++def validate_thresholds(values: dict, current: dict) -> dict:+ """Merge ``values`` over ``current`` and validate the result. Raises ValueError with a readable message."""+ unknown = set(values) - DEFAULT_THRESHOLDS.keys()+ if unknown:+ raise ValueError(f"Unknown threshold: {', '.join(sorted(unknown))}.")+ merged = {**current, **values}+ for key, value in merged.items():+ if isinstance(value, bool) or not isinstance(value, (int, float)):+ raise ValueError(f"{key} must be a number.")+ if key in _INTEGER_KEYS and value != int(value):+ raise ValueError(f"{key} must be a whole number.")+ low, high = _LIMITS[key]+ if not low <= value <= high:+ raise ValueError(f"{key} must be between {low:g} and {high:g}.")+ merged[key] = int(value) if key in _INTEGER_KEYS else value+ for metric in ("disk", "inodes", "memory"):+ if merged[f"{metric}_warning_pct"] >= merged[f"{metric}_critical_pct"]:+ raise ValueError(f"The {metric} warning threshold must be below its critical threshold.")+ if merged["load_warning_per_core"] >= merged["load_critical_per_core"]:+ raise ValueError("The load warning threshold must be below its critical threshold.")+ return merged+++def _level(value: float | None, warning: float, critical: float | None) -> int:+ """0 = fine, 1 = warning, 2 = critical."""+ if value is None:+ return 0+ if critical is not None and value >= critical:+ return 2+ return 1 if value >= warning else 0+++def host_rules(db, server, thresholds: dict, active_keys: set[str]) -> dict[str, tuple[str, str, str]]:+ """Evaluate the host rules against the server's most recent check-ins (newest first)."""+ sustained = thresholds["sustained_checks"]+ samples = db.query(HostMetric).filter(HostMetric.server_id == server.id).order_by(HostMetric.time.desc(), HostMetric.id.desc()).limit(sustained).all()+ if not samples:+ return {}+ cores = server.cpu_cores or None++ def load_per_core(sample):+ return sample.load_5m / cores if cores and sample.load_5m is not None else None++ checks = (+ # (rule, value getter, warning, critical, title, description)+ ("disk", lambda s: s.disk_usage_pct, thresholds["disk_warning_pct"], thresholds["disk_critical_pct"],+ "Disk usage on {host} is {value:.0f}%", "The root filesystem is {value:.1f}% full. Free up space before it fills completely."),+ ("inodes", lambda s: s.disk_inodes_pct, thresholds["inodes_warning_pct"], thresholds["inodes_critical_pct"],+ "Inode usage on {host} is {value:.0f}%", "{value:.1f}% of inodes on the root filesystem are used. Many small files (caches, logs, mail queues) are the usual cause."),+ ("memory", lambda s: s.ram_pct, thresholds["memory_warning_pct"], thresholds["memory_critical_pct"],+ "Memory usage on {host} is {value:.0f}%", "{value:.1f}% of RAM is in use. Check the containers and processes using the most memory."),+ ("swap", lambda s: s.swap_pct, thresholds["swap_warning_pct"], None,+ "Swap usage on {host} is {value:.0f}%", "{value:.1f}% of swap has been in use for several check-ins, an early sign of memory pressure."),+ ("load", load_per_core, thresholds["load_warning_per_core"], thresholds["load_critical_per_core"],+ "Load on {host} is {value:.1f}× its cores", "The 5-minute load average is {value:.2f} per CPU core across " + f"{cores or '?'} cores. Work is queuing for the CPU."),+ )+ firing: dict[str, tuple[str, str, str]] = {}+ for rule, value_of, warning, critical, title, description in checks:+ key = f"{HOST_RULE_PREFIX}{rule}"+ latest_value = value_of(samples[0])+ levels = [_level(value_of(sample), warning, critical) for sample in samples]+ # Opening an alert, or escalating it to critical, needs every recent check-in to breach.+ sustained_level = min(levels) if len(samples) >= sustained else 0+ level = max(sustained_level, min(levels[0], 1)) if key in active_keys else sustained_level+ if level:+ firing[key] = (+ CRITICAL if level == 2 else WARNING,+ title.format(host=server.hostname, value=latest_value),+ description.format(value=latest_value),+ )+ return firing+++def container_rules(db, server_id: int, containers: list[dict], thresholds: dict, now: datetime) -> dict[str, tuple[str, str, str]]:+ """Evaluate the container rules for one check-in's container records."""+ window = thresholds["restart_loop_window_minutes"]+ # Lowest restart count seen per container within the window; the rise since then is the number of restarts.+ # Keyed by container ID because a recreated container starts counting from zero again.+ ids = [container["container_id"] for container in containers]+ earliest = dict(+ db.query(ContainerMetric.container_id, func.min(ContainerMetric.restart_count))+ .filter(ContainerMetric.server_id == server_id, ContainerMetric.container_id.in_(ids), ContainerMetric.time >= now - timedelta(minutes=window))+ .group_by(ContainerMetric.container_id)+ .all()+ ) if ids else {}++ firing: dict[str, tuple[str, str, str]] = {}+ for container in containers:+ # Compose recreates containers with new IDs but the same name, so key alerts by name.+ name = container.get("name") or container["container_id"]+ restarts = int(container.get("restart_count") or 0) - earliest.get(container["container_id"], int(container.get("restart_count") or 0))+ if restarts >= thresholds["restart_loop_count"]:+ firing[f"{CONTAINER_RULE_PREFIX}{name}:restart_loop"] = (+ CRITICAL, f"Container {name} is in a restart loop",+ f"{name} restarted {restarts} times in the last {window} minutes (restart count {container.get('restart_count', 0)}). Check its logs for the reason it exits.",+ )+ if str(container.get("healthcheck_status") or "").lower() == "unhealthy":+ firing[f"{CONTAINER_RULE_PREFIX}{name}:unhealthy"] = (+ CRITICAL, f"Container {name} is unhealthy",+ f"The Docker HEALTHCHECK for {name} is failing. Inspect it with: docker inspect --format '{{{{json .State.Health}}}}' {name}",+ )+ return firingdiff --git a/src/frontend/src/App.jsx b/src/frontend/src/App.jsxindex bc42b08..ced5a0d 100644--- a/src/frontend/src/App.jsx+++ b/src/frontend/src/App.jsx@@ -431,6 +431,12 @@ function RegisterPage({ onRegister }) {// Izvan ili na vrhu komponente postavi guard za izbjegavanje paralelnih pozivalet isFetchingOverview = false+const ALERT_STYLES = {+ CRITICAL: { box: 'border-red-400/25 bg-red-500/5', badge: 'bg-red-500/15 text-red-300', title: 'text-red-200' },+ WARNING: { box: 'border-amber-400/20 bg-amber-400/5', badge: 'bg-amber-400/15 text-amber-300', title: 'text-amber-200' },+ INFO: { box: 'border-sky-400/20 bg-sky-400/5', badge: 'bg-sky-400/15 text-sky-300', title: 'text-sky-200' },+}+function DashboardPage({ user, onLogout }) {const [overview, setOverview] = useState({ servers: [], alerts: [] })const [error, setError] = useState('')@@ -536,7 +542,7 @@ const refresh = async () => {['Average memory', latestMetrics.length ? `${average('ram_pct')}%` : '—', 'RAM utilization'],['Average disk', latestMetrics.length ? `${average('disk_usage_pct')}%` : '—', 'Disk utilization'],['Containers running', String(runningContainers), 'Across monitored servers'],- ['Active incidents', String(overview.alerts?.length || 0), 'Needs attention'],+ ['Active incidents', String(overview.alert_count ?? overview.alerts?.length ?? 0), 'Needs attention'],].map(([label, value, detail]) => <div key={label} className="glass-panel rounded-[24px] p-5"><div className="text-sm text-[#9aa0a6]">{label}</div><div className="mt-3 text-3xl font-semibold text-white">{value}</div><div className="mt-2 text-xs text-[#7f8589]">{detail}</div></div>)}</section>@@ -562,7 +568,7 @@ const refresh = async () => {</article>})}</div>}</section>- <section className="glass-panel rounded-[28px] p-5 md:p-6"><div className="mb-4"><div className="text-xs uppercase tracking-[0.2em] text-[#9aa0a6]">Alerts</div><h2 className="mt-1 text-xl font-semibold text-white">Active incidents</h2></div>{overview.alerts?.length ? <div className="space-y-2">{overview.alerts.map((alert) => <div key={alert.id} className="rounded-xl border border-amber-400/20 bg-amber-400/5 p-3"><div className="font-medium text-amber-200">{alert.title}</div><div className="mt-1 text-sm text-[#bcbcbc]">{alert.description}</div></div>)}</div> : <p className="text-sm text-[#9aa0a6]">No active incidents recorded.</p>}</section>+ <section className="glass-panel rounded-[28px] p-5 md:p-6"><div className="mb-4"><div className="text-xs uppercase tracking-[0.2em] text-[#9aa0a6]">Alerts</div><h2 className="mt-1 text-xl font-semibold text-white">Active incidents</h2></div>{overview.alerts?.length ? <div className="space-y-2">{overview.alerts.map((alert) => { const style = ALERT_STYLES[alert.severity] || ALERT_STYLES.WARNING; return <div key={alert.id} className={`rounded-xl border p-3 ${style.box}`}><div className="flex items-center gap-2"><span className={`rounded-full px-2 py-0.5 text-[10px] font-semibold tracking-wider ${style.badge}`}>{alert.severity}</span><div className={`font-medium ${style.title}`}>{alert.title}</div></div><div className="mt-1 text-xs text-[#9aa0a6]">{alert.hostname || `Server ${alert.server_id}`} · since {new Date(alert.created_at).toLocaleString([], { dateStyle: 'medium', timeStyle: 'short' })}</div><div className="mt-1 text-sm text-[#bcbcbc]">{alert.description}</div></div> })}{overview.alert_count > overview.alerts.length ? <p className="text-xs text-[#9aa0a6]">Showing the {overview.alerts.length} most severe of {overview.alert_count} incidents.</p> : null}</div> : <p className="text-sm text-[#9aa0a6]">No active incidents recorded.</p>}</section></div></AppShell>)@@ -1316,14 +1322,97 @@ function AdminAgentTab() {)}-function AdminAlertsTab() {- const thresholds = [- ['Disk usage above', 90, '%'],- ['Memory usage above', 90, '%'],- ['CPU usage above (for 10 min)', 95, '%'],- ['Container restarts within 30 min', 3, 'restarts'],- ]+// [label, hint, warning key, critical key (null when the rule only warns), unit, step]+const thresholdRows = [+ ['Disk usage', 'Root filesystem', 'disk_warning_pct', 'disk_critical_pct', '%', 1],+ ['Inode usage', 'Root filesystem', 'inodes_warning_pct', 'inodes_critical_pct', '%', 1],+ ['Memory usage', null, 'memory_warning_pct', 'memory_critical_pct', '%', 1],+ ['Swap usage', 'Warning only', 'swap_warning_pct', null, '%', 1],+ ['Load per CPU core', '5-minute average', 'load_warning_per_core', 'load_critical_per_core', '×', 0.1],+]++const thresholdInputClass = 'w-20 rounded-xl border border-[#1f2326] bg-[#0d1013] px-2 py-1 text-right text-white outline-none focus:border-white/30 disabled:opacity-50'++function ThresholdInput({ label, value, step, onChange, disabled }) {+ return <input type="number" aria-label={label} value={value ?? ''} step={step} min={0} disabled={disabled} onChange={(event) => onChange(event.target.value === '' ? '' : Number(event.target.value))} className={thresholdInputClass} />+}++function IncidentThresholdsSection() {+ const [saved, setSaved] = useState(null)+ const [values, setValues] = useState(null)+ const [saving, setSaving] = useState(false)+ const [message, setMessage] = useState(null)++ useEffect(() => {+ apiRequest('/admin/settings', { method: 'GET' })+ .then((data) => {+ setSaved(data.alert_thresholds)+ setValues(data.alert_thresholds)+ })+ .catch((error) => setMessage({ ok: false, text: error.message || 'Could not load the thresholds.' }))+ }, [])++ const set = (key) => (value) => setValues((current) => ({ ...current, [key]: value }))++ const save = async () => {+ setSaving(true)+ setMessage(null)+ try {+ const data = await apiRequest('/admin/settings', { method: 'PUT', body: JSON.stringify({ alert_thresholds: values }) })+ setSaved(data.alert_thresholds)+ setValues(data.alert_thresholds)+ setMessage({ ok: true, text: 'Saved. New thresholds apply from each server\'s next check-in.' })+ } catch (error) {+ setMessage({ ok: false, text: error.message || 'Could not save the thresholds.' })+ } finally {+ setSaving(false)+ }+ }++ const loading = values === null+ const unchanged = loading || JSON.stringify(values) === JSON.stringify(saved)+ const incomplete = !loading && Object.values(values).some((value) => value === '')++ return (+ <AdminSection+ title="Incident thresholds"+ description="When monitoring opens an incident. Applies to every server."+ action={+ <button type="button" onClick={save} disabled={unchanged || incomplete || saving} className="rounded-2xl bg-linear-to-r from-white to-[#cfcfcf] px-4 py-2.5 text-sm font-medium text-[#0b0b0b] disabled:cursor-not-allowed disabled:opacity-40">+ {saving ? 'Saving…' : 'Save changes'}+ </button>+ }+ >+ <div className="space-y-2">+ <div className="flex justify-end gap-2 px-4 text-[10px] uppercase tracking-wider text-[#7f8589]"><span className="w-20 text-right text-amber-300/80">Warning</span><span className="w-20 text-right text-red-300/80">Critical</span><span className="w-4" /></div>+ {thresholdRows.map(([label, hint, warningKey, criticalKey, unit, step]) => (+ <div key={label} className="flex items-center justify-between gap-4 rounded-2xl border border-[#1f2326] bg-[#0d1013]/60 px-4 py-2.5 text-sm">+ <span><span className="text-[#bcbcbc]">{label}</span>{hint && <span className="block text-xs text-[#7f8589]">{hint}</span>}</span>+ <span className="flex items-center gap-2">+ <ThresholdInput label={`${label} warning`} value={values?.[warningKey]} step={step} onChange={set(warningKey)} disabled={loading} />+ {criticalKey ? <ThresholdInput label={`${label} critical`} value={values?.[criticalKey]} step={step} onChange={set(criticalKey)} disabled={loading} /> : <span className="w-20 text-right text-[#7f8589]">—</span>}+ <span className="w-4 text-xs text-[#9aa0a6]">{unit}</span>+ </span>+ </div>+ ))}+ <label className="flex items-center justify-between gap-4 rounded-2xl border border-[#1f2326] bg-[#0d1013]/60 px-4 py-2.5 text-sm">+ <span><span className="text-[#bcbcbc]">Check-ins in a row</span><span className="block text-xs text-[#7f8589]">How long a host metric must stay over before an incident opens</span></span>+ <ThresholdInput label="Check-ins in a row" value={values?.sustained_checks} step={1} onChange={set('sustained_checks')} disabled={loading} />+ </label>+ <div className="flex items-center justify-between gap-4 rounded-2xl border border-[#1f2326] bg-[#0d1013]/60 px-4 py-2.5 text-sm">+ <span><span className="text-[#bcbcbc]">Container restart loop</span><span className="block text-xs text-[#7f8589]">Critical incident</span></span>+ <span className="flex items-center gap-2 text-xs text-[#9aa0a6]">+ <ThresholdInput label="Restarts" value={values?.restart_loop_count} step={1} onChange={set('restart_loop_count')} disabled={loading} />restarts in+ <ThresholdInput label="Restart window in minutes" value={values?.restart_loop_window_minutes} step={1} onChange={set('restart_loop_window_minutes')} disabled={loading} />min+ </span>+ </div>+ </div>+ {message && <p className={`mt-3 text-xs ${message.ok ? 'text-emerald-300' : 'text-amber-200'}`}>{message.text}</p>}+ </AdminSection>+ )+}+function AdminAlertsTab() {return (<div className="grid gap-5 xl:grid-cols-2"><AdminSection title="Webhook" description="Receives a JSON POST for every critical incident. Replaces ADMIN_ALERT_WEBHOOK_URL in .env." action={<SaveButton />}>@@ -1333,19 +1422,7 @@ function AdminAlertsTab() {</div></AdminSection>- <AdminSection title="Incident thresholds" description="When monitoring opens an incident." action={<SaveButton />}>- <div className="space-y-2">- {thresholds.map(([label, value, unit]) => (- <label key={label} className="flex items-center justify-between gap-4 rounded-2xl border border-[#1f2326] bg-[#0d1013]/60 px-4 py-2.5 text-sm">- <span className="text-[#bcbcbc]">{label}</span>- <span className="flex items-center gap-2">- <input type="number" defaultValue={value} min={0} className="w-20 rounded-xl border border-[#1f2326] bg-[#0d1013] px-2 py-1 text-right text-white outline-none focus:border-white/30" />- <span className="w-16 text-xs text-[#9aa0a6]">{unit}</span>- </span>- </label>- ))}- </div>- </AdminSection>+ <IncidentThresholdsSection /></div>)}diff --git a/tests/test_admin.py b/tests/test_admin.pyindex 4e3055b..a25f3af 100644--- a/tests/test_admin.py+++ b/tests/test_admin.py@@ -9,7 +9,7 @@ from fastapi import HTTPExceptionfrom sqlalchemy import create_enginefrom sqlalchemy.orm import sessionmaker-from src.backend import main+from src.backend import main, rulesfrom src.backend.auth import register_userfrom src.backend.models import AppSetting, Base, ContainerMetric, HostMetric, PendingCollector, Server, User@@ -164,7 +164,7 @@ def test_admin_can_turn_agent_chat_off_and_on(fleet, monkeypatch):result = main.update_admin_settings(main.AdminSettingsRequest(agent_chat_enabled=False), session_id="s", db=fleet)- assert result == {"metrics_retention_days": main.DEFAULT_METRICS_RETENTION_DAYS, "agent_chat_enabled": False}+ assert result == {"metrics_retention_days": main.DEFAULT_METRICS_RETENTION_DAYS, "agent_chat_enabled": False, "alert_thresholds": rules.DEFAULT_THRESHOLDS}_login_as(monkeypatch, 2)assert main.agent_config(session_id="s", db=fleet) == {"chat_enabled": False}diff --git a/tests/test_alerts.py b/tests/test_alerts.pynew file mode 100644index 0000000..d002870--- /dev/null+++ b/tests/test_alerts.py@@ -0,0 +1,125 @@+import os++os.environ.setdefault("DATABASE_URL", "sqlite://")+os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0")++import pytest+from sqlalchemy import create_engine+from sqlalchemy.orm import sessionmaker++from src.backend import alerts, main+from src.backend.models import Base, IncidentAlert, Server, User++TOKEN = "collector-token"+++def db():+ engine = create_engine("sqlite://")+ Base.metadata.create_all(engine)+ session = sessionmaker(bind=engine)()+ session.add(User(id=1, name="Owner", email="[email protected]", password_hash="x", role="admin"))+ session.add(Server(id=5, owner_id=1, hostname="nas", check_interval_seconds=60, collector_token_hash=main._hash_secret(TOKEN)))+ session.commit()+ return session+++def _ingest(db, containers):+ main.ingest_metrics(main.MetricsIngestRequest(server_id=5, containers=containers), authorization=f"Bearer {TOKEN}", session_id=None, db=db)+++def _container(name, status="running", health=None, restarts=0):+ return {"container_id": f"id-{name}", "name": name, "status": status, "healthcheck_status": health, "restart_count": restarts}+++def _active(db):+ return db.query(IncidentAlert).filter_by(status="active").all()+++def test_raise_alert_dedupes_and_refreshes(db):+ assert alerts.raise_alert(db, 5, "disk", alerts.WARNING, "Disk 86%", "first") is True+ assert alerts.raise_alert(db, 5, "disk", alerts.CRITICAL, "Disk 96%", "second") is False++ [alert] = _active(db)+ assert (alert.severity, alert.title, alert.description) == (alerts.CRITICAL, "Disk 96%", "second")+++def test_raise_alert_rejects_unknown_severity(db):+ with pytest.raises(ValueError):+ alerts.raise_alert(db, 5, "disk", "SEVERE", "t", "d")+++def test_resolved_rule_can_fire_again_as_new_alert(db):+ alerts.raise_alert(db, 5, "disk", alerts.WARNING, "t", "d")+ assert alerts.resolve_alerts(db, 5, ["disk"]) == 1+ assert alerts.raise_alert(db, 5, "disk", alerts.WARNING, "t", "d") is True++ assert db.query(IncidentAlert).count() == 2+ assert db.query(IncidentAlert).filter_by(status="resolved").one().resolved_at is not None+++def test_unhealthy_container_alerts_once_and_resolves(db):+ for _ in range(3):+ _ingest(db, [_container("web", health="unhealthy")])++ [alert] = _active(db)+ assert alert.rule_key == "container:web:unhealthy"+ assert alert.severity == alerts.CRITICAL++ _ingest(db, [_container("web", health="healthy")])+ assert _active(db) == []+++def test_restart_loop_alerts_once_when_count_climbs(db):+ for restarts in range(0, 5):+ _ingest(db, [_container("db", status="restarting", restarts=restarts)])++ [alert] = _active(db)+ assert alert.rule_key == "container:db:restart_loop"+ assert "restarted 4 times" in alert.description+++def test_restart_loop_ignores_old_restarts_and_status_alone(db):+ """A high restart count from long ago, or one restarting sample, is not a loop."""+ for _ in range(3):+ _ingest(db, [_container("db", status="restarting", restarts=40)])++ assert _active(db) == []+++def test_removed_container_alert_resolves_when_others_report(db):+ _ingest(db, [_container("web", health="unhealthy"), _container("db")])+ _ingest(db, [_container("db")])++ assert _active(db) == []+++def test_empty_container_list_keeps_container_alerts(db):+ """No containers means Docker was unreachable, not that every problem cleared."""+ _ingest(db, [_container("web", health="unhealthy")])+ _ingest(db, [])++ assert len(_active(db)) == 1+++def test_container_sync_leaves_other_rules_alone(db):+ alerts.raise_alert(db, 5, alerts.SERVER_OFFLINE_RULE, alerts.CRITICAL, "offline", "d")+ alerts.sync_alerts(db, 5, alerts.CONTAINER_RULE_PREFIX, {})++ assert len(_active(db)) == 1+++def test_overview_lists_critical_first_with_hostname_and_true_count(db, monkeypatch):+ monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1})+ monkeypatch.setattr(main, "OVERVIEW_ALERT_LIMIT", 2)+ alerts.raise_alert(db, 5, "a", alerts.CRITICAL, "old critical", "d")+ db.commit()+ alerts.raise_alert(db, 5, "b", alerts.INFO, "new info", "d")+ alerts.raise_alert(db, 5, "c", alerts.WARNING, "new warning", "d")+ db.commit()++ overview = main.monitoring_overview(session_id="s", db=db)++ assert overview["alert_count"] == 3+ assert [a["title"] for a in overview["alerts"]] == ["old critical", "new warning"]+ assert overview["alerts"][0]["hostname"] == "nas"diff --git a/tests/test_rules.py b/tests/test_rules.pynew file mode 100644index 0000000..d5ea971--- /dev/null+++ b/tests/test_rules.py@@ -0,0 +1,122 @@+import os++os.environ.setdefault("DATABASE_URL", "sqlite://")+os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0")++import pytest+from fastapi import HTTPException+from sqlalchemy import create_engine+from sqlalchemy.orm import sessionmaker++from src.backend import main, rules+from src.backend.alerts import CRITICAL, WARNING+from src.backend.models import AuditLog, Base, IncidentAlert, Server, User++TOKEN = "collector-token"+++def db(monkeypatch):+ engine = create_engine("sqlite://")+ Base.metadata.create_all(engine)+ session = sessionmaker(bind=engine)()+ session.add(User(id=1, name="Owner", email="[email protected]", password_hash="x", role="admin"))+ session.add(Server(id=5, owner_id=1, hostname="nas", cpu_cores=4, check_interval_seconds=60, collector_token_hash=main._hash_secret(TOKEN)))+ session.commit()+ monkeypatch.setattr(main, "_require_admin", lambda session_id, db: {"id": 1})+ monkeypatch.setattr(main, "record_audit_log", lambda db, **kw: db.add(AuditLog(user_id=kw["user_id"], level=kw["level"], message=kw["message"], source=kw["source"])))+ return session+++def _ingest(db, **host):+ main.ingest_metrics(main.MetricsIngestRequest(server_id=5, host=host), authorization=f"Bearer {TOKEN}", session_id=None, db=db)+++def _active(db):+ return {a.rule_key: a for a in db.query(IncidentAlert).filter_by(status="active")}+++def test_host_alert_needs_sustained_breach(db):+ _ingest(db, disk_usage_pct=90)+ _ingest(db, disk_usage_pct=90)+ assert _active(db) == {}++ _ingest(db, disk_usage_pct=90)+ alert = _active(db)["host:disk"]+ assert alert.severity == WARNING+ assert alert.title == "Disk usage on nas is 90%"+++def test_single_spike_does_not_alert(db):+ for value in (40, 99, 40, 99, 40):+ _ingest(db, ram_pct=value)++ assert _active(db) == {}+++def test_open_alert_stays_open_until_value_drops_and_escalates_only_when_sustained(db):+ for _ in range(3):+ _ingest(db, disk_usage_pct=90)+ _ingest(db, disk_usage_pct=97)+ assert _active(db)["host:disk"].severity == WARNING++ _ingest(db, disk_usage_pct=97)+ _ingest(db, disk_usage_pct=97)+ assert _active(db)["host:disk"].severity == CRITICAL++ _ingest(db, disk_usage_pct=88)+ assert _active(db)["host:disk"].severity == WARNING++ _ingest(db, disk_usage_pct=50)+ assert _active(db) == {}+++def test_load_is_compared_per_core(db):+ for _ in range(3):+ _ingest(db, load_5m=5.0) # 1.25 per core on 4 cores+ assert _active(db) == {}++ for _ in range(3):+ _ingest(db, load_5m=9.0) # 2.25 per core+ assert _active(db)["host:load"].severity == CRITICAL+++def test_swap_only_warns(db):+ for _ in range(3):+ _ingest(db, swap_pct=100)++ assert _active(db)["host:swap"].severity == WARNING+++def test_missing_metric_does_not_alert(db):+ for _ in range(3):+ _ingest(db)++ assert _active(db) == {}+++def test_saved_thresholds_apply(db):+ main.update_admin_settings(main.AdminSettingsRequest(alert_thresholds={"sustained_checks": 1, "memory_warning_pct": 50}), session_id="s", db=db)++ _ingest(db, ram_pct=60)++ assert _active(db)["host:memory"].severity == WARNING+++def test_threshold_update_is_validated_and_audited(db):+ for bad in ({"disk_warning_pct": 99}, {"disk_warning_pct": 0}, {"sustained_checks": 1.5}, {"nope": 1}, {"memory_warning_pct": "90"}):+ with pytest.raises(HTTPException) as exc:+ main.update_admin_settings(main.AdminSettingsRequest(alert_thresholds=bad), session_id="s", db=db)+ assert exc.value.status_code == 400++ result = main.update_admin_settings(main.AdminSettingsRequest(alert_thresholds={"disk_critical_pct": 98}), session_id="s", db=db)++ assert result["alert_thresholds"] == {**rules.DEFAULT_THRESHOLDS, "disk_critical_pct": 98}+ assert db.query(AuditLog).one().message == "Changed alert thresholds: disk_critical_pct=98"+++def test_corrupt_saved_thresholds_fall_back_to_defaults(db):+ main._set_setting(db, main.ALERT_THRESHOLDS_KEY, "not json")+ db.commit()++ assert main.get_alert_thresholds(db) == rules.DEFAULT_THRESHOLDS