Skip to content

Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.

21.5 KB
Raw Blame
"""Incidents as the Agent page shows them: an inbox, and for each incident a diagnosis built from the data.

Nothing here is guessed or written by a language model. The summary is the rule's own description, the evidence is
the metric the rule looks at, the signals are checks against stored samples, and the fixes come from a fixed
playbook per kind of rule, with the container or mount name filled in.
"""
from __future__ import annotations

import math
import re
from datetime import datetime, timedelta, timezone

from .alerts import CRITICAL, INFO, SERVER_OFFLINE_RULE, WARNING
from .dashboard import _event_source
from .models import ContainerMetric, HostMetric, IncidentAlert, MountMetric, Server, SystemEvent, User

MAX_LISTED = 200
EVIDENCE_POINTS = 120
EVIDENCE_LEAD = timedelta(minutes=60)
MAX_LINES = 14
MAX_TIMELINE = 12
_SAFE_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}")


def _utc(moment: datetime | None) -> datetime | None:
    return None if moment is None else moment if moment.tzinfo else moment.replace(tzinfo=timezone.utc)


def rule_kind(rule_key: str | None) -> str:
    """The kind of rule behind an alert, without the container, mount or port it is about:
    ``container:gitlab:restart_loop`` -> ``container:restart_loop``, ``host:disk:/mnt/x`` -> ``host:disk``."""
    if not rule_key:
        return "other"
    parts = rule_key.split(":")
    if parts[0] == "container" and len(parts) >= 3:
        return f"container:{parts[-1]}"
    if parts[0] == "trend":
        return "trend:memory"
    if parts[0] in ("host", "system", "network") and len(parts) >= 2:
        return f"{parts[0]}:{parts[1]}"
    if parts[0] == "check" and len(parts) >= 3:
        return f"check:{parts[-1]}"
    return rule_key


def subject(rule_key: str | None) -> str | None:
    """The container, mount or port a rule is about, when it names one."""
    parts = (rule_key or "").split(":")
    if parts[0] in ("container", "trend") and len(parts) >= 3:
        return ":".join(parts[1:-1])
    if parts[0] in ("host", "network") and len(parts) >= 3:
        return ":".join(parts[2:])
    return None


def reference(alert_id: int) -> str:
    return f"INC-{alert_id:04d}"


def is_muted(alert: IncidentAlert, now: datetime) -> bool:
    return alert.muted_until is not None and _utc(alert.muted_until) > now


def state_of(alert: IncidentAlert) -> str:
    """What the inbox says under the title: diagnosed (there are steps to take) or watching (nothing to do yet)."""
    return "watching" if alert.severity == INFO or rule_kind(alert.rule_key) in ("trend:memory", "host:swap") else "diagnosed"


def _command_name(name: str | None) -> str:
    """A container name that is safe to show inside a command to copy; anything unusual becomes a placeholder."""
    return name if name and _SAFE_NAME.fullmatch(name) else "<container>"


def _gb(megabytes: float) -> str:
    return f"{megabytes / 1024:.1f} GB" if megabytes >= 1024 else f"{megabytes:.0f} MB"


def _step(title: str, text: str, command: str | None = None, label: str | None = None, recommended: bool = False) -> dict:
    return {"title": title, "text": text, "command": command, "label": label, "recommended": recommended}


def _downsample(rows: list[tuple[datetime, float | None]], limit: int = EVIDENCE_POINTS) -> list[dict]:
    if len(rows) > limit:
        size = math.ceil(len(rows) / limit)
        grouped = []
        for start in range(0, len(rows), size):
            chunk = [value for _, value in rows[start:start + size] if value is not None]
            # The highest value in each group: an incident is about peaks, and an average would hide them.
            grouped.append((rows[start][0], max(chunk) if chunk else None))
        rows = grouped
    return [{"time": _utc(moment).isoformat(), "value": None if value is None else round(value, 2)} for moment, value in rows]


def _container_samples(db, server_id: int, name: str, since: datetime, until: datetime) -> list[ContainerMetric]:
    return db.query(ContainerMetric).filter(ContainerMetric.server_id == server_id, ContainerMetric.name == name, ContainerMetric.time >= since, ContainerMetric.time <= until).order_by(ContainerMetric.time).limit(5000).all()


def _container_diagnosis(db, alert: IncidentAlert, server: Server, kind: str, since: datetime, until: datetime) -> dict:
    name = subject(alert.rule_key) or ""
    samples = _container_samples(db, server.id, name, since, until)
    shown = _command_name(name)
    latest = samples[-1] if samples else None
    host_mb = (server.total_ram_gb or 0) * 1024
    limit = latest.memory_limit_mb if latest and latest.memory_limit_mb and (not host_mb or latest.memory_limit_mb < host_mb * 0.98) else None
    restarts = [(sample.time, sample.restart_count) for sample in samples]
    marks = [{"time": _utc(moment).isoformat(), "label": "restart"} for (_, before), (moment, after) in zip(restarts, restarts[1:]) if after > before]
    peak = max((sample.memory_used_mb for sample in samples if sample.memory_used_mb is not None), default=None)
    oom = any(sample.oom_killed for sample in samples)
    codes = sorted({sample.exit_code for sample in samples if sample.exit_code is not None})
    signals = []
    if restarts:
        rose = restarts[-1][1] - restarts[0][1]
        signals.append({"text": f"The restart count rose by {rose} in this window.", "source": f"{len(marks)} of {len(samples)} check-ins", "ok": rose > 0})
    if limit and peak is not None:
        signals.append({"text": f"Memory reached {_gb(peak)} of its {_gb(limit)} limit.", "source": "container memory", "ok": peak >= limit * 0.9})
    if oom:
        signals.append({"text": "Docker reports the last exit as an out-of-memory kill.", "source": "container state", "ok": True})
    elif codes:
        signals.append({"text": f"Exit code{'s' if len(codes) > 1 else ''} seen: {', '.join(map(str, codes))}{' (137 means it was killed)' if 137 in codes else ''}.", "source": "container state", "ok": True})
    if latest and latest.healthcheck_status:
        signals.append({"text": f"Its healthcheck reports {latest.healthcheck_status}.", "source": "docker healthcheck", "ok": latest.healthcheck_status == "unhealthy"})
    memory_bound = oom or (limit is not None and peak is not None and peak >= limit * 0.9)
    steps = []
    if memory_bound and limit:
        bigger = max(1, math.ceil(limit * 1.5 / 1024))
        steps.append(_step(f"Raise the memory limit to {bigger} GB", f"{name} runs into its {_gb(limit)} limit. In its compose file, raise the limit and recreate the container:", f"mem_limit: {bigger}g", "compose", True))
    elif memory_bound:
        steps.append(_step("Find what is using the memory", f"{name} has no memory limit of its own, so the host ran out. See which containers hold the most:", "docker stats --no-stream --format 'table {{.Name}}\\t{{.MemUsage}}' | sort -k2 -h", "shell", True))
    steps.append(_step("Read why it exits" if kind != "container:unhealthy" else "Read what the healthcheck says", "The container's own log usually names the reason in its last lines before each exit." if kind != "container:unhealthy" else "The last healthcheck output is kept by Docker.",
                       f"docker logs --since 30m --tail 200 {shown}" if kind != "container:unhealthy" else f"docker inspect -f '{{{{json .State.Health}}}}' {shown}", "shell", not steps))
    steps.append(_step("Confirm it's fixed", "Infradeck resolves this incident on its own once the container stays up with no new restarts.", f"docker inspect -f '{{{{.RestartCount}}}} {{{{.State.Status}}}}' {shown}", "shell"))
    series = _downsample([(sample.time, sample.memory_used_mb) for sample in samples])
    return {
        "evidence": {"title": "container memory", "unit": "MB", "series": series, "limit": limit, "limit_label": "Container limit", "marks": marks[-40:], "mark_label": "Restart"} if series else None,
        "signals": signals, "steps": steps, "container": name,
        "note": (f"limit is {_gb(limit)}, peak {_gb(peak)}" if limit and peak is not None and memory_bound else f"{len(marks)} restarts in this window" if marks else None),
    }


def _host_series(db, server: Server, column, since: datetime, until: datetime) -> list[tuple[datetime, float | None]]:
    return [(moment, value) for moment, value in db.query(HostMetric.time, column).filter(HostMetric.server_id == server.id, HostMetric.time >= since, HostMetric.time <= until).order_by(HostMetric.time).limit(20000)]


def _top_containers(db, server: Server, column, count: int = 3) -> list[tuple[str, float]]:
    latest = db.query(HostMetric.time).filter(HostMetric.server_id == server.id).order_by(HostMetric.time.desc()).limit(1).scalar()
    if latest is None:
        return []
    rows = db.query(ContainerMetric.name, column).filter(ContainerMetric.server_id == server.id, ContainerMetric.time == latest, column.isnot(None)).order_by(column.desc()).limit(count).all()
    return [(name, value) for name, value in rows if value]


def _host_diagnosis(db, alert: IncidentAlert, server: Server, kind: str, since: datetime, until: datetime) -> dict:
    confirm = _step("Confirm it's fixed", "Infradeck resolves this incident on its own once the value stays under the threshold for a few check-ins.")
    if kind in ("host:memory", "host:swap"):
        swap = kind == "host:swap"
        rows = _host_series(db, server, HostMetric.swap_pct if swap else HostMetric.ram_pct, since, until)
        top = _top_containers(db, server, ContainerMetric.swap_mb if swap else ContainerMetric.memory_used_mb)
        values = [value for _, value in rows if value is not None]
        signals = [{"text": f"{'Swap' if swap else 'Memory'} use peaked at {max(values):.0f}% in this window.", "source": "host metrics", "ok": True}] if values else []
        if top:
            signals.append({"text": f"{top[0][0]} holds the most {'swap' if swap else 'memory'}: {_gb(top[0][1])}." + (f" Then {', '.join(f'{name} ({_gb(value)})' for name, value in top[1:])}." if len(top) > 1 else ""), "source": "container metrics", "ok": True})
        steps = [_step("See what holds the memory", "The biggest containers right now are listed above. To see it on the machine:", "docker stats --no-stream --format 'table {{.Name}}\\t{{.MemUsage}}\\t{{.MemPerc}}'", "shell", True)]
        if top:
            steps.append(_step(f"Give {top[0][0]} a memory limit", "A limit stops one container from pushing the rest of the machine into swap. In its compose file:", f"mem_limit: {max(1, math.ceil(top[0][1] * 1.3 / 1024))}g", "compose"))
        return {"evidence": {"title": "swap in use" if swap else "memory in use", "unit": "%", "series": _downsample(rows), "limit": None, "marks": []}, "signals": signals, "steps": [*steps, confirm],
                "note": f"top {'swap' if swap else 'memory'} user is {top[0][0]} ({_gb(top[0][1])})" if top else None}
    if kind == "host:load":
        rows = _host_series(db, server, HostMetric.load_1m, since, until)
        top = _top_containers(db, server, ContainerMetric.cpu_pct)
        signals = [{"text": f"The machine has {server.cpu_cores} cores; load above that means work is waiting.", "source": "host metrics", "ok": True}] if server.cpu_cores else []
        if top:
            signals.append({"text": f"{top[0][0]} uses the most CPU right now: {top[0][1]:.0f}%.", "source": "container metrics", "ok": True})
        return {"evidence": {"title": "load (1 min)", "unit": "", "series": _downsample(rows), "limit": server.cpu_cores, "limit_label": f"{server.cpu_cores} cores", "marks": []}, "signals": signals,
                "steps": [_step("See what is using the CPU", "Load counts processes that are running or waiting for the CPU or the disk.", "docker stats --no-stream --format 'table {{.Name}}\\t{{.CPUPerc}}' | sort -k2 -rh | head", "shell", True), confirm],
                "note": f"busiest container is {top[0][0]} ({top[0][1]:.0f}% CPU)" if top else None}
    if kind in ("host:disk", "host:inodes"):
        mount = subject(alert.rule_key) or "/"
        column = MountMetric.used_pct if kind == "host:disk" else MountMetric.inodes_pct
        rows = [(moment, value) for moment, value in db.query(MountMetric.time, column).filter(MountMetric.server_id == server.id, MountMetric.mount == mount, MountMetric.time >= since, MountMetric.time <= until).order_by(MountMetric.time).limit(20000)]
        values = [value for _, value in rows if value is not None]
        signals = [{"text": f"{mount} went from {values[0]:.1f}% to {values[-1]:.1f}% in this window.", "source": "mount metrics", "ok": values[-1] >= values[0]}] if len(values) > 1 else []
        docker = mount in ("/", "/var", "/var/lib", "/var/lib/docker")
        steps = [_step("See what Docker can give back", "Unused images and build cache are the usual cause on a Docker host.", "docker system df", "shell", True),
                 _step("Remove what nothing uses", "This deletes every image no container uses and the unused build cache. Images you keep for later will be pulled again.", "docker image prune -a && docker builder prune", "shell")] if docker else []
        steps.append(_step("Find the biggest folders", "Shows the largest directories on that filesystem.", f"sudo du -xh --max-depth=2 {mount if mount.startswith('/') else '/'} | sort -rh | head -20", "shell", not docker))
        return {"evidence": {"title": f"{mount} used" if kind == "host:disk" else f"{mount} inodes used", "unit": "%", "series": _downsample(rows), "limit": None, "marks": []}, "signals": signals, "steps": [*steps, confirm], "note": None}
    return {}


PLAYBOOKS: dict[str, list[dict]] = {
    SERVER_OFFLINE_RULE: [
        _step("Check that the machine is up", "If it answers a ping, the machine is fine and the collector or the network path is the problem.", None, None, True),
        _step("Look at the collector on that machine", "It logs why it could not send.", "docker compose -p infradeck-collector logs --tail 50 collector", "shell"),
        _step("Confirm it's back", "This incident resolves on the collector's next check-in."),
    ],
    "network:port": [
        _step("Decide whether it should be reachable", "If the service is only used on this machine or behind a reverse proxy, bind it to localhost in its compose file:", 'ports: ["127.0.0.1:8080:8080"]', "compose", True),
        _step("Or allow it on purpose", "If other machines need it, add a firewall rule for it so the choice is written down. Note that ports published by Docker bypass ufw.", "sudo ufw allow from 10.0.0.0/24 to any port <port>", "shell"),
    ],
    "network:firewall": [
        _step("Turn the firewall back on", "If it was switched off by accident:", "sudo ufw enable && sudo ufw status verbose", "shell", True),
        _step("Check what changed", "The Network tab lists which ports are reachable from other machines now."),
    ],
    "network:internet": [_step("Check the uplink", "The collector could not open a connection to its targets.", "ping -c 3 1.1.1.1", "shell", True)],
    "network:dns": [_step("Check name resolution", "Connections by address work but names don't resolve.", "resolvectl status | head -20", "shell", True)],
    "system:reboot_required": [_step("Reboot when it suits you", "Installed updates, usually a new kernel, only take effect after a restart.", "sudo systemctl reboot", "shell", True)],
    "system:ntp": [_step("Check the time service", "One of chrony, systemd-timesyncd or ntpd should be running.", "timedatectl status", "shell", True)],
    "system:oom": [_step("See what the kernel killed", "The kernel log names the process and how much memory it held.", "sudo dmesg -T | grep -i 'out of memory' | tail", "shell", True)],
    "system:kernel": [_step("Read the kernel log", "The fault flag clears after a reboot; the log says what raised it.", "sudo dmesg -T --level=err,crit,alert,emerg | tail -30", "shell", True)],
    "system:docker_unreachable": [_step("Check the Docker daemon and the proxy", "Container data stops while either is down.", "sudo systemctl status docker --no-pager && docker ps --filter name=docker-proxy", "shell", True)],
    "system:docker_slow": [_step("See what keeps Docker busy", "A daemon that takes seconds to list containers is usually starved of CPU or disk.", "docker stats --no-stream | head -15", "shell", True)],
    "system:collect_slow": [_step("Give the collector more time", "Raise this server's check interval in Admin, or reduce the number of containers on the machine.", None, None, True)],
    "system:reboot": [_step("Find out why it restarted", "The previous boot's last log lines usually say.", "journalctl -b -1 -n 50 --no-pager", "shell", True)],
    "system:credentials": [_step("Make sure it was you", "The enrollment key gave this server new credentials. If you did not re-install its collector, rotate the enrollment key and remove the server.", None, None, True)],
    "container:removed": [_step("Bring it back if it should exist", "The container is no longer on the machine.", "docker compose up -d", "shell", True)],
    "container:image": [_step("Check what changed", "The container now runs a different image build than before.", "docker image history --no-trunc <image> | head", "shell", True)],
    "trend:memory": [_step("Watch it, then restart it", "Memory has climbed steadily. Restarting the container frees it; a limit keeps a leak from taking the host down.", "mem_limit: 2g", "compose", True)],
}


def diagnose(db, alert: IncidentAlert, server: Server, now: datetime | None = None) -> dict:
    """Everything the incident page shows under the title, worked out from stored data."""
    now = _utc(now or datetime.now(timezone.utc))
    created, resolved = _utc(alert.created_at), _utc(alert.resolved_at)
    since, until = created - EVIDENCE_LEAD, resolved or now
    kind = rule_kind(alert.rule_key)
    found: dict = {}
    if kind.startswith("container:") and kind not in ("container:removed", "container:image"):
        found = _container_diagnosis(db, alert, server, kind, since, until)
    elif kind.startswith("host:"):
        found = _host_diagnosis(db, alert, server, kind, since, until)
    steps = found.get("steps") or PLAYBOOKS.get(kind) or []
    events = db.query(SystemEvent).filter(SystemEvent.server_id == server.id, SystemEvent.time >= since - timedelta(hours=1), SystemEvent.time <= until).order_by(SystemEvent.time.desc()).limit(200).all()
    lines = [{"time": _utc(event.time).isoformat(), "source": _event_source(event.kind), "text": event.detail, "strong": event.kind in ("oom_kill", "kernel_error", "kernel_fault", "firewall_weakened")} for event in reversed(events[:MAX_LINES])]
    timeline = [{"time": _utc(event.time).isoformat(), "text": event.detail, "tone": "plain"} for event in events[:MAX_TIMELINE - 2]]
    timeline.append({"time": created.isoformat(), "text": f"Incident opened · {alert.title}", "tone": "signal" if alert.severity == CRITICAL else "warn" if alert.severity == WARNING else "plain"})
    if alert.acknowledged_at:
        timeline.append({"time": _utc(alert.acknowledged_at).isoformat(), "text": "Acknowledged", "tone": "plain"})
    if resolved:
        timeline.append({"time": resolved.isoformat(), "text": f"Resolved · {alert.resolution or 'the condition cleared'}", "tone": "ok"})
    timeline.sort(key=lambda item: item["time"])
    signals = found.get("signals") or []
    return {
        "summary": alert.description, "state": state_of(alert), "kind": kind, "container": found.get("container"), "note": found.get("note"),
        "evidence": found.get("evidence"), "lines": lines, "signals": signals, "agreeing": sum(1 for signal in signals if signal["ok"]),
        "steps": steps, "timeline": timeline[-MAX_TIMELINE:], "window": {"start": since.isoformat(), "end": until.isoformat()},
    }


def summary(alert: IncidentAlert, hostname: str | None, now: datetime, users: dict[int, str] | None = None) -> dict:
    return {
        "id": alert.id, "reference": reference(alert.id), "server_id": alert.server_id, "hostname": hostname, "severity": alert.severity, "title": alert.title, "description": alert.description,
        "rule_key": alert.rule_key, "kind": rule_kind(alert.rule_key), "status": alert.status, "state": state_of(alert), "created_at": _utc(alert.created_at).isoformat(),
        "resolved_at": _utc(alert.resolved_at).isoformat() if alert.resolved_at else None, "resolution": alert.resolution,
        "acknowledged_at": _utc(alert.acknowledged_at).isoformat() if alert.acknowledged_at else None, "acknowledged_by": (users or {}).get(alert.acknowledged_by),
        "muted_until": _utc(alert.muted_until).isoformat() if is_muted(alert, now) else None, "notified": bool(alert.notified_at),
    }


def inbox(db, now: datetime | None = None) -> dict:
    """Open incidents (most severe first), the ones resolved lately, and the muted ones, each as a summary."""
    now = _utc(now or datetime.now(timezone.utc))
    hostnames = dict(db.query(Server.id, Server.hostname).all())
    users = dict(db.query(User.id, User.name).all())
    rank = {CRITICAL: 0, WARNING: 1, INFO: 2}
    active = db.query(IncidentAlert).filter(IncidentAlert.status == "active").order_by(IncidentAlert.created_at.desc()).limit(MAX_LISTED).all()
    active.sort(key=lambda alert: (rank.get(alert.severity, 3), -_utc(alert.created_at).timestamp()))
    recent = db.query(IncidentAlert).filter(IncidentAlert.status == "resolved", IncidentAlert.resolved_at.isnot(None)).order_by(IncidentAlert.resolved_at.desc()).limit(MAX_LISTED).all()
    row = lambda alert: summary(alert, hostnames.get(alert.server_id), now, users)  # noqa: E731
    return {
        "open": [row(alert) for alert in active if not is_muted(alert, now)],
        "muted": [row(alert) for alert in active if is_muted(alert, now)],
        "resolved": [row(alert) for alert in recent],
        "resolved_count": db.query(IncidentAlert).filter(IncidentAlert.status == "resolved").count(),
    }