Skip to content

Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.

19.2 KB
Raw Blame
"""Fleet-wide statistics for the Statistics page: incidents over time and how reliably servers reported.

Only aggregated numbers leave this module, never raw rows. Incidents are counted by the time they opened, in
buckets that suit the range (hours up to a day, days up to a month, weeks up to a year, months beyond), and the
amount of data read is capped.
"""
from __future__ import annotations

from datetime import datetime, timedelta, timezone

from sqlalchemy import Integer, case, cast, extract, func, select

from . import history, rollups
from .alerts import CRITICAL, INFO, SEVERITIES, WARNING
from .models import ContainerMetric, ContainerRollup, HostMetric, IncidentAlert, Server

MAX_INCIDENTS_READ = 20000
# The bucket an incident count is grouped into, by range.
INCIDENT_BUCKET = {"1h": None, "6h": rollups.HOUR, "24h": rollups.HOUR, "7d": rollups.DAY, "30d": rollups.DAY, "90d": rollups.WEEK, "1y": rollups.WEEK, "all": rollups.MONTH}
KIND_LABELS = {
    "host:disk": "Disk space", "host:inodes": "Inodes", "host:memory": "Memory", "host:swap": "Swap", "host:load": "Load",
    "system:reboot": "Reboots", "system:oom": "Out-of-memory kills", "system:kernel": "Kernel faults", "system:ntp": "Clock not synced",
    "system:docker_unreachable": "Docker unreachable", "system:docker_slow": "Docker slow", "system:collect_slow": "Collector too slow",
    "system:reboot_required": "Reboot needed",
}


def incident_kind(rule_key: str | None) -> str:
    """A readable group for an incident's rule, so the page never has to show raw rule keys."""
    if not rule_key:
        return "Other"
    if rule_key == "server_offline":
        return "Server offline"
    parts = rule_key.split(":")
    if len(parts) >= 2 and f"{parts[0]}:{parts[1]}" in KIND_LABELS:
        return KIND_LABELS[f"{parts[0]}:{parts[1]}"]
    return {"trend": "Possible memory leaks", "network": "Network", "container": "Containers"}.get(parts[0], "Other")


def _utc(moment: datetime) -> datetime:
    return rollups._utc(moment)


def window(db, range_name: str, now: datetime) -> datetime:
    """When the range starts: a fixed distance back, or the first incident for all time."""
    length = history.RANGES[range_name]
    if length is not None:
        return now - length
    first = db.execute(select(func.min(IncidentAlert.created_at))).scalar()
    return _utc(first) if first else now


def incident_stats(db, range_name: str, now: datetime | None = None) -> dict:
    now = _utc(now or datetime.now(timezone.utc))
    start = window(db, range_name, now)
    bucket = INCIDENT_BUCKET[range_name]
    rows = db.execute(
        select(IncidentAlert.created_at, IncidentAlert.resolved_at, IncidentAlert.severity, IncidentAlert.rule_key, IncidentAlert.status)
        .where(IncidentAlert.created_at >= start).order_by(IncidentAlert.created_at).limit(MAX_INCIDENTS_READ + 1)
    ).all()
    truncated = len(rows) > MAX_INCIDENTS_READ
    rows = rows[:MAX_INCIDENTS_READ]
    columns: dict[datetime, dict[str, int]] = {}
    kinds: dict[str, int] = {}
    minutes: list[float] = []
    for created, resolved, severity, rule_key, status in rows:
        created = _utc(created)
        key = rollups.floor(bucket, created) if bucket else rollups.floor(rollups.HOUR, created)
        column = columns.setdefault(key, {level: 0 for level in SEVERITIES})
        column[severity if severity in column else INFO] += 1
        kinds[incident_kind(rule_key)] = kinds.get(incident_kind(rule_key), 0) + 1
        if resolved is not None and status == "resolved":
            minutes.append(max((_utc(resolved) - created).total_seconds() / 60, 0))
    open_now = {level: count for level, count in db.execute(select(IncidentAlert.severity, func.count()).where(IncidentAlert.status == "active").group_by(IncidentAlert.severity))}
    return {
        "range": range_name, "start": start.isoformat(), "end": now.isoformat(), "bucket": bucket or "hour",
        "opened": len(rows), "resolved": len(minutes), "truncated": truncated,
        "mean_minutes_to_resolve": (sum(minutes) / len(minutes)) if minutes else None,
        "open_now": {level.lower(): open_now.get(level, 0) for level in (CRITICAL, WARNING, INFO)},
        "by_bucket": [{"time": moment.isoformat(), **{level.lower(): counts[level] for level in (CRITICAL, WARNING, INFO)}} for moment, counts in sorted(columns.items())],
        "by_kind": [{"kind": kind, "count": count} for kind, count in sorted(kinds.items(), key=lambda item: (-item[1], item[0]))],
    }


def availability(db, range_name: str, now: datetime | None = None) -> list[dict]:
    """Per server: the share of the check-ins it should have made in the range that actually arrived.

    Approximate: it assumes the check interval it has now held for the whole range, and the range starts when the
    server's first data does, so a new server isn't blamed for the time before it existed.
    """
    now = _utc(now or datetime.now(timezone.utc))
    result = []
    for server in db.execute(select(Server).order_by(Server.hostname)).scalars():
        series = history.history(db, history.HOST, server.id, None, range_name, (), server.check_interval_seconds, now=now)
        received = sum(point["samples"] for point in series["points"])
        span = (now - datetime.fromisoformat(series["start"])).total_seconds()
        expected = max(span / server.check_interval_seconds, 1)
        result.append({"server_id": server.id, "hostname": server.hostname, "checkins": received, "expected": round(expected),
                       "percent": round(min(100.0, 100.0 * received / expected), 2) if series["data_since"] else None})
    return result


HEATMAP_DAYS = 30
HEATMAP_METRICS = ("cpu", "ram")
MAX_OFFSET_MINUTES = 14 * 60


def hour_weekday_heatmap(db, server: Server, metric: str, offset_minutes: int = 0, now: datetime | None = None) -> dict:
    """The average of a metric for each hour of the day and weekday over the last 30 days, for a time zone.

    ``offset_minutes`` is the viewer's offset from UTC (a fixed one: daylight saving changes inside the 30 days
    are ignored). Row 0 is Monday. Hours nobody reported stay None.
    """
    now = _utc(now or datetime.now(timezone.utc))
    items, _ = history._tiered_items(db, history.HOST, (metric,), server.id, None, rollups.HOUR, now - timedelta(days=HEATMAP_DAYS), now)
    totals = [[[0.0, 0] for _ in range(24)] for _ in range(7)]
    for start, agg in items:
        local = start + timedelta(minutes=offset_minutes)
        total, count, _maximum = agg.values.get(metric, (0.0, 0, None))
        cell = totals[local.weekday()][local.hour]
        cell[0] += total
        cell[1] += count
    since = history.data_since(db, history.HOST, server.id, None)
    return {
        "server_id": server.id, "metric": metric, "days": HEATMAP_DAYS, "offset_minutes": offset_minutes,
        "data_since": since.isoformat() if since else None,
        "cells": [[(cell[0] / cell[1]) if cell[1] else None for cell in row] for row in totals],
        "samples": [[cell[1] for cell in row] for row in totals],
    }


def fleet_stats(db, range_name: str, now: datetime | None = None) -> dict:
    return {"incidents": incident_stats(db, range_name, now), "availability": availability(db, range_name, now)}


# --- containers over a range ------------------------------------------------------------------------------

SPARKLINE_POINTS = 40
MAX_ROLLUP_ROWS = 200_000
# Which summaries answer a range for every container of a server at once. A week is read from the hourly ones
# rather than from check-ins: a week of check-ins for a few dozen containers is over a million rows.
CONTAINER_TIER = {"7d": rollups.HOUR, "30d": rollups.HOUR, "90d": rollups.DAY, "1y": rollups.DAY, "all": rollups.DAY}
BUSIEST_TIER = {"7d": rollups.HOUR, "30d": rollups.HOUR, "90d": rollups.WEEK, "1y": rollups.WEEK, "all": rollups.MONTH}


def _epoch_bucket(column, step: int, origin: int = 0):
    """The whole number of ``step``-second periods a timestamp is past ``origin``, worked out by the database."""
    return (cast(extract("epoch", column), Integer) - origin) // step


class _Usage:
    """What one container did over a range, added up from samples or summaries."""

    __slots__ = ("name", "cpu", "cpu_n", "cpu_max", "memory", "memory_n", "memory_max", "restarts_high", "restarts_low", "samples", "down", "spark", "swap", "swap_n", "swap_max")

    def __init__(self):
        self.name, self.cpu, self.cpu_n, self.cpu_max, self.memory, self.memory_n, self.memory_max = "", 0.0, 0, None, 0.0, 0, None
        self.restarts_high, self.restarts_low, self.samples, self.down, self.spark = None, None, 0, 0, {}
        self.swap, self.swap_n, self.swap_max = 0.0, 0, None

    def add_swap(self, total, count, maximum):
        if total is not None and count:
            self.swap, self.swap_n = self.swap + total, self.swap_n + count
        self.swap_max = maximum if maximum is not None and (self.swap_max is None or maximum > self.swap_max) else self.swap_max

    def add(self, name, cpu_total, cpu_n, cpu_max, memory_total, memory_n, memory_max, restarts_high, restarts_low, samples, down, slot):
        self.name = name or self.name
        if cpu_total is not None and cpu_n:
            self.cpu, self.cpu_n = self.cpu + cpu_total, self.cpu_n + cpu_n
            cell = self.spark.setdefault(slot, [0.0, 0])
            cell[0], cell[1] = cell[0] + cpu_total, cell[1] + cpu_n
        if memory_total is not None and memory_n:
            self.memory, self.memory_n = self.memory + memory_total, self.memory_n + memory_n
        self.cpu_max = cpu_max if cpu_max is not None and (self.cpu_max is None or cpu_max > self.cpu_max) else self.cpu_max
        self.memory_max = memory_max if memory_max is not None and (self.memory_max is None or memory_max > self.memory_max) else self.memory_max
        for value in (restarts_high, restarts_low):
            if value is not None:
                self.restarts_high = value if self.restarts_high is None else max(self.restarts_high, value)
                self.restarts_low = value if self.restarts_low is None else min(self.restarts_low, value)
        self.samples, self.down = self.samples + (samples or 0), self.down + (down or 0)


def _raw_usage(db, usage: dict, server_id: int, since: datetime, until: datetime, origin: int, slot_seconds: int) -> None:
    table = ContainerMetric
    slot = _epoch_bucket(table.time, slot_seconds, origin).label("slot")
    query = select(
        table.container_id, func.max(table.name), slot, func.sum(table.cpu_pct), func.count(table.cpu_pct), func.max(table.cpu_pct),
        func.sum(table.memory_used_mb), func.count(table.memory_used_mb), func.max(table.memory_used_mb), func.max(table.restart_count), func.min(table.restart_count),
        func.count(), func.sum(case((table.status != "running", 1), else_=0)),
        func.sum(table.swap_mb), func.count(table.swap_mb), func.max(table.swap_mb),
    ).where(table.server_id == server_id, table.time >= since, table.time < until).group_by(table.container_id, slot)
    for container_id, name, index, *figures in db.execute(query):
        item = usage.setdefault(container_id, _Usage())
        item.add(name, *figures[:-3], max(0, min(SPARKLINE_POINTS - 1, int(index))))
        item.add_swap(*figures[-3:])


def container_usage(db, server: Server, range_name: str, now: datetime | None = None) -> dict:
    """Every container of a server over a range: average and peak CPU and memory, restarts in the range, the share of
    check-ins it was down, and a small CPU series. The busiest comes first."""
    now = _utc(now or datetime.now(timezone.utc))
    length = history.RANGES[range_name]
    tier = CONTAINER_TIER.get(range_name)
    first = db.execute(select(func.min(ContainerMetric.time)).where(ContainerMetric.server_id == server.id)).scalar()
    oldest = db.execute(select(func.min(ContainerRollup.bucket_start)).where(ContainerRollup.server_id == server.id)).scalar()
    known = [_utc(moment) for moment in (first, oldest) if moment]
    since = min(known) if known else now
    start = max(now - length, since) if length else since
    origin = int(start.timestamp())
    slot_seconds = max(1, -(-int((now - start).total_seconds() + 1) // SPARKLINE_POINTS))
    usage: dict[str, _Usage] = {}
    raw_from = start
    if tier:
        done = rollups._read_watermark(db, rollups.HOUR if tier == rollups.HOUR else tier)
        closed = min(done, now) if done and done > start else start
        table = ContainerRollup
        query = select(
            table.container_id, table.name, table.bucket_start, table.avg_cpu, table.n_cpu, table.max_cpu, table.avg_memory_mb, table.n_memory_mb, table.max_memory_mb,
            table.max_restarts, table.samples, table.down_samples, table.avg_swap_mb, table.n_swap_mb, table.max_swap_mb,
        ).where(table.server_id == server.id, table.tier == tier, table.bucket_start >= rollups.floor(tier, start), table.bucket_start < closed).limit(MAX_ROLLUP_ROWS)
        for container_id, name, bucket, avg_cpu, n_cpu, max_cpu, avg_memory, n_memory, max_memory, restarts, samples, down, avg_swap, n_swap, max_swap in db.execute(query):
            slot = max(0, min(SPARKLINE_POINTS - 1, (int(_utc(bucket).timestamp()) - origin) // slot_seconds))
            usage.setdefault(container_id, _Usage()).add(
                name, avg_cpu * n_cpu if avg_cpu is not None else None, n_cpu, max_cpu, avg_memory * n_memory if avg_memory is not None else None, n_memory, max_memory,
                restarts, restarts, samples, down, slot)
            usage[container_id].add_swap(avg_swap * n_swap if avg_swap is not None and n_swap else None, n_swap or 0, max_swap)
        # What the summaries haven't reached yet comes from the check-ins themselves, when those are still recent.
        raw_from = max(closed, now - timedelta(days=2)) if closed > start else start if now - start <= timedelta(days=2) else closed
    _raw_usage(db, usage, server.id, raw_from, now + timedelta(seconds=1), origin, slot_seconds)
    containers = []
    for container_id, item in usage.items():
        containers.append({
            "container_id": container_id, "name": item.name,
            "avg_cpu": item.cpu / item.cpu_n if item.cpu_n else None, "max_cpu": item.cpu_max,
            "avg_memory_mb": item.memory / item.memory_n if item.memory_n else None, "max_memory_mb": item.memory_max,
            "avg_swap_mb": item.swap / item.swap_n if item.swap_n else None, "max_swap_mb": item.swap_max,
            "restarts": item.restarts_high - item.restarts_low if item.restarts_high is not None else None,
            "samples": item.samples, "down_samples": item.down,
            "sparkline": [round(item.spark[slot][0] / item.spark[slot][1], 2) if slot in item.spark else None for slot in range(SPARKLINE_POINTS)],
        })
    containers.sort(key=lambda item: (-(item["avg_cpu"] if item["avg_cpu"] is not None else -1), item["name"]))
    return {"server_id": server.id, "range": range_name, "start": start.isoformat(), "end": now.isoformat(), "containers": containers}


def busiest_containers(db, server: Server, range_name: str, now: datetime | None = None) -> dict:
    """Per period of the range, the container that used the most CPU and how much, so a spike on the server's
    chart can name what caused it. Periods in which no container reported are left out."""
    now = _utc(now or datetime.now(timezone.utc))
    length = history.RANGES[range_name]
    tier = BUSIEST_TIER.get(range_name)
    best: dict[int, tuple[float, str]] = {}

    def offer(moment: int, name: str, cpu: float | None) -> None:
        if cpu is not None and (moment not in best or cpu > best[moment][0]):
            best[moment] = (cpu, name)

    if tier:
        step = history.NATURAL_SECONDS[tier]
        table = ContainerRollup
        query = select(table.bucket_start, table.name, table.max_cpu).where(table.server_id == server.id, table.tier == tier, table.max_cpu.isnot(None))
        if length:
            query = query.where(table.bucket_start >= rollups.floor(tier, now - length))
        for bucket, name, cpu in db.execute(query.limit(MAX_ROLLUP_ROWS)):
            offer(int(_utc(bucket).timestamp()), name, cpu)
        done = rollups._read_watermark(db, tier)
        raw_from = max(done, now - timedelta(days=2)) if done else now - min(length or timedelta(days=2), timedelta(days=2))
    else:
        step = history.nice_step(max(length.total_seconds() / history.MAX_POINTS, server.check_interval_seconds))
        raw_from = now - length
    # Check-ins the summaries haven't reached (or, for the short ranges, all of them), grouped by the database.
    raw_step = step if not tier else history.NATURAL_SECONDS[rollups.HOUR] if tier == rollups.HOUR else None
    if raw_step:
        table = ContainerMetric
        bucket = _epoch_bucket(table.time, raw_step).label("bucket")
        query = select(bucket, table.container_id, func.max(table.name), func.max(table.cpu_pct)).where(table.server_id == server.id, table.time >= raw_from, table.time <= now).group_by(bucket, table.container_id)
        for index, _container_id, name, cpu in db.execute(query):
            offer(int(index) * raw_step, name, cpu)
    return {"server_id": server.id, "range": range_name, "bucket_seconds": step,
            "points": [{"time": datetime.fromtimestamp(moment, timezone.utc).isoformat(), "name": name, "cpu": round(cpu, 2)} for moment, (cpu, name) in sorted(best.items())]}


SPIKE_PERCENT = 90.0
MAX_SPIKES = 500
MAX_SPIKE_ROWS = 150_000


def cpu_spikes(db, server: Server, range_name: str, now: datetime | None = None) -> dict:
    """Every stretch in which the server's CPU stayed at or above 90%, with how long it lasted, read from the
    check-ins themselves so a spike shorter than a chart point still has a length. Only ranges that still have
    check-ins (up to a week) can be answered; longer ones give no spikes and say so."""
    now = _utc(now or datetime.now(timezone.utc))
    length = history.RANGES[range_name]
    if history.SOURCE[range_name] != history.DETAILED or length is None:
        return {"server_id": server.id, "range": range_name, "available": False, "threshold": SPIKE_PERCENT, "spikes": []}
    rows = db.execute(select(HostMetric.time, HostMetric.cpu_utilization_pct).where(HostMetric.server_id == server.id, HostMetric.time >= now - length, HostMetric.time <= now).order_by(HostMetric.time).limit(MAX_SPIKE_ROWS))
    spikes, current, previous = [], None, None
    # A check-in that is late by more than a few intervals ends a spike: nobody knows what happened in between.
    patience = timedelta(seconds=server.check_interval_seconds * 3)
    for moment, cpu in rows:
        moment = _utc(moment)
        high = cpu is not None and cpu >= SPIKE_PERCENT
        if current and (not high or moment - previous > patience):
            spikes.append(current)
            current = None
        if high:
            if current is None:
                current = {"start": moment, "end": moment, "peak": cpu}
            current["end"], current["peak"] = moment, max(current["peak"], cpu)
        previous = moment
    if current:
        spikes.append(current)
    return {"server_id": server.id, "range": range_name, "available": True, "threshold": SPIKE_PERCENT, "spikes": [
        # One check-in at 90% stands for about one interval, not for no time at all.
        {"time": spike["start"].isoformat(), "seconds": round((spike["end"] - spike["start"]).total_seconds()) + server.check_interval_seconds, "peak": round(spike["peak"], 1)}
        for spike in spikes[-MAX_SPIKES:]]}