Skip to content

Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.

6.6 KB
Raw Blame
"""Read models for the Overview page: sparklines, the all-clear summary and a server's event timeline."""
from __future__ import annotations

from datetime import datetime, timedelta, timezone

from .alerts import INFO, SERVER_OFFLINE_RULE
from .models import ContainerMetric, HostMetric, IncidentAlert, Server, SystemEvent
from .rules import CONTAINER_EVENT_KINDS, NETWORK_EVENT_KINDS

SPARKLINE_POINTS = 40
MAX_EVENTS = 2000


def _utc(moment: datetime) -> datetime:
    return moment if moment.tzinfo else moment.replace(tzinfo=timezone.utc)


def downsample(values: list[float | None], limit: int = SPARKLINE_POINTS) -> list[float | None]:
    """At most ``limit`` points, each the average of its share of ``values``; gaps stay gaps."""
    if len(values) <= limit:
        return [None if value is None else round(value, 2) for value in values]
    result = []
    for index in range(limit):
        chunk = [value for value in values[index * len(values) // limit:(index + 1) * len(values) // limit] if value is not None]
        result.append(round(sum(chunk) / len(chunk), 2) if chunk else None)
    return result


def fleet_sparklines(db, minutes: int, now: datetime | None = None) -> dict[int, list[float | None]]:
    """Recent CPU use of every server, oldest first, for the fleet rack."""
    since = (now or datetime.now(timezone.utc)) - timedelta(minutes=minutes)
    series: dict[int, list[float | None]] = {}
    for server_id, cpu in db.query(HostMetric.server_id, HostMetric.cpu_utilization_pct).filter(HostMetric.time >= since).order_by(HostMetric.time):
        series.setdefault(server_id, []).append(cpu)
    return {server_id: downsample(values) for server_id, values in series.items()}


def container_sparklines(db, server_id: int, minutes: int, now: datetime | None = None) -> dict[str, list[float | None]]:
    """Recent CPU use of every container on one server, oldest first, for the container table."""
    since = (now or datetime.now(timezone.utc)) - timedelta(minutes=minutes)
    series: dict[str, list[float | None]] = {}
    rows = db.query(ContainerMetric.container_id, ContainerMetric.cpu_pct).filter(ContainerMetric.server_id == server_id, ContainerMetric.time >= since).order_by(ContainerMetric.time)
    for container_id, cpu in rows:
        series.setdefault(container_id, []).append(cpu)
    return {container_id: downsample(values) for container_id, values in series.items()}


def longest_quiet_seconds(spans: list[tuple[datetime, datetime]], start: datetime, end: datetime) -> float:
    """The longest stretch between ``start`` and ``end`` that none of the (opened, closed) ``spans`` covers."""
    longest, cursor = 0.0, start
    for opened, closed in sorted(spans):
        if opened > cursor:
            longest = max(longest, (min(opened, end) - cursor).total_seconds())
        cursor = max(cursor, closed)
        if cursor >= end:
            break
    return max(longest, (end - cursor).total_seconds(), 0.0)


def quiet_summary(db, now: datetime | None = None) -> dict:
    """What the Overview says when nothing needs attention: the last incident and how long it has been quiet.

    Notices (INFO) don't count: they never asked for attention in the first place.
    """
    now = _utc(now or datetime.now(timezone.utc))
    serious = IncidentAlert.severity != INFO
    last = db.query(IncidentAlert).filter(serious, IncidentAlert.status == "resolved", IncidentAlert.resolved_at.isnot(None)).order_by(IncidentAlert.resolved_at.desc()).first()
    month_start = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
    first_server = db.query(Server.created_at).order_by(Server.created_at).first()
    # A month the install only partly existed in is measured from when it started watching.
    window_start = max(month_start, _utc(first_server[0])) if first_server else now
    spans = [
        (_utc(alert.created_at), _utc(alert.resolved_at) if alert.resolved_at else now)
        for alert in db.query(IncidentAlert).filter(serious, (IncidentAlert.resolved_at.is_(None)) | (IncidentAlert.resolved_at >= window_start))
    ]
    hostname = db.query(Server.hostname).filter(Server.id == last.server_id).scalar() if last else None
    return {
        "last_incident": {"title": last.title, "hostname": hostname, "severity": last.severity, "created_at": _utc(last.created_at).isoformat(), "resolved_at": _utc(last.resolved_at).isoformat()} if last else None,
        "quiet_since": (_utc(last.resolved_at) if last else _utc(first_server[0]) if first_server else now).isoformat(),
        "longest_quiet_seconds_this_month": round(longest_quiet_seconds(spans, window_start, now)),
    }


def _event_source(kind: str) -> str:
    return "container" if kind in CONTAINER_EVENT_KINDS else "network" if kind in NETWORK_EVENT_KINDS else "system"


def server_events(db, server: Server, days: int, now: datetime | None = None) -> list[dict]:
    """Everything that happened on one server, newest first: its recorded events plus alerts opening and clearing.

    A server going offline and coming back is told as the collector's story rather than as an alert.
    """
    now = _utc(now or datetime.now(timezone.utc))
    since = now - timedelta(days=days)
    events: list[dict] = []
    for event in db.query(SystemEvent).filter(SystemEvent.server_id == server.id, SystemEvent.time >= since).order_by(SystemEvent.time.desc()).limit(MAX_EVENTS):
        events.append({"time": _utc(event.time).isoformat(), "source": _event_source(event.kind), "kind": event.kind, "detail": event.detail})
    alerts = db.query(IncidentAlert).filter(IncidentAlert.server_id == server.id, (IncidentAlert.created_at >= since) | (IncidentAlert.resolved_at >= since)).order_by(IncidentAlert.created_at.desc()).limit(MAX_EVENTS)
    for alert in alerts:
        offline = alert.rule_key == SERVER_OFFLINE_RULE
        created, resolved = _utc(alert.created_at), _utc(alert.resolved_at) if alert.resolved_at else None
        common = {"alert_id": alert.id, "rule_key": alert.rule_key, "severity": alert.severity, "title": alert.title}
        if created >= since:
            events.append({**common, "time": created.isoformat(), "source": "collector" if offline else "alert", "kind": "collector_stopped" if offline else "alert_opened",
                           "detail": alert.description, "resolved": resolved is not None})
        if resolved and resolved >= since:
            events.append({**common, "time": resolved.isoformat(), "source": "collector" if offline else "alert", "kind": "collector_resumed" if offline else "alert_resolved",
                           "detail": alert.description, "duration_seconds": round((resolved - created).total_seconds())})
    events.sort(key=lambda event: event["time"], reverse=True)
    return events[:MAX_EVENTS]