Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.
Python ai-agentsaiopsdashboarddockerfastapihomelabincident-managementllmmonitoringopenaireactroot-cause-analysisself-hosteduptime-monitoring
"""The evidence pack: everything the agent is told about one incident, collected from stored data without any model.
Every fact has a stable ID (``metric:main``, ``metric:host.cpu``, ``event:812``, ``check:2``, ``incident:31``), and a
diagnosis may only point at facts by these IDs, so a claim that cites something not in the pack can be found and
dropped. The same incident and data always give the same pack, and its size is bounded: series are cut to a few
dozen points, lists to their newest entries, text to a few hundred characters. All text is masked (masking.py)
before the pack is stored or sent anywhere.
"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from src.backend import incidents
from src.backend.models import ContainerMetric, HostMetric, IncidentAlert, MountMetric, Server, SystemEvent
from .bounds import clip as _clip, series as _series, utc as _utc
from .masking import mask
LEAD = timedelta(minutes=60)
MAX_EVENTS = 30
MAX_IMAGE_CHANGES = 10
MAX_SIMILAR = 5
SIMILAR_DAYS = 90
IMAGE_HISTORY_DAYS = 30
def build(db, incident: IncidentAlert, now: datetime | None = None) -> dict:
"""The pack for ``incident``: what it is, the machine, metrics around it, events, image changes, the rule's own
checks and similar incidents before it. Masked and bounded."""
now = _utc(now or datetime.now(timezone.utc))
server = db.get(Server, incident.server_id)
created, resolved = _utc(incident.created_at), _utc(incident.resolved_at)
since, until = created - LEAD, resolved or now
kind = incidents.rule_kind(incident.rule_key)
subject = incidents.subject(incident.rule_key)
rule = incidents.diagnose(db, incident, server, now) if server else {}
metrics = []
main = rule.get("evidence")
if main and main.get("series"):
rows = [(datetime.fromisoformat(point["time"]), point["value"]) for point in main["series"]]
metrics.append(_series("metric:main", f"{main['title']} (what the rule watches)", main.get("unit") or "", rows, main.get("limit")))
host_rows = db.query(HostMetric.time, HostMetric.cpu_utilization_pct, HostMetric.ram_pct, HostMetric.swap_pct, HostMetric.load_1m) \
.filter(HostMetric.server_id == incident.server_id, HostMetric.time >= since, HostMetric.time <= until).order_by(HostMetric.time).limit(20000).all()
for index, (fact, label, unit) in enumerate((("host.cpu", "host CPU", "%"), ("host.memory", "host memory", "%"), ("host.swap", "host swap", "%"), ("host.load", "host load (1 min)", "")), start=1):
metrics.append(_series(f"metric:{fact}", label, unit, [(row[0], row[index]) for row in host_rows], server.cpu_cores if fact == "host.load" and server else None))
if kind.startswith("container:") and subject:
container_rows = db.query(ContainerMetric.time, ContainerMetric.cpu_pct, ContainerMetric.memory_used_mb, ContainerMetric.restart_count) \
.filter(ContainerMetric.server_id == incident.server_id, ContainerMetric.name == subject, ContainerMetric.time >= since, ContainerMetric.time <= until).order_by(ContainerMetric.time).limit(20000).all()
metrics.append(_series("metric:container.cpu", f"{subject} CPU", "%", [(row[0], row[1]) for row in container_rows]))
metrics.append(_series("metric:container.restarts", f"{subject} restart count (running total)", "", [(row[0], row[3]) for row in container_rows]))
if kind in ("host:disk", "host:inodes") and subject:
mounts = db.query(MountMetric.time, MountMetric.used_gb, MountMetric.total_gb).filter(MountMetric.server_id == incident.server_id, MountMetric.mount == subject, MountMetric.time >= since, MountMetric.time <= until).order_by(MountMetric.time).limit(20000).all()
metrics.append(_series("metric:mount.used_gb", f"{subject} used (total {mounts[-1][2]:.0f} GB)" if mounts and mounts[-1][2] else f"{subject} used", "GB", [(row[0], row[1]) for row in mounts]))
metrics = [metric for metric in metrics if metric]
latest = db.query(HostMetric.time).filter(HostMetric.server_id == incident.server_id).order_by(HostMetric.time.desc()).limit(1).scalar()
containers = []
if latest is not None:
for row in db.query(ContainerMetric).filter(ContainerMetric.server_id == incident.server_id, ContainerMetric.time == latest).order_by(ContainerMetric.memory_used_mb.desc()).limit(8):
containers.append({"name": row.name, "image": _clip(row.image, 120), "status": row.status, "health": row.healthcheck_status, "cpu_pct": row.cpu_pct, "memory_mb": row.memory_used_mb,
"memory_limit_mb": row.memory_limit_mb, "restarts": row.restart_count, "exit_code": row.exit_code, "oom_killed": row.oom_killed})
events = db.query(SystemEvent).filter(SystemEvent.server_id == incident.server_id, SystemEvent.time >= since - timedelta(hours=1), SystemEvent.time <= until) \
.order_by(SystemEvent.time.desc()).limit(MAX_EVENTS).all()
image_changes = []
if subject and kind.startswith("container:"):
image_changes = db.query(SystemEvent).filter(SystemEvent.server_id == incident.server_id, SystemEvent.kind == "image_changed", SystemEvent.event_key == subject,
SystemEvent.time >= created - timedelta(days=IMAGE_HISTORY_DAYS), SystemEvent.time <= until).order_by(SystemEvent.time.desc()).limit(MAX_IMAGE_CHANGES).all()
seen = {event.id for event in events}
similar = db.query(IncidentAlert).filter(IncidentAlert.server_id == incident.server_id, IncidentAlert.id != incident.id, IncidentAlert.created_at >= created - timedelta(days=SIMILAR_DAYS),
IncidentAlert.created_at < created).order_by(IncidentAlert.created_at.desc()).limit(200).all()
similar = [item for item in similar if incidents.rule_kind(item.rule_key) == kind][:MAX_SIMILAR]
pack = {
"incident": {"id": f"incident:{incident.id}", "reference": incidents.reference(incident.id), "rule": incident.rule_key, "kind": kind, "subject": subject, "severity": incident.severity,
"title": _clip(incident.title), "description": _clip(incident.description, 600), "opened_at": created.isoformat(timespec="minutes"),
"resolved_at": resolved.isoformat(timespec="minutes") if resolved else None, "status": incident.status},
"server": {"hostname": server.hostname if server else None, "os": server.os_info if server else None, "cpu_cores": server.cpu_cores if server else None,
"memory_gb": round(server.total_ram_gb, 1) if server and server.total_ram_gb else None},
"window": {"from": since.isoformat(timespec="minutes"), "to": until.isoformat(timespec="minutes")},
"metrics": metrics,
"containers_now": containers,
"events": [{"id": f"event:{event.id}", "time": _utc(event.time).isoformat(timespec="minutes"), "kind": event.kind, "text": _clip(event.detail)} for event in reversed(events)],
"image_changes": [{"id": f"event:{event.id}", "time": _utc(event.time).isoformat(timespec="minutes"), "text": _clip(event.detail)} for event in reversed(image_changes) if event.id not in seen]
+ [{"id": f"event:{event.id}", "time": _utc(event.time).isoformat(timespec="minutes"), "text": _clip(event.detail), "also_in": "events"} for event in reversed(image_changes) if event.id in seen],
"rule_checks": [{"id": f"check:{index}", "text": _clip(signal["text"]), "holds": signal["ok"], "source": signal.get("source")} for index, signal in enumerate(rule.get("signals") or [], start=1)],
"similar_incidents": [{"id": f"incident:{item.id}", "reference": incidents.reference(item.id), "title": _clip(item.title), "opened_at": _utc(item.created_at).isoformat(timespec="minutes"),
"status": item.status, "resolution": _clip(item.resolution) if item.resolution else None} for item in similar],
}
return mask(pack)
def index(pack: dict) -> dict[str, str]:
"""Every fact ID in ``pack`` with a short label, for checking a diagnosis's references and showing them."""
labels = {pack["incident"]["id"]: f"{pack['incident']['reference']} itself"}
for metric in pack["metrics"]:
labels[metric["id"]] = metric["label"]
for event in pack["events"] + pack["image_changes"]:
labels[event["id"]] = f"{event['time'][11:16]} {event.get('kind', 'image change').replace('_', ' ')}"
for check in pack["rule_checks"]:
labels[check["id"]] = check["text"]
for item in pack["similar_incidents"]:
labels[item["id"]] = f"{item['reference']} · {item['title']}"
return labels