Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.
Python ai-agentsaiopsdashboarddockerfastapihomelabincident-managementllmmonitoringopenaireactroot-cause-analysisself-hosteduptime-monitoring
import os
from datetime import datetime, timedelta, timezone
os.environ.setdefault("DATABASE_URL", "sqlite://")
os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0")
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from src.collector import collector
from src.backend.routes import monitoring as monitoring_routes
from src.backend import main
from src.backend.models import Base, IncidentAlert, Server, SystemEvent, User
TOKEN = "collector-token"
BOOT_A, BOOT_B = "boot-a", "boot-b"
@pytest.fixture
def db():
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
session = sessionmaker(bind=engine)()
session.add(User(id=1, name="Owner", email="[email protected]", password_hash="x", role="admin"))
session.add(Server(id=5, owner_id=1, hostname="nas", check_interval_seconds=60, collector_token_hash=main._hash_secret(TOKEN)))
session.commit()
return session
def _ingest(db, **system):
host = {"boot_id": BOOT_A, "oom_kill_total": 0, "kernel_tainted": 0, "ntp_synced": True, "reboot_required": False, **system}
monitoring_routes.ingest_metrics(main.MetricsIngestRequest(server_id=5, host=host), authorization=f"Bearer {TOKEN}", db=db)
def _active(db):
return {alert.rule_key: alert for alert in db.query(IncidentAlert).filter_by(status="active")}
def _kinds(db):
return [event.kind for event in db.query(SystemEvent).order_by(SystemEvent.id)]
def test_first_check_in_records_no_events(db):
_ingest(db)
assert _kinds(db) == [] and _active(db) == {}
def test_new_boot_id_records_a_reboot_and_raises_info_alert(db):
_ingest(db)
_ingest(db, boot_id=BOOT_B)
assert _kinds(db) == ["reboot"]
assert _active(db)["system:reboot"].severity == "INFO"
def test_reboot_alert_clears_after_the_event_window(db):
_ingest(db)
_ingest(db, boot_id=BOOT_B)
for event in db.query(SystemEvent):
event.time = event.time.replace(year=2000)
db.commit()
_ingest(db, boot_id=BOOT_B)
assert "system:reboot" not in _active(db)
def test_oom_counter_rise_raises_alert_but_a_reboot_resetting_it_does_not(db):
_ingest(db, oom_kill_total=4)
_ingest(db, oom_kill_total=6)
assert _kinds(db) == ["oom_kill"]
assert _active(db)["system:oom"].severity == "WARNING"
_ingest(db, boot_id=BOOT_B, oom_kill_total=0)
assert _kinds(db) == ["oom_kill", "reboot"]
def test_repeated_oom_kills_are_critical(db):
for total in range(4):
_ingest(db, oom_kill_total=total)
assert _active(db)["system:oom"].severity == "CRITICAL"
def test_kernel_fault_taint_alerts_until_reboot_but_module_taint_is_ignored(db):
_ingest(db, kernel_tainted=1 | 4096) # proprietary + out-of-tree module only
assert _active(db) == {} and _kinds(db) == []
_ingest(db, kernel_tainted=1 | 1 << 4 | 1 << 7)
assert _kinds(db) == ["kernel_fault"]
assert "machine check" in db.query(SystemEvent).one().detail
_ingest(db, kernel_tainted=1 | 1 << 4 | 1 << 7)
assert "system:kernel" in _active(db) and _kinds(db) == ["kernel_fault"]
_ingest(db, boot_id=BOOT_B, kernel_tainted=0)
assert "system:kernel" not in _active(db)
def test_ntp_alert_needs_sustained_unsynced_checks_and_resolves(db):
_ingest(db, ntp_synced=False)
_ingest(db, ntp_synced=False)
assert "system:ntp" not in _active(db)
_ingest(db, ntp_synced=False)
assert "system:ntp" in _active(db)
_ingest(db, ntp_synced=True)
assert "system:ntp" not in _active(db)
def test_unknown_values_never_alert(db):
for _ in range(4):
_ingest(db, boot_id=None, oom_kill_total=None, kernel_tainted=None, ntp_synced=None, reboot_required=None)
assert _active(db) == {}
def test_malformed_values_from_a_collector_are_dropped(db):
_ingest(db, oom_kill_total="many", kernel_tainted=True, ntp_synced="yes", reboot_required=1, boot_id="x" * 100)
from src.backend.models import HostMetric
row = db.query(HostMetric).one()
assert (row.oom_kill_total, row.kernel_tainted, row.ntp_synced, row.reboot_required) == (None, None, None, None)
assert row.boot_id == "x" * 64
def test_reboot_required_raises_info_alert(db):
_ingest(db, reboot_required=True)
assert _active(db)["system:reboot_required"].severity == "INFO"
_ingest(db, reboot_required=False)
assert "system:reboot_required" not in _active(db)
def test_collector_reads_kernel_state_from_proc(tmp_path, monkeypatch):
(tmp_path / "sys/kernel/random").mkdir(parents=True)
(tmp_path / "sys/kernel/random/boot_id").write_text("abc-123\n")
(tmp_path / "sys/kernel/tainted").write_text("128\n")
(tmp_path / "vmstat").write_text("nr_free_pages 1\noom_kill 7\n")
monkeypatch.setattr(collector, "HOST_PROC", tmp_path)
monkeypatch.setattr(collector, "HOST_ROOT", tmp_path)
(tmp_path / "etc").mkdir()
(tmp_path / "etc/debian_version").write_text("12\n")
state = collector.collect_system()
assert (state["boot_id"], state["kernel_tainted"], state["oom_kill_total"], state["reboot_required"]) == ("abc-123", 128, 7, False)
(tmp_path / "run").mkdir()
(tmp_path / "run/reboot-required").touch()
assert collector.collect_system()["reboot_required"] is True
def test_collector_reports_unknown_when_files_are_missing(tmp_path, monkeypatch):
monkeypatch.setattr(collector, "HOST_PROC", tmp_path)
monkeypatch.setattr(collector, "HOST_ROOT", tmp_path)
state = collector.collect_system()
assert (state["boot_id"], state["kernel_tainted"], state["oom_kill_total"], state["reboot_required"]) == (None, None, None, None)
# --- the System row's event list ---------------------------------------------------------------
def _event(db, kind, detail, minutes_ago):
db.add(SystemEvent(server_id=5, time=datetime.now(timezone.utc) - timedelta(minutes=minutes_ago), kind=kind, detail=detail))
db.commit()
def _shown(db, monkeypatch):
monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1})
return monitoring_routes.monitoring_overview(session_id="s", db=db)["servers"][0]["system_events"]
def test_repeated_kernel_errors_collapse_into_one_row_with_a_count(db, monkeypatch):
for minutes_ago in (3000, 2000, 100):
_event(db, "kernel_error", "ACPI BIOS Error (bug): Could not resolve symbol", minutes_ago)
_event(db, "kernel_error", "i915: Unclaimed access detected", 50)
rows = {row["detail"]: row for row in _shown(db, monkeypatch)}
assert len(rows) == 2
acpi = rows["ACPI BIOS Error (bug): Could not resolve symbol"]
assert acpi["count"] == 3
# The row's time is the latest occurrence; the first one is kept for the tooltip.
assert datetime.fromisoformat(acpi["time"]) > datetime.fromisoformat(acpi["first_time"])
assert rows["i915: Unclaimed access detected"]["count"] == 1
def test_reboots_and_oom_kills_are_never_collapsed(db, monkeypatch):
for minutes_ago in (500, 100):
_event(db, "reboot", "The host rebooted after a clean shutdown", minutes_ago)
_event(db, "oom_kill", "The kernel OOM killer stopped java (pid 1)", minutes_ago + 1)
rows = _shown(db, monkeypatch)
assert sorted((row["kind"], row["count"]) for row in rows) == [("oom_kill", 1), ("oom_kill", 1), ("reboot", 1), ("reboot", 1)]
def test_noisy_kernel_errors_cannot_crowd_out_real_events(db, monkeypatch):
_event(db, "oom_kill", "The kernel OOM killer stopped java (pid 1)", 60 * 24 * 6)
for number in range(40):
_event(db, "kernel_error", f"driver message {number}", number + 1)
rows = _shown(db, monkeypatch)
assert [row["kind"] for row in rows].count("oom_kill") == 1
assert [row["kind"] for row in rows].count("kernel_error") == main.SYSTEM_EVENTS_SHOWN
def test_events_are_listed_newest_first_across_kinds(db, monkeypatch):
_event(db, "reboot", "The host rebooted", 300)
_event(db, "kernel_error", "late message", 10)
_event(db, "oom_kill", "The kernel OOM killer stopped x (pid 2)", 100)
assert [row["kind"] for row in _shown(db, monkeypatch)] == ["kernel_error", "oom_kill", "reboot"]
def test_events_older_than_seven_days_are_not_shown(db, monkeypatch):
_event(db, "reboot", "The host rebooted", 60 * 24 * 8)
_event(db, "kernel_error", "old message", 60 * 24 * 8)
assert _shown(db, monkeypatch) == []