Skip to content

Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.

10.7 KB
Raw Blame
import json
import os
import struct
import time
from datetime import datetime, timedelta, timezone

os.environ.setdefault("DATABASE_URL", "sqlite://")
os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0")

import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

from src.collector import collector, system_helper as helper
from src.backend.routes import monitoring as monitoring_routes
from src.backend import main
from src.backend.models import Base, IncidentAlert, Server, SystemEvent, User

TOKEN = "collector-token"
BOOT = "boot-a"
NOW = 1_000_000.0


def _klog(*lines):
    return "\n".join(lines) + "\n"


def _parse(text, boot_id=BOOT):
    return helper.parse_kernel_log(text, boot_id, monotonic_now=500.0, wall_now=NOW)


# --- kernel log parsing -------------------------------------------------------------------

def test_oom_kill_is_parsed_with_process_name_and_wall_time():
    [event] = _parse(_klog("<3>[  400.000000] Out of memory: Killed process 4321 (java) total-vm:100kB"))

    assert (event["kind"], event["key"]) == ("oom_kill", "boot-a:oom:400.000000")
    assert event["detail"] == "The kernel OOM killer stopped java (pid 4321)"
    # 100 s before the snapshot: monotonic 500 vs 400.
    assert datetime.fromisoformat(event["time"]) == datetime.fromtimestamp(NOW - 100, timezone.utc)


def test_cgroup_oom_is_recognised():
    [event] = _parse(_klog("<3>[   10.500000] Memory cgroup out of memory: Killed process 77 (postgres) total-vm:1kB"))
    assert event["kind"] == "oom_kill" and "postgres" in event["detail"]


def test_kernel_errors_are_kept_but_info_lines_are_not():
    events = _parse(_klog("<6>[    1.000000] eth0: link up", "<4>[    2.000000] a warning", "<3>[    3.000000] nvme0: I/O error, dev nvme0n1", "<0>[    4.000000] kernel panic"))
    assert [event["detail"] for event in events] == ["nvme0: I/O error, dev nvme0n1", "kernel panic"]
    assert {event["kind"] for event in events} == {"kernel_error"}


def test_hostile_log_content_is_neutralised():
    name = "x\x1b[31m;$(reboot)\x00" + "A" * 500
    [oom] = _parse(_klog(f"<3>[    5.000000] Out of memory: Killed process 1 ({name})"))
    [err] = _parse(_klog("<3>[    6.000000] bad \x1b[2J\x07 line " + "B" * 1000))

    assert all(char.isprintable() for char in oom["detail"] + err["detail"])
    assert len(oom["detail"]) < 150 and len(err["detail"]) <= helper.MAX_TEXT


def test_malformed_and_untimestamped_lines_are_ignored():
    assert _parse(_klog("garbage", "<3>no timestamp here", "<x>[1.000000] y", " continuation line", "<3>[abc] z", "")) == []


def test_output_is_capped_to_the_newest_events():
    lines = [f"<3>[{n:>8}.000000] error number {n}" for n in range(1, 40)]
    events = _parse(_klog(*lines))
    assert len(events) == helper.MAX_ERROR_EVENTS and events[-1]["detail"] == "error number 39"


# --- wtmp ---------------------------------------------------------------------------------

def _record(kind, user, when):
    return helper._UTMP.pack(kind, 0, b"~", b"~~", user.encode(), b"", 0, 0, 0, int(when), 0, 0, 0, 0, 0)


def _wtmp(tmp_path, *records):
    path = tmp_path / "wtmp"
    path.write_bytes(b"".join(_record(*record) for record in records))
    return path


def test_utmp_layout_is_384_bytes():
    assert helper._UTMP.size == 384


def test_clean_shutdown_is_detected(tmp_path):
    path = _wtmp(tmp_path, (2, "reboot", 100), (1, "shutdown", 900), (2, "reboot", 1000))
    assert helper.classify_last_shutdown(helper.read_wtmp_records(path), boot_time=1002) == "clean"


def test_missing_shutdown_record_is_unclean_when_shutdowns_were_recorded_before(tmp_path):
    path = _wtmp(tmp_path, (2, "reboot", 100), (1, "shutdown", 200), (2, "reboot", 300), (2, "reboot", 1000))
    assert helper.classify_last_shutdown(helper.read_wtmp_records(path), boot_time=1000) == "unclean"


def test_never_recorded_shutdowns_are_unknown_not_unclean(tmp_path):
    path = _wtmp(tmp_path, (2, "reboot", 100), (2, "reboot", 300), (2, "reboot", 1000))
    assert helper.classify_last_shutdown(helper.read_wtmp_records(path), boot_time=1000) == "unknown"


def test_unknown_when_wtmp_is_missing_short_or_out_of_date(tmp_path):
    assert helper.read_wtmp_records(tmp_path / "nope") == []
    assert helper.classify_last_shutdown([], 1000) == "unknown"
    one_boot = helper.read_wtmp_records(_wtmp(tmp_path, (2, "reboot", 1000)))
    assert helper.classify_last_shutdown(one_boot, 1000) == "unknown"
    stale = helper.read_wtmp_records(_wtmp(tmp_path, (1, "shutdown", 10), (2, "reboot", 100), (2, "reboot", 200)))
    assert helper.classify_last_shutdown(stale, boot_time=5000) == "unknown"


def test_truncated_or_symlinked_wtmp_is_safe(tmp_path):
    path = _wtmp(tmp_path, (2, "reboot", 1))
    path.write_bytes(path.read_bytes() + b"junk")
    assert len(helper.read_wtmp_records(path)) == 1
    link = tmp_path / "link"
    link.symlink_to(path)
    assert helper.read_wtmp_records(link) == []


def test_snapshot_is_written_atomically_with_a_fixed_schema(tmp_path):
    out = tmp_path / "system.json"
    helper.write_snapshot({"version": 1, "events": []}, out)
    assert json.loads(out.read_text()) == {"version": 1, "events": []}
    assert not (tmp_path / "system.json.tmp").exists() and out.stat().st_mode & 0o777 == 0o644


# --- collector reading the helper's file --------------------------------------------------

def _helper_file(tmp_path, **overrides):
    data = {"version": 1, "generated_at": datetime.now(timezone.utc).isoformat(), "boot_id": BOOT, "last_shutdown": "clean",
            "events": [{"kind": "oom_kill"}], **overrides}
    path = tmp_path / "system.json"
    path.write_text(json.dumps(data))
    return path


def test_collector_uses_a_fresh_file_from_the_current_boot(tmp_path):
    assert collector.collect_helper(BOOT, _helper_file(tmp_path)) == {"last_shutdown": "clean", "helper_events": [{"kind": "oom_kill"}]}


@pytest.mark.parametrize("overrides", [{"boot_id": "other"}, {"generated_at": "2001-01-01T00:00:00+00:00"}, {"generated_at": "nonsense"}])
def test_collector_ignores_stale_or_foreign_files(tmp_path, overrides):
    assert collector.collect_helper(BOOT, _helper_file(tmp_path, **overrides)) == {}


def test_collector_refuses_missing_symlinked_oversized_and_garbage_files(tmp_path):
    assert collector.collect_helper(BOOT, tmp_path / "missing.json") == {}
    real = _helper_file(tmp_path)
    link = tmp_path / "link.json"
    link.symlink_to(real)
    assert collector.collect_helper(BOOT, link) == {}
    big = tmp_path / "big.json"
    big.write_text(" " * (collector.HELPER_FILE_MAX_BYTES + 10))
    assert collector.collect_helper(BOOT, big) == {}
    bad = tmp_path / "bad.json"
    bad.write_text("[1,2")
    assert collector.collect_helper(BOOT, bad) == {}
    arr = tmp_path / "arr.json"
    arr.write_text("[]")
    assert collector.collect_helper(BOOT, arr) == {}


# --- backend ingest -----------------------------------------------------------------------

@pytest.fixture
def db():
    engine = create_engine("sqlite://")
    Base.metadata.create_all(engine)
    session = sessionmaker(bind=engine)()
    session.add(User(id=1, name="Owner", email="[email protected]", password_hash="x", role="admin"))
    session.add(Server(id=5, owner_id=1, hostname="nas", check_interval_seconds=60, collector_token_hash=main._hash_secret(TOKEN)))
    session.commit()
    return session


def _ingest(db, **host):
    host = {"boot_id": BOOT, "oom_kill_total": 0, "kernel_tainted": 0, "ntp_synced": True, "reboot_required": False, **host}
    monitoring_routes.ingest_metrics(main.MetricsIngestRequest(server_id=5, host=host), authorization=f"Bearer {TOKEN}", db=db)


def _event(kind="oom_kill", key="boot-a:oom:1.000000", minutes_ago=5, detail="The kernel OOM killer stopped java (pid 1)"):
    return {"kind": kind, "key": key, "detail": detail, "time": (datetime.now(timezone.utc) - timedelta(minutes=minutes_ago)).isoformat()}


def _events(db):
    return [(e.kind, e.event_key) for e in db.query(SystemEvent).order_by(SystemEvent.id)]


def _alerts(db):
    return {a.rule_key: a for a in db.query(IncidentAlert).filter_by(status="active")}


def test_helper_events_are_stored_once_even_when_republished(db):
    _ingest(db, helper_events=[_event()], last_shutdown="unknown")
    _ingest(db, helper_events=[_event()], last_shutdown="unknown")

    assert _events(db) == [("oom_kill", "boot-a:oom:1.000000")]
    assert "java" in _alerts(db)["system:oom"].description


def test_with_the_helper_the_oom_counter_does_not_add_a_second_event(db):
    _ingest(db, oom_kill_total=0, helper_events=[], last_shutdown="unknown")
    _ingest(db, oom_kill_total=1, helper_events=[_event()], last_shutdown="unknown")

    assert _events(db) == [("oom_kill", "boot-a:oom:1.000000")]


@pytest.mark.parametrize("bad", [
    {"kind": "reboot", "key": "k", "detail": "d", "time": "2026-01-01T00:00:00+00:00"},  # kind not allowed from a helper
    _event(key="has spaces"), _event(key="x" * 200), _event(minutes_ago=60 * 24 * 30), _event(minutes_ago=-600),
    {**_event(), "time": "garbage"}, {**_event(), "detail": 5}, "string", None, 12,
])
def test_malformed_helper_events_are_dropped(db, bad):
    _ingest(db, helper_events=[bad], last_shutdown="unknown")
    assert _events(db) == []


def test_helper_event_count_and_text_are_bounded(db):
    many = [_event(key=f"boot-a:err:{n}", kind="kernel_error", detail="e" * 5000) for n in range(500)]
    _ingest(db, helper_events=many, last_shutdown="unknown")

    rows = db.query(SystemEvent).all()
    assert len(rows) == main.MAX_HELPER_EVENTS_PER_CHECKIN == 20 and all(len(row.detail) <= 300 for row in rows)   # the first 50 offered, 20 stored per check-in


def test_unclean_shutdown_makes_the_reboot_critical(db):
    _ingest(db)
    _ingest(db, boot_id="boot-b", helper_events=[], last_shutdown="unclean")

    alert = _alerts(db)["system:reboot"]
    assert alert.severity == "CRITICAL" and "unclean" in alert.title
    assert db.query(SystemEvent).one().detail.startswith("The host restarted after an unclean shutdown")


def test_clean_shutdown_keeps_the_reboot_informational(db):
    _ingest(db)
    _ingest(db, boot_id="boot-b", helper_events=[], last_shutdown="clean")

    assert _alerts(db)["system:reboot"].severity == "INFO"
    assert "clean shutdown" in db.query(SystemEvent).one().detail


def test_helper_arriving_after_the_reboot_was_noticed_updates_the_event_and_alert(db):
    _ingest(db)
    _ingest(db, boot_id="boot-b")  # collector saw the new boot before the helper had written anything
    assert _alerts(db)["system:reboot"].severity == "INFO"

    _ingest(db, boot_id="boot-b", helper_events=[], last_shutdown="unclean")

    assert _alerts(db)["system:reboot"].severity == "CRITICAL"


def test_invalid_last_shutdown_values_are_dropped(db):
    from src.backend.models import HostMetric
    _ingest(db, last_shutdown="<script>")
    assert db.query(HostMetric).one().last_shutdown is None