Skip to content

Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.

19.8 KB
Raw Blame
import os
from datetime import datetime, timedelta, timezone

os.environ.setdefault("DATABASE_URL", "sqlite://")
os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0")

import pytest
from sqlalchemy import create_engine, func
from sqlalchemy.orm import sessionmaker

from src.backend.routes import monitoring as monitoring_routes
from src.backend import main, rollups
from src.backend.models import (
    AppSetting, Base, ContainerMetric, ContainerRollup, HostMetric, HostRollup, MountMetric, MountRollup, Server, User,
)


def T(year, month, day, hour=0, minute=0):
    return datetime(year, month, day, hour, minute, tzinfo=timezone.utc)


@pytest.fixture
def db():
    engine = create_engine("sqlite://")
    Base.metadata.create_all(engine)
    session = sessionmaker(bind=engine)()
    session.add(User(id=1, name="Owner", email="[email protected]", password_hash="x", role="admin"))
    session.add_all(Server(id=n, owner_id=1, hostname=f"srv{n}", check_interval_seconds=60) for n in (1, 2))
    session.commit()
    return session


def ago(days, hour=10):
    """A moment a few days back, for the tests that use the real clock (maintenance and server deletion)."""
    return (datetime.now(timezone.utc) - timedelta(days=days)).replace(hour=hour, minute=0, second=0, microsecond=0)


def host(db, when, server=1, **fields):
    db.add(HostMetric(server_id=server, time=when, **fields))
    db.commit()


def catch_up(db, now):
    """Run the job until it has nothing left to do (a long history takes several chunks)."""
    for _ in range(100):
        if not any(rollups.run_rollups(db, now).values()):
            return
    raise AssertionError("the job never finished")


def hosts(db, tier, server=1):
    rows = db.query(HostRollup).filter_by(tier=tier, server_id=server).order_by(HostRollup.bucket_start).all()
    return [(rollups._utc(row.bucket_start), row) for row in rows]


# --- bucket arithmetic ---------------------------------------------------------------------

def test_buckets_start_on_utc_boundaries_and_weeks_on_monday():
    assert rollups.floor(rollups.HOUR, T(2026, 3, 4, 17, 59)) == T(2026, 3, 4, 17)
    assert rollups.floor(rollups.DAY, T(2026, 3, 4, 17, 59)) == T(2026, 3, 4)
    sunday = T(2026, 3, 8, 23, 59)
    assert sunday.weekday() == 6
    assert rollups.floor(rollups.WEEK, sunday) == T(2026, 3, 2) and T(2026, 3, 2).weekday() == 0
    assert rollups.floor(rollups.WEEK, T(2026, 3, 9)) == T(2026, 3, 9)  # the next Monday starts a new week
    assert rollups.floor(rollups.MONTH, T(2026, 3, 31, 23)) == T(2026, 3, 1)


def test_a_time_with_another_offset_is_converted_to_utc_first():
    plus_two = datetime(2026, 3, 5, 1, 30, tzinfo=timezone(timedelta(hours=2)))  # 23:30 UTC the day before
    assert rollups.floor(rollups.DAY, plus_two) == T(2026, 3, 4)
    assert rollups.floor(rollups.DAY, datetime(2026, 3, 4, 12, 0)) == T(2026, 3, 4)  # naive counts as UTC


def test_the_following_bucket_handles_month_ends_and_leap_years():
    assert rollups.following(rollups.MONTH, T(2026, 12, 1)) == T(2027, 1, 1)
    assert rollups.following(rollups.MONTH, T(2028, 2, 1)) == T(2028, 3, 1)
    assert rollups.following(rollups.WEEK, T(2026, 12, 28)) == T(2027, 1, 4)
    assert rollups.following(rollups.HOUR, T(2026, 3, 8, 23)) == T(2026, 3, 9)


# --- hourly summaries ----------------------------------------------------------------------

def test_an_hour_is_summarised_with_average_maximum_and_count(db):
    for minute, cpu in ((0, 10.0), (20, 20.0), (40, 60.0)):
        host(db, T(2026, 3, 4, 10, minute), cpu_utilization_pct=cpu, ram_pct=50.0)

    assert rollups.run_rollups(db, T(2026, 3, 4, 11, 10))[rollups.HOUR] == 1

    [(start, row)] = hosts(db, rollups.HOUR)
    assert start == T(2026, 3, 4, 10)
    assert (row.samples, row.avg_cpu, row.max_cpu, row.avg_ram, row.max_ram) == (3, 30.0, 60.0, 50.0, 50.0)
    assert row.avg_swap is None and row.max_swap is None  # never reported stays unknown, not zero


def test_a_metric_missing_from_some_samples_is_averaged_over_the_ones_that_have_it(db):
    host(db, T(2026, 3, 4, 10, 0), cpu_utilization_pct=10.0)
    host(db, T(2026, 3, 4, 10, 1))
    host(db, T(2026, 3, 4, 10, 2), cpu_utilization_pct=30.0)

    rollups.run_rollups(db, T(2026, 3, 4, 11, 10))

    [(_, row)] = hosts(db, rollups.HOUR)
    assert (row.samples, row.avg_cpu, row.max_cpu) == (3, 20.0, 30.0)


def test_a_history_of_nothing_but_unknowns_still_counts_check_ins_and_does_not_break(db):
    for minute in range(5):
        host(db, T(2026, 3, 4, 10, minute))
    db.add(MountMetric(server_id=1, time=T(2026, 3, 4, 10, 0), mount="/"))
    db.add(ContainerMetric(server_id=1, time=T(2026, 3, 4, 10, 0), container_id="c", name="web", status="running"))
    db.commit()

    rollups.run_rollups(db, T(2026, 3, 4, 11, 10))

    [(_, row)] = hosts(db, rollups.HOUR)
    assert row.samples == 5 and row.avg_cpu is None and row.max_cpu is None
    assert db.query(MountRollup).one().avg_used_pct is None
    container = db.query(ContainerRollup).one()
    assert (container.samples, container.down_samples, container.avg_cpu, container.max_restarts) == (1, 0, None, 0)   # restarts default to 0


def test_only_completed_buckets_are_written_after_a_short_grace_period(db):
    host(db, T(2026, 3, 4, 10, 30), cpu_utilization_pct=10.0)

    assert rollups.run_rollups(db, T(2026, 3, 4, 10, 59))[rollups.HOUR] == 0
    assert rollups.run_rollups(db, T(2026, 3, 4, 11, 4))[rollups.HOUR] == 0   # ended, but check-ins may still be arriving
    assert hosts(db, rollups.HOUR) == []
    assert rollups.run_rollups(db, T(2026, 3, 4, 11, 6))[rollups.HOUR] == 1
    assert len(hosts(db, rollups.HOUR)) == 1


def test_running_twice_changes_nothing(db):
    for minute in range(0, 60, 10):
        host(db, T(2026, 3, 4, 10, minute), cpu_utilization_pct=float(minute))
    db.add(ContainerMetric(server_id=1, time=T(2026, 3, 4, 10, 0), container_id="c", name="web", status="running", cpu_pct=5.0))
    db.commit()
    now = T(2026, 3, 6, 0, 30)

    rollups.run_rollups(db, now)
    first = [(r.tier, r.bucket_start, r.samples, r.avg_cpu) for r in db.query(HostRollup).order_by(HostRollup.id)]
    again = rollups.run_rollups(db, now)

    assert again == {"hour": 0, "day": 0, "week": 0, "month": 0}
    assert [(r.tier, r.bucket_start, r.samples, r.avg_cpu) for r in db.query(HostRollup).order_by(HostRollup.id)] == first


def test_rebuilding_a_bucket_replaces_it_instead_of_duplicating_it(db):
    host(db, T(2026, 3, 4, 10, 0), cpu_utilization_pct=10.0)
    rollups.run_rollups(db, T(2026, 3, 4, 11, 10))
    db.query(AppSetting).filter(AppSetting.key == "rollup_done_hour").delete()   # forget how far it got
    host(db, T(2026, 3, 4, 10, 30), cpu_utilization_pct=30.0)

    rollups.run_rollups(db, T(2026, 3, 4, 11, 10))

    [(_, row)] = hosts(db, rollups.HOUR)
    assert (row.samples, row.avg_cpu) == (2, 20.0)


def test_servers_mounts_and_containers_are_summarised_separately(db):
    host(db, T(2026, 3, 4, 10, 0), server=1, cpu_utilization_pct=10.0)
    host(db, T(2026, 3, 4, 10, 0), server=2, cpu_utilization_pct=90.0)
    for mount, used in (("/", 40.0), ("/data", 80.0)):
        db.add(MountMetric(server_id=1, time=T(2026, 3, 4, 10, 0), mount=mount, used_pct=used, used_gb=used, total_gb=100.0))
    for name, cpu in (("web", 5.0), ("db", 50.0)):
        db.add(ContainerMetric(server_id=1, time=T(2026, 3, 4, 10, 0), container_id=f"id-{name}", name=name, status="running", cpu_pct=cpu, memory_used_mb=100.0, memory_limit_mb=512.0, restart_count=2))
    db.commit()

    rollups.run_rollups(db, T(2026, 3, 4, 11, 10))

    assert [row.avg_cpu for _, row in hosts(db, rollups.HOUR, 1)] == [10.0] and [row.avg_cpu for _, row in hosts(db, rollups.HOUR, 2)] == [90.0]
    mounts = {row.mount: (row.avg_used_pct, row.max_total_gb) for row in db.query(MountRollup)}
    assert mounts == {"/": (40.0, 100.0), "/data": (80.0, 100.0)}
    containers = {row.name: (row.container_id, row.avg_cpu, row.max_memory_limit_mb, row.max_restarts) for row in db.query(ContainerRollup)}
    assert containers == {"web": ("id-web", 5.0, 512.0, 2), "db": ("id-db", 50.0, 512.0, 2)}


def test_down_samples_count_the_check_ins_where_a_container_was_not_running(db):
    for minute, status in ((0, "running"), (1, "exited"), (2, "exited"), (3, "restarting")):
        db.add(ContainerMetric(server_id=1, time=T(2026, 3, 4, 10, minute), container_id="c", name="web", status=status))
    db.commit()

    rollups.run_rollups(db, T(2026, 3, 4, 11, 10))

    row = db.query(ContainerRollup).one()
    assert (row.samples, row.down_samples) == (4, 3)


# --- daily, weekly and monthly summaries -----------------------------------------------------

def test_a_day_is_the_samples_weighted_average_of_its_hours_and_keeps_the_maximum(db):
    host(db, T(2026, 3, 4, 10, 0), cpu_utilization_pct=100.0)
    for minute in range(3):
        host(db, T(2026, 3, 4, 11, minute), cpu_utilization_pct=0.0)

    rollups.run_rollups(db, T(2026, 3, 5, 0, 10))

    [(start, day)] = hosts(db, rollups.DAY)
    assert start == T(2026, 3, 4)
    assert (day.samples, day.avg_cpu, day.max_cpu) == (4, 25.0, 100.0)   # 1 sample of 100 and 3 of 0, not (100 + 0) / 2


def test_each_average_remembers_how_many_check_ins_reported_it(db):
    host(db, T(2026, 3, 4, 10, 0), cpu_utilization_pct=10.0, ram_pct=40.0)
    host(db, T(2026, 3, 4, 10, 1), cpu_utilization_pct=20.0)
    host(db, T(2026, 3, 4, 10, 2))

    rollups.run_rollups(db, T(2026, 3, 5, 0, 10))

    for tier in (rollups.HOUR, rollups.DAY):
        [(_, row)] = hosts(db, tier)
        assert (row.samples, row.n_cpu, row.n_ram, row.n_swap) == (3, 2, 1, 0)


def test_a_container_that_was_stopped_part_of_the_time_is_averaged_over_the_time_it_ran(db):
    for hour, status, cpu in ((10, "running", 90.0), (11, "exited", None), (12, "exited", None), (13, "exited", None)):
        db.add(ContainerMetric(server_id=1, time=T(2026, 3, 4, hour), container_id="c", name="web", status=status, cpu_pct=cpu))
    db.add(ContainerMetric(server_id=1, time=T(2026, 3, 5, 10), container_id="c", name="web", status="running", cpu_pct=10.0))
    db.commit()

    catch_up(db, T(2026, 3, 9, 0, 10))

    day = db.query(ContainerRollup).filter_by(tier="day", bucket_start=T(2026, 3, 4)).one()
    week = db.query(ContainerRollup).filter_by(tier="week").one()
    assert (day.samples, day.down_samples, day.avg_cpu, day.n_cpu) == (4, 3, 90.0, 1)
    assert (week.samples, week.down_samples, week.avg_cpu, week.n_cpu) == (5, 3, 50.0, 2)   # (90 + 10) / 2, not (90*4 + 10) / 5


def test_hours_that_never_reported_a_metric_do_not_pull_a_day_toward_zero(db):
    host(db, T(2026, 3, 4, 10, 0), cpu_utilization_pct=80.0)
    host(db, T(2026, 3, 4, 11, 0), docker_latency_ms=50)
    host(db, T(2026, 3, 4, 11, 1), docker_latency_ms=70)

    rollups.run_rollups(db, T(2026, 3, 5, 0, 10))

    [(_, day)] = hosts(db, rollups.DAY)
    assert (day.avg_cpu, day.max_cpu, day.avg_docker_ms, day.max_docker_ms, day.samples) == (80.0, 80.0, 60.0, 70, 3)


def test_a_week_runs_monday_to_sunday_and_is_written_only_when_sunday_has_ended(db):
    assert T(2026, 3, 2).weekday() == 0
    for day in range(2, 9):                                    # Monday 2 March to Sunday 8 March
        host(db, T(2026, 3, day, 12, 0), cpu_utilization_pct=float(day))
    host(db, T(2026, 3, 9, 12, 0), cpu_utilization_pct=1000.0)  # the following Monday belongs to the next week

    rollups.run_rollups(db, T(2026, 3, 8, 23, 59))
    assert hosts(db, rollups.WEEK) == []                        # still Sunday

    rollups.run_rollups(db, T(2026, 3, 9, 0, 10))
    [(start, week)] = hosts(db, rollups.WEEK)
    assert start == T(2026, 3, 2)
    assert (week.samples, week.avg_cpu, week.max_cpu) == (7, 5.0, 8.0)   # days 2..8 average 5, the Monday after is excluded


def test_a_month_is_built_from_its_days_including_a_leap_february(db):
    for day in range(1, 30):
        host(db, T(2028, 2, day, 12, 0), cpu_utilization_pct=10.0)

    catch_up(db, T(2028, 3, 1, 0, 10))

    [(start, month)] = hosts(db, rollups.MONTH)
    assert start == T(2028, 2, 1) and (month.samples, month.avg_cpu) == (29, 10.0)


def test_data_either_side_of_new_year_lands_in_different_months_and_weeks(db):
    host(db, T(2026, 12, 31, 12, 0), cpu_utilization_pct=10.0)   # a Thursday
    host(db, T(2027, 1, 1, 12, 0), cpu_utilization_pct=20.0)

    catch_up(db, T(2027, 2, 1, 1, 0))

    assert [(start, row.avg_cpu) for start, row in hosts(db, rollups.MONTH)] == [(T(2026, 12, 1), 10.0), (T(2027, 1, 1), 20.0)]
    assert [(start, row.avg_cpu) for start, row in hosts(db, rollups.WEEK)] == [(T(2026, 12, 28), 15.0)]   # one ISO week spans both


def test_every_tier_is_built_for_mounts_and_containers_too(db):
    for day in (2, 3):
        db.add(MountMetric(server_id=1, time=T(2026, 3, day, 12), mount="/", used_pct=40.0 + day, used_gb=10.0, total_gb=100.0))
        db.add(ContainerMetric(server_id=1, time=T(2026, 3, day, 12), container_id="c", name="web", status="running", cpu_pct=float(day), memory_used_mb=100.0))
    db.add(ContainerMetric(server_id=1, time=T(2026, 3, 3, 13), container_id="c", name="web", status="exited"))
    db.commit()

    catch_up(db, T(2026, 4, 2))

    for model in (MountRollup, ContainerRollup):
        assert {row.tier for row in db.query(model)} == {"hour", "day", "week", "month"}
    week = db.query(ContainerRollup).filter_by(tier="week").one()
    assert (week.samples, week.down_samples, week.avg_cpu, week.max_cpu, week.avg_memory_mb) == (3, 1, 2.5, 3.0, 100.0)
    assert db.query(MountRollup).filter_by(tier="week").one().max_used_pct == 43.0


# --- gaps, catching up and the watermark -----------------------------------------------------

def test_a_gap_stays_a_gap(db):
    host(db, T(2026, 3, 4, 10, 0), server=1, cpu_utilization_pct=10.0)
    host(db, T(2026, 3, 4, 10, 0), server=2, cpu_utilization_pct=20.0)
    host(db, T(2026, 3, 4, 14, 0), server=1, cpu_utilization_pct=30.0)   # server 2 was silent at 14:00 and 11:00-13:00
    host(db, T(2026, 3, 6, 9, 0), server=1, cpu_utilization_pct=40.0)    # nothing at all on the 5th

    rollups.run_rollups(db, T(2026, 3, 7, 0, 10))

    assert [start for start, _ in hosts(db, rollups.HOUR, 1)] == [T(2026, 3, 4, 10), T(2026, 3, 4, 14), T(2026, 3, 6, 9)]
    assert [start for start, _ in hosts(db, rollups.HOUR, 2)] == [T(2026, 3, 4, 10)]
    assert [start for start, _ in hosts(db, rollups.DAY, 1)] == [T(2026, 3, 4), T(2026, 3, 6)]


def test_the_first_run_summarises_the_history_that_already_exists(db):
    for day in range(2, 11):                                   # Monday 2 March to Tuesday 10 March
        for hour in range(0, 24, 6):
            host(db, T(2026, 3, day, hour, 0), cpu_utilization_pct=float(day))

    result = rollups.run_rollups(db, T(2026, 3, 11, 0, 10))

    assert result[rollups.HOUR] == 216 and result[rollups.DAY] == 9 and result[rollups.WEEK] == 1   # only 2-8 March is a complete week
    assert len(hosts(db, rollups.HOUR)) == 36 and len(hosts(db, rollups.DAY)) == 9


def test_new_samples_are_picked_up_by_the_next_run(db):
    host(db, T(2026, 3, 4, 10, 0), cpu_utilization_pct=10.0)
    rollups.run_rollups(db, T(2026, 3, 4, 11, 10))
    host(db, T(2026, 3, 4, 12, 0), cpu_utilization_pct=20.0)

    assert rollups.run_rollups(db, T(2026, 3, 4, 13, 10))[rollups.HOUR] == 2   # the empty 11:00 hour and the 12:00 one
    assert [row.avg_cpu for _, row in hosts(db, rollups.HOUR)] == [10.0, 20.0]


def test_after_downtime_the_job_catches_up_a_chunk_at_a_time(db, monkeypatch):
    monkeypatch.setitem(rollups.MAX_BUCKETS_PER_RUN, rollups.HOUR, 5)
    for hour in range(12):
        host(db, T(2026, 3, 4, hour, 0), cpu_utilization_pct=float(hour))
    now = T(2026, 3, 4, 12, 10)

    built = [rollups.run_rollups(db, now)[rollups.HOUR] for _ in range(4)]

    assert built == [5, 5, 2, 0]
    assert [row.avg_cpu for _, row in hosts(db, rollups.HOUR)] == [float(hour) for hour in range(12)]


def test_a_long_empty_stretch_cannot_stall_the_job(db, monkeypatch):
    monkeypatch.setitem(rollups.MAX_BUCKETS_PER_RUN, rollups.HOUR, 5)
    host(db, T(2026, 3, 4, 0, 0), cpu_utilization_pct=1.0)
    host(db, T(2026, 3, 5, 6, 0), cpu_utilization_pct=2.0)     # 30 hours later, with nothing between

    for _ in range(10):
        rollups.run_rollups(db, T(2026, 3, 5, 8, 0))

    assert [row.avg_cpu for _, row in hosts(db, rollups.HOUR)] == [1.0, 2.0]


def test_the_watermark_is_stored_per_tier_and_a_damaged_one_is_ignored(db):
    host(db, T(2026, 3, 4, 10, 0), cpu_utilization_pct=10.0)
    rollups.run_rollups(db, T(2026, 3, 6, 0, 10))

    assert db.get(AppSetting, "rollup_done_hour").value.startswith("2026-03-06T00:00:00")
    assert db.get(AppSetting, "rollup_done_day").value.startswith("2026-03-06T00:00:00")

    db.get(AppSetting, "rollup_done_hour").value = "not a date"
    db.commit()
    rollups.run_rollups(db, T(2026, 3, 6, 0, 10))                # starts over from the first sample, harmlessly
    assert len(hosts(db, rollups.HOUR)) == 1 and len(hosts(db, rollups.DAY)) == 1


# --- purging only what has been summarised ---------------------------------------------------

def test_without_any_summaries_no_check_in_is_purged(db):
    host(db, T(2026, 3, 1, 10, 0), cpu_utilization_pct=1.0)
    host(db, T(2026, 3, 20, 10, 0), cpu_utilization_pct=2.0)

    main.retention.purge(db, now=T(2026, 3, 20, 12, 0))

    assert db.query(HostMetric).count() == 2


def test_samples_that_were_not_summarised_yet_are_never_purged(db):
    for day in range(1, 10):
        host(db, T(2026, 3, day, 10, 0), cpu_utilization_pct=float(day))
    db.add(AppSetting(key="rollup_done_hour", value=T(2026, 3, 2, 12).isoformat()))   # only the first day and a half are summarised
    db.commit()

    main.retention.purge(db, now=T(2026, 3, 12, 12, 0))      # the week alone would delete everything before 5 March

    # 1 and 2 March are summarised and old enough; 3 and 4 March are old enough but not summarised yet, so they stay
    assert [rollups._utc(row.time).day for row in db.query(HostMetric).order_by(HostMetric.time)] == [3, 4, 5, 6, 7, 8, 9]


def test_maintenance_summarises_before_it_purges(db):
    for days_back, cpu in ((9, 1.0), (8, 2.0)):
        host(db, ago(days_back), cpu_utilization_pct=cpu)
    db.add(ContainerMetric(server_id=1, time=ago(9), container_id="c", name="web", status="running"))
    host(db, datetime.now(timezone.utc) - timedelta(minutes=1), cpu_utilization_pct=99.0)   # recent: kept
    db.commit()

    assert main.metrics_maintenance(db, purge_due=True) is True

    assert [row.cpu_utilization_pct for row in db.query(HostMetric)] == [99.0] and db.query(ContainerMetric).count() == 0
    assert [row.avg_cpu for _, row in hosts(db, rollups.HOUR)] == [1.0, 2.0] and db.query(ContainerRollup).filter_by(tier="hour").count() == 1


def test_a_failing_rollup_job_stops_the_purge(db, monkeypatch):
    host(db, ago(9), cpu_utilization_pct=1.0)

    def broken(*args, **kwargs):
        raise RuntimeError("database trouble")

    monkeypatch.setattr(rollups, "run_rollups", broken)

    assert main.metrics_maintenance(db, purge_due=True) is False
    assert db.query(HostMetric).count() == 1


def test_between_purges_maintenance_only_summarises(db):
    host(db, ago(2), cpu_utilization_pct=1.0)

    assert main.metrics_maintenance(db, purge_due=False) is False
    assert len(hosts(db, rollups.HOUR)) == 1 and db.query(HostMetric).count() == 1


def test_deleting_a_server_deletes_its_summaries_but_not_another_servers(db, monkeypatch):
    monkeypatch.setattr(main, "_require_admin", lambda session_id, db: {"id": 1})
    for server in (1, 2):
        host(db, ago(2), server=server, cpu_utilization_pct=10.0)
        db.add(MountMetric(server_id=server, time=ago(2), mount="/", used_pct=1.0))
        db.add(ContainerMetric(server_id=server, time=ago(2), container_id="c", name="web", status="running"))
    db.commit()
    rollups.run_rollups(db)

    monitoring_routes.delete_monitored_server(1, session_id="s", db=db)

    for model in (HostRollup, MountRollup, ContainerRollup):
        assert db.query(model).filter_by(server_id=1).count() == 0
        assert db.query(model).filter_by(server_id=2).count() > 0