Skip to content

Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.

8.6 KB
Raw Blame
import os
from datetime import datetime, timedelta, timezone

os.environ.setdefault("DATABASE_URL", "sqlite://")
os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0")

import pytest
from fastapi import HTTPException
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

from src.collector import collector
from src.backend.routes import monitoring as monitoring_routes
from src.backend import main
from src.backend.models import Base, ContainerMetric, Server, User


@pytest.fixture(autouse=True)
def _fresh_collector_state():
    collector._cpu_previous.clear()
    collector._one_shot_unsupported = False
    yield
    collector._cpu_previous.clear()
    collector._one_shot_unsupported = False


def _one_shot(total, system, cpus=4, **extra):
    """Docker's one-shot answer: current counters and no earlier reading."""
    return {"cpu_stats": {"cpu_usage": {"total_usage": total}, "system_cpu_usage": system, "online_cpus": cpus}, "precpu_stats": {}, **extra}


def _session():
    engine = create_engine("sqlite://")
    Base.metadata.create_all(engine)
    session = sessionmaker(bind=engine)()
    session.add_all([
        User(id=1, name="Owner", email="[email protected]", password_hash="x", role="admin"),
        User(id=2, name="Other", email="[email protected]", password_hash="x"),
    ])
    session.add(Server(id=10, owner_id=1, hostname="nas"))
    now = datetime.now(timezone.utc)
    for index in range(3):
        session.add(ContainerMetric(server_id=10, container_id="abc", name="web", status="running", cpu_pct=float(index), memory_used_mb=100.0 + index, memory_limit_mb=1024.0, pids=4, time=now - timedelta(minutes=3 - index)))
    session.commit()
    return session


def test_container_detail_returns_latest_and_chronological_history(monkeypatch):
    session = _session()
    monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1})

    result = monitoring_routes.container_detail(10, "abc", limit=60, session_id="s", db=session)

    assert result["name"] == "web"
    assert result["latest"]["cpu_pct"] == 2.0
    assert result["latest"]["memory_limit_mb"] == 1024.0
    assert [sample["cpu_pct"] for sample in result["history"]] == [0.0, 1.0, 2.0]


def test_container_detail_is_visible_to_regular_users(monkeypatch):
    session = _session()
    monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 2})

    assert monitoring_routes.container_detail(10, "abc", limit=60, session_id="s", db=session)["name"] == "web"


def test_container_detail_returns_404_for_unknown_server(monkeypatch):
    session = _session()
    monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1})

    with pytest.raises(HTTPException) as exc:
        monitoring_routes.container_detail(99, "abc", limit=60, session_id="s", db=session)
    assert exc.value.status_code == 404


def test_container_usage_parses_docker_stats(monkeypatch):
    stats = {
        "cpu_stats": {"cpu_usage": {"total_usage": 300}, "system_cpu_usage": 1000, "online_cpus": 4},
        "precpu_stats": {"cpu_usage": {"total_usage": 100}, "system_cpu_usage": 600},
        "memory_stats": {"usage": 300 * collector.MiB, "limit": 1000 * collector.MiB, "stats": {"inactive_file": 100 * collector.MiB}},
        "networks": {"eth0": {"rx_bytes": 2 * collector.MiB, "tx_bytes": collector.MiB}},
        "blkio_stats": {"io_service_bytes_recursive": [{"op": "read", "value": 5 * collector.MiB}, {"op": "write", "value": 3 * collector.MiB}]},
        "pids_stats": {"current": 7},
    }
    monkeypatch.setattr(collector, "_docker_request", lambda socket_path, endpoint: stats)

    usage = collector._container_usage("/sock", "abc")

    assert usage["cpu_pct"] == 200.0
    assert usage["memory_used_mb"] == 200.0
    assert usage["memory_pct"] == 20.0
    assert (usage["net_rx_mb"], usage["net_tx_mb"]) == (2.0, 1.0)
    assert (usage["block_read_mb"], usage["block_write_mb"]) == (5.0, 3.0)
    assert usage["pids"] == 7


def test_the_quick_one_shot_request_is_tried_first(monkeypatch):
    asked = []
    monkeypatch.setattr(collector, "_docker_request", lambda socket_path, endpoint: asked.append(endpoint) or _one_shot(100, 1000))

    collector._container_usage("/sock", "abc")

    assert asked == ["/containers/abc/stats?stream=false&one-shot=true"]


def test_cpu_is_the_change_since_the_previous_check_in_and_the_first_has_none(monkeypatch):
    readings = iter([_one_shot(100, 1000), _one_shot(300, 1400), _one_shot(300, 1800)])
    monkeypatch.setattr(collector, "_docker_request", lambda socket_path, endpoint: next(readings))

    first = collector._container_usage("/sock", "abc")
    second = collector._container_usage("/sock", "abc")
    idle = collector._container_usage("/sock", "abc")

    assert "cpu_pct" not in first            # unknown, not zero
    assert second["cpu_pct"] == 200.0        # 200 of 400 CPU time across 4 cores
    assert "cpu_pct" not in idle             # no CPU used at all gives no figure, as before


def test_containers_do_not_share_cpu_history(monkeypatch):
    readings = {"a": iter([_one_shot(100, 1000), _one_shot(200, 1400)]), "b": iter([_one_shot(5000, 1000), _one_shot(5040, 1400)])}
    monkeypatch.setattr(collector, "_docker_request", lambda socket_path, endpoint: next(readings[endpoint.split("/")[2]]))

    for name in ("a", "b"):
        collector._container_usage("/sock", name)

    assert collector._container_usage("/sock", "a")["cpu_pct"] == 100.0
    assert collector._container_usage("/sock", "b")["cpu_pct"] == 40.0


def test_a_counter_that_goes_backwards_skips_that_sample_and_recovers(monkeypatch):
    readings = iter([_one_shot(900, 5000), _one_shot(50, 5400), _one_shot(150, 5800)])
    monkeypatch.setattr(collector, "_docker_request", lambda socket_path, endpoint: next(readings))

    collector._container_usage("/sock", "abc")
    restarted = collector._container_usage("/sock", "abc")
    after = collector._container_usage("/sock", "abc")

    assert "cpu_pct" not in restarted        # the container restarted
    assert after["cpu_pct"] == 100.0         # measured from the new baseline


def test_the_normal_request_is_used_when_one_shot_is_refused_and_remembered(monkeypatch):
    classic = {"cpu_stats": {"cpu_usage": {"total_usage": 300}, "system_cpu_usage": 1000, "online_cpus": 4}, "precpu_stats": {"cpu_usage": {"total_usage": 100}, "system_cpu_usage": 600}}
    asked = []

    def fake(socket_path, endpoint):
        asked.append(endpoint)
        if "one-shot" in endpoint:
            raise OSError("Docker API returned HTTP 403")
        return classic

    monkeypatch.setattr(collector, "_docker_request", fake)

    assert collector._container_usage("/sock", "abc")["cpu_pct"] == 200.0
    assert collector._one_shot_unsupported
    asked.clear()
    assert collector._container_usage("/sock", "abc")["cpu_pct"] == 200.0
    assert asked == ["/containers/abc/stats?stream=false"]   # no wasted one-shot attempt next time


def test_a_container_that_vanished_does_not_make_one_shot_look_unsupported(monkeypatch):
    def gone(socket_path, endpoint):
        raise OSError("Docker API returned HTTP 404")

    monkeypatch.setattr(collector, "_docker_request", gone)

    assert collector._container_usage("/sock", "abc") == {}
    assert not collector._one_shot_unsupported


@pytest.mark.parametrize("bad", ["12", None, True, [1], {"a": 1}, float("nan")])
def test_hostile_cpu_counters_give_no_figure_and_no_crash(monkeypatch, bad):
    readings = iter([_one_shot(100, 1000), _one_shot(bad, 1400), _one_shot(200, bad), _one_shot(300, 1800)])
    monkeypatch.setattr(collector, "_docker_request", lambda socket_path, endpoint: next(readings))

    results = [collector._container_usage("/sock", "abc") for _ in range(3)]

    assert all("cpu_pct" not in usage for usage in results[:2])


def test_cpu_state_for_containers_that_are_gone_is_dropped(monkeypatch):
    collector._cpu_previous.update({"gone" + "0" * 8: (1, 1), "kept": (2, 2)})

    def fake(socket_path, endpoint):
        if endpoint.startswith("/containers/json"):
            return [{"Id": "kept", "Names": ["/kept"], "Image": "x", "State": "running"}]
        if endpoint.endswith("/json"):
            return {"Id": "kept", "Name": "/kept", "State": {"Status": "running"}, "Config": {"Image": "x"}}
        return _one_shot(10, 100)

    monkeypatch.setattr(collector, "_docker_request", fake)
    monkeypatch.setattr(collector.Path, "exists", lambda self: True)

    collector.collect_containers("/sock")

    assert list(collector._cpu_previous) == ["kept"]


def test_format_ports_lists_published_bindings():
    ports = {"80/tcp": [{"HostIp": "0.0.0.0", "HostPort": "8080"}, {"HostIp": "::", "HostPort": "8080"}], "443/tcp": None}
    assert collector._format_ports(ports) == "8080->80/tcp"