Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.
Python ai-agentsaiopsdashboarddockerfastapihomelabincident-managementllmmonitoringopenaireactroot-cause-analysisself-hosteduptime-monitoring
import math
import os
import socket
from datetime import datetime, timedelta, timezone
os.environ.setdefault("DATABASE_URL", "sqlite://")
os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0")
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from src.collector import collector
from src.backend.routes import monitoring as monitoring_routes
from src.backend import alerts, main, rules
from src.backend.models import Base, ContainerMetric, HostMetric, IncidentAlert, Server, SystemEvent, User
TOKEN = "collector-token"
IMAGE_A, IMAGE_B = "sha256:" + "a" * 64, "sha256:" + "b" * 64
@pytest.fixture
def db():
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
session = sessionmaker(bind=engine)()
session.add(User(id=1, name="Owner", email="[email protected]", password_hash="x", role="admin"))
session.add(Server(id=5, owner_id=1, hostname="nas", check_interval_seconds=60, collector_token_hash=main._hash_secret(TOKEN)))
session.commit()
return session
def _ingest(db, containers=(), **host):
monitoring_routes.ingest_metrics(main.MetricsIngestRequest(server_id=5, host=host, containers=list(containers)), authorization=f"Bearer {TOKEN}", db=db)
def _c(name, status="running", **extra):
return {"container_id": f"id-{name}", "name": name, "status": status, **extra}
def _stamp(minutes_ago):
return (datetime.now(timezone.utc) - timedelta(minutes=minutes_ago)).strftime("%Y-%m-%dT%H:%M:%SZ")
def _active(db):
return {alert.rule_key: alert for alert in db.query(IncidentAlert).filter_by(status="active")}
def _events(db, *kinds):
return [(event.kind, event.detail, event.event_key) for event in db.query(SystemEvent).filter(SystemEvent.kind.in_(kinds or rules.CONTAINER_EVENT_KINDS)).order_by(SystemEvent.id)]
# --- collector: Docker status (#22) -----------------------------------------------------------------
def _clear_docker_env(monkeypatch, tmp_path):
monkeypatch.delenv("INFRADECK_DOCKER_HOST", raising=False)
monkeypatch.setenv("DOCKER_SOCKET_PATH", str(tmp_path / "no-such-docker.sock"))
def test_a_host_without_docker_is_not_configured_not_broken(monkeypatch, tmp_path):
_clear_docker_env(monkeypatch, tmp_path)
assert collector.collect_docker() == ([], {"docker_status": "not_configured", "docker_reason": None, "docker_latency_ms": None})
def test_a_configured_but_missing_proxy_socket_is_unreachable(monkeypatch, tmp_path):
monkeypatch.setenv("INFRADECK_DOCKER_HOST", str(tmp_path / "proxy.sock"))
_containers, status = collector.collect_docker()
assert (status["docker_status"], status["docker_reason"]) == ("unreachable", "missing")
def test_a_stale_socket_nobody_listens_on_is_reported_as_refused(monkeypatch, tmp_path):
path = str(tmp_path / "stale.sock")
listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
listener.bind(path)
listener.close() # leaves the socket file behind, like a stopped proxy
monkeypatch.setenv("INFRADECK_DOCKER_HOST", path)
containers, status = collector.collect_docker()
assert containers == [] and (status["docker_status"], status["docker_reason"]) == ("unreachable", "refused")
@pytest.mark.parametrize("error, reason", [
(TimeoutError(), "timeout"), (ConnectionRefusedError(), "refused"), (FileNotFoundError(), "missing"), (PermissionError(), "denied"),
(OSError("Docker API returned HTTP 502"), "error"), (ValueError("bad json"), "error"), (RuntimeError("secret detail"), "error"),
])
def test_failures_map_to_a_fixed_vocabulary_never_raw_text(error, reason):
assert collector._docker_failure_reason(error) == reason
def test_a_working_daemon_reports_ok_with_its_latency(monkeypatch, tmp_path):
socket_path = tmp_path / "docker.sock"
socket_path.touch()
cid = "c" * 64
answers = {"/containers/json?all=1": [{"Id": cid, "Names": ["/web"]}], f"/containers/{cid}/json": {"Name": "/web", "State": {"Status": "running"}, "Config": {"Image": "nginx"}}}
monkeypatch.setattr(collector, "_docker_request", lambda path, endpoint: answers.get(endpoint, {}))
monkeypatch.setenv("INFRADECK_DOCKER_HOST", str(socket_path))
containers, status = collector.collect_docker()
assert [c["name"] for c in containers] == ["web"]
assert status["docker_status"] == "ok" and isinstance(status["docker_latency_ms"], int) and status["docker_latency_ms"] >= 0
def test_a_daemon_error_during_listing_is_unreachable(monkeypatch, tmp_path):
socket_path = tmp_path / "docker.sock"
socket_path.touch()
monkeypatch.setattr(collector, "_docker_request", lambda path, endpoint: (_ for _ in ()).throw(OSError("Docker API returned HTTP 502")))
monkeypatch.setenv("INFRADECK_DOCKER_HOST", str(socket_path))
containers, status = collector.collect_docker()
assert containers == [] and (status["docker_status"], status["docker_reason"]) == ("unreachable", "error")
def test_one_container_vanishing_mid_collection_does_not_fail_the_rest(monkeypatch, tmp_path):
socket_path = tmp_path / "docker.sock"
socket_path.touch()
a, b = "a" * 64, "b" * 64
def request(path, endpoint):
if endpoint == "/containers/json?all=1":
return [{"Id": a, "Names": ["/gone"]}, {"Id": b, "Names": ["/web"]}]
if endpoint == f"/containers/{a}/json":
raise OSError("Docker API returned HTTP 502") # removed between listing and inspecting
return {"Name": "/web", "State": {"Status": "exited"}}
monkeypatch.setattr(collector, "_docker_request", request)
monkeypatch.setenv("INFRADECK_DOCKER_HOST", str(socket_path))
containers, status = collector.collect_docker()
assert [c["name"] for c in containers] == ["web"] and status["docker_status"] == "ok"
def test_when_nothing_can_be_inspected_the_daemon_is_failing_not_empty(monkeypatch, tmp_path):
socket_path = tmp_path / "docker.sock"
socket_path.touch()
def request(path, endpoint):
if endpoint == "/containers/json?all=1":
return [{"Id": "a" * 64, "Names": ["/web"]}]
raise OSError("Docker API returned HTTP 502")
monkeypatch.setattr(collector, "_docker_request", request)
monkeypatch.setenv("INFRADECK_DOCKER_HOST", str(socket_path))
containers, status = collector.collect_docker()
assert containers == [] and status["docker_status"] == "unreachable"
# --- collector: exit status and image ID (#19, #21) -------------------------------------------------
def _collect_one(monkeypatch, tmp_path, **state_and_detail):
socket_path = tmp_path / "docker.sock"
socket_path.touch()
cid = "d" * 64
detail = {"Name": "/job", "Config": {"Image": "busybox"}, "State": {"Status": "exited"}, **state_and_detail}
monkeypatch.setattr(collector, "_docker_request", lambda path, endpoint: [{"Id": cid, "Names": ["/job"]}] if "json?all" in endpoint else detail)
return collector.collect_containers(str(socket_path))[0]
def test_a_stopped_container_reports_how_it_ended(monkeypatch, tmp_path):
container = _collect_one(monkeypatch, tmp_path, Image=IMAGE_A, State={"Status": "exited", "ExitCode": 137, "OOMKilled": True, "FinishedAt": "2026-10-04T10:11:12.123456789Z"})
assert (container["exit_code"], container["oom_killed"], container["finished_at"], container["image_id"]) == (137, True, "2026-10-04T10:11:12Z", IMAGE_A)
def test_a_running_container_does_not_report_a_stale_exit_status(monkeypatch, tmp_path):
container = _collect_one(monkeypatch, tmp_path, State={"Status": "running", "ExitCode": 1, "OOMKilled": True, "FinishedAt": "2026-10-04T10:11:12Z"})
assert (container["exit_code"], container["oom_killed"], container["finished_at"]) == (None, None, None)
@pytest.mark.parametrize("state", [
{"ExitCode": "1"}, {"ExitCode": True}, {"ExitCode": 99999999}, {"ExitCode": -5}, {"ExitCode": 1.5}, {"ExitCode": None},
{"ExitCode": 1, "OOMKilled": "yes"}, {"ExitCode": 1, "FinishedAt": "0001-01-01T00:00:00Z"}, {"ExitCode": 1, "FinishedAt": "garbage"}, {"ExitCode": 1, "FinishedAt": 12345},
])
def test_malformed_exit_details_become_unknown(monkeypatch, tmp_path, state):
container = _collect_one(monkeypatch, tmp_path, State={"Status": "exited", **state})
assert container["exit_code"] in (None, 1) and container["oom_killed"] is None and container["finished_at"] is None
@pytest.mark.parametrize("image_id", ["sha256:abc", "nginx:latest", "sha256:" + "G" * 64, "sha256:" + "a" * 65, "", None, 5, ["x"]])
def test_only_a_well_formed_image_id_is_passed_on(monkeypatch, tmp_path, image_id):
assert _collect_one(monkeypatch, tmp_path, Image=image_id)["image_id"] is None
# --- backend: Docker health rules (#22) -------------------------------------------------------------
def test_docker_unreachable_alerts_only_after_it_worked_and_only_when_sustained(db):
_ingest(db, docker_status="ok", docker_latency_ms=20)
_ingest(db, docker_status="unreachable", docker_reason="refused")
_ingest(db, docker_status="unreachable", docker_reason="refused")
assert "system:docker_unreachable" not in _active(db)
_ingest(db, docker_status="unreachable", docker_reason="refused")
alert = _active(db)["system:docker_unreachable"]
assert alert.severity == alerts.CRITICAL and "refused" in alert.description
_ingest(db, docker_status="ok", docker_latency_ms=15)
assert "system:docker_unreachable" not in _active(db)
def test_a_host_that_never_had_working_docker_never_alerts(db):
for _ in range(5):
_ingest(db, docker_status="unreachable", docker_reason="missing")
for _ in range(5):
_ingest(db, docker_status="not_configured")
assert not [key for key in _active(db) if "docker" in key]
def test_an_open_docker_alert_survives_the_purge_of_the_samples_that_proved_it_worked(db):
_ingest(db, docker_status="ok", docker_latency_ms=10)
for _ in range(3):
_ingest(db, docker_status="unreachable", docker_reason="timeout")
assert "system:docker_unreachable" in _active(db)
db.query(HostMetric).filter(HostMetric.docker_status == "ok").delete()
db.commit()
_ingest(db, docker_status="unreachable", docker_reason="timeout")
assert "system:docker_unreachable" in _active(db)
def test_slow_docker_warns_when_sustained_and_clears_when_fast(db):
for _ in range(2):
_ingest(db, docker_status="ok", docker_latency_ms=9000)
assert "system:docker_slow" not in _active(db)
_ingest(db, docker_status="ok", docker_latency_ms=9000)
assert _active(db)["system:docker_slow"].severity == alerts.WARNING
_ingest(db, docker_status="ok", docker_latency_ms=30)
assert "system:docker_slow" not in _active(db)
def test_missing_docker_data_from_old_collectors_never_alerts(db):
for _ in range(5):
_ingest(db, [_c("web")])
assert not [key for key in _active(db) if "docker" in key]
def test_malformed_docker_fields_are_dropped(db):
from src.backend.models import HostMetric as Host
_ingest(db, docker_status="<script>", docker_reason="rm -rf /", docker_latency_ms="fast")
_ingest(db, docker_status="ok", docker_reason="timeout", docker_latency_ms=10**9)
rows = db.query(Host).order_by(Host.id).all()
assert [(r.docker_status, r.docker_reason, r.docker_latency_ms) for r in rows] == [(None, None, None), ("ok", "timeout", None)]
# --- backend: exit codes (#19) ----------------------------------------------------------------------
@pytest.mark.parametrize("code, oom, severity, words", [
(1, False, alerts.WARNING, "exited with an error"),
(127, False, alerts.WARNING, "command not found"),
(137, False, alerts.INFO, "stopped by a signal"),
(143, False, alerts.INFO, "stopped by a signal"),
(137, True, alerts.CRITICAL, "running out of memory"),
])
def test_stopped_containers_are_classified_by_how_they_ended(db, code, oom, severity, words):
_ingest(db, [_c("job", "exited", exit_code=code, oom_killed=oom, finished_at=_stamp(5))])
alert = _active(db)["container:job:exit"]
assert alert.severity == severity and words in (alert.title + alert.description)
@pytest.mark.parametrize("container", [
_c("job", "exited", exit_code=0, oom_killed=False, finished_at="NOW"),
_c("job", "exited", exit_code=1, finished_at=None),
_c("job", "exited", exit_code=None, finished_at="NOW"),
_c("job", "running", exit_code=1, finished_at="NOW"),
_c("job", "created", exit_code=1, finished_at="NOW"),
])
def test_clean_exits_unknown_and_running_containers_never_alert(db, container):
if container.get("finished_at") == "NOW":
container = {**container, "finished_at": _stamp(5)}
_ingest(db, [container])
assert "container:job:exit" not in _active(db)
def test_an_old_exit_is_not_an_incident_and_a_restart_clears_it(db):
_ingest(db, [_c("old", "exited", exit_code=1, finished_at=_stamp(60 * 5))])
assert "container:old:exit" not in _active(db)
_ingest(db, [_c("job", "exited", exit_code=1, finished_at=_stamp(5))])
assert "container:job:exit" in _active(db)
_ingest(db, [_c("job", "running")])
assert "container:job:exit" not in _active(db)
# --- backend: what a collector may send about containers --------------------------------------------
def _clean(raw):
return main._clean_container(raw, datetime.now(timezone.utc))
@pytest.mark.parametrize("raw", [None, 5, "x", [], {}, {"container_id": "a"}, {"status": "running"}, {"container_id": 5, "status": "running"}, {"container_id": "a", "status": 5}, {"container_id": "", "status": "running"}, {"container_id": "a", "status": " "}])
def test_unusable_container_records_are_dropped(raw):
assert _clean(raw) is None
def test_container_text_is_made_printable_and_bounded():
clean = _clean({"container_id": "id\x00" + "x" * 500, "status": "RUNNING\n", "name": "web\x1b[31m" + "n" * 600, "image": "i" * 600, "ports": "p" * 2000, "healthcheck_status": "h" * 200, "started_at": "s" * 200})
assert len(clean["container_id"]) == 128 and len(clean["name"]) == main.MAX_CONTAINER_NAME and len(clean["image"]) == 255 and len(clean["ports"]) == 512
assert clean["status"] == "running" and "\x1b" not in clean["name"] and "\x00" not in clean["container_id"]
assert len(clean["healthcheck_status"]) == 64 and len(clean["started_at"]) == 64
def test_a_missing_name_falls_back_to_the_container_id():
assert _clean({"container_id": "abcdef0123456789", "status": "running"})["name"] == "abcdef012345"
@pytest.mark.parametrize("bad", ["12", True, -1, math.nan, math.inf, 1e13, None, [1], {"a": 1}])
def test_numeric_fields_must_be_real_in_range_numbers(bad):
clean = _clean({"container_id": "a", "status": "running", "cpu_pct": bad, "memory_used_mb": bad, "pids": bad, "restart_count": bad, "exit_code": bad})
assert clean["cpu_pct"] is None and clean["memory_used_mb"] is None and clean["pids"] is None and clean["restart_count"] == 0
assert clean["exit_code"] is None or clean["exit_code"] == -1
def test_sane_numbers_pass_through():
clean = _clean({"container_id": "a", "status": "exited", "cpu_pct": 12.5, "memory_pct": 3, "pids": 7, "restart_count": 2, "exit_code": 137, "oom_killed": True, "image_id": IMAGE_A})
assert (clean["cpu_pct"], clean["memory_pct"], clean["pids"], clean["restart_count"], clean["exit_code"], clean["oom_killed"], clean["image_id"]) == (12.5, 3.0, 7, 2, 137, True, IMAGE_A)
@pytest.mark.parametrize("value", ["garbage", "2026-10-04", "2026-10-04T10:11:12", "2026-10-04T10:11:12.5Z", "1999-12-31T23:59:59Z", "2999-01-01T00:00:00Z", 5, None, ["x"]])
def test_finish_times_must_be_exact_recent_utc_timestamps(value):
assert _clean({"container_id": "a", "status": "exited", "finished_at": value})["finished_at"] is None
def test_a_valid_finish_time_becomes_an_aware_datetime():
finished = _clean({"container_id": "a", "status": "exited", "finished_at": "2026-10-04T10:11:12Z"})["finished_at"]
assert finished == datetime(2026, 10, 4, 10, 11, 12, tzinfo=timezone.utc)
def test_unknown_fields_are_not_passed_to_the_database():
clean = _clean({"container_id": "a", "status": "running", "id": 99, "server_id": 7, "time": "now", "evil": "x"})
assert not {"id", "server_id", "time", "evil"} & clean.keys()
def test_the_api_schema_already_refuses_container_entries_that_are_not_objects():
from pydantic import ValidationError
for junk in ([None], [5], ["x"], [[1]]):
with pytest.raises(ValidationError):
main.MetricsIngestRequest(server_id=5, containers=junk)
def test_hostile_container_records_never_break_ingest_or_other_data(db):
hostile = [{"container_id": "a" * 1000, "status": "running", "cpu_pct": "NaN", "exit_code": "1", "finished_at": {"x": 1}},
{"container_id": "ok", "status": "running", "name": "fine", "memory_used_mb": float("inf")},
{"container_id": "dropped", "status": 5}]
_ingest(db, hostile, cpu_utilization_pct=12)
stored = db.query(ContainerMetric).order_by(ContainerMetric.id).all()
# Unusable records vanish; the odd ones are kept with unusable values turned into "unknown".
assert [c.name for c in stored] == ["a" * 12, "fine"]
assert stored[0].cpu_pct is None and stored[0].exit_code is None and stored[0].finished_at is None and stored[1].memory_used_mb is None
assert db.query(HostMetric).one().cpu_utilization_pct == 12 # the host's own data was still recorded
def test_no_more_than_500_containers_are_accepted_per_check_in(db):
_ingest(db, [_c(f"c{number}") for number in range(600)])
assert db.query(ContainerMetric).count() == main.MAX_CONTAINERS_PER_CHECKIN
# --- backend: inventory changes and image drift (#16, #21) ------------------------------------------
def test_the_first_check_in_is_only_a_baseline(db):
_ingest(db, [_c("web", image_id=IMAGE_A), _c("db")])
assert _events(db) == []
def test_new_and_missing_containers_are_recorded(db):
_ingest(db, [_c("web"), _c("db")])
_ingest(db, [_c("web"), _c("cache", image="redis:7")])
assert _events(db) == [("container_added", "Container cache appeared (redis:7)", "cache"), ("container_removed", "Container db is no longer present", "db")]
def test_a_removed_container_opens_an_informational_incident_that_clears_with_the_window(db):
_ingest(db, [_c("web"), _c("db")])
_ingest(db, [_c("web")])
assert _active(db)["container:db:removed"].severity == alerts.INFO
db.query(SystemEvent).update({"time": datetime.now(timezone.utc) - timedelta(hours=3)})
db.commit()
_ingest(db, [_c("web")])
assert "container:db:removed" not in _active(db)
def test_a_container_that_comes_back_clears_its_removal_incident(db):
_ingest(db, [_c("web"), _c("db")])
_ingest(db, [_c("web")])
_ingest(db, [_c("web"), _c("db")])
assert "container:db:removed" not in _active(db)
def test_containers_are_matched_by_name_so_recreation_is_not_a_change(db):
_ingest(db, [{"container_id": "old-id", "name": "web", "status": "running", "image_id": IMAGE_A}])
_ingest(db, [{"container_id": "new-id", "name": "web", "status": "running", "image_id": IMAGE_A}])
assert _events(db) == []
def test_an_empty_list_never_looks_like_everything_vanished_or_reappeared(db):
_ingest(db, [_c("web"), _c("db")])
_ingest(db, [], docker_status="unreachable", docker_reason="refused")
_ingest(db, [_c("web"), _c("db")])
assert _events(db) == []
def test_a_changed_image_is_recorded_and_raises_an_informational_incident(db):
_ingest(db, [_c("web", image_id=IMAGE_A)])
_ingest(db, [_c("web", image_id=IMAGE_B)])
[(kind, detail, name)] = _events(db)
assert (kind, name) == ("image_changed", "web") and "aaaaaaaaaaaa to bbbbbbbbbbbb" in detail
assert _active(db)["container:web:image"].severity == alerts.INFO
def test_the_same_image_or_an_unknown_one_is_not_a_change(db):
_ingest(db, [_c("same", image_id=IMAGE_A), _c("unknown")])
_ingest(db, [_c("same", image_id=IMAGE_A), _c("unknown", image_id=IMAGE_B)])
_ingest(db, [_c("same", image_id=IMAGE_A), _c("unknown", image_id=None)])
assert _events(db) == []
def test_a_burst_of_changes_is_capped_per_check_in_with_one_notice(db):
_ingest(db, [_c("web")])
_ingest(db, [_c("web")] + [_c(f"ci-{number}") for number in range(25)])
kinds = [kind for kind, _detail, _name in _events(db)]
assert kinds.count("container_added") == rules.MAX_CONTAINER_EVENTS_PER_CHECKIN and kinds.count("container_notice") == 1
def test_sustained_churn_is_capped_per_hour(db):
_ingest(db, [_c("web")])
for round_number in range(10):
_ingest(db, [_c("web")] + [_c(f"ci-{round_number}-{number}") for number in range(20)])
kinds = [kind for kind, _detail, _name in _events(db)]
assert len(kinds) <= rules.MAX_CONTAINER_EVENTS_PER_HOUR + 1 and kinds.count("container_notice") == 1
def test_container_changes_have_their_own_list_and_do_not_crowd_the_system_row(db, monkeypatch):
monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1})
db.add(SystemEvent(server_id=5, time=datetime.now(timezone.utc) - timedelta(days=2), kind="reboot", detail="The host rebooted"))
db.commit()
_ingest(db, [_c("web")])
_ingest(db, [_c("web")] + [_c(f"ci-{number}") for number in range(8)])
server = monitoring_routes.monitoring_overview(session_id="s", db=db)["servers"][0]
assert [event["kind"] for event in server["system_events"]] == ["reboot"]
assert len(server["container_events"]) == 8 and {event["kind"] for event in server["container_events"]} == {"container_added"}
def test_the_container_change_list_is_bounded(db, monkeypatch):
monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1})
for number in range(40):
db.add(SystemEvent(server_id=5, time=datetime.now(timezone.utc) - timedelta(minutes=number), kind="container_added", detail=f"Container c{number} appeared", event_key=f"c{number}"))
db.commit()
assert len(monitoring_routes.monitoring_overview(session_id="s", db=db)["servers"][0]["container_events"]) == main.CONTAINER_EVENTS_SHOWN
def test_container_events_are_kept_for_the_history_retention(db):
now = datetime.now(timezone.utc)
db.add_all([SystemEvent(server_id=5, time=now - timedelta(days=days), kind=kind, detail="x", event_key="x") for kind in rules.CONTAINER_EVENT_KINDS for days in (40, 400)])
db.commit()
assert main.retention.purge(db) == len(rules.CONTAINER_EVENT_KINDS) # the ones from 400 days ago; a year is the default
assert db.query(SystemEvent).count() == len(rules.CONTAINER_EVENT_KINDS)
# --- backend: memory leak trends (#20) --------------------------------------------------------------
def _seed_memory(db, name="app", hours=12, per_hour=6, value=lambda hour: 200 + 20 * hour, restart=0, container_id="cid", limit=None, status="running", skip_hours=()):
"""Samples spread over the window, ending a few minutes before now, plus the check-in that makes them current."""
now = datetime.now(timezone.utc)
since = now - timedelta(hours=hours)
last = None
for hour in range(hours):
if hour in skip_hours:
continue
for k in range(per_hour):
last = since + timedelta(minutes=hour * 60 + k * 10 + 5)
db.add(ContainerMetric(server_id=5, container_id=container_id, name=name, status=status, restart_count=restart, memory_used_mb=float(value(hour + k / per_hour)), memory_limit_mb=limit, time=last))
db.add(HostMetric(server_id=5, time=last))
db.commit()
return now
def _trend(db, now=None):
main.analyze_memory_trends(db, now=now)
return {key: alert for key, alert in _active(db).items() if key.startswith("trend:")}
def test_steady_growth_is_flagged_as_a_possible_leak(db):
now = _seed_memory(db, limit=2048.0)
alert = _trend(db, now)["trend:app:memory"]
assert alert.severity == alerts.WARNING and "possible" not in alert.title.lower() and "memory leak" in alert.title
assert "20.0 MB per hour" in alert.description and "2048 MB limit" in alert.description
def test_flat_noisy_and_sawtooth_memory_is_not_a_leak(db):
_seed_memory(db, name="flat", value=lambda hour: 300)
_seed_memory(db, name="sawtooth", value=lambda hour: 200 + 150 * (int(hour) % 3))
now = _seed_memory(db, name="noisy", value=lambda hour: 300 + (40 if int(hour * 6) % 2 else -40))
assert _trend(db, now) == {}
def test_a_restart_or_recreation_in_the_window_resets_the_trend(db):
_seed_memory(db, name="recreated", container_id="one")
db.query(ContainerMetric).filter(ContainerMetric.name == "recreated", ContainerMetric.memory_used_mb > 300).update({"container_id": "two"})
_seed_memory(db, name="restarted")
db.query(ContainerMetric).filter(ContainerMetric.name == "restarted", ContainerMetric.memory_used_mb > 300).update({"restart_count": 1})
db.commit()
assert _trend(db) == {}
def test_too_little_history_or_a_slow_or_proportionally_small_rise_is_not_flagged(db):
_seed_memory(db, name="gappy", skip_hours=(1, 2, 3, 4, 5))
_seed_memory(db, name="slow", value=lambda hour: 200 + 2 * hour)
now = _seed_memory(db, name="huge", value=lambda hour: 5000 + 12 * hour) # 12 MB/h but only ~2.6% over the window
assert _trend(db, now) == {}
def test_containers_that_are_not_running_now_are_not_analysed(db):
now = _seed_memory(db, status="exited")
assert _trend(db, now) == {}
def test_the_flag_clears_when_the_growth_stops(db):
now = _seed_memory(db)
assert "trend:app:memory" in _trend(db, now)
db.query(ContainerMetric).delete()
db.query(HostMetric).delete()
db.commit()
now = _seed_memory(db, value=lambda hour: 300)
assert _trend(db, now) == {}
def test_analysis_is_bounded_to_100_containers_per_server(db):
now = datetime.now(timezone.utc)
since = now - timedelta(hours=12)
last = None
for number in range(120):
for hour in range(12):
for k in range(6):
last = since + timedelta(minutes=hour * 60 + k * 10 + 5)
db.add(ContainerMetric(server_id=5, container_id=f"id{number}", name=f"app{number:03d}", status="running", restart_count=0, memory_used_mb=float(200 + 20 * (hour + k / 6)), time=last))
db.add(HostMetric(server_id=5, time=last))
db.commit()
assert len(_trend(db, now)) == rules.MAX_TREND_CONTAINERS
def test_a_fit_needs_distinct_points():
assert rules._linear_fit([(1.0, 5.0), (1.0, 6.0)]) is None
slope, intercept, r_squared = rules._linear_fit([(0.0, 10.0), (1.0, 20.0), (2.0, 30.0)])
assert (round(slope, 6), round(intercept, 6), round(r_squared, 6)) == (10.0, 10.0, 1.0)
def test_the_new_thresholds_have_safe_defaults_and_are_validated():
defaults = rules.DEFAULT_THRESHOLDS
assert (defaults["docker_slow_ms"], defaults["leak_window_hours"], defaults["leak_min_mb_per_hour"], defaults["leak_min_growth_pct"]) == (2000, 12, 10, 20)
for bad in ({"docker_slow_ms": 50}, {"docker_slow_ms": 70000}, {"docker_slow_ms": 1500.5}, {"leak_window_hours": 5}, {"leak_window_hours": 73}, {"leak_window_hours": 12.5},
{"leak_min_mb_per_hour": 0}, {"leak_min_mb_per_hour": 10001}, {"leak_min_growth_pct": 0}, {"leak_min_growth_pct": 101}, {"leak_min_growth_pct": "20"}, {"leak_window_hours": True}):
with pytest.raises(ValueError):
rules.validate_thresholds(bad, defaults)
ok = rules.validate_thresholds({"docker_slow_ms": 5000, "leak_window_hours": 24, "leak_min_mb_per_hour": 2.5, "leak_min_growth_pct": 50}, defaults)
assert (ok["docker_slow_ms"], ok["leak_window_hours"], ok["leak_min_mb_per_hour"], ok["leak_min_growth_pct"]) == (5000, 24, 2.5, 50)
def test_the_longest_container_name_still_fits_every_incident_key(db):
name = "n" * 600
_ingest(db, [_c(name, "exited", exit_code=1, finished_at=_stamp(5), restart_count=0, healthcheck_status="unhealthy")])
keys = list(_active(db))
assert keys and max(len(key) for key in keys) < 255
# --- collector keeping up with its check interval (#44) ---------------------------------------------
def test_a_collector_too_slow_for_its_interval_warns_when_sustained_and_clears(db):
for _ in range(2):
_ingest(db, collect_duration_ms=55_000) # 60 s interval, limit 80% = 48 s
assert "system:collect_slow" not in _active(db)
_ingest(db, collect_duration_ms=55_000)
alert = _active(db)["system:collect_slow"]
assert alert.severity == alerts.WARNING and "55.0 s" in alert.description and "60 s" in alert.description
_ingest(db, collect_duration_ms=900)
assert "system:collect_slow" not in _active(db)
def test_collection_within_the_limit_never_warns(db):
for _ in range(6):
_ingest(db, collect_duration_ms=47_999)
assert "system:collect_slow" not in _active(db)
def test_one_slow_check_in_among_fast_ones_does_not_warn(db):
for duration in (900, 55_000, 900, 55_000, 900):
_ingest(db, collect_duration_ms=duration)
assert "system:collect_slow" not in _active(db)
def test_old_collectors_that_do_not_send_the_duration_never_trigger_it(db):
for _ in range(6):
_ingest(db)
assert "system:collect_slow" not in _active(db)
for duration in (55_000, None, 55_000):
_ingest(db, collect_duration_ms=duration)
assert "system:collect_slow" not in _active(db)
def test_the_limit_follows_the_servers_check_interval(db):
db.get(Server, 5).check_interval_seconds = 15
db.commit()
for _ in range(3):
_ingest(db, collect_duration_ms=13_000) # fine at 60 s, too slow at 15 s
assert "system:collect_slow" in _active(db)
def test_the_limit_is_an_editable_validated_threshold():
assert rules.DEFAULT_THRESHOLDS["collect_slow_pct"] == 80
for bad in (0, 101, -5, "80", True, None):
with pytest.raises(ValueError):
rules.validate_thresholds({"collect_slow_pct": bad}, rules.DEFAULT_THRESHOLDS)
assert rules.validate_thresholds({"collect_slow_pct": 60}, rules.DEFAULT_THRESHOLDS)["collect_slow_pct"] == 60
@pytest.mark.parametrize("bad", ["12", -1, 10**12, True, 1.5, None, [1], float("nan")])
def test_a_hostile_collection_time_is_dropped_not_stored(db, bad):
_ingest(db, collect_duration_ms=bad)
assert db.query(HostMetric).order_by(HostMetric.id.desc()).first().collect_duration_ms is None
def test_a_sane_collection_time_is_stored(db):
_ingest(db, collect_duration_ms=1234)
assert db.query(HostMetric).order_by(HostMetric.id.desc()).first().collect_duration_ms == 1234