Skip to content

Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.

15.2 KB
Raw Blame
import http.client
import json
import socket
import socketserver
import threading
import time
from http.server import BaseHTTPRequestHandler

import pytest

from src.collector import collector, docker_proxy as proxy

CID = "a" * 64
SECRET = "SUPER_SECRET_TOKEN_123"

INSPECT = {
    "Id": CID, "Name": "/web", "RestartCount": 2, "Image": "sha256:" + "ab" * 32,
    "Config": {"Image": "nginx:1", "Env": [f"DB_PASSWORD={SECRET}"], "Labels": {"k": "v", "secret": SECRET, "com.docker.compose.project": "shop"}, "Cmd": ["nginx"]},
    "State": {"Status": "running", "StartedAt": "2026-01-01T00:00:00Z", "Pid": 99, "ExitCode": 137, "OOMKilled": True, "FinishedAt": "2026-01-02T03:04:05.123456789Z", "Error": f"failed: {SECRET}", "Health": {"Status": "healthy", "Log": [{"Output": SECRET}]}},
    "NetworkSettings": {"Ports": {"80/tcp": [{"HostIp": "0.0.0.0", "HostPort": "8080"}]}, "SecondaryIPAddresses": None, "Networks": {"x": {"Gateway": "1"}}},
    "Mounts": [{"Type": "bind", "Source": "/srv/web", "Destination": "/data", "Mode": SECRET, "RW": True}, {"Type": "volume", "Name": "web_cache", "Source": "/var/lib/docker/volumes/web_cache/_data", "Destination": "/cache", "Driver": "local"}, "junk"], "HostConfig": {"Privileged": True}, "GraphDriver": {"Data": {}},
}
LISTING = [{"Id": CID, "Names": ["/web"], "Image": "nginx:1", "State": "running", "Labels": {"secret": SECRET}, "Mounts": []}]
STATS = {"cpu_stats": {"cpu_usage": {"total_usage": 10}, "system_cpu_usage": 100, "online_cpus": 2}, "precpu_stats": {"cpu_usage": {"total_usage": 5}, "system_cpu_usage": 50},
         "memory_stats": {"usage": 1048576, "limit": 2097152, "stats": {"inactive_file": 0}}, "networks": {"eth0": {"rx_bytes": 1, "tx_bytes": 2}},
         "blkio_stats": {"io_service_bytes_recursive": []}, "pids_stats": {"current": 3}, "name": "/web", "id": CID}


# What Docker returns for one-shot: the same readings but no earlier one to compare with.
ONE_SHOT_STATS = {**STATS, "precpu_stats": {}}


@pytest.fixture(autouse=True)
def _fresh_collector_state():
    collector._cpu_previous.clear()
    collector._one_shot_unsupported = False
    yield
    collector._cpu_previous.clear()
    collector._one_shot_unsupported = False


# --- routing: the allow-list ---------------------------------------------------------------

@pytest.mark.parametrize("target", [
    "/containers/json?all=1", f"/containers/{CID}/json", f"/containers/{CID}/stats?stream=false", f"/containers/{CID[:12]}/json",
    f"/containers/{CID}/stats?stream=false&one-shot=true", "/version", "/system/df",
])
def test_the_six_read_requests_are_allowed(target):
    assert proxy.route("GET", target) is not None


@pytest.mark.parametrize("method", ["POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS", "get", "CONNECT"])
def test_only_get_is_allowed(method):
    assert proxy.route(method, "/containers/json?all=1") is None


@pytest.mark.parametrize("target", [
    # other endpoints, including the ones that read container contents or start things
    "/containers/json",   # only the full list the collector asks for (?all=1) is allowed
    "/", "/version?x=1", "/v1.43/version", "/info", "/images/json", "/volumes", "/networks", "/system/df?type=image", "/system/info", "/events", "/_ping", "/containers", "/containers/",
    f"/containers/{CID}/export", f"/containers/{CID}/archive?path=/etc/passwd", f"/containers/{CID}/logs", f"/containers/{CID}/top",
    f"/containers/{CID}/attach", f"/containers/{CID}/exec", f"/containers/{CID}/start", f"/containers/{CID}/kill", f"/containers/{CID}/changes",
    "/containers/create", "/containers/prune", "/exec/abc/start", "/build", "/commit", "/swarm", "/secrets", "/configs",
    # query abuse
    "/containers/json?all=1&filters=x", "/containers/json?all=0", "/containers/json?all=1&all=1", "/containers/json?size=1", "/containers/json?",
    f"/containers/{CID}/stats", f"/containers/{CID}/stats?stream=true", f"/containers/{CID}/stats?stream=false&one-shot=1", f"/containers/{CID}/json?size=1",
    # one-shot: exactly one spelling, in exactly one order
    f"/containers/{CID}/stats?stream=false&one-shot=false", f"/containers/{CID}/stats?stream=false&one-shot=", f"/containers/{CID}/stats?stream=false&one-shot",
    f"/containers/{CID}/stats?one-shot=true&stream=false", f"/containers/{CID}/stats?one-shot=true", f"/containers/{CID}/stats?stream=true&one-shot=true",
    f"/containers/{CID}/stats?stream=false&one-shot=true&one-shot=true", f"/containers/{CID}/stats?stream=false&one-shot=true&stream=true",
    f"/containers/{CID}/stats?stream=false&one-shot=True", f"/containers/{CID}/stats?stream=false&one-shot=TRUE", f"/containers/{CID}/stats?stream=false&One-Shot=true",
    f"/containers/{CID}/stats?stream=false&one-shot=true&", f"/containers/{CID}/stats?stream=false&one-shot=true&x=1", f"/containers/{CID}/stats?stream=false&&one-shot=true",
    f"/containers/{CID}/stats?stream=false&one-shot=true#f", f"/containers/{CID}/stats?stream=false&one-shot=true ", f"/containers/{CID}/stats?stream=false&one-shot=%74rue",
    # traversal, encoding, padding
    f"/containers/{CID}/../../images/json", "/containers/../images/json", f"/containers/{CID}/json/..", "//containers/json", "/containers//json",
    "/containers/json/", f"/containers/{CID}/json/", "/containers/%6Ason", "/containers/json%00", f"/containers/{CID}%2fjson", "/%2e%2e/containers/json",
    "/containers/json#frag", "/containers/json ", " /containers/json", "/containers/json\r\nX: y", "/containers/json\n", "/containers/jsón",
    "/v1.43/containers/json", "http://localhost/containers/json", "*",
    # ids that are not plain hex
    "/containers/web/json", "/containers/ABC123ABC123/json", "/containers/abc/json", f"/containers/{'a' * 65}/json", "/containers/zzzzzzzzzzzz/json", "/containers/;rm/json",
])
def test_everything_else_is_refused(target):
    assert proxy.route("GET", target) is None


# --- response filtering --------------------------------------------------------------------

def test_inspect_output_drops_environment_and_everything_unneeded():
    result = proxy.project_inspect(INSPECT)

    assert SECRET not in json.dumps(result)
    assert result == {
        "Id": CID, "Name": "/web", "RestartCount": 2, "Image": "sha256:" + "ab" * 32, "Config": {"Image": "nginx:1", "Labels": {"com.docker.compose.project": "shop"}},
        # Of a mount only where it comes from and where it lands; of the labels only the compose project.
        "Mounts": [{"Type": "bind", "Source": "/srv/web", "Destination": "/data"}, {"Type": "volume", "Name": "web_cache", "Source": "/var/lib/docker/volumes/web_cache/_data", "Destination": "/cache"}],
        "State": {"Status": "running", "StartedAt": "2026-01-01T00:00:00Z", "ExitCode": 137, "OOMKilled": True, "FinishedAt": "2026-01-02T03:04:05.123456789Z", "Health": {"Status": "healthy"}},
        "NetworkSettings": {"Ports": {"80/tcp": [{"HostIp": "0.0.0.0", "HostPort": "8080"}]}},
    }


def test_version_output_keeps_only_the_version_numbers():
    assert proxy.project_version({"Version": "27.3.1", "ApiVersion": "1.47", "KernelVersion": "6.8.0", "Os": "linux", "Components": [{"Details": {"x": SECRET}}]}) == {"Version": "27.3.1", "ApiVersion": "1.47"}
    assert proxy.project_version(None) == {}


def test_disk_usage_is_reduced_to_totals_with_no_names():
    usage = {
        "LayersSize": 9_000, "Images": [
            {"Id": "sha256:a", "RepoTags": [SECRET], "Size": 5_000, "SharedSize": 1_000, "Containers": 2},
            {"Id": "sha256:b", "RepoTags": ["old:1"], "Size": 4_000, "SharedSize": 1_000, "Containers": 0},
            {"Id": "sha256:c", "Size": 500, "SharedSize": 900, "Containers": 0}, "junk",
        ],
        "Containers": [{"Names": [SECRET]}], "Volumes": [{"Name": SECRET, "UsageData": {"Size": 7}}],
        "BuildCache": [{"ID": SECRET, "Size": 300, "InUse": True, "Shared": False}, {"Size": 200, "InUse": False, "Shared": False}, {"Size": 999, "InUse": False, "Shared": True}],
    }
    result = proxy.project_disk_usage(usage)

    assert SECRET not in json.dumps(result)
    assert result == {"images_bytes": 9_000, "images_unused_bytes": 3_000, "images_unused": 2, "images": 3, "build_cache_bytes": 500, "build_cache_unused_bytes": 200}
    assert proxy.project_disk_usage([]) == {} and proxy.project_disk_usage({"Images": "x", "LayersSize": -5})["images_bytes"] == 0


def test_list_and_stats_output_are_cut_down():
    assert SECRET not in json.dumps(proxy.project_list(LISTING))
    assert proxy.project_list(LISTING) == [{"Id": CID, "Names": ["/web"], "Image": "nginx:1", "State": "running"}]
    assert set(proxy.project_stats(STATS)) == {"cpu_stats", "precpu_stats", "memory_stats", "networks", "blkio_stats", "pids_stats"}


@pytest.mark.parametrize("junk", [None, 5, "x", [], {}, [1, "a", None], {"State": "bad", "Config": 3, "NetworkSettings": []}])
def test_unexpected_upstream_shapes_do_not_crash_the_filters(junk):
    proxy.project_list(junk), proxy.project_inspect(junk), proxy.project_stats(junk)


# --- end to end: collector -> proxy -> fake Docker daemon ----------------------------------

class _FakeDocker(BaseHTTPRequestHandler):
    seen: list[str] = []

    def do_GET(self):
        _FakeDocker.seen.append(self.path)
        routes = {"/containers/json?all=1": LISTING, f"/containers/{CID}/json": INSPECT, f"/containers/{CID}/stats?stream=false": STATS,
                  f"/containers/{CID}/stats?stream=false&one-shot=true": ONE_SHOT_STATS}
        body = json.dumps(routes.get(self.path)).encode() if self.path in routes else b"nope"
        self.send_response(200 if self.path in routes else 404)
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        self.wfile.write(body)

    do_POST = do_DELETE = do_GET

    def log_message(self, *args):
        pass


@pytest.fixture
def stack(tmp_path):
    socket_path = str(tmp_path / "docker.sock")
    daemon = socketserver.ThreadingUnixStreamServer(socket_path, _FakeDocker)
    daemon.daemon_threads = True
    _FakeDocker.seen = []
    listen_path = str(tmp_path / "proxy" / "docker.sock")
    server = proxy.make_server(listen_path, socket_path)
    for item in (daemon, server):
        threading.Thread(target=item.serve_forever, daemon=True).start()
    yield listen_path, daemon
    for item in (server, daemon):
        item.shutdown()
        item.server_close()


def _raw(listen_path, method, path):
    connection = collector._UnixHTTPConnection(listen_path)
    connection.request(method, path)
    response = connection.getresponse()
    return response.status, response.read()


def test_collector_works_through_the_proxy_and_never_sees_secrets(stack):
    port, _ = stack

    [container] = collector.collect_containers(port)

    assert (container["name"], container["image"], container["status"], container["restart_count"], container["healthcheck_status"]) == ("web", "nginx:1", "running", 2, "healthy")
    assert container["ports"] == "8080->80/tcp" and container["memory_used_mb"] == 1.0 and container["pids"] == 3
    assert container["project"] == "shop" and container["volumes"] == "/srv/web, web_cache"
    assert SECRET not in json.dumps(container)


def test_the_one_shot_request_is_passed_on_and_answers_are_cut_down(stack):
    port, _ = stack

    status, body = _raw(port, "GET", f"/containers/{CID}/stats?stream=false&one-shot=true")

    assert status == 200 and _FakeDocker.seen == [f"/containers/{CID}/stats?stream=false&one-shot=true"]
    assert set(json.loads(body)) == {"cpu_stats", "precpu_stats", "memory_stats", "networks", "blkio_stats", "pids_stats"}


def test_cpu_comes_from_the_change_between_two_collections_through_the_proxy(stack):
    port, _ = stack

    [first] = collector.collect_containers(port)
    [second] = collector.collect_containers(port)

    assert "cpu_pct" not in first and "cpu_pct" not in second  # the fake daemon reports identical counters twice
    assert collector._cpu_previous == {CID: (10, 100)} and not collector._one_shot_unsupported
    assert f"/containers/{CID}/stats?stream=false" not in _FakeDocker.seen  # only the one-shot form was used


def test_refused_requests_never_reach_the_docker_daemon(stack):
    port, _ = stack
    attacks = [("POST", "/containers/create"), ("DELETE", f"/containers/{CID}"), ("GET", f"/containers/{CID}/archive?path=/etc"), ("GET", "/containers/../info"),
               ("GET", f"/containers/{CID}/json?size=1"),
               ("GET", f"/containers/{CID}/stats?one-shot=true&stream=false"), ("GET", f"/containers/{CID}/stats?stream=false&one-shot=false"), ("POST", f"/containers/{CID}/start"), ("GET", "/containers/%6Ason")]
    for method, path in attacks:
        assert _raw(port, method, path)[0] == 403, (method, path)

    assert _FakeDocker.seen == []


def test_upstream_failures_become_a_generic_502(stack):
    port, daemon = stack
    status, body = _raw(port, "GET", f"/containers/{'b' * 64}/json")  # fake daemon answers 404
    assert (status, body) == (502, b"Upstream error")

    daemon.shutdown()
    daemon.server_close()
    assert _raw(port, "GET", "/containers/json?all=1")[0] == 502


def test_oversized_upstream_answers_are_rejected(stack, monkeypatch):
    port, _ = stack
    monkeypatch.setattr(proxy, "MAX_RESPONSE_BYTES", 10)
    assert _raw(port, "GET", "/containers/json?all=1")[0] == 502


def test_proxy_socket_is_not_world_accessible_and_replaces_a_stale_one(tmp_path):
    listen = tmp_path / "run" / "docker.sock"
    listen.parent.mkdir()
    listen.write_text("stale")
    server = proxy.make_server(str(listen), str(tmp_path / "unused"))
    try:
        assert listen.stat().st_mode & 0o777 == 0o660
    finally:
        server.server_close()


def test_collector_still_supports_the_plain_socket_path(tmp_path):
    assert collector.collect_containers(str(tmp_path / "missing.sock")) == []


@pytest.fixture
def bare_proxy(tmp_path):
    """A proxy with no Docker daemon behind it, for tests of how it treats its own clients."""
    def start():
        listen_path = str(tmp_path / "proxy" / "docker.sock")
        server = proxy.make_server(listen_path, str(tmp_path / "unused"))
        threading.Thread(target=server.serve_forever, daemon=True).start()
        servers.append(server)
        return listen_path
    servers = []
    yield start
    for server in servers:
        server.shutdown()
        server.server_close()


def _connect(listen_path):
    client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    client.settimeout(5)
    client.connect(listen_path)
    return client


def test_a_stalled_client_is_dropped(bare_proxy, monkeypatch):
    monkeypatch.setattr(proxy.ProxyHandler, "timeout", 0.2)
    client = _connect(bare_proxy())
    try:
        assert client.recv(1) == b""   # closed by the proxy, not by the 5 s timeout here
    finally:
        client.close()


def test_connections_beyond_the_limit_are_closed_until_a_slot_frees(bare_proxy, monkeypatch):
    monkeypatch.setattr(proxy, "MAX_CONNECTIONS", 1)
    listen_path = bare_proxy()
    holder = _connect(listen_path)
    try:
        with pytest.raises((http.client.RemoteDisconnected, ConnectionError)):
            _raw(listen_path, "GET", "/containers/json?all=1")
    finally:
        holder.close()

    deadline = time.monotonic() + 5
    while True:
        try:
            assert _raw(listen_path, "GET", "/containers/json?all=1")[0] == 502   # served again; no daemon behind it
            break
        except (http.client.RemoteDisconnected, ConnectionError):
            assert time.monotonic() < deadline
            time.sleep(0.05)