Skip to content

Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.

94.6 KB
Raw Blame
"""Read-only host and Docker metrics collector for a registered Infradeck server.

Only INFRADECK_API_URL is required. On first start the collector asks to join and
appears on the dashboard under "New servers detected"; once someone clicks Add
it receives its token (no restart) and saves it under INFRADECK_STATE_DIR.
INFRADECK_JOIN_TOKEN (from the dashboard's Connect a server command) proves who asked it to join.
INFRADECK_ENROLL_KEY skips the approval step for automated installs, and
INFRADECK_SERVER_ID / INFRADECK_COLLECTOR_TOKEN still work as explicit credentials.
Run with ``python -m src.collector.collector`` or as a standalone script (standard
library only; it ships as a container image). Docker access is optional; when
the socket is unavailable, host metrics continue to be sent.
"""
from __future__ import annotations

import collections
import concurrent.futures
import ctypes
import ctypes.util
import hashlib
import http.client
import json
import math
import os
import platform
import re
import socket
import ssl
import struct
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

MiB = 1024 * 1024
GiB = 1024**3
# In a container the host is mounted at /host/proc and /hostfs; fall back to our own view otherwise.
# Installs from before the rename to Infradeck set HOMELAB_* variables; they keep working unchanged.
for _name, _value in list(os.environ.items()):
    if _name.startswith("HOMELAB_"):
        os.environ.setdefault("INFRADECK_" + _name[len("HOMELAB_"):], _value)

HOST_PROC = Path(os.getenv("HOST_PROC") or ("/host/proc" if Path("/host/proc").is_dir() else "/proc"))
HOST_ROOT = Path(os.getenv("HOST_ROOT") or ("/hostfs" if Path("/hostfs").is_dir() else "/"))


def _cpu_times() -> tuple[int, int]:
    values = [int(value) for value in (HOST_PROC / "stat").read_text().splitlines()[0].split()[1:]]
    idle = values[3] + (values[4] if len(values) > 4 else 0)
    return sum(values), idle


def _memory() -> dict[str, float]:
    values: dict[str, int] = {}
    for line in (HOST_PROC / "meminfo").read_text().splitlines():
        key, raw = line.split(":", 1)
        parts = raw.split()
        if parts:
            values[key] = int(parts[0]) * 1024
    total = values.get("MemTotal", 0)
    available = values.get("MemAvailable", 0)
    swap_total = values.get("SwapTotal", 0)
    swap_free = values.get("SwapFree", 0)
    return {
        "ram_used_gb": round((total - available) / GiB, 3) if total else 0,
        "ram_available_gb": round(available / GiB, 3),
        "ram_pct": round((total - available) * 100 / total, 2) if total else 0,
        "swap_pct": round((swap_total - swap_free) * 100 / swap_total, 2) if swap_total else 0,
    }


def _host_net_dirs() -> list[Path]:
    """Where the host's network tables are. PID 1's view is the host's; a container's own /proc/net is not.

    Only a collector running directly on the host may fall back to /proc/net. Inside a container that
    fallback would silently report the container's own interfaces and sockets as the host's.
    """
    return [HOST_PROC / "1/net"] + ([HOST_PROC / "net"] if HOST_PROC == Path("/proc") else [])


def _read_net_table(name: str) -> list[str] | None:
    """The lines of a host network table (e.g. ``tcp``, ``dev``), or None if it can't be read."""
    for directory in _host_net_dirs():
        try:
            with open(directory / name, "rb") as handle:
                data = handle.read(MAX_NET_TABLE_BYTES + 1)
        except OSError:
            continue
        if len(data) <= MAX_NET_TABLE_BYTES:
            return data.decode("utf-8", errors="replace").splitlines()
    return None


MAX_NET_TABLE_BYTES = 8 * 1024 * 1024
# Name prefixes of interfaces that are never physical, used only when sysfs can't say.
_VIRTUAL_INTERFACES = ("lo", "veth", "docker", "br-", "virbr", "tun", "tap", "wg", "dummy", "ifb", "vboxnet", "vmnet")


def _physical_interfaces() -> set[str] | None:
    """Interfaces backed by real hardware (they have a ``device`` entry in sysfs), or None if sysfs isn't visible."""
    base = HOST_ROOT / "sys/class/net"
    try:
        return {entry.name for entry in base.iterdir() if (entry / "device").exists()}
    except OSError:
        return None


def _network_totals() -> tuple[int, int] | None:
    """Bytes sent and received over the host's physical interfaces, or None if the host's table is unreadable.

    Virtual interfaces (Docker bridges, veth pairs, loopback) are left out: their traffic is already
    counted on the physical interface and would be counted twice.
    """
    lines = _read_net_table("dev")
    if lines is None:
        return None
    physical = _physical_interfaces()
    sent = received = 0
    for line in lines[2:]:
        if ":" not in line:
            continue
        interface, raw = line.split(":", 1)
        interface = interface.strip()
        if (interface not in physical) if physical else interface.startswith(_VIRTUAL_INTERFACES):
            continue
        try:
            fields = [int(value) for value in raw.split()]
        except ValueError:
            continue
        if len(fields) >= 9:
            received += fields[0]
            sent += fields[8]
    return sent, received


EPHEMERAL_PORT_START = 32768  # Linux's default ephemeral range starts here
MAX_LISTENERS = 512
MAX_NET_TABLE_LINES = 500_000
_ZERO_V6 = "0" * 32


def _listener_scope(address: str) -> str | None:
    """loopback, any (all interfaces) or specific, from a hex address in /proc/net/tcp(6) byte order."""
    if len(address) == 8:  # IPv4: stored little-endian, so the first octet is the last byte
        return "any" if address == "00000000" else "loopback" if address.endswith("7F") else "specific"
    if len(address) == 32:
        if address == _ZERO_V6:
            return "any"
        if address == "00000000000000000000000001000000":
            return "loopback"
        if address.startswith("0000000000000000FFFF0000"):  # IPv4-mapped IPv6
            return _listener_scope(address[24:])
        return "specific"
    return None


def listener_tracked(proto: str, port: int, scope: str) -> bool:
    """Ephemeral ports are skipped: loopback ones, and every UDP one, are programs talking out, not services."""
    return not (port >= EPHEMERAL_PORT_START and (scope == "loopback" or proto == "udp"))


def _scan_listeners(table: str, proto: str) -> set[tuple[str, int, str]] | None:
    """Listeners in one host network table, streamed so a busy server's huge connection table costs no memory.

    None if the table can't be read or is longer than MAX_NET_TABLE_LINES: a truncated read could miss
    listeners and make them look closed, so it is treated as unknown instead.
    """
    for directory in _host_net_dirs():
        try:
            handle = open(directory / table, "r", encoding="utf-8", errors="replace")
        except OSError:
            continue
        found: set[tuple[str, int, str]] = set()
        with handle:
            try:
                for number, line in enumerate(handle):
                    if number > MAX_NET_TABLE_LINES:
                        return None
                    if number:
                        found |= _parse_listener_line(line, proto)
            except OSError:
                continue
        return found
    return None


def _parse_listener_line(line: str, proto: str) -> set[tuple[str, int, str]]:
    fields = line.split()
    if len(fields) < 4:
        return set()
    address, _, port_hex = fields[1].partition(":")
    # TCP LISTEN is state 0A; a bound, unconnected UDP socket is state 07 with no remote address.
    listening = fields[3] == "0A" if proto == "tcp" else fields[3] == "07" and fields[2].startswith(("00000000:0000", _ZERO_V6 + ":0000"))
    if not listening:
        return set()
    try:
        port = int(port_hex, 16)
    except ValueError:
        return set()
    scope = _listener_scope(address)
    return {(proto, port, scope)} if scope and 1 <= port <= 65535 and listener_tracked(proto, port, scope) else set()


def collect_listeners() -> list[dict[str, Any]] | None:
    """What is listening on the host (protocol, port, scope), read passively from /proc. None if unreadable.

    No process names are collected: matching sockets to other users' processes needs privileges the
    collector deliberately doesn't have. If any table is unreadable the whole answer is unknown, so a
    partial picture can never be mistaken for ports having closed.
    """
    found: set[tuple[str, int, str]] = set()
    for table, proto in (("tcp", "tcp"), ("tcp6", "tcp"), ("udp", "udp"), ("udp6", "udp")):
        listeners = _scan_listeners(table, proto)
        if listeners is None:
            return None
        found |= listeners
    return [{"proto": proto, "port": port, "scope": scope} for proto, port, scope in sorted(found)[:MAX_LISTENERS]]


_HOSTNAME = re.compile(r"(?=.{1,253}$)([A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?)(\.[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?)*")
_IPV4 = re.compile(r"(\d{1,3})(\.\d{1,3}){3}")
MAX_CONNECTIVITY_TARGETS = 5
CONNECT_TIMEOUT_SECONDS = 3.0
CONNECTIVITY_DEADLINE_SECONDS = 6.0


def _connectivity_targets() -> list[tuple[str, int]]:
    """``host:port`` entries from INFRADECK_CONNECTIVITY_TARGETS (hostnames or IPv4 only). Empty means the check is off."""
    targets: list[tuple[str, int]] = []
    for entry in os.getenv("INFRADECK_CONNECTIVITY_TARGETS", "").split(","):
        host, _, port = entry.strip().rpartition(":")
        # A name whose last label is all digits isn't a valid hostname; it is a mistyped IP address.
        valid_host = _IPV4.fullmatch(host) and all(int(part) <= 255 for part in host.split(".")) or (_HOSTNAME.fullmatch(host) and not host.rsplit(".", 1)[-1].isdigit())
        if valid_host and port.isdigit() and 1 <= int(port) <= 65535 and (host, int(port)) not in targets:
            targets.append((host, int(port)))
        elif entry.strip():
            print(f"Ignoring invalid connectivity target {entry.strip()[:60]!r}", flush=True)
    return targets[:MAX_CONNECTIVITY_TARGETS]


def _probe(host: str, port: int, results: dict[tuple[str, int], dict[str, Any]]) -> None:
    """Resolve the name (if it is one) and open a TCP connection. Nothing is sent over it."""
    outcome: dict[str, Any] = {"dns": None if _IPV4.fullmatch(host) else False, "tcp": False, "ms": None}
    results[(host, port)] = outcome  # visible even if this thread outlives the deadline
    started = time.monotonic()
    try:
        address = host if outcome["dns"] is None else socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)[0][4][0]
        if outcome["dns"] is not None:
            outcome["dns"] = True
        socket.create_connection((address, port), CONNECT_TIMEOUT_SECONDS).close()
        outcome["tcp"], outcome["ms"] = True, int((time.monotonic() - started) * 1000)
    except (OSError, IndexError):
        pass


def summarise_connectivity(results: list[dict[str, Any]]) -> str:
    """ok (all reachable), degraded (some), dns_failed (names don't resolve, nothing else wrong) or down."""
    if all(result["tcp"] for result in results):
        return "ok"
    if any(result["tcp"] for result in results):
        return "degraded"
    return "dns_failed" if all(result["dns"] is False for result in results) else "down"


def collect_connectivity() -> dict[str, Any]:
    """Can this host reach the internet? Off unless INFRADECK_CONNECTIVITY_TARGETS is set on this collector.

    The targets come from the collector's own environment, never from the dashboard, so a compromised
    dashboard can't point these probes anywhere.
    """
    targets = _connectivity_targets()
    if not targets:
        return {"internet_status": None, "internet_latency_ms": None}
    results: dict[tuple[str, int], dict[str, Any]] = {}
    threads = [threading.Thread(target=_probe, args=(host, port, results), daemon=True) for host, port in targets]
    deadline = time.monotonic() + CONNECTIVITY_DEADLINE_SECONDS
    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join(max(0.0, deadline - time.monotonic()))
    outcomes = [dict(results.get(target) or {"dns": False, "tcp": False, "ms": None}) for target in targets]
    latencies = sorted(outcome["ms"] for outcome in outcomes if outcome["ms"] is not None)
    return {"internet_status": summarise_connectivity(outcomes), "internet_latency_ms": latencies[len(latencies) // 2] if latencies else None}


# Firewall configuration files that are looked at, by a fixed name. Only a hash and a rule count leave the host.
FIREWALL_FILES = (("ufw", "etc/ufw/user.rules"), ("ufw6", "etc/ufw/user6.rules"), ("ufw-conf", "etc/ufw/ufw.conf"),
                  ("nftables", "etc/nftables.conf"), ("iptables", "etc/iptables/rules.v4"), ("ip6tables", "etc/iptables/rules.v6"))
MAX_FIREWALL_FILE_BYTES = 1024 * 1024


def _read_regular_file(path: Path, limit: int) -> bytes | None:
    """File contents up to ``limit`` bytes; None for symlinks, non-files, unreadable or larger files."""
    try:
        descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
    except OSError:
        return None
    try:
        data = os.read(descriptor, limit + 1)
    except OSError:
        return None
    finally:
        os.close(descriptor)
    return data if len(data) <= limit else None


def collect_firewall() -> dict[str, Any] | None:
    """A fingerprint of the host's firewall configuration files; never their contents (which reveal the network layout).

    This sees configuration, not the live ruleset: reading that needs CAP_NET_ADMIN, which could also change
    the firewall, so the collector doesn't have it. Returns None if no known firewall file exists.
    """
    files: list[dict[str, Any]] = []
    ufw_enabled: bool | None = None
    for name, relative in FIREWALL_FILES:
        data = _read_regular_file(HOST_ROOT / relative, MAX_FIREWALL_FILE_BYTES)
        if data is None:
            continue
        text = data.decode("utf-8", errors="replace")
        lines = [line.strip() for line in text.splitlines()]
        if name == "ufw-conf":
            for line in lines:
                if line.upper().startswith("ENABLED="):
                    ufw_enabled = line.split("=", 1)[1].strip().lower() == "yes"
            rules = 0
        elif name in ("nftables",):
            rules = sum(1 for line in lines if line and not line.startswith("#"))
        else:
            rules = sum(1 for line in lines if line.startswith("-A "))
        files.append({"name": name, "sha": hashlib.sha256(data).hexdigest()[:16], "rules": rules})
    return {"files": files, "ufw_enabled": ufw_enabled} if files else None


# Filesystems worth reporting: anything backed by a block device, plus these pooled/network ones.
NON_DEVICE_FS_TYPES = {"zfs", "nfs", "nfs4", "cifs", "smb3", "fuse.mergerfs", "fuse.sshfs"}
SKIPPED_FS_TYPES = {"squashfs", "overlay", "tmpfs", "devtmpfs"}
SKIPPED_MOUNT_PREFIXES = ("/proc", "/sys", "/dev", "/run", "/snap", "/var/snap", "/var/lib/docker", "/var/lib/containerd", "/var/lib/kubelet")


def _decode_mount_path(path: str) -> str:
    """/proc mounts escapes spaces, tabs, newlines and backslashes as octal."""
    for escaped, char in (("\\040", " "), ("\\011", "\t"), ("\\012", "\n"), ("\\134", "\\")):
        path = path.replace(escaped, char)
    return path


def _mount_entries() -> list[tuple[str, str, str]]:
    """(source, mount point, fs type) as the host sees them."""
    # PID 1's table is the host's; "self" would be this container's own mounts.
    for candidate in (HOST_PROC / "1/mounts", HOST_PROC / "self/mounts"):
        try:
            lines = candidate.read_text().splitlines()
        except OSError:
            continue
        return [(fields[0], _decode_mount_path(fields[1]), fields[2]) for fields in (line.split() for line in lines) if len(fields) >= 3]
    return []


def collect_mounts() -> list[dict[str, Any]]:
    """Usage of every real filesystem on the host, one entry per device."""
    by_device: dict[tuple[str, int], dict[str, Any]] = {}
    for source, mount, fs_type in _mount_entries():
        if fs_type in SKIPPED_FS_TYPES or not (source.startswith("/dev/") or fs_type in NON_DEVICE_FS_TYPES):
            continue
        if mount != "/" and mount.startswith(SKIPPED_MOUNT_PREFIXES):
            continue
        local_path = HOST_ROOT / mount.lstrip("/")
        # A mount made after the collector started is only an empty directory here; skip it rather
        # than report the parent filesystem under its name.
        if mount != "/" and not os.path.ismount(local_path):
            continue
        try:
            stats = os.statvfs(local_path)
        except OSError:
            continue
        total = stats.f_blocks * stats.f_frsize
        if not total:
            continue
        used = total - stats.f_bfree * stats.f_frsize
        # Space available to non-root users is what fills up, so percentages use it like `df` does.
        usable = used + stats.f_bavail * stats.f_frsize
        entry = {
            "mount": mount[:255],
            "fs_type": fs_type[:64],
            "total_gb": round(total / GiB, 2),
            "used_gb": round(used / GiB, 2),
            "used_pct": round(100 * used / usable, 2) if usable else 0,
            "inodes_pct": round(100 * (stats.f_files - stats.f_ffree) / stats.f_files, 2) if stats.f_files else None,
        }
        # Bind mounts and btrfs subvolumes show the same device several times; keep its shortest path.
        key = (source, total)
        if key not in by_device or len(mount) < len(by_device[key]["mount"]):
            by_device[key] = entry
    return sorted(by_device.values(), key=lambda entry: entry["mount"])


def _read_text(path: Path) -> str | None:
    try:
        return path.read_text().strip()
    except OSError:
        return None


def _oom_kill_total() -> int | None:
    """Processes the kernel OOM killer has stopped since boot (needs Linux 4.13+)."""
    for line in (_read_text(HOST_PROC / "vmstat") or "").splitlines():
        if line.startswith("oom_kill "):
            return int(line.split()[1])
    return None


class _Timex(ctypes.Structure):
    _fields_ = [
        ("modes", ctypes.c_uint), ("offset", ctypes.c_long), ("freq", ctypes.c_long), ("maxerror", ctypes.c_long),
        ("esterror", ctypes.c_long), ("status", ctypes.c_int), ("constant", ctypes.c_long), ("precision", ctypes.c_long),
        ("tolerance", ctypes.c_long), ("time_sec", ctypes.c_long), ("time_usec", ctypes.c_long), ("tick", ctypes.c_long),
        ("ppsfreq", ctypes.c_long), ("jitter", ctypes.c_long), ("shift", ctypes.c_int), ("stabil", ctypes.c_long),
        ("jitcnt", ctypes.c_long), ("calcnt", ctypes.c_long), ("errcnt", ctypes.c_long), ("stbcnt", ctypes.c_long),
        ("tai", ctypes.c_int), ("padding", ctypes.c_int * 11),
    ]


_TIME_ERROR = 5


_STA_NANO = 0x2000


def _read_clock() -> tuple[int, float] | None:
    """(kernel clock state, offset from the time source in milliseconds) from a read-only adjtimex, or None."""
    try:
        libc = ctypes.CDLL(ctypes.util.find_library("c") or "libc.so.6", use_errno=True)
        timex = _Timex()
        state = libc.adjtimex(ctypes.byref(timex))
    except (OSError, AttributeError):
        return None
    if state < 0:
        return None
    # The offset is in nanoseconds when the kernel says so, microseconds otherwise.
    return state, timex.offset / (1_000_000 if timex.status & _STA_NANO else 1_000)


def _ntp_synced() -> bool | None:
    """Whether the kernel clock is disciplined by a time service; None when it can't be read."""
    # A read-only adjtimex (modes=0) covers chrony, ntpd and timesyncd alike. Docker's default
    # seccomp profile may refuse it, in which case timesyncd's marker file is the fallback.
    clock = _read_clock()
    if clock is not None:
        return clock[0] != _TIME_ERROR
    if (HOST_ROOT / "run/systemd/timesyncd/synchronized").exists():
        return True
    return None


def _reboot_required() -> bool | None:
    """Debian and Ubuntu flag pending reboots with a file; other distros aren't checked, so None."""
    if (HOST_ROOT / "run/reboot-required").exists() or (HOST_ROOT / "var/run/reboot-required").exists():
        return True
    return False if (HOST_ROOT / "etc/debian_version").exists() else None


HELPER_FILE_MAX_BYTES = 256 * 1024
HELPER_MAX_AGE_SECONDS = 300


def _helper_path() -> Path:
    return Path(os.getenv("INFRADECK_HELPER_FILE", "/helper/system.json"))


def collect_helper(boot_id: str | None, path: Path | None = None, now: float | None = None) -> dict[str, Any]:
    """Summary from the optional system helper; {} when it isn't installed, stale or from another boot.

    The file is treated as untrusted input: symlinks are refused, size is capped, and only
    expected fields are passed on (the backend validates them again).
    """
    path = path or _helper_path()
    try:
        fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
    except OSError:
        return {}
    try:
        raw = os.read(fd, HELPER_FILE_MAX_BYTES + 1)
    except OSError:
        return {}
    finally:
        os.close(fd)
    try:
        data = json.loads(raw) if len(raw) <= HELPER_FILE_MAX_BYTES else None
        generated = datetime.fromisoformat(data["generated_at"]).timestamp()
    except (ValueError, KeyError, TypeError):
        return {}
    if not isinstance(data, dict) or data.get("boot_id") != boot_id or abs((now if now is not None else time.time()) - generated) > HELPER_MAX_AGE_SECONDS:
        return {}
    events = data.get("events")
    return {"last_shutdown": data.get("last_shutdown"), "helper_events": events if isinstance(events, list) else []}


def collect_system() -> dict[str, Any]:
    """Kernel-reported system state; the optional helper adds what needs log access."""
    tainted = _read_text(HOST_PROC / "sys/kernel/tainted")
    boot_id = _read_text(HOST_PROC / "sys/kernel/random/boot_id")
    return {
        **collect_helper(boot_id),
        "boot_id": boot_id,
        "oom_kill_total": _oom_kill_total(),
        "kernel_tainted": int(tainted) if tainted and tainted.isdigit() else None,
        "ntp_synced": _ntp_synced(),
        "reboot_required": _reboot_required(),
    }


def _read_version():
    """The release this collector belongs to: VERSION next to this file in the image, or at the repository root
    when run from a checkout. "dev" when neither exists."""
    here = os.path.dirname(os.path.abspath(__file__))
    for path in (os.path.join(here, "VERSION"), os.path.join(here, "..", "..", "VERSION")):
        try:
            with open(path, encoding="utf-8") as file:
                return file.read().strip()
        except OSError:
            continue
    return "dev"


COLLECTOR_VERSION = _read_version()
MAX_INTERFACES = 16
MAX_TEMPERATURES = 8
_KERNEL_IMAGE = re.compile(r"vmlinuz-(\d[\w.+~-]*)")
_UPDATES_PENDING = re.compile(r"(\d+) (?:updates?|packages?) can be (?:applied|updated|installed)")
_UPDATES_SECURITY = re.compile(r"(\d+) (?:of these updates|additional security updates?|updates?) (?:is a|are|can be applied with)[^\n]*security", re.IGNORECASE)
# Counters from the previous check-in, per interface: (received bytes, sent bytes, monotonic time).
_interface_previous: dict[str, tuple[int, int, float]] = {}


def _version_key(version: str) -> list[tuple[int, Any]]:
    """Sorts 6.8.0-9 before 6.8.0-85: digit runs compare as numbers."""
    return [(0, int(part)) if part.isdigit() else (1, part) for part in re.split(r"(\d+)", version) if part]


def _mtime(relative: str) -> float | None:
    try:
        return (HOST_ROOT / relative).stat().st_mtime
    except OSError:
        return None


def _cpu_model() -> str | None:
    for line in (_read_text(HOST_PROC / "cpuinfo") or "").splitlines():
        if line.lower().startswith(("model name", "hardware")) and ":" in line:
            return " ".join(line.split(":", 1)[1].split())[:128] or None
    return None


def _swap_total_gb() -> float | None:
    for line in (_read_text(HOST_PROC / "meminfo") or "").splitlines():
        if line.startswith("SwapTotal:"):
            return round(int(line.split()[1]) * 1024 / GiB, 2)
    return None


def _timezone() -> str | None:
    name = _read_text(HOST_ROOT / "etc/timezone")
    if not name:
        try:
            name = os.readlink(HOST_ROOT / "etc/localtime").partition("zoneinfo/")[2]
        except OSError:
            name = None
    return name[:64] if name else None


def _time_service() -> str | None:
    for service, marker in (("chrony", "run/chrony"), ("systemd-timesyncd", "run/systemd/timesync"), ("ntpd", "var/lib/ntp"), ("chrony", "var/lib/chrony")):
        if (HOST_ROOT / marker).exists():
            return service
    return None


def collect_updates(kernel: str | None, reboot_required: bool | None) -> dict[str, Any]:
    """Pending package updates, from the files apt leaves behind (Debian and Ubuntu); {} elsewhere.

    Nothing is run on the host, so the numbers are as old as apt's last own check.
    """
    if not (HOST_ROOT / "etc/debian_version").exists():
        return {}
    info: dict[str, Any] = {"updates_manager": "apt", "updates_checked_at": _mtime("var/lib/apt/periodic/update-success-stamp"), "updates_applied_at": _mtime("var/log/apt/history.log")}
    summary = _read_regular_file(HOST_ROOT / "var/lib/update-notifier/updates-available", 16 * 1024)
    if summary is not None:
        text = summary.decode("utf-8", errors="replace")
        pending, security = _UPDATES_PENDING.search(text), _UPDATES_SECURITY.search(text)
        info["updates_pending"] = int(pending.group(1)) if pending else 0
        info["updates_security"] = int(security.group(1)) if security else 0
    settings = _read_regular_file(HOST_ROOT / "etc/apt/apt.conf.d/20auto-upgrades", 16 * 1024)
    if settings is not None:
        info["auto_updates"] = bool(re.search(r'Unattended-Upgrade\s+"1"', settings.decode("utf-8", errors="replace")))
    if reboot_required and kernel:
        try:
            installed = [match.group(1) for entry in (HOST_ROOT / "boot").iterdir() if (match := _KERNEL_IMAGE.fullmatch(entry.name))]
        except OSError:
            installed = []
        newest = max(installed, key=_version_key, default=None)
        if newest and _version_key(newest) > _version_key(kernel):
            info["reboot_kernel"] = newest[:64]
    return info


def collect_temperatures() -> list[dict[str, Any]]:
    """Hardware temperature sensors the kernel exposes in sysfs; [] when sysfs isn't visible or has none."""
    found: list[dict[str, Any]] = []
    try:
        sensors = sorted((HOST_ROOT / "sys/class/hwmon").iterdir())
    except OSError:
        return found
    for sensor in sensors:
        label, reading = _read_text(sensor / "name"), _read_text(sensor / "temp1_input")
        if label and reading and reading.lstrip("-").isdigit() and -50_000 <= int(reading) <= 200_000:
            found.append({"label": label[:32], "celsius": round(int(reading) / 1000, 1)})
    return found[:MAX_TEMPERATURES]


def _interface_networks() -> dict[str, str]:
    """The directly connected IPv4 network of each interface, from the host's routing table."""
    networks: dict[str, str] = {}
    for line in (_read_net_table("route") or [])[1:]:
        fields = line.split()
        if len(fields) < 8 or fields[2] != "00000000" or fields[1] == "00000000":
            continue  # routes through a gateway and the default route say nothing about the interface's own network
        try:
            destination = socket.inet_ntoa(bytes.fromhex(fields[1])[::-1])
            prefix = bin(int.from_bytes(bytes.fromhex(fields[7])[::-1], "big")).count("1")
        except (ValueError, OSError):
            continue
        networks.setdefault(fields[0], f"{destination}/{prefix}")
    return networks


def collect_interfaces(now: float | None = None) -> list[dict[str, Any]]:
    """The host's network interfaces with their current traffic; veth pairs (one per container) are left out."""
    lines = _read_net_table("dev")
    if lines is None:
        return []
    now = time.monotonic() if now is None else now
    networks = _interface_networks()
    interfaces, seen = [], set()
    for line in lines[2:]:
        name, _, raw = line.partition(":")
        name = name.strip()
        if not raw or name == "lo" or name.startswith("veth"):
            continue
        try:
            fields = [int(value) for value in raw.split()]
        except ValueError:
            continue
        if len(fields) < 9:
            continue
        seen.add(name)
        received, sent = fields[0], fields[8]
        entry: dict[str, Any] = {"name": name[:16], "network": networks.get(name)}
        previous = _interface_previous.get(name)
        if previous and now > previous[2] and received >= previous[0] and sent >= previous[1]:
            elapsed = now - previous[2]
            entry["rx_mbps"] = round((received - previous[0]) / MiB / elapsed, 3)
            entry["tx_mbps"] = round((sent - previous[1]) / MiB / elapsed, 3)
        speed = _read_text(HOST_ROOT / "sys/class/net" / name / "speed")
        if speed and speed.isdigit() and int(speed) > 0:
            entry["speed_mbit"] = int(speed)
        _interface_previous[name] = (received, sent, now)
        # Interfaces that carry an address or have ever moved data; idle bridges without either are noise.
        if entry["network"] or received or sent:
            interfaces.append(entry)
    for stale in set(_interface_previous) - seen:
        del _interface_previous[stale]
    interfaces.sort(key=lambda item: (item["network"] is None, item["name"]))
    return interfaces[:MAX_INTERFACES]


_PORT_LIST = r"\{?\s*((?:\d{1,5}(?:\s*[-:]\s*\d{1,5})?\s*,?\s*)+)\}?"
_NFT_ACCEPT = re.compile(rf"\b(tcp|udp)\s+dport\s+{_PORT_LIST}[^\n;]*\baccept\b")
_IPTABLES_ACCEPT = re.compile(r"^-A \S*(?:input|INPUT)\S*\s.*-p (tcp|udp)\b.*--dports? ([\d,:]+).*-j ACCEPT", re.MULTILINE)


def _ports_in(spec: str) -> set[int]:
    ports: set[int] = set()
    for part in re.split(r"[,\s]+", spec.strip()):
        low, _, high = part.replace(":", "-").partition("-")
        if low.isdigit() and (not high or high.isdigit()):
            first, last = int(low), int(high or low)
            if 1 <= first <= last <= 65535 and last - first <= 4096:
                ports.update(range(first, last + 1))
    return ports


def firewall_allowed(listeners: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
    """Which of the listening ports a firewall configuration file explicitly accepts, and in which file.

    Only ports the host is already reported to listen on are named, so this adds nothing about the rest of the
    ruleset. Like the fingerprint, it reads configuration, not the live rules.
    """
    wanted = {(item["proto"], item["port"]) for item in listeners or []}
    allowed: dict[tuple[str, int], str] = {}
    for name, relative in FIREWALL_FILES:
        data = _read_regular_file(HOST_ROOT / relative, MAX_FIREWALL_FILE_BYTES) if name != "ufw-conf" else None
        if data is None:
            continue
        text = data.decode("utf-8", errors="replace")
        for proto, spec in (_NFT_ACCEPT if name == "nftables" else _IPTABLES_ACCEPT).findall(text):
            for port in _ports_in(spec):
                if (proto, port) in wanted:
                    allowed.setdefault((proto, port), name)
    return [{"proto": proto, "port": port, "file": name} for (proto, port), name in sorted(allowed.items())]


def disk_io_errors() -> int | None:
    """I/O errors the kernel has counted on SCSI and SATA disks since boot, or None when no disk exposes a counter
    (NVMe drives don't)."""
    total, found = 0, False
    try:
        disks = list((HOST_ROOT / "sys/block").iterdir())
    except OSError:
        return None
    for disk in disks:
        raw = _read_text(disk / "device/ioerr_cnt")
        if raw:
            try:
                total += int(raw, 16)
                found = True
            except ValueError:
                continue
    return total if found else None


PORT_OWNERS_MAX_AGE_SECONDS = 300
MAX_PORT_OWNERS = 512


def _port_owners_path() -> Path:
    return Path(os.getenv("INFRADECK_PORT_OWNERS_FILE", "/helper/ports.json"))


def collect_port_owners(path: Path | None = None, now: float | None = None) -> list[dict[str, Any]] | None:
    """Which process listens on each port, from the optional port-owners helper; None when it isn't running.

    Read like the system helper's file: no symlinks, a size cap, and nothing older than a few minutes.
    """
    path = path or _port_owners_path()
    try:
        fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
    except OSError:
        return None
    try:
        raw = os.read(fd, HELPER_FILE_MAX_BYTES + 1)
    except OSError:
        return None
    finally:
        os.close(fd)
    try:
        data = json.loads(raw) if len(raw) <= HELPER_FILE_MAX_BYTES else None
        generated = datetime.fromisoformat(data["generated_at"]).timestamp()
    except (ValueError, KeyError, TypeError):
        return None
    if abs((now if now is not None else time.time()) - generated) > PORT_OWNERS_MAX_AGE_SECONDS or not isinstance(data.get("listeners"), list):
        return None
    owners = []
    for item in data["listeners"][:MAX_PORT_OWNERS]:
        if isinstance(item, dict) and item.get("proto") in ("tcp", "udp") and isinstance(item.get("port"), int) and isinstance(item.get("process"), str):
            owners.append({"proto": item["proto"], "port": item["port"], "process": item["process"][:64]})
    return owners


def collect_system_info(listeners: list[dict[str, Any]] | None, reboot_required: bool | None, mounts: list[dict[str, Any]] | None = None) -> dict[str, Any]:
    """Facts about the machine for the dashboard's System and Network tabs. Everything is read from /proc, sysfs and
    a few world-readable files; what can't be read is simply left out."""
    kernel = _read_text(HOST_PROC / "sys/kernel/osrelease")
    clock = _read_clock()
    reported = {mount["mount"] for mount in mounts or []}
    info: dict[str, Any] = {
        "kernel": kernel[:64] if kernel else None,
        "arch": platform.machine()[:16] or None,
        "cpu_model": _cpu_model(),
        "swap_total_gb": _swap_total_gb(),
        "timezone": _timezone(),
        "time_service": _time_service(),
        "clock_offset_ms": round(clock[1], 3) if clock else None,
        "collector_version": COLLECTOR_VERSION,
        "docker_version": _docker_version["value"],
        "helper_installed": _helper_path().exists(),
        "temperatures": collect_temperatures(),
        "interfaces": collect_interfaces(),
        "mount_devices": {mount[:255]: source.rsplit("/", 1)[-1][:64] for source, mount, _ in _mount_entries() if source.startswith("/dev/") and mount in reported},
        "firewall_allowed": firewall_allowed(listeners),
        "docker_disk": _docker_disk["value"],
        "disk_io_errors": disk_io_errors(),
        "listener_owners": collect_port_owners(),
        **collect_updates(kernel, reboot_required),
    }
    return {key: value for key, value in info.items() if value is not None}


def registration_details() -> dict[str, Any]:
    """What a collector found on its machine, shown next to the Add button before anyone approves it."""
    socket_path = os.getenv("INFRADECK_DOCKER_HOST") or os.getenv("DOCKER_SOCKET_PATH", "/var/run/docker.sock")
    try:
        containers = len(_docker_request(socket_path, "/containers/json?all=1")) if Path(socket_path).exists() else None
    except (OSError, ValueError, TypeError, http.client.HTTPException):
        containers = None
    return {
        "cpu_cores": _host_cpu_cores(), "containers": containers, "mounts": len(collect_mounts()),
        "helper_installed": _helper_path().exists(),
        "dashboard_host": os.getenv("INFRADECK_DASHBOARD_HOST", "").lower() in {"1", "true", "yes"},
    }


def collect_host() -> tuple[dict[str, Any], dict[str, Any]]:
    before_total, before_idle = _cpu_times()
    time.sleep(0.15)
    after_total, after_idle = _cpu_times()
    elapsed, idle = after_total - before_total, after_idle - before_idle
    cpu_pct = round(100 * (elapsed - idle) / elapsed, 2) if elapsed else 0
    memory = _memory()
    disk = os.statvfs(HOST_ROOT)
    inode_total = disk.f_files
    inode_pct = (100 * (inode_total - disk.f_ffree) / inode_total) if inode_total else None
    disk_used = (disk.f_blocks - disk.f_bfree) * disk.f_frsize
    disk_usable = disk_used + disk.f_bavail * disk.f_frsize
    # Same formula as `df` and the per-mount figures.
    disk_pct = (100 * disk_used / disk_usable) if disk_usable else 0
    try:
        loads = os.getloadavg()
    except (AttributeError, OSError):
        loads = (0.0, 0.0, 0.0)
    network = _network_totals()
    uptime = float((HOST_PROC / "uptime").read_text().split()[0])
    host = {
        "cpu_utilization_pct": cpu_pct,
        "load_1m": round(loads[0], 2), "load_5m": round(loads[1], 2), "load_15m": round(loads[2], 2),
        **memory,
        "disk_usage_pct": round(disk_pct, 2),
        "disk_inodes_pct": round(inode_pct, 2) if inode_pct is not None else None,
        "net_bytes_sent_mb": round(network[0] / MiB, 3) if network else None,
        "net_bytes_recv_mb": round(network[1] / MiB, 3) if network else None,
        "system_uptime_seconds": round(uptime, 1),
        "mounts": collect_mounts(),
        "listeners": collect_listeners(),
        "firewall": collect_firewall(),
        **collect_connectivity(),
        **collect_system(),
    }
    server = {
        "hostname": _host_hostname(),
        "os_info": _host_os_info(),
        "cpu_cores": _host_cpu_cores(),
        "total_ram_gb": round(_memory_total_gb(), 3),
        "system": collect_system_info(host["listeners"], host.get("reboot_required"), host["mounts"]),
    }
    # Set only on the collector in the dashboard's own compose stack; the dashboard's top charts follow it.
    if os.getenv("INFRADECK_DASHBOARD_HOST", "").lower() in {"1", "true", "yes"}:
        server["dashboard_host"] = True
    return server, host


def _memory_total_gb() -> float:
    for line in (HOST_PROC / "meminfo").read_text().splitlines():
        if line.startswith("MemTotal:"):
            return int(line.split()[1]) * 1024 / GiB
    return 0.0


def _host_hostname() -> str:
    override = os.getenv("INFRADECK_SERVER_NAME")
    if override:
        return override
    try:
        return (HOST_ROOT / "etc/hostname").read_text().strip() or platform.node()
    except OSError:
        return platform.node()


def _host_os_info() -> str:
    try:
        for line in (HOST_ROOT / "etc/os-release").read_text().splitlines():
            if line.startswith("PRETTY_NAME="):
                return line.split("=", 1)[1].strip().strip('"')
    except OSError:
        pass
    return f"{platform.system()} {platform.release()}"


def _host_cpu_cores() -> int:
    try:
        count = sum(line.startswith("processor:") for line in (HOST_PROC / "cpuinfo").read_text().splitlines())
        if count:
            return count
    except OSError:
        pass
    return os.cpu_count() or 1


class _UnixHTTPConnection(http.client.HTTPConnection):
    def __init__(self, socket_path: str, timeout: float = 3):
        super().__init__("localhost", timeout=timeout)
        self.socket_path = socket_path

    def connect(self) -> None:
        connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        connection.settimeout(self.timeout)
        connection.connect(self.socket_path)
        self.sock = connection


def _docker_request(socket_path: str, endpoint: str) -> Any:
    connection = _UnixHTTPConnection(socket_path)
    connection.request("GET", endpoint, headers={"Host": "localhost", "Connection": "close"})
    response = connection.getresponse()
    body = response.read()
    connection.close()
    if response.status != 200:
        raise OSError(f"Docker API returned HTTP {response.status}")
    return json.loads(body or b"null")


_IMAGE_ID = re.compile(r"sha256:[0-9a-f]{64}")
_DOCKER_TIME = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}")
# Containers that have stopped (or are between restarts) carry a meaningful exit status.
_STOPPED = {"exited", "dead", "restarting"}


def _docker_time(value: Any) -> str | None:
    """Docker's nanosecond timestamp cut to whole seconds, or None for the "never" value (year 1)."""
    match = _DOCKER_TIME.match(value) if isinstance(value, str) else None
    return f"{match.group(0)}Z" if match and int(match.group(0)[:4]) >= 2000 else None


def _docker_failure_reason(error: BaseException) -> str:
    """A coarse, fixed-vocabulary reason, never the raw error text."""
    if isinstance(error, TimeoutError):
        return "timeout"
    if isinstance(error, ConnectionRefusedError):
        return "refused"
    if isinstance(error, FileNotFoundError):
        return "missing"
    if isinstance(error, PermissionError):
        return "denied"
    return "error"


def collect_containers(socket_path: str | None = None, info: dict[str, Any] | None = None) -> list[dict[str, Any]]:
    """Every container with its state and usage. ``info`` receives how long listing them took."""
    # The read-only proxy is preferred: it keeps the Docker socket, which is root on the host, out of this container.
    socket_path = socket_path or os.getenv("INFRADECK_DOCKER_HOST") or os.getenv("DOCKER_SOCKET_PATH", "/var/run/docker.sock")
    if not Path(socket_path).exists():
        return []
    listing_started = time.monotonic()
    containers = _docker_request(socket_path, "/containers/json?all=1")
    if info is not None:
        info["latency_ms"] = int((time.monotonic() - listing_started) * 1000)
    result = []
    for stale in set(_cpu_previous) - {item.get("Id") for item in containers}:
        del _cpu_previous[stale]
    for item in containers:
        container_id = item.get("Id", "")
        if not container_id:
            continue
        try:
            detail = _docker_request(socket_path, f"/containers/{container_id}/json")
        except (OSError, ValueError, http.client.HTTPException):
            # A container removed between listing and inspecting it must not fail the whole collection.
            continue
        state = detail.get("State") or {}
        status = state.get("Status") or item.get("State") or "unknown"
        health = (state.get("Health") or {}).get("Status")
        usage_stats = _container_usage(socket_path, container_id) if status == "running" else {}
        stopped = str(status).lower() in _STOPPED
        exit_code = state.get("ExitCode")
        image_id = detail.get("Image")
        result.append({
            "container_id": container_id,
            "name": (detail.get("Name") or (item.get("Names") or [container_id[:12]])[0]).lstrip("/"),
            "image": detail.get("Config", {}).get("Image") or item.get("Image"),
            "status": status.lower(),
            "restart_count": int(detail.get("RestartCount", 0)),
            "healthcheck_status": health,
            "started_at": state.get("StartedAt") if status == "running" else None,
            "ports": _format_ports((detail.get("NetworkSettings") or {}).get("Ports")),
            "project": _compose_project(detail),
            "volumes": _format_volumes(detail.get("Mounts")),
            "exit_code": exit_code if stopped and isinstance(exit_code, int) and not isinstance(exit_code, bool) and -1 <= exit_code <= 65535 else None,
            "oom_killed": state.get("OOMKilled") if stopped and isinstance(state.get("OOMKilled"), bool) else None,
            "finished_at": _docker_time(state.get("FinishedAt")) if stopped else None,
            "image_id": image_id if isinstance(image_id, str) and _IMAGE_ID.fullmatch(image_id) else None,
            **usage_stats,
            **({"swap_mb": swap} if status == "running" and (swap := container_swap_mb(container_id)) is not None else {}),
        })
    if containers and not result:
        # Listing worked but nothing could be inspected: the daemon is failing, not empty.
        raise OSError("no container could be inspected")
    return result


# The Docker daemon's version, asked for rarely: it only changes when Docker is upgraded.
_docker_version: dict[str, Any] = {"value": None, "countdown": 0}
DOCKER_VERSION_EVERY = 120


def _remember_docker_version(socket_path: str) -> None:
    if _docker_version["countdown"] > 0:
        _docker_version["countdown"] -= 1
        return
    _docker_version["countdown"] = DOCKER_VERSION_EVERY
    try:
        version = _docker_request(socket_path, "/version").get("Version")
    except (OSError, ValueError, AttributeError, http.client.HTTPException):
        return  # an older proxy image refuses the request; the version just stays unknown
    _docker_version["value"] = version[:32] if isinstance(version, str) and version else None


# What Docker's images and build cache take on disk. Asking makes the daemon walk every layer, which can take
# seconds on a big host, so it is asked for rarely and never delays a check-in by more than its own timeout.
_docker_disk: dict[str, Any] = {"value": None, "next": 0.0}
DOCKER_DISK_EVERY_SECONDS = 30 * 60
_DOCKER_DISK_KEYS = ("images_bytes", "images_unused_bytes", "build_cache_bytes", "build_cache_unused_bytes")


def _remember_docker_disk(socket_path: str, now: float | None = None) -> None:
    now = time.monotonic() if now is None else now
    if now < _docker_disk["next"]:
        return
    _docker_disk["next"] = now + DOCKER_DISK_EVERY_SECONDS
    try:
        usage = _docker_request(socket_path, "/system/df")
    except (OSError, ValueError, http.client.HTTPException):
        return  # refused by an older proxy image, too slow or too large: stays unknown
    if not isinstance(usage, dict) or not all(isinstance(usage.get(key), int) for key in _DOCKER_DISK_KEYS):
        return
    _docker_disk["value"] = {
        **{key.replace("_bytes", "_gb"): round(usage[key] / GiB, 2) for key in _DOCKER_DISK_KEYS},
        "images": usage.get("images") if isinstance(usage.get("images"), int) else None,
        "images_unused": usage.get("images_unused") if isinstance(usage.get("images_unused"), int) else None,
    }


def collect_docker(socket_path: str | None = None) -> tuple[list[dict[str, Any]], dict[str, Any]]:
    """Containers plus a verdict on the Docker API: ``ok``, ``unreachable`` or ``not_configured``.

    "No Docker on this host" is not an error, and an empty list is not proof of a problem, so the
    backend only alerts on ``unreachable`` for a host that had Docker working before.
    """
    explicit = socket_path or os.getenv("INFRADECK_DOCKER_HOST")
    target = explicit or os.getenv("DOCKER_SOCKET_PATH", "/var/run/docker.sock")
    if not Path(target).exists():
        # A proxy that is configured but missing is a failure; a host that simply has no Docker socket is not.
        if explicit:
            return [], {"docker_status": "unreachable", "docker_reason": "missing", "docker_latency_ms": None}
        return [], {"docker_status": "not_configured", "docker_reason": None, "docker_latency_ms": None}
    info: dict[str, Any] = {}
    try:
        containers = collect_containers(target, info)
        _remember_docker_version(target)
        _remember_docker_disk(target)
    except (OSError, ValueError, http.client.HTTPException) as error:
        print(f"Docker metrics unavailable ({type(error).__name__}); sending host metrics only", flush=True)
        return [], {"docker_status": "unreachable", "docker_reason": _docker_failure_reason(error), "docker_latency_ms": info.get("latency_ms")}
    return containers, {"docker_status": "ok", "docker_reason": None, "docker_latency_ms": info.get("latency_ms")}


COMPOSE_PROJECT_LABEL = "com.docker.compose.project"
_CONTAINER_ID = re.compile(r"[0-9a-f]{64}")


def container_swap_mb(container_id: str) -> float | None:
    """How much of a container's memory is swapped out, from its cgroup (v2), or None when that can't be read.

    Docker's stats don't report it. The cgroup files are world-readable and sit under the host's /sys, at a path
    that depends on whether Docker uses the systemd or the cgroupfs driver.
    """
    if not _CONTAINER_ID.fullmatch(container_id):
        return None
    for relative in (f"sys/fs/cgroup/system.slice/docker-{container_id}.scope/memory.swap.current", f"sys/fs/cgroup/docker/{container_id}/memory.swap.current"):
        raw = _read_text(HOST_ROOT / relative)
        if raw and raw.isdigit():
            return round(int(raw) / MiB, 2)
    return None


def _compose_project(detail: dict[str, Any]) -> str | None:
    """The compose project a container belongs to, or None for one started on its own."""
    labels = (detail.get("Config") or {}).get("Labels")
    project = labels.get(COMPOSE_PROJECT_LABEL) if isinstance(labels, dict) else None
    return project[:128] if isinstance(project, str) and project else None


def _format_volumes(mounts: Any) -> str | None:
    """Where a container's data lives on the host: bind-mount paths and volume names, never what is in them."""
    sources = set()
    for mount in mounts if isinstance(mounts, list) else []:
        if not isinstance(mount, dict):
            continue
        source = mount.get("Name") if mount.get("Type") == "volume" else mount.get("Source")
        if isinstance(source, str) and source:
            sources.add(source)
    return ", ".join(sorted(sources))[:512] or None


def _format_ports(ports: dict[str, Any] | None) -> str | None:
    mappings = []
    for container_port, bindings in (ports or {}).items():
        for binding in bindings or []:
            mappings.append(f"{binding.get('HostPort')}->{container_port}")
    return ", ".join(sorted(set(mappings)))[:512] or None


# Cumulative CPU counters (container, whole system) from the previous check-in, per container. The one-shot stats
# call returns at once but carries no earlier reading, so CPU % is the change since the last check-in. The first
# check-in after a start has no figure for a container (unknown, not zero).
_cpu_previous: dict[str, tuple[float, float]] = {}
# Set once the proxy or daemon refuses the one-shot request while the normal one works (an older proxy image).
_one_shot_unsupported = False


def _fetch_stats(socket_path: str, container_id: str) -> Any:
    """One container's stats: the quick one-shot request, falling back to the normal one that waits about a second."""
    global _one_shot_unsupported
    base = f"/containers/{container_id}/stats?stream=false"
    tried_one_shot = not _one_shot_unsupported
    if tried_one_shot:
        try:
            return _docker_request(socket_path, f"{base}&one-shot=true")
        except (OSError, ValueError):
            pass
    stats = _docker_request(socket_path, base)
    if tried_one_shot:
        _one_shot_unsupported = True
    return stats


def _number(value: Any) -> float | None:
    return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) else None


def _cpu_percent(container_id: str, stats: dict[str, Any]) -> float | None:
    cpu = stats.get("cpu_stats") or {}
    previous = stats.get("precpu_stats") or {}
    total = _number((cpu.get("cpu_usage") or {}).get("total_usage"))
    system = _number(cpu.get("system_cpu_usage"))
    before_total = _number((previous.get("cpu_usage") or {}).get("total_usage"))
    before_system = _number(previous.get("system_cpu_usage"))
    if total is not None and system is not None:
        remembered = _cpu_previous.get(container_id)
        _cpu_previous[container_id] = (total, system)
        if not before_system:
            before_total, before_system = remembered if remembered else (None, None)
    if None in (total, system, before_total, before_system):
        return None
    cpu_delta, system_delta = total - before_total, system - before_system
    # A counter that went backwards means the container restarted; skip that sample.
    if cpu_delta <= 0 or system_delta <= 0:
        return None
    # cgroup v2 omits percpu_usage, so prefer online_cpus like `docker stats` does.
    cores = cpu.get("online_cpus") or len((cpu.get("cpu_usage") or {}).get("percpu_usage") or []) or 1
    return round(cpu_delta / system_delta * cores * 100, 2)


def _container_usage(socket_path: str, container_id: str) -> dict[str, Any]:
    try:
        stats = _fetch_stats(socket_path, container_id)
    except (OSError, ValueError):
        return {}
    usage: dict[str, Any] = {}
    try:
        cpu_pct = _cpu_percent(container_id, stats)
        if cpu_pct is not None:
            usage["cpu_pct"] = cpu_pct

        memory = stats.get("memory_stats") or {}
        # Page cache is reclaimable; subtract it the same way `docker stats` does.
        details = memory.get("stats") or {}
        used = memory.get("usage", 0) - details.get("inactive_file", details.get("total_inactive_file", 0))
        limit = memory.get("limit", 0)
        usage["memory_used_mb"] = round(max(used, 0) / MiB, 2)
        usage["memory_limit_mb"] = round(limit / MiB, 2) if limit else None
        usage["memory_pct"] = round(max(used, 0) * 100 / limit, 2) if limit else None

        networks = (stats.get("networks") or {}).values()
        usage["net_rx_mb"] = round(sum(n.get("rx_bytes", 0) for n in networks) / MiB, 2)
        usage["net_tx_mb"] = round(sum(n.get("tx_bytes", 0) for n in networks) / MiB, 2)

        io_entries = (stats.get("blkio_stats") or {}).get("io_service_bytes_recursive") or []
        usage["block_read_mb"] = round(sum(e.get("value", 0) for e in io_entries if str(e.get("op", "")).lower() == "read") / MiB, 2)
        usage["block_write_mb"] = round(sum(e.get("value", 0) for e in io_entries if str(e.get("op", "")).lower() == "write") / MiB, 2)

        usage["pids"] = (stats.get("pids_stats") or {}).get("current")
    except (ValueError, KeyError, TypeError, AttributeError):
        pass
    return usage


# ---------- availability checks ----------
# The dashboard asks this collector to try services every so often and reports each run: an HTTP(s) URL, a keyword in
# a page, a value in a JSON reply, a TCP port, a TLS certificate's expiry, a DNS record, or an ICMP ping. Only what
# the dashboard sends is run, after checking it here too: http/https URLs and host/port pairs, GET or HEAD without
# custom headers or body, at most 1 MiB of a response read and none of it sent back. No shell is ever involved; ping
# uses an unprivileged ICMP datagram socket (no raw sockets, no CAP_NET_RAW), DNS a plain UDP query built by hand.

CHECK_TYPES = ("http", "keyword", "tcp", "tls", "dns", "json", "ping")
URL_CHECK_TYPES = ("http", "keyword", "json")
DNS_RECORD_NUMS = {"A": 1, "NS": 2, "CNAME": 5, "MX": 15, "TXT": 16, "AAAA": 28}
JSON_OPS = ("equals", "contains", "gt", "lt", "exists")
DEFAULT_TLS_PORT = 443
MAX_CHECKS = 50
MIN_CHECK_INTERVAL, MAX_CHECK_INTERVAL = 20, 86400
MAX_CHECK_TIMEOUT = 48
MAX_BODY_BYTES = 1024 * 1024
MAX_REDIRECTS = 10
MAX_PENDING_RESULTS = 500
CHECK_WORKERS = 8
IDLE_REPORT_SECONDS = 60
_CHECK_HOST = re.compile(r"[A-Za-z0-9]([A-Za-z0-9.-]{0,251}[A-Za-z0-9])?|\[[0-9A-Fa-f:.]+\]")
_CHECK_CODES = re.compile(r"[1-5]\d\d(-[1-5]\d\d)?")


def _whole(value: Any, low: int, high: int) -> int | None:
    return value if isinstance(value, int) and not isinstance(value, bool) and low <= value <= high else None


def _check_host_port(target: str, default_port: int | None) -> tuple[str, int] | None:
    """(host, port) from "host:port", "[ipv6]:port", or a bare host with ``default_port``; None if unusable."""
    if target.startswith("["):
        inner, _, rest = target[1:].partition("]")
        if not _is_ip_literal(inner):
            return None
        if rest == "":
            port = default_port
        elif rest.startswith(":") and rest[1:].isdigit():
            port = int(rest[1:])
        else:
            return None
        host = inner
    else:
        head, sep, tail = target.rpartition(":")
        if sep and tail.isdigit() and head:
            host, port = head, int(tail)
        elif not sep:
            host, port = target, default_port
        else:
            return None
        if not _CHECK_HOST.fullmatch(host):
            return None
    if port is None or not 1 <= int(port) <= 65535:
        return None
    return host, int(port)


def _is_ip_literal(text: str) -> bool:
    try:
        socket.inet_pton(socket.AF_INET, text)
        return True
    except OSError:
        pass
    try:
        socket.inet_pton(socket.AF_INET6, text)
        return True
    except OSError:
        return False


def validate_check(raw: Any) -> dict[str, Any] | None:
    """One check as the dashboard sent it, reduced to known fields of the right type and range; None if unusable."""
    if not isinstance(raw, dict) or raw.get("type") not in CHECK_TYPES or not isinstance(raw.get("target"), str):
        return None
    check_id = _whole(raw.get("id"), 1, 2**31 - 1)
    interval = _whole(raw.get("interval"), MIN_CHECK_INTERVAL, MAX_CHECK_INTERVAL)
    retry_interval = _whole(raw.get("retry_interval"), MIN_CHECK_INTERVAL, MAX_CHECK_INTERVAL)
    timeout = _whole(raw.get("timeout"), 1, MAX_CHECK_TIMEOUT)
    if None in (check_id, interval, retry_interval, timeout):
        return None
    kind, target = raw["type"], raw["target"].strip()
    check: dict[str, Any] = {"id": check_id, "type": kind, "target": target, "interval": interval, "retry_interval": retry_interval, "timeout": timeout}
    if kind in ("tcp", "tls"):
        parsed = _check_host_port(target, DEFAULT_TLS_PORT if kind == "tls" else None)
        if not parsed:
            return None
        check.update(host=parsed[0], port=parsed[1])
        if kind == "tls":
            warn_days = _whole(raw.get("warn_days"), 1, 3650)
            if warn_days is None:
                return None
            check["warn_days"] = warn_days
        return check
    if kind == "ping":
        if not (_is_ip_literal(target.strip("[]")) or _CHECK_HOST.fullmatch(target)):
            return None
        check["host"] = target.strip("[]")
        return check
    if kind == "dns":
        if not _CHECK_HOST.fullmatch(target) or raw.get("record_type") not in DNS_RECORD_NUMS:
            return None
        resolver = str(raw.get("resolver") or "").strip()
        if resolver and not _is_ip_literal(resolver):
            return None
        expected = raw.get("expected")
        check.update(record_type=raw["record_type"], resolver=resolver, expected=expected if isinstance(expected, str) else "")
        return check
    # url types: http, keyword, json
    parts = urllib.parse.urlsplit(target)
    if parts.scheme not in ("http", "https") or not parts.hostname or len(target) > 2048 or any(char.isspace() or not char.isprintable() for char in target):
        return None
    if parts.username or parts.password:
        return None
    codes = raw.get("accepted_codes")
    if not isinstance(codes, list) or not 1 <= len(codes) <= 10 or not all(isinstance(code, str) and _CHECK_CODES.fullmatch(code) for code in codes):
        return None
    check.update(accepted_codes=codes, follow_redirects=raw.get("follow_redirects") is True, ignore_tls=raw.get("ignore_tls") is True,
                 method="HEAD" if raw.get("method") == "HEAD" and kind == "http" else "GET")
    if kind == "keyword":
        keyword = raw.get("keyword")
        if not isinstance(keyword, str) or not 1 <= len(keyword) <= 200:
            return None
        check.update(keyword=keyword, case_sensitive=raw.get("case_sensitive") is True)
    if kind == "json":
        path, operator, expect = raw.get("json_path"), raw.get("json_op"), raw.get("json_expect")
        if not isinstance(path, str) or not 1 <= len(path) <= 200 or operator not in JSON_OPS:
            return None
        if operator != "exists" and not (isinstance(expect, str) and expect):
            return None
        check.update(json_path=path, json_op=operator, json_expect=expect if isinstance(expect, str) else "", method="GET")
    return check


def _code_accepted(code: int, accepted: list[str]) -> bool:
    for item in accepted:
        low, _, high = item.partition("-")
        if int(low) <= code <= int(high or low):
            return True
    return False


class _Redirects(urllib.request.HTTPRedirectHandler):
    """Follows at most MAX_REDIRECTS redirects, and only to http and https, or none at all."""
    max_redirections = MAX_REDIRECTS

    def __init__(self, follow: bool) -> None:
        self.follow = follow

    def redirect_request(self, req, fp, code, msg, headers, newurl):  # noqa: ANN001
        if not self.follow or urllib.parse.urlsplit(newurl).scheme not in ("http", "https"):
            return None   # urllib then raises HTTPError with the 3xx code, which the accepted codes decide on
        return super().redirect_request(req, fp, code, msg, headers, newurl)


def _cert_days(response: Any) -> int | None:
    """Days until the server's TLS certificate expires, when it was verified."""
    try:
        cert = response.fp.raw._sock.getpeercert()
        expires = ssl.cert_time_to_seconds(cert["notAfter"])
    except (AttributeError, KeyError, TypeError, ValueError, OSError):
        return None
    return int((expires - time.time()) // 86400)


def _failure(exc: BaseException, timeout: int) -> str:
    reason = getattr(exc, "reason", exc)
    if isinstance(reason, (TimeoutError, socket.timeout)) or isinstance(exc, (TimeoutError, socket.timeout)):
        return f"Timed out after {timeout} s"
    if isinstance(reason, ssl.SSLCertVerificationError):
        return f"TLS certificate not valid: {reason.verify_message}"[:200]
    if isinstance(reason, ssl.SSLError) or isinstance(exc, ssl.SSLError):   # an SSLError's own reason is a string
        return "TLS handshake failed"
    if isinstance(reason, socket.gaierror):
        return "Host name not found"
    if isinstance(reason, ConnectionRefusedError):
        return "Connection refused"
    if isinstance(reason, ConnectionResetError):
        return "Connection reset"
    if isinstance(reason, OSError) and reason.strerror:
        return reason.strerror[:200]
    return type(reason).__name__


class _Deadline:
    """A check's timeout as a hard limit for the whole run (#204). A socket timeout only limits each read, so a
    server that sends its answer a byte at a time could hold a check, and one of the CHECK_WORKERS, for as long as
    it likes. When the time is up this shuts down every socket the check opened, which ends any read or TLS
    handshake still waiting on it. A name lookup can't be interrupted; it ends when the system resolver gives up,
    and the connection after it is then shut down at once."""

    def __init__(self, seconds: float) -> None:
        self.expired = False
        self.sockets: list[socket.socket] = []
        self.lock = threading.Lock()
        self.timer = threading.Timer(seconds, self._expire)
        self.timer.daemon = True

    def __enter__(self) -> "_Deadline":
        self.timer.start()
        return self

    def __exit__(self, *_exc: object) -> None:
        self.timer.cancel()

    def track(self, sock: socket.socket) -> socket.socket:
        with self.lock:
            self.sockets.append(sock)
            if self.expired:
                _shut(sock)
        return sock

    def _expire(self) -> None:
        with self.lock:
            self.expired = True
            for sock in self.sockets:
                _shut(sock)


def _shut(sock: socket.socket) -> None:
    try:
        sock.shutdown(socket.SHUT_RDWR)
    except OSError:
        pass


def _tracked(base: type, deadline: _Deadline) -> type:
    """``base`` (an http.client connection class) whose sockets ``deadline`` can shut down."""
    class Tracked(base):
        def __init__(self, *args: Any, **kwargs: Any) -> None:
            super().__init__(*args, **kwargs)
            self._create_connection = lambda *a, **k: deadline.track(socket.create_connection(*a, **k))
    return Tracked


class _DeadlineHTTP(urllib.request.HTTPHandler):
    def __init__(self, deadline: _Deadline) -> None:
        super().__init__()
        self.deadline = deadline

    def http_open(self, req):  # noqa: ANN001, ANN201
        return self.do_open(_tracked(http.client.HTTPConnection, self.deadline), req)


class _DeadlineHTTPS(urllib.request.HTTPSHandler):
    def __init__(self, context: ssl.SSLContext, deadline: _Deadline) -> None:
        super().__init__(context=context)
        self.deadline = deadline

    def https_open(self, req):  # noqa: ANN001, ANN201
        return self.do_open(_tracked(http.client.HTTPSConnection, self.deadline), req, context=self._context)


def run_http(check: dict[str, Any]) -> dict[str, Any]:
    """One run of an HTTP(s) or keyword check: {ok, response_ms, code, detail, cert_days}."""
    context = ssl.create_default_context()
    if check["ignore_tls"]:
        context.check_hostname = False
        context.verify_mode = ssl.CERT_NONE
    request = urllib.request.Request(check["target"], method=check["method"], headers={"User-Agent": f"Infradeck-Check/{COLLECTOR_VERSION}", "Accept": "*/*"})
    started = time.monotonic()
    result: dict[str, Any] = {"ok": False, "response_ms": None, "code": None, "detail": None, "cert_days": None}
    body = b""
    with _Deadline(check["timeout"]) as deadline:
        opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _DeadlineHTTP(deadline), _DeadlineHTTPS(context, deadline), _Redirects(check["follow_redirects"]))
        try:
            with opener.open(request, timeout=check["timeout"]) as response:
                result["code"] = response.status
                if not check["ignore_tls"]:
                    result["cert_days"] = _cert_days(response)
                if check["type"] in ("keyword", "json"):
                    while len(body) < MAX_BODY_BYTES:
                        chunk = response.read(min(65536, MAX_BODY_BYTES - len(body)))
                        if not chunk:
                            break
                        body += chunk
        except urllib.error.HTTPError as exc:
            result["code"] = exc.code
            exc.close()
        except (urllib.error.URLError, OSError, http.client.HTTPException, ValueError) as exc:
            result["detail"] = f"Timed out after {check['timeout']} s" if deadline.expired else _failure(exc, check["timeout"])
            return result
    if deadline.expired:   # the answer was cut off, so it can't be judged
        result["detail"] = f"Timed out after {check['timeout']} s"
        return result
    result["response_ms"] = int((time.monotonic() - started) * 1000)
    if not _code_accepted(result["code"], check["accepted_codes"]):
        result["detail"] = f"HTTP {result['code']}"
        return result
    if check["type"] == "keyword":
        text = body.decode("utf-8", errors="replace")
        found = check["keyword"] in text if check["case_sensitive"] else check["keyword"].casefold() in text.casefold()
        if not found:
            result["detail"] = "Keyword not found"
            return result
    if check["type"] == "json":
        try:
            data = json.loads(body.decode("utf-8"))
        except (ValueError, UnicodeDecodeError):
            result["detail"] = "The reply is not JSON"
            return result
        try:
            value = _json_walk(data, check["json_path"])
        except (KeyError, IndexError, TypeError):
            result["detail"] = f"No value at {check['json_path']}"[:200]
            return result
        if not _json_match(value, check["json_op"], check.get("json_expect", "")):
            result["detail"] = f"{check['json_path']} is {json.dumps(value)[:80]}"[:200]
            return result
    result["ok"] = True
    return result


def _json_walk(data: Any, path: str) -> Any:
    """The value at a dotted/indexed path like ``data.status`` or ``items[0].ok``; raises if it isn't there."""
    current = data
    for part in re.findall(r"[^.\[\]]+|\[\d+\]", path):
        if part.startswith("["):
            current = current[int(part[1:-1])]
        elif isinstance(current, list) and part.lstrip("-").isdigit():
            current = current[int(part)]
        elif isinstance(current, dict):
            current = current[part]
        else:
            raise KeyError(part)
    return current


def _json_match(value: Any, operator: str, expect: str) -> bool:
    if operator == "exists":
        return True
    if isinstance(value, bool):
        shown = "true" if value else "false"
    elif value is None:
        shown = "null"
    elif isinstance(value, (dict, list)):
        shown = json.dumps(value, separators=(",", ":"))
    else:
        shown = str(value)
    if operator == "equals":
        return shown.strip().casefold() == expect.strip().casefold()
    if operator == "contains":
        return expect in shown
    try:
        return (float(shown) > float(expect)) if operator == "gt" else (float(shown) < float(expect))
    except ValueError:
        return False


def run_tls(check: dict[str, Any]) -> dict[str, Any]:
    """Connect, read the server's certificate, and be down when it is invalid, expired, or close to expiring."""
    context = ssl.create_default_context()
    started = time.monotonic()
    with _Deadline(check["timeout"]) as deadline:
        try:
            with deadline.track(socket.create_connection((check["host"], check["port"]), timeout=check["timeout"])) as sock:
                with context.wrap_socket(sock, server_hostname=check["host"]) as tls:
                    cert = tls.getpeercert()
        except ssl.SSLCertVerificationError as exc:
            return {"ok": False, "response_ms": None, "code": None, "detail": f"Certificate not valid: {getattr(exc, 'verify_message', None) or exc}"[:200], "cert_days": None}
        except (ssl.SSLError, OSError, http.client.HTTPException) as exc:
            detail = f"Timed out after {check['timeout']} s" if deadline.expired else _failure(exc, check["timeout"])
            return {"ok": False, "response_ms": None, "code": None, "detail": detail, "cert_days": None}
    response_ms = int((time.monotonic() - started) * 1000)
    try:
        days = int((ssl.cert_time_to_seconds(cert["notAfter"]) - time.time()) // 86400)
    except (KeyError, ValueError, TypeError):
        return {"ok": False, "response_ms": response_ms, "code": None, "detail": "The certificate has no expiry date", "cert_days": None}
    ok = days >= check.get("warn_days", 14)
    detail = None if ok else (f"Certificate expired {abs(days)} d ago" if days < 0 else f"Certificate expires in {days} d")
    return {"ok": ok, "response_ms": response_ms, "code": None, "detail": detail, "cert_days": days}


def _system_resolver() -> str:
    try:
        for line in Path("/etc/resolv.conf").read_text().splitlines():
            parts = line.split()
            if len(parts) >= 2 and parts[0] == "nameserver" and _is_ip_literal(parts[1]):
                return parts[1]
    except OSError:
        pass
    return ""


def _skip_name(data: bytes, off: int) -> int:
    while off < len(data):
        length = data[off]
        if length == 0:
            return off + 1
        if length & 0xC0 == 0xC0:   # a compression pointer ends the name
            return off + 2
        off += 1 + length
    return off


def _read_name(data: bytes, off: int) -> str:
    labels, hops = [], 0
    while off < len(data) and hops < 64:
        length = data[off]
        if length == 0:
            break
        if length & 0xC0 == 0xC0:
            off = ((length & 0x3F) << 8) | data[off + 1]
            hops += 1
            continue
        labels.append(data[off + 1:off + 1 + length].decode("ascii", "replace"))
        off += 1 + length
    return ".".join(labels)


def _render_rdata(want_type: int, data: bytes, off: int, rdlen: int) -> str:
    rdata = data[off:off + rdlen]
    if want_type == 1 and rdlen == 4:
        return socket.inet_ntop(socket.AF_INET, rdata)
    if want_type == 28 and rdlen == 16:
        return socket.inet_ntop(socket.AF_INET6, rdata)
    if want_type in (2, 5):   # NS, CNAME
        return _read_name(data, off)
    if want_type == 15 and rdlen >= 3:   # MX: preference + exchange
        return f"{struct.unpack('>H', rdata[:2])[0]} {_read_name(data, off + 2)}"
    if want_type == 16:   # TXT: one or more length-prefixed strings
        parts, index = [], 0
        while index < rdlen:
            piece = rdata[index + 1:index + 1 + rdata[index]]
            parts.append(piece.decode("utf-8", "replace"))
            index += 1 + rdata[index]
        return "".join(parts)
    return ""


class _DnsError(Exception):
    """A DNS reply that says the lookup failed, with a message safe to show."""


def _dns_query(name: str, want_type: int, resolver: str, timeout: int) -> list[str]:
    tid = os.urandom(2)
    header = tid + struct.pack(">HHHHH", 0x0100, 1, 0, 0, 0)   # one question, recursion desired
    question = b"".join(bytes([len(label)]) + label for label in (part.encode("idna") for part in name.split(".") if part)) + b"\x00"
    packet = header + question + struct.pack(">HH", want_type, 1)
    family = socket.AF_INET6 if ":" in resolver else socket.AF_INET
    with socket.socket(family, socket.SOCK_DGRAM) as sock:
        sock.settimeout(timeout)
        sock.sendto(packet, (resolver, 53))
        deadline = time.monotonic() + timeout
        data = b""
        while time.monotonic() < deadline:
            sock.settimeout(max(0.05, deadline - time.monotonic()))
            data, _ = sock.recvfrom(4096)
            if len(data) >= 12 and data[:2] == tid:
                break
    if len(data) < 12 or data[:2] != tid:
        raise _DnsError("No answer from the DNS server")
    return _parse_dns(data, want_type)


def _parse_dns(data: bytes, want_type: int) -> list[str]:
    flags, qd, an = struct.unpack(">HHH", data[2:8])
    rcode = flags & 0x0F
    if rcode == 3:
        raise _DnsError("The name does not exist (NXDOMAIN)")
    if rcode != 0:
        raise _DnsError(f"The DNS server returned an error (rcode {rcode})")
    off = 12
    for _ in range(qd):
        off = _skip_name(data, off) + 4
    answers = []
    for _ in range(an):
        off = _skip_name(data, off)
        rtype, _rclass, _ttl, rdlen = struct.unpack(">HHIH", data[off:off + 10])
        off += 10
        if rtype == want_type:
            rendered = _render_rdata(want_type, data, off, rdlen)
            if rendered:
                answers.append(rendered)
        off += rdlen
    return answers


def run_dns(check: dict[str, Any]) -> dict[str, Any]:
    """Ask a DNS server for a record and be down when it doesn't resolve or doesn't match the expected value."""
    resolver = check.get("resolver") or _system_resolver()
    if not resolver:
        return {"ok": False, "response_ms": None, "code": None, "detail": "No DNS server to ask", "cert_days": None}
    want_type = DNS_RECORD_NUMS[check["record_type"]]
    started = time.monotonic()
    try:
        answers = _dns_query(check["target"].rstrip("."), want_type, resolver, check["timeout"])
    except _DnsError as exc:
        return {"ok": False, "response_ms": None, "code": None, "detail": str(exc)[:200], "cert_days": None}
    except (OSError, UnicodeError, struct.error) as exc:
        return {"ok": False, "response_ms": None, "code": None, "detail": _failure(exc, check["timeout"]), "cert_days": None}
    response_ms = int((time.monotonic() - started) * 1000)
    if not answers:
        return {"ok": False, "response_ms": response_ms, "code": None, "detail": f"No {check['record_type']} record", "cert_days": None}
    expected = (check.get("expected") or "").strip()
    if expected and not any(expected.casefold() == answer.casefold() or (check["record_type"] == "TXT" and expected in answer) for answer in answers):
        return {"ok": False, "response_ms": response_ms, "code": None, "detail": f"Resolved to {', '.join(answers)}"[:200], "cert_days": None}
    return {"ok": True, "response_ms": response_ms, "code": None, "detail": None, "cert_days": None}


def _icmp_checksum(data: bytes) -> int:
    if len(data) % 2:
        data += b"\x00"
    total = sum(struct.unpack(">%dH" % (len(data) // 2), data))
    total = (total >> 16) + (total & 0xFFFF)
    total += total >> 16
    return ~total & 0xFFFF


def run_ping(check: dict[str, Any]) -> dict[str, Any]:
    """One ICMP echo over an unprivileged datagram socket (no raw socket, no shell). The container allows this only
    for its own group via net.ipv4.ping_group_range; without that the result says ICMP isn't permitted."""
    try:
        family, _, _, _, sockaddr = socket.getaddrinfo(check["host"], None)[0]
    except (socket.gaierror, IndexError):
        return {"ok": False, "response_ms": None, "code": None, "detail": "Host name not found", "cert_days": None}
    address = sockaddr[0]
    if family == socket.AF_INET6:
        proto, echo_type, reply_type = socket.IPPROTO_ICMPV6, 128, 129
    else:
        proto, echo_type, reply_type = socket.IPPROTO_ICMP, 8, 0
    identifier = os.getpid() & 0xFFFF
    body = struct.pack(">BBHHH", echo_type, 0, 0, identifier, 1) + b"infradeck"
    packet = struct.pack(">BBHHH", echo_type, 0, _icmp_checksum(body), identifier, 1) + b"infradeck"
    started = time.monotonic()
    try:
        with socket.socket(family, socket.SOCK_DGRAM, proto) as sock:
            sock.settimeout(check["timeout"])
            sock.sendto(packet, (address, 0))
            deadline = time.monotonic() + check["timeout"]
            while time.monotonic() < deadline:
                sock.settimeout(max(0.05, deadline - time.monotonic()))
                reply, _ = sock.recvfrom(1024)
                if reply and reply[0] == reply_type:
                    return {"ok": True, "response_ms": int((time.monotonic() - started) * 1000), "code": None, "detail": None, "cert_days": None}
    except PermissionError:
        return {"ok": False, "response_ms": None, "code": None, "detail": "ICMP is not permitted on this collector (ping_group_range)", "cert_days": None}
    except (OSError, struct.error) as exc:
        return {"ok": False, "response_ms": None, "code": None, "detail": _failure(exc, check["timeout"]), "cert_days": None}
    return {"ok": False, "response_ms": None, "code": None, "detail": "No reply", "cert_days": None}


def run_tcp(check: dict[str, Any]) -> dict[str, Any]:
    started = time.monotonic()
    try:
        with socket.create_connection((check["host"], check["port"]), timeout=check["timeout"]):
            pass
    except OSError as exc:
        return {"ok": False, "response_ms": None, "code": None, "detail": _failure(exc, check["timeout"]), "cert_days": None}
    return {"ok": True, "response_ms": int((time.monotonic() - started) * 1000), "code": None, "detail": None, "cert_days": None}


_RUNNERS = {"tcp": lambda c: run_tcp(c), "tls": lambda c: run_tls(c), "dns": lambda c: run_dns(c), "ping": lambda c: run_ping(c)}


def run_check(check: dict[str, Any]) -> dict[str, Any]:
    try:
        outcome = _RUNNERS.get(check["type"], run_http)(check)
    except Exception as exc:  # one broken check must never stop the others
        outcome = {"ok": False, "response_ms": None, "code": None, "detail": f"Check failed: {type(exc).__name__}", "cert_days": None}
    return {"check_id": check["id"], "time": datetime.now(timezone.utc).isoformat(), **{key: value for key, value in outcome.items() if value is not None or key == "ok"}}


class _ApiRedirect(urllib.error.URLError):
    """The dashboard's address answered with a redirect, which calls to it never follow."""


class _NoRedirects(urllib.request.HTTPRedirectHandler):
    def redirect_request(self, req, fp, code, msg, headers, newurl):  # noqa: ANN001
        target = urllib.parse.urlsplit(urllib.parse.urljoin(req.full_url, newurl))
        raise _ApiRedirect(f"INFRADECK_API_URL redirects to {target.scheme}://{target.netloc}; set it to the final address")


# Calls to the dashboard carry this collector's token, so they never follow a redirect: a proxy in front of the API
# (a sign-in page, an http-to-https redirect) would otherwise get the token sent to wherever it points (#205).
_API_OPENER = urllib.request.build_opener(_NoRedirects())


def _api(request: urllib.request.Request) -> dict[str, Any]:
    """Send ``request`` to the dashboard and read its JSON answer, at most MAX_BODY_BYTES of it."""
    if urllib.parse.urlsplit(request.full_url).scheme not in ("http", "https"):
        raise ValueError("INFRADECK_API_URL must start with http:// or https://")
    with _API_OPENER.open(request, timeout=15) as response:
        body = response.read(MAX_BODY_BYTES + 1)
    if len(body) > MAX_BODY_BYTES:
        raise ValueError("The dashboard's answer is too large")
    return json.loads(body.decode())


def send_check_results(api_url: str, server_id: int, token: str, results: list[dict[str, Any]], tests: list[dict[str, Any]] | None = None) -> dict[str, Any]:
    request = urllib.request.Request(
        f"{api_url.rstrip('/')}/monitoring/metrics/checks",
        data=json.dumps({"server_id": server_id, "results": results, "tests": tests or []}).encode(),
        headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
        method="POST",
    )
    return _api(request)


MAX_TESTS = 5


class CheckRunner(threading.Thread):
    """Runs the checks the dashboard gave this collector, each on its interval (the retry interval after a
    failure), and reports the results every few seconds; with nothing to report it asks for its checks every
    10 s (as the dashboard says), so a test from the New check form is picked up quickly. A test runs once, at
    once, and its result goes back with the next report. Results that couldn't be sent wait, up to the newest 500."""

    def __init__(self, api_url: str, credentials: tuple[int, str]) -> None:
        super().__init__(name="checks", daemon=True)
        self.api_url = api_url
        self.credentials = credentials   # the main loop replaces this after joining again
        self.checks: dict[int, dict[str, Any]] = {}
        self.next_run: dict[int, float] = {}
        self.running: set[int] = set()
        self.pending: collections.deque = collections.deque(maxlen=MAX_PENDING_RESULTS)
        self.lock = threading.Lock()
        self.pool = concurrent.futures.ThreadPoolExecutor(max_workers=CHECK_WORKERS, thread_name_prefix="check")
        self.report_every = 5
        self.idle_every = IDLE_REPORT_SECONDS
        self.test_results: collections.deque = collections.deque(maxlen=MAX_TESTS * 2)

    def run_tests(self, raw: Any) -> None:
        """Run each test once, right away; its result is reported apart from the checks' results."""
        for test in (validate_check(item) for item in (raw if isinstance(raw, list) else [])[:MAX_TESTS]):
            if test:
                self.pool.submit(run_check, test).add_done_callback(lambda future: self._tested(future))

    def _tested(self, future: concurrent.futures.Future) -> None:
        with self.lock:
            self.test_results.append(future.result())

    def set_checks(self, raw: Any) -> None:
        valid = [check for check in (validate_check(item) for item in (raw if isinstance(raw, list) else [])[:MAX_CHECKS]) if check]
        with self.lock:
            new = {check["id"]: check for check in valid}
            for check_id, check in new.items():
                if self.checks.get(check_id) != check:
                    self.next_run[check_id] = time.monotonic()   # new or changed: run it now
            self.checks = new
            for check_id in list(self.next_run):
                if check_id not in new:
                    del self.next_run[check_id]

    def _done(self, check: dict[str, Any], future: concurrent.futures.Future) -> None:
        result = future.result()
        with self.lock:
            self.running.discard(check["id"])
            self.pending.append(result)
            if check["id"] in self.checks:
                self.next_run[check["id"]] = time.monotonic() + (check["interval"] if result["ok"] else check["retry_interval"])

    def run_due(self) -> None:
        now = time.monotonic()
        with self.lock:
            due = [check for check_id, check in self.checks.items() if self.next_run.get(check_id, 0) <= now and check_id not in self.running]
            self.running.update(check["id"] for check in due)
        for check in due:
            self.pool.submit(run_check, check).add_done_callback(lambda future, check=check: self._done(check, future))

    def report(self) -> bool:
        with self.lock:
            batch, tested = list(self.pending), list(self.test_results)
            self.pending.clear()
            self.test_results.clear()
        server_id, token = self.credentials

        def put_back() -> None:
            with self.lock:
                self.pending.extendleft(reversed(batch))
                self.test_results.extendleft(reversed(tested))

        try:
            response = send_check_results(self.api_url, server_id, token, batch, tested)
        except urllib.error.HTTPError as exc:
            put_back()
            if exc.code == 404:
                self.report_every = self.idle_every = 300   # a dashboard without availability checks
            return False
        except (urllib.error.URLError, OSError, ValueError, http.client.HTTPException):
            put_back()
            return False
        if not isinstance(response, dict):
            response = {}
        self.set_checks(response.get("checks"))
        self.run_tests(response.get("tests"))
        self.report_every = _whole(response.get("report_every"), 2, 300) or 5
        self.idle_every = _whole(response.get("idle_every"), 5, 300) or IDLE_REPORT_SECONDS
        if batch:
            print(f"Reported {len(batch)} check results", flush=True)
        return True

    def run(self) -> None:
        last_report = -IDLE_REPORT_SECONDS
        while True:
            self.run_due()
            since = time.monotonic() - last_report
            # Test results go back at once (the dashboard allows a report every 2 s); check results every few seconds.
            if since >= self.idle_every or (self.test_results and since >= 2) or (self.pending and since >= self.report_every):
                self.report()
                last_report = time.monotonic()
            time.sleep(1)


def send_payload(api_url: str, server_id: int, token: str, payload: dict[str, Any]) -> dict[str, Any]:
    request = urllib.request.Request(
        f"{api_url.rstrip('/')}/monitoring/metrics/ingest",
        data=json.dumps({"server_id": server_id, **payload}).encode(),
        headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
        method="POST",
    )
    return _api(request)


def _machine_id() -> str | None:
    override = os.getenv("INFRADECK_MACHINE_ID")
    if override:
        return override.strip()
    for candidate in ("etc/machine-id", "var/lib/dbus/machine-id"):
        try:
            value = (HOST_ROOT / candidate).read_text().strip()
        except OSError:
            continue
        if value:
            return value
    return None


LEGACY_STATE_DIR = Path("/var/lib/homelab-collector")


def _credentials_path() -> Path:
    """Where this collector keeps its credentials. A collector installed before the rename to Infradeck has its state
    volume mounted at the old path; it keeps using that, so an image update doesn't make it ask to join again."""
    configured = os.getenv("INFRADECK_STATE_DIR")
    if configured:
        return Path(configured) / "credentials.json"
    current = Path("/var/lib/infradeck-collector") / "credentials.json"
    legacy = LEGACY_STATE_DIR / "credentials.json"
    return legacy if not current.exists() and LEGACY_STATE_DIR.is_dir() else current


def load_credentials(path: Path | None = None) -> tuple[int, str] | None:
    path = path or _credentials_path()
    try:
        data = json.loads(path.read_text())
        return int(data["server_id"]), str(data["collector_token"])
    except (OSError, ValueError, KeyError, TypeError):
        return None


def save_credentials(server_id: int, token: str, path: Path | None = None) -> None:
    path = path or _credentials_path()
    path.parent.mkdir(parents=True, exist_ok=True)
    temporary = path.with_suffix(".tmp")
    # Created with mode 0600 from the start, so there is no moment when the token sits in a file others can read.
    descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
    with os.fdopen(descriptor, "w") as handle:
        handle.write(json.dumps({"server_id": server_id, "collector_token": token}))
    temporary.chmod(0o600)
    temporary.replace(path)


def enroll(api_url: str, enroll_key: str) -> tuple[int, str]:
    request = urllib.request.Request(
        f"{api_url.rstrip('/')}/monitoring/collectors/enroll",
        data=json.dumps({"enroll_key": enroll_key, "hostname": _host_hostname(), "machine_id": _machine_id()}).encode(),
        headers={"Content-Type": "application/json"},
        method="POST",
    )
    data = _api(request)
    return int(data["server_id"]), str(data["collector_token"])


def _post_json(url: str, payload: dict[str, Any], bearer: str | None = None) -> dict[str, Any]:
    headers = {"Content-Type": "application/json"}
    if bearer:
        headers["Authorization"] = f"Bearer {bearer}"
    request = urllib.request.Request(url, data=json.dumps(payload).encode(), headers=headers, method="POST")
    return _api(request)


def _error_detail(exc: urllib.error.HTTPError) -> str:
    """The backend's explanation, e.g. that a join token has expired, without trusting its size or shape."""
    try:
        detail = json.loads(exc.read(2048)).get("detail")
        return " ".join(str(detail).split())[:200] if detail else "no details"
    except (ValueError, AttributeError, OSError):
        return "no details"


def request_approval(api_url: str) -> tuple[int, str]:
    """Ask to join, then wait until someone clicks Add on the dashboard. No restart needed."""
    base = api_url.rstrip("/")
    while True:
        try:
            registration = _post_json(f"{base}/monitoring/collectors/register", {"hostname": _host_hostname(), "machine_id": _machine_id(), "os_info": _host_os_info(), "join_token": os.getenv("INFRADECK_JOIN_TOKEN", "").strip() or None, "details": registration_details()})
        except urllib.error.HTTPError as exc:
            print(f"Registration refused (HTTP {exc.code}): {_error_detail(exc)}; retrying in 5 minutes", flush=True)
            time.sleep(300)
            continue
        except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
            print(f"Could not reach {base} ({type(exc).__name__}): {exc}; retrying", flush=True)
            time.sleep(15)
            continue

        secret = registration["request_secret"]
        poll_seconds = max(2, int(registration.get("poll_interval_seconds", 5)))
        print(f"Waiting for approval: open the dashboard and click Add on '{_host_hostname()}' (pairing code {registration['pairing_code']})", flush=True)
        last_status = "pending"
        while True:
            time.sleep(poll_seconds if last_status == "pending" else 60)
            try:
                result = _post_json(f"{base}/monitoring/collectors/register/poll", {}, bearer=secret)
            except urllib.error.HTTPError as exc:
                if exc.code == 404:
                    print("Registration request expired; registering again", flush=True)
                    break
                print(f"Approval check failed (HTTP {exc.code})", flush=True)
                continue
            except (urllib.error.URLError, TimeoutError, OSError, ValueError) as exc:
                print(f"Approval check failed ({type(exc).__name__}): {exc}", flush=True)
                continue
            status = result.get("status")
            if status == "approved":
                return int(result["server_id"]), str(result["collector_token"])
            if status != last_status:
                print("Dismissed on the dashboard; it can still be added from there" if status == "dismissed" else "Waiting for approval again", flush=True)
                last_status = status


def resolve_credentials(api_url: str) -> tuple[int, str]:
    """Explicit env credentials win, then saved credentials, then an enrollment key, then dashboard approval."""
    if os.getenv("INFRADECK_SERVER_ID") and os.getenv("INFRADECK_COLLECTOR_TOKEN"):
        return int(os.environ["INFRADECK_SERVER_ID"]), os.environ["INFRADECK_COLLECTOR_TOKEN"]
    saved = load_credentials()
    if saved:
        return saved
    credentials = None
    enroll_key = os.getenv("INFRADECK_ENROLL_KEY", "").strip()
    if enroll_key:
        try:
            credentials = enroll(api_url, enroll_key)
        except urllib.error.HTTPError as exc:
            print(f"Enrollment key rejected (HTTP {exc.code}); falling back to dashboard approval", flush=True)
        except (urllib.error.URLError, TimeoutError, OSError, ValueError, KeyError) as exc:
            print(f"Enrollment failed ({type(exc).__name__}): {exc}; falling back to dashboard approval", flush=True)
    if credentials is None:
        credentials = request_approval(api_url)
    try:
        save_credentials(*credentials)
    except OSError as exc:
        print(f"Could not save credentials ({exc}); mount a volume at {_credentials_path().parent} so restarts don't need approval again", flush=True)
    print(f"Registered as server {credentials[0]}", flush=True)
    return credentials


def run_collector() -> None:
    api_url = os.getenv("INFRADECK_API_URL", "http://localhost:8060")
    server_id, token = resolve_credentials(api_url)
    interval = max(15, int(os.getenv("INFRADECK_CHECK_INTERVAL_SECONDS", "60")))
    print(f"Collector started for server {server_id}; API={api_url}", flush=True)
    checks = CheckRunner(api_url, (server_id, token))
    checks.start()
    while True:
        started = time.monotonic()
        try:
            server, host = collect_host()
            containers, docker = collect_docker()
            host.update(docker)
            # How long gathering took, so the dashboard can warn when it no longer fits in the check interval.
            host["collect_duration_ms"] = int((time.monotonic() - started) * 1000)
            response = send_payload(api_url, server_id, token, {"server": server, "host": host, "containers": containers})
            interval = max(15, int(response.get("interval_seconds", interval)))
            print(f"Sent host metrics and {len(containers)} container records; next check in {interval}s", flush=True)
            delay = max(0, interval - (time.monotonic() - started))
        except urllib.error.HTTPError as exc:
            if exc.code in (401, 404) and not os.getenv("INFRADECK_SERVER_ID"):
                # Removed from the dashboard: ask to join again. This needs a fresh Add, so a
                # deleted server never comes back on its own.
                print(f"Backend no longer accepts server {server_id} (HTTP {exc.code}); requesting approval again", flush=True)
                _credentials_path().unlink(missing_ok=True)
                server_id, token = request_approval(api_url)
                checks.credentials = (server_id, token)
                try:
                    save_credentials(server_id, token)
                except OSError:
                    pass
                delay = 0
            elif exc.code in (401, 404):
                print(f"Backend rejected server {server_id} (HTTP {exc.code}); check INFRADECK_SERVER_ID / INFRADECK_COLLECTOR_TOKEN", flush=True)
                delay = 300
            else:
                print(f"Send failed (HTTP {exc.code})", flush=True)
                delay = min(60, max(15, interval))
        except (urllib.error.URLError, TimeoutError, OSError, ValueError, KeyError, http.client.HTTPException) as exc:
            print(f"Collection/send failed ({type(exc).__name__}): {exc}", flush=True)
            delay = min(60, max(15, interval))
        time.sleep(delay)


if __name__ == "__main__":
    run_collector()