Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.
Python ai-agentsaiopsdashboarddockerfastapihomelabincident-managementllmmonitoringopenaireactroot-cause-analysisself-hosteduptime-monitoring
import json
import os
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
os.environ.setdefault("DATABASE_URL", "sqlite://")
os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0")
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from src.collector import collector
from src.backend.routes import monitoring as monitoring_routes
from src.backend import alerts, main, rules
from src.backend.models import Base, FirewallState, HostMetric, IncidentAlert, ListenerState, Server, SystemEvent, User
TOKEN = "collector-token"
# --- collector: host network totals ----------------------------------------------------------------
def _dev_line(name, received, sent):
return f"{name:>8}: {received} 1 0 0 0 0 0 0 {sent} 1 0 0 0 0 0 0"
DEV_HEADER = "Inter-| Receive | Transmit\n face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n"
def _fake_host(tmp_path, monkeypatch, host_dev=None, own_dev=None, physical=("eno2", "wlo1")):
proc, root = tmp_path / "proc", tmp_path / "root"
(proc / "1/net").mkdir(parents=True)
(proc / "net").mkdir()
if host_dev is not None:
(proc / "1/net/dev").write_text(DEV_HEADER + "\n".join(host_dev) + "\n")
if own_dev is not None:
(proc / "net/dev").write_text(DEV_HEADER + "\n".join(own_dev) + "\n")
for name in ("lo", "docker0", "veth123", "eno2", "wlo1"):
(root / "sys/class/net" / name).mkdir(parents=True)
if name in physical:
(root / "sys/class/net" / name / "device").mkdir()
monkeypatch.setattr(collector, "HOST_PROC", proc)
monkeypatch.setattr(collector, "HOST_ROOT", root)
return proc, root
def test_network_totals_come_from_the_hosts_physical_interfaces(tmp_path, monkeypatch):
_fake_host(tmp_path, monkeypatch,
host_dev=[_dev_line("lo", 999, 999), _dev_line("eno2", 1000, 400), _dev_line("wlo1", 500, 100), _dev_line("docker0", 777, 777), _dev_line("veth123", 888, 888)],
own_dev=[_dev_line("lo", 5, 5), _dev_line("eth0", 7, 3)])
# The collector container's own eth0 is never used, and Docker's virtual interfaces would double count.
assert collector._network_totals() == (500, 1500)
def test_without_sysfs_virtual_interfaces_are_skipped_by_name(tmp_path, monkeypatch):
_proc, root = _fake_host(tmp_path, monkeypatch, host_dev=[_dev_line("lo", 9, 9), _dev_line("eno2", 100, 50), _dev_line("veth9", 70, 70), _dev_line("br-abc", 60, 60), _dev_line("docker0", 40, 40)])
import shutil
shutil.rmtree(root / "sys")
assert collector._network_totals() == (50, 100)
def test_inside_a_container_an_unreadable_host_table_is_unknown_not_the_containers_own(tmp_path, monkeypatch):
_fake_host(tmp_path, monkeypatch, host_dev=None, own_dev=[_dev_line("eth0", 7, 3)])
assert collector._network_totals() is None
def test_only_a_collector_on_the_host_itself_may_fall_back_to_proc_net(monkeypatch):
monkeypatch.setattr(collector, "HOST_PROC", Path("/host/proc"))
assert collector._host_net_dirs() == [Path("/host/proc/1/net")]
monkeypatch.setattr(collector, "HOST_PROC", Path("/proc"))
assert collector._host_net_dirs() == [Path("/proc/1/net"), Path("/proc/net")]
def test_garbage_and_oversized_network_tables_are_survived(tmp_path, monkeypatch):
proc, _root = _fake_host(tmp_path, monkeypatch, host_dev=[_dev_line("eno2", 100, 50), "garbage without a colon", "eno2: not numbers at all", _dev_line("wlo1", 1, 1)])
assert collector._network_totals() == (51, 101)
(proc / "1/net/dev").write_bytes(b"x" * (collector.MAX_NET_TABLE_BYTES + 10))
assert collector._network_totals() is None
# --- collector: listening ports ---------------------------------------------------------------------
TCP_HEADER = " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n"
ZERO6 = "0" * 32
def _sock(address, port, state, remote="00000000:0000"):
return f" 0: {address}:{port:04X} {remote} {state} 00000000:00000000 00:00000000 00000000 0 0 1 1 0"
def _net_tables(tmp_path, monkeypatch, **tables):
proc = tmp_path / "proc"
(proc / "1/net").mkdir(parents=True)
for name, lines in tables.items():
(proc / "1/net" / name).write_text(TCP_HEADER + "\n".join(lines) + "\n")
monkeypatch.setattr(collector, "HOST_PROC", proc)
def test_listeners_are_read_with_their_scope(tmp_path, monkeypatch):
_net_tables(tmp_path, monkeypatch,
tcp=[_sock("0100007F", 5432, "0A"), _sock("00000000", 80, "0A"), _sock("0100A8C0", 8080, "0A"), _sock("00000000", 443, "01", "0100A8C0:C350")],
tcp6=[_sock(ZERO6, 22, "0A"), _sock("00000000000000000000000001000000", 631, "0A"), _sock("0000000000000000FFFF00000100007F", 9000, "0A")],
udp=[_sock("00000000", 5353, "07"), _sock("00000000", 5355, "01", "0100A8C0:0035")], udp6=[])
found = {(item["proto"], item["port"], item["scope"]) for item in collector.collect_listeners()}
assert found == {("tcp", 5432, "loopback"), ("tcp", 80, "any"), ("tcp", 8080, "specific"), ("tcp", 22, "any"), ("tcp", 631, "loopback"), ("tcp", 9000, "loopback"), ("udp", 5353, "any")}
def test_the_same_port_on_ipv4_and_ipv6_is_one_listener(tmp_path, monkeypatch):
_net_tables(tmp_path, monkeypatch, tcp=[_sock("00000000", 80, "0A")], tcp6=[_sock(ZERO6, 80, "0A")], udp=[], udp6=[])
assert collector.collect_listeners() == [{"proto": "tcp", "port": 80, "scope": "any"}]
@pytest.mark.parametrize("proto, port, scope, tracked", [
("tcp", 80, "any", True), ("tcp", 5432, "loopback", True), ("udp", 5353, "any", True), ("udp", 53, "loopback", True),
("tcp", 45983, "loopback", False), ("udp", 40000, "any", False), ("udp", 40000, "specific", False),
("tcp", 40000, "any", True), ("tcp", 40000, "specific", True),
])
def test_ephemeral_ports_are_skipped_but_exposed_high_tcp_ports_are_not(proto, port, scope, tracked):
assert collector.listener_tracked(proto, port, scope) is tracked
assert rules.listener_tracked(proto, port, scope) is tracked
def test_missing_or_garbage_tables_are_unknown_or_ignored(tmp_path, monkeypatch):
_net_tables(tmp_path, monkeypatch)
assert collector.collect_listeners() is None # nothing readable
proc = tmp_path / "proc"
for name in ("tcp6", "udp", "udp6"):
(proc / "1/net" / name).write_text(TCP_HEADER)
(proc / "1/net/tcp").write_text(TCP_HEADER + "\n".join(["garbage", "", "1: nonsense", "0: ZZZ:GG 00000000:0000 0A", "0: 0100007F:0000 00000000:0000 0A", "0: 0100007F:FFFFF 00000000:0000 0A", _sock("00000000", 22, "0A")]) + "\n")
assert collector.collect_listeners() == [{"proto": "tcp", "port": 22, "scope": "any"}]
def test_one_unreadable_table_makes_the_whole_answer_unknown_not_partial(tmp_path, monkeypatch):
_net_tables(tmp_path, monkeypatch, tcp=[_sock("00000000", 22, "0A")], tcp6=[], udp=[]) # udp6 is missing
assert collector.collect_listeners() is None
def test_a_table_longer_than_the_cap_is_unknown_because_a_truncated_read_could_fake_closed_ports(tmp_path, monkeypatch):
monkeypatch.setattr(collector, "MAX_NET_TABLE_LINES", 100)
_net_tables(tmp_path, monkeypatch, tcp=[_sock("00000000", 22, "0A")] + [_sock("0100A8C0", 40000 + number, "01", "0100A8C0:C350") for number in range(150)], tcp6=[], udp=[], udp6=[])
assert collector.collect_listeners() is None
def test_a_busy_hosts_connection_table_does_not_hide_its_listeners(tmp_path, monkeypatch):
connections = [_sock("0100A8C0", 20000 + number, "01", "0100A8C0:C350") for number in range(5000)]
_net_tables(tmp_path, monkeypatch, tcp=[_sock("00000000", 443, "0A")] + connections, tcp6=[], udp=[], udp6=[])
assert collector.collect_listeners() == [{"proto": "tcp", "port": 443, "scope": "any"}]
def test_a_container_never_reports_its_own_sockets_as_the_hosts(tmp_path, monkeypatch):
proc = tmp_path / "proc"
(proc / "net").mkdir(parents=True)
(proc / "net/tcp").write_text(TCP_HEADER + _sock("00000000", 9999, "0A") + "\n")
monkeypatch.setattr(collector, "HOST_PROC", proc) # a container's /proc/net only has the container's sockets
assert collector.collect_listeners() is None
def test_the_listener_report_is_capped(tmp_path, monkeypatch):
monkeypatch.setattr(collector, "MAX_NET_TABLE_LINES", 10_000)
_net_tables(tmp_path, monkeypatch, tcp=[_sock("00000000", port, "0A") for port in range(1, 700)], tcp6=[], udp=[], udp6=[])
assert len(collector.collect_listeners()) == collector.MAX_LISTENERS
# --- collector: connectivity ------------------------------------------------------------------------
def test_connectivity_is_off_unless_targets_are_configured(monkeypatch):
monkeypatch.delenv("INFRADECK_CONNECTIVITY_TARGETS", raising=False)
assert collector.collect_connectivity() == {"internet_status": None, "internet_latency_ms": None}
@pytest.mark.parametrize("value, expected", [
("1.1.1.1:443", [("1.1.1.1", 443)]),
("example.com:443, 9.9.9.9:53 ,example.com:443", [("example.com", 443), ("9.9.9.9", 53)]),
("a.com:1,b.com:2,c.com:3,d.com:4,e.com:5,f.com:6,g.com:7", [("a.com", 1), ("b.com", 2), ("c.com", 3), ("d.com", 4), ("e.com", 5)]),
])
def test_valid_targets_are_parsed_deduplicated_and_capped(monkeypatch, value, expected):
monkeypatch.setenv("INFRADECK_CONNECTIVITY_TARGETS", value)
assert collector._connectivity_targets() == expected
@pytest.mark.parametrize("bad", ["", "nohost", ":443", "host:", "host:0", "host:65536", "host:abc", "host:-1", "http://example.com:443", "exa mple.com:443", "example.com;rm -rf /:443",
"999.1.1.1:443", "1.1.1:443", "[::1]:443", "-bad.com:443", "a" * 300 + ".com:443", "user@host:443", "host:443/path", "host\n:443"])
def test_malformed_targets_are_ignored(monkeypatch, bad):
monkeypatch.setenv("INFRADECK_CONNECTIVITY_TARGETS", bad)
assert collector._connectivity_targets() == []
@pytest.mark.parametrize("results, status", [
([{"dns": None, "tcp": True}, {"dns": True, "tcp": True}], "ok"),
([{"dns": None, "tcp": True}, {"dns": True, "tcp": False}], "degraded"),
([{"dns": False, "tcp": False}, {"dns": False, "tcp": False}], "dns_failed"),
([{"dns": None, "tcp": False}, {"dns": True, "tcp": False}], "down"),
([{"dns": False, "tcp": False}, {"dns": None, "tcp": False}], "down"),
])
def test_results_are_summarised(results, status):
assert collector.summarise_connectivity(results) == status
def test_a_reachable_target_reports_ok_with_its_latency(monkeypatch):
monkeypatch.setenv("INFRADECK_CONNECTIVITY_TARGETS", "example.com:443,1.1.1.1:443")
monkeypatch.setattr(collector.socket, "getaddrinfo", lambda host, port, **kw: [(None, None, None, "", ("93.184.216.34", port))])
class Connection:
def close(self):
pass
monkeypatch.setattr(collector.socket, "create_connection", lambda address, timeout: Connection())
result = collector.collect_connectivity()
assert result["internet_status"] == "ok" and isinstance(result["internet_latency_ms"], int)
def test_unresolvable_names_are_reported_as_dns_failure(monkeypatch):
monkeypatch.setenv("INFRADECK_CONNECTIVITY_TARGETS", "example.com:443")
monkeypatch.setattr(collector.socket, "getaddrinfo", lambda *a, **k: (_ for _ in ()).throw(OSError("no dns")))
assert collector.collect_connectivity()["internet_status"] == "dns_failed"
def test_a_hung_probe_cannot_stall_the_check_in(monkeypatch):
monkeypatch.setenv("INFRADECK_CONNECTIVITY_TARGETS", "1.1.1.1:443")
monkeypatch.setattr(collector, "CONNECTIVITY_DEADLINE_SECONDS", 0.3)
monkeypatch.setattr(collector.socket, "create_connection", lambda address, timeout: time.sleep(3))
started = time.monotonic()
result = collector.collect_connectivity()
assert time.monotonic() - started < 1.5 and result["internet_status"] == "down"
def test_probes_only_connect_and_never_send_data(monkeypatch):
monkeypatch.setenv("INFRADECK_CONNECTIVITY_TARGETS", "1.1.1.1:443")
sent = []
class Connection:
def close(self):
pass
def send(self, data):
sent.append(data)
sendall = send
monkeypatch.setattr(collector.socket, "create_connection", lambda address, timeout: Connection())
collector.collect_connectivity()
assert sent == []
# --- collector: firewall files ----------------------------------------------------------------------
def _firewall_root(tmp_path, monkeypatch, files):
root = tmp_path / "root"
for relative, content in files.items():
path = root / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(content if isinstance(content, bytes) else content.encode())
root.mkdir(exist_ok=True)
monkeypatch.setattr(collector, "HOST_ROOT", root)
return root
UFW_RULES = "*filter\n:ufw-user-input - [0:0]\n-A ufw-user-input -p tcp --dport 22 -j ACCEPT\n-A ufw-user-input -p tcp --dport 8443 -s 10.0.0.0/8 -j ACCEPT\nCOMMIT\n"
def test_firewall_files_are_reduced_to_a_hash_a_rule_count_and_the_ufw_flag(tmp_path, monkeypatch):
_firewall_root(tmp_path, monkeypatch, {"etc/ufw/user.rules": UFW_RULES, "etc/ufw/ufw.conf": "# comment\nENABLED=yes\nLOGLEVEL=low\n", "etc/nftables.conf": "#!/usr/sbin/nft -f\n\nflush ruleset\ntable inet filter {\n}\n"})
report = collector.collect_firewall()
by_name = {item["name"]: item for item in report["files"]}
assert by_name["ufw"]["rules"] == 2 and by_name["nftables"]["rules"] == 3 and report["ufw_enabled"] is True
assert all(len(item["sha"]) == 16 for item in report["files"])
def test_the_firewall_report_never_contains_the_rules_themselves(tmp_path, monkeypatch):
_firewall_root(tmp_path, monkeypatch, {"etc/ufw/user.rules": UFW_RULES + "# secret-internal-subnet 10.99.0.0/16\n"})
dumped = json.dumps(collector.collect_firewall())
assert "10.0.0.0" not in dumped and "8443" not in dumped and "secret-internal-subnet" not in dumped
def test_ufw_disabled_is_reported_and_changes_the_fingerprint(tmp_path, monkeypatch):
_firewall_root(tmp_path, monkeypatch, {"etc/ufw/ufw.conf": "ENABLED=yes\n"})
enabled = collector.collect_firewall()
_firewall_root(tmp_path, monkeypatch, {"etc/ufw/ufw.conf": "ENABLED=no\n"})
disabled = collector.collect_firewall()
assert (enabled["ufw_enabled"], disabled["ufw_enabled"]) == (True, False) and enabled["files"][0]["sha"] != disabled["files"][0]["sha"]
def test_no_known_firewall_file_means_none(tmp_path, monkeypatch):
_firewall_root(tmp_path, monkeypatch, {})
assert collector.collect_firewall() is None
def test_symlinked_oversized_and_binary_firewall_files_are_handled_safely(tmp_path, monkeypatch):
root = _firewall_root(tmp_path, monkeypatch, {"etc/ufw/user6.rules": "x", "etc/iptables/rules.v6": b"\xff\xfe\x00" * 10})
secret = tmp_path / "secret.txt"
secret.write_text("root:hash")
(root / "etc/ufw/user.rules").symlink_to(secret) # must not be followed
(root / "etc/nftables.conf").write_bytes(b"#" * (collector.MAX_FIREWALL_FILE_BYTES + 1)) # too large
names = {item["name"] for item in collector.collect_firewall()["files"]}
assert names == {"ufw6", "ip6tables"}
# --- backend ---------------------------------------------------------------------------------------
@pytest.fixture
def db():
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
session = sessionmaker(bind=engine)()
session.add(User(id=1, name="Owner", email="[email protected]", password_hash="x", role="admin"))
session.add(Server(id=5, owner_id=1, hostname="nas", check_interval_seconds=60, collector_token_hash=main._hash_secret(TOKEN)))
session.commit()
return session
def _ingest(db, **host):
monitoring_routes.ingest_metrics(main.MetricsIngestRequest(server_id=5, host=host), authorization=f"Bearer {TOKEN}", db=db)
def _l(proto, port, scope="any"):
return {"proto": proto, "port": port, "scope": scope}
BASE_PORTS = [_l("tcp", 22), _l("tcp", 5432, "loopback")]
def _active(db):
return {alert.rule_key: alert for alert in db.query(IncidentAlert).filter_by(status="active")}
def _events(db, *kinds):
return [(event.kind, event.detail, event.event_key) for event in db.query(SystemEvent).filter(SystemEvent.kind.in_(kinds or rules.NETWORK_EVENT_KINDS)).order_by(SystemEvent.id)]
@pytest.mark.parametrize("raw", [None, 5, "x", {}, {"proto": "tcp"}])
def test_a_listener_report_must_be_a_list(raw):
assert rules.validate_listeners(raw) is None
def test_listener_entries_are_validated_deduplicated_and_filtered():
raw = [_l("tcp", 80), _l("tcp", 80), _l("sctp", 80), _l("tcp", 0), _l("tcp", 65536), _l("tcp", "80"), _l("tcp", True), _l("tcp", 1.5), _l("tcp", 80, "everywhere"),
{"proto": "tcp"}, None, 5, "x", _l("tcp", 45000, "loopback"), _l("udp", 45000), _l("udp", 53, "specific")]
assert rules.validate_listeners(raw) == [("tcp", 80, "any"), ("udp", 53, "specific")]
def test_the_listener_list_is_capped():
assert len(rules.validate_listeners([_l("tcp", port) for port in range(1, 2000)])) == rules.MAX_LISTENERS
def test_firewall_reports_are_validated():
sha = "0123456789abcdef"
assert rules.validate_firewall(None) == {"files": {}, "ufw_enabled": None}
assert rules.validate_firewall("x") is None and rules.validate_firewall({"files": "x"}) is None and rules.validate_firewall({}) is None
good = {"files": [{"name": "ufw", "sha": sha, "rules": 3}], "ufw_enabled": True}
assert rules.validate_firewall(good) == {"files": {"ufw": (sha, 3)}, "ufw_enabled": True}
junk = {"files": [{"name": "passwd", "sha": sha, "rules": 1}, {"name": "ufw", "sha": "NOT-HEX", "rules": 1}, {"name": "ufw6", "sha": sha, "rules": -1}, {"name": "ufw6", "sha": sha, "rules": "5"},
{"name": "ufw6", "sha": sha, "rules": True}, {"name": "nftables", "sha": sha, "rules": 10**9}, {"name": "iptables", "sha": sha, "rules": 2}, {"name": "iptables", "sha": sha, "rules": 9}, 5, None], "ufw_enabled": "yes"}
assert rules.validate_firewall(junk) == {"files": {"iptables": (sha, 2)}, "ufw_enabled": None}
def test_the_first_listener_report_is_only_a_baseline(db):
_ingest(db, listeners=BASE_PORTS)
assert _events(db) == [] and db.query(ListenerState).count() == 2
def test_a_new_exposed_listener_is_recorded_and_raises_a_warning(db):
_ingest(db, listeners=BASE_PORTS)
_ingest(db, listeners=BASE_PORTS + [_l("tcp", 8080)])
assert _events(db) == [("port_opened", "New listener: tcp/8080 (all interfaces)", "tcp/8080/any")]
alert = _active(db)["network:port:tcp/8080"]
assert alert.severity == alerts.WARNING and "8080" in alert.title
def test_a_new_loopback_listener_is_an_event_without_an_incident(db):
_ingest(db, listeners=BASE_PORTS)
_ingest(db, listeners=BASE_PORTS + [_l("tcp", 6379, "loopback")])
assert [kind for kind, _d, _k in _events(db)] == ["port_opened"] and not [key for key in _active(db) if key.startswith("network:port")]
def test_a_closed_listener_is_recorded_and_clears_its_incident(db):
_ingest(db, listeners=BASE_PORTS)
_ingest(db, listeners=BASE_PORTS + [_l("tcp", 8080)])
_ingest(db, listeners=BASE_PORTS)
assert [kind for kind, _d, _k in _events(db)] == ["port_opened", "port_closed"]
assert "network:port:tcp/8080" not in _active(db) and db.query(ListenerState).count() == 2
def test_the_listener_incident_clears_with_the_event_window(db):
_ingest(db, listeners=BASE_PORTS)
_ingest(db, listeners=BASE_PORTS + [_l("tcp", 8080)])
db.query(SystemEvent).update({"time": datetime.now(timezone.utc) - timedelta(hours=3)})
db.commit()
_ingest(db, listeners=BASE_PORTS + [_l("tcp", 8080)])
assert "network:port:tcp/8080" not in _active(db)
def test_unchanged_listeners_create_no_events_or_duplicate_rows(db):
for _ in range(3):
_ingest(db, listeners=BASE_PORTS)
assert _events(db) == [] and db.query(ListenerState).count() == 2
def test_an_empty_or_missing_listener_report_never_looks_like_everything_closed(db):
_ingest(db, listeners=BASE_PORTS)
_ingest(db, listeners=[])
_ingest(db, listeners=None)
_ingest(db) # an old collector that doesn't send the field
_ingest(db, listeners="garbage")
assert _events(db) == [] and db.query(ListenerState).count() == 2
def test_a_burst_of_new_listeners_is_capped_with_one_notice(db):
_ingest(db, listeners=BASE_PORTS)
_ingest(db, listeners=BASE_PORTS + [_l("tcp", port) for port in range(9000, 9030)])
kinds = [kind for kind, _d, _k in _events(db)]
assert kinds.count("port_opened") == rules.MAX_NETWORK_EVENTS_PER_CHECKIN and kinds.count("network_notice") == 1
def test_sustained_port_flapping_is_capped_per_hour(db):
_ingest(db, listeners=BASE_PORTS)
for round_number in range(12):
extra = [_l("tcp", 9000 + number + round_number * 100) for number in range(15)]
_ingest(db, listeners=BASE_PORTS + extra)
kinds = [kind for kind, _d, _k in _events(db)]
assert len(kinds) <= rules.MAX_NETWORK_EVENTS_PER_HOUR + 1 and kinds.count("network_notice") == 1
def test_listener_state_is_removed_with_the_server(db, monkeypatch):
monkeypatch.setattr(main, "_require_admin", lambda session_id, db: {"id": 1})
_ingest(db, listeners=BASE_PORTS, firewall={"files": [{"name": "ufw", "sha": "0123456789abcdef", "rules": 1}], "ufw_enabled": None})
monitoring_routes.delete_monitored_server(5, session_id="s", db=db)
assert db.query(ListenerState).count() == 0 and db.query(FirewallState).count() == 0
def _fw(*files, enabled=None):
return {"files": [{"name": name, "sha": sha, "rules": count} for name, sha, count in files], "ufw_enabled": enabled}
SHA_A, SHA_B, SHA_C = "a" * 16, "b" * 16, "c" * 16
def test_the_first_firewall_report_is_only_a_baseline(db):
_ingest(db, firewall=_fw(("ufw", SHA_A, 4), ("ufw-conf", SHA_B, 0), enabled=True))
assert _events(db) == [] and db.query(FirewallState).count() == 2 and not [k for k in _active(db) if k.startswith("network:")]
def test_unchanged_firewall_files_report_nothing(db):
for _ in range(3):
_ingest(db, firewall=_fw(("ufw", SHA_A, 4)))
assert _events(db) == []
def test_changed_rules_are_informational_and_fewer_rules_are_a_warning(db):
_ingest(db, firewall=_fw(("ufw", SHA_A, 4)))
_ingest(db, firewall=_fw(("ufw", SHA_B, 6)))
assert _events(db) == [("firewall_changed", "ufw IPv4 rules changed (4 to 6 rules)", "ufw")]
assert _active(db)["network:firewall"].severity == alerts.INFO
_ingest(db, firewall=_fw(("ufw", SHA_C, 2)))
assert [kind for kind, _d, _k in _events(db)][-1] == "firewall_weakened"
assert _active(db)["network:firewall"].severity == alerts.WARNING
def test_disabling_ufw_is_a_warning_and_enabling_it_is_informational(db):
_ingest(db, firewall=_fw(("ufw-conf", SHA_A, 0), enabled=True))
_ingest(db, firewall=_fw(("ufw-conf", SHA_B, 0), enabled=False))
assert _events(db) == [("firewall_weakened", "ufw was enabled and is now disabled", "ufw-conf")]
assert _active(db)["network:firewall"].severity == alerts.WARNING
_ingest(db, firewall=_fw(("ufw-conf", SHA_C, 0), enabled=True))
assert _events(db)[-1][:2] == ("firewall_changed", "ufw was disabled and is now enabled")
def test_a_firewall_file_disappearing_is_a_warning_and_one_appearing_is_informational(db):
_ingest(db, firewall=_fw(("ufw", SHA_A, 4), ("nftables", SHA_B, 9)))
_ingest(db, firewall=_fw(("nftables", SHA_B, 9), ("iptables", SHA_C, 1)))
assert _events(db) == [("firewall_changed", "iptables rules.v4 appeared", "iptables"), ("firewall_weakened", "ufw IPv4 rules is no longer present", "ufw")]
assert db.query(FirewallState).count() == 2
def test_all_firewall_files_vanishing_is_noticed(db):
_ingest(db, firewall=_fw(("ufw", SHA_A, 4)))
_ingest(db, firewall=None)
assert [kind for kind, _d, _k in _events(db)] == ["firewall_weakened"] and db.query(FirewallState).count() == 0
def test_missing_or_malformed_firewall_reports_are_ignored(db):
_ingest(db, firewall=_fw(("ufw", SHA_A, 4)))
_ingest(db)
_ingest(db, firewall="garbage")
_ingest(db, firewall={"files": "x"})
_ingest(db, firewall=_fw(("passwd", SHA_B, 1), ("ufw", "not-a-hash", 1))) # valid shape, but no usable entry
assert [kind for kind, _d, _k in _events(db)] == ["firewall_weakened"] # only the last one: the real file is now absent
def test_the_firewall_incident_clears_with_the_event_window(db):
_ingest(db, firewall=_fw(("ufw", SHA_A, 4)))
_ingest(db, firewall=_fw(("ufw", SHA_B, 5)))
db.query(SystemEvent).update({"time": datetime.now(timezone.utc) - timedelta(hours=3)})
db.commit()
_ingest(db, firewall=_fw(("ufw", SHA_B, 5)))
assert "network:firewall" not in _active(db)
def test_sustained_internet_loss_warns_and_recovery_clears(db):
for _ in range(2):
_ingest(db, internet_status="down")
assert "network:internet" not in _active(db)
_ingest(db, internet_status="down")
assert _active(db)["network:internet"].severity == alerts.WARNING
_ingest(db, internet_status="ok", internet_latency_ms=20)
assert "network:internet" not in _active(db)
def test_sustained_dns_failure_is_its_own_warning(db):
for _ in range(3):
_ingest(db, internet_status="dns_failed")
assert "network:dns" in _active(db) and "network:internet" not in _active(db)
def test_degraded_unchecked_and_malformed_internet_status_never_alerts(db):
for _ in range(4):
_ingest(db, internet_status="degraded")
for _ in range(4):
_ingest(db, internet_status=None)
for _ in range(4):
_ingest(db, internet_status="<script>", internet_latency_ms="fast")
assert not [key for key in _active(db) if key.startswith("network:")]
assert db.query(HostMetric).filter(HostMetric.internet_status == "<script>").count() == 0
def test_internet_latency_must_be_a_sane_number(db):
_ingest(db, internet_status="ok", internet_latency_ms=10**9)
_ingest(db, internet_status="ok", internet_latency_ms=42)
assert [row.internet_latency_ms for row in db.query(HostMetric).order_by(HostMetric.id)] == [None, 42]
def test_the_overview_shows_listeners_firewall_and_network_events(db, monkeypatch):
monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1})
_ingest(db, listeners=BASE_PORTS, firewall=_fw(("ufw-conf", SHA_A, 0), enabled=False), internet_status="ok", internet_latency_ms=12)
_ingest(db, listeners=BASE_PORTS + [_l("tcp", 8080)], firewall=_fw(("ufw-conf", SHA_A, 0), enabled=False), internet_status="ok", internet_latency_ms=12)
server = monitoring_routes.monitoring_overview(session_id="s", db=db)["servers"][0]
assert [(item["proto"], item["port"], item["scope"]) for item in server["network"]["listeners"]] == [("tcp", 22, "any"), ("tcp", 5432, "loopback"), ("tcp", 8080, "any")]
assert server["network"]["firewall"][0]["label"] == "ufw settings" and server["network"]["firewall"][0]["enabled"] is False
assert [event["kind"] for event in server["network"]["events"]] == ["port_opened"]
assert (server["host"]["internet_status"], server["host"]["internet_latency_ms"]) == ("ok", 12)
def test_network_events_have_their_own_list_and_do_not_crowd_the_other_rows(db, monkeypatch):
monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1})
db.add(SystemEvent(server_id=5, time=datetime.now(timezone.utc) - timedelta(days=2), kind="reboot", detail="The host rebooted"))
db.commit()
_ingest(db, listeners=BASE_PORTS)
_ingest(db, listeners=BASE_PORTS + [_l("tcp", port) for port in range(9000, 9008)])
server = monitoring_routes.monitoring_overview(session_id="s", db=db)["servers"][0]
assert [event["kind"] for event in server["system_events"]] == ["reboot"] and server["container_events"] == []
assert len(server["network"]["events"]) == 8
def test_the_network_lists_are_bounded(db, monkeypatch):
monkeypatch.setattr(main, "_current_user", lambda session_id, db: {"id": 1})
now = datetime.now(timezone.utc)
for number in range(300):
db.add(ListenerState(server_id=5, proto="tcp", port=1000 + number, scope="any", first_seen=now, last_seen=now))
for number in range(40):
db.add(SystemEvent(server_id=5, time=now - timedelta(minutes=number), kind="port_opened", detail=f"New listener: tcp/{number}", event_key=f"tcp/{number}/any"))
db.commit()
network = monitoring_routes.monitoring_overview(session_id="s", db=db)["servers"][0]["network"]
assert len(network["listeners"]) == main.LISTENERS_SHOWN and len(network["events"]) == main.NETWORK_EVENTS_SHOWN
def test_network_events_are_kept_for_the_history_retention(db):
now = datetime.now(timezone.utc)
db.add_all([SystemEvent(server_id=5, time=now - timedelta(days=days), kind=kind, detail="x", event_key="x") for kind in rules.NETWORK_EVENT_KINDS for days in (40, 400)])
db.commit()
assert main.retention.purge(db) == len(rules.NETWORK_EVENT_KINDS) # the ones from 400 days ago; a year is the default
def test_crafted_event_keys_cannot_raise_incidents_for_closed_or_loopback_ports(db):
now = datetime.now(timezone.utc)
db.add_all([
SystemEvent(server_id=5, time=now, kind="port_opened", detail="x", event_key="tcp/8080/any"), # not actually open
SystemEvent(server_id=5, time=now, kind="port_opened", detail="x", event_key="garbage"),
SystemEvent(server_id=5, time=now, kind="port_opened", detail="x", event_key="tcp/notaport/any"),
])
db.add_all([ListenerState(server_id=5, proto="tcp", port=5432, scope="loopback", first_seen=now, last_seen=now), ListenerState(server_id=5, proto="tcp", port=22, scope="any", first_seen=now, last_seen=now)])
db.add(SystemEvent(server_id=5, time=now, kind="port_opened", detail="x", event_key="tcp/5432/loopback"))
db.commit()
_ingest(db, listeners=BASE_PORTS)
assert not [key for key in _active(db) if key.startswith("network:port")]