Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.
Python ai-agentsaiopsdashboarddockerfastapihomelabincident-managementllmmonitoringopenaireactroot-cause-analysisself-hosteduptime-monitoring
fix(checks): make a check's timeout a hard limit and show checks without results as stale (#216)
Karlo Kajba Šimanića8238039563c
diff --git a/SECURITY.md b/SECURITY.mdindex 50b2435..87e81b5 100644--- a/SECURITY.md+++ b/SECURITY.md@@ -411,6 +411,11 @@ URL, a keyword or a value in a JSON reply, a TCP port, a TLS certificate, a DNSnever leaves the collector: a result is only up/down, the status code, the response time, the daysuntil a TLS certificate expires, and a short reason (≤200 characters; for a JSON check that reason caninclude up to ~80 characters of the value that failed the comparison).+- **A check's timeout is a hard limit for the whole run.** When it is up, the collector shuts down the+ check's connections, so a target that answers a byte at a time can't hold a check, or the collector's+ few check workers, past it. (A name lookup ends when the system resolver gives up.) A check whose+ results stop arriving turns **stale** on the Availability page within a few intervals instead of+ keeping its last state; an incident it had stays open until results come back.- **Ping uses an unprivileged ICMP datagram socket**, allowed for the container's own group by anamespaced `net.ipv4.ping_group_range` sysctl. It grants no capability and no raw socket (no packetcrafting or sniffing); without the sysctl a ping check simply reports that ICMP isn't permitted. DNSdiff --git a/src/backend/availability.py b/src/backend/availability.pyindex fe42096..f690253 100644--- a/src/backend/availability.py+++ b/src/backend/availability.py@@ -408,13 +408,16 @@ def evaluate(db, check: Check, now: datetime | None = None, alerting: bool = Truelse:states = server_states(db, check, now)known = {server: state for server, state in states.items() if state != "unknown"}- if check.rule == "all":+ if known and check.rule == "all":# A server that should be reporting but hasn't yet keeps the check from counting as down everywhere;# one that can't (offline, or a collector too old to run checks) doesn't hold it up.waiting = [server for server, state in states.items() if state == "unknown" and server in _reporting(db, now)]known.update({server: "retrying" for server in waiting})if not known:- new = "pending" if check.state in ("pending", "paused") else check.state+ # No server has a fresh result: the check isn't run, or its results don't arrive (a collector that is+ # online but whose checks are stuck, too). Its last verdict no longer says anything, so it isn't shown+ # as up (or down) any more (#204).+ new = "pending" if check.state in ("pending", "paused") else "stale"elif check.rule == "all":new = "down" if all(state == "down" for state in known.values()) else "up" if any(state == "up" for state in known.values()) else check.state if check.state in ("up", "down") else "pending"else:@@ -429,11 +432,24 @@ def evaluate(db, check: Check, now: datetime | None = None, alerting: bool = Truwhy = (latest.detail if latest and latest.detail else "No answer") + f" from {', '.join(failing) or 'its servers'}"raise_alert(db, servers[0], rule_key(check), check.severity, f"{check.name} is down"[:255],f"{why}. Failed {check.retries + 1} checks in a row. {redacted_target(check.target)}"[:1000])- elif servers:+ elif servers and new != "stale": # without results nothing is known to be fixed, so an open incident stays openresolve_alerts(db, servers[0], [rule_key(check)], now)return new+def mark_stale(db, now: datetime | None = None) -> int:+ """Re-evaluate the checks whose results stopped arriving, which nothing else would look at again, so they turn+ stale instead of keeping their last verdict. Run every minute. Returns how many changed."""+ now = now or datetime.now(timezone.utc)+ changed = 0+ for check in db.query(Check).filter(Check.paused.is_(False), Check.state.notin_(("pending", "stale"))):+ last = _utc(check.last_result_at or check.state_since or check.created_at)+ if last and now - last > timedelta(seconds=check.interval_seconds * 3 + check.timeout_seconds + 60):+ changed += evaluate(db, check, now, alerting=False) == "stale"+ db.commit()+ return changed++def forget_check(db, check: Check) -> None:"""Delete a check with its history; its open incident is resolved, so the record of it stays."""servers = servers_of(check)diff --git a/src/backend/main.py b/src/backend/main.pyindex 6116850..813e2d3 100644--- a/src/backend/main.py+++ b/src/backend/main.py@@ -1007,6 +1007,7 @@ def _offline_watch_loop() -> None:db = SessionLocal()try:mark_offline_servers(db)+ availability.mark_stale(db)purge_stale_pending(db)purge_due = last_metrics_purge is None or time.monotonic() - last_metrics_purge >= METRICS_PURGE_INTERVAL_SECONDSif metrics_maintenance(db, purge_due):diff --git a/src/collector/collector.py b/src/collector/collector.pyindex cba1600..3e3c7ce 100644--- a/src/collector/collector.py+++ b/src/collector/collector.py@@ -1364,7 +1364,7 @@ def _failure(exc: BaseException, timeout: int) -> str:return f"Timed out after {timeout} s"if isinstance(reason, ssl.SSLCertVerificationError):return f"TLS certificate not valid: {reason.verify_message}"[:200]- if isinstance(reason, ssl.SSLError):+ if isinstance(reason, ssl.SSLError) or isinstance(exc, ssl.SSLError): # an SSLError's own reason is a stringreturn "TLS handshake failed"if isinstance(reason, socket.gaierror):return "Host name not found"@@ -1377,34 +1377,106 @@ def _failure(exc: BaseException, timeout: int) -> str:return type(reason).__name__+class _Deadline:+ """A check's timeout as a hard limit for the whole run (#204). A socket timeout only limits each read, so a+ server that sends its answer a byte at a time could hold a check, and one of the CHECK_WORKERS, for as long as+ it likes. When the time is up this shuts down every socket the check opened, which ends any read or TLS+ handshake still waiting on it. A name lookup can't be interrupted; it ends when the system resolver gives up,+ and the connection after it is then shut down at once."""++ def __init__(self, seconds: float) -> None:+ self.expired = False+ self.sockets: list[socket.socket] = []+ self.lock = threading.Lock()+ self.timer = threading.Timer(seconds, self._expire)+ self.timer.daemon = True++ def __enter__(self) -> "_Deadline":+ self.timer.start()+ return self++ def __exit__(self, *_exc: object) -> None:+ self.timer.cancel()++ def track(self, sock: socket.socket) -> socket.socket:+ with self.lock:+ self.sockets.append(sock)+ if self.expired:+ _shut(sock)+ return sock++ def _expire(self) -> None:+ with self.lock:+ self.expired = True+ for sock in self.sockets:+ _shut(sock)+++def _shut(sock: socket.socket) -> None:+ try:+ sock.shutdown(socket.SHUT_RDWR)+ except OSError:+ pass+++def _tracked(base: type, deadline: _Deadline) -> type:+ """``base`` (an http.client connection class) whose sockets ``deadline`` can shut down."""+ class Tracked(base):+ def __init__(self, *args: Any, **kwargs: Any) -> None:+ super().__init__(*args, **kwargs)+ self._create_connection = lambda *a, **k: deadline.track(socket.create_connection(*a, **k))+ return Tracked+++class _DeadlineHTTP(urllib.request.HTTPHandler):+ def __init__(self, deadline: _Deadline) -> None:+ super().__init__()+ self.deadline = deadline++ def http_open(self, req): # noqa: ANN001, ANN201+ return self.do_open(_tracked(http.client.HTTPConnection, self.deadline), req)+++class _DeadlineHTTPS(urllib.request.HTTPSHandler):+ def __init__(self, context: ssl.SSLContext, deadline: _Deadline) -> None:+ super().__init__(context=context)+ self.deadline = deadline++ def https_open(self, req): # noqa: ANN001, ANN201+ return self.do_open(_tracked(http.client.HTTPSConnection, self.deadline), req, context=self._context)++def run_http(check: dict[str, Any]) -> dict[str, Any]:"""One run of an HTTP(s) or keyword check: {ok, response_ms, code, detail, cert_days}."""context = ssl.create_default_context()if check["ignore_tls"]:context.check_hostname = Falsecontext.verify_mode = ssl.CERT_NONE- opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPSHandler(context=context), _Redirects(check["follow_redirects"]))request = urllib.request.Request(check["target"], method=check["method"], headers={"User-Agent": f"Infradeck-Check/{COLLECTOR_VERSION}", "Accept": "*/*"})started = time.monotonic()result: dict[str, Any] = {"ok": False, "response_ms": None, "code": None, "detail": None, "cert_days": None}body = b""- try:- with opener.open(request, timeout=check["timeout"]) as response:- result["code"] = response.status- if not check["ignore_tls"]:- result["cert_days"] = _cert_days(response)- if check["type"] in ("keyword", "json"):- deadline = started + check["timeout"]- while len(body) < MAX_BODY_BYTES and time.monotonic() < deadline:- chunk = response.read(min(65536, MAX_BODY_BYTES - len(body)))- if not chunk:- break- body += chunk- except urllib.error.HTTPError as exc:- result["code"] = exc.code- exc.close()- except (urllib.error.URLError, OSError, http.client.HTTPException, ValueError) as exc:- result["detail"] = _failure(exc, check["timeout"])+ with _Deadline(check["timeout"]) as deadline:+ opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _DeadlineHTTP(deadline), _DeadlineHTTPS(context, deadline), _Redirects(check["follow_redirects"]))+ try:+ with opener.open(request, timeout=check["timeout"]) as response:+ result["code"] = response.status+ if not check["ignore_tls"]:+ result["cert_days"] = _cert_days(response)+ if check["type"] in ("keyword", "json"):+ while len(body) < MAX_BODY_BYTES:+ chunk = response.read(min(65536, MAX_BODY_BYTES - len(body)))+ if not chunk:+ break+ body += chunk+ except urllib.error.HTTPError as exc:+ result["code"] = exc.code+ exc.close()+ except (urllib.error.URLError, OSError, http.client.HTTPException, ValueError) as exc:+ result["detail"] = f"Timed out after {check['timeout']} s" if deadline.expired else _failure(exc, check["timeout"])+ return result+ if deadline.expired: # the answer was cut off, so it can't be judged+ result["detail"] = f"Timed out after {check['timeout']} s"return resultresult["response_ms"] = int((time.monotonic() - started) * 1000)if not _code_accepted(result["code"], check["accepted_codes"]):@@ -1474,14 +1546,16 @@ def run_tls(check: dict[str, Any]) -> dict[str, Any]:"""Connect, read the server's certificate, and be down when it is invalid, expired, or close to expiring."""context = ssl.create_default_context()started = time.monotonic()- try:- with socket.create_connection((check["host"], check["port"]), timeout=check["timeout"]) as sock:- with context.wrap_socket(sock, server_hostname=check["host"]) as tls:- cert = tls.getpeercert()- except ssl.SSLCertVerificationError as exc:- return {"ok": False, "response_ms": None, "code": None, "detail": f"Certificate not valid: {getattr(exc, 'verify_message', None) or exc}"[:200], "cert_days": None}- except (ssl.SSLError, OSError, http.client.HTTPException) as exc:- return {"ok": False, "response_ms": None, "code": None, "detail": _failure(exc, check["timeout"]), "cert_days": None}+ with _Deadline(check["timeout"]) as deadline:+ try:+ with deadline.track(socket.create_connection((check["host"], check["port"]), timeout=check["timeout"])) as sock:+ with context.wrap_socket(sock, server_hostname=check["host"]) as tls:+ cert = tls.getpeercert()+ except ssl.SSLCertVerificationError as exc:+ return {"ok": False, "response_ms": None, "code": None, "detail": f"Certificate not valid: {getattr(exc, 'verify_message', None) or exc}"[:200], "cert_days": None}+ except (ssl.SSLError, OSError, http.client.HTTPException) as exc:+ detail = f"Timed out after {check['timeout']} s" if deadline.expired else _failure(exc, check["timeout"])+ return {"ok": False, "response_ms": None, "code": None, "detail": detail, "cert_days": None}response_ms = int((time.monotonic() - started) * 1000)try:days = int((ssl.cert_time_to_seconds(cert["notAfter"]) - time.time()) // 86400)diff --git a/src/frontend/src/availability/AvailabilityPage.jsx b/src/frontend/src/availability/AvailabilityPage.jsxindex 438899b..122abf4 100644--- a/src/frontend/src/availability/AvailabilityPage.jsx+++ b/src/frontend/src/availability/AvailabilityPage.jsx@@ -128,7 +128,7 @@ function CheckRow({ check, selected, onSelect }) {<td className="px-3 text-right text-[13px] tabular-nums text-paper">{check.paused ? '—' : percent(check.uptime_30d)}</td><td className="px-3"><span className="flex items-center gap-2.5">{!check.paused && <Sparkline check={check} />}- <span className={`text-[13px] tabular-nums ${down ? 'text-signal' : isSlow(check) ? 'text-warn' : 'text-paper'}`}>{check.paused ? '—' : down ? (check.last_detail?.replace(/^HTTP /, '') || 'down') : check.response_ms != null ? `${check.response_ms} ms` : '—'}</span>+ <span className={`text-[13px] tabular-nums ${down ? 'text-signal' : check.state === 'stale' ? 'text-warn' : isSlow(check) ? 'text-warn' : 'text-paper'}`}>{check.paused ? '—' : down ? (check.last_detail?.replace(/^HTTP /, '') || 'down') : check.state === 'stale' ? 'no results' : check.response_ms != null ? `${check.response_ms} ms` : '—'}</span></span></td><td className={`px-3 text-[13px] ${check.cert_days != null && check.cert_days <= 21 ? 'text-warn' : 'text-paper'}`}>{check.cert_days != null ? `${check.cert_days} days` : check.target.startsWith('http://') ? <span className="text-muted">no TLS</span> : <span className="text-muted">—</span>}</td>@@ -144,7 +144,7 @@ function PhoneCheck({ check, onSelect }) {<button type="button" onClick={() => onSelect(check)} className={`block w-full border-t border-line px-4 py-3 text-left ${down ? 'shadow-[inset_3px_0_0_var(--color-signal)]' : ''}`}><span className="flex items-baseline gap-2"><span className="min-w-0 flex-1 truncate text-[15px] font-medium text-paper">{check.name}</span>- <span className={`shrink-0 text-[13px] tabular-nums ${check.paused ? 'text-muted' : down ? 'text-signal' : 'text-paper'}`}>{check.paused ? 'paused' : down ? (check.last_detail || 'down') : percent(check.uptime_24h)}</span>+ <span className={`shrink-0 text-[13px] tabular-nums ${check.paused ? 'text-muted' : down ? 'text-signal' : check.state === 'stale' ? 'text-warn' : 'text-paper'}`}>{check.paused ? 'paused' : down ? (check.last_detail || 'down') : check.state === 'stale' ? 'no results' : percent(check.uptime_24h)}</span></span><span className="mono mt-0.5 block truncate text-[11px] text-muted">{TYPE_LABEL[check.type]} · {check.target}</span><span className="mt-2 flex items-center justify-between gap-3"><Strip check={check} size="sm" /><span className="shrink-0 text-xs tabular-nums text-muted">{check.response_ms != null && !check.paused ? `${check.response_ms} ms` : ''}</span></span>@@ -337,7 +337,7 @@ function Detail({ check, canEdit, onEdit, onPause, onDelete, refreshedAt }) {<h3 className="m-0 text-[13px] font-semibold text-paper">Last 30 checks</h3><div className="mt-2.5"><Strip check={check} /></div><p className="mb-0 mt-2.5 text-xs text-muted">- {check.paused ? 'Paused: no checks run and no alerts are sent.' : check.state === 'pending' ? 'Waiting for the first results from its servers.' : `${check.state === 'down' ? 'Down' : 'Up'} since ${check.state_since ? new Date(check.state_since).toLocaleString() : '—'}.`}+ {check.paused ? 'Paused: no checks run and no alerts are sent.' : check.state === 'pending' ? 'Waiting for the first results from its servers.' : check.state === 'stale' ? `No recent results since ${check.state_since ? new Date(check.state_since).toLocaleString() : '—'}: its servers aren't running it, or their results don't arrive.` : `${check.state === 'down' ? 'Down' : 'Up'} since ${check.state_since ? new Date(check.state_since).toLocaleString() : '—'}.`}{check.last_result_at && ` Last result ${ago(check.last_result_at)}${check.last_detail ? `: ${check.last_detail}` : ''}.`}{check.quiet_until && ` New check: it alerts from ${new Date(check.quiet_until).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })}, so a wrong address doesn't page anyone.`}</p>diff --git a/tests/collectors/test_collector_checks.py b/tests/collectors/test_collector_checks.pyindex bdbbbd9..60afe4d 100644--- a/tests/collectors/test_collector_checks.py+++ b/tests/collectors/test_collector_checks.py@@ -100,6 +100,55 @@ def test_an_unreachable_service_is_down_with_a_short_reason():assert collector.run_tcp(check(type="tcp", target=f"127.0.0.1:{port}"))["detail"] == "Connection refused"+def dripping():+ """Starts a server that sends ``answer`` one byte every 0.2 s, so no single read ever times out (#204)."""+ listeners = []++ def start(answer: bytes) -> int:+ listener = socket.socket()+ listener.bind(("127.0.0.1", 0))+ listener.listen()+ listeners.append(listener)++ def drip(conn):+ with conn:+ try:+ conn.recv(1024)+ for byte in answer:+ conn.send(bytes([byte]))+ time.sleep(0.2)+ except OSError:+ pass++ def accept():+ while True:+ try:+ conn, _ = listener.accept()+ except OSError:+ return+ threading.Thread(target=drip, args=(conn,), daemon=True).start()++ threading.Thread(target=accept, daemon=True).start()+ return listener.getsockname()[1]++ yield start+ for listener in listeners:+ listener.close()+++def test_a_server_that_answers_a_byte_at_a_time_cant_stretch_the_timeout(dripping):+ web_port = dripping(b"HTTP/1.1 200 OK\r\nX-Slow: " + b"a" * 100)+ tls_port = dripping(b"\x16\x03\x03\x40\x00" + b"a" * 100) # a TLS handshake record that never finishes++ started = time.monotonic()+ web = collector.run_http(check(target=f"http://127.0.0.1:{web_port}/", timeout=1))+ tls = collector.run_tls({"host": "127.0.0.1", "port": tls_port, "timeout": 1})++ assert (web["ok"], web["detail"]) == (False, "Timed out after 1 s") and (tls["ok"], tls["detail"]) == (False, "Timed out after 1 s")+ assert time.monotonic() - started < 4 # each within its own second, not the 20 s the answer takes++def test_an_open_tcp_port_is_up(site):port = site.rsplit(":", 1)[1]diff --git a/tests/monitoring/test_availability.py b/tests/monitoring/test_availability.pyindex 2c011a9..7eda35c 100644--- a/tests/monitoring/test_availability.py+++ b/tests/monitoring/test_availability.py@@ -242,6 +242,24 @@ def test_coming_back_up_resolves_the_incident(app):assert incident(app, check["id"]).status == "resolved"+def test_a_check_whose_results_stop_turns_stale_instead_of_staying_up(app):+ up, down = create(app, server_ids=[1], name="Up"), create(app, server_ids=[1], name="Down")+ for check in (up, down):+ age(app, check["id"])+ online(app, 1) # the collector still checks in: only its checks went quiet (#204)+ report(app, 1, [run(up["id"], True)] + runs(down["id"], [False, False, False]))+ assert (check_row(app, up["id"]).state, check_row(app, down["id"]).state) == ("up", "down")++ with app.Session() as db:+ assert availability.mark_stale(db) == 0 # results are fresh: nothing changes+ assert availability.mark_stale(db, datetime.now(timezone.utc) + timedelta(minutes=10)) == 2+ assert (check_row(app, up["id"]).state, check_row(app, down["id"]).state) == ("stale", "stale")+ assert incident(app, down["id"]).status == "active" # nothing is known to be fixed, so its incident stays open++ report(app, 1, [run(down["id"], True)]) # results again+ assert check_row(app, down["id"]).state == "up" and incident(app, down["id"]).status == "resolved"++def test_a_new_check_stays_quiet_for_ten_minutes(app):check = create(app, server_ids=[1])