Skip to content

Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.

fix(checks): make a check's timeout a hard limit and show checks without results as stale (#216)

Karlo Kajba Šimanića8238039563c
diff --git a/SECURITY.md b/SECURITY.md
index 50b2435..87e81b5 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -411,6 +411,11 @@ URL, a keyword or a value in a JSON reply, a TCP port, a TLS certificate, a DNS
never leaves the collector: a result is only up/down, the status code, the response time, the days
until a TLS certificate expires, and a short reason (≤200 characters; for a JSON check that reason can
include up to ~80 characters of the value that failed the comparison).
+- **A check's timeout is a hard limit for the whole run.** When it is up, the collector shuts down the
+ check's connections, so a target that answers a byte at a time can't hold a check, or the collector's
+ few check workers, past it. (A name lookup ends when the system resolver gives up.) A check whose
+ results stop arriving turns **stale** on the Availability page within a few intervals instead of
+ keeping its last state; an incident it had stays open until results come back.
- **Ping uses an unprivileged ICMP datagram socket**, allowed for the container's own group by a
namespaced `net.ipv4.ping_group_range` sysctl. It grants no capability and no raw socket (no packet
crafting or sniffing); without the sysctl a ping check simply reports that ICMP isn't permitted. DNS
diff --git a/src/backend/availability.py b/src/backend/availability.py
index fe42096..f690253 100644
--- a/src/backend/availability.py
+++ b/src/backend/availability.py
@@ -408,13 +408,16 @@ def evaluate(db, check: Check, now: datetime | None = None, alerting: bool = Tru
else:
states = server_states(db, check, now)
known = {server: state for server, state in states.items() if state != "unknown"}
- if check.rule == "all":
+ if known and check.rule == "all":
# A server that should be reporting but hasn't yet keeps the check from counting as down everywhere;
# one that can't (offline, or a collector too old to run checks) doesn't hold it up.
waiting = [server for server, state in states.items() if state == "unknown" and server in _reporting(db, now)]
known.update({server: "retrying" for server in waiting})
if not known:
- new = "pending" if check.state in ("pending", "paused") else check.state
+ # No server has a fresh result: the check isn't run, or its results don't arrive (a collector that is
+ # online but whose checks are stuck, too). Its last verdict no longer says anything, so it isn't shown
+ # as up (or down) any more (#204).
+ new = "pending" if check.state in ("pending", "paused") else "stale"
elif check.rule == "all":
new = "down" if all(state == "down" for state in known.values()) else "up" if any(state == "up" for state in known.values()) else check.state if check.state in ("up", "down") else "pending"
else:
@@ -429,11 +432,24 @@ def evaluate(db, check: Check, now: datetime | None = None, alerting: bool = Tru
why = (latest.detail if latest and latest.detail else "No answer") + f" from {', '.join(failing) or 'its servers'}"
raise_alert(db, servers[0], rule_key(check), check.severity, f"{check.name} is down"[:255],
f"{why}. Failed {check.retries + 1} checks in a row. {redacted_target(check.target)}"[:1000])
- elif servers:
+ elif servers and new != "stale": # without results nothing is known to be fixed, so an open incident stays open
resolve_alerts(db, servers[0], [rule_key(check)], now)
return new
+def mark_stale(db, now: datetime | None = None) -> int:
+ """Re-evaluate the checks whose results stopped arriving, which nothing else would look at again, so they turn
+ stale instead of keeping their last verdict. Run every minute. Returns how many changed."""
+ now = now or datetime.now(timezone.utc)
+ changed = 0
+ for check in db.query(Check).filter(Check.paused.is_(False), Check.state.notin_(("pending", "stale"))):
+ last = _utc(check.last_result_at or check.state_since or check.created_at)
+ if last and now - last > timedelta(seconds=check.interval_seconds * 3 + check.timeout_seconds + 60):
+ changed += evaluate(db, check, now, alerting=False) == "stale"
+ db.commit()
+ return changed
+
+
def forget_check(db, check: Check) -> None:
"""Delete a check with its history; its open incident is resolved, so the record of it stays."""
servers = servers_of(check)
diff --git a/src/backend/main.py b/src/backend/main.py
index 6116850..813e2d3 100644
--- a/src/backend/main.py
+++ b/src/backend/main.py
@@ -1007,6 +1007,7 @@ def _offline_watch_loop() -> None:
db = SessionLocal()
try:
mark_offline_servers(db)
+ availability.mark_stale(db)
purge_stale_pending(db)
purge_due = last_metrics_purge is None or time.monotonic() - last_metrics_purge >= METRICS_PURGE_INTERVAL_SECONDS
if metrics_maintenance(db, purge_due):
diff --git a/src/collector/collector.py b/src/collector/collector.py
index cba1600..3e3c7ce 100644
--- a/src/collector/collector.py
+++ b/src/collector/collector.py
@@ -1364,7 +1364,7 @@ def _failure(exc: BaseException, timeout: int) -> str:
return f"Timed out after {timeout} s"
if isinstance(reason, ssl.SSLCertVerificationError):
return f"TLS certificate not valid: {reason.verify_message}"[:200]
- if isinstance(reason, ssl.SSLError):
+ if isinstance(reason, ssl.SSLError) or isinstance(exc, ssl.SSLError): # an SSLError's own reason is a string
return "TLS handshake failed"
if isinstance(reason, socket.gaierror):
return "Host name not found"
@@ -1377,34 +1377,106 @@ def _failure(exc: BaseException, timeout: int) -> str:
return type(reason).__name__
+class _Deadline:
+ """A check's timeout as a hard limit for the whole run (#204). A socket timeout only limits each read, so a
+ server that sends its answer a byte at a time could hold a check, and one of the CHECK_WORKERS, for as long as
+ it likes. When the time is up this shuts down every socket the check opened, which ends any read or TLS
+ handshake still waiting on it. A name lookup can't be interrupted; it ends when the system resolver gives up,
+ and the connection after it is then shut down at once."""
+
+ def __init__(self, seconds: float) -> None:
+ self.expired = False
+ self.sockets: list[socket.socket] = []
+ self.lock = threading.Lock()
+ self.timer = threading.Timer(seconds, self._expire)
+ self.timer.daemon = True
+
+ def __enter__(self) -> "_Deadline":
+ self.timer.start()
+ return self
+
+ def __exit__(self, *_exc: object) -> None:
+ self.timer.cancel()
+
+ def track(self, sock: socket.socket) -> socket.socket:
+ with self.lock:
+ self.sockets.append(sock)
+ if self.expired:
+ _shut(sock)
+ return sock
+
+ def _expire(self) -> None:
+ with self.lock:
+ self.expired = True
+ for sock in self.sockets:
+ _shut(sock)
+
+
+def _shut(sock: socket.socket) -> None:
+ try:
+ sock.shutdown(socket.SHUT_RDWR)
+ except OSError:
+ pass
+
+
+def _tracked(base: type, deadline: _Deadline) -> type:
+ """``base`` (an http.client connection class) whose sockets ``deadline`` can shut down."""
+ class Tracked(base):
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ super().__init__(*args, **kwargs)
+ self._create_connection = lambda *a, **k: deadline.track(socket.create_connection(*a, **k))
+ return Tracked
+
+
+class _DeadlineHTTP(urllib.request.HTTPHandler):
+ def __init__(self, deadline: _Deadline) -> None:
+ super().__init__()
+ self.deadline = deadline
+
+ def http_open(self, req): # noqa: ANN001, ANN201
+ return self.do_open(_tracked(http.client.HTTPConnection, self.deadline), req)
+
+
+class _DeadlineHTTPS(urllib.request.HTTPSHandler):
+ def __init__(self, context: ssl.SSLContext, deadline: _Deadline) -> None:
+ super().__init__(context=context)
+ self.deadline = deadline
+
+ def https_open(self, req): # noqa: ANN001, ANN201
+ return self.do_open(_tracked(http.client.HTTPSConnection, self.deadline), req, context=self._context)
+
+
def run_http(check: dict[str, Any]) -> dict[str, Any]:
"""One run of an HTTP(s) or keyword check: {ok, response_ms, code, detail, cert_days}."""
context = ssl.create_default_context()
if check["ignore_tls"]:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
- opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPSHandler(context=context), _Redirects(check["follow_redirects"]))
request = urllib.request.Request(check["target"], method=check["method"], headers={"User-Agent": f"Infradeck-Check/{COLLECTOR_VERSION}", "Accept": "*/*"})
started = time.monotonic()
result: dict[str, Any] = {"ok": False, "response_ms": None, "code": None, "detail": None, "cert_days": None}
body = b""
- try:
- with opener.open(request, timeout=check["timeout"]) as response:
- result["code"] = response.status
- if not check["ignore_tls"]:
- result["cert_days"] = _cert_days(response)
- if check["type"] in ("keyword", "json"):
- deadline = started + check["timeout"]
- while len(body) < MAX_BODY_BYTES and time.monotonic() < deadline:
- chunk = response.read(min(65536, MAX_BODY_BYTES - len(body)))
- if not chunk:
- break
- body += chunk
- except urllib.error.HTTPError as exc:
- result["code"] = exc.code
- exc.close()
- except (urllib.error.URLError, OSError, http.client.HTTPException, ValueError) as exc:
- result["detail"] = _failure(exc, check["timeout"])
+ with _Deadline(check["timeout"]) as deadline:
+ opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _DeadlineHTTP(deadline), _DeadlineHTTPS(context, deadline), _Redirects(check["follow_redirects"]))
+ try:
+ with opener.open(request, timeout=check["timeout"]) as response:
+ result["code"] = response.status
+ if not check["ignore_tls"]:
+ result["cert_days"] = _cert_days(response)
+ if check["type"] in ("keyword", "json"):
+ while len(body) < MAX_BODY_BYTES:
+ chunk = response.read(min(65536, MAX_BODY_BYTES - len(body)))
+ if not chunk:
+ break
+ body += chunk
+ except urllib.error.HTTPError as exc:
+ result["code"] = exc.code
+ exc.close()
+ except (urllib.error.URLError, OSError, http.client.HTTPException, ValueError) as exc:
+ result["detail"] = f"Timed out after {check['timeout']} s" if deadline.expired else _failure(exc, check["timeout"])
+ return result
+ if deadline.expired: # the answer was cut off, so it can't be judged
+ result["detail"] = f"Timed out after {check['timeout']} s"
return result
result["response_ms"] = int((time.monotonic() - started) * 1000)
if not _code_accepted(result["code"], check["accepted_codes"]):
@@ -1474,14 +1546,16 @@ def run_tls(check: dict[str, Any]) -> dict[str, Any]:
"""Connect, read the server's certificate, and be down when it is invalid, expired, or close to expiring."""
context = ssl.create_default_context()
started = time.monotonic()
- try:
- with socket.create_connection((check["host"], check["port"]), timeout=check["timeout"]) as sock:
- with context.wrap_socket(sock, server_hostname=check["host"]) as tls:
- cert = tls.getpeercert()
- except ssl.SSLCertVerificationError as exc:
- return {"ok": False, "response_ms": None, "code": None, "detail": f"Certificate not valid: {getattr(exc, 'verify_message', None) or exc}"[:200], "cert_days": None}
- except (ssl.SSLError, OSError, http.client.HTTPException) as exc:
- return {"ok": False, "response_ms": None, "code": None, "detail": _failure(exc, check["timeout"]), "cert_days": None}
+ with _Deadline(check["timeout"]) as deadline:
+ try:
+ with deadline.track(socket.create_connection((check["host"], check["port"]), timeout=check["timeout"])) as sock:
+ with context.wrap_socket(sock, server_hostname=check["host"]) as tls:
+ cert = tls.getpeercert()
+ except ssl.SSLCertVerificationError as exc:
+ return {"ok": False, "response_ms": None, "code": None, "detail": f"Certificate not valid: {getattr(exc, 'verify_message', None) or exc}"[:200], "cert_days": None}
+ except (ssl.SSLError, OSError, http.client.HTTPException) as exc:
+ detail = f"Timed out after {check['timeout']} s" if deadline.expired else _failure(exc, check["timeout"])
+ return {"ok": False, "response_ms": None, "code": None, "detail": detail, "cert_days": None}
response_ms = int((time.monotonic() - started) * 1000)
try:
days = int((ssl.cert_time_to_seconds(cert["notAfter"]) - time.time()) // 86400)
diff --git a/src/frontend/src/availability/AvailabilityPage.jsx b/src/frontend/src/availability/AvailabilityPage.jsx
index 438899b..122abf4 100644
--- a/src/frontend/src/availability/AvailabilityPage.jsx
+++ b/src/frontend/src/availability/AvailabilityPage.jsx
@@ -128,7 +128,7 @@ function CheckRow({ check, selected, onSelect }) {
<td className="px-3 text-right text-[13px] tabular-nums text-paper">{check.paused ? '—' : percent(check.uptime_30d)}</td>
<td className="px-3">
<span className="flex items-center gap-2.5">{!check.paused && <Sparkline check={check} />}
- <span className={`text-[13px] tabular-nums ${down ? 'text-signal' : isSlow(check) ? 'text-warn' : 'text-paper'}`}>{check.paused ? '—' : down ? (check.last_detail?.replace(/^HTTP /, '') || 'down') : check.response_ms != null ? `${check.response_ms} ms` : '—'}</span>
+ <span className={`text-[13px] tabular-nums ${down ? 'text-signal' : check.state === 'stale' ? 'text-warn' : isSlow(check) ? 'text-warn' : 'text-paper'}`}>{check.paused ? '—' : down ? (check.last_detail?.replace(/^HTTP /, '') || 'down') : check.state === 'stale' ? 'no results' : check.response_ms != null ? `${check.response_ms} ms` : '—'}</span>
</span>
</td>
<td className={`px-3 text-[13px] ${check.cert_days != null && check.cert_days <= 21 ? 'text-warn' : 'text-paper'}`}>{check.cert_days != null ? `${check.cert_days} days` : check.target.startsWith('http://') ? <span className="text-muted">no TLS</span> : <span className="text-muted">—</span>}</td>
@@ -144,7 +144,7 @@ function PhoneCheck({ check, onSelect }) {
<button type="button" onClick={() => onSelect(check)} className={`block w-full border-t border-line px-4 py-3 text-left ${down ? 'shadow-[inset_3px_0_0_var(--color-signal)]' : ''}`}>
<span className="flex items-baseline gap-2">
<span className="min-w-0 flex-1 truncate text-[15px] font-medium text-paper">{check.name}</span>
- <span className={`shrink-0 text-[13px] tabular-nums ${check.paused ? 'text-muted' : down ? 'text-signal' : 'text-paper'}`}>{check.paused ? 'paused' : down ? (check.last_detail || 'down') : percent(check.uptime_24h)}</span>
+ <span className={`shrink-0 text-[13px] tabular-nums ${check.paused ? 'text-muted' : down ? 'text-signal' : check.state === 'stale' ? 'text-warn' : 'text-paper'}`}>{check.paused ? 'paused' : down ? (check.last_detail || 'down') : check.state === 'stale' ? 'no results' : percent(check.uptime_24h)}</span>
</span>
<span className="mono mt-0.5 block truncate text-[11px] text-muted">{TYPE_LABEL[check.type]} · {check.target}</span>
<span className="mt-2 flex items-center justify-between gap-3"><Strip check={check} size="sm" /><span className="shrink-0 text-xs tabular-nums text-muted">{check.response_ms != null && !check.paused ? `${check.response_ms} ms` : ''}</span></span>
@@ -337,7 +337,7 @@ function Detail({ check, canEdit, onEdit, onPause, onDelete, refreshedAt }) {
<h3 className="m-0 text-[13px] font-semibold text-paper">Last 30 checks</h3>
<div className="mt-2.5"><Strip check={check} /></div>
<p className="mb-0 mt-2.5 text-xs text-muted">
- {check.paused ? 'Paused: no checks run and no alerts are sent.' : check.state === 'pending' ? 'Waiting for the first results from its servers.' : `${check.state === 'down' ? 'Down' : 'Up'} since ${check.state_since ? new Date(check.state_since).toLocaleString() : '—'}.`}
+ {check.paused ? 'Paused: no checks run and no alerts are sent.' : check.state === 'pending' ? 'Waiting for the first results from its servers.' : check.state === 'stale' ? `No recent results since ${check.state_since ? new Date(check.state_since).toLocaleString() : '—'}: its servers aren't running it, or their results don't arrive.` : `${check.state === 'down' ? 'Down' : 'Up'} since ${check.state_since ? new Date(check.state_since).toLocaleString() : '—'}.`}
{check.last_result_at && ` Last result ${ago(check.last_result_at)}${check.last_detail ? `: ${check.last_detail}` : ''}.`}
{check.quiet_until && ` New check: it alerts from ${new Date(check.quiet_until).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })}, so a wrong address doesn't page anyone.`}
</p>
diff --git a/tests/collectors/test_collector_checks.py b/tests/collectors/test_collector_checks.py
index bdbbbd9..60afe4d 100644
--- a/tests/collectors/test_collector_checks.py
+++ b/tests/collectors/test_collector_checks.py
@@ -100,6 +100,55 @@ def test_an_unreachable_service_is_down_with_a_short_reason():
assert collector.run_tcp(check(type="tcp", target=f"127.0.0.1:{port}"))["detail"] == "Connection refused"
+def dripping():
+ """Starts a server that sends ``answer`` one byte every 0.2 s, so no single read ever times out (#204)."""
+ listeners = []
+
+ def start(answer: bytes) -> int:
+ listener = socket.socket()
+ listener.bind(("127.0.0.1", 0))
+ listener.listen()
+ listeners.append(listener)
+
+ def drip(conn):
+ with conn:
+ try:
+ conn.recv(1024)
+ for byte in answer:
+ conn.send(bytes([byte]))
+ time.sleep(0.2)
+ except OSError:
+ pass
+
+ def accept():
+ while True:
+ try:
+ conn, _ = listener.accept()
+ except OSError:
+ return
+ threading.Thread(target=drip, args=(conn,), daemon=True).start()
+
+ threading.Thread(target=accept, daemon=True).start()
+ return listener.getsockname()[1]
+
+ yield start
+ for listener in listeners:
+ listener.close()
+
+
+def test_a_server_that_answers_a_byte_at_a_time_cant_stretch_the_timeout(dripping):
+ web_port = dripping(b"HTTP/1.1 200 OK\r\nX-Slow: " + b"a" * 100)
+ tls_port = dripping(b"\x16\x03\x03\x40\x00" + b"a" * 100) # a TLS handshake record that never finishes
+
+ started = time.monotonic()
+ web = collector.run_http(check(target=f"http://127.0.0.1:{web_port}/", timeout=1))
+ tls = collector.run_tls({"host": "127.0.0.1", "port": tls_port, "timeout": 1})
+
+ assert (web["ok"], web["detail"]) == (False, "Timed out after 1 s") and (tls["ok"], tls["detail"]) == (False, "Timed out after 1 s")
+ assert time.monotonic() - started < 4 # each within its own second, not the 20 s the answer takes
+
+
def test_an_open_tcp_port_is_up(site):
port = site.rsplit(":", 1)[1]
diff --git a/tests/monitoring/test_availability.py b/tests/monitoring/test_availability.py
index 2c011a9..7eda35c 100644
--- a/tests/monitoring/test_availability.py
+++ b/tests/monitoring/test_availability.py
@@ -242,6 +242,24 @@ def test_coming_back_up_resolves_the_incident(app):
assert incident(app, check["id"]).status == "resolved"
+def test_a_check_whose_results_stop_turns_stale_instead_of_staying_up(app):
+ up, down = create(app, server_ids=[1], name="Up"), create(app, server_ids=[1], name="Down")
+ for check in (up, down):
+ age(app, check["id"])
+ online(app, 1) # the collector still checks in: only its checks went quiet (#204)
+ report(app, 1, [run(up["id"], True)] + runs(down["id"], [False, False, False]))
+ assert (check_row(app, up["id"]).state, check_row(app, down["id"]).state) == ("up", "down")
+
+ with app.Session() as db:
+ assert availability.mark_stale(db) == 0 # results are fresh: nothing changes
+ assert availability.mark_stale(db, datetime.now(timezone.utc) + timedelta(minutes=10)) == 2
+ assert (check_row(app, up["id"]).state, check_row(app, down["id"]).state) == ("stale", "stale")
+ assert incident(app, down["id"]).status == "active" # nothing is known to be fixed, so its incident stays open
+
+ report(app, 1, [run(down["id"], True)]) # results again
+ assert check_row(app, down["id"]).state == "up" and incident(app, down["id"]).status == "resolved"
+
+
def test_a_new_check_stays_quiet_for_ten_minutes(app):
check = create(app, server_ids=[1])