Skip to content

Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.

feat(availability): test a check from its servers before saving it

karloks20058f72de263f4d
diff --git a/ODSTUPANJA.md b/ODSTUPANJA.md
index 68869ab..e2b7be5 100644
--- a/ODSTUPANJA.md
+++ b/ODSTUPANJA.md
@@ -179,7 +179,7 @@ Prvi dio milestonea: provjere HTTP(s), Keyword i TCP port, pokreću ih collector
| Down now: „alert sent to ntfy" i sažetak agenta | Razlog pada, s kojih servera i koliko puta zaredom; za novu provjeru „no alert yet" | Ne bilježi se slanje po kanalu; agent za provjere još ne postoji | #137, #145 |
| Down now: „Pause alerts" | „Pause" pauzira cijelu provjeru | Odvojeno pauziranje obavijesti nije napravljeno | #137 |
| New check: obavijesti po kanalu, „While down, remind every", „Alert on certificate expiry" | Pad otvara incident koji ide na iste kanale kao svi incidenti; bira se samo Critical/Warning | Obavijesti idu kroz postojeće incidente | #137 |
-| New check: „Test passed · 200 OK" i „Test again" prije spremanja | Nema testa; nova provjera prvih 10 minuta ne šalje obavijesti | Prvi dio | #141 |
+| New check: „Test passed · 200 OK · 142 ms from both servers" odmah | Test traje 5–15 s: collectori pitaju za posao svakih 10 s kad nemaju što javiti; nova provjera i dalje prvih 10 minuta ne šalje obavijesti | Collector ne prima veze izvana, nego sam pita dashboard | #141 |
| New check: timeout fiksno 48 s | Timeout se sam smanji na 80 % intervala kad je interval kraći; greške se pokazuju uz polje i u podnožju | Zadani timeout inače blokira interval ispod 49 s | #141 |
| New check: tagovi kao čipovi | Tekstno polje, tagovi odvojeni zarezom | Jednostavnije | #141 |
| Mobitel: Availability kao zaseban tab | Availability je u More (s brojem palih provjera na tabu More); popis provjera je kompaktan, s trakom zadnjih 30 provjera | Tab bar ima mjesta za četiri stavke | #146 |
diff --git a/src/backend/availability.py b/src/backend/availability.py
index fe4ff91..d95e01f 100644
--- a/src/backend/availability.py
+++ b/src/backend/availability.py
@@ -141,6 +141,76 @@ def for_collector(db, server: Server) -> list[dict]:
return result[:MAX_CHECKS_PER_SERVER]
+
+# ---------- testing a check before it is saved ----------
+# The form's Test button: the check goes once to the collectors of the chosen servers, through Redis, with the
+# answer to their next report (they ask every 10 s when idle), and their results come back the same way. Nothing
+# is stored in the database; a test is forgotten after two minutes.
+
+TEST_TTL = 120
+MAX_TESTS_QUEUED = 5
+IDLE_EVERY = 10
+
+
+def _spec(fields: dict, check_id: int) -> dict:
+ options = {key: value for key, value in json.loads(fields["options"]).items() if key != "upside_down"}
+ return {"id": check_id, "type": fields["type"], "target": fields["target"], "interval": fields["interval_seconds"], "retry_interval": fields["retry_interval_seconds"],
+ "timeout": fields["timeout_seconds"], **options}
+
+
+def start_test(redis, fields: dict, servers: list[int]) -> int:
+ """Queue one run of a check (validated ``fields``) for each server's collector. Returns the test's id."""
+ test_id = int(redis.incr("infradeck:checktest:seq"))
+ spec = json.dumps(_spec(fields, test_id))
+ upside_down = json.loads(fields["options"]).get("upside_down") is True
+ redis.set(f"infradeck:checktest:{test_id}", json.dumps({"servers": servers, "upside_down": upside_down}), ex=TEST_TTL)
+ for server_id in servers:
+ queue = f"infradeck:checktest:queue:{server_id}"
+ redis.rpush(queue, spec)
+ redis.ltrim(queue, -MAX_TESTS_QUEUED, -1)
+ redis.expire(queue, 60)
+ return test_id
+
+
+def tests_for(redis, server_id: int) -> list[dict]:
+ """The tests waiting for this server's collector; handing them over takes them off the queue."""
+ queue = f"infradeck:checktest:queue:{server_id}"
+ items = redis.lrange(queue, 0, -1)
+ if items:
+ redis.delete(queue)
+ return [json.loads(item) for item in items]
+
+
+def store_tests(redis, server_id: int, results: list) -> None:
+ """Keep this server's results for tests it was given. Untrusted like every result: anything else is dropped."""
+ for item in results[:MAX_TESTS_QUEUED * 2]:
+ if not isinstance(item, dict) or not isinstance(item.get("check_id"), int) or isinstance(item.get("check_id"), bool) or not isinstance(item.get("ok"), bool):
+ continue
+ raw = redis.get(f"infradeck:checktest:{item['check_id']}")
+ meta = json.loads(raw) if raw else None
+ if not meta or server_id not in meta["servers"]:
+ continue
+ number = lambda key, low, high: item.get(key) if isinstance(item.get(key), int) and not isinstance(item.get(key), bool) and low <= item[key] <= high else None # noqa: E731
+ detail = item.get("detail")
+ detail = "".join(char for char in detail if char.isprintable())[:200] if isinstance(detail, str) else None
+ ok = item["ok"] != meta["upside_down"]
+ key = f"infradeck:checktest:{item['check_id']}:results"
+ redis.hset(key, str(server_id), json.dumps({"ok": ok, "response_ms": number("response_ms", 0, 600_000), "code": number("code", 100, 599),
+ "detail": detail or None, "cert_days": number("cert_days", -10_000, 10_000)}))
+ redis.expire(key, TEST_TTL)
+
+
+def test_status(redis, test_id: int, names: dict[int, str]) -> dict | None:
+ """Where a test is: each server's result, or waiting. None when the test is unknown or forgotten."""
+ raw = redis.get(f"infradeck:checktest:{test_id}")
+ if not raw:
+ return None
+ meta = json.loads(raw)
+ results = {int(server): json.loads(value) for server, value in (redis.hgetall(f"infradeck:checktest:{test_id}:results") or {}).items()}
+ servers = [{"id": server, "hostname": names.get(server, f"server {server}"), **({"state": "done", **results[server]} if server in results else {"state": "waiting"})}
+ for server in meta["servers"]]
+ return {"id": test_id, "servers": servers, "done": all(server["state"] == "done" for server in servers)}
+
# ---------- results ----------
def store(db, server: Server, results: list, now: datetime | None = None) -> set[int]:
diff --git a/src/backend/main.py b/src/backend/main.py
index 4644f33..bd33ed1 100644
--- a/src/backend/main.py
+++ b/src/backend/main.py
@@ -1979,6 +1979,40 @@ def get_check(check_id: PathId, range: str = Query(default="24h"), session_id: A
return availability.detail(db, _check_or_404(db, check_id), range)
[email protected]("/availability/test")
+def test_check(payload: dict, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):
+ """Run a check once from its servers before saving it. Only servers whose collector can run checks are asked."""
+ user = _require_admin(session_id, db)
+ servers = {server.id: server for server in db.query(Server)}
+ try:
+ fields = availability.validate(payload, set(servers))
+ except ValueError as exc:
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
+ able = [server_id for server_id in json.loads(fields["server_ids"]) if availability.can_run_checks(servers[server_id])]
+ if not able:
+ raise HTTPException(status_code=400, detail="None of the chosen servers can run checks yet: their collector is older than 0.10.0.")
+ try:
+ # The expiry is set when the counter is created, so a failure between two calls can't leave one that never ends.
+ redis_client.set(f"infradeck:checktest:user:{user['id']}", 0, ex=60, nx=True)
+ if redis_client.incr(f"infradeck:checktest:user:{user['id']}") > 20:
+ raise HTTPException(status_code=429, detail="At most 20 tests a minute. Try again shortly.")
+ test_id = availability.start_test(redis_client, fields, able)
+ except HTTPException:
+ raise
+ except Exception as exc:
+ raise HTTPException(status_code=503, detail="Testing needs Redis, which isn't answering.") from exc
+ return availability.test_status(redis_client, test_id, {server_id: servers[server_id].hostname for server_id in servers})
+
+
[email protected]("/availability/test/{test_id}")
+def get_check_test(test_id: PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):
+ _require_admin(session_id, db)
+ status = availability.test_status(redis_client, test_id, dict(db.query(Server.id, Server.hostname).all()))
+ if status is None:
+ raise HTTPException(status_code=404, detail="This test is over. Run it again.")
+ return status
+
+
@app.post("/availability/checks")
def create_check(payload: dict, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):
_require_admin(session_id, db)
@@ -2537,6 +2571,7 @@ def ingest_metrics(payload: MetricsIngestRequest, authorization: Annotated[str |
class CheckResultsRequest(BaseModel):
server_id: int = Field(ge=1, le=2**31 - 1)
results: list = Field(default=[], max_length=availability.MAX_RESULTS_PER_REPORT)
+ tests: list = Field(default=[], max_length=availability.MAX_TESTS_QUEUED * 2)
def _enforce_report_spacing(server: Server) -> None:
@@ -2565,7 +2600,12 @@ def report_check_results(payload: CheckResultsRequest, authorization: Annotated[
check.last_result_at = now
availability.evaluate(db, check, now, alerting)
db.commit()
- return {"checks": availability.for_collector(db, server), "report_every": 5}
+ try: # tests from the form's Test button go through Redis; without it there are none, the checks still work
+ availability.store_tests(redis_client, server.id, payload.tests)
+ tests = availability.tests_for(redis_client, server.id)
+ except Exception:
+ tests = []
+ return {"checks": availability.for_collector(db, server), "tests": tests, "report_every": 5, "idle_every": availability.IDLE_EVERY}
@app.get("/dashboard/summary")
diff --git a/src/collector/collector.py b/src/collector/collector.py
index d7114a2..a221665 100644
--- a/src/collector/collector.py
+++ b/src/collector/collector.py
@@ -1349,10 +1349,10 @@ def run_check(check: dict[str, Any]) -> dict[str, Any]:
return {"check_id": check["id"], "time": datetime.now(timezone.utc).isoformat(), **{key: value for key, value in outcome.items() if value is not None or key == "ok"}}
-def send_check_results(api_url: str, server_id: int, token: str, results: list[dict[str, Any]]) -> dict[str, Any]:
+def send_check_results(api_url: str, server_id: int, token: str, results: list[dict[str, Any]], tests: list[dict[str, Any]] | None = None) -> dict[str, Any]:
request = urllib.request.Request(
f"{api_url.rstrip('/')}/monitoring/metrics/checks",
- data=json.dumps({"server_id": server_id, "results": results}).encode(),
+ data=json.dumps({"server_id": server_id, "results": results, "tests": tests or []}).encode(),
headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
method="POST",
)
@@ -1360,10 +1360,14 @@ def send_check_results(api_url: str, server_id: int, token: str, results: list[d
return json.loads(response.read(MAX_BODY_BYTES).decode())
+MAX_TESTS = 5
+
+
class CheckRunner(threading.Thread):
"""Runs the checks the dashboard gave this collector, each on its interval (the retry interval after a
- failure), and reports the results every few seconds; with nothing to report it asks for its checks once a
- minute. Results that couldn't be sent wait, up to the newest 500."""
+ failure), and reports the results every few seconds; with nothing to report it asks for its checks every
+ 10 s (as the dashboard says), so a test from the New check form is picked up quickly. A test runs once, at
+ once, and its result goes back with the next report. Results that couldn't be sent wait, up to the newest 500."""
def __init__(self, api_url: str, credentials: tuple[int, str]) -> None:
super().__init__(name="checks", daemon=True)
@@ -1376,6 +1380,18 @@ class CheckRunner(threading.Thread):
self.lock = threading.Lock()
self.pool = concurrent.futures.ThreadPoolExecutor(max_workers=CHECK_WORKERS, thread_name_prefix="check")
self.report_every = 5
+ self.idle_every = IDLE_REPORT_SECONDS
+ self.test_results: collections.deque = collections.deque(maxlen=MAX_TESTS * 2)
+
+ def run_tests(self, raw: Any) -> None:
+ """Run each test once, right away; its result is reported apart from the checks' results."""
+ for test in (validate_check(item) for item in (raw if isinstance(raw, list) else [])[:MAX_TESTS]):
+ if test:
+ self.pool.submit(run_check, test).add_done_callback(lambda future: self._tested(future))
+
+ def _tested(self, future: concurrent.futures.Future) -> None:
+ with self.lock:
+ self.test_results.append(future.result())
def set_checks(self, raw: Any) -> None:
valid = [check for check in (validate_check(item) for item in (raw if isinstance(raw, list) else [])[:MAX_CHECKS]) if check]
@@ -1407,24 +1423,32 @@ class CheckRunner(threading.Thread):
def report(self) -> bool:
with self.lock:
- batch = list(self.pending)
+ batch, tested = list(self.pending), list(self.test_results)
self.pending.clear()
+ self.test_results.clear()
server_id, token = self.credentials
- try:
- response = send_check_results(self.api_url, server_id, token, batch)
- except urllib.error.HTTPError as exc:
+
+ def put_back() -> None:
with self.lock:
self.pending.extendleft(reversed(batch))
+ self.test_results.extendleft(reversed(tested))
+
+ try:
+ response = send_check_results(self.api_url, server_id, token, batch, tested)
+ except urllib.error.HTTPError as exc:
+ put_back()
if exc.code == 404:
- self.report_every = 300 # a dashboard without availability checks
+ self.report_every = self.idle_every = 300 # a dashboard without availability checks
return False
except (urllib.error.URLError, OSError, ValueError, http.client.HTTPException):
- with self.lock:
- self.pending.extendleft(reversed(batch))
+ put_back()
return False
- self.set_checks(response.get("checks") if isinstance(response, dict) else None)
- every = _whole(response.get("report_every"), 2, 300) if isinstance(response, dict) else None
- self.report_every = every or 5
+ if not isinstance(response, dict):
+ response = {}
+ self.set_checks(response.get("checks"))
+ self.run_tests(response.get("tests"))
+ self.report_every = _whole(response.get("report_every"), 2, 300) or 5
+ self.idle_every = _whole(response.get("idle_every"), 5, 300) or IDLE_REPORT_SECONDS
if batch:
print(f"Reported {len(batch)} check results", flush=True)
return True
@@ -1434,7 +1458,8 @@ class CheckRunner(threading.Thread):
while True:
self.run_due()
since = time.monotonic() - last_report
- if since >= IDLE_REPORT_SECONDS or (self.pending and since >= self.report_every):
+ # Test results go back at once (the dashboard allows a report every 2 s); check results every few seconds.
+ if since >= self.idle_every or (self.test_results and since >= 2) or (self.pending and since >= self.report_every):
self.report()
last_report = time.monotonic()
time.sleep(1)
diff --git a/src/frontend/src/availability/CheckForm.jsx b/src/frontend/src/availability/CheckForm.jsx
index f426f60..43e6383 100644
--- a/src/frontend/src/availability/CheckForm.jsx
+++ b/src/frontend/src/availability/CheckForm.jsx
@@ -1,6 +1,6 @@
// The New check / Edit form in the drawer: the type, what to check, from which servers, the timing in numbers
// with one sentence saying what they mean together, what counts as up, and how bad it is when it's down.
-import { useState } from 'react'
+import { useEffect, useRef, useState } from 'react'
import { Link } from 'react-router-dom'
import { apiRequest } from '../api.js'
import { Check, Field, Switch, useAction } from '../ui.jsx'
@@ -46,6 +46,22 @@ function problems(form, web) {
const duration = (seconds) => (seconds < 90 ? `${Math.round(seconds)} seconds` : seconds < 5400 ? `${Math.round(seconds / 60)} ${Math.round(seconds / 60) === 1 ? 'minute' : 'minutes'}` : `${Math.round(seconds / 3600)} hours`)
+// How long the form waits for a test: the collectors ask every 10 s when idle, then the check may take its timeout.
+const TEST_WAIT_MS = 75_000
+
+// "Test passed · 200 OK · 142 ms from both servers", or what went wrong and where.
+function describeTest(test) {
+ const done = test.servers.filter((server) => server.state === 'done')
+ const waiting = test.servers.filter((server) => server.state === 'waiting')
+ const names = (servers) => (servers.length === test.servers.length && servers.length > 1 ? (servers.length === 2 ? 'both servers' : 'all servers') : servers.map((server) => server.hostname).join(', '))
+ const failed = done.filter((server) => !server.ok)
+ if (failed.length) return { tone: 'warn', text: `Test failed · ${failed.map((server) => `${server.detail || (server.code ? `HTTP ${server.code}` : 'no answer')} from ${server.hostname}`).join(', ')}` }
+ if (waiting.length) return { tone: 'muted', text: test.expired ? `No answer from ${names(waiting)}: is its collector running?` : `Testing from ${names(test.servers)}…` }
+ const times = done.map((server) => server.response_ms).filter((value) => value != null)
+ const code = done.find((server) => server.code)?.code
+ return { tone: 'ok', text: `Test passed${code ? ` · ${code} OK` : ''}${times.length ? ` · ${Math.round(times.reduce((a, b) => a + b, 0) / times.length)} ms` : ''} from ${names(done)}` }
+}
+
// "Marked down after 3 failed checks in a row, so about 1 minute after it breaks."
function TimingSentence({ form }) {
const interval = Number(form.interval_seconds), retries = Number(form.retries), retry = Number(form.retry_interval_seconds)
@@ -77,7 +93,10 @@ export function CheckForm({ check, servers, onSaved, onCancel }) {
const [form, setForm] = useState(() => initial(check, servers))
const [state, run] = useAction()
const [tried, setTried] = useState(false)
- const set = (changes) => setForm((current) => {
+ const [test, setTest] = useState(null) // null, { running: true }, a test from the backend, or { error }
+ // A finished test says nothing about a changed form, so it goes once anything changes.
+ const set = (changes) => { setTest((current) => (current && (current.done || current.expired || current.error) ? null : current)); setFormFields(changes) }
+ const setFormFields = (changes) => setForm((current) => {
const next = { ...current, ...changes }
// A shorter interval takes the timeout down with it, so the default 48 s doesn't block a 30 s interval.
if ('interval_seconds' in changes && /^\d+$/.test(changes.interval_seconds) && Number(next.timeout_seconds) >= Number(changes.interval_seconds)) {
@@ -91,18 +110,48 @@ export function CheckForm({ check, servers, onSaved, onCancel }) {
const hint = (key) => shown(key) && <span className="mt-1.5 block text-[11px] leading-4 text-warn">{issues[key]}</span>
const toggleServer = (id, on) => set({ server_ids: on ? [...form.server_ids, id] : form.server_ids.filter((item) => item !== id) })
- const save = (event) => {
- event.preventDefault()
- setTried(true)
- if (Object.keys(issues).length) return
+ const bodyOf = () => {
const whole = (value) => (/^\d+$/.test(String(value).trim()) ? Number(value) : value)
- const body = {
+ return {
type: form.type, name: form.name.trim(), target: form.target.trim(), tags: form.tags.split(',').map((tag) => tag.trim()).filter(Boolean),
server_ids: form.server_ids, rule: form.rule, severity: form.severity, upside_down: form.upside_down,
interval_seconds: whole(form.interval_seconds), retries: whole(form.retries), retry_interval_seconds: whole(form.retry_interval_seconds), timeout_seconds: whole(form.timeout_seconds),
...(web ? { accepted_codes: form.accepted_codes.split(',').map((code) => code.trim()).filter(Boolean), follow_redirects: form.follow_redirects, ignore_tls: form.ignore_tls } : {}),
...(form.type === 'keyword' ? { keyword: form.keyword, case_sensitive: form.case_sensitive } : {}),
}
+ }
+
+ // Test: the check runs once from its servers; the answer is asked for every 1.5 s until all servers replied.
+ const polling = useRef(null)
+ useEffect(() => () => clearTimeout(polling.current), [])
+ const runTest = async () => {
+ setTried(true)
+ if (Object.keys(issues).length) return
+ clearTimeout(polling.current)
+ setTest({ running: true })
+ try {
+ const started = Date.now()
+ let current = await apiRequest('/availability/test', { method: 'POST', body: JSON.stringify(bodyOf()) })
+ setTest(current)
+ const poll = async () => {
+ if (current.done) return
+ if (Date.now() - started > TEST_WAIT_MS) { setTest({ ...current, expired: true }); return }
+ try { current = await apiRequest(`/availability/test/${current.id}`, { method: 'GET' }); setTest(current) } catch (error) { setTest({ error: error.message }); return }
+ polling.current = setTimeout(poll, 1500)
+ }
+ polling.current = setTimeout(poll, 1500)
+ } catch (error) {
+ setTest({ error: error.message || 'The test could not start.' })
+ }
+ }
+ const testLine = test && (test.running ? { tone: 'muted', text: 'Starting the test…' } : test.error ? { tone: 'warn', text: test.error } : describeTest(test))
+ const testing = Boolean(test && (test.running || (test.servers && !test.done && !test.expired)))
+
+ const save = (event) => {
+ event.preventDefault()
+ setTried(true)
+ if (Object.keys(issues).length) return
+ const body = bodyOf()
run(async () => {
const result = await apiRequest(check ? `/availability/checks/${check.id}` : '/availability/checks', { method: check ? 'PATCH' : 'POST', body: JSON.stringify(body) })
onSaved(result)
@@ -187,8 +236,12 @@ export function CheckForm({ check, servers, onSaved, onCancel }) {
</Field>
<div className="sticky -bottom-5 -mx-6 -mb-5 flex items-center justify-end gap-2 border-t border-line bg-bg px-6 py-3">
- <p role="alert" className="m-0 mr-auto min-w-0 text-xs leading-4 text-warn">{state.error || (tried && Object.keys(issues).length ? `Fix ${Object.keys(issues).length === 1 ? 'the field' : `${Object.keys(issues).length} fields`} marked above.` : '')}</p>
+ {state.error || (tried && Object.keys(issues).length) || !testLine
+ ? <p role="alert" className="m-0 mr-auto min-w-0 text-xs leading-4 text-warn">{state.error || (tried && Object.keys(issues).length ? `Fix ${Object.keys(issues).length === 1 ? 'the field' : `${Object.keys(issues).length} fields`} marked above.` : '')}</p>
+ : <p role="status" className={`m-0 mr-auto flex min-w-0 items-center gap-2 text-xs leading-4 ${testLine.tone === 'ok' ? 'text-ok' : testLine.tone === 'warn' ? 'text-warn' : 'text-soft'}`}>
+ <span className={`dot shrink-0 ${testLine.tone === 'ok' ? 'dot-ok' : testLine.tone === 'warn' ? 'dot-warn' : ''}`} /><span className="min-w-0">{testLine.text}</span></p>}
<button type="button" className="btn" onClick={onCancel}>Cancel</button>
+ <button type="button" className="btn" onClick={runTest} disabled={testing}>{testing ? 'Testing…' : test ? 'Test again' : 'Test'}</button>
<button type="submit" className="btn btn-primary" disabled={state.busy}>{state.busy ? 'Saving…' : 'Save check'}</button>
</div>
</form>
diff --git a/tests/collectors/test_collector_checks.py b/tests/collectors/test_collector_checks.py
index 39ed525..8196698 100644
--- a/tests/collectors/test_collector_checks.py
+++ b/tests/collectors/test_collector_checks.py
@@ -111,7 +111,7 @@ def test_an_open_tcp_port_is_up(site):
def test_the_runner_runs_due_checks_and_reports_them(monkeypatch, site):
sent = []
- def fake_send(api_url, server_id, token, results):
+ def fake_send(api_url, server_id, token, results, tests=None):
sent.append(list(results))
return {"checks": [{**GOOD, "target": f"{site}/ok"}], "report_every": 5}
@@ -151,3 +151,34 @@ def test_checks_from_the_dashboard_are_validated_and_capped():
assert len(runner.checks) == collector.MAX_CHECKS and 999 not in runner.checks
runner.set_checks(None)
assert runner.checks == {} and runner.next_run == {}
+
+
+def test_a_test_from_the_form_runs_once_at_once_and_goes_back_on_its_own(monkeypatch, site):
+ sent = []
+ answers = iter([{"checks": [], "tests": [{**GOOD, "id": 41, "target": f"{site}/broken"}], "idle_every": 10}, {"checks": [], "tests": [], "idle_every": 10}])
+
+ def fake_send(api_url, server_id, token, results, tests=None):
+ sent.append((list(results), list(tests or [])))
+ return next(answers)
+
+ monkeypatch.setattr(collector, "send_check_results", fake_send)
+ runner = collector.CheckRunner("http://dashboard/api", (3, "token"))
+
+ runner.report()
+ deadline = time.monotonic() + 5
+ while not runner.test_results and time.monotonic() < deadline:
+ time.sleep(0.05)
+ runner.report()
+
+ assert runner.idle_every == 10 and runner.checks == {} # a test is not a check: it isn't kept or repeated
+ assert sent[1][0] == [] and [(item["check_id"], item["ok"], item["detail"]) for item in sent[1][1]] == [(41, False, "HTTP 503")]
+
+
+def test_a_test_that_isnt_well_formed_is_not_run(monkeypatch):
+ runner = collector.CheckRunner("http://dashboard/api", (3, "token"))
+ submitted = []
+ monkeypatch.setattr(runner.pool, "submit", lambda *args: submitted.append(args) or type("Future", (), {"add_done_callback": lambda self, done: None})())
+
+ runner.run_tests([{**GOOD, "target": "file:///etc/shadow"}, "junk", None] + [{**GOOD, "id": index} for index in range(1, 10)])
+
+ assert len(submitted) == 2 # only the first five are looked at, and three of those aren't checks
diff --git a/tests/monitoring/test_availability.py b/tests/monitoring/test_availability.py
index a065bb0..a4059eb 100644
--- a/tests/monitoring/test_availability.py
+++ b/tests/monitoring/test_availability.py
@@ -397,3 +397,60 @@ def test_the_detail_lists_when_a_check_went_down_and_came_back(app):
assert [event["kind"] for event in detail["events"]] == ["up", "down"]
assert detail["events"][1]["text"].startswith("Down · No answer from alpha")
assert detail["last_down"]["ongoing"] is False
+
+
+
+# --- testing a check before saving it -----------------------------------------------------------------
+
+def test_a_test_goes_to_the_collectors_and_their_results_come_back(app):
+ with app.Session() as db:
+ db.get(Server, 2).system_info = json.dumps({"collector_version": "0.10.0"})
+ db.commit()
+ started = app.post("/availability/test", json={**NEW_CHECK, "upside_down": False}).json()
+ assert started["done"] is False and [server["state"] for server in started["servers"]] == ["waiting", "waiting"]
+
+ handed = report(app, 1, []).json()["tests"]
+ assert [test["id"] for test in handed] == [started["id"]] and handed[0]["target"] == "https://git.example.com/"
+ assert report(app, 1, []).json()["tests"] == [] # handed over once
+ assert report(app, 1, []).json()["idle_every"] == 10
+
+ app.post("/monitoring/metrics/checks", json={"server_id": 1, "results": [], "tests": [{"check_id": started["id"], "ok": True, "response_ms": 142, "code": 200}]},
+ headers={"Authorization": f"Bearer {TOKENS[1]}"})
+ app.post("/monitoring/metrics/checks", json={"server_id": 2, "results": [], "tests": [{"check_id": started["id"], "ok": False, "detail": "Connection refused"}]},
+ headers={"Authorization": f"Bearer {TOKENS[2]}"})
+ status = app.get(f"/availability/test/{started['id']}").json()
+
+ assert status["done"] is True
+ assert [(server["hostname"], server["ok"], server.get("code"), server.get("detail")) for server in status["servers"]] == [("alpha", True, 200, None), ("bravo", False, None, "Connection refused")]
+ with app.Session() as db:
+ assert db.query(Check).count() == 0 and db.query(CheckResult).count() == 0 # a test saves nothing
+
+
+def test_a_collector_cant_answer_a_test_it_wasnt_given(app):
+ started = app.post("/availability/test", json={**NEW_CHECK, "server_ids": [1]}).json()
+
+ app.post("/monitoring/metrics/checks", json={"server_id": 2, "results": [], "tests": [{"check_id": started["id"], "ok": True}]}, headers={"Authorization": f"Bearer {TOKENS[2]}"})
+
+ assert app.get(f"/availability/test/{started['id']}").json()["servers"][0]["state"] == "waiting"
+
+
+def test_only_servers_that_can_run_checks_are_asked(app):
+ assert app.post("/availability/test", json={**NEW_CHECK, "server_ids": [2]}).status_code == 400 # bravo's collector is 0.9.2
+ started = app.post("/availability/test", json={**NEW_CHECK, "server_ids": [1, 2]}).json()
+ assert [server["hostname"] for server in started["servers"]] == ["alpha"]
+ assert report(app, 2, []).json()["tests"] == []
+
+
+def test_a_test_is_checked_like_a_check_and_only_admins_run_it(app):
+ assert app.post("/availability/test", json={**NEW_CHECK, "target": "file:///etc/passwd"}).status_code == 400
+ app.user.update(id=2, role="member")
+ assert app.post("/availability/test", json=NEW_CHECK).status_code == 403
+ assert app.get("/availability/test/1").status_code == 403
+
+
+def test_an_upside_down_test_passes_when_the_service_is_unreachable(app):
+ started = app.post("/availability/test", json={**NEW_CHECK, "server_ids": [1], "upside_down": True}).json()
+ app.post("/monitoring/metrics/checks", json={"server_id": 1, "results": [], "tests": [{"check_id": started["id"], "ok": False, "detail": "Connection refused"}]},
+ headers={"Authorization": f"Bearer {TOKENS[1]}"})
+
+ assert app.get(f"/availability/test/{started['id']}").json()["servers"][0]["ok"] is True