Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.
Python ai-agentsaiopsdashboarddockerfastapihomelabincident-managementllmmonitoringopenaireactroot-cause-analysisself-hosteduptime-monitoring
feat(availability): test a check from its servers before saving it
karloks20058f72de263f4d
diff --git a/ODSTUPANJA.md b/ODSTUPANJA.mdindex 68869ab..e2b7be5 100644--- a/ODSTUPANJA.md+++ b/ODSTUPANJA.md@@ -179,7 +179,7 @@ Prvi dio milestonea: provjere HTTP(s), Keyword i TCP port, pokreću ih collector| Down now: „alert sent to ntfy" i sažetak agenta | Razlog pada, s kojih servera i koliko puta zaredom; za novu provjeru „no alert yet" | Ne bilježi se slanje po kanalu; agent za provjere još ne postoji | #137, #145 || Down now: „Pause alerts" | „Pause" pauzira cijelu provjeru | Odvojeno pauziranje obavijesti nije napravljeno | #137 || New check: obavijesti po kanalu, „While down, remind every", „Alert on certificate expiry" | Pad otvara incident koji ide na iste kanale kao svi incidenti; bira se samo Critical/Warning | Obavijesti idu kroz postojeće incidente | #137 |-| New check: „Test passed · 200 OK" i „Test again" prije spremanja | Nema testa; nova provjera prvih 10 minuta ne šalje obavijesti | Prvi dio | #141 |+| New check: „Test passed · 200 OK · 142 ms from both servers" odmah | Test traje 5–15 s: collectori pitaju za posao svakih 10 s kad nemaju što javiti; nova provjera i dalje prvih 10 minuta ne šalje obavijesti | Collector ne prima veze izvana, nego sam pita dashboard | #141 || New check: timeout fiksno 48 s | Timeout se sam smanji na 80 % intervala kad je interval kraći; greške se pokazuju uz polje i u podnožju | Zadani timeout inače blokira interval ispod 49 s | #141 || New check: tagovi kao čipovi | Tekstno polje, tagovi odvojeni zarezom | Jednostavnije | #141 || Mobitel: Availability kao zaseban tab | Availability je u More (s brojem palih provjera na tabu More); popis provjera je kompaktan, s trakom zadnjih 30 provjera | Tab bar ima mjesta za četiri stavke | #146 |diff --git a/src/backend/availability.py b/src/backend/availability.pyindex fe4ff91..d95e01f 100644--- a/src/backend/availability.py+++ b/src/backend/availability.py@@ -141,6 +141,76 @@ def for_collector(db, server: Server) -> list[dict]:return result[:MAX_CHECKS_PER_SERVER]++# ---------- testing a check before it is saved ----------+# The form's Test button: the check goes once to the collectors of the chosen servers, through Redis, with the+# answer to their next report (they ask every 10 s when idle), and their results come back the same way. Nothing+# is stored in the database; a test is forgotten after two minutes.++TEST_TTL = 120+MAX_TESTS_QUEUED = 5+IDLE_EVERY = 10+++def _spec(fields: dict, check_id: int) -> dict:+ options = {key: value for key, value in json.loads(fields["options"]).items() if key != "upside_down"}+ return {"id": check_id, "type": fields["type"], "target": fields["target"], "interval": fields["interval_seconds"], "retry_interval": fields["retry_interval_seconds"],+ "timeout": fields["timeout_seconds"], **options}+++def start_test(redis, fields: dict, servers: list[int]) -> int:+ """Queue one run of a check (validated ``fields``) for each server's collector. Returns the test's id."""+ test_id = int(redis.incr("infradeck:checktest:seq"))+ spec = json.dumps(_spec(fields, test_id))+ upside_down = json.loads(fields["options"]).get("upside_down") is True+ redis.set(f"infradeck:checktest:{test_id}", json.dumps({"servers": servers, "upside_down": upside_down}), ex=TEST_TTL)+ for server_id in servers:+ queue = f"infradeck:checktest:queue:{server_id}"+ redis.rpush(queue, spec)+ redis.ltrim(queue, -MAX_TESTS_QUEUED, -1)+ redis.expire(queue, 60)+ return test_id+++def tests_for(redis, server_id: int) -> list[dict]:+ """The tests waiting for this server's collector; handing them over takes them off the queue."""+ queue = f"infradeck:checktest:queue:{server_id}"+ items = redis.lrange(queue, 0, -1)+ if items:+ redis.delete(queue)+ return [json.loads(item) for item in items]+++def store_tests(redis, server_id: int, results: list) -> None:+ """Keep this server's results for tests it was given. Untrusted like every result: anything else is dropped."""+ for item in results[:MAX_TESTS_QUEUED * 2]:+ if not isinstance(item, dict) or not isinstance(item.get("check_id"), int) or isinstance(item.get("check_id"), bool) or not isinstance(item.get("ok"), bool):+ continue+ raw = redis.get(f"infradeck:checktest:{item['check_id']}")+ meta = json.loads(raw) if raw else None+ if not meta or server_id not in meta["servers"]:+ continue+ number = lambda key, low, high: item.get(key) if isinstance(item.get(key), int) and not isinstance(item.get(key), bool) and low <= item[key] <= high else None # noqa: E731+ detail = item.get("detail")+ detail = "".join(char for char in detail if char.isprintable())[:200] if isinstance(detail, str) else None+ ok = item["ok"] != meta["upside_down"]+ key = f"infradeck:checktest:{item['check_id']}:results"+ redis.hset(key, str(server_id), json.dumps({"ok": ok, "response_ms": number("response_ms", 0, 600_000), "code": number("code", 100, 599),+ "detail": detail or None, "cert_days": number("cert_days", -10_000, 10_000)}))+ redis.expire(key, TEST_TTL)+++def test_status(redis, test_id: int, names: dict[int, str]) -> dict | None:+ """Where a test is: each server's result, or waiting. None when the test is unknown or forgotten."""+ raw = redis.get(f"infradeck:checktest:{test_id}")+ if not raw:+ return None+ meta = json.loads(raw)+ results = {int(server): json.loads(value) for server, value in (redis.hgetall(f"infradeck:checktest:{test_id}:results") or {}).items()}+ servers = [{"id": server, "hostname": names.get(server, f"server {server}"), **({"state": "done", **results[server]} if server in results else {"state": "waiting"})}+ for server in meta["servers"]]+ return {"id": test_id, "servers": servers, "done": all(server["state"] == "done" for server in servers)}+# ---------- results ----------def store(db, server: Server, results: list, now: datetime | None = None) -> set[int]:diff --git a/src/backend/main.py b/src/backend/main.pyindex 4644f33..bd33ed1 100644--- a/src/backend/main.py+++ b/src/backend/main.py@@ -1979,6 +1979,40 @@ def get_check(check_id: PathId, range: str = Query(default="24h"), session_id: Areturn availability.detail(db, _check_or_404(db, check_id), range)[email protected]("/availability/test")+def test_check(payload: dict, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ """Run a check once from its servers before saving it. Only servers whose collector can run checks are asked."""+ user = _require_admin(session_id, db)+ servers = {server.id: server for server in db.query(Server)}+ try:+ fields = availability.validate(payload, set(servers))+ except ValueError as exc:+ raise HTTPException(status_code=400, detail=str(exc)) from exc+ able = [server_id for server_id in json.loads(fields["server_ids"]) if availability.can_run_checks(servers[server_id])]+ if not able:+ raise HTTPException(status_code=400, detail="None of the chosen servers can run checks yet: their collector is older than 0.10.0.")+ try:+ # The expiry is set when the counter is created, so a failure between two calls can't leave one that never ends.+ redis_client.set(f"infradeck:checktest:user:{user['id']}", 0, ex=60, nx=True)+ if redis_client.incr(f"infradeck:checktest:user:{user['id']}") > 20:+ raise HTTPException(status_code=429, detail="At most 20 tests a minute. Try again shortly.")+ test_id = availability.start_test(redis_client, fields, able)+ except HTTPException:+ raise+ except Exception as exc:+ raise HTTPException(status_code=503, detail="Testing needs Redis, which isn't answering.") from exc+ return availability.test_status(redis_client, test_id, {server_id: servers[server_id].hostname for server_id in servers})++[email protected]("/availability/test/{test_id}")+def get_check_test(test_id: PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ _require_admin(session_id, db)+ status = availability.test_status(redis_client, test_id, dict(db.query(Server.id, Server.hostname).all()))+ if status is None:+ raise HTTPException(status_code=404, detail="This test is over. Run it again.")+ return status++@app.post("/availability/checks")def create_check(payload: dict, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):_require_admin(session_id, db)@@ -2537,6 +2571,7 @@ def ingest_metrics(payload: MetricsIngestRequest, authorization: Annotated[str |class CheckResultsRequest(BaseModel):server_id: int = Field(ge=1, le=2**31 - 1)results: list = Field(default=[], max_length=availability.MAX_RESULTS_PER_REPORT)+ tests: list = Field(default=[], max_length=availability.MAX_TESTS_QUEUED * 2)def _enforce_report_spacing(server: Server) -> None:@@ -2565,7 +2600,12 @@ def report_check_results(payload: CheckResultsRequest, authorization: Annotated[check.last_result_at = nowavailability.evaluate(db, check, now, alerting)db.commit()- return {"checks": availability.for_collector(db, server), "report_every": 5}+ try: # tests from the form's Test button go through Redis; without it there are none, the checks still work+ availability.store_tests(redis_client, server.id, payload.tests)+ tests = availability.tests_for(redis_client, server.id)+ except Exception:+ tests = []+ return {"checks": availability.for_collector(db, server), "tests": tests, "report_every": 5, "idle_every": availability.IDLE_EVERY}@app.get("/dashboard/summary")diff --git a/src/collector/collector.py b/src/collector/collector.pyindex d7114a2..a221665 100644--- a/src/collector/collector.py+++ b/src/collector/collector.py@@ -1349,10 +1349,10 @@ def run_check(check: dict[str, Any]) -> dict[str, Any]:return {"check_id": check["id"], "time": datetime.now(timezone.utc).isoformat(), **{key: value for key, value in outcome.items() if value is not None or key == "ok"}}-def send_check_results(api_url: str, server_id: int, token: str, results: list[dict[str, Any]]) -> dict[str, Any]:+def send_check_results(api_url: str, server_id: int, token: str, results: list[dict[str, Any]], tests: list[dict[str, Any]] | None = None) -> dict[str, Any]:request = urllib.request.Request(f"{api_url.rstrip('/')}/monitoring/metrics/checks",- data=json.dumps({"server_id": server_id, "results": results}).encode(),+ data=json.dumps({"server_id": server_id, "results": results, "tests": tests or []}).encode(),headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},method="POST",)@@ -1360,10 +1360,14 @@ def send_check_results(api_url: str, server_id: int, token: str, results: list[dreturn json.loads(response.read(MAX_BODY_BYTES).decode())+MAX_TESTS = 5++class CheckRunner(threading.Thread):"""Runs the checks the dashboard gave this collector, each on its interval (the retry interval after a- failure), and reports the results every few seconds; with nothing to report it asks for its checks once a- minute. Results that couldn't be sent wait, up to the newest 500."""+ failure), and reports the results every few seconds; with nothing to report it asks for its checks every+ 10 s (as the dashboard says), so a test from the New check form is picked up quickly. A test runs once, at+ once, and its result goes back with the next report. Results that couldn't be sent wait, up to the newest 500."""def __init__(self, api_url: str, credentials: tuple[int, str]) -> None:super().__init__(name="checks", daemon=True)@@ -1376,6 +1380,18 @@ class CheckRunner(threading.Thread):self.lock = threading.Lock()self.pool = concurrent.futures.ThreadPoolExecutor(max_workers=CHECK_WORKERS, thread_name_prefix="check")self.report_every = 5+ self.idle_every = IDLE_REPORT_SECONDS+ self.test_results: collections.deque = collections.deque(maxlen=MAX_TESTS * 2)++ def run_tests(self, raw: Any) -> None:+ """Run each test once, right away; its result is reported apart from the checks' results."""+ for test in (validate_check(item) for item in (raw if isinstance(raw, list) else [])[:MAX_TESTS]):+ if test:+ self.pool.submit(run_check, test).add_done_callback(lambda future: self._tested(future))++ def _tested(self, future: concurrent.futures.Future) -> None:+ with self.lock:+ self.test_results.append(future.result())def set_checks(self, raw: Any) -> None:valid = [check for check in (validate_check(item) for item in (raw if isinstance(raw, list) else [])[:MAX_CHECKS]) if check]@@ -1407,24 +1423,32 @@ class CheckRunner(threading.Thread):def report(self) -> bool:with self.lock:- batch = list(self.pending)+ batch, tested = list(self.pending), list(self.test_results)self.pending.clear()+ self.test_results.clear()server_id, token = self.credentials- try:- response = send_check_results(self.api_url, server_id, token, batch)- except urllib.error.HTTPError as exc:++ def put_back() -> None:with self.lock:self.pending.extendleft(reversed(batch))+ self.test_results.extendleft(reversed(tested))++ try:+ response = send_check_results(self.api_url, server_id, token, batch, tested)+ except urllib.error.HTTPError as exc:+ put_back()if exc.code == 404:- self.report_every = 300 # a dashboard without availability checks+ self.report_every = self.idle_every = 300 # a dashboard without availability checksreturn Falseexcept (urllib.error.URLError, OSError, ValueError, http.client.HTTPException):- with self.lock:- self.pending.extendleft(reversed(batch))+ put_back()return False- self.set_checks(response.get("checks") if isinstance(response, dict) else None)- every = _whole(response.get("report_every"), 2, 300) if isinstance(response, dict) else None- self.report_every = every or 5+ if not isinstance(response, dict):+ response = {}+ self.set_checks(response.get("checks"))+ self.run_tests(response.get("tests"))+ self.report_every = _whole(response.get("report_every"), 2, 300) or 5+ self.idle_every = _whole(response.get("idle_every"), 5, 300) or IDLE_REPORT_SECONDSif batch:print(f"Reported {len(batch)} check results", flush=True)return True@@ -1434,7 +1458,8 @@ class CheckRunner(threading.Thread):while True:self.run_due()since = time.monotonic() - last_report- if since >= IDLE_REPORT_SECONDS or (self.pending and since >= self.report_every):+ # Test results go back at once (the dashboard allows a report every 2 s); check results every few seconds.+ if since >= self.idle_every or (self.test_results and since >= 2) or (self.pending and since >= self.report_every):self.report()last_report = time.monotonic()time.sleep(1)diff --git a/src/frontend/src/availability/CheckForm.jsx b/src/frontend/src/availability/CheckForm.jsxindex f426f60..43e6383 100644--- a/src/frontend/src/availability/CheckForm.jsx+++ b/src/frontend/src/availability/CheckForm.jsx@@ -1,6 +1,6 @@// The New check / Edit form in the drawer: the type, what to check, from which servers, the timing in numbers// with one sentence saying what they mean together, what counts as up, and how bad it is when it's down.-import { useState } from 'react'+import { useEffect, useRef, useState } from 'react'import { Link } from 'react-router-dom'import { apiRequest } from '../api.js'import { Check, Field, Switch, useAction } from '../ui.jsx'@@ -46,6 +46,22 @@ function problems(form, web) {const duration = (seconds) => (seconds < 90 ? `${Math.round(seconds)} seconds` : seconds < 5400 ? `${Math.round(seconds / 60)} ${Math.round(seconds / 60) === 1 ? 'minute' : 'minutes'}` : `${Math.round(seconds / 3600)} hours`)+// How long the form waits for a test: the collectors ask every 10 s when idle, then the check may take its timeout.+const TEST_WAIT_MS = 75_000++// "Test passed · 200 OK · 142 ms from both servers", or what went wrong and where.+function describeTest(test) {+ const done = test.servers.filter((server) => server.state === 'done')+ const waiting = test.servers.filter((server) => server.state === 'waiting')+ const names = (servers) => (servers.length === test.servers.length && servers.length > 1 ? (servers.length === 2 ? 'both servers' : 'all servers') : servers.map((server) => server.hostname).join(', '))+ const failed = done.filter((server) => !server.ok)+ if (failed.length) return { tone: 'warn', text: `Test failed · ${failed.map((server) => `${server.detail || (server.code ? `HTTP ${server.code}` : 'no answer')} from ${server.hostname}`).join(', ')}` }+ if (waiting.length) return { tone: 'muted', text: test.expired ? `No answer from ${names(waiting)}: is its collector running?` : `Testing from ${names(test.servers)}…` }+ const times = done.map((server) => server.response_ms).filter((value) => value != null)+ const code = done.find((server) => server.code)?.code+ return { tone: 'ok', text: `Test passed${code ? ` · ${code} OK` : ''}${times.length ? ` · ${Math.round(times.reduce((a, b) => a + b, 0) / times.length)} ms` : ''} from ${names(done)}` }+}+// "Marked down after 3 failed checks in a row, so about 1 minute after it breaks."function TimingSentence({ form }) {const interval = Number(form.interval_seconds), retries = Number(form.retries), retry = Number(form.retry_interval_seconds)@@ -77,7 +93,10 @@ export function CheckForm({ check, servers, onSaved, onCancel }) {const [form, setForm] = useState(() => initial(check, servers))const [state, run] = useAction()const [tried, setTried] = useState(false)- const set = (changes) => setForm((current) => {+ const [test, setTest] = useState(null) // null, { running: true }, a test from the backend, or { error }+ // A finished test says nothing about a changed form, so it goes once anything changes.+ const set = (changes) => { setTest((current) => (current && (current.done || current.expired || current.error) ? null : current)); setFormFields(changes) }+ const setFormFields = (changes) => setForm((current) => {const next = { ...current, ...changes }// A shorter interval takes the timeout down with it, so the default 48 s doesn't block a 30 s interval.if ('interval_seconds' in changes && /^\d+$/.test(changes.interval_seconds) && Number(next.timeout_seconds) >= Number(changes.interval_seconds)) {@@ -91,18 +110,48 @@ export function CheckForm({ check, servers, onSaved, onCancel }) {const hint = (key) => shown(key) && <span className="mt-1.5 block text-[11px] leading-4 text-warn">{issues[key]}</span>const toggleServer = (id, on) => set({ server_ids: on ? [...form.server_ids, id] : form.server_ids.filter((item) => item !== id) })- const save = (event) => {- event.preventDefault()- setTried(true)- if (Object.keys(issues).length) return+ const bodyOf = () => {const whole = (value) => (/^\d+$/.test(String(value).trim()) ? Number(value) : value)- const body = {+ return {type: form.type, name: form.name.trim(), target: form.target.trim(), tags: form.tags.split(',').map((tag) => tag.trim()).filter(Boolean),server_ids: form.server_ids, rule: form.rule, severity: form.severity, upside_down: form.upside_down,interval_seconds: whole(form.interval_seconds), retries: whole(form.retries), retry_interval_seconds: whole(form.retry_interval_seconds), timeout_seconds: whole(form.timeout_seconds),...(web ? { accepted_codes: form.accepted_codes.split(',').map((code) => code.trim()).filter(Boolean), follow_redirects: form.follow_redirects, ignore_tls: form.ignore_tls } : {}),...(form.type === 'keyword' ? { keyword: form.keyword, case_sensitive: form.case_sensitive } : {}),}+ }++ // Test: the check runs once from its servers; the answer is asked for every 1.5 s until all servers replied.+ const polling = useRef(null)+ useEffect(() => () => clearTimeout(polling.current), [])+ const runTest = async () => {+ setTried(true)+ if (Object.keys(issues).length) return+ clearTimeout(polling.current)+ setTest({ running: true })+ try {+ const started = Date.now()+ let current = await apiRequest('/availability/test', { method: 'POST', body: JSON.stringify(bodyOf()) })+ setTest(current)+ const poll = async () => {+ if (current.done) return+ if (Date.now() - started > TEST_WAIT_MS) { setTest({ ...current, expired: true }); return }+ try { current = await apiRequest(`/availability/test/${current.id}`, { method: 'GET' }); setTest(current) } catch (error) { setTest({ error: error.message }); return }+ polling.current = setTimeout(poll, 1500)+ }+ polling.current = setTimeout(poll, 1500)+ } catch (error) {+ setTest({ error: error.message || 'The test could not start.' })+ }+ }+ const testLine = test && (test.running ? { tone: 'muted', text: 'Starting the test…' } : test.error ? { tone: 'warn', text: test.error } : describeTest(test))+ const testing = Boolean(test && (test.running || (test.servers && !test.done && !test.expired)))++ const save = (event) => {+ event.preventDefault()+ setTried(true)+ if (Object.keys(issues).length) return+ const body = bodyOf()run(async () => {const result = await apiRequest(check ? `/availability/checks/${check.id}` : '/availability/checks', { method: check ? 'PATCH' : 'POST', body: JSON.stringify(body) })onSaved(result)@@ -187,8 +236,12 @@ export function CheckForm({ check, servers, onSaved, onCancel }) {</Field><div className="sticky -bottom-5 -mx-6 -mb-5 flex items-center justify-end gap-2 border-t border-line bg-bg px-6 py-3">- <p role="alert" className="m-0 mr-auto min-w-0 text-xs leading-4 text-warn">{state.error || (tried && Object.keys(issues).length ? `Fix ${Object.keys(issues).length === 1 ? 'the field' : `${Object.keys(issues).length} fields`} marked above.` : '')}</p>+ {state.error || (tried && Object.keys(issues).length) || !testLine+ ? <p role="alert" className="m-0 mr-auto min-w-0 text-xs leading-4 text-warn">{state.error || (tried && Object.keys(issues).length ? `Fix ${Object.keys(issues).length === 1 ? 'the field' : `${Object.keys(issues).length} fields`} marked above.` : '')}</p>+ : <p role="status" className={`m-0 mr-auto flex min-w-0 items-center gap-2 text-xs leading-4 ${testLine.tone === 'ok' ? 'text-ok' : testLine.tone === 'warn' ? 'text-warn' : 'text-soft'}`}>+ <span className={`dot shrink-0 ${testLine.tone === 'ok' ? 'dot-ok' : testLine.tone === 'warn' ? 'dot-warn' : ''}`} /><span className="min-w-0">{testLine.text}</span></p>}<button type="button" className="btn" onClick={onCancel}>Cancel</button>+ <button type="button" className="btn" onClick={runTest} disabled={testing}>{testing ? 'Testing…' : test ? 'Test again' : 'Test'}</button><button type="submit" className="btn btn-primary" disabled={state.busy}>{state.busy ? 'Saving…' : 'Save check'}</button></div></form>diff --git a/tests/collectors/test_collector_checks.py b/tests/collectors/test_collector_checks.pyindex 39ed525..8196698 100644--- a/tests/collectors/test_collector_checks.py+++ b/tests/collectors/test_collector_checks.py@@ -111,7 +111,7 @@ def test_an_open_tcp_port_is_up(site):def test_the_runner_runs_due_checks_and_reports_them(monkeypatch, site):sent = []- def fake_send(api_url, server_id, token, results):+ def fake_send(api_url, server_id, token, results, tests=None):sent.append(list(results))return {"checks": [{**GOOD, "target": f"{site}/ok"}], "report_every": 5}@@ -151,3 +151,34 @@ def test_checks_from_the_dashboard_are_validated_and_capped():assert len(runner.checks) == collector.MAX_CHECKS and 999 not in runner.checksrunner.set_checks(None)assert runner.checks == {} and runner.next_run == {}+++def test_a_test_from_the_form_runs_once_at_once_and_goes_back_on_its_own(monkeypatch, site):+ sent = []+ answers = iter([{"checks": [], "tests": [{**GOOD, "id": 41, "target": f"{site}/broken"}], "idle_every": 10}, {"checks": [], "tests": [], "idle_every": 10}])++ def fake_send(api_url, server_id, token, results, tests=None):+ sent.append((list(results), list(tests or [])))+ return next(answers)++ monkeypatch.setattr(collector, "send_check_results", fake_send)+ runner = collector.CheckRunner("http://dashboard/api", (3, "token"))++ runner.report()+ deadline = time.monotonic() + 5+ while not runner.test_results and time.monotonic() < deadline:+ time.sleep(0.05)+ runner.report()++ assert runner.idle_every == 10 and runner.checks == {} # a test is not a check: it isn't kept or repeated+ assert sent[1][0] == [] and [(item["check_id"], item["ok"], item["detail"]) for item in sent[1][1]] == [(41, False, "HTTP 503")]+++def test_a_test_that_isnt_well_formed_is_not_run(monkeypatch):+ runner = collector.CheckRunner("http://dashboard/api", (3, "token"))+ submitted = []+ monkeypatch.setattr(runner.pool, "submit", lambda *args: submitted.append(args) or type("Future", (), {"add_done_callback": lambda self, done: None})())++ runner.run_tests([{**GOOD, "target": "file:///etc/shadow"}, "junk", None] + [{**GOOD, "id": index} for index in range(1, 10)])++ assert len(submitted) == 2 # only the first five are looked at, and three of those aren't checksdiff --git a/tests/monitoring/test_availability.py b/tests/monitoring/test_availability.pyindex a065bb0..a4059eb 100644--- a/tests/monitoring/test_availability.py+++ b/tests/monitoring/test_availability.py@@ -397,3 +397,60 @@ def test_the_detail_lists_when_a_check_went_down_and_came_back(app):assert [event["kind"] for event in detail["events"]] == ["up", "down"]assert detail["events"][1]["text"].startswith("Down · No answer from alpha")assert detail["last_down"]["ongoing"] is False++++# --- testing a check before saving it -----------------------------------------------------------------++def test_a_test_goes_to_the_collectors_and_their_results_come_back(app):+ with app.Session() as db:+ db.get(Server, 2).system_info = json.dumps({"collector_version": "0.10.0"})+ db.commit()+ started = app.post("/availability/test", json={**NEW_CHECK, "upside_down": False}).json()+ assert started["done"] is False and [server["state"] for server in started["servers"]] == ["waiting", "waiting"]++ handed = report(app, 1, []).json()["tests"]+ assert [test["id"] for test in handed] == [started["id"]] and handed[0]["target"] == "https://git.example.com/"+ assert report(app, 1, []).json()["tests"] == [] # handed over once+ assert report(app, 1, []).json()["idle_every"] == 10++ app.post("/monitoring/metrics/checks", json={"server_id": 1, "results": [], "tests": [{"check_id": started["id"], "ok": True, "response_ms": 142, "code": 200}]},+ headers={"Authorization": f"Bearer {TOKENS[1]}"})+ app.post("/monitoring/metrics/checks", json={"server_id": 2, "results": [], "tests": [{"check_id": started["id"], "ok": False, "detail": "Connection refused"}]},+ headers={"Authorization": f"Bearer {TOKENS[2]}"})+ status = app.get(f"/availability/test/{started['id']}").json()++ assert status["done"] is True+ assert [(server["hostname"], server["ok"], server.get("code"), server.get("detail")) for server in status["servers"]] == [("alpha", True, 200, None), ("bravo", False, None, "Connection refused")]+ with app.Session() as db:+ assert db.query(Check).count() == 0 and db.query(CheckResult).count() == 0 # a test saves nothing+++def test_a_collector_cant_answer_a_test_it_wasnt_given(app):+ started = app.post("/availability/test", json={**NEW_CHECK, "server_ids": [1]}).json()++ app.post("/monitoring/metrics/checks", json={"server_id": 2, "results": [], "tests": [{"check_id": started["id"], "ok": True}]}, headers={"Authorization": f"Bearer {TOKENS[2]}"})++ assert app.get(f"/availability/test/{started['id']}").json()["servers"][0]["state"] == "waiting"+++def test_only_servers_that_can_run_checks_are_asked(app):+ assert app.post("/availability/test", json={**NEW_CHECK, "server_ids": [2]}).status_code == 400 # bravo's collector is 0.9.2+ started = app.post("/availability/test", json={**NEW_CHECK, "server_ids": [1, 2]}).json()+ assert [server["hostname"] for server in started["servers"]] == ["alpha"]+ assert report(app, 2, []).json()["tests"] == []+++def test_a_test_is_checked_like_a_check_and_only_admins_run_it(app):+ assert app.post("/availability/test", json={**NEW_CHECK, "target": "file:///etc/passwd"}).status_code == 400+ app.user.update(id=2, role="member")+ assert app.post("/availability/test", json=NEW_CHECK).status_code == 403+ assert app.get("/availability/test/1").status_code == 403+++def test_an_upside_down_test_passes_when_the_service_is_unreachable(app):+ started = app.post("/availability/test", json={**NEW_CHECK, "server_ids": [1], "upside_down": True}).json()+ app.post("/monitoring/metrics/checks", json={"server_id": 1, "results": [], "tests": [{"check_id": started["id"], "ok": False, "detail": "Connection refused"}]},+ headers={"Authorization": f"Bearer {TOKENS[1]}"})++ assert app.get(f"/availability/test/{started['id']}").json()["servers"][0]["ok"] is True