Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.
Python ai-agentsaiopsdashboarddockerfastapihomelabincident-managementllmmonitoringopenaireactroot-cause-analysisself-hosteduptime-monitoring
refactor(main): move availability routes into routes/availability.py
karloks20054c76b652bc8c
diff --git a/src/backend/main.py b/src/backend/main.pyindex 317b288..a2342a8 100644--- a/src/backend/main.py+++ b/src/backend/main.py@@ -1934,120 +1934,6 @@ def update_check_interval(server_id: PathId, payload: dict, session_id: Annotatereturn {"id": server.id, "check_interval_seconds": server.check_interval_seconds}-# ---------- Availability (Milestone 11) ------------def _check_or_404(db, check_id: int) -> Check:- check = db.get(Check, check_id)- if not check:- raise HTTPException(status_code=404, detail="Check not found.")- return check--[email protected]("/availability")-def get_availability(session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- user = _current_user(session_id, db)- is_admin = _is_admin(db, user["id"])- # Non-admins can't edit checks, so they get targets with credentials and query strings (possible secrets) removed.- return {**availability.overview(db, redact=not is_admin), "can_edit": is_admin}---def _save_check(db, check: Check, payload: dict) -> dict:- try:- fields = availability.validate(payload, {server_id for (server_id,) in db.query(Server.id)})- except ValueError as exc:- print(f"Check not saved: {exc}", flush=True)- raise HTTPException(status_code=400, detail=str(exc)) from exc- servers = json.loads(fields["server_ids"])- for server_id in servers:- others = sum(server_id in availability.servers_of(other) for other in db.query(Check).filter(Check.id != (check.id or 0)))- if others >= availability.MAX_CHECKS_PER_SERVER:- raise HTTPException(status_code=400, detail=f"A server can run at most {availability.MAX_CHECKS_PER_SERVER} checks.")- old_servers = availability.servers_of(check) if check.id else []- for key, value in fields.items():- setattr(check, key, value)- if check.id is None:- db.add(check)- elif old_servers and old_servers[0] != servers[0]:- resolve_alerts(db, old_servers[0], [availability.rule_key(check)])- db.flush()- availability.evaluate(db, check)- db.commit()- return availability.summary(db, check, dict(db.query(Server.id, Server.hostname).all()))--[email protected]("/availability/checks/{check_id}")-def get_check(check_id: PathId, range: str = Query(default="24h"), session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- _current_user(session_id, db)- return availability.detail(db, _check_or_404(db, check_id), range)--[email protected]("/availability/test")-def test_check(payload: dict, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- """Run a check once from its servers before saving it. Only servers whose collector can run checks are asked."""- user = _require_admin(session_id, db)- servers = {server.id: server for server in db.query(Server)}- try:- fields = availability.validate(payload, set(servers))- except ValueError as exc:- raise HTTPException(status_code=400, detail=str(exc)) from exc- able = [server_id for server_id in json.loads(fields["server_ids"]) if availability.can_run_checks(servers[server_id])]- if not able:- raise HTTPException(status_code=400, detail="None of the chosen servers can run checks yet: their collector is older than 0.10.0.")- try:- # The expiry is set when the counter is created, so a failure between two calls can't leave one that never ends.- redis_client.set(f"infradeck:checktest:user:{user['id']}", 0, ex=60, nx=True)- if redis_client.incr(f"infradeck:checktest:user:{user['id']}") > 20:- raise HTTPException(status_code=429, detail="At most 20 tests a minute. Try again shortly.")- test_id = availability.start_test(redis_client, fields, able)- except HTTPException:- raise- except Exception as exc:- raise HTTPException(status_code=503, detail="Testing needs Redis, which isn't answering.") from exc- return availability.test_status(redis_client, test_id, {server_id: servers[server_id].hostname for server_id in servers})--[email protected]("/availability/test/{test_id}")-def get_check_test(test_id: PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- _require_admin(session_id, db)- status = availability.test_status(redis_client, test_id, dict(db.query(Server.id, Server.hostname).all()))- if status is None:- raise HTTPException(status_code=404, detail="This test is over. Run it again.")- return status--[email protected]("/availability/checks")-def create_check(payload: dict, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- _require_admin(session_id, db)- return _save_check(db, Check(), payload)--[email protected]("/availability/checks/{check_id}")-def update_check(check_id: PathId, payload: dict, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- _require_admin(session_id, db)- return _save_check(db, _check_or_404(db, check_id), payload)--[email protected]("/availability/checks/{check_id}/pause")-def pause_check(check_id: PathId, payload: dict, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- _require_admin(session_id, db)- check = _check_or_404(db, check_id)- check.paused = payload.get("paused") is True- if not check.paused:- check.state = "pending"- availability.evaluate(db, check)- db.commit()- return availability.summary(db, check, dict(db.query(Server.id, Server.hostname).all()))--[email protected]("/availability/checks/{check_id}")-def delete_check(check_id: PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- _require_admin(session_id, db)- check = _check_or_404(db, check_id)- availability.forget_check(db, check)- db.commit()- return {"status": "deleted", "id": check_id}--MAX_HELPER_EVENTS_PER_CHECKIN = 20MAX_HELPER_EVENTS_PER_WEEK = 500HELPER_NOTICE_KIND = "helper_notice"@@ -3514,3 +3400,8 @@ def admin_agent_test(session_id: Annotated[str | None, Cookie(alias=SESSION_COOK@app.get("/")def root():return {"message": "Infradeck API"}+++# Routes split out of this file; imported last so they can reference this module's shared helpers (see routes/).+from .routes import availability as availability_routes # noqa: E402+app.include_router(availability_routes.router)diff --git a/src/backend/routes/__init__.py b/src/backend/routes/__init__.pynew file mode 100644index 0000000..52a0f49--- /dev/null+++ b/src/backend/routes/__init__.py@@ -0,0 +1,7 @@+"""HTTP routes, grouped by area, mounted on the app in main.py.++Handlers reach the app's shared helpers and singletons through the ``main`` module (``main._current_user``,+``main.redis_client`` and the like) rather than importing them by value, so the tests that monkeypatch those names+on ``main`` still affect the routers. ``get_db`` is imported directly from ``db`` because it is the same object the+tests override.+"""diff --git a/src/backend/routes/availability.py b/src/backend/routes/availability.pynew file mode 100644index 0000000..4b0641b--- /dev/null+++ b/src/backend/routes/availability.py@@ -0,0 +1,128 @@+"""Availability page: the service checks the collectors run, and the Test button (Milestone 11). See availability.py+for the logic; this module is only the HTTP layer."""+from __future__ import annotations++import json+from typing import Annotated++from fastapi import APIRouter, Cookie, Depends, HTTPException, Query++from .. import availability, main+from ..alerts import resolve_alerts+from ..config import SESSION_COOKIE_NAME+from ..db import get_db+from ..models import Check, Server++router = APIRouter()+++def _check_or_404(db, check_id: int) -> Check:+ check = db.get(Check, check_id)+ if not check:+ raise HTTPException(status_code=404, detail="Check not found.")+ return check++[email protected]("/availability")+def get_availability(session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ user = main._current_user(session_id, db)+ is_admin = main._is_admin(db, user["id"])+ # Non-admins can't edit checks, so they get targets with credentials and query strings (possible secrets) removed.+ return {**availability.overview(db, redact=not is_admin), "can_edit": is_admin}+++def _save_check(db, check: Check, payload: dict) -> dict:+ try:+ fields = availability.validate(payload, {server_id for (server_id,) in db.query(Server.id)})+ except ValueError as exc:+ print(f"Check not saved: {exc}", flush=True)+ raise HTTPException(status_code=400, detail=str(exc)) from exc+ servers = json.loads(fields["server_ids"])+ for server_id in servers:+ others = sum(server_id in availability.servers_of(other) for other in db.query(Check).filter(Check.id != (check.id or 0)))+ if others >= availability.MAX_CHECKS_PER_SERVER:+ raise HTTPException(status_code=400, detail=f"A server can run at most {availability.MAX_CHECKS_PER_SERVER} checks.")+ old_servers = availability.servers_of(check) if check.id else []+ for key, value in fields.items():+ setattr(check, key, value)+ if check.id is None:+ db.add(check)+ elif old_servers and old_servers[0] != servers[0]:+ resolve_alerts(db, old_servers[0], [availability.rule_key(check)])+ db.flush()+ availability.evaluate(db, check)+ db.commit()+ return availability.summary(db, check, dict(db.query(Server.id, Server.hostname).all()))++[email protected]("/availability/checks/{check_id}")+def get_check(check_id: main.PathId, range: str = Query(default="24h"), session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ main._current_user(session_id, db)+ return availability.detail(db, _check_or_404(db, check_id), range)++[email protected]("/availability/test")+def test_check(payload: dict, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ """Run a check once from its servers before saving it. Only servers whose collector can run checks are asked."""+ user = main._require_admin(session_id, db)+ servers = {server.id: server for server in db.query(Server)}+ try:+ fields = availability.validate(payload, set(servers))+ except ValueError as exc:+ raise HTTPException(status_code=400, detail=str(exc)) from exc+ able = [server_id for server_id in json.loads(fields["server_ids"]) if availability.can_run_checks(servers[server_id])]+ if not able:+ raise HTTPException(status_code=400, detail="None of the chosen servers can run checks yet: their collector is older than 0.10.0.")+ try:+ # The expiry is set when the counter is created, so a failure between two calls can't leave one that never ends.+ main.redis_client.set(f"infradeck:checktest:user:{user['id']}", 0, ex=60, nx=True)+ if main.redis_client.incr(f"infradeck:checktest:user:{user['id']}") > 20:+ raise HTTPException(status_code=429, detail="At most 20 tests a minute. Try again shortly.")+ test_id = availability.start_test(main.redis_client, fields, able)+ except HTTPException:+ raise+ except Exception as exc:+ raise HTTPException(status_code=503, detail="Testing needs Redis, which isn't answering.") from exc+ return availability.test_status(main.redis_client, test_id, {server_id: servers[server_id].hostname for server_id in servers})++[email protected]("/availability/test/{test_id}")+def get_check_test(test_id: main.PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ main._require_admin(session_id, db)+ status = availability.test_status(main.redis_client, test_id, dict(db.query(Server.id, Server.hostname).all()))+ if status is None:+ raise HTTPException(status_code=404, detail="This test is over. Run it again.")+ return status++[email protected]("/availability/checks")+def create_check(payload: dict, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ main._require_admin(session_id, db)+ return _save_check(db, Check(), payload)++[email protected]("/availability/checks/{check_id}")+def update_check(check_id: main.PathId, payload: dict, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ main._require_admin(session_id, db)+ return _save_check(db, _check_or_404(db, check_id), payload)++[email protected]("/availability/checks/{check_id}/pause")+def pause_check(check_id: main.PathId, payload: dict, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ main._require_admin(session_id, db)+ check = _check_or_404(db, check_id)+ check.paused = payload.get("paused") is True+ if not check.paused:+ check.state = "pending"+ availability.evaluate(db, check)+ db.commit()+ return availability.summary(db, check, dict(db.query(Server.id, Server.hostname).all()))++[email protected]("/availability/checks/{check_id}")+def delete_check(check_id: main.PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ main._require_admin(session_id, db)+ check = _check_or_404(db, check_id)+ availability.forget_check(db, check)+ db.commit()+ return {"status": "deleted", "id": check_id}