Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.
Python ai-agentsaiopsdashboarddockerfastapihomelabincident-managementllmmonitoringopenaireactroot-cause-analysisself-hosteduptime-monitoring
refactor(main): move incident routes into routes/incidents.py
karloks2005643f687368b6
diff --git a/src/backend/main.py b/src/backend/main.pyindex a2342a8..f87a769 100644--- a/src/backend/main.py+++ b/src/backend/main.py@@ -2740,99 +2740,6 @@ def _incident_context(db, incident_id: int | None) -> str:+ ("\n".join(facts) + "\n" if facts else "") + "[QUESTION]\n")-class MuteRequest(BaseModel):- minutes: int = Field(default=60, ge=5, le=7 * 24 * 60)---def _incident_or_404(db, incident_id: int) -> tuple[IncidentAlert, Server]:- alert = db.get(IncidentAlert, incident_id)- server = db.get(Server, alert.server_id) if alert else None- if not alert or not server:- raise HTTPException(status_code=404, detail="Incident not found.")- return alert, server--[email protected]("/incidents")-def list_incidents(session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- user = _current_user(session_id, db)- return {**incidents.inbox(db), "can_act": _is_admin(db, user["id"])}--[email protected]("/incidents/{incident_id}")-def get_incident(incident_id: PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- user = _current_user(session_id, db)- alert, server = _incident_or_404(db, incident_id)- users = dict(db.query(User.id, User.name).all())- # Admins always see the agent's diagnosis; others only while Admin shows it to them (checked here, not in the page).- visible = agent_settings.sees_diagnoses(db, "admin" if _is_admin(db, user["id"]) else "user")- report = report_json(latest_report(db, alert.id)) if visible else None- if report and report["status"] == "ok" and alert.status == "active":- report["watch"] = agent_verify.evaluate(db, alert, report["verify"])- return {**incidents.summary(alert, server.hostname, datetime.now(timezone.utc), users), "diagnosis": incidents.diagnose(db, alert, server), "cpu_cores": server.cpu_cores,- "agent": {"report": report, "hidden": not visible, "key_configured": bool(configured_key()), "budget_paused": agent_usage.budget(db)["state"] == "paused"}}--[email protected]("/incidents/{incident_id}/diagnose")-def request_diagnosis(incident_id: PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- """Ask the agent worker to diagnose this incident (again). It works through the queue, so the report appears- on the incident when the worker gets to it."""- user = _require_operator(session_id, db)- alert, _server = _incident_or_404(db, incident_id)- if not configured_key():- raise HTTPException(status_code=409, detail="The agent has no API key. An admin sets OPENAI_API_KEY in .env.")- if agent_usage.budget(db)["state"] == "paused":- raise HTTPException(status_code=429, detail="This month's budget for the agent is used up. An admin can raise it in Admin › Incident agent.")- enforce_limit(redis_client, scope="diagnose-user", subject=user["id"], limit=10, window_seconds=600)- try:- agent_jobs.publish(redis_client, agent_jobs.REQUESTED, alert.id, alert.severity)- except Exception as exc:- raise HTTPException(status_code=503, detail="Redis is not answering, so the request can't be queued.") from exc- record_audit_log(db, user_id=user["id"], level="INFO", message=f"Asked the agent to diagnose {incidents.reference(alert.id)}", source="agent")- return {"queued": True}---def _incident_action(db, incident_id: int, session_id, change, message: str) -> dict:- user = _require_operator(session_id, db)- alert, server = _incident_or_404(db, incident_id)- change(alert, user)- db.commit()- record_audit_log(db, user_id=user["id"], level="INFO", message=f"{message} {incidents.reference(alert.id)} ({alert.title})"[:255], source="incidents")- return incidents.summary(alert, server.hostname, datetime.now(timezone.utc), dict(db.query(User.id, User.name).all()))--[email protected]("/incidents/{incident_id}/acknowledge")-def acknowledge_incident(incident_id: PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- def change(alert, user):- alert.acknowledged_at, alert.acknowledged_by = datetime.now(timezone.utc), user["id"]- return _incident_action(db, incident_id, session_id, change, "Acknowledged")--[email protected]("/incidents/{incident_id}/resolve")-def resolve_incident(incident_id: PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- """Close an incident by hand. If the rule behind it is still firing at the next check-in, a new one opens."""- def change(alert, user):- if alert.status != "active":- raise HTTPException(status_code=409, detail="This incident is already resolved.")- alert.status, alert.resolved_at, alert.resolution = "resolved", datetime.now(timezone.utc), f"Marked resolved by {user['name']}"[:255]- return _incident_action(db, incident_id, session_id, change, "Marked resolved")--[email protected]("/incidents/{incident_id}/mute")-def mute_incident(incident_id: PathId, payload: MuteRequest | None = None, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- minutes = (payload or MuteRequest()).minutes-- def change(alert, user):- alert.muted_until = datetime.now(timezone.utc) + timedelta(minutes=minutes)- return _incident_action(db, incident_id, session_id, change, f"Muted for {minutes} min")--[email protected]("/incidents/{incident_id}/unmute")-def unmute_incident(incident_id: PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):- def change(alert, user):- alert.muted_until = None- return _incident_action(db, incident_id, session_id, change, "Unmuted")--# ---------------------------------------------------------------------------------------------------------# The account page: profile, preferences, API tokens, notification channels@@ -3403,5 +3310,6 @@ def root():# Routes split out of this file; imported last so they can reference this module's shared helpers (see routes/).-from .routes import availability as availability_routes # noqa: E402+from .routes import availability as availability_routes, incidents as incidents_routes # noqa: E402app.include_router(availability_routes.router)+app.include_router(incidents_routes.router)diff --git a/src/backend/routes/incidents.py b/src/backend/routes/incidents.pynew file mode 100644index 0000000..f3f3baa--- /dev/null+++ b/src/backend/routes/incidents.py@@ -0,0 +1,114 @@+"""Incidents (the Agent page): the inbox, one incident with its diagnosis, asking the agent to diagnose again, and+the operator actions (acknowledge, resolve, mute, unmute). Incident logic lives in incidents.py and the agent+package; this module is the HTTP layer. ``_agent_note`` and ``_incident_context`` stay in main (the Overview and+chat use them)."""+from __future__ import annotations++from datetime import datetime, timedelta, timezone+from typing import Annotated++from fastapi import APIRouter, Cookie, Depends, HTTPException+from pydantic import BaseModel, Field++from .. import incidents, main+from ...agent import jobs as agent_jobs, settings as agent_settings, usage as agent_usage, verify as agent_verify+from ...agent.diagnosis import latest as latest_report, report_json+from ...agent.providers import configured_key+from ..config import SESSION_COOKIE_NAME+from ..db import get_db+from ..models import IncidentAlert, Server, User++router = APIRouter()+++class MuteRequest(BaseModel):+ minutes: int = Field(default=60, ge=5, le=7 * 24 * 60)+++def _incident_or_404(db, incident_id: int) -> tuple[IncidentAlert, Server]:+ alert = db.get(IncidentAlert, incident_id)+ server = db.get(Server, alert.server_id) if alert else None+ if not alert or not server:+ raise HTTPException(status_code=404, detail="Incident not found.")+ return alert, server++[email protected]("/incidents")+def list_incidents(session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ user = main._current_user(session_id, db)+ return {**incidents.inbox(db), "can_act": main._is_admin(db, user["id"])}++[email protected]("/incidents/{incident_id}")+def get_incident(incident_id: main.PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ user = main._current_user(session_id, db)+ alert, server = _incident_or_404(db, incident_id)+ users = dict(db.query(User.id, User.name).all())+ # Admins always see the agent's diagnosis; others only while Admin shows it to them (checked here, not in the page).+ visible = agent_settings.sees_diagnoses(db, "admin" if main._is_admin(db, user["id"]) else "user")+ report = report_json(latest_report(db, alert.id)) if visible else None+ if report and report["status"] == "ok" and alert.status == "active":+ report["watch"] = agent_verify.evaluate(db, alert, report["verify"])+ return {**incidents.summary(alert, server.hostname, datetime.now(timezone.utc), users), "diagnosis": incidents.diagnose(db, alert, server), "cpu_cores": server.cpu_cores,+ "agent": {"report": report, "hidden": not visible, "key_configured": bool(configured_key()), "budget_paused": agent_usage.budget(db)["state"] == "paused"}}++[email protected]("/incidents/{incident_id}/diagnose")+def request_diagnosis(incident_id: main.PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ """Ask the agent worker to diagnose this incident (again). It works through the queue, so the report appears+ on the incident when the worker gets to it."""+ user = main._require_operator(session_id, db)+ alert, _server = _incident_or_404(db, incident_id)+ if not configured_key():+ raise HTTPException(status_code=409, detail="The agent has no API key. An admin sets OPENAI_API_KEY in .env.")+ if agent_usage.budget(db)["state"] == "paused":+ raise HTTPException(status_code=429, detail="This month's budget for the agent is used up. An admin can raise it in Admin › Incident agent.")+ main.enforce_limit(main.redis_client, scope="diagnose-user", subject=user["id"], limit=10, window_seconds=600)+ try:+ agent_jobs.publish(main.redis_client, agent_jobs.REQUESTED, alert.id, alert.severity)+ except Exception as exc:+ raise HTTPException(status_code=503, detail="Redis is not answering, so the request can't be queued.") from exc+ main.record_audit_log(db, user_id=user["id"], level="INFO", message=f"Asked the agent to diagnose {incidents.reference(alert.id)}", source="agent")+ return {"queued": True}+++def _incident_action(db, incident_id: int, session_id, change, message: str) -> dict:+ user = main._require_operator(session_id, db)+ alert, server = _incident_or_404(db, incident_id)+ change(alert, user)+ db.commit()+ main.record_audit_log(db, user_id=user["id"], level="INFO", message=f"{message} {incidents.reference(alert.id)} ({alert.title})"[:255], source="incidents")+ return incidents.summary(alert, server.hostname, datetime.now(timezone.utc), dict(db.query(User.id, User.name).all()))++[email protected]("/incidents/{incident_id}/acknowledge")+def acknowledge_incident(incident_id: main.PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ def change(alert, user):+ alert.acknowledged_at, alert.acknowledged_by = datetime.now(timezone.utc), user["id"]+ return _incident_action(db, incident_id, session_id, change, "Acknowledged")++[email protected]("/incidents/{incident_id}/resolve")+def resolve_incident(incident_id: main.PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ """Close an incident by hand. If the rule behind it is still firing at the next check-in, a new one opens."""+ def change(alert, user):+ if alert.status != "active":+ raise HTTPException(status_code=409, detail="This incident is already resolved.")+ alert.status, alert.resolved_at, alert.resolution = "resolved", datetime.now(timezone.utc), f"Marked resolved by {user['name']}"[:255]+ return _incident_action(db, incident_id, session_id, change, "Marked resolved")++[email protected]("/incidents/{incident_id}/mute")+def mute_incident(incident_id: main.PathId, payload: MuteRequest | None = None, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ minutes = (payload or MuteRequest()).minutes++ def change(alert, user):+ alert.muted_until = datetime.now(timezone.utc) + timedelta(minutes=minutes)+ return _incident_action(db, incident_id, session_id, change, f"Muted for {minutes} min")++[email protected]("/incidents/{incident_id}/unmute")+def unmute_incident(incident_id: main.PathId, session_id: Annotated[str | None, Cookie(alias=SESSION_COOKIE_NAME)] = None, db=Depends(get_db)):+ def change(alert, user):+ alert.muted_until = None+ return _incident_action(db, incident_id, session_id, change, "Unmuted")