Self-hosted monitoring for homelabs: host and container metrics and uptime checks from all your machines in one dashboard, with an optional AI agent that diagnoses incidents and suggests fixes.
Python ai-agentsaiopsdashboarddockerfastapihomelabincident-managementllmmonitoringopenaireactroot-cause-analysisself-hosteduptime-monitoring
fix(notify): look at new incidents first so a busy day can't hold back their notifications (#215)
Karlo Kajba Šimanić9ea6770ae503
diff --git a/src/backend/main.py b/src/backend/main.pyindex b7dd55d..6116850 100644--- a/src/backend/main.py+++ b/src/backend/main.py@@ -1623,6 +1623,7 @@ def _deliver(db, channel: NotificationChannel, owner: User, title: str, body: stNOTIFY_WINDOW = timedelta(hours=24)NOTIFY_MARK_SECONDS = 7 * 24 * 3600+NOTIFY_SCAN_LIMIT = 2000 # incidents looked at per round; only a flood of incidents reaches it# A new warning or critical incident waits this long for the agent's diagnosis, so the notification can carry it.@@ -1663,7 +1664,10 @@ def dispatch_notifications(db, now: datetime | None = None) -> int:by_user.setdefault(channel.user_id, []).append(channel)if not by_user:return 0- recent = db.query(IncidentAlert).filter((IncidentAlert.created_at >= now - NOTIFY_WINDOW) | (IncidentAlert.resolved_at >= now - NOTIFY_WINDOW), IncidentAlert.severity != "INFO").order_by(IncidentAlert.created_at).limit(200).all()+ # Critical first, then the latest change first, so a new incident is always looked at at once (#212). An+ # incident everyone was already told about costs one Redis look-up per user, so the cap only bounds a flood.+ recent = db.query(IncidentAlert).filter((IncidentAlert.created_at >= now - NOTIFY_WINDOW) | (IncidentAlert.resolved_at >= now - NOTIFY_WINDOW), IncidentAlert.severity != "INFO") \+ .order_by(case((IncidentAlert.severity == "CRITICAL", 0), else_=1), func.coalesce(IncidentAlert.resolved_at, IncidentAlert.created_at).desc()).limit(NOTIFY_SCAN_LIMIT).all()hostnames = dict(db.query(Server.id, Server.hostname).all())coming = Nonesent = 0@@ -1673,6 +1677,12 @@ def dispatch_notifications(db, now: datetime | None = None) -> int:event = "resolved" if alert.status == "resolved" else alert.severity.lower()if event not in notify.EVENTS:continue+ try:+ waiting = [user_id for user_id in by_user if not redis_client.exists(f"infradeck:notified:{alert.id}:{user_id}:{event}")]+ except Exception: # without the markers everyone would be told again every few seconds+ return sent+ if not waiting:+ continuereport = latest_report(db, alert.id)if event != "resolved" and report is None:coming = _diagnosis_coming(db) if coming is None else coming@@ -1680,7 +1690,8 @@ def dispatch_notifications(db, now: datetime | None = None) -> int:if coming and now - created < DIAGNOSIS_WAIT:continue # checked again in 15 s; after DIAGNOSIS_WAIT it goes out without the diagnosissummary = _agent_summary(report) if event != "resolved" else None- for user_id, channels in by_user.items():+ for user_id in waiting:+ channels = by_user[user_id]preferences = _preferences(owners[user_id])routing = preferences["routing"].get(event, {})wanted = [channel for channel in channels if routing.get(str(channel.id), notify.DEFAULT_ROUTING[event])]diff --git a/tests/accounts/test_new_pages.py b/tests/accounts/test_new_pages.pyindex cb8b85f..041c8a0 100644--- a/tests/accounts/test_new_pages.py+++ b/tests/accounts/test_new_pages.py@@ -261,6 +261,23 @@ def test_incidents_are_sent_once_following_routing_quiet_hours_and_muting(app, oassert main.dispatch_notifications(app.db) == 1 and b"INC-0050" in outbox[-1][1]+def test_a_new_incident_is_sent_however_many_came_before_it_that_day(app, outbox):+ viewer = signed_in(app, 3)+ viewer.post("/account/channels", json={"kind": "ntfy", "target": "https://ntfy.example/t"})+ now = datetime.now(timezone.utc)+ for index in range(250): # a busy day, more than the 200 incidents a round used to look at (#212)+ app.db.add(IncidentAlert(id=1000 + index, server_id=1, severity="WARNING", title=f"Container c{index} exited", description="d", status="active",+ rule_key=f"container:c{index}:exited", created_at=now - timedelta(hours=20) + timedelta(minutes=index)))+ app.db.commit()+ assert main.dispatch_notifications(app.db) == 252 # all of them in one round, and the two from the fixture++ app.db.add(IncidentAlert(id=5000, server_id=1, severity="CRITICAL", title="Disk / on alpha is full", description="d", status="active", rule_key="host:disk:/",+ created_at=now - timedelta(minutes=5)))+ app.db.commit()+ assert main.dispatch_notifications(app.db) == 1 and b"INC-5000" in outbox[-1][1]+ assert main.dispatch_notifications(app.db) == 0 # and still each incident once++# --- admin -------------------------------------------------------------------------------------------def test_users_are_listed_and_changed_but_not_the_owner_or_yourself(app):