Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Stop tenant monitor lookups falling back to the default org
rcourtman94f96eb04341
diff --git a/internal/api/agent_handlers_base.go b/internal/api/agent_handlers_base.goindex d30dcf5d35..90c8c9b86f 100644--- a/internal/api/agent_handlers_base.go+++ b/internal/api/agent_handlers_base.go@@ -2,10 +2,13 @@ package apiimport ("context"+ "net/http"+ "strings""sync""github.com/rcourtman/pulse-go-rewrite/internal/monitoring""github.com/rcourtman/pulse-go-rewrite/internal/websocket"+ "github.com/rs/zerolog/log")// baseAgentHandlers provides the shared monitor-management and broadcast logic@@ -53,21 +56,47 @@ func (b *baseAgentHandlers) SetMultiTenantMonitor(mtm *monitoring.MultiTenantMon}}-// getMonitor returns the monitor for the organization in the request context,-// falling back to the default-org monitor.+// getMonitor returns the monitor for the organization in the request context.+// Only the default organization may fall back to the primary monitor. Any+// other organization whose tenant monitor cannot be resolved gets nil, so its+// requests never read or change the default organization's agents.func (b *baseAgentHandlers) getMonitor(ctx context.Context) *monitoring.Monitor {b.stateMu.RLock()mtMonitor := b.mtMonitordefaultMonitor := b.defaultMonitorb.stateMu.RUnlock()- orgID := GetOrgID(ctx)- if mtMonitor != nil {- if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {- return m+ orgID := strings.TrimSpace(GetOrgID(ctx))+ if orgID == "" || orgID == "default" {+ if mtMonitor != nil {+ if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {+ return m+ }}+ return defaultMonitor+ }++ if mtMonitor == nil {+ log.Warn().Str("org_id", orgID).Msg("Tenant monitor manager unavailable for agent request")+ return nil+ }+ m, err := mtMonitor.GetMonitor(orgID)+ if err != nil || m == nil {+ log.Warn().Err(err).Str("org_id", orgID).Msg("Failed to resolve tenant monitor for agent request")+ return nil+ }+ return m+}++// requireMonitor resolves the request organization's monitor, writing a 503+// and returning nil when it is unavailable.+func (b *baseAgentHandlers) requireMonitor(w http.ResponseWriter, r *http.Request) *monitoring.Monitor {+ monitor := b.getMonitor(r.Context())+ if monitor == nil {+ writeErrorResponse(w, http.StatusServiceUnavailable, "tenant_unavailable", "Tenant monitor is not available", nil)+ return nil}- return defaultMonitor+ return monitor}// broadcastState pushes monitor state to tenant-scoped WebSocket clients whendiff --git a/internal/api/agent_ingest.go b/internal/api/agent_ingest.goindex 9195a50cd2..e0fb470e45 100644--- a/internal/api/agent_ingest.go+++ b/internal/api/agent_ingest.go@@ -100,8 +100,12 @@ func (h *UnifiedAgentHandlers) HandleReport(w http.ResponseWriter, r *http.Requereport.Timestamp = time.Now().UTC()}+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }tokenRecord := getAPITokenRecordFromRequest(r)- host, err := h.getMonitor(r.Context()).ApplyHostReport(report, tokenRecord)+ host, err := monitor.ApplyHostReport(report, tokenRecord)if err != nil {writeErrorResponse(w, http.StatusBadRequest, "invalid_report", err.Error(), nil)return@@ -129,7 +133,7 @@ func (h *UnifiedAgentHandlers) HandleReport(w http.ResponseWriter, r *http.Reque// identity before returning remote config, otherwise an old false override// survives remove/reinstall and immediately shuts down an agent launched// with --enable-commands (#1728).- if err := reconcileInstallTokenCommandPolicy(h.getMonitor(r.Context()), tokenRecord, host); err != nil {+ if err := reconcileInstallTokenCommandPolicy(monitor, tokenRecord, host); err != nil {log.Error().Err(err).Str("agent_id", host.ID).Str("token_id", tokenID(tokenRecord)).@@ -145,7 +149,7 @@ func (h *UnifiedAgentHandlers) HandleReport(w http.ResponseWriter, r *http.Requeh.broadcastState(r.Context())// Include any server-side config overrides in the response- serverConfig := h.getMonitor(r.Context()).GetHostAgentConfig(host.ID)+ serverConfig := monitor.GetHostAgentConfig(host.ID)serverConfig = sanitizeHostAgentConfigForToken(serverConfig, tokenRecord, host)resp := map[string]any{@@ -280,7 +284,11 @@ func (h *UnifiedAgentHandlers) HandleLookup(w http.ResponseWriter, r *http.Reque// Use the live host snapshot (not the global mock snapshot) so agent// registrations can still be validated while Pulse is in mock/demo mode.- hosts := h.getMonitor(r.Context()).GetLiveHostsSnapshot()+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ hosts := monitor.GetLiveHostsSnapshot()var (host models.Host@@ -415,7 +423,11 @@ func (h *UnifiedAgentHandlers) HandleDeleteHost(w http.ResponseWriter, r *http.R}// Remove the agent from state.- host, err := h.getMonitor(r.Context()).RemoveHostAgent(agentID)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ host, err := monitor.RemoveHostAgent(agentID)if err != nil {writeErrorResponse(w, http.StatusInternalServerError, "agent_removal_failed", err.Error(), nil)return@@ -447,7 +459,11 @@ func (h *UnifiedAgentHandlers) HandleAllowReenroll(w http.ResponseWriter, r *httreturn}- if err := h.getMonitor(r.Context()).AllowHostAgentReenroll(agentID); err != nil {+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ if err := monitor.AllowHostAgentReenroll(agentID); err != nil {writeErrorResponse(w, http.StatusBadRequest, "agent_reenroll_failed", err.Error(), nil)return}@@ -505,6 +521,9 @@ func (h *UnifiedAgentHandlers) resolveConfigAgent(ctx context.Context, agentID s// Use the live host snapshot so agents can still fetch config while// Pulse is running in mock/demo mode.monitor := h.getMonitor(ctx)+ if monitor == nil {+ return models.Host{}, false+ }hosts := monitor.GetLiveHostsSnapshot()if record == nil || record.HasScope(config.ScopeSettingsWrite) || record.HasScope(config.ScopeAgentManage) {@@ -617,6 +636,11 @@ func (h *UnifiedAgentHandlers) handleGetConfig(w http.ResponseWriter, r *http.Rereturn}+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+host, ok := h.resolveConfigAgent(r.Context(), agentID, record)if !ok {writeErrorResponse(w, http.StatusNotFound, "agent_not_found", "Agent has not registered with Pulse yet", nil)@@ -627,7 +651,7 @@ func (h *UnifiedAgentHandlers) handleGetConfig(w http.ResponseWriter, r *http.ReagentID = host.ID- config := h.getMonitor(r.Context()).GetHostAgentConfig(agentID)+ config := monitor.GetHostAgentConfig(agentID)config = sanitizeHostAgentConfigForToken(config, record, host)signedConfig, err := h.signAgentConfig(agentID, config)if err != nil {@@ -765,7 +789,11 @@ func (h *UnifiedAgentHandlers) ensureAgentTokenMatch(w http.ResponseWriter, r *h}// Use the live host snapshot so mock/demo mode doesn't block agent auth checks.- hosts := h.getMonitor(r.Context()).GetLiveHostsSnapshot()+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return false+ }+ hosts := monitor.GetLiveHostsSnapshot()for _, host := range hosts {if host.ID != agentID {continue@@ -797,7 +825,11 @@ func (h *UnifiedAgentHandlers) handlePatchConfig(w http.ResponseWriter, r *http.return}- if err := h.getMonitor(r.Context()).UpdateHostAgentConfig(agentID, req.CommandsEnabled); err != nil {+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ if err := monitor.UpdateHostAgentConfig(agentID, req.CommandsEnabled); err != nil {writeErrorResponse(w, http.StatusInternalServerError, "update_failed", err.Error(), nil)return}@@ -849,7 +881,10 @@ func (h *UnifiedAgentHandlers) HandleUninstall(w http.ResponseWriter, r *http.Relog.Info().Str("agentId", agentID).Msg("Received unregistration request from agent uninstaller")- monitor := h.getMonitor(r.Context())+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }record := getAPITokenRecordFromRequest(r)var err errorif record != nil {@@ -915,7 +950,11 @@ func (h *UnifiedAgentHandlers) HandleLink(w http.ResponseWriter, r *http.Requestreturn}- if err := h.getMonitor(r.Context()).LinkHostAgent(agentID, nodeID); err != nil {+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ if err := monitor.LinkHostAgent(agentID, nodeID); err != nil {writeErrorResponse(w, http.StatusBadRequest, "link_failed", err.Error(), nil)return}@@ -957,7 +996,11 @@ func (h *UnifiedAgentHandlers) HandleUnlink(w http.ResponseWriter, r *http.Requereturn}- if err := h.getMonitor(r.Context()).UnlinkHostAgent(agentID); err != nil {+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ if err := monitor.UnlinkHostAgent(agentID); err != nil {writeErrorResponse(w, http.StatusNotFound, "unlink_failed", err.Error(), nil)return}diff --git a/internal/api/alerting/alerts.go b/internal/api/alerting/alerts.goindex 8cc4165417..8f975db233 100644--- a/internal/api/alerting/alerts.go+++ b/internal/api/alerting/alerts.go@@ -132,13 +132,39 @@ func (h *AlertHandlers) getMonitor(ctx context.Context) AlertMonitor {defaultMonitor := h.defaultMonitorh.stateMu.RUnlock()- orgID := apicontext.OrgID(ctx)- if mtMonitor != nil {- if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {- return NewAlertMonitorWrapper(m)+ orgID := strings.TrimSpace(apicontext.OrgID(ctx))+ if orgID == "" || orgID == "default" {+ if mtMonitor != nil {+ if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {+ return NewAlertMonitorWrapper(m)+ }}+ return defaultMonitor+ }++ // Never fall back to the default organization's runtime for another+ // organization: its requests would read or change the wrong tenant.+ if mtMonitor == nil {+ log.Warn().Str("org_id", orgID).Msg("Tenant monitor manager unavailable for alert request")+ return nil}- return defaultMonitor+ m, err := mtMonitor.GetMonitor(orgID)+ if err != nil || m == nil {+ log.Warn().Err(err).Str("org_id", orgID).Msg("Failed to resolve tenant monitor for alert request")+ return nil+ }+ return NewAlertMonitorWrapper(m)+}++// requireMonitor resolves the request organization's monitor, writing a 503+// and returning nil when it is unavailable.+func (h *AlertHandlers) requireMonitor(w http.ResponseWriter, r *http.Request) AlertMonitor {+ monitor := h.getMonitor(r.Context())+ if monitor == nil {+ http.Error(w, "Tenant monitor is not available", http.StatusServiceUnavailable)+ return nil+ }+ return monitor}// MonitorForContext resolves the tenant-scoped alert runtime used by handlers.@@ -206,7 +232,11 @@ func validateAlertIdentifier(alertIdentifier string) bool {// GetAlertConfig returns the current alert configurationfunc (h *AlertHandlers) GetAlertConfig(w http.ResponseWriter, r *http.Request) {- config := h.getMonitor(r.Context()).GetAlertManager().GetConfig()+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ config := monitor.GetAlertManager().GetConfig()if err := utils.WriteJSONResponse(w, config); err != nil {log.Error().Err(err).Msg("Failed to write alert config response")@@ -216,9 +246,8 @@ func (h *AlertHandlers) GetAlertConfig(w http.ResponseWriter, r *http.Request) {// GetDeadManStatus returns the live external-watchdog read model without// echoing the secret-bearing ping URL.func (h *AlertHandlers) GetDeadManStatus(w http.ResponseWriter, r *http.Request) {- monitor := h.getMonitor(r.Context())+ monitor := h.requireMonitor(w, r)if monitor == nil {- http.Error(w, "Alert monitor unavailable", http.StatusServiceUnavailable)return}if err := utils.WriteJSONResponse(w, monitor.DeadManStatus()); err != nil {@@ -236,9 +265,8 @@ type deadManConfigView struct {// GetDeadManConfig returns only a presence marker for the credential-bearing// ping URL. The sentinel supports an edit round-trip without disclosing it.func (h *AlertHandlers) GetDeadManConfig(w http.ResponseWriter, r *http.Request) {- monitor := h.getMonitor(r.Context())+ monitor := h.requireMonitor(w, r)if monitor == nil {- http.Error(w, "Alert monitor unavailable", http.StatusServiceUnavailable)return}configured := strings.TrimSpace(monitor.DeadManConfig().PingURL) != ""@@ -254,9 +282,8 @@ func (h *AlertHandlers) GetDeadManConfig(w http.ResponseWriter, r *http.Request)// UpdateDeadManConfig validates and durably encrypts the watchdog destination// before its live worker is reconfigured.func (h *AlertHandlers) UpdateDeadManConfig(w http.ResponseWriter, r *http.Request) {- monitor := h.getMonitor(r.Context())+ monitor := h.requireMonitor(w, r)if monitor == nil {- http.Error(w, "Alert monitor unavailable", http.StatusServiceUnavailable)return}r.Body = http.MaxBytesReader(w, r.Body, notifications.MaxDeadManPingURLLength+1024)@@ -306,7 +333,11 @@ func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request// Keys the client left out keep their stored values, so a thresholds// save no longer turns off flapping detection or alert TTL cleanup.- updatedConfig, err := h.getMonitor(r.Context()).GetAlertManager().ApplyConfigUpdate(body)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ updatedConfig, err := monitor.GetAlertManager().ApplyConfigUpdate(body)if errors.Is(err, alerts.ErrConfigSnapshot) {log.Error().Err(err).Msg("Failed to snapshot applied alert configuration")http.Error(w, err.Error(), http.StatusInternalServerError)@@ -318,7 +349,7 @@ func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request}// Update notification manager with schedule settings- notificationMgr := h.getMonitor(r.Context()).GetNotificationManager()+ notificationMgr := monitor.GetNotificationManager()notificationMgr.SetEnabled(updatedConfig.Enabled && updatedConfig.ActivationState == alerts.ActivationActive)notificationMgr.SetCooldown(updatedConfig.Schedule.Cooldown)notificationMgr.SetGroupingConfig(@@ -335,7 +366,7 @@ func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request// what was actually persisted. On the next restart or config reload,// the override silently vanished. Surface the error so the client can// show a real save-failed signal.- if err := h.getMonitor(r.Context()).GetConfigPersistence().SaveAlertConfig(updatedConfig); err != nil {+ if err := monitor.GetConfigPersistence().SaveAlertConfig(updatedConfig); err != nil {log.Error().Err(err).Msg("Failed to save alert configuration")http.Error(w, fmt.Sprintf("Failed to save alert configuration: %v", err), http.StatusInternalServerError)return@@ -352,7 +383,11 @@ func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request// ActivateAlerts activates alert notificationsfunc (h *AlertHandlers) ActivateAlerts(w http.ResponseWriter, r *http.Request) {// Get current config- config := h.getMonitor(r.Context()).GetAlertManager().GetConfig()+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ config := monitor.GetAlertManager().GetConfig()// Check if already activeif config.ActivationState == alerts.ActivationActive {@@ -373,25 +408,25 @@ func (h *AlertHandlers) ActivateAlerts(w http.ResponseWriter, r *http.Request) {config.ActivationTime = &now// Update config- h.getMonitor(r.Context()).GetAlertManager().UpdateConfig(config)- h.getMonitor(r.Context()).GetNotificationManager().SetEnabled(+ monitor.GetAlertManager().UpdateConfig(config)+ monitor.GetNotificationManager().SetEnabled(config.Enabled && config.ActivationState == alerts.ActivationActive,)// Save to persistent storage- if err := h.getMonitor(r.Context()).GetConfigPersistence().SaveAlertConfig(config); err != nil {+ if err := monitor.GetConfigPersistence().SaveAlertConfig(config); err != nil {log.Error().Err(err).Msg("Failed to save alert configuration after activation")http.Error(w, "Failed to save configuration", http.StatusInternalServerError)return}// Notify about existing critical alerts after activation- activeAlerts := h.getMonitor(r.Context()).GetAlertManager().GetActiveAlerts()+ activeAlerts := monitor.GetAlertManager().GetActiveAlerts()criticalCount := 0for _, alert := range activeAlerts {if alert.Level == alerts.AlertLevelCritical && !alert.Acknowledged {// Re-dispatch critical alerts to trigger notifications- h.getMonitor(r.Context()).GetAlertManager().NotifyExistingAlert(alert.ID)+ monitor.GetAlertManager().NotifyExistingAlert(alert.ID)criticalCount++}}@@ -416,7 +451,11 @@ func (h *AlertHandlers) ActivateAlerts(w http.ResponseWriter, r *http.Request) {// GetActiveAlerts returns all active alertsfunc (h *AlertHandlers) GetActiveAlerts(w http.ResponseWriter, r *http.Request) {- alerts := h.getMonitor(r.Context()).GetAlertManager().GetActiveAlerts()+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ alerts := monitor.GetAlertManager().GetActiveAlerts()if err := utils.WriteJSONResponse(w, alerts); err != nil {log.Error().Err(err).Msg("Failed to write active alerts response")@@ -426,7 +465,11 @@ func (h *AlertHandlers) GetActiveAlerts(w http.ResponseWriter, r *http.Request)func (h *AlertHandlers) GetAlertIntentPolicies(w http.ResponseWriter, r *http.Request) {h.intentPolicyMu.Lock()defer h.intentPolicyMu.Unlock()- manager, ok := h.getMonitor(r.Context()).GetAlertManager().(alertIntentPolicyManager)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ manager, ok := monitor.GetAlertManager().(alertIntentPolicyManager)if !ok {http.Error(w, "alert intent policies are unavailable", http.StatusServiceUnavailable)return@@ -456,7 +499,10 @@ func (h *AlertHandlers) UpdateAlertIntentPolicies(w http.ResponseWriter, r *http// back a later successful update.h.intentPolicyMu.Lock()defer h.intentPolicyMu.Unlock()- monitor := h.getMonitor(r.Context())+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }manager, ok := monitor.GetAlertManager().(alertIntentPolicyManager)if !ok {http.Error(w, "alert intent policies are unavailable", http.StatusServiceUnavailable)@@ -504,7 +550,11 @@ func (h *AlertHandlers) PreviewAlertIntentPolicy(w http.ResponseWriter, r *http.}h.intentPolicyMu.Lock()defer h.intentPolicyMu.Unlock()- manager, ok := h.getMonitor(r.Context()).GetAlertManager().(alertIntentPolicyManager)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ manager, ok := monitor.GetAlertManager().(alertIntentPolicyManager)if !ok {http.Error(w, "alert intent policy preview is unavailable", http.StatusServiceUnavailable)return@@ -539,9 +589,8 @@ func (h *AlertHandlers) GetAlertDeliveryDiagnosis(w http.ResponseWriter, r *httpreturn}- monitor := h.getMonitor(r.Context())+ monitor := h.requireMonitor(w, r)if monitor == nil {- http.Error(w, "monitor is not initialized", http.StatusServiceUnavailable)return}@@ -614,9 +663,8 @@ func (h *AlertHandlers) GetAlertEvents(w http.ResponseWriter, r *http.Request) {filter.Limit = limit}- monitor := h.getMonitor(r.Context())+ monitor := h.requireMonitor(w, r)if monitor == nil {- http.Error(w, "monitor is not initialized", http.StatusServiceUnavailable)return}manager := monitor.GetAlertManager()@@ -778,12 +826,12 @@ func (h *AlertHandlers) GetAlertHistory(w http.ResponseWriter, r *http.Request)return}- if h.getMonitor(r.Context()) == nil {- http.Error(w, "monitor is not initialized", http.StatusServiceUnavailable)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {return}- manager := h.getMonitor(r.Context()).GetAlertManager()+ manager := monitor.GetAlertManager()var history []alerts.Alertif startTime != nil {history = manager.GetAlertHistorySince(*startTime, fetchLimit)@@ -825,6 +873,10 @@ func (h *AlertHandlers) GetAlertIncidentTimeline(w http.ResponseWriter, r *http.}}+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }if alertID != "" {if !validateAlertIdentifier(alertID) {http.Error(w, "Invalid alert identifier", http.StatusBadRequest)@@ -846,7 +898,7 @@ func (h *AlertHandlers) GetAlertIncidentTimeline(w http.ResponseWriter, r *http.return}- store := h.getMonitor(r.Context()).GetIncidentStore()+ store := monitor.GetIncidentStore()if store == nil {http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)return@@ -863,7 +915,7 @@ func (h *AlertHandlers) GetAlertIncidentTimeline(w http.ResponseWriter, r *http.}if !startedAt.IsZero() && (incident == nil || len(incident.Events) == 0) {- if alert, resolvedAt := findAlertOccurrenceForTimeline(h.getMonitor(r.Context()).GetAlertManager(), alertID, startedAt); alert != nil {+ if alert, resolvedAt := findAlertOccurrenceForTimeline(monitor.GetAlertManager(), alertID, startedAt); alert != nil {incident = store.EnsureAlertOccurrence(alert, resolvedAt)}}@@ -885,7 +937,7 @@ func (h *AlertHandlers) GetAlertIncidentTimeline(w http.ResponseWriter, r *http.}return}- store := h.getMonitor(r.Context()).GetIncidentStore()+ store := monitor.GetIncidentStore()if store == nil {http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)return@@ -994,7 +1046,11 @@ func (h *AlertHandlers) SaveAlertIncidentNote(w http.ResponseWriter, r *http.Reqreturn}- store := h.getMonitor(r.Context()).GetIncidentStore()+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ store := monitor.GetIncidentStore()if store == nil {http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)return@@ -1118,7 +1174,11 @@ func exportIncidents(incidents []*memory.Incident) []*incidentView {// ClearAlertHistory clears all alert historyfunc (h *AlertHandlers) ClearAlertHistory(w http.ResponseWriter, r *http.Request) {- if err := h.getMonitor(r.Context()).GetAlertManager().ClearAlertHistory(); err != nil {+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ if err := monitor.GetAlertManager().ClearAlertHistory(); err != nil {http.Error(w, err.Error(), http.StatusInternalServerError)return}@@ -1187,13 +1247,17 @@ func (h *AlertHandlers) AcknowledgeAlertByBody(w http.ResponseWriter, r *http.Reuser = "unknown" // Fallback if auth header not set (shouldn't happen in normal flow)}- if err := h.getMonitor(r.Context()).GetAlertManager().AcknowledgeAlert(alertID, user); err != nil {+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ if err := monitor.GetAlertManager().AcknowledgeAlert(alertID, user); err != nil {log.Error().Err(err).Str("alertID", alertID).Msg("Failed to acknowledge alert")http.Error(w, err.Error(), http.StatusNotFound)return}- h.getMonitor(r.Context()).SyncAlertState()+ monitor.SyncAlertState()log.Info().Str("alertID", alertID).Str("user", user).Msg("Alert acknowledged successfully")@@ -1231,13 +1295,17 @@ func (h *AlertHandlers) UnacknowledgeAlertByBody(w http.ResponseWriter, r *http.return}- if err := h.getMonitor(r.Context()).GetAlertManager().UnacknowledgeAlert(alertID); err != nil {+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ if err := monitor.GetAlertManager().UnacknowledgeAlert(alertID); err != nil {log.Error().Err(err).Str("alertID", alertID).Msg("Failed to unacknowledge alert")http.Error(w, err.Error(), http.StatusNotFound)return}- h.getMonitor(r.Context()).SyncAlertState()+ monitor.SyncAlertState()log.Info().Str("alertID", alertID).Msg("Alert unacknowledged successfully")@@ -1273,7 +1341,11 @@ func (h *AlertHandlers) SnoozeAlertByBody(w http.ResponseWriter, r *http.Requestif user == "" {user = "unknown"}- if err := h.getMonitor(r.Context()).GetAlertManager().SnoozeAlert(alertID, user, req.Until); err != nil {+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ if err := monitor.GetAlertManager().SnoozeAlert(alertID, user, req.Until); err != nil {status := http.StatusBadRequestif errors.Is(err, alerts.ErrAlertNotFound) {status = http.StatusNotFound@@ -1281,7 +1353,7 @@ func (h *AlertHandlers) SnoozeAlertByBody(w http.ResponseWriter, r *http.Requesthttp.Error(w, err.Error(), status)return}- h.getMonitor(r.Context()).SyncAlertState()+ monitor.SyncAlertState()if err := utils.WriteJSONResponse(w, map[string]any{"success": true, "snoozedUntil": req.Until.UTC()}); err != nil {log.Error().Err(err).Str("alertID", alertID).Msg("Failed to write snooze response")}@@ -1307,7 +1379,11 @@ func (h *AlertHandlers) UnsnoozeAlertByBody(w http.ResponseWriter, r *http.Requeif user == "" {user = "unknown"}- if err := h.getMonitor(r.Context()).GetAlertManager().UnsnoozeAlert(alertID, user); err != nil {+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ if err := monitor.GetAlertManager().UnsnoozeAlert(alertID, user); err != nil {status := http.StatusBadRequestif errors.Is(err, alerts.ErrAlertNotFound) {status = http.StatusNotFound@@ -1315,7 +1391,7 @@ func (h *AlertHandlers) UnsnoozeAlertByBody(w http.ResponseWriter, r *http.Requehttp.Error(w, err.Error(), status)return}- h.getMonitor(r.Context()).SyncAlertState()+ monitor.SyncAlertState()if err := utils.WriteJSONResponse(w, map[string]bool{"success": true}); err != nil {log.Error().Err(err).Str("alertID", alertID).Msg("Failed to write unsnooze response")}@@ -1347,12 +1423,16 @@ func (h *AlertHandlers) ClearAlertByBody(w http.ResponseWriter, r *http.Request)return}- if !h.getMonitor(r.Context()).GetAlertManager().ClearAlert(alertID) {+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ if !monitor.GetAlertManager().ClearAlert(alertID) {http.Error(w, "Alert not found", http.StatusNotFound)return}- h.getMonitor(r.Context()).SyncAlertState()+ monitor.SyncAlertState()log.Info().Str("alertID", alertID).Msg("Alert cleared successfully")@@ -1398,12 +1478,16 @@ func (h *AlertHandlers) BulkAcknowledgeAlerts(w http.ResponseWriter, r *http.Reqresults []map[string]interface{}anySuccess bool)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }for _, alertID := range alertIdentifiers {result := map[string]interface{}{"alertIdentifier": alertID,"success": true,}- if err := h.getMonitor(r.Context()).GetAlertManager().AcknowledgeAlert(alertID, user); err != nil {+ if err := monitor.GetAlertManager().AcknowledgeAlert(alertID, user); err != nil {result["success"] = falseresult["error"] = err.Error()} else {@@ -1413,7 +1497,7 @@ func (h *AlertHandlers) BulkAcknowledgeAlerts(w http.ResponseWriter, r *http.Req}if anySuccess {- h.getMonitor(r.Context()).SyncAlertState()+ monitor.SyncAlertState()}// Send response immediately@@ -1456,12 +1540,16 @@ func (h *AlertHandlers) BulkClearAlerts(w http.ResponseWriter, r *http.Request)results []map[string]interface{}anySuccess bool)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }for _, alertID := range alertIdentifiers {result := map[string]interface{}{"alertIdentifier": alertID,"success": true,}- if h.getMonitor(r.Context()).GetAlertManager().ClearAlert(alertID) {+ if monitor.GetAlertManager().ClearAlert(alertID) {anySuccess = true} else {result["success"] = false@@ -1471,7 +1559,7 @@ func (h *AlertHandlers) BulkClearAlerts(w http.ResponseWriter, r *http.Request)}if anySuccess {- h.getMonitor(r.Context()).SyncAlertState()+ monitor.SyncAlertState()}// Send response immediatelydiff --git a/internal/api/alerting/notifications.go b/internal/api/alerting/notifications.goindex 2c30e52232..e5756bb858 100644--- a/internal/api/alerting/notifications.go+++ b/internal/api/alerting/notifications.go@@ -127,13 +127,39 @@ func (h *NotificationHandlers) getMonitor(ctx context.Context) NotificationMonitdefaultMonitor := h.defaultMonitorh.stateMu.RUnlock()- orgID := apicontext.OrgID(ctx)- if mtMonitor != nil {- if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {- return NewNotificationMonitorWrapper(m)+ orgID := strings.TrimSpace(apicontext.OrgID(ctx))+ if orgID == "" || orgID == "default" {+ if mtMonitor != nil {+ if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {+ return NewNotificationMonitorWrapper(m)+ }}+ return defaultMonitor+ }++ // Never fall back to the default organization's runtime for another+ // organization: its requests would read or change the wrong tenant.+ if mtMonitor == nil {+ log.Warn().Str("org_id", orgID).Msg("Tenant monitor manager unavailable for notification request")+ return nil}- return defaultMonitor+ m, err := mtMonitor.GetMonitor(orgID)+ if err != nil || m == nil {+ log.Warn().Err(err).Str("org_id", orgID).Msg("Failed to resolve tenant monitor for notification request")+ return nil+ }+ return NewNotificationMonitorWrapper(m)+}++// requireMonitor resolves the request organization's monitor, writing a 503+// and returning nil when it is unavailable.+func (h *NotificationHandlers) requireMonitor(w http.ResponseWriter, r *http.Request) NotificationMonitor {+ monitor := h.getMonitor(r.Context())+ if monitor == nil {+ http.Error(w, "Tenant monitor is not available", http.StatusServiceUnavailable)+ return nil+ }+ return monitor}// MonitorForContext resolves the tenant-scoped notification runtime used by@@ -169,7 +195,11 @@ func (h *NotificationHandlers) getReadState(ctx context.Context) unifiedresource// GetEmailConfig returns the current email configurationfunc (h *NotificationHandlers) GetEmailConfig(w http.ResponseWriter, r *http.Request) {- config := h.getMonitor(r.Context()).GetNotificationManager().GetEmailConfig()+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ config := monitor.GetNotificationManager().GetEmailConfig()// For security, don't return the passwordconfig.Password = ""@@ -215,7 +245,10 @@ func (h *NotificationHandlers) UpdateEmailConfig(w http.ResponseWriter, r *http.return}- monitor := h.getMonitor(r.Context())+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }manager := monitor.GetNotificationManager()existingConfig := manager.GetEmailConfig()@@ -277,7 +310,11 @@ func redactAppriseConfig(config notifications.AppriseConfig) appriseConfigRespon// GetAppriseConfig returns the current Apprise configuration.func (h *NotificationHandlers) GetAppriseConfig(w http.ResponseWriter, r *http.Request) {- config := h.getMonitor(r.Context()).GetNotificationManager().GetAppriseConfig()+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ config := monitor.GetNotificationManager().GetAppriseConfig()w.Header().Set("Content-Type", "application/json")if err := json.NewEncoder(w).Encode(redactAppriseConfig(config)); err != nil {@@ -308,7 +345,10 @@ func (h *NotificationHandlers) UpdateAppriseConfig(w http.ResponseWriter, r *httMinimumSeverity *string `json:"minimumSeverity"`}_ = json.Unmarshal(body, &presenceCheck)- monitor := h.getMonitor(r.Context())+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }manager := monitor.GetNotificationManager()existingConfig := manager.GetAppriseConfig()@@ -350,7 +390,11 @@ func (h *NotificationHandlers) UpdateAppriseConfig(w http.ResponseWriter, r *htt// GetWebhooks returns all webhook configurations with secrets maskedfunc (h *NotificationHandlers) GetWebhooks(w http.ResponseWriter, r *http.Request) {- webhooks := h.getMonitor(r.Context()).GetNotificationManager().GetWebhooks()+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ webhooks := monitor.GetNotificationManager().GetWebhooks()maskedWebhooks := make([]map[string]interface{}, len(webhooks))for i, webhook := range webhooks {@@ -470,7 +514,10 @@ func (h *NotificationHandlers) CreateWebhook(w http.ResponseWriter, r *http.Requreturn}- monitor := h.getMonitor(r.Context())+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }manager := monitor.GetNotificationManager()// Validate webhook URL@@ -585,7 +632,10 @@ func (h *NotificationHandlers) UpdateWebhook(w http.ResponseWriter, r *http.Requ}_ = json.Unmarshal(bodyBytes, &routingPresence)- monitor := h.getMonitor(r.Context())+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }manager := monitor.GetNotificationManager()// Restore each masked value returned by GetWebhooks, not its entire map.@@ -667,7 +717,10 @@ func (h *NotificationHandlers) DeleteWebhook(w http.ResponseWriter, r *http.Requreturn}- monitor := h.getMonitor(r.Context())+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }manager := monitor.GetNotificationManager()existingWebhooks := manager.GetWebhooks()candidateWebhooks := make([]notifications.WebhookConfig, 0, len(existingWebhooks))@@ -794,6 +847,11 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.R}}+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+// Handle webhook testingif req.Method == "webhook" && req.WebhookID != "" {log.Info().@@ -801,7 +859,7 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.RMsg("Testing specific webhook")// Get the webhook by ID and test it- webhooks := h.getMonitor(r.Context()).GetNotificationManager().GetWebhooks()+ webhooks := monitor.GetNotificationManager().GetWebhooks()var foundWebhook *notifications.WebhookConfigfor _, wh := range webhooks {if wh.ID == req.WebhookID {@@ -816,7 +874,7 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.R}// Send test webhook- if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestWebhook(*foundWebhook); err != nil {+ if err := monitor.GetNotificationManager().SendTestWebhook(*foundWebhook); err != nil {writeTestNotificationError(w, err, http.StatusBadRequest)return}@@ -829,7 +887,7 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.R// If password is empty, use the saved passwordif emailConfig.Password == "" {- savedConfig := h.getMonitor(r.Context()).GetNotificationManager().GetEmailConfig()+ savedConfig := monitor.GetNotificationManager().GetEmailConfig()emailConfig.Password = savedConfig.Password}@@ -842,7 +900,7 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.RBool("hasPassword", emailConfig.Password != "").Msg("Testing email with provided config")- if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestNotificationWithConfig(req.Method, &emailConfig, nodeInfo); err != nil {+ if err := monitor.GetNotificationManager().SendTestNotificationWithConfig(req.Method, &emailConfig, nodeInfo); err != nil {writeTestNotificationError(w, err, http.StatusBadRequest)return}@@ -855,17 +913,17 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.R// If the API key is empty, use the saved keyif appriseConfig.APIKey == "" {- savedConfig := h.getMonitor(r.Context()).GetNotificationManager().GetAppriseConfig()+ savedConfig := monitor.GetNotificationManager().GetAppriseConfig()appriseConfig.APIKey = savedConfig.APIKey}- if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestAppriseWithConfig(appriseConfig); err != nil {+ if err := monitor.GetNotificationManager().SendTestAppriseWithConfig(appriseConfig); err != nil {writeTestNotificationError(w, err, http.StatusBadRequest)return}} else {// Use saved config- if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestNotification(req.Method); err != nil {+ if err := monitor.GetNotificationManager().SendTestNotification(req.Method); err != nil {writeTestNotificationError(w, err, http.StatusBadRequest)return}@@ -878,7 +936,7 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.R"status": "success","message": "Test notification sent",}- if !h.getMonitor(r.Context()).GetNotificationManager().IsEnabled() {+ if !monitor.GetNotificationManager().IsEnabled() {response["deliveryPaused"] = trueresponse["message"] = "Test notification sent, but alert delivery is paused: real alerts are not being sent"}@@ -896,7 +954,11 @@ func (h *NotificationHandlers) GetWebhookTemplates(w http.ResponseWriter, r *htt// GetWebhookHistory returns recent webhook delivery history with URLs redactedfunc (h *NotificationHandlers) GetWebhookHistory(w http.ResponseWriter, r *http.Request) {- history := h.getMonitor(r.Context()).GetNotificationManager().GetWebhookHistory()+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ history := monitor.GetNotificationManager().GetWebhookHistory()// Redact secrets from URLs in historyfor i := range history {@@ -948,9 +1010,13 @@ func (h *NotificationHandlers) TestWebhook(w http.ResponseWriter, r *http.Reques// secrets are the masked values from GetWebhooks. Test with the saved// values, as the saved webhook would send; this route requires the same// settings:write scope as UpdateWebhook, which keeps them the same way.+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }saved := notifications.WebhookConfig{}if basicWebhook.ID != "" {- for _, candidate := range h.getMonitor(r.Context()).GetNotificationManager().GetWebhooks() {+ for _, candidate := range monitor.GetNotificationManager().GetWebhooks() {if candidate.ID == basicWebhook.ID {saved = candidatebreak@@ -980,7 +1046,7 @@ func (h *NotificationHandlers) TestWebhook(w http.ResponseWriter, r *http.RequesMsg("Testing webhook")// Test the webhook- status, response, err := h.getMonitor(r.Context()).GetNotificationManager().TestEnhancedWebhook(webhook)+ status, response, err := monitor.GetNotificationManager().TestEnhancedWebhook(webhook)result := map[string]interface{}{"status": status,@@ -1003,7 +1069,7 @@ func (h *NotificationHandlers) TestWebhook(w http.ResponseWriter, r *http.Requesresult["success"] = true// Same honesty as TestNotification: a webhook test bypasses the// activation gate, so its success must not imply live alerts flow.- if !h.getMonitor(r.Context()).GetNotificationManager().IsEnabled() {+ if !monitor.GetNotificationManager().IsEnabled() {result["deliveryPaused"] = true}}@@ -1014,7 +1080,10 @@ func (h *NotificationHandlers) TestWebhook(w http.ResponseWriter, r *http.Reques// GetNotificationHealth returns health status of notification systemfunc (h *NotificationHandlers) GetNotificationHealth(w http.ResponseWriter, r *http.Request) {- monitor := h.getMonitor(r.Context())+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }manager := monitor.GetNotificationManager()failureClasses := notifications.NotificationFailureClassCounts{}.AsMap()@@ -1103,7 +1172,11 @@ func (h *NotificationHandlers) GetNotificationHealth(w http.ResponseWriter, r *h// queue's retention windows, so the payload names the window rather than// presenting itself as lifetime history.func (h *NotificationHandlers) GetDeliveryLog(w http.ResponseWriter, r *http.Request) {- manager := h.getMonitor(r.Context()).GetNotificationManager()+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ manager := monitor.GetNotificationManager()limit := 0if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" {diff --git a/internal/api/alerting/tenant_monitor_fail_closed_test.go b/internal/api/alerting/tenant_monitor_fail_closed_test.gonew file mode 100644index 0000000000..2112c87634--- /dev/null+++ b/internal/api/alerting/tenant_monitor_fail_closed_test.go@@ -0,0 +1,101 @@+package alerting++import (+ "bytes"+ "context"+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext"+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"+ "github.com/stretchr/testify/assert"+)++// An empty tenant manager has no persistence, so every organization lookup+// it is asked for fails. The default-org mocks carry no expectations: any+// call that falls back to them fails the test.++func tenantRequest(method, target, orgID string) *http.Request {+ req := httptest.NewRequest(method, target, bytes.NewReader([]byte(`{}`)))+ return req.WithContext(context.WithValue(req.Context(), apicontext.OrgIDContextKey, orgID))+}++func TestAlertHandlersGetMonitor_NonDefaultOrgFailsClosed(t *testing.T) {+ defaultMonitor := new(MockAlertMonitor)+ tenantCtx := context.WithValue(context.Background(), apicontext.OrgIDContextKey, "tenant-a")++ h := NewAlertHandlers(&monitoring.MultiTenantMonitor{}, defaultMonitor, nil)+ assert.Nil(t, h.getMonitor(tenantCtx))+ assert.Nil(t, h.MonitorForContext(tenantCtx))+ assert.Same(t, defaultMonitor, h.getMonitor(context.Background()))++ noManager := NewAlertHandlers(nil, defaultMonitor, nil)+ assert.Nil(t, noManager.getMonitor(tenantCtx))+ assert.Same(t, defaultMonitor, noManager.getMonitor(context.Background()))+}++func TestAlertHandlers_UnresolvableTenantReturns503(t *testing.T) {+ defaultMonitor := new(MockAlertMonitor)+ h := NewAlertHandlers(&monitoring.MultiTenantMonitor{}, defaultMonitor, nil)++ cases := []struct {+ name string+ method string+ target string+ handler http.HandlerFunc+ }{+ {"get config", http.MethodGet, "/api/alerts/config", h.GetAlertConfig},+ {"update config", http.MethodPut, "/api/alerts/config", h.UpdateAlertConfig},+ {"active alerts", http.MethodGet, "/api/alerts/active", h.GetActiveAlerts},+ {"clear history", http.MethodDelete, "/api/alerts/history", h.ClearAlertHistory},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ rec := httptest.NewRecorder()+ tc.handler(rec, tenantRequest(tc.method, tc.target, "tenant-a"))+ assert.Equal(t, http.StatusServiceUnavailable, rec.Code, rec.Body.String())+ })+ }+ defaultMonitor.AssertExpectations(t)+}++func TestNotificationHandlersGetMonitor_NonDefaultOrgFailsClosed(t *testing.T) {+ defaultMonitor := new(MockNotificationMonitor)+ tenantCtx := context.WithValue(context.Background(), apicontext.OrgIDContextKey, "tenant-a")++ h := NewNotificationHandlers(&monitoring.MultiTenantMonitor{}, defaultMonitor)+ assert.Nil(t, h.getMonitor(tenantCtx))+ assert.Nil(t, h.MonitorForContext(tenantCtx))+ assert.Same(t, defaultMonitor, h.getMonitor(context.Background()))++ noManager := NewNotificationHandlers(nil, defaultMonitor)+ assert.Nil(t, noManager.getMonitor(tenantCtx))+ assert.Same(t, defaultMonitor, noManager.getMonitor(context.Background()))+}++func TestNotificationHandlers_UnresolvableTenantReturns503(t *testing.T) {+ defaultMonitor := new(MockNotificationMonitor)+ h := NewNotificationHandlers(&monitoring.MultiTenantMonitor{}, defaultMonitor)++ cases := []struct {+ name string+ method string+ target string+ handler http.HandlerFunc+ }{+ {"get email", http.MethodGet, "/api/notifications/email", h.GetEmailConfig},+ {"update email", http.MethodPut, "/api/notifications/email", h.UpdateEmailConfig},+ {"get webhooks", http.MethodGet, "/api/notifications/webhooks", h.GetWebhooks},+ {"delete webhook", http.MethodDelete, "/api/notifications/webhooks/wh-1", h.DeleteWebhook},+ {"health", http.MethodGet, "/api/notifications/health", h.GetNotificationHealth},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ rec := httptest.NewRecorder()+ tc.handler(rec, tenantRequest(tc.method, tc.target, "tenant-a"))+ assert.Equal(t, http.StatusServiceUnavailable, rec.Code, rec.Body.String())+ })+ }+ defaultMonitor.AssertExpectations(t)+}diff --git a/internal/api/docker_agents.go b/internal/api/docker_agents.goindex 114cba6e37..29c8224f7d 100644--- a/internal/api/docker_agents.go+++ b/internal/api/docker_agents.go@@ -1,7 +1,6 @@package apiimport (- "context""encoding/json""errors""fmt"@@ -131,7 +130,11 @@ func (h *DockerAgentHandlers) HandleReport(w http.ResponseWriter, r *http.Reques}tokenRecord := getAPITokenRecordFromRequest(r)- host, err := h.getMonitor(r.Context()).ApplyDockerReport(report, tokenRecord)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ host, err := monitor.ApplyDockerReport(report, tokenRecord)if err != nil {writeErrorResponse(w, http.StatusBadRequest, "invalid_report", err.Error(), nil)return@@ -152,7 +155,7 @@ func (h *DockerAgentHandlers) HandleReport(w http.ResponseWriter, r *http.Reques"lastSeen": host.LastSeen,}- if payload, cmd := h.getMonitor(r.Context()).FetchDockerCommandForHost(host.ID); cmd != nil {+ if payload, cmd := monitor.FetchDockerCommandForHost(host.ID); cmd != nil {commandResponse := map[string]any{"id": cmd.ID,"type": cmd.Type,@@ -271,14 +274,17 @@ func (h *DockerAgentHandlers) HandleCommandAck(w http.ResponseWriter, r *http.Rereturn}- mon := h.getMonitor(r.Context())+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }agentID := strings.TrimSpace(req.AgentID)if agentID == "" {writeErrorResponse(w, http.StatusBadRequest, "missing_agent_id", "Agent ID is required", nil)return}- commandStatus, agentID, shouldRemove, err := mon.AcknowledgeDockerHostCommand(commandID, agentID, status, req.Message)+ commandStatus, agentID, shouldRemove, err := monitor.AcknowledgeDockerHostCommand(commandID, agentID, status, req.Message)if err != nil {writeErrorResponse(w, http.StatusBadRequest, "docker_command_ack_failed", err.Error(), nil)return@@ -292,7 +298,7 @@ func (h *DockerAgentHandlers) HandleCommandAck(w http.ResponseWriter, r *http.ReoldContainerID = strings.TrimSpace(oldContainerID)newContainerID = strings.TrimSpace(newContainerID)if oldContainerID != "" && newContainerID != "" && oldContainerID != newContainerID {- if err := mon.CopyDockerContainerMetadata(agentID, oldContainerID, newContainerID); err != nil {+ if err := monitor.CopyDockerContainerMetadata(agentID, oldContainerID, newContainerID); err != nil {log.Warn().Err(err).Str("dockerAgentID", agentID).@@ -304,12 +310,12 @@ func (h *DockerAgentHandlers) HandleCommandAck(w http.ResponseWriter, r *http.Re}if shouldRemove {- if _, removeErr := mon.RemoveDockerHost(agentID); removeErr != nil {+ if _, removeErr := monitor.RemoveDockerHost(agentID); removeErr != nil {log.Error().Err(removeErr).Str("dockerAgentID", agentID).Str("commandID", commandID).Msg("Failed to remove docker host after command completion")} else {// Clear the removal block since the agent has confirmed it stopped successfully.// This allows immediate re-enrollment without waiting for the 24-hour TTL.- if reenrollErr := mon.AllowDockerHostReenroll(agentID); reenrollErr != nil {+ if reenrollErr := monitor.AllowDockerHostReenroll(agentID); reenrollErr != nil {log.Warn().Err(reenrollErr).Str("dockerAgentID", agentID).Msg("Failed to clear removal block after successful stop")}}@@ -346,14 +352,18 @@ func (h *DockerAgentHandlers) HandleDeleteHost(w http.ResponseWriter, r *http.ReforceParam := strings.ToLower(r.URL.Query().Get("force"))force := forceParam == "true" || strings.ToLower(r.URL.Query().Get("mode")) == "force"- priorHost, hostExists := h.getMonitor(r.Context()).GetDockerHost(agentID)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ priorHost, hostExists := monitor.GetDockerHost(agentID)if shouldHide {if !hostExists {writeErrorResponse(w, http.StatusNotFound, "docker_agent_not_found", "Docker / Podman module not found", nil)return}- host, err := h.getMonitor(r.Context()).HideDockerHost(agentID)+ host, err := monitor.HideDockerHost(agentID)if err != nil {writeErrorResponse(w, http.StatusNotFound, "docker_agent_not_found", err.Error(), nil)return@@ -392,7 +402,7 @@ func (h *DockerAgentHandlers) HandleDeleteHost(w http.ResponseWriter, r *http.Rereturn}- host, err := h.getMonitor(r.Context()).RemoveDockerHost(agentID)+ host, err := monitor.RemoveDockerHost(agentID)if err != nil {writeErrorResponse(w, http.StatusNotFound, "docker_agent_not_found", err.Error(), nil)return@@ -422,7 +432,11 @@ func (h *DockerAgentHandlers) HandleAllowReenroll(w http.ResponseWriter, r *httpreturn}- if err := h.getMonitor(r.Context()).AllowDockerHostReenroll(agentID); err != nil {+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ if err := monitor.AllowDockerHostReenroll(agentID); err != nil {writeErrorResponse(w, http.StatusBadRequest, "docker_agent_reenroll_failed", err.Error(), nil)return}@@ -447,7 +461,7 @@ func (h *DockerAgentHandlers) handleHostLifecycleAction(pathSuffix string,successMsg string,logMsg string,- action func(ctx context.Context, agentID string) (string, error),+ action func(monitor *monitoring.Monitor, agentID string) (string, error),) {if r.Method != http.MethodPut {writeErrorResponse(w, http.StatusMethodNotAllowed, "method_not_allowed", "Only PUT is allowed", nil)@@ -460,7 +474,11 @@ func (h *DockerAgentHandlers) handleHostLifecycleAction(return}- hostID, err := action(r.Context(), agentID)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ hostID, err := action(monitor, agentID)if err != nil {writeErrorResponse(w, http.StatusNotFound, "docker_agent_not_found", err.Error(), nil)return@@ -480,8 +498,8 @@ func (h *DockerAgentHandlers) handleHostLifecycleAction(// HandleUnhideHost unhides a previously hidden Docker / Podman host.func (h *DockerAgentHandlers) HandleUnhideHost(w http.ResponseWriter, r *http.Request) {h.handleHostLifecycleAction(w, r, "/unhide", "Docker / Podman module unhidden", "Failed to serialize docker host unhide response",- func(ctx context.Context, agentID string) (string, error) {- host, err := h.getMonitor(ctx).UnhideDockerHost(agentID)+ func(monitor *monitoring.Monitor, agentID string) (string, error) {+ host, err := monitor.UnhideDockerHost(agentID)if err != nil {return "", err}@@ -492,8 +510,8 @@ func (h *DockerAgentHandlers) HandleUnhideHost(w http.ResponseWriter, r *http.Re// HandleMarkPendingUninstall marks a Docker / Podman host as pending uninstall.func (h *DockerAgentHandlers) HandleMarkPendingUninstall(w http.ResponseWriter, r *http.Request) {h.handleHostLifecycleAction(w, r, "/pending-uninstall", "Docker / Podman module marked as pending uninstall", "Failed to serialize docker host pending uninstall response",- func(ctx context.Context, agentID string) (string, error) {- host, err := h.getMonitor(ctx).MarkDockerHostPendingUninstall(agentID)+ func(monitor *monitoring.Monitor, agentID string) (string, error) {+ host, err := monitor.MarkDockerHostPendingUninstall(agentID)if err != nil {return "", err}@@ -528,7 +546,11 @@ func (h *DockerAgentHandlers) HandleSetCustomDisplayName(w http.ResponseWriter,customName := strings.TrimSpace(req.DisplayName)- host, err := h.getMonitor(r.Context()).SetDockerHostCustomDisplayName(agentID, customName)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ host, err := monitor.SetDockerHostCustomDisplayName(agentID, customName)if err != nil {writeErrorResponse(w, http.StatusNotFound, "docker_agent_not_found", err.Error(), nil)return@@ -612,7 +634,11 @@ func (h *DockerAgentHandlers) HandleCheckUpdates(w http.ResponseWriter, r *http.}// Queue the check updates command- commandStatus, err := h.getMonitor(r.Context()).QueueDockerCheckUpdatesCommand(agentID)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ commandStatus, err := monitor.QueueDockerCheckUpdatesCommand(agentID)if err != nil {writeErrorResponse(w, http.StatusBadRequest, "check_updates_command_failed", err.Error(), nil)returndiff --git a/internal/api/kubernetes_agents.go b/internal/api/kubernetes_agents.goindex f439fe7e61..ee518fbafb 100644--- a/internal/api/kubernetes_agents.go+++ b/internal/api/kubernetes_agents.go@@ -52,7 +52,11 @@ func (h *KubernetesAgentHandlers) HandleReport(w http.ResponseWriter, r *http.Re}tokenRecord := getAPITokenRecordFromRequest(r)- cluster, err := h.getMonitor(r.Context()).ApplyKubernetesReport(report, tokenRecord)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ cluster, err := monitor.ApplyKubernetesReport(report, tokenRecord)if err != nil {writeErrorResponse(w, http.StatusBadRequest, "invalid_report", err.Error(), nil)return@@ -138,7 +142,11 @@ func (h *KubernetesAgentHandlers) HandleDeleteCluster(w http.ResponseWriter, r *return}- cluster, err := h.getMonitor(r.Context()).RemoveKubernetesCluster(clusterID)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ cluster, err := monitor.RemoveKubernetesCluster(clusterID)if err != nil {writeErrorResponse(w, http.StatusNotFound, "k8s_cluster_not_found", err.Error(), nil)return@@ -170,7 +178,11 @@ func (h *KubernetesAgentHandlers) HandleAllowReenroll(w http.ResponseWriter, r *return}- if err := h.getMonitor(r.Context()).AllowKubernetesClusterReenroll(clusterID); err != nil {+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ if err := monitor.AllowKubernetesClusterReenroll(clusterID); err != nil {writeErrorResponse(w, http.StatusBadRequest, "k8s_cluster_reenroll_failed", err.Error(), nil)return}@@ -192,7 +204,7 @@ func (h *KubernetesAgentHandlers) handleClusterLifecycleAction(pathSuffix string,successMsg string,logMsg string,- action func(ctx context.Context, clusterID string) (string, error),+ action func(monitor *monitoring.Monitor, clusterID string) (string, error),) {if r.Method != http.MethodPut {writeErrorResponse(w, http.StatusMethodNotAllowed, "method_not_allowed", "Only PUT is allowed", nil)@@ -207,7 +219,11 @@ func (h *KubernetesAgentHandlers) handleClusterLifecycleAction(return}- canonicalID, err := action(r.Context(), clusterID)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ canonicalID, err := action(monitor, clusterID)if err != nil {writeErrorResponse(w, http.StatusNotFound, "k8s_cluster_not_found", err.Error(), nil)return@@ -227,8 +243,8 @@ func (h *KubernetesAgentHandlers) handleClusterLifecycleAction(// HandleUnhideCluster unhides a previously hidden kubernetes cluster.func (h *KubernetesAgentHandlers) HandleUnhideCluster(w http.ResponseWriter, r *http.Request) {h.handleClusterLifecycleAction(w, r, "/unhide", "Kubernetes cluster unhidden", "Failed to serialize kubernetes cluster unhide response",- func(ctx context.Context, clusterID string) (string, error) {- cluster, err := h.getMonitor(ctx).UnhideKubernetesCluster(clusterID)+ func(monitor *monitoring.Monitor, clusterID string) (string, error) {+ cluster, err := monitor.UnhideKubernetesCluster(clusterID)if err != nil {return "", err}@@ -239,8 +255,8 @@ func (h *KubernetesAgentHandlers) HandleUnhideCluster(w http.ResponseWriter, r *// HandleMarkPendingUninstall marks a cluster as pending uninstall.func (h *KubernetesAgentHandlers) HandleMarkPendingUninstall(w http.ResponseWriter, r *http.Request) {h.handleClusterLifecycleAction(w, r, "/pending-uninstall", "Kubernetes cluster marked as pending uninstall", "Failed to serialize kubernetes cluster pending uninstall response",- func(ctx context.Context, clusterID string) (string, error) {- cluster, err := h.getMonitor(ctx).MarkKubernetesClusterPendingUninstall(clusterID)+ func(monitor *monitoring.Monitor, clusterID string) (string, error) {+ cluster, err := monitor.MarkKubernetesClusterPendingUninstall(clusterID)if err != nil {return "", err}@@ -277,7 +293,11 @@ func (h *KubernetesAgentHandlers) HandleSetCustomDisplayName(w http.ResponseWritcustomName := strings.TrimSpace(req.DisplayName)- cluster, err := h.getMonitor(r.Context()).SetKubernetesClusterCustomDisplayName(clusterID, customName)+ monitor := h.requireMonitor(w, r)+ if monitor == nil {+ return+ }+ cluster, err := monitor.SetKubernetesClusterCustomDisplayName(clusterID, customName)if err != nil {writeErrorResponse(w, http.StatusNotFound, "k8s_cluster_not_found", err.Error(), nil)returndiff --git a/internal/api/system_settings.go b/internal/api/system_settings.goindex c5951c71c1..ebff08f982 100644--- a/internal/api/system_settings.go+++ b/internal/api/system_settings.go@@ -206,13 +206,39 @@ func (h *SystemSettingsHandler) getMonitor(ctx context.Context) SystemSettingsModefaultMonitor := h.defaultMonitorh.stateMu.RUnlock()- if mtMonitor != nil {- orgID := GetOrgID(ctx)- if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {- return m+ orgID := strings.TrimSpace(GetOrgID(ctx))+ if orgID == "" || orgID == "default" {+ if mtMonitor != nil {+ if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {+ return m+ }}+ return defaultMonitor+ }++ // Never fall back to the default organization's runtime for another+ // organization: its requests would read or change the wrong tenant.+ if mtMonitor == nil {+ log.Warn().Str("org_id", orgID).Msg("Tenant monitor manager unavailable for system settings request")+ return nil+ }+ m, err := mtMonitor.GetMonitor(orgID)+ if err != nil || m == nil {+ log.Warn().Err(err).Str("org_id", orgID).Msg("Failed to resolve tenant monitor for system settings request")+ return nil+ }+ return m+}++// requireMonitor resolves the request organization's monitor, writing a 503+// and returning nil when it is unavailable.+func (h *SystemSettingsHandler) requireMonitor(w http.ResponseWriter, r *http.Request) SystemSettingsMonitor {+ monitor := h.getMonitor(r.Context())+ if monitor == nil {+ writeErrorResponse(w, http.StatusServiceUnavailable, "tenant_unavailable", "Tenant monitor is not available", nil)+ return nil}- return defaultMonitor+ return monitor}// SetConfig updates the configuration reference used by the handler.diff --git a/internal/api/system_settings_handlers_test.go b/internal/api/system_settings_handlers_test.goindex f9acf98b2a..cf2ec7c6ed 100644--- a/internal/api/system_settings_handlers_test.go+++ b/internal/api/system_settings_handlers_test.go@@ -619,7 +619,7 @@ func TestSystemSettingsHandler_GetMonitor_UsesTenantMonitorInterface(t *testing.}}-func TestSystemSettingsHandler_GetMonitor_FallsBackToLegacyOnTenantError(t *testing.T) {+func TestSystemSettingsHandler_GetMonitor_TenantErrorFailsClosed(t *testing.T) {defaultMonitor := &mockMonitor{}provider := &mockTenantMonitorProvider{err: errors.New("boom")}@@ -629,8 +629,8 @@ func TestSystemSettingsHandler_GetMonitor_FallsBackToLegacyOnTenantError(t *test}got := handler.getMonitor(context.WithValue(context.Background(), OrgIDContextKey, "acme"))- if got != defaultMonitor {- t.Fatalf("expected legacy monitor fallback, got %#v", got)+ if got != nil {+ t.Fatalf("expected nil monitor for unresolvable tenant, got %#v", got)}if provider.orgID != "acme" {t.Fatalf("expected tenant org lookup for acme, got %q", provider.orgID)diff --git a/internal/api/tenant_monitor_fail_closed_test.go b/internal/api/tenant_monitor_fail_closed_test.gonew file mode 100644index 0000000000..819197de44--- /dev/null+++ b/internal/api/tenant_monitor_fail_closed_test.go@@ -0,0 +1,116 @@+package api++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"+)++// unresolvableTenantMonitor returns a tenant manager with no persistence, so+// every organization lookup it is asked for fails.+func unresolvableTenantMonitor() *monitoring.MultiTenantMonitor {+ return &monitoring.MultiTenantMonitor{}+}++func tenantContext(orgID string) context.Context {+ return context.WithValue(context.Background(), OrgIDContextKey, orgID)+}++func TestAgentHandlersGetMonitor_NonDefaultOrgFailsClosed(t *testing.T) {+ defaultMonitor := &monitoring.Monitor{}++ t.Run("tenant resolution error", func(t *testing.T) {+ base := newBaseAgentHandlers(unresolvableTenantMonitor(), defaultMonitor, nil)+ if got := base.getMonitor(tenantContext("tenant-a")); got != nil {+ t.Fatalf("expected nil monitor for unresolvable tenant, got default-org fallback")+ }+ })++ t.Run("no tenant manager", func(t *testing.T) {+ base := newBaseAgentHandlers(nil, defaultMonitor, nil)+ if got := base.getMonitor(tenantContext("tenant-a")); got != nil {+ t.Fatalf("expected nil monitor without a tenant manager, got default-org fallback")+ }+ })++ t.Run("default org keeps primary monitor", func(t *testing.T) {+ base := newBaseAgentHandlers(unresolvableTenantMonitor(), defaultMonitor, nil)+ for _, ctx := range []context.Context{context.Background(), tenantContext("default"), tenantContext("")} {+ if got := base.getMonitor(ctx); got != defaultMonitor {+ t.Fatalf("expected default monitor for default org, got %p", got)+ }+ }+ })+}++func TestAgentHandlers_UnresolvableTenantReturns503(t *testing.T) {+ defaultMonitor := &monitoring.Monitor{}++ cases := []struct {+ name string+ path string+ handler http.HandlerFunc+ }{+ {+ name: "unified agent",+ path: agentRoutePrefix + "agent-1/allow-reenroll",+ handler: NewUnifiedAgentHandlers(unresolvableTenantMonitor(), defaultMonitor, nil).HandleAllowReenroll,+ },+ {+ name: "docker runtime",+ path: "/api/agents/docker/runtimes/host-1/allow-reenroll",+ handler: NewDockerAgentHandlers(unresolvableTenantMonitor(), defaultMonitor, nil, nil).HandleAllowReenroll,+ },+ {+ name: "kubernetes cluster",+ path: "/api/agents/kubernetes/clusters/cluster-1/allow-reenroll",+ handler: NewKubernetesAgentHandlers(unresolvableTenantMonitor(), defaultMonitor, nil).HandleAllowReenroll,+ },+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ req := httptest.NewRequest(http.MethodPost, tc.path, nil).WithContext(tenantContext("tenant-a"))+ rec := httptest.NewRecorder()+ tc.handler(rec, req)++ if rec.Code != http.StatusServiceUnavailable {+ t.Fatalf("expected 503, got %d: %s", rec.Code, rec.Body.String())+ }+ var body struct {+ Code string `json:"code"`+ }+ if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {+ t.Fatalf("decode error body: %v", err)+ }+ if body.Code != "tenant_unavailable" {+ t.Fatalf("expected tenant_unavailable, got %q", body.Code)+ }+ })+ }+}++func TestDockerAgentHandlers_LifecycleActionUnresolvableTenantReturns503(t *testing.T) {+ handler := NewDockerAgentHandlers(unresolvableTenantMonitor(), &monitoring.Monitor{}, nil, nil)+ req := httptest.NewRequest(http.MethodPut, "/api/agents/docker/runtimes/host-1/unhide", nil).+ WithContext(tenantContext("tenant-a"))+ rec := httptest.NewRecorder()++ handler.HandleUnhideHost(rec, req)++ if rec.Code != http.StatusServiceUnavailable {+ t.Fatalf("expected 503, got %d: %s", rec.Code, rec.Body.String())+ }+}++func TestSystemSettingsHandler_GetMonitor_NoTenantManagerFailsClosed(t *testing.T) {+ handler := &SystemSettingsHandler{defaultMonitor: &mockMonitor{}}++ if got := handler.getMonitor(tenantContext("acme")); got != nil {+ t.Fatalf("expected nil monitor without a tenant manager, got %#v", got)+ }+}