Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Stop tenant monitor lookups falling back to the default org

rcourtman94f96eb04341
diff --git a/internal/api/agent_handlers_base.go b/internal/api/agent_handlers_base.go
index d30dcf5d35..90c8c9b86f 100644
--- a/internal/api/agent_handlers_base.go
+++ b/internal/api/agent_handlers_base.go
@@ -2,10 +2,13 @@ package api
import (
"context"
+ "net/http"
+ "strings"
"sync"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
+ "github.com/rs/zerolog/log"
)
// baseAgentHandlers provides the shared monitor-management and broadcast logic
@@ -53,21 +56,47 @@ func (b *baseAgentHandlers) SetMultiTenantMonitor(mtm *monitoring.MultiTenantMon
}
}
-// getMonitor returns the monitor for the organization in the request context,
-// falling back to the default-org monitor.
+// getMonitor returns the monitor for the organization in the request context.
+// Only the default organization may fall back to the primary monitor. Any
+// other organization whose tenant monitor cannot be resolved gets nil, so its
+// requests never read or change the default organization's agents.
func (b *baseAgentHandlers) getMonitor(ctx context.Context) *monitoring.Monitor {
b.stateMu.RLock()
mtMonitor := b.mtMonitor
defaultMonitor := b.defaultMonitor
b.stateMu.RUnlock()
- orgID := GetOrgID(ctx)
- if mtMonitor != nil {
- if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
- return m
+ orgID := strings.TrimSpace(GetOrgID(ctx))
+ if orgID == "" || orgID == "default" {
+ if mtMonitor != nil {
+ if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
+ return m
+ }
}
+ return defaultMonitor
+ }
+
+ if mtMonitor == nil {
+ log.Warn().Str("org_id", orgID).Msg("Tenant monitor manager unavailable for agent request")
+ return nil
+ }
+ m, err := mtMonitor.GetMonitor(orgID)
+ if err != nil || m == nil {
+ log.Warn().Err(err).Str("org_id", orgID).Msg("Failed to resolve tenant monitor for agent request")
+ return nil
+ }
+ return m
+}
+
+// requireMonitor resolves the request organization's monitor, writing a 503
+// and returning nil when it is unavailable.
+func (b *baseAgentHandlers) requireMonitor(w http.ResponseWriter, r *http.Request) *monitoring.Monitor {
+ monitor := b.getMonitor(r.Context())
+ if monitor == nil {
+ writeErrorResponse(w, http.StatusServiceUnavailable, "tenant_unavailable", "Tenant monitor is not available", nil)
+ return nil
}
- return defaultMonitor
+ return monitor
}
// broadcastState pushes monitor state to tenant-scoped WebSocket clients when
diff --git a/internal/api/agent_ingest.go b/internal/api/agent_ingest.go
index 9195a50cd2..e0fb470e45 100644
--- a/internal/api/agent_ingest.go
+++ b/internal/api/agent_ingest.go
@@ -100,8 +100,12 @@ func (h *UnifiedAgentHandlers) HandleReport(w http.ResponseWriter, r *http.Reque
report.Timestamp = time.Now().UTC()
}
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
tokenRecord := getAPITokenRecordFromRequest(r)
- host, err := h.getMonitor(r.Context()).ApplyHostReport(report, tokenRecord)
+ host, err := monitor.ApplyHostReport(report, tokenRecord)
if err != nil {
writeErrorResponse(w, http.StatusBadRequest, "invalid_report", err.Error(), nil)
return
@@ -129,7 +133,7 @@ func (h *UnifiedAgentHandlers) HandleReport(w http.ResponseWriter, r *http.Reque
// identity before returning remote config, otherwise an old false override
// survives remove/reinstall and immediately shuts down an agent launched
// with --enable-commands (#1728).
- if err := reconcileInstallTokenCommandPolicy(h.getMonitor(r.Context()), tokenRecord, host); err != nil {
+ if err := reconcileInstallTokenCommandPolicy(monitor, tokenRecord, host); err != nil {
log.Error().Err(err).
Str("agent_id", host.ID).
Str("token_id", tokenID(tokenRecord)).
@@ -145,7 +149,7 @@ func (h *UnifiedAgentHandlers) HandleReport(w http.ResponseWriter, r *http.Reque
h.broadcastState(r.Context())
// Include any server-side config overrides in the response
- serverConfig := h.getMonitor(r.Context()).GetHostAgentConfig(host.ID)
+ serverConfig := monitor.GetHostAgentConfig(host.ID)
serverConfig = sanitizeHostAgentConfigForToken(serverConfig, tokenRecord, host)
resp := map[string]any{
@@ -280,7 +284,11 @@ func (h *UnifiedAgentHandlers) HandleLookup(w http.ResponseWriter, r *http.Reque
// Use the live host snapshot (not the global mock snapshot) so agent
// registrations can still be validated while Pulse is in mock/demo mode.
- hosts := h.getMonitor(r.Context()).GetLiveHostsSnapshot()
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ hosts := monitor.GetLiveHostsSnapshot()
var (
host models.Host
@@ -415,7 +423,11 @@ func (h *UnifiedAgentHandlers) HandleDeleteHost(w http.ResponseWriter, r *http.R
}
// Remove the agent from state.
- host, err := h.getMonitor(r.Context()).RemoveHostAgent(agentID)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ host, err := monitor.RemoveHostAgent(agentID)
if err != nil {
writeErrorResponse(w, http.StatusInternalServerError, "agent_removal_failed", err.Error(), nil)
return
@@ -447,7 +459,11 @@ func (h *UnifiedAgentHandlers) HandleAllowReenroll(w http.ResponseWriter, r *htt
return
}
- if err := h.getMonitor(r.Context()).AllowHostAgentReenroll(agentID); err != nil {
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ if err := monitor.AllowHostAgentReenroll(agentID); err != nil {
writeErrorResponse(w, http.StatusBadRequest, "agent_reenroll_failed", err.Error(), nil)
return
}
@@ -505,6 +521,9 @@ func (h *UnifiedAgentHandlers) resolveConfigAgent(ctx context.Context, agentID s
// Use the live host snapshot so agents can still fetch config while
// Pulse is running in mock/demo mode.
monitor := h.getMonitor(ctx)
+ if monitor == nil {
+ return models.Host{}, false
+ }
hosts := monitor.GetLiveHostsSnapshot()
if record == nil || record.HasScope(config.ScopeSettingsWrite) || record.HasScope(config.ScopeAgentManage) {
@@ -617,6 +636,11 @@ func (h *UnifiedAgentHandlers) handleGetConfig(w http.ResponseWriter, r *http.Re
return
}
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+
host, ok := h.resolveConfigAgent(r.Context(), agentID, record)
if !ok {
writeErrorResponse(w, http.StatusNotFound, "agent_not_found", "Agent has not registered with Pulse yet", nil)
@@ -627,7 +651,7 @@ func (h *UnifiedAgentHandlers) handleGetConfig(w http.ResponseWriter, r *http.Re
agentID = host.ID
- config := h.getMonitor(r.Context()).GetHostAgentConfig(agentID)
+ config := monitor.GetHostAgentConfig(agentID)
config = sanitizeHostAgentConfigForToken(config, record, host)
signedConfig, err := h.signAgentConfig(agentID, config)
if err != nil {
@@ -765,7 +789,11 @@ func (h *UnifiedAgentHandlers) ensureAgentTokenMatch(w http.ResponseWriter, r *h
}
// Use the live host snapshot so mock/demo mode doesn't block agent auth checks.
- hosts := h.getMonitor(r.Context()).GetLiveHostsSnapshot()
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return false
+ }
+ hosts := monitor.GetLiveHostsSnapshot()
for _, host := range hosts {
if host.ID != agentID {
continue
@@ -797,7 +825,11 @@ func (h *UnifiedAgentHandlers) handlePatchConfig(w http.ResponseWriter, r *http.
return
}
- if err := h.getMonitor(r.Context()).UpdateHostAgentConfig(agentID, req.CommandsEnabled); err != nil {
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ if err := monitor.UpdateHostAgentConfig(agentID, req.CommandsEnabled); err != nil {
writeErrorResponse(w, http.StatusInternalServerError, "update_failed", err.Error(), nil)
return
}
@@ -849,7 +881,10 @@ func (h *UnifiedAgentHandlers) HandleUninstall(w http.ResponseWriter, r *http.Re
log.Info().Str("agentId", agentID).Msg("Received unregistration request from agent uninstaller")
- monitor := h.getMonitor(r.Context())
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
record := getAPITokenRecordFromRequest(r)
var err error
if record != nil {
@@ -915,7 +950,11 @@ func (h *UnifiedAgentHandlers) HandleLink(w http.ResponseWriter, r *http.Request
return
}
- if err := h.getMonitor(r.Context()).LinkHostAgent(agentID, nodeID); err != nil {
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ if err := monitor.LinkHostAgent(agentID, nodeID); err != nil {
writeErrorResponse(w, http.StatusBadRequest, "link_failed", err.Error(), nil)
return
}
@@ -957,7 +996,11 @@ func (h *UnifiedAgentHandlers) HandleUnlink(w http.ResponseWriter, r *http.Reque
return
}
- if err := h.getMonitor(r.Context()).UnlinkHostAgent(agentID); err != nil {
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ if err := monitor.UnlinkHostAgent(agentID); err != nil {
writeErrorResponse(w, http.StatusNotFound, "unlink_failed", err.Error(), nil)
return
}
diff --git a/internal/api/alerting/alerts.go b/internal/api/alerting/alerts.go
index 8cc4165417..8f975db233 100644
--- a/internal/api/alerting/alerts.go
+++ b/internal/api/alerting/alerts.go
@@ -132,13 +132,39 @@ func (h *AlertHandlers) getMonitor(ctx context.Context) AlertMonitor {
defaultMonitor := h.defaultMonitor
h.stateMu.RUnlock()
- orgID := apicontext.OrgID(ctx)
- if mtMonitor != nil {
- if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
- return NewAlertMonitorWrapper(m)
+ orgID := strings.TrimSpace(apicontext.OrgID(ctx))
+ if orgID == "" || orgID == "default" {
+ if mtMonitor != nil {
+ if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
+ return NewAlertMonitorWrapper(m)
+ }
}
+ return defaultMonitor
+ }
+
+ // Never fall back to the default organization's runtime for another
+ // organization: its requests would read or change the wrong tenant.
+ if mtMonitor == nil {
+ log.Warn().Str("org_id", orgID).Msg("Tenant monitor manager unavailable for alert request")
+ return nil
}
- return defaultMonitor
+ m, err := mtMonitor.GetMonitor(orgID)
+ if err != nil || m == nil {
+ log.Warn().Err(err).Str("org_id", orgID).Msg("Failed to resolve tenant monitor for alert request")
+ return nil
+ }
+ return NewAlertMonitorWrapper(m)
+}
+
+// requireMonitor resolves the request organization's monitor, writing a 503
+// and returning nil when it is unavailable.
+func (h *AlertHandlers) requireMonitor(w http.ResponseWriter, r *http.Request) AlertMonitor {
+ monitor := h.getMonitor(r.Context())
+ if monitor == nil {
+ http.Error(w, "Tenant monitor is not available", http.StatusServiceUnavailable)
+ return nil
+ }
+ return monitor
}
// MonitorForContext resolves the tenant-scoped alert runtime used by handlers.
@@ -206,7 +232,11 @@ func validateAlertIdentifier(alertIdentifier string) bool {
// GetAlertConfig returns the current alert configuration
func (h *AlertHandlers) GetAlertConfig(w http.ResponseWriter, r *http.Request) {
- config := h.getMonitor(r.Context()).GetAlertManager().GetConfig()
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ config := monitor.GetAlertManager().GetConfig()
if err := utils.WriteJSONResponse(w, config); err != nil {
log.Error().Err(err).Msg("Failed to write alert config response")
@@ -216,9 +246,8 @@ func (h *AlertHandlers) GetAlertConfig(w http.ResponseWriter, r *http.Request) {
// GetDeadManStatus returns the live external-watchdog read model without
// echoing the secret-bearing ping URL.
func (h *AlertHandlers) GetDeadManStatus(w http.ResponseWriter, r *http.Request) {
- monitor := h.getMonitor(r.Context())
+ monitor := h.requireMonitor(w, r)
if monitor == nil {
- http.Error(w, "Alert monitor unavailable", http.StatusServiceUnavailable)
return
}
if err := utils.WriteJSONResponse(w, monitor.DeadManStatus()); err != nil {
@@ -236,9 +265,8 @@ type deadManConfigView struct {
// GetDeadManConfig returns only a presence marker for the credential-bearing
// ping URL. The sentinel supports an edit round-trip without disclosing it.
func (h *AlertHandlers) GetDeadManConfig(w http.ResponseWriter, r *http.Request) {
- monitor := h.getMonitor(r.Context())
+ monitor := h.requireMonitor(w, r)
if monitor == nil {
- http.Error(w, "Alert monitor unavailable", http.StatusServiceUnavailable)
return
}
configured := strings.TrimSpace(monitor.DeadManConfig().PingURL) != ""
@@ -254,9 +282,8 @@ func (h *AlertHandlers) GetDeadManConfig(w http.ResponseWriter, r *http.Request)
// UpdateDeadManConfig validates and durably encrypts the watchdog destination
// before its live worker is reconfigured.
func (h *AlertHandlers) UpdateDeadManConfig(w http.ResponseWriter, r *http.Request) {
- monitor := h.getMonitor(r.Context())
+ monitor := h.requireMonitor(w, r)
if monitor == nil {
- http.Error(w, "Alert monitor unavailable", http.StatusServiceUnavailable)
return
}
r.Body = http.MaxBytesReader(w, r.Body, notifications.MaxDeadManPingURLLength+1024)
@@ -306,7 +333,11 @@ func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request
// Keys the client left out keep their stored values, so a thresholds
// save no longer turns off flapping detection or alert TTL cleanup.
- updatedConfig, err := h.getMonitor(r.Context()).GetAlertManager().ApplyConfigUpdate(body)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ updatedConfig, err := monitor.GetAlertManager().ApplyConfigUpdate(body)
if errors.Is(err, alerts.ErrConfigSnapshot) {
log.Error().Err(err).Msg("Failed to snapshot applied alert configuration")
http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -318,7 +349,7 @@ func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request
}
// Update notification manager with schedule settings
- notificationMgr := h.getMonitor(r.Context()).GetNotificationManager()
+ notificationMgr := monitor.GetNotificationManager()
notificationMgr.SetEnabled(updatedConfig.Enabled && updatedConfig.ActivationState == alerts.ActivationActive)
notificationMgr.SetCooldown(updatedConfig.Schedule.Cooldown)
notificationMgr.SetGroupingConfig(
@@ -335,7 +366,7 @@ func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request
// what was actually persisted. On the next restart or config reload,
// the override silently vanished. Surface the error so the client can
// show a real save-failed signal.
- if err := h.getMonitor(r.Context()).GetConfigPersistence().SaveAlertConfig(updatedConfig); err != nil {
+ if err := monitor.GetConfigPersistence().SaveAlertConfig(updatedConfig); err != nil {
log.Error().Err(err).Msg("Failed to save alert configuration")
http.Error(w, fmt.Sprintf("Failed to save alert configuration: %v", err), http.StatusInternalServerError)
return
@@ -352,7 +383,11 @@ func (h *AlertHandlers) UpdateAlertConfig(w http.ResponseWriter, r *http.Request
// ActivateAlerts activates alert notifications
func (h *AlertHandlers) ActivateAlerts(w http.ResponseWriter, r *http.Request) {
// Get current config
- config := h.getMonitor(r.Context()).GetAlertManager().GetConfig()
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ config := monitor.GetAlertManager().GetConfig()
// Check if already active
if config.ActivationState == alerts.ActivationActive {
@@ -373,25 +408,25 @@ func (h *AlertHandlers) ActivateAlerts(w http.ResponseWriter, r *http.Request) {
config.ActivationTime = &now
// Update config
- h.getMonitor(r.Context()).GetAlertManager().UpdateConfig(config)
- h.getMonitor(r.Context()).GetNotificationManager().SetEnabled(
+ monitor.GetAlertManager().UpdateConfig(config)
+ monitor.GetNotificationManager().SetEnabled(
config.Enabled && config.ActivationState == alerts.ActivationActive,
)
// Save to persistent storage
- if err := h.getMonitor(r.Context()).GetConfigPersistence().SaveAlertConfig(config); err != nil {
+ if err := monitor.GetConfigPersistence().SaveAlertConfig(config); err != nil {
log.Error().Err(err).Msg("Failed to save alert configuration after activation")
http.Error(w, "Failed to save configuration", http.StatusInternalServerError)
return
}
// Notify about existing critical alerts after activation
- activeAlerts := h.getMonitor(r.Context()).GetAlertManager().GetActiveAlerts()
+ activeAlerts := monitor.GetAlertManager().GetActiveAlerts()
criticalCount := 0
for _, alert := range activeAlerts {
if alert.Level == alerts.AlertLevelCritical && !alert.Acknowledged {
// Re-dispatch critical alerts to trigger notifications
- h.getMonitor(r.Context()).GetAlertManager().NotifyExistingAlert(alert.ID)
+ monitor.GetAlertManager().NotifyExistingAlert(alert.ID)
criticalCount++
}
}
@@ -416,7 +451,11 @@ func (h *AlertHandlers) ActivateAlerts(w http.ResponseWriter, r *http.Request) {
// GetActiveAlerts returns all active alerts
func (h *AlertHandlers) GetActiveAlerts(w http.ResponseWriter, r *http.Request) {
- alerts := h.getMonitor(r.Context()).GetAlertManager().GetActiveAlerts()
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ alerts := monitor.GetAlertManager().GetActiveAlerts()
if err := utils.WriteJSONResponse(w, alerts); err != nil {
log.Error().Err(err).Msg("Failed to write active alerts response")
@@ -426,7 +465,11 @@ func (h *AlertHandlers) GetActiveAlerts(w http.ResponseWriter, r *http.Request)
func (h *AlertHandlers) GetAlertIntentPolicies(w http.ResponseWriter, r *http.Request) {
h.intentPolicyMu.Lock()
defer h.intentPolicyMu.Unlock()
- manager, ok := h.getMonitor(r.Context()).GetAlertManager().(alertIntentPolicyManager)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ manager, ok := monitor.GetAlertManager().(alertIntentPolicyManager)
if !ok {
http.Error(w, "alert intent policies are unavailable", http.StatusServiceUnavailable)
return
@@ -456,7 +499,10 @@ func (h *AlertHandlers) UpdateAlertIntentPolicies(w http.ResponseWriter, r *http
// back a later successful update.
h.intentPolicyMu.Lock()
defer h.intentPolicyMu.Unlock()
- monitor := h.getMonitor(r.Context())
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
manager, ok := monitor.GetAlertManager().(alertIntentPolicyManager)
if !ok {
http.Error(w, "alert intent policies are unavailable", http.StatusServiceUnavailable)
@@ -504,7 +550,11 @@ func (h *AlertHandlers) PreviewAlertIntentPolicy(w http.ResponseWriter, r *http.
}
h.intentPolicyMu.Lock()
defer h.intentPolicyMu.Unlock()
- manager, ok := h.getMonitor(r.Context()).GetAlertManager().(alertIntentPolicyManager)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ manager, ok := monitor.GetAlertManager().(alertIntentPolicyManager)
if !ok {
http.Error(w, "alert intent policy preview is unavailable", http.StatusServiceUnavailable)
return
@@ -539,9 +589,8 @@ func (h *AlertHandlers) GetAlertDeliveryDiagnosis(w http.ResponseWriter, r *http
return
}
- monitor := h.getMonitor(r.Context())
+ monitor := h.requireMonitor(w, r)
if monitor == nil {
- http.Error(w, "monitor is not initialized", http.StatusServiceUnavailable)
return
}
@@ -614,9 +663,8 @@ func (h *AlertHandlers) GetAlertEvents(w http.ResponseWriter, r *http.Request) {
filter.Limit = limit
}
- monitor := h.getMonitor(r.Context())
+ monitor := h.requireMonitor(w, r)
if monitor == nil {
- http.Error(w, "monitor is not initialized", http.StatusServiceUnavailable)
return
}
manager := monitor.GetAlertManager()
@@ -778,12 +826,12 @@ func (h *AlertHandlers) GetAlertHistory(w http.ResponseWriter, r *http.Request)
return
}
- if h.getMonitor(r.Context()) == nil {
- http.Error(w, "monitor is not initialized", http.StatusServiceUnavailable)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
return
}
- manager := h.getMonitor(r.Context()).GetAlertManager()
+ manager := monitor.GetAlertManager()
var history []alerts.Alert
if startTime != nil {
history = manager.GetAlertHistorySince(*startTime, fetchLimit)
@@ -825,6 +873,10 @@ func (h *AlertHandlers) GetAlertIncidentTimeline(w http.ResponseWriter, r *http.
}
}
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
if alertID != "" {
if !validateAlertIdentifier(alertID) {
http.Error(w, "Invalid alert identifier", http.StatusBadRequest)
@@ -846,7 +898,7 @@ func (h *AlertHandlers) GetAlertIncidentTimeline(w http.ResponseWriter, r *http.
return
}
- store := h.getMonitor(r.Context()).GetIncidentStore()
+ store := monitor.GetIncidentStore()
if store == nil {
http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
return
@@ -863,7 +915,7 @@ func (h *AlertHandlers) GetAlertIncidentTimeline(w http.ResponseWriter, r *http.
}
if !startedAt.IsZero() && (incident == nil || len(incident.Events) == 0) {
- if alert, resolvedAt := findAlertOccurrenceForTimeline(h.getMonitor(r.Context()).GetAlertManager(), alertID, startedAt); alert != nil {
+ if alert, resolvedAt := findAlertOccurrenceForTimeline(monitor.GetAlertManager(), alertID, startedAt); alert != nil {
incident = store.EnsureAlertOccurrence(alert, resolvedAt)
}
}
@@ -885,7 +937,7 @@ func (h *AlertHandlers) GetAlertIncidentTimeline(w http.ResponseWriter, r *http.
}
return
}
- store := h.getMonitor(r.Context()).GetIncidentStore()
+ store := monitor.GetIncidentStore()
if store == nil {
http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
return
@@ -994,7 +1046,11 @@ func (h *AlertHandlers) SaveAlertIncidentNote(w http.ResponseWriter, r *http.Req
return
}
- store := h.getMonitor(r.Context()).GetIncidentStore()
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ store := monitor.GetIncidentStore()
if store == nil {
http.Error(w, "Incident store unavailable", http.StatusServiceUnavailable)
return
@@ -1118,7 +1174,11 @@ func exportIncidents(incidents []*memory.Incident) []*incidentView {
// ClearAlertHistory clears all alert history
func (h *AlertHandlers) ClearAlertHistory(w http.ResponseWriter, r *http.Request) {
- if err := h.getMonitor(r.Context()).GetAlertManager().ClearAlertHistory(); err != nil {
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ if err := monitor.GetAlertManager().ClearAlertHistory(); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
@@ -1187,13 +1247,17 @@ func (h *AlertHandlers) AcknowledgeAlertByBody(w http.ResponseWriter, r *http.Re
user = "unknown" // Fallback if auth header not set (shouldn't happen in normal flow)
}
- if err := h.getMonitor(r.Context()).GetAlertManager().AcknowledgeAlert(alertID, user); err != nil {
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ if err := monitor.GetAlertManager().AcknowledgeAlert(alertID, user); err != nil {
log.Error().Err(err).Str("alertID", alertID).Msg("Failed to acknowledge alert")
http.Error(w, err.Error(), http.StatusNotFound)
return
}
- h.getMonitor(r.Context()).SyncAlertState()
+ monitor.SyncAlertState()
log.Info().Str("alertID", alertID).Str("user", user).Msg("Alert acknowledged successfully")
@@ -1231,13 +1295,17 @@ func (h *AlertHandlers) UnacknowledgeAlertByBody(w http.ResponseWriter, r *http.
return
}
- if err := h.getMonitor(r.Context()).GetAlertManager().UnacknowledgeAlert(alertID); err != nil {
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ if err := monitor.GetAlertManager().UnacknowledgeAlert(alertID); err != nil {
log.Error().Err(err).Str("alertID", alertID).Msg("Failed to unacknowledge alert")
http.Error(w, err.Error(), http.StatusNotFound)
return
}
- h.getMonitor(r.Context()).SyncAlertState()
+ monitor.SyncAlertState()
log.Info().Str("alertID", alertID).Msg("Alert unacknowledged successfully")
@@ -1273,7 +1341,11 @@ func (h *AlertHandlers) SnoozeAlertByBody(w http.ResponseWriter, r *http.Request
if user == "" {
user = "unknown"
}
- if err := h.getMonitor(r.Context()).GetAlertManager().SnoozeAlert(alertID, user, req.Until); err != nil {
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ if err := monitor.GetAlertManager().SnoozeAlert(alertID, user, req.Until); err != nil {
status := http.StatusBadRequest
if errors.Is(err, alerts.ErrAlertNotFound) {
status = http.StatusNotFound
@@ -1281,7 +1353,7 @@ func (h *AlertHandlers) SnoozeAlertByBody(w http.ResponseWriter, r *http.Request
http.Error(w, err.Error(), status)
return
}
- h.getMonitor(r.Context()).SyncAlertState()
+ monitor.SyncAlertState()
if err := utils.WriteJSONResponse(w, map[string]any{"success": true, "snoozedUntil": req.Until.UTC()}); err != nil {
log.Error().Err(err).Str("alertID", alertID).Msg("Failed to write snooze response")
}
@@ -1307,7 +1379,11 @@ func (h *AlertHandlers) UnsnoozeAlertByBody(w http.ResponseWriter, r *http.Reque
if user == "" {
user = "unknown"
}
- if err := h.getMonitor(r.Context()).GetAlertManager().UnsnoozeAlert(alertID, user); err != nil {
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ if err := monitor.GetAlertManager().UnsnoozeAlert(alertID, user); err != nil {
status := http.StatusBadRequest
if errors.Is(err, alerts.ErrAlertNotFound) {
status = http.StatusNotFound
@@ -1315,7 +1391,7 @@ func (h *AlertHandlers) UnsnoozeAlertByBody(w http.ResponseWriter, r *http.Reque
http.Error(w, err.Error(), status)
return
}
- h.getMonitor(r.Context()).SyncAlertState()
+ monitor.SyncAlertState()
if err := utils.WriteJSONResponse(w, map[string]bool{"success": true}); err != nil {
log.Error().Err(err).Str("alertID", alertID).Msg("Failed to write unsnooze response")
}
@@ -1347,12 +1423,16 @@ func (h *AlertHandlers) ClearAlertByBody(w http.ResponseWriter, r *http.Request)
return
}
- if !h.getMonitor(r.Context()).GetAlertManager().ClearAlert(alertID) {
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ if !monitor.GetAlertManager().ClearAlert(alertID) {
http.Error(w, "Alert not found", http.StatusNotFound)
return
}
- h.getMonitor(r.Context()).SyncAlertState()
+ monitor.SyncAlertState()
log.Info().Str("alertID", alertID).Msg("Alert cleared successfully")
@@ -1398,12 +1478,16 @@ func (h *AlertHandlers) BulkAcknowledgeAlerts(w http.ResponseWriter, r *http.Req
results []map[string]interface{}
anySuccess bool
)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
for _, alertID := range alertIdentifiers {
result := map[string]interface{}{
"alertIdentifier": alertID,
"success": true,
}
- if err := h.getMonitor(r.Context()).GetAlertManager().AcknowledgeAlert(alertID, user); err != nil {
+ if err := monitor.GetAlertManager().AcknowledgeAlert(alertID, user); err != nil {
result["success"] = false
result["error"] = err.Error()
} else {
@@ -1413,7 +1497,7 @@ func (h *AlertHandlers) BulkAcknowledgeAlerts(w http.ResponseWriter, r *http.Req
}
if anySuccess {
- h.getMonitor(r.Context()).SyncAlertState()
+ monitor.SyncAlertState()
}
// Send response immediately
@@ -1456,12 +1540,16 @@ func (h *AlertHandlers) BulkClearAlerts(w http.ResponseWriter, r *http.Request)
results []map[string]interface{}
anySuccess bool
)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
for _, alertID := range alertIdentifiers {
result := map[string]interface{}{
"alertIdentifier": alertID,
"success": true,
}
- if h.getMonitor(r.Context()).GetAlertManager().ClearAlert(alertID) {
+ if monitor.GetAlertManager().ClearAlert(alertID) {
anySuccess = true
} else {
result["success"] = false
@@ -1471,7 +1559,7 @@ func (h *AlertHandlers) BulkClearAlerts(w http.ResponseWriter, r *http.Request)
}
if anySuccess {
- h.getMonitor(r.Context()).SyncAlertState()
+ monitor.SyncAlertState()
}
// Send response immediately
diff --git a/internal/api/alerting/notifications.go b/internal/api/alerting/notifications.go
index 2c30e52232..e5756bb858 100644
--- a/internal/api/alerting/notifications.go
+++ b/internal/api/alerting/notifications.go
@@ -127,13 +127,39 @@ func (h *NotificationHandlers) getMonitor(ctx context.Context) NotificationMonit
defaultMonitor := h.defaultMonitor
h.stateMu.RUnlock()
- orgID := apicontext.OrgID(ctx)
- if mtMonitor != nil {
- if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
- return NewNotificationMonitorWrapper(m)
+ orgID := strings.TrimSpace(apicontext.OrgID(ctx))
+ if orgID == "" || orgID == "default" {
+ if mtMonitor != nil {
+ if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
+ return NewNotificationMonitorWrapper(m)
+ }
}
+ return defaultMonitor
+ }
+
+ // Never fall back to the default organization's runtime for another
+ // organization: its requests would read or change the wrong tenant.
+ if mtMonitor == nil {
+ log.Warn().Str("org_id", orgID).Msg("Tenant monitor manager unavailable for notification request")
+ return nil
}
- return defaultMonitor
+ m, err := mtMonitor.GetMonitor(orgID)
+ if err != nil || m == nil {
+ log.Warn().Err(err).Str("org_id", orgID).Msg("Failed to resolve tenant monitor for notification request")
+ return nil
+ }
+ return NewNotificationMonitorWrapper(m)
+}
+
+// requireMonitor resolves the request organization's monitor, writing a 503
+// and returning nil when it is unavailable.
+func (h *NotificationHandlers) requireMonitor(w http.ResponseWriter, r *http.Request) NotificationMonitor {
+ monitor := h.getMonitor(r.Context())
+ if monitor == nil {
+ http.Error(w, "Tenant monitor is not available", http.StatusServiceUnavailable)
+ return nil
+ }
+ return monitor
}
// MonitorForContext resolves the tenant-scoped notification runtime used by
@@ -169,7 +195,11 @@ func (h *NotificationHandlers) getReadState(ctx context.Context) unifiedresource
// GetEmailConfig returns the current email configuration
func (h *NotificationHandlers) GetEmailConfig(w http.ResponseWriter, r *http.Request) {
- config := h.getMonitor(r.Context()).GetNotificationManager().GetEmailConfig()
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ config := monitor.GetNotificationManager().GetEmailConfig()
// For security, don't return the password
config.Password = ""
@@ -215,7 +245,10 @@ func (h *NotificationHandlers) UpdateEmailConfig(w http.ResponseWriter, r *http.
return
}
- monitor := h.getMonitor(r.Context())
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
manager := monitor.GetNotificationManager()
existingConfig := manager.GetEmailConfig()
@@ -277,7 +310,11 @@ func redactAppriseConfig(config notifications.AppriseConfig) appriseConfigRespon
// GetAppriseConfig returns the current Apprise configuration.
func (h *NotificationHandlers) GetAppriseConfig(w http.ResponseWriter, r *http.Request) {
- config := h.getMonitor(r.Context()).GetNotificationManager().GetAppriseConfig()
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ config := monitor.GetNotificationManager().GetAppriseConfig()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(redactAppriseConfig(config)); err != nil {
@@ -308,7 +345,10 @@ func (h *NotificationHandlers) UpdateAppriseConfig(w http.ResponseWriter, r *htt
MinimumSeverity *string `json:"minimumSeverity"`
}
_ = json.Unmarshal(body, &presenceCheck)
- monitor := h.getMonitor(r.Context())
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
manager := monitor.GetNotificationManager()
existingConfig := manager.GetAppriseConfig()
@@ -350,7 +390,11 @@ func (h *NotificationHandlers) UpdateAppriseConfig(w http.ResponseWriter, r *htt
// GetWebhooks returns all webhook configurations with secrets masked
func (h *NotificationHandlers) GetWebhooks(w http.ResponseWriter, r *http.Request) {
- webhooks := h.getMonitor(r.Context()).GetNotificationManager().GetWebhooks()
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ webhooks := monitor.GetNotificationManager().GetWebhooks()
maskedWebhooks := make([]map[string]interface{}, len(webhooks))
for i, webhook := range webhooks {
@@ -470,7 +514,10 @@ func (h *NotificationHandlers) CreateWebhook(w http.ResponseWriter, r *http.Requ
return
}
- monitor := h.getMonitor(r.Context())
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
manager := monitor.GetNotificationManager()
// Validate webhook URL
@@ -585,7 +632,10 @@ func (h *NotificationHandlers) UpdateWebhook(w http.ResponseWriter, r *http.Requ
}
_ = json.Unmarshal(bodyBytes, &routingPresence)
- monitor := h.getMonitor(r.Context())
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
manager := monitor.GetNotificationManager()
// Restore each masked value returned by GetWebhooks, not its entire map.
@@ -667,7 +717,10 @@ func (h *NotificationHandlers) DeleteWebhook(w http.ResponseWriter, r *http.Requ
return
}
- monitor := h.getMonitor(r.Context())
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
manager := monitor.GetNotificationManager()
existingWebhooks := manager.GetWebhooks()
candidateWebhooks := make([]notifications.WebhookConfig, 0, len(existingWebhooks))
@@ -794,6 +847,11 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.R
}
}
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+
// Handle webhook testing
if req.Method == "webhook" && req.WebhookID != "" {
log.Info().
@@ -801,7 +859,7 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.R
Msg("Testing specific webhook")
// Get the webhook by ID and test it
- webhooks := h.getMonitor(r.Context()).GetNotificationManager().GetWebhooks()
+ webhooks := monitor.GetNotificationManager().GetWebhooks()
var foundWebhook *notifications.WebhookConfig
for _, wh := range webhooks {
if wh.ID == req.WebhookID {
@@ -816,7 +874,7 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.R
}
// Send test webhook
- if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestWebhook(*foundWebhook); err != nil {
+ if err := monitor.GetNotificationManager().SendTestWebhook(*foundWebhook); err != nil {
writeTestNotificationError(w, err, http.StatusBadRequest)
return
}
@@ -829,7 +887,7 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.R
// If password is empty, use the saved password
if emailConfig.Password == "" {
- savedConfig := h.getMonitor(r.Context()).GetNotificationManager().GetEmailConfig()
+ savedConfig := monitor.GetNotificationManager().GetEmailConfig()
emailConfig.Password = savedConfig.Password
}
@@ -842,7 +900,7 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.R
Bool("hasPassword", emailConfig.Password != "").
Msg("Testing email with provided config")
- if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestNotificationWithConfig(req.Method, &emailConfig, nodeInfo); err != nil {
+ if err := monitor.GetNotificationManager().SendTestNotificationWithConfig(req.Method, &emailConfig, nodeInfo); err != nil {
writeTestNotificationError(w, err, http.StatusBadRequest)
return
}
@@ -855,17 +913,17 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.R
// If the API key is empty, use the saved key
if appriseConfig.APIKey == "" {
- savedConfig := h.getMonitor(r.Context()).GetNotificationManager().GetAppriseConfig()
+ savedConfig := monitor.GetNotificationManager().GetAppriseConfig()
appriseConfig.APIKey = savedConfig.APIKey
}
- if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestAppriseWithConfig(appriseConfig); err != nil {
+ if err := monitor.GetNotificationManager().SendTestAppriseWithConfig(appriseConfig); err != nil {
writeTestNotificationError(w, err, http.StatusBadRequest)
return
}
} else {
// Use saved config
- if err := h.getMonitor(r.Context()).GetNotificationManager().SendTestNotification(req.Method); err != nil {
+ if err := monitor.GetNotificationManager().SendTestNotification(req.Method); err != nil {
writeTestNotificationError(w, err, http.StatusBadRequest)
return
}
@@ -878,7 +936,7 @@ func (h *NotificationHandlers) TestNotification(w http.ResponseWriter, r *http.R
"status": "success",
"message": "Test notification sent",
}
- if !h.getMonitor(r.Context()).GetNotificationManager().IsEnabled() {
+ if !monitor.GetNotificationManager().IsEnabled() {
response["deliveryPaused"] = true
response["message"] = "Test notification sent, but alert delivery is paused: real alerts are not being sent"
}
@@ -896,7 +954,11 @@ func (h *NotificationHandlers) GetWebhookTemplates(w http.ResponseWriter, r *htt
// GetWebhookHistory returns recent webhook delivery history with URLs redacted
func (h *NotificationHandlers) GetWebhookHistory(w http.ResponseWriter, r *http.Request) {
- history := h.getMonitor(r.Context()).GetNotificationManager().GetWebhookHistory()
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ history := monitor.GetNotificationManager().GetWebhookHistory()
// Redact secrets from URLs in history
for i := range history {
@@ -948,9 +1010,13 @@ func (h *NotificationHandlers) TestWebhook(w http.ResponseWriter, r *http.Reques
// secrets are the masked values from GetWebhooks. Test with the saved
// values, as the saved webhook would send; this route requires the same
// settings:write scope as UpdateWebhook, which keeps them the same way.
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
saved := notifications.WebhookConfig{}
if basicWebhook.ID != "" {
- for _, candidate := range h.getMonitor(r.Context()).GetNotificationManager().GetWebhooks() {
+ for _, candidate := range monitor.GetNotificationManager().GetWebhooks() {
if candidate.ID == basicWebhook.ID {
saved = candidate
break
@@ -980,7 +1046,7 @@ func (h *NotificationHandlers) TestWebhook(w http.ResponseWriter, r *http.Reques
Msg("Testing webhook")
// Test the webhook
- status, response, err := h.getMonitor(r.Context()).GetNotificationManager().TestEnhancedWebhook(webhook)
+ status, response, err := monitor.GetNotificationManager().TestEnhancedWebhook(webhook)
result := map[string]interface{}{
"status": status,
@@ -1003,7 +1069,7 @@ func (h *NotificationHandlers) TestWebhook(w http.ResponseWriter, r *http.Reques
result["success"] = true
// Same honesty as TestNotification: a webhook test bypasses the
// activation gate, so its success must not imply live alerts flow.
- if !h.getMonitor(r.Context()).GetNotificationManager().IsEnabled() {
+ if !monitor.GetNotificationManager().IsEnabled() {
result["deliveryPaused"] = true
}
}
@@ -1014,7 +1080,10 @@ func (h *NotificationHandlers) TestWebhook(w http.ResponseWriter, r *http.Reques
// GetNotificationHealth returns health status of notification system
func (h *NotificationHandlers) GetNotificationHealth(w http.ResponseWriter, r *http.Request) {
- monitor := h.getMonitor(r.Context())
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
manager := monitor.GetNotificationManager()
failureClasses := notifications.NotificationFailureClassCounts{}.AsMap()
@@ -1103,7 +1172,11 @@ func (h *NotificationHandlers) GetNotificationHealth(w http.ResponseWriter, r *h
// queue's retention windows, so the payload names the window rather than
// presenting itself as lifetime history.
func (h *NotificationHandlers) GetDeliveryLog(w http.ResponseWriter, r *http.Request) {
- manager := h.getMonitor(r.Context()).GetNotificationManager()
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ manager := monitor.GetNotificationManager()
limit := 0
if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" {
diff --git a/internal/api/alerting/tenant_monitor_fail_closed_test.go b/internal/api/alerting/tenant_monitor_fail_closed_test.go
new file mode 100644
index 0000000000..2112c87634
--- /dev/null
+++ b/internal/api/alerting/tenant_monitor_fail_closed_test.go
@@ -0,0 +1,101 @@
+package alerting
+
+import (
+ "bytes"
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext"
+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
+ "github.com/stretchr/testify/assert"
+)
+
+// An empty tenant manager has no persistence, so every organization lookup
+// it is asked for fails. The default-org mocks carry no expectations: any
+// call that falls back to them fails the test.
+
+func tenantRequest(method, target, orgID string) *http.Request {
+ req := httptest.NewRequest(method, target, bytes.NewReader([]byte(`{}`)))
+ return req.WithContext(context.WithValue(req.Context(), apicontext.OrgIDContextKey, orgID))
+}
+
+func TestAlertHandlersGetMonitor_NonDefaultOrgFailsClosed(t *testing.T) {
+ defaultMonitor := new(MockAlertMonitor)
+ tenantCtx := context.WithValue(context.Background(), apicontext.OrgIDContextKey, "tenant-a")
+
+ h := NewAlertHandlers(&monitoring.MultiTenantMonitor{}, defaultMonitor, nil)
+ assert.Nil(t, h.getMonitor(tenantCtx))
+ assert.Nil(t, h.MonitorForContext(tenantCtx))
+ assert.Same(t, defaultMonitor, h.getMonitor(context.Background()))
+
+ noManager := NewAlertHandlers(nil, defaultMonitor, nil)
+ assert.Nil(t, noManager.getMonitor(tenantCtx))
+ assert.Same(t, defaultMonitor, noManager.getMonitor(context.Background()))
+}
+
+func TestAlertHandlers_UnresolvableTenantReturns503(t *testing.T) {
+ defaultMonitor := new(MockAlertMonitor)
+ h := NewAlertHandlers(&monitoring.MultiTenantMonitor{}, defaultMonitor, nil)
+
+ cases := []struct {
+ name string
+ method string
+ target string
+ handler http.HandlerFunc
+ }{
+ {"get config", http.MethodGet, "/api/alerts/config", h.GetAlertConfig},
+ {"update config", http.MethodPut, "/api/alerts/config", h.UpdateAlertConfig},
+ {"active alerts", http.MethodGet, "/api/alerts/active", h.GetActiveAlerts},
+ {"clear history", http.MethodDelete, "/api/alerts/history", h.ClearAlertHistory},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ rec := httptest.NewRecorder()
+ tc.handler(rec, tenantRequest(tc.method, tc.target, "tenant-a"))
+ assert.Equal(t, http.StatusServiceUnavailable, rec.Code, rec.Body.String())
+ })
+ }
+ defaultMonitor.AssertExpectations(t)
+}
+
+func TestNotificationHandlersGetMonitor_NonDefaultOrgFailsClosed(t *testing.T) {
+ defaultMonitor := new(MockNotificationMonitor)
+ tenantCtx := context.WithValue(context.Background(), apicontext.OrgIDContextKey, "tenant-a")
+
+ h := NewNotificationHandlers(&monitoring.MultiTenantMonitor{}, defaultMonitor)
+ assert.Nil(t, h.getMonitor(tenantCtx))
+ assert.Nil(t, h.MonitorForContext(tenantCtx))
+ assert.Same(t, defaultMonitor, h.getMonitor(context.Background()))
+
+ noManager := NewNotificationHandlers(nil, defaultMonitor)
+ assert.Nil(t, noManager.getMonitor(tenantCtx))
+ assert.Same(t, defaultMonitor, noManager.getMonitor(context.Background()))
+}
+
+func TestNotificationHandlers_UnresolvableTenantReturns503(t *testing.T) {
+ defaultMonitor := new(MockNotificationMonitor)
+ h := NewNotificationHandlers(&monitoring.MultiTenantMonitor{}, defaultMonitor)
+
+ cases := []struct {
+ name string
+ method string
+ target string
+ handler http.HandlerFunc
+ }{
+ {"get email", http.MethodGet, "/api/notifications/email", h.GetEmailConfig},
+ {"update email", http.MethodPut, "/api/notifications/email", h.UpdateEmailConfig},
+ {"get webhooks", http.MethodGet, "/api/notifications/webhooks", h.GetWebhooks},
+ {"delete webhook", http.MethodDelete, "/api/notifications/webhooks/wh-1", h.DeleteWebhook},
+ {"health", http.MethodGet, "/api/notifications/health", h.GetNotificationHealth},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ rec := httptest.NewRecorder()
+ tc.handler(rec, tenantRequest(tc.method, tc.target, "tenant-a"))
+ assert.Equal(t, http.StatusServiceUnavailable, rec.Code, rec.Body.String())
+ })
+ }
+ defaultMonitor.AssertExpectations(t)
+}
diff --git a/internal/api/docker_agents.go b/internal/api/docker_agents.go
index 114cba6e37..29c8224f7d 100644
--- a/internal/api/docker_agents.go
+++ b/internal/api/docker_agents.go
@@ -1,7 +1,6 @@
package api
import (
- "context"
"encoding/json"
"errors"
"fmt"
@@ -131,7 +130,11 @@ func (h *DockerAgentHandlers) HandleReport(w http.ResponseWriter, r *http.Reques
}
tokenRecord := getAPITokenRecordFromRequest(r)
- host, err := h.getMonitor(r.Context()).ApplyDockerReport(report, tokenRecord)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ host, err := monitor.ApplyDockerReport(report, tokenRecord)
if err != nil {
writeErrorResponse(w, http.StatusBadRequest, "invalid_report", err.Error(), nil)
return
@@ -152,7 +155,7 @@ func (h *DockerAgentHandlers) HandleReport(w http.ResponseWriter, r *http.Reques
"lastSeen": host.LastSeen,
}
- if payload, cmd := h.getMonitor(r.Context()).FetchDockerCommandForHost(host.ID); cmd != nil {
+ if payload, cmd := monitor.FetchDockerCommandForHost(host.ID); cmd != nil {
commandResponse := map[string]any{
"id": cmd.ID,
"type": cmd.Type,
@@ -271,14 +274,17 @@ func (h *DockerAgentHandlers) HandleCommandAck(w http.ResponseWriter, r *http.Re
return
}
- mon := h.getMonitor(r.Context())
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
agentID := strings.TrimSpace(req.AgentID)
if agentID == "" {
writeErrorResponse(w, http.StatusBadRequest, "missing_agent_id", "Agent ID is required", nil)
return
}
- commandStatus, agentID, shouldRemove, err := mon.AcknowledgeDockerHostCommand(commandID, agentID, status, req.Message)
+ commandStatus, agentID, shouldRemove, err := monitor.AcknowledgeDockerHostCommand(commandID, agentID, status, req.Message)
if err != nil {
writeErrorResponse(w, http.StatusBadRequest, "docker_command_ack_failed", err.Error(), nil)
return
@@ -292,7 +298,7 @@ func (h *DockerAgentHandlers) HandleCommandAck(w http.ResponseWriter, r *http.Re
oldContainerID = strings.TrimSpace(oldContainerID)
newContainerID = strings.TrimSpace(newContainerID)
if oldContainerID != "" && newContainerID != "" && oldContainerID != newContainerID {
- if err := mon.CopyDockerContainerMetadata(agentID, oldContainerID, newContainerID); err != nil {
+ if err := monitor.CopyDockerContainerMetadata(agentID, oldContainerID, newContainerID); err != nil {
log.Warn().
Err(err).
Str("dockerAgentID", agentID).
@@ -304,12 +310,12 @@ func (h *DockerAgentHandlers) HandleCommandAck(w http.ResponseWriter, r *http.Re
}
if shouldRemove {
- if _, removeErr := mon.RemoveDockerHost(agentID); removeErr != nil {
+ if _, removeErr := monitor.RemoveDockerHost(agentID); removeErr != nil {
log.Error().Err(removeErr).Str("dockerAgentID", agentID).Str("commandID", commandID).Msg("Failed to remove docker host after command completion")
} else {
// Clear the removal block since the agent has confirmed it stopped successfully.
// This allows immediate re-enrollment without waiting for the 24-hour TTL.
- if reenrollErr := mon.AllowDockerHostReenroll(agentID); reenrollErr != nil {
+ if reenrollErr := monitor.AllowDockerHostReenroll(agentID); reenrollErr != nil {
log.Warn().Err(reenrollErr).Str("dockerAgentID", agentID).Msg("Failed to clear removal block after successful stop")
}
}
@@ -346,14 +352,18 @@ func (h *DockerAgentHandlers) HandleDeleteHost(w http.ResponseWriter, r *http.Re
forceParam := strings.ToLower(r.URL.Query().Get("force"))
force := forceParam == "true" || strings.ToLower(r.URL.Query().Get("mode")) == "force"
- priorHost, hostExists := h.getMonitor(r.Context()).GetDockerHost(agentID)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ priorHost, hostExists := monitor.GetDockerHost(agentID)
if shouldHide {
if !hostExists {
writeErrorResponse(w, http.StatusNotFound, "docker_agent_not_found", "Docker / Podman module not found", nil)
return
}
- host, err := h.getMonitor(r.Context()).HideDockerHost(agentID)
+ host, err := monitor.HideDockerHost(agentID)
if err != nil {
writeErrorResponse(w, http.StatusNotFound, "docker_agent_not_found", err.Error(), nil)
return
@@ -392,7 +402,7 @@ func (h *DockerAgentHandlers) HandleDeleteHost(w http.ResponseWriter, r *http.Re
return
}
- host, err := h.getMonitor(r.Context()).RemoveDockerHost(agentID)
+ host, err := monitor.RemoveDockerHost(agentID)
if err != nil {
writeErrorResponse(w, http.StatusNotFound, "docker_agent_not_found", err.Error(), nil)
return
@@ -422,7 +432,11 @@ func (h *DockerAgentHandlers) HandleAllowReenroll(w http.ResponseWriter, r *http
return
}
- if err := h.getMonitor(r.Context()).AllowDockerHostReenroll(agentID); err != nil {
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ if err := monitor.AllowDockerHostReenroll(agentID); err != nil {
writeErrorResponse(w, http.StatusBadRequest, "docker_agent_reenroll_failed", err.Error(), nil)
return
}
@@ -447,7 +461,7 @@ func (h *DockerAgentHandlers) handleHostLifecycleAction(
pathSuffix string,
successMsg string,
logMsg string,
- action func(ctx context.Context, agentID string) (string, error),
+ action func(monitor *monitoring.Monitor, agentID string) (string, error),
) {
if r.Method != http.MethodPut {
writeErrorResponse(w, http.StatusMethodNotAllowed, "method_not_allowed", "Only PUT is allowed", nil)
@@ -460,7 +474,11 @@ func (h *DockerAgentHandlers) handleHostLifecycleAction(
return
}
- hostID, err := action(r.Context(), agentID)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ hostID, err := action(monitor, agentID)
if err != nil {
writeErrorResponse(w, http.StatusNotFound, "docker_agent_not_found", err.Error(), nil)
return
@@ -480,8 +498,8 @@ func (h *DockerAgentHandlers) handleHostLifecycleAction(
// HandleUnhideHost unhides a previously hidden Docker / Podman host.
func (h *DockerAgentHandlers) HandleUnhideHost(w http.ResponseWriter, r *http.Request) {
h.handleHostLifecycleAction(w, r, "/unhide", "Docker / Podman module unhidden", "Failed to serialize docker host unhide response",
- func(ctx context.Context, agentID string) (string, error) {
- host, err := h.getMonitor(ctx).UnhideDockerHost(agentID)
+ func(monitor *monitoring.Monitor, agentID string) (string, error) {
+ host, err := monitor.UnhideDockerHost(agentID)
if err != nil {
return "", err
}
@@ -492,8 +510,8 @@ func (h *DockerAgentHandlers) HandleUnhideHost(w http.ResponseWriter, r *http.Re
// HandleMarkPendingUninstall marks a Docker / Podman host as pending uninstall.
func (h *DockerAgentHandlers) HandleMarkPendingUninstall(w http.ResponseWriter, r *http.Request) {
h.handleHostLifecycleAction(w, r, "/pending-uninstall", "Docker / Podman module marked as pending uninstall", "Failed to serialize docker host pending uninstall response",
- func(ctx context.Context, agentID string) (string, error) {
- host, err := h.getMonitor(ctx).MarkDockerHostPendingUninstall(agentID)
+ func(monitor *monitoring.Monitor, agentID string) (string, error) {
+ host, err := monitor.MarkDockerHostPendingUninstall(agentID)
if err != nil {
return "", err
}
@@ -528,7 +546,11 @@ func (h *DockerAgentHandlers) HandleSetCustomDisplayName(w http.ResponseWriter,
customName := strings.TrimSpace(req.DisplayName)
- host, err := h.getMonitor(r.Context()).SetDockerHostCustomDisplayName(agentID, customName)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ host, err := monitor.SetDockerHostCustomDisplayName(agentID, customName)
if err != nil {
writeErrorResponse(w, http.StatusNotFound, "docker_agent_not_found", err.Error(), nil)
return
@@ -612,7 +634,11 @@ func (h *DockerAgentHandlers) HandleCheckUpdates(w http.ResponseWriter, r *http.
}
// Queue the check updates command
- commandStatus, err := h.getMonitor(r.Context()).QueueDockerCheckUpdatesCommand(agentID)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ commandStatus, err := monitor.QueueDockerCheckUpdatesCommand(agentID)
if err != nil {
writeErrorResponse(w, http.StatusBadRequest, "check_updates_command_failed", err.Error(), nil)
return
diff --git a/internal/api/kubernetes_agents.go b/internal/api/kubernetes_agents.go
index f439fe7e61..ee518fbafb 100644
--- a/internal/api/kubernetes_agents.go
+++ b/internal/api/kubernetes_agents.go
@@ -52,7 +52,11 @@ func (h *KubernetesAgentHandlers) HandleReport(w http.ResponseWriter, r *http.Re
}
tokenRecord := getAPITokenRecordFromRequest(r)
- cluster, err := h.getMonitor(r.Context()).ApplyKubernetesReport(report, tokenRecord)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ cluster, err := monitor.ApplyKubernetesReport(report, tokenRecord)
if err != nil {
writeErrorResponse(w, http.StatusBadRequest, "invalid_report", err.Error(), nil)
return
@@ -138,7 +142,11 @@ func (h *KubernetesAgentHandlers) HandleDeleteCluster(w http.ResponseWriter, r *
return
}
- cluster, err := h.getMonitor(r.Context()).RemoveKubernetesCluster(clusterID)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ cluster, err := monitor.RemoveKubernetesCluster(clusterID)
if err != nil {
writeErrorResponse(w, http.StatusNotFound, "k8s_cluster_not_found", err.Error(), nil)
return
@@ -170,7 +178,11 @@ func (h *KubernetesAgentHandlers) HandleAllowReenroll(w http.ResponseWriter, r *
return
}
- if err := h.getMonitor(r.Context()).AllowKubernetesClusterReenroll(clusterID); err != nil {
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ if err := monitor.AllowKubernetesClusterReenroll(clusterID); err != nil {
writeErrorResponse(w, http.StatusBadRequest, "k8s_cluster_reenroll_failed", err.Error(), nil)
return
}
@@ -192,7 +204,7 @@ func (h *KubernetesAgentHandlers) handleClusterLifecycleAction(
pathSuffix string,
successMsg string,
logMsg string,
- action func(ctx context.Context, clusterID string) (string, error),
+ action func(monitor *monitoring.Monitor, clusterID string) (string, error),
) {
if r.Method != http.MethodPut {
writeErrorResponse(w, http.StatusMethodNotAllowed, "method_not_allowed", "Only PUT is allowed", nil)
@@ -207,7 +219,11 @@ func (h *KubernetesAgentHandlers) handleClusterLifecycleAction(
return
}
- canonicalID, err := action(r.Context(), clusterID)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ canonicalID, err := action(monitor, clusterID)
if err != nil {
writeErrorResponse(w, http.StatusNotFound, "k8s_cluster_not_found", err.Error(), nil)
return
@@ -227,8 +243,8 @@ func (h *KubernetesAgentHandlers) handleClusterLifecycleAction(
// HandleUnhideCluster unhides a previously hidden kubernetes cluster.
func (h *KubernetesAgentHandlers) HandleUnhideCluster(w http.ResponseWriter, r *http.Request) {
h.handleClusterLifecycleAction(w, r, "/unhide", "Kubernetes cluster unhidden", "Failed to serialize kubernetes cluster unhide response",
- func(ctx context.Context, clusterID string) (string, error) {
- cluster, err := h.getMonitor(ctx).UnhideKubernetesCluster(clusterID)
+ func(monitor *monitoring.Monitor, clusterID string) (string, error) {
+ cluster, err := monitor.UnhideKubernetesCluster(clusterID)
if err != nil {
return "", err
}
@@ -239,8 +255,8 @@ func (h *KubernetesAgentHandlers) HandleUnhideCluster(w http.ResponseWriter, r *
// HandleMarkPendingUninstall marks a cluster as pending uninstall.
func (h *KubernetesAgentHandlers) HandleMarkPendingUninstall(w http.ResponseWriter, r *http.Request) {
h.handleClusterLifecycleAction(w, r, "/pending-uninstall", "Kubernetes cluster marked as pending uninstall", "Failed to serialize kubernetes cluster pending uninstall response",
- func(ctx context.Context, clusterID string) (string, error) {
- cluster, err := h.getMonitor(ctx).MarkKubernetesClusterPendingUninstall(clusterID)
+ func(monitor *monitoring.Monitor, clusterID string) (string, error) {
+ cluster, err := monitor.MarkKubernetesClusterPendingUninstall(clusterID)
if err != nil {
return "", err
}
@@ -277,7 +293,11 @@ func (h *KubernetesAgentHandlers) HandleSetCustomDisplayName(w http.ResponseWrit
customName := strings.TrimSpace(req.DisplayName)
- cluster, err := h.getMonitor(r.Context()).SetKubernetesClusterCustomDisplayName(clusterID, customName)
+ monitor := h.requireMonitor(w, r)
+ if monitor == nil {
+ return
+ }
+ cluster, err := monitor.SetKubernetesClusterCustomDisplayName(clusterID, customName)
if err != nil {
writeErrorResponse(w, http.StatusNotFound, "k8s_cluster_not_found", err.Error(), nil)
return
diff --git a/internal/api/system_settings.go b/internal/api/system_settings.go
index c5951c71c1..ebff08f982 100644
--- a/internal/api/system_settings.go
+++ b/internal/api/system_settings.go
@@ -206,13 +206,39 @@ func (h *SystemSettingsHandler) getMonitor(ctx context.Context) SystemSettingsMo
defaultMonitor := h.defaultMonitor
h.stateMu.RUnlock()
- if mtMonitor != nil {
- orgID := GetOrgID(ctx)
- if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
- return m
+ orgID := strings.TrimSpace(GetOrgID(ctx))
+ if orgID == "" || orgID == "default" {
+ if mtMonitor != nil {
+ if m, err := mtMonitor.GetMonitor(orgID); err == nil && m != nil {
+ return m
+ }
}
+ return defaultMonitor
+ }
+
+ // Never fall back to the default organization's runtime for another
+ // organization: its requests would read or change the wrong tenant.
+ if mtMonitor == nil {
+ log.Warn().Str("org_id", orgID).Msg("Tenant monitor manager unavailable for system settings request")
+ return nil
+ }
+ m, err := mtMonitor.GetMonitor(orgID)
+ if err != nil || m == nil {
+ log.Warn().Err(err).Str("org_id", orgID).Msg("Failed to resolve tenant monitor for system settings request")
+ return nil
+ }
+ return m
+}
+
+// requireMonitor resolves the request organization's monitor, writing a 503
+// and returning nil when it is unavailable.
+func (h *SystemSettingsHandler) requireMonitor(w http.ResponseWriter, r *http.Request) SystemSettingsMonitor {
+ monitor := h.getMonitor(r.Context())
+ if monitor == nil {
+ writeErrorResponse(w, http.StatusServiceUnavailable, "tenant_unavailable", "Tenant monitor is not available", nil)
+ return nil
}
- return defaultMonitor
+ return monitor
}
// SetConfig updates the configuration reference used by the handler.
diff --git a/internal/api/system_settings_handlers_test.go b/internal/api/system_settings_handlers_test.go
index f9acf98b2a..cf2ec7c6ed 100644
--- a/internal/api/system_settings_handlers_test.go
+++ b/internal/api/system_settings_handlers_test.go
@@ -619,7 +619,7 @@ func TestSystemSettingsHandler_GetMonitor_UsesTenantMonitorInterface(t *testing.
}
}
-func TestSystemSettingsHandler_GetMonitor_FallsBackToLegacyOnTenantError(t *testing.T) {
+func TestSystemSettingsHandler_GetMonitor_TenantErrorFailsClosed(t *testing.T) {
defaultMonitor := &mockMonitor{}
provider := &mockTenantMonitorProvider{err: errors.New("boom")}
@@ -629,8 +629,8 @@ func TestSystemSettingsHandler_GetMonitor_FallsBackToLegacyOnTenantError(t *test
}
got := handler.getMonitor(context.WithValue(context.Background(), OrgIDContextKey, "acme"))
- if got != defaultMonitor {
- t.Fatalf("expected legacy monitor fallback, got %#v", got)
+ if got != nil {
+ t.Fatalf("expected nil monitor for unresolvable tenant, got %#v", got)
}
if provider.orgID != "acme" {
t.Fatalf("expected tenant org lookup for acme, got %q", provider.orgID)
diff --git a/internal/api/tenant_monitor_fail_closed_test.go b/internal/api/tenant_monitor_fail_closed_test.go
new file mode 100644
index 0000000000..819197de44
--- /dev/null
+++ b/internal/api/tenant_monitor_fail_closed_test.go
@@ -0,0 +1,116 @@
+package api
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
+)
+
+// unresolvableTenantMonitor returns a tenant manager with no persistence, so
+// every organization lookup it is asked for fails.
+func unresolvableTenantMonitor() *monitoring.MultiTenantMonitor {
+ return &monitoring.MultiTenantMonitor{}
+}
+
+func tenantContext(orgID string) context.Context {
+ return context.WithValue(context.Background(), OrgIDContextKey, orgID)
+}
+
+func TestAgentHandlersGetMonitor_NonDefaultOrgFailsClosed(t *testing.T) {
+ defaultMonitor := &monitoring.Monitor{}
+
+ t.Run("tenant resolution error", func(t *testing.T) {
+ base := newBaseAgentHandlers(unresolvableTenantMonitor(), defaultMonitor, nil)
+ if got := base.getMonitor(tenantContext("tenant-a")); got != nil {
+ t.Fatalf("expected nil monitor for unresolvable tenant, got default-org fallback")
+ }
+ })
+
+ t.Run("no tenant manager", func(t *testing.T) {
+ base := newBaseAgentHandlers(nil, defaultMonitor, nil)
+ if got := base.getMonitor(tenantContext("tenant-a")); got != nil {
+ t.Fatalf("expected nil monitor without a tenant manager, got default-org fallback")
+ }
+ })
+
+ t.Run("default org keeps primary monitor", func(t *testing.T) {
+ base := newBaseAgentHandlers(unresolvableTenantMonitor(), defaultMonitor, nil)
+ for _, ctx := range []context.Context{context.Background(), tenantContext("default"), tenantContext("")} {
+ if got := base.getMonitor(ctx); got != defaultMonitor {
+ t.Fatalf("expected default monitor for default org, got %p", got)
+ }
+ }
+ })
+}
+
+func TestAgentHandlers_UnresolvableTenantReturns503(t *testing.T) {
+ defaultMonitor := &monitoring.Monitor{}
+
+ cases := []struct {
+ name string
+ path string
+ handler http.HandlerFunc
+ }{
+ {
+ name: "unified agent",
+ path: agentRoutePrefix + "agent-1/allow-reenroll",
+ handler: NewUnifiedAgentHandlers(unresolvableTenantMonitor(), defaultMonitor, nil).HandleAllowReenroll,
+ },
+ {
+ name: "docker runtime",
+ path: "/api/agents/docker/runtimes/host-1/allow-reenroll",
+ handler: NewDockerAgentHandlers(unresolvableTenantMonitor(), defaultMonitor, nil, nil).HandleAllowReenroll,
+ },
+ {
+ name: "kubernetes cluster",
+ path: "/api/agents/kubernetes/clusters/cluster-1/allow-reenroll",
+ handler: NewKubernetesAgentHandlers(unresolvableTenantMonitor(), defaultMonitor, nil).HandleAllowReenroll,
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ req := httptest.NewRequest(http.MethodPost, tc.path, nil).WithContext(tenantContext("tenant-a"))
+ rec := httptest.NewRecorder()
+ tc.handler(rec, req)
+
+ if rec.Code != http.StatusServiceUnavailable {
+ t.Fatalf("expected 503, got %d: %s", rec.Code, rec.Body.String())
+ }
+ var body struct {
+ Code string `json:"code"`
+ }
+ if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
+ t.Fatalf("decode error body: %v", err)
+ }
+ if body.Code != "tenant_unavailable" {
+ t.Fatalf("expected tenant_unavailable, got %q", body.Code)
+ }
+ })
+ }
+}
+
+func TestDockerAgentHandlers_LifecycleActionUnresolvableTenantReturns503(t *testing.T) {
+ handler := NewDockerAgentHandlers(unresolvableTenantMonitor(), &monitoring.Monitor{}, nil, nil)
+ req := httptest.NewRequest(http.MethodPut, "/api/agents/docker/runtimes/host-1/unhide", nil).
+ WithContext(tenantContext("tenant-a"))
+ rec := httptest.NewRecorder()
+
+ handler.HandleUnhideHost(rec, req)
+
+ if rec.Code != http.StatusServiceUnavailable {
+ t.Fatalf("expected 503, got %d: %s", rec.Code, rec.Body.String())
+ }
+}
+
+func TestSystemSettingsHandler_GetMonitor_NoTenantManagerFailsClosed(t *testing.T) {
+ handler := &SystemSettingsHandler{defaultMonitor: &mockMonitor{}}
+
+ if got := handler.getMonitor(tenantContext("acme")); got != nil {
+ t.Fatalf("expected nil monitor without a tenant manager, got %#v", got)
+ }
+}