Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Resolve the request organization for infra updates

rcourtman0bba6c94bbba
diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md
index 7dde0dabc9..22be03f5d2 100644
--- a/docs/release-control/v6/internal/subsystems/api-contracts.md
+++ b/docs/release-control/v6/internal/subsystems/api-contracts.md
@@ -3320,6 +3320,10 @@ a new API state machine, queue contract, or verification-accounting field.
projection and configuration export immediately. Assistant URL updates must
resolve a provider per organization and refresh existing services after a
monitor replacement; a default provider must not be shared across tenants.
+ Infrastructure update detection (`internal/api/update_detection.go`) and
+ scheduler health resolve the request organization's monitor and read state
+ on every request; an organization whose monitor cannot be resolved gets no
+ updates and no queued update check, never the default organization's.
Deploy
preflights and jobs share `handleDeployJobStatus` /
`handleDeployJobEvents` in `internal/api/deploy_handlers.go`; recovery
diff --git a/internal/api/router.go b/internal/api/router.go
index ba33c16d7a..b16874978a 100644
--- a/internal/api/router.go
+++ b/internal/api/router.go
@@ -662,7 +662,7 @@ func (r *Router) setupRoutes() {
r.config.DataPath,
)
stripeWebhookHandlers := r.stripeWebhookHandlers
- infraUpdateHandlers := NewUpdateDetectionHandlers(r.monitor, r.defaultReadState())
+ infraUpdateHandlers := NewTenantUpdateDetectionHandlers(r.tenantMonitorAndReadState)
auditHandlers := NewAuditHandlers()
auditHandlers.SetResourceStoreProvider(r.resourceHandlers.getStore)
@@ -1701,6 +1701,20 @@ func (r *Router) configureProxmoxGuestDockerDetection(m *monitoring.Monitor) {
}
}
+// tenantMonitorAndReadState resolves the request organization's monitor and
+// read state. A non-default organization whose monitor cannot be resolved gets
+// neither, never the default organization's.
+func (r *Router) tenantMonitorAndReadState(ctx context.Context) (*monitoring.Monitor, unifiedresources.ReadState) {
+ if orgID := GetOrgID(ctx); orgID == "" || orgID == "default" {
+ return r.monitor, r.defaultReadState()
+ }
+ monitor := r.getTenantMonitor(ctx)
+ if monitor == nil {
+ return nil, nil
+ }
+ return monitor, monitor.GetUnifiedReadState()
+}
+
func (r *Router) defaultReadState() unifiedresources.ReadState {
if r == nil {
return nil
@@ -5084,12 +5098,13 @@ func (r *Router) handleSchedulerHealth(w http.ResponseWriter, req *http.Request)
return
}
- if r.monitor == nil {
+ monitor := r.getTenantMonitor(req.Context())
+ if monitor == nil {
http.Error(w, "Monitor not available", http.StatusServiceUnavailable)
return
}
- health := r.monitor.SchedulerHealth()
+ health := monitor.SchedulerHealth()
if err := utils.WriteJSONResponse(w, health); err != nil {
log.Error().Err(err).Msg("Failed to write scheduler health response")
}
diff --git a/internal/api/update_detection.go b/internal/api/update_detection.go
index aff4362c06..c828e0142f 100644
--- a/internal/api/update_detection.go
+++ b/internal/api/update_detection.go
@@ -1,6 +1,7 @@
package api
import (
+ "context"
"encoding/json"
"net/http"
"strings"
@@ -16,13 +17,30 @@ import (
type UpdateDetectionHandlers struct {
monitor *monitoring.Monitor
readState unifiedresources.ReadState
+ // tenant resolves the request organization's monitor and read state. The
+ // router sets it so an organization never reads or queues commands against
+ // another organization's agents; the fixed fields serve a single monitor.
+ tenant func(context.Context) (*monitoring.Monitor, unifiedresources.ReadState)
}
-// NewUpdateDetectionHandlers creates a new update detection handlers group.
+// NewUpdateDetectionHandlers creates handlers bound to one monitor.
func NewUpdateDetectionHandlers(monitor *monitoring.Monitor, readState unifiedresources.ReadState) *UpdateDetectionHandlers {
return &UpdateDetectionHandlers{monitor: monitor, readState: readState}
}
+// NewTenantUpdateDetectionHandlers creates handlers that resolve the monitor
+// and read state of each request's organization.
+func NewTenantUpdateDetectionHandlers(resolve func(context.Context) (*monitoring.Monitor, unifiedresources.ReadState)) *UpdateDetectionHandlers {
+ return &UpdateDetectionHandlers{tenant: resolve}
+}
+
+func (h *UpdateDetectionHandlers) sources(r *http.Request) (*monitoring.Monitor, unifiedresources.ReadState) {
+ if h.tenant != nil {
+ return h.tenant(r.Context())
+ }
+ return h.monitor, h.readState
+}
+
// ContainerUpdateInfo represents a container with an available update
type ContainerUpdateInfo struct {
AgentID string `json:"agentId"`
@@ -116,7 +134,8 @@ func (h *UpdateDetectionHandlers) HandleGetInfraUpdates(w http.ResponseWriter, r
return
}
- if h.monitor == nil {
+ monitor, readState := h.sources(r)
+ if monitor == nil {
if err := utils.WriteJSONResponse(w, emptyInfraUpdatesResponse()); err != nil {
log.Error().Err(err).Msg("Failed to serialize empty updates response")
}
@@ -129,7 +148,7 @@ func (h *UpdateDetectionHandlers) HandleGetInfraUpdates(w http.ResponseWriter, r
resourceTypeFilter := strings.ToLower(query.Get("resourceType"))
// Collect updates from Docker / Podman modules.
- updates := h.collectDockerUpdates(agentIDFilter)
+ updates := collectDockerUpdates(readState, agentIDFilter)
// Filter by resource type if specified
if resourceTypeFilter != "" && resourceTypeFilter != string(infraUpdateResourceTypeDocker) {
@@ -153,13 +172,14 @@ func (h *UpdateDetectionHandlers) HandleGetInfraUpdateForResource(w http.Respons
return
}
- if h.monitor == nil {
+ monitor, readState := h.sources(r)
+ if monitor == nil {
writeErrorResponse(w, http.StatusNotFound, "not_found", "No update found for resource", nil)
return
}
// ResourceID format: docker:<agentId>/<containerId>
- updates := h.collectDockerUpdates("")
+ updates := collectDockerUpdates(readState, "")
for _, update := range updates {
dockerResourceID := "docker:" + update.AgentID + "/" + update.ContainerID
if dockerResourceID == resourceID || update.ContainerID == resourceID {
@@ -181,14 +201,15 @@ func (h *UpdateDetectionHandlers) HandleGetInfraUpdatesSummary(w http.ResponseWr
return
}
- if h.monitor == nil {
+ monitor, readState := h.sources(r)
+ if monitor == nil {
if err := utils.WriteJSONResponse(w, emptyInfraUpdatesSummaryResponse()); err != nil {
log.Error().Err(err).Msg("Failed to serialize empty summary response")
}
return
}
- updates := h.collectDockerUpdates("")
+ updates := collectDockerUpdates(readState, "")
// Aggregate by agent
summaries := make(map[string]infraUpdateAgentSummary)
@@ -225,7 +246,8 @@ func (h *UpdateDetectionHandlers) HandleTriggerInfraUpdateCheck(w http.ResponseW
return
}
- if h.monitor == nil {
+ monitor, readState := h.sources(r)
+ if monitor == nil {
writeErrorResponse(w, http.StatusServiceUnavailable, "service_unavailable", "Update detection not available", nil)
return
}
@@ -245,7 +267,7 @@ func (h *UpdateDetectionHandlers) HandleTriggerInfraUpdateCheck(w http.ResponseW
// Handle agent-level check
if req.AgentID != "" {
- commandStatus, err := h.monitor.QueueDockerCheckUpdatesCommand(req.AgentID)
+ commandStatus, err := monitor.QueueDockerCheckUpdatesCommand(req.AgentID)
if err != nil {
writeErrorResponse(w, http.StatusBadRequest, "check_updates_failed", err.Error(), nil)
return
@@ -265,7 +287,7 @@ func (h *UpdateDetectionHandlers) HandleTriggerInfraUpdateCheck(w http.ResponseW
// Handle resource-level check (currently we just check the whole agent)
if req.ResourceID != "" {
// Try to find which agent this resource belongs to
- updates := h.collectDockerUpdates("")
+ updates := collectDockerUpdates(readState, "")
var agentID string
for _, update := range updates {
if update.ContainerID == req.ResourceID || ("docker:"+update.AgentID+"/"+update.ContainerID) == req.ResourceID {
@@ -279,7 +301,7 @@ func (h *UpdateDetectionHandlers) HandleTriggerInfraUpdateCheck(w http.ResponseW
return
}
- commandStatus, err := h.monitor.QueueDockerCheckUpdatesCommand(agentID)
+ commandStatus, err := monitor.QueueDockerCheckUpdatesCommand(agentID)
if err != nil {
writeErrorResponse(w, http.StatusBadRequest, "check_updates_failed", err.Error(), nil)
return
@@ -307,7 +329,8 @@ func (h *UpdateDetectionHandlers) HandleGetInfraUpdatesForAgent(w http.ResponseW
return
}
- if h.monitor == nil {
+ monitor, readState := h.sources(r)
+ if monitor == nil {
response := emptyInfraUpdatesForAgentResponse()
response.AgentID = agentID
if err := utils.WriteJSONResponse(w, response.normalizeCollections()); err != nil {
@@ -316,7 +339,7 @@ func (h *UpdateDetectionHandlers) HandleGetInfraUpdatesForAgent(w http.ResponseW
return
}
- updates := h.collectDockerUpdates(agentID)
+ updates := collectDockerUpdates(readState, agentID)
response := emptyInfraUpdatesForAgentResponse()
response.Updates = updates
@@ -329,19 +352,19 @@ func (h *UpdateDetectionHandlers) HandleGetInfraUpdatesForAgent(w http.ResponseW
}
// collectDockerUpdates gathers update information from Docker / Podman modules via ReadState.
-func (h *UpdateDetectionHandlers) collectDockerUpdates(agentIDFilter string) []ContainerUpdateInfo {
- if h.readState == nil {
+func collectDockerUpdates(readState unifiedresources.ReadState, agentIDFilter string) []ContainerUpdateInfo {
+ if readState == nil {
return nil
}
// Build agent ID -> display name map for lookups.
agentNames := make(map[string]string)
- for _, dh := range h.readState.DockerHosts() {
+ for _, dh := range readState.DockerHosts() {
agentNames[dh.HostSourceID()] = dh.Name()
}
var updates []ContainerUpdateInfo
- for _, c := range h.readState.DockerContainers() {
+ for _, c := range readState.DockerContainers() {
agentID := c.HostSourceID()
// Filter by agent ID if specified.
diff --git a/internal/api/update_detection_tenant_test.go b/internal/api/update_detection_tenant_test.go
new file mode 100644
index 0000000000..ccb98a6dc6
--- /dev/null
+++ b/internal/api/update_detection_tenant_test.go
@@ -0,0 +1,179 @@
+package api
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+)
+
+func dockerUpdateSnapshot(hostID string) models.StateSnapshot {
+ return models.StateSnapshot{
+ DockerHosts: []models.DockerHost{{
+ ID: hostID,
+ DisplayName: hostID,
+ Containers: []models.DockerContainer{{
+ ID: hostID + "-c1",
+ Name: "/web",
+ Image: "nginx:latest",
+ UpdateStatus: &models.DockerContainerUpdateStatus{
+ UpdateAvailable: true,
+ LastChecked: time.Now(),
+ },
+ }},
+ }},
+ }
+}
+
+func dockerUpdateReadState(hostID string) unifiedresources.ReadState {
+ registry := unifiedresources.NewRegistry(nil)
+ registry.IngestSnapshot(dockerUpdateSnapshot(hostID))
+ return registry
+}
+
+func withOrg(req *http.Request, orgID string) *http.Request {
+ return req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, orgID))
+}
+
+func TestTenantUpdateDetectionHandlersServeRequestOrganization(t *testing.T) {
+ defaultMonitor, tenantMonitor := &monitoring.Monitor{}, &monitoring.Monitor{}
+ defaultState, tenantState := dockerUpdateReadState("default-host"), dockerUpdateReadState("acme-host")
+ handler := NewTenantUpdateDetectionHandlers(func(ctx context.Context) (*monitoring.Monitor, unifiedresources.ReadState) {
+ switch GetOrgID(ctx) {
+ case "", "default":
+ return defaultMonitor, defaultState
+ case "acme":
+ return tenantMonitor, tenantState
+ }
+ return nil, nil
+ })
+
+ for orgID, wantHost := range map[string]string{"default": "default-host", "acme": "acme-host"} {
+ rr := httptest.NewRecorder()
+ handler.HandleGetInfraUpdates(rr, withOrg(httptest.NewRequest(http.MethodGet, "/api/infra-updates", nil), orgID))
+ var resp infraUpdatesResponse
+ if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil {
+ t.Fatalf("org %s: decode: %v", orgID, err)
+ }
+ if resp.Total != 1 || resp.Updates[0].AgentID != wantHost {
+ t.Fatalf("org %s: expected only %s updates, got %+v", orgID, wantHost, resp.Updates)
+ }
+ }
+
+ // A known agent of another organization is not found from this one.
+ rr := httptest.NewRecorder()
+ handler.HandleGetInfraUpdateForResource(rr,
+ withOrg(httptest.NewRequest(http.MethodGet, "/api/infra-updates/docker:default-host/default-host-c1", nil), "acme"),
+ "docker:default-host/default-host-c1")
+ if rr.Code != http.StatusNotFound {
+ t.Fatalf("expected default-org resource to be hidden from acme, got %d", rr.Code)
+ }
+}
+
+func TestTenantUpdateDetectionHandlersFailClosedWithoutTenantMonitor(t *testing.T) {
+ handler := NewTenantUpdateDetectionHandlers(func(context.Context) (*monitoring.Monitor, unifiedresources.ReadState) {
+ return nil, nil
+ })
+
+ rr := httptest.NewRecorder()
+ handler.HandleGetInfraUpdatesSummary(rr, withOrg(httptest.NewRequest(http.MethodGet, "/api/infra-updates/summary", nil), "acme"))
+ var summary infraUpdatesSummaryResponse
+ if err := json.NewDecoder(rr.Body).Decode(&summary); err != nil {
+ t.Fatalf("decode: %v", err)
+ }
+ if rr.Code != http.StatusOK || summary.TotalUpdates != 0 {
+ t.Fatalf("expected empty summary, got %d %+v", rr.Code, summary)
+ }
+
+ rr = httptest.NewRecorder()
+ handler.HandleTriggerInfraUpdateCheck(rr, withOrg(httptest.NewRequest(http.MethodPost, "/api/infra-updates/check", strings.NewReader(`{"agentId":"default-host"}`)), "acme"))
+ if rr.Code != http.StatusServiceUnavailable {
+ t.Fatalf("expected no command to be queued without the tenant monitor, got %d", rr.Code)
+ }
+}
+
+func TestRouterTenantMonitorAndReadStateNeverFallsBackToDefault(t *testing.T) {
+ router := &Router{monitor: &monitoring.Monitor{}}
+
+ if monitor, readState := router.tenantMonitorAndReadState(context.WithValue(context.Background(), OrgIDContextKey, "acme")); monitor != nil || readState != nil {
+ t.Fatalf("expected no monitor or read state for an unresolved tenant, got %v %v", monitor, readState)
+ }
+ if monitor, _ := router.tenantMonitorAndReadState(context.WithValue(context.Background(), OrgIDContextKey, "default")); monitor != router.monitor {
+ t.Fatalf("expected the default monitor for the default organization")
+ }
+
+ rr := httptest.NewRecorder()
+ router.handleSchedulerHealth(rr, withOrg(httptest.NewRequest(http.MethodGet, "/api/monitoring/scheduler/health", nil), "acme"))
+ if rr.Code != http.StatusServiceUnavailable {
+ t.Fatalf("expected tenant scheduler health not to report the default monitor, got %d", rr.Code)
+ }
+}
+
+func monitorWithDockerUpdate(t *testing.T, hostID string) *monitoring.Monitor {
+ t.Helper()
+ monitor, _, _ := newTestMonitor(t)
+ adapter := unifiedresources.NewMonitorAdapter(nil)
+ adapter.PopulateFromSnapshot(dockerUpdateSnapshot(hostID))
+ setUnexportedField(t, monitor, "resourceStore", monitoring.ResourceStoreInterface(adapter))
+ return monitor
+}
+
+// The routed check guards the router wiring itself: each organization's
+// infra-update reads come from its own monitor, through the real middleware.
+func TestRoutedInfraUpdatesServeRequestOrganization(t *testing.T) {
+ defer SetMultiTenantEnabled(false)
+ SetMultiTenantEnabled(true)
+ t.Setenv("PULSE_DEV", "true")
+
+ const rawToken = "infra-updates-tenant-token-123.12345678"
+ record, err := config.NewAPITokenRecord(rawToken, "two-org-token", []string{config.ScopeMonitoringRead})
+ if err != nil {
+ t.Fatalf("create api token record: %v", err)
+ }
+ record.OrgIDs = []string{"default", "org-a"}
+
+ dataDir := t.TempDir()
+ cfg := &config.Config{DataPath: dataDir, ConfigPath: dataDir, APITokens: []config.APITokenRecord{*record}}
+ cfg.SortAPITokens()
+ if err := config.NewMultiTenantPersistence(dataDir).SaveOrganization(&models.Organization{ID: "org-a", DisplayName: "ORG-A"}); err != nil {
+ t.Fatalf("save organization: %v", err)
+ }
+
+ router := newMultiTenantRouter(t, cfg)
+ router.monitor = monitorWithDockerUpdate(t, "default-host")
+ mtm := &monitoring.MultiTenantMonitor{}
+ setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{"org-a": monitorWithDockerUpdate(t, "org-a-host")})
+ router.mtMonitor = mtm
+ server := httptest.NewServer(router.Handler())
+ t.Cleanup(server.Close)
+
+ for orgID, wantHost := range map[string]string{"default": "default-host", "org-a": "org-a-host"} {
+ req, err := http.NewRequest(http.MethodGet, server.URL+"/api/infra-updates", nil)
+ if err != nil {
+ t.Fatalf("create request: %v", err)
+ }
+ req.Header.Set("X-Pulse-Org-ID", orgID)
+ req.Header.Set("Authorization", "Bearer "+rawToken)
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("org %s: request failed: %v", orgID, err)
+ }
+ var resp infraUpdatesResponse
+ decodeErr := json.NewDecoder(res.Body).Decode(&resp)
+ res.Body.Close()
+ if res.StatusCode != http.StatusOK || decodeErr != nil {
+ t.Fatalf("org %s: status %d, decode %v", orgID, res.StatusCode, decodeErr)
+ }
+ if resp.Total != 1 || resp.Updates[0].AgentID != wantHost {
+ t.Fatalf("org %s: expected only %s updates, got %+v", orgID, wantHost, resp.Updates)
+ }
+ }
+}