Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Resolve the request organization for infra updates
rcourtman0bba6c94bbba
diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.mdindex 7dde0dabc9..22be03f5d2 100644--- a/docs/release-control/v6/internal/subsystems/api-contracts.md+++ b/docs/release-control/v6/internal/subsystems/api-contracts.md@@ -3320,6 +3320,10 @@ a new API state machine, queue contract, or verification-accounting field.projection and configuration export immediately. Assistant URL updates mustresolve a provider per organization and refresh existing services after amonitor replacement; a default provider must not be shared across tenants.+ Infrastructure update detection (`internal/api/update_detection.go`) and+ scheduler health resolve the request organization's monitor and read state+ on every request; an organization whose monitor cannot be resolved gets no+ updates and no queued update check, never the default organization's.Deploypreflights and jobs share `handleDeployJobStatus` /`handleDeployJobEvents` in `internal/api/deploy_handlers.go`; recoverydiff --git a/internal/api/router.go b/internal/api/router.goindex ba33c16d7a..b16874978a 100644--- a/internal/api/router.go+++ b/internal/api/router.go@@ -662,7 +662,7 @@ func (r *Router) setupRoutes() {r.config.DataPath,)stripeWebhookHandlers := r.stripeWebhookHandlers- infraUpdateHandlers := NewUpdateDetectionHandlers(r.monitor, r.defaultReadState())+ infraUpdateHandlers := NewTenantUpdateDetectionHandlers(r.tenantMonitorAndReadState)auditHandlers := NewAuditHandlers()auditHandlers.SetResourceStoreProvider(r.resourceHandlers.getStore)@@ -1701,6 +1701,20 @@ func (r *Router) configureProxmoxGuestDockerDetection(m *monitoring.Monitor) {}}+// tenantMonitorAndReadState resolves the request organization's monitor and+// read state. A non-default organization whose monitor cannot be resolved gets+// neither, never the default organization's.+func (r *Router) tenantMonitorAndReadState(ctx context.Context) (*monitoring.Monitor, unifiedresources.ReadState) {+ if orgID := GetOrgID(ctx); orgID == "" || orgID == "default" {+ return r.monitor, r.defaultReadState()+ }+ monitor := r.getTenantMonitor(ctx)+ if monitor == nil {+ return nil, nil+ }+ return monitor, monitor.GetUnifiedReadState()+}+func (r *Router) defaultReadState() unifiedresources.ReadState {if r == nil {return nil@@ -5084,12 +5098,13 @@ func (r *Router) handleSchedulerHealth(w http.ResponseWriter, req *http.Request)return}- if r.monitor == nil {+ monitor := r.getTenantMonitor(req.Context())+ if monitor == nil {http.Error(w, "Monitor not available", http.StatusServiceUnavailable)return}- health := r.monitor.SchedulerHealth()+ health := monitor.SchedulerHealth()if err := utils.WriteJSONResponse(w, health); err != nil {log.Error().Err(err).Msg("Failed to write scheduler health response")}diff --git a/internal/api/update_detection.go b/internal/api/update_detection.goindex aff4362c06..c828e0142f 100644--- a/internal/api/update_detection.go+++ b/internal/api/update_detection.go@@ -1,6 +1,7 @@package apiimport (+ "context""encoding/json""net/http""strings"@@ -16,13 +17,30 @@ import (type UpdateDetectionHandlers struct {monitor *monitoring.MonitorreadState unifiedresources.ReadState+ // tenant resolves the request organization's monitor and read state. The+ // router sets it so an organization never reads or queues commands against+ // another organization's agents; the fixed fields serve a single monitor.+ tenant func(context.Context) (*monitoring.Monitor, unifiedresources.ReadState)}-// NewUpdateDetectionHandlers creates a new update detection handlers group.+// NewUpdateDetectionHandlers creates handlers bound to one monitor.func NewUpdateDetectionHandlers(monitor *monitoring.Monitor, readState unifiedresources.ReadState) *UpdateDetectionHandlers {return &UpdateDetectionHandlers{monitor: monitor, readState: readState}}+// NewTenantUpdateDetectionHandlers creates handlers that resolve the monitor+// and read state of each request's organization.+func NewTenantUpdateDetectionHandlers(resolve func(context.Context) (*monitoring.Monitor, unifiedresources.ReadState)) *UpdateDetectionHandlers {+ return &UpdateDetectionHandlers{tenant: resolve}+}++func (h *UpdateDetectionHandlers) sources(r *http.Request) (*monitoring.Monitor, unifiedresources.ReadState) {+ if h.tenant != nil {+ return h.tenant(r.Context())+ }+ return h.monitor, h.readState+}+// ContainerUpdateInfo represents a container with an available updatetype ContainerUpdateInfo struct {AgentID string `json:"agentId"`@@ -116,7 +134,8 @@ func (h *UpdateDetectionHandlers) HandleGetInfraUpdates(w http.ResponseWriter, rreturn}- if h.monitor == nil {+ monitor, readState := h.sources(r)+ if monitor == nil {if err := utils.WriteJSONResponse(w, emptyInfraUpdatesResponse()); err != nil {log.Error().Err(err).Msg("Failed to serialize empty updates response")}@@ -129,7 +148,7 @@ func (h *UpdateDetectionHandlers) HandleGetInfraUpdates(w http.ResponseWriter, rresourceTypeFilter := strings.ToLower(query.Get("resourceType"))// Collect updates from Docker / Podman modules.- updates := h.collectDockerUpdates(agentIDFilter)+ updates := collectDockerUpdates(readState, agentIDFilter)// Filter by resource type if specifiedif resourceTypeFilter != "" && resourceTypeFilter != string(infraUpdateResourceTypeDocker) {@@ -153,13 +172,14 @@ func (h *UpdateDetectionHandlers) HandleGetInfraUpdateForResource(w http.Responsreturn}- if h.monitor == nil {+ monitor, readState := h.sources(r)+ if monitor == nil {writeErrorResponse(w, http.StatusNotFound, "not_found", "No update found for resource", nil)return}// ResourceID format: docker:<agentId>/<containerId>- updates := h.collectDockerUpdates("")+ updates := collectDockerUpdates(readState, "")for _, update := range updates {dockerResourceID := "docker:" + update.AgentID + "/" + update.ContainerIDif dockerResourceID == resourceID || update.ContainerID == resourceID {@@ -181,14 +201,15 @@ func (h *UpdateDetectionHandlers) HandleGetInfraUpdatesSummary(w http.ResponseWrreturn}- if h.monitor == nil {+ monitor, readState := h.sources(r)+ if monitor == nil {if err := utils.WriteJSONResponse(w, emptyInfraUpdatesSummaryResponse()); err != nil {log.Error().Err(err).Msg("Failed to serialize empty summary response")}return}- updates := h.collectDockerUpdates("")+ updates := collectDockerUpdates(readState, "")// Aggregate by agentsummaries := make(map[string]infraUpdateAgentSummary)@@ -225,7 +246,8 @@ func (h *UpdateDetectionHandlers) HandleTriggerInfraUpdateCheck(w http.ResponseWreturn}- if h.monitor == nil {+ monitor, readState := h.sources(r)+ if monitor == nil {writeErrorResponse(w, http.StatusServiceUnavailable, "service_unavailable", "Update detection not available", nil)return}@@ -245,7 +267,7 @@ func (h *UpdateDetectionHandlers) HandleTriggerInfraUpdateCheck(w http.ResponseW// Handle agent-level checkif req.AgentID != "" {- commandStatus, err := h.monitor.QueueDockerCheckUpdatesCommand(req.AgentID)+ commandStatus, err := monitor.QueueDockerCheckUpdatesCommand(req.AgentID)if err != nil {writeErrorResponse(w, http.StatusBadRequest, "check_updates_failed", err.Error(), nil)return@@ -265,7 +287,7 @@ func (h *UpdateDetectionHandlers) HandleTriggerInfraUpdateCheck(w http.ResponseW// Handle resource-level check (currently we just check the whole agent)if req.ResourceID != "" {// Try to find which agent this resource belongs to- updates := h.collectDockerUpdates("")+ updates := collectDockerUpdates(readState, "")var agentID stringfor _, update := range updates {if update.ContainerID == req.ResourceID || ("docker:"+update.AgentID+"/"+update.ContainerID) == req.ResourceID {@@ -279,7 +301,7 @@ func (h *UpdateDetectionHandlers) HandleTriggerInfraUpdateCheck(w http.ResponseWreturn}- commandStatus, err := h.monitor.QueueDockerCheckUpdatesCommand(agentID)+ commandStatus, err := monitor.QueueDockerCheckUpdatesCommand(agentID)if err != nil {writeErrorResponse(w, http.StatusBadRequest, "check_updates_failed", err.Error(), nil)return@@ -307,7 +329,8 @@ func (h *UpdateDetectionHandlers) HandleGetInfraUpdatesForAgent(w http.ResponseWreturn}- if h.monitor == nil {+ monitor, readState := h.sources(r)+ if monitor == nil {response := emptyInfraUpdatesForAgentResponse()response.AgentID = agentIDif err := utils.WriteJSONResponse(w, response.normalizeCollections()); err != nil {@@ -316,7 +339,7 @@ func (h *UpdateDetectionHandlers) HandleGetInfraUpdatesForAgent(w http.ResponseWreturn}- updates := h.collectDockerUpdates(agentID)+ updates := collectDockerUpdates(readState, agentID)response := emptyInfraUpdatesForAgentResponse()response.Updates = updates@@ -329,19 +352,19 @@ func (h *UpdateDetectionHandlers) HandleGetInfraUpdatesForAgent(w http.ResponseW}// collectDockerUpdates gathers update information from Docker / Podman modules via ReadState.-func (h *UpdateDetectionHandlers) collectDockerUpdates(agentIDFilter string) []ContainerUpdateInfo {- if h.readState == nil {+func collectDockerUpdates(readState unifiedresources.ReadState, agentIDFilter string) []ContainerUpdateInfo {+ if readState == nil {return nil}// Build agent ID -> display name map for lookups.agentNames := make(map[string]string)- for _, dh := range h.readState.DockerHosts() {+ for _, dh := range readState.DockerHosts() {agentNames[dh.HostSourceID()] = dh.Name()}var updates []ContainerUpdateInfo- for _, c := range h.readState.DockerContainers() {+ for _, c := range readState.DockerContainers() {agentID := c.HostSourceID()// Filter by agent ID if specified.diff --git a/internal/api/update_detection_tenant_test.go b/internal/api/update_detection_tenant_test.gonew file mode 100644index 0000000000..ccb98a6dc6--- /dev/null+++ b/internal/api/update_detection_tenant_test.go@@ -0,0 +1,179 @@+package api++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+)++func dockerUpdateSnapshot(hostID string) models.StateSnapshot {+ return models.StateSnapshot{+ DockerHosts: []models.DockerHost{{+ ID: hostID,+ DisplayName: hostID,+ Containers: []models.DockerContainer{{+ ID: hostID + "-c1",+ Name: "/web",+ Image: "nginx:latest",+ UpdateStatus: &models.DockerContainerUpdateStatus{+ UpdateAvailable: true,+ LastChecked: time.Now(),+ },+ }},+ }},+ }+}++func dockerUpdateReadState(hostID string) unifiedresources.ReadState {+ registry := unifiedresources.NewRegistry(nil)+ registry.IngestSnapshot(dockerUpdateSnapshot(hostID))+ return registry+}++func withOrg(req *http.Request, orgID string) *http.Request {+ return req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, orgID))+}++func TestTenantUpdateDetectionHandlersServeRequestOrganization(t *testing.T) {+ defaultMonitor, tenantMonitor := &monitoring.Monitor{}, &monitoring.Monitor{}+ defaultState, tenantState := dockerUpdateReadState("default-host"), dockerUpdateReadState("acme-host")+ handler := NewTenantUpdateDetectionHandlers(func(ctx context.Context) (*monitoring.Monitor, unifiedresources.ReadState) {+ switch GetOrgID(ctx) {+ case "", "default":+ return defaultMonitor, defaultState+ case "acme":+ return tenantMonitor, tenantState+ }+ return nil, nil+ })++ for orgID, wantHost := range map[string]string{"default": "default-host", "acme": "acme-host"} {+ rr := httptest.NewRecorder()+ handler.HandleGetInfraUpdates(rr, withOrg(httptest.NewRequest(http.MethodGet, "/api/infra-updates", nil), orgID))+ var resp infraUpdatesResponse+ if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil {+ t.Fatalf("org %s: decode: %v", orgID, err)+ }+ if resp.Total != 1 || resp.Updates[0].AgentID != wantHost {+ t.Fatalf("org %s: expected only %s updates, got %+v", orgID, wantHost, resp.Updates)+ }+ }++ // A known agent of another organization is not found from this one.+ rr := httptest.NewRecorder()+ handler.HandleGetInfraUpdateForResource(rr,+ withOrg(httptest.NewRequest(http.MethodGet, "/api/infra-updates/docker:default-host/default-host-c1", nil), "acme"),+ "docker:default-host/default-host-c1")+ if rr.Code != http.StatusNotFound {+ t.Fatalf("expected default-org resource to be hidden from acme, got %d", rr.Code)+ }+}++func TestTenantUpdateDetectionHandlersFailClosedWithoutTenantMonitor(t *testing.T) {+ handler := NewTenantUpdateDetectionHandlers(func(context.Context) (*monitoring.Monitor, unifiedresources.ReadState) {+ return nil, nil+ })++ rr := httptest.NewRecorder()+ handler.HandleGetInfraUpdatesSummary(rr, withOrg(httptest.NewRequest(http.MethodGet, "/api/infra-updates/summary", nil), "acme"))+ var summary infraUpdatesSummaryResponse+ if err := json.NewDecoder(rr.Body).Decode(&summary); err != nil {+ t.Fatalf("decode: %v", err)+ }+ if rr.Code != http.StatusOK || summary.TotalUpdates != 0 {+ t.Fatalf("expected empty summary, got %d %+v", rr.Code, summary)+ }++ rr = httptest.NewRecorder()+ handler.HandleTriggerInfraUpdateCheck(rr, withOrg(httptest.NewRequest(http.MethodPost, "/api/infra-updates/check", strings.NewReader(`{"agentId":"default-host"}`)), "acme"))+ if rr.Code != http.StatusServiceUnavailable {+ t.Fatalf("expected no command to be queued without the tenant monitor, got %d", rr.Code)+ }+}++func TestRouterTenantMonitorAndReadStateNeverFallsBackToDefault(t *testing.T) {+ router := &Router{monitor: &monitoring.Monitor{}}++ if monitor, readState := router.tenantMonitorAndReadState(context.WithValue(context.Background(), OrgIDContextKey, "acme")); monitor != nil || readState != nil {+ t.Fatalf("expected no monitor or read state for an unresolved tenant, got %v %v", monitor, readState)+ }+ if monitor, _ := router.tenantMonitorAndReadState(context.WithValue(context.Background(), OrgIDContextKey, "default")); monitor != router.monitor {+ t.Fatalf("expected the default monitor for the default organization")+ }++ rr := httptest.NewRecorder()+ router.handleSchedulerHealth(rr, withOrg(httptest.NewRequest(http.MethodGet, "/api/monitoring/scheduler/health", nil), "acme"))+ if rr.Code != http.StatusServiceUnavailable {+ t.Fatalf("expected tenant scheduler health not to report the default monitor, got %d", rr.Code)+ }+}++func monitorWithDockerUpdate(t *testing.T, hostID string) *monitoring.Monitor {+ t.Helper()+ monitor, _, _ := newTestMonitor(t)+ adapter := unifiedresources.NewMonitorAdapter(nil)+ adapter.PopulateFromSnapshot(dockerUpdateSnapshot(hostID))+ setUnexportedField(t, monitor, "resourceStore", monitoring.ResourceStoreInterface(adapter))+ return monitor+}++// The routed check guards the router wiring itself: each organization's+// infra-update reads come from its own monitor, through the real middleware.+func TestRoutedInfraUpdatesServeRequestOrganization(t *testing.T) {+ defer SetMultiTenantEnabled(false)+ SetMultiTenantEnabled(true)+ t.Setenv("PULSE_DEV", "true")++ const rawToken = "infra-updates-tenant-token-123.12345678"+ record, err := config.NewAPITokenRecord(rawToken, "two-org-token", []string{config.ScopeMonitoringRead})+ if err != nil {+ t.Fatalf("create api token record: %v", err)+ }+ record.OrgIDs = []string{"default", "org-a"}++ dataDir := t.TempDir()+ cfg := &config.Config{DataPath: dataDir, ConfigPath: dataDir, APITokens: []config.APITokenRecord{*record}}+ cfg.SortAPITokens()+ if err := config.NewMultiTenantPersistence(dataDir).SaveOrganization(&models.Organization{ID: "org-a", DisplayName: "ORG-A"}); err != nil {+ t.Fatalf("save organization: %v", err)+ }++ router := newMultiTenantRouter(t, cfg)+ router.monitor = monitorWithDockerUpdate(t, "default-host")+ mtm := &monitoring.MultiTenantMonitor{}+ setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{"org-a": monitorWithDockerUpdate(t, "org-a-host")})+ router.mtMonitor = mtm+ server := httptest.NewServer(router.Handler())+ t.Cleanup(server.Close)++ for orgID, wantHost := range map[string]string{"default": "default-host", "org-a": "org-a-host"} {+ req, err := http.NewRequest(http.MethodGet, server.URL+"/api/infra-updates", nil)+ if err != nil {+ t.Fatalf("create request: %v", err)+ }+ req.Header.Set("X-Pulse-Org-ID", orgID)+ req.Header.Set("Authorization", "Bearer "+rawToken)+ res, err := http.DefaultClient.Do(req)+ if err != nil {+ t.Fatalf("org %s: request failed: %v", orgID, err)+ }+ var resp infraUpdatesResponse+ decodeErr := json.NewDecoder(res.Body).Decode(&resp)+ res.Body.Close()+ if res.StatusCode != http.StatusOK || decodeErr != nil {+ t.Fatalf("org %s: status %d, decode %v", orgID, res.StatusCode, decodeErr)+ }+ if resp.Total != 1 || resp.Updates[0].AgentID != wantHost {+ t.Fatalf("org %s: expected only %s updates, got %+v", orgID, wantHost, resp.Updates)+ }+ }+}