Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Replace presumptive backup ACL recipes with evidenced access guidance
pulse-triage[bot]e9924fac0562
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 8bfb34cad6..04a7a26314 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -5751,3 +5751,24 @@ cancellation, replacement, retirement, late completion and ordinary recovery.`pkg/proxmox/replication_lifecycle_test.go` binds envelope and cancellationcontrols to the real HTTP client. Synthetic proofs do not establish #2606'sreported eleven-hour cause, native PVE recovery or published availability.++### Non-destructive PVE backup access guidance++Backup storage-list/content access warnings name the attempted endpoint and+point operators to the actual saved PVE connection, installed-version endpoint+permissions and both user/token scopes without disabling privilege separation.+Observed HTTP 401 is an authentication rejection; 403 is an access denial, not+proof of a particular missing role. Unknown errors and proxy bodies quoting+another status remain unconfirmed. The private HTTP error wrapper exposes only+its numeric wire status for this distinction, never its body or credentials.+Warnings and associated log guidance contain no unconditional ACL command,+assumed account, token ID, token secret or role prescription. Endpoint text is+bounded and control-quoted. Collection, inventory/error completeness, polling+cadence and setup permissions are unchanged.++`monitor_backup_poll_test.go` and `canonical_guardrails_test.go` retain the safe+saved-identity/manual-token contract. `pve_backup_warning_test.go` exercises+401/403/proxy errors through the real client and storage-list/content producer,+as well as unknown evidence and hostile endpoint text. The HTTP status helper+has wrapped-error/body-injection controls in `api_error_status_test.go`. These+are local source controls, not a native permission diagnosis or availability.diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex a565924a36..6f7ae2a907 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -375,18 +375,13 @@ func TestDockerInventoryConvertersPreserveNativeRuntimeFields(t *testing.T) {}}-func TestPVEBackupPermissionWarningsPreserveTokenACLRepair(t *testing.T) {- warning := pveBackupPermissionWarning(&config.PVEInstance{- TokenName: "pulse-monitor@pve!pulse-example",- })-- for _, snippet := range []string{- "pveum aclmod /storage -user pulse-monitor@pve -role PVEDatastoreAdmin",- "pveum aclmod /storage -token 'pulse-monitor@pve!pulse-example' -role PVEDatastoreAdmin",- } {- if !strings.Contains(warning, snippet) {- t.Fatalf("expected warning to contain %q, got %q", snippet, warning)- }+func TestPVEBackupPermissionWarningsPreserveNonDestructiveGuidance(t *testing.T) {+ warning := pveBackupPermissionWarning(&config.PVEInstance{TokenName: "different-reader@pve!manual"})+ if !strings.Contains(warning, "saved PVE connection") || !strings.Contains(warning, "privilege separation") || !strings.Contains(warning, "rejected endpoint") {+ t.Fatalf("missing identity/scope guidance: %q", warning)+ }+ if strings.Contains(warning, "pveum") || strings.Contains(warning, "PVEDatastoreAdmin") || strings.Contains(warning, "pulse-monitor@pve") {+ t.Fatalf("unconditional ACL recipe survived: %q", warning)}}diff --git a/internal/monitoring/monitor_backup_poll_test.go b/internal/monitoring/monitor_backup_poll_test.goindex 3728549ebb..2092ee43c9 100644--- a/internal/monitoring/monitor_backup_poll_test.go+++ b/internal/monitoring/monitor_backup_poll_test.go@@ -8,29 +8,19 @@ import ("github.com/rcourtman/pulse-go-rewrite/internal/config")-func TestPVEBackupPermissionWarningIncludesTokenACLWhenAvailable(t *testing.T) {- warning := pveBackupPermissionWarning(&config.PVEInstance{- TokenName: "pulse-monitor@pve!pulse-example",- })-- for _, snippet := range []string{- "pveum aclmod /storage -user pulse-monitor@pve -role PVEDatastoreAdmin",- "pveum aclmod /storage -token 'pulse-monitor@pve!pulse-example' -role PVEDatastoreAdmin",- } {- if !strings.Contains(warning, snippet) {- t.Fatalf("expected warning to contain %q, got %q", snippet, warning)+func TestPVEBackupPermissionWarningUsesSavedIdentityWithoutACLMutation(t *testing.T) {+ for _, cfg := range []*config.PVEInstance{nil, {}, {TokenName: "custom-reader@pve!manual"}} {+ warning := pveBackupPermissionWarning(cfg)+ for _, snippet := range []string{"saved PVE connection", "both user and token scopes", "without disabling privilege separation", "rejected endpoint", "installed PVE version", "Do not delete nodes"} {+ if !strings.Contains(warning, snippet) {+ t.Fatalf("missing safe guidance %q: %q", snippet, warning)+ }+ }+ for _, unsafe := range []string{"pveum", "PVEDatastoreAdmin", "pulse-monitor@pve", "custom-reader@pve!manual"} {+ if strings.Contains(warning, unsafe) {+ t.Fatalf("warning prescribes or discloses %q: %q", unsafe, warning)+ }}- }-}--func TestPVEBackupPermissionWarningFallsBackForManualTokens(t *testing.T) {- warning := pveBackupPermissionWarning(&config.PVEInstance{})-- if !strings.Contains(warning, "pveum aclmod /storage -user pulse-monitor@pve -role PVEDatastoreAdmin") {- t.Fatalf("expected user ACL guidance, got %q", warning)- }- if !strings.Contains(warning, "if using a privilege-separated API token") {- t.Fatalf("expected privilege-separated token fallback guidance, got %q", warning)}}diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex e634998786..e045509c2d 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -108,20 +108,43 @@ func (m *Monitor) updatePVEBackupTemplateSubjectsFromClusterResources(instanceNam.updatePVEBackupTemplateSubjectsForType(instanceName, "lxc", lxcTemplates)}-func quotePVEACLTokenID(tokenID string) string {- return "'" + strings.ReplaceAll(tokenID, "'", `'"'"'`) + "'"+func pveBackupPermissionWarning(instanceCfg *config.PVEInstance) string {+ identity := "the configured user and API token in the saved PVE connection"+ if instanceCfg != nil && strings.TrimSpace(instanceCfg.TokenName) == "" {+ identity = "the configured login credentials or manually supplied API token in the saved PVE connection"+ }+ return "Check " + identity + ". For a privilege-separated token, check both user and token scopes without disabling privilege separation. Verify the permissions required by the rejected endpoint on your installed PVE version; a failed read alone does not identify a missing role. Do not delete nodes, remove registration state or rotate credentials to diagnose an empty backup table."}-func pveBackupPermissionWarning(instanceCfg *config.PVEInstance) string {- warning := "Missing PVEDatastoreAdmin permission on /storage. Run: pveum aclmod /storage -user pulse-monitor@pve -role PVEDatastoreAdmin"- if instanceCfg == nil {- return warning + "; if using a privilege-separated API token, also grant PVEDatastoreAdmin on /storage to that token."+func pveBackupAccessWarning(instanceCfg *config.PVEInstance, endpoint string, err error) string {+ status, known := proxmox.APIErrorStatus(err)+ cause := "PVE backup inventory access failed; authentication or permission cause is unconfirmed."+ if known && status == 401 {+ cause = "PVE rejected backup inventory authentication (HTTP 401); this is not evidence of a missing role."+ } else if known && status == 403 {+ cause = "PVE denied backup inventory access (HTTP 403); the required permission and scope need verification."+ }+ // Endpoint comes from the actual attempted node/storage operation, never+ // from provider error text. Bound and quote it to keep controls out of logs+ // and warnings. No credential, token ID or provider body is reproduced.+ endpointRunes := []rune(strings.ToValidUTF8(endpoint, "?"))+ if len(endpointRunes) > 256 {+ endpoint = string(endpointRunes[:256]) + "…"+ } else {+ endpoint = string(endpointRunes)}- tokenID := strings.TrimSpace(instanceCfg.TokenName)- if tokenID == "" || !strings.Contains(tokenID, "!") {- return warning + "; if using a privilege-separated API token, also grant PVEDatastoreAdmin on /storage to that token."+ return cause + " Rejected endpoint: " + strconv.Quote(endpoint) + ". " + pveBackupPermissionWarning(instanceCfg)+}++func (m *Monitor) recordPVEBackupAccessWarning(instanceName, endpoint string, err error) {+ warning := pveBackupAccessWarning(m.getInstanceConfig(instanceName), endpoint, err)+ m.mu.Lock()+ if m.backupPermissionWarnings == nil {+ m.backupPermissionWarnings = make(map[string]string)}- return warning + " && pveum aclmod /storage -token " + quotePVEACLTokenID(tokenID) + " -role PVEDatastoreAdmin"+ m.backupPermissionWarnings[instanceName] = warning+ m.mu.Unlock()+ log.Warn().Str("instance", instanceName).Str("guidance", warning).Msg("PVE backup inventory access failed")}func (m *Monitor) backupInventoryScopeForAlerts() *alerts.BackupInventoryScope {@@ -234,6 +257,7 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceNameif isPVEBackupPermissionError(err) {hadPermissionError = truepermissionFailureCount+++ m.recordPVEBackupAccessWarning(instanceName, fmt.Sprintf("/nodes/%s/storage", node.Node), err)}monErr := errors.NewMonitorError(errors.ErrorTypeAPI, "get_storage_for_backups", instanceName, err).WithNode(node.Node)log.Warn().Err(monErr).Str("node", node.Node).Msg("failed to get storage for backups - skipping node")@@ -267,15 +291,7 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceNameif isPVEBackupPermissionError(err) {hadPermissionError = truepermissionFailureCount++- warning := pveBackupPermissionWarning(m.getInstanceConfig(instanceName))- m.mu.Lock()- m.backupPermissionWarnings[instanceName] = warning- m.mu.Unlock()- log.Warn().- Str("instance", instanceName).- Str("node", node.Node).- Str("storage", storage.Storage).- Msg("Backup permission denied - PVEDatastoreAdmin role may be missing on /storage")+ m.recordPVEBackupAccessWarning(instanceName, fmt.Sprintf("/nodes/%s/storage/%s/content", node.Node, storage.Storage), err)} else {log.Debug().Err(monErr).Str("node", node.Node).diff --git a/internal/monitoring/pve_backup_warning_test.go b/internal/monitoring/pve_backup_warning_test.gonew file mode 100644index 0000000000..164b707001--- /dev/null+++ b/internal/monitoring/pve_backup_warning_test.go@@ -0,0 +1,88 @@+package monitoring++import (+ "context"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++type backupWarningClient struct {+ stubPVEClient+ contentErr error+ storageErr error+}++func (c *backupWarningClient) GetStorage(context.Context, string) ([]proxmox.Storage, error) {+ if c.storageErr != nil {+ return nil, c.storageErr+ }+ return []proxmox.Storage{{Storage: "backup-store", Type: "dir", Content: "backup", Enabled: 1, Active: 1}}, nil+}+func (c *backupWarningClient) GetStorageContent(context.Context, string, string) ([]proxmox.StorageContent, error) {+ return nil, c.contentErr+}++func TestPVEBackupWarningProducerUsesActualStatusAndEndpoint(t *testing.T) {+ for _, tc := range []struct {+ name string+ status int+ list bool+ body, expected string+ }{+ {"custom-token-auth", 401, false, "authentication failed; private-synthetic-body", "authentication (HTTP 401)"},+ {"custom-token-denied", 403, false, "permission denied; private-synthetic-body", "access (HTTP 403)"},+ {"manual-list-auth", 401, true, "authentication failed; private-synthetic-body", "authentication (HTTP 401)"},+ {"proxy-quoted-denial", 502, false, "API error 403: permission denied; private-synthetic-body", "cause is unconfirmed"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, tc.body, tc.status) }))+ defer server.Close()+ api, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "custom-user@pve!manual", TokenValue: "synthetic-secret"})+ if err != nil {+ t.Fatal(err)+ }+ _, readErr := api.GetStorageContent(context.Background(), "node-a", "backup-store")+ if readErr == nil {+ t.Fatal("HTTP error not observed")+ }+ client := &backupWarningClient{contentErr: readErr}+ endpoint := "/nodes/node-a/storage/backup-store/content"+ tokenName := "custom-user@pve!manual"+ if tc.list {+ client.storageErr = readErr+ endpoint = "/nodes/node-a/storage"+ tokenName = ""+ }+ m := newUnreachableTestMonitor(t, &config.Config{PVEInstances: []config.PVEInstance{{Name: "site-a", TokenName: tokenName, TokenValue: "synthetic-secret"}}})+ m.pollStorageBackupsWithNodes(context.Background(), "site-a", client, []proxmox.Node{{Node: "node-a", Status: "online"}}, map[string]string{"node-a": "online"})+ m.mu.RLock()+ warning := m.backupPermissionWarnings["site-a"]+ m.mu.RUnlock()+ for _, snippet := range []string{tc.expected, endpoint, "saved PVE connection", "both user and token scopes", "without disabling privilege separation", "installed PVE version"} {+ if !strings.Contains(warning, snippet) {+ t.Fatalf("missing %q in producer warning: %q", snippet, warning)+ }+ }+ for _, unsafe := range []string{"pveum", "PVEDatastoreAdmin", "pulse-monitor@pve", "custom-user@pve!manual", "synthetic-secret", "private-synthetic-body"} {+ if strings.Contains(warning, unsafe) {+ t.Fatalf("unsafe prescription/disclosure %q: %q", unsafe, warning)+ }+ }+ })+ }+}+func TestPVEBackupWarningUnknownEvidenceAndBoundedEndpoint(t *testing.T) {+ warning := pveBackupAccessWarning(nil, "/nodes/"+strings.Repeat("界", 300)+"\n/storage", fmt.Errorf("403 permission denied; synthetic-secret"))+ if !strings.Contains(warning, "cause is unconfirmed") || strings.Contains(warning, "HTTP 403") || strings.Contains(warning, "synthetic-secret") {+ t.Fatalf("text-only error became HTTP evidence: %q", warning)+ }+ if strings.ContainsAny(warning, "\r\n") || !strings.Contains(warning, "…") || len([]rune(warning)) > 1100 {+ t.Fatalf("endpoint not safely bounded: %q", warning)+ }+}diff --git a/pkg/proxmox/api_error_status.go b/pkg/proxmox/api_error_status.gonew file mode 100644index 0000000000..b7bf33f491--- /dev/null+++ b/pkg/proxmox/api_error_status.go@@ -0,0 +1,17 @@+package proxmox++import "errors"++// APIErrorStatus returns the observed HTTP status, never a number quoted in a+// provider's error body. Unknown/local errors have no HTTP-status evidence.+func APIErrorStatus(err error) (int, bool) {+ var response *apiResponseError+ if errors.As(err, &response) && response != nil {+ return response.statusCode, true+ }+ var auth *authHTTPError+ if errors.As(err, &auth) && auth != nil {+ return auth.status, true+ }+ return 0, false+}diff --git a/pkg/proxmox/api_error_status_test.go b/pkg/proxmox/api_error_status_test.gonew file mode 100644index 0000000000..15bc387fa7--- /dev/null+++ b/pkg/proxmox/api_error_status_test.go@@ -0,0 +1,30 @@+package proxmox++import (+ "errors"+ "fmt"+ "testing"+)++func TestAPIErrorStatusUsesWireEvidenceNotBody(t *testing.T) {+ for _, tc := range []struct {+ name string+ err error+ status int+ known bool+ }{+ {"wire-401", fmt.Errorf("wrapped: %w", &apiResponseError{statusCode: 401, cause: errors.New("API error 403: quoted body")}), 401, true},+ {"wire-403", &apiResponseError{statusCode: 403, cause: errors.New("body says 401")}, 403, true},+ {"proxy-502", &apiResponseError{statusCode: 502, cause: errors.New("API error 403: permission denied")}, 502, true},+ {"login-401", &authHTTPError{status: 401, body: "untrusted 403"}, 401, true},+ {"text-only", errors.New("403 permission denied"), 0, false},+ {"nil", nil, 0, false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ status, known := APIErrorStatus(tc.err)+ if status != tc.status || known != tc.known {+ t.Fatalf("status/known=%d/%v want %d/%v", status, known, tc.status, tc.known)+ }+ })+ }+}