Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Replace presumptive backup ACL recipes with evidenced access guidance

pulse-triage[bot]e9924fac0562
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 8bfb34cad6..04a7a26314 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -5751,3 +5751,24 @@ cancellation, replacement, retirement, late completion and ordinary recovery.
`pkg/proxmox/replication_lifecycle_test.go` binds envelope and cancellation
controls to the real HTTP client. Synthetic proofs do not establish #2606's
reported eleven-hour cause, native PVE recovery or published availability.
+
+### Non-destructive PVE backup access guidance
+
+Backup storage-list/content access warnings name the attempted endpoint and
+point operators to the actual saved PVE connection, installed-version endpoint
+permissions and both user/token scopes without disabling privilege separation.
+Observed HTTP 401 is an authentication rejection; 403 is an access denial, not
+proof of a particular missing role. Unknown errors and proxy bodies quoting
+another status remain unconfirmed. The private HTTP error wrapper exposes only
+its numeric wire status for this distinction, never its body or credentials.
+Warnings and associated log guidance contain no unconditional ACL command,
+assumed account, token ID, token secret or role prescription. Endpoint text is
+bounded and control-quoted. Collection, inventory/error completeness, polling
+cadence and setup permissions are unchanged.
+
+`monitor_backup_poll_test.go` and `canonical_guardrails_test.go` retain the safe
+saved-identity/manual-token contract. `pve_backup_warning_test.go` exercises
+401/403/proxy errors through the real client and storage-list/content producer,
+as well as unknown evidence and hostile endpoint text. The HTTP status helper
+has wrapped-error/body-injection controls in `api_error_status_test.go`. These
+are local source controls, not a native permission diagnosis or availability.
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index a565924a36..6f7ae2a907 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -375,18 +375,13 @@ func TestDockerInventoryConvertersPreserveNativeRuntimeFields(t *testing.T) {
}
}
-func TestPVEBackupPermissionWarningsPreserveTokenACLRepair(t *testing.T) {
- warning := pveBackupPermissionWarning(&config.PVEInstance{
- TokenName: "pulse-monitor@pve!pulse-example",
- })
-
- for _, snippet := range []string{
- "pveum aclmod /storage -user pulse-monitor@pve -role PVEDatastoreAdmin",
- "pveum aclmod /storage -token 'pulse-monitor@pve!pulse-example' -role PVEDatastoreAdmin",
- } {
- if !strings.Contains(warning, snippet) {
- t.Fatalf("expected warning to contain %q, got %q", snippet, warning)
- }
+func TestPVEBackupPermissionWarningsPreserveNonDestructiveGuidance(t *testing.T) {
+ warning := pveBackupPermissionWarning(&config.PVEInstance{TokenName: "different-reader@pve!manual"})
+ if !strings.Contains(warning, "saved PVE connection") || !strings.Contains(warning, "privilege separation") || !strings.Contains(warning, "rejected endpoint") {
+ t.Fatalf("missing identity/scope guidance: %q", warning)
+ }
+ if strings.Contains(warning, "pveum") || strings.Contains(warning, "PVEDatastoreAdmin") || strings.Contains(warning, "pulse-monitor@pve") {
+ t.Fatalf("unconditional ACL recipe survived: %q", warning)
}
}
diff --git a/internal/monitoring/monitor_backup_poll_test.go b/internal/monitoring/monitor_backup_poll_test.go
index 3728549ebb..2092ee43c9 100644
--- a/internal/monitoring/monitor_backup_poll_test.go
+++ b/internal/monitoring/monitor_backup_poll_test.go
@@ -8,29 +8,19 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/config"
)
-func TestPVEBackupPermissionWarningIncludesTokenACLWhenAvailable(t *testing.T) {
- warning := pveBackupPermissionWarning(&config.PVEInstance{
- TokenName: "pulse-monitor@pve!pulse-example",
- })
-
- for _, snippet := range []string{
- "pveum aclmod /storage -user pulse-monitor@pve -role PVEDatastoreAdmin",
- "pveum aclmod /storage -token 'pulse-monitor@pve!pulse-example' -role PVEDatastoreAdmin",
- } {
- if !strings.Contains(warning, snippet) {
- t.Fatalf("expected warning to contain %q, got %q", snippet, warning)
+func TestPVEBackupPermissionWarningUsesSavedIdentityWithoutACLMutation(t *testing.T) {
+ for _, cfg := range []*config.PVEInstance{nil, {}, {TokenName: "custom-reader@pve!manual"}} {
+ warning := pveBackupPermissionWarning(cfg)
+ for _, snippet := range []string{"saved PVE connection", "both user and token scopes", "without disabling privilege separation", "rejected endpoint", "installed PVE version", "Do not delete nodes"} {
+ if !strings.Contains(warning, snippet) {
+ t.Fatalf("missing safe guidance %q: %q", snippet, warning)
+ }
+ }
+ for _, unsafe := range []string{"pveum", "PVEDatastoreAdmin", "pulse-monitor@pve", "custom-reader@pve!manual"} {
+ if strings.Contains(warning, unsafe) {
+ t.Fatalf("warning prescribes or discloses %q: %q", unsafe, warning)
+ }
}
- }
-}
-
-func TestPVEBackupPermissionWarningFallsBackForManualTokens(t *testing.T) {
- warning := pveBackupPermissionWarning(&config.PVEInstance{})
-
- if !strings.Contains(warning, "pveum aclmod /storage -user pulse-monitor@pve -role PVEDatastoreAdmin") {
- t.Fatalf("expected user ACL guidance, got %q", warning)
- }
- if !strings.Contains(warning, "if using a privilege-separated API token") {
- t.Fatalf("expected privilege-separated token fallback guidance, got %q", warning)
}
}
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index e634998786..e045509c2d 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -108,20 +108,43 @@ func (m *Monitor) updatePVEBackupTemplateSubjectsFromClusterResources(instanceNa
m.updatePVEBackupTemplateSubjectsForType(instanceName, "lxc", lxcTemplates)
}
-func quotePVEACLTokenID(tokenID string) string {
- return "'" + strings.ReplaceAll(tokenID, "'", `'"'"'`) + "'"
+func pveBackupPermissionWarning(instanceCfg *config.PVEInstance) string {
+ identity := "the configured user and API token in the saved PVE connection"
+ if instanceCfg != nil && strings.TrimSpace(instanceCfg.TokenName) == "" {
+ identity = "the configured login credentials or manually supplied API token in the saved PVE connection"
+ }
+ return "Check " + identity + ". For a privilege-separated token, check both user and token scopes without disabling privilege separation. Verify the permissions required by the rejected endpoint on your installed PVE version; a failed read alone does not identify a missing role. Do not delete nodes, remove registration state or rotate credentials to diagnose an empty backup table."
}
-func pveBackupPermissionWarning(instanceCfg *config.PVEInstance) string {
- warning := "Missing PVEDatastoreAdmin permission on /storage. Run: pveum aclmod /storage -user pulse-monitor@pve -role PVEDatastoreAdmin"
- if instanceCfg == nil {
- return warning + "; if using a privilege-separated API token, also grant PVEDatastoreAdmin on /storage to that token."
+func pveBackupAccessWarning(instanceCfg *config.PVEInstance, endpoint string, err error) string {
+ status, known := proxmox.APIErrorStatus(err)
+ cause := "PVE backup inventory access failed; authentication or permission cause is unconfirmed."
+ if known && status == 401 {
+ cause = "PVE rejected backup inventory authentication (HTTP 401); this is not evidence of a missing role."
+ } else if known && status == 403 {
+ cause = "PVE denied backup inventory access (HTTP 403); the required permission and scope need verification."
+ }
+ // Endpoint comes from the actual attempted node/storage operation, never
+ // from provider error text. Bound and quote it to keep controls out of logs
+ // and warnings. No credential, token ID or provider body is reproduced.
+ endpointRunes := []rune(strings.ToValidUTF8(endpoint, "?"))
+ if len(endpointRunes) > 256 {
+ endpoint = string(endpointRunes[:256]) + "…"
+ } else {
+ endpoint = string(endpointRunes)
}
- tokenID := strings.TrimSpace(instanceCfg.TokenName)
- if tokenID == "" || !strings.Contains(tokenID, "!") {
- return warning + "; if using a privilege-separated API token, also grant PVEDatastoreAdmin on /storage to that token."
+ return cause + " Rejected endpoint: " + strconv.Quote(endpoint) + ". " + pveBackupPermissionWarning(instanceCfg)
+}
+
+func (m *Monitor) recordPVEBackupAccessWarning(instanceName, endpoint string, err error) {
+ warning := pveBackupAccessWarning(m.getInstanceConfig(instanceName), endpoint, err)
+ m.mu.Lock()
+ if m.backupPermissionWarnings == nil {
+ m.backupPermissionWarnings = make(map[string]string)
}
- return warning + " && pveum aclmod /storage -token " + quotePVEACLTokenID(tokenID) + " -role PVEDatastoreAdmin"
+ m.backupPermissionWarnings[instanceName] = warning
+ m.mu.Unlock()
+ log.Warn().Str("instance", instanceName).Str("guidance", warning).Msg("PVE backup inventory access failed")
}
func (m *Monitor) backupInventoryScopeForAlerts() *alerts.BackupInventoryScope {
@@ -234,6 +257,7 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
if isPVEBackupPermissionError(err) {
hadPermissionError = true
permissionFailureCount++
+ m.recordPVEBackupAccessWarning(instanceName, fmt.Sprintf("/nodes/%s/storage", node.Node), err)
}
monErr := errors.NewMonitorError(errors.ErrorTypeAPI, "get_storage_for_backups", instanceName, err).WithNode(node.Node)
log.Warn().Err(monErr).Str("node", node.Node).Msg("failed to get storage for backups - skipping node")
@@ -267,15 +291,7 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
if isPVEBackupPermissionError(err) {
hadPermissionError = true
permissionFailureCount++
- warning := pveBackupPermissionWarning(m.getInstanceConfig(instanceName))
- m.mu.Lock()
- m.backupPermissionWarnings[instanceName] = warning
- m.mu.Unlock()
- log.Warn().
- Str("instance", instanceName).
- Str("node", node.Node).
- Str("storage", storage.Storage).
- Msg("Backup permission denied - PVEDatastoreAdmin role may be missing on /storage")
+ m.recordPVEBackupAccessWarning(instanceName, fmt.Sprintf("/nodes/%s/storage/%s/content", node.Node, storage.Storage), err)
} else {
log.Debug().Err(monErr).
Str("node", node.Node).
diff --git a/internal/monitoring/pve_backup_warning_test.go b/internal/monitoring/pve_backup_warning_test.go
new file mode 100644
index 0000000000..164b707001
--- /dev/null
+++ b/internal/monitoring/pve_backup_warning_test.go
@@ -0,0 +1,88 @@
+package monitoring
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+type backupWarningClient struct {
+ stubPVEClient
+ contentErr error
+ storageErr error
+}
+
+func (c *backupWarningClient) GetStorage(context.Context, string) ([]proxmox.Storage, error) {
+ if c.storageErr != nil {
+ return nil, c.storageErr
+ }
+ return []proxmox.Storage{{Storage: "backup-store", Type: "dir", Content: "backup", Enabled: 1, Active: 1}}, nil
+}
+func (c *backupWarningClient) GetStorageContent(context.Context, string, string) ([]proxmox.StorageContent, error) {
+ return nil, c.contentErr
+}
+
+func TestPVEBackupWarningProducerUsesActualStatusAndEndpoint(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ status int
+ list bool
+ body, expected string
+ }{
+ {"custom-token-auth", 401, false, "authentication failed; private-synthetic-body", "authentication (HTTP 401)"},
+ {"custom-token-denied", 403, false, "permission denied; private-synthetic-body", "access (HTTP 403)"},
+ {"manual-list-auth", 401, true, "authentication failed; private-synthetic-body", "authentication (HTTP 401)"},
+ {"proxy-quoted-denial", 502, false, "API error 403: permission denied; private-synthetic-body", "cause is unconfirmed"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, tc.body, tc.status) }))
+ defer server.Close()
+ api, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "custom-user@pve!manual", TokenValue: "synthetic-secret"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, readErr := api.GetStorageContent(context.Background(), "node-a", "backup-store")
+ if readErr == nil {
+ t.Fatal("HTTP error not observed")
+ }
+ client := &backupWarningClient{contentErr: readErr}
+ endpoint := "/nodes/node-a/storage/backup-store/content"
+ tokenName := "custom-user@pve!manual"
+ if tc.list {
+ client.storageErr = readErr
+ endpoint = "/nodes/node-a/storage"
+ tokenName = ""
+ }
+ m := newUnreachableTestMonitor(t, &config.Config{PVEInstances: []config.PVEInstance{{Name: "site-a", TokenName: tokenName, TokenValue: "synthetic-secret"}}})
+ m.pollStorageBackupsWithNodes(context.Background(), "site-a", client, []proxmox.Node{{Node: "node-a", Status: "online"}}, map[string]string{"node-a": "online"})
+ m.mu.RLock()
+ warning := m.backupPermissionWarnings["site-a"]
+ m.mu.RUnlock()
+ for _, snippet := range []string{tc.expected, endpoint, "saved PVE connection", "both user and token scopes", "without disabling privilege separation", "installed PVE version"} {
+ if !strings.Contains(warning, snippet) {
+ t.Fatalf("missing %q in producer warning: %q", snippet, warning)
+ }
+ }
+ for _, unsafe := range []string{"pveum", "PVEDatastoreAdmin", "pulse-monitor@pve", "custom-user@pve!manual", "synthetic-secret", "private-synthetic-body"} {
+ if strings.Contains(warning, unsafe) {
+ t.Fatalf("unsafe prescription/disclosure %q: %q", unsafe, warning)
+ }
+ }
+ })
+ }
+}
+func TestPVEBackupWarningUnknownEvidenceAndBoundedEndpoint(t *testing.T) {
+ warning := pveBackupAccessWarning(nil, "/nodes/"+strings.Repeat("界", 300)+"\n/storage", fmt.Errorf("403 permission denied; synthetic-secret"))
+ if !strings.Contains(warning, "cause is unconfirmed") || strings.Contains(warning, "HTTP 403") || strings.Contains(warning, "synthetic-secret") {
+ t.Fatalf("text-only error became HTTP evidence: %q", warning)
+ }
+ if strings.ContainsAny(warning, "\r\n") || !strings.Contains(warning, "…") || len([]rune(warning)) > 1100 {
+ t.Fatalf("endpoint not safely bounded: %q", warning)
+ }
+}
diff --git a/pkg/proxmox/api_error_status.go b/pkg/proxmox/api_error_status.go
new file mode 100644
index 0000000000..b7bf33f491
--- /dev/null
+++ b/pkg/proxmox/api_error_status.go
@@ -0,0 +1,17 @@
+package proxmox
+
+import "errors"
+
+// APIErrorStatus returns the observed HTTP status, never a number quoted in a
+// provider's error body. Unknown/local errors have no HTTP-status evidence.
+func APIErrorStatus(err error) (int, bool) {
+ var response *apiResponseError
+ if errors.As(err, &response) && response != nil {
+ return response.statusCode, true
+ }
+ var auth *authHTTPError
+ if errors.As(err, &auth) && auth != nil {
+ return auth.status, true
+ }
+ return 0, false
+}
diff --git a/pkg/proxmox/api_error_status_test.go b/pkg/proxmox/api_error_status_test.go
new file mode 100644
index 0000000000..15bc387fa7
--- /dev/null
+++ b/pkg/proxmox/api_error_status_test.go
@@ -0,0 +1,30 @@
+package proxmox
+
+import (
+ "errors"
+ "fmt"
+ "testing"
+)
+
+func TestAPIErrorStatusUsesWireEvidenceNotBody(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ err error
+ status int
+ known bool
+ }{
+ {"wire-401", fmt.Errorf("wrapped: %w", &apiResponseError{statusCode: 401, cause: errors.New("API error 403: quoted body")}), 401, true},
+ {"wire-403", &apiResponseError{statusCode: 403, cause: errors.New("body says 401")}, 403, true},
+ {"proxy-502", &apiResponseError{statusCode: 502, cause: errors.New("API error 403: permission denied")}, 502, true},
+ {"login-401", &authHTTPError{status: 401, body: "untrusted 403"}, 401, true},
+ {"text-only", errors.New("403 permission denied"), 0, false},
+ {"nil", nil, 0, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ status, known := APIErrorStatus(tc.err)
+ if status != tc.status || known != tc.known {
+ t.Fatalf("status/known=%d/%v want %d/%v", status, known, tc.status, tc.known)
+ }
+ })
+ }
+}