Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Retain observed PBS files across timeout and throttle responses
pulse-triage[bot]404a859356de
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 00d2282c9f..489e8e2e39 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -2816,17 +2816,35 @@ Direct PBS backup polling classifies typed API and authentication failures bythe response status exposed by `pbs.HTTPStatus`, including wrapped errors.A 5xx gateway or server response remains transient even when its body quotesan upstream “API error 403” or “API error 404”; that text must not erase the-last known backup inventory. Genuine 4xx responses remain terminal under the-existing cache policy. The legacy untyped-error fallback is unchanged.+last known backup inventory. Typed HTTP 408 (request timeout) and 429 (rate+limit) likewise retain already accepted files in their own unread scope. They+cannot renew point evidence, cache age or complete/permission-granted provider+claims, and introduce no immediate retry or cadence change. Cold unread scopes+invent no files; readable siblings still advance and accepted empty reads still+confirm absence. Other 4xx responses, including 401/403/404, retain the existing+terminal cache policy. The legacy untyped-error fallback is unchanged.++Namespace compatibility treats only an actual HTTP 404 as an unavailable legacy+endpoint; a quoted 404 in another response cannot turn failed discovery into+an accepted empty namespace list. Wrapped typed errors retain their status.Retaining cached inventory does not establish a successful poll or freshbackup evidence.-Verification: `TestPollPBSBackups_PreservesCacheOnTransientDatastoreError` and-`TestPollPBSBackups_DropsStaleCacheOnTerminalDatastoreError` in-`internal/monitoring/monitor_backups_readstate_test.go` exercise actual HTTP-fixtures for 500, 502 quoting 403, 503 quoting 404, and genuine 401/403/404.-These tests prove cache retention/removal, not installed PBS wake, service-restart, or notification receipt.+Verification: `TestPBSRetryableReadLifecycle` and+`TestPBSRetryableReadWithoutCacheDoesNotInventFiles` exercise real synthetic+namespace discovery, group and snapshot HTTP 408/429 replies for VM/LXC+history, including bodies quoting 404, repeated ordinary polls, unchanged+historical points/cache age, partial/unknown provider evidence, sibling+progress, complete recovery and accepted removal. The existing namespace+transient/mixed-denial controls remain. `TestNamespaceCompatibilityUsesActualHTTPStatus`+pins actual 400/401/403/404/408/429/500/503 replies in `pkg/pbs`.+`TestPBSNamespaceContinuityLifecycle` exports actual producer frames for+production Backups/Coverage desktop and narrow interaction/pixel proof.+`TestPollPBSBackups_PreservesCacheOnTransientDatastoreError` and+`TestPollPBSBackups_DropsStaleCacheOnTerminalDatastoreError` retain 500,+502 quoting 403, 503 quoting 404, and genuine 401/403/404 controls. These are+source safety proofs, not installed PBS wake, guest/backup recovery,+notification receipt, archive restorability or release availability.Docker alert lifecycle events pass through the shared resource history identitywriter. A full Docker source reference must reach the same canonical containerdiff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.jsonindex d6a82dbcf5..1445b51f5d 100644--- a/docs/release-control/v6/internal/subsystems/registry.json+++ b/docs/release-control/v6/internal/subsystems/registry.json@@ -6848,7 +6848,8 @@"pkg/pbs/auth_session_concurrency_test.go","pkg/pbs/client_http_test.go","pkg/pbs/client_test.go",- "pkg/pbs/host_diagnostics_regression_test.go"+ "pkg/pbs/host_diagnostics_regression_test.go",+ "pkg/pbs/namespace_status_test.go"]},{@@ -6968,6 +6969,7 @@"internal/monitoring/monitor_notification_startup_test.go","internal/monitoring/monitor_pve_cluster_refresh_test.go","internal/monitoring/monitor_pve_guest_lxc_test.go",+ "internal/monitoring/pbs_retryable_read_test.go","internal/monitoring/physical_disk_roundtrip_test.go","internal/monitoring/proxmox_large_cluster_poll_budget_test.go","internal/monitoring/pve_protection_observation_test.go",diff --git a/internal/monitoring/backup_guard.go b/internal/monitoring/backup_guard.goindex f6e1b298d2..02f985db3f 100644--- a/internal/monitoring/backup_guard.go+++ b/internal/monitoring/backup_guard.go@@ -1,6 +1,7 @@package monitoringimport (+ "net/http""strings""github.com/rcourtman/pulse-go-rewrite/pkg/pbs"@@ -16,13 +17,16 @@ func shouldPreservePBSBackupsWithTerminal(datastoreCount, datastoreFetches, data}// shouldReuseCachedPBSBackups keeps cached data for transient fetch errors and-// avoids reuse for terminal API 4xx errors.+// avoids reuse for terminal API 4xx errors. A response-level timeout or rate+// limit is not a rejected scope or an accepted empty inventory. Retaining it+// neither retries a request nor renews the cached artifact's evidence.func shouldReuseCachedPBSBackups(err error) bool {if err == nil {return false}if status, ok := pbs.HTTPStatus(err); ok {- return status < 400 || status >= 500+ return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests ||+ status < 400 || status >= 500}// Retain compatibility with untyped errors from older callers.if strings.Contains(strings.ToLower(err.Error()), "api error 4") {diff --git a/internal/monitoring/monitor_backups_readstate_test.go b/internal/monitoring/monitor_backups_readstate_test.goindex 72ab5def00..0828cc62da 100644--- a/internal/monitoring/monitor_backups_readstate_test.go+++ b/internal/monitoring/monitor_backups_readstate_test.go@@ -1059,6 +1059,8 @@ func TestPollPBSBackups_PreservesCacheOnTransientDatastoreError(t *testing.T) {status intbody string}{+ {"response timeout quoting missing datastore", http.StatusRequestTimeout, "upstream API error 404: request timed out"},+ {"rate limit quoting forbidden", http.StatusTooManyRequests, "upstream API error 403: retry later"},{"server error", http.StatusInternalServerError, "temporary server issue"},{"gateway quoting forbidden", http.StatusBadGateway, "upstream API error 403: permission denied"},{"unavailable quoting missing datastore", http.StatusServiceUnavailable, "upstream API error 404: datastore does not exist"},diff --git a/internal/monitoring/pbs_namespace_continuity_test.go b/internal/monitoring/pbs_namespace_continuity_test.goindex e3d4b72dea..5285231612 100644--- a/internal/monitoring/pbs_namespace_continuity_test.go+++ b/internal/monitoring/pbs_namespace_continuity_test.go@@ -21,10 +21,12 @@ import (type pbsNamespaceContinuityFixture struct {*pbsPosturePollFixture- rootStatus, tenantStatus atomic.Int32- rootEmpty, tenantEmpty atomic.Bool- rootAt, tenantAt atomic.Int64- rootReads, tenantReads atomic.Int32+ rootStatus, tenantStatus atomic.Int32+ rootEmpty, tenantEmpty atomic.Bool+ rootAt, tenantAt atomic.Int64+ rootReads, tenantReads atomic.Int32+ tenantSnapshotStatus, namespaceStatus atomic.Int32+ failedReads atomic.Int32}func newPBSNamespaceContinuityFixture(t *testing.T, kind string) *pbsNamespaceContinuityFixture {@@ -40,21 +42,36 @@ func newPBSNamespaceContinuityFixture(t *testing.T, kind string) *pbsNamespaceCo} else {f.rootReads.Add(1)}+ fail := func(status int32) {+ f.failedReads.Add(1)+ w.Header().Set("Retry-After", "3600")+ http.Error(w, "upstream API error 404: fixture unavailable", int(status))+ }var data any = []any{}switch r.URL.Path {+ case "/api2/json/admin/datastore/archive/namespace":+ if status := f.namespaceStatus.Load(); status != 0 {+ fail(status)+ return+ }+ data = []map[string]any{{"ns": "tenant"}}case "/api2/json/admin/datastore/archive/groups":if status == http.StatusOK {_, _ = w.Write([]byte(`{"data":`))return}if status != 0 {- http.Error(w, "fixture unavailable", int(status))+ fail(status)return}if !empty {data = []map[string]any{{"backup-type": kind, "backup-id": id, "last-backup": at, "backup-count": 1}}}case "/api2/json/admin/datastore/archive/snapshots":+ if id == "101" && f.tenantSnapshotStatus.Load() != 0 {+ fail(f.tenantSnapshotStatus.Load())+ return+ }data = []map[string]any{{"backup-type": kind, "backup-id": id, "backup-time": at, "size": 1024, "files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}, "comment": "guest-" + id}}default:http.NotFound(w, r)@@ -98,7 +115,7 @@ func (f *pbsNamespaceContinuityFixture) live() map[string]models.PBSBackup {func TestPBSNamespaceTransientGroupReadKeepsLiveInventory(t *testing.T) {for _, kind := range []string{"vm", "ct"} {for _, ns := range []string{"", "tenant"} {- for _, status := range []int{http.StatusServiceUnavailable, http.StatusOK} {+ for _, status := range []int{http.StatusServiceUnavailable, http.StatusOK, http.StatusRequestTimeout, http.StatusTooManyRequests} {t.Run(fmt.Sprintf("%s/ns=%s/status=%d", kind, ns, status), func(t *testing.T) {f := newPBSNamespaceContinuityFixture(t, kind)f.collect(t, true, nil)@@ -309,11 +326,13 @@ func TestPBSNamespaceContinuityLifecycle(t *testing.T) {f := newPBSNamespaceContinuityFixture(t, "vm")frames := map[string]any{}original := map[string]models.PBSBackup{}- for _, phase := range []string{"observed", "partial", "recovered", "denied", "restored", "confirmed-empty"} {+ for _, phase := range []string{"observed", "throttled", "timed-out", "recovered", "denied", "restored", "confirmed-empty"} {switch phase {- case "partial":- f.tenantStatus.Store(http.StatusServiceUnavailable)+ case "throttled":+ f.tenantStatus.Store(http.StatusTooManyRequests)f.rootAt.Add(15 * 60)+ case "timed-out":+ f.tenantStatus.Store(http.StatusRequestTimeout)case "recovered", "restored":f.tenantStatus.Store(0)case "denied":@@ -337,7 +356,7 @@ func TestPBSNamespaceContinuityLifecycle(t *testing.T) {if phase == "observed" {original = rows}- if phase == "partial" && !reflect.DeepEqual(rows["101"], original["101"]) {+ if (phase == "throttled" || phase == "timed-out") && !reflect.DeepEqual(rows["101"], original["101"]) {t.Fatal("unread row changed")}postures := f.postures(t)@@ -356,7 +375,15 @@ func TestPBSNamespaceContinuityLifecycle(t *testing.T) {if err != nil {t.Fatal(err)}- data, err := json.Marshal(map[string]any{"runtime_sha256": fmt.Sprintf("%x", sha256.Sum256(source)), "observed_at": time.Now().UTC(), "resource_ids": f.ids, "frames": frames})+ runtimeHashes := map[string]string{"internal/monitoring/monitor_backups.go": fmt.Sprintf("%x", sha256.Sum256(source))}+ for _, file := range []string{"backup_guard.go", "../../pkg/pbs/client.go"} {+ content, err := os.ReadFile(file)+ if err != nil {+ t.Fatal(err)+ }+ runtimeHashes[file] = fmt.Sprintf("%x", sha256.Sum256(content))+ }+ data, err := json.Marshal(map[string]any{"runtime_sha256": runtimeHashes, "observed_at": time.Now().UTC(), "resource_ids": f.ids, "frames": frames})if err != nil {t.Fatal(err)}diff --git a/internal/monitoring/pbs_retryable_read_test.go b/internal/monitoring/pbs_retryable_read_test.gonew file mode 100644index 0000000000..abb1ffe4f3--- /dev/null+++ b/internal/monitoring/pbs_retryable_read_test.go@@ -0,0 +1,126 @@+package monitoring++import (+ "context"+ "fmt"+ "net/http"+ "reflect"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"+ "github.com/rcourtman/pulse-go-rewrite/pkg/pbs"+)++// These are ordinary reads of a synthetic PBS API, not a provider refusal,+// retry/replay experiment, native backup or guest operation. A rate/timeout+// response must retain only already observed files, never refresh their proof.+func TestPBSRetryableReadLifecycle(t *testing.T) {+ for _, kind := range []string{"vm", "ct"} {+ for _, stage := range []string{"namespace", "groups", "snapshots"} {+ for _, status := range []int{http.StatusRequestTimeout, http.StatusTooManyRequests} {+ t.Run(fmt.Sprintf("%s/%s/%d", kind, stage, status), func(t *testing.T) {+ f := newPBSNamespaceContinuityFixture(t, kind)+ f.collect(t, true, nil)+ before, points := f.live(), f.points(t, "pbs-main")+ key := pbsBackupGroupKey{datastore: "archive", namespace: "tenant", backupType: kind, backupID: "101"}+ cacheAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)+ f.rootAt.Add(15 * 60)+ f.tenantAt.Add(15 * 60) // Require a snapshot read without changing any TTL.+ f.setRetryableFailure(stage, status)+ for range 2 {+ f.collectRetryable(t, stage, status)+ got := f.live()+ if len(got) != 2 || !reflect.DeepEqual(got["101"], before["101"]) {+ t.Error("temporary read loss erased or changed the last accepted file")+ }+ if got["100"].BackupTime.Unix() != before["100"].BackupTime.Add(15*time.Minute).Unix() {+ t.Error("readable sibling did not advance")+ }+ if !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(cacheAt) {+ t.Error("failed read renewed the snapshot cache")+ }+ for _, prior := range points {+ if prior.Details["namespace"] != "tenant" {+ continue+ }+ found := false+ for _, now := range f.points(t, "pbs-main") {+ found = found || (prior.ID == now.ID && reflect.DeepEqual(prior, now))+ }+ if !found {+ t.Error("unread historical point lost or its evidence renewed")+ }+ }+ for _, p := range f.postures(t) {+ if p.State == recovery.ProtectionStateProtected || len(p.ProviderStates) != 1 ||+ p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsUnknown ||+ p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial {+ t.Errorf("temporary failure invented protection or denied access: %+v", p)+ }+ }+ }+ if f.failedReads.Load() != 2 {+ t.Fatal("failure triggered an extra request or hid a failed read")+ }+ f.setRetryableFailure(stage, 0)+ f.collect(t, true, nil)+ if f.live()["101"].BackupTime.Unix() != f.tenantAt.Load() {+ t.Fatal("ordinary recovery did not replace the retained file")+ }+ for _, p := range f.postures(t) {+ if p.State != recovery.ProtectionStateProtected {+ t.Error("complete read failed to restore protection")+ }+ }+ f.rootEmpty.Store(true)+ f.tenantEmpty.Store(true)+ f.collect(t, true, nil)+ if len(f.live()) != 0 || len(f.points(t, "pbs-main")) != 0 {+ t.Fatal("accepted empty read retained files that are now absent")+ }+ })+ }+ }+ }+}++func (f *pbsNamespaceContinuityFixture) setRetryableFailure(stage string, status int) {+ switch stage {+ case "namespace":+ f.namespaceStatus.Store(int32(status))+ case "groups":+ f.tenantStatus.Store(int32(status))+ case "snapshots":+ f.tenantSnapshotStatus.Store(int32(status))+ }+}++func (f *pbsNamespaceContinuityFixture) collectRetryable(t *testing.T, stage string, status int) {+ t.Helper()+ var discoveryErr error+ if stage == "namespace" {+ _, err := f.client.ListNamespaces(context.Background(), "archive", "", 0)+ if got, known := pbs.HTTPStatus(err); !known || got != status {+ t.Fatalf("namespace response lost actual status %d: %v", status, err)+ }+ discoveryErr = fmt.Errorf("discover namespaces: %w", err)+ }+ f.collect(t, stage != "namespace", discoveryErr)+}++func TestPBSRetryableReadWithoutCacheDoesNotInventFiles(t *testing.T) {+ for _, stage := range []string{"namespace", "groups", "snapshots"} {+ for _, status := range []int{http.StatusRequestTimeout, http.StatusTooManyRequests} {+ t.Run(fmt.Sprintf("%s/%d", stage, status), func(t *testing.T) {+ f := newPBSNamespaceContinuityFixture(t, "vm")+ f.setRetryableFailure(stage, status)+ f.collectRetryable(t, stage, status)+ if len(f.live()) != 1 || f.live()["101"].ID != "" || len(f.points(t, "pbs-main")) != 1 {+ t.Fatal("unread cold scope invented files or lost the readable root")+ }+ })+ }+ }+}diff --git a/pkg/pbs/client.go b/pkg/pbs/client.goindex e0330f58df..4118e0501f 100644--- a/pkg/pbs/client.go+++ b/pkg/pbs/client.go@@ -1950,7 +1950,7 @@ func (c *Client) ListNamespaces(ctx context.Context, datastore string, parentNamresp, err := c.get(ctx, path)if err != nil {// If namespace endpoint doesn't exist (older PBS versions), return empty list- if strings.Contains(err.Error(), "404") {+ if isPBSNotFoundError(err) {return []Namespace{}, nil}return nil, errdiff --git a/pkg/pbs/namespace_status_test.go b/pkg/pbs/namespace_status_test.gonew file mode 100644index 0000000000..4464172c73--- /dev/null+++ b/pkg/pbs/namespace_status_test.go@@ -0,0 +1,28 @@+package pbs++import (+ "context"+ "net/http"+ "testing"+)++func TestNamespaceCompatibilityUsesActualHTTPStatus(t *testing.T) {+ for _, status := range []int{400, 401, 403, 404, 408, 429, 500, 503} {+ t.Run(http.StatusText(status), func(t *testing.T) {+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {+ http.Error(w, "upstream API error 404: not the response status", status)+ })+ defer server.Close()+ namespaces, err := client.ListNamespaces(context.Background(), "archive", "", 0)+ if status == http.StatusNotFound {+ if err != nil || len(namespaces) != 0 {+ t.Fatalf("legacy endpoint compatibility lost: namespaces=%+v err=%v", namespaces, err)+ }+ return+ }+ if got, known := HTTPStatus(err); !known || got != status {+ t.Fatalf("body text concealed actual HTTP %d: namespaces=%+v err=%v", status, namespaces, err)+ }+ })+ }+}