Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Retain observed PBS files across timeout and throttle responses

pulse-triage[bot]404a859356de
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 00d2282c9f..489e8e2e39 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -2816,17 +2816,35 @@ Direct PBS backup polling classifies typed API and authentication failures by
the response status exposed by `pbs.HTTPStatus`, including wrapped errors.
A 5xx gateway or server response remains transient even when its body quotes
an upstream “API error 403” or “API error 404”; that text must not erase the
-last known backup inventory. Genuine 4xx responses remain terminal under the
-existing cache policy. The legacy untyped-error fallback is unchanged.
+last known backup inventory. Typed HTTP 408 (request timeout) and 429 (rate
+limit) likewise retain already accepted files in their own unread scope. They
+cannot renew point evidence, cache age or complete/permission-granted provider
+claims, and introduce no immediate retry or cadence change. Cold unread scopes
+invent no files; readable siblings still advance and accepted empty reads still
+confirm absence. Other 4xx responses, including 401/403/404, retain the existing
+terminal cache policy. The legacy untyped-error fallback is unchanged.
+
+Namespace compatibility treats only an actual HTTP 404 as an unavailable legacy
+endpoint; a quoted 404 in another response cannot turn failed discovery into
+an accepted empty namespace list. Wrapped typed errors retain their status.
Retaining cached inventory does not establish a successful poll or fresh
backup evidence.
-Verification: `TestPollPBSBackups_PreservesCacheOnTransientDatastoreError` and
-`TestPollPBSBackups_DropsStaleCacheOnTerminalDatastoreError` in
-`internal/monitoring/monitor_backups_readstate_test.go` exercise actual HTTP
-fixtures for 500, 502 quoting 403, 503 quoting 404, and genuine 401/403/404.
-These tests prove cache retention/removal, not installed PBS wake, service
-restart, or notification receipt.
+Verification: `TestPBSRetryableReadLifecycle` and
+`TestPBSRetryableReadWithoutCacheDoesNotInventFiles` exercise real synthetic
+namespace discovery, group and snapshot HTTP 408/429 replies for VM/LXC
+history, including bodies quoting 404, repeated ordinary polls, unchanged
+historical points/cache age, partial/unknown provider evidence, sibling
+progress, complete recovery and accepted removal. The existing namespace
+transient/mixed-denial controls remain. `TestNamespaceCompatibilityUsesActualHTTPStatus`
+pins actual 400/401/403/404/408/429/500/503 replies in `pkg/pbs`.
+`TestPBSNamespaceContinuityLifecycle` exports actual producer frames for
+production Backups/Coverage desktop and narrow interaction/pixel proof.
+`TestPollPBSBackups_PreservesCacheOnTransientDatastoreError` and
+`TestPollPBSBackups_DropsStaleCacheOnTerminalDatastoreError` retain 500,
+502 quoting 403, 503 quoting 404, and genuine 401/403/404 controls. These are
+source safety proofs, not installed PBS wake, guest/backup recovery,
+notification receipt, archive restorability or release availability.
Docker alert lifecycle events pass through the shared resource history identity
writer. A full Docker source reference must reach the same canonical container
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index d6a82dbcf5..1445b51f5d 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -6848,7 +6848,8 @@
"pkg/pbs/auth_session_concurrency_test.go",
"pkg/pbs/client_http_test.go",
"pkg/pbs/client_test.go",
- "pkg/pbs/host_diagnostics_regression_test.go"
+ "pkg/pbs/host_diagnostics_regression_test.go",
+ "pkg/pbs/namespace_status_test.go"
]
},
{
@@ -6968,6 +6969,7 @@
"internal/monitoring/monitor_notification_startup_test.go",
"internal/monitoring/monitor_pve_cluster_refresh_test.go",
"internal/monitoring/monitor_pve_guest_lxc_test.go",
+ "internal/monitoring/pbs_retryable_read_test.go",
"internal/monitoring/physical_disk_roundtrip_test.go",
"internal/monitoring/proxmox_large_cluster_poll_budget_test.go",
"internal/monitoring/pve_protection_observation_test.go",
diff --git a/internal/monitoring/backup_guard.go b/internal/monitoring/backup_guard.go
index f6e1b298d2..02f985db3f 100644
--- a/internal/monitoring/backup_guard.go
+++ b/internal/monitoring/backup_guard.go
@@ -1,6 +1,7 @@
package monitoring
import (
+ "net/http"
"strings"
"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
@@ -16,13 +17,16 @@ func shouldPreservePBSBackupsWithTerminal(datastoreCount, datastoreFetches, data
}
// shouldReuseCachedPBSBackups keeps cached data for transient fetch errors and
-// avoids reuse for terminal API 4xx errors.
+// avoids reuse for terminal API 4xx errors. A response-level timeout or rate
+// limit is not a rejected scope or an accepted empty inventory. Retaining it
+// neither retries a request nor renews the cached artifact's evidence.
func shouldReuseCachedPBSBackups(err error) bool {
if err == nil {
return false
}
if status, ok := pbs.HTTPStatus(err); ok {
- return status < 400 || status >= 500
+ return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests ||
+ status < 400 || status >= 500
}
// Retain compatibility with untyped errors from older callers.
if strings.Contains(strings.ToLower(err.Error()), "api error 4") {
diff --git a/internal/monitoring/monitor_backups_readstate_test.go b/internal/monitoring/monitor_backups_readstate_test.go
index 72ab5def00..0828cc62da 100644
--- a/internal/monitoring/monitor_backups_readstate_test.go
+++ b/internal/monitoring/monitor_backups_readstate_test.go
@@ -1059,6 +1059,8 @@ func TestPollPBSBackups_PreservesCacheOnTransientDatastoreError(t *testing.T) {
status int
body string
}{
+ {"response timeout quoting missing datastore", http.StatusRequestTimeout, "upstream API error 404: request timed out"},
+ {"rate limit quoting forbidden", http.StatusTooManyRequests, "upstream API error 403: retry later"},
{"server error", http.StatusInternalServerError, "temporary server issue"},
{"gateway quoting forbidden", http.StatusBadGateway, "upstream API error 403: permission denied"},
{"unavailable quoting missing datastore", http.StatusServiceUnavailable, "upstream API error 404: datastore does not exist"},
diff --git a/internal/monitoring/pbs_namespace_continuity_test.go b/internal/monitoring/pbs_namespace_continuity_test.go
index e3d4b72dea..5285231612 100644
--- a/internal/monitoring/pbs_namespace_continuity_test.go
+++ b/internal/monitoring/pbs_namespace_continuity_test.go
@@ -21,10 +21,12 @@ import (
type pbsNamespaceContinuityFixture struct {
*pbsPosturePollFixture
- rootStatus, tenantStatus atomic.Int32
- rootEmpty, tenantEmpty atomic.Bool
- rootAt, tenantAt atomic.Int64
- rootReads, tenantReads atomic.Int32
+ rootStatus, tenantStatus atomic.Int32
+ rootEmpty, tenantEmpty atomic.Bool
+ rootAt, tenantAt atomic.Int64
+ rootReads, tenantReads atomic.Int32
+ tenantSnapshotStatus, namespaceStatus atomic.Int32
+ failedReads atomic.Int32
}
func newPBSNamespaceContinuityFixture(t *testing.T, kind string) *pbsNamespaceContinuityFixture {
@@ -40,21 +42,36 @@ func newPBSNamespaceContinuityFixture(t *testing.T, kind string) *pbsNamespaceCo
} else {
f.rootReads.Add(1)
}
+ fail := func(status int32) {
+ f.failedReads.Add(1)
+ w.Header().Set("Retry-After", "3600")
+ http.Error(w, "upstream API error 404: fixture unavailable", int(status))
+ }
var data any = []any{}
switch r.URL.Path {
+ case "/api2/json/admin/datastore/archive/namespace":
+ if status := f.namespaceStatus.Load(); status != 0 {
+ fail(status)
+ return
+ }
+ data = []map[string]any{{"ns": "tenant"}}
case "/api2/json/admin/datastore/archive/groups":
if status == http.StatusOK {
_, _ = w.Write([]byte(`{"data":`))
return
}
if status != 0 {
- http.Error(w, "fixture unavailable", int(status))
+ fail(status)
return
}
if !empty {
data = []map[string]any{{"backup-type": kind, "backup-id": id, "last-backup": at, "backup-count": 1}}
}
case "/api2/json/admin/datastore/archive/snapshots":
+ if id == "101" && f.tenantSnapshotStatus.Load() != 0 {
+ fail(f.tenantSnapshotStatus.Load())
+ return
+ }
data = []map[string]any{{"backup-type": kind, "backup-id": id, "backup-time": at, "size": 1024, "files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}, "comment": "guest-" + id}}
default:
http.NotFound(w, r)
@@ -98,7 +115,7 @@ func (f *pbsNamespaceContinuityFixture) live() map[string]models.PBSBackup {
func TestPBSNamespaceTransientGroupReadKeepsLiveInventory(t *testing.T) {
for _, kind := range []string{"vm", "ct"} {
for _, ns := range []string{"", "tenant"} {
- for _, status := range []int{http.StatusServiceUnavailable, http.StatusOK} {
+ for _, status := range []int{http.StatusServiceUnavailable, http.StatusOK, http.StatusRequestTimeout, http.StatusTooManyRequests} {
t.Run(fmt.Sprintf("%s/ns=%s/status=%d", kind, ns, status), func(t *testing.T) {
f := newPBSNamespaceContinuityFixture(t, kind)
f.collect(t, true, nil)
@@ -309,11 +326,13 @@ func TestPBSNamespaceContinuityLifecycle(t *testing.T) {
f := newPBSNamespaceContinuityFixture(t, "vm")
frames := map[string]any{}
original := map[string]models.PBSBackup{}
- for _, phase := range []string{"observed", "partial", "recovered", "denied", "restored", "confirmed-empty"} {
+ for _, phase := range []string{"observed", "throttled", "timed-out", "recovered", "denied", "restored", "confirmed-empty"} {
switch phase {
- case "partial":
- f.tenantStatus.Store(http.StatusServiceUnavailable)
+ case "throttled":
+ f.tenantStatus.Store(http.StatusTooManyRequests)
f.rootAt.Add(15 * 60)
+ case "timed-out":
+ f.tenantStatus.Store(http.StatusRequestTimeout)
case "recovered", "restored":
f.tenantStatus.Store(0)
case "denied":
@@ -337,7 +356,7 @@ func TestPBSNamespaceContinuityLifecycle(t *testing.T) {
if phase == "observed" {
original = rows
}
- if phase == "partial" && !reflect.DeepEqual(rows["101"], original["101"]) {
+ if (phase == "throttled" || phase == "timed-out") && !reflect.DeepEqual(rows["101"], original["101"]) {
t.Fatal("unread row changed")
}
postures := f.postures(t)
@@ -356,7 +375,15 @@ func TestPBSNamespaceContinuityLifecycle(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- data, err := json.Marshal(map[string]any{"runtime_sha256": fmt.Sprintf("%x", sha256.Sum256(source)), "observed_at": time.Now().UTC(), "resource_ids": f.ids, "frames": frames})
+ runtimeHashes := map[string]string{"internal/monitoring/monitor_backups.go": fmt.Sprintf("%x", sha256.Sum256(source))}
+ for _, file := range []string{"backup_guard.go", "../../pkg/pbs/client.go"} {
+ content, err := os.ReadFile(file)
+ if err != nil {
+ t.Fatal(err)
+ }
+ runtimeHashes[file] = fmt.Sprintf("%x", sha256.Sum256(content))
+ }
+ data, err := json.Marshal(map[string]any{"runtime_sha256": runtimeHashes, "observed_at": time.Now().UTC(), "resource_ids": f.ids, "frames": frames})
if err != nil {
t.Fatal(err)
}
diff --git a/internal/monitoring/pbs_retryable_read_test.go b/internal/monitoring/pbs_retryable_read_test.go
new file mode 100644
index 0000000000..abb1ffe4f3
--- /dev/null
+++ b/internal/monitoring/pbs_retryable_read_test.go
@@ -0,0 +1,126 @@
+package monitoring
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
+)
+
+// These are ordinary reads of a synthetic PBS API, not a provider refusal,
+// retry/replay experiment, native backup or guest operation. A rate/timeout
+// response must retain only already observed files, never refresh their proof.
+func TestPBSRetryableReadLifecycle(t *testing.T) {
+ for _, kind := range []string{"vm", "ct"} {
+ for _, stage := range []string{"namespace", "groups", "snapshots"} {
+ for _, status := range []int{http.StatusRequestTimeout, http.StatusTooManyRequests} {
+ t.Run(fmt.Sprintf("%s/%s/%d", kind, stage, status), func(t *testing.T) {
+ f := newPBSNamespaceContinuityFixture(t, kind)
+ f.collect(t, true, nil)
+ before, points := f.live(), f.points(t, "pbs-main")
+ key := pbsBackupGroupKey{datastore: "archive", namespace: "tenant", backupType: kind, backupID: "101"}
+ cacheAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)
+ f.rootAt.Add(15 * 60)
+ f.tenantAt.Add(15 * 60) // Require a snapshot read without changing any TTL.
+ f.setRetryableFailure(stage, status)
+ for range 2 {
+ f.collectRetryable(t, stage, status)
+ got := f.live()
+ if len(got) != 2 || !reflect.DeepEqual(got["101"], before["101"]) {
+ t.Error("temporary read loss erased or changed the last accepted file")
+ }
+ if got["100"].BackupTime.Unix() != before["100"].BackupTime.Add(15*time.Minute).Unix() {
+ t.Error("readable sibling did not advance")
+ }
+ if !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(cacheAt) {
+ t.Error("failed read renewed the snapshot cache")
+ }
+ for _, prior := range points {
+ if prior.Details["namespace"] != "tenant" {
+ continue
+ }
+ found := false
+ for _, now := range f.points(t, "pbs-main") {
+ found = found || (prior.ID == now.ID && reflect.DeepEqual(prior, now))
+ }
+ if !found {
+ t.Error("unread historical point lost or its evidence renewed")
+ }
+ }
+ for _, p := range f.postures(t) {
+ if p.State == recovery.ProtectionStateProtected || len(p.ProviderStates) != 1 ||
+ p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsUnknown ||
+ p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial {
+ t.Errorf("temporary failure invented protection or denied access: %+v", p)
+ }
+ }
+ }
+ if f.failedReads.Load() != 2 {
+ t.Fatal("failure triggered an extra request or hid a failed read")
+ }
+ f.setRetryableFailure(stage, 0)
+ f.collect(t, true, nil)
+ if f.live()["101"].BackupTime.Unix() != f.tenantAt.Load() {
+ t.Fatal("ordinary recovery did not replace the retained file")
+ }
+ for _, p := range f.postures(t) {
+ if p.State != recovery.ProtectionStateProtected {
+ t.Error("complete read failed to restore protection")
+ }
+ }
+ f.rootEmpty.Store(true)
+ f.tenantEmpty.Store(true)
+ f.collect(t, true, nil)
+ if len(f.live()) != 0 || len(f.points(t, "pbs-main")) != 0 {
+ t.Fatal("accepted empty read retained files that are now absent")
+ }
+ })
+ }
+ }
+ }
+}
+
+func (f *pbsNamespaceContinuityFixture) setRetryableFailure(stage string, status int) {
+ switch stage {
+ case "namespace":
+ f.namespaceStatus.Store(int32(status))
+ case "groups":
+ f.tenantStatus.Store(int32(status))
+ case "snapshots":
+ f.tenantSnapshotStatus.Store(int32(status))
+ }
+}
+
+func (f *pbsNamespaceContinuityFixture) collectRetryable(t *testing.T, stage string, status int) {
+ t.Helper()
+ var discoveryErr error
+ if stage == "namespace" {
+ _, err := f.client.ListNamespaces(context.Background(), "archive", "", 0)
+ if got, known := pbs.HTTPStatus(err); !known || got != status {
+ t.Fatalf("namespace response lost actual status %d: %v", status, err)
+ }
+ discoveryErr = fmt.Errorf("discover namespaces: %w", err)
+ }
+ f.collect(t, stage != "namespace", discoveryErr)
+}
+
+func TestPBSRetryableReadWithoutCacheDoesNotInventFiles(t *testing.T) {
+ for _, stage := range []string{"namespace", "groups", "snapshots"} {
+ for _, status := range []int{http.StatusRequestTimeout, http.StatusTooManyRequests} {
+ t.Run(fmt.Sprintf("%s/%d", stage, status), func(t *testing.T) {
+ f := newPBSNamespaceContinuityFixture(t, "vm")
+ f.setRetryableFailure(stage, status)
+ f.collectRetryable(t, stage, status)
+ if len(f.live()) != 1 || f.live()["101"].ID != "" || len(f.points(t, "pbs-main")) != 1 {
+ t.Fatal("unread cold scope invented files or lost the readable root")
+ }
+ })
+ }
+ }
+}
diff --git a/pkg/pbs/client.go b/pkg/pbs/client.go
index e0330f58df..4118e0501f 100644
--- a/pkg/pbs/client.go
+++ b/pkg/pbs/client.go
@@ -1950,7 +1950,7 @@ func (c *Client) ListNamespaces(ctx context.Context, datastore string, parentNam
resp, err := c.get(ctx, path)
if err != nil {
// If namespace endpoint doesn't exist (older PBS versions), return empty list
- if strings.Contains(err.Error(), "404") {
+ if isPBSNotFoundError(err) {
return []Namespace{}, nil
}
return nil, err
diff --git a/pkg/pbs/namespace_status_test.go b/pkg/pbs/namespace_status_test.go
new file mode 100644
index 0000000000..4464172c73
--- /dev/null
+++ b/pkg/pbs/namespace_status_test.go
@@ -0,0 +1,28 @@
+package pbs
+
+import (
+ "context"
+ "net/http"
+ "testing"
+)
+
+func TestNamespaceCompatibilityUsesActualHTTPStatus(t *testing.T) {
+ for _, status := range []int{400, 401, 403, 404, 408, 429, 500, 503} {
+ t.Run(http.StatusText(status), func(t *testing.T) {
+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "upstream API error 404: not the response status", status)
+ })
+ defer server.Close()
+ namespaces, err := client.ListNamespaces(context.Background(), "archive", "", 0)
+ if status == http.StatusNotFound {
+ if err != nil || len(namespaces) != 0 {
+ t.Fatalf("legacy endpoint compatibility lost: namespaces=%+v err=%v", namespaces, err)
+ }
+ return
+ }
+ if got, known := HTTPStatus(err); !known || got != status {
+ t.Fatalf("body text concealed actual HTTP %d: namespaces=%+v err=%v", status, namespaces, err)
+ }
+ })
+ }
+}