Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Reject ambiguous PBS running-writer inventories
pulse-triage[bot]f55e27912158
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex bd14594ef4..ea60a08235 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -39,8 +39,12 @@ not original guest recovery, an established #3248 cause or release availability.### Unread PBS writer inventories do not establish a terminal backup-The running backup/sync task reader requires an explicit `data` array from a-fully read response within the existing 8 MiB body bound. Missing/null arrays,+The running backup/sync task reader shares the whole-response backup inventory+admission above: an explicit `data` array in one unambiguous, fully read and+successfully closed HTTP 200 response within the existing 8 MiB body bound.+Duplicate envelope fields (including escaped spellings), case aliases of+`data`/`errors`, invalid UTF-8 and other success statuses are unread; none can+turn a discarded writer or error into an accepted empty set. Missing/null arrays,error-bearing envelopes, trailing JSON, unread/failed-close bodies and rowswithout a nonblank worker type return no writer set, even after an earlierfamily or page succeeded. Known non-writer types remain filtered; a missing@@ -56,10 +60,13 @@ snapshot identity or borrowing a completion time. This is not guest liveness,a tested restore, an installed recovery or an established cause of #3248.`TestClient_ListRunningDataTasksRequiresReadableInventory`, the later-page and-complete-response controls exercise the actual reader. Normal-suite+`TestClient_ListRunningDataTasksRequiresUnambiguousResponse`, HTTP-status and+complete-response controls exercise the actual reader, including both writer+families and later-page ambiguity. Normal-suite`TestPBSWriterInventoryDoesNotInventTerminalFailure` connects VM/CT HTTP readsthrough polling, raw state, durable points and posture, including failed reads-after prior no-writer evidence and an unaffected completed sibling. Its opt-in+after prior no-writer evidence, contradictory envelopes/non-200 responses and+an unaffected completed sibling. Its opt-inframes feed the actual existing Backups consumer at desktop/narrow widths forinteraction and native pixel proof. Fixtures perform only local GETs.diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex 3476a701cb..f5707eac6b 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -43,8 +43,12 @@ not original guest recovery, an established #3248 cause or release availability.### Unread PBS writer inventories do not establish a terminal backup-The running backup/sync task reader requires an explicit `data` array from a-fully read response within the existing 8 MiB body bound. Missing/null arrays,+The running backup/sync task reader shares the whole-response backup inventory+admission above: an explicit `data` array in one unambiguous, fully read and+successfully closed HTTP 200 response within the existing 8 MiB body bound.+Duplicate envelope fields (including escaped spellings), case aliases of+`data`/`errors`, invalid UTF-8 and other success statuses are unread; none can+turn a discarded writer or error into an accepted empty set. Missing/null arrays,error-bearing envelopes, trailing JSON, unread/failed-close bodies and rowswithout a nonblank worker type return no writer set, even after an earlierfamily or page succeeded. Known non-writer types remain filtered; a missing@@ -60,10 +64,13 @@ snapshot identity or borrowing a completion time. This is not guest liveness,a tested restore, an installed recovery or an established cause of #3248.`TestClient_ListRunningDataTasksRequiresReadableInventory`, the later-page and-complete-response controls exercise the actual reader. Normal-suite+`TestClient_ListRunningDataTasksRequiresUnambiguousResponse`, HTTP-status and+complete-response controls exercise the actual reader, including both writer+families and later-page ambiguity. Normal-suite`TestPBSWriterInventoryDoesNotInventTerminalFailure` connects VM/CT HTTP readsthrough polling, raw state, durable points and posture, including failed reads-after prior no-writer evidence and an unaffected completed sibling. Its opt-in+after prior no-writer evidence, contradictory envelopes/non-200 responses and+an unaffected completed sibling. Its opt-inframes feed the actual existing Backups consumer at desktop/narrow widths forinteraction and native pixel proof. Fixtures perform only local GETs.diff --git a/internal/monitoring/pbs_writer_inventory_test.go b/internal/monitoring/pbs_writer_inventory_test.goindex 2a1f6ca9c4..8fae2f77ba 100644--- a/internal/monitoring/pbs_writer_inventory_test.go+++ b/internal/monitoring/pbs_writer_inventory_test.go@@ -2,6 +2,7 @@ package monitoringimport ("context"+ "crypto/sha256""encoding/json""fmt""net/http"@@ -24,8 +25,27 @@ import (func TestPBSWriterInventoryDoesNotInventTerminalFailure(t *testing.T) {for _, kind := range []string{"vm", "ct"} {for _, family := range []string{"backup", "syncjob"} {- for _, invalid := range []string{`{}`, `{"data":null}`, `{"data":[{}]}`, `{"data":[null]}`, `{"data":[],"errors":{"private":"fixture-only"}}`} {- t.Run(kind+"/"+family+"/"+invalid, func(t *testing.T) {+ for _, invalid := range []struct {+ name, body string+ status int+ }{+ {"missing-data", `{}`, 0},+ {"null-data", `{"data":null}`, 0},+ {"missing-worker", `{"data":[{}]}`, 0},+ {"null-worker", `{"data":[null]}`, 0},+ {"provider-error", `{"data":[],"errors":{"private":"fixture-only"}}`, 0},+ {"duplicate-data", `{"data":null,"data":[]}`, 0},+ {"discarded-writer", `{"data":[{"worker_type":"backup","worker_id":"archive:vm/100"}],"data":[]}`, 0},+ {"case-data", `{"data":null,"DATA":[]}`, 0},+ {"duplicate-errors", `{"data":[],"errors":{"private":"fixture-only"},"errors":null}`, 0},+ {"case-errors", `{"data":[],"errors":{"private":"fixture-only"},"Errors":null}`, 0},+ {"duplicate-metadata", `{"data":[],"total":1,"total":0}`, 0},+ {"escaped-duplicate", `{"data":null,"d\u0061ta":[]}`, 0},+ {"invalid-utf8", "{\"data\":[],\"metadata\":\"\xff\"}", 0},+ {"created-response", `{"data":[]}`, http.StatusCreated},+ {"partial-response", `{"data":[]}`, http.StatusPartialContent},+ } {+ t.Run(kind+"/"+family+"/"+invalid.name, func(t *testing.T) {f := newPBSPosturePollFixture(t, 2, kind)var phase atomic.Int32var writes atomic.Int32@@ -49,7 +69,10 @@ func TestPBSWriterInventoryDoesNotInventTerminalFailure(t *testing.T) {_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{row}})case strings.HasSuffix(r.URL.Path, "/tasks"):if (phase.Load() == 1 || phase.Load() == 3) && r.URL.Query().Get("typefilter") == family {- _, _ = w.Write([]byte(invalid))+ if invalid.status != 0 {+ w.WriteHeader(invalid.status)+ }+ _, _ = w.Write([]byte(invalid.body))} else if (phase.Load() == 0 || phase.Load() == 4) && r.URL.Query().Get("typefilter") == "backup" {_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{"worker_type": "backup", "worker_id": "archive:" + kind + "/100"}}})} else {@@ -117,8 +140,14 @@ func TestPBSWriterInventoryDoesNotInventTerminalFailure(t *testing.T) {}}require.Zero(t, writes.Load())- if dir := os.Getenv("PULSE_PBS_WRITER_FRAMES"); dir != "" && invalid == `{}` && family == "syncjob" {- data, err := json.Marshal(frames)+ if dir := os.Getenv("PULSE_PBS_WRITER_FRAMES"); dir != "" && invalid.name == "duplicate-data" && family == "syncjob" {+ hashes := map[string]string{}+ for _, file := range []string{"monitor_backups.go", "backup_guard.go", "../../pkg/pbs/client.go", "../../pkg/pbs/backup_inventory_response.go"} {+ content, err := os.ReadFile(file)+ require.NoError(t, err)+ hashes[file] = fmt.Sprintf("%x", sha256.Sum256(content))+ }+ data, err := json.Marshal(map[string]any{"runtime_sha256": hashes, "resource_ids": f.ids, "frames": frames})require.NoError(t, err)require.NoError(t, os.WriteFile(dir+"/frames-"+kind+".json", data, 0600))}diff --git a/pkg/pbs/client.go b/pkg/pbs/client.goindex 894c742144..323df4fb4f 100644--- a/pkg/pbs/client.go+++ b/pkg/pbs/client.go@@ -1210,27 +1210,15 @@ func (c *Client) ListRunningDataTasks(ctx context.Context) ([]RunningDataTask, eif err != nil {return nil, err}- body, readErr := readResponseBodyLimited(resp.Body)- closeErr := resp.Body.Close()- if readErr != nil {- return nil, fmt.Errorf("failed to read running PBS %s tasks: %w", requestedType, readErr)- }- if closeErr != nil {- return nil, fmt.Errorf("failed to close running PBS %s task response: %w", requestedType, closeErr)- }- var result struct {- Data []map[string]interface{} `json:"data"`- Errors json.RawMessage `json:"errors"`- }- if json.Unmarshal(body, &result) != nil || result.Data == nil ||- (len(result.Errors) > 0 && !bytes.Equal(bytes.TrimSpace(result.Errors), []byte("null"))) {- // Only an explicit, fully read inventory can establish no writer.- // Keep provider bodies out of diagnostics and do not return a- // partial writer set if a later family/page is unreadable.- return nil, fmt.Errorf("invalid running PBS %s task inventory", requestedType)+ // Writer absence can turn an incomplete artifact into a failed+ // backup. Admit the same whole, unambiguous inventory envelope as+ // the snapshot reader; last-field-wins decoding is not evidence.+ rows, err := readBackupInventory[map[string]interface{}](resp)+ if err != nil {+ return nil, fmt.Errorf("invalid running PBS %s task inventory: %w", requestedType, err)}- for _, raw := range result.Data {+ for _, raw := range rows {workerType := strings.ToLower(stringJobField(raw, "worker-type", "worker_type", "type"))if workerType == "" {return nil, fmt.Errorf("running PBS %s task inventory contains no worker type", requestedType)@@ -1244,7 +1232,7 @@ func (c *Client) ListRunningDataTasks(ctx context.Context) ([]RunningDataTask, eStartTime: int64JobField(raw, "starttime", "start-time", "start_time"),})}- if len(result.Data) < limit {+ if len(rows) < limit {break}if page == maxPages-1 {diff --git a/pkg/pbs/client_http_test.go b/pkg/pbs/client_http_test.goindex f9ae9b4c1a..7bb1c3ec68 100644--- a/pkg/pbs/client_http_test.go+++ b/pkg/pbs/client_http_test.go@@ -4,6 +4,7 @@ import ("context""encoding/json""errors"+ "fmt""io""net/http""net/http/httptest"@@ -1169,3 +1170,86 @@ func TestBackupInventoryAcceptsCompleteEmptyResponse(t *testing.T) {}}}++// No-writer evidence must not depend on which duplicate/aliased field a+// decoder picks. Reject the entire set, including earlier accepted pages or+// families, rather than letting an ambiguous empty result end a backup.+func TestClient_ListRunningDataTasksRequiresUnambiguousResponse(t *testing.T) {+ oldLimit, oldMaxPages := pbsRunningTaskPageLimit, pbsRunningTaskMaxPages+ pbsRunningTaskPageLimit, pbsRunningTaskMaxPages = 2, 2+ t.Cleanup(func() {+ pbsRunningTaskPageLimit, pbsRunningTaskMaxPages = oldLimit, oldMaxPages+ })+ for _, family := range []string{"backup", "syncjob"} {+ for _, laterPage := range []bool{false, true} {+ for name, body := range map[string]string{+ "duplicate-data": `{"data":null,"data":[]}`,+ "discarded-writer": `{"data":[{"worker_type":"backup","worker_id":"private-provider-detail"}],"data":[]}`,+ "case-data": `{"data":null,"DATA":[]}`,+ "duplicate-errors": `{"data":[],"errors":{"private":"private-provider-detail"},"errors":null}`,+ "case-errors": `{"data":[],"errors":{"private":"private-provider-detail"},"Errors":null}`,+ "duplicate-metadata": `{"data":[],"total":1,"total":0}`,+ "escaped-duplicate": `{"data":null,"d\u0061ta":[]}`,+ "invalid-utf8": "{\"data\":[],\"metadata\":\"\xff\"}",+ } {+ t.Run(fmt.Sprintf("%s/later=%t/%s", family, laterPage, name), func(t *testing.T) {+ reads := 0+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {+ reads+++ q := r.URL.Query()+ if r.Method != http.MethodGet || r.URL.Path != "/api2/json/nodes/localhost/tasks" || q.Get("running") != "true" || q.Get("limit") != "2" {+ t.Errorf("writer query changed: %s %s", r.Method, r.URL)+ }+ if q.Get("typefilter") == family && (!laterPage || q.Get("start") == "2") {+ _, _ = w.Write([]byte(body))+ return+ }+ rows := []map[string]any{{"worker_type": q.Get("typefilter"), "worker_id": "archive:vm/100"}}+ if q.Get("typefilter") == family && laterPage {+ rows = append(rows, map[string]any{"worker_type": family, "worker_id": "archive:vm/101"})+ }+ _ = json.NewEncoder(w).Encode(map[string]any{"data": rows})+ })+ defer server.Close()+ tasks, err := client.ListRunningDataTasks(context.Background())+ if err == nil || tasks != nil {+ t.Fatalf("ambiguous inventory accepted/partial writers returned: tasks=%v err=%v", tasks, err)+ }+ if strings.Contains(err.Error(), "private-provider-detail") {+ t.Fatal("provider payload leaked into format diagnostic")+ }+ wantReads := 1+ if family == "syncjob" {+ wantReads+++ }+ if laterPage {+ wantReads+++ }+ if reads != wantReads {+ t.Fatalf("reads=%d want=%d; no ambiguous inventory retry", reads, wantReads)+ }+ })+ }+ }+ }+}++func TestClient_ListRunningDataTasksRequiresHTTP200(t *testing.T) {+ for _, family := range []string{"backup", "syncjob"} {+ for _, status := range []int{http.StatusCreated, http.StatusAccepted, http.StatusNonAuthoritativeInfo, http.StatusNoContent, http.StatusPartialContent} {+ t.Run(fmt.Sprintf("%s/%d", family, status), func(t *testing.T) {+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Query().Get("typefilter") == family {+ w.WriteHeader(status)+ }+ _, _ = w.Write([]byte(`{"data":[]}`))+ })+ defer server.Close()+ tasks, err := client.ListRunningDataTasks(context.Background())+ if err == nil || tasks != nil {+ t.Fatalf("non-200 writer set admitted: tasks=%v err=%v", tasks, err)+ }+ })+ }+ }+}