Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Reject ambiguous PBS running-writer inventories

pulse-triage[bot]f55e27912158
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index bd14594ef4..ea60a08235 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -39,8 +39,12 @@ not original guest recovery, an established #3248 cause or release availability.
### Unread PBS writer inventories do not establish a terminal backup
-The running backup/sync task reader requires an explicit `data` array from a
-fully read response within the existing 8 MiB body bound. Missing/null arrays,
+The running backup/sync task reader shares the whole-response backup inventory
+admission above: an explicit `data` array in one unambiguous, fully read and
+successfully closed HTTP 200 response within the existing 8 MiB body bound.
+Duplicate envelope fields (including escaped spellings), case aliases of
+`data`/`errors`, invalid UTF-8 and other success statuses are unread; none can
+turn a discarded writer or error into an accepted empty set. Missing/null arrays,
error-bearing envelopes, trailing JSON, unread/failed-close bodies and rows
without a nonblank worker type return no writer set, even after an earlier
family or page succeeded. Known non-writer types remain filtered; a missing
@@ -56,10 +60,13 @@ snapshot identity or borrowing a completion time. This is not guest liveness,
a tested restore, an installed recovery or an established cause of #3248.
`TestClient_ListRunningDataTasksRequiresReadableInventory`, the later-page and
-complete-response controls exercise the actual reader. Normal-suite
+`TestClient_ListRunningDataTasksRequiresUnambiguousResponse`, HTTP-status and
+complete-response controls exercise the actual reader, including both writer
+families and later-page ambiguity. Normal-suite
`TestPBSWriterInventoryDoesNotInventTerminalFailure` connects VM/CT HTTP reads
through polling, raw state, durable points and posture, including failed reads
-after prior no-writer evidence and an unaffected completed sibling. Its opt-in
+after prior no-writer evidence, contradictory envelopes/non-200 responses and
+an unaffected completed sibling. Its opt-in
frames feed the actual existing Backups consumer at desktop/narrow widths for
interaction and native pixel proof. Fixtures perform only local GETs.
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index 3476a701cb..f5707eac6b 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -43,8 +43,12 @@ not original guest recovery, an established #3248 cause or release availability.
### Unread PBS writer inventories do not establish a terminal backup
-The running backup/sync task reader requires an explicit `data` array from a
-fully read response within the existing 8 MiB body bound. Missing/null arrays,
+The running backup/sync task reader shares the whole-response backup inventory
+admission above: an explicit `data` array in one unambiguous, fully read and
+successfully closed HTTP 200 response within the existing 8 MiB body bound.
+Duplicate envelope fields (including escaped spellings), case aliases of
+`data`/`errors`, invalid UTF-8 and other success statuses are unread; none can
+turn a discarded writer or error into an accepted empty set. Missing/null arrays,
error-bearing envelopes, trailing JSON, unread/failed-close bodies and rows
without a nonblank worker type return no writer set, even after an earlier
family or page succeeded. Known non-writer types remain filtered; a missing
@@ -60,10 +64,13 @@ snapshot identity or borrowing a completion time. This is not guest liveness,
a tested restore, an installed recovery or an established cause of #3248.
`TestClient_ListRunningDataTasksRequiresReadableInventory`, the later-page and
-complete-response controls exercise the actual reader. Normal-suite
+`TestClient_ListRunningDataTasksRequiresUnambiguousResponse`, HTTP-status and
+complete-response controls exercise the actual reader, including both writer
+families and later-page ambiguity. Normal-suite
`TestPBSWriterInventoryDoesNotInventTerminalFailure` connects VM/CT HTTP reads
through polling, raw state, durable points and posture, including failed reads
-after prior no-writer evidence and an unaffected completed sibling. Its opt-in
+after prior no-writer evidence, contradictory envelopes/non-200 responses and
+an unaffected completed sibling. Its opt-in
frames feed the actual existing Backups consumer at desktop/narrow widths for
interaction and native pixel proof. Fixtures perform only local GETs.
diff --git a/internal/monitoring/pbs_writer_inventory_test.go b/internal/monitoring/pbs_writer_inventory_test.go
index 2a1f6ca9c4..8fae2f77ba 100644
--- a/internal/monitoring/pbs_writer_inventory_test.go
+++ b/internal/monitoring/pbs_writer_inventory_test.go
@@ -2,6 +2,7 @@ package monitoring
import (
"context"
+ "crypto/sha256"
"encoding/json"
"fmt"
"net/http"
@@ -24,8 +25,27 @@ import (
func TestPBSWriterInventoryDoesNotInventTerminalFailure(t *testing.T) {
for _, kind := range []string{"vm", "ct"} {
for _, family := range []string{"backup", "syncjob"} {
- for _, invalid := range []string{`{}`, `{"data":null}`, `{"data":[{}]}`, `{"data":[null]}`, `{"data":[],"errors":{"private":"fixture-only"}}`} {
- t.Run(kind+"/"+family+"/"+invalid, func(t *testing.T) {
+ for _, invalid := range []struct {
+ name, body string
+ status int
+ }{
+ {"missing-data", `{}`, 0},
+ {"null-data", `{"data":null}`, 0},
+ {"missing-worker", `{"data":[{}]}`, 0},
+ {"null-worker", `{"data":[null]}`, 0},
+ {"provider-error", `{"data":[],"errors":{"private":"fixture-only"}}`, 0},
+ {"duplicate-data", `{"data":null,"data":[]}`, 0},
+ {"discarded-writer", `{"data":[{"worker_type":"backup","worker_id":"archive:vm/100"}],"data":[]}`, 0},
+ {"case-data", `{"data":null,"DATA":[]}`, 0},
+ {"duplicate-errors", `{"data":[],"errors":{"private":"fixture-only"},"errors":null}`, 0},
+ {"case-errors", `{"data":[],"errors":{"private":"fixture-only"},"Errors":null}`, 0},
+ {"duplicate-metadata", `{"data":[],"total":1,"total":0}`, 0},
+ {"escaped-duplicate", `{"data":null,"d\u0061ta":[]}`, 0},
+ {"invalid-utf8", "{\"data\":[],\"metadata\":\"\xff\"}", 0},
+ {"created-response", `{"data":[]}`, http.StatusCreated},
+ {"partial-response", `{"data":[]}`, http.StatusPartialContent},
+ } {
+ t.Run(kind+"/"+family+"/"+invalid.name, func(t *testing.T) {
f := newPBSPosturePollFixture(t, 2, kind)
var phase atomic.Int32
var writes atomic.Int32
@@ -49,7 +69,10 @@ func TestPBSWriterInventoryDoesNotInventTerminalFailure(t *testing.T) {
_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{row}})
case strings.HasSuffix(r.URL.Path, "/tasks"):
if (phase.Load() == 1 || phase.Load() == 3) && r.URL.Query().Get("typefilter") == family {
- _, _ = w.Write([]byte(invalid))
+ if invalid.status != 0 {
+ w.WriteHeader(invalid.status)
+ }
+ _, _ = w.Write([]byte(invalid.body))
} else if (phase.Load() == 0 || phase.Load() == 4) && r.URL.Query().Get("typefilter") == "backup" {
_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{"worker_type": "backup", "worker_id": "archive:" + kind + "/100"}}})
} else {
@@ -117,8 +140,14 @@ func TestPBSWriterInventoryDoesNotInventTerminalFailure(t *testing.T) {
}
}
require.Zero(t, writes.Load())
- if dir := os.Getenv("PULSE_PBS_WRITER_FRAMES"); dir != "" && invalid == `{}` && family == "syncjob" {
- data, err := json.Marshal(frames)
+ if dir := os.Getenv("PULSE_PBS_WRITER_FRAMES"); dir != "" && invalid.name == "duplicate-data" && family == "syncjob" {
+ hashes := map[string]string{}
+ for _, file := range []string{"monitor_backups.go", "backup_guard.go", "../../pkg/pbs/client.go", "../../pkg/pbs/backup_inventory_response.go"} {
+ content, err := os.ReadFile(file)
+ require.NoError(t, err)
+ hashes[file] = fmt.Sprintf("%x", sha256.Sum256(content))
+ }
+ data, err := json.Marshal(map[string]any{"runtime_sha256": hashes, "resource_ids": f.ids, "frames": frames})
require.NoError(t, err)
require.NoError(t, os.WriteFile(dir+"/frames-"+kind+".json", data, 0600))
}
diff --git a/pkg/pbs/client.go b/pkg/pbs/client.go
index 894c742144..323df4fb4f 100644
--- a/pkg/pbs/client.go
+++ b/pkg/pbs/client.go
@@ -1210,27 +1210,15 @@ func (c *Client) ListRunningDataTasks(ctx context.Context) ([]RunningDataTask, e
if err != nil {
return nil, err
}
- body, readErr := readResponseBodyLimited(resp.Body)
- closeErr := resp.Body.Close()
- if readErr != nil {
- return nil, fmt.Errorf("failed to read running PBS %s tasks: %w", requestedType, readErr)
- }
- if closeErr != nil {
- return nil, fmt.Errorf("failed to close running PBS %s task response: %w", requestedType, closeErr)
- }
- var result struct {
- Data []map[string]interface{} `json:"data"`
- Errors json.RawMessage `json:"errors"`
- }
- if json.Unmarshal(body, &result) != nil || result.Data == nil ||
- (len(result.Errors) > 0 && !bytes.Equal(bytes.TrimSpace(result.Errors), []byte("null"))) {
- // Only an explicit, fully read inventory can establish no writer.
- // Keep provider bodies out of diagnostics and do not return a
- // partial writer set if a later family/page is unreadable.
- return nil, fmt.Errorf("invalid running PBS %s task inventory", requestedType)
+ // Writer absence can turn an incomplete artifact into a failed
+ // backup. Admit the same whole, unambiguous inventory envelope as
+ // the snapshot reader; last-field-wins decoding is not evidence.
+ rows, err := readBackupInventory[map[string]interface{}](resp)
+ if err != nil {
+ return nil, fmt.Errorf("invalid running PBS %s task inventory: %w", requestedType, err)
}
- for _, raw := range result.Data {
+ for _, raw := range rows {
workerType := strings.ToLower(stringJobField(raw, "worker-type", "worker_type", "type"))
if workerType == "" {
return nil, fmt.Errorf("running PBS %s task inventory contains no worker type", requestedType)
@@ -1244,7 +1232,7 @@ func (c *Client) ListRunningDataTasks(ctx context.Context) ([]RunningDataTask, e
StartTime: int64JobField(raw, "starttime", "start-time", "start_time"),
})
}
- if len(result.Data) < limit {
+ if len(rows) < limit {
break
}
if page == maxPages-1 {
diff --git a/pkg/pbs/client_http_test.go b/pkg/pbs/client_http_test.go
index f9ae9b4c1a..7bb1c3ec68 100644
--- a/pkg/pbs/client_http_test.go
+++ b/pkg/pbs/client_http_test.go
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
+ "fmt"
"io"
"net/http"
"net/http/httptest"
@@ -1169,3 +1170,86 @@ func TestBackupInventoryAcceptsCompleteEmptyResponse(t *testing.T) {
}
}
}
+
+// No-writer evidence must not depend on which duplicate/aliased field a
+// decoder picks. Reject the entire set, including earlier accepted pages or
+// families, rather than letting an ambiguous empty result end a backup.
+func TestClient_ListRunningDataTasksRequiresUnambiguousResponse(t *testing.T) {
+ oldLimit, oldMaxPages := pbsRunningTaskPageLimit, pbsRunningTaskMaxPages
+ pbsRunningTaskPageLimit, pbsRunningTaskMaxPages = 2, 2
+ t.Cleanup(func() {
+ pbsRunningTaskPageLimit, pbsRunningTaskMaxPages = oldLimit, oldMaxPages
+ })
+ for _, family := range []string{"backup", "syncjob"} {
+ for _, laterPage := range []bool{false, true} {
+ for name, body := range map[string]string{
+ "duplicate-data": `{"data":null,"data":[]}`,
+ "discarded-writer": `{"data":[{"worker_type":"backup","worker_id":"private-provider-detail"}],"data":[]}`,
+ "case-data": `{"data":null,"DATA":[]}`,
+ "duplicate-errors": `{"data":[],"errors":{"private":"private-provider-detail"},"errors":null}`,
+ "case-errors": `{"data":[],"errors":{"private":"private-provider-detail"},"Errors":null}`,
+ "duplicate-metadata": `{"data":[],"total":1,"total":0}`,
+ "escaped-duplicate": `{"data":null,"d\u0061ta":[]}`,
+ "invalid-utf8": "{\"data\":[],\"metadata\":\"\xff\"}",
+ } {
+ t.Run(fmt.Sprintf("%s/later=%t/%s", family, laterPage, name), func(t *testing.T) {
+ reads := 0
+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+ reads++
+ q := r.URL.Query()
+ if r.Method != http.MethodGet || r.URL.Path != "/api2/json/nodes/localhost/tasks" || q.Get("running") != "true" || q.Get("limit") != "2" {
+ t.Errorf("writer query changed: %s %s", r.Method, r.URL)
+ }
+ if q.Get("typefilter") == family && (!laterPage || q.Get("start") == "2") {
+ _, _ = w.Write([]byte(body))
+ return
+ }
+ rows := []map[string]any{{"worker_type": q.Get("typefilter"), "worker_id": "archive:vm/100"}}
+ if q.Get("typefilter") == family && laterPage {
+ rows = append(rows, map[string]any{"worker_type": family, "worker_id": "archive:vm/101"})
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": rows})
+ })
+ defer server.Close()
+ tasks, err := client.ListRunningDataTasks(context.Background())
+ if err == nil || tasks != nil {
+ t.Fatalf("ambiguous inventory accepted/partial writers returned: tasks=%v err=%v", tasks, err)
+ }
+ if strings.Contains(err.Error(), "private-provider-detail") {
+ t.Fatal("provider payload leaked into format diagnostic")
+ }
+ wantReads := 1
+ if family == "syncjob" {
+ wantReads++
+ }
+ if laterPage {
+ wantReads++
+ }
+ if reads != wantReads {
+ t.Fatalf("reads=%d want=%d; no ambiguous inventory retry", reads, wantReads)
+ }
+ })
+ }
+ }
+ }
+}
+
+func TestClient_ListRunningDataTasksRequiresHTTP200(t *testing.T) {
+ for _, family := range []string{"backup", "syncjob"} {
+ for _, status := range []int{http.StatusCreated, http.StatusAccepted, http.StatusNonAuthoritativeInfo, http.StatusNoContent, http.StatusPartialContent} {
+ t.Run(fmt.Sprintf("%s/%d", family, status), func(t *testing.T) {
+ client, server := newTestClient(t, func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Query().Get("typefilter") == family {
+ w.WriteHeader(status)
+ }
+ _, _ = w.Write([]byte(`{"data":[]}`))
+ })
+ defer server.Close()
+ tasks, err := client.ListRunningDataTasks(context.Background())
+ if err == nil || tasks != nil {
+ t.Fatalf("non-200 writer set admitted: tasks=%v err=%v", tasks, err)
+ }
+ })
+ }
+ }
+}