Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep unread PVE task history from erasing backup intent

pulse-triage[bot]eab2db62cfcd
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index 700845be31..cf23f4c56c 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -21,6 +21,36 @@
## Purpose
+### Unread PVE task history cannot establish absence or completion
+
+The paginated backup-history collector admits only a complete, bounded HTTP200
+response with an explicit task array and unambiguous, scoped task identities and
+times. Missing/null inventories, duplicate or case-conflicting fields, invalid
+rows, contradictory errors, trailing data and unfinished bodies are unread,
+not empty history. Audit authority likewise requires a whole unambiguous
+response granting `Sys.Audit` on the requested node; a valid prefix cannot
+qualify collection completeness. Valid empty arrays, independent accepted pages,
+provider metadata, multi-guest jobs and existing pagination bounds remain valid.
+
+Live tasks on a transiently unread/offline node retain only their last accepted
+connection-scoped observations. Accepted IDs win, other nodes still advance, and
+carried rows neither renew `ObservedAt` nor re-enter durable ingestion. Accepted
+empty reads and removed nodes withdraw their live rows. Explicit access denial
+withdraws that node's live rows and accepted page prefix; a later failed read
+cannot resurrect them. History coverage remains incomplete until its existing
+collection/authority checks pass. Retained running intent continues to keep its
+writer's partial archive from claiming completion. No new endpoint, field, token
+scope, policy, store/schema migration or guest/backup operation is introduced.
+
+Normal-suite `backup_history_response_test.go` covers adverse envelopes, rows,
+body completion, audit evidence and later-page failures; pagination controls
+remain required. `backup_task_inventory_lifecycle_test.go` exercises the actual
+HTTP client and poller through unread/empty/recovery/denial, connection isolation,
+fresh-page precedence and VM/LXC partial-archive ingestion. Desktop/narrow browser
+proof consumes those actual task frames in the production Backups table. These
+are source continuity and attribution proofs, not restore availability, original
+guest recovery or the cause of a reported freeze or stale date.
+
### Fleet protection reads bound their request fan-out
Coverage reads keep the existing 200-subject batch size and issue at most four
diff --git a/internal/monitoring/backup_task_history.go b/internal/monitoring/backup_task_history.go
index 896ac7764d..acdeeb707c 100644
--- a/internal/monitoring/backup_task_history.go
+++ b/internal/monitoring/backup_task_history.go
@@ -38,7 +38,7 @@ type backupTaskHistoryClient interface {
GetBackupTaskHistory(context.Context, time.Time) (proxmox.BackupTaskHistory, error)
}
-func (m *Monitor) readBackupTaskHistory(ctx context.Context, instance string, client PVEClientInterface) ([]proxmox.Task, model.BackupHistoryCollection, error) {
+func (m *Monitor) readBackupTaskHistory(ctx context.Context, instance string, client PVEClientInterface) (proxmox.BackupTaskHistory, model.BackupHistoryCollection, error) {
observation := model.BackupHistoryCollection{Provider: recovery.ProviderProxmoxPVE, Instance: instance}
m.mu.RLock()
interval := time.Duration(0)
@@ -92,14 +92,42 @@ func (m *Monitor) readBackupTaskHistory(ctx context.Context, instance string, cl
if len(history.Nodes) == 0 && err == nil {
observation.Reason = "no_nodes_observed"
}
- return history.Tasks, observation, err
+ return history, observation, err
}
// Older provider adapters remain useful for live monitoring but cannot
// establish report completeness without collection-quality metadata.
tasks, err := client.GetBackupTasks(ctx)
observation.ObservedAt = time.Now().UTC()
observation.Reason = "collection_quality_unavailable"
- return tasks, observation, err
+ return proxmox.BackupTaskHistory{Tasks: tasks}, observation, err
+}
+
+// Unread is not an authoritative empty inventory. Carry forward only the last
+// accepted live observations from that connection's failed/offline nodes. New
+// accepted pages win by exact ID; retained rows never re-enter durable ingestion
+// or acquire a new ObservedAt. A valid empty read, removed node or explicit
+// access denial withdraws its old live rows instead of resurrecting them.
+func (m *Monitor) retainUnreadBackupTasks(instance string, nodes []proxmox.BackupTaskHistoryNode, tasks []models.BackupTask) []models.BackupTask {
+ unread := make(map[string]bool)
+ for _, node := range nodes {
+ if node.Reason == "collection_failed" || node.Reason == "node_offline" {
+ unread[node.Node] = true
+ }
+ }
+ if len(unread) == 0 {
+ return tasks
+ }
+ seen := make(map[string]bool, len(tasks))
+ for _, task := range tasks {
+ seen[task.ID] = true
+ }
+ for _, task := range m.state.GetSnapshot().PVEBackups.BackupTasks {
+ if task.Instance == instance && unread[task.Node] && !seen[task.ID] {
+ tasks = append(tasks, task)
+ seen[task.ID] = true
+ }
+ }
+ return tasks
}
func (m *Monitor) persistBackupTaskHistory(ctx context.Context, points []recovery.RecoveryPoint, observation model.BackupHistoryCollection) {
diff --git a/internal/monitoring/backup_task_inventory_lifecycle_test.go b/internal/monitoring/backup_task_inventory_lifecycle_test.go
new file mode 100644
index 0000000000..3a6b462878
--- /dev/null
+++ b/internal/monitoring/backup_task_inventory_lifecycle_test.go
@@ -0,0 +1,190 @@
+package monitoring
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+ "github.com/stretchr/testify/require"
+)
+
+// Actual HTTP client -> history collector -> monitor -> owning snapshot.
+// Fixtures perform reads only and never start a task or contact a real PVE node.
+func TestBackupHistoryUnreadNodeRetainsAcceptedTasksWithoutRenewal(t *testing.T) {
+ var phase atomic.Int32
+ start := time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Second)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ p := phase.Load()
+ switch r.URL.Path {
+ case "/api2/json/nodes":
+ fmt.Fprint(w, `{"data":[{"node":"node-a","status":"online"},{"node":"node-b","status":"online"}]}`)
+ case "/api2/json/access/permissions":
+ json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{r.URL.Query().Get("path"): map[string]int{"Sys.Audit": 1}}})
+ case "/api2/json/nodes/node-a/tasks":
+ switch p {
+ case 1, 3:
+ fmt.Fprint(w, `{"data":null}`)
+ case 2:
+ fmt.Fprint(w, `{"data":[]}`)
+ case 5:
+ w.WriteHeader(http.StatusForbidden)
+ case 6:
+ w.WriteHeader(http.StatusServiceUnavailable)
+ default:
+ json.NewEncoder(w).Encode(map[string]any{"data": []proxmox.Task{{UPID: "task-a", Node: "node-a", Type: "vzdump", ID: "101", StartTime: start.Unix(), Status: "running"}}})
+ }
+ case "/api2/json/nodes/node-b/tasks":
+ status := "OK"
+ if p > 0 {
+ status = "ERROR: independent archive unavailable"
+ }
+ json.NewEncoder(w).Encode(map[string]any{"data": []proxmox.Task{{UPID: "task-b", Node: "node-b", Type: "vzdump", ID: "102", StartTime: start.Unix(), EndTime: start.Add(time.Minute).Unix(), Status: status}}})
+ default:
+ w.WriteHeader(http.StatusNotFound)
+ }
+ }))
+ defer server.Close()
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pam!readonly", TokenValue: "fixture"})
+ require.NoError(t, err)
+ m := &Monitor{state: models.NewState()}
+ peer := models.BackupTask{ID: "peer-task", Instance: "independent-site", Node: "node-a", VMID: 101, Type: "vzdump", Status: "OK", StartTime: start, EndTime: start.Add(time.Minute), ObservedAt: start}
+ m.state.UpdateBackupTasksForInstance(peer.Instance, []models.BackupTask{peer})
+ frames := map[string]any{}
+ var accepted models.BackupTask
+ for i, name := range []string{"accepted", "unread", "empty", "unread-after-empty", "recovered", "denied", "unavailable-after-denial", "recovered-again"} {
+ phase.Store(int32(i))
+ m.pollBackupTasks(context.Background(), "site", client)
+ snapshot := m.state.GetSnapshot()
+ byID := map[string]models.BackupTask{}
+ for _, task := range snapshot.PVEBackups.BackupTasks {
+ byID[task.ID] = task
+ }
+ require.Equal(t, peer, byID[peer.ID], "same node and VMID on another connection must remain untouched")
+ b := byID["site-task-b"]
+ require.Equal(t, "node-b", b.Node)
+ require.Equal(t, "site", b.Instance)
+ require.Equal(t, 102, b.VMID)
+ if i > 0 {
+ require.Equal(t, "ERROR: independent archive unavailable", b.Status, "usable independent node evidence must advance")
+ }
+ a, present := byID["site-task-a"]
+ switch i {
+ case 0, 4, 7:
+ require.True(t, present)
+ require.Equal(t, "running", a.Status)
+ require.Equal(t, start, a.StartTime)
+ require.True(t, a.EndTime.IsZero())
+ require.False(t, a.ObservedAt.IsZero())
+ accepted = a
+ case 1:
+ require.True(t, present, "unread is not an authoritative empty task inventory")
+ require.Equal(t, accepted, a, "carry-forward cannot renew time, completion or scope")
+ default:
+ require.False(t, present, "valid empty/authority withdrawal cannot resurrect from an old unread cache")
+ }
+ frames[name] = map[string]any{"pve": map[string]any{"data": snapshot.PVEBackups}}
+ }
+ if os.Getenv("PULSE_TASK_INVENTORY_FRAMES") == "1" {
+ b, err := json.Marshal(frames)
+ require.NoError(t, err)
+ t.Log("TASK_INVENTORY_FRAMES=" + string(b))
+ }
+}
+
+func TestBackupHistoryRetentionIsNodeScopedAndFreshEvidenceWins(t *testing.T) {
+ old := time.Now().UTC().Add(-time.Hour)
+ m := &Monitor{state: models.NewState()}
+ m.state.UpdateBackupTasksForInstance("site", []models.BackupTask{
+ {ID: "site-running", Instance: "site", Node: "unread", Type: "vzdump", VMID: 101, Status: "running", StartTime: old, ObservedAt: old},
+ {ID: "site-retained", Instance: "site", Node: "unread", Type: "vzdump", VMID: 102, Status: "OK", StartTime: old, EndTime: old.Add(time.Minute), ObservedAt: old},
+ {ID: "site-offline", Instance: "site", Node: "offline", Type: "vzdump", VMID: 103, Status: "running", StartTime: old, ObservedAt: old},
+ {ID: "site-removed", Instance: "site", Node: "removed", Type: "vzdump", VMID: 104, Status: "OK", StartTime: old, ObservedAt: old},
+ })
+ client := &historyPVEClient{history: proxmox.BackupTaskHistory{
+ Nodes: []proxmox.BackupTaskHistoryNode{{Node: "unread", Reason: "collection_failed"}, {Node: "offline", Reason: "node_offline"}, {Node: "empty", Complete: true}},
+ Tasks: []proxmox.Task{{UPID: "running", Node: "unread", Type: "vzdump", ID: "101", Status: "OK", StartTime: old.Unix(), EndTime: old.Add(time.Minute).Unix()}},
+ }}
+ m.pollBackupTasks(context.Background(), "site", client)
+ byID := map[string]models.BackupTask{}
+ for _, task := range m.state.GetSnapshot().PVEBackups.BackupTasks {
+ byID[task.ID] = task
+ }
+ require.Len(t, byID, 3)
+ require.Equal(t, "OK", byID["site-running"].Status, "accepted earlier page wins over retained running state")
+ require.True(t, byID["site-running"].ObservedAt.After(old))
+ require.Equal(t, old, byID["site-retained"].ObservedAt)
+ require.Equal(t, old, byID["site-offline"].ObservedAt)
+ require.NotContains(t, byID, "site-removed", "a removed node is not a failed read")
+}
+
+func TestBackupHistoryUnreadTaskCannotCompleteRunningArchive(t *testing.T) {
+ for _, kind := range []string{"qemu", "lxc"} {
+ t.Run(kind, func(t *testing.T) {
+ f, storageClient := localArchiveIdentityFixture(t, kind)
+ now := time.Now().UTC().Truncate(time.Second)
+ f.client.at = now.Add(-time.Minute)
+ var phase atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/api2/json/nodes":
+ fmt.Fprint(w, `{"data":[{"node":"node-a","status":"online"},{"node":"node-b","status":"online"}]}`)
+ case "/api2/json/access/permissions":
+ json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{r.URL.Query().Get("path"): map[string]int{"Sys.Audit": 1}}})
+ case "/api2/json/nodes/node-a/tasks":
+ fmt.Fprint(w, `{"data":[]}`)
+ case "/api2/json/nodes/node-b/tasks":
+ if phase.Load() == 1 {
+ fmt.Fprint(w, `{"data":null}`)
+ return
+ }
+ task := proxmox.Task{UPID: "writer", Node: "node-b", Type: "vzdump", ID: "101", StartTime: now.Add(-10 * time.Minute).Unix(), Status: "running"}
+ if phase.Load() == 2 {
+ task.Status, task.EndTime = "OK", now.Unix()
+ }
+ json.NewEncoder(w).Encode(map[string]any{"data": []proxmox.Task{task}})
+ default:
+ w.WriteHeader(http.StatusNotFound)
+ }
+ }))
+ defer server.Close()
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pam!readonly", TokenValue: "fixture"})
+ require.NoError(t, err)
+ var before models.BackupTask
+ for i := range 3 {
+ phase.Store(int32(i))
+ f.m.pollBackupTasks(context.Background(), "cluster", client)
+ f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", storageClient, f.nodes, f.online)
+ f.wait(t)
+ snapshot, points := f.m.state.GetSnapshot(), f.points(t)
+ require.Len(t, snapshot.PVEBackups.BackupTasks, 1)
+ if i == 0 {
+ before = snapshot.PVEBackups.BackupTasks[0]
+ } else if i == 1 {
+ require.Equal(t, before, snapshot.PVEBackups.BackupTasks[0], "unread task cannot refresh backup intent")
+ }
+ require.Len(t, snapshot.PVEBackups.StorageBackups, 2)
+ for _, archive := range snapshot.PVEBackups.StorageBackups {
+ point := points["pve-backup:"+archive.ID]
+ partial := archive.StorageNode == "node-b" && i < 2
+ require.Equal(t, partial, archive.InProgress)
+ if partial {
+ require.Equal(t, recovery.OutcomeRunning, point.Outcome)
+ require.Nil(t, point.CompletedAt, "an unread task list cannot turn the writer's partial archive into success")
+ } else {
+ require.Equal(t, recovery.OutcomeSuccess, point.Outcome)
+ require.NotNil(t, point.CompletedAt, "independent completed archive and ordinary completion stay usable")
+ }
+ }
+ }
+ })
+ }
+}
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index e2217c0451..bb765d8281 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -2605,7 +2605,7 @@ func appendPBSBackupsWithinLimit(backups []models.PBSBackup, additions []models.
func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, client PVEClientInterface) {
log.Debug().Str("instance", instanceName).Msg("polling backup tasks")
- tasks, collection, err := m.readBackupTaskHistory(ctx, instanceName, client)
+ history, collection, err := m.readBackupTaskHistory(ctx, instanceName, client)
if err != nil {
m.persistBackupTaskHistory(ctx, nil, collection)
monErr := errors.WrapAPIError("get_backup_tasks", instanceName, err, 0)
@@ -2619,7 +2619,7 @@ func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, clie
liveJobIDs := make(map[string]bool)
var jobTasks []models.BackupTask
jobUPIDs := make(map[string]string)
- for _, task := range tasks {
+ for _, task := range history.Tasks {
// Extract VMID from task ID (format: "UPID:node:pid:starttime:type:vmid:user@realm:")
vmid := 0
if task.ID != "" {
@@ -2674,7 +2674,7 @@ func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, clie
}
// Update state with new backup tasks for this instance
- m.state.UpdateBackupTasksForInstance(instanceName, liveTasks)
+ m.state.UpdateBackupTasksForInstance(instanceName, m.retainUnreadBackupTasks(instanceName, history.Nodes, liveTasks))
// Retain execution history before qualifying collection coverage. Task rows
// are not reconciled against the provider's shrinking inventory window.
diff --git a/pkg/proxmox/backup_history.go b/pkg/proxmox/backup_history.go
index ddde6f03c3..c0f662cc5d 100644
--- a/pkg/proxmox/backup_history.go
+++ b/pkg/proxmox/backup_history.go
@@ -2,8 +2,8 @@ package proxmox
import (
"context"
- "encoding/json"
"fmt"
+ "net/http"
"net/url"
"time"
)
@@ -38,24 +38,24 @@ func (c *Client) GetBackupTaskHistory(ctx context.Context, since time.Time) (Bac
continue
}
seen := map[string]bool{}
+ var nodeTasks []Task
for page := 0; page < 30; page++ {
params := url.Values{"typefilter": {"vzdump"}, "source": {"all"}, "limit": {"200"}, "start": {fmt.Sprint(page * 200)}, "since": {fmt.Sprint(since.Unix())}, "until": {fmt.Sprint(until)}}
resp, readErr := c.get(ctx, "/nodes/"+url.PathEscape(node.Node)+"/tasks?"+params.Encode())
if readErr != nil {
observation.Reason = "collection_failed"
+ if status, known := APIErrorStatus(readErr); known && (status == http.StatusUnauthorized || status == http.StatusForbidden) {
+ observation.Reason = "access_denied"
+ }
break
}
- var result struct {
- Data []Task `json:"data"`
- }
- readErr = json.NewDecoder(resp.Body).Decode(&result)
- resp.Body.Close()
+ tasks, readErr := readBackupTaskInventory(resp, node.Node)
if readErr != nil {
observation.Reason = "collection_failed"
break
}
added := 0
- for _, task := range result.Data {
+ for _, task := range tasks {
if task.Type != "vzdump" || task.UPID == "" || seen[task.UPID] {
continue
}
@@ -63,10 +63,10 @@ func (c *Client) GetBackupTaskHistory(ctx context.Context, since time.Time) (Bac
if task.Node == "" {
task.Node = node.Node
}
- history.Tasks = append(history.Tasks, task)
+ nodeTasks = append(nodeTasks, task)
added++
}
- if len(result.Data) < 200 {
+ if len(tasks) < 200 {
observation.Complete = true
break
}
@@ -79,6 +79,9 @@ func (c *Client) GetBackupTaskHistory(ctx context.Context, since time.Time) (Bac
observation.Complete = false
observation.Reason = "audit_permission_unconfirmed"
}
+ if observation.Reason != "access_denied" {
+ history.Tasks = append(history.Tasks, nodeTasks...)
+ }
history.Nodes = append(history.Nodes, observation)
}
return history, nil
@@ -90,14 +93,7 @@ func (c *Client) backupHistoryAuditPermission(ctx context.Context, node string)
if err != nil {
return false
}
- defer resp.Body.Close()
- var result struct {
- Data map[string]map[string]int `json:"data"`
- }
- if json.NewDecoder(resp.Body).Decode(&result) != nil {
- return false
- }
- return result.Data[path]["Sys.Audit"] == 1
+ return readBackupHistoryAuditPermission(resp, path)
}
func (cc *ClusterClient) GetBackupTaskHistory(ctx context.Context, since time.Time) (BackupTaskHistory, error) {
diff --git a/pkg/proxmox/backup_history_response.go b/pkg/proxmox/backup_history_response.go
new file mode 100644
index 0000000000..4b6d05cb28
--- /dev/null
+++ b/pkg/proxmox/backup_history_response.go
@@ -0,0 +1,101 @@
+package proxmox
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "net/http"
+ "strconv"
+ "strings"
+ "unicode/utf8"
+)
+
+var errInvalidBackupHistoryResponse = errors.New("invalid Proxmox backup history response")
+
+// History completeness and audit authority require the whole bounded response,
+// not a decodable prefix. Never include provider data in a format diagnostic.
+func readBackupHistoryData(resp *http.Response) (json.RawMessage, error) {
+ body, readErr := readResponseBodyLimited(resp.Body)
+ closeErr := resp.Body.Close()
+ if readErr != nil || closeErr != nil {
+ return nil, errors.Join(readErr, closeErr)
+ }
+ if resp.StatusCode != http.StatusOK || !utf8.Valid(body) {
+ return nil, errInvalidBackupHistoryResponse
+ }
+ envelope, valid := inventoryResponseObject(body, "data", "errors")
+ if !valid {
+ return nil, errInvalidBackupHistoryResponse
+ }
+ if failure, exists := envelope["errors"]; exists && !bytes.Equal(bytes.TrimSpace(failure), []byte("null")) {
+ return nil, errInvalidBackupHistoryResponse
+ }
+ data := bytes.TrimSpace(envelope["data"])
+ if len(data) == 0 || bytes.Equal(data, []byte("null")) {
+ return nil, errInvalidBackupHistoryResponse
+ }
+ return data, nil
+}
+
+func readBackupTaskInventory(resp *http.Response, node string) ([]Task, error) {
+ data, err := readBackupHistoryData(resp)
+ if err != nil {
+ return nil, err
+ }
+ if data[0] != '[' {
+ return nil, errInvalidBackupHistoryResponse
+ }
+ var rows []json.RawMessage
+ if json.Unmarshal(data, &rows) != nil {
+ return nil, errInvalidBackupHistoryResponse
+ }
+ tasks := make([]Task, 0, len(rows))
+ seen := make(map[string]bool, len(rows))
+ for _, row := range rows {
+ fields, valid := inventoryResponseObject(row, "upid", "node", "pid", "pstart", "starttime", "type", "id", "user", "status", "endtime")
+ if !valid {
+ return nil, errInvalidBackupHistoryResponse
+ }
+ // Null scalar fields cannot silently become zero/empty observations.
+ for _, name := range []string{"upid", "node", "pid", "pstart", "starttime", "type", "id", "user", "status", "endtime"} {
+ if raw, present := fields[name]; present && bytes.Equal(bytes.TrimSpace(raw), []byte("null")) {
+ return nil, errInvalidBackupHistoryResponse
+ }
+ }
+ var task Task
+ if json.Unmarshal(row, &task) != nil || strings.TrimSpace(task.UPID) == "" || seen[task.UPID] ||
+ strings.TrimSpace(task.Type) == "" || task.Type != strings.TrimSpace(task.Type) || task.StartTime <= 0 ||
+ task.EndTime < 0 || (task.EndTime > 0 && task.EndTime < task.StartTime) || (task.Node != "" && task.Node != node) {
+ return nil, errInvalidBackupHistoryResponse
+ }
+ if task.Type == "vzdump" && task.ID != "" {
+ vmid, err := strconv.ParseUint(task.ID, 10, 32)
+ if err != nil || vmid == 0 {
+ return nil, errInvalidBackupHistoryResponse
+ }
+ }
+ seen[task.UPID] = true
+ // The requested node supplies an absent legacy field, never overrides a
+ // conflicting origin. An empty ID still denotes a multi-guest job.
+ task.Node = node
+ tasks = append(tasks, task)
+ }
+ return tasks, nil
+}
+
+func readBackupHistoryAuditPermission(resp *http.Response, path string) bool {
+ data, err := readBackupHistoryData(resp)
+ if err != nil {
+ return false
+ }
+ paths, valid := inventoryResponseObject(data)
+ if !valid {
+ return false
+ }
+ permissions, valid := inventoryResponseObject(paths[path], "Sys.Audit")
+ if !valid {
+ return false
+ }
+ var granted int
+ return json.Unmarshal(permissions["Sys.Audit"], &granted) == nil && granted == 1
+}
diff --git a/pkg/proxmox/backup_history_response_test.go b/pkg/proxmox/backup_history_response_test.go
new file mode 100644
index 0000000000..5adffc8812
--- /dev/null
+++ b/pkg/proxmox/backup_history_response_test.go
@@ -0,0 +1,211 @@
+package proxmox
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+const acceptedHistoryRow = `{"upid":"accepted","node":"member","type":"vzdump","id":"101","starttime":1700000000,"endtime":1700000060,"status":"OK"}`
+
+func TestBackupHistoryRejectsUnverifiedTaskInventory(t *testing.T) {
+ valid := `{"data":[` + acceptedHistoryRow + `]}`
+ for _, tc := range []struct {
+ name, body string
+ incomplete bool
+ }{
+ {"missing-data", `{}`, false},
+ {"null-data", `{"data":null}`, false},
+ {"null-envelope", `null`, false},
+ {"non-array", `{"data":{}}`, false},
+ {"error-as-success", `{"data":[],"errors":{"message":"fixture-private-detail"}}`, false},
+ {"case-data", `{"Data":[]}`, false},
+ {"duplicate-data", `{"data":[` + acceptedHistoryRow + `],"data":[]}`, false},
+ {"escaped-duplicate", `{"data":[],"\u0064ata":[]}`, false},
+ {"case-overwrite", `{"data":[` + acceptedHistoryRow + `],"Data":[]}`, false},
+ {"case-errors", `{"data":[],"Errors":null}`, false},
+ {"trailing-value", valid + ` {"data":[]}`, false},
+ {"trailing-text", valid + `fixture-private-detail`, false},
+ {"unfinished-response", valid, true},
+ {"oversized-response", valid + strings.Repeat(" ", int(maxResponseBodyBytes)), false},
+ {"invalid-utf8", `{"data":[],"metadata":"` + string([]byte{0xff}) + `"}`, false},
+ {"null-row", `{"data":[` + acceptedHistoryRow + `,null]}`, false},
+ {"duplicate-row", `{"data":[` + acceptedHistoryRow + `,` + acceptedHistoryRow + `]}`, false},
+ {"missing-upid", `{"data":[{"type":"vzdump","starttime":1700000000}]}`, false},
+ {"blank-upid", `{"data":[{"upid":" ","type":"vzdump","starttime":1700000000}]}`, false},
+ {"duplicate-upid-field", `{"data":[{"upid":"first",` + acceptedHistoryRow[1:] + `]}`, false},
+ {"case-status-field", `{"data":[{"Status":"running",` + acceptedHistoryRow[1:] + `]}`, false},
+ {"missing-type", `{"data":[{"upid":"accepted","starttime":1700000000}]}`, false},
+ {"missing-start", `{"data":[{"upid":"accepted","type":"vzdump"}]}`, false},
+ {"null-start", `{"data":[{"upid":"accepted","type":"vzdump","starttime":null}]}`, false},
+ {"negative-start", `{"data":[{"upid":"accepted","type":"vzdump","starttime":-1}]}`, false},
+ {"reversed-time", `{"data":[{"upid":"accepted","type":"vzdump","starttime":1700000000,"endtime":1600000000}]}`, false},
+ {"null-end", `{"data":[{"upid":"accepted","type":"vzdump","starttime":1700000000,"endtime":null}]}`, false},
+ {"null-status", `{"data":[{"upid":"accepted","type":"vzdump","starttime":1700000000,"status":null}]}`, false},
+ {"invalid-guest", `{"data":[{"upid":"accepted","type":"vzdump","starttime":1700000000,"id":"not-a-guest"}]}`, false},
+ {"foreign-node", `{"data":[{"upid":"accepted","node":"other","type":"vzdump","starttime":1700000000}]}`, false},
+ {"bad-row-after-good", `{"data":[` + acceptedHistoryRow + `,{"upid":"second","type":false}]}`, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ var taskReads, auditReads atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/api2/json/nodes":
+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online"}]}`)
+ case "/api2/json/nodes/member/tasks":
+ taskReads.Add(1)
+ if tc.incomplete {
+ w.Header().Set("Content-Length", fmt.Sprint(len(tc.body)+100))
+ }
+ fmt.Fprint(w, tc.body)
+ case "/api2/json/access/permissions":
+ auditReads.Add(1)
+ fmt.Fprint(w, `{"data":{"/nodes/member":{"Sys.Audit":1}}}`)
+ default:
+ w.WriteHeader(http.StatusNotFound)
+ }
+ }))
+ defer server.Close()
+ history, err := backupTestClient(t, server.URL).GetBackupTaskHistory(context.Background(), time.Unix(1600000000, 0))
+ require.NoError(t, err, "per-node failures remain explicit alongside independent evidence")
+ require.Empty(t, history.Tasks, "no prefix or partially valid page may publish tasks")
+ require.Equal(t, []BackupTaskHistoryNode{{Node: "member", Reason: "collection_failed"}}, history.Nodes)
+ require.EqualValues(t, 1, taskReads.Load(), "invalid successful responses are not retried")
+ require.Zero(t, auditReads.Load(), "an unread inventory cannot qualify by checking permissions")
+ })
+ }
+}
+
+func TestBackupHistoryRequiresCompleteAuditEvidence(t *testing.T) {
+ valid := `{"data":{"/nodes/member":{"Sys.Audit":1}}}`
+ for _, tc := range []struct {
+ name, body string
+ incomplete bool
+ }{
+ {"missing", `{}`, false},
+ {"null", `{"data":null}`, false},
+ {"trailing", valid + ` {}`, false},
+ {"unfinished", valid, true},
+ {"duplicate-data", `{"data":{},"data":{"/nodes/member":{"Sys.Audit":1}}}`, false},
+ {"case-data", `{"Data":{"/nodes/member":{"Sys.Audit":1}}}`, false},
+ {"duplicate-node", `{"data":{"/nodes/member":{},"/nodes/member":{"Sys.Audit":1}}}`, false},
+ {"duplicate-grant", `{"data":{"/nodes/member":{"Sys.Audit":0,"Sys.Audit":1}}}`, false},
+ {"case-grant", `{"data":{"/nodes/member":{"Sys.Audit":1,"sys.audit":0}}}`, false},
+ {"contradictory-error", `{"data":{"/nodes/member":{"Sys.Audit":1}},"errors":{"message":"fixture-private-detail"}}`, false},
+ {"other-path", `{"data":{"/nodes/other":{"Sys.Audit":1}}}`, false},
+ {"not-granted", `{"data":{"/nodes/member":{"Sys.Audit":0}}}`, false},
+ {"oversized", valid + strings.Repeat(" ", int(maxResponseBodyBytes)), false},
+ {"invalid-utf8", `{"data":{"/nodes/member":{"Sys.Audit":1}},"metadata":"` + string([]byte{0xff}) + `"}`, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/api2/json/nodes":
+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online"}]}`)
+ case "/api2/json/nodes/member/tasks":
+ fmt.Fprint(w, `{"data":[`+acceptedHistoryRow+`]}`)
+ case "/api2/json/access/permissions":
+ if tc.incomplete {
+ w.Header().Set("Content-Length", fmt.Sprint(len(tc.body)+100))
+ }
+ fmt.Fprint(w, tc.body)
+ default:
+ w.WriteHeader(http.StatusNotFound)
+ }
+ }))
+ defer server.Close()
+ history, err := backupTestClient(t, server.URL).GetBackupTaskHistory(context.Background(), time.Unix(1600000000, 0))
+ require.NoError(t, err)
+ require.Len(t, history.Tasks, 1, "unconfirmed audit does not erase accepted task evidence")
+ require.Equal(t, []BackupTaskHistoryNode{{Node: "member", Reason: "audit_permission_unconfirmed"}}, history.Nodes)
+ })
+ }
+}
+
+func TestBackupHistoryAcceptsExplicitInventoryAndScopedAuthority(t *testing.T) {
+ for _, body := range []string{`{"data":[]}`, `{"data":[],"errors":null,"total":0}`, `{"data":[` + acceptedHistoryRow + `],"provider-metadata":{"revision":1}}`, `{"data":[{"upid":"running","type":"vzdump","id":"","starttime":1700000000}]}`} {
+ t.Run(body, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/api2/json/nodes":
+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online"}]}`)
+ case "/api2/json/nodes/member/tasks":
+ fmt.Fprint(w, body)
+ case "/api2/json/access/permissions":
+ require.Equal(t, "/nodes/member", r.URL.Query().Get("path"))
+ fmt.Fprint(w, `{"data":{"/nodes/member":{"Sys.Audit":1,"VM.Audit":1},"/":{"Other":1}},"errors":null}`)
+ }
+ }))
+ defer server.Close()
+ history, err := backupTestClient(t, server.URL).GetBackupTaskHistory(context.Background(), time.Unix(1600000000, 0))
+ require.NoError(t, err)
+ require.Equal(t, []BackupTaskHistoryNode{{Node: "member", Complete: true}}, history.Nodes)
+ if strings.Contains(body, "upid") {
+ require.Len(t, history.Tasks, 1)
+ require.Equal(t, "member", history.Tasks[0].Node)
+ } else {
+ require.Empty(t, history.Tasks)
+ }
+ })
+ }
+}
+
+func TestBackupHistoryReadClosureAndStatus(t *testing.T) {
+ for _, status := range []int{http.StatusNoContent, http.StatusPartialContent} {
+ resp := &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(`{"data":[]}`))}
+ _, err := readBackupTaskInventory(resp, "member")
+ require.Error(t, err)
+ }
+ body := &snapshotCloseFailure{Reader: strings.NewReader(`{"data":[]}`)}
+ _, err := readBackupTaskInventory(&http.Response{StatusCode: http.StatusOK, Body: body}, "member")
+ require.Error(t, err)
+ require.True(t, body.closed)
+ for _, resp := range []*http.Response{
+ {StatusCode: http.StatusPartialContent, Body: io.NopCloser(strings.NewReader(`{"data":{"/nodes/member":{"Sys.Audit":1}}}`))},
+ {StatusCode: http.StatusOK, Body: &snapshotCloseFailure{Reader: strings.NewReader(`{"data":{"/nodes/member":{"Sys.Audit":1}}}`)}},
+ } {
+ require.False(t, readBackupHistoryAuditPermission(resp, "/nodes/member"))
+ }
+}
+
+func TestBackupHistoryMalformedLaterPageDoesNotQualifyPrefix(t *testing.T) {
+ for _, second := range []string{`{}`, `{"data":null}`, `{"data":[{"upid":"invalid","type":"vzdump"}]}`} {
+ t.Run(second, func(t *testing.T) {
+ var auditReads atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/api2/json/nodes":
+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online"}]}`)
+ case "/api2/json/nodes/member/tasks":
+ if r.URL.Query().Get("start") != "0" {
+ fmt.Fprint(w, second)
+ return
+ }
+ tasks := make([]Task, 200)
+ for i := range tasks {
+ tasks[i] = Task{UPID: fmt.Sprint(i), Type: "vzdump", ID: "101", StartTime: 1700000000}
+ }
+ json.NewEncoder(w).Encode(map[string]any{"data": tasks})
+ case "/api2/json/access/permissions":
+ auditReads.Add(1)
+ fmt.Fprint(w, `{"data":{"/nodes/member":{"Sys.Audit":1}}}`)
+ }
+ }))
+ defer server.Close()
+ history, err := backupTestClient(t, server.URL).GetBackupTaskHistory(context.Background(), time.Unix(1600000000, 0))
+ require.NoError(t, err)
+ require.Len(t, history.Tasks, 200, "accepted independent pages remain evidence, never completeness")
+ require.Equal(t, []BackupTaskHistoryNode{{Node: "member", Reason: "collection_failed"}}, history.Nodes)
+ require.Zero(t, auditReads.Load())
+ })
+ }
+}
diff --git a/pkg/proxmox/backup_history_test.go b/pkg/proxmox/backup_history_test.go
index 0aeb7b6a78..b3b1e4e4e1 100644
--- a/pkg/proxmox/backup_history_test.go
+++ b/pkg/proxmox/backup_history_test.go
@@ -20,7 +20,7 @@ func TestBackupHistoryPaginationAndCollectionQuality(t *testing.T) {
repeat bool
expected int
reason string
- }{{"full", true, false, false, 201, ""}, {"permission", false, false, false, 201, "audit_permission_unconfirmed"}, {"partial", true, true, false, 200, "collection_failed"}, {"repeated-page", true, false, true, 200, "history_truncated"}} {
+ }{{"full", true, false, false, 201, ""}, {"permission", false, false, false, 201, "audit_permission_unconfirmed"}, {"denied", true, true, false, 0, "access_denied"}, {"repeated-page", true, false, true, 200, "history_truncated"}} {
t.Run(test.name, func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@@ -49,7 +49,7 @@ func TestBackupHistoryPaginationAndCollectionQuality(t *testing.T) {
}
tasks := make([]Task, count)
for i := range tasks {
- tasks[i] = Task{UPID: fmt.Sprintf("task-%d", i+offset), Type: "vzdump"}
+ tasks[i] = Task{UPID: fmt.Sprintf("task-%d", i+offset), Type: "vzdump", StartTime: 1700000000}
}
json.NewEncoder(w).Encode(map[string]any{"data": tasks})
default: