Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep unread PVE task history from erasing backup intent
pulse-triage[bot]eab2db62cfcd
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex 700845be31..cf23f4c56c 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -21,6 +21,36 @@## Purpose+### Unread PVE task history cannot establish absence or completion++The paginated backup-history collector admits only a complete, bounded HTTP200+response with an explicit task array and unambiguous, scoped task identities and+times. Missing/null inventories, duplicate or case-conflicting fields, invalid+rows, contradictory errors, trailing data and unfinished bodies are unread,+not empty history. Audit authority likewise requires a whole unambiguous+response granting `Sys.Audit` on the requested node; a valid prefix cannot+qualify collection completeness. Valid empty arrays, independent accepted pages,+provider metadata, multi-guest jobs and existing pagination bounds remain valid.++Live tasks on a transiently unread/offline node retain only their last accepted+connection-scoped observations. Accepted IDs win, other nodes still advance, and+carried rows neither renew `ObservedAt` nor re-enter durable ingestion. Accepted+empty reads and removed nodes withdraw their live rows. Explicit access denial+withdraws that node's live rows and accepted page prefix; a later failed read+cannot resurrect them. History coverage remains incomplete until its existing+collection/authority checks pass. Retained running intent continues to keep its+writer's partial archive from claiming completion. No new endpoint, field, token+scope, policy, store/schema migration or guest/backup operation is introduced.++Normal-suite `backup_history_response_test.go` covers adverse envelopes, rows,+body completion, audit evidence and later-page failures; pagination controls+remain required. `backup_task_inventory_lifecycle_test.go` exercises the actual+HTTP client and poller through unread/empty/recovery/denial, connection isolation,+fresh-page precedence and VM/LXC partial-archive ingestion. Desktop/narrow browser+proof consumes those actual task frames in the production Backups table. These+are source continuity and attribution proofs, not restore availability, original+guest recovery or the cause of a reported freeze or stale date.+### Fleet protection reads bound their request fan-outCoverage reads keep the existing 200-subject batch size and issue at most fourdiff --git a/internal/monitoring/backup_task_history.go b/internal/monitoring/backup_task_history.goindex 896ac7764d..acdeeb707c 100644--- a/internal/monitoring/backup_task_history.go+++ b/internal/monitoring/backup_task_history.go@@ -38,7 +38,7 @@ type backupTaskHistoryClient interface {GetBackupTaskHistory(context.Context, time.Time) (proxmox.BackupTaskHistory, error)}-func (m *Monitor) readBackupTaskHistory(ctx context.Context, instance string, client PVEClientInterface) ([]proxmox.Task, model.BackupHistoryCollection, error) {+func (m *Monitor) readBackupTaskHistory(ctx context.Context, instance string, client PVEClientInterface) (proxmox.BackupTaskHistory, model.BackupHistoryCollection, error) {observation := model.BackupHistoryCollection{Provider: recovery.ProviderProxmoxPVE, Instance: instance}m.mu.RLock()interval := time.Duration(0)@@ -92,14 +92,42 @@ func (m *Monitor) readBackupTaskHistory(ctx context.Context, instance string, clif len(history.Nodes) == 0 && err == nil {observation.Reason = "no_nodes_observed"}- return history.Tasks, observation, err+ return history, observation, err}// Older provider adapters remain useful for live monitoring but cannot// establish report completeness without collection-quality metadata.tasks, err := client.GetBackupTasks(ctx)observation.ObservedAt = time.Now().UTC()observation.Reason = "collection_quality_unavailable"- return tasks, observation, err+ return proxmox.BackupTaskHistory{Tasks: tasks}, observation, err+}++// Unread is not an authoritative empty inventory. Carry forward only the last+// accepted live observations from that connection's failed/offline nodes. New+// accepted pages win by exact ID; retained rows never re-enter durable ingestion+// or acquire a new ObservedAt. A valid empty read, removed node or explicit+// access denial withdraws its old live rows instead of resurrecting them.+func (m *Monitor) retainUnreadBackupTasks(instance string, nodes []proxmox.BackupTaskHistoryNode, tasks []models.BackupTask) []models.BackupTask {+ unread := make(map[string]bool)+ for _, node := range nodes {+ if node.Reason == "collection_failed" || node.Reason == "node_offline" {+ unread[node.Node] = true+ }+ }+ if len(unread) == 0 {+ return tasks+ }+ seen := make(map[string]bool, len(tasks))+ for _, task := range tasks {+ seen[task.ID] = true+ }+ for _, task := range m.state.GetSnapshot().PVEBackups.BackupTasks {+ if task.Instance == instance && unread[task.Node] && !seen[task.ID] {+ tasks = append(tasks, task)+ seen[task.ID] = true+ }+ }+ return tasks}func (m *Monitor) persistBackupTaskHistory(ctx context.Context, points []recovery.RecoveryPoint, observation model.BackupHistoryCollection) {diff --git a/internal/monitoring/backup_task_inventory_lifecycle_test.go b/internal/monitoring/backup_task_inventory_lifecycle_test.gonew file mode 100644index 0000000000..3a6b462878--- /dev/null+++ b/internal/monitoring/backup_task_inventory_lifecycle_test.go@@ -0,0 +1,190 @@+package monitoring++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "os"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+ "github.com/stretchr/testify/require"+)++// Actual HTTP client -> history collector -> monitor -> owning snapshot.+// Fixtures perform reads only and never start a task or contact a real PVE node.+func TestBackupHistoryUnreadNodeRetainsAcceptedTasksWithoutRenewal(t *testing.T) {+ var phase atomic.Int32+ start := time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Second)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ p := phase.Load()+ switch r.URL.Path {+ case "/api2/json/nodes":+ fmt.Fprint(w, `{"data":[{"node":"node-a","status":"online"},{"node":"node-b","status":"online"}]}`)+ case "/api2/json/access/permissions":+ json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{r.URL.Query().Get("path"): map[string]int{"Sys.Audit": 1}}})+ case "/api2/json/nodes/node-a/tasks":+ switch p {+ case 1, 3:+ fmt.Fprint(w, `{"data":null}`)+ case 2:+ fmt.Fprint(w, `{"data":[]}`)+ case 5:+ w.WriteHeader(http.StatusForbidden)+ case 6:+ w.WriteHeader(http.StatusServiceUnavailable)+ default:+ json.NewEncoder(w).Encode(map[string]any{"data": []proxmox.Task{{UPID: "task-a", Node: "node-a", Type: "vzdump", ID: "101", StartTime: start.Unix(), Status: "running"}}})+ }+ case "/api2/json/nodes/node-b/tasks":+ status := "OK"+ if p > 0 {+ status = "ERROR: independent archive unavailable"+ }+ json.NewEncoder(w).Encode(map[string]any{"data": []proxmox.Task{{UPID: "task-b", Node: "node-b", Type: "vzdump", ID: "102", StartTime: start.Unix(), EndTime: start.Add(time.Minute).Unix(), Status: status}}})+ default:+ w.WriteHeader(http.StatusNotFound)+ }+ }))+ defer server.Close()+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pam!readonly", TokenValue: "fixture"})+ require.NoError(t, err)+ m := &Monitor{state: models.NewState()}+ peer := models.BackupTask{ID: "peer-task", Instance: "independent-site", Node: "node-a", VMID: 101, Type: "vzdump", Status: "OK", StartTime: start, EndTime: start.Add(time.Minute), ObservedAt: start}+ m.state.UpdateBackupTasksForInstance(peer.Instance, []models.BackupTask{peer})+ frames := map[string]any{}+ var accepted models.BackupTask+ for i, name := range []string{"accepted", "unread", "empty", "unread-after-empty", "recovered", "denied", "unavailable-after-denial", "recovered-again"} {+ phase.Store(int32(i))+ m.pollBackupTasks(context.Background(), "site", client)+ snapshot := m.state.GetSnapshot()+ byID := map[string]models.BackupTask{}+ for _, task := range snapshot.PVEBackups.BackupTasks {+ byID[task.ID] = task+ }+ require.Equal(t, peer, byID[peer.ID], "same node and VMID on another connection must remain untouched")+ b := byID["site-task-b"]+ require.Equal(t, "node-b", b.Node)+ require.Equal(t, "site", b.Instance)+ require.Equal(t, 102, b.VMID)+ if i > 0 {+ require.Equal(t, "ERROR: independent archive unavailable", b.Status, "usable independent node evidence must advance")+ }+ a, present := byID["site-task-a"]+ switch i {+ case 0, 4, 7:+ require.True(t, present)+ require.Equal(t, "running", a.Status)+ require.Equal(t, start, a.StartTime)+ require.True(t, a.EndTime.IsZero())+ require.False(t, a.ObservedAt.IsZero())+ accepted = a+ case 1:+ require.True(t, present, "unread is not an authoritative empty task inventory")+ require.Equal(t, accepted, a, "carry-forward cannot renew time, completion or scope")+ default:+ require.False(t, present, "valid empty/authority withdrawal cannot resurrect from an old unread cache")+ }+ frames[name] = map[string]any{"pve": map[string]any{"data": snapshot.PVEBackups}}+ }+ if os.Getenv("PULSE_TASK_INVENTORY_FRAMES") == "1" {+ b, err := json.Marshal(frames)+ require.NoError(t, err)+ t.Log("TASK_INVENTORY_FRAMES=" + string(b))+ }+}++func TestBackupHistoryRetentionIsNodeScopedAndFreshEvidenceWins(t *testing.T) {+ old := time.Now().UTC().Add(-time.Hour)+ m := &Monitor{state: models.NewState()}+ m.state.UpdateBackupTasksForInstance("site", []models.BackupTask{+ {ID: "site-running", Instance: "site", Node: "unread", Type: "vzdump", VMID: 101, Status: "running", StartTime: old, ObservedAt: old},+ {ID: "site-retained", Instance: "site", Node: "unread", Type: "vzdump", VMID: 102, Status: "OK", StartTime: old, EndTime: old.Add(time.Minute), ObservedAt: old},+ {ID: "site-offline", Instance: "site", Node: "offline", Type: "vzdump", VMID: 103, Status: "running", StartTime: old, ObservedAt: old},+ {ID: "site-removed", Instance: "site", Node: "removed", Type: "vzdump", VMID: 104, Status: "OK", StartTime: old, ObservedAt: old},+ })+ client := &historyPVEClient{history: proxmox.BackupTaskHistory{+ Nodes: []proxmox.BackupTaskHistoryNode{{Node: "unread", Reason: "collection_failed"}, {Node: "offline", Reason: "node_offline"}, {Node: "empty", Complete: true}},+ Tasks: []proxmox.Task{{UPID: "running", Node: "unread", Type: "vzdump", ID: "101", Status: "OK", StartTime: old.Unix(), EndTime: old.Add(time.Minute).Unix()}},+ }}+ m.pollBackupTasks(context.Background(), "site", client)+ byID := map[string]models.BackupTask{}+ for _, task := range m.state.GetSnapshot().PVEBackups.BackupTasks {+ byID[task.ID] = task+ }+ require.Len(t, byID, 3)+ require.Equal(t, "OK", byID["site-running"].Status, "accepted earlier page wins over retained running state")+ require.True(t, byID["site-running"].ObservedAt.After(old))+ require.Equal(t, old, byID["site-retained"].ObservedAt)+ require.Equal(t, old, byID["site-offline"].ObservedAt)+ require.NotContains(t, byID, "site-removed", "a removed node is not a failed read")+}++func TestBackupHistoryUnreadTaskCannotCompleteRunningArchive(t *testing.T) {+ for _, kind := range []string{"qemu", "lxc"} {+ t.Run(kind, func(t *testing.T) {+ f, storageClient := localArchiveIdentityFixture(t, kind)+ now := time.Now().UTC().Truncate(time.Second)+ f.client.at = now.Add(-time.Minute)+ var phase atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/api2/json/nodes":+ fmt.Fprint(w, `{"data":[{"node":"node-a","status":"online"},{"node":"node-b","status":"online"}]}`)+ case "/api2/json/access/permissions":+ json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{r.URL.Query().Get("path"): map[string]int{"Sys.Audit": 1}}})+ case "/api2/json/nodes/node-a/tasks":+ fmt.Fprint(w, `{"data":[]}`)+ case "/api2/json/nodes/node-b/tasks":+ if phase.Load() == 1 {+ fmt.Fprint(w, `{"data":null}`)+ return+ }+ task := proxmox.Task{UPID: "writer", Node: "node-b", Type: "vzdump", ID: "101", StartTime: now.Add(-10 * time.Minute).Unix(), Status: "running"}+ if phase.Load() == 2 {+ task.Status, task.EndTime = "OK", now.Unix()+ }+ json.NewEncoder(w).Encode(map[string]any{"data": []proxmox.Task{task}})+ default:+ w.WriteHeader(http.StatusNotFound)+ }+ }))+ defer server.Close()+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pam!readonly", TokenValue: "fixture"})+ require.NoError(t, err)+ var before models.BackupTask+ for i := range 3 {+ phase.Store(int32(i))+ f.m.pollBackupTasks(context.Background(), "cluster", client)+ f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", storageClient, f.nodes, f.online)+ f.wait(t)+ snapshot, points := f.m.state.GetSnapshot(), f.points(t)+ require.Len(t, snapshot.PVEBackups.BackupTasks, 1)+ if i == 0 {+ before = snapshot.PVEBackups.BackupTasks[0]+ } else if i == 1 {+ require.Equal(t, before, snapshot.PVEBackups.BackupTasks[0], "unread task cannot refresh backup intent")+ }+ require.Len(t, snapshot.PVEBackups.StorageBackups, 2)+ for _, archive := range snapshot.PVEBackups.StorageBackups {+ point := points["pve-backup:"+archive.ID]+ partial := archive.StorageNode == "node-b" && i < 2+ require.Equal(t, partial, archive.InProgress)+ if partial {+ require.Equal(t, recovery.OutcomeRunning, point.Outcome)+ require.Nil(t, point.CompletedAt, "an unread task list cannot turn the writer's partial archive into success")+ } else {+ require.Equal(t, recovery.OutcomeSuccess, point.Outcome)+ require.NotNil(t, point.CompletedAt, "independent completed archive and ordinary completion stay usable")+ }+ }+ }+ })+ }+}diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex e2217c0451..bb765d8281 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -2605,7 +2605,7 @@ func appendPBSBackupsWithinLimit(backups []models.PBSBackup, additions []models.func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, client PVEClientInterface) {log.Debug().Str("instance", instanceName).Msg("polling backup tasks")- tasks, collection, err := m.readBackupTaskHistory(ctx, instanceName, client)+ history, collection, err := m.readBackupTaskHistory(ctx, instanceName, client)if err != nil {m.persistBackupTaskHistory(ctx, nil, collection)monErr := errors.WrapAPIError("get_backup_tasks", instanceName, err, 0)@@ -2619,7 +2619,7 @@ func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, clieliveJobIDs := make(map[string]bool)var jobTasks []models.BackupTaskjobUPIDs := make(map[string]string)- for _, task := range tasks {+ for _, task := range history.Tasks {// Extract VMID from task ID (format: "UPID:node:pid:starttime:type:vmid:user@realm:")vmid := 0if task.ID != "" {@@ -2674,7 +2674,7 @@ func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, clie}// Update state with new backup tasks for this instance- m.state.UpdateBackupTasksForInstance(instanceName, liveTasks)+ m.state.UpdateBackupTasksForInstance(instanceName, m.retainUnreadBackupTasks(instanceName, history.Nodes, liveTasks))// Retain execution history before qualifying collection coverage. Task rows// are not reconciled against the provider's shrinking inventory window.diff --git a/pkg/proxmox/backup_history.go b/pkg/proxmox/backup_history.goindex ddde6f03c3..c0f662cc5d 100644--- a/pkg/proxmox/backup_history.go+++ b/pkg/proxmox/backup_history.go@@ -2,8 +2,8 @@ package proxmoximport ("context"- "encoding/json""fmt"+ "net/http""net/url""time")@@ -38,24 +38,24 @@ func (c *Client) GetBackupTaskHistory(ctx context.Context, since time.Time) (Baccontinue}seen := map[string]bool{}+ var nodeTasks []Taskfor page := 0; page < 30; page++ {params := url.Values{"typefilter": {"vzdump"}, "source": {"all"}, "limit": {"200"}, "start": {fmt.Sprint(page * 200)}, "since": {fmt.Sprint(since.Unix())}, "until": {fmt.Sprint(until)}}resp, readErr := c.get(ctx, "/nodes/"+url.PathEscape(node.Node)+"/tasks?"+params.Encode())if readErr != nil {observation.Reason = "collection_failed"+ if status, known := APIErrorStatus(readErr); known && (status == http.StatusUnauthorized || status == http.StatusForbidden) {+ observation.Reason = "access_denied"+ }break}- var result struct {- Data []Task `json:"data"`- }- readErr = json.NewDecoder(resp.Body).Decode(&result)- resp.Body.Close()+ tasks, readErr := readBackupTaskInventory(resp, node.Node)if readErr != nil {observation.Reason = "collection_failed"break}added := 0- for _, task := range result.Data {+ for _, task := range tasks {if task.Type != "vzdump" || task.UPID == "" || seen[task.UPID] {continue}@@ -63,10 +63,10 @@ func (c *Client) GetBackupTaskHistory(ctx context.Context, since time.Time) (Bacif task.Node == "" {task.Node = node.Node}- history.Tasks = append(history.Tasks, task)+ nodeTasks = append(nodeTasks, task)added++}- if len(result.Data) < 200 {+ if len(tasks) < 200 {observation.Complete = truebreak}@@ -79,6 +79,9 @@ func (c *Client) GetBackupTaskHistory(ctx context.Context, since time.Time) (Bacobservation.Complete = falseobservation.Reason = "audit_permission_unconfirmed"}+ if observation.Reason != "access_denied" {+ history.Tasks = append(history.Tasks, nodeTasks...)+ }history.Nodes = append(history.Nodes, observation)}return history, nil@@ -90,14 +93,7 @@ func (c *Client) backupHistoryAuditPermission(ctx context.Context, node string)if err != nil {return false}- defer resp.Body.Close()- var result struct {- Data map[string]map[string]int `json:"data"`- }- if json.NewDecoder(resp.Body).Decode(&result) != nil {- return false- }- return result.Data[path]["Sys.Audit"] == 1+ return readBackupHistoryAuditPermission(resp, path)}func (cc *ClusterClient) GetBackupTaskHistory(ctx context.Context, since time.Time) (BackupTaskHistory, error) {diff --git a/pkg/proxmox/backup_history_response.go b/pkg/proxmox/backup_history_response.gonew file mode 100644index 0000000000..4b6d05cb28--- /dev/null+++ b/pkg/proxmox/backup_history_response.go@@ -0,0 +1,101 @@+package proxmox++import (+ "bytes"+ "encoding/json"+ "errors"+ "net/http"+ "strconv"+ "strings"+ "unicode/utf8"+)++var errInvalidBackupHistoryResponse = errors.New("invalid Proxmox backup history response")++// History completeness and audit authority require the whole bounded response,+// not a decodable prefix. Never include provider data in a format diagnostic.+func readBackupHistoryData(resp *http.Response) (json.RawMessage, error) {+ body, readErr := readResponseBodyLimited(resp.Body)+ closeErr := resp.Body.Close()+ if readErr != nil || closeErr != nil {+ return nil, errors.Join(readErr, closeErr)+ }+ if resp.StatusCode != http.StatusOK || !utf8.Valid(body) {+ return nil, errInvalidBackupHistoryResponse+ }+ envelope, valid := inventoryResponseObject(body, "data", "errors")+ if !valid {+ return nil, errInvalidBackupHistoryResponse+ }+ if failure, exists := envelope["errors"]; exists && !bytes.Equal(bytes.TrimSpace(failure), []byte("null")) {+ return nil, errInvalidBackupHistoryResponse+ }+ data := bytes.TrimSpace(envelope["data"])+ if len(data) == 0 || bytes.Equal(data, []byte("null")) {+ return nil, errInvalidBackupHistoryResponse+ }+ return data, nil+}++func readBackupTaskInventory(resp *http.Response, node string) ([]Task, error) {+ data, err := readBackupHistoryData(resp)+ if err != nil {+ return nil, err+ }+ if data[0] != '[' {+ return nil, errInvalidBackupHistoryResponse+ }+ var rows []json.RawMessage+ if json.Unmarshal(data, &rows) != nil {+ return nil, errInvalidBackupHistoryResponse+ }+ tasks := make([]Task, 0, len(rows))+ seen := make(map[string]bool, len(rows))+ for _, row := range rows {+ fields, valid := inventoryResponseObject(row, "upid", "node", "pid", "pstart", "starttime", "type", "id", "user", "status", "endtime")+ if !valid {+ return nil, errInvalidBackupHistoryResponse+ }+ // Null scalar fields cannot silently become zero/empty observations.+ for _, name := range []string{"upid", "node", "pid", "pstart", "starttime", "type", "id", "user", "status", "endtime"} {+ if raw, present := fields[name]; present && bytes.Equal(bytes.TrimSpace(raw), []byte("null")) {+ return nil, errInvalidBackupHistoryResponse+ }+ }+ var task Task+ if json.Unmarshal(row, &task) != nil || strings.TrimSpace(task.UPID) == "" || seen[task.UPID] ||+ strings.TrimSpace(task.Type) == "" || task.Type != strings.TrimSpace(task.Type) || task.StartTime <= 0 ||+ task.EndTime < 0 || (task.EndTime > 0 && task.EndTime < task.StartTime) || (task.Node != "" && task.Node != node) {+ return nil, errInvalidBackupHistoryResponse+ }+ if task.Type == "vzdump" && task.ID != "" {+ vmid, err := strconv.ParseUint(task.ID, 10, 32)+ if err != nil || vmid == 0 {+ return nil, errInvalidBackupHistoryResponse+ }+ }+ seen[task.UPID] = true+ // The requested node supplies an absent legacy field, never overrides a+ // conflicting origin. An empty ID still denotes a multi-guest job.+ task.Node = node+ tasks = append(tasks, task)+ }+ return tasks, nil+}++func readBackupHistoryAuditPermission(resp *http.Response, path string) bool {+ data, err := readBackupHistoryData(resp)+ if err != nil {+ return false+ }+ paths, valid := inventoryResponseObject(data)+ if !valid {+ return false+ }+ permissions, valid := inventoryResponseObject(paths[path], "Sys.Audit")+ if !valid {+ return false+ }+ var granted int+ return json.Unmarshal(permissions["Sys.Audit"], &granted) == nil && granted == 1+}diff --git a/pkg/proxmox/backup_history_response_test.go b/pkg/proxmox/backup_history_response_test.gonew file mode 100644index 0000000000..5adffc8812--- /dev/null+++ b/pkg/proxmox/backup_history_response_test.go@@ -0,0 +1,211 @@+package proxmox++import (+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+)++const acceptedHistoryRow = `{"upid":"accepted","node":"member","type":"vzdump","id":"101","starttime":1700000000,"endtime":1700000060,"status":"OK"}`++func TestBackupHistoryRejectsUnverifiedTaskInventory(t *testing.T) {+ valid := `{"data":[` + acceptedHistoryRow + `]}`+ for _, tc := range []struct {+ name, body string+ incomplete bool+ }{+ {"missing-data", `{}`, false},+ {"null-data", `{"data":null}`, false},+ {"null-envelope", `null`, false},+ {"non-array", `{"data":{}}`, false},+ {"error-as-success", `{"data":[],"errors":{"message":"fixture-private-detail"}}`, false},+ {"case-data", `{"Data":[]}`, false},+ {"duplicate-data", `{"data":[` + acceptedHistoryRow + `],"data":[]}`, false},+ {"escaped-duplicate", `{"data":[],"\u0064ata":[]}`, false},+ {"case-overwrite", `{"data":[` + acceptedHistoryRow + `],"Data":[]}`, false},+ {"case-errors", `{"data":[],"Errors":null}`, false},+ {"trailing-value", valid + ` {"data":[]}`, false},+ {"trailing-text", valid + `fixture-private-detail`, false},+ {"unfinished-response", valid, true},+ {"oversized-response", valid + strings.Repeat(" ", int(maxResponseBodyBytes)), false},+ {"invalid-utf8", `{"data":[],"metadata":"` + string([]byte{0xff}) + `"}`, false},+ {"null-row", `{"data":[` + acceptedHistoryRow + `,null]}`, false},+ {"duplicate-row", `{"data":[` + acceptedHistoryRow + `,` + acceptedHistoryRow + `]}`, false},+ {"missing-upid", `{"data":[{"type":"vzdump","starttime":1700000000}]}`, false},+ {"blank-upid", `{"data":[{"upid":" ","type":"vzdump","starttime":1700000000}]}`, false},+ {"duplicate-upid-field", `{"data":[{"upid":"first",` + acceptedHistoryRow[1:] + `]}`, false},+ {"case-status-field", `{"data":[{"Status":"running",` + acceptedHistoryRow[1:] + `]}`, false},+ {"missing-type", `{"data":[{"upid":"accepted","starttime":1700000000}]}`, false},+ {"missing-start", `{"data":[{"upid":"accepted","type":"vzdump"}]}`, false},+ {"null-start", `{"data":[{"upid":"accepted","type":"vzdump","starttime":null}]}`, false},+ {"negative-start", `{"data":[{"upid":"accepted","type":"vzdump","starttime":-1}]}`, false},+ {"reversed-time", `{"data":[{"upid":"accepted","type":"vzdump","starttime":1700000000,"endtime":1600000000}]}`, false},+ {"null-end", `{"data":[{"upid":"accepted","type":"vzdump","starttime":1700000000,"endtime":null}]}`, false},+ {"null-status", `{"data":[{"upid":"accepted","type":"vzdump","starttime":1700000000,"status":null}]}`, false},+ {"invalid-guest", `{"data":[{"upid":"accepted","type":"vzdump","starttime":1700000000,"id":"not-a-guest"}]}`, false},+ {"foreign-node", `{"data":[{"upid":"accepted","node":"other","type":"vzdump","starttime":1700000000}]}`, false},+ {"bad-row-after-good", `{"data":[` + acceptedHistoryRow + `,{"upid":"second","type":false}]}`, false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ var taskReads, auditReads atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/api2/json/nodes":+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online"}]}`)+ case "/api2/json/nodes/member/tasks":+ taskReads.Add(1)+ if tc.incomplete {+ w.Header().Set("Content-Length", fmt.Sprint(len(tc.body)+100))+ }+ fmt.Fprint(w, tc.body)+ case "/api2/json/access/permissions":+ auditReads.Add(1)+ fmt.Fprint(w, `{"data":{"/nodes/member":{"Sys.Audit":1}}}`)+ default:+ w.WriteHeader(http.StatusNotFound)+ }+ }))+ defer server.Close()+ history, err := backupTestClient(t, server.URL).GetBackupTaskHistory(context.Background(), time.Unix(1600000000, 0))+ require.NoError(t, err, "per-node failures remain explicit alongside independent evidence")+ require.Empty(t, history.Tasks, "no prefix or partially valid page may publish tasks")+ require.Equal(t, []BackupTaskHistoryNode{{Node: "member", Reason: "collection_failed"}}, history.Nodes)+ require.EqualValues(t, 1, taskReads.Load(), "invalid successful responses are not retried")+ require.Zero(t, auditReads.Load(), "an unread inventory cannot qualify by checking permissions")+ })+ }+}++func TestBackupHistoryRequiresCompleteAuditEvidence(t *testing.T) {+ valid := `{"data":{"/nodes/member":{"Sys.Audit":1}}}`+ for _, tc := range []struct {+ name, body string+ incomplete bool+ }{+ {"missing", `{}`, false},+ {"null", `{"data":null}`, false},+ {"trailing", valid + ` {}`, false},+ {"unfinished", valid, true},+ {"duplicate-data", `{"data":{},"data":{"/nodes/member":{"Sys.Audit":1}}}`, false},+ {"case-data", `{"Data":{"/nodes/member":{"Sys.Audit":1}}}`, false},+ {"duplicate-node", `{"data":{"/nodes/member":{},"/nodes/member":{"Sys.Audit":1}}}`, false},+ {"duplicate-grant", `{"data":{"/nodes/member":{"Sys.Audit":0,"Sys.Audit":1}}}`, false},+ {"case-grant", `{"data":{"/nodes/member":{"Sys.Audit":1,"sys.audit":0}}}`, false},+ {"contradictory-error", `{"data":{"/nodes/member":{"Sys.Audit":1}},"errors":{"message":"fixture-private-detail"}}`, false},+ {"other-path", `{"data":{"/nodes/other":{"Sys.Audit":1}}}`, false},+ {"not-granted", `{"data":{"/nodes/member":{"Sys.Audit":0}}}`, false},+ {"oversized", valid + strings.Repeat(" ", int(maxResponseBodyBytes)), false},+ {"invalid-utf8", `{"data":{"/nodes/member":{"Sys.Audit":1}},"metadata":"` + string([]byte{0xff}) + `"}`, false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/api2/json/nodes":+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online"}]}`)+ case "/api2/json/nodes/member/tasks":+ fmt.Fprint(w, `{"data":[`+acceptedHistoryRow+`]}`)+ case "/api2/json/access/permissions":+ if tc.incomplete {+ w.Header().Set("Content-Length", fmt.Sprint(len(tc.body)+100))+ }+ fmt.Fprint(w, tc.body)+ default:+ w.WriteHeader(http.StatusNotFound)+ }+ }))+ defer server.Close()+ history, err := backupTestClient(t, server.URL).GetBackupTaskHistory(context.Background(), time.Unix(1600000000, 0))+ require.NoError(t, err)+ require.Len(t, history.Tasks, 1, "unconfirmed audit does not erase accepted task evidence")+ require.Equal(t, []BackupTaskHistoryNode{{Node: "member", Reason: "audit_permission_unconfirmed"}}, history.Nodes)+ })+ }+}++func TestBackupHistoryAcceptsExplicitInventoryAndScopedAuthority(t *testing.T) {+ for _, body := range []string{`{"data":[]}`, `{"data":[],"errors":null,"total":0}`, `{"data":[` + acceptedHistoryRow + `],"provider-metadata":{"revision":1}}`, `{"data":[{"upid":"running","type":"vzdump","id":"","starttime":1700000000}]}`} {+ t.Run(body, func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/api2/json/nodes":+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online"}]}`)+ case "/api2/json/nodes/member/tasks":+ fmt.Fprint(w, body)+ case "/api2/json/access/permissions":+ require.Equal(t, "/nodes/member", r.URL.Query().Get("path"))+ fmt.Fprint(w, `{"data":{"/nodes/member":{"Sys.Audit":1,"VM.Audit":1},"/":{"Other":1}},"errors":null}`)+ }+ }))+ defer server.Close()+ history, err := backupTestClient(t, server.URL).GetBackupTaskHistory(context.Background(), time.Unix(1600000000, 0))+ require.NoError(t, err)+ require.Equal(t, []BackupTaskHistoryNode{{Node: "member", Complete: true}}, history.Nodes)+ if strings.Contains(body, "upid") {+ require.Len(t, history.Tasks, 1)+ require.Equal(t, "member", history.Tasks[0].Node)+ } else {+ require.Empty(t, history.Tasks)+ }+ })+ }+}++func TestBackupHistoryReadClosureAndStatus(t *testing.T) {+ for _, status := range []int{http.StatusNoContent, http.StatusPartialContent} {+ resp := &http.Response{StatusCode: status, Body: io.NopCloser(strings.NewReader(`{"data":[]}`))}+ _, err := readBackupTaskInventory(resp, "member")+ require.Error(t, err)+ }+ body := &snapshotCloseFailure{Reader: strings.NewReader(`{"data":[]}`)}+ _, err := readBackupTaskInventory(&http.Response{StatusCode: http.StatusOK, Body: body}, "member")+ require.Error(t, err)+ require.True(t, body.closed)+ for _, resp := range []*http.Response{+ {StatusCode: http.StatusPartialContent, Body: io.NopCloser(strings.NewReader(`{"data":{"/nodes/member":{"Sys.Audit":1}}}`))},+ {StatusCode: http.StatusOK, Body: &snapshotCloseFailure{Reader: strings.NewReader(`{"data":{"/nodes/member":{"Sys.Audit":1}}}`)}},+ } {+ require.False(t, readBackupHistoryAuditPermission(resp, "/nodes/member"))+ }+}++func TestBackupHistoryMalformedLaterPageDoesNotQualifyPrefix(t *testing.T) {+ for _, second := range []string{`{}`, `{"data":null}`, `{"data":[{"upid":"invalid","type":"vzdump"}]}`} {+ t.Run(second, func(t *testing.T) {+ var auditReads atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/api2/json/nodes":+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online"}]}`)+ case "/api2/json/nodes/member/tasks":+ if r.URL.Query().Get("start") != "0" {+ fmt.Fprint(w, second)+ return+ }+ tasks := make([]Task, 200)+ for i := range tasks {+ tasks[i] = Task{UPID: fmt.Sprint(i), Type: "vzdump", ID: "101", StartTime: 1700000000}+ }+ json.NewEncoder(w).Encode(map[string]any{"data": tasks})+ case "/api2/json/access/permissions":+ auditReads.Add(1)+ fmt.Fprint(w, `{"data":{"/nodes/member":{"Sys.Audit":1}}}`)+ }+ }))+ defer server.Close()+ history, err := backupTestClient(t, server.URL).GetBackupTaskHistory(context.Background(), time.Unix(1600000000, 0))+ require.NoError(t, err)+ require.Len(t, history.Tasks, 200, "accepted independent pages remain evidence, never completeness")+ require.Equal(t, []BackupTaskHistoryNode{{Node: "member", Reason: "collection_failed"}}, history.Nodes)+ require.Zero(t, auditReads.Load())+ })+ }+}diff --git a/pkg/proxmox/backup_history_test.go b/pkg/proxmox/backup_history_test.goindex 0aeb7b6a78..b3b1e4e4e1 100644--- a/pkg/proxmox/backup_history_test.go+++ b/pkg/proxmox/backup_history_test.go@@ -20,7 +20,7 @@ func TestBackupHistoryPaginationAndCollectionQuality(t *testing.T) {repeat boolexpected intreason string- }{{"full", true, false, false, 201, ""}, {"permission", false, false, false, 201, "audit_permission_unconfirmed"}, {"partial", true, true, false, 200, "collection_failed"}, {"repeated-page", true, false, true, 200, "history_truncated"}} {+ }{{"full", true, false, false, 201, ""}, {"permission", false, false, false, 201, "audit_permission_unconfirmed"}, {"denied", true, true, false, 0, "access_denied"}, {"repeated-page", true, false, true, 200, "history_truncated"}} {t.Run(test.name, func(t *testing.T) {server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {w.Header().Set("Content-Type", "application/json")@@ -49,7 +49,7 @@ func TestBackupHistoryPaginationAndCollectionQuality(t *testing.T) {}tasks := make([]Task, count)for i := range tasks {- tasks[i] = Task{UPID: fmt.Sprintf("task-%d", i+offset), Type: "vzdump"}+ tasks[i] = Task{UPID: fmt.Sprintf("task-%d", i+offset), Type: "vzdump", StartTime: 1700000000}}json.NewEncoder(w).Encode(map[string]any{"data": tasks})default: