Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep contradictory backup task results from completing archives

pulse-triage[bot]7967164537dd
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 4a0c338b95..712f770762 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,29 @@
## Purpose
+### Backup task results agree with their completion clocks
+
+The PVE backup-history reader accepts a live task only with an absent/zero end
+clock and an omitted/empty or native `running` status. Any exit-result text
+requires a positive, ordered end clock. An ended task cannot still say `running`
+or `active`. A contradictory row rejects its whole page through the existing
+unread path, without inferring completion, rewriting result text or inventing
+an end time. Ended rows with no result remain unknown, not successful; native
+warning/error results, zero-duration tasks and explicit empty inventories remain
+valid. Provider payloads stay out of format errors.
+
+`TestBackupHistoryTaskCompletionMustBeConsistent` covers native live, completed,
+unknown and contradictory clocks/results. The normal-suite
+`TestBackupHistoryUnreadTaskCannotCompleteRunningArchive` crosses the actual
+HTTP reader and VM/LXC poller: rejected results retain the accepted writer and
+its original observation, keep its partial archive uncompleted in recovery
+history, and leave the independent node's completed copy usable. Ordinary
+consistent completion recovers without resetting state. Desktop/narrow browser
+proof consumes these actual produced archive frames through Backups. No task,
+guest, backup or restore operation, policy, schema or public payload is changed.
+This is truthful evidence handling, not a diagnosis or original guest recovery.
+
+
### PBS snapshot reads retain their exact requested group
A snapshot array is accepted only when every row's native backup kind and
@@ -3983,6 +4006,7 @@ cleanup so readers cannot retain orphaned runtime or alert projections.
21a. `pkg/proxmox/cluster_client.go`
21b. `pkg/proxmox/client.go`
21c. `pkg/proxmox/io_counters.go`
+21d. `pkg/proxmox/backup_history_response.go`
22. `pkg/proxmox/zfs.go`
22a. `pkg/pbs/client.go`
23. `internal/monitoring/guest_memory_sources.go`
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index f57b22d34d..fb9a4ecd35 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -6393,6 +6393,7 @@
"pkg/agents/host/report.go",
"pkg/agents/kubernetes/report.go",
"pkg/proxmox/backup_history.go",
+ "pkg/proxmox/backup_history_response.go",
"pkg/proxmox/ceph.go",
"pkg/proxmox/client.go",
"pkg/proxmox/cluster_client.go",
@@ -6406,6 +6407,20 @@
"exact_files": [],
"require_explicit_path_policy_coverage": true,
"path_policies": [
+ {
+ "id": "backup-history-response",
+ "label": "bounded backup inventory and completion evidence proof",
+ "match_prefixes": [],
+ "match_files": [
+ "pkg/proxmox/backup_history_response.go"
+ ],
+ "allow_same_subsystem_tests": false,
+ "test_prefixes": [],
+ "exact_files": [
+ "internal/monitoring/backup_task_inventory_lifecycle_test.go",
+ "pkg/proxmox/backup_history_response_test.go"
+ ]
+ },
{
"id": "backup-history-collection",
"label": "paginated backup execution history and collection quality proof",
@@ -6846,7 +6861,7 @@
"test_prefixes": [],
"exact_files": [
"pkg/pbs/auth_session_concurrency_test.go",
- "pkg/pbs/backup_snapshot_identity_test.go",
+ "pkg/pbs/backup_snapshot_identity_test.go",
"pkg/pbs/client_http_test.go",
"pkg/pbs/client_test.go",
"pkg/pbs/host_diagnostics_regression_test.go",
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index ab525f7551..40bc0d936a 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -59,6 +59,25 @@ proof must consume the resource producer through the actual Backups surface and
its existing drawer at desktop and narrow widths. This improves truthful
inspection, not installed memory relief, a successful backup or guest recovery.
+### Contradictory task completion cannot complete an archive
+
+A PVE task's exit result and end clock must agree before its page can replace
+accepted live intent. A result without an end time, or an ended task still
+labelled running/active, is unread rather than proof that the archive writer
+finished. The existing node-scoped carry-forward preserves the last accepted
+writer without renewing it or re-ingesting it into durable history. Independent
+readable evidence progresses; ordinary consistent completion restores the same
+archive's usability. No end time, outcome or healthy restore point is fabricated.
+
+The normal-suite client completion-matrix and VM/LXC archive-lifecycle tests
+exercise missing/empty native live statuses, successful/failed/warning/unknown
+ended results, contradictory clocks, exact retained observations, incomplete
+history and independent-node/recovery controls. Desktop and 320px browser proof
+must consume the actual HTTP-client/poller outputs through the production
+Backups inventory, inspect the running archive and independent completed point,
+and follow normal completion without losing the open disclosure. Policy,
+retention, identity, payloads and guest/backup/restore operations are unchanged.
+
### Unread PVE task history cannot establish absence or completion
The paginated backup-history collector admits only a complete, bounded HTTP200
diff --git a/internal/monitoring/backup_task_inventory_lifecycle_test.go b/internal/monitoring/backup_task_inventory_lifecycle_test.go
index 012ccf3eb8..8f9b22be14 100644
--- a/internal/monitoring/backup_task_inventory_lifecycle_test.go
+++ b/internal/monitoring/backup_task_inventory_lifecycle_test.go
@@ -14,6 +14,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+ "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -128,73 +129,89 @@ func TestBackupHistoryRetentionIsNodeScopedAndFreshEvidenceWins(t *testing.T) {
func TestBackupHistoryUnreadTaskCannotCompleteRunningArchive(t *testing.T) {
for _, kind := range []string{"qemu", "lxc"} {
- t.Run(kind, func(t *testing.T) {
- f, storageClient := localArchiveIdentityFixture(t, kind)
- now := time.Now().UTC().Truncate(time.Second)
- f.client.at = now.Add(-time.Minute)
- var phase atomic.Int32
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- switch r.URL.Path {
- case "/api2/json/nodes":
- fmt.Fprint(w, `{"data":[{"node":"node-a","status":"online"},{"node":"node-b","status":"online"}]}`)
- case "/api2/json/access/permissions":
- json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{r.URL.Query().Get("path"): map[string]int{"Sys.Audit": 1}}})
- case "/api2/json/nodes/node-a/tasks":
- fmt.Fprint(w, `{"data":[]}`)
- case "/api2/json/nodes/node-b/tasks":
- if phase.Load() == 1 {
- fmt.Fprint(w, `{"data":null}`)
- return
+ for _, adverse := range []string{"unread", "ok-without-end", "warning-without-end", "error-without-end", "running-with-end", "active-with-end"} {
+ t.Run(kind+"/"+adverse, func(t *testing.T) {
+ f, storageClient := localArchiveIdentityFixture(t, kind)
+ now := time.Now().UTC().Truncate(time.Second)
+ f.client.at = now.Add(-time.Minute)
+ var phase atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/api2/json/nodes":
+ fmt.Fprint(w, `{"data":[{"node":"node-a","status":"online"},{"node":"node-b","status":"online"}]}`)
+ case "/api2/json/access/permissions":
+ json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{r.URL.Query().Get("path"): map[string]int{"Sys.Audit": 1}}})
+ case "/api2/json/nodes/node-a/tasks":
+ fmt.Fprint(w, `{"data":[]}`)
+ case "/api2/json/nodes/node-b/tasks":
+ if phase.Load() == 1 && adverse == "unread" {
+ fmt.Fprint(w, `{"data":null}`)
+ return
+ }
+ task := proxmox.Task{UPID: "writer", Node: "node-b", Type: "vzdump", ID: "101", StartTime: now.Add(-10 * time.Minute).Unix(), Status: "running"}
+ if phase.Load() == 1 {
+ switch adverse {
+ case "ok-without-end":
+ task.Status = "OK"
+ case "warning-without-end":
+ task.Status = "WARNINGS: 1"
+ case "error-without-end":
+ task.Status = "ERROR: interrupted"
+ case "running-with-end":
+ task.EndTime = now.Unix()
+ case "active-with-end":
+ task.Status, task.EndTime = "active", now.Unix()
+ }
+ }
+ if phase.Load() == 2 {
+ task.Status, task.EndTime = "OK", now.Unix()
+ }
+ json.NewEncoder(w).Encode(map[string]any{"data": []proxmox.Task{task}})
+ default:
+ w.WriteHeader(http.StatusNotFound)
}
- task := proxmox.Task{UPID: "writer", Node: "node-b", Type: "vzdump", ID: "101", StartTime: now.Add(-10 * time.Minute).Unix(), Status: "running"}
- if phase.Load() == 2 {
- task.Status, task.EndTime = "OK", now.Unix()
+ }))
+ defer server.Close()
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pam!readonly", TokenValue: "fixture"})
+ require.NoError(t, err)
+ var before models.BackupTask
+ var beforePoint recovery.RecoveryPoint
+ for i := range 3 {
+ phase.Store(int32(i))
+ f.m.pollBackupTasks(context.Background(), "cluster", client)
+ f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", storageClient, f.nodes, f.online)
+ f.wait(t)
+ snapshot, points := f.m.state.GetSnapshot(), f.points(t)
+ taskPoint, exists := points["pve-task:cluster-writer"]
+ require.True(t, exists, "the actual task must reach retained execution history")
+ require.Len(t, snapshot.PVEBackups.BackupTasks, 1)
+ if i == 0 {
+ before = snapshot.PVEBackups.BackupTasks[0]
+ beforePoint = taskPoint
+ } else if i == 1 {
+ assert.Equal(t, before, snapshot.PVEBackups.BackupTasks[0], "unread task cannot refresh backup intent")
+ assert.Equal(t, beforePoint, taskPoint, "carried task cannot renew durable point evidence")
+ coverage, err := f.store.BackupHistoryCoverage(context.Background(), now.Add(-time.Hour), time.Now().UTC())
+ require.NoError(t, err)
+ require.Len(t, coverage.Sources, 1)
+ assert.False(t, coverage.Sources[0].Complete, "malformed task reads cannot qualify report history")
}
- json.NewEncoder(w).Encode(map[string]any{"data": []proxmox.Task{task}})
- default:
- w.WriteHeader(http.StatusNotFound)
- }
- }))
- defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pam!readonly", TokenValue: "fixture"})
- require.NoError(t, err)
- var before models.BackupTask
- var beforePoint recovery.RecoveryPoint
- for i := range 3 {
- phase.Store(int32(i))
- f.m.pollBackupTasks(context.Background(), "cluster", client)
- f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", storageClient, f.nodes, f.online)
- f.wait(t)
- snapshot, points := f.m.state.GetSnapshot(), f.points(t)
- taskPoint, exists := points["pve-task:cluster-writer"]
- require.True(t, exists, "the actual task must reach retained execution history")
- require.Len(t, snapshot.PVEBackups.BackupTasks, 1)
- if i == 0 {
- before = snapshot.PVEBackups.BackupTasks[0]
- beforePoint = taskPoint
- } else if i == 1 {
- require.Equal(t, before, snapshot.PVEBackups.BackupTasks[0], "unread task cannot refresh backup intent")
- require.Equal(t, beforePoint, taskPoint, "carried task cannot renew durable point evidence")
- coverage, err := f.store.BackupHistoryCoverage(context.Background(), now.Add(-time.Hour), time.Now().UTC())
- require.NoError(t, err)
- require.Len(t, coverage.Sources, 1)
- require.False(t, coverage.Sources[0].Complete, "malformed task reads cannot qualify report history")
- }
- require.Len(t, snapshot.PVEBackups.StorageBackups, 2)
- for _, archive := range snapshot.PVEBackups.StorageBackups {
- point := points["pve-backup:"+archive.ID]
- partial := archive.StorageNode == "node-b" && i < 2
- require.Equal(t, partial, archive.InProgress)
- if partial {
- require.Equal(t, recovery.OutcomeRunning, point.Outcome)
- require.Nil(t, point.CompletedAt, "an unread task list cannot turn the writer's partial archive into success")
- } else {
- require.Equal(t, recovery.OutcomeSuccess, point.Outcome)
- require.NotNil(t, point.CompletedAt, "independent completed archive and ordinary completion stay usable")
+ require.Len(t, snapshot.PVEBackups.StorageBackups, 2)
+ for _, archive := range snapshot.PVEBackups.StorageBackups {
+ point := points["pve-backup:"+archive.ID]
+ partial := archive.StorageNode == "node-b" && i < 2
+ assert.Equal(t, partial, archive.InProgress)
+ if partial {
+ assert.Equal(t, recovery.OutcomeRunning, point.Outcome)
+ assert.Nil(t, point.CompletedAt, "an unread task list cannot turn the writer's partial archive into success")
+ } else {
+ require.Equal(t, recovery.OutcomeSuccess, point.Outcome)
+ require.NotNil(t, point.CompletedAt, "independent completed archive and ordinary completion stay usable")
+ }
}
}
- }
- })
+ })
+ }
}
}
diff --git a/pkg/proxmox/backup_history_response.go b/pkg/proxmox/backup_history_response.go
index 4b6d05cb28..d8cf6911d9 100644
--- a/pkg/proxmox/backup_history_response.go
+++ b/pkg/proxmox/backup_history_response.go
@@ -68,6 +68,16 @@ func readBackupTaskInventory(resp *http.Response, node string) ([]Task, error) {
task.EndTime < 0 || (task.EndTime > 0 && task.EndTime < task.StartTime) || (task.Node != "" && task.Node != node) {
return nil, errInvalidBackupHistoryResponse
}
+ // PVE may omit the status of a running task, but an exit result is
+ // not completion evidence without an end clock. Equally, a live
+ // status cannot contradict an observed end. Treat the page as unread
+ // rather than replacing accepted running intent and thereby allowing
+ // its still-growing archive to appear complete.
+ status := strings.ToLower(strings.TrimSpace(task.Status))
+ if (task.EndTime == 0 && status != "" && status != "running") ||
+ (task.EndTime > 0 && (status == "running" || status == "active")) {
+ return nil, errInvalidBackupHistoryResponse
+ }
if task.Type == "vzdump" && task.ID != "" {
vmid, err := strconv.ParseUint(task.ID, 10, 32)
if err != nil || vmid == 0 {
diff --git a/pkg/proxmox/backup_history_response_test.go b/pkg/proxmox/backup_history_response_test.go
index 5adffc8812..043aee8281 100644
--- a/pkg/proxmox/backup_history_response_test.go
+++ b/pkg/proxmox/backup_history_response_test.go
@@ -209,3 +209,47 @@ func TestBackupHistoryMalformedLaterPageDoesNotQualifyPrefix(t *testing.T) {
})
}
}
+
+// Native running tasks have no end time and may omit status. A result label
+// without its completion clock, or a running label with one, is unread evidence.
+func TestBackupHistoryTaskCompletionMustBeConsistent(t *testing.T) {
+ for _, tc := range []struct {
+ name, status string
+ end int64
+ valid bool
+ }{
+ {"running-status-omitted", "", 0, true},
+ {"running", "running", 0, true},
+ {"running-case", "RUNNING", 0, true},
+ {"completed-ok", "OK", 1700000060, true},
+ {"completed-warning", "WARNINGS: 1", 1700000060, true},
+ {"completed-error", "ERROR: interrupted", 1700000060, true},
+ {"completed-other-result", "interrupted by signal", 1700000060, true},
+ {"completed-result-unknown", "", 1700000060, true},
+ {"instant-completion", "OK", 1700000000, true},
+ {"ok-without-end", "OK", 0, false},
+ {"success-without-end", "success", 0, false},
+ {"warning-without-end", "WARNINGS: 1", 0, false},
+ {"error-without-end", "ERROR: interrupted", 0, false},
+ {"other-result-without-end", "interrupted by signal", 0, false},
+ {"running-with-end", "running", 1700000060, false},
+ {"running-case-with-end", "RUNNING", 1700000060, false},
+ {"active-with-end", "active", 1700000060, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ row := Task{UPID: "writer", Type: "vzdump", ID: "101", StartTime: 1700000000, EndTime: tc.end, Status: tc.status}
+ body, err := json.Marshal(map[string]any{"data": []Task{row}})
+ require.NoError(t, err)
+ resp := &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(string(body)))}
+ tasks, err := readBackupTaskInventory(resp, "member")
+ if !tc.valid {
+ require.ErrorIs(t, err, errInvalidBackupHistoryResponse)
+ require.Empty(t, tasks, "contradictory completion cannot publish a partial row")
+ return
+ }
+ require.NoError(t, err)
+ row.Node = "member"
+ require.Equal(t, []Task{row}, tasks, "valid native clocks and result text stay unchanged")
+ })
+ }
+}
diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.py
index c10a46518a..63fd4d6444 100644
--- a/scripts/release_control/subsystem_lookup_test.py
+++ b/scripts/release_control/subsystem_lookup_test.py
@@ -4370,6 +4370,26 @@ class SubsystemLookupTest(unittest.TestCase):
],
)
+ def test_backup_history_response_has_explicit_ownership_and_proofs(self) -> None:
+ runtime = "pkg/proxmox/backup_history_response.go"
+ proofs = [
+ "internal/monitoring/backup_task_inventory_lifecycle_test.go",
+ "pkg/proxmox/backup_history_response_test.go",
+ ]
+ result = lookup_paths([runtime, *proofs])
+ self.assertEqual(result["unowned_runtime_files"], [])
+ match = result["files"][0]["matches"]
+ self.assertEqual(len(match), 1)
+ self.assertEqual(match[0]["subsystem"], "monitoring")
+ self.assertEqual(match[0]["verification_requirement"]["id"], "backup-history-response")
+ self.assertEqual(match[0]["verification_requirement"]["exact_files"], proofs)
+ for entry in result["files"][1:]:
+ self.assertTrue(any(
+ item["subsystem"] == "monitoring" and
+ item["verification_requirement"]["id"] == "backup-history-response"
+ for item in entry["matches"]
+ ))
+
def test_lookup_paths_assigns_pbs_client_runtime_to_monitoring(self) -> None:
result = lookup_paths([
"pkg/pbs/client.go",