Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Populate replication outcomes from observed source-job status

pulse-triage[bot]02ca5a04148a
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 43388a6bb6..50c478a03b 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -5477,3 +5477,22 @@ that actually accepted the same occurrence. The connected
HTTP 200/503 split, reopens the persistent queue, and requires the old accepted
destination's recovery plus the new firing, with no recovery to the unannounced
destination. Quiet hours and disabled recovery controls still apply.
+
+### Observed Proxmox replication sync outcome
+
+Replication configuration is enriched only from the source node's per-job
+status endpoint. A successful timestamp plus an explicitly valid zero failure
+count and no failure/unknown-state evidence yields `lastSyncStatus: ok`;
+error text, positive failures or an explicit failed state yields `error`,
+including error-with-zero-failures. Missing, denied, malformed, never-synced,
+invalid-count or unknown-state responses do not acquire success from defaults.
+Job/source mismatches and ambiguous array identities are rejected before any
+status fields merge. Single legacy rows without IDs retain endpoint binding;
+a multi-row response requires exactly one matching job.
+
+`replication_test.go` covers native field shapes, failure/unknown controls,
+HTTP enrichment and identity. `monitor_full_coverage_test.go` carries the real
+client through four normal fourteen-job/five-node polls into model/wire state,
+with zero-count error, denied status, recovery and another-instance isolation.
+The unchanged replication table consumes those explicit outcomes. These local
+HTTP controls are not native restart recovery; issue #2606 remains separate.
diff --git a/internal/monitoring/monitor_full_coverage_test.go b/internal/monitoring/monitor_full_coverage_test.go
index bce1b5d0b8..aee9fc3165 100644
--- a/internal/monitoring/monitor_full_coverage_test.go
+++ b/internal/monitoring/monitor_full_coverage_test.go
@@ -1135,3 +1135,91 @@ func TestPollPBSBackupsCachedIncompleteSnapshotLosesStaleTaskObservation(t *test
t.Fatalf("cached snapshot retained stale task observation: %+v", backups[0])
}
}
+
+// Real client -> collector -> state -> JSON projection, across normal polls.
+// Local HTTP is an API-shape control, not a PVE reboot/installed acceptance.
+func TestReplicationObservedOutcomeSurvivesNormalPolling(t *testing.T) {
+ var phase atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ if r.URL.Path == "/api2/json/cluster/replication" {
+ jobs := make([]map[string]interface{}, 14)
+ for i := range jobs {
+ jobs[i] = map[string]interface{}{"id": fmt.Sprintf("%d-0", 100+i), "guest": 100 + i, "source": fmt.Sprintf("node%d", i%5), "target": fmt.Sprintf("node%d", (i+1)%5)}
+ }
+ json.NewEncoder(w).Encode(map[string]interface{}{"data": jobs})
+ return
+ }
+ for i := range 14 {
+ if r.URL.Path != fmt.Sprintf("/api2/json/nodes/node%d/replication/%d-0/status", i%5, 100+i) {
+ continue
+ }
+ if phase.Load() == 2 && i == 0 {
+ w.WriteHeader(403)
+ return
+ }
+ status := map[string]interface{}{"id": fmt.Sprintf("%d-0", 100+i), "source": fmt.Sprintf("node%d", i%5), "last_sync": 1735689600 + int64(phase.Load())*300, "fail_count": 0}
+ if phase.Load() == 1 && i == 0 {
+ status["error"] = "sync failed"
+ }
+ json.NewEncoder(w).Encode(map[string]interface{}{"data": status})
+ return
+ }
+ t.Errorf("unexpected source/job path: %s", r.URL.Path)
+ w.WriteHeader(404)
+ }))
+ defer server.Close()
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "test@pve!token", TokenValue: "fixture", Timeout: time.Second})
+ if err != nil {
+ t.Fatal(err)
+ }
+ m := &Monitor{state: models.NewState()}
+ // An identically numbered job on another configured instance stays separate.
+ m.state.UpdateReplicationJobsForInstance("other", []models.ReplicationJob{{ID: "other-100-0", Instance: "other", JobID: "100-0", LastSyncStatus: "error"}})
+ for step := range 4 {
+ phase.Store(int32(step))
+ m.pollReplicationStatus(context.Background(), "cluster", client, []models.VM{{VMID: 100, Name: "guest", Type: "qemu", Node: "node0"}})
+ snapshot := m.state.GetSnapshot()
+ if len(snapshot.ReplicationJobs) != 15 {
+ t.Fatalf("phase %d lost inventory: %+v", step, snapshot.ReplicationJobs)
+ }
+ for _, job := range snapshot.ReplicationJobs {
+ if job.Instance == "other" {
+ if job.LastSyncStatus != "error" {
+ t.Fatal("poll changed another instance")
+ }
+ continue
+ }
+ want := "ok"
+ if job.JobID == "100-0" {
+ if step == 1 {
+ want = "error"
+ }
+ if step == 2 {
+ want = ""
+ }
+ }
+ if job.LastSyncStatus != want || job.ID != "cluster-"+job.JobID || job.LastPolled.IsZero() {
+ t.Fatalf("phase %d outcome/identity: %+v, want %q", step, job, want)
+ }
+ if job.JobID == "100-0" && (job.GuestName != "guest" || job.SourceNode != "node0" || job.TargetNode != "node1") {
+ t.Fatalf("projection changed guest/source/target: %+v", job)
+ }
+ if !(step == 2 && job.JobID == "100-0") && job.LastSyncUnix != 1735689600+int64(step)*300 {
+ t.Fatalf("phase %d retained old sync time: %+v", step, job)
+ }
+ encoded, err := json.Marshal(job)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var projected map[string]interface{}
+ json.Unmarshal(encoded, &projected)
+ if want != "" && projected["lastSyncStatus"] != want {
+ t.Fatal("wire projection lost outcome")
+ }
+ if want == "" && projected["lastSyncStatus"] != nil {
+ t.Fatal("unavailable status acquired a wire outcome")
+ }
+ }
+ }
+}
diff --git a/pkg/proxmox/replication.go b/pkg/proxmox/replication.go
index f5e3babca4..54a2a18ac4 100644
--- a/pkg/proxmox/replication.go
+++ b/pkg/proxmox/replication.go
@@ -164,7 +164,28 @@ func (c *Client) enrichReplicationJobStatus(ctx context.Context, job *Replicatio
Msg("Replication status response has empty data array")
return
}
- status = statusResp.Data[0]
+ // Per-job endpoints normally return one row without an ID. If a
+ // provider returns several rows, bind by job identity rather than
+ // silently taking the first job's health.
+ for _, candidate := range statusResp.Data {
+ candidateID := stringFromAny(firstNonNilRaw(candidate, "id", "jobid"))
+ if candidateID == jobID || (len(statusResp.Data) == 1 && candidateID == "") {
+ if status != nil {
+ return
+ } // Ambiguous duplicate identity.
+ status = candidate
+ }
+ }
+ if status == nil {
+ return
+ }
+ }
+
+ if statusID := stringFromAny(firstNonNilRaw(status, "id", "jobid")); statusID != "" && statusID != jobID {
+ return
+ }
+ if statusSource := stringFromAny(decodeRaw(status["source"])); statusSource != "" && statusSource != sourceNode {
+ return
}
// Parse and merge status fields
@@ -198,6 +219,8 @@ func (c *Client) enrichReplicationJobStatus(ctx context.Context, job *Replicatio
job.Error = errMsg
}
+ job.LastSyncStatus = replicationSyncOutcome(status)
+
log.Debug().
Str("jobID", jobID).
Str("source", sourceNode).
@@ -207,6 +230,35 @@ func (c *Client) enrichReplicationJobStatus(ctx context.Context, job *Replicatio
Msg("Successfully enriched replication job with status")
}
+// replicationSyncOutcome describes only the successfully read source-node
+// status. Configuration, a zero-value fail count, and an old display timestamp
+// are not proof of a successful sync. Error evidence wins even with zero fails.
+func replicationSyncOutcome(status map[string]json.RawMessage) string {
+ errMsg := stringFromAny(decodeRaw(status["error"]))
+ if errMsg == "" {
+ errMsg = stringFromAny(decodeRaw(status["last_sync_error"]))
+ }
+ fails, failErr := strconv.ParseInt(stringFromAny(firstNonNilRaw(status, "fail_count", "fail-count")), 10, 64)
+ outcome := strings.ToLower(stringFromAny(firstNonNilRaw(status, "last_sync_status", "last-sync-status", "last_sync_state", "last-sync-state")))
+ state := strings.ToLower(stringFromAny(firstNonNilRaw(status, "state", "status")))
+ if errMsg != "" || (failErr == nil && fails > 0) || outcome == "error" || outcome == "failed" || state == "error" || state == "failed" {
+ return "error"
+ }
+ _, lastSync := parseReplicationTime(decodeRaw(status["last_sync"]))
+ if lastSync <= 0 || failErr != nil || fails != 0 {
+ return ""
+ }
+ // PVE's native status commonly has last_sync/fail_count but no outcome
+ // string. Unknown/running states must not be rewritten as success.
+ if outcome != "" && outcome != "ok" && outcome != "success" && outcome != "completed" {
+ return ""
+ }
+ if state != "" && state != "ok" && state != "success" && state != "completed" && state != "idle" {
+ return ""
+ }
+ return "ok"
+}
+
func parseReplicationJob(entry map[string]json.RawMessage) ReplicationJob {
job := ReplicationJob{
Enabled: true,
diff --git a/pkg/proxmox/replication_test.go b/pkg/proxmox/replication_test.go
index a4d570909a..f832a9d3a7 100644
--- a/pkg/proxmox/replication_test.go
+++ b/pkg/proxmox/replication_test.go
@@ -963,3 +963,87 @@ func TestGetReplicationStatus_StatusEndpointFails(t *testing.T) {
t.Error("LastSyncTime should be nil (status endpoint failed)")
}
}
+
+func TestReplicationObservedSyncOutcome(t *testing.T) {
+ cases := []struct{ name, status, want string }{
+ {"successful native status", `{"last_sync":1735689600,"fail_count":0,"state":"ok"}`, "ok"},
+ {"successful without state", `{"last_sync":1735689600,"fail_count":0}`, "ok"},
+ {"successful string scalars", `{"last_sync":"1735689600","fail_count":"0","state":"OK"}`, "ok"},
+ {"never synced", `{"last_sync":0,"fail_count":0,"state":"ok"}`, ""},
+ {"absent sync", `{"fail_count":0,"state":"ok"}`, ""},
+ {"absent fail count", `{"last_sync":1735689600,"state":"ok"}`, ""},
+ {"negative count", `{"last_sync":1735689600,"fail_count":-1}`, ""},
+ {"fractional count", `{"last_sync":1735689600,"fail_count":0.4}`, ""},
+ {"invalid count", `{"last_sync":1735689600,"fail_count":"bad"}`, ""},
+ {"unknown state", `{"last_sync":1735689600,"fail_count":0,"state":"future-state"}`, ""},
+ {"running", `{"last_sync":1735689600,"fail_count":0,"state":"syncing"}`, ""},
+ {"zero-count error", `{"last_sync":1735689600,"fail_count":0,"error":"sync failed"}`, "error"},
+ {"zero-count error state", `{"last_sync":1735689600,"fail_count":0,"state":"error"}`, "error"},
+ {"failed explicit outcome", `{"last_sync":1735689600,"fail_count":0,"last_sync_status":"failed"}`, "error"},
+ {"failure overrides explicit success", `{"last_sync":1735689600,"fail_count":2,"last_sync_status":"ok"}`, "error"},
+ {"error overrides success", `{"last_sync":1735689600,"fail_count":0,"state":"ok","error":"failed"}`, "error"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ var status map[string]json.RawMessage
+ if err := json.Unmarshal([]byte(tc.status), &status); err != nil {
+ t.Fatal(err)
+ }
+ if got := replicationSyncOutcome(status); got != tc.want {
+ t.Fatalf("outcome = %q, want %q", got, tc.want)
+ }
+ })
+ }
+}
+
+func TestReplicationStatusBindsOutcomeToSourceJob(t *testing.T) {
+ cases := []struct {
+ name, body, outcome string
+ httpStatus int
+ }{
+ {"single legacy array", `{"data":[{"last_sync":1735689600,"fail_count":0}]}`, "ok", 200},
+ {"single object", `{"data":{"id":"100-0","source":"pve1","last_sync":1735689600,"fail_count":0}}`, "ok", 200},
+ {"matching row not first", `{"data":[{"id":"200-0","last_sync":1735689600,"fail_count":4},{"id":"100-0","last_sync":1735689600,"fail_count":0}]}`, "ok", 200},
+ {"different job", `{"data":[{"id":"200-0","last_sync":1735689600,"fail_count":0}]}`, "", 200},
+ {"different source", `{"data":{"source":"pve2","last_sync":1735689600,"fail_count":0}}`, "", 200},
+ {"duplicate job", `{"data":[{"id":"100-0","last_sync":1735689600,"fail_count":0},{"id":"100-0","last_sync":1735689600,"fail_count":1}]}`, "", 200},
+ {"ambiguous no identity", `{"data":[{"last_sync":1735689600,"fail_count":0},{"last_sync":1735689600,"fail_count":0}]}`, "", 200},
+ {"empty", `{"data":[]}`, "", 200},
+ {"malformed", `{not-json}`, "", 200},
+ {"unavailable permission", `{"errors":"permission denied"}`, "", 403},
+ {"error with zero fails", `{"data":{"last_sync":1735689600,"fail_count":0,"error":"sync failed"}}`, "error", 200},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/api2/json/cluster/replication":
+ fmt.Fprint(w, `{"data":[{"id":"100-0","guest":100,"source":"pve1","target":"pve2","schedule":"*/5"}]}`)
+ case "/api2/json/nodes/pve1/replication/100-0/status":
+ w.WriteHeader(tc.httpStatus)
+ fmt.Fprint(w, tc.body)
+ default:
+ t.Errorf("wrong source/job endpoint: %s", r.URL.Path)
+ w.WriteHeader(404)
+ }
+ }))
+ defer server.Close()
+ client, err := NewClient(ClientConfig{Host: server.URL, TokenName: "test@pve!token", TokenValue: "fixture", Timeout: time.Second})
+ if err != nil {
+ t.Fatal(err)
+ }
+ jobs, err := client.GetReplicationStatus(context.Background())
+ if err != nil || len(jobs) != 1 {
+ t.Fatalf("config inventory lost: %v %+v", err, jobs)
+ }
+ job := jobs[0]
+ if job.ID != "100-0" || job.Source != "pve1" || job.Target != "pve2" || job.LastSyncStatus != tc.outcome {
+ t.Fatalf("outcome/job identity = %+v, want %q", job, tc.outcome)
+ }
+ if tc.outcome == "" && job.LastSyncTime != nil {
+ t.Fatal("unmatched/unavailable status published another job's timestamp")
+ }
+ })
+ }
+}