Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Populate replication outcomes from observed source-job status
pulse-triage[bot]02ca5a04148a
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 43388a6bb6..50c478a03b 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -5477,3 +5477,22 @@ that actually accepted the same occurrence. The connectedHTTP 200/503 split, reopens the persistent queue, and requires the old accepteddestination's recovery plus the new firing, with no recovery to the unannounceddestination. Quiet hours and disabled recovery controls still apply.++### Observed Proxmox replication sync outcome++Replication configuration is enriched only from the source node's per-job+status endpoint. A successful timestamp plus an explicitly valid zero failure+count and no failure/unknown-state evidence yields `lastSyncStatus: ok`;+error text, positive failures or an explicit failed state yields `error`,+including error-with-zero-failures. Missing, denied, malformed, never-synced,+invalid-count or unknown-state responses do not acquire success from defaults.+Job/source mismatches and ambiguous array identities are rejected before any+status fields merge. Single legacy rows without IDs retain endpoint binding;+a multi-row response requires exactly one matching job.++`replication_test.go` covers native field shapes, failure/unknown controls,+HTTP enrichment and identity. `monitor_full_coverage_test.go` carries the real+client through four normal fourteen-job/five-node polls into model/wire state,+with zero-count error, denied status, recovery and another-instance isolation.+The unchanged replication table consumes those explicit outcomes. These local+HTTP controls are not native restart recovery; issue #2606 remains separate.diff --git a/internal/monitoring/monitor_full_coverage_test.go b/internal/monitoring/monitor_full_coverage_test.goindex bce1b5d0b8..aee9fc3165 100644--- a/internal/monitoring/monitor_full_coverage_test.go+++ b/internal/monitoring/monitor_full_coverage_test.go@@ -1135,3 +1135,91 @@ func TestPollPBSBackupsCachedIncompleteSnapshotLosesStaleTaskObservation(t *testt.Fatalf("cached snapshot retained stale task observation: %+v", backups[0])}}++// Real client -> collector -> state -> JSON projection, across normal polls.+// Local HTTP is an API-shape control, not a PVE reboot/installed acceptance.+func TestReplicationObservedOutcomeSurvivesNormalPolling(t *testing.T) {+ var phase atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ if r.URL.Path == "/api2/json/cluster/replication" {+ jobs := make([]map[string]interface{}, 14)+ for i := range jobs {+ jobs[i] = map[string]interface{}{"id": fmt.Sprintf("%d-0", 100+i), "guest": 100 + i, "source": fmt.Sprintf("node%d", i%5), "target": fmt.Sprintf("node%d", (i+1)%5)}+ }+ json.NewEncoder(w).Encode(map[string]interface{}{"data": jobs})+ return+ }+ for i := range 14 {+ if r.URL.Path != fmt.Sprintf("/api2/json/nodes/node%d/replication/%d-0/status", i%5, 100+i) {+ continue+ }+ if phase.Load() == 2 && i == 0 {+ w.WriteHeader(403)+ return+ }+ status := map[string]interface{}{"id": fmt.Sprintf("%d-0", 100+i), "source": fmt.Sprintf("node%d", i%5), "last_sync": 1735689600 + int64(phase.Load())*300, "fail_count": 0}+ if phase.Load() == 1 && i == 0 {+ status["error"] = "sync failed"+ }+ json.NewEncoder(w).Encode(map[string]interface{}{"data": status})+ return+ }+ t.Errorf("unexpected source/job path: %s", r.URL.Path)+ w.WriteHeader(404)+ }))+ defer server.Close()+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "test@pve!token", TokenValue: "fixture", Timeout: time.Second})+ if err != nil {+ t.Fatal(err)+ }+ m := &Monitor{state: models.NewState()}+ // An identically numbered job on another configured instance stays separate.+ m.state.UpdateReplicationJobsForInstance("other", []models.ReplicationJob{{ID: "other-100-0", Instance: "other", JobID: "100-0", LastSyncStatus: "error"}})+ for step := range 4 {+ phase.Store(int32(step))+ m.pollReplicationStatus(context.Background(), "cluster", client, []models.VM{{VMID: 100, Name: "guest", Type: "qemu", Node: "node0"}})+ snapshot := m.state.GetSnapshot()+ if len(snapshot.ReplicationJobs) != 15 {+ t.Fatalf("phase %d lost inventory: %+v", step, snapshot.ReplicationJobs)+ }+ for _, job := range snapshot.ReplicationJobs {+ if job.Instance == "other" {+ if job.LastSyncStatus != "error" {+ t.Fatal("poll changed another instance")+ }+ continue+ }+ want := "ok"+ if job.JobID == "100-0" {+ if step == 1 {+ want = "error"+ }+ if step == 2 {+ want = ""+ }+ }+ if job.LastSyncStatus != want || job.ID != "cluster-"+job.JobID || job.LastPolled.IsZero() {+ t.Fatalf("phase %d outcome/identity: %+v, want %q", step, job, want)+ }+ if job.JobID == "100-0" && (job.GuestName != "guest" || job.SourceNode != "node0" || job.TargetNode != "node1") {+ t.Fatalf("projection changed guest/source/target: %+v", job)+ }+ if !(step == 2 && job.JobID == "100-0") && job.LastSyncUnix != 1735689600+int64(step)*300 {+ t.Fatalf("phase %d retained old sync time: %+v", step, job)+ }+ encoded, err := json.Marshal(job)+ if err != nil {+ t.Fatal(err)+ }+ var projected map[string]interface{}+ json.Unmarshal(encoded, &projected)+ if want != "" && projected["lastSyncStatus"] != want {+ t.Fatal("wire projection lost outcome")+ }+ if want == "" && projected["lastSyncStatus"] != nil {+ t.Fatal("unavailable status acquired a wire outcome")+ }+ }+ }+}diff --git a/pkg/proxmox/replication.go b/pkg/proxmox/replication.goindex f5e3babca4..54a2a18ac4 100644--- a/pkg/proxmox/replication.go+++ b/pkg/proxmox/replication.go@@ -164,7 +164,28 @@ func (c *Client) enrichReplicationJobStatus(ctx context.Context, job *ReplicatioMsg("Replication status response has empty data array")return}- status = statusResp.Data[0]+ // Per-job endpoints normally return one row without an ID. If a+ // provider returns several rows, bind by job identity rather than+ // silently taking the first job's health.+ for _, candidate := range statusResp.Data {+ candidateID := stringFromAny(firstNonNilRaw(candidate, "id", "jobid"))+ if candidateID == jobID || (len(statusResp.Data) == 1 && candidateID == "") {+ if status != nil {+ return+ } // Ambiguous duplicate identity.+ status = candidate+ }+ }+ if status == nil {+ return+ }+ }++ if statusID := stringFromAny(firstNonNilRaw(status, "id", "jobid")); statusID != "" && statusID != jobID {+ return+ }+ if statusSource := stringFromAny(decodeRaw(status["source"])); statusSource != "" && statusSource != sourceNode {+ return}// Parse and merge status fields@@ -198,6 +219,8 @@ func (c *Client) enrichReplicationJobStatus(ctx context.Context, job *Replicatiojob.Error = errMsg}+ job.LastSyncStatus = replicationSyncOutcome(status)+log.Debug().Str("jobID", jobID).Str("source", sourceNode).@@ -207,6 +230,35 @@ func (c *Client) enrichReplicationJobStatus(ctx context.Context, job *ReplicatioMsg("Successfully enriched replication job with status")}+// replicationSyncOutcome describes only the successfully read source-node+// status. Configuration, a zero-value fail count, and an old display timestamp+// are not proof of a successful sync. Error evidence wins even with zero fails.+func replicationSyncOutcome(status map[string]json.RawMessage) string {+ errMsg := stringFromAny(decodeRaw(status["error"]))+ if errMsg == "" {+ errMsg = stringFromAny(decodeRaw(status["last_sync_error"]))+ }+ fails, failErr := strconv.ParseInt(stringFromAny(firstNonNilRaw(status, "fail_count", "fail-count")), 10, 64)+ outcome := strings.ToLower(stringFromAny(firstNonNilRaw(status, "last_sync_status", "last-sync-status", "last_sync_state", "last-sync-state")))+ state := strings.ToLower(stringFromAny(firstNonNilRaw(status, "state", "status")))+ if errMsg != "" || (failErr == nil && fails > 0) || outcome == "error" || outcome == "failed" || state == "error" || state == "failed" {+ return "error"+ }+ _, lastSync := parseReplicationTime(decodeRaw(status["last_sync"]))+ if lastSync <= 0 || failErr != nil || fails != 0 {+ return ""+ }+ // PVE's native status commonly has last_sync/fail_count but no outcome+ // string. Unknown/running states must not be rewritten as success.+ if outcome != "" && outcome != "ok" && outcome != "success" && outcome != "completed" {+ return ""+ }+ if state != "" && state != "ok" && state != "success" && state != "completed" && state != "idle" {+ return ""+ }+ return "ok"+}+func parseReplicationJob(entry map[string]json.RawMessage) ReplicationJob {job := ReplicationJob{Enabled: true,diff --git a/pkg/proxmox/replication_test.go b/pkg/proxmox/replication_test.goindex a4d570909a..f832a9d3a7 100644--- a/pkg/proxmox/replication_test.go+++ b/pkg/proxmox/replication_test.go@@ -963,3 +963,87 @@ func TestGetReplicationStatus_StatusEndpointFails(t *testing.T) {t.Error("LastSyncTime should be nil (status endpoint failed)")}}++func TestReplicationObservedSyncOutcome(t *testing.T) {+ cases := []struct{ name, status, want string }{+ {"successful native status", `{"last_sync":1735689600,"fail_count":0,"state":"ok"}`, "ok"},+ {"successful without state", `{"last_sync":1735689600,"fail_count":0}`, "ok"},+ {"successful string scalars", `{"last_sync":"1735689600","fail_count":"0","state":"OK"}`, "ok"},+ {"never synced", `{"last_sync":0,"fail_count":0,"state":"ok"}`, ""},+ {"absent sync", `{"fail_count":0,"state":"ok"}`, ""},+ {"absent fail count", `{"last_sync":1735689600,"state":"ok"}`, ""},+ {"negative count", `{"last_sync":1735689600,"fail_count":-1}`, ""},+ {"fractional count", `{"last_sync":1735689600,"fail_count":0.4}`, ""},+ {"invalid count", `{"last_sync":1735689600,"fail_count":"bad"}`, ""},+ {"unknown state", `{"last_sync":1735689600,"fail_count":0,"state":"future-state"}`, ""},+ {"running", `{"last_sync":1735689600,"fail_count":0,"state":"syncing"}`, ""},+ {"zero-count error", `{"last_sync":1735689600,"fail_count":0,"error":"sync failed"}`, "error"},+ {"zero-count error state", `{"last_sync":1735689600,"fail_count":0,"state":"error"}`, "error"},+ {"failed explicit outcome", `{"last_sync":1735689600,"fail_count":0,"last_sync_status":"failed"}`, "error"},+ {"failure overrides explicit success", `{"last_sync":1735689600,"fail_count":2,"last_sync_status":"ok"}`, "error"},+ {"error overrides success", `{"last_sync":1735689600,"fail_count":0,"state":"ok","error":"failed"}`, "error"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ var status map[string]json.RawMessage+ if err := json.Unmarshal([]byte(tc.status), &status); err != nil {+ t.Fatal(err)+ }+ if got := replicationSyncOutcome(status); got != tc.want {+ t.Fatalf("outcome = %q, want %q", got, tc.want)+ }+ })+ }+}++func TestReplicationStatusBindsOutcomeToSourceJob(t *testing.T) {+ cases := []struct {+ name, body, outcome string+ httpStatus int+ }{+ {"single legacy array", `{"data":[{"last_sync":1735689600,"fail_count":0}]}`, "ok", 200},+ {"single object", `{"data":{"id":"100-0","source":"pve1","last_sync":1735689600,"fail_count":0}}`, "ok", 200},+ {"matching row not first", `{"data":[{"id":"200-0","last_sync":1735689600,"fail_count":4},{"id":"100-0","last_sync":1735689600,"fail_count":0}]}`, "ok", 200},+ {"different job", `{"data":[{"id":"200-0","last_sync":1735689600,"fail_count":0}]}`, "", 200},+ {"different source", `{"data":{"source":"pve2","last_sync":1735689600,"fail_count":0}}`, "", 200},+ {"duplicate job", `{"data":[{"id":"100-0","last_sync":1735689600,"fail_count":0},{"id":"100-0","last_sync":1735689600,"fail_count":1}]}`, "", 200},+ {"ambiguous no identity", `{"data":[{"last_sync":1735689600,"fail_count":0},{"last_sync":1735689600,"fail_count":0}]}`, "", 200},+ {"empty", `{"data":[]}`, "", 200},+ {"malformed", `{not-json}`, "", 200},+ {"unavailable permission", `{"errors":"permission denied"}`, "", 403},+ {"error with zero fails", `{"data":{"last_sync":1735689600,"fail_count":0,"error":"sync failed"}}`, "error", 200},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/api2/json/cluster/replication":+ fmt.Fprint(w, `{"data":[{"id":"100-0","guest":100,"source":"pve1","target":"pve2","schedule":"*/5"}]}`)+ case "/api2/json/nodes/pve1/replication/100-0/status":+ w.WriteHeader(tc.httpStatus)+ fmt.Fprint(w, tc.body)+ default:+ t.Errorf("wrong source/job endpoint: %s", r.URL.Path)+ w.WriteHeader(404)+ }+ }))+ defer server.Close()+ client, err := NewClient(ClientConfig{Host: server.URL, TokenName: "test@pve!token", TokenValue: "fixture", Timeout: time.Second})+ if err != nil {+ t.Fatal(err)+ }+ jobs, err := client.GetReplicationStatus(context.Background())+ if err != nil || len(jobs) != 1 {+ t.Fatalf("config inventory lost: %v %+v", err, jobs)+ }+ job := jobs[0]+ if job.ID != "100-0" || job.Source != "pve1" || job.Target != "pve2" || job.LastSyncStatus != tc.outcome {+ t.Fatalf("outcome/job identity = %+v, want %q", job, tc.outcome)+ }+ if tc.outcome == "" && job.LastSyncTime != nil {+ t.Fatal("unmatched/unavailable status published another job's timestamp")+ }+ })+ }+}