Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep carried PBS job failures across Pulse restarts
rcourtmand3a22bc3f7d6
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.mdindex 9dfd7843bf..b0559922df 100644--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md@@ -10761,6 +10761,15 @@ through both monitor initialization paths, including explicit and legacydefault organizations. These notifications refresh inventory presentation;they do not grant enrollment, removal, re-enrollment or command authority.+### PBS job failure record preserves lifecycle authority (3 November 2026)++The monitor now opens `config.PBSJobFailureStore` beside its other data-dir+stores, so carried PBS job failures survive a Pulse restart. The record holds+only PBS job results the poller already read; it adds no agent enrollment,+credential, command, collection or remediation authority, never triggers a+poll or a job, leaves agent records untouched, and a load failure only logs+instead of blocking startup.+### PBS job failure carry preserves lifecycle authority (3 November 2026)`State.PBSJobHealthEvidence` is a read-only copy of the job results alreadydiff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 3e4e6deacf..9928324f93 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -157,8 +157,16 @@ entry) makes a family unreadable; task-history trouble does not keep aremoved job's failure. Recovery is judged from the run's own status, so asuccessful run first seen after its next schedule passed still clears thecarry. A poll that fails before reading jobs keeps the instance's last job-results in state, so the carry survives a PBS outage. Carried failures live-in memory: a Pulse restart forgets them until PBS reports the job again.+results in state, so the carry survives a PBS outage. Carried failures also+survive a Pulse restart: `config.PBSJobFailureStore` keeps each instance's+failures in `pbs_job_failures.json` in the data directory, with only the+failed run's fields (no poll times, so an unchanged failure never rewrites the+file), and the first poll after a restart, when state holds nothing for the+instance, carries from it. A record is bound to the server's host, so a+different server added under a reused connection name starts clean; records+of connections no longer configured are dropped; each instance keeps at most+256 failures, and a write that would exceed the record's 1 MiB read limit is+refused. An unreadable record only logs and never blocks startup.`pbsJobHealthPosture` now ranks a failed lastrun above overdue, so a failed job that has also missed its next run readsfailed, not late. `pbsTaskStatusOutcome` reads a PBS status the waydiff --git a/internal/config/pbs_job_failures.go b/internal/config/pbs_job_failures.gonew file mode 100644index 0000000000..3f2a41bdd7--- /dev/null+++ b/internal/config/pbs_job_failures.go@@ -0,0 +1,261 @@+package config++import (+ "bytes"+ "encoding/json"+ "fmt"+ "os"+ "path/filepath"+ "sort"+ "strings"+ "sync"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rs/zerolog/log"+)++const (+ pbsJobFailuresFileName = "pbs_job_failures.json"+ maxPBSJobFailuresFileSizeBytes = 1 * 1024 * 1024+ // A PBS server with more failing jobs than this is far past what one+ // record needs to carry; the cap keeps the file well inside its read limit.+ maxPBSJobFailuresPerInstance = 256+)++// PBSJobFailureStore keeps, per PBS instance, the jobs whose last completed run+// failed, so the PBS poller can still carry a failure after Pulse restarts.+// PBS's /admin job status drops a job's previous result while its next run is+// in progress; without this record, the first poll after a restart that meets+// a running job would read as recovery and clear the job's alert, only for it+// to fire again if that run failed too.+type PBSJobFailureStore struct {+ mu sync.RWMutex+ entries map[string]pbsJobFailureRecord+ dataPath string+ fs FileSystem+ loadErr error+}++// Instance ids come from connection names, so a record is bound to the server+// it was read from: a different server added under a reused name starts clean.+type pbsJobFailureRecord struct {+ Host string `json:"host"`+ Failures []models.PBSJobHealthEvidence `json:"failures"`+}++func NewPBSJobFailureStore(dataPath string, fs FileSystem) *PBSJobFailureStore {+ store := &PBSJobFailureStore{+ entries: make(map[string]pbsJobFailureRecord),+ dataPath: dataPath,+ fs: fs,+ }+ if store.fs == nil {+ store.fs = defaultFileSystem{}+ }+ if err := store.Load(); err != nil {+ store.loadErr = err+ log.Warn().Err(err).Msg("Failed to load PBS job failure record")+ }+ return store+}++func (s *PBSJobFailureStore) LoadError() error {+ if s == nil {+ return nil+ }+ s.mu.RLock()+ defer s.mu.RUnlock()+ return s.loadErr+}++func (s *PBSJobFailureStore) Load() error {+ if strings.TrimSpace(s.dataPath) == "" {+ return nil+ }+ filePath := filepath.Join(s.dataPath, pbsJobFailuresFileName)+ data, err := readLimitedRegularFileFS(s.fs, filePath, maxPBSJobFailuresFileSizeBytes)+ if err != nil {+ if os.IsNotExist(err) {+ return nil+ }+ return fmt.Errorf("failed to read PBS job failure record: %w", err)+ }++ entries := make(map[string]pbsJobFailureRecord)+ if err := json.Unmarshal(data, &entries); err != nil {+ return fmt.Errorf("failed to unmarshal PBS job failure record: %w", err)+ }+ normalized := make(map[string]pbsJobFailureRecord, len(entries))+ for instanceID, record := range entries {+ instanceID = strings.TrimSpace(instanceID)+ failures := stablePBSJobFailures(record.Failures)+ if instanceID != "" && len(failures) > 0 {+ normalized[instanceID] = pbsJobFailureRecord{Host: normalizePBSJobFailureHost(record.Host), Failures: failures}+ }+ }++ s.mu.Lock()+ s.entries = normalized+ s.loadErr = nil+ s.mu.Unlock()+ return nil+}++// Get returns a copy of the failures recorded for one PBS instance, or none+// when the record was read from a different server.+func (s *PBSJobFailureStore) Get(instanceID, host string) []models.PBSJobHealthEvidence {+ instanceID = strings.TrimSpace(instanceID)+ if s == nil || instanceID == "" {+ return nil+ }+ s.mu.RLock()+ defer s.mu.RUnlock()+ record, ok := s.entries[instanceID]+ if !ok || record.Host != normalizePBSJobFailureHost(host) {+ return nil+ }+ return append([]models.PBSJobHealthEvidence(nil), record.Failures...)+}++// Set records the instance's failed jobs, keeping only failures and only the+// fields that describe the failed run. Poll times are left out so an+// unchanged failure does not rewrite the file on every poll; an empty set+// removes the instance.+func (s *PBSJobFailureStore) Set(instanceID, host string, evidence []models.PBSJobHealthEvidence) error {+ instanceID = strings.TrimSpace(instanceID)+ if s == nil || instanceID == "" {+ return nil+ }+ next := pbsJobFailureRecord{Host: normalizePBSJobFailureHost(host), Failures: stablePBSJobFailures(evidence)}++ s.mu.Lock()+ defer s.mu.Unlock()+ previous, existed := s.entries[instanceID]+ if !existed && len(next.Failures) == 0 {+ return nil+ }+ if existed && len(next.Failures) > 0 && samePBSJobFailureRecord(previous, next) {+ return nil+ }+ if s.entries == nil {+ s.entries = make(map[string]pbsJobFailureRecord)+ }+ if len(next.Failures) == 0 {+ delete(s.entries, instanceID)+ } else {+ s.entries[instanceID] = next+ }+ if err := s.saveLocked(); err != nil {+ if existed {+ s.entries[instanceID] = previous+ } else {+ delete(s.entries, instanceID)+ }+ return err+ }+ return nil+}++// Retain drops records for instances no longer configured, so a removed or+// renamed connection neither grows the file nor lends its failures to a later+// connection of the same name.+func (s *PBSJobFailureStore) Retain(instanceIDs []string) error {+ if s == nil {+ return nil+ }+ keep := make(map[string]bool, len(instanceIDs))+ for _, id := range instanceIDs {+ keep[strings.TrimSpace(id)] = true+ }+ s.mu.Lock()+ defer s.mu.Unlock()+ removed := make(map[string]pbsJobFailureRecord)+ for id, record := range s.entries {+ if !keep[id] {+ removed[id] = record+ delete(s.entries, id)+ }+ }+ if len(removed) == 0 {+ return nil+ }+ if err := s.saveLocked(); err != nil {+ for id, record := range removed {+ s.entries[id] = record+ }+ return err+ }+ return nil+}++func (s *PBSJobFailureStore) saveLocked() error {+ // Without a data directory the record lives in memory only; writing it+ // relative to the working directory would scatter files.+ if strings.TrimSpace(s.dataPath) == "" {+ return nil+ }+ data, err := json.Marshal(s.entries)+ if err != nil {+ return fmt.Errorf("failed to marshal PBS job failure record: %w", err)+ }+ // Never write a record the loader would refuse to read back.+ if len(data) > maxPBSJobFailuresFileSizeBytes {+ return fmt.Errorf("PBS job failure record would be %d bytes, over the %d byte limit", len(data), maxPBSJobFailuresFileSizeBytes)+ }+ if err := persistMetadata(s.fs, s.dataPath, pbsJobFailuresFileName, data); err != nil {+ return fmt.Errorf("failed to save PBS job failure record: %w", err)+ }+ return nil+}++func normalizePBSJobFailureHost(host string) string {+ return strings.TrimRight(strings.ToLower(strings.TrimSpace(host)), "/")+}++func stablePBSJobFailures(evidence []models.PBSJobHealthEvidence) []models.PBSJobHealthEvidence {+ out := make([]models.PBSJobHealthEvidence, 0, len(evidence))+ for _, entry := range evidence {+ if entry.Posture != "critical" || entry.PostureReason != "last-run-failed" || strings.TrimSpace(entry.ID) == "" {+ continue+ }+ out = append(out, models.PBSJobHealthEvidence{+ ID: entry.ID,+ Family: entry.Family,+ Store: entry.Store,+ Remote: entry.Remote,+ Namespace: entry.Namespace,+ Enabled: entry.Enabled,+ LastRunState: entry.LastRunState,+ LastRunUPID: entry.LastRunUPID,+ LastRunEndtime: entry.LastRunEndtime,+ UPID: entry.UPID,+ TaskStatus: entry.TaskStatus,+ TaskEndTime: entry.TaskEndTime,+ Confidence: entry.Confidence,+ EvidenceSource: entry.EvidenceSource,+ EvidenceScope: entry.EvidenceScope,+ Freshness: models.PBSJobHealthFreshness{LastRunEndTime: entry.Freshness.LastRunEndTime},+ Posture: entry.Posture,+ PostureReason: entry.PostureReason,+ })+ }+ sort.Slice(out, func(i, j int) bool {+ if out[i].Family != out[j].Family {+ return out[i].Family < out[j].Family+ }+ return out[i].ID < out[j].ID+ })+ if len(out) > maxPBSJobFailuresPerInstance {+ out = out[:maxPBSJobFailuresPerInstance]+ }+ return out+}++func samePBSJobFailureRecord(left, right pbsJobFailureRecord) bool {+ if left.Host != right.Host || len(left.Failures) != len(right.Failures) {+ return false+ }+ a, errA := json.Marshal(left.Failures)+ b, errB := json.Marshal(right.Failures)+ return errA == nil && errB == nil && bytes.Equal(a, b)+}diff --git a/internal/config/pbs_job_failures_test.go b/internal/config/pbs_job_failures_test.gonew file mode 100644index 0000000000..8d1b6b1cf4--- /dev/null+++ b/internal/config/pbs_job_failures_test.go@@ -0,0 +1,188 @@+package config++import (+ "fmt"+ "os"+ "path/filepath"+ "strings"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+)++func failedPBSJob(id string) models.PBSJobHealthEvidence {+ ended := time.Date(2026, 11, 3, 6, 0, 0, 0, time.UTC)+ return models.PBSJobHealthEvidence{+ ID: id,+ Family: "sync",+ Store: "offsite",+ Remote: "pbs-main",+ Enabled: true,+ LastRunState: "connection error",+ LastRunEndtime: ended.Unix(),+ Posture: "critical",+ PostureReason: "last-run-failed",+ Freshness: models.PBSJobHealthFreshness{+ ObservedAt: time.Now(),+ LastRunEndTime: ended,+ AgeSeconds: 42,+ State: "observed",+ },+ }+}++func TestPBSJobFailureStoreKeepsFailuresAcrossARestart(t *testing.T) {+ dir := t.TempDir()+ store := NewPBSJobFailureStore(dir, nil)+ healthy := failedPBSJob("v-1")+ healthy.Family, healthy.Posture, healthy.PostureReason = "verify", "healthy", "last-run-success"+ if err := store.Set("pbs-main", "https://pbs-a:8007", []models.PBSJobHealthEvidence{failedPBSJob("s-1"), healthy}); err != nil {+ t.Fatalf("Set: %v", err)+ }++ reopened := NewPBSJobFailureStore(dir, nil)+ if err := reopened.LoadError(); err != nil {+ t.Fatalf("LoadError: %v", err)+ }+ got := reopened.Get("pbs-main", "https://pbs-a:8007")+ if len(got) != 1 || got[0].ID != "s-1" || got[0].Posture != "critical" || got[0].Remote != "pbs-main" {+ t.Fatalf("reloaded failures = %+v, want only the failed sync", got)+ }+ // Poll times are not part of the record, so an unchanged failure never+ // rewrites the file.+ if !got[0].Freshness.ObservedAt.IsZero() || got[0].Freshness.AgeSeconds != 0 {+ t.Fatalf("poll-time fields were persisted: %+v", got[0].Freshness)+ }+ if got[0].Freshness.LastRunEndTime.IsZero() {+ t.Fatal("the failed run's end time must be kept")+ }+}++func TestPBSJobFailureStoreDoesNotRewriteAnUnchangedFailure(t *testing.T) {+ dir := t.TempDir()+ store := NewPBSJobFailureStore(dir, nil)+ if err := store.Set("pbs-main", "https://pbs-a:8007", []models.PBSJobHealthEvidence{failedPBSJob("s-1")}); err != nil {+ t.Fatalf("Set: %v", err)+ }+ path := filepath.Join(dir, pbsJobFailuresFileName)+ old := time.Now().Add(-time.Hour)+ if err := os.Chtimes(path, old, old); err != nil {+ t.Fatalf("Chtimes: %v", err)+ }+ next := failedPBSJob("s-1")+ next.Freshness.ObservedAt = time.Now().Add(time.Minute)+ next.Freshness.AgeSeconds = 102+ if err := store.Set("pbs-main", "https://pbs-a:8007", []models.PBSJobHealthEvidence{next}); err != nil {+ t.Fatalf("Set: %v", err)+ }+ info, err := os.Stat(path)+ if err != nil {+ t.Fatalf("Stat: %v", err)+ }+ if !info.ModTime().Equal(old) {+ t.Fatalf("an unchanged failure rewrote the record at %v", info.ModTime())+ }+}++func TestPBSJobFailureStoreForgetsAnInstanceWithNoFailures(t *testing.T) {+ dir := t.TempDir()+ store := NewPBSJobFailureStore(dir, nil)+ if err := store.Set("pbs-main", "https://pbs-a:8007", []models.PBSJobHealthEvidence{failedPBSJob("s-1")}); err != nil {+ t.Fatalf("Set: %v", err)+ }+ if err := store.Set("pbs-main", "https://pbs-a:8007", nil); err != nil {+ t.Fatalf("Set: %v", err)+ }+ if got := NewPBSJobFailureStore(dir, nil).Get("pbs-main", "https://pbs-a:8007"); len(got) != 0 {+ t.Fatalf("cleared instance reloaded %+v", got)+ }+ var missing *PBSJobFailureStore+ if missing.Get("pbs-main", "https://pbs-a:8007") != nil || missing.Set("pbs-main", "https://pbs-a:8007", nil) != nil || missing.Retain(nil) != nil {+ t.Fatal("a nil store must be inert")+ }+}++func TestPBSJobFailureStoreReportsAnUnreadableRecord(t *testing.T) {+ dir := t.TempDir()+ if err := os.WriteFile(filepath.Join(dir, pbsJobFailuresFileName), []byte("{not json"), 0o600); err != nil {+ t.Fatalf("WriteFile: %v", err)+ }+ store := NewPBSJobFailureStore(dir, nil)+ if store.LoadError() == nil {+ t.Fatal("expected a load error for a corrupt record")+ }+ if got := store.Get("pbs-main", "https://pbs-a:8007"); len(got) != 0 {+ t.Fatalf("corrupt record yielded %+v", got)+ }+}++func TestPBSJobFailureStoreWithoutADataDirectoryWritesNothing(t *testing.T) {+ cwd := t.TempDir()+ t.Chdir(cwd)+ store := NewPBSJobFailureStore("", nil)+ if err := store.Set("pbs-main", "https://pbs-a:8007", []models.PBSJobHealthEvidence{failedPBSJob("s-1")}); err != nil {+ t.Fatalf("Set: %v", err)+ }+ if got := store.Get("pbs-main", "https://pbs-a:8007"); len(got) != 1 {+ t.Fatalf("in-memory record = %+v, want the failure", got)+ }+ if _, err := os.Stat(filepath.Join(cwd, pbsJobFailuresFileName)); !os.IsNotExist(err) {+ t.Fatalf("record written to the working directory: %v", err)+ }+}++func TestPBSJobFailureStoreBindsARecordToItsServer(t *testing.T) {+ store := NewPBSJobFailureStore(t.TempDir(), nil)+ if err := store.Set("pbs-main", "https://PBS-A:8007/", []models.PBSJobHealthEvidence{failedPBSJob("s-1")}); err != nil {+ t.Fatalf("Set: %v", err)+ }+ if got := store.Get("pbs-main", "https://pbs-a:8007"); len(got) != 1 {+ t.Fatalf("same server, other spelling = %+v", got)+ }+ if got := store.Get("pbs-main", "https://pbs-b:8007"); got != nil {+ t.Fatalf("another server under the same name got %+v", got)+ }+}++func TestPBSJobFailureStoreRetainDropsRemovedInstances(t *testing.T) {+ dir := t.TempDir()+ store := NewPBSJobFailureStore(dir, nil)+ for _, id := range []string{"pbs-main", "pbs-old"} {+ if err := store.Set(id, "https://pbs-a:8007", []models.PBSJobHealthEvidence{failedPBSJob("s-1")}); err != nil {+ t.Fatalf("Set: %v", err)+ }+ }+ if err := store.Retain([]string{"pbs-main"}); err != nil {+ t.Fatalf("Retain: %v", err)+ }+ reopened := NewPBSJobFailureStore(dir, nil)+ if len(reopened.Get("pbs-main", "https://pbs-a:8007")) != 1 || len(reopened.Get("pbs-old", "https://pbs-a:8007")) != 0 {+ t.Fatal("Retain must keep configured instances and drop the rest")+ }+}++func TestPBSJobFailureStoreNeverWritesAnUnreadableRecord(t *testing.T) {+ dir := t.TempDir()+ store := NewPBSJobFailureStore(dir, nil)+ many := make([]models.PBSJobHealthEvidence, 0, 400)+ for i := 0; i < 400; i++ {+ job := failedPBSJob(fmt.Sprintf("s-%03d-%s", i, strings.Repeat("x", 200)))+ job.Remote = strings.Repeat("r", 400)+ many = append(many, job)+ }+ var lastErr error+ for i := 0; i < 20 && lastErr == nil; i++ {+ lastErr = store.Set(fmt.Sprintf("pbs-%02d", i), "https://pbs-a:8007", many)+ }+ if lastErr == nil {+ t.Fatal("expected the size bound to refuse a write")+ }+ reopened := NewPBSJobFailureStore(dir, nil)+ if err := reopened.LoadError(); err != nil {+ t.Fatalf("the record on disk must stay readable: %v", err)+ }+ if got := reopened.Get("pbs-00", "https://pbs-a:8007"); len(got) != maxPBSJobFailuresPerInstance {+ t.Fatalf("per-instance cap = %d, want %d", len(got), maxPBSJobFailuresPerInstance)+ }+}diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.goindex 36c145ec2b..08e537af94 100644--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@ -1332,6 +1332,7 @@ type Monitor struct {guestMetadataStore *config.GuestMetadataStoredockerMetadataStore *config.DockerMetadataStoredockerReportOrderStore *config.DockerReportOrderStore+ pbsJobFailureStore *config.PBSJobFailureStorehostMetadataStore *config.HostMetadataStorehostContinuityStore *config.HostContinuityStorehostAgentLifecycleMu sync.RWMutex@@ -1838,6 +1839,9 @@ func New(cfg *config.Config) (*Monitor, error) {if err := dockerReportOrderStore.LoadError(); err != nil {return nil, fmt.Errorf("load Docker report order journal: %w", err)}+ // An unreadable record only loses carried PBS job failures; PBS reports+ // them again on the jobs' next runs, so it never blocks startup.+ pbsJobFailureStore := config.NewPBSJobFailureStore(cfg.DataPath, nil)// Queued deliveries can run before a router exists. Restore the instance// policy before creating workers, including for tenant persistence paths.@@ -2021,6 +2025,7 @@ func New(cfg *config.Config) (*Monitor, error) {guestMetadataStore: config.NewGuestMetadataStore(cfg.DataPath, nil),dockerMetadataStore: config.NewDockerMetadataStore(cfg.DataPath, nil),dockerReportOrderStore: dockerReportOrderStore,+ pbsJobFailureStore: pbsJobFailureStore,hostMetadataStore: config.NewHostMetadataStore(cfg.DataPath, nil),hostContinuityStore: hostContinuityStore,startTime: time.Now(),diff --git a/internal/monitoring/monitor_pbs_coverage_test.go b/internal/monitoring/monitor_pbs_coverage_test.goindex ce20b43388..0cb7b2bbe3 100644--- a/internal/monitoring/monitor_pbs_coverage_test.go+++ b/internal/monitoring/monitor_pbs_coverage_test.go@@ -984,3 +984,57 @@ func TestPBSJobHealthPostureReadsStatusLikeProxmox(t *testing.T) {}}}++// A Pulse restart empties state; the persisted record still carries the+// failure, so the first poll that meets the job's next run in progress does+// not read as recovery and clear the job's alert.+func TestCarryPBSJobFailuresSurvivesARestart(t *testing.T) {+ dir := t.TempDir()+ before := &Monitor{state: models.NewState(), pbsJobFailureStore: config.NewPBSJobFailureStore(dir, nil)}+ before.carryPBSJobFailuresForInstance("pbs-main", "https://pbs-a:8007", []models.PBSJobHealthEvidence{+ carriedJob("s-1", "sync", "connection error", "critical", "last-run-failed"),+ }, false)++ after := &Monitor{state: models.NewState(), pbsJobFailureStore: config.NewPBSJobFailureStore(dir, nil)}+ running := []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "", "unknown", "no-run-observed")}+ if posture := postureOf(after.carryPBSJobFailuresForInstance("pbs-main", "https://pbs-a:8007", running, false), "sync", "s-1"); posture != "critical/last-run-failed" {+ t.Fatalf("after restart posture = %s, want the carried failure", posture)+ }++ // The next run succeeds: the failure clears and so does its record.+ succeeded := []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "OK", "healthy", "last-run-success")}+ if posture := postureOf(after.carryPBSJobFailuresForInstance("pbs-main", "https://pbs-a:8007", succeeded, false), "sync", "s-1"); posture != "healthy/last-run-success" {+ t.Fatalf("posture = %s, want the observed success", posture)+ }+ if left := config.NewPBSJobFailureStore(dir, nil).Get("pbs-main", "https://pbs-a:8007"); len(left) != 0 {+ t.Fatalf("a recovered job stayed recorded: %+v", left)+ }+}++// A record belongs to the server it was read from and to a configured+// connection: a different server under a reused name, or a removed+// connection, carries nothing.+func TestCarryPBSJobFailuresIgnoresAnotherServerAndPrunesRemovedConnections(t *testing.T) {+ dir := t.TempDir()+ store := config.NewPBSJobFailureStore(dir, nil)+ failed := []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "connection error", "critical", "last-run-failed")}+ if err := store.Set("pbs-main", "https://pbs-a:8007", failed); err != nil {+ t.Fatalf("Set: %v", err)+ }+ if err := store.Set("pbs-old", "https://pbs-old:8007", failed); err != nil {+ t.Fatalf("Set: %v", err)+ }++ m := &Monitor{+ state: models.NewState(),+ pbsJobFailureStore: config.NewPBSJobFailureStore(dir, nil),+ config: &config.Config{PBSInstances: []config.PBSInstance{{Name: "main"}}},+ }+ running := []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "", "unknown", "no-run-observed")}+ if posture := postureOf(m.carryPBSJobFailuresForInstance("pbs-main", "https://pbs-b:8007", running, false), "sync", "s-1"); posture != "unknown/no-run-observed" {+ t.Fatalf("another server under the same name inherited %s", posture)+ }+ if left := config.NewPBSJobFailureStore(dir, nil).Get("pbs-old", "https://pbs-old:8007"); len(left) != 0 {+ t.Fatalf("a removed connection kept its record: %+v", left)+ }+}diff --git a/internal/monitoring/monitor_pbs_pmg.go b/internal/monitoring/monitor_pbs_pmg.goindex dd6d9bf0fc..2e68e91edb 100644--- a/internal/monitoring/monitor_pbs_pmg.go+++ b/internal/monitoring/monitor_pbs_pmg.go@@ -253,6 +253,31 @@ func carryPBSJobFailures(previous, current []models.PBSJobHealthEvidence, readFareturn out}+// carryPBSJobFailuresForInstance carries failures from the instance's last+// stored results, or, on the first poll after a restart when state holds none,+// from the persisted failure record of the same server, and records what is+// still failing. Records of connections no longer configured are dropped.+func (m *Monitor) carryPBSJobFailuresForInstance(instanceID, host string, current []models.PBSJobHealthEvidence, readFailed bool) []models.PBSJobHealthEvidence {+ previous := m.state.PBSJobHealthEvidence(instanceID)+ if len(previous) == 0 {+ previous = m.pbsJobFailureStore.Get(instanceID, host)+ }+ carried := carryPBSJobFailures(previous, current, readFailed)+ if err := m.pbsJobFailureStore.Set(instanceID, host, carried); err != nil {+ log.Warn().Err(err).Str("instance", instanceID).Msg("Failed to record PBS job failures")+ }+ if m.config != nil {+ configured := make([]string, 0, len(m.config.PBSInstances))+ for _, cfg := range m.config.PBSInstances {+ configured = append(configured, PBSMonitorResourceID(cfg.Name))+ }+ if err := m.pbsJobFailureStore.Retain(configured); err != nil {+ log.Warn().Err(err).Msg("Failed to prune PBS job failure record")+ }+ }+ return carried+}+// pbsJobReportsOutcome says whether this poll observed how the job's newest// run ended: success, warning or failure, rather than running or an// unreadable state. It reads the run's own status, not the posture, so a@@ -584,8 +609,9 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, clieStr("instance", instanceName).Msg("Failed to collect PBS job health evidence")}- pbsInst.JobHealthEvidence = carryPBSJobFailures(- m.state.PBSJobHealthEvidence(pbsInst.ID),+ pbsInst.JobHealthEvidence = m.carryPBSJobFailuresForInstance(+ pbsInst.ID,+ pbsInst.Host,pbsJobHealthEvidenceFromFacts(jobFacts, time.Now()),err != nil,)