Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep carried PBS job failures across Pulse restarts

rcourtmand3a22bc3f7d6
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index 9dfd7843bf..b0559922df 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -10761,6 +10761,15 @@ through both monitor initialization paths, including explicit and legacy
default organizations. These notifications refresh inventory presentation;
they do not grant enrollment, removal, re-enrollment or command authority.
+### PBS job failure record preserves lifecycle authority (3 November 2026)
+
+The monitor now opens `config.PBSJobFailureStore` beside its other data-dir
+stores, so carried PBS job failures survive a Pulse restart. The record holds
+only PBS job results the poller already read; it adds no agent enrollment,
+credential, command, collection or remediation authority, never triggers a
+poll or a job, leaves agent records untouched, and a load failure only logs
+instead of blocking startup.
+
### PBS job failure carry preserves lifecycle authority (3 November 2026)
`State.PBSJobHealthEvidence` is a read-only copy of the job results already
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 3e4e6deacf..9928324f93 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -157,8 +157,16 @@ entry) makes a family unreadable; task-history trouble does not keep a
removed job's failure. Recovery is judged from the run's own status, so a
successful run first seen after its next schedule passed still clears the
carry. A poll that fails before reading jobs keeps the instance's last job
-results in state, so the carry survives a PBS outage. Carried failures live
-in memory: a Pulse restart forgets them until PBS reports the job again.
+results in state, so the carry survives a PBS outage. Carried failures also
+survive a Pulse restart: `config.PBSJobFailureStore` keeps each instance's
+failures in `pbs_job_failures.json` in the data directory, with only the
+failed run's fields (no poll times, so an unchanged failure never rewrites the
+file), and the first poll after a restart, when state holds nothing for the
+instance, carries from it. A record is bound to the server's host, so a
+different server added under a reused connection name starts clean; records
+of connections no longer configured are dropped; each instance keeps at most
+256 failures, and a write that would exceed the record's 1 MiB read limit is
+refused. An unreadable record only logs and never blocks startup.
`pbsJobHealthPosture` now ranks a failed last
run above overdue, so a failed job that has also missed its next run reads
failed, not late. `pbsTaskStatusOutcome` reads a PBS status the way
diff --git a/internal/config/pbs_job_failures.go b/internal/config/pbs_job_failures.go
new file mode 100644
index 0000000000..3f2a41bdd7
--- /dev/null
+++ b/internal/config/pbs_job_failures.go
@@ -0,0 +1,261 @@
+package config
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+ "sync"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rs/zerolog/log"
+)
+
+const (
+ pbsJobFailuresFileName = "pbs_job_failures.json"
+ maxPBSJobFailuresFileSizeBytes = 1 * 1024 * 1024
+ // A PBS server with more failing jobs than this is far past what one
+ // record needs to carry; the cap keeps the file well inside its read limit.
+ maxPBSJobFailuresPerInstance = 256
+)
+
+// PBSJobFailureStore keeps, per PBS instance, the jobs whose last completed run
+// failed, so the PBS poller can still carry a failure after Pulse restarts.
+// PBS's /admin job status drops a job's previous result while its next run is
+// in progress; without this record, the first poll after a restart that meets
+// a running job would read as recovery and clear the job's alert, only for it
+// to fire again if that run failed too.
+type PBSJobFailureStore struct {
+ mu sync.RWMutex
+ entries map[string]pbsJobFailureRecord
+ dataPath string
+ fs FileSystem
+ loadErr error
+}
+
+// Instance ids come from connection names, so a record is bound to the server
+// it was read from: a different server added under a reused name starts clean.
+type pbsJobFailureRecord struct {
+ Host string `json:"host"`
+ Failures []models.PBSJobHealthEvidence `json:"failures"`
+}
+
+func NewPBSJobFailureStore(dataPath string, fs FileSystem) *PBSJobFailureStore {
+ store := &PBSJobFailureStore{
+ entries: make(map[string]pbsJobFailureRecord),
+ dataPath: dataPath,
+ fs: fs,
+ }
+ if store.fs == nil {
+ store.fs = defaultFileSystem{}
+ }
+ if err := store.Load(); err != nil {
+ store.loadErr = err
+ log.Warn().Err(err).Msg("Failed to load PBS job failure record")
+ }
+ return store
+}
+
+func (s *PBSJobFailureStore) LoadError() error {
+ if s == nil {
+ return nil
+ }
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ return s.loadErr
+}
+
+func (s *PBSJobFailureStore) Load() error {
+ if strings.TrimSpace(s.dataPath) == "" {
+ return nil
+ }
+ filePath := filepath.Join(s.dataPath, pbsJobFailuresFileName)
+ data, err := readLimitedRegularFileFS(s.fs, filePath, maxPBSJobFailuresFileSizeBytes)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil
+ }
+ return fmt.Errorf("failed to read PBS job failure record: %w", err)
+ }
+
+ entries := make(map[string]pbsJobFailureRecord)
+ if err := json.Unmarshal(data, &entries); err != nil {
+ return fmt.Errorf("failed to unmarshal PBS job failure record: %w", err)
+ }
+ normalized := make(map[string]pbsJobFailureRecord, len(entries))
+ for instanceID, record := range entries {
+ instanceID = strings.TrimSpace(instanceID)
+ failures := stablePBSJobFailures(record.Failures)
+ if instanceID != "" && len(failures) > 0 {
+ normalized[instanceID] = pbsJobFailureRecord{Host: normalizePBSJobFailureHost(record.Host), Failures: failures}
+ }
+ }
+
+ s.mu.Lock()
+ s.entries = normalized
+ s.loadErr = nil
+ s.mu.Unlock()
+ return nil
+}
+
+// Get returns a copy of the failures recorded for one PBS instance, or none
+// when the record was read from a different server.
+func (s *PBSJobFailureStore) Get(instanceID, host string) []models.PBSJobHealthEvidence {
+ instanceID = strings.TrimSpace(instanceID)
+ if s == nil || instanceID == "" {
+ return nil
+ }
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ record, ok := s.entries[instanceID]
+ if !ok || record.Host != normalizePBSJobFailureHost(host) {
+ return nil
+ }
+ return append([]models.PBSJobHealthEvidence(nil), record.Failures...)
+}
+
+// Set records the instance's failed jobs, keeping only failures and only the
+// fields that describe the failed run. Poll times are left out so an
+// unchanged failure does not rewrite the file on every poll; an empty set
+// removes the instance.
+func (s *PBSJobFailureStore) Set(instanceID, host string, evidence []models.PBSJobHealthEvidence) error {
+ instanceID = strings.TrimSpace(instanceID)
+ if s == nil || instanceID == "" {
+ return nil
+ }
+ next := pbsJobFailureRecord{Host: normalizePBSJobFailureHost(host), Failures: stablePBSJobFailures(evidence)}
+
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ previous, existed := s.entries[instanceID]
+ if !existed && len(next.Failures) == 0 {
+ return nil
+ }
+ if existed && len(next.Failures) > 0 && samePBSJobFailureRecord(previous, next) {
+ return nil
+ }
+ if s.entries == nil {
+ s.entries = make(map[string]pbsJobFailureRecord)
+ }
+ if len(next.Failures) == 0 {
+ delete(s.entries, instanceID)
+ } else {
+ s.entries[instanceID] = next
+ }
+ if err := s.saveLocked(); err != nil {
+ if existed {
+ s.entries[instanceID] = previous
+ } else {
+ delete(s.entries, instanceID)
+ }
+ return err
+ }
+ return nil
+}
+
+// Retain drops records for instances no longer configured, so a removed or
+// renamed connection neither grows the file nor lends its failures to a later
+// connection of the same name.
+func (s *PBSJobFailureStore) Retain(instanceIDs []string) error {
+ if s == nil {
+ return nil
+ }
+ keep := make(map[string]bool, len(instanceIDs))
+ for _, id := range instanceIDs {
+ keep[strings.TrimSpace(id)] = true
+ }
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ removed := make(map[string]pbsJobFailureRecord)
+ for id, record := range s.entries {
+ if !keep[id] {
+ removed[id] = record
+ delete(s.entries, id)
+ }
+ }
+ if len(removed) == 0 {
+ return nil
+ }
+ if err := s.saveLocked(); err != nil {
+ for id, record := range removed {
+ s.entries[id] = record
+ }
+ return err
+ }
+ return nil
+}
+
+func (s *PBSJobFailureStore) saveLocked() error {
+ // Without a data directory the record lives in memory only; writing it
+ // relative to the working directory would scatter files.
+ if strings.TrimSpace(s.dataPath) == "" {
+ return nil
+ }
+ data, err := json.Marshal(s.entries)
+ if err != nil {
+ return fmt.Errorf("failed to marshal PBS job failure record: %w", err)
+ }
+ // Never write a record the loader would refuse to read back.
+ if len(data) > maxPBSJobFailuresFileSizeBytes {
+ return fmt.Errorf("PBS job failure record would be %d bytes, over the %d byte limit", len(data), maxPBSJobFailuresFileSizeBytes)
+ }
+ if err := persistMetadata(s.fs, s.dataPath, pbsJobFailuresFileName, data); err != nil {
+ return fmt.Errorf("failed to save PBS job failure record: %w", err)
+ }
+ return nil
+}
+
+func normalizePBSJobFailureHost(host string) string {
+ return strings.TrimRight(strings.ToLower(strings.TrimSpace(host)), "/")
+}
+
+func stablePBSJobFailures(evidence []models.PBSJobHealthEvidence) []models.PBSJobHealthEvidence {
+ out := make([]models.PBSJobHealthEvidence, 0, len(evidence))
+ for _, entry := range evidence {
+ if entry.Posture != "critical" || entry.PostureReason != "last-run-failed" || strings.TrimSpace(entry.ID) == "" {
+ continue
+ }
+ out = append(out, models.PBSJobHealthEvidence{
+ ID: entry.ID,
+ Family: entry.Family,
+ Store: entry.Store,
+ Remote: entry.Remote,
+ Namespace: entry.Namespace,
+ Enabled: entry.Enabled,
+ LastRunState: entry.LastRunState,
+ LastRunUPID: entry.LastRunUPID,
+ LastRunEndtime: entry.LastRunEndtime,
+ UPID: entry.UPID,
+ TaskStatus: entry.TaskStatus,
+ TaskEndTime: entry.TaskEndTime,
+ Confidence: entry.Confidence,
+ EvidenceSource: entry.EvidenceSource,
+ EvidenceScope: entry.EvidenceScope,
+ Freshness: models.PBSJobHealthFreshness{LastRunEndTime: entry.Freshness.LastRunEndTime},
+ Posture: entry.Posture,
+ PostureReason: entry.PostureReason,
+ })
+ }
+ sort.Slice(out, func(i, j int) bool {
+ if out[i].Family != out[j].Family {
+ return out[i].Family < out[j].Family
+ }
+ return out[i].ID < out[j].ID
+ })
+ if len(out) > maxPBSJobFailuresPerInstance {
+ out = out[:maxPBSJobFailuresPerInstance]
+ }
+ return out
+}
+
+func samePBSJobFailureRecord(left, right pbsJobFailureRecord) bool {
+ if left.Host != right.Host || len(left.Failures) != len(right.Failures) {
+ return false
+ }
+ a, errA := json.Marshal(left.Failures)
+ b, errB := json.Marshal(right.Failures)
+ return errA == nil && errB == nil && bytes.Equal(a, b)
+}
diff --git a/internal/config/pbs_job_failures_test.go b/internal/config/pbs_job_failures_test.go
new file mode 100644
index 0000000000..8d1b6b1cf4
--- /dev/null
+++ b/internal/config/pbs_job_failures_test.go
@@ -0,0 +1,188 @@
+package config
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+)
+
+func failedPBSJob(id string) models.PBSJobHealthEvidence {
+ ended := time.Date(2026, 11, 3, 6, 0, 0, 0, time.UTC)
+ return models.PBSJobHealthEvidence{
+ ID: id,
+ Family: "sync",
+ Store: "offsite",
+ Remote: "pbs-main",
+ Enabled: true,
+ LastRunState: "connection error",
+ LastRunEndtime: ended.Unix(),
+ Posture: "critical",
+ PostureReason: "last-run-failed",
+ Freshness: models.PBSJobHealthFreshness{
+ ObservedAt: time.Now(),
+ LastRunEndTime: ended,
+ AgeSeconds: 42,
+ State: "observed",
+ },
+ }
+}
+
+func TestPBSJobFailureStoreKeepsFailuresAcrossARestart(t *testing.T) {
+ dir := t.TempDir()
+ store := NewPBSJobFailureStore(dir, nil)
+ healthy := failedPBSJob("v-1")
+ healthy.Family, healthy.Posture, healthy.PostureReason = "verify", "healthy", "last-run-success"
+ if err := store.Set("pbs-main", "https://pbs-a:8007", []models.PBSJobHealthEvidence{failedPBSJob("s-1"), healthy}); err != nil {
+ t.Fatalf("Set: %v", err)
+ }
+
+ reopened := NewPBSJobFailureStore(dir, nil)
+ if err := reopened.LoadError(); err != nil {
+ t.Fatalf("LoadError: %v", err)
+ }
+ got := reopened.Get("pbs-main", "https://pbs-a:8007")
+ if len(got) != 1 || got[0].ID != "s-1" || got[0].Posture != "critical" || got[0].Remote != "pbs-main" {
+ t.Fatalf("reloaded failures = %+v, want only the failed sync", got)
+ }
+ // Poll times are not part of the record, so an unchanged failure never
+ // rewrites the file.
+ if !got[0].Freshness.ObservedAt.IsZero() || got[0].Freshness.AgeSeconds != 0 {
+ t.Fatalf("poll-time fields were persisted: %+v", got[0].Freshness)
+ }
+ if got[0].Freshness.LastRunEndTime.IsZero() {
+ t.Fatal("the failed run's end time must be kept")
+ }
+}
+
+func TestPBSJobFailureStoreDoesNotRewriteAnUnchangedFailure(t *testing.T) {
+ dir := t.TempDir()
+ store := NewPBSJobFailureStore(dir, nil)
+ if err := store.Set("pbs-main", "https://pbs-a:8007", []models.PBSJobHealthEvidence{failedPBSJob("s-1")}); err != nil {
+ t.Fatalf("Set: %v", err)
+ }
+ path := filepath.Join(dir, pbsJobFailuresFileName)
+ old := time.Now().Add(-time.Hour)
+ if err := os.Chtimes(path, old, old); err != nil {
+ t.Fatalf("Chtimes: %v", err)
+ }
+ next := failedPBSJob("s-1")
+ next.Freshness.ObservedAt = time.Now().Add(time.Minute)
+ next.Freshness.AgeSeconds = 102
+ if err := store.Set("pbs-main", "https://pbs-a:8007", []models.PBSJobHealthEvidence{next}); err != nil {
+ t.Fatalf("Set: %v", err)
+ }
+ info, err := os.Stat(path)
+ if err != nil {
+ t.Fatalf("Stat: %v", err)
+ }
+ if !info.ModTime().Equal(old) {
+ t.Fatalf("an unchanged failure rewrote the record at %v", info.ModTime())
+ }
+}
+
+func TestPBSJobFailureStoreForgetsAnInstanceWithNoFailures(t *testing.T) {
+ dir := t.TempDir()
+ store := NewPBSJobFailureStore(dir, nil)
+ if err := store.Set("pbs-main", "https://pbs-a:8007", []models.PBSJobHealthEvidence{failedPBSJob("s-1")}); err != nil {
+ t.Fatalf("Set: %v", err)
+ }
+ if err := store.Set("pbs-main", "https://pbs-a:8007", nil); err != nil {
+ t.Fatalf("Set: %v", err)
+ }
+ if got := NewPBSJobFailureStore(dir, nil).Get("pbs-main", "https://pbs-a:8007"); len(got) != 0 {
+ t.Fatalf("cleared instance reloaded %+v", got)
+ }
+ var missing *PBSJobFailureStore
+ if missing.Get("pbs-main", "https://pbs-a:8007") != nil || missing.Set("pbs-main", "https://pbs-a:8007", nil) != nil || missing.Retain(nil) != nil {
+ t.Fatal("a nil store must be inert")
+ }
+}
+
+func TestPBSJobFailureStoreReportsAnUnreadableRecord(t *testing.T) {
+ dir := t.TempDir()
+ if err := os.WriteFile(filepath.Join(dir, pbsJobFailuresFileName), []byte("{not json"), 0o600); err != nil {
+ t.Fatalf("WriteFile: %v", err)
+ }
+ store := NewPBSJobFailureStore(dir, nil)
+ if store.LoadError() == nil {
+ t.Fatal("expected a load error for a corrupt record")
+ }
+ if got := store.Get("pbs-main", "https://pbs-a:8007"); len(got) != 0 {
+ t.Fatalf("corrupt record yielded %+v", got)
+ }
+}
+
+func TestPBSJobFailureStoreWithoutADataDirectoryWritesNothing(t *testing.T) {
+ cwd := t.TempDir()
+ t.Chdir(cwd)
+ store := NewPBSJobFailureStore("", nil)
+ if err := store.Set("pbs-main", "https://pbs-a:8007", []models.PBSJobHealthEvidence{failedPBSJob("s-1")}); err != nil {
+ t.Fatalf("Set: %v", err)
+ }
+ if got := store.Get("pbs-main", "https://pbs-a:8007"); len(got) != 1 {
+ t.Fatalf("in-memory record = %+v, want the failure", got)
+ }
+ if _, err := os.Stat(filepath.Join(cwd, pbsJobFailuresFileName)); !os.IsNotExist(err) {
+ t.Fatalf("record written to the working directory: %v", err)
+ }
+}
+
+func TestPBSJobFailureStoreBindsARecordToItsServer(t *testing.T) {
+ store := NewPBSJobFailureStore(t.TempDir(), nil)
+ if err := store.Set("pbs-main", "https://PBS-A:8007/", []models.PBSJobHealthEvidence{failedPBSJob("s-1")}); err != nil {
+ t.Fatalf("Set: %v", err)
+ }
+ if got := store.Get("pbs-main", "https://pbs-a:8007"); len(got) != 1 {
+ t.Fatalf("same server, other spelling = %+v", got)
+ }
+ if got := store.Get("pbs-main", "https://pbs-b:8007"); got != nil {
+ t.Fatalf("another server under the same name got %+v", got)
+ }
+}
+
+func TestPBSJobFailureStoreRetainDropsRemovedInstances(t *testing.T) {
+ dir := t.TempDir()
+ store := NewPBSJobFailureStore(dir, nil)
+ for _, id := range []string{"pbs-main", "pbs-old"} {
+ if err := store.Set(id, "https://pbs-a:8007", []models.PBSJobHealthEvidence{failedPBSJob("s-1")}); err != nil {
+ t.Fatalf("Set: %v", err)
+ }
+ }
+ if err := store.Retain([]string{"pbs-main"}); err != nil {
+ t.Fatalf("Retain: %v", err)
+ }
+ reopened := NewPBSJobFailureStore(dir, nil)
+ if len(reopened.Get("pbs-main", "https://pbs-a:8007")) != 1 || len(reopened.Get("pbs-old", "https://pbs-a:8007")) != 0 {
+ t.Fatal("Retain must keep configured instances and drop the rest")
+ }
+}
+
+func TestPBSJobFailureStoreNeverWritesAnUnreadableRecord(t *testing.T) {
+ dir := t.TempDir()
+ store := NewPBSJobFailureStore(dir, nil)
+ many := make([]models.PBSJobHealthEvidence, 0, 400)
+ for i := 0; i < 400; i++ {
+ job := failedPBSJob(fmt.Sprintf("s-%03d-%s", i, strings.Repeat("x", 200)))
+ job.Remote = strings.Repeat("r", 400)
+ many = append(many, job)
+ }
+ var lastErr error
+ for i := 0; i < 20 && lastErr == nil; i++ {
+ lastErr = store.Set(fmt.Sprintf("pbs-%02d", i), "https://pbs-a:8007", many)
+ }
+ if lastErr == nil {
+ t.Fatal("expected the size bound to refuse a write")
+ }
+ reopened := NewPBSJobFailureStore(dir, nil)
+ if err := reopened.LoadError(); err != nil {
+ t.Fatalf("the record on disk must stay readable: %v", err)
+ }
+ if got := reopened.Get("pbs-00", "https://pbs-a:8007"); len(got) != maxPBSJobFailuresPerInstance {
+ t.Fatalf("per-instance cap = %d, want %d", len(got), maxPBSJobFailuresPerInstance)
+ }
+}
diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go
index 36c145ec2b..08e537af94 100644
--- a/internal/monitoring/monitor.go
+++ b/internal/monitoring/monitor.go
@@ -1332,6 +1332,7 @@ type Monitor struct {
guestMetadataStore *config.GuestMetadataStore
dockerMetadataStore *config.DockerMetadataStore
dockerReportOrderStore *config.DockerReportOrderStore
+ pbsJobFailureStore *config.PBSJobFailureStore
hostMetadataStore *config.HostMetadataStore
hostContinuityStore *config.HostContinuityStore
hostAgentLifecycleMu sync.RWMutex
@@ -1838,6 +1839,9 @@ func New(cfg *config.Config) (*Monitor, error) {
if err := dockerReportOrderStore.LoadError(); err != nil {
return nil, fmt.Errorf("load Docker report order journal: %w", err)
}
+ // An unreadable record only loses carried PBS job failures; PBS reports
+ // them again on the jobs' next runs, so it never blocks startup.
+ pbsJobFailureStore := config.NewPBSJobFailureStore(cfg.DataPath, nil)
// Queued deliveries can run before a router exists. Restore the instance
// policy before creating workers, including for tenant persistence paths.
@@ -2021,6 +2025,7 @@ func New(cfg *config.Config) (*Monitor, error) {
guestMetadataStore: config.NewGuestMetadataStore(cfg.DataPath, nil),
dockerMetadataStore: config.NewDockerMetadataStore(cfg.DataPath, nil),
dockerReportOrderStore: dockerReportOrderStore,
+ pbsJobFailureStore: pbsJobFailureStore,
hostMetadataStore: config.NewHostMetadataStore(cfg.DataPath, nil),
hostContinuityStore: hostContinuityStore,
startTime: time.Now(),
diff --git a/internal/monitoring/monitor_pbs_coverage_test.go b/internal/monitoring/monitor_pbs_coverage_test.go
index ce20b43388..0cb7b2bbe3 100644
--- a/internal/monitoring/monitor_pbs_coverage_test.go
+++ b/internal/monitoring/monitor_pbs_coverage_test.go
@@ -984,3 +984,57 @@ func TestPBSJobHealthPostureReadsStatusLikeProxmox(t *testing.T) {
}
}
}
+
+// A Pulse restart empties state; the persisted record still carries the
+// failure, so the first poll that meets the job's next run in progress does
+// not read as recovery and clear the job's alert.
+func TestCarryPBSJobFailuresSurvivesARestart(t *testing.T) {
+ dir := t.TempDir()
+ before := &Monitor{state: models.NewState(), pbsJobFailureStore: config.NewPBSJobFailureStore(dir, nil)}
+ before.carryPBSJobFailuresForInstance("pbs-main", "https://pbs-a:8007", []models.PBSJobHealthEvidence{
+ carriedJob("s-1", "sync", "connection error", "critical", "last-run-failed"),
+ }, false)
+
+ after := &Monitor{state: models.NewState(), pbsJobFailureStore: config.NewPBSJobFailureStore(dir, nil)}
+ running := []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "", "unknown", "no-run-observed")}
+ if posture := postureOf(after.carryPBSJobFailuresForInstance("pbs-main", "https://pbs-a:8007", running, false), "sync", "s-1"); posture != "critical/last-run-failed" {
+ t.Fatalf("after restart posture = %s, want the carried failure", posture)
+ }
+
+ // The next run succeeds: the failure clears and so does its record.
+ succeeded := []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "OK", "healthy", "last-run-success")}
+ if posture := postureOf(after.carryPBSJobFailuresForInstance("pbs-main", "https://pbs-a:8007", succeeded, false), "sync", "s-1"); posture != "healthy/last-run-success" {
+ t.Fatalf("posture = %s, want the observed success", posture)
+ }
+ if left := config.NewPBSJobFailureStore(dir, nil).Get("pbs-main", "https://pbs-a:8007"); len(left) != 0 {
+ t.Fatalf("a recovered job stayed recorded: %+v", left)
+ }
+}
+
+// A record belongs to the server it was read from and to a configured
+// connection: a different server under a reused name, or a removed
+// connection, carries nothing.
+func TestCarryPBSJobFailuresIgnoresAnotherServerAndPrunesRemovedConnections(t *testing.T) {
+ dir := t.TempDir()
+ store := config.NewPBSJobFailureStore(dir, nil)
+ failed := []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "connection error", "critical", "last-run-failed")}
+ if err := store.Set("pbs-main", "https://pbs-a:8007", failed); err != nil {
+ t.Fatalf("Set: %v", err)
+ }
+ if err := store.Set("pbs-old", "https://pbs-old:8007", failed); err != nil {
+ t.Fatalf("Set: %v", err)
+ }
+
+ m := &Monitor{
+ state: models.NewState(),
+ pbsJobFailureStore: config.NewPBSJobFailureStore(dir, nil),
+ config: &config.Config{PBSInstances: []config.PBSInstance{{Name: "main"}}},
+ }
+ running := []models.PBSJobHealthEvidence{carriedJob("s-1", "sync", "", "unknown", "no-run-observed")}
+ if posture := postureOf(m.carryPBSJobFailuresForInstance("pbs-main", "https://pbs-b:8007", running, false), "sync", "s-1"); posture != "unknown/no-run-observed" {
+ t.Fatalf("another server under the same name inherited %s", posture)
+ }
+ if left := config.NewPBSJobFailureStore(dir, nil).Get("pbs-old", "https://pbs-old:8007"); len(left) != 0 {
+ t.Fatalf("a removed connection kept its record: %+v", left)
+ }
+}
diff --git a/internal/monitoring/monitor_pbs_pmg.go b/internal/monitoring/monitor_pbs_pmg.go
index dd6d9bf0fc..2e68e91edb 100644
--- a/internal/monitoring/monitor_pbs_pmg.go
+++ b/internal/monitoring/monitor_pbs_pmg.go
@@ -253,6 +253,31 @@ func carryPBSJobFailures(previous, current []models.PBSJobHealthEvidence, readFa
return out
}
+// carryPBSJobFailuresForInstance carries failures from the instance's last
+// stored results, or, on the first poll after a restart when state holds none,
+// from the persisted failure record of the same server, and records what is
+// still failing. Records of connections no longer configured are dropped.
+func (m *Monitor) carryPBSJobFailuresForInstance(instanceID, host string, current []models.PBSJobHealthEvidence, readFailed bool) []models.PBSJobHealthEvidence {
+ previous := m.state.PBSJobHealthEvidence(instanceID)
+ if len(previous) == 0 {
+ previous = m.pbsJobFailureStore.Get(instanceID, host)
+ }
+ carried := carryPBSJobFailures(previous, current, readFailed)
+ if err := m.pbsJobFailureStore.Set(instanceID, host, carried); err != nil {
+ log.Warn().Err(err).Str("instance", instanceID).Msg("Failed to record PBS job failures")
+ }
+ if m.config != nil {
+ configured := make([]string, 0, len(m.config.PBSInstances))
+ for _, cfg := range m.config.PBSInstances {
+ configured = append(configured, PBSMonitorResourceID(cfg.Name))
+ }
+ if err := m.pbsJobFailureStore.Retain(configured); err != nil {
+ log.Warn().Err(err).Msg("Failed to prune PBS job failure record")
+ }
+ }
+ return carried
+}
+
// pbsJobReportsOutcome says whether this poll observed how the job's newest
// run ended: success, warning or failure, rather than running or an
// unreadable state. It reads the run's own status, not the posture, so a
@@ -584,8 +609,9 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, clie
Str("instance", instanceName).
Msg("Failed to collect PBS job health evidence")
}
- pbsInst.JobHealthEvidence = carryPBSJobFailures(
- m.state.PBSJobHealthEvidence(pbsInst.ID),
+ pbsInst.JobHealthEvidence = m.carryPBSJobFailuresForInstance(
+ pbsInst.ID,
+ pbsInst.Host,
pbsJobHealthEvidenceFromFacts(jobFacts, time.Now()),
err != nil,
)