Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Resume Patrol's persisted schedule after restart

rcourtmanc38545226a66
diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md
index adac4daab1..0aa0e0a286 100644
--- a/docs/release-control/v6/internal/subsystems/ai-runtime.md
+++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md
@@ -25,6 +25,24 @@ that same result. Successful reads retain their content and execution provenance
## Purpose
+### Patrol resumes its actual next-check deadline after restart
+
+The main Patrol timer checkpoints its next deadline and effective interval in
+the existing run-history persistence envelope. A restart retains that deadline
+after the normal collection/configuration startup grace. An overdue deadline
+causes one immediate check after that grace, without replaying missed intervals.
+An explicit interval change resets and checkpoints the new timer deadline.
+Scoped checks and capped run-history retention cannot move an existing deadline.
+Older stores without a cursor resume from the last successful full run and the
+current interval. Disabled preparation never checkpoints a check that did not run.
+
+`patrol_schedule_test.go` exercises the real timer loop with Go simulated time,
+including repeated restarts, due/overdue execution, interval changes and first
+startup. History adapter and configuration tests verify file reconstruction,
+failed-write retry and concurrent history/deadline preservation. Live native
+Patrol and independent workload observations remain necessary for operational
+outcome qualification.
+
### Investigations retain collected backup and storage history evidence
Guest investigation manifests include the existing read-only storage tool so
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index 8184232917..30d6c3f918 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -21,6 +21,16 @@
## Purpose
+### Patrol timer checkpoints preserve run and recovery history
+
+The existing Patrol history envelope retains the next scheduled deadline and
+its effective interval. History and timer writes share a serialized
+read-modify-write boundary, preserving each other's records, daily tallies and
+cursor. A failed history read must fail the timer write instead of replacing
+unreadable evidence. The configuration and Patrol history tests cover concurrent
+writes, file reconstruction and failed-write retry. This cursor schedules a
+check only; it grants no additional recovery execution authority.
+
### Recurring findings preserve the ownership of recovery evidence
Storage and recovery findings use the same occurrence-bound investigation
diff --git a/internal/ai/patrol.go b/internal/ai/patrol.go
index 2cad890162..d053975a8f 100644
--- a/internal/ai/patrol.go
+++ b/internal/ai/patrol.go
@@ -583,7 +583,7 @@ type PatrolService struct {
lastBlockedReason string
lastBlockedCause PatrolFailureCause
lastBlockedAt time.Time
- nextScheduledAt time.Time // Tracks actual next patrol time (accounts for ticker resets)
+ nextScheduledAt time.Time // Live timer projection; enabled deadlines are checkpointed in runHistoryStore.
// Patrol run history with persistence support
runHistoryStore *PatrolRunHistoryStore
diff --git a/internal/ai/patrol_history_persistence.go b/internal/ai/patrol_history_persistence.go
index b298f2ae1a..65053fb4f1 100644
--- a/internal/ai/patrol_history_persistence.go
+++ b/internal/ai/patrol_history_persistence.go
@@ -13,6 +13,8 @@ import (
type PatrolHistoryPersistence interface {
SavePatrolRunHistory(runs []PatrolRunRecord) error
LoadPatrolRunHistory() ([]PatrolRunRecord, error)
+ SavePatrolSchedule(next time.Time, interval time.Duration) error
+ LoadPatrolSchedule() (time.Time, time.Duration, error)
}
// PatrolHistoryPersistenceAdapter bridges ConfigPersistence to PatrolHistoryPersistence interface
@@ -25,6 +27,18 @@ func NewPatrolHistoryPersistenceAdapter(cfg *config.ConfigPersistence) *PatrolHi
return &PatrolHistoryPersistenceAdapter{config: cfg}
}
+func (a *PatrolHistoryPersistenceAdapter) SavePatrolSchedule(next time.Time, interval time.Duration) error {
+ return a.config.SavePatrolSchedule(next, interval)
+}
+
+func (a *PatrolHistoryPersistenceAdapter) LoadPatrolSchedule() (time.Time, time.Duration, error) {
+ data, err := a.config.LoadPatrolRunHistory()
+ if err != nil {
+ return time.Time{}, 0, err
+ }
+ return data.NextScheduledAt, data.ScheduleInterval, nil
+}
+
// SavePatrolRunHistory saves patrol run history to disk via ConfigPersistence
func (a *PatrolHistoryPersistenceAdapter) SavePatrolRunHistory(runs []PatrolRunRecord) error {
// Convert from ai.PatrolRunRecord to config.PatrolRunRecord
@@ -194,10 +208,12 @@ func convertConfigToolCallsToAI(calls []config.ToolCallRecord) []ToolCallRecord
// PatrolRunHistoryStore provides thread-safe storage for patrol run history with optional persistence
type PatrolRunHistoryStore struct {
- mu sync.RWMutex
- runs []PatrolRunRecord
- maxRuns int
- persistence PatrolHistoryPersistence
+ mu sync.RWMutex
+ runs []PatrolRunRecord
+ maxRuns int
+ persistence PatrolHistoryPersistence
+ nextScheduledAt time.Time
+ scheduleInterval time.Duration
// Debounce save operations
saveTimer *time.Timer
savePending bool
@@ -261,10 +277,51 @@ func (s *PatrolRunHistoryStore) SetPersistence(p PatrolHistoryPersistence) error
s.mu.Unlock()
log.Info().Int("count", len(runs)).Msg("loaded patrol run history from disk")
}
+ next, interval, err := p.LoadPatrolSchedule()
+ if err != nil {
+ return err
+ }
+ s.mu.Lock()
+ s.nextScheduledAt, s.scheduleInterval = next, interval
+ s.mu.Unlock()
}
return nil
}
+func (s *PatrolRunHistoryStore) Schedule() (time.Time, time.Duration) {
+ s.mu.RLock()
+ defer s.mu.RUnlock()
+ return s.nextScheduledAt, s.scheduleInterval
+}
+
+// SetSchedule persists before the scheduler waits or starts its next run. Run
+// history still uses its ordinary debounce; a restart must not reset this cursor.
+func (s *PatrolRunHistoryStore) SetSchedule(next time.Time, interval time.Duration) error {
+ s.mu.Lock()
+ if s.nextScheduledAt.Equal(next) && s.scheduleInterval == interval && s.lastSaveError == nil {
+ s.mu.Unlock()
+ return nil
+ }
+ s.nextScheduledAt, s.scheduleInterval = next, interval
+ persistence := s.persistence
+ s.mu.Unlock()
+ if persistence == nil {
+ return nil
+ }
+ err := persistence.SavePatrolSchedule(next, interval)
+ s.mu.Lock()
+ s.lastSaveError = err
+ if err == nil {
+ s.lastSaveTime = time.Now()
+ }
+ onErr := s.onSaveError
+ s.mu.Unlock()
+ if err != nil && onErr != nil {
+ onErr(err)
+ }
+ return err
+}
+
// Add adds a new patrol run to the history
func (s *PatrolRunHistoryStore) Add(run PatrolRunRecord) {
if run.DurationMs == 0 && run.Duration > 0 {
diff --git a/internal/ai/patrol_history_persistence_test.go b/internal/ai/patrol_history_persistence_test.go
index 294ab78e6f..a0f1150baa 100644
--- a/internal/ai/patrol_history_persistence_test.go
+++ b/internal/ai/patrol_history_persistence_test.go
@@ -18,12 +18,74 @@ import (
// mockPatrolHistoryPersistence implements PatrolHistoryPersistence for testing
type mockPatrolHistoryPersistence struct {
- mu sync.Mutex
- runs []PatrolRunRecord
- saveErr error
- loadErr error
- saveCalls atomic.Int32
- loadCalls atomic.Int32
+ mu sync.Mutex
+ runs []PatrolRunRecord
+ saveErr error
+ loadErr error
+ saveCalls atomic.Int32
+ loadCalls atomic.Int32
+ nextScheduledAt time.Time
+ scheduleInterval time.Duration
+ scheduleSaves atomic.Int32
+}
+
+func TestPatrolHistorySchedulePersistenceAndRetry(t *testing.T) {
+ want := time.Now().Add(5 * time.Minute).UTC()
+ persistence := &mockPatrolHistoryPersistence{saveErr: errors.New("schedule write failed")}
+ store := NewPatrolRunHistoryStore(10)
+ if err := store.SetPersistence(persistence); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.SetSchedule(want, 10*time.Minute); err == nil {
+ t.Fatal("schedule persistence failure was hidden")
+ }
+ if err, _, _ := store.GetPersistenceStatus(); err == nil {
+ t.Fatal("schedule persistence failure was not retained")
+ }
+ persistence.mu.Lock()
+ persistence.saveErr = nil
+ persistence.mu.Unlock()
+ if err := store.SetSchedule(want, 10*time.Minute); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.SetSchedule(want, 10*time.Minute); err != nil {
+ t.Fatal(err)
+ }
+ if persistence.scheduleSaves.Load() != 2 {
+ t.Fatal("identical failed deadline must retry; identical saved deadline must not rewrite")
+ }
+ restored := NewPatrolRunHistoryStore(10)
+ if err := restored.SetPersistence(persistence); err != nil {
+ t.Fatal(err)
+ }
+ next, interval := restored.Schedule()
+ if !next.Equal(want) || interval != 10*time.Minute {
+ t.Fatalf("restored deadline %v/%v, want %v/10m", next, interval, want)
+ }
+}
+
+func TestPatrolHistoryScheduleSurvivesFileReconstruction(t *testing.T) {
+ dir := t.TempDir()
+ want := time.Now().Add(5 * time.Minute).UTC()
+ store := NewPatrolRunHistoryStore(10)
+ if err := store.SetPersistence(NewPatrolHistoryPersistenceAdapter(config.NewConfigPersistence(dir))); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.SetSchedule(want, 10*time.Minute); err != nil {
+ t.Fatal(err)
+ }
+ store.Add(PatrolRunRecord{ID: "scoped", Type: "scoped", CompletedAt: time.Now()})
+ if err := store.FlushPersistence(); err != nil {
+ t.Fatal(err)
+ }
+ restored := NewPatrolRunHistoryStore(10)
+ if err := restored.SetPersistence(NewPatrolHistoryPersistenceAdapter(config.NewConfigPersistence(dir))); err != nil {
+ t.Fatal(err)
+ }
+ next, interval := restored.Schedule()
+ if !next.Equal(want) || interval != 10*time.Minute || restored.Count() != 1 {
+ t.Fatalf("reconstructed history lost deadline or records: %v/%v count=%d", next, interval, restored.Count())
+ }
}
type errorPatrolHistoryPersistence struct {
@@ -38,6 +100,31 @@ func (e *errorPatrolHistoryPersistence) LoadPatrolRunHistory() ([]PatrolRunRecor
return nil, nil
}
+func (e *errorPatrolHistoryPersistence) SavePatrolSchedule(time.Time, time.Duration) error {
+ return e.err
+}
+
+func (e *errorPatrolHistoryPersistence) LoadPatrolSchedule() (time.Time, time.Duration, error) {
+ return time.Time{}, 0, nil
+}
+
+func (m *mockPatrolHistoryPersistence) SavePatrolSchedule(next time.Time, interval time.Duration) error {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.scheduleSaves.Add(1)
+ if m.saveErr != nil {
+ return m.saveErr
+ }
+ m.nextScheduledAt, m.scheduleInterval = next, interval
+ return nil
+}
+
+func (m *mockPatrolHistoryPersistence) LoadPatrolSchedule() (time.Time, time.Duration, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ return m.nextScheduledAt, m.scheduleInterval, m.loadErr
+}
+
func (m *mockPatrolHistoryPersistence) SavePatrolRunHistory(runs []PatrolRunRecord) error {
m.saveCalls.Add(1)
m.mu.Lock()
diff --git a/internal/ai/patrol_run.go b/internal/ai/patrol_run.go
index 17f68eed3a..e34dafe56f 100644
--- a/internal/ai/patrol_run.go
+++ b/internal/ai/patrol_run.go
@@ -117,20 +117,13 @@ func (p *PatrolService) patrolLoop(ctx context.Context) {
p.mu.Unlock()
}
- // Run initial patrol shortly after startup, but only if one hasn't run recently
- initialDelay := initialPatrolStartDelay
- initialTimer := time.NewTimer(initialDelay)
+ // Let configuration and collection initialize before resuming a due check.
+ // This grace period must not become a new full scheduling interval.
+ initialTimer := time.NewTimer(initialPatrolStartDelay)
defer initialTimer.Stop()
select {
case <-initialTimer.C:
- // Check if a patrol ran recently (within last hour) to avoid wasting tokens on restarts
- runHistory := p.GetRunHistory(10)
- skipInitial := shouldSkipInitialFullPatrol(runHistory, time.Now())
-
- if !skipInitial {
- p.runPatrolWithTrigger(ctx, TriggerReasonStartup, nil)
- }
case <-p.stopCh:
if !initialTimer.Stop() {
select {
@@ -154,23 +147,30 @@ func (p *PatrolService) patrolLoop(ctx context.Context) {
configCh := p.configChanged
p.mu.RUnlock()
- ticker := time.NewTicker(interval)
- defer ticker.Stop()
-
- p.mu.Lock()
- p.nextScheduledAt = time.Now().Add(interval)
- p.mu.Unlock()
+ next, trigger := p.initialPatrolSchedule(time.Now(), interval)
+ if trigger != TriggerReasonStartup {
+ p.setNextPatrolSchedule(next, interval)
+ }
+ timer := time.NewTimer(time.Until(next))
+ defer timer.Stop()
for {
select {
- case <-ticker.C:
- // Update next scheduled time before the run starts — time.Now() closely
- // matches the tick time here, and the ticker will fire again at roughly
- // this moment + interval regardless of how long the run takes.
- p.mu.Lock()
- p.nextScheduledAt = time.Now().Add(interval)
- p.mu.Unlock()
- p.runPatrolWithTrigger(ctx, TriggerReasonScheduled, nil)
+ case <-timer.C:
+ if trigger == TriggerReasonStartup {
+ p.runPatrolWithTrigger(ctx, trigger, nil)
+ // Keep the first-install cadence anchored after its startup check.
+ next = time.Now().Add(interval)
+ } else {
+ // Persist the next deadline before starting work. Collapse missed
+ // intervals into one due check rather than replaying a backlog.
+ next = nextPatrolDeadline(next, time.Now(), interval)
+ p.setNextPatrolSchedule(next, interval)
+ p.runPatrolWithTrigger(ctx, trigger, nil)
+ }
+ trigger = TriggerReasonScheduled
+ p.setNextPatrolSchedule(next, interval)
+ timer.Reset(time.Until(next))
case alert := <-p.adHocTrigger:
// Run immediate targeted patrol for this alert
@@ -178,20 +178,22 @@ func (p *PatrolService) patrolLoop(ctx context.Context) {
p.runTargetedPatrol(ctx, alert)
case <-configCh:
- // Config changed - reset ticker with new interval
+ // An operator interval change deliberately resets the timer. Persist
+ // that exact reset too, even if no new full run has completed yet.
p.mu.RLock()
newInterval := p.config.GetInterval()
p.mu.RUnlock()
if newInterval != interval {
interval = newInterval
- ticker.Reset(interval)
- p.mu.Lock()
- p.nextScheduledAt = time.Now().Add(interval)
- p.mu.Unlock()
+ if trigger != TriggerReasonStartup {
+ next = time.Now().Add(interval)
+ p.setNextPatrolSchedule(next, interval)
+ timer.Reset(time.Until(next))
+ }
log.Info().
Dur("interval", interval).
- Msg("Patrol ticker reset to new interval")
+ Msg("Patrol schedule reset to new interval")
}
case <-p.stopCh:
@@ -244,25 +246,40 @@ func (p *PatrolService) remindSweepLoop(ctx context.Context) {
}
}
-func shouldSkipInitialFullPatrol(runHistory []PatrolRunRecord, now time.Time) bool {
- for _, run := range runHistory {
- if run.CompletedAt.IsZero() {
- continue
+func (p *PatrolService) initialPatrolSchedule(now time.Time, interval time.Duration) (time.Time, TriggerReason) {
+ if p.runHistoryStore != nil {
+ if next, savedInterval := p.runHistoryStore.Schedule(); !next.IsZero() && savedInterval == interval {
+ return next, TriggerReasonScheduled
}
- timeSinceLastRun := now.Sub(run.CompletedAt)
- if timeSinceLastRun >= 1*time.Hour {
- continue
- }
- if isSuccessfulFullPatrolRun(run) {
- log.Info().
- Dur("time_since_last", timeSinceLastRun).
- Str("run_type", run.Type).
- Str("trigger_reason", run.TriggerReason).
- Msg("AI Patrol: Skipping initial patrol - recent successful full run exists")
- return true
+ }
+ // Upgrade old history without a timer cursor. Use all retained full runs:
+ // recent scoped/verification activity must not hide or move the cadence.
+ if last := p.LastSuccessfulFullPatrolAt(); !last.IsZero() {
+ return last.Add(interval), TriggerReasonScheduled
+ }
+ return now, TriggerReasonStartup
+}
+
+func nextPatrolDeadline(due, now time.Time, interval time.Duration) time.Time {
+ next := due.Add(interval)
+ if !next.After(now) {
+ return now.Add(interval)
+ }
+ return next
+}
+
+func (p *PatrolService) setNextPatrolSchedule(next time.Time, interval time.Duration) {
+ p.mu.Lock()
+ p.nextScheduledAt = next
+ enabled, store := p.config.Enabled, p.runHistoryStore
+ p.mu.Unlock()
+ // Disabled timer ticks are not checks. In particular, first-time provider
+ // setup must not leave a durable cursor that suppresses the startup check.
+ if enabled && store != nil {
+ if err := store.SetSchedule(next, interval); err != nil {
+ log.Error().Err(err).Time("next_scheduled_at", next).Msg("Failed to persist Patrol schedule")
}
}
- return false
}
// LastSuccessfulFullPatrolAt returns the completion time of the most recent
diff --git a/internal/ai/patrol_run_test.go b/internal/ai/patrol_run_test.go
index de708b56b0..7af89acb81 100644
--- a/internal/ai/patrol_run_test.go
+++ b/internal/ai/patrol_run_test.go
@@ -1746,37 +1746,29 @@ func TestGetStatus_BlockedReason(t *testing.T) {
}
}
-func TestShouldSkipInitialFullPatrol_IgnoresRecentScopedErrorRun(t *testing.T) {
+func TestInitialPatrolSchedule_IgnoresRecentScopedErrorRun(t *testing.T) {
now := time.Now()
- skip := shouldSkipInitialFullPatrol([]PatrolRunRecord{
- {
- ID: "scoped-error",
- Type: "scoped",
- TriggerReason: "alert_fired",
- CompletedAt: now.Add(-5 * time.Minute),
- ErrorCount: 1,
- Status: "error",
- },
- }, now)
- if skip {
- t.Fatal("expected recent scoped error run not to suppress initial full patrol")
+ p := NewPatrolService(nil, nil)
+ p.runHistoryStore.Add(PatrolRunRecord{
+ ID: "scoped-error", Type: "scoped", TriggerReason: "alert_fired",
+ CompletedAt: now.Add(-5 * time.Minute), ErrorCount: 1, Status: "error",
+ })
+ next, trigger := p.initialPatrolSchedule(now, 15*time.Minute)
+ if trigger != TriggerReasonStartup || !next.Equal(now) {
+ t.Fatalf("scoped error postponed initial full patrol: next=%v trigger=%s", next, trigger)
}
}
-func TestShouldSkipInitialFullPatrol_SkipsAfterRecentSuccessfulFullRun(t *testing.T) {
+func TestInitialPatrolSchedule_ResumesAfterRecentSuccessfulFullRun(t *testing.T) {
now := time.Now()
- skip := shouldSkipInitialFullPatrol([]PatrolRunRecord{
- {
- ID: "full-success",
- Type: "patrol",
- TriggerReason: "scheduled",
- CompletedAt: now.Add(-10 * time.Minute),
- ErrorCount: 0,
- Status: "healthy",
- },
- }, now)
- if !skip {
- t.Fatal("expected recent successful full patrol to suppress initial full patrol")
+ p := NewPatrolService(nil, nil)
+ p.runHistoryStore.Add(PatrolRunRecord{
+ ID: "full-success", Type: "patrol", TriggerReason: "scheduled",
+ CompletedAt: now.Add(-10 * time.Minute), Status: "healthy",
+ })
+ next, trigger := p.initialPatrolSchedule(now, 15*time.Minute)
+ if trigger != TriggerReasonScheduled || !next.Equal(now.Add(5*time.Minute)) {
+ t.Fatalf("full patrol cadence was not resumed: next=%v trigger=%s", next, trigger)
}
}
diff --git a/internal/ai/patrol_schedule_test.go b/internal/ai/patrol_schedule_test.go
new file mode 100644
index 0000000000..1623dcb46d
--- /dev/null
+++ b/internal/ai/patrol_schedule_test.go
@@ -0,0 +1,187 @@
+package ai
+
+import (
+ "context"
+ "testing"
+ "testing/synctest"
+ "time"
+)
+
+func TestPatrolLoopRestartRetainsRemainingInterval(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ p := NewPatrolService(nil, nil)
+ p.config = PatrolConfig{Enabled: true, Interval: 10 * time.Minute}
+ p.stopCh = make(chan struct{})
+ p.configChanged = make(chan struct{}, 1)
+ completed := time.Now().Add(-5 * time.Minute)
+ p.runHistoryStore.Add(PatrolRunRecord{
+ ID: "before-restart", Type: "patrol", TriggerReason: "scheduled",
+ StartedAt: completed.Add(-time.Minute), CompletedAt: completed, Status: "healthy",
+ })
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go p.patrolLoop(ctx)
+ time.Sleep(initialPatrolStartDelay + time.Second)
+ synctest.Wait()
+ status := p.GetStatus()
+ want := completed.Add(10 * time.Minute)
+ if status.NextPatrolAt == nil || !status.NextPatrolAt.Equal(want) {
+ t.Fatalf("restart postponed the existing cadence: next=%v, want=%v", status.NextPatrolAt, want)
+ }
+ })
+}
+
+func newScheduleTestPatrol(t *testing.T, persistence *mockPatrolHistoryPersistence, interval time.Duration) *PatrolService {
+ t.Helper()
+ p := NewPatrolService(nil, nil)
+ p.config = PatrolConfig{Enabled: true, Interval: interval, RuntimeBlockedReason: "scheduler fixture has no provider"}
+ p.stopCh = make(chan struct{})
+ p.configChanged = make(chan struct{}, 1)
+ if err := p.SetRunHistoryPersistence(persistence); err != nil {
+ t.Fatal(err)
+ }
+ return p
+}
+
+func startScheduleTestLoop(p *PatrolService) func() {
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ p.patrolLoop(ctx)
+ }()
+ return func() { cancel(); <-done }
+}
+
+func requireNextPatrol(t *testing.T, p *PatrolService, want time.Time) {
+ t.Helper()
+ status := p.GetStatus()
+ if status.NextPatrolAt == nil || !status.NextPatrolAt.Equal(want) {
+ t.Fatalf("next check = %v, want %v", status.NextPatrolAt, want)
+ }
+}
+
+func scheduleTestAttemptedAt(p *PatrolService) time.Time {
+ p.mu.RLock()
+ defer p.mu.RUnlock()
+ return p.lastBlockedAt
+}
+
+func TestPatrolLoopRepeatedRestartsPreserveDeadlineAndFire(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ due := time.Now().Add(5 * time.Minute)
+ persistence := &mockPatrolHistoryPersistence{nextScheduledAt: due, scheduleInterval: 10 * time.Minute}
+ for range 3 {
+ p := newScheduleTestPatrol(t, persistence, 10*time.Minute)
+ stop := startScheduleTestLoop(p)
+ time.Sleep(initialPatrolStartDelay + time.Second)
+ synctest.Wait()
+ requireNextPatrol(t, p, due)
+ if !scheduleTestAttemptedAt(p).IsZero() {
+ t.Fatal("restart ran a duplicate check before its deadline")
+ }
+ stop()
+ }
+ p := newScheduleTestPatrol(t, persistence, 10*time.Minute)
+ stop := startScheduleTestLoop(p)
+ defer stop()
+ time.Sleep(time.Until(due) - time.Second)
+ synctest.Wait()
+ if !scheduleTestAttemptedAt(p).IsZero() {
+ t.Fatal("scheduled check ran early")
+ }
+ time.Sleep(2 * time.Second)
+ synctest.Wait()
+ if at := scheduleTestAttemptedAt(p); !at.Equal(due) {
+ t.Fatalf("actual check attempt at %v, want %v", at, due)
+ }
+ requireNextPatrol(t, p, due.Add(10*time.Minute))
+ if got := persistence.scheduleSaves.Load(); got != 1 {
+ t.Fatalf("deadline writes = %d, want one advancement and no restart rewrites", got)
+ }
+ })
+}
+
+func TestPatrolLoopOverdueRestartRunsOnceWithoutBacklog(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ persistence := &mockPatrolHistoryPersistence{nextScheduledAt: time.Now().Add(-25 * time.Minute), scheduleInterval: 10 * time.Minute}
+ p := newScheduleTestPatrol(t, persistence, 10*time.Minute)
+ stop := startScheduleTestLoop(p)
+ defer stop()
+ wantAttempt := time.Now().Add(initialPatrolStartDelay)
+ time.Sleep(initialPatrolStartDelay + time.Second)
+ synctest.Wait()
+ if at := scheduleTestAttemptedAt(p); !at.Equal(wantAttempt) {
+ t.Fatalf("overdue check at %v, want startup readiness at %v", at, wantAttempt)
+ }
+ requireNextPatrol(t, p, wantAttempt.Add(10*time.Minute))
+ time.Sleep(time.Minute)
+ synctest.Wait()
+ if !scheduleTestAttemptedAt(p).Equal(wantAttempt) || persistence.scheduleSaves.Load() != 1 {
+ t.Fatal("overdue restart replayed missed intervals")
+ }
+ })
+}
+
+func TestPatrolLoopIntervalResetSurvivesRestart(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ persistence := &mockPatrolHistoryPersistence{nextScheduledAt: time.Now().Add(5 * time.Minute), scheduleInterval: 10 * time.Minute}
+ p := newScheduleTestPatrol(t, persistence, 10*time.Minute)
+ stop := startScheduleTestLoop(p)
+ time.Sleep(initialPatrolStartDelay + time.Second)
+ synctest.Wait()
+ cfg := p.config
+ cfg.Interval = 20 * time.Minute
+ want := time.Now().Add(cfg.Interval)
+ p.SetConfig(cfg)
+ synctest.Wait()
+ requireNextPatrol(t, p, want)
+ stop()
+ p = newScheduleTestPatrol(t, persistence, cfg.Interval)
+ stop = startScheduleTestLoop(p)
+ defer stop()
+ time.Sleep(initialPatrolStartDelay + time.Second)
+ synctest.Wait()
+ requireNextPatrol(t, p, want)
+ })
+}
+
+func TestPatrolLoopDisabledPreparationKeepsFirstStartupCheck(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ persistence := &mockPatrolHistoryPersistence{}
+ p := newScheduleTestPatrol(t, persistence, 10*time.Minute)
+ p.config.Enabled = false
+ stop := startScheduleTestLoop(p)
+ time.Sleep(2 * time.Minute)
+ synctest.Wait()
+ stop()
+ if !persistence.nextScheduledAt.IsZero() {
+ t.Fatal("disabled preparation persisted a check that never ran")
+ }
+ p = newScheduleTestPatrol(t, persistence, 10*time.Minute)
+ stop = startScheduleTestLoop(p)
+ defer stop()
+ want := time.Now().Add(initialPatrolStartDelay)
+ time.Sleep(initialPatrolStartDelay + time.Second)
+ synctest.Wait()
+ if at := scheduleTestAttemptedAt(p); !at.Equal(want) {
+ t.Fatalf("first startup check at %v, want %v", at, want)
+ }
+ })
+}
+
+func TestInitialPatrolScheduleLegacyHistoryAndChangedInterval(t *testing.T) {
+ now := time.Now()
+ p := NewPatrolService(nil, nil)
+ p.runHistoryStore.Add(PatrolRunRecord{ID: "full", Type: "patrol", CompletedAt: now.Add(-8 * time.Minute)})
+ for range 20 {
+ p.runHistoryStore.Add(PatrolRunRecord{Type: "scoped", CompletedAt: now})
+ }
+ if err := p.runHistoryStore.SetSchedule(now.Add(12*time.Minute), 20*time.Minute); err != nil {
+ t.Fatal(err)
+ }
+ next, trigger := p.initialPatrolSchedule(now, 10*time.Minute)
+ if !next.Equal(now.Add(2*time.Minute)) || trigger != TriggerReasonScheduled {
+ t.Fatalf("changed interval/retained full run schedule = %v %s", next, trigger)
+ }
+}
diff --git a/internal/config/patrol_schedule_persistence_test.go b/internal/config/patrol_schedule_persistence_test.go
new file mode 100644
index 0000000000..76441c9021
--- /dev/null
+++ b/internal/config/patrol_schedule_persistence_test.go
@@ -0,0 +1,72 @@
+package config_test
+
+import (
+ "os"
+ "path/filepath"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+)
+
+func TestPatrolScheduleAndHistoryWritersPreserveEachOther(t *testing.T) {
+ cp := config.NewConfigPersistence(t.TempDir())
+ now := time.Now().UTC()
+ runs := []config.PatrolRunRecord{{ID: "kept", Type: "patrol", StartedAt: now.Add(-time.Minute), CompletedAt: now, Status: "healthy"}}
+ if err := cp.SavePatrolRunHistory(runs); err != nil {
+ t.Fatal(err)
+ }
+ var wg sync.WaitGroup
+ start := make(chan struct{})
+ wg.Add(2)
+ go func() {
+ defer wg.Done()
+ <-start
+ for range 20 {
+ if err := cp.SavePatrolRunHistory(runs); err != nil {
+ t.Error(err)
+ return
+ }
+ }
+ }()
+ go func() {
+ defer wg.Done()
+ <-start
+ for i := range 20 {
+ if err := cp.SavePatrolSchedule(now.Add(time.Duration(i)*time.Minute), 10*time.Minute); err != nil {
+ t.Error(err)
+ return
+ }
+ }
+ }()
+ close(start)
+ wg.Wait()
+ data, err := cp.LoadPatrolRunHistory()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(data.Runs) != 1 || data.Runs[0].ID != "kept" || !data.NextScheduledAt.Equal(now.Add(19*time.Minute)) || data.ScheduleInterval != 10*time.Minute {
+ t.Fatalf("concurrent history/schedule writes lost state: %+v", data)
+ }
+ if data.DailyRuns[now.Format("2006-01-02")] != 1 {
+ t.Fatalf("schedule writes changed daily run tally: %v", data.DailyRuns)
+ }
+}
+
+func TestPatrolScheduleWritePreservesUnreadableHistory(t *testing.T) {
+ dir := t.TempDir()
+ cp := config.NewConfigPersistence(dir)
+ path := filepath.Join(dir, "ai_patrol_runs.json")
+ before := []byte("{not valid history")
+ if err := os.WriteFile(path, before, 0600); err != nil {
+ t.Fatal(err)
+ }
+ if err := cp.SavePatrolSchedule(time.Now(), 10*time.Minute); err == nil {
+ t.Fatal("saving the deadline overwrote unreadable history")
+ }
+ after, err := os.ReadFile(path)
+ if err != nil || string(after) != string(before) {
+ t.Fatalf("unreadable history changed: %q, %v", after, err)
+ }
+}
diff --git a/internal/config/persistence.go b/internal/config/persistence.go
index 62f8fcd0bc..354698ae54 100644
--- a/internal/config/persistence.go
+++ b/internal/config/persistence.go
@@ -27,6 +27,7 @@ import (
// ConfigPersistence handles saving and loading configuration
type ConfigPersistence struct {
mu sync.RWMutex
+ patrolHistoryMu sync.Mutex // Serializes history/schedule read-modify-write updates.
tx *importTransaction
configDir string
alertFile string
@@ -2721,6 +2722,9 @@ type PatrolRunHistoryData struct {
Version int `json:"version"`
LastSaved time.Time `json:"last_saved"`
Runs []PatrolRunRecord `json:"runs"`
+ // The timer deadline is independent of capped run history and scoped work.
+ NextScheduledAt time.Time `json:"next_scheduled_at,omitzero"`
+ ScheduleInterval time.Duration `json:"schedule_interval_ns,omitempty"`
// DailyRuns counts completed patrol runs per UTC day. Runs is deliberately
// capped for the operator-facing history, so on any install patrolling on a
@@ -3557,6 +3561,7 @@ func loadHistoryData[T any](
getCount func(*T) int,
getLastSaved func(*T) time.Time,
logLabel string,
+ strictDecode ...bool,
) (*T, error) {
mu.Lock()
defer mu.Unlock()
@@ -3583,6 +3588,9 @@ func loadHistoryData[T any](
var historyData T
if err := json.Unmarshal(data, &historyData); err != nil {
log.Error().Err(err).Str("file", filePath).Msgf("Failed to parse %s file", logLabel)
+ if len(strictDecode) > 0 && strictDecode[0] {
+ return nil, fmt.Errorf("parse %s: %w", logLabel, err)
+ }
return emptyDataFactory(), nil
}
@@ -3693,6 +3701,8 @@ func (c *ConfigPersistence) LoadWorkflowPromptActivityHistory() (*WorkflowPrompt
// SavePatrolRunHistory persists patrol run history to disk, carrying the
// daily run and finding tallies forward across the capped run list it replaces.
func (c *ConfigPersistence) SavePatrolRunHistory(runs []PatrolRunRecord) error {
+ c.patrolHistoryMu.Lock()
+ defer c.patrolHistoryMu.Unlock()
now := time.Now()
data := PatrolRunHistoryData{
Version: 1,
@@ -3705,6 +3715,8 @@ func (c *ConfigPersistence) SavePatrolRunHistory(runs []PatrolRunRecord) error {
if existing, err := c.LoadPatrolRunHistory(); err != nil {
log.Warn().Err(err).Str("file", c.aiPatrolRunsFile).Msg("Failed to read existing patrol run history; daily run and finding tallies restart from this save")
} else if existing != nil {
+ data.NextScheduledAt = existing.NextScheduledAt
+ data.ScheduleInterval = existing.ScheduleInterval
data.DailyRuns = existing.DailyRuns
data.RunTallyThrough = existing.RunTallyThrough
data.DailyNewFindings = existing.DailyNewFindings
@@ -3714,6 +3726,21 @@ func (c *ConfigPersistence) SavePatrolRunHistory(runs []PatrolRunRecord) error {
return saveHistoryData(c, c.aiPatrolRunsFile, data, len(runs), "patrol run history", "Patrol run history")
}
+// SavePatrolSchedule retains the timer cursor in the existing history envelope.
+// Serialize with history writes so neither writer can discard the other's data.
+func (c *ConfigPersistence) SavePatrolSchedule(next time.Time, interval time.Duration) error {
+ c.patrolHistoryMu.Lock()
+ defer c.patrolHistoryMu.Unlock()
+ data, err := c.loadPatrolRunHistory(true)
+ if err != nil {
+ return fmt.Errorf("load patrol history before saving schedule: %w", err)
+ }
+ data.NextScheduledAt = next
+ data.ScheduleInterval = interval
+ data.LastSaved = time.Now()
+ return saveHistoryData(c, c.aiPatrolRunsFile, data, len(data.Runs), "patrol schedule", "Patrol schedule")
+}
+
func newEmptyPatrolRunHistoryData() *PatrolRunHistoryData {
return &PatrolRunHistoryData{
Version: 1,
@@ -3723,6 +3750,10 @@ func newEmptyPatrolRunHistoryData() *PatrolRunHistoryData {
// LoadPatrolRunHistory loads patrol run history from disk
func (c *ConfigPersistence) LoadPatrolRunHistory() (*PatrolRunHistoryData, error) {
+ return c.loadPatrolRunHistory(false)
+}
+
+func (c *ConfigPersistence) loadPatrolRunHistory(strictDecode bool) (*PatrolRunHistoryData, error) {
return loadHistoryData(
c.fs,
&c.mu,
@@ -3754,6 +3785,7 @@ func (c *ConfigPersistence) LoadPatrolRunHistory() (*PatrolRunHistoryData, error
return data.LastSaved
},
"Patrol run history",
+ strictDecode,
)
}