Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Stop tenant incident stores racing on shared ID counters
rcourtman88d44099aa3c
diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.mdindex 9ab54a9d35..540a33d923 100644--- a/docs/release-control/v6/internal/subsystems/ai-runtime.md+++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md@@ -128,6 +128,25 @@ on component mount. Closing clears the registered input so later keyboardcommands cannot target a detached composer. A handoff must leave Escape andkeyboard input in Assistant, not the underlying alert search.+### Process-wide memory ID sequences++`internal/ai/memory` mints incident, incident-event, remediation-record and+context-memory IDs from package-level sequences that every store in the process+shares. Multi-tenant monitoring gives each tenant monitor its own+`IncidentStore`, and their alert checks call `RecordAlertFired` from separate+goroutines, while a store's mutex covers only its own state. Each sequence is+therefore an `atomic.Int64` minted through `nextSequencedID` in+`internal/ai/memory/ids.go`, never a plain package counter. The ID shape stays+`<prefix><local second>-<n>`, but `n` no longer wraps at 1000. The old+`counter%1000` suffix repeated an ID whenever one generator minted more than a+thousand in a wall-clock second, and incident notes resolve an incident by ID+while timeline merges deduplicate events by ID. Within one process IDs are now+unique by construction. Across restarts only the timestamp separates them, so a+repeated local second (DST fall-back or a backward clock step) can still meet a+restarted sequence's suffix. Persisted IDs and the stored format are unchanged.+`internal/ai/memory/ids_test.go` mints from every generator concurrently and+records alerts into two tenant stores concurrently; run it under `-race`.+### Canonical incident-history queriesIncident context reads `IncidentStore.QueryIncidents`, whose page containsdiff --git a/internal/ai/memory/changes.go b/internal/ai/memory/changes.goindex 0771e9cd71..3b0a4e4bc0 100644--- a/internal/ai/memory/changes.go+++ b/internal/ai/memory/changes.go@@ -131,17 +131,3 @@ func (d *ChangeDetector) loadFromDisk() error {d.trimChanges()return nil}--// Helper functions--func intToString(n int) string {- if n == 0 {- return "0"- }- var result string- for n > 0 {- result = string(rune('0'+n%10)) + result- n /= 10- }- return result-}diff --git a/internal/ai/memory/context.go b/internal/ai/memory/context.goindex 1b5547d882..9a42d03758 100644--- a/internal/ai/memory/context.go+++ b/internal/ai/memory/context.go@@ -711,18 +711,15 @@ func (s *ContextStore) ForceSave() error {var memoryCounter, incidentMemCounter, patternMemCounter atomic.Int64func generateMemoryID() string {- n := memoryCounter.Add(1)- return fmt.Sprintf("mem-%s-%d", time.Now().Format("20060102150405"), n%1000)+ return nextSequencedID("mem-", &memoryCounter)}func generateIncidentMemoryID() string {- n := incidentMemCounter.Add(1)- return fmt.Sprintf("inc-mem-%s-%d", time.Now().Format("20060102150405"), n%1000)+ return nextSequencedID("inc-mem-", &incidentMemCounter)}func generatePatternMemoryID() string {- n := patternMemCounter.Add(1)- return fmt.Sprintf("pat-mem-%s-%d", time.Now().Format("20060102150405"), n%1000)+ return nextSequencedID("pat-mem-", &patternMemCounter)}func calculatePatternConfidence(occurrences int) float64 {diff --git a/internal/ai/memory/ids.go b/internal/ai/memory/ids.gonew file mode 100644index 0000000000..2be409a1da--- /dev/null+++ b/internal/ai/memory/ids.go@@ -0,0 +1,16 @@+package memory++import (+ "strconv"+ "sync/atomic"+ "time"+)++// nextSequencedID mints prefix + local wall-clock second + "-" + the next value+// of seq. Each sequence is package-level and shared by every store in the+// process, including one store per tenant monitor, so it must stay atomic.+// The suffix never wraps, so one sequence never repeats an ID within a process+// however many IDs share a second.+func nextSequencedID(prefix string, seq *atomic.Int64) string {+ return prefix + time.Now().Format("20060102150405") + "-" + strconv.FormatInt(seq.Add(1), 10)+}diff --git a/internal/ai/memory/ids_test.go b/internal/ai/memory/ids_test.gonew file mode 100644index 0000000000..530381625e--- /dev/null+++ b/internal/ai/memory/ids_test.go@@ -0,0 +1,122 @@+package memory++import (+ "fmt"+ "regexp"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+)++// TestSequencedIDsStayUniqueUnderConcurrentMinting mints from every+// package-level ID sequence on many goroutines at once. Under -race it proves+// the sequences are synchronized. The uniqueness check proves the suffix does+// not wrap: more than a thousand IDs from one generator land in the same+// wall-clock second, which the old counter%1000 suffix turned into duplicates.+func TestSequencedIDsStayUniqueUnderConcurrentMinting(t *testing.T) {+ generators := []struct {+ prefix string+ mint func() string+ }{+ {"inc-", generateIncidentID},+ {"inc-evt-", generateIncidentEventID},+ {"rem-", generateRecordID},+ {"mem-", generateMemoryID},+ {"inc-mem-", generateIncidentMemoryID},+ {"pat-mem-", generatePatternMemoryID},+ }+ const goroutines, perGoroutine = 16, 150++ for _, gen := range generators {+ t.Run(strings.TrimSuffix(gen.prefix, "-"), func(t *testing.T) {+ shape := regexp.MustCompile("^" + regexp.QuoteMeta(gen.prefix) + `\d{14}-[1-9]\d*$`)+ minted := make([][]string, goroutines)+ start := make(chan struct{})+ var wg sync.WaitGroup+ for g := range goroutines {+ wg.Add(1)+ go func() {+ defer wg.Done()+ <-start+ batch := make([]string, perGoroutine)+ for i := range batch {+ batch[i] = gen.mint()+ }+ minted[g] = batch+ }()+ }+ close(start)+ wg.Wait()++ seen := make(map[string]bool, goroutines*perGoroutine)+ for _, batch := range minted {+ for _, id := range batch {+ if !shape.MatchString(id) {+ t.Fatalf("ID %q does not match %s", id, shape)+ }+ if seen[id] {+ t.Fatalf("duplicate ID %q among %d concurrent mints", id, goroutines*perGoroutine)+ }+ seen[id] = true+ }+ }+ })+ }+}++// TestIncidentStoresMintIDsConcurrentlyAcrossTenants mirrors multi-tenant+// monitoring: each tenant monitor owns its own IncidentStore, and their alert+// checks call RecordAlertFired from separate goroutines. Each store's mutex+// covers only its own state, so the shared ID sequence must be synchronized on+// its own. Run under -race.+func TestIncidentStoresMintIDsConcurrentlyAcrossTenants(t *testing.T) {+ const tenants, checkersPerTenant, alertsPerChecker = 2, 4, 50+ stores := make([]*IncidentStore, tenants)+ for i := range stores {+ stores[i] = NewIncidentStore(IncidentStoreConfig{MaxIncidents: checkersPerTenant * alertsPerChecker})+ }++ startedAt := time.Now().Add(-time.Minute)+ start := make(chan struct{})+ var wg sync.WaitGroup+ for tenant, store := range stores {+ for checker := range checkersPerTenant {+ wg.Add(1)+ go func() {+ defer wg.Done()+ <-start+ for n := range alertsPerChecker {+ store.RecordAlertFired(&alerts.Alert{+ ID: fmt.Sprintf("tenant-%d-checker-%d-alert-%d", tenant, checker, n),+ Type: "cpu",+ Level: alerts.AlertLevelWarning,+ ResourceID: fmt.Sprintf("tenant-%d-vm-%d-%d", tenant, checker, n),+ ResourceName: "vm",+ StartTime: startedAt,+ })+ }+ }()+ }+ }+ close(start)+ wg.Wait()++ seen := make(map[string]bool, tenants*checkersPerTenant*alertsPerChecker)+ for tenant, store := range stores {+ store.mu.RLock()+ incidents := len(store.incidents)+ for _, shell := range store.incidents {+ if seen[shell.ID] {+ t.Errorf("tenant %d incident ID %q was already minted", tenant, shell.ID)+ }+ seen[shell.ID] = true+ }+ store.mu.RUnlock()+ if incidents != checkersPerTenant*alertsPerChecker {+ t.Errorf("tenant %d recorded %d incidents, want %d", tenant, incidents, checkersPerTenant*alertsPerChecker)+ }+ }+}diff --git a/internal/ai/memory/incidents.go b/internal/ai/memory/incidents.goindex 8729c83cef..5c763b2d92 100644--- a/internal/ai/memory/incidents.go+++ b/internal/ai/memory/incidents.go@@ -1747,18 +1747,14 @@ func cloneTime(value time.Time) *time.Time {return &cloned}-var incidentCounter int64+var incidentCounter, incidentEventCounter atomic.Int64func generateIncidentID() string {- incidentCounter++- return "inc-" + time.Now().Format("20060102150405") + "-" + intToString(int(incidentCounter%1000))+ return nextSequencedID("inc-", &incidentCounter)}-var incidentEventCounter int64-func generateIncidentEventID() string {- incidentEventCounter++- return "inc-evt-" + time.Now().Format("20060102150405") + "-" + intToString(int(incidentEventCounter%1000))+ return nextSequencedID("inc-evt-", &incidentEventCounter)}func formatAlertSummary(alert *alerts.Alert) string {diff --git a/internal/ai/memory/memory_coverage_test.go b/internal/ai/memory/memory_coverage_test.goindex 653f05796d..7ae826ea6a 100644--- a/internal/ai/memory/memory_coverage_test.go+++ b/internal/ai/memory/memory_coverage_test.go@@ -11,18 +11,11 @@ import ("github.com/rcourtman/pulse-go-rewrite/internal/alerts")-func TestChangeDetector_DefaultsAndHelpers(t *testing.T) {+func TestChangeDetector_Defaults(t *testing.T) {detector := NewChangeDetector(ChangeDetectorConfig{})if detector.maxChanges != 1000 {t.Fatalf("expected default maxChanges=1000, got %d", detector.maxChanges)}-- if got := intToString(0); got != "0" {- t.Errorf("intToString(0) = %q", got)- }- if got := intToString(42); got != "42" {- t.Errorf("intToString(42) = %q", got)- }}func TestNewChangeDetector_LoadsFromDisk(t *testing.T) {diff --git a/internal/ai/memory/remediation.go b/internal/ai/memory/remediation.goindex da93f5f4f5..97d2c8ac81 100644--- a/internal/ai/memory/remediation.go+++ b/internal/ai/memory/remediation.go@@ -5,6 +5,7 @@ import ("fmt""os""sync"+ "sync/atomic""time""github.com/rcourtman/pulse-go-rewrite/internal/utils"@@ -349,11 +350,10 @@ func (r *RemediationLog) loadFromDisk() error {// Helper functions-var recordCounter int64+var recordCounter atomic.Int64func generateRecordID() string {- recordCounter++- return "rem-" + time.Now().Format("20060102150405") + "-" + intToString(int(recordCounter%1000))+ return nextSequencedID("rem-", &recordCounter)}func truncateOutput(output string, maxLen int) string {