Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Recover unread investigation history from its original store
pulse-triage[bot]6db0255a79db
diff --git a/internal/api/ai_handlers.go b/internal/api/ai_handlers.goindex 61b0ea13d9..8f64cae99e 100644--- a/internal/api/ai_handlers.go+++ b/internal/api/ai_handlers.go@@ -1842,6 +1842,14 @@ func (h *AISettingsHandler) WireOrchestratorAfterChatStart() {// setupInvestigationOrchestrator creates and wires the investigation orchestrator for an AI servicefunc (h *AISettingsHandler) setupInvestigationOrchestrator(orgID string, svc *ai.Service) {+ h.investigationMu.Lock()+ defer h.investigationMu.Unlock()+ h.setupInvestigationOrchestratorLocked(orgID, svc)+}++// setupInvestigationOrchestratorLocked publishes a store only after its retained+// history has loaded. The same lock serializes startup and read-side recovery.+func (h *AISettingsHandler) setupInvestigationOrchestratorLocked(orgID string, svc *ai.Service) {// Check factory exists first — if nil, this is OSS (no orchestrator available).// Clear any stale orchestrator from a prior setup so the patrol service// doesn't keep using a removed enterprise component.@@ -1868,34 +1876,43 @@ func (h *AISettingsHandler) setupInvestigationOrchestrator(orgID string, svc *aireturn}- // Get or create investigation store for this org- h.investigationMu.Lock()+ // A missing tenant directory must not silently become an empty memory-only+ // store. It would lose the original run and permit new writes over unread+ // history. Never fall back to another organization's persistence.store, exists := h.investigationStores[orgID]- if !exists {- // Get data directory from persistence- var dataDir string+ if !exists || store == nil {+ var persistence *config.ConfigPersistencemtPersistence, _, _, defaultPersistence, _, _ := h.stateRefs()- if defaultPersistence != nil && orgID == "default" {- dataDir = defaultPersistence.DataDir()+ if orgID == "default" && defaultPersistence != nil {+ persistence = defaultPersistence} else if mtPersistence != nil {- if p, err := mtPersistence.GetPersistence(orgID); err == nil {- dataDir = p.DataDir()+ var err error+ persistence, err = mtPersistence.GetPersistence(orgID)+ if err != nil {+ log.Warn().Err(err).Str("orgID", orgID).Msg("Investigation persistence unavailable")+ return}}+ if persistence == nil || strings.TrimSpace(persistence.DataDir()) == "" {+ log.Warn().Str("orgID", orgID).Msg("Investigation persistence unavailable")+ return+ }if storeFactory := getCreateInvestigationStore(); storeFactory != nil {- store = storeFactory(dataDir)+ store = storeFactory(persistence.DataDir())}if store == nil {log.Warn().Str("orgID", orgID).Msg("Investigation store not available (requires Pulse Pro)")- h.investigationMu.Unlock()return}if err := store.LoadFromDisk(); err != nil {log.Warn().Err(err).Str("orgID", orgID).Msg("Failed to load investigation store")+ return+ }+ if h.investigationStores == nil {+ h.investigationStores = make(map[string]aicontracts.InvestigationStore)}h.investigationStores[orgID] = store}- h.investigationMu.Unlock()// Get chat service for this org using org-scoped contextctx := context.WithValue(context.Background(), OrgIDContextKey, orgID)@@ -8318,6 +8335,45 @@ func (h *AISettingsHandler) HandleUpdatePatrolAutonomyMonitorOnly(w http.Respons// forward compatibility with any future ID scheme.const maxFindingIDLength = 256+// investigationOrchestratorForRead retries an incomplete startup without+// starting a new investigation. Failed stores were never published, so a retry+// can reload the same org's original files after their read failure clears.+func (h *AISettingsHandler) investigationOrchestratorForRead(ctx context.Context, svc *ai.Service) aicontracts.InvestigationOrchestrator {+ patrol := svc.GetPatrolService()+ if orchestrator := patrol.GetInvestigationOrchestrator(); orchestrator != nil {+ return orchestrator+ }+ if !isAIInvestigationEnabled() {+ return nil+ }+ h.investigationMu.Lock()+ defer h.investigationMu.Unlock()+ // Concurrent retries must not replace a newly recovered live orchestrator.+ if orchestrator := patrol.GetInvestigationOrchestrator(); orchestrator != nil {+ return orchestrator+ }+ h.setupInvestigationOrchestratorLocked(GetOrgID(ctx), svc)+ return patrol.GetInvestigationOrchestrator()+}++// writeFindingInvestigationReadError distinguishes proven absence in the current+// occurrence from a missing run the finding still references. Do not substitute+// a historical occurrence or turn unread retained history into a 404.+func writeFindingInvestigationReadError(w http.ResponseWriter, finding *ai.Finding) {+ knownRun := finding != nil && finding.InvestigationSessionID != ""+ if finding != nil && finding.InvestigationRecord != nil {+ record := finding.InvestigationRecord+ knownRun = knownRun || finding.IsCurrentInvestigation(&aicontracts.InvestigationSession{+ FindingID: record.FindingID, SessionID: record.SessionID, StartedAt: record.StartedAt,+ })+ }+ if knownRun {+ writeErrorResponse(w, http.StatusServiceUnavailable, "history_unavailable", "Saved investigation history could not be read", nil)+ return+ }+ writeErrorResponse(w, http.StatusNotFound, "not_found", "No investigation found for this finding", nil)+}+// HandleGetInvestigation returns investigation details for a finding (GET /api/ai/findings/{id}/investigation)func (h *AISettingsHandler) HandleGetInvestigation(w http.ResponseWriter, r *http.Request) {if r.Method != http.MethodGet {@@ -8349,7 +8405,7 @@ func (h *AISettingsHandler) HandleGetInvestigation(w http.ResponseWriter, r *htt}// Get investigation from orchestrator- orchestrator := patrol.GetInvestigationOrchestrator()+ orchestrator := h.investigationOrchestratorForRead(r.Context(), aiService)if orchestrator == nil {writeErrorResponse(w, http.StatusServiceUnavailable, "not_initialized", "Investigation orchestrator not initialized", nil)return@@ -8358,7 +8414,7 @@ func (h *AISettingsHandler) HandleGetInvestigation(w http.ResponseWriter, r *httinvestigation := orchestrator.GetInvestigationByFinding(findingID)finding := patrol.GetFindings().Get(findingID)if investigation == nil || (finding != nil && !finding.IsCurrentInvestigation(investigation)) {- writeErrorResponse(w, http.StatusNotFound, "not_found", "No investigation found for this finding", nil)+ writeFindingInvestigationReadError(w, finding)return}orgID := GetOrgID(r.Context())@@ -8403,7 +8459,7 @@ func (h *AISettingsHandler) HandleGetInvestigationMessages(w http.ResponseWriter}// Get investigation from orchestrator- orchestrator := patrol.GetInvestigationOrchestrator()+ orchestrator := h.investigationOrchestratorForRead(r.Context(), aiService)if orchestrator == nil {writeErrorResponse(w, http.StatusServiceUnavailable, "not_initialized", "Investigation orchestrator not initialized", nil)return@@ -8412,7 +8468,7 @@ func (h *AISettingsHandler) HandleGetInvestigationMessages(w http.ResponseWriterinvestigation := orchestrator.GetInvestigationByFinding(findingID)finding := patrol.GetFindings().Get(findingID)if investigation == nil || (finding != nil && !finding.IsCurrentInvestigation(investigation)) {- writeErrorResponse(w, http.StatusNotFound, "not_found", "No investigation found for this finding", nil)+ writeFindingInvestigationReadError(w, finding)return}diff --git a/internal/api/ai_handlers_investigation_additional_test.go b/internal/api/ai_handlers_investigation_additional_test.goindex 2a184ce7e3..20f99d8446 100644--- a/internal/api/ai_handlers_investigation_additional_test.go+++ b/internal/api/ai_handlers_investigation_additional_test.go@@ -461,6 +461,7 @@ func TestSetupInvestigationOrchestrator_WiresTenantBudgetChecker(t *testing.T) {t.Cleanup(func() { SetCreateInvestigationOrchestrator(prevOrchestratorFactory) })handler := &AISettingsHandler{+ mtPersistence: config.NewMultiTenantPersistence(t.TempDir()),investigationStores: make(map[string]aicontracts.InvestigationStore),}@@ -920,6 +921,7 @@ func TestSetupInvestigationOrchestratorUsesSubscriptionAgentLimits(t *testing.T)tenantSvc.SetStateProvider(&MockStateProvider{})handler := &AISettingsHandler{+ mtPersistence: config.NewMultiTenantPersistence(t.TempDir()),investigationStores: make(map[string]aicontracts.InvestigationStore),}handler.chatHandler = &AIHandler{@@ -971,6 +973,7 @@ func orchestratorPatrolFor(t *testing.T, audits unifiedresources.ResourceStore)tenantSvc := ai.NewService(nil, nil)tenantSvc.SetStateProvider(&MockStateProvider{})handler := &AISettingsHandler{+ mtPersistence: config.NewMultiTenantPersistence(t.TempDir()),investigationStores: make(map[string]aicontracts.InvestigationStore),}if audits != nil {diff --git a/internal/api/ai_history_readiness_test.go b/internal/api/ai_history_readiness_test.gonew file mode 100644index 0000000000..faf678cdb3--- /dev/null+++ b/internal/api/ai_history_readiness_test.go@@ -0,0 +1,393 @@+package api++import (+ "context"+ "crypto/sha256"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "reflect"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/ai"+ "github.com/rcourtman/pulse-go-rewrite/internal/ai/chat"+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts"+)++// The private store is supplied by a factory. This fixture reads real JSON+// files through that public seam; it does not model the private file format.+type historyReadDiskStore struct {+ *testInvestigationStore+ path string+ org string+}++func (s *historyReadDiskStore) LoadFromDisk() error {+ raw, err := os.ReadFile(s.path)+ if err != nil {+ return err+ }+ var record aicontracts.InvestigationSession+ if err := json.Unmarshal(raw, &record); err != nil {+ return err+ }+ s.sessions[record.ID] = &record+ s.byFinding[record.FindingID] = []string{record.ID}+ return nil+}++func (s *historyReadDiskStore) Create(string, string) *aicontracts.InvestigationSession {+ panic("reading retained history must not create a replacement investigation")+}++type historyReadOrchestrator struct {+ stubInvestigationOrchestrator+ store aicontracts.InvestigationStore+}++func (o *historyReadOrchestrator) GetInvestigationByFinding(id string) *aicontracts.InvestigationSession {+ return o.store.GetLatestByFinding(id)+}++func (o *historyReadOrchestrator) InvestigateFinding(context.Context, *aicontracts.Finding, string) error {+ panic("history read started an investigation")+}++func (o *historyReadOrchestrator) ReinvestigateFinding(context.Context, string, string) error {+ panic("history read started a re-investigation")+}++type historyReadMessages struct {+ stubChatService+ store *chat.SessionStore+}++func (s *historyReadMessages) GetMessages(_ context.Context, id string) ([]ai.ChatMessage, error) {+ messages, err := s.store.GetMessages(id)+ if err != nil {+ return nil, err+ }+ result := make([]ai.ChatMessage, len(messages))+ for i, message := range messages {+ result[i] = adaptChatMessage(message.ClientSafe())+ }+ return result, nil+}++type historyReadFixture struct {+ root string+ handler *AISettingsHandler+ services map[string]*ai.Service+ records map[string]aicontracts.InvestigationSession+ historyPaths map[string]string+ messagePaths map[string]string+ storeCalls atomic.Int32+ factoryCalls atomic.Int32+}++func newHistoryReadFixture(t *testing.T) *historyReadFixture {+ t.Helper()+ f := &historyReadFixture{+ root: t.TempDir(), services: make(map[string]*ai.Service), records: make(map[string]aicontracts.InvestigationSession),+ historyPaths: make(map[string]string), messagePaths: make(map[string]string),+ }+ mtp := config.NewMultiTenantPersistence(f.root)+ f.handler = &AISettingsHandler{+ mtPersistence: mtp, aiServices: make(map[string]*ai.Service),+ defaultConfig: &config.Config{AuthUser: "retained-admin", AuthPass: "retained-hash"},+ chatHandler: &AIHandler{services: make(map[string]AIService)},+ }+ messagesByOrg := make(map[string]*historyReadMessages)+ orgForDir := make(map[string]string)+ for _, org := range []string{"default", "tenant-a", "tenant-b"} {+ persistence, err := mtp.GetPersistence(org)+ if err != nil {+ t.Fatal(err)+ }+ orgForDir[persistence.DataDir()] = org+ // Retain an actual encrypted config alongside the history. Recovery+ // must not rewrite credentials or touch the administrator's access.+ if err := persistence.SaveSSOConfig(&config.SSOConfig{Providers: []config.SSOProvider{{+ ID: "retained-oidc", Type: config.SSOProviderTypeOIDC,+ OIDC: &config.OIDCProviderConfig{ClientID: org, ClientSecret: "synthetic-retained-secret"},+ }}}); err != nil {+ t.Fatal(err)+ }+ messages, err := chat.NewSessionStore(persistence.DataDir())+ if err != nil {+ t.Fatal(err)+ }+ original, err := messages.Create()+ if err != nil {+ t.Fatal(err)+ }+ if err := messages.AddMessage(original.ID, chat.Message{ID: "original-message", Role: "assistant", Content: "Retained evidence for " + org, Timestamp: time.Date(2026, 10, 20, 10, 0, 0, 0, time.UTC)}); err != nil {+ t.Fatal(err)+ }+ entries, err := filepath.Glob(filepath.Join(persistence.DataDir(), "ai_sessions", "*.json"))+ if err != nil {+ t.Fatal(err)+ }+ for _, entry := range entries {+ raw, err := os.ReadFile(entry)+ if err != nil {+ t.Fatal(err)+ }+ if strings.Contains(string(raw), "original-message") {+ f.messagePaths[org] = entry+ }+ }+ if f.messagePaths[org] == "" {+ t.Fatal("original persisted chat file missing")+ }+ completed := time.Date(2026, 10, 20, 10, 1, 0, 0, time.UTC)+ record := aicontracts.InvestigationSession{+ ID: "original-investigation-" + org, FindingID: "shared-history", SessionID: original.ID,+ Status: aicontracts.InvestigationStatusCompleted, Outcome: aicontracts.OutcomeUnclassified,+ StartedAt: completed.Add(-time.Minute), CompletedAt: &completed, TurnCount: 3,+ Summary: "Retained review for " + org, ToolsUsed: []string{"pulse_query"}, EvidenceIDs: []string{"original-evidence"},+ }+ f.records[org] = record+ path := filepath.Join(persistence.DataDir(), "retained-history-fixture.json")+ raw, err := json.Marshal(record)+ if err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(path, raw, 0600); err != nil {+ t.Fatal(err)+ }+ f.historyPaths[org] = path+ svc := ai.NewService(nil, nil)+ svc.SetOrgID(org)+ svc.SetStateProvider(&MockStateProvider{})+ finding := investigatedFinding(record.FindingID)+ finding.InvestigationSessionID = record.SessionID+ finding.InvestigationStatus = string(record.Status)+ finding.InvestigationRecord = &aicontracts.InvestigationRecord{ID: record.ID, FindingID: record.FindingID, SessionID: record.SessionID,+ Status: record.Status, Outcome: record.Outcome, Conclusion: record.Summary, StartedAt: record.StartedAt}+ svc.GetPatrolService().GetFindings().Add(finding)+ f.services[org] = svc+ chatSvc := chat.NewService(chat.Config{AIConfig: config.NewDefaultAIConfig(), DataDir: persistence.DataDir(), OrgID: org})+ messagesByOrg[org] = &historyReadMessages{store: messages}+ if org == "default" {+ f.handler.defaultPersistence, f.handler.defaultAIService = persistence, svc+ f.handler.chatHandler.defaultService = chatSvc+ } else {+ f.handler.aiServices[org] = svc+ f.handler.chatHandler.services[org] = chatSvc+ }+ }+ prevStore, prevOrchestrator := getCreateInvestigationStore(), getCreateInvestigationOrchestrator()+ aiInvestigationEnabledMu.RLock()+ prevEnabled := aiInvestigationEnabledFunc+ aiInvestigationEnabledMu.RUnlock()+ t.Cleanup(func() {+ SetCreateInvestigationStore(prevStore)+ SetCreateInvestigationOrchestrator(prevOrchestrator)+ SetAIInvestigationEnabled(prevEnabled)+ })+ SetAIInvestigationEnabled(func() bool { return true })+ SetCreateInvestigationStore(func(dir string) aicontracts.InvestigationStore {+ f.storeCalls.Add(1)+ org, ok := orgForDir[dir]+ if !ok {+ panic("history factory received another organization's directory")+ }+ return &historyReadDiskStore{testInvestigationStore: newTestInvestigationStore(), path: f.historyPaths[org], org: org}+ })+ SetCreateInvestigationOrchestrator(func(deps aicontracts.OrchestratorDeps) aicontracts.InvestigationOrchestrator {+ f.factoryCalls.Add(1)+ store := deps.Store.(*historyReadDiskStore)+ f.services[store.org].SetChatService(messagesByOrg[store.org])+ return &historyReadOrchestrator{store: store}+ })+ return f+}++func (f *historyReadFixture) read(org, suffix string) *httptest.ResponseRecorder {+ req := httptest.NewRequest(http.MethodGet, "/api/ai/findings/shared-history/investigation"+suffix, nil)+ req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, org))+ rec := httptest.NewRecorder()+ if suffix == "/messages" {+ f.handler.HandleGetInvestigationMessages(rec, req)+ } else {+ f.handler.HandleGetInvestigation(rec, req)+ }+ return rec+}++// Make the read fail deterministically even under root, without changing a+// byte of the original file. Restoring it is recovery, not a replacement run.+func blockHistoryFixtureFile(t *testing.T, path string) func() {+ t.Helper()+ if err := os.Rename(path, path+".retained"); err != nil {+ t.Fatal(err)+ }+ if err := os.Mkdir(path, 0700); err != nil {+ t.Fatal(err)+ }+ return func() {+ if err := os.Remove(path); err != nil {+ t.Fatal(err)+ }+ if err := os.Rename(path+".retained", path); err != nil {+ t.Fatal(err)+ }+ }+}++func historyFixtureFiles(t *testing.T, root string) map[string]string {+ t.Helper()+ result := make(map[string]string)+ err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {+ if err != nil || entry.IsDir() {+ return err+ }+ raw, err := os.ReadFile(path)+ if err == nil {+ relative, _ := filepath.Rel(root, path)+ result[relative] = fmt.Sprintf("%x", sha256.Sum256(raw))+ }+ return err+ })+ if err != nil {+ t.Fatal(err)+ }+ return result+}++func TestInvestigationReadRecovery(t *testing.T) {+ f := newHistoryReadFixture(t)+ before := historyFixtureFiles(t, f.root)+ restore := blockHistoryFixtureFile(t, f.historyPaths["tenant-a"])+ f.handler.setupInvestigationOrchestrator("tenant-a", f.services["tenant-a"])+ if f.factoryCalls.Load() != 0 || len(f.handler.investigationStores) != 0 || f.services["tenant-a"].GetPatrolService().GetInvestigationOrchestrator() != nil {+ t.Fatal("failed load was installed for investigation or exposed to its writer")+ }+ for _, suffix := range []string{"", "/messages"} {+ rec := f.read("tenant-a", suffix)+ if rec.Code != http.StatusServiceUnavailable || strings.Contains(rec.Body.String(), f.root) {+ t.Fatalf("unread history should be a redacted 503, got %d: %s", rec.Code, rec.Body.String())+ }+ }+ if rec := f.read("tenant-b", ""); rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "Retained review for tenant-b") {+ t.Fatalf("other org lost its independent history: %d %s", rec.Code, rec.Body.String())+ }+ restore()+ // Race a restored startup against both consuming reads. Only one factory+ // may publish the recovered store/orchestrator for this organization.+ var wg sync.WaitGroup+ for i := range 8 {+ wg.Add(1)+ go func() {+ defer wg.Done()+ suffix := ""+ if i%2 == 1 {+ suffix = "/messages"+ }+ rec := f.read("tenant-a", suffix)+ if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), f.records["tenant-a"].SessionID) || strings.Contains(rec.Body.String(), "tenant-b") {+ t.Errorf("original org/session did not recover: %d %s", rec.Code, rec.Body.String())+ }+ }()+ }+ wg.Wait()+ if got := f.factoryCalls.Load(); got != 2 {+ t.Fatalf("orchestrator factory calls = %d, want one per recovered org", got)+ }+ if after := historyFixtureFiles(t, f.root); !reflect.DeepEqual(before, after) {+ t.Fatal("history, chat files or encrypted credentials changed during recovery")+ }+ if f.handler.defaultConfig.AuthUser != "retained-admin" || f.handler.defaultConfig.AuthPass != "retained-hash" {+ t.Fatal("history recovery changed local administrator access")+ }+}++func TestInvestigationReadMessageRecovery(t *testing.T) {+ f := newHistoryReadFixture(t)+ before := historyFixtureFiles(t, f.root)+ if rec := f.read("default", "/messages"); rec.Code != http.StatusOK {+ t.Fatalf("initial original thread: %d %s", rec.Code, rec.Body.String())+ }+ restore := blockHistoryFixtureFile(t, f.messagePaths["default"])+ rec := f.read("default", "/messages")+ if rec.Code != http.StatusInternalServerError || strings.Contains(rec.Body.String(), f.root) || strings.Contains(rec.Body.String(), "original-message") {+ t.Fatalf("message failure should not be empty success or expose private data: %d %s", rec.Code, rec.Body.String())+ }+ restore()+ rec = f.read("default", "/messages")+ if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "original-message") || !strings.Contains(rec.Body.String(), f.records["default"].SessionID) {+ t.Fatalf("original thread did not recover: %d %s", rec.Code, rec.Body.String())+ }+ if !reflect.DeepEqual(before, historyFixtureFiles(t, f.root)) || f.factoryCalls.Load() != 1 {+ t.Fatal("message retry rewrote retained files or replaced the live orchestrator")+ }+}++func TestInvestigationReadReferencedRunIsNotAbsence(t *testing.T) {+ for _, mode := range []string{"missing", "different-session", "cached-record", "absent", "previous-occurrence"} {+ t.Run(mode, func(t *testing.T) {+ f := newHistoryReadFixture(t)+ svc := f.services["default"]+ finding := svc.GetPatrolService().GetFindings().Get("shared-history")+ session := f.records["default"]+ want := http.StatusServiceUnavailable+ switch mode {+ case "missing":+ session.FindingID = "other-finding"+ case "different-session":+ session.SessionID = "another-session"+ case "cached-record":+ finding.InvestigationSessionID = ""+ session.FindingID = "other-finding"+ case "absent", "previous-occurrence":+ finding.InvestigationSessionID, finding.InvestigationRecord = "", nil+ want = http.StatusNotFound+ if mode == "absent" {+ session.FindingID = "other-finding"+ } else {+ regressed := session.StartedAt.Add(time.Hour)+ finding.LastRegressionAt = ®ressed+ }+ }+ svc.GetPatrolService().GetFindings().ClearAll()+ svc.GetPatrolService().GetFindings().Add(finding)+ svc.GetPatrolService().SetInvestigationOrchestrator(&stubInvestigationOrchestrator{session: &session})+ for _, suffix := range []string{"", "/messages"} {+ rec := f.read("default", suffix)+ if rec.Code != want {+ t.Fatalf("%s: %d %s, want %d", suffix, rec.Code, rec.Body.String(), want)+ }+ }+ })+ }+}++func TestInvestigationReadRequiresScopedPersistenceAndEnabledRuntime(t *testing.T) {+ f := newHistoryReadFixture(t)+ for _, org := range []string{"unavailable", "../tenant-a"} {+ svc := ai.NewService(nil, nil)+ svc.SetStateProvider(&MockStateProvider{})+ f.handler.chatHandler.services[org] = chat.NewService(chat.Config{AIConfig: config.NewDefaultAIConfig()})+ if org == "unavailable" {+ f.handler.mtPersistence = nil+ }+ f.handler.setupInvestigationOrchestrator(org, svc)+ if svc.GetPatrolService().GetInvestigationOrchestrator() != nil || f.storeCalls.Load() != 0 {+ t.Fatal("missing/invalid org persistence fell back to default or a memory-only store")+ }+ f.handler.mtPersistence = config.NewMultiTenantPersistence(f.root)+ }+ SetAIInvestigationEnabled(func() bool { return false })+ if rec := f.read("default", ""); rec.Code != http.StatusServiceUnavailable || f.storeCalls.Load() != 0 {+ t.Fatalf("disabled runtime was created by a read: %d %s", rec.Code, rec.Body.String())+ }+}