Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package api
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"sync"
"testing"
"time"
"unsafe"
"github.com/gorilla/websocket"
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
"github.com/rcourtman/pulse-go-rewrite/internal/ai"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
internalauth "github.com/rcourtman/pulse-go-rewrite/pkg/auth"
)
type stubMetadataProvider struct{}
func (stubMetadataProvider) SetGuestURL(string, string) error { return nil }
func (stubMetadataProvider) SetDockerURL(string, string) error { return nil }
func (stubMetadataProvider) SetHostURL(string, string) error { return nil }
type recordingMetadataProvider struct {
guestURLs map[string]string
}
func TestTenantAgentServerForOrganizationFailsClosedAcrossTenants(t *testing.T) {
admissions := map[string]agentexec.AgentAdmission{
"token-a": {OrganizationID: "tenant-a", TokenID: "token-a", AgentID: "agent-a", Hostname: "host-a"},
"token-b": {OrganizationID: "tenant-b", TokenID: "token-b", AgentID: "agent-b", Hostname: "host-b"},
}
server := agentexec.NewServerWithAdmissionValidator(
func(token, _, _ string) (agentexec.AgentAdmission, bool) {
admission, ok := admissions[token]
return admission, ok
},
func(agentexec.AgentAdmission) bool { return true },
)
websocketServer := httptest.NewServer(http.HandlerFunc(server.HandleWebSocket))
defer websocketServer.Close()
register := func(admission agentexec.AgentAdmission) *websocket.Conn {
t.Helper()
url := "ws" + strings.TrimPrefix(websocketServer.URL, "http")
conn, _, err := websocket.DefaultDialer.Dial(url, http.Header{
"Origin": []string{websocketServer.URL},
})
if err != nil {
t.Fatalf("dial agent websocket: %v", err)
}
message, err := agentexec.NewMessage(agentexec.MsgTypeAgentRegister, "", agentexec.AgentRegisterPayload{
AgentID: admission.AgentID,
Hostname: admission.Hostname,
Token: admission.TokenID,
})
if err != nil {
t.Fatalf("create registration message: %v", err)
}
if err := conn.WriteJSON(message); err != nil {
t.Fatalf("write registration: %v", err)
}
var response agentexec.Message
if err := conn.ReadJSON(&response); err != nil {
t.Fatalf("read registration response: %v", err)
}
var registered agentexec.RegisteredPayload
if err := response.DecodePayload(®istered); err != nil || !registered.Success {
t.Fatalf("registration failed: payload=%+v err=%v", registered, err)
}
return conn
}
connA := register(admissions["token-a"])
defer connA.Close()
connB := register(admissions["token-b"])
defer connB.Close()
tenantA := tenantAgentServerForOrganization(server, "tenant-a")
agents := tenantA.GetConnectedAgents()
if len(agents) != 1 || agents[0].AgentID != "agent-a" {
t.Fatalf("tenant-a command view leaked another tenant: %#v", agents)
}
if _, err := tenantA.ExecuteCommand(context.Background(), "agent-b", agentexec.ExecuteCommandPayload{
Command: "true",
Trusted: true,
}); err == nil || !strings.Contains(err.Error(), "not connected") {
t.Fatalf("cross-tenant command dispatch did not fail closed: %v", err)
}
}
func (p *recordingMetadataProvider) SetGuestURL(id, url string) error {
p.guestURLs[id] = url
return nil
}
func (*recordingMetadataProvider) SetDockerURL(string, string) error { return nil }
func (*recordingMetadataProvider) SetHostURL(string, string) error { return nil }
func TestAISettingsHandlerMetadataProviderFactoryScopesExistingServices(t *testing.T) {
handler := NewAISettingsHandler(config.NewMultiTenantPersistence(t.TempDir()), nil, nil)
tenantContext := context.WithValue(context.Background(), OrgIDContextKey, "tenant-a")
defaultService := handler.GetAIService(context.Background())
tenantService := handler.GetAIService(tenantContext)
if defaultService == nil || tenantService == nil {
t.Fatal("expected default and tenant AI services")
}
providers := map[string]*recordingMetadataProvider{}
handler.SetMetadataProviderFactory(func(orgID string) ai.MetadataProvider {
provider := &recordingMetadataProvider{guestURLs: make(map[string]string)}
providers[orgID] = provider
return provider
})
if err := defaultService.SetResourceURL("vm", "shared-id", "https://default.internal"); err != nil {
t.Fatalf("set default URL: %v", err)
}
if err := tenantService.SetResourceURL("vm", "shared-id", "https://tenant.internal"); err != nil {
t.Fatalf("set tenant URL: %v", err)
}
if got := providers["default"].guestURLs["shared-id"]; got != "https://default.internal" {
t.Fatalf("default provider URL = %q", got)
}
if got := providers["tenant-a"].guestURLs["shared-id"]; got != "https://tenant.internal" {
t.Fatalf("tenant provider URL = %q", got)
}
newTenantContext := context.WithValue(context.Background(), OrgIDContextKey, "tenant-b")
newTenantService := handler.GetAIService(newTenantContext)
if err := newTenantService.SetResourceURL("vm", "shared-id", "https://new-tenant.internal"); err != nil {
t.Fatalf("set new tenant URL: %v", err)
}
if got := providers["tenant-b"].guestURLs["shared-id"]; got != "https://new-tenant.internal" {
t.Fatalf("new tenant provider URL = %q", got)
}
}
type stubThresholdProvider struct{}
func (stubThresholdProvider) GetNodeCPUThreshold() float64 { return 80 }
func (stubThresholdProvider) GetNodeMemoryThreshold() float64 { return 85 }
func (stubThresholdProvider) GetGuestMemoryThreshold() float64 {
return 90
}
func (stubThresholdProvider) GetGuestDiskThreshold() float64 { return 95 }
func (stubThresholdProvider) GetStorageThreshold() float64 { return 92 }
func (stubThresholdProvider) GetDiskTemperatureThreshold(alerts.DiskTemperatureHost, string) (float64, float64) {
return 70, 65
}
func (stubThresholdProvider) GetTrueNASDiskTemperatureThreshold(string, string) (float64, float64) {
return 70, 65
}
type stubMetricsHistoryProvider struct{}
func (stubMetricsHistoryProvider) GetNodeMetrics(string, string, time.Duration) []ai.MetricPoint {
return nil
}
func (stubMetricsHistoryProvider) GetGuestMetrics(string, string, time.Duration) []ai.MetricPoint {
return nil
}
func (stubMetricsHistoryProvider) GetAllGuestMetrics(string, time.Duration) map[string][]ai.MetricPoint {
return nil
}
func (stubMetricsHistoryProvider) GetAllStorageMetrics(string, time.Duration) map[string][]ai.MetricPoint {
return nil
}
type stubUnifiedResourceProvider struct {
resources []unifiedresources.Resource
}
func (s stubUnifiedResourceProvider) GetAll() []unifiedresources.Resource {
return s.resources
}
func (s stubUnifiedResourceProvider) GetInfrastructure() []unifiedresources.Resource {
return nil
}
func (s stubUnifiedResourceProvider) GetWorkloads() []unifiedresources.Resource {
return nil
}
func (s stubUnifiedResourceProvider) GetByType(unifiedresources.ResourceType) []unifiedresources.Resource {
return nil
}
func (stubUnifiedResourceProvider) GetStats() unifiedresources.ResourceStats {
return unifiedresources.ResourceStats{}
}
func (s stubUnifiedResourceProvider) GetTopByCPU(int, []unifiedresources.ResourceType) []unifiedresources.Resource {
return nil
}
func (s stubUnifiedResourceProvider) GetTopByMemory(int, []unifiedresources.ResourceType) []unifiedresources.Resource {
return nil
}
func (s stubUnifiedResourceProvider) GetTopByDisk(int, []unifiedresources.ResourceType) []unifiedresources.Resource {
return nil
}
func (s stubUnifiedResourceProvider) GetRelated(string) map[string][]unifiedresources.Resource {
return map[string][]unifiedresources.Resource{}
}
func (stubUnifiedResourceProvider) FindContainerHost(string) string { return "" }
func TestAISettingsHandler_setSSECORSHeaders(t *testing.T) {
handler := newTestAISettingsHandler(&config.Config{AllowedOrigins: "*"}, nil, nil)
req := httptest.NewRequest(http.MethodGet, "/api/ai/stream", nil)
req.Header.Set("Origin", "https://example.com")
rec := httptest.NewRecorder()
handler.setSSECORSHeaders(rec, req)
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
t.Fatalf("expected allow origin, got %q", got)
}
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
t.Fatalf("expected no credentials header for wildcard origins, got %q", got)
}
if rec.Header().Get("Access-Control-Allow-Methods") == "" {
t.Fatalf("expected allow methods header")
}
if rec.Header().Get("Access-Control-Allow-Headers") == "" {
t.Fatalf("expected allow headers header")
}
if got := rec.Header().Get("Vary"); got != "" {
t.Fatalf("expected no Vary header for wildcard policy, got %q", got)
}
handler = newTestAISettingsHandler(&config.Config{AllowedOrigins: "https://allowed.com"}, nil, nil)
req = httptest.NewRequest(http.MethodGet, "/api/ai/stream", nil)
req.Header.Set("Origin", "https://allowed.com")
rec = httptest.NewRecorder()
handler.setSSECORSHeaders(rec, req)
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://allowed.com" {
t.Fatalf("expected allow origin for matched origin, got %q", got)
}
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
t.Fatalf("expected credentials header for matched origin, got %q", got)
}
if got := rec.Header().Get("Vary"); got != "Origin" {
t.Fatalf("expected Vary=Origin for matched origin, got %q", got)
}
handler = newTestAISettingsHandler(&config.Config{AllowedOrigins: "https://allowed.com"}, nil, nil)
req = httptest.NewRequest(http.MethodGet, "/api/ai/stream", nil)
req.Header.Set("Origin", "https://nope.com")
rec = httptest.NewRecorder()
handler.setSSECORSHeaders(rec, req)
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
t.Fatalf("expected no allow origin for mismatched origin, got %q", got)
}
if rec.Header().Get("Access-Control-Allow-Methods") == "" {
t.Fatalf("expected allow methods header for mismatched origin")
}
if got := rec.Header().Get("Vary"); got != "Origin" {
t.Fatalf("expected Vary=Origin for mismatched origin, got %q", got)
}
handler = newTestAISettingsHandler(&config.Config{AllowedOrigins: "*"}, nil, nil)
req = httptest.NewRequest(http.MethodGet, "/api/ai/stream", nil)
rec = httptest.NewRecorder()
handler.setSSECORSHeaders(rec, req)
if len(rec.Header()) != 0 {
t.Fatalf("expected no headers when origin is missing, got %v", rec.Header())
}
}
func TestAISettingsHandler_GetAIService_MultiTenantProviders(t *testing.T) {
tmp := t.TempDir()
mtp := config.NewMultiTenantPersistence(tmp)
handler := NewAISettingsHandler(mtp, nil, nil)
handler.SetStateProvider(&stubStateProvider{})
handler.SetUnifiedResourceProvider(stubUnifiedResourceProvider{})
handler.SetMetadataProvider(stubMetadataProvider{})
handler.SetPatrolThresholdProvider(stubThresholdProvider{})
handler.SetMetricsHistoryProvider(stubMetricsHistoryProvider{})
handler.SetBaselineStore(ai.NewBaselineStore(ai.DefaultBaselineConfig()))
handler.SetChangeDetector(ai.NewChangeDetector(ai.ChangeDetectorConfig{}))
handler.SetRemediationLog(ai.NewRemediationLog(ai.RemediationLogConfig{MaxRecords: 1}))
handler.SetIncidentStore(memory.NewIncidentStore(memory.IncidentStoreConfig{DataDir: t.TempDir()}))
handler.SetPatternDetector(ai.NewPatternDetector(ai.DefaultPatternConfig()))
handler.SetCorrelationDetector(ai.NewCorrelationDetector(ai.DefaultCorrelationConfig()))
ctx := context.WithValue(context.Background(), OrgIDContextKey, "tenant-1")
svc := handler.GetAIService(ctx)
if svc == nil {
t.Fatalf("expected tenant service to be created")
}
if svc2 := handler.GetAIService(ctx); svc2 != svc {
t.Fatalf("expected cached tenant service")
}
}
// Report GETs read Patrol findings through ExistingAIService, so it must never
// construct a tenant service: construction can list provider models and start
// background workers.
func TestAISettingsHandler_ExistingAIServiceNeverConstructs(t *testing.T) {
handler := NewAISettingsHandler(config.NewMultiTenantPersistence(t.TempDir()), nil, nil)
t.Cleanup(handler.StopServices)
tenant := context.WithValue(context.Background(), OrgIDContextKey, "tenant-1")
if svc := handler.ExistingAIService(tenant); svc != nil {
t.Fatalf("expected no service before the tenant's first GetAIService, got %p", svc)
}
handler.aiServicesMu.RLock()
built := len(handler.aiServices)
handler.aiServicesMu.RUnlock()
if built != 0 {
t.Fatalf("ExistingAIService constructed %d tenant services, want 0", built)
}
created := handler.GetAIService(tenant)
if created == nil {
t.Fatal("expected GetAIService to construct the tenant service")
}
if got := handler.ExistingAIService(tenant); got != created {
t.Fatalf("expected the running tenant service %p, got %p", created, got)
}
if got, want := handler.ExistingAIService(context.Background()), handler.GetAIService(context.Background()); got != want {
t.Fatalf("expected the default service %p, got %p", want, got)
}
}
func TestAISettingsHandler_GetAIService_UsesTenantReadState(t *testing.T) {
tmp := t.TempDir()
mtp := config.NewMultiTenantPersistence(tmp)
mtm := monitoring.NewMultiTenantMonitor(&config.Config{}, mtp, nil)
t.Cleanup(mtm.Stop)
tenantAdapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
tenantMonitor := &monitoring.Monitor{}
tenantMonitor.SetResourceStore(tenantAdapter)
setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{"tenant-1": tenantMonitor})
handler := NewAISettingsHandler(mtp, mtm, nil)
handler.SetReadState(unifiedresources.NewRegistry(nil))
ctx := context.WithValue(context.Background(), OrgIDContextKey, "tenant-1")
svc := handler.GetAIService(ctx)
if svc == nil {
t.Fatalf("expected tenant AI service")
}
field := reflect.ValueOf(svc).Elem().FieldByName("readState")
ptr := unsafe.Pointer(field.UnsafeAddr())
current := reflect.NewAt(field.Type(), ptr).Elem().Interface().(unifiedresources.ReadState)
if current != tenantAdapter {
t.Fatalf("expected tenant read state adapter, got %#v", current)
}
}
func TestAISettingsHandler_GetAIService_UsesTenantUnifiedResourceProvider(t *testing.T) {
tmp := t.TempDir()
mtp := config.NewMultiTenantPersistence(tmp)
mtm := monitoring.NewMultiTenantMonitor(&config.Config{}, mtp, nil)
t.Cleanup(mtm.Stop)
tenantAdapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
tenantMonitor := &monitoring.Monitor{}
tenantMonitor.SetResourceStore(tenantAdapter)
setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{"tenant-1": tenantMonitor})
handler := NewAISettingsHandler(mtp, mtm, nil)
handler.SetUnifiedResourceProvider(stubUnifiedResourceProvider{})
ctx := context.WithValue(context.Background(), OrgIDContextKey, "tenant-1")
svc := handler.GetAIService(ctx)
if svc == nil {
t.Fatalf("expected tenant AI service")
}
field := reflect.ValueOf(svc).Elem().FieldByName("unifiedResourceProvider")
ptr := unsafe.Pointer(field.UnsafeAddr())
current := reflect.NewAt(field.Type(), ptr).Elem().Interface().(ai.UnifiedResourceProvider)
if current != tenantAdapter {
t.Fatalf("expected tenant unified provider adapter, got %#v", current)
}
}
func TestAISettingsHandler_GetAIService_NonDefaultWithoutTenantReadStateFailsClosed(t *testing.T) {
tmp := t.TempDir()
mtp := config.NewMultiTenantPersistence(tmp)
mtm := monitoring.NewMultiTenantMonitor(&config.Config{}, mtp, nil)
t.Cleanup(mtm.Stop)
handler := NewAISettingsHandler(mtp, mtm, nil)
handler.SetReadState(unifiedresources.NewRegistry(nil))
ctx := context.WithValue(context.Background(), OrgIDContextKey, "tenant-1")
svc := handler.GetAIService(ctx)
if svc == nil {
t.Fatalf("expected tenant AI service")
}
field := reflect.ValueOf(svc).Elem().FieldByName("readState")
ptr := unsafe.Pointer(field.UnsafeAddr())
current := reflect.NewAt(field.Type(), ptr).Elem().Interface()
if current != nil {
t.Fatalf("expected nil tenant read state when tenant monitor read-state is unavailable, got %#v", current)
}
}
func TestAISettingsHandler_GetAIService_NonDefaultWithoutTenantUnifiedProviderFailsClosed(t *testing.T) {
tmp := t.TempDir()
mtp := config.NewMultiTenantPersistence(tmp)
mtm := monitoring.NewMultiTenantMonitor(&config.Config{}, mtp, nil)
t.Cleanup(mtm.Stop)
handler := NewAISettingsHandler(mtp, mtm, nil)
handler.SetUnifiedResourceProvider(stubUnifiedResourceProvider{})
ctx := context.WithValue(context.Background(), OrgIDContextKey, "tenant-1")
svc := handler.GetAIService(ctx)
if svc == nil {
t.Fatalf("expected tenant AI service")
}
field := reflect.ValueOf(svc).Elem().FieldByName("unifiedResourceProvider")
ptr := unsafe.Pointer(field.UnsafeAddr())
current := reflect.NewAt(field.Type(), ptr).Elem().Interface()
if current != nil {
t.Fatalf("expected nil tenant unified provider when tenant monitor provider is unavailable, got %#v", current)
}
}
func TestAISettingsHandler_SetUnifiedResourceProvider_ReappliesTenantScopedProvider(t *testing.T) {
tmp := t.TempDir()
mtp := config.NewMultiTenantPersistence(tmp)
mtm := monitoring.NewMultiTenantMonitor(&config.Config{}, mtp, nil)
t.Cleanup(mtm.Stop)
tenantAdapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
tenantMonitor := &monitoring.Monitor{}
tenantMonitor.SetResourceStore(tenantAdapter)
setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{"tenant-1": tenantMonitor})
handler := NewAISettingsHandler(mtp, mtm, nil)
ctx := context.WithValue(context.Background(), OrgIDContextKey, "tenant-1")
svc := handler.GetAIService(ctx)
if svc == nil {
t.Fatalf("expected tenant AI service")
}
handler.SetUnifiedResourceProvider(stubUnifiedResourceProvider{})
field := reflect.ValueOf(svc).Elem().FieldByName("unifiedResourceProvider")
ptr := unsafe.Pointer(field.UnsafeAddr())
current := reflect.NewAt(field.Type(), ptr).Elem().Interface().(ai.UnifiedResourceProvider)
if current != tenantAdapter {
t.Fatalf("expected tenant scoped provider to be preserved, got %#v", current)
}
}
func TestAISettingsHandler_GetAIService_TenantUsesCanonicalReadStateWithoutSnapshotProvider(t *testing.T) {
tmp := t.TempDir()
mtp := config.NewMultiTenantPersistence(tmp)
defaultMonitor, defaultState, _ := newTestMonitor(t)
defaultState.VMs = []models.VM{{ID: "vm-default"}}
tenantAdapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
tenantMonitor := &monitoring.Monitor{}
tenantMonitor.SetResourceStore(tenantAdapter)
mtm := &monitoring.MultiTenantMonitor{}
setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{
"default": defaultMonitor,
"tenant-1": tenantMonitor,
})
handler := NewAISettingsHandler(mtp, mtm, nil)
handler.SetStateProvider(defaultMonitor)
ctx := context.WithValue(context.Background(), OrgIDContextKey, "tenant-1")
svc := handler.GetAIService(ctx)
if svc == nil {
t.Fatalf("expected tenant AI service")
}
if svc.GetStateProvider() != nil {
t.Fatalf("expected tenant AI service to avoid snapshot provider bridge")
}
if svc.GetPatrolService() == nil {
t.Fatalf("expected patrol service to initialize from tenant read state")
}
field := reflect.ValueOf(svc).Elem().FieldByName("readState")
ptr := unsafe.Pointer(field.UnsafeAddr())
current := reflect.NewAt(field.Type(), ptr).Elem().Interface().(unifiedresources.ReadState)
if current != tenantAdapter {
t.Fatalf("expected tenant read state adapter, got %#v", current)
}
}
// In DEMO_MODE the demo middleware admits every GET and HEAD as a read, and
// the CSRF check skips them, so a mutating handler reachable by a safe method
// is a write any demo visitor can make. These routes dispatch without a mux
// method pattern, so each handler must refuse GET and HEAD itself.
func TestDemoModeSafeMethodsCannotReachRouteMutations(t *testing.T) {
dataPath := t.TempDir()
hashed, err := internalauth.HashPassword("demo-router-test-password")
if err != nil {
t.Fatalf("HashPassword: %v", err)
}
cfg := &config.Config{
DataPath: dataPath,
ConfigPath: dataPath,
DemoMode: true,
AuthUser: "demo",
AuthPass: hashed,
}
router := NewRouter(cfg, nil, nil, nil, nil, "1.0.0")
cleanupTestRouter(t, router)
handler := router.Handler()
router.samlManager.services["okta"] = newTestSAMLService(t, "okta", `<?xml version="1.0"?>
<EntityDescriptor xmlns="urn:oasis:names:tc:SAML:2.0:metadata" entityID="idp">
<IDPSSODescriptor protocolSupportEnumeration="urn:oasis:names:tc:SAML:2.0:protocol">
<SingleSignOnService Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" Location="https://idp.example.com/sso"/>
<SingleLogoutService Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" Location="https://idp.example.com/slo"/>
</IDPSSODescriptor>
</EntityDescriptor>`)
samlSession := generateSessionToken()
GetSessionStore().CreateSAMLSession(samlSession, time.Hour, "agent", "127.0.0.1", "demo", &SAMLTokenInfo{
ProviderID: "okta",
NameID: "name-id",
SessionIndex: "sess-1",
})
cases := []struct {
path string
body string
allow string
}{
{path: "/api/saml/okta/logout", allow: http.MethodPost},
}
for _, tc := range cases {
for _, method := range []string{http.MethodGet, http.MethodHead} {
req := httptest.NewRequest(method, tc.path, strings.NewReader(tc.body))
req.SetBasicAuth("demo", "demo-router-test-password")
req.AddCookie(&http.Cookie{Name: "pulse_session", Value: samlSession})
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusMethodNotAllowed {
t.Fatalf("%s %s status = %d, want %d: %s", method, tc.path, rec.Code, http.StatusMethodNotAllowed, rec.Body.String())
}
if allow := rec.Header().Get("Allow"); allow != tc.allow {
t.Fatalf("%s %s Allow = %q, want %q", method, tc.path, allow, tc.allow)
}
}
}
if !ValidateSession(samlSession) {
t.Fatal("safe methods cleared the SAML session")
}
}
func TestAISettingsHandler_GetConfig_NonDefaultFallsBackWhenMultiTenantUnavailable(t *testing.T) {
handler := newTestAISettingsHandler(&config.Config{APIToken: "token"}, config.NewConfigPersistence(t.TempDir()), nil)
ctx := context.WithValue(context.Background(), OrgIDContextKey, "acme")
if got := handler.getConfig(ctx); got == nil {
t.Fatalf("expected legacy config fallback for non-default org without tenant monitor")
}
}
func TestAISettingsHandler_GetPersistence_NonDefaultFallsBackWhenMultiTenantUnavailable(t *testing.T) {
persistence := config.NewConfigPersistence(t.TempDir())
handler := newTestAISettingsHandler(&config.Config{DataPath: t.TempDir()}, persistence, nil)
ctx := context.WithValue(context.Background(), OrgIDContextKey, "acme")
if got := handler.getPersistence(ctx); got != persistence {
t.Fatalf("expected legacy persistence fallback for non-default org without tenant persistence, got %#v", got)
}
}
func TestAISettingsHandler_GetConfig_NonDefaultInvalidOrgFailsClosedWhenMultiTenantAvailable(t *testing.T) {
mtp := config.NewMultiTenantPersistence(t.TempDir())
mtm := monitoring.NewMultiTenantMonitor(&config.Config{}, mtp, nil)
t.Cleanup(mtm.Stop)
handler := NewAISettingsHandler(mtp, mtm, nil)
handler.SetConfig(&config.Config{APIToken: "token"})
ctx := context.WithValue(context.Background(), OrgIDContextKey, "../bad")
if got := handler.getConfig(ctx); got != nil {
t.Fatalf("expected nil config for invalid non-default org in multi-tenant mode, got %#v", got)
}
}
func TestAISettingsHandler_GetPersistence_NonDefaultInvalidOrgFailsClosedWhenMultiTenantAvailable(t *testing.T) {
mtp := config.NewMultiTenantPersistence(t.TempDir())
mtm := monitoring.NewMultiTenantMonitor(&config.Config{}, mtp, nil)
t.Cleanup(mtm.Stop)
handler := NewAISettingsHandler(mtp, mtm, nil)
ctx := context.WithValue(context.Background(), OrgIDContextKey, "../bad")
if got := handler.getPersistence(ctx); got != nil {
t.Fatalf("expected nil persistence for invalid non-default org in multi-tenant mode, got %#v", got)
}
}
func TestAISettingsHandler_GetAIService_NonDefaultInvalidOrgReturnsFailClosedTenantService(t *testing.T) {
setMockModeForTest(t, false)
mtp := config.NewMultiTenantPersistence(t.TempDir())
mtm := monitoring.NewMultiTenantMonitor(&config.Config{}, mtp, nil)
t.Cleanup(mtm.Stop)
handler := NewAISettingsHandler(mtp, mtm, nil)
defaultSvc := handler.GetAIService(context.Background())
if defaultSvc == nil {
t.Fatal("expected default AI service to be available")
}
ctx := context.WithValue(context.Background(), OrgIDContextKey, "../bad")
tenantSvc := handler.GetAIService(ctx)
if tenantSvc == nil {
t.Fatal("expected fail-closed tenant service")
}
if tenantSvc == defaultSvc {
t.Fatal("expected non-default invalid org to not fall back to default legacy service")
}
if got := tenantSvc.GetOrgID(); got != "../bad" {
t.Fatalf("expected tenant org id to be preserved, got %q", got)
}
if tenantSvc.IsEnabled() {
t.Fatal("expected fail-closed tenant service to be disabled")
}
if tenantSvc.HasLicenseFeature(ai.FeatureAIAutoFix) {
t.Fatal("expected fail-closed tenant service license checker to deny features")
}
}
func TestNewAISettingsHandler_DefaultServiceAlwaysInitialized(t *testing.T) {
setMockModeForTest(t, false)
handler := NewAISettingsHandler(nil, nil, nil)
svc := handler.GetAIService(context.Background())
if svc == nil {
t.Fatal("expected default AI service to be initialized even without persistence")
}
if got := svc.GetOrgID(); got != "default" {
t.Fatalf("expected default org id, got %q", got)
}
}
func TestAISettingsHandler_GetAIService_NonDefaultWithTenantMonitorWithoutPersistenceFailsClosed(t *testing.T) {
setMockModeForTest(t, false)
mtp := config.NewMultiTenantPersistence(t.TempDir())
mtm := monitoring.NewMultiTenantMonitor(&config.Config{}, mtp, nil)
t.Cleanup(mtm.Stop)
handler := NewAISettingsHandler(nil, mtm, nil)
defaultSvc := handler.GetAIService(context.Background())
if defaultSvc == nil {
t.Fatal("expected default AI service to be available")
}
ctx := context.WithValue(context.Background(), OrgIDContextKey, "tenant-1")
tenantSvc := handler.GetAIService(ctx)
if tenantSvc == nil {
t.Fatal("expected fail-closed tenant service")
}
if tenantSvc == defaultSvc {
t.Fatal("expected non-default org to not fall back to default service when tenant monitor is present")
}
if got := tenantSvc.GetOrgID(); got != "tenant-1" {
t.Fatalf("expected tenant org id, got %q", got)
}
if tenantSvc.IsEnabled() {
t.Fatal("expected fail-closed tenant service to be disabled")
}
}
// TestHandleGetPatrolDigest_PayloadContract pins the wire shape the Patrol
// "This week" card and docs/API.md describe: snake_case rollup groups, an
// explicit window with coverage flags, and by_outcome serialised as an object.
func TestHandleGetPatrolDigest_PayloadContract(t *testing.T) {
t.Parallel()
handler := createTestAIHandler(t)
req := httptest.NewRequest(http.MethodGet, "/api/ai/patrol/digest?days=14", nil)
rec := httptest.NewRecorder()
handler.HandleGetPatrolDigest(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
}
var payload map[string]json.RawMessage
if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
for _, key := range []string{"generated_at", "window", "mode", "runs", "findings", "investigations", "actions", "alerts", "spend"} {
if _, ok := payload[key]; !ok {
t.Fatalf("digest payload missing %q: %s", key, rec.Body.String())
}
}
var window struct {
Days int `json:"days"`
HistoryComplete bool `json:"history_complete"`
}
if err := json.Unmarshal(payload["window"], &window); err != nil {
t.Fatal(err)
}
if window.Days != 14 || !window.HistoryComplete {
t.Fatalf("window = %+v", window)
}
var investigations map[string]json.RawMessage
if err := json.Unmarshal(payload["investigations"], &investigations); err != nil {
t.Fatal(err)
}
if string(investigations["by_outcome"]) != "{}" {
t.Fatalf("by_outcome must serialise as an empty object, got %s", investigations["by_outcome"])
}
var findings map[string]json.RawMessage
if err := json.Unmarshal(payload["findings"], &findings); err != nil {
t.Fatal(err)
}
for _, key := range []string{"new", "open_by_severity", "resolved", "auto_resolved", "dismissed", "suppressed"} {
if _, ok := findings[key]; !ok {
t.Fatalf("findings rollup missing %q: %s", key, payload["findings"])
}
}
if strings.TrimSpace(string(payload["mode"])) != `"monitor"` {
t.Fatalf("mode = %s, want monitor when no autonomy is configured", payload["mode"])
}
}
// TestBuildPatrolDigestReportsAvailability pins the contract the scheduled
// Patrol summary relies on: without a Patrol service the digest is the zero
// shape and reported as unavailable, so a schedule fails clearly instead of
// emailing an empty week.
func TestBuildPatrolDigestReportsAvailability(t *testing.T) {
t.Parallel()
handler := createTestAIHandler(t)
digest, available := handler.BuildPatrolDigest(context.Background(), 0)
if available {
t.Fatal("digest must report unavailable without a Patrol service")
}
if digest.Window.Days != ai.PatrolDigestDefaultDays || digest.Runs.Total != 0 || digest.Mode != config.PatrolAutonomyMonitor {
t.Fatalf("zero digest = %+v", digest)
}
if digest.Investigations.ByOutcome == nil {
t.Fatal("by_outcome must stay an object for clients and email rendering")
}
patrol := &ai.PatrolService{}
setUnexportedField(t, patrol, "runHistoryStore", ai.NewPatrolRunHistoryStore(10))
setUnexportedField(t, handler.defaultAIService, "patrolService", patrol)
if _, available := handler.BuildPatrolDigest(context.Background(), 45); !available {
t.Fatal("digest must report available once a Patrol service exists")
}
}
// Pulse #2350: the usage export must carry the prompt-cache buckets and price
// them at the provider's cache rates, in both formats.
func TestHandleExportAICostHistoryCarriesPromptCacheBuckets(t *testing.T) {
t.Parallel()
tmp := t.TempDir()
cfg := &config.Config{DataPath: tmp}
persistence := config.NewConfigPersistence(tmp)
handler := newTestAISettingsHandler(cfg, persistence, nil)
store := handler.defaultAIService.CostStore()
if store == nil {
t.Fatal("expected the AI service to own a cost store")
}
store.Record(cost.UsageEvent{
Timestamp: time.Now(),
Provider: "anthropic",
RequestModel: "anthropic:claude-sonnet-5",
UseCase: "patrol",
InputTokens: 100_000,
OutputTokens: 10_000,
CacheCreationInputTokens: 200_000,
CacheReadInputTokens: 1_000_000,
})
// 0.1M*2.00 + 0.01M*10.00 + 0.2M*2.50 + 1.0M*0.20 at Anthropic cache rates.
const wantUSD = 1.0
req := newLoopbackRequest(http.MethodGet, "/api/ai/cost/export?days=7&format=json", nil)
rec := httptest.NewRecorder()
handler.HandleExportAICostHistory(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("json export status = %d: %s", rec.Code, rec.Body.String())
}
var exported struct {
Events []struct {
cost.UsageEvent
EstimatedUSD float64 `json:"estimated_usd"`
PricingKnown bool `json:"pricing_known"`
} `json:"events"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &exported); err != nil {
t.Fatalf("decode json export: %v", err)
}
if len(exported.Events) != 1 {
t.Fatalf("expected one exported event, got %d", len(exported.Events))
}
got := exported.Events[0]
if got.CacheCreationInputTokens != 200_000 || got.CacheReadInputTokens != 1_000_000 || got.InputTokens != 100_000 {
t.Fatalf("json export lost cache buckets: %+v", got.UsageEvent)
}
if !got.PricingKnown || got.EstimatedUSD < wantUSD-1e-6 || got.EstimatedUSD > wantUSD+1e-6 {
t.Fatalf("json export estimated_usd = %f (known=%v), want %f", got.EstimatedUSD, got.PricingKnown, wantUSD)
}
req = newLoopbackRequest(http.MethodGet, "/api/ai/cost/export?days=7&format=csv", nil)
rec = httptest.NewRecorder()
handler.HandleExportAICostHistory(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("csv export status = %d: %s", rec.Code, rec.Body.String())
}
lines := strings.Split(strings.TrimSpace(rec.Body.String()), "\n")
if len(lines) != 2 {
t.Fatalf("expected header and one row, got %d lines: %q", len(lines), rec.Body.String())
}
if !strings.Contains(lines[0], "input_tokens,output_tokens,cache_creation_input_tokens,cache_read_input_tokens,estimated_usd,") {
t.Fatalf("csv header missing cache columns after output_tokens: %s", lines[0])
}
if !strings.Contains(lines[1], ",100000,10000,200000,1000000,1.000000,true,") {
t.Fatalf("csv row missing cache buckets or cache-rate pricing: %s", lines[1])
}
}
// sessionMutationRecordingAIService records every Assistant session mutation
// and every running check that reaches the service, so a rejected request can
// be proven to stop before the handler touches the service at all.
type sessionMutationRecordingAIService struct {
capturingAIService
mu sync.Mutex
mutations []string
runningCalls int
}
func newSessionMutationRecordingAIService() *sessionMutationRecordingAIService {
return &sessionMutationRecordingAIService{capturingAIService: capturingAIService{running: true}}
}
func (s *sessionMutationRecordingAIService) record(op, sessionID string) {
s.mu.Lock()
defer s.mu.Unlock()
s.mutations = append(s.mutations, op+":"+sessionID)
}
func (s *sessionMutationRecordingAIService) recorded() []string {
s.mu.Lock()
defer s.mu.Unlock()
return append([]string(nil), s.mutations...)
}
func (s *sessionMutationRecordingAIService) IsRunning() bool {
s.mu.Lock()
defer s.mu.Unlock()
s.runningCalls++
return s.capturingAIService.running
}
func (s *sessionMutationRecordingAIService) runningChecks() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.runningCalls
}
func (s *sessionMutationRecordingAIService) AbortSession(ctx context.Context, sessionID string) error {
s.record("abort", sessionID)
return nil
}
var assistantSessionMutationSubresources = []string{"abort"}
func TestAssistantSessionMutationsRequirePost(t *testing.T) {
for _, sub := range assistantSessionMutationSubresources {
for _, method := range []string{http.MethodGet, http.MethodHead, http.MethodPut, http.MethodDelete} {
t.Run(sub+"/"+method, func(t *testing.T) {
svc := newSessionMutationRecordingAIService()
handler := &AIHandler{}
setUnexportedField(t, handler, "defaultService", svc)
router := &Router{aiHandler: handler}
req := httptest.NewRequest(method, "/api/ai/sessions/session-1/"+sub, nil)
rec := httptest.NewRecorder()
router.routeAISessions(rec, req)
if rec.Code != http.StatusMethodNotAllowed {
t.Fatalf("%s %s status = %d, want %d", method, sub, rec.Code, http.StatusMethodNotAllowed)
}
if allow := rec.Header().Get("Allow"); allow != http.MethodPost {
t.Fatalf("%s %s Allow = %q, want %q", method, sub, allow, http.MethodPost)
}
if got := svc.recorded(); len(got) != 0 {
t.Fatalf("%s %s reached the service: %v", method, sub, got)
}
if checks := svc.runningChecks(); checks != 0 {
t.Fatalf("%s %s resolved the service (%d running checks) before rejecting the method", method, sub, checks)
}
})
}
t.Run(sub+"/POST", func(t *testing.T) {
svc := newSessionMutationRecordingAIService()
handler := &AIHandler{}
setUnexportedField(t, handler, "defaultService", svc)
router := &Router{aiHandler: handler}
req := httptest.NewRequest(http.MethodPost, "/api/ai/sessions/session-1/"+sub, strings.NewReader(""))
rec := httptest.NewRecorder()
router.routeAISessions(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("POST %s status = %d, want %d: %s", sub, rec.Code, http.StatusOK, rec.Body.String())
}
if got := svc.recorded(); !reflect.DeepEqual(got, []string{sub + ":session-1"}) {
t.Fatalf("POST %s mutations = %v, want [%s:session-1]", sub, got, sub)
}
})
}
}
func TestAssistantSessionReadSubresourcesKeepTheirMethods(t *testing.T) {
svc := newSessionMutationRecordingAIService()
handler := &AIHandler{}
setUnexportedField(t, handler, "defaultService", svc)
router := &Router{aiHandler: handler}
messagesReq := httptest.NewRequest(http.MethodGet, "/api/ai/sessions/session-1/messages", nil)
messagesRec := httptest.NewRecorder()
router.routeAISessions(messagesRec, messagesReq)
if messagesRec.Code != http.StatusOK {
t.Fatalf("GET messages status = %d, want %d: %s", messagesRec.Code, http.StatusOK, messagesRec.Body.String())
}
if got := svc.recorded(); len(got) != 0 {
t.Fatalf("read sub-resources mutated the session: %v", got)
}
}
// TestDemoModeRouterRejectsAssistantSessionMutationsOverSafeMethods drives the
// production handler chain: the demo-mode guard admits GET and HEAD as reads,
// so the session mutation itself must refuse them.
func TestDemoModeRouterRejectsAssistantSessionMutationsOverSafeMethods(t *testing.T) {
tempDir := t.TempDir()
hashed, err := internalauth.HashPassword("Password!1")
if err != nil {
t.Fatalf("hash password: %v", err)
}
cfg := &config.Config{
DataPath: tempDir,
ConfigPath: tempDir,
AuthUser: "admin",
AuthPass: hashed,
DemoMode: true,
}
router := NewRouter(cfg, nil, nil, nil, nil, "1.0.0")
t.Cleanup(router.shutdownBackgroundWorkers)
svc := newSessionMutationRecordingAIService()
setUnexportedField(t, router.aiHandler, "defaultService", svc)
serve := func(method, path string) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest(method, path, nil)
req.SetBasicAuth("admin", "Password!1")
rec := httptest.NewRecorder()
router.Handler().ServeHTTP(rec, req)
return rec
}
for _, sub := range assistantSessionMutationSubresources {
for _, method := range []string{http.MethodGet, http.MethodHead} {
rec := serve(method, "/api/ai/sessions/session-1/"+sub)
if rec.Header().Get("X-Demo-Mode") != "true" {
t.Fatalf("%s %s did not pass through the demo-mode guard", method, sub)
}
if rec.Code != http.StatusMethodNotAllowed {
t.Fatalf("demo %s %s status = %d, want %d: %s", method, sub, rec.Code, http.StatusMethodNotAllowed, rec.Body.String())
}
}
if rec := serve(http.MethodPost, "/api/ai/sessions/session-1/"+sub); rec.Code != http.StatusForbidden {
t.Fatalf("demo POST %s status = %d, want %d: %s", sub, rec.Code, http.StatusForbidden, rec.Body.String())
}
}
if got := svc.recorded(); len(got) != 0 {
t.Fatalf("demo-mode requests mutated Assistant sessions: %v", got)
}
if rec := serve(http.MethodGet, "/api/ai/sessions/session-1/messages"); rec.Code != http.StatusOK {
t.Fatalf("demo GET messages status = %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
}
}
// countingUnifiedSeedProvider serves a unified listing to the resources API
// and counts the listing clones taken from it.
type countingUnifiedSeedProvider struct {
listing func() ([]unifiedresources.Resource, time.Time)
clones int
}
func (p *countingUnifiedSeedProvider) ReadSnapshot() models.StateSnapshot {
return models.StateSnapshot{}
}
func (p *countingUnifiedSeedProvider) UnifiedResourceSnapshot() ([]unifiedresources.Resource, time.Time) {
p.clones++
return p.listing()
}
// putOperatorStateThroughAPI writes operator state the way the resource
// editor does and returns the canonical ID the resources API stored it under.
func putOperatorStateThroughAPI(t *testing.T, handlers *ResourceHandlers, ref string, body map[string]any) string {
t.Helper()
payload, _ := json.Marshal(body)
rec := httptest.NewRecorder()
handlers.HandleResourceOperatorState(rec, httptest.NewRequest(http.MethodPut, "/api/resources/"+ref+"/operator-state", strings.NewReader(string(payload))))
if rec.Code != http.StatusOK {
t.Fatalf("PUT %s operator state: %d %s", ref, rec.Code, rec.Body.String())
}
var written resourceOperatorStateAPI
if err := json.Unmarshal(rec.Body.Bytes(), &written); err != nil {
t.Fatalf("decode PUT %s: %v", ref, err)
}
return written.CanonicalID
}
func activeMaintenanceBody(now time.Time, reason string) map[string]any {
return map[string]any{
"monitoringMode": "normal",
"lifecycleState": "active",
"maintenanceStartAt": now.Add(-time.Minute).Format(time.RFC3339),
"maintenanceEndAt": now.Add(time.Hour).Format(time.RFC3339),
"maintenanceReason": reason,
}
}
// A Patrol finding raised before an operator linked its agent into a VM keeps
// the agent's canonical ID, and Patrol re-projects it with that reference. Its
// operator state must be the merged VM's, which the resources API writes and
// alert intent reads for the same reference, even though the agent kept a row
// of its own from before the link. Resolving finding references must not clone
// the unified listing.
func TestPatrolFindingOperatorStateFollowsLinkFoldedReference(t *testing.T) {
now := time.Now().UTC()
handlers := NewResourceHandlers(&config.Config{DataPath: t.TempDir()})
t.Cleanup(func() { _ = handlers.CloseStores() })
store, err := handlers.getStore("default")
if err != nil {
t.Fatalf("resource store: %v", err)
}
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store))
monitor := &monitoring.Monitor{}
monitor.SetResourceStore(adapter)
freshness := now
seed := &countingUnifiedSeedProvider{listing: func() ([]unifiedresources.Resource, time.Time) {
return adapter.GetAll(), freshness
}}
handlers.SetStateProvider(seed)
router := &Router{monitor: monitor, resourceHandlers: handlers}
publish := func(at time.Time) {
freshness = at
adapter.PopulateSnapshotAndSupplemental(models.StateSnapshot{
Hosts: []models.Host{{ID: "host-app-guest", Hostname: "app-guest", MachineID: "machine-app-guest", Status: "online", LastSeen: now}},
LastUpdate: at,
}, map[unifiedresources.DataSource][]unifiedresources.IngestRecord{
unifiedresources.SourceVMware: {{
SourceID: "vc-1:vm:vm-42",
Resource: unifiedresources.Resource{
Type: unifiedresources.ResourceTypeVM,
Technology: "vmware",
Name: "app-guest",
Status: unifiedresources.StatusOnline,
LastSeen: now,
VMware: &unifiedresources.VMwareData{ConnectionID: "vc-1", ManagedObjectID: "vm-42", EntityType: "vm"},
},
Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"app-guest"}},
}},
})
}
publish(now)
agentID := unifiedresources.MachineIdentityCanonicalID(unifiedresources.ResourceTypeAgent, "machine-app-guest")
vmID := ""
for _, resource := range adapter.GetAll() {
if resource.VMware != nil {
vmID = resource.ID
}
}
if vmID == "" {
t.Fatal("published estate has no vSphere VM")
}
// The agent's own row predates the link and holds only defaults.
if got := putOperatorStateThroughAPI(t, handlers, agentID, map[string]any{"monitoringMode": "normal", "lifecycleState": "active"}); got != agentID {
t.Fatalf("pre-link write landed on %q, want the agent %q", got, agentID)
}
if err := store.AddLink(unifiedresources.ResourceLink{ResourceA: vmID, ResourceB: agentID, PrimaryID: vmID}); err != nil {
t.Fatalf("add link: %v", err)
}
// Identity pins persist on the first rebuild; damage from them shows on the second.
publish(now.Add(time.Second))
publish(now.Add(2 * time.Second))
if resolved, ok := adapter.ResolveCanonicalResourceID(agentID); !ok || resolved != vmID {
t.Fatalf("published registry resolves folded agent %s to %q (ok=%v), want VM %s", agentID, resolved, ok, vmID)
}
// The operator puts the merged row into maintenance through the folded ID.
if got := putOperatorStateThroughAPI(t, handlers, agentID, activeMaintenanceBody(now, "moving hosts")); got != vmID {
t.Fatalf("maintenance written by folded ID landed on %q, want merged VM %q", got, vmID)
}
// A reference no registry resolves keeps its own row.
if got := putOperatorStateThroughAPI(t, handlers, "agent:retired-host", map[string]any{"monitoringMode": "muted", "lifecycleState": "active"}); got != "agent:retired-host" {
t.Fatalf("unresolved write landed on %q", got)
}
seed.clones = 0
provider := router.patrolResourceOperatorStateProvider("default")
for _, ref := range []string{agentID, vmID, "vc-1:vm:vm-42"} {
projection, ok := provider.OperatorStateProjection(ref, now)
if !ok || projection.MaintenanceWindow == nil || projection.MaintenanceWindow.Reason != "moving hosts" {
t.Fatalf("finding on %s projected %+v (ok=%v), want merged VM %s's maintenance window", ref, projection, ok, vmID)
}
}
projection, ok := provider.OperatorStateProjection("agent:retired-host", now)
if !ok || projection.MonitoringMode != "muted" {
t.Fatalf("unresolved reference projected %+v (ok=%v), want its own muted row", projection, ok)
}
if seed.clones != 0 {
t.Fatalf("finding projection cloned the unified listing %d times; it must resolve through the monitor's read state", seed.clones)
}
}
// After a restart, a host that has not reported again is listed only through
// saved-host continuity. A Patrol finding naming it by its agent reference
// must read the operator state the resources API wrote for that reference,
// still without cloning the unified listing per finding.
func TestPatrolFindingOperatorStateResolvesContinuityBackedHost(t *testing.T) {
now := time.Now().UTC()
monitorCfg := &config.Config{DataPath: t.TempDir()}
seedMonitor, err := monitoring.New(monitorCfg)
if err != nil {
t.Fatalf("monitoring.New seed monitor: %v", err)
}
t.Cleanup(seedMonitor.Stop)
if _, err := seedMonitor.ApplyHostReport(agentshost.Report{
Agent: agentshost.AgentInfo{ID: "agent-1", Version: "6.0.0-rc.1", IntervalSeconds: 30},
Host: agentshost.HostInfo{ID: "machine-1", MachineID: "machine-1", Hostname: "host-1.local", Platform: "linux"},
Timestamp: now,
}, &config.APITokenRecord{ID: "token-1", Name: "Token One"}); err != nil {
t.Fatalf("ApplyHostReport seed continuity: %v", err)
}
monitor, err := monitoring.New(monitorCfg)
if err != nil {
t.Fatalf("monitoring.New restarted monitor: %v", err)
}
t.Cleanup(monitor.Stop)
handlers := NewResourceHandlers(&config.Config{DataPath: t.TempDir()})
t.Cleanup(func() { _ = handlers.CloseStores() })
store, err := handlers.getStore("default")
if err != nil {
t.Fatalf("resource store: %v", err)
}
monitor.SetResourceStore(unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store)))
seed := &countingUnifiedSeedProvider{listing: monitor.UnifiedResourceSnapshot}
handlers.SetStateProvider(seed)
router := &Router{monitor: monitor, resourceHandlers: handlers}
const ref = "agent:machine-1"
canonicalID := putOperatorStateThroughAPI(t, handlers, ref, activeMaintenanceBody(now, "disk swap"))
if canonicalID == ref {
t.Fatalf("resources API did not resolve continuity host reference %s", ref)
}
seed.clones = 0
projection, ok := router.patrolResourceOperatorStateProvider("default").OperatorStateProjection(ref, now)
if !ok || projection.MaintenanceWindow == nil || projection.MaintenanceWindow.Reason != "disk swap" {
t.Fatalf("finding on %s projected %+v (ok=%v), want the maintenance window the API stored on %s", ref, projection, ok, canonicalID)
}
if seed.clones != 0 {
t.Fatalf("finding projection cloned the unified listing %d times; it must resolve through the monitor's read state", seed.clones)
}
}
// patrolMaintenanceTree publishes a node with one guest under it and returns
// the router, the operator-state store, and the two canonical IDs.
func patrolMaintenanceTree(t *testing.T) (*Router, unifiedresources.ResourceStore, string, string) {
t.Helper()
handlers := NewResourceHandlers(&config.Config{DataPath: t.TempDir()})
t.Cleanup(func() { _ = handlers.CloseStores() })
store, err := handlers.getStore("default")
if err != nil {
t.Fatalf("resource store: %v", err)
}
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store))
const nodeRef, guestRef = "node:pve-a", "vm:pve-a:101"
adapter.PopulateSupplementalRecords(unifiedresources.SourceProxmox, []unifiedresources.IngestRecord{
{SourceID: nodeRef, Resource: unifiedresources.Resource{ID: nodeRef, Type: unifiedresources.ResourceTypeAgent, Name: "pve-a"}},
{SourceID: guestRef, ParentSourceID: nodeRef, Resource: unifiedresources.Resource{ID: guestRef, Type: unifiedresources.ResourceTypeVM, Name: "database"}},
})
nodeID, nodeOK := adapter.ResolveCanonicalResourceID(nodeRef)
guestID, guestOK := adapter.ResolveCanonicalResourceID(guestRef)
if !nodeOK || !guestOK {
t.Fatalf("published tree does not resolve node (%v) or guest (%v)", nodeOK, guestOK)
}
if ancestors := adapter.ResolveCanonicalResourceAncestors(guestID); len(ancestors) == 0 || ancestors[0] != nodeID {
t.Fatalf("guest ancestors = %v, want the node %s first", ancestors, nodeID)
}
monitor := &monitoring.Monitor{}
monitor.SetResourceStore(adapter)
return &Router{monitor: monitor, resourceHandlers: handlers}, store, nodeID, guestID
}
// 2026-10-13 is a Tuesday.
var patrolMaintenanceNow = time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
func tuesdayLunchRecurrence() *unifiedresources.RecurringMaintenanceWindow {
return &unifiedresources.RecurringMaintenanceWindow{
Timezone: "UTC",
Weekdays: []string{"tuesday"},
StartMinute: 11 * 60,
EndMinute: 13 * 60,
}
}
func setOperatorState(t *testing.T, store unifiedresources.ResourceStore, state unifiedresources.ResourceOperatorState) {
t.Helper()
if err := store.SetResourceOperatorState(state); err != nil {
t.Fatalf("set operator state on %s: %v", state.CanonicalID, err)
}
}
// A recurring window carries no one-shot start or end. Patrol must project
// the occurrence that is active, not dereference the absent one-shot dates.
func TestPatrolOperatorStateProjectsActiveRecurringMaintenance(t *testing.T) {
router, store, nodeID, _ := patrolMaintenanceTree(t)
setOperatorState(t, store, unifiedresources.ResourceOperatorState{
CanonicalID: nodeID,
MaintenanceRecurrence: tuesdayLunchRecurrence(),
MaintenanceReason: "weekly patching",
})
provider := router.patrolResourceOperatorStateProvider("default")
projection, ok := provider.OperatorStateProjection(nodeID, patrolMaintenanceNow)
window := projection.MaintenanceWindow
if !ok || window == nil {
t.Fatalf("recurring occurrence active at %s projected %+v (ok=%v), want a maintenance window", patrolMaintenanceNow, projection, ok)
}
wantStart := time.Date(2026, 10, 13, 11, 0, 0, 0, time.UTC)
wantEnd := time.Date(2026, 10, 13, 13, 0, 0, 0, time.UTC)
if !window.StartAt.Equal(wantStart) || !window.EndAt.Equal(wantEnd) || window.Reason != "weekly patching" || window.Inherited {
t.Fatalf("projected window %+v, want the %s-%s occurrence on the resource itself", window, wantStart, wantEnd)
}
// Control: outside the occurrence the same row projects no window.
projection, ok = provider.OperatorStateProjection(nodeID, wantEnd.Add(time.Minute))
if !ok || projection.MaintenanceWindow != nil {
t.Fatalf("after the occurrence projected %+v (ok=%v), want the row without a window", projection, ok)
}
// Control: a one-shot window still projects its own dates.
start, end := patrolMaintenanceNow.Add(-time.Minute), patrolMaintenanceNow.Add(time.Hour)
setOperatorState(t, store, unifiedresources.ResourceOperatorState{
CanonicalID: nodeID,
MaintenanceStartAt: &start,
MaintenanceEndAt: &end,
})
projection, _ = provider.OperatorStateProjection(nodeID, patrolMaintenanceNow)
if window := projection.MaintenanceWindow; window == nil || !window.StartAt.Equal(start) || !window.EndAt.Equal(end) {
t.Fatalf("one-shot window projected %+v, want %s-%s", projection.MaintenanceWindow, start, end)
}
}
// A window scoped to resource_and_descendants silences Patrol findings on the
// guests under it, as it silences their alerts. Only maintenance crosses: the
// node's remediation lock and monitoring mode stay the node's own.
func TestPatrolOperatorStateInheritsDescendantScopedMaintenance(t *testing.T) {
router, store, nodeID, guestID := patrolMaintenanceTree(t)
setOperatorState(t, store, unifiedresources.ResourceOperatorState{
CanonicalID: nodeID,
MaintenanceRecurrence: tuesdayLunchRecurrence(),
MaintenanceScope: unifiedresources.MaintenanceScopeResourceAndDescendants,
MaintenanceReason: "hypervisor patching",
MonitoringMode: unifiedresources.MonitoringModeMuted,
NeverAutoRemediate: true,
})
provider := router.patrolResourceOperatorStateProvider("default")
projection, ok := provider.OperatorStateProjection(guestID, patrolMaintenanceNow)
window := projection.MaintenanceWindow
if !ok || window == nil || !window.Inherited || window.SourceID != nodeID || window.Reason != "hypervisor patching" {
t.Fatalf("guest under a descendant-scoped node window projected %+v (ok=%v), want the node's window inherited", projection, ok)
}
if projection.NeverAutoRemediate || projection.MonitoringMode != "" {
t.Fatalf("guest inherited non-maintenance state %+v; only maintenance may cross", projection)
}
// A guest window ending later than the node's keeps the guest covered
// until the later end; an earlier one yields to the node's.
guestStart, guestEnd := patrolMaintenanceNow.Add(-time.Minute), patrolMaintenanceNow.Add(3*time.Hour)
setOperatorState(t, store, unifiedresources.ResourceOperatorState{CanonicalID: guestID, MaintenanceStartAt: &guestStart, MaintenanceEndAt: &guestEnd, MaintenanceReason: "guest upgrade"})
projection, _ = provider.OperatorStateProjection(guestID, patrolMaintenanceNow)
if window := projection.MaintenanceWindow; window == nil || window.Inherited || !window.EndAt.Equal(guestEnd) {
t.Fatalf("guest's own later window projected %+v, want its own window to %s", projection.MaintenanceWindow, guestEnd)
}
shortEnd := patrolMaintenanceNow.Add(10 * time.Minute)
setOperatorState(t, store, unifiedresources.ResourceOperatorState{CanonicalID: guestID, MaintenanceStartAt: &guestStart, MaintenanceEndAt: &shortEnd})
projection, _ = provider.OperatorStateProjection(guestID, patrolMaintenanceNow)
if window := projection.MaintenanceWindow; window == nil || !window.Inherited || window.SourceID != nodeID {
t.Fatalf("guest's own earlier window projected %+v, want the node's later window", projection.MaintenanceWindow)
}
// Control: a resource-only node window does not reach the guest.
if err := store.ClearResourceOperatorState(guestID); err != nil {
t.Fatalf("clear guest operator state: %v", err)
}
setOperatorState(t, store, unifiedresources.ResourceOperatorState{
CanonicalID: nodeID,
MaintenanceRecurrence: tuesdayLunchRecurrence(),
MaintenanceScope: unifiedresources.MaintenanceScopeResource,
})
if projection, ok := provider.OperatorStateProjection(guestID, patrolMaintenanceNow); ok || projection.MaintenanceWindow != nil {
t.Fatalf("resource-only node window reached the guest: %+v (ok=%v)", projection, ok)
}
if projection, _ := provider.OperatorStateProjection(nodeID, patrolMaintenanceNow); projection.MaintenanceWindow == nil {
t.Fatal("resource-only node window no longer covers the node itself, so the guest control proves nothing")
}
}