Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package api
import (
"context"
"net/http"
"net/http/httptest"
"reflect"
"sync"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/ai"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/auth"
)
func TestRouterGetMonitor_Defaults(t *testing.T) {
defaultMonitor, _, _ := newTestMonitor(t)
router := &Router{monitor: defaultMonitor}
req := httptest.NewRequest(http.MethodGet, "/", nil)
monitor, err := router.getMonitor(req)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if monitor != defaultMonitor {
t.Fatalf("expected default monitor to be returned")
}
}
func TestRouterGetMonitor_WithTenant(t *testing.T) {
defaultMonitor, _, _ := newTestMonitor(t)
tenantMonitor, _, _ := newTestMonitor(t)
mtm := &monitoring.MultiTenantMonitor{}
setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{
"tenant-1": tenantMonitor,
})
router := &Router{monitor: defaultMonitor, mtMonitor: mtm}
req := httptest.NewRequest(http.MethodGet, "/", nil)
ctx := context.WithValue(req.Context(), OrgIDContextKey, "tenant-1")
req = req.WithContext(ctx)
monitor, err := router.getMonitor(req)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if monitor != tenantMonitor {
t.Fatalf("expected tenant monitor to be returned")
}
}
func TestRouterGetMonitor_NonDefaultWithoutTenantMonitorFailsClosed(t *testing.T) {
defaultMonitor, _, _ := newTestMonitor(t)
router := &Router{monitor: defaultMonitor}
req := httptest.NewRequest(http.MethodGet, "/", nil)
req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "tenant-1"))
monitor, err := router.getMonitor(req)
if err == nil {
t.Fatal("expected non-default org without tenant monitor to fail closed")
}
if monitor != nil {
t.Fatalf("monitor = %#v, want nil", monitor)
}
}
func TestMultiTenantStateProvider_UnifiedReadStateForTenant(t *testing.T) {
defaultMonitor, defaultState, _ := newTestMonitor(t)
defaultState.Hosts = []models.Host{{ID: "host-default", Hostname: "default-host", Status: "online"}}
syncTestResourceStore(t, defaultMonitor, defaultState)
tenantMonitor, tenantState, _ := newTestMonitor(t)
tenantState.Hosts = []models.Host{{ID: "host-tenant", Hostname: "tenant-host", Status: "online"}}
syncTestResourceStore(t, tenantMonitor, tenantState)
mtm := &monitoring.MultiTenantMonitor{}
setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{
"tenant-1": tenantMonitor,
})
provider := NewMultiTenantStateProvider(mtm, defaultMonitor)
defaultReadState := provider.UnifiedReadStateForTenant("default")
if defaultReadState == nil {
t.Fatal("expected default unified read state")
}
if defaultReadState != defaultMonitor.GetUnifiedReadState() {
t.Fatal("expected default monitor-scoped read state")
}
if hosts := defaultReadState.Hosts(); len(hosts) != 1 || hosts[0].Name() != "default-host" {
t.Fatalf("unexpected default hosts: %#v", hosts)
}
tenantReadState := provider.UnifiedReadStateForTenant("tenant-1")
if tenantReadState == nil {
t.Fatal("expected tenant unified read state")
}
if tenantReadState != tenantMonitor.GetUnifiedReadState() {
t.Fatal("expected tenant monitor-scoped read state")
}
if hosts := tenantReadState.Hosts(); len(hosts) != 1 || hosts[0].Name() != "tenant-host" {
t.Fatalf("unexpected tenant hosts: %#v", hosts)
}
}
func TestMultiTenantStateProvider_UnifiedResourceSnapshotForTenant(t *testing.T) {
defaultMonitor, defaultState, _ := newTestMonitor(t)
defaultState.Hosts = []models.Host{{ID: "host-default", Hostname: "default-host", Status: "online"}}
syncTestResourceStore(t, defaultMonitor, defaultState)
tenantMonitor, tenantState, _ := newTestMonitor(t)
tenantState.Hosts = []models.Host{{ID: "host-tenant", Hostname: "tenant-host", Status: "online"}}
syncTestResourceStore(t, tenantMonitor, tenantState)
mtm := &monitoring.MultiTenantMonitor{}
setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{
"tenant-1": tenantMonitor,
})
provider := NewMultiTenantStateProvider(mtm, defaultMonitor)
defaultResources, _ := provider.UnifiedResourceSnapshotForTenant("default")
if len(defaultResources) != 1 || defaultResources[0].Name != "default-host" {
t.Fatalf("unexpected default unified resources: %#v", defaultResources)
}
tenantResources, _ := provider.UnifiedResourceSnapshotForTenant("tenant-1")
if len(tenantResources) != 1 || tenantResources[0].Name != "tenant-host" {
t.Fatalf("unexpected tenant unified resources: %#v", tenantResources)
}
}
func TestMultiTenantStateProvider_FallbackOnError(t *testing.T) {
defaultMonitor, defaultState, _ := newTestMonitor(t)
defaultState.VMs = []models.VM{{ID: "vm-default"}}
mtp := config.NewMultiTenantPersistence(t.TempDir())
mtm := monitoring.NewMultiTenantMonitor(&config.Config{}, mtp, nil)
defer mtm.Stop()
provider := NewMultiTenantStateProvider(mtm, defaultMonitor)
resources, freshness := provider.UnifiedResourceSnapshotForTenant("../bad")
if len(resources) != 0 {
t.Fatalf("expected empty unified resources for tenant error, got %#v", resources)
}
if !freshness.IsZero() {
t.Fatalf("expected zero freshness for tenant error, got %v", freshness)
}
if readState := provider.UnifiedReadStateForTenant("../bad"); readState != nil {
t.Fatalf("expected nil read state for tenant error, got %#v", readState)
}
}
func TestSetMultiTenantMonitor_WiresHandlers(t *testing.T) {
defaultMonitor, _, _ := newTestMonitor(t)
mtm := &monitoring.MultiTenantMonitor{}
setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{
"default": defaultMonitor,
})
router := &Router{
alertHandlers: &AlertHandlers{},
notificationHandlers: &NotificationHandlers{},
configHandlers: &ConfigHandlers{},
dockerAgentHandlers: &DockerAgentHandlers{},
unifiedAgentHandlers: &UnifiedAgentHandlers{},
kubernetesAgentHandlers: &KubernetesAgentHandlers{},
systemSettingsHandler: &SystemSettingsHandler{},
aiSettingsHandler: &AISettingsHandler{},
aiHandler: &AIHandler{},
resourceHandlers: &ResourceHandlers{},
}
router.SetMultiTenantMonitor(mtm)
if router.mtMonitor != mtm {
t.Fatalf("expected router mtMonitor to be updated")
}
if router.monitor != defaultMonitor {
t.Fatalf("expected router monitor to be set to default monitor")
}
if got := router.alertHandlers.MonitorForContext(context.Background()); got == nil || got.GetAlertManager() != defaultMonitor.GetAlertManager() {
t.Fatalf("expected alertHandlers to resolve the default tenant monitor")
}
if got := router.notificationHandlers.MonitorForContext(context.Background()); got == nil || got.GetNotificationManager() != defaultMonitor.GetNotificationManager() {
t.Fatalf("expected notificationHandlers to resolve the default tenant monitor")
}
if router.configHandlers.Monitor(context.Background()) != defaultMonitor {
t.Fatalf("expected configHandlers to resolve the default multi-tenant monitor")
}
if router.dockerAgentHandlers.mtMonitor != mtm {
t.Fatalf("expected dockerAgentHandlers mtMonitor to be set")
}
if router.unifiedAgentHandlers.mtMonitor != mtm {
t.Fatalf("expected unifiedAgentHandlers mtMonitor to be set")
}
if router.kubernetesAgentHandlers.mtMonitor != mtm {
t.Fatalf("expected kubernetesAgentHandlers mtMonitor to be set")
}
if router.systemSettingsHandler.mtMonitor != mtm {
t.Fatalf("expected systemSettingsHandler mtMonitor to be set")
}
if router.aiSettingsHandler.mtMonitor != mtm {
t.Fatalf("expected aiSettingsHandler mtMonitor to be set")
}
if router.aiHandler.mtMonitor != mtm {
t.Fatalf("expected aiHandler mtMonitor to be set")
}
if router.resourceHandlers.tenantStateProvider == nil {
t.Fatalf("expected tenant state provider to be set")
}
}
func TestRouterConfigureMonitorDependencies_UsesTenantSpecificResourceAdapters(t *testing.T) {
defaultAdapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
router := &Router{
monitorResourceAdapter: defaultAdapter,
monitorResourceAdapters: make(map[string]*unifiedresources.MonitorAdapter),
}
defaultMonitor := &monitoring.Monitor{}
router.configureMonitorDependencies(defaultMonitor)
defaultReadState := defaultMonitor.GetUnifiedReadState()
if defaultReadState == nil {
t.Fatal("expected default monitor read state to be configured")
}
if defaultReadState != defaultAdapter {
t.Fatalf("expected default monitor to use default adapter, got %#v", defaultReadState)
}
tenantMonitor := &monitoring.Monitor{}
tenantMonitor.SetOrgID("tenant-1")
router.configureMonitorDependencies(tenantMonitor)
tenantReadState := tenantMonitor.GetUnifiedReadState()
if tenantReadState == nil {
t.Fatal("expected tenant monitor read state to be configured")
}
if tenantReadState == defaultAdapter {
t.Fatal("expected tenant monitor adapter to differ from default adapter")
}
router.configureMonitorDependencies(tenantMonitor)
if second := tenantMonitor.GetUnifiedReadState(); second != tenantReadState {
t.Fatal("expected tenant monitor to reuse stable adapter for same org")
}
}
func TestRouterMonitorAdapterForMonitorUsesPersistentStore(t *testing.T) {
router := &Router{
resourceHandlers: NewResourceHandlers(&config.Config{DataPath: t.TempDir()}),
monitorResourceAdapters: make(map[string]*unifiedresources.MonitorAdapter),
}
monitor := &monitoring.Monitor{}
monitor.SetOrgID("tenant-1")
adapter := router.monitorAdapterForMonitor(monitor)
if adapter == nil {
t.Fatal("expected monitor adapter")
}
adapter.PopulateSupplementalRecords(unifiedresources.SourceDocker, []unifiedresources.IngestRecord{
{
SourceID: "docker-host-1",
Resource: unifiedresources.Resource{
Type: unifiedresources.ResourceTypeDockerService,
Name: "svc-1",
Status: unifiedresources.StatusOnline,
},
},
})
store, err := router.resourceHandlers.getStore("tenant-1")
if err != nil {
t.Fatalf("getStore: %v", err)
}
resources := adapter.GetAll()
if len(resources) != 1 {
t.Fatalf("expected 1 resource from adapter, got %d", len(resources))
}
changes, err := store.GetRecentChanges(resources[0].ID, time.Time{}, 10)
if err != nil {
t.Fatalf("GetRecentChanges: %v", err)
}
if len(changes) != 1 {
t.Fatalf("expected 1 stored change, got %d", len(changes))
}
}
func TestRouterDefaultUnifiedResourceProvider_PrefersMonitorScopedAdapter(t *testing.T) {
defaultAdapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
defaultMonitor := &monitoring.Monitor{}
defaultMonitor.SetResourceStore(defaultAdapter)
router := &Router{
monitor: defaultMonitor,
monitorResourceAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)),
monitorResourceAdapters: map[string]*unifiedresources.MonitorAdapter{},
}
provider := router.defaultUnifiedResourceProvider()
if provider == nil {
t.Fatal("expected default unified provider")
}
if provider != defaultAdapter {
t.Fatalf("expected monitor-scoped adapter, got %#v", provider)
}
}
func TestRouterPersistenceForOrg_UsesTenantPersistence(t *testing.T) {
tempDir := t.TempDir()
mtp := config.NewMultiTenantPersistence(tempDir)
tenantPersistence, err := mtp.GetPersistence("tenant-1")
if err != nil {
t.Fatalf("GetPersistence tenant-1: %v", err)
}
router := &Router{
persistence: config.NewConfigPersistence(tempDir),
multiTenant: mtp,
}
ctx := context.WithValue(context.Background(), OrgIDContextKey, "tenant-1")
got := router.persistenceForOrg(ctx)
if got != tenantPersistence {
t.Fatalf("expected tenant persistence, got %#v", got)
}
}
func TestRouterPersistenceForOrg_NonDefaultDoesNotFallbackToDefault(t *testing.T) {
router := &Router{persistence: config.NewConfigPersistence(t.TempDir())}
ctx := context.WithValue(context.Background(), OrgIDContextKey, "tenant-1")
if got := router.persistenceForOrg(ctx); got != nil {
t.Fatalf("expected nil persistence for non-default org without mt persistence, got %#v", got)
}
}
func TestRouterStopPatrolForOrg_ClearsLifecycleMarker(t *testing.T) {
router := &Router{
startedPatrolOrgs: map[string]bool{
"default": true,
"tenant-1": true,
},
}
router.StopPatrolForOrg("tenant-1")
if router.startedPatrolOrgs["tenant-1"] {
t.Fatal("expected tenant patrol marker to be cleared")
}
if !router.startedPatrolOrgs["default"] {
t.Fatal("expected default marker to remain set")
}
}
func TestRouterStopPatrol_ClearsAllLifecycleMarkers(t *testing.T) {
router := &Router{
aiSettingsHandler: &AISettingsHandler{},
startedPatrolOrgs: map[string]bool{
"default": true,
"tenant-1": true,
},
}
router.StopPatrol()
if len(router.startedPatrolOrgs) != 0 {
t.Fatalf("expected all patrol markers cleared, got %#v", router.startedPatrolOrgs)
}
}
func TestStartPatrolForContext_DoesNotOverwriteOtherTenantPatrolComponents(t *testing.T) {
setMockModeForTest(t, true)
tempDir := t.TempDir()
mtp := config.NewMultiTenantPersistence(tempDir)
defaultMonitor, _, _ := newTestMonitor(t)
tenantOneMonitor, _, _ := newTestMonitor(t)
tenantTwoMonitor, _, _ := newTestMonitor(t)
tenantOneMonitor.SetResourceStore(unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)))
tenantTwoMonitor.SetResourceStore(unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)))
mtm := &monitoring.MultiTenantMonitor{}
setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{
"default": defaultMonitor,
"tenant-1": tenantOneMonitor,
"tenant-2": tenantTwoMonitor,
})
router := &Router{
monitor: defaultMonitor,
mtMonitor: mtm,
multiTenant: mtp,
aiSettingsHandler: NewAISettingsHandler(mtp, mtm, nil),
startedPatrolOrgs: make(map[string]bool),
monitorResourceAdapters: make(map[string]*unifiedresources.MonitorAdapter),
}
router.aiSettingsHandler.SetStateProvider(&stubStateProvider{})
defer router.ShutdownAIIntelligence()
ctxTenantOne := context.WithValue(context.Background(), OrgIDContextKey, "tenant-1")
if ok := router.startPatrolForContext(ctxTenantOne, "tenant-1"); !ok {
t.Fatal("expected tenant-1 patrol start to succeed")
}
svcTenantOne := router.aiSettingsHandler.GetAIService(ctxTenantOne)
if svcTenantOne == nil {
t.Fatal("expected tenant-1 AI service")
}
patrolTenantOne := svcTenantOne.GetPatrolService()
if patrolTenantOne == nil {
t.Fatal("expected tenant-1 patrol service")
}
baselineOne := patrolTenantOne.GetBaselineStore()
changeDetectorOne := patrolTenantOne.GetChangeDetector()
remediationLogOne := patrolTenantOne.GetRemediationLog()
if baselineOne == nil || changeDetectorOne == nil || remediationLogOne == nil {
t.Fatal("expected tenant-1 patrol components to be initialized")
}
ctxTenantTwo := context.WithValue(context.Background(), OrgIDContextKey, "tenant-2")
if ok := router.startPatrolForContext(ctxTenantTwo, "tenant-2"); !ok {
t.Fatal("expected tenant-2 patrol start to succeed")
}
if got := patrolTenantOne.GetBaselineStore(); got != baselineOne {
t.Fatal("expected tenant-1 baseline store to remain unchanged after tenant-2 startup")
}
if got := patrolTenantOne.GetChangeDetector(); got != changeDetectorOne {
t.Fatal("expected tenant-1 change detector to remain unchanged after tenant-2 startup")
}
if got := patrolTenantOne.GetRemediationLog(); got != remediationLogOne {
t.Fatal("expected tenant-1 remediation log to remain unchanged after tenant-2 startup")
}
}
func TestStartPatrolForContext_RejectsMismatchedAIServiceOrg(t *testing.T) {
setMockModeForTest(t, true)
mtp := config.NewMultiTenantPersistence(t.TempDir())
defaultMonitor, _, _ := newTestMonitor(t)
tenantMonitor, _, _ := newTestMonitor(t)
mtm := &monitoring.MultiTenantMonitor{}
setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{
"default": defaultMonitor,
"tenant-1": tenantMonitor,
})
handler := NewAISettingsHandler(mtp, mtm, nil)
legacySvc := ai.NewService(config.NewConfigPersistence(t.TempDir()), nil)
legacySvc.SetOrgID("default")
handler.aiServices["tenant-1"] = legacySvc
router := &Router{
monitor: defaultMonitor,
mtMonitor: mtm,
aiSettingsHandler: handler,
startedPatrolOrgs: make(map[string]bool),
monitorResourceAdapters: make(map[string]*unifiedresources.MonitorAdapter),
}
ctx := context.WithValue(context.Background(), OrgIDContextKey, "tenant-1")
if ok := router.startPatrolForContext(ctx, "tenant-1"); ok {
t.Fatal("expected patrol start to fail when AI service org scope mismatches tenant org")
}
}
// Public URL detection runs before the route handler on every request. Its
// token lookup must share authentication's metadata writer ownership, not
// mutate a live record concurrently with auth readers or normalize its scopes
// again after dropping the writer lock.
func TestPublicURLTokenCaptureUsesAuthenticationOwnership(t *testing.T) {
t.Setenv("PULSE_AUTH_BYPASS", "false")
cases := []struct {
raw string
scopes []string
want bool
}{
{"capture-monitoring-fixture.12345678", []string{config.ScopeMonitoringRead}, false},
{"capture-settings-fixture.12345678", []string{config.ScopeSettingsWrite}, true},
}
cfg := &config.Config{}
for _, tc := range cases {
record, err := config.NewAPITokenRecord(tc.raw, "capture fixture", tc.scopes)
if err != nil {
t.Fatal(err)
}
cfg.APITokens = append(cfg.APITokens, *record)
}
before := make([]config.APITokenRecord, len(cfg.APITokens))
for i := range cfg.APITokens {
before[i] = cfg.APITokens[i].Clone()
}
start := make(chan struct{})
var wg sync.WaitGroup
for _, tc := range cases {
for _, header := range []string{"X-API-Token", "Authorization"} {
wg.Add(2)
go func() {
defer wg.Done()
<-start
for i := 0; i < 32; i++ {
req := httptest.NewRequest(http.MethodGet, "http://localhost/api/agent/changes", nil)
value := tc.raw
if header == "Authorization" {
value = "Bearer " + value
}
req.Header.Set(header, value)
if got := canCapturePublicURL(cfg, req); got != tc.want {
t.Errorf("capture permission = %v, want %v", got, tc.want)
}
}
}()
go func() {
defer wg.Done()
<-start
for i := 0; i < 32; i++ {
req := httptest.NewRequest(http.MethodGet, "/api/agent/changes", nil)
req.Header.Set("Authorization", "Bearer "+tc.raw)
if !CheckAuth(cfg, httptest.NewRecorder(), req) {
t.Error("valid token lost authentication")
}
}
}()
}
}
close(start)
wg.Wait()
config.Mu.RLock()
defer config.Mu.RUnlock()
for i, record := range cfg.APITokens {
if record.ID != before[i].ID || record.Hash != before[i].Hash || !reflect.DeepEqual(record.Scopes, before[i].Scopes) {
t.Fatal("capture changed credential identity or scope")
}
if record.LastUsedAt == nil {
t.Fatal("successful validation lost last-used metadata")
}
}
}
func TestPublicURLTokenCapturePreservesTokenRefusals(t *testing.T) {
for _, kind := range []string{"expired", "invalid_role", "wrong_token"} {
t.Run(kind, func(t *testing.T) {
const raw = "capture-refusal-fixture.12345678"
record, err := config.NewAPITokenRecord(raw, "capture refusal", []string{config.ScopeSettingsWrite})
if err != nil {
t.Fatal(err)
}
if kind == "expired" {
expired := time.Now().Add(-time.Hour)
record.ExpiresAt = &expired
}
if kind == "invalid_role" {
record.Metadata = map[string]string{auth.RuntimeRoleMetadataKey: auth.RuntimeRoleMonitoringCollector}
}
cfg := &config.Config{APITokens: []config.APITokenRecord{*record}}
for _, header := range []string{"X-API-Token", "Authorization"} {
req := httptest.NewRequest(http.MethodGet, "http://localhost/api/agent/changes", nil)
value := raw
if kind == "wrong_token" {
value = "different-fixture-token.12345678"
}
if header == "Authorization" {
value = "Bearer " + value
}
req.Header.Set(header, value)
if canCapturePublicURL(cfg, req) {
t.Fatal("rejected credential gained URL capture authority")
}
}
if cfg.APITokens[0].LastUsedAt != nil || !reflect.DeepEqual(cfg.APITokens[0].Scopes, record.Scopes) {
t.Fatal("refused capture mutated token metadata")
}
})
}
}
// The router builds one organization access policy and shares it with tenant
// selection, action authority, the organization handlers and the WebSocket
// hub, so they cannot disagree about who may use the default organization.
func TestNewRouterSharesOrgAccessPolicy(t *testing.T) {
previous := IsMultiTenantEnabled()
SetMultiTenantEnabled(true)
t.Cleanup(func() { SetMultiTenantEnabled(previous) })
t.Setenv("PULSE_DEV", "true")
dataDir := t.TempDir()
router := newMultiTenantRouter(t, &config.Config{DataPath: dataDir, ConfigPath: dataDir, AuthUser: "admin"})
if router.orgAccess == nil || router.orgAccess.instanceAdmin == nil {
t.Fatal("router must build the shared organization access policy")
}
if !router.orgAccess.CheckAccess(nil, "admin", "default").Allowed {
t.Fatal("instance administrator must reach the default organization")
}
if router.orgAccess.CheckAccess(nil, "bob", "default").Allowed {
t.Fatal("a user without membership must not reach the default organization")
}
}