Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

10.9 KB
Raw Blame
package api

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/agentcapabilities"
	"github.com/rcourtman/pulse-go-rewrite/internal/ai"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
	"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

func monitorWithNode(t *testing.T, nodeID string, cpu float64) *monitoring.Monitor {
	t.Helper()
	monitor, _, _ := newTestMonitor(t)
	adapter := unifiedresources.NewMonitorAdapter(nil)
	adapter.PopulateFromSnapshot(models.StateSnapshot{Nodes: []models.Node{{ID: nodeID, Name: nodeID, Status: "online", CPU: cpu}}})
	setUnexportedField(t, monitor, "resourceStore", monitoring.ResourceStoreInterface(adapter))
	return monitor
}

func readStateNodeID(t *testing.T, monitor *monitoring.Monitor) string {
	t.Helper()
	nodes := monitor.GetUnifiedReadState().Nodes()
	if len(nodes) != 1 {
		t.Fatalf("expected one node, got %d", len(nodes))
	}
	return nodes[0].ID()
}

func patrolServiceWithFindings(t *testing.T, orgID string, resourceIDs ...string) *ai.Service {
	t.Helper()
	svc := ai.NewService(nil, nil)
	svc.SetOrgID(orgID)
	svc.SetStateProvider(&MockStateProvider{})
	setUnexportedField(t, svc, "cfg", &config.AIConfig{Enabled: true})
	setUnexportedField(t, svc, "provider", &stubProvider{})
	for _, resourceID := range resourceIDs {
		svc.GetPatrolService().GetFindings().Add(&ai.Finding{
			ID:         orgID + "-" + resourceID,
			ResourceID: resourceID,
			Title:      orgID + " finding",
			Severity:   ai.FindingSeverityWarning,
			DetectedAt: time.Now(),
			LastSeenAt: time.Now(),
		})
	}
	return svc
}

type tenantPatrolRig struct {
	router         *Router
	server         *httptest.Server
	token          string
	defaultMonitor *monitoring.Monitor
	tenantMonitor  *monitoring.Monitor
	defaultService *ai.Service
	tenantService  *ai.Service
}

// newTenantPatrolRig serves a real router for two organizations, each with
// its own monitor and Patrol, behind the real auth and tenant middleware.
func newTenantPatrolRig(t *testing.T) tenantPatrolRig {
	t.Helper()
	SetMultiTenantEnabled(true)
	t.Cleanup(func() { SetMultiTenantEnabled(false) })
	t.Setenv("PULSE_DEV", "true")

	const rawToken = "patrol-tenant-scope-token-123.12345678"
	record, err := config.NewAPITokenRecord(rawToken, "two-org-token", []string{config.ScopeMonitoringRead, config.ScopeAIExecute})
	if err != nil {
		t.Fatalf("create api token record: %v", err)
	}
	record.OrgIDs = []string{"default", "org-a"}

	dataDir := t.TempDir()
	cfg := &config.Config{DataPath: dataDir, ConfigPath: dataDir, APITokens: []config.APITokenRecord{*record}}
	cfg.SortAPITokens()
	if err := config.NewMultiTenantPersistence(dataDir).SaveOrganization(&models.Organization{ID: "org-a", DisplayName: "ORG-A"}); err != nil {
		t.Fatalf("save organization: %v", err)
	}

	router := newMultiTenantRouter(t, cfg)
	defaultMonitor := monitorWithNode(t, "default-node", 0.9)
	tenantMonitor := monitorWithNode(t, "org-a-node", 0.9)
	router.monitor = defaultMonitor
	mtm := &monitoring.MultiTenantMonitor{}
	setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{"default": defaultMonitor, "org-a": tenantMonitor})
	router.mtMonitor = mtm
	router.aiSettingsHandler.SetMultiTenantMonitor(mtm)
	router.aiSettingsHandler.SetReadState(defaultMonitor.GetUnifiedReadState())
	router.resourceHandlers.SetTenantStateProvider(NewMultiTenantStateProvider(mtm, defaultMonitor))

	defaultService := patrolServiceWithFindings(t, "default", "default-1", "default-2")
	tenantService := patrolServiceWithFindings(t, "org-a", "org-a-1")
	router.aiSettingsHandler.stateMu.Lock()
	router.aiSettingsHandler.defaultAIService = defaultService
	router.aiSettingsHandler.stateMu.Unlock()
	router.aiSettingsHandler.aiServicesMu.Lock()
	router.aiSettingsHandler.aiServices["org-a"] = tenantService
	router.aiSettingsHandler.aiServicesMu.Unlock()

	server := httptest.NewServer(router.Handler())
	t.Cleanup(server.Close)
	return tenantPatrolRig{
		router:         router,
		server:         server,
		token:          rawToken,
		defaultMonitor: defaultMonitor,
		tenantMonitor:  tenantMonitor,
		defaultService: defaultService,
		tenantService:  tenantService,
	}
}

func (rig tenantPatrolRig) get(t *testing.T, path, orgID string, out any) {
	t.Helper()
	req, err := http.NewRequest(http.MethodGet, rig.server.URL+path, nil)
	if err != nil {
		t.Fatalf("create request: %v", err)
	}
	req.Header.Set("X-Pulse-Org-ID", orgID)
	req.Header.Set("Authorization", "Bearer "+rig.token)
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		t.Fatalf("org %s %s: request failed: %v", orgID, path, err)
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		t.Fatalf("org %s %s: status %d", orgID, path, res.StatusCode)
	}
	if err := json.NewDecoder(res.Body).Decode(out); err != nil {
		t.Fatalf("org %s %s: decode: %v", orgID, path, err)
	}
}

// Agent Patrol-control status counts only the request organization's
// findings, whichever organization's Patrol was wired last.
func TestRoutedAgentPatrolStatusCountsRequestOrganizationFindings(t *testing.T) {
	rig := newTenantPatrolRig(t)

	for orgID, want := range map[string]int{"default": 2, "org-a": 1} {
		var status AgentOperationsLoopStatus
		rig.get(t, agentcapabilities.PatrolControlStatusCapabilityPath, orgID, &status)
		if status.ActiveFindingCount != want {
			t.Fatalf("org %s: expected %d active findings, got %d", orgID, want, status.ActiveFindingCount)
		}
	}
}

func TestPatrolFindingNotificationManagerNeverFallsBackToDefault(t *testing.T) {
	defaultMonitor, _, _ := newTestMonitor(t)
	tenantMonitor, _, _ := newTestMonitor(t)
	defaultManager := notifications.NewNotificationManagerWithDataDir("", t.TempDir())
	tenantManager := notifications.NewNotificationManagerWithDataDir("", t.TempDir())
	t.Cleanup(defaultManager.Stop)
	t.Cleanup(tenantManager.Stop)
	setUnexportedField(t, defaultMonitor, "notificationMgr", defaultManager)
	setUnexportedField(t, tenantMonitor, "notificationMgr", tenantManager)
	mtm := &monitoring.MultiTenantMonitor{}
	setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{"default": defaultMonitor, "org-a": tenantMonitor})
	router := &Router{monitor: defaultMonitor, mtMonitor: mtm}

	if got := router.patrolFindingNotificationManager("org-a"); got != tenantManager {
		t.Fatalf("expected org-a findings to use org-a's notification manager")
	}
	if got := router.patrolFindingNotificationManager("default"); got != defaultManager {
		t.Fatalf("expected default findings to use the default notification manager")
	}
	if got := (&Router{monitor: defaultMonitor}).patrolFindingNotificationManager("org-a"); got != nil {
		t.Fatalf("expected no notification manager for an unresolved organization")
	}
}

func TestLearnBaselinesUsesOrganizationInventory(t *testing.T) {
	defaultMonitor, defaultState, defaultHistory := newTestMonitor(t)
	tenantMonitor, tenantState, tenantHistory := newTestMonitor(t)
	defaultState.Nodes = []models.Node{{ID: "default-node", Name: "default-node"}}
	tenantState.Nodes = []models.Node{{ID: "org-a-node", Name: "org-a-node"}}
	syncTestResourceStore(t, defaultMonitor, defaultState)
	syncTestResourceStore(t, tenantMonitor, tenantState)
	now := time.Now()
	// The tenant's history also holds the default node's ID, and only the
	// tenant's history holds its own node: learning from the default inventory
	// or from the default history both show up below.
	defaultHistory.AddNodeMetric("default-node", "cpu", 0.5, now)
	tenantHistory.AddNodeMetric("default-node", "cpu", 0.5, now)
	tenantHistory.AddNodeMetric("org-a-node", "cpu", 0.5, now)
	mtm := &monitoring.MultiTenantMonitor{}
	setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{"default": defaultMonitor, "org-a": tenantMonitor})
	router := &Router{monitor: defaultMonitor, mtMonitor: mtm}

	store := ai.NewBaselineStore(ai.BaselineConfig{MinSamples: 1})
	router.learnBaselines(store, router.resolveMonitorForOrg("org-a"))
	if _, ok := store.GetBaseline("org-a-node", "cpu"); !ok {
		t.Fatalf("expected org-a to learn its own node")
	}
	if _, ok := store.GetBaseline("default-node", "cpu"); ok {
		t.Fatalf("org-a learned a default-organization resource")
	}
}

func TestDeployCandidatesReadRequestOrganizationMonitor(t *testing.T) {
	defaultMonitor := monitorWithNode(t, "default-node", 0.1)
	handler := &DeployHandlers{monitor: defaultMonitor}
	handler.SetMonitorResolver(func(ctx context.Context) *monitoring.Monitor {
		if GetOrgID(ctx) == "default" {
			return defaultMonitor
		}
		return nil
	})

	rr := httptest.NewRecorder()
	handler.HandleCandidates(rr, withOrg(httptest.NewRequest(http.MethodGet, "/api/clusters/c1/agent-deploy/candidates", nil), "org-a"))
	if rr.Code != http.StatusServiceUnavailable {
		t.Fatalf("expected an unresolved organization not to read the default cluster, got %d", rr.Code)
	}
}

// A tenant's Patrol notifies through its own channels: it gets the finding
// notification callback but never the instance relay's mobile push.
func TestStartPatrolForContextKeepsTenantFindingsOffInstanceRelay(t *testing.T) {
	setMockModeForTest(t, true)

	mtp := config.NewMultiTenantPersistence(t.TempDir())
	defaultMonitor, _, _ := newTestMonitor(t)
	tenantMonitor, _, _ := newTestMonitor(t)
	tenantMonitor.SetResourceStore(unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)))
	// Finding callbacks are wired only once the unified store exists, which
	// needs the organization's alert manager.
	alertManager := alerts.NewManagerWithDataDir(t.TempDir())
	t.Cleanup(alertManager.Stop)
	setUnexportedField(t, tenantMonitor, "alertManager", alertManager)
	mtm := &monitoring.MultiTenantMonitor{}
	setUnexportedField(t, mtm, "monitors", map[string]*monitoring.Monitor{"default": defaultMonitor, "tenant-1": tenantMonitor})

	router := &Router{
		monitor:                 defaultMonitor,
		mtMonitor:               mtm,
		multiTenant:             mtp,
		aiSettingsHandler:       NewAISettingsHandler(mtp, mtm, nil),
		aiHandler:               &AIHandler{},
		startedPatrolOrgs:       make(map[string]bool),
		monitorResourceAdapters: make(map[string]*unifiedresources.MonitorAdapter),
	}
	router.aiSettingsHandler.SetStateProvider(&stubStateProvider{})
	defer router.ShutdownAIIntelligence()

	ctx := context.WithValue(context.Background(), OrgIDContextKey, "tenant-1")
	if ok := router.startPatrolForContext(ctx, "tenant-1"); !ok {
		t.Fatal("expected tenant-1 patrol start to succeed")
	}
	patrol := reflect.ValueOf(router.aiSettingsHandler.GetAIService(ctx).GetPatrolService()).Elem()
	if patrol.FieldByName("findingNotifyCallback").IsNil() {
		t.Fatal("expected tenant-1 findings to be wired to notifications")
	}
	if !patrol.FieldByName("pushNotifyCallback").IsNil() {
		t.Fatal("tenant-1 findings must not be pushed through the instance relay")
	}
}