Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

12.0 KB
Raw Blame
package ai

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/ai/circuit"
	"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
)

type mockPatrolProvider struct {
	response string
	err      error
	lastReq  providers.ChatRequest
	calls    int
}

func TestRunPatrol_EarlyRuntimeFailureDoesNotAcquireHalfOpenProbe(t *testing.T) {
	svc := NewService(config.NewConfigPersistence(t.TempDir()), nil)
	svc.cfg = &config.AIConfig{Enabled: true, PatrolModel: "mock:model"}
	ps := NewPatrolService(svc, nil)
	ps.SetConfig(PatrolConfig{Enabled: true, AnalyzeNodes: true})
	breaker := circuit.NewBreaker("patrol", circuit.Config{
		FailureThreshold: 1,
		SuccessThreshold: 1,
		InitialBackoff:   time.Nanosecond,
	})
	breaker.RecordFailure(errors.New("transient provider failure"))
	ps.SetCircuitBreaker(breaker)

	ps.runPatrolWithTrigger(context.Background(), TriggerReasonManual, nil)

	if breaker.State() != circuit.StateOpen {
		t.Fatalf("runtime-input failure must not acquire a half-open provider probe, got %s", breaker.State())
	}
}

func (m *mockPatrolProvider) Chat(ctx context.Context, req providers.ChatRequest) (*providers.ChatResponse, error) {
	m.lastReq = req
	m.calls++
	if m.err != nil {
		return nil, m.err
	}
	return &providers.ChatResponse{Content: m.response}, nil
}

func (m *mockPatrolProvider) ChatStream(ctx context.Context, req providers.ChatRequest, callback providers.StreamCallback) error {
	return nil
}

func (m *mockPatrolProvider) SupportsThinking(model string) bool       { return false }
func (m *mockPatrolProvider) TestConnection(ctx context.Context) error { return nil }
func (m *mockPatrolProvider) Name() string                             { return "mock" }
func (m *mockPatrolProvider) ListModels(ctx context.Context) ([]providers.ModelInfo, error) {
	return nil, nil
}

type mockPatrolStateProvider struct {
	state models.StateSnapshot
}

func (m mockPatrolStateProvider) ReadSnapshot() models.StateSnapshot { return m.state }

func TestPatrolService_ReviewAlertBatch(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	provider := &mockPatrolProvider{response: "ALERT 1: RESOLVE: cpu back to normal\nALERT 2: KEEP: still high"}
	aiSvc := &Service{
		provider: provider,
		cfg:      &config.AIConfig{PatrolModel: "mock:model"},
	}

	alertsToReview := []AlertInfo{
		{
			ID:           "a1",
			Type:         "cpu",
			ResourceName: "node1",
			ResourceType: "node",
			Message:      "high cpu",
			Value:        90,
			Threshold:    80,
			Duration:     "5m",
		},
		{
			ID:           "a-usage",
			Type:         "usage",
			ResourceName: "local",
			ResourceType: "usage",
			ResourceID:   "storage-1",
			Message:      "high usage",
			Value:        92,
			Threshold:    90,
			Duration:     "20m",
		},
	}
	state := models.StateSnapshot{
		Nodes:   []models.Node{{ID: "node1", Name: "node1", CPU: 10}},
		Storage: []models.Storage{{ID: "storage-1", Name: "local", Usage: 91, Status: "active"}},
	}

	verdicts := ps.reviewAlertsWithModelState(context.Background(), alertsToReview, patrolRuntimeStateForTest(ps, state), aiSvc, "patrol-run-123")
	if provider.calls != 1 {
		t.Fatalf("expected one batched model call for two alerts, got %d", provider.calls)
	}
	if len(verdicts) != 2 {
		t.Fatalf("expected one verdict per alert, got %d", len(verdicts))
	}
	if !verdicts[0].resolve {
		t.Fatalf("expected first alert to resolve")
	}
	if !strings.Contains(verdicts[0].reason, "cpu back to normal") {
		t.Fatalf("expected resolution reason, got %q", verdicts[0].reason)
	}
	if verdicts[1].resolve {
		t.Fatalf("expected second alert to stay unresolved")
	}
	prompt := provider.lastReq.Messages[1].Content
	if !strings.Contains(prompt, "Resource: node1 (node)") {
		t.Fatalf("expected prompt to include canonical node resource type, got %q", prompt)
	}
	if !strings.Contains(prompt, "Resource: local (storage)") {
		t.Fatalf("expected prompt to normalize usage alerts to storage, got %q", prompt)
	}
	if provider.lastReq.ExecutionID != "patrol-run-123" {
		t.Fatalf("execution_id=%q want patrol-run-123", provider.lastReq.ExecutionID)
	}
}

func TestPatrolService_ReviewAlertBatch_SingleAlertBareResolveFallback(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	provider := &mockPatrolProvider{response: "RESOLVE: looks good"}
	aiSvc := &Service{
		provider: provider,
		cfg:      &config.AIConfig{PatrolModel: "mock:model"},
	}

	alert := AlertInfo{
		ID:           "a1",
		Type:         "cpu",
		ResourceName: "node1",
		ResourceType: "node",
		Value:        90,
		Threshold:    80,
	}
	state := models.StateSnapshot{Nodes: []models.Node{{ID: "node1", Name: "node1", CPU: 10}}}

	verdicts := ps.reviewAlertsWithModelState(context.Background(), []AlertInfo{alert}, patrolRuntimeStateForTest(ps, state), aiSvc, "patrol-run-bare")
	if len(verdicts) != 1 || !verdicts[0].resolve {
		t.Fatalf("expected bare RESOLVE response to resolve the single alert, got %+v", verdicts)
	}
	if !strings.Contains(verdicts[0].reason, "looks good") {
		t.Fatalf("expected resolution reason, got %q", verdicts[0].reason)
	}
}

func TestPatrolService_ReviewAlertBatch_UnparseableKeepsAll(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	provider := &mockPatrolProvider{response: "Everything appears to be operating within normal parameters."}
	aiSvc := &Service{
		provider: provider,
		cfg:      &config.AIConfig{PatrolModel: "mock:model"},
	}

	alertsToReview := []AlertInfo{
		{ID: "a1", Type: "cpu", ResourceName: "node1", ResourceType: "node"},
		{ID: "a2", Type: "memory", ResourceName: "node1", ResourceType: "node"},
	}
	state := models.StateSnapshot{Nodes: []models.Node{{ID: "node1", Name: "node1"}}}

	verdicts := ps.reviewAlertsWithModelState(context.Background(), alertsToReview, patrolRuntimeStateForTest(ps, state), aiSvc, "")
	for i, v := range verdicts {
		if v.resolve {
			t.Fatalf("expected unparseable response to keep alert %d, got %+v", i, v)
		}
	}
}

func TestPatrolService_ReviewAlertBatch_ChunksLargeBacklogs(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	provider := &mockPatrolProvider{response: "ALERT 1: KEEP: still investigating"}
	aiSvc := &Service{
		provider: provider,
		cfg:      &config.AIConfig{PatrolModel: "mock:model"},
	}

	alertsToReview := make([]AlertInfo, patrolAlertReviewBatchSize+1)
	for i := range alertsToReview {
		alertsToReview[i] = AlertInfo{ID: fmt.Sprintf("a%d", i), Type: "cpu", ResourceName: "node1", ResourceType: "node"}
	}
	state := models.StateSnapshot{Nodes: []models.Node{{ID: "node1", Name: "node1"}}}

	verdicts := ps.reviewAlertsWithModelState(context.Background(), alertsToReview, patrolRuntimeStateForTest(ps, state), aiSvc, "")
	if provider.calls != 2 {
		t.Fatalf("expected %d alerts to review in 2 chunked calls, got %d calls", len(alertsToReview), provider.calls)
	}
	if len(verdicts) != len(alertsToReview) {
		t.Fatalf("expected %d verdicts, got %d", len(alertsToReview), len(verdicts))
	}
}

func TestApplyAlertReviewResponse_FirstLinePerAlertWins(t *testing.T) {
	verdicts := []patrolAlertReviewVerdict{{alert: AlertInfo{ID: "a1"}}}
	matched := applyAlertReviewResponse(verdicts, "ALERT 1: RESOLVE: fixed\nALERT 1: KEEP: on second thought\nALERT 9: RESOLVE: out of range")
	if !matched {
		t.Fatalf("expected verdict lines to match")
	}
	if !verdicts[0].resolve {
		t.Fatalf("expected first verdict line to win, got %+v", verdicts[0])
	}
}

func TestPatrolService_GetResourceCurrentState(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Storage: []models.Storage{{ID: "s1", Name: "store1", Usage: 75.5, Status: "ok"}},
	}
	alert := AlertInfo{ResourceType: "storage", ResourceID: "s1", ResourceName: "store1"}
	if got := ps.getResourceCurrentStateState(alert, patrolRuntimeStateForTest(ps, state)); got == "" {
		t.Fatalf("expected storage state")
	}

	alert = AlertInfo{ResourceType: "unknown"}
	if got := ps.getResourceCurrentStateState(alert, patrolRuntimeStateForTest(ps, state)); got == "" {
		t.Fatalf("expected fallback for unknown resource")
	}
}

func TestPatrolService_TriggerPatrolForAlert(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	ps.adHocTrigger = make(chan *alerts.Alert, 1)

	ps.TriggerPatrolForAlert(nil) // should no-op

	alert := &alerts.Alert{ID: "a1", Type: "cpu", ResourceID: "node1"}
	ps.TriggerPatrolForAlert(alert)
	select {
	case got := <-ps.adHocTrigger:
		if got.ID != alert.ID {
			t.Fatalf("expected alert to be queued")
		}
	default:
		t.Fatalf("expected alert to be queued")
	}

	// Trigger manager path
	tm := NewTriggerManager(TriggerManagerConfig{MaxPendingTriggers: 1})
	ps.SetTriggerManager(tm)
	ps.TriggerPatrolForAlert(alert)
	if tm.GetPendingCount() != 1 {
		t.Fatalf("expected trigger manager to queue alert")
	}
}

func TestPatrolService_TriggerPatrolForAlert_RespectsAlertTriggerConfig(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	ps.SetEventTriggerConfig(PatrolEventTriggerConfig{
		AlertTriggersEnabled:   false,
		AnomalyTriggersEnabled: true,
	})

	tm := NewTriggerManager(TriggerManagerConfig{MaxPendingTriggers: 1})
	ps.SetTriggerManager(tm)

	ps.TriggerPatrolForAlert(&alerts.Alert{ID: "a2", Type: "cpu", ResourceID: "node1"})
	if tm.GetPendingCount() != 0 {
		t.Fatalf("expected alert-driven patrol to be skipped when alert trigger source is disabled")
	}
}

func TestPatrolService_TriggerPatrolForAlert_RespectsRuntimeBlock(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	ps.SetEventTriggerConfig(PatrolEventTriggerConfig{
		AlertTriggersEnabled:   true,
		AnomalyTriggersEnabled: true,
	})
	ps.SetEventTriggerBlock(BackgroundAutomationEventTriggerBlock())

	tm := NewTriggerManager(TriggerManagerConfig{MaxPendingTriggers: 1})
	ps.SetTriggerManager(tm)

	ps.TriggerPatrolForAlert(&alerts.Alert{ID: "a3", Type: "cpu", ResourceID: "node1"})
	if tm.GetPendingCount() != 0 {
		t.Fatalf("expected alert-driven patrol to be skipped when event triggers are blocked by runtime policy")
	}

	status := tm.GetStatus()
	if !status.AlertTriggersEnabled || !status.AnomalyTriggersEnabled {
		t.Fatal("expected runtime block not to rewrite configured trigger preferences")
	}
	if !status.EventTriggersBlocked {
		t.Fatal("expected trigger manager status to expose event trigger runtime block")
	}
}

func TestPatrolService_RunTargetedPatrol_Disabled(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	ps.config.Enabled = false

	alert := &alerts.Alert{ID: "a2", Type: "cpu", ResourceID: "node1"}
	ps.runTargetedPatrol(context.Background(), alert)
}

func TestPatrolService_RunScopedPatrol_EarlyPaths(t *testing.T) {
	ps := NewPatrolService(nil, mockPatrolStateProvider{state: models.StateSnapshot{}})
	ps.config.Enabled = false
	ps.runScopedPatrol(context.Background(), PatrolScope{ResourceIDs: []string{"node1"}})

	// Re-queue when run already in progress
	ps.config.Enabled = true
	ps.runInProgress = true
	ps.runStartedAt = time.Now()
	ps.SetTriggerManager(NewTriggerManager(TriggerManagerConfig{MaxPendingTriggers: 2}))
	ps.runScopedPatrol(context.Background(), PatrolScope{ResourceIDs: []string{"node1"}})
	if ps.triggerManager.GetPendingCount() != 1 {
		t.Fatalf("expected scoped patrol to be re-queued")
	}

	// Drop permanently when retries exhausted
	ps.runInProgress = true
	ps.runStartedAt = time.Now()
	ps.runScopedPatrol(context.Background(), PatrolScope{ResourceIDs: []string{"node1"}, RetryCount: scopedPatrolMaxRetries})

	// No resources matched
	ps.runInProgress = false
	ps.stateProvider = mockPatrolStateProvider{state: models.StateSnapshot{}}
	ps.runScopedPatrol(context.Background(), PatrolScope{ResourceIDs: []string{"node1"}})

	// AI unavailable path
	ps.runInProgress = false
	ps.stateProvider = mockPatrolStateProvider{state: models.StateSnapshot{Nodes: []models.Node{{ID: "node1", Name: "node1"}}}}
	ps.runScopedPatrol(context.Background(), PatrolScope{ResourceIDs: []string{"node1"}, ResourceTypes: []string{"node"}})
	status := ps.GetStatus()
	if status.BlockedReason != patrolProviderNotConfiguredReason {
		t.Fatalf("blocked reason = %q, want %q", status.BlockedReason, patrolProviderNotConfiguredReason)
	}
}