Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

20.4 KB
Raw Blame
package hostagent

import (
	"compress/gzip"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"sync"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/availabilityprobe"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
	"github.com/rs/zerolog"
)

func testProbeModule(t *testing.T, probe func(context.Context, config.AvailabilityTarget) (availabilityprobe.Outcome, error)) *availabilityProbeModule {
	t.Helper()
	logger := zerolog.New(io.Discard)
	module := newAvailabilityProbeModule(logger, nil)
	if probe != nil {
		module.probe = probe
	}
	return module
}

func availabilitySetting(entries ...map[string]interface{}) map[string]interface{} {
	values := make([]interface{}, 0, len(entries))
	for _, entry := range entries {
		values = append(values, entry)
	}
	return map[string]interface{}{availabilitySettingsKey: values}
}

func TestAvailabilityTargetsFromSettingsDecodesServerPayload(t *testing.T) {
	// Mirrors the payload shape built by the server for an assigned agent,
	// including a key the agent does not know about.
	settings := availabilitySetting(map[string]interface{}{
		"id":                  "udp-check",
		"name":                "UDP check",
		"targetKind":          "device",
		"address":             "sensor.local",
		"protocol":            "udp",
		"port":                float64(5353),
		"udpMode":             "response-required",
		"udpRequest":          "ping",
		"udpExpectedResponse": "pong",
		"enabled":             true,
		"pollIntervalSeconds": float64(45),
		"timeoutMillis":       float64(1500),
		"unknownFutureField":  "ignored",
	})

	targets, err := availabilityTargetsFromSettings(settings)
	if err != nil {
		t.Fatalf("availabilityTargetsFromSettings() error = %v", err)
	}
	if len(targets) != 1 {
		t.Fatalf("targets = %+v, want one assignment", targets)
	}
	target := targets[0]
	if target.ID != "udp-check" || target.Address != "sensor.local" {
		t.Fatalf("target = %+v", target)
	}
	if target.Protocol != config.AvailabilityProbeUDP || target.Port != 5353 {
		t.Fatalf("target protocol/port = %v/%d", target.Protocol, target.Port)
	}
	if target.UDPRequest != "ping" || target.UDPExpected != "pong" {
		t.Fatalf("target udp payloads = %q/%q", target.UDPRequest, target.UDPExpected)
	}
	if target.EffectivePollIntervalSecs() != 45 || target.EffectiveTimeoutMillis() != 1500 {
		t.Fatalf("target schedule = %ds/%dms", target.EffectivePollIntervalSecs(), target.EffectiveTimeoutMillis())
	}
	if !target.Enabled {
		t.Fatal("target should be enabled")
	}
	// Server-only concerns are never carried in an agent assignment.
	if target.ProbeAgentID != "" || target.LinkedResourceID != "" {
		t.Fatalf("target carried server-side fields: %+v", target)
	}
}

func TestAvailabilityTargetsFromSettingsHandlesMissingAndUnusableValues(t *testing.T) {
	targets, err := availabilityTargetsFromSettings(nil)
	if err != nil || targets != nil {
		t.Fatalf("nil settings = (%+v, %v), want no assignments", targets, err)
	}

	targets, err = availabilityTargetsFromSettings(map[string]interface{}{"interval": "30s"})
	if err != nil || targets != nil {
		t.Fatalf("settings without the key = (%+v, %v), want no assignments", targets, err)
	}

	targets, err = availabilityTargetsFromSettings(map[string]interface{}{availabilitySettingsKey: nil})
	if err != nil || targets != nil {
		t.Fatalf("null value = (%+v, %v), want no assignments", targets, err)
	}

	if _, err := availabilityTargetsFromSettings(map[string]interface{}{availabilitySettingsKey: "not-a-list"}); err == nil {
		t.Fatal("garbage value error = nil, want a decode failure")
	}

	// An entry without an ID cannot be reported against and is dropped.
	targets, err = availabilityTargetsFromSettings(availabilitySetting(
		map[string]interface{}{"address": "orphan.local", "protocol": "icmp", "enabled": true},
		map[string]interface{}{"id": "keep", "address": "keep.local", "protocol": "icmp", "enabled": true},
	))
	if err != nil {
		t.Fatalf("availabilityTargetsFromSettings() error = %v", err)
	}
	if len(targets) != 1 || targets[0].ID != "keep" {
		t.Fatalf("targets = %+v, want only the identified assignment", targets)
	}
}

func TestAvailabilityTargetsFromSettingsPreservesHTTPExecutionSecrets(t *testing.T) {
	settings := availabilitySetting(map[string]interface{}{
		"id": "http-contract", "address": "https://service.local/health", "protocol": "https", "enabled": true,
		"http": map[string]interface{}{
			"method": "POST", "body": `{"operation":"health"}`,
			"expectedStatusMin": float64(200), "expectedStatusMax": float64(299),
			"authentication": map[string]interface{}{"type": "bearer", "bearerToken": "agent-secret-token"},
			"headers":        []interface{}{map[string]interface{}{"id": "tenant", "name": "X-Tenant", "value": "tenant-a"}},
			"jsonPath":       "status", "jsonEquals": "healthy",
		},
	})

	targets, err := availabilityTargetsFromSettings(settings)
	if err != nil || len(targets) != 1 || targets[0].HTTP == nil {
		t.Fatalf("availabilityTargetsFromSettings() = %+v, %v", targets, err)
	}
	contract := targets[0].HTTP
	if contract.Body == nil || *contract.Body != `{"operation":"health"}` ||
		contract.Authentication.BearerToken == nil || *contract.Authentication.BearerToken != "agent-secret-token" ||
		len(contract.Headers) != 1 || contract.Headers[0].Value == nil || *contract.Headers[0].Value != "tenant-a" {
		t.Fatalf("decoded HTTP contract = %+v, want complete execution values", contract)
	}

	decodedAgain, err := availabilityTargetsFromSettings(settings)
	if err != nil || !availabilityAssignmentsEqual(targets, decodedAgain) {
		t.Fatalf("identical pointer-backed HTTP assignments were reported as changed: %+v, %v", decodedAgain, err)
	}
}

func TestApplyRemoteAvailabilityTargetsReconcilesAssignments(t *testing.T) {
	agent := &Agent{
		logger:       zerolog.New(io.Discard),
		availability: testProbeModule(t, nil),
	}

	assigned := availabilitySetting(map[string]interface{}{
		"id": "remote-a", "address": "a.local", "protocol": "icmp", "enabled": true,
	})
	agent.ApplyRemoteConfig(assigned, nil)
	if got := agent.availability.assignments(); len(got) != 1 || got[0].ID != "remote-a" {
		t.Fatalf("assignments = %+v, want remote-a", got)
	}

	// A repeated fetch of the same assignment must not restart the schedule.
	if agent.availability.applyTargets(agent.availability.assignments()) {
		t.Fatal("unchanged assignments reported a schedule change")
	}

	// An unreadable payload leaves the current schedule alone.
	agent.ApplyRemoteConfig(map[string]interface{}{availabilitySettingsKey: "not-a-list"}, nil)
	if got := agent.availability.assignments(); len(got) != 1 {
		t.Fatalf("assignments after a garbage payload = %+v, want the previous schedule", got)
	}

	// The server omits the key once nothing is assigned: that unassigns.
	agent.ApplyRemoteConfig(map[string]interface{}{"interval": "30s"}, nil)
	if got := agent.availability.assignments(); len(got) != 0 {
		t.Fatalf("assignments after unassignment = %+v, want none", got)
	}
}

func TestNewSeedsAvailabilityAssignmentsFromStartupConfig(t *testing.T) {
	recorder := &availabilityReportRecorder{}
	agent := newAvailabilityDeliveryAgent(t, recorder, config.AvailabilityTarget{
		ID:       "boot-target",
		Address:  "boot.local",
		Protocol: config.AvailabilityProbeICMP,
		Enabled:  true,
	})

	// The assignment fetched before startup must be live without waiting for
	// the first remote-config refresh.
	if got := agent.availability.assignments(); len(got) != 1 || got[0].ID != "boot-target" {
		t.Fatalf("assignments = %+v, want the startup assignment", got)
	}
	status, ok := agent.availability.moduleStatus()
	if !ok || status.Name != availabilityModuleName {
		t.Fatalf("module status = (%+v, %v), want the availability module", status, ok)
	}
}

func TestAvailabilityProbeIntervalClampsToServerBounds(t *testing.T) {
	tests := []struct {
		seconds int
		want    time.Duration
	}{
		{seconds: 0, want: time.Duration(config.DefaultAvailabilityPollIntervalSecs) * time.Second},
		{seconds: 1, want: availabilityMinInterval},
		{seconds: 45, want: 45 * time.Second},
		{seconds: 86400, want: availabilityMaxInterval},
	}
	for _, test := range tests {
		target := config.AvailabilityTarget{PollIntervalSecs: test.seconds}
		if got := availabilityProbeInterval(target); got != test.want {
			t.Fatalf("availabilityProbeInterval(%ds) = %v, want %v", test.seconds, got, test.want)
		}
	}
}

func TestAvailabilityModuleSchedulerRunsAssignedTargetsAndReschedules(t *testing.T) {
	var (
		mu      sync.Mutex
		checked []string
	)
	module := testProbeModule(t, func(_ context.Context, target config.AvailabilityTarget) (availabilityprobe.Outcome, error) {
		mu.Lock()
		checked = append(checked, target.ID)
		mu.Unlock()
		if target.ID == "down" {
			return availabilityprobe.OutcomeUnreachable, errors.New("icmp probe timed out")
		}
		return availabilityprobe.OutcomeReachable, nil
	})
	module.applyTargets([]config.AvailabilityTarget{
		{ID: "up", Address: "up.local", Protocol: config.AvailabilityProbeICMP, Enabled: true},
		{ID: "down", Address: "down.local", Protocol: config.AvailabilityProbeICMP, Enabled: true},
		{ID: "paused", Address: "paused.local", Protocol: config.AvailabilityProbeICMP, Enabled: false},
	})

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	done := make(chan struct{})
	go func() {
		defer close(done)
		module.Run(ctx)
	}()

	waitForPending := func(want int) {
		t.Helper()
		deadline := time.After(5 * time.Second)
		for {
			if module.pending.Len() >= want {
				return
			}
			select {
			case <-time.After(5 * time.Millisecond):
			case <-deadline:
				t.Fatalf("timed out waiting for %d queued results, have %d", want, module.pending.Len())
			}
		}
	}

	// Enabled targets probe immediately rather than after a full interval.
	waitForPending(2)
	results := module.snapshotForReport()
	if len(results) != 2 {
		t.Fatalf("results = %+v, want one per enabled target", results)
	}
	byTarget := make(map[string]agentshost.AvailabilityProbeResult, len(results))
	for _, result := range results {
		byTarget[result.TargetID] = result
	}
	if got := byTarget["up"].Outcome; got != string(availabilityprobe.OutcomeReachable) {
		t.Fatalf("up outcome = %q", got)
	}
	if got := byTarget["down"]; got.Outcome != string(availabilityprobe.OutcomeUnreachable) || got.Error == "" {
		t.Fatalf("down result = %+v, want a failure with an explanation", got)
	}
	if byTarget["up"].CheckedAt.IsZero() {
		t.Fatal("result is missing its observation time")
	}
	if _, ok := byTarget["paused"]; ok {
		t.Fatalf("disabled target was probed: %+v", results)
	}

	module.commitDelivered()
	if module.pending.Len() != 0 {
		t.Fatalf("pending = %d after delivery, want 0", module.pending.Len())
	}

	// Reassignment stops the old workers and starts the new set at once.
	mu.Lock()
	checked = nil
	mu.Unlock()
	module.applyTargets([]config.AvailabilityTarget{
		{ID: "replacement", Address: "new.local", Protocol: config.AvailabilityProbeICMP, Enabled: true},
	})
	waitForPending(1)
	reassigned := module.snapshotForReport()
	if len(reassigned) != 1 || reassigned[0].TargetID != "replacement" {
		t.Fatalf("results after reassignment = %+v, want only the replacement", reassigned)
	}
	mu.Lock()
	for _, id := range checked {
		if id != "replacement" {
			t.Fatalf("target %q ran after it was unassigned", id)
		}
	}
	mu.Unlock()

	cancel()
	select {
	case <-done:
	case <-time.After(5 * time.Second):
		t.Fatal("scheduler did not stop on context cancellation")
	}
}

func TestAvailabilityModuleQueueDropsOldestBeyondCapacity(t *testing.T) {
	module := testProbeModule(t, nil)
	for i := 0; i < availabilityPendingCapacity+50; i++ {
		module.enqueue(agentshost.AvailabilityProbeResult{
			TargetID:  "target",
			Outcome:   string(availabilityprobe.OutcomeReachable),
			CheckedAt: time.Unix(int64(i), 0).UTC(),
		})
	}

	if got := module.pending.Len(); got != availabilityPendingCapacity {
		t.Fatalf("pending = %d, want the capacity %d", got, availabilityPendingCapacity)
	}
	results := module.snapshotForReport()
	if len(results) != availabilityPendingCapacity {
		t.Fatalf("results = %d, want %d", len(results), availabilityPendingCapacity)
	}
	// The oldest 50 observations were dropped, not the newest.
	if got := results[0].CheckedAt; !got.Equal(time.Unix(50, 0).UTC()) {
		t.Fatalf("oldest retained result = %v, want the 51st observation", got)
	}
	if got := results[len(results)-1].CheckedAt; !got.Equal(time.Unix(availabilityPendingCapacity+49, 0).UTC()) {
		t.Fatalf("newest retained result = %v", got)
	}
}

func TestAvailabilityModuleConcurrentEnqueuePreservesSequenceOrder(t *testing.T) {
	module := testProbeModule(t, nil)
	start := make(chan struct{})
	var workers sync.WaitGroup
	for i := 0; i < availabilityPendingCapacity*4; i++ {
		workers.Add(1)
		go func(id int) {
			defer workers.Done()
			<-start
			module.enqueue(agentshost.AvailabilityProbeResult{TargetID: fmt.Sprintf("target-%d", id)})
		}(i)
	}
	close(start)
	workers.Wait()

	items := module.pending.Items()
	if len(items) != availabilityPendingCapacity {
		t.Fatalf("queued results = %d, want capacity %d", len(items), availabilityPendingCapacity)
	}
	for i := 1; i < len(items); i++ {
		if items[i].sequence <= items[i-1].sequence {
			t.Fatalf("queue sequence is not increasing at %d: %d followed %d", i, items[i].sequence, items[i-1].sequence)
		}
	}
}

func TestAvailabilityModuleStatusOnlyWhileAssigned(t *testing.T) {
	module := testProbeModule(t, nil)
	if _, ok := module.moduleStatus(); ok {
		t.Fatal("module status reported without an assignment")
	}

	module.applyTargets([]config.AvailabilityTarget{
		{ID: "remote", Address: "remote.local", Protocol: config.AvailabilityProbeICMP, Enabled: true},
	})
	status, ok := module.moduleStatus()
	if !ok {
		t.Fatal("module status missing for an assigned agent")
	}
	if status.Name != availabilityModuleName || !status.Enabled {
		t.Fatalf("status = %+v", status)
	}
	if status.State != availabilityModuleStateStarting {
		t.Fatalf("state = %q before the scheduler runs, want %q", status.State, availabilityModuleStateStarting)
	}
	if status.UpdatedAt.IsZero() {
		t.Fatal("status is missing its assignment time")
	}

	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan struct{})
	go func() {
		defer close(done)
		module.Run(ctx)
	}()
	deadline := time.After(5 * time.Second)
	for {
		status, _ = module.moduleStatus()
		if status.State == availabilityModuleStateRunning {
			break
		}
		select {
		case <-time.After(5 * time.Millisecond):
		case <-deadline:
			t.Fatalf("state = %q, want %q once the scheduler is up", status.State, availabilityModuleStateRunning)
		}
	}
	cancel()
	<-done

	module.applyTargets(nil)
	if _, ok := module.moduleStatus(); ok {
		t.Fatal("module status survived unassignment")
	}
}

func TestAgentModuleStatusAppendsAvailabilityToRuntimeModules(t *testing.T) {
	agent := &Agent{
		logger:       zerolog.New(io.Discard),
		availability: testProbeModule(t, nil),
		cfg: Config{
			ModuleStatus: func() []agentshost.ModuleStatus {
				return []agentshost.ModuleStatus{{Name: "host", Enabled: true, State: "running"}}
			},
		},
	}

	statuses := agent.currentModuleStatus()
	if len(statuses) != 1 || statuses[0].Name != "host" {
		t.Fatalf("modules = %+v, want only the runtime modules", statuses)
	}

	agent.availability.applyTargets([]config.AvailabilityTarget{
		{ID: "remote", Address: "remote.local", Protocol: config.AvailabilityProbeICMP, Enabled: true},
	})
	statuses = agent.currentModuleStatus()
	if len(statuses) != 2 || statuses[1].Name != availabilityModuleName {
		t.Fatalf("modules = %+v, want the availability module appended", statuses)
	}
}

// availabilityReportRecorder is a Pulse report endpoint that can be switched
// into failure so retention across a delivery outage is observable.
type availabilityReportRecorder struct {
	mu      sync.Mutex
	reports []agentshost.Report
	fail    bool
}

func (r *availabilityReportRecorder) setFail(fail bool) {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.fail = fail
}

func (r *availabilityReportRecorder) received() []agentshost.Report {
	r.mu.Lock()
	defer r.mu.Unlock()
	return append([]agentshost.Report(nil), r.reports...)
}

func (r *availabilityReportRecorder) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	r.mu.Lock()
	fail := r.fail
	r.mu.Unlock()
	if fail {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	var body io.Reader = req.Body
	if req.Header.Get("Content-Encoding") == "gzip" {
		gz, err := gzip.NewReader(req.Body)
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		defer gz.Close()
		body = gz
	}
	var report agentshost.Report
	if err := json.NewDecoder(body).Decode(&report); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}

	r.mu.Lock()
	r.reports = append(r.reports, report)
	r.mu.Unlock()
	_ = json.NewEncoder(w).Encode(map[string]any{"success": true})
}

func newAvailabilityDeliveryAgent(t *testing.T, recorder *availabilityReportRecorder, targets ...config.AvailabilityTarget) *Agent {
	t.Helper()
	server := httptest.NewServer(recorder)
	t.Cleanup(server.Close)

	logger := zerolog.New(io.Discard)
	agent, err := New(Config{
		PulseURL:            server.URL,
		APIToken:            "test-token",
		Interval:            time.Minute,
		HostnameOverride:    "probe-agent",
		StateDir:            t.TempDir(),
		Logger:              &logger,
		Collector:           &mockCollector{},
		AvailabilityTargets: targets,
		packageUpdates:      newPackageUpdateManager("windows", nil),
		storageCleanup:      newStorageCleanupManager("windows", nil),
	})
	if err != nil {
		t.Fatalf("New() error = %v", err)
	}
	return agent
}

func TestAvailabilityResultsSurviveFailedDeliveryAndAreSentOnce(t *testing.T) {
	recorder := &availabilityReportRecorder{}
	agent := newAvailabilityDeliveryAgent(t, recorder)
	ctx := context.Background()

	for _, id := range []string{"target-a", "target-b"} {
		agent.availability.enqueue(agentshost.AvailabilityProbeResult{
			TargetID:  id,
			Outcome:   string(availabilityprobe.OutcomeReachable),
			CheckedAt: time.Now().UTC(),
		})
	}

	recorder.setFail(true)
	if err := agent.process(ctx); err != nil {
		t.Fatalf("process() during outage error = %v", err)
	}
	if got := agent.availability.pending.Len(); got != 2 {
		t.Fatalf("pending = %d after a failed delivery, want the results retained", got)
	}
	buffered, ok := agent.reportBuffer.Peek()
	if !ok {
		t.Fatal("failed report was not buffered")
	}
	if len(buffered.AvailabilityResults) != 0 {
		t.Fatalf("buffered report kept %d results; a retry would double count them", len(buffered.AvailabilityResults))
	}

	// A result observed during the outage joins the same batch.
	agent.availability.enqueue(agentshost.AvailabilityProbeResult{
		TargetID:  "target-c",
		Outcome:   string(availabilityprobe.OutcomeUnreachable),
		CheckedAt: time.Now().UTC(),
		Error:     "icmp probe timed out",
	})

	recorder.setFail(false)
	if err := agent.process(ctx); err != nil {
		t.Fatalf("process() after recovery error = %v", err)
	}
	if got := agent.availability.pending.Len(); got != 0 {
		t.Fatalf("pending = %d after a successful delivery, want 0", got)
	}

	// A later report must not repeat what the server already counted.
	if err := agent.process(ctx); err != nil {
		t.Fatalf("process() error = %v", err)
	}

	delivered := map[string]int{}
	for _, report := range recorder.received() {
		for _, result := range report.AvailabilityResults {
			delivered[result.TargetID]++
		}
	}
	if len(delivered) != 3 {
		t.Fatalf("delivered targets = %+v, want all three observations", delivered)
	}
	for id, count := range delivered {
		if count != 1 {
			t.Fatalf("target %q was delivered %d times, want exactly once", id, count)
		}
	}
}

func TestAvailabilityResultsAreNotRetiredWhenTheTokenIsRejected(t *testing.T) {
	recorder := &availabilityReportRecorder{}
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		w.WriteHeader(http.StatusUnauthorized)
	}))
	defer server.Close()

	agent := newAvailabilityDeliveryAgent(t, recorder)
	agent.trimmedPulseURL = server.URL
	agent.availability.enqueue(agentshost.AvailabilityProbeResult{
		TargetID:  "target-a",
		Outcome:   string(availabilityprobe.OutcomeReachable),
		CheckedAt: time.Now().UTC(),
	})

	// A rejected token drops the report instead of buffering it, so the
	// results have to stay queued for whenever the token is replaced.
	if err := agent.process(context.Background()); err != nil {
		t.Fatalf("process() error = %v", err)
	}
	if got := agent.availability.pending.Len(); got != 1 {
		t.Fatalf("pending = %d after a rejected report, want the result retained", got)
	}
	if !agent.reportBuffer.IsEmpty() {
		t.Fatal("a rejected report should not be buffered")
	}
}