Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

11.4 KB
Raw Blame
//go:build unix

package notifications_test

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"os/exec"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
	"github.com/stretchr/testify/require"
)

const checkpointProcessMarker = "GROUPED_PROCESS "

type checkpointProcessState struct {
	Phase   string                           `json:"phase"`
	Stats   map[string]int                   `json:"stats"`
	Log     []notifications.DeliveryLogEntry `json:"entries"`
	Started time.Time                        `json:"started"`
	Active  int                              `json:"active"`
}

// This registered helper runs real alert/notification managers in a separate
// process with persistent state, guest-local receivers and an actual SIGTERM.
// It is not the installed server executable or a system-service qualification.
func TestGroupedWarningProcessHelper(t *testing.T) {
	dir := os.Getenv("PULSE_GROUPED_TEST_DIR")
	if dir == "" {
		t.Skip("subprocess helper")
	}
	t.Setenv("PULSE_DATA_DIR", dir)
	url := os.Getenv("PULSE_GROUPED_TEST_RECEIVER")
	cooldown, err := strconv.Atoi(os.Getenv("PULSE_GROUPED_TEST_COOLDOWN"))
	require.NoError(t, err)
	var am *alerts.Manager
	var nm *notifications.NotificationManager
	fired := make(chan *alerts.Alert, 8)
	open := func() {
		nm = notifications.NewNotificationManagerWithDeferredQueue("", dir)
		require.NotNil(t, nm.GetQueue())
		require.NoError(t, nm.UpdateAllowedPrivateCIDRs("127.0.0.1/32"))
		nm.SetCooldown(cooldown)
		nm.SetGroupingConfig(true, 4, true, true)
		for _, id := range []string{"left", "right"} {
			nm.AddWebhook(notifications.WebhookConfig{ID: id, Name: "Fixture " + id, URL: url + "/" + id, Enabled: true, Service: "generic"})
		}
		am = alerts.NewManagerWithDataDir(dir, alerts.WithDurableAlertStore())
		cfg := am.GetConfig()
		cfg.Enabled = true
		cfg.ActivationState = alerts.ActivationActive
		cfg.FlappingEnabled = false
		cfg.Schedule.Cooldown = cooldown
		cfg.Schedule.MaxAlertsHour = 100
		cfg.TimeThresholds = map[string]int{"agent": 0, "storage": 0}
		cfg.SuppressionWindow = 0
		am.UpdateConfig(cfg)
		am.SetAlertCallback(func(a *alerts.Alert) { nm.SendAlert(a); fired <- a.Clone() })
		nm.StartQueueProcessing()
	}
	stop := func() {
		if am != nil {
			am.Stop()
			am = nil
		}
		if nm != nil {
			nm.Stop()
			nm = nil
		}
	}
	defer stop()
	open()
	started := time.Now().UTC()
	host := models.Host{ID: "process-host", Hostname: "Process host", Sensors: models.HostSensorSummary{Custom: []models.HostCustomSensorMetric{
		{ID: "probe-one", Name: "Probe one", Status: "warning", ObservedAt: started},
		{ID: "probe-two", Name: "Probe two", Status: "warning", ObservedAt: started},
	}}}
	if os.Getenv("PULSE_GROUPED_TEST_SEED") == "1" {
		am.CheckHost(host)
		for range 2 {
			select {
			case a := <-fired:
				require.Equal(t, alerts.AlertLevelWarning, a.Level)
			case <-time.After(5 * time.Second):
				t.Fatal("CheckHost did not dispatch both warnings")
			}
		}
		require.NoError(t, am.SaveActiveAlerts())
	} else {
		active := am.GetActiveAlerts()
		require.Len(t, active, 2)
		for _, a := range active {
			require.NotNil(t, a.LastNotified, "dispatch marker must survive restore")
		}
		am.CheckHost(host)
	}
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGTERM)
	defer signal.Stop(signals)
	commands := make(chan string)
	go func() {
		scanner := bufio.NewScanner(os.Stdin)
		for scanner.Scan() {
			commands <- scanner.Text()
		}
		close(commands)
	}()
	emit := func(phase string) {
		stats, err := nm.GetQueue().GetQueueStats()
		require.NoError(t, err)
		entries, err := nm.GetQueue().GetDeliveryLog(time.Now().Add(-time.Hour), 100)
		require.NoError(t, err)
		encoded, err := json.Marshal(checkpointProcessState{phase, stats, entries, started, len(am.GetActiveAlerts())})
		require.NoError(t, err)
		fmt.Println(checkpointProcessMarker + string(encoded))
	}
	emit("ready")
	for {
		select {
		case <-signals:
			stop()
			fmt.Println(checkpointProcessMarker + `{"phase":"stopped"}`)
			return
		case command, ok := <-commands:
			if !ok {
				return
			}
			switch command {
			case "reload":
				stop()
				open()
				am.CheckHost(host)
				emit("reloaded")
			case "status":
				emit("status")
			case "resolve":
				// Recovery dispatch uses the same public occurrence-cancellation
				// and receipt-gated entry points as the monitor callback.
				nm.SetGroupingConfig(false, 0, false, false)
				for _, a := range am.GetActiveAlerts() {
					nm.CancelResolvedAlert(&a)
					nm.SendResolvedAlert(&alerts.ResolvedAlert{Alert: &a, ResolvedTime: time.Now().UTC()})
				}
				emit("resolved")
			case "stop":
				stop()
				fmt.Println(checkpointProcessMarker + `{"phase":"stopped"}`)
				return
			default:
				t.Fatalf("unknown fixture command %q", command)
			}
		}
	}
}

func TestGroupedWarningGracefulProcessRestartAndReload(t *testing.T) {
	for _, tc := range []struct {
		mode     string
		cooldown int
	}{{"restart", 0}, {"reload", 5}} {
		t.Run(tc.mode, func(t *testing.T) {
			type delivery struct {
				At      time.Time
				Path    string
				Event   string          `json:"event"`
				Source  string          `json:"source"`
				Alerts  []*alerts.Alert `json:"alerts"`
				Count   int             `json:"count"`
				Grouped bool            `json:"grouped"`
			}
			received := make(chan delivery, 16)
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				var d delivery
				if err := json.NewDecoder(r.Body).Decode(&d); err != nil {
					t.Error(err)
					w.WriteHeader(400)
					return
				}
				d.At, d.Path = time.Now().UTC(), r.URL.Path
				received <- d
				w.WriteHeader(200)
			}))
			t.Cleanup(server.Close)
			dir := t.TempDir()
			type child struct {
				cmd    *exec.Cmd
				input  io.WriteCloser
				states chan checkpointProcessState
				stderr *bytes.Buffer
			}
			start := func(seed bool) *child {
				cmd := exec.Command(os.Args[0], "-test.run=^TestGroupedWarningProcessHelper$", "-test.timeout=45s", "-test.v")
				cmd.Env = append(os.Environ(), "PULSE_GROUPED_TEST_DIR="+dir, "PULSE_GROUPED_TEST_RECEIVER="+server.URL, "PULSE_GROUPED_TEST_COOLDOWN="+strconv.Itoa(tc.cooldown), "PULSE_GROUPED_TEST_SEED="+map[bool]string{true: "1", false: "0"}[seed])
				stdout, err := cmd.StdoutPipe()
				require.NoError(t, err)
				input, err := cmd.StdinPipe()
				require.NoError(t, err)
				c := &child{cmd, input, make(chan checkpointProcessState, 32), &bytes.Buffer{}}
				cmd.Stderr = c.stderr
				require.NoError(t, cmd.Start())
				go func() {
					defer close(c.states)
					scanner := bufio.NewScanner(stdout)
					for scanner.Scan() {
						line := scanner.Text()
						if strings.HasPrefix(line, checkpointProcessMarker) {
							var state checkpointProcessState
							if json.Unmarshal([]byte(strings.TrimPrefix(line, checkpointProcessMarker)), &state) == nil {
								c.states <- state
							}
						}
					}
				}()
				return c
			}
			read := func(c *child, phase string) checkpointProcessState {
				t.Helper()
				select {
				case state, ok := <-c.states:
					require.True(t, ok, "child ended before %s", phase)
					require.Equal(t, phase, state.Phase)
					return state
				case <-time.After(15 * time.Second):
					t.Fatalf("no child phase %s", phase)
					return checkpointProcessState{}
				}
			}
			finish := func(c *child, sig bool) {
				t.Helper()
				if sig {
					require.NoError(t, c.cmd.Process.Signal(syscall.SIGTERM))
				} else {
					_, err := fmt.Fprintln(c.input, "stop")
					require.NoError(t, err)
				}
				read(c, "stopped")
				require.NoError(t, c.input.Close())
				err := c.cmd.Wait()
				if err != nil {
					t.Log(c.stderr.String())
				}
				require.NoError(t, err, "helper must exit normally after graceful stop")
			}
			c := start(true)
			// Cleanup only the child owned by this test, even on an assertion failure.
			t.Cleanup(func() {
				if c != nil && c.cmd.ProcessState == nil {
					_ = c.input.Close()
					_ = c.cmd.Process.Signal(syscall.SIGTERM)
					_ = c.cmd.Wait()
				}
			})
			seed := read(c, "ready")
			require.Equal(t, 2, seed.Active)
			require.Empty(t, seed.Log, "buffer admission is not a provider attempt")
			if tc.mode == "restart" {
				finish(c, true)
				c = start(false)
				read(c, "ready")
			} else {
				_, err := fmt.Fprintln(c.input, "reload")
				require.NoError(t, err)
				read(c, "reloaded")
			}
			_, err := fmt.Fprintln(c.input, "status")
			require.NoError(t, err)
			pending := read(c, "status")
			require.Equal(t, 2, pending.Stats["pending"], "restored dispatch markers must not strand accepted warnings")
			require.Empty(t, pending.Log)
			t.Logf("GROUPED_BROWSER_SNAPSHOT %s", mustCheckpointJSON(t, pending))
			seen := map[string]bool{}
			for range 2 {
				select {
				case d := <-received:
					require.False(t, d.At.Before(seed.Started.Add(4*time.Second)), "shutdown shortened grouping window")
					require.True(t, d.Grouped)
					require.Equal(t, 2, d.Count)
					require.Len(t, d.Alerts, 2)
					require.Empty(t, d.Event, "the existing generic grouped firing payload has no event field")
					require.Equal(t, "pulse-monitoring", d.Source)
					require.NotEqual(t, d.Alerts[0].ID, d.Alerts[1].ID)
					for _, a := range d.Alerts {
						require.Equal(t, alerts.AlertLevelWarning, a.Level)
					}
					require.False(t, seen[d.Path])
					seen[d.Path] = true
				case <-time.After(10 * time.Second):
					t.Fatal("wanted firing did not reach both receivers after restart/reload")
				}
			}
			// Wait for durable status/audit completion, not merely receiver arrival.
			waitSent := func(want int) checkpointProcessState {
				t.Helper()
				deadline := time.Now().Add(5 * time.Second)
				for {
					_, err := fmt.Fprintln(c.input, "status")
					require.NoError(t, err)
					state := read(c, "status")
					if state.Stats["sent"] == want && len(state.Log) == want {
						return state
					}
					require.True(t, time.Now().Before(deadline), "durable completion missing")
					time.Sleep(5 * time.Millisecond)
				}
			}
			sent := waitSent(2)
			t.Logf("GROUPED_BROWSER_SNAPSHOT %s", mustCheckpointJSON(t, sent))
			// Another real process boundary proves each occurrence/destination
			// firing receipt, not the in-memory delivery map, authorises recovery.
			finish(c, true)
			c = start(false)
			read(c, "ready")
			_, err = fmt.Fprintln(c.input, "resolve")
			require.NoError(t, err)
			read(c, "resolved")
			recoveries := map[string]int{}
			for range 4 {
				select {
				case d := <-received:
					require.Equal(t, "resolved", d.Event)
					require.Len(t, d.Alerts, 1)
					require.True(t, seen[d.Path])
					recoveries[d.Path]++
				case <-time.After(10 * time.Second):
					t.Fatal("receipt-gated recovery missing after process restart")
				}
			}
			require.Equal(t, map[string]int{"/left": 2, "/right": 2}, recoveries)
			resolved := waitSent(6)
			t.Logf("GROUPED_BROWSER_SNAPSHOT %s", mustCheckpointJSON(t, resolved))
			finish(c, false)
			c = nil
			select {
			case d := <-received:
				t.Fatalf("duplicate receiver message %+v", d)
			default:
			}
			t.Log("CheckHost warning markers, original grouped deadline, normal SIGTERM exit/reload, two scoped grouped HTTP firings and four receipt-gated recoveries verified without edited queue clocks or provider access")
		})
	}
}

func mustCheckpointJSON(t *testing.T, value any) string {
	t.Helper()
	b, err := json.Marshal(value)
	require.NoError(t, err)
	return string(b)
}