Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

6.8 KB
Raw Blame
package alerts

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
)

// Unlike an orderly restart, process exit must not get a shutdown checkpoint
// or store Close to rescue a lifecycle transition. Keep the recovery mirror
// deliberately stale to prove SQLite, not the mirror, preserves the incident.
// This tests process interruption, not power loss or installed delivery.
func TestStorageLifecycleAcrossProcessExit(t *testing.T) {
	const helperEnv = "PULSE_TEST_STORAGE_CRASH_PHASE"
	storage := models.Storage{ID: "pbs-crash-backups", Name: "backups", Type: "pbs", Status: "online", Total: 1000, Used: 850, Free: 150, Usage: 85}
	id := canonicalMetricStateID(storage.ID, "usage")
	start := func(t *testing.T, dir string) *Manager {
		t.Helper()
		m := NewManagerWithDataDir(dir)
		m.EnableEventLog()
		if !m.activeStateAuthoritative.Load() {
			t.Fatal("SQLite authority not enabled")
		}
		m.UpdateConfig(AlertConfig{Enabled: true, ActivationState: ActivationActive,
			TimeThresholds: map[string]int{"storage": 0}, StorageDefault: HysteresisThreshold{Trigger: 80, Clear: 70}})
		return m
	}
	observe := func(m *Manager, s models.Storage) {
		for range 5 {
			m.CheckStorage(s)
		}
	}
	if phase := os.Getenv(helperEnv); phase != "" {
		// Use test output rather than application logging: a stuck child must
		// identify its last application boundary even when logging is disabled.
		stage := func(name string) { t.Logf("child phase=%s stage=%s", phase, name) }
		stage("validate")
		if phase != "fired" && phase != "resolved" {
			t.Fatal("invalid child phase")
		}
		stage("manager-start")
		m := start(t, os.Getenv("PULSE_TEST_STORAGE_CRASH_DIR"))
		stage("manager-ready")
		if phase == "resolved" {
			stage("seed-firing")
			observe(m, storage)
			testRequireActiveAlert(t, m, id)
		}
		stage("initial-checkpoint")
		if err := m.SaveActiveAlerts(); err != nil {
			t.Fatal(err)
		}
		// Freeze checkpoints, leaving either an empty or a firing JSON mirror.
		// Lifecycle commits must remain independent of periodic/async saves.
		stage("checkpoint-lock")
		m.saveMu.Lock()
		stage("checkpoint-locked")
		if phase == "resolved" {
			storage.Used, storage.Free, storage.Usage = 0, 1000, 0
		}
		stage("lifecycle-observation")
		observe(m, storage)
		stage("lifecycle-observed")
		if testHasActiveAlert(t, m, id) != (phase == "fired") {
			t.Fatal("child did not reach expected lifecycle state")
		}
		stage("process-exit")
		os.Exit(0) // Intentionally bypass all cleanups, Stop and store Close.
	}

	for _, phase := range []string{"fired", "resolved"} {
		t.Run(phase, func(t *testing.T) {
			dir := t.TempDir()
			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
			defer cancel()
			cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestStorageLifecycleAcrossProcessExit$", "-test.v")
			cmd.Env = append(os.Environ(), helperEnv+"="+phase, "PULSE_TEST_STORAGE_CRASH_DIR="+dir)
			output, err := storageProcessExitChildResult(ctx, cmd, phase)
			if err != nil {
				t.Fatal(err)
			}
			t.Logf("child output:\n%s", output)
			mirror, err := os.ReadFile(filepath.Join(dir, "alerts", "active-alerts.json"))
			if err != nil {
				t.Fatal(err)
			}
			var stale []*Alert
			if err := json.Unmarshal(mirror, &stale); err != nil {
				t.Fatal(err)
			}
			wantStale := 0
			if phase == "resolved" {
				wantStale = 1
			}
			if len(stale) != wantStale {
				t.Fatalf("recovery mirror was not stale: %s", mirror)
			}
			m := start(t, dir)
			t.Cleanup(m.Stop)
			if testHasActiveAlert(t, m, id) != (phase == "fired") {
				t.Fatal("process exit lost firing incident or resurrected resolved incident")
			}
			events, err := m.AlertEvents(eventlog.Filter{Types: []string{eventlog.TypeFired, eventlog.TypeResolved}})
			if err != nil {
				t.Fatal(err)
			}
			want := 1
			if phase == "resolved" {
				want = 2
			}
			counts := make(map[string]int)
			for _, event := range events {
				counts[event.Type]++
			}
			if counts[eventlog.TypeFired] != 1 || counts[eventlog.TypeResolved] != want-1 {
				t.Fatalf("unexpected lifecycle history: %v", counts)
			}
			if len(events) != want {
				t.Fatalf("durable lifecycle events = %d, want %d", len(events), want)
			}
			missing := storage
			missing.Total, missing.Used, missing.Free, missing.Usage = 0, 0, 0, 0
			observe(m, missing)
			if testHasActiveAlert(t, m, id) != (phase == "fired") {
				t.Fatal("missing post-restart counters changed the incident state")
			}
		})
	}
}

// Keep the command failure and actual context result separate. In particular,
// "signal: killed" alone is not evidence that this harness's deadline fired.
func storageProcessExitChildResult(ctx context.Context, cmd *exec.Cmd, phase string) ([]byte, error) {
	started := time.Now()
	output, err := cmd.CombinedOutput()
	if err != nil {
		return output, fmt.Errorf("child phase=%s failed after %s (context=%v): %w\n%s",
			phase, time.Since(started), ctx.Err(), errors.Join(err, ctx.Err()), output)
	}
	return output, nil
}

// These child controls use no Manager or database. They prove timeout and
// nonzero-exit diagnostics without reenacting a persistence failure or making
// the real lifecycle test's deadline more permissive.
func TestStorageProcessExitChildDiagnostics(t *testing.T) {
	const helperEnv = "PULSE_TEST_STORAGE_CHILD_DIAGNOSTIC"
	if mode := os.Getenv(helperEnv); mode != "" {
		fmt.Fprintf(os.Stderr, "helper-stage=%s\n", mode)
		switch mode {
		case "timeout":
			<-time.After(time.Minute)
		case "exit":
			os.Exit(17)
		case "success":
			os.Exit(0)
		default:
			t.Fatal("invalid diagnostic mode")
		}
		return
	}
	for _, mode := range []string{"timeout", "exit", "success"} {
		t.Run(mode, func(t *testing.T) {
			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
			defer cancel()
			cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestStorageProcessExitChildDiagnostics$")
			cmd.Env = append(os.Environ(), helperEnv+"="+mode)
			output, err := storageProcessExitChildResult(ctx, cmd, mode)
			if !strings.Contains(string(output), "helper-stage="+mode) {
				t.Fatalf("missing executed-child stage: %q; error: %v", output, err)
			}
			switch mode {
			case "timeout":
				if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), "child phase=timeout") || !strings.Contains(err.Error(), "helper-stage=timeout") {
					t.Fatalf("timeout lost context, phase or stage: %v", err)
				}
			case "exit":
				var exitErr *exec.ExitError
				if !errors.As(err, &exitErr) || exitErr.ExitCode() != 17 || !strings.Contains(err.Error(), "context=<nil>") || !strings.Contains(err.Error(), "helper-stage=exit") {
					t.Fatalf("nonzero exit lost command result or invented a deadline: %v", err)
				}
			case "success":
				if err != nil {
					t.Fatal(err)
				}
			}
		})
	}
}