Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

13.6 KB
Raw Blame
package unifiedresources

import (
	"database/sql"
	"errors"
	"os"
	"path/filepath"
	"sync"
	"testing"
	"time"

	_ "modernc.org/sqlite"
)

func TestSQLiteActionDispatchBindingMigrationKeepsLegacyReceiptPendingInert(t *testing.T) {
	dir := t.TempDir()
	path := filepath.Join(dir, "resources", "unified_resources.db")
	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
		t.Fatal(err)
	}
	db, err := sql.Open("sqlite", path)
	if err != nil {
		t.Fatal(err)
	}
	_, err = db.Exec(`CREATE TABLE action_dispatch_attempts(attempt_id TEXT PRIMARY KEY,action_id TEXT NOT NULL UNIQUE,state TEXT NOT NULL,created_at DATETIME NOT NULL,updated_at DATETIME NOT NULL,lease_owner TEXT,lease_expires_at DATETIME,dispatch_count INTEGER NOT NULL DEFAULT 0);INSERT INTO action_dispatch_attempts(attempt_id,action_id,state,created_at,updated_at,dispatch_count)VALUES('legacy.dispatch.1','legacy','receipt_pending',CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,1)`)
	if err != nil {
		t.Fatal(err)
	}
	if err := db.Close(); err != nil {
		t.Fatal(err)
	}
	store, err := NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	attempt, found, err := store.GetActionDispatchAttempt("legacy")
	if err != nil || !found {
		t.Fatalf("attempt=%#v found=%v err=%v", attempt, found, err)
	}
	if attempt.State != ActionDispatchReceiptPending || attempt.OperationKind != "" || attempt.OperationVersion != 0 || attempt.RequestDigest != "" || attempt.AgentID != "" {
		t.Fatalf("migrated attempt=%#v", attempt)
	}
	if _, claimed, err := store.ClaimActionDispatch("legacy", "worker", time.Now(), time.Minute); err != nil || claimed {
		t.Fatalf("claimed=%v err=%v", claimed, err)
	}
}

func admitDispatchTestAction(t *testing.T, store ResourceStore, id string, now time.Time) ActionDispatchAttempt {
	t.Helper()
	record := atomicLifecycleTestRecord(id, ActionStatePlanned)
	record.CreatedAt = now
	record.UpdatedAt = now
	record.Plan.PlannedAt = now
	record.Plan.ExpiresAt = now.Add(time.Hour)
	record.Plan.RequiresApproval = false
	record.Plan.ApprovalPolicy = ApprovalNone
	if _, _, err := store.CreateActionAudit(record, atomicLifecycleInitialEvents(record)); err != nil {
		t.Fatal(err)
	}
	started, event, err := BeginActionExecution(record, "operator", now)
	if err != nil {
		t.Fatal(err)
	}
	attempt, err := NewActionDispatchAttempt(id, now)
	if err != nil {
		t.Fatal(err)
	}
	if err := store.RecordActionExecutionAdmission(started, event, attempt); err != nil {
		t.Fatal(err)
	}
	return attempt
}

func TestMemoryStoreActionStateQueryDeepCopiesPolicyProvenance(t *testing.T) {
	store := NewMemoryStore()
	record := policyProvenanceTestRecord(t, "act_state_policy_copy")
	if _, created, err := store.CreateActionAudit(record, policyProvenanceInitialEvents(record)); err != nil || !created {
		t.Fatalf("CreateActionAudit: created=%v err=%v", created, err)
	}
	first, err := store.GetActionAuditsByStates([]ActionState{ActionStatePending}, 10)
	if err != nil || len(first) != 1 {
		t.Fatalf("first=%#v err=%v", first, err)
	}
	first[0].Plan.PolicyDecision.Authorities[0].ReasonCodes[0] = PolicyReasonCapabilityApprovalMFA
	second, err := store.GetActionAuditsByStates([]ActionState{ActionStatePending}, 10)
	if err != nil || len(second) != 1 || second[0].Plan.PolicyDecision.Authorities[0].ReasonCodes[0] != PolicyReasonCapabilityApprovalAdmin {
		t.Fatalf("state query shared mutable provenance: second=%#v err=%v", second, err)
	}
}

func runActionDispatchCrashBoundaries(t *testing.T, store ResourceStore) {
	t.Helper()
	now := time.Date(2026, 7, 11, 14, 0, 0, 0, time.UTC)
	attempt := admitDispatchTestAction(t, store, "act_dispatch_crash", now)
	claimed, ok, err := store.ClaimActionDispatch(attempt.ActionID, "worker-a", now, time.Second)
	if err != nil || !ok || claimed.DispatchCount != 0 {
		t.Fatalf("initial claim=%#v ok=%v err=%v", claimed, ok, err)
	}
	// Crash after claim but before MarkActionDispatchStarted: the same durable
	// attempt is safely reclaimed and dispatch count remains zero.
	reclaimed, ok, err := store.ClaimActionDispatch(attempt.ActionID, "worker-b", now.Add(2*time.Second), time.Second)
	if err != nil || !ok || reclaimed.ID != attempt.ID || reclaimed.DispatchCount != 0 {
		t.Fatalf("reclaim=%#v ok=%v err=%v", reclaimed, ok, err)
	}
	started, err := store.MarkActionDispatchStarted(attempt.ID, "worker-b", now.Add(2*time.Second))
	if err != nil || started.State != ActionDispatchReceiptPending || started.DispatchCount != 1 {
		t.Fatalf("started=%#v err=%v", started, err)
	}
	if _, err := store.MarkActionDispatchStarted(attempt.ID, "worker-b", now.Add(3*time.Second)); !errors.Is(err, ErrActionDispatchNotClaimable) {
		t.Fatalf("duplicate pre-send boundary error=%v", err)
	}
	// Crash after the pre-send boundary: recovery cannot claim or resend.
	resumed, ok, err := store.RecoverActionDispatch(attempt.ActionID, now.Add(time.Hour))
	if err != nil || ok || resumed.State != ActionDispatchReceiptPending || resumed.DispatchCount != 1 {
		t.Fatalf("post-start recovery=%#v ok=%v err=%v", resumed, ok, err)
	}
}

func TestMemoryStoreActionDispatchCrashBoundaries(t *testing.T) {
	runActionDispatchCrashBoundaries(t, NewMemoryStore())
}

func TestSQLiteActionDispatchCrashBoundaries(t *testing.T) {
	store, err := NewSQLiteResourceStore(t.TempDir(), "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	runActionDispatchCrashBoundaries(t, store)
}

func runConcurrentActionDispatchClaimHasOneWinner(t *testing.T, store ResourceStore) {
	t.Helper()
	now := time.Now().UTC()
	attempt := admitDispatchTestAction(t, store, "act_dispatch_claim", now)
	start := make(chan struct{})
	var wg sync.WaitGroup
	var mu sync.Mutex
	winners := 0
	for _, owner := range []string{"worker-a", "worker-b"} {
		wg.Add(1)
		go func(owner string) {
			defer wg.Done()
			<-start
			_, claimed, claimErr := store.ClaimActionDispatch(attempt.ActionID, owner, now, time.Minute)
			if claimErr != nil {
				t.Errorf("ClaimActionDispatch: %v", claimErr)
			}
			if claimed {
				mu.Lock()
				winners++
				mu.Unlock()
			}
		}(owner)
	}
	close(start)
	wg.Wait()
	if winners != 1 {
		t.Fatalf("claim winners=%d, want 1", winners)
	}
}

func TestMemoryStoreConcurrentActionDispatchClaimHasOneWinner(t *testing.T) {
	runConcurrentActionDispatchClaimHasOneWinner(t, NewMemoryStore())
}

func TestSQLiteConcurrentActionDispatchClaimHasOneWinner(t *testing.T) {
	store, err := NewSQLiteResourceStore(t.TempDir(), "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	runConcurrentActionDispatchClaimHasOneWinner(t, store)
}

func TestSQLiteActionDispatchAdmissionRollsBackAuditAndOutboxTogether(t *testing.T) {
	store, err := NewSQLiteResourceStore(t.TempDir(), "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	now := time.Now().UTC()
	record := atomicLifecycleTestRecord("act_dispatch_rollback", ActionStatePlanned)
	record.Plan.RequiresApproval = false
	record.Plan.ApprovalPolicy = ApprovalNone
	if _, _, err := store.CreateActionAudit(record, atomicLifecycleInitialEvents(record)); err != nil {
		t.Fatal(err)
	}
	if _, err := store.db.Exec(`CREATE TRIGGER fail_dispatch_outbox BEFORE INSERT ON action_dispatch_outbox BEGIN SELECT RAISE(FAIL, 'outbox failure'); END`); err != nil {
		t.Fatal(err)
	}
	started, event, _ := BeginActionExecution(record, "operator", now)
	attempt, _ := NewActionDispatchAttempt(record.ID, now)
	if err := store.RecordActionExecutionAdmission(started, event, attempt); err == nil {
		t.Fatal("expected admission failure")
	}
	current, found, err := store.GetActionAudit(record.ID)
	if err != nil || !found || current.State != ActionStatePlanned {
		t.Fatalf("current=%#v found=%v err=%v", current, found, err)
	}
	if _, found, err := store.GetActionDispatchAttempt(record.ID); err != nil || found {
		t.Fatalf("attempt found=%v err=%v", found, err)
	}
}

func TestSQLiteActionDispatchReceiptIsIdempotentAndCorrelatedAcrossRestart(t *testing.T) {
	dir := t.TempDir()
	now := time.Now().UTC()
	store, err := NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	attempt := admitDispatchTestAction(t, store, "act_dispatch_receipt", now)
	if _, ok, err := store.ClaimActionDispatch(attempt.ActionID, "worker", now, time.Minute); err != nil || !ok {
		t.Fatalf("claim ok=%v err=%v", ok, err)
	}
	if _, err := store.MarkActionDispatchStarted(attempt.ID, "worker", now); err != nil {
		t.Fatal(err)
	}
	if err := store.Close(); err != nil {
		t.Fatal(err)
	}
	store, err = NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	receipt := ActionDispatchReceipt{AttemptID: attempt.ID, ActionID: attempt.ActionID, TransportRequestID: attempt.ID, ReceivedAt: now.Add(time.Minute)}
	if _, err := store.RecordActionDispatchReceipt(receipt); err != nil {
		t.Fatal(err)
	}
	if _, err := store.RecordActionDispatchReceipt(receipt); err != nil {
		t.Fatalf("duplicate receipt: %v", err)
	}
	receipt.TransportRequestID = "different"
	if _, err := store.RecordActionDispatchReceipt(receipt); !errors.Is(err, ErrActionDispatchReceiptConflict) {
		t.Fatalf("conflicting receipt error=%v", err)
	}
}

func runEarlyActionDispatchReceiptIsRejected(t *testing.T, store ResourceStore) {
	t.Helper()
	now := time.Now().UTC()
	attempt := admitDispatchTestAction(t, store, "act_dispatch_early_receipt", now)
	if _, ok, err := store.ClaimActionDispatch(attempt.ActionID, "worker", now, time.Minute); err != nil || !ok {
		t.Fatalf("claim ok=%v err=%v", ok, err)
	}
	receipt := ActionDispatchReceipt{AttemptID: attempt.ID, ActionID: attempt.ActionID, TransportRequestID: attempt.ID, ReceivedAt: now}
	if _, err := store.RecordActionDispatchReceipt(receipt); !errors.Is(err, ErrActionDispatchReceiptConflict) {
		t.Fatalf("early receipt error=%v", err)
	}
	current, found, err := store.GetActionDispatchAttempt(attempt.ActionID)
	if err != nil || !found || current.State != ActionDispatchClaimed || current.DispatchCount != 0 {
		t.Fatalf("current=%#v found=%v err=%v", current, found, err)
	}
}

func TestMemoryStoreEarlyActionDispatchReceiptIsRejected(t *testing.T) {
	runEarlyActionDispatchReceiptIsRejected(t, NewMemoryStore())
}

func TestSQLiteEarlyActionDispatchReceiptIsRejected(t *testing.T) {
	store, err := NewSQLiteResourceStore(t.TempDir(), "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	runEarlyActionDispatchReceiptIsRejected(t, store)
}

func prepareCorrelatedDispatchCompletion(t *testing.T, store ResourceStore, id string, now time.Time) (ActionDispatchAttempt, ActionAuditRecord, ActionLifecycleEvent, ActionDispatchReceipt) {
	t.Helper()
	attempt := admitDispatchTestAction(t, store, id, now)
	if _, claimed, err := store.ClaimActionDispatch(id, "worker", now, time.Minute); err != nil || !claimed {
		t.Fatalf("claim claimed=%v err=%v", claimed, err)
	}
	if _, err := store.MarkActionDispatchStarted(attempt.ID, "worker", now); err != nil {
		t.Fatal(err)
	}
	executing, found, err := store.GetActionAudit(id)
	if err != nil || !found {
		t.Fatalf("executing found=%v err=%v", found, err)
	}
	completed, event, err := CompleteActionExecution(executing, &ExecutionResult{Success: true, Output: "correlated"}, "operator", now.Add(time.Second))
	if err != nil {
		t.Fatal(err)
	}
	receipt := ActionDispatchReceipt{AttemptID: attempt.ID, ActionID: id, TransportRequestID: attempt.ID, ReceivedAt: now.Add(time.Second)}
	return attempt, completed, event, receipt
}

func TestSQLiteCorrelatedDispatchCompletionRollsBackReceiptOnTerminalWriteFailure(t *testing.T) {
	store, err := NewSQLiteResourceStore(t.TempDir(), "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	now := time.Now().UTC()
	attempt, completed, event, receipt := prepareCorrelatedDispatchCompletion(t, store, "act_dispatch_atomic_rollback", now)
	if _, err := store.db.Exec(`CREATE TRIGGER fail_terminal_event BEFORE INSERT ON action_lifecycle_events WHEN NEW.state IN ('completed', 'failed') BEGIN SELECT RAISE(FAIL, 'terminal event failure'); END`); err != nil {
		t.Fatal(err)
	}
	if err := store.RecordActionDispatchCompletion(receipt, completed, event); err == nil {
		t.Fatal("expected atomic completion failure")
	}
	if _, found, err := store.GetActionDispatchReceipt(attempt.ID); err != nil || found {
		t.Fatalf("receipt found=%v err=%v", found, err)
	}
	currentAttempt, found, err := store.GetActionDispatchAttempt(attempt.ActionID)
	if err != nil || !found || currentAttempt.State != ActionDispatchReceiptPending {
		t.Fatalf("attempt=%#v found=%v err=%v", currentAttempt, found, err)
	}
	currentAudit, found, err := store.GetActionAudit(attempt.ActionID)
	if err != nil || !found || currentAudit.State != ActionStateExecuting || currentAudit.Result != nil {
		t.Fatalf("audit=%#v found=%v err=%v", currentAudit, found, err)
	}
}

func TestSQLiteCorrelatedDispatchCompletionSurvivesRestartWithoutReceiptOnlyState(t *testing.T) {
	dir := t.TempDir()
	now := time.Now().UTC()
	store, err := NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	attempt, completed, event, receipt := prepareCorrelatedDispatchCompletion(t, store, "act_dispatch_atomic_restart", now)
	if err := store.RecordActionDispatchCompletion(receipt, completed, event); err != nil {
		t.Fatal(err)
	}
	if err := store.Close(); err != nil {
		t.Fatal(err)
	}
	store, err = NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	storedReceipt, found, err := store.GetActionDispatchReceipt(attempt.ID)
	if err != nil || !found || storedReceipt.AttemptID != attempt.ID {
		t.Fatalf("receipt=%#v found=%v err=%v", storedReceipt, found, err)
	}
	storedAudit, found, err := store.GetActionAudit(attempt.ActionID)
	if err != nil || !found || storedAudit.State != ActionStateCompleted || storedAudit.Result == nil || !storedAudit.Result.Success {
		t.Fatalf("audit=%#v found=%v err=%v", storedAudit, found, err)
	}
}