Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

94.8 KB
Raw Blame
package actionlifecycle

import (
	"context"
	"errors"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/actionplanner"
	unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

type delayedCreateStore struct {
	unified.ResourceStore
	calls         atomic.Int32
	secondArrived chan struct{}
	releaseSecond chan struct{}
}

// Force a cold lookup to exercise the atomic creation boundary even when a
// competing request has persisted between the lookup and insert.
func (s *delayedCreateStore) GetActionAuditByRequest(unified.ActionRequest, *unified.ActionOrigin) (unified.ActionAuditRecord, bool, error) {
	return unified.ActionAuditRecord{}, false, nil
}

func (s *delayedCreateStore) CreateActionAudit(record unified.ActionAuditRecord, events []unified.ActionLifecycleEvent) (unified.ActionAuditRecord, bool, error) {
	if s.calls.Add(1) == 2 {
		close(s.secondArrived)
		<-s.releaseSecond
	}
	return s.ResourceStore.CreateActionAudit(record, events)
}

type blockingExecutor struct {
	calls   atomic.Int32
	entered chan struct{}
	release chan struct{}
}

type claimBoundaryStore struct {
	unified.ResourceStore
	entered chan struct{}
	release chan struct{}
	once    atomic.Bool
}

func (s *claimBoundaryStore) ClaimActionDispatch(actionID, owner string, now time.Time, lease time.Duration) (unified.ActionDispatchAttempt, bool, error) {
	attempt, claimed, err := s.ResourceStore.ClaimActionDispatch(actionID, owner, now, lease)
	if claimed && s.once.CompareAndSwap(false, true) {
		close(s.entered)
		<-s.release
	}
	return attempt, claimed, err
}

func (e *blockingExecutor) ExecuteAction(_ context.Context, record unified.ActionAuditRecord) (*unified.ExecutionResult, error) {
	if e.calls.Add(1) == 1 {
		close(e.entered)
	}
	<-e.release
	return &unified.ExecutionResult{Success: true, Output: record.ID}, nil
}

func serviceForStore(t *testing.T, store unified.ResourceStore, resource unified.Resource, executor Executor) *Service {
	t.Helper()
	registry := unified.NewRegistry(store)
	registry.IngestResources([]unified.Resource{resource})
	return &Service{
		Registry:            func(string) (*unified.ResourceRegistry, error) { return registry, nil },
		Store:               func(string) (Store, error) { return store, nil },
		Executor:            executor,
		DecisionAuthorizer:  DecisionAuthorizerFunc(func(context.Context, string, unified.ActionAuditRecord, unified.ActionDecision) error { return nil }),
		ExecutionAuthorizer: ExecutionAuthorizerFunc(func(context.Context, string, unified.ActionAuditRecord, unified.ActionActor) error { return nil }),
	}
}

func TestWithPolicyMutationConcurrentFirstUseSharesCoordinator(t *testing.T) {
	const writers = 64

	service := &Service{}
	start := make(chan struct{})
	coordinators := make(chan *PolicyAdmissionCoordinator, writers)
	var ready sync.WaitGroup
	var done sync.WaitGroup
	ready.Add(writers)
	done.Add(writers)

	for range writers {
		go func() {
			defer done.Done()
			ready.Done()
			<-start
			coordinators <- service.admissionCoordinator()
		}()
	}

	ready.Wait()
	close(start)
	done.Wait()
	close(coordinators)

	var shared *PolicyAdmissionCoordinator
	for coordinator := range coordinators {
		if shared == nil {
			shared = coordinator
			continue
		}
		if coordinator != shared {
			t.Fatal("concurrent first use returned more than one policy admission coordinator")
		}
	}
	if shared == nil {
		t.Fatal("concurrent first use did not initialize a policy admission coordinator")
	}
}

func testActionActor(subject, orgID string) unified.ActionActor {
	if strings.TrimSpace(subject) == "" {
		subject = "operator"
	}
	return unified.ActionActor{SubjectID: subject, Kind: unified.ActionActorUser, CredentialID: "session:test", OrgID: orgID}
}

func testActionDecision(t *testing.T, service *Service, orgID, actionID string, approval unified.ActionApprovalRecord) unified.ActionDecision {
	t.Helper()
	record, found, _ := service.Get(orgID, actionID)
	planHash := "sha256:test"
	if found {
		planHash = record.Plan.PlanHash
	}
	actor := testActionActor(approval.Actor, orgID)
	return unified.ActionDecision{
		Actor:   actor,
		Outcome: approval.Outcome,
		Reason:  approval.Reason,
		Evidence: unified.ApprovalEvidence{
			Version: 1, Method: unified.MethodSession, Actor: actor, OrgID: orgID,
			ActionID: actionID, PlanHash: planHash, Outcome: approval.Outcome, IssuedAt: time.Now().UTC(),
		},
	}
}

func testBoundLifecycleApproval(record unified.ActionAuditRecord, subject string, method unified.ApprovalMethod, outcome unified.ApprovalOutcome, at time.Time) unified.ActionApprovalRecord {
	kind := unified.ActionActorUser
	credential := "session:test"
	if method == unified.MethodPolicy {
		kind = unified.ActionActorPolicy
		credential = "policy:test"
	}
	actor := unified.ActionActor{SubjectID: subject, Kind: kind, CredentialID: credential, OrgID: "default"}
	evidence := unified.ApprovalEvidence{Version: 1, Method: method, Actor: actor, OrgID: "default", ActionID: record.ID, PlanHash: record.Plan.PlanHash, Outcome: outcome, IssuedAt: at}
	return unified.ActionApprovalRecord{Actor: subject, ActorBinding: actor, Method: method, Timestamp: at, Outcome: outcome, Evidence: &evidence}
}

func runConcurrentPlanReplayCannotRewindTerminalAction(t *testing.T, store unified.ResourceStore) {
	t.Helper()
	delayed := &delayedCreateStore{ResourceStore: store, secondArrived: make(chan struct{}), releaseSecond: make(chan struct{})}
	executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
	service := serviceForStore(t, delayed, testResource(time.Now().UTC(), unified.ApprovalAdmin), executor)
	firstPlan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	secondResult := make(chan error, 1)
	go func() {
		_, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
		secondResult <- err
	}()
	<-delayed.secondArrived
	if _, err := service.Decide(context.Background(), "default", firstPlan.ActionID, testActionDecision(t, service, "default", firstPlan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved})); err != nil {
		t.Fatal(err)
	}
	if _, err := service.Execute(context.Background(), "default", firstPlan.ActionID, testActionActor("operator", "default"), "proof"); err != nil {
		t.Fatal(err)
	}
	close(delayed.releaseSecond)
	if err := <-secondResult; err != nil {
		t.Fatal(err)
	}
	current, found, err := store.GetActionAudit(firstPlan.ActionID)
	if err != nil || !found || current.State != unified.ActionStateCompleted {
		t.Fatalf("current=%#v found=%v err=%v", current, found, err)
	}
	if executor.calls != 1 {
		t.Fatalf("executor calls=%d, want 1", executor.calls)
	}
}

func TestConcurrentPlanReplayCannotRewindTerminalActionMemoryStore(t *testing.T) {
	runConcurrentPlanReplayCannotRewindTerminalAction(t, unified.NewMemoryStore())
}

func TestConcurrentPlanReplayCannotRewindTerminalActionSQLiteStore(t *testing.T) {
	store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	runConcurrentPlanReplayCannotRewindTerminalAction(t, store)
}

func runConcurrentExecuteAdmitsExecutorExactlyOnce(t *testing.T, store unified.ResourceStore) {
	t.Helper()
	executor := &blockingExecutor{entered: make(chan struct{}), release: make(chan struct{})}
	service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	first := make(chan error, 1)
	go func() {
		_, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "first")
		first <- err
	}()
	<-executor.entered
	secondRecord, secondErr := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "second")
	if secondErr != nil {
		t.Fatalf("duplicate Execute: %v", secondErr)
	}
	if secondRecord.State != unified.ActionStateExecuting {
		t.Fatalf("duplicate state=%q", secondRecord.State)
	}
	close(executor.release)
	if err := <-first; err != nil {
		t.Fatal(err)
	}
	if executor.calls.Load() != 1 {
		t.Fatalf("executor calls=%d, want 1", executor.calls.Load())
	}
}

func TestConcurrentExecuteAdmitsExecutorExactlyOnceMemoryStore(t *testing.T) {
	runConcurrentExecuteAdmitsExecutorExactlyOnce(t, unified.NewMemoryStore())
}

func TestConcurrentExecuteAdmitsExecutorExactlyOnceSQLiteStore(t *testing.T) {
	store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	runConcurrentExecuteAdmitsExecutorExactlyOnce(t, store)
}

func runConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t *testing.T, store unified.ResourceStore) {
	t.Helper()
	blocked := &claimBoundaryStore{ResourceStore: store, entered: make(chan struct{}), release: make(chan struct{})}
	executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
	service := serviceForStore(t, blocked, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	first := make(chan error, 1)
	go func() {
		_, executeErr := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "first")
		first <- executeErr
	}()
	<-blocked.entered
	second, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "duplicate")
	if err != nil || second.State != unified.ActionStateExecuting {
		t.Fatalf("duplicate=%#v err=%v", second, err)
	}
	close(blocked.release)
	if err := <-first; err != nil {
		t.Fatal(err)
	}
	if executor.calls != 1 {
		t.Fatalf("executor calls=%d, want 1", executor.calls)
	}
}

func TestMemoryStoreConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t *testing.T) {
	runConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t, unified.NewMemoryStore())
}

func TestSQLiteConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t *testing.T) {
	store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	runConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t, store)
}

func TestPlanReplayReturnsAuthoritativeApprovedExecutingAndTerminalRecords(t *testing.T) {
	states := []unified.ActionState{unified.ActionStateApproved, unified.ActionStateExecuting, unified.ActionStateRejected, unified.ActionStateCompleted, unified.ActionStateFailed}
	for _, state := range states {
		t.Run(string(state), func(t *testing.T) {
			store := unified.NewMemoryStore()
			service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalAdmin), &stubExecutor{})
			plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
			if err != nil {
				t.Fatal(err)
			}
			record, _, _ := store.GetActionAudit(plan.ActionID)
			decisionOutcome := unified.OutcomeApproved
			if state == unified.ActionStateRejected {
				decisionOutcome = unified.OutcomeRejected
			}
			decisionAt := time.Now().UTC()
			decided, decisionEvent, err := unified.ApplyActionDecision(record, testBoundLifecycleApproval(record, "operator", unified.MethodSession, decisionOutcome, decisionAt), decisionAt)
			if err != nil || store.RecordActionDecision(decided, decisionEvent) != nil {
				t.Fatalf("decision: %v", err)
			}
			if state != unified.ActionStateApproved && state != unified.ActionStateRejected {
				started, startEvent, err := unified.BeginActionExecution(decided, "operator", time.Now().UTC())
				if err != nil || store.RecordActionExecutionStart(started, startEvent) != nil {
					t.Fatalf("start: %v", err)
				}
				if state == unified.ActionStateCompleted || state == unified.ActionStateFailed {
					result := &unified.ExecutionResult{Success: state == unified.ActionStateCompleted}
					completed, doneEvent, err := unified.CompleteActionExecution(started, result, "operator", time.Now().UTC())
					if err != nil || store.RecordActionExecutionResult(completed, doneEvent) != nil {
						t.Fatalf("complete: %v", err)
					}
				}
			}
			returned, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
			if err != nil {
				t.Fatal(err)
			}
			if returned.ActionID != plan.ActionID {
				t.Fatalf("action id=%q", returned.ActionID)
			}
			current, _, _ := store.GetActionAudit(plan.ActionID)
			if current.State != state {
				t.Fatalf("state=%q want=%q", current.State, state)
			}
		})
	}
}

func TestPlanReplayRejectsConflictingOriginForDeterministicActionID(t *testing.T) {
	store := unified.NewMemoryStore()
	service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalAdmin), &stubExecutor{})
	first := PlanOptions{Actor: testActionActor("requester", "default"), Origin: &unified.ActionOrigin{Surface: "patrol", FindingID: "finding-1", InvestigationID: "inv-1", ProposalID: "proposal-1"}}
	if _, err := service.PlanWithOptions(context.Background(), "default", restartRequest(), first); err != nil {
		t.Fatal(err)
	}
	conflict := first
	conflict.Origin = &unified.ActionOrigin{Surface: "patrol", FindingID: "finding-2", InvestigationID: "inv-2", ProposalID: "proposal-1"}
	if _, err := service.PlanWithOptions(context.Background(), "default", restartRequest(), conflict); !errors.Is(err, unified.ErrActionIdentityConflict) {
		t.Fatalf("error=%v", err)
	}
}

func TestExecutePublishesPersistedExecutingTransition(t *testing.T) {
	store := unified.NewMemoryStore()
	executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
	service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
	var states []unified.ActionState
	service.OnActionTransition = func(_ string, record unified.ActionAuditRecord) { states = append(states, record.State) }
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	if _, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), ""); err != nil {
		t.Fatal(err)
	}
	want := []unified.ActionState{unified.ActionStatePlanned, unified.ActionStateExecuting, unified.ActionStateCompleted}
	if len(states) != len(want) {
		t.Fatalf("states=%v", states)
	}
	for i := range want {
		if states[i] != want[i] {
			t.Fatalf("states=%v want=%v", states, want)
		}
	}
}

func TestExecuteAfterSQLiteRestartDoesNotReadmitExecutingAction(t *testing.T) {
	dir := t.TempDir()
	first, err := unified.NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
	service := serviceForStore(t, first, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	record, _, _ := first.GetActionAudit(plan.ActionID)
	started, event, _ := unified.BeginActionExecution(record, "operator", time.Now().UTC())
	if err := first.RecordActionExecutionStart(started, event); err != nil {
		t.Fatal(err)
	}
	first.Close()
	second, err := unified.NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	defer second.Close()
	service = serviceForStore(t, second, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
	current, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "retry")
	if err != nil || current.State != unified.ActionStateExecuting || executor.calls != 0 {
		t.Fatalf("current=%#v err=%v calls=%d", current, err, executor.calls)
	}
}

func TestExecuteTimeoutAfterSendWaitsForCorrelatedLateResponseWithoutResend(t *testing.T) {
	store := unified.NewMemoryStore()
	executor := &reconcilingExecutor{executeErr: context.DeadlineExceeded, result: &unified.ExecutionResult{Success: true}}
	service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	current, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "timeout proof")
	if !errors.Is(err, context.DeadlineExceeded) || current.State != unified.ActionStateExecuting {
		t.Fatalf("current=%#v err=%v", current, err)
	}
	attempt, found, err := store.GetActionDispatchAttempt(plan.ActionID)
	if err != nil || !found || attempt.State != unified.ActionDispatchReceiptPending || attempt.DispatchCount != 1 {
		t.Fatalf("attempt=%#v found=%v err=%v", attempt, found, err)
	}
	if _, found, err := store.GetActionDispatchReceipt(attempt.ID); err != nil || found {
		t.Fatalf("receipt found=%v err=%v", found, err)
	}
	// Duplicate execute is a query-only resume and cannot call ExecuteAction.
	current, err = service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "duplicate resume")
	if err != nil || current.State != unified.ActionStateExecuting || executor.executeCalls != 1 || executor.reconcileCalls != 1 {
		t.Fatalf("current=%#v err=%v execute=%d reconcile=%d", current, err, executor.executeCalls, executor.reconcileCalls)
	}
	executor.found = true
	current, err = service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "late response")
	if err != nil || current.State != unified.ActionStateCompleted || executor.executeCalls != 1 || executor.reconcileCalls != 2 {
		t.Fatalf("current=%#v err=%v execute=%d reconcile=%d", current, err, executor.executeCalls, executor.reconcileCalls)
	}
	if receipt, found, err := store.GetActionDispatchReceipt(attempt.ID); err != nil || !found || receipt.AttemptID != attempt.ID {
		t.Fatalf("receipt=%#v found=%v err=%v", receipt, found, err)
	}
}

func TestSQLiteRestartRecoveryReconcilesReceiptPendingWithoutResend(t *testing.T) {
	dir := t.TempDir()
	now := time.Now().UTC()
	first, err := unified.NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	executor := &reconcilingExecutor{executeErr: context.DeadlineExceeded, result: &unified.ExecutionResult{Success: true}}
	service := serviceForStore(t, first, testResource(now, unified.ApprovalNone), executor)
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	if _, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "before restart"); !errors.Is(err, context.DeadlineExceeded) {
		t.Fatalf("Execute error=%v", err)
	}
	if err := first.Close(); err != nil {
		t.Fatal(err)
	}
	second, err := unified.NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	defer second.Close()
	executor.found = true
	service = serviceForStore(t, second, testResource(now, unified.ApprovalNone), executor)
	recovered, err := service.RecoverExecutingActions(context.Background(), "default", "system:restart-recovery", 100)
	if err != nil || len(recovered) != 1 || recovered[0].State != unified.ActionStateCompleted {
		t.Fatalf("recovered=%#v err=%v", recovered, err)
	}
	if executor.executeCalls != 1 || executor.reconcileCalls != 1 {
		t.Fatalf("execute=%d reconcile=%d", executor.executeCalls, executor.reconcileCalls)
	}
	current, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "duplicate after recovery")
	if err != nil || current.State != unified.ActionStateCompleted || executor.executeCalls != 1 {
		t.Fatalf("current=%#v err=%v execute=%d", current, err, executor.executeCalls)
	}
}

func TestSQLiteRestartRecoveryNotFoundRemainsReceiptPendingWithoutResend(t *testing.T) {
	dir := t.TempDir()
	now := time.Now().UTC()
	store, err := unified.NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	executor := &reconcilingExecutor{executeErr: context.DeadlineExceeded, found: false}
	service := serviceForStore(t, store, testResource(now, unified.ApprovalNone), executor)
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	if _, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "initial"); !errors.Is(err, context.DeadlineExceeded) {
		t.Fatalf("execute err=%v", err)
	}
	for n := 0; n < 2; n++ {
		recovered, err := service.RecoverExecutingActions(context.Background(), "default", "system:restart-recovery", 100)
		if err != nil || len(recovered) != 1 || recovered[0].State != unified.ActionStateExecuting {
			t.Fatalf("pass=%d recovered=%#v err=%v", n, recovered, err)
		}
	}
	attempt, found, err := store.GetActionDispatchAttempt(plan.ActionID)
	if err != nil || !found || attempt.State != unified.ActionDispatchReceiptPending {
		t.Fatalf("attempt=%#v found=%v err=%v", attempt, found, err)
	}
	if _, found, err := store.GetActionDispatchReceipt(attempt.ID); err != nil || found {
		t.Fatalf("receipt found=%v err=%v", found, err)
	}
	audit, found, err := store.GetActionAudit(plan.ActionID)
	if err != nil || !found || audit.State != unified.ActionStateExecuting || audit.Result != nil {
		t.Fatalf("audit=%#v found=%v err=%v", audit, found, err)
	}
	if executor.executeCalls != 1 || executor.reconcileCalls != 2 {
		t.Fatalf("execute=%d reconcile=%d", executor.executeCalls, executor.reconcileCalls)
	}
}

func TestActionDetailAndInboxMaterializeExplicitExpiry(t *testing.T) {
	now := time.Date(2026, 7, 11, 14, 0, 0, 0, time.UTC)
	store := unified.NewMemoryStore()
	service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), &stubExecutor{})
	service.Now = func() time.Time { return now }
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	service.Now = func() time.Time { return plan.ExpiresAt.Add(time.Second) }
	detail, found, err := service.Detail("default", plan.ActionID)
	if err != nil || !found || detail.Audit.State != unified.ActionStateExpired {
		t.Fatalf("detail=%#v found=%v err=%v", detail, found, err)
	}
	if len(detail.Events) == 0 || detail.Events[0].State != unified.ActionStateExpired {
		t.Fatalf("events=%#v", detail.Events)
	}
	pending, err := service.List("default", ActionListPending, 100)
	if err != nil || len(pending) != 0 {
		t.Fatalf("pending=%#v err=%v", pending, err)
	}
	settled, err := service.List("default", ActionListSettled, 100)
	if err != nil || len(settled) != 1 || settled[0].State != unified.ActionStateExpired {
		t.Fatalf("settled=%#v err=%v", settled, err)
	}
	if _, err := service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Outcome: unified.OutcomeApproved})); !errors.Is(err, unified.ErrActionPlanExpired) {
		t.Fatalf("Decide error=%v", err)
	}
}

func policyTestLease(record unified.ActionAuditRecord, now time.Time) unified.ActionPolicyAuthorizationLease {
	lease := unified.ActionPolicyAuthorizationLease{
		Version: 1, OrgID: "default", ActionID: record.ID, ResourceID: record.Request.ResourceID,
		CapabilityName: record.Request.CapabilityName, PlanHash: record.Plan.PlanHash,
		CapabilityPolicyVersion: record.Plan.PolicyVersion, AutoAuthorization: unified.AutoAuthorizeLowRisk,
		ApprovalPolicy: record.Plan.ApprovalPolicy, TenantPolicyVersion: "tenant:v1",
		EffectiveAutonomyLevel: "assisted", LicenseAllowsAutoFix: true,
		ResourcePolicyVersion: "resource:v1", CapabilityNames: []string{"restart"},
		IssuedAt: now, ExpiresAt: now.Add(time.Minute),
	}
	lease.Digest = unified.ActionPolicyAuthorizationDigest(lease)
	return lease
}

func runPolicyAdmissionCommitsAtomically(t *testing.T, store unified.ResourceStore) {
	now := time.Now().UTC()
	executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
	service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
	service.Now = func() time.Time { return now }
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	completed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
		return policyTestLease(record, at), "bounded policy", nil
	})
	if err != nil || completed.State != unified.ActionStateCompleted || executor.calls != 1 {
		t.Fatalf("completed=%#v err=%v calls=%d", completed, err, executor.calls)
	}
	if len(completed.Approvals) != 1 || completed.Approvals[0].Method != unified.MethodPolicy || completed.Approvals[0].PolicyLease == nil {
		t.Fatalf("approvals=%#v", completed.Approvals)
	}
	events, err := store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 10)
	if err != nil {
		t.Fatal(err)
	}
	seenApproved, seenExecuting := false, false
	for _, event := range events {
		seenApproved = seenApproved || event.State == unified.ActionStateApproved
		seenExecuting = seenExecuting || event.State == unified.ActionStateExecuting
	}
	if !seenApproved || !seenExecuting {
		t.Fatalf("events=%#v", events)
	}
}

func TestPolicyAdmissionCommitsApprovalAndExecutingAtomicallyMemoryStore(t *testing.T) {
	runPolicyAdmissionCommitsAtomically(t, unified.NewMemoryStore())
}
func TestPolicyAdmissionCommitsApprovalAndExecutingAtomicallySQLite(t *testing.T) {
	store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	runPolicyAdmissionCommitsAtomically(t, store)
}

func TestExecuteUnderPolicyRefusesLostReadinessBeforeAutomaticAdmission(t *testing.T) {
	now := time.Now().UTC()
	store := unified.NewMemoryStore()
	executor := &stubExecutor{
		result:    &unified.ExecutionResult{Success: true},
		readiness: &unified.ResourceActionReadiness{Name: "restart", Available: true},
	}
	service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
	service.Now = func() time.Time { return now }
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	executor.readiness = &unified.ResourceActionReadiness{
		Name:       "restart",
		Available:  false,
		ReasonCode: "command_agent_disconnected",
		Reason:     "command agent disconnected",
	}

	failed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
		return policyTestLease(record, at), "bounded policy", nil
	})
	if !errors.Is(err, unified.ErrActionExecutionUnavailable) || failed.State != unified.ActionStateFailed {
		t.Fatalf("failed=%#v err=%v", failed, err)
	}
	if executor.calls != 0 || len(failed.Approvals) != 0 {
		t.Fatalf("executor calls=%d approvals=%#v, want no dispatch or policy approval", executor.calls, failed.Approvals)
	}
	if _, found, getErr := store.GetActionDispatchAttempt(plan.ActionID); getErr != nil || found {
		t.Fatalf("dispatch attempt found=%v err=%v, want none", found, getErr)
	}
}

func runPolicyBarrierRevocations(t *testing.T, storeFactory func(t *testing.T) unified.ResourceStore) {
	cases := []struct {
		name   string
		mutate func(*unified.ActionPolicyAuthorizationLease) error
	}{
		{"policy_row_deleted", func(*unified.ActionPolicyAuthorizationLease) error {
			return unified.ErrActionPolicyAuthorizationInvalid
		}},
		{"policy_disabled", func(*unified.ActionPolicyAuthorizationLease) error {
			return unified.ErrActionPolicyAuthorizationRevoked
		}},
		{"capability_removed_from_allowlist", func(*unified.ActionPolicyAuthorizationLease) error {
			return unified.ErrActionPolicyAuthorizationRevoked
		}},
		{"recurring_window_closed", func(*unified.ActionPolicyAuthorizationLease) error {
			return unified.ErrActionPolicyAuthorizationRevoked
		}},
		{"mode_downgraded_to_approval", func(*unified.ActionPolicyAuthorizationLease) error {
			return unified.ErrActionPolicyAuthorizationRevoked
		}},
		{"mode_downgraded_to_monitor", func(*unified.ActionPolicyAuthorizationLease) error {
			return unified.ErrActionPolicyAuthorizationRevoked
		}},
		{"full_mode_unlock_removed", func(*unified.ActionPolicyAuthorizationLease) error {
			return unified.ErrActionPolicyAuthorizationRevoked
		}},
		{"license_removes_effective_mode", func(lease *unified.ActionPolicyAuthorizationLease) error {
			lease.LicenseAllowsAutoFix = false
			lease.Digest = unified.ActionPolicyAuthorizationDigest(*lease)
			return nil
		}},
		{"capability_removed", func(*unified.ActionPolicyAuthorizationLease) error { return unified.ErrActionPlanDrift }},
		{"safety_class_changed_to_never", func(*unified.ActionPolicyAuthorizationLease) error {
			return unified.ErrActionPolicyAuthorizationRevoked
		}},
		{"approval_floor_changed_to_dry_run", func(*unified.ActionPolicyAuthorizationLease) error {
			return unified.ErrActionPolicyAuthorizationRevoked
		}},
		{"approval_floor_changed_to_mfa", func(*unified.ActionPolicyAuthorizationLease) error {
			return unified.ErrActionPolicyAuthorizationRevoked
		}},
		{"never_auto_remediate_enabled", func(*unified.ActionPolicyAuthorizationLease) error { return unified.ErrResourceRemediationLocked }},
		{"emergency_stop_enabled", func(lease *unified.ActionPolicyAuthorizationLease) error {
			lease.EmergencyStop = true
			lease.Digest = unified.ActionPolicyAuthorizationDigest(*lease)
			return nil
		}},
		{"policy_store_unreadable", func(*unified.ActionPolicyAuthorizationLease) error { return errors.New("policy store unavailable") }},
		{"lease_wrong_org", func(lease *unified.ActionPolicyAuthorizationLease) error {
			lease.OrgID = "other"
			lease.Digest = unified.ActionPolicyAuthorizationDigest(*lease)
			return nil
		}},
		{"lease_expired", func(lease *unified.ActionPolicyAuthorizationLease) error {
			lease.ExpiresAt = lease.IssuedAt
			lease.Digest = unified.ActionPolicyAuthorizationDigest(*lease)
			return nil
		}},
		{"lease_digest_tampered", func(lease *unified.ActionPolicyAuthorizationLease) error {
			lease.Digest = "sha256:tampered"
			return nil
		}},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			store := storeFactory(t)
			now := time.Now().UTC()
			executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
			service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
			service.Now = func() time.Time { return now }
			plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
			if err != nil {
				t.Fatal(err)
			}
			failed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
				lease := policyTestLease(record, at)
				if mutationErr := tc.mutate(&lease); mutationErr != nil {
					return unified.ActionPolicyAuthorizationLease{}, "", mutationErr
				}
				return lease, "policy", nil
			})
			if err == nil || failed.State != unified.ActionStateFailed || executor.calls != 0 {
				t.Fatalf("failed=%#v err=%v calls=%d", failed, err, executor.calls)
			}
			if len(failed.Approvals) != 0 {
				t.Fatalf("policy approval persisted: %#v", failed.Approvals)
			}
			events, _ := store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 20)
			for _, event := range events {
				if event.State == unified.ActionStateExecuting {
					t.Fatalf("executing event persisted: %#v", events)
				}
			}
		})
	}
}

func TestExecuteUnderPolicyBarrierRevocationsMemoryStore(t *testing.T) {
	runPolicyBarrierRevocations(t, func(*testing.T) unified.ResourceStore { return unified.NewMemoryStore() })
}
func TestExecuteUnderPolicyBarrierRevocationsSQLite(t *testing.T) {
	runPolicyBarrierRevocations(t, func(t *testing.T) unified.ResourceStore {
		store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
		if err != nil {
			t.Fatal(err)
		}
		t.Cleanup(func() { _ = store.Close() })
		return store
	})
}

func runHumanApprovalSurvivesPolicyRevocation(t *testing.T, store unified.ResourceStore) {
	now := time.Now().UTC()
	executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
	service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	if _, err = service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved})); err != nil {
		t.Fatal(err)
	}
	if _, err = service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), ""); err != nil || executor.calls != 1 {
		t.Fatalf("err=%v calls=%d", err, executor.calls)
	}
}
func TestHumanApprovalSurvivesAutomaticPolicyRevocationMemoryStore(t *testing.T) {
	runHumanApprovalSurvivesPolicyRevocation(t, unified.NewMemoryStore())
}
func TestHumanApprovalSurvivesAutomaticPolicyRevocationSQLite(t *testing.T) {
	store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	runHumanApprovalSurvivesPolicyRevocation(t, store)
}

func runEmergencyStopBlocksHumanAndPolicy(t *testing.T, store unified.ResourceStore) {
	now := time.Now().UTC()
	executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
	service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
	service.EmergencyStop = func(string) (bool, error) { return true, nil }
	human := restartRequest()
	human.RequestID = "human-stop"
	plan, err := service.Plan(context.Background(), "default", human, testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	if _, err = service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved})); !errors.Is(err, unified.ErrActionEmergencyStop) {
		t.Fatalf("human approval error=%v", err)
	}
	if record, found, getErr := service.Get("default", plan.ActionID); getErr != nil || !found || record.State != unified.ActionStatePending {
		t.Fatalf("blocked approval must remain pending: found=%v state=%s err=%v", found, record.State, getErr)
	}
	policy := restartRequest()
	policy.RequestID = "policy-stop"
	policyPlan, err := service.Plan(context.Background(), "default", policy, testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	if _, err = service.ExecuteUnderPolicy(context.Background(), "default", policyPlan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
		return policyTestLease(record, at), "policy", nil
	}); !errors.Is(err, unified.ErrActionEmergencyStop) {
		t.Fatalf("policy error=%v", err)
	}
	if executor.calls != 0 {
		t.Fatalf("executor calls=%d", executor.calls)
	}
}
func TestEmergencyStopBlocksHumanAndPolicyAdmissionMemoryStore(t *testing.T) {
	runEmergencyStopBlocksHumanAndPolicy(t, unified.NewMemoryStore())
}
func TestEmergencyStopBlocksHumanAndPolicyAdmissionSQLite(t *testing.T) {
	store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
	if err != nil {
		t.Fatal(err)
	}
	defer store.Close()
	runEmergencyStopBlocksHumanAndPolicy(t, store)
}

func TestLegacyPolicyApprovalWithoutLeaseFailsClosedAfterRestart(t *testing.T) {
	dir := t.TempDir()
	first, err := unified.NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	now := time.Now().UTC()
	request := restartRequest()
	request.Actor = testActionActor("requester", "default")
	request.RequestedBy = request.Actor.SubjectID
	plan, err := (actionplanner.Planner{Now: func() time.Time { return now }}).Plan(request, testResource(now, unified.ApprovalAdmin))
	if err != nil {
		t.Fatal(err)
	}
	legacyApproved := unified.ActionAuditRecord{
		ID: plan.ActionID, CreatedAt: now, UpdatedAt: now, State: unified.ActionStateApproved,
		Request: request, Plan: plan,
		Approvals: []unified.ActionApprovalRecord{{Actor: "pulse_patrol_policy", Method: unified.MethodPolicy, Timestamp: now, Outcome: unified.OutcomeApproved}},
	}
	if err := first.RecordActionAudit(legacyApproved); err != nil {
		t.Fatalf("legacy audit: %v", err)
	}
	if err = first.Close(); err != nil {
		t.Fatal(err)
	}
	second, err := unified.NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	defer second.Close()
	executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
	service := serviceForStore(t, second, testResource(now, unified.ApprovalAdmin), executor)
	failed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
		return policyTestLease(record, at), "policy", nil
	})
	if !errors.Is(err, unified.ErrActionPolicyAuthorizationInvalid) || failed.State != unified.ActionStateFailed || executor.calls != 0 {
		t.Fatalf("failed=%#v err=%v calls=%d", failed, err, executor.calls)
	}
}

func TestQueuedPolicyActionRevalidatesAfterSQLiteRestart(t *testing.T) {
	dir := t.TempDir()
	first, err := unified.NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	now := time.Now().UTC()
	service := serviceForStore(t, first, testResource(now, unified.ApprovalAdmin), &stubExecutor{})
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	if err = first.Close(); err != nil {
		t.Fatal(err)
	}
	second, err := unified.NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	defer second.Close()
	executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
	service = serviceForStore(t, second, testResource(now, unified.ApprovalAdmin), executor)
	failed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(context.Context, unified.ActionAuditRecord, time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
		return unified.ActionPolicyAuthorizationLease{}, "", unified.ErrActionPolicyAuthorizationRevoked
	})
	if !errors.Is(err, unified.ErrActionPolicyAuthorizationRevoked) || failed.State != unified.ActionStateFailed || executor.calls != 0 {
		t.Fatalf("failed=%#v err=%v calls=%d", failed, err, executor.calls)
	}
}

type stubExecutor struct {
	result      *unified.ExecutionResult
	err         error
	calls       int
	received    unified.ActionAuditRecord
	readiness   *unified.ResourceActionReadiness
	feasibility *unified.ResourceActionReadiness
}

type reconcilingExecutor struct {
	executeCalls   int
	reconcileCalls int
	executeErr     error
	found          bool
	result         *unified.ExecutionResult
}
type bindingReconcilingExecutor struct{ reconcilingExecutor }

func (e *bindingReconcilingExecutor) BindActionDispatch(_ context.Context, _ unified.ActionAuditRecord, a unified.ActionDispatchAttempt) (unified.ActionDispatchAttempt, error) {
	return a, nil
}

func (e *reconcilingExecutor) ExecuteAction(_ context.Context, _ unified.ActionAuditRecord) (*unified.ExecutionResult, error) {
	e.executeCalls++
	return nil, e.executeErr
}

func (e *reconcilingExecutor) ReconcileActionDispatch(_ context.Context, record unified.ActionAuditRecord, attempt unified.ActionDispatchAttempt) (*unified.ExecutionResult, unified.ActionDispatchReceipt, bool, error) {
	e.reconcileCalls++
	if !e.found {
		return nil, unified.ActionDispatchReceipt{}, false, nil
	}
	return e.result, unified.ActionDispatchReceipt{
		AttemptID: attempt.ID, ActionID: record.ID, TransportRequestID: attempt.ID, ReceivedAt: time.Now().UTC(),
	}, true, nil
}

func TestLegacyUnboundReceiptPendingAttemptIsInertForBindingOwnedExecutor(t *testing.T) {
	now := time.Now().UTC()
	store := unified.NewMemoryStore()
	executor := &bindingReconcilingExecutor{}
	service := serviceForStore(t, store, testResource(now, unified.ApprovalNone), executor)
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	record, found, err := store.GetActionAudit(plan.ActionID)
	if err != nil || !found {
		t.Fatal(err)
	}
	started, event, err := unified.BeginActionExecution(record, "operator", now)
	if err != nil {
		t.Fatal(err)
	}
	attempt, err := unified.NewActionDispatchAttempt(started.ID, now)
	if err != nil {
		t.Fatal(err)
	}
	if err := store.RecordActionExecutionAdmission(started, event, attempt); err != nil {
		t.Fatal(err)
	}
	claimed, won, err := store.ClaimActionDispatch(started.ID, "legacy-worker", now, time.Minute)
	if err != nil || !won {
		t.Fatalf("claim=%#v won=%v err=%v", claimed, won, err)
	}
	if _, err := store.MarkActionDispatchStarted(claimed.ID, "legacy-worker", now); err != nil {
		t.Fatal(err)
	}
	recovered, err := service.RecoverExecutingActions(context.Background(), "default", "system:recovery", 10)
	if err != nil || len(recovered) != 1 || recovered[0].State != unified.ActionStateExecuting {
		t.Fatalf("recovered=%#v err=%v", recovered, err)
	}
	if executor.executeCalls != 0 || executor.reconcileCalls != 0 {
		t.Fatalf("execute=%d reconcile=%d", executor.executeCalls, executor.reconcileCalls)
	}
}

func (s *stubExecutor) ExecuteAction(_ context.Context, record unified.ActionAuditRecord) (*unified.ExecutionResult, error) {
	s.calls++
	s.received = record
	return s.result, s.err
}

func (s *stubExecutor) CheckActionAvailable(_ context.Context, _ unified.ActionRequest, _ unified.Resource) unified.ResourceActionReadiness {
	if s.readiness == nil {
		return unified.ResourceActionReadiness{}
	}
	return *s.readiness
}

func (s *stubExecutor) CheckActionFeasible(_ context.Context, _ string, _ unified.ActionRequest, _ unified.Resource) unified.ResourceActionReadiness {
	if s.feasibility == nil {
		return unified.ResourceActionReadiness{}
	}
	return *s.feasibility
}

type serviceEnv struct {
	store     unified.ResourceStore
	registry  *unified.ResourceRegistry
	executor  *stubExecutor
	completed []unified.ActionAuditRecord
	service   *Service
}

func testResource(now time.Time, minimumApproval unified.ActionApprovalLevel) unified.Resource {
	return unified.Resource{
		ID:        "vm:42",
		Type:      unified.ResourceTypeVM,
		Name:      "web-42",
		Status:    unified.StatusWarning,
		LastSeen:  now,
		UpdatedAt: now,
		Sources:   []unified.DataSource{unified.SourceProxmox},
		Capabilities: []unified.ResourceCapability{
			{
				Name:                 "restart",
				Type:                 unified.CapabilityTypeCommon,
				Description:          "Restart the VM",
				MinimumApprovalLevel: minimumApproval,
				InternalHandler:      "proxmox.vm.restart",
				Params: []unified.CapabilityParam{
					{Name: "mode", Type: "string", Required: true, Enum: []string{"graceful", "force"}},
				},
			},
		},
	}
}

func newServiceEnv(t *testing.T, resource unified.Resource) *serviceEnv {
	t.Helper()
	env := &serviceEnv{
		store:    unified.NewMemoryStore(),
		executor: &stubExecutor{result: &unified.ExecutionResult{Success: true, Output: "restarted"}},
	}
	env.registry = unified.NewRegistry(env.store)
	env.registry.IngestResources([]unified.Resource{resource})
	env.service = &Service{
		Registry: func(orgID string) (*unified.ResourceRegistry, error) {
			if orgID != "default" {
				return nil, errors.New("unknown org")
			}
			return env.registry, nil
		},
		Store: func(orgID string) (Store, error) {
			if orgID != "default" {
				return nil, errors.New("unknown org")
			}
			return env.store, nil
		},
		Executor:            env.executor,
		DecisionAuthorizer:  DecisionAuthorizerFunc(func(context.Context, string, unified.ActionAuditRecord, unified.ActionDecision) error { return nil }),
		ExecutionAuthorizer: ExecutionAuthorizerFunc(func(context.Context, string, unified.ActionAuditRecord, unified.ActionActor) error { return nil }),
		OnActionCompleted: func(record unified.ActionAuditRecord) {
			env.completed = append(env.completed, record)
		},
	}
	return env
}

func restartRequest() unified.ActionRequest {
	return unified.ActionRequest{
		RequestID:      "req-1",
		ResourceID:     "vm:42",
		CapabilityName: "restart",
		Params:         map[string]any{"mode": "graceful"},
		Reason:         "Recover after confirmed outage",
		RequestedBy:    "agent:test",
	}
}

func TestPlanPersistsPendingAuditAndLifecycle(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))

	plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}
	if !plan.RequiresApproval {
		t.Fatal("expected admin-gated capability to require approval")
	}

	record, ok, err := env.store.GetActionAudit(plan.ActionID)
	if err != nil || !ok {
		t.Fatalf("GetActionAudit: ok=%v err=%v", ok, err)
	}
	if record.State != unified.ActionStatePending {
		t.Fatalf("audit state = %q, want pending_approval", record.State)
	}
	events, err := env.store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 10)
	if err != nil {
		t.Fatalf("GetActionLifecycleEvents: %v", err)
	}
	if len(events) != 2 {
		t.Fatalf("lifecycle events = %d, want planned + pending", len(events))
	}

	// Idempotent replan must not duplicate lifecycle events.
	if _, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default")); err != nil {
		t.Fatalf("replan: %v", err)
	}
	events, err = env.store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 10)
	if err != nil {
		t.Fatalf("GetActionLifecycleEvents after replan: %v", err)
	}
	if len(events) != 2 {
		t.Fatalf("replan duplicated lifecycle events: %d", len(events))
	}
}

func TestPlanFollowsAdvertisedLifecycleSynonym(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))

	// The fixture advertises only "restart"; a request for the lifecycle
	// synonym "reboot" must plan the advertised verb.
	req := restartRequest()
	req.CapabilityName = "reboot"
	plan, err := env.service.Plan(context.Background(), "default", req, testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan with lifecycle synonym: %v", err)
	}
	record, ok, err := env.store.GetActionAudit(plan.ActionID)
	if err != nil || !ok {
		t.Fatalf("GetActionAudit: ok=%v err=%v", ok, err)
	}
	if record.Request.CapabilityName != "restart" {
		t.Fatalf("persisted capability = %q, want the advertised verb %q", record.Request.CapabilityName, "restart")
	}

	replay, err := env.service.Plan(context.Background(), "default", req, testActionActor("requester", "default"))
	if err != nil || replay.ActionID != plan.ActionID {
		t.Fatalf("synonym replay changed plan: %#v %v", replay, err)
	}

	// Verbs without an advertised synonym still fail closed.
	unknown := restartRequest()
	unknown.RequestID = "independent-unknown-capability"
	unknown.CapabilityName = "shutdown"
	var capErr *CapabilityNotFoundError
	if _, err := env.service.Plan(context.Background(), "default", unknown, testActionActor("requester", "default")); !errors.As(err, &capErr) {
		t.Fatalf("non-synonym verb error = %v, want CapabilityNotFoundError", err)
	}
}

func TestPlanFailsClosedOnUnknownResourceAndCapability(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))

	missing := restartRequest()
	missing.ResourceID = "vm:404"
	var notFound *ResourceNotFoundError
	if _, err := env.service.Plan(context.Background(), "default", missing, testActionActor("requester", "default")); !errors.As(err, &notFound) {
		t.Fatalf("unknown resource error = %v, want ResourceNotFoundError", err)
	}

	unknownCap := restartRequest()
	unknownCap.CapabilityName = "detonate"
	_, err := env.service.Plan(context.Background(), "default", unknownCap, testActionActor("requester", "default"))
	var capErr *CapabilityNotFoundError
	if !errors.As(err, &capErr) || !errors.Is(err, actionplanner.ErrCapabilityNotFound) {
		t.Fatalf("unknown capability error = %v, want CapabilityNotFoundError wrapping ErrCapabilityNotFound", err)
	}
	if capErr.ResourceID != "vm:42" || capErr.CapabilityName != "detonate" {
		t.Fatalf("capability error detail = %#v", capErr)
	}

	empty := restartRequest()
	empty.ResourceID = "  "
	var validation *actionplanner.ValidationError
	if _, err := env.service.Plan(context.Background(), "default", empty, testActionActor("requester", "default")); !errors.As(err, &validation) {
		t.Fatalf("empty resource id error = %v, want ValidationError", err)
	}

	audits, err := env.store.GetActionAudits("vm:42", time.Time{}, 10)
	if err != nil {
		t.Fatalf("GetActionAudits: %v", err)
	}
	if len(audits) != 0 {
		t.Fatalf("failed plans must not persist audits, got %d", len(audits))
	}
}

func TestPlanAvailabilityRefusalPersistsNothing(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
	env.executor.readiness = &unified.ResourceActionReadiness{
		Name:       "restart",
		Available:  false,
		ReasonCode: "agent_disconnected",
		Reason:     "no connected command agent",
	}

	_, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	var refused *AvailabilityRefusedError
	if !errors.As(err, &refused) {
		t.Fatalf("error = %v, want AvailabilityRefusedError", err)
	}
	if refused.Readiness.ReasonCode != "agent_disconnected" {
		t.Fatalf("readiness = %#v", refused.Readiness)
	}
	audits, err := env.store.GetActionAudits("vm:42", time.Time{}, 10)
	if err != nil {
		t.Fatalf("GetActionAudits: %v", err)
	}
	if len(audits) != 0 {
		t.Fatalf("refused availability must not persist audits, got %d", len(audits))
	}
}

func TestPlanAgentFeasibilityRefusalPersistsNothing(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
	env.executor.feasibility = &unified.ResourceActionReadiness{
		Name: "restart", Available: false, ReasonCode: "target_state_changed",
		Reason: "The target changed before approval.",
	}
	_, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	var refused *AvailabilityRefusedError
	if !errors.As(err, &refused) || refused.Readiness.ReasonCode != "target_state_changed" {
		t.Fatalf("error=%v refused=%#v", err, refused)
	}
	audits, queryErr := env.store.GetActionAudits("vm:42", time.Time{}, 10)
	if queryErr != nil || len(audits) != 0 {
		t.Fatalf("audits=%#v err=%v", audits, queryErr)
	}
}

func TestDecideApprovesPendingAction(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))

	plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}

	updated, err := env.service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{
		Actor:   "[email protected]",
		Method:  unified.MethodAPI,
		Outcome: unified.OutcomeApproved,
		Reason:  "confirmed outage",
	}))

	if err != nil {
		t.Fatalf("Decide: %v", err)
	}
	if updated.State != unified.ActionStateApproved {
		t.Fatalf("state = %q, want approved", updated.State)
	}
	if len(updated.Approvals) != 1 || updated.Approvals[0].Actor != "[email protected]" {
		t.Fatalf("approvals = %#v", updated.Approvals)
	}

	var notFound *ActionNotFoundError
	if _, err := env.service.Decide(context.Background(), "default", "act_missing", testActionDecision(t, env.service, "default", "act_missing", unified.ActionApprovalRecord{
		Actor: "[email protected]", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved,
	})); !errors.As(err, &notFound) {
		t.Fatalf("unknown action error = %v, want ActionNotFoundError", err)
	}
}

func TestApprovalRequiresCurrentReadinessButRejectionRemainsAvailable(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
	plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	env.executor.readiness = &unified.ResourceActionReadiness{
		Name:       "restart",
		Available:  false,
		ReasonCode: "agent_disconnected",
		Reason:     "Connect the command agent for web-42.",
	}

	readiness := env.service.AssessCurrentReadiness(context.Background(), "default", mustActionRecord(t, env.service, plan.ActionID))
	if readiness.Ready || readiness.Code != "agent_disconnected" || readiness.Message != "Connect the command agent for web-42." {
		t.Fatalf("readiness=%#v", readiness)
	}
	decision := testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Outcome: unified.OutcomeApproved})
	if _, err := env.service.Decide(context.Background(), "default", plan.ActionID, decision); !errors.Is(err, unified.ErrActionExecutionUnavailable) {
		t.Fatalf("approval error=%v", err)
	}
	current := mustActionRecord(t, env.service, plan.ActionID)
	if current.State != unified.ActionStatePending || len(current.Approvals) != 0 {
		t.Fatalf("refused approval mutated record: %#v", current)
	}

	rejection := testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Outcome: unified.OutcomeRejected})
	rejected, err := env.service.Decide(context.Background(), "default", plan.ActionID, rejection)
	if err != nil || rejected.State != unified.ActionStateRejected {
		t.Fatalf("rejected=%#v err=%v", rejected, err)
	}
}

func TestApprovalRechecksAgentFeasibilityBeforePersistingDecision(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
	plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	env.executor.feasibility = &unified.ResourceActionReadiness{
		Name: "restart", Available: false, ReasonCode: "target_state_changed",
		Reason: "The target changed after planning.",
	}
	decision := testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Outcome: unified.OutcomeApproved})
	if _, err := env.service.Decide(context.Background(), "default", plan.ActionID, decision); !errors.Is(err, unified.ErrActionExecutionUnavailable) {
		t.Fatalf("approval error=%v", err)
	}
	current := mustActionRecord(t, env.service, plan.ActionID)
	if current.State != unified.ActionStatePending || len(current.Approvals) != 0 {
		t.Fatalf("refused approval mutated record: %#v", current)
	}
}

func TestRefreshReplacesExpiredPlanAndPreservesTrustedOrigin(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
	origin := &unified.ActionOrigin{Surface: "operational_trust_attention", OperationalRecordID: "attention-1", EvidenceIDs: []string{"evidence-1"}}
	plan, err := env.service.PlanWithOptions(context.Background(), "default", restartRequest(), PlanOptions{Actor: testActionActor("requester", "default"), Origin: origin})
	if err != nil {
		t.Fatal(err)
	}
	readinessChecks := 0
	env.service.Now = func() time.Time {
		readinessChecks++
		if readinessChecks == 1 {
			return plan.ExpiresAt.Add(time.Second)
		}
		return time.Now().UTC()
	}
	replacement, err := env.service.Refresh(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"))
	if err != nil {
		t.Fatal(err)
	}
	if replacement.ID == plan.ActionID || replacement.Request.RequestID != "refresh:"+plan.ActionID {
		t.Fatalf("replacement=%#v", replacement)
	}
	if replacement.Origin == nil || replacement.Origin.Surface != origin.Surface || replacement.Origin.OperationalRecordID != origin.OperationalRecordID {
		t.Fatalf("origin=%#v", replacement.Origin)
	}
	previous := mustActionRecord(t, env.service, plan.ActionID)
	if previous.State != unified.ActionStateExpired {
		t.Fatalf("previous state=%q", previous.State)
	}

	replayed, err := env.service.Refresh(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"))
	if err != nil || replayed.ID != replacement.ID {
		t.Fatalf("replayed=%#v err=%v", replayed, err)
	}
}

func mustActionRecord(t *testing.T, service *Service, actionID string) unified.ActionAuditRecord {
	t.Helper()
	record, found, err := service.Get("default", actionID)
	if err != nil || !found {
		t.Fatalf("Get(%q): found=%v err=%v", actionID, found, err)
	}
	return record
}

func TestExecuteRunsApprovedActionToTerminalAudit(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))

	plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}
	if _, err := env.service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{
		Actor: "[email protected]", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved,
	})); err != nil {
		t.Fatalf("Decide: %v", err)
	}

	completed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("[email protected]", "default"), "approved restart")
	if err != nil {
		t.Fatalf("Execute: %v", err)
	}
	if completed.State != unified.ActionStateCompleted {
		t.Fatalf("state = %q, want completed", completed.State)
	}
	if env.executor.calls != 1 {
		t.Fatalf("executor calls = %d, want 1", env.executor.calls)
	}
	if len(env.completed) != 1 || env.completed[0].ID != plan.ActionID {
		t.Fatalf("completion publisher observed %#v", env.completed)
	}

	record, ok, err := env.store.GetActionAudit(plan.ActionID)
	if err != nil || !ok {
		t.Fatalf("GetActionAudit: ok=%v err=%v", ok, err)
	}
	if record.State != unified.ActionStateCompleted || record.Result == nil || !record.Result.Success {
		t.Fatalf("terminal audit = state %q result %#v", record.State, record.Result)
	}
}

func TestExecuteRefusesLostReadinessBeforeAdmission(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
	env.executor.readiness = &unified.ResourceActionReadiness{Name: "restart", Available: true}

	plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}
	env.executor.readiness = &unified.ResourceActionReadiness{
		Name:       "restart",
		Available:  false,
		ReasonCode: "command_agent_disconnected",
		Reason:     "command agent disconnected",
	}

	failed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("requester", "default"), "")
	var refused *AvailabilityRefusedError
	if !errors.As(err, &refused) || !errors.Is(err, unified.ErrActionExecutionUnavailable) {
		t.Fatalf("error = %v, want AvailabilityRefusedError wrapping ErrActionExecutionUnavailable", err)
	}
	if failed.State != unified.ActionStateFailed || failed.Result == nil ||
		failed.Result.ActionResultV2 == nil ||
		failed.Result.ActionResultV2.Execution.ReasonCode != "action_execution_unavailable" ||
		!strings.HasPrefix(failed.Result.ErrorMessage, "action_execution_unavailable: command agent disconnected") {
		t.Fatalf("failed audit = %#v", failed)
	}
	if env.executor.calls != 0 {
		t.Fatalf("executor calls = %d, want no dispatch", env.executor.calls)
	}
	if len(env.completed) != 1 || env.completed[0].State != unified.ActionStateFailed {
		t.Fatalf("completion publisher observed %#v", env.completed)
	}
	if _, found, getErr := env.store.GetActionDispatchAttempt(plan.ActionID); getErr != nil || found {
		t.Fatalf("dispatch attempt found=%v err=%v, want none", found, getErr)
	}
}

func TestExecuteRefusesUnapprovedActionWithoutDispatch(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))

	plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}
	if _, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("[email protected]", "default"), ""); !errors.Is(err, unified.ErrActionNotApproved) {
		t.Fatalf("error = %v, want ErrActionNotApproved", err)
	}
	if env.executor.calls != 0 {
		t.Fatalf("executor must not run for unapproved actions, calls = %d", env.executor.calls)
	}
}

func TestExecuteRefusesRemediationLockedResource(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalNone))

	plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}
	if err := env.store.SetResourceOperatorState(unified.ResourceOperatorState{
		CanonicalID:        "vm:42",
		NeverAutoRemediate: true,
	}); err != nil {
		t.Fatalf("SetResourceOperatorState: %v", err)
	}

	failed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), "")
	if !errors.Is(err, unified.ErrResourceRemediationLocked) {
		t.Fatalf("error = %v, want ErrResourceRemediationLocked", err)
	}
	if env.executor.calls != 0 {
		t.Fatalf("executor must not run on locked resources, calls = %d", env.executor.calls)
	}
	if failed.State != unified.ActionStateFailed {
		t.Fatalf("refusal must persist a terminal failed audit, state = %q", failed.State)
	}
	if len(env.completed) != 1 || env.completed[0].State != unified.ActionStateFailed {
		t.Fatalf("refusal must publish the failed record, got %#v", env.completed)
	}
}

// A rebuild moves the operator's rows to the successor ID before it publishes
// the registry generation that lists it, so an action planned against the old
// ID can be dispatched, and the planner can be asked about it, while the
// registry the lifecycle consults still lists the old ID. The lock the
// operator set on it has moved with its row; the gates must still find it.
func TestLockMovedBySuccessionStillRefusesTheIDThePreviousGenerationLists(t *testing.T) {
	locks := map[string]unified.ResourceOperatorState{
		"never auto-remediate": {NeverAutoRemediate: true},
		"retired":              {LifecycleState: unified.LifecycleStateRetired},
	}
	for lockName, lock := range locks {
		t.Run(lockName, func(t *testing.T) {
			now := time.Now().UTC()
			env := newServiceEnv(t, testResource(now, unified.ApprovalNone))

			plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
			if err != nil {
				t.Fatalf("Plan: %v", err)
			}
			state := lock
			state.CanonicalID = "vm:42"
			if err := env.store.SetResourceOperatorState(state); err != nil {
				t.Fatalf("SetResourceOperatorState: %v", err)
			}
			if err := env.store.(*unified.MemoryStore).ApplyCanonicalIDSuccessions([]unified.CanonicalIDSuccession{{OldCanonicalID: "vm:42", NewCanonicalID: "vm:42-successor"}}); err != nil {
				t.Fatalf("ApplyCanonicalIDSuccessions: %v", err)
			}
			if _, listed := env.registry.Get("vm:42"); !listed {
				t.Fatal("the registry no longer lists vm:42; the fixture does not model the window before the new generation publishes")
			}
			if row, found, err := env.store.GetResourceOperatorState("vm:42-successor"); err != nil || !found || !row.BlocksRemediation() {
				t.Fatalf("the successor does not hold the moved lock: found=%v err=%v state=%+v", found, err, row)
			}

			// The planner and the Patrol broker read the policy through the lifecycle.
			planned, found, err := env.service.ResourceOperatorState("default", "vm:42")
			if err != nil || !found || !planned.BlocksRemediation() {
				t.Fatalf("the policy read for vm:42 lost the lock: found=%v err=%v state=%+v", found, err, planned)
			}

			failed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), "")
			if !errors.Is(err, unified.ErrResourceRemediationLocked) {
				t.Fatalf("error = %v, want ErrResourceRemediationLocked", err)
			}
			if env.executor.calls != 0 {
				t.Fatalf("executor must not run on a locked resource whose row moved, calls = %d", env.executor.calls)
			}
			if failed.State != unified.ActionStateFailed {
				t.Fatalf("refusal must persist a terminal failed audit, state = %q", failed.State)
			}
		})
	}
}

func TestExecuteRefusesRetiredResourceWithoutDispatch(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalNone))

	plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}
	if err := env.store.SetResourceOperatorState(unified.ResourceOperatorState{
		CanonicalID:    "vm:42",
		LifecycleState: unified.LifecycleStateRetired,
	}); err != nil {
		t.Fatalf("SetResourceOperatorState: %v", err)
	}

	failed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), "")
	if !errors.Is(err, unified.ErrResourceRemediationLocked) {
		t.Fatalf("error = %v, want ErrResourceRemediationLocked", err)
	}
	if env.executor.calls != 0 {
		t.Fatalf("executor must not run for retired resources, calls = %d", env.executor.calls)
	}
	if failed.State != unified.ActionStateFailed {
		t.Fatalf("retirement refusal must persist a failed audit, state = %q", failed.State)
	}
}

// admissionOrderStore stamps the moment the audit store commits an execution
// admission, so a test can order it against a concurrent policy save.
type admissionOrderStore struct {
	unified.ResourceStore
	seq       *atomic.Int64
	admission atomic.Int64
}

func (s *admissionOrderStore) RecordActionExecutionAdmission(record unified.ActionAuditRecord, event unified.ActionLifecycleEvent, attempt unified.ActionDispatchAttempt) error {
	err := s.ResourceStore.RecordActionExecutionAdmission(record, event, attempt)
	if err == nil {
		s.admission.Store(s.seq.Add(1))
	}
	return err
}

// windowExecutor runs one hook inside the readiness phase: after the operator
// lock was read, before admission is committed.
type windowExecutor struct {
	*stubExecutor
	window func()
}

func (e *windowExecutor) CheckActionAvailable(ctx context.Context, req unified.ActionRequest, resource unified.Resource) unified.ResourceActionReadiness {
	if window := e.window; window != nil {
		e.window = nil
		window()
	}
	return e.stubExecutor.CheckActionAvailable(ctx, req, resource)
}

// dispatchHookExecutor runs one hook once the dispatch is durably admitted,
// immediately before the executor would act.
type dispatchHookExecutor struct {
	*stubExecutor
	onDispatch func()
}

func (e *dispatchHookExecutor) ExecuteAction(ctx context.Context, record unified.ActionAuditRecord) (*unified.ExecutionResult, error) {
	if hook := e.onDispatch; hook != nil {
		e.onDispatch = nil
		hook()
	}
	return e.stubExecutor.ExecuteAction(ctx, record)
}

func approveForExecution(t *testing.T, service *Service, actionID string) {
	t.Helper()
	if _, err := service.Decide(context.Background(), "default", actionID, testActionDecision(t, service, "default", actionID, unified.ActionApprovalRecord{
		Actor: "[email protected]", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved,
	})); err != nil {
		t.Fatalf("Decide: %v", err)
	}
}

func saveRemediationLock(store unified.ResourceStore) error {
	return store.SetResourceOperatorState(unified.ResourceOperatorState{CanonicalID: "vm:42", NeverAutoRemediate: true})
}

// A lock saved while a human-approved action is between its readiness check and
// its admission commit must not be missed: the save either lands before the
// readiness check (the dispatch is refused) or queues behind the admission (the
// dispatch is already committed). What it may never do is acknowledge to the
// operator before the admission commits and still let the dispatch through.
func runExecuteAdmissionLinearizesWithOperatorLockSave(t *testing.T, newStore func(t *testing.T) unified.ResourceStore) {
	now := time.Now().UTC()
	var seq atomic.Int64
	store := &admissionOrderStore{ResourceStore: newStore(t), seq: &seq}
	executor := &windowExecutor{stubExecutor: &stubExecutor{result: &unified.ExecutionResult{Success: true, Output: "restarted"}}}
	service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)

	// Both actions are approved up front: once the lock is saved, Decide itself
	// refuses to record an approval for the locked resource.
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}
	approveForExecution(t, service, plan.ActionID)
	second, err := service.Plan(context.Background(), "default", unified.ActionRequest{
		RequestID: "req-2", ResourceID: "vm:42", CapabilityName: "restart",
		Params: map[string]any{"mode": "graceful"}, Reason: "Second restart", RequestedBy: "agent:test",
	}, testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("second Plan: %v", err)
	}
	approveForExecution(t, service, second.ActionID)

	var ackSeq atomic.Int64
	acked := make(chan struct{})
	entered := make(chan struct{})
	saveErr := make(chan error, 1)
	executor.window = func() {
		// Deterministic half: a policy writer must be excluded while Execute sits
		// between the lock read and the admission commit.
		coordinator := service.admissionCoordinator()
		if coordinator.mu.TryLock() {
			coordinator.mu.Unlock()
			t.Error("a policy writer could enter between the readiness check and the admission commit: Execute is not holding the admission coordinator")
		}
		// Concurrent half: a real writer gets its chance to finish inside the
		// window. A lifecycle holding the coordinator keeps it queued, so the
		// bounded wait only elapses on the correct path.
		go func() {
			err := service.WithPolicyMutation(func() error {
				close(entered)
				return saveRemediationLock(store)
			})
			ackSeq.Store(seq.Add(1))
			saveErr <- err
			close(acked)
		}()
		select {
		case <-acked:
		case <-entered:
			waitFor(t, acked, "lock save to acknowledge")
		case <-time.After(100 * time.Millisecond):
		}
	}

	completed, execErr := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("[email protected]", "default"), "approved restart")
	waitFor(t, acked, "lock save to acknowledge after Execute")
	if err := <-saveErr; err != nil {
		t.Fatalf("lock save: %v", err)
	}

	if executor.calls > 0 && ackSeq.Load() < store.admission.Load() {
		t.Fatalf("operator lock save was acknowledged (seq %d) before the dispatch was admitted (seq %d), yet the action still reached the executor (calls=%d, state=%q, err=%v)",
			ackSeq.Load(), store.admission.Load(), executor.calls, completed.State, execErr)
	}
	if execErr != nil || completed.State != unified.ActionStateCompleted || executor.calls != 1 {
		t.Fatalf("admitted dispatch must complete once the save queues behind it: state=%q calls=%d err=%v", completed.State, executor.calls, execErr)
	}

	// The queued save is durable and refuses every later dispatch, including an
	// action that was approved before the lock existed.
	refused, err := service.Execute(context.Background(), "default", second.ActionID, testActionActor("[email protected]", "default"), "")
	if !errors.Is(err, unified.ErrResourceRemediationLocked) || refused.State != unified.ActionStateFailed || executor.calls != 1 {
		t.Fatalf("later dispatch after the lock save: state=%q calls=%d err=%v, want refused with ErrResourceRemediationLocked", refused.State, executor.calls, err)
	}
}

func waitFor(t *testing.T, done <-chan struct{}, what string) {
	t.Helper()
	select {
	case <-done:
	case <-time.After(10 * time.Second):
		t.Fatalf("timed out waiting for %s", what)
	}
}

func TestExecuteAdmissionLinearizesWithOperatorLockSaveMemoryStore(t *testing.T) {
	runExecuteAdmissionLinearizesWithOperatorLockSave(t, func(*testing.T) unified.ResourceStore { return unified.NewMemoryStore() })
}

func TestExecuteAdmissionLinearizesWithOperatorLockSaveSQLite(t *testing.T) {
	runExecuteAdmissionLinearizesWithOperatorLockSave(t, func(t *testing.T) unified.ResourceStore {
		store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
		if err != nil {
			t.Fatal(err)
		}
		t.Cleanup(func() { _ = store.Close() })
		return store
	})
}

// Admission is the linearization point: a lock saved after the dispatch is
// durably admitted never recalls it, and the save must not wait for the
// dispatch, so Execute may not hold the admission coordinator across the
// executor call.
func TestExecuteAdmittedDispatchIsNotRecalledByLaterOperatorLock(t *testing.T) {
	now := time.Now().UTC()
	store := unified.NewMemoryStore()
	executor := &dispatchHookExecutor{stubExecutor: &stubExecutor{result: &unified.ExecutionResult{Success: true, Output: "restarted"}}}
	service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)

	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}
	approveForExecution(t, service, plan.ActionID)

	saved := make(chan error, 1)
	executor.onDispatch = func() {
		go func() {
			saved <- service.WithPolicyMutation(func() error { return saveRemediationLock(store) })
		}()
		select {
		case err := <-saved:
			if err != nil {
				t.Errorf("lock save during dispatch: %v", err)
			}
		case <-time.After(5 * time.Second):
			t.Error("lock save blocked behind an admitted dispatch: Execute must release the admission coordinator before calling the executor")
		}
	}

	completed, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("[email protected]", "default"), "approved restart")
	if err != nil || completed.State != unified.ActionStateCompleted || executor.calls != 1 {
		t.Fatalf("admitted dispatch: state=%q calls=%d err=%v, want completed exactly once", completed.State, executor.calls, err)
	}
	if state, found, getErr := store.GetResourceOperatorState("vm:42"); getErr != nil || !found || !state.BlocksRemediation() {
		t.Fatalf("lock save during dispatch must persist: found=%v state=%#v err=%v", found, state, getErr)
	}
}

// The no-recall boundary also covers an attempt that was admitted but never
// sent (a crash between admission and send): restart recovery drives the
// committed authority and does not re-read the operator lock. Recalling known
// unsent work would need an atomic pre-send terminalization the stores do not
// have, so this pins the contract instead of leaving it implicit.
func TestRecoverExecutingActionsDoesNotRecallQueuedAttemptAfterOperatorLock(t *testing.T) {
	now := time.Now().UTC()
	store := unified.NewMemoryStore()
	executor := &stubExecutor{result: &unified.ExecutionResult{Success: true, Output: "restarted"}}
	service := serviceForStore(t, store, testResource(now, unified.ApprovalNone), executor)

	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}
	record, found, err := store.GetActionAudit(plan.ActionID)
	if err != nil || !found {
		t.Fatalf("GetActionAudit: found=%v err=%v", found, err)
	}
	started, event, err := unified.BeginActionExecution(record, "operator", now)
	if err != nil {
		t.Fatal(err)
	}
	attempt, err := unified.NewActionDispatchAttempt(started.ID, now)
	if err != nil {
		t.Fatal(err)
	}
	if err := store.RecordActionExecutionAdmission(started, event, attempt); err != nil {
		t.Fatal(err)
	}
	if queued, ok, getErr := store.GetActionDispatchAttempt(started.ID); getErr != nil || !ok || queued.State != unified.ActionDispatchQueued {
		t.Fatalf("setup attempt = %#v found=%v err=%v, want queued", queued, ok, getErr)
	}

	if err := service.WithPolicyMutation(func() error { return saveRemediationLock(store) }); err != nil {
		t.Fatalf("lock save: %v", err)
	}
	recovered, err := service.RecoverExecutingActions(context.Background(), "default", "system:recovery", 10)
	if err != nil || len(recovered) != 1 || recovered[0].State != unified.ActionStateCompleted || executor.calls != 1 {
		t.Fatalf("recovered=%#v calls=%d err=%v, want the admitted attempt dispatched once", recovered, executor.calls, err)
	}
}

func TestExecuteRefusesDriftedPlan(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalNone))

	plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}

	// The capability policy tightens between plan and dispatch: the
	// replanned contract no longer matches the recorded plan hash.
	drifted := testResource(now, unified.ApprovalAdmin)
	env.registry = unified.NewRegistry(env.store)
	env.registry.IngestResources([]unified.Resource{drifted})

	failed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), "")
	if !errors.Is(err, unified.ErrActionPlanDrift) {
		t.Fatalf("error = %v, want ErrActionPlanDrift", err)
	}
	if env.executor.calls != 0 {
		t.Fatalf("executor must not run on drifted plans, calls = %d", env.executor.calls)
	}
	if failed.State != unified.ActionStateFailed {
		t.Fatalf("drift refusal must persist a terminal failed audit, state = %q", failed.State)
	}
}

func TestExecuteFailsClosedWithoutExecutor(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalNone))

	plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}
	env.service.Executor = nil
	if _, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), ""); !errors.Is(err, ErrExecutorUnavailable) {
		t.Fatalf("error = %v, want ErrExecutorUnavailable", err)
	}
}

func TestLifecycleFailsClosedWithoutStore(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
	env.service.Store = func(string) (Store, error) { return nil, errors.New("db offline") }

	if _, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default")); !errors.Is(err, ErrStoreUnavailable) {
		t.Fatalf("Plan error = %v, want ErrStoreUnavailable", err)
	}
	if _, err := env.service.Decide(context.Background(), "default", "act_x", testActionDecision(t, env.service, "default", "act_x", unified.ActionApprovalRecord{Outcome: unified.OutcomeApproved})); !errors.Is(err, ErrStoreUnavailable) {
		t.Fatalf("Decide error = %v, want ErrStoreUnavailable", err)
	}
	if _, err := env.service.Execute(context.Background(), "default", "act_x", testActionActor("agent:test", "default"), ""); !errors.Is(err, ErrStoreUnavailable) {
		t.Fatalf("Execute error = %v, want ErrStoreUnavailable", err)
	}
}

func TestCapabilitiesUsesSameRegistryResolutionAsPlanning(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))

	capabilities, err := env.service.Capabilities(context.Background(), "default", "vm:42")
	if err != nil {
		t.Fatalf("Capabilities: %v", err)
	}
	if len(capabilities) != 1 || capabilities[0].Name != "restart" {
		t.Fatalf("capabilities = %#v", capabilities)
	}

	var notFound *ResourceNotFoundError
	if _, err := env.service.Capabilities(context.Background(), "default", "vm:404"); !errors.As(err, &notFound) {
		t.Fatalf("unknown resource error = %v, want ResourceNotFoundError", err)
	}
}

func TestPlanWithOptionsPersistsOriginAcrossLifecycle(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))

	origin := &unified.ActionOrigin{
		Surface:         "patrol",
		FindingID:       "finding-9",
		InvestigationID: "inv-9",
		ProposalID:      "prop-9",
	}
	plan, err := env.service.PlanWithOptions(context.Background(), "default", restartRequest(), PlanOptions{Actor: testActionActor("requester", "default"), Origin: origin})
	if err != nil {
		t.Fatalf("PlanWithOptions: %v", err)
	}

	record, ok, err := env.store.GetActionAudit(plan.ActionID)
	if err != nil || !ok {
		t.Fatalf("GetActionAudit: ok=%v err=%v", ok, err)
	}
	if record.Origin == nil || record.Origin.FindingID != "finding-9" {
		t.Fatalf("plan audit origin = %#v", record.Origin)
	}

	// Origin must survive the decision transition: the decision persists
	// the record loaded from the store, so the broker-owned metadata
	// stays reconcilable at approval time.
	updated, err := env.service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{
		Actor: "[email protected]", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved,
	}))

	if err != nil {
		t.Fatalf("Decide: %v", err)
	}
	if updated.Origin == nil || updated.Origin.ProposalID != "prop-9" {
		t.Fatalf("decision record origin = %#v", updated.Origin)
	}

	// Plain Plan must never stamp an origin.
	plain := restartRequest()
	plain.RequestID = "req-plain"
	plainPlan, err := env.service.Plan(context.Background(), "default", plain, testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}
	plainRecord, ok, err := env.store.GetActionAudit(plainPlan.ActionID)
	if err != nil || !ok {
		t.Fatalf("GetActionAudit(plain): ok=%v err=%v", ok, err)
	}
	if plainRecord.Origin != nil {
		t.Fatalf("plain plan must not carry an origin, got %#v", plainRecord.Origin)
	}
}

func TestOnActionTransitionFiresAfterEachPersistedState(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
	var transitions []unified.ActionState
	var transitionOrgs []string
	env.service.OnActionTransition = func(orgID string, record unified.ActionAuditRecord) {
		transitions = append(transitions, record.State)
		transitionOrgs = append(transitionOrgs, orgID)
	}

	plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}
	if _, err := env.service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{
		Actor: "[email protected]", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved,
	})); err != nil {
		t.Fatalf("Decide: %v", err)
	}
	if _, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("[email protected]", "default"), ""); err != nil {
		t.Fatalf("Execute: %v", err)
	}

	want := []unified.ActionState{unified.ActionStatePending, unified.ActionStateApproved, unified.ActionStateExecuting, unified.ActionStateCompleted}
	if len(transitions) != len(want) {
		t.Fatalf("transitions = %v, want %v", transitions, want)
	}
	for i := range want {
		if transitions[i] != want[i] {
			t.Fatalf("transitions = %v, want %v", transitions, want)
		}
		if transitionOrgs[i] != "default" {
			t.Fatalf("transition org = %q, want default", transitionOrgs[i])
		}
	}
}

func TestOnActionTransitionFiresForPersistedRefusals(t *testing.T) {
	now := time.Now().UTC()
	env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
	var transitions []unified.ActionState
	env.service.OnActionTransition = func(orgID string, record unified.ActionAuditRecord) {
		transitions = append(transitions, record.State)
	}

	plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatalf("Plan: %v", err)
	}
	if err := env.store.SetResourceOperatorState(unified.ResourceOperatorState{
		CanonicalID:        "vm:42",
		NeverAutoRemediate: true,
	}); err != nil {
		t.Fatalf("SetResourceOperatorState: %v", err)
	}
	if _, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), ""); !errors.Is(err, unified.ErrResourceRemediationLocked) {
		t.Fatalf("error = %v, want ErrResourceRemediationLocked", err)
	}

	want := []unified.ActionState{unified.ActionStatePlanned, unified.ActionStateFailed}
	if len(transitions) != len(want) || transitions[0] != want[0] || transitions[1] != want[1] {
		t.Fatalf("transitions = %v, want %v", transitions, want)
	}
}

func TestActionPolicyProvenanceParticipatesInFreshnessWithoutAuthorizingDispatch(t *testing.T) {
	now := time.Date(2026, 7, 11, 21, 0, 0, 0, time.UTC)
	env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
	factor := unified.ActionPolicyAuthorityFactor{
		Kind: unified.ActionPolicyAuthorityTenant, SourceID: "patrol-tenant-policy",
		Revision: "tenant-policy:sha256:0123456789abcdef01234567", Status: unified.ActionPolicyAuthorityConsulted,
		ReasonCodes: []unified.ActionPolicyReasonCode{unified.PolicyReasonTenantModeAssisted, unified.PolicyReasonTenantFullLocked},
	}
	plan, err := env.service.PlanWithOptions(context.Background(), "default", restartRequest(), PlanOptions{Actor: testActionActor("requester", "default"), PolicyFactors: []unified.ActionPolicyAuthorityFactor{factor}})
	if err != nil {
		t.Fatal(err)
	}
	record, found, err := env.service.Get("default", plan.ActionID)
	if err != nil || !found {
		t.Fatalf("Get: found=%v err=%v", found, err)
	}
	if err := env.service.ValidatePlanFresh("default", record); err != nil {
		t.Fatalf("captured descriptive snapshot should replan deterministically: %v", err)
	}
	record.Plan.PolicyDecision.Authorities[1].Revision = "tenant-policy:sha256:1123456789abcdef01234567"
	record.Plan.PolicyDecision.DecisionID = unified.ActionPolicyDecisionDigest(record.Plan.PolicyDecision)
	if err := env.service.ValidatePlanFresh("default", record); !errors.Is(err, unified.ErrActionPlanDrift) {
		t.Fatalf("provenance drift error=%v", err)
	}
	if _, _, _, err := unified.BeginPolicyActionExecution(record, unified.ActionApprovalRecord{Actor: "pulse_patrol_policy", Method: unified.MethodPolicy, Outcome: unified.OutcomeApproved, Timestamp: now}, unified.ActionPolicyAuthorizationLease{}, now); !errors.Is(err, unified.ErrActionPolicyAuthorizationInvalid) {
		t.Fatalf("descriptive provenance authorized dispatch: %v", err)
	}
}

// Issue1649: an executing action whose agent never returns completion evidence
// used to have no exit at all. Expiry skips executing rows and only a terminal
// agent receipt could complete one, so a Docker update interrupted by an agent
// restart stayed "executing" forever with no operator route out. Reconciliation
// now self-heals the known stranded shapes; ForceFail is the manual override
// for anything left, and it must refuse to rewrite settled truth.
func TestIssue1649ForceFailTerminalizesWedgedExecutingActionWithAudit(t *testing.T) {
	store := unified.NewMemoryStore()
	executor := &reconcilingExecutor{executeErr: context.DeadlineExceeded, found: false}
	service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
	var completed []unified.ActionAuditRecord
	var transitions []unified.ActionAuditRecord
	service.OnActionCompleted = func(record unified.ActionAuditRecord) { completed = append(completed, record) }
	service.OnActionTransition = func(_ string, record unified.ActionAuditRecord) { transitions = append(transitions, record) }
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	if _, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "wedge it"); !errors.Is(err, context.DeadlineExceeded) {
		t.Fatalf("Execute error=%v", err)
	}
	if current, _, err := service.Get("default", plan.ActionID); err != nil || current.State != unified.ActionStateExecuting {
		t.Fatalf("precondition current=%#v err=%v", current, err)
	}

	failed, err := service.ForceFail("default", plan.ActionID, "operator", "verified by hand, container is running the new image")
	if err != nil || failed.State != unified.ActionStateFailed || failed.Result == nil || failed.Result.ActionResultV2 == nil {
		t.Fatalf("ForceFail failed=%#v err=%v", failed, err)
	}
	truth := failed.Result.ActionResultV2
	if truth.Execution.Status != unified.ActionExecutionInconclusive || truth.Execution.ReasonCode != ForceFailReasonCode {
		t.Fatalf("execution truth=%#v", truth.Execution)
	}
	if truth.Verification.Status != unified.ActionVerificationInconclusive || truth.Verification.EvidenceClass != unified.ActionEvidenceNone {
		t.Fatalf("verification truth=%#v", truth.Verification)
	}
	if !strings.Contains(failed.Result.ErrorMessage, "verified by hand") {
		t.Fatalf("operator reason missing from audit result: %q", failed.Result.ErrorMessage)
	}
	if executor.executeCalls != 1 {
		t.Fatalf("ForceFail touched the transport: execute=%d", executor.executeCalls)
	}
	persisted, found, err := store.GetActionAudit(plan.ActionID)
	if err != nil || !found || persisted.State != unified.ActionStateFailed {
		t.Fatalf("persisted=%#v found=%v err=%v", persisted, found, err)
	}
	events, err := store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 100)
	if err != nil || len(events) == 0 {
		t.Fatalf("events=%#v err=%v", events, err)
	}
	terminal := events[0]
	if terminal.State != unified.ActionStateFailed || terminal.Actor != "operator" || !strings.Contains(terminal.Message, "Operator force-failed") {
		t.Fatalf("terminal audit event=%#v", terminal)
	}
	if len(completed) != 1 || completed[0].State != unified.ActionStateFailed {
		t.Fatalf("completed=%#v", completed)
	}
	if len(transitions) == 0 || transitions[len(transitions)-1].State != unified.ActionStateFailed {
		t.Fatalf("transitions=%#v", transitions)
	}
	// The row is settled, so the recovery loop stops reconciling it and the
	// override cannot be replayed over the persisted outcome.
	recovered, err := service.RecoverExecutingActions(context.Background(), "default", "system:restart-recovery", 100)
	if err != nil || len(recovered) != 0 {
		t.Fatalf("recovered=%#v err=%v", recovered, err)
	}
	if _, err := service.ForceFail("default", plan.ActionID, "operator", "again"); !errors.Is(err, unified.ErrActionExecutionFinal) {
		t.Fatalf("replayed ForceFail error=%v", err)
	}
}

func TestIssue1649ForceFailRefusesTerminalAndUndispatchedActions(t *testing.T) {
	store := unified.NewMemoryStore()
	service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), &stubExecutor{result: &unified.ExecutionResult{Success: true, Output: "restarted"}})
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	// Planned but never dispatched: expiry still owns this row, so the
	// override must not manufacture an unknown-effect failure for it.
	if _, err := service.ForceFail("default", plan.ActionID, "operator", ""); !errors.Is(err, unified.ErrActionNotExecuting) {
		t.Fatalf("undispatched ForceFail error=%v", err)
	}
	current, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "run it")
	if err != nil || current.State != unified.ActionStateCompleted {
		t.Fatalf("current=%#v err=%v", current, err)
	}
	if _, err := service.ForceFail("default", plan.ActionID, "operator", ""); !errors.Is(err, unified.ErrActionExecutionFinal) {
		t.Fatalf("completed ForceFail error=%v", err)
	}
	settled, found, err := store.GetActionAudit(plan.ActionID)
	if err != nil || !found || settled.State != unified.ActionStateCompleted || !settled.Result.Success {
		t.Fatalf("refused override rewrote settled truth: %#v found=%v err=%v", settled, found, err)
	}
	var notFound *ActionNotFoundError
	if _, err := service.ForceFail("default", "act_missing", "operator", ""); !errors.As(err, &notFound) {
		t.Fatalf("missing action ForceFail error=%v", err)
	}
}

// A refresh's replacement names the action it replaced, which is how a
// surface that projects actions tells the replaced plan's later transitions
// from its replacement's.
func TestRefreshedActionIDNamesTheReplacedAction(t *testing.T) {
	now := time.Date(2026, 10, 23, 18, 0, 0, 0, time.UTC)
	store := unified.NewMemoryStore()
	service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), &stubExecutor{})
	service.Now = func() time.Time { return now }
	plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
	if err != nil {
		t.Fatal(err)
	}
	service.Now = func() time.Time { return plan.ExpiresAt.Add(time.Second) }
	replacement, err := service.Refresh(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"))
	if err != nil {
		t.Fatal(err)
	}
	if replaced, ok := RefreshedActionID(replacement); !ok || replaced != plan.ActionID {
		t.Fatalf("RefreshedActionID(replacement) = %q, %v, want %q", replaced, ok, plan.ActionID)
	}
	original, found, err := store.GetActionAudit(plan.ActionID)
	if err != nil || !found {
		t.Fatalf("original: found=%v err=%v", found, err)
	}
	if replaced, ok := RefreshedActionID(original); ok {
		t.Fatalf("a first plan names a replaced action: %q", replaced)
	}
}

// The inbox and decision-queue sweeps publish each plan they expire, as a
// single action's expiry is published, so a surface that projects the action,
// such as a Patrol finding's queued fix, does not outlive the plan. Each expiry
// is published once.
func TestExpirySweepsPublishEachExpiredPlan(t *testing.T) {
	sweeps := map[string]func(*Service) error{
		"inbox": func(s *Service) error {
			_, err := s.List("default", ActionListPending, 100)
			return err
		},
		"decision queue": func(s *Service) error {
			_, err := s.DecisionQueue("default", 100)
			return err
		},
	}
	for name, sweep := range sweeps {
		t.Run(name, func(t *testing.T) {
			now := time.Date(2026, 10, 23, 18, 0, 0, 0, time.UTC)
			store := unified.NewMemoryStore()
			service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), &stubExecutor{})
			service.Now = func() time.Time { return now }
			plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
			if err != nil {
				t.Fatal(err)
			}
			var published []unified.ActionAuditRecord
			var orgs []string
			service.OnActionTransition = func(orgID string, record unified.ActionAuditRecord) {
				published = append(published, record)
				orgs = append(orgs, orgID)
			}

			if err := sweep(service); err != nil || len(published) != 0 {
				t.Fatalf("a sweep before expiry: err=%v published=%+v", err, published)
			}
			service.Now = func() time.Time { return plan.ExpiresAt.Add(time.Second) }
			if err := sweep(service); err != nil {
				t.Fatal(err)
			}
			if len(published) != 1 || published[0].ID != plan.ActionID || published[0].State != unified.ActionStateExpired || orgs[0] != "default" {
				t.Fatalf("published = %+v for orgs %v, want one expiry of %s", published, orgs, plan.ActionID)
			}
			if err := sweep(service); err != nil || len(published) != 1 {
				t.Fatalf("a later sweep published the expiry again: err=%v published=%+v", err, published)
			}
		})
	}
}