Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package actionlifecycle
import (
"context"
"errors"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/actionplanner"
unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
type delayedCreateStore struct {
unified.ResourceStore
calls atomic.Int32
secondArrived chan struct{}
releaseSecond chan struct{}
}
// Force a cold lookup to exercise the atomic creation boundary even when a
// competing request has persisted between the lookup and insert.
func (s *delayedCreateStore) GetActionAuditByRequest(unified.ActionRequest, *unified.ActionOrigin) (unified.ActionAuditRecord, bool, error) {
return unified.ActionAuditRecord{}, false, nil
}
func (s *delayedCreateStore) CreateActionAudit(record unified.ActionAuditRecord, events []unified.ActionLifecycleEvent) (unified.ActionAuditRecord, bool, error) {
if s.calls.Add(1) == 2 {
close(s.secondArrived)
<-s.releaseSecond
}
return s.ResourceStore.CreateActionAudit(record, events)
}
type blockingExecutor struct {
calls atomic.Int32
entered chan struct{}
release chan struct{}
}
type claimBoundaryStore struct {
unified.ResourceStore
entered chan struct{}
release chan struct{}
once atomic.Bool
}
func (s *claimBoundaryStore) ClaimActionDispatch(actionID, owner string, now time.Time, lease time.Duration) (unified.ActionDispatchAttempt, bool, error) {
attempt, claimed, err := s.ResourceStore.ClaimActionDispatch(actionID, owner, now, lease)
if claimed && s.once.CompareAndSwap(false, true) {
close(s.entered)
<-s.release
}
return attempt, claimed, err
}
func (e *blockingExecutor) ExecuteAction(_ context.Context, record unified.ActionAuditRecord) (*unified.ExecutionResult, error) {
if e.calls.Add(1) == 1 {
close(e.entered)
}
<-e.release
return &unified.ExecutionResult{Success: true, Output: record.ID}, nil
}
func serviceForStore(t *testing.T, store unified.ResourceStore, resource unified.Resource, executor Executor) *Service {
t.Helper()
registry := unified.NewRegistry(store)
registry.IngestResources([]unified.Resource{resource})
return &Service{
Registry: func(string) (*unified.ResourceRegistry, error) { return registry, nil },
Store: func(string) (Store, error) { return store, nil },
Executor: executor,
DecisionAuthorizer: DecisionAuthorizerFunc(func(context.Context, string, unified.ActionAuditRecord, unified.ActionDecision) error { return nil }),
ExecutionAuthorizer: ExecutionAuthorizerFunc(func(context.Context, string, unified.ActionAuditRecord, unified.ActionActor) error { return nil }),
}
}
func TestWithPolicyMutationConcurrentFirstUseSharesCoordinator(t *testing.T) {
const writers = 64
service := &Service{}
start := make(chan struct{})
coordinators := make(chan *PolicyAdmissionCoordinator, writers)
var ready sync.WaitGroup
var done sync.WaitGroup
ready.Add(writers)
done.Add(writers)
for range writers {
go func() {
defer done.Done()
ready.Done()
<-start
coordinators <- service.admissionCoordinator()
}()
}
ready.Wait()
close(start)
done.Wait()
close(coordinators)
var shared *PolicyAdmissionCoordinator
for coordinator := range coordinators {
if shared == nil {
shared = coordinator
continue
}
if coordinator != shared {
t.Fatal("concurrent first use returned more than one policy admission coordinator")
}
}
if shared == nil {
t.Fatal("concurrent first use did not initialize a policy admission coordinator")
}
}
func testActionActor(subject, orgID string) unified.ActionActor {
if strings.TrimSpace(subject) == "" {
subject = "operator"
}
return unified.ActionActor{SubjectID: subject, Kind: unified.ActionActorUser, CredentialID: "session:test", OrgID: orgID}
}
func testActionDecision(t *testing.T, service *Service, orgID, actionID string, approval unified.ActionApprovalRecord) unified.ActionDecision {
t.Helper()
record, found, _ := service.Get(orgID, actionID)
planHash := "sha256:test"
if found {
planHash = record.Plan.PlanHash
}
actor := testActionActor(approval.Actor, orgID)
return unified.ActionDecision{
Actor: actor,
Outcome: approval.Outcome,
Reason: approval.Reason,
Evidence: unified.ApprovalEvidence{
Version: 1, Method: unified.MethodSession, Actor: actor, OrgID: orgID,
ActionID: actionID, PlanHash: planHash, Outcome: approval.Outcome, IssuedAt: time.Now().UTC(),
},
}
}
func testBoundLifecycleApproval(record unified.ActionAuditRecord, subject string, method unified.ApprovalMethod, outcome unified.ApprovalOutcome, at time.Time) unified.ActionApprovalRecord {
kind := unified.ActionActorUser
credential := "session:test"
if method == unified.MethodPolicy {
kind = unified.ActionActorPolicy
credential = "policy:test"
}
actor := unified.ActionActor{SubjectID: subject, Kind: kind, CredentialID: credential, OrgID: "default"}
evidence := unified.ApprovalEvidence{Version: 1, Method: method, Actor: actor, OrgID: "default", ActionID: record.ID, PlanHash: record.Plan.PlanHash, Outcome: outcome, IssuedAt: at}
return unified.ActionApprovalRecord{Actor: subject, ActorBinding: actor, Method: method, Timestamp: at, Outcome: outcome, Evidence: &evidence}
}
func runConcurrentPlanReplayCannotRewindTerminalAction(t *testing.T, store unified.ResourceStore) {
t.Helper()
delayed := &delayedCreateStore{ResourceStore: store, secondArrived: make(chan struct{}), releaseSecond: make(chan struct{})}
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
service := serviceForStore(t, delayed, testResource(time.Now().UTC(), unified.ApprovalAdmin), executor)
firstPlan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
secondResult := make(chan error, 1)
go func() {
_, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
secondResult <- err
}()
<-delayed.secondArrived
if _, err := service.Decide(context.Background(), "default", firstPlan.ActionID, testActionDecision(t, service, "default", firstPlan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved})); err != nil {
t.Fatal(err)
}
if _, err := service.Execute(context.Background(), "default", firstPlan.ActionID, testActionActor("operator", "default"), "proof"); err != nil {
t.Fatal(err)
}
close(delayed.releaseSecond)
if err := <-secondResult; err != nil {
t.Fatal(err)
}
current, found, err := store.GetActionAudit(firstPlan.ActionID)
if err != nil || !found || current.State != unified.ActionStateCompleted {
t.Fatalf("current=%#v found=%v err=%v", current, found, err)
}
if executor.calls != 1 {
t.Fatalf("executor calls=%d, want 1", executor.calls)
}
}
func TestConcurrentPlanReplayCannotRewindTerminalActionMemoryStore(t *testing.T) {
runConcurrentPlanReplayCannotRewindTerminalAction(t, unified.NewMemoryStore())
}
func TestConcurrentPlanReplayCannotRewindTerminalActionSQLiteStore(t *testing.T) {
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatal(err)
}
defer store.Close()
runConcurrentPlanReplayCannotRewindTerminalAction(t, store)
}
func runConcurrentExecuteAdmitsExecutorExactlyOnce(t *testing.T, store unified.ResourceStore) {
t.Helper()
executor := &blockingExecutor{entered: make(chan struct{}), release: make(chan struct{})}
service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
first := make(chan error, 1)
go func() {
_, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "first")
first <- err
}()
<-executor.entered
secondRecord, secondErr := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "second")
if secondErr != nil {
t.Fatalf("duplicate Execute: %v", secondErr)
}
if secondRecord.State != unified.ActionStateExecuting {
t.Fatalf("duplicate state=%q", secondRecord.State)
}
close(executor.release)
if err := <-first; err != nil {
t.Fatal(err)
}
if executor.calls.Load() != 1 {
t.Fatalf("executor calls=%d, want 1", executor.calls.Load())
}
}
func TestConcurrentExecuteAdmitsExecutorExactlyOnceMemoryStore(t *testing.T) {
runConcurrentExecuteAdmitsExecutorExactlyOnce(t, unified.NewMemoryStore())
}
func TestConcurrentExecuteAdmitsExecutorExactlyOnceSQLiteStore(t *testing.T) {
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatal(err)
}
defer store.Close()
runConcurrentExecuteAdmitsExecutorExactlyOnce(t, store)
}
func runConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t *testing.T, store unified.ResourceStore) {
t.Helper()
blocked := &claimBoundaryStore{ResourceStore: store, entered: make(chan struct{}), release: make(chan struct{})}
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
service := serviceForStore(t, blocked, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
first := make(chan error, 1)
go func() {
_, executeErr := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "first")
first <- executeErr
}()
<-blocked.entered
second, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "duplicate")
if err != nil || second.State != unified.ActionStateExecuting {
t.Fatalf("duplicate=%#v err=%v", second, err)
}
close(blocked.release)
if err := <-first; err != nil {
t.Fatal(err)
}
if executor.calls != 1 {
t.Fatalf("executor calls=%d, want 1", executor.calls)
}
}
func TestMemoryStoreConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t *testing.T) {
runConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t, unified.NewMemoryStore())
}
func TestSQLiteConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t *testing.T) {
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatal(err)
}
defer store.Close()
runConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t, store)
}
func TestPlanReplayReturnsAuthoritativeApprovedExecutingAndTerminalRecords(t *testing.T) {
states := []unified.ActionState{unified.ActionStateApproved, unified.ActionStateExecuting, unified.ActionStateRejected, unified.ActionStateCompleted, unified.ActionStateFailed}
for _, state := range states {
t.Run(string(state), func(t *testing.T) {
store := unified.NewMemoryStore()
service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalAdmin), &stubExecutor{})
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
record, _, _ := store.GetActionAudit(plan.ActionID)
decisionOutcome := unified.OutcomeApproved
if state == unified.ActionStateRejected {
decisionOutcome = unified.OutcomeRejected
}
decisionAt := time.Now().UTC()
decided, decisionEvent, err := unified.ApplyActionDecision(record, testBoundLifecycleApproval(record, "operator", unified.MethodSession, decisionOutcome, decisionAt), decisionAt)
if err != nil || store.RecordActionDecision(decided, decisionEvent) != nil {
t.Fatalf("decision: %v", err)
}
if state != unified.ActionStateApproved && state != unified.ActionStateRejected {
started, startEvent, err := unified.BeginActionExecution(decided, "operator", time.Now().UTC())
if err != nil || store.RecordActionExecutionStart(started, startEvent) != nil {
t.Fatalf("start: %v", err)
}
if state == unified.ActionStateCompleted || state == unified.ActionStateFailed {
result := &unified.ExecutionResult{Success: state == unified.ActionStateCompleted}
completed, doneEvent, err := unified.CompleteActionExecution(started, result, "operator", time.Now().UTC())
if err != nil || store.RecordActionExecutionResult(completed, doneEvent) != nil {
t.Fatalf("complete: %v", err)
}
}
}
returned, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
if returned.ActionID != plan.ActionID {
t.Fatalf("action id=%q", returned.ActionID)
}
current, _, _ := store.GetActionAudit(plan.ActionID)
if current.State != state {
t.Fatalf("state=%q want=%q", current.State, state)
}
})
}
}
func TestPlanReplayRejectsConflictingOriginForDeterministicActionID(t *testing.T) {
store := unified.NewMemoryStore()
service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalAdmin), &stubExecutor{})
first := PlanOptions{Actor: testActionActor("requester", "default"), Origin: &unified.ActionOrigin{Surface: "patrol", FindingID: "finding-1", InvestigationID: "inv-1", ProposalID: "proposal-1"}}
if _, err := service.PlanWithOptions(context.Background(), "default", restartRequest(), first); err != nil {
t.Fatal(err)
}
conflict := first
conflict.Origin = &unified.ActionOrigin{Surface: "patrol", FindingID: "finding-2", InvestigationID: "inv-2", ProposalID: "proposal-1"}
if _, err := service.PlanWithOptions(context.Background(), "default", restartRequest(), conflict); !errors.Is(err, unified.ErrActionIdentityConflict) {
t.Fatalf("error=%v", err)
}
}
func TestExecutePublishesPersistedExecutingTransition(t *testing.T) {
store := unified.NewMemoryStore()
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
var states []unified.ActionState
service.OnActionTransition = func(_ string, record unified.ActionAuditRecord) { states = append(states, record.State) }
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
if _, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), ""); err != nil {
t.Fatal(err)
}
want := []unified.ActionState{unified.ActionStatePlanned, unified.ActionStateExecuting, unified.ActionStateCompleted}
if len(states) != len(want) {
t.Fatalf("states=%v", states)
}
for i := range want {
if states[i] != want[i] {
t.Fatalf("states=%v want=%v", states, want)
}
}
}
func TestExecuteAfterSQLiteRestartDoesNotReadmitExecutingAction(t *testing.T) {
dir := t.TempDir()
first, err := unified.NewSQLiteResourceStore(dir, "default")
if err != nil {
t.Fatal(err)
}
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
service := serviceForStore(t, first, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
record, _, _ := first.GetActionAudit(plan.ActionID)
started, event, _ := unified.BeginActionExecution(record, "operator", time.Now().UTC())
if err := first.RecordActionExecutionStart(started, event); err != nil {
t.Fatal(err)
}
first.Close()
second, err := unified.NewSQLiteResourceStore(dir, "default")
if err != nil {
t.Fatal(err)
}
defer second.Close()
service = serviceForStore(t, second, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
current, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "retry")
if err != nil || current.State != unified.ActionStateExecuting || executor.calls != 0 {
t.Fatalf("current=%#v err=%v calls=%d", current, err, executor.calls)
}
}
func TestExecuteTimeoutAfterSendWaitsForCorrelatedLateResponseWithoutResend(t *testing.T) {
store := unified.NewMemoryStore()
executor := &reconcilingExecutor{executeErr: context.DeadlineExceeded, result: &unified.ExecutionResult{Success: true}}
service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
current, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "timeout proof")
if !errors.Is(err, context.DeadlineExceeded) || current.State != unified.ActionStateExecuting {
t.Fatalf("current=%#v err=%v", current, err)
}
attempt, found, err := store.GetActionDispatchAttempt(plan.ActionID)
if err != nil || !found || attempt.State != unified.ActionDispatchReceiptPending || attempt.DispatchCount != 1 {
t.Fatalf("attempt=%#v found=%v err=%v", attempt, found, err)
}
if _, found, err := store.GetActionDispatchReceipt(attempt.ID); err != nil || found {
t.Fatalf("receipt found=%v err=%v", found, err)
}
// Duplicate execute is a query-only resume and cannot call ExecuteAction.
current, err = service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "duplicate resume")
if err != nil || current.State != unified.ActionStateExecuting || executor.executeCalls != 1 || executor.reconcileCalls != 1 {
t.Fatalf("current=%#v err=%v execute=%d reconcile=%d", current, err, executor.executeCalls, executor.reconcileCalls)
}
executor.found = true
current, err = service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "late response")
if err != nil || current.State != unified.ActionStateCompleted || executor.executeCalls != 1 || executor.reconcileCalls != 2 {
t.Fatalf("current=%#v err=%v execute=%d reconcile=%d", current, err, executor.executeCalls, executor.reconcileCalls)
}
if receipt, found, err := store.GetActionDispatchReceipt(attempt.ID); err != nil || !found || receipt.AttemptID != attempt.ID {
t.Fatalf("receipt=%#v found=%v err=%v", receipt, found, err)
}
}
func TestSQLiteRestartRecoveryReconcilesReceiptPendingWithoutResend(t *testing.T) {
dir := t.TempDir()
now := time.Now().UTC()
first, err := unified.NewSQLiteResourceStore(dir, "default")
if err != nil {
t.Fatal(err)
}
executor := &reconcilingExecutor{executeErr: context.DeadlineExceeded, result: &unified.ExecutionResult{Success: true}}
service := serviceForStore(t, first, testResource(now, unified.ApprovalNone), executor)
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
if _, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "before restart"); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("Execute error=%v", err)
}
if err := first.Close(); err != nil {
t.Fatal(err)
}
second, err := unified.NewSQLiteResourceStore(dir, "default")
if err != nil {
t.Fatal(err)
}
defer second.Close()
executor.found = true
service = serviceForStore(t, second, testResource(now, unified.ApprovalNone), executor)
recovered, err := service.RecoverExecutingActions(context.Background(), "default", "system:restart-recovery", 100)
if err != nil || len(recovered) != 1 || recovered[0].State != unified.ActionStateCompleted {
t.Fatalf("recovered=%#v err=%v", recovered, err)
}
if executor.executeCalls != 1 || executor.reconcileCalls != 1 {
t.Fatalf("execute=%d reconcile=%d", executor.executeCalls, executor.reconcileCalls)
}
current, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "duplicate after recovery")
if err != nil || current.State != unified.ActionStateCompleted || executor.executeCalls != 1 {
t.Fatalf("current=%#v err=%v execute=%d", current, err, executor.executeCalls)
}
}
func TestSQLiteRestartRecoveryNotFoundRemainsReceiptPendingWithoutResend(t *testing.T) {
dir := t.TempDir()
now := time.Now().UTC()
store, err := unified.NewSQLiteResourceStore(dir, "default")
if err != nil {
t.Fatal(err)
}
defer store.Close()
executor := &reconcilingExecutor{executeErr: context.DeadlineExceeded, found: false}
service := serviceForStore(t, store, testResource(now, unified.ApprovalNone), executor)
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
if _, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "initial"); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("execute err=%v", err)
}
for n := 0; n < 2; n++ {
recovered, err := service.RecoverExecutingActions(context.Background(), "default", "system:restart-recovery", 100)
if err != nil || len(recovered) != 1 || recovered[0].State != unified.ActionStateExecuting {
t.Fatalf("pass=%d recovered=%#v err=%v", n, recovered, err)
}
}
attempt, found, err := store.GetActionDispatchAttempt(plan.ActionID)
if err != nil || !found || attempt.State != unified.ActionDispatchReceiptPending {
t.Fatalf("attempt=%#v found=%v err=%v", attempt, found, err)
}
if _, found, err := store.GetActionDispatchReceipt(attempt.ID); err != nil || found {
t.Fatalf("receipt found=%v err=%v", found, err)
}
audit, found, err := store.GetActionAudit(plan.ActionID)
if err != nil || !found || audit.State != unified.ActionStateExecuting || audit.Result != nil {
t.Fatalf("audit=%#v found=%v err=%v", audit, found, err)
}
if executor.executeCalls != 1 || executor.reconcileCalls != 2 {
t.Fatalf("execute=%d reconcile=%d", executor.executeCalls, executor.reconcileCalls)
}
}
func TestActionDetailAndInboxMaterializeExplicitExpiry(t *testing.T) {
now := time.Date(2026, 7, 11, 14, 0, 0, 0, time.UTC)
store := unified.NewMemoryStore()
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), &stubExecutor{})
service.Now = func() time.Time { return now }
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
service.Now = func() time.Time { return plan.ExpiresAt.Add(time.Second) }
detail, found, err := service.Detail("default", plan.ActionID)
if err != nil || !found || detail.Audit.State != unified.ActionStateExpired {
t.Fatalf("detail=%#v found=%v err=%v", detail, found, err)
}
if len(detail.Events) == 0 || detail.Events[0].State != unified.ActionStateExpired {
t.Fatalf("events=%#v", detail.Events)
}
pending, err := service.List("default", ActionListPending, 100)
if err != nil || len(pending) != 0 {
t.Fatalf("pending=%#v err=%v", pending, err)
}
settled, err := service.List("default", ActionListSettled, 100)
if err != nil || len(settled) != 1 || settled[0].State != unified.ActionStateExpired {
t.Fatalf("settled=%#v err=%v", settled, err)
}
if _, err := service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Outcome: unified.OutcomeApproved})); !errors.Is(err, unified.ErrActionPlanExpired) {
t.Fatalf("Decide error=%v", err)
}
}
func policyTestLease(record unified.ActionAuditRecord, now time.Time) unified.ActionPolicyAuthorizationLease {
lease := unified.ActionPolicyAuthorizationLease{
Version: 1, OrgID: "default", ActionID: record.ID, ResourceID: record.Request.ResourceID,
CapabilityName: record.Request.CapabilityName, PlanHash: record.Plan.PlanHash,
CapabilityPolicyVersion: record.Plan.PolicyVersion, AutoAuthorization: unified.AutoAuthorizeLowRisk,
ApprovalPolicy: record.Plan.ApprovalPolicy, TenantPolicyVersion: "tenant:v1",
EffectiveAutonomyLevel: "assisted", LicenseAllowsAutoFix: true,
ResourcePolicyVersion: "resource:v1", CapabilityNames: []string{"restart"},
IssuedAt: now, ExpiresAt: now.Add(time.Minute),
}
lease.Digest = unified.ActionPolicyAuthorizationDigest(lease)
return lease
}
func runPolicyAdmissionCommitsAtomically(t *testing.T, store unified.ResourceStore) {
now := time.Now().UTC()
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
service.Now = func() time.Time { return now }
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
completed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
return policyTestLease(record, at), "bounded policy", nil
})
if err != nil || completed.State != unified.ActionStateCompleted || executor.calls != 1 {
t.Fatalf("completed=%#v err=%v calls=%d", completed, err, executor.calls)
}
if len(completed.Approvals) != 1 || completed.Approvals[0].Method != unified.MethodPolicy || completed.Approvals[0].PolicyLease == nil {
t.Fatalf("approvals=%#v", completed.Approvals)
}
events, err := store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 10)
if err != nil {
t.Fatal(err)
}
seenApproved, seenExecuting := false, false
for _, event := range events {
seenApproved = seenApproved || event.State == unified.ActionStateApproved
seenExecuting = seenExecuting || event.State == unified.ActionStateExecuting
}
if !seenApproved || !seenExecuting {
t.Fatalf("events=%#v", events)
}
}
func TestPolicyAdmissionCommitsApprovalAndExecutingAtomicallyMemoryStore(t *testing.T) {
runPolicyAdmissionCommitsAtomically(t, unified.NewMemoryStore())
}
func TestPolicyAdmissionCommitsApprovalAndExecutingAtomicallySQLite(t *testing.T) {
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatal(err)
}
defer store.Close()
runPolicyAdmissionCommitsAtomically(t, store)
}
func TestExecuteUnderPolicyRefusesLostReadinessBeforeAutomaticAdmission(t *testing.T) {
now := time.Now().UTC()
store := unified.NewMemoryStore()
executor := &stubExecutor{
result: &unified.ExecutionResult{Success: true},
readiness: &unified.ResourceActionReadiness{Name: "restart", Available: true},
}
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
service.Now = func() time.Time { return now }
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
executor.readiness = &unified.ResourceActionReadiness{
Name: "restart",
Available: false,
ReasonCode: "command_agent_disconnected",
Reason: "command agent disconnected",
}
failed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
return policyTestLease(record, at), "bounded policy", nil
})
if !errors.Is(err, unified.ErrActionExecutionUnavailable) || failed.State != unified.ActionStateFailed {
t.Fatalf("failed=%#v err=%v", failed, err)
}
if executor.calls != 0 || len(failed.Approvals) != 0 {
t.Fatalf("executor calls=%d approvals=%#v, want no dispatch or policy approval", executor.calls, failed.Approvals)
}
if _, found, getErr := store.GetActionDispatchAttempt(plan.ActionID); getErr != nil || found {
t.Fatalf("dispatch attempt found=%v err=%v, want none", found, getErr)
}
}
func runPolicyBarrierRevocations(t *testing.T, storeFactory func(t *testing.T) unified.ResourceStore) {
cases := []struct {
name string
mutate func(*unified.ActionPolicyAuthorizationLease) error
}{
{"policy_row_deleted", func(*unified.ActionPolicyAuthorizationLease) error {
return unified.ErrActionPolicyAuthorizationInvalid
}},
{"policy_disabled", func(*unified.ActionPolicyAuthorizationLease) error {
return unified.ErrActionPolicyAuthorizationRevoked
}},
{"capability_removed_from_allowlist", func(*unified.ActionPolicyAuthorizationLease) error {
return unified.ErrActionPolicyAuthorizationRevoked
}},
{"recurring_window_closed", func(*unified.ActionPolicyAuthorizationLease) error {
return unified.ErrActionPolicyAuthorizationRevoked
}},
{"mode_downgraded_to_approval", func(*unified.ActionPolicyAuthorizationLease) error {
return unified.ErrActionPolicyAuthorizationRevoked
}},
{"mode_downgraded_to_monitor", func(*unified.ActionPolicyAuthorizationLease) error {
return unified.ErrActionPolicyAuthorizationRevoked
}},
{"full_mode_unlock_removed", func(*unified.ActionPolicyAuthorizationLease) error {
return unified.ErrActionPolicyAuthorizationRevoked
}},
{"license_removes_effective_mode", func(lease *unified.ActionPolicyAuthorizationLease) error {
lease.LicenseAllowsAutoFix = false
lease.Digest = unified.ActionPolicyAuthorizationDigest(*lease)
return nil
}},
{"capability_removed", func(*unified.ActionPolicyAuthorizationLease) error { return unified.ErrActionPlanDrift }},
{"safety_class_changed_to_never", func(*unified.ActionPolicyAuthorizationLease) error {
return unified.ErrActionPolicyAuthorizationRevoked
}},
{"approval_floor_changed_to_dry_run", func(*unified.ActionPolicyAuthorizationLease) error {
return unified.ErrActionPolicyAuthorizationRevoked
}},
{"approval_floor_changed_to_mfa", func(*unified.ActionPolicyAuthorizationLease) error {
return unified.ErrActionPolicyAuthorizationRevoked
}},
{"never_auto_remediate_enabled", func(*unified.ActionPolicyAuthorizationLease) error { return unified.ErrResourceRemediationLocked }},
{"emergency_stop_enabled", func(lease *unified.ActionPolicyAuthorizationLease) error {
lease.EmergencyStop = true
lease.Digest = unified.ActionPolicyAuthorizationDigest(*lease)
return nil
}},
{"policy_store_unreadable", func(*unified.ActionPolicyAuthorizationLease) error { return errors.New("policy store unavailable") }},
{"lease_wrong_org", func(lease *unified.ActionPolicyAuthorizationLease) error {
lease.OrgID = "other"
lease.Digest = unified.ActionPolicyAuthorizationDigest(*lease)
return nil
}},
{"lease_expired", func(lease *unified.ActionPolicyAuthorizationLease) error {
lease.ExpiresAt = lease.IssuedAt
lease.Digest = unified.ActionPolicyAuthorizationDigest(*lease)
return nil
}},
{"lease_digest_tampered", func(lease *unified.ActionPolicyAuthorizationLease) error {
lease.Digest = "sha256:tampered"
return nil
}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
store := storeFactory(t)
now := time.Now().UTC()
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
service.Now = func() time.Time { return now }
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
failed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
lease := policyTestLease(record, at)
if mutationErr := tc.mutate(&lease); mutationErr != nil {
return unified.ActionPolicyAuthorizationLease{}, "", mutationErr
}
return lease, "policy", nil
})
if err == nil || failed.State != unified.ActionStateFailed || executor.calls != 0 {
t.Fatalf("failed=%#v err=%v calls=%d", failed, err, executor.calls)
}
if len(failed.Approvals) != 0 {
t.Fatalf("policy approval persisted: %#v", failed.Approvals)
}
events, _ := store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 20)
for _, event := range events {
if event.State == unified.ActionStateExecuting {
t.Fatalf("executing event persisted: %#v", events)
}
}
})
}
}
func TestExecuteUnderPolicyBarrierRevocationsMemoryStore(t *testing.T) {
runPolicyBarrierRevocations(t, func(*testing.T) unified.ResourceStore { return unified.NewMemoryStore() })
}
func TestExecuteUnderPolicyBarrierRevocationsSQLite(t *testing.T) {
runPolicyBarrierRevocations(t, func(t *testing.T) unified.ResourceStore {
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = store.Close() })
return store
})
}
func runHumanApprovalSurvivesPolicyRevocation(t *testing.T, store unified.ResourceStore) {
now := time.Now().UTC()
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
if _, err = service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved})); err != nil {
t.Fatal(err)
}
if _, err = service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), ""); err != nil || executor.calls != 1 {
t.Fatalf("err=%v calls=%d", err, executor.calls)
}
}
func TestHumanApprovalSurvivesAutomaticPolicyRevocationMemoryStore(t *testing.T) {
runHumanApprovalSurvivesPolicyRevocation(t, unified.NewMemoryStore())
}
func TestHumanApprovalSurvivesAutomaticPolicyRevocationSQLite(t *testing.T) {
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatal(err)
}
defer store.Close()
runHumanApprovalSurvivesPolicyRevocation(t, store)
}
func runEmergencyStopBlocksHumanAndPolicy(t *testing.T, store unified.ResourceStore) {
now := time.Now().UTC()
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
service.EmergencyStop = func(string) (bool, error) { return true, nil }
human := restartRequest()
human.RequestID = "human-stop"
plan, err := service.Plan(context.Background(), "default", human, testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
if _, err = service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved})); !errors.Is(err, unified.ErrActionEmergencyStop) {
t.Fatalf("human approval error=%v", err)
}
if record, found, getErr := service.Get("default", plan.ActionID); getErr != nil || !found || record.State != unified.ActionStatePending {
t.Fatalf("blocked approval must remain pending: found=%v state=%s err=%v", found, record.State, getErr)
}
policy := restartRequest()
policy.RequestID = "policy-stop"
policyPlan, err := service.Plan(context.Background(), "default", policy, testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
if _, err = service.ExecuteUnderPolicy(context.Background(), "default", policyPlan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
return policyTestLease(record, at), "policy", nil
}); !errors.Is(err, unified.ErrActionEmergencyStop) {
t.Fatalf("policy error=%v", err)
}
if executor.calls != 0 {
t.Fatalf("executor calls=%d", executor.calls)
}
}
func TestEmergencyStopBlocksHumanAndPolicyAdmissionMemoryStore(t *testing.T) {
runEmergencyStopBlocksHumanAndPolicy(t, unified.NewMemoryStore())
}
func TestEmergencyStopBlocksHumanAndPolicyAdmissionSQLite(t *testing.T) {
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatal(err)
}
defer store.Close()
runEmergencyStopBlocksHumanAndPolicy(t, store)
}
func TestLegacyPolicyApprovalWithoutLeaseFailsClosedAfterRestart(t *testing.T) {
dir := t.TempDir()
first, err := unified.NewSQLiteResourceStore(dir, "default")
if err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
request := restartRequest()
request.Actor = testActionActor("requester", "default")
request.RequestedBy = request.Actor.SubjectID
plan, err := (actionplanner.Planner{Now: func() time.Time { return now }}).Plan(request, testResource(now, unified.ApprovalAdmin))
if err != nil {
t.Fatal(err)
}
legacyApproved := unified.ActionAuditRecord{
ID: plan.ActionID, CreatedAt: now, UpdatedAt: now, State: unified.ActionStateApproved,
Request: request, Plan: plan,
Approvals: []unified.ActionApprovalRecord{{Actor: "pulse_patrol_policy", Method: unified.MethodPolicy, Timestamp: now, Outcome: unified.OutcomeApproved}},
}
if err := first.RecordActionAudit(legacyApproved); err != nil {
t.Fatalf("legacy audit: %v", err)
}
if err = first.Close(); err != nil {
t.Fatal(err)
}
second, err := unified.NewSQLiteResourceStore(dir, "default")
if err != nil {
t.Fatal(err)
}
defer second.Close()
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
service := serviceForStore(t, second, testResource(now, unified.ApprovalAdmin), executor)
failed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
return policyTestLease(record, at), "policy", nil
})
if !errors.Is(err, unified.ErrActionPolicyAuthorizationInvalid) || failed.State != unified.ActionStateFailed || executor.calls != 0 {
t.Fatalf("failed=%#v err=%v calls=%d", failed, err, executor.calls)
}
}
func TestQueuedPolicyActionRevalidatesAfterSQLiteRestart(t *testing.T) {
dir := t.TempDir()
first, err := unified.NewSQLiteResourceStore(dir, "default")
if err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
service := serviceForStore(t, first, testResource(now, unified.ApprovalAdmin), &stubExecutor{})
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
if err = first.Close(); err != nil {
t.Fatal(err)
}
second, err := unified.NewSQLiteResourceStore(dir, "default")
if err != nil {
t.Fatal(err)
}
defer second.Close()
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
service = serviceForStore(t, second, testResource(now, unified.ApprovalAdmin), executor)
failed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(context.Context, unified.ActionAuditRecord, time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
return unified.ActionPolicyAuthorizationLease{}, "", unified.ErrActionPolicyAuthorizationRevoked
})
if !errors.Is(err, unified.ErrActionPolicyAuthorizationRevoked) || failed.State != unified.ActionStateFailed || executor.calls != 0 {
t.Fatalf("failed=%#v err=%v calls=%d", failed, err, executor.calls)
}
}
type stubExecutor struct {
result *unified.ExecutionResult
err error
calls int
received unified.ActionAuditRecord
readiness *unified.ResourceActionReadiness
feasibility *unified.ResourceActionReadiness
}
type reconcilingExecutor struct {
executeCalls int
reconcileCalls int
executeErr error
found bool
result *unified.ExecutionResult
}
type bindingReconcilingExecutor struct{ reconcilingExecutor }
func (e *bindingReconcilingExecutor) BindActionDispatch(_ context.Context, _ unified.ActionAuditRecord, a unified.ActionDispatchAttempt) (unified.ActionDispatchAttempt, error) {
return a, nil
}
func (e *reconcilingExecutor) ExecuteAction(_ context.Context, _ unified.ActionAuditRecord) (*unified.ExecutionResult, error) {
e.executeCalls++
return nil, e.executeErr
}
func (e *reconcilingExecutor) ReconcileActionDispatch(_ context.Context, record unified.ActionAuditRecord, attempt unified.ActionDispatchAttempt) (*unified.ExecutionResult, unified.ActionDispatchReceipt, bool, error) {
e.reconcileCalls++
if !e.found {
return nil, unified.ActionDispatchReceipt{}, false, nil
}
return e.result, unified.ActionDispatchReceipt{
AttemptID: attempt.ID, ActionID: record.ID, TransportRequestID: attempt.ID, ReceivedAt: time.Now().UTC(),
}, true, nil
}
func TestLegacyUnboundReceiptPendingAttemptIsInertForBindingOwnedExecutor(t *testing.T) {
now := time.Now().UTC()
store := unified.NewMemoryStore()
executor := &bindingReconcilingExecutor{}
service := serviceForStore(t, store, testResource(now, unified.ApprovalNone), executor)
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
record, found, err := store.GetActionAudit(plan.ActionID)
if err != nil || !found {
t.Fatal(err)
}
started, event, err := unified.BeginActionExecution(record, "operator", now)
if err != nil {
t.Fatal(err)
}
attempt, err := unified.NewActionDispatchAttempt(started.ID, now)
if err != nil {
t.Fatal(err)
}
if err := store.RecordActionExecutionAdmission(started, event, attempt); err != nil {
t.Fatal(err)
}
claimed, won, err := store.ClaimActionDispatch(started.ID, "legacy-worker", now, time.Minute)
if err != nil || !won {
t.Fatalf("claim=%#v won=%v err=%v", claimed, won, err)
}
if _, err := store.MarkActionDispatchStarted(claimed.ID, "legacy-worker", now); err != nil {
t.Fatal(err)
}
recovered, err := service.RecoverExecutingActions(context.Background(), "default", "system:recovery", 10)
if err != nil || len(recovered) != 1 || recovered[0].State != unified.ActionStateExecuting {
t.Fatalf("recovered=%#v err=%v", recovered, err)
}
if executor.executeCalls != 0 || executor.reconcileCalls != 0 {
t.Fatalf("execute=%d reconcile=%d", executor.executeCalls, executor.reconcileCalls)
}
}
func (s *stubExecutor) ExecuteAction(_ context.Context, record unified.ActionAuditRecord) (*unified.ExecutionResult, error) {
s.calls++
s.received = record
return s.result, s.err
}
func (s *stubExecutor) CheckActionAvailable(_ context.Context, _ unified.ActionRequest, _ unified.Resource) unified.ResourceActionReadiness {
if s.readiness == nil {
return unified.ResourceActionReadiness{}
}
return *s.readiness
}
func (s *stubExecutor) CheckActionFeasible(_ context.Context, _ string, _ unified.ActionRequest, _ unified.Resource) unified.ResourceActionReadiness {
if s.feasibility == nil {
return unified.ResourceActionReadiness{}
}
return *s.feasibility
}
type serviceEnv struct {
store unified.ResourceStore
registry *unified.ResourceRegistry
executor *stubExecutor
completed []unified.ActionAuditRecord
service *Service
}
func testResource(now time.Time, minimumApproval unified.ActionApprovalLevel) unified.Resource {
return unified.Resource{
ID: "vm:42",
Type: unified.ResourceTypeVM,
Name: "web-42",
Status: unified.StatusWarning,
LastSeen: now,
UpdatedAt: now,
Sources: []unified.DataSource{unified.SourceProxmox},
Capabilities: []unified.ResourceCapability{
{
Name: "restart",
Type: unified.CapabilityTypeCommon,
Description: "Restart the VM",
MinimumApprovalLevel: minimumApproval,
InternalHandler: "proxmox.vm.restart",
Params: []unified.CapabilityParam{
{Name: "mode", Type: "string", Required: true, Enum: []string{"graceful", "force"}},
},
},
},
}
}
func newServiceEnv(t *testing.T, resource unified.Resource) *serviceEnv {
t.Helper()
env := &serviceEnv{
store: unified.NewMemoryStore(),
executor: &stubExecutor{result: &unified.ExecutionResult{Success: true, Output: "restarted"}},
}
env.registry = unified.NewRegistry(env.store)
env.registry.IngestResources([]unified.Resource{resource})
env.service = &Service{
Registry: func(orgID string) (*unified.ResourceRegistry, error) {
if orgID != "default" {
return nil, errors.New("unknown org")
}
return env.registry, nil
},
Store: func(orgID string) (Store, error) {
if orgID != "default" {
return nil, errors.New("unknown org")
}
return env.store, nil
},
Executor: env.executor,
DecisionAuthorizer: DecisionAuthorizerFunc(func(context.Context, string, unified.ActionAuditRecord, unified.ActionDecision) error { return nil }),
ExecutionAuthorizer: ExecutionAuthorizerFunc(func(context.Context, string, unified.ActionAuditRecord, unified.ActionActor) error { return nil }),
OnActionCompleted: func(record unified.ActionAuditRecord) {
env.completed = append(env.completed, record)
},
}
return env
}
func restartRequest() unified.ActionRequest {
return unified.ActionRequest{
RequestID: "req-1",
ResourceID: "vm:42",
CapabilityName: "restart",
Params: map[string]any{"mode": "graceful"},
Reason: "Recover after confirmed outage",
RequestedBy: "agent:test",
}
}
func TestPlanPersistsPendingAuditAndLifecycle(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
if !plan.RequiresApproval {
t.Fatal("expected admin-gated capability to require approval")
}
record, ok, err := env.store.GetActionAudit(plan.ActionID)
if err != nil || !ok {
t.Fatalf("GetActionAudit: ok=%v err=%v", ok, err)
}
if record.State != unified.ActionStatePending {
t.Fatalf("audit state = %q, want pending_approval", record.State)
}
events, err := env.store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 10)
if err != nil {
t.Fatalf("GetActionLifecycleEvents: %v", err)
}
if len(events) != 2 {
t.Fatalf("lifecycle events = %d, want planned + pending", len(events))
}
// Idempotent replan must not duplicate lifecycle events.
if _, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default")); err != nil {
t.Fatalf("replan: %v", err)
}
events, err = env.store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 10)
if err != nil {
t.Fatalf("GetActionLifecycleEvents after replan: %v", err)
}
if len(events) != 2 {
t.Fatalf("replan duplicated lifecycle events: %d", len(events))
}
}
func TestPlanFollowsAdvertisedLifecycleSynonym(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
// The fixture advertises only "restart"; a request for the lifecycle
// synonym "reboot" must plan the advertised verb.
req := restartRequest()
req.CapabilityName = "reboot"
plan, err := env.service.Plan(context.Background(), "default", req, testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan with lifecycle synonym: %v", err)
}
record, ok, err := env.store.GetActionAudit(plan.ActionID)
if err != nil || !ok {
t.Fatalf("GetActionAudit: ok=%v err=%v", ok, err)
}
if record.Request.CapabilityName != "restart" {
t.Fatalf("persisted capability = %q, want the advertised verb %q", record.Request.CapabilityName, "restart")
}
replay, err := env.service.Plan(context.Background(), "default", req, testActionActor("requester", "default"))
if err != nil || replay.ActionID != plan.ActionID {
t.Fatalf("synonym replay changed plan: %#v %v", replay, err)
}
// Verbs without an advertised synonym still fail closed.
unknown := restartRequest()
unknown.RequestID = "independent-unknown-capability"
unknown.CapabilityName = "shutdown"
var capErr *CapabilityNotFoundError
if _, err := env.service.Plan(context.Background(), "default", unknown, testActionActor("requester", "default")); !errors.As(err, &capErr) {
t.Fatalf("non-synonym verb error = %v, want CapabilityNotFoundError", err)
}
}
func TestPlanFailsClosedOnUnknownResourceAndCapability(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
missing := restartRequest()
missing.ResourceID = "vm:404"
var notFound *ResourceNotFoundError
if _, err := env.service.Plan(context.Background(), "default", missing, testActionActor("requester", "default")); !errors.As(err, ¬Found) {
t.Fatalf("unknown resource error = %v, want ResourceNotFoundError", err)
}
unknownCap := restartRequest()
unknownCap.CapabilityName = "detonate"
_, err := env.service.Plan(context.Background(), "default", unknownCap, testActionActor("requester", "default"))
var capErr *CapabilityNotFoundError
if !errors.As(err, &capErr) || !errors.Is(err, actionplanner.ErrCapabilityNotFound) {
t.Fatalf("unknown capability error = %v, want CapabilityNotFoundError wrapping ErrCapabilityNotFound", err)
}
if capErr.ResourceID != "vm:42" || capErr.CapabilityName != "detonate" {
t.Fatalf("capability error detail = %#v", capErr)
}
empty := restartRequest()
empty.ResourceID = " "
var validation *actionplanner.ValidationError
if _, err := env.service.Plan(context.Background(), "default", empty, testActionActor("requester", "default")); !errors.As(err, &validation) {
t.Fatalf("empty resource id error = %v, want ValidationError", err)
}
audits, err := env.store.GetActionAudits("vm:42", time.Time{}, 10)
if err != nil {
t.Fatalf("GetActionAudits: %v", err)
}
if len(audits) != 0 {
t.Fatalf("failed plans must not persist audits, got %d", len(audits))
}
}
func TestPlanAvailabilityRefusalPersistsNothing(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
env.executor.readiness = &unified.ResourceActionReadiness{
Name: "restart",
Available: false,
ReasonCode: "agent_disconnected",
Reason: "no connected command agent",
}
_, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
var refused *AvailabilityRefusedError
if !errors.As(err, &refused) {
t.Fatalf("error = %v, want AvailabilityRefusedError", err)
}
if refused.Readiness.ReasonCode != "agent_disconnected" {
t.Fatalf("readiness = %#v", refused.Readiness)
}
audits, err := env.store.GetActionAudits("vm:42", time.Time{}, 10)
if err != nil {
t.Fatalf("GetActionAudits: %v", err)
}
if len(audits) != 0 {
t.Fatalf("refused availability must not persist audits, got %d", len(audits))
}
}
func TestPlanAgentFeasibilityRefusalPersistsNothing(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
env.executor.feasibility = &unified.ResourceActionReadiness{
Name: "restart", Available: false, ReasonCode: "target_state_changed",
Reason: "The target changed before approval.",
}
_, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
var refused *AvailabilityRefusedError
if !errors.As(err, &refused) || refused.Readiness.ReasonCode != "target_state_changed" {
t.Fatalf("error=%v refused=%#v", err, refused)
}
audits, queryErr := env.store.GetActionAudits("vm:42", time.Time{}, 10)
if queryErr != nil || len(audits) != 0 {
t.Fatalf("audits=%#v err=%v", audits, queryErr)
}
}
func TestDecideApprovesPendingAction(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
updated, err := env.service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{
Actor: "[email protected]",
Method: unified.MethodAPI,
Outcome: unified.OutcomeApproved,
Reason: "confirmed outage",
}))
if err != nil {
t.Fatalf("Decide: %v", err)
}
if updated.State != unified.ActionStateApproved {
t.Fatalf("state = %q, want approved", updated.State)
}
if len(updated.Approvals) != 1 || updated.Approvals[0].Actor != "[email protected]" {
t.Fatalf("approvals = %#v", updated.Approvals)
}
var notFound *ActionNotFoundError
if _, err := env.service.Decide(context.Background(), "default", "act_missing", testActionDecision(t, env.service, "default", "act_missing", unified.ActionApprovalRecord{
Actor: "[email protected]", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved,
})); !errors.As(err, ¬Found) {
t.Fatalf("unknown action error = %v, want ActionNotFoundError", err)
}
}
func TestApprovalRequiresCurrentReadinessButRejectionRemainsAvailable(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
env.executor.readiness = &unified.ResourceActionReadiness{
Name: "restart",
Available: false,
ReasonCode: "agent_disconnected",
Reason: "Connect the command agent for web-42.",
}
readiness := env.service.AssessCurrentReadiness(context.Background(), "default", mustActionRecord(t, env.service, plan.ActionID))
if readiness.Ready || readiness.Code != "agent_disconnected" || readiness.Message != "Connect the command agent for web-42." {
t.Fatalf("readiness=%#v", readiness)
}
decision := testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Outcome: unified.OutcomeApproved})
if _, err := env.service.Decide(context.Background(), "default", plan.ActionID, decision); !errors.Is(err, unified.ErrActionExecutionUnavailable) {
t.Fatalf("approval error=%v", err)
}
current := mustActionRecord(t, env.service, plan.ActionID)
if current.State != unified.ActionStatePending || len(current.Approvals) != 0 {
t.Fatalf("refused approval mutated record: %#v", current)
}
rejection := testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Outcome: unified.OutcomeRejected})
rejected, err := env.service.Decide(context.Background(), "default", plan.ActionID, rejection)
if err != nil || rejected.State != unified.ActionStateRejected {
t.Fatalf("rejected=%#v err=%v", rejected, err)
}
}
func TestApprovalRechecksAgentFeasibilityBeforePersistingDecision(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
env.executor.feasibility = &unified.ResourceActionReadiness{
Name: "restart", Available: false, ReasonCode: "target_state_changed",
Reason: "The target changed after planning.",
}
decision := testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Outcome: unified.OutcomeApproved})
if _, err := env.service.Decide(context.Background(), "default", plan.ActionID, decision); !errors.Is(err, unified.ErrActionExecutionUnavailable) {
t.Fatalf("approval error=%v", err)
}
current := mustActionRecord(t, env.service, plan.ActionID)
if current.State != unified.ActionStatePending || len(current.Approvals) != 0 {
t.Fatalf("refused approval mutated record: %#v", current)
}
}
func TestRefreshReplacesExpiredPlanAndPreservesTrustedOrigin(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
origin := &unified.ActionOrigin{Surface: "operational_trust_attention", OperationalRecordID: "attention-1", EvidenceIDs: []string{"evidence-1"}}
plan, err := env.service.PlanWithOptions(context.Background(), "default", restartRequest(), PlanOptions{Actor: testActionActor("requester", "default"), Origin: origin})
if err != nil {
t.Fatal(err)
}
readinessChecks := 0
env.service.Now = func() time.Time {
readinessChecks++
if readinessChecks == 1 {
return plan.ExpiresAt.Add(time.Second)
}
return time.Now().UTC()
}
replacement, err := env.service.Refresh(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"))
if err != nil {
t.Fatal(err)
}
if replacement.ID == plan.ActionID || replacement.Request.RequestID != "refresh:"+plan.ActionID {
t.Fatalf("replacement=%#v", replacement)
}
if replacement.Origin == nil || replacement.Origin.Surface != origin.Surface || replacement.Origin.OperationalRecordID != origin.OperationalRecordID {
t.Fatalf("origin=%#v", replacement.Origin)
}
previous := mustActionRecord(t, env.service, plan.ActionID)
if previous.State != unified.ActionStateExpired {
t.Fatalf("previous state=%q", previous.State)
}
replayed, err := env.service.Refresh(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"))
if err != nil || replayed.ID != replacement.ID {
t.Fatalf("replayed=%#v err=%v", replayed, err)
}
}
func mustActionRecord(t *testing.T, service *Service, actionID string) unified.ActionAuditRecord {
t.Helper()
record, found, err := service.Get("default", actionID)
if err != nil || !found {
t.Fatalf("Get(%q): found=%v err=%v", actionID, found, err)
}
return record
}
func TestExecuteRunsApprovedActionToTerminalAudit(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
if _, err := env.service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{
Actor: "[email protected]", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved,
})); err != nil {
t.Fatalf("Decide: %v", err)
}
completed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("[email protected]", "default"), "approved restart")
if err != nil {
t.Fatalf("Execute: %v", err)
}
if completed.State != unified.ActionStateCompleted {
t.Fatalf("state = %q, want completed", completed.State)
}
if env.executor.calls != 1 {
t.Fatalf("executor calls = %d, want 1", env.executor.calls)
}
if len(env.completed) != 1 || env.completed[0].ID != plan.ActionID {
t.Fatalf("completion publisher observed %#v", env.completed)
}
record, ok, err := env.store.GetActionAudit(plan.ActionID)
if err != nil || !ok {
t.Fatalf("GetActionAudit: ok=%v err=%v", ok, err)
}
if record.State != unified.ActionStateCompleted || record.Result == nil || !record.Result.Success {
t.Fatalf("terminal audit = state %q result %#v", record.State, record.Result)
}
}
func TestExecuteRefusesLostReadinessBeforeAdmission(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
env.executor.readiness = &unified.ResourceActionReadiness{Name: "restart", Available: true}
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
env.executor.readiness = &unified.ResourceActionReadiness{
Name: "restart",
Available: false,
ReasonCode: "command_agent_disconnected",
Reason: "command agent disconnected",
}
failed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("requester", "default"), "")
var refused *AvailabilityRefusedError
if !errors.As(err, &refused) || !errors.Is(err, unified.ErrActionExecutionUnavailable) {
t.Fatalf("error = %v, want AvailabilityRefusedError wrapping ErrActionExecutionUnavailable", err)
}
if failed.State != unified.ActionStateFailed || failed.Result == nil ||
failed.Result.ActionResultV2 == nil ||
failed.Result.ActionResultV2.Execution.ReasonCode != "action_execution_unavailable" ||
!strings.HasPrefix(failed.Result.ErrorMessage, "action_execution_unavailable: command agent disconnected") {
t.Fatalf("failed audit = %#v", failed)
}
if env.executor.calls != 0 {
t.Fatalf("executor calls = %d, want no dispatch", env.executor.calls)
}
if len(env.completed) != 1 || env.completed[0].State != unified.ActionStateFailed {
t.Fatalf("completion publisher observed %#v", env.completed)
}
if _, found, getErr := env.store.GetActionDispatchAttempt(plan.ActionID); getErr != nil || found {
t.Fatalf("dispatch attempt found=%v err=%v, want none", found, getErr)
}
}
func TestExecuteRefusesUnapprovedActionWithoutDispatch(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
if _, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("[email protected]", "default"), ""); !errors.Is(err, unified.ErrActionNotApproved) {
t.Fatalf("error = %v, want ErrActionNotApproved", err)
}
if env.executor.calls != 0 {
t.Fatalf("executor must not run for unapproved actions, calls = %d", env.executor.calls)
}
}
func TestExecuteRefusesRemediationLockedResource(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
if err := env.store.SetResourceOperatorState(unified.ResourceOperatorState{
CanonicalID: "vm:42",
NeverAutoRemediate: true,
}); err != nil {
t.Fatalf("SetResourceOperatorState: %v", err)
}
failed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), "")
if !errors.Is(err, unified.ErrResourceRemediationLocked) {
t.Fatalf("error = %v, want ErrResourceRemediationLocked", err)
}
if env.executor.calls != 0 {
t.Fatalf("executor must not run on locked resources, calls = %d", env.executor.calls)
}
if failed.State != unified.ActionStateFailed {
t.Fatalf("refusal must persist a terminal failed audit, state = %q", failed.State)
}
if len(env.completed) != 1 || env.completed[0].State != unified.ActionStateFailed {
t.Fatalf("refusal must publish the failed record, got %#v", env.completed)
}
}
// A rebuild moves the operator's rows to the successor ID before it publishes
// the registry generation that lists it, so an action planned against the old
// ID can be dispatched, and the planner can be asked about it, while the
// registry the lifecycle consults still lists the old ID. The lock the
// operator set on it has moved with its row; the gates must still find it.
func TestLockMovedBySuccessionStillRefusesTheIDThePreviousGenerationLists(t *testing.T) {
locks := map[string]unified.ResourceOperatorState{
"never auto-remediate": {NeverAutoRemediate: true},
"retired": {LifecycleState: unified.LifecycleStateRetired},
}
for lockName, lock := range locks {
t.Run(lockName, func(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
state := lock
state.CanonicalID = "vm:42"
if err := env.store.SetResourceOperatorState(state); err != nil {
t.Fatalf("SetResourceOperatorState: %v", err)
}
if err := env.store.(*unified.MemoryStore).ApplyCanonicalIDSuccessions([]unified.CanonicalIDSuccession{{OldCanonicalID: "vm:42", NewCanonicalID: "vm:42-successor"}}); err != nil {
t.Fatalf("ApplyCanonicalIDSuccessions: %v", err)
}
if _, listed := env.registry.Get("vm:42"); !listed {
t.Fatal("the registry no longer lists vm:42; the fixture does not model the window before the new generation publishes")
}
if row, found, err := env.store.GetResourceOperatorState("vm:42-successor"); err != nil || !found || !row.BlocksRemediation() {
t.Fatalf("the successor does not hold the moved lock: found=%v err=%v state=%+v", found, err, row)
}
// The planner and the Patrol broker read the policy through the lifecycle.
planned, found, err := env.service.ResourceOperatorState("default", "vm:42")
if err != nil || !found || !planned.BlocksRemediation() {
t.Fatalf("the policy read for vm:42 lost the lock: found=%v err=%v state=%+v", found, err, planned)
}
failed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), "")
if !errors.Is(err, unified.ErrResourceRemediationLocked) {
t.Fatalf("error = %v, want ErrResourceRemediationLocked", err)
}
if env.executor.calls != 0 {
t.Fatalf("executor must not run on a locked resource whose row moved, calls = %d", env.executor.calls)
}
if failed.State != unified.ActionStateFailed {
t.Fatalf("refusal must persist a terminal failed audit, state = %q", failed.State)
}
})
}
}
func TestExecuteRefusesRetiredResourceWithoutDispatch(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
if err := env.store.SetResourceOperatorState(unified.ResourceOperatorState{
CanonicalID: "vm:42",
LifecycleState: unified.LifecycleStateRetired,
}); err != nil {
t.Fatalf("SetResourceOperatorState: %v", err)
}
failed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), "")
if !errors.Is(err, unified.ErrResourceRemediationLocked) {
t.Fatalf("error = %v, want ErrResourceRemediationLocked", err)
}
if env.executor.calls != 0 {
t.Fatalf("executor must not run for retired resources, calls = %d", env.executor.calls)
}
if failed.State != unified.ActionStateFailed {
t.Fatalf("retirement refusal must persist a failed audit, state = %q", failed.State)
}
}
// admissionOrderStore stamps the moment the audit store commits an execution
// admission, so a test can order it against a concurrent policy save.
type admissionOrderStore struct {
unified.ResourceStore
seq *atomic.Int64
admission atomic.Int64
}
func (s *admissionOrderStore) RecordActionExecutionAdmission(record unified.ActionAuditRecord, event unified.ActionLifecycleEvent, attempt unified.ActionDispatchAttempt) error {
err := s.ResourceStore.RecordActionExecutionAdmission(record, event, attempt)
if err == nil {
s.admission.Store(s.seq.Add(1))
}
return err
}
// windowExecutor runs one hook inside the readiness phase: after the operator
// lock was read, before admission is committed.
type windowExecutor struct {
*stubExecutor
window func()
}
func (e *windowExecutor) CheckActionAvailable(ctx context.Context, req unified.ActionRequest, resource unified.Resource) unified.ResourceActionReadiness {
if window := e.window; window != nil {
e.window = nil
window()
}
return e.stubExecutor.CheckActionAvailable(ctx, req, resource)
}
// dispatchHookExecutor runs one hook once the dispatch is durably admitted,
// immediately before the executor would act.
type dispatchHookExecutor struct {
*stubExecutor
onDispatch func()
}
func (e *dispatchHookExecutor) ExecuteAction(ctx context.Context, record unified.ActionAuditRecord) (*unified.ExecutionResult, error) {
if hook := e.onDispatch; hook != nil {
e.onDispatch = nil
hook()
}
return e.stubExecutor.ExecuteAction(ctx, record)
}
func approveForExecution(t *testing.T, service *Service, actionID string) {
t.Helper()
if _, err := service.Decide(context.Background(), "default", actionID, testActionDecision(t, service, "default", actionID, unified.ActionApprovalRecord{
Actor: "[email protected]", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved,
})); err != nil {
t.Fatalf("Decide: %v", err)
}
}
func saveRemediationLock(store unified.ResourceStore) error {
return store.SetResourceOperatorState(unified.ResourceOperatorState{CanonicalID: "vm:42", NeverAutoRemediate: true})
}
// A lock saved while a human-approved action is between its readiness check and
// its admission commit must not be missed: the save either lands before the
// readiness check (the dispatch is refused) or queues behind the admission (the
// dispatch is already committed). What it may never do is acknowledge to the
// operator before the admission commits and still let the dispatch through.
func runExecuteAdmissionLinearizesWithOperatorLockSave(t *testing.T, newStore func(t *testing.T) unified.ResourceStore) {
now := time.Now().UTC()
var seq atomic.Int64
store := &admissionOrderStore{ResourceStore: newStore(t), seq: &seq}
executor := &windowExecutor{stubExecutor: &stubExecutor{result: &unified.ExecutionResult{Success: true, Output: "restarted"}}}
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
// Both actions are approved up front: once the lock is saved, Decide itself
// refuses to record an approval for the locked resource.
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
approveForExecution(t, service, plan.ActionID)
second, err := service.Plan(context.Background(), "default", unified.ActionRequest{
RequestID: "req-2", ResourceID: "vm:42", CapabilityName: "restart",
Params: map[string]any{"mode": "graceful"}, Reason: "Second restart", RequestedBy: "agent:test",
}, testActionActor("requester", "default"))
if err != nil {
t.Fatalf("second Plan: %v", err)
}
approveForExecution(t, service, second.ActionID)
var ackSeq atomic.Int64
acked := make(chan struct{})
entered := make(chan struct{})
saveErr := make(chan error, 1)
executor.window = func() {
// Deterministic half: a policy writer must be excluded while Execute sits
// between the lock read and the admission commit.
coordinator := service.admissionCoordinator()
if coordinator.mu.TryLock() {
coordinator.mu.Unlock()
t.Error("a policy writer could enter between the readiness check and the admission commit: Execute is not holding the admission coordinator")
}
// Concurrent half: a real writer gets its chance to finish inside the
// window. A lifecycle holding the coordinator keeps it queued, so the
// bounded wait only elapses on the correct path.
go func() {
err := service.WithPolicyMutation(func() error {
close(entered)
return saveRemediationLock(store)
})
ackSeq.Store(seq.Add(1))
saveErr <- err
close(acked)
}()
select {
case <-acked:
case <-entered:
waitFor(t, acked, "lock save to acknowledge")
case <-time.After(100 * time.Millisecond):
}
}
completed, execErr := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("[email protected]", "default"), "approved restart")
waitFor(t, acked, "lock save to acknowledge after Execute")
if err := <-saveErr; err != nil {
t.Fatalf("lock save: %v", err)
}
if executor.calls > 0 && ackSeq.Load() < store.admission.Load() {
t.Fatalf("operator lock save was acknowledged (seq %d) before the dispatch was admitted (seq %d), yet the action still reached the executor (calls=%d, state=%q, err=%v)",
ackSeq.Load(), store.admission.Load(), executor.calls, completed.State, execErr)
}
if execErr != nil || completed.State != unified.ActionStateCompleted || executor.calls != 1 {
t.Fatalf("admitted dispatch must complete once the save queues behind it: state=%q calls=%d err=%v", completed.State, executor.calls, execErr)
}
// The queued save is durable and refuses every later dispatch, including an
// action that was approved before the lock existed.
refused, err := service.Execute(context.Background(), "default", second.ActionID, testActionActor("[email protected]", "default"), "")
if !errors.Is(err, unified.ErrResourceRemediationLocked) || refused.State != unified.ActionStateFailed || executor.calls != 1 {
t.Fatalf("later dispatch after the lock save: state=%q calls=%d err=%v, want refused with ErrResourceRemediationLocked", refused.State, executor.calls, err)
}
}
func waitFor(t *testing.T, done <-chan struct{}, what string) {
t.Helper()
select {
case <-done:
case <-time.After(10 * time.Second):
t.Fatalf("timed out waiting for %s", what)
}
}
func TestExecuteAdmissionLinearizesWithOperatorLockSaveMemoryStore(t *testing.T) {
runExecuteAdmissionLinearizesWithOperatorLockSave(t, func(*testing.T) unified.ResourceStore { return unified.NewMemoryStore() })
}
func TestExecuteAdmissionLinearizesWithOperatorLockSaveSQLite(t *testing.T) {
runExecuteAdmissionLinearizesWithOperatorLockSave(t, func(t *testing.T) unified.ResourceStore {
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = store.Close() })
return store
})
}
// Admission is the linearization point: a lock saved after the dispatch is
// durably admitted never recalls it, and the save must not wait for the
// dispatch, so Execute may not hold the admission coordinator across the
// executor call.
func TestExecuteAdmittedDispatchIsNotRecalledByLaterOperatorLock(t *testing.T) {
now := time.Now().UTC()
store := unified.NewMemoryStore()
executor := &dispatchHookExecutor{stubExecutor: &stubExecutor{result: &unified.ExecutionResult{Success: true, Output: "restarted"}}}
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
approveForExecution(t, service, plan.ActionID)
saved := make(chan error, 1)
executor.onDispatch = func() {
go func() {
saved <- service.WithPolicyMutation(func() error { return saveRemediationLock(store) })
}()
select {
case err := <-saved:
if err != nil {
t.Errorf("lock save during dispatch: %v", err)
}
case <-time.After(5 * time.Second):
t.Error("lock save blocked behind an admitted dispatch: Execute must release the admission coordinator before calling the executor")
}
}
completed, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("[email protected]", "default"), "approved restart")
if err != nil || completed.State != unified.ActionStateCompleted || executor.calls != 1 {
t.Fatalf("admitted dispatch: state=%q calls=%d err=%v, want completed exactly once", completed.State, executor.calls, err)
}
if state, found, getErr := store.GetResourceOperatorState("vm:42"); getErr != nil || !found || !state.BlocksRemediation() {
t.Fatalf("lock save during dispatch must persist: found=%v state=%#v err=%v", found, state, getErr)
}
}
// The no-recall boundary also covers an attempt that was admitted but never
// sent (a crash between admission and send): restart recovery drives the
// committed authority and does not re-read the operator lock. Recalling known
// unsent work would need an atomic pre-send terminalization the stores do not
// have, so this pins the contract instead of leaving it implicit.
func TestRecoverExecutingActionsDoesNotRecallQueuedAttemptAfterOperatorLock(t *testing.T) {
now := time.Now().UTC()
store := unified.NewMemoryStore()
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true, Output: "restarted"}}
service := serviceForStore(t, store, testResource(now, unified.ApprovalNone), executor)
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
record, found, err := store.GetActionAudit(plan.ActionID)
if err != nil || !found {
t.Fatalf("GetActionAudit: found=%v err=%v", found, err)
}
started, event, err := unified.BeginActionExecution(record, "operator", now)
if err != nil {
t.Fatal(err)
}
attempt, err := unified.NewActionDispatchAttempt(started.ID, now)
if err != nil {
t.Fatal(err)
}
if err := store.RecordActionExecutionAdmission(started, event, attempt); err != nil {
t.Fatal(err)
}
if queued, ok, getErr := store.GetActionDispatchAttempt(started.ID); getErr != nil || !ok || queued.State != unified.ActionDispatchQueued {
t.Fatalf("setup attempt = %#v found=%v err=%v, want queued", queued, ok, getErr)
}
if err := service.WithPolicyMutation(func() error { return saveRemediationLock(store) }); err != nil {
t.Fatalf("lock save: %v", err)
}
recovered, err := service.RecoverExecutingActions(context.Background(), "default", "system:recovery", 10)
if err != nil || len(recovered) != 1 || recovered[0].State != unified.ActionStateCompleted || executor.calls != 1 {
t.Fatalf("recovered=%#v calls=%d err=%v, want the admitted attempt dispatched once", recovered, executor.calls, err)
}
}
func TestExecuteRefusesDriftedPlan(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
// The capability policy tightens between plan and dispatch: the
// replanned contract no longer matches the recorded plan hash.
drifted := testResource(now, unified.ApprovalAdmin)
env.registry = unified.NewRegistry(env.store)
env.registry.IngestResources([]unified.Resource{drifted})
failed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), "")
if !errors.Is(err, unified.ErrActionPlanDrift) {
t.Fatalf("error = %v, want ErrActionPlanDrift", err)
}
if env.executor.calls != 0 {
t.Fatalf("executor must not run on drifted plans, calls = %d", env.executor.calls)
}
if failed.State != unified.ActionStateFailed {
t.Fatalf("drift refusal must persist a terminal failed audit, state = %q", failed.State)
}
}
func TestExecuteFailsClosedWithoutExecutor(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
env.service.Executor = nil
if _, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), ""); !errors.Is(err, ErrExecutorUnavailable) {
t.Fatalf("error = %v, want ErrExecutorUnavailable", err)
}
}
func TestLifecycleFailsClosedWithoutStore(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
env.service.Store = func(string) (Store, error) { return nil, errors.New("db offline") }
if _, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default")); !errors.Is(err, ErrStoreUnavailable) {
t.Fatalf("Plan error = %v, want ErrStoreUnavailable", err)
}
if _, err := env.service.Decide(context.Background(), "default", "act_x", testActionDecision(t, env.service, "default", "act_x", unified.ActionApprovalRecord{Outcome: unified.OutcomeApproved})); !errors.Is(err, ErrStoreUnavailable) {
t.Fatalf("Decide error = %v, want ErrStoreUnavailable", err)
}
if _, err := env.service.Execute(context.Background(), "default", "act_x", testActionActor("agent:test", "default"), ""); !errors.Is(err, ErrStoreUnavailable) {
t.Fatalf("Execute error = %v, want ErrStoreUnavailable", err)
}
}
func TestCapabilitiesUsesSameRegistryResolutionAsPlanning(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
capabilities, err := env.service.Capabilities(context.Background(), "default", "vm:42")
if err != nil {
t.Fatalf("Capabilities: %v", err)
}
if len(capabilities) != 1 || capabilities[0].Name != "restart" {
t.Fatalf("capabilities = %#v", capabilities)
}
var notFound *ResourceNotFoundError
if _, err := env.service.Capabilities(context.Background(), "default", "vm:404"); !errors.As(err, ¬Found) {
t.Fatalf("unknown resource error = %v, want ResourceNotFoundError", err)
}
}
func TestPlanWithOptionsPersistsOriginAcrossLifecycle(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
origin := &unified.ActionOrigin{
Surface: "patrol",
FindingID: "finding-9",
InvestigationID: "inv-9",
ProposalID: "prop-9",
}
plan, err := env.service.PlanWithOptions(context.Background(), "default", restartRequest(), PlanOptions{Actor: testActionActor("requester", "default"), Origin: origin})
if err != nil {
t.Fatalf("PlanWithOptions: %v", err)
}
record, ok, err := env.store.GetActionAudit(plan.ActionID)
if err != nil || !ok {
t.Fatalf("GetActionAudit: ok=%v err=%v", ok, err)
}
if record.Origin == nil || record.Origin.FindingID != "finding-9" {
t.Fatalf("plan audit origin = %#v", record.Origin)
}
// Origin must survive the decision transition: the decision persists
// the record loaded from the store, so the broker-owned metadata
// stays reconcilable at approval time.
updated, err := env.service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{
Actor: "[email protected]", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved,
}))
if err != nil {
t.Fatalf("Decide: %v", err)
}
if updated.Origin == nil || updated.Origin.ProposalID != "prop-9" {
t.Fatalf("decision record origin = %#v", updated.Origin)
}
// Plain Plan must never stamp an origin.
plain := restartRequest()
plain.RequestID = "req-plain"
plainPlan, err := env.service.Plan(context.Background(), "default", plain, testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
plainRecord, ok, err := env.store.GetActionAudit(plainPlan.ActionID)
if err != nil || !ok {
t.Fatalf("GetActionAudit(plain): ok=%v err=%v", ok, err)
}
if plainRecord.Origin != nil {
t.Fatalf("plain plan must not carry an origin, got %#v", plainRecord.Origin)
}
}
func TestOnActionTransitionFiresAfterEachPersistedState(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
var transitions []unified.ActionState
var transitionOrgs []string
env.service.OnActionTransition = func(orgID string, record unified.ActionAuditRecord) {
transitions = append(transitions, record.State)
transitionOrgs = append(transitionOrgs, orgID)
}
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
if _, err := env.service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{
Actor: "[email protected]", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved,
})); err != nil {
t.Fatalf("Decide: %v", err)
}
if _, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("[email protected]", "default"), ""); err != nil {
t.Fatalf("Execute: %v", err)
}
want := []unified.ActionState{unified.ActionStatePending, unified.ActionStateApproved, unified.ActionStateExecuting, unified.ActionStateCompleted}
if len(transitions) != len(want) {
t.Fatalf("transitions = %v, want %v", transitions, want)
}
for i := range want {
if transitions[i] != want[i] {
t.Fatalf("transitions = %v, want %v", transitions, want)
}
if transitionOrgs[i] != "default" {
t.Fatalf("transition org = %q, want default", transitionOrgs[i])
}
}
}
func TestOnActionTransitionFiresForPersistedRefusals(t *testing.T) {
now := time.Now().UTC()
env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
var transitions []unified.ActionState
env.service.OnActionTransition = func(orgID string, record unified.ActionAuditRecord) {
transitions = append(transitions, record.State)
}
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatalf("Plan: %v", err)
}
if err := env.store.SetResourceOperatorState(unified.ResourceOperatorState{
CanonicalID: "vm:42",
NeverAutoRemediate: true,
}); err != nil {
t.Fatalf("SetResourceOperatorState: %v", err)
}
if _, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), ""); !errors.Is(err, unified.ErrResourceRemediationLocked) {
t.Fatalf("error = %v, want ErrResourceRemediationLocked", err)
}
want := []unified.ActionState{unified.ActionStatePlanned, unified.ActionStateFailed}
if len(transitions) != len(want) || transitions[0] != want[0] || transitions[1] != want[1] {
t.Fatalf("transitions = %v, want %v", transitions, want)
}
}
func TestActionPolicyProvenanceParticipatesInFreshnessWithoutAuthorizingDispatch(t *testing.T) {
now := time.Date(2026, 7, 11, 21, 0, 0, 0, time.UTC)
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
factor := unified.ActionPolicyAuthorityFactor{
Kind: unified.ActionPolicyAuthorityTenant, SourceID: "patrol-tenant-policy",
Revision: "tenant-policy:sha256:0123456789abcdef01234567", Status: unified.ActionPolicyAuthorityConsulted,
ReasonCodes: []unified.ActionPolicyReasonCode{unified.PolicyReasonTenantModeAssisted, unified.PolicyReasonTenantFullLocked},
}
plan, err := env.service.PlanWithOptions(context.Background(), "default", restartRequest(), PlanOptions{Actor: testActionActor("requester", "default"), PolicyFactors: []unified.ActionPolicyAuthorityFactor{factor}})
if err != nil {
t.Fatal(err)
}
record, found, err := env.service.Get("default", plan.ActionID)
if err != nil || !found {
t.Fatalf("Get: found=%v err=%v", found, err)
}
if err := env.service.ValidatePlanFresh("default", record); err != nil {
t.Fatalf("captured descriptive snapshot should replan deterministically: %v", err)
}
record.Plan.PolicyDecision.Authorities[1].Revision = "tenant-policy:sha256:1123456789abcdef01234567"
record.Plan.PolicyDecision.DecisionID = unified.ActionPolicyDecisionDigest(record.Plan.PolicyDecision)
if err := env.service.ValidatePlanFresh("default", record); !errors.Is(err, unified.ErrActionPlanDrift) {
t.Fatalf("provenance drift error=%v", err)
}
if _, _, _, err := unified.BeginPolicyActionExecution(record, unified.ActionApprovalRecord{Actor: "pulse_patrol_policy", Method: unified.MethodPolicy, Outcome: unified.OutcomeApproved, Timestamp: now}, unified.ActionPolicyAuthorizationLease{}, now); !errors.Is(err, unified.ErrActionPolicyAuthorizationInvalid) {
t.Fatalf("descriptive provenance authorized dispatch: %v", err)
}
}
// Issue1649: an executing action whose agent never returns completion evidence
// used to have no exit at all. Expiry skips executing rows and only a terminal
// agent receipt could complete one, so a Docker update interrupted by an agent
// restart stayed "executing" forever with no operator route out. Reconciliation
// now self-heals the known stranded shapes; ForceFail is the manual override
// for anything left, and it must refuse to rewrite settled truth.
func TestIssue1649ForceFailTerminalizesWedgedExecutingActionWithAudit(t *testing.T) {
store := unified.NewMemoryStore()
executor := &reconcilingExecutor{executeErr: context.DeadlineExceeded, found: false}
service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
var completed []unified.ActionAuditRecord
var transitions []unified.ActionAuditRecord
service.OnActionCompleted = func(record unified.ActionAuditRecord) { completed = append(completed, record) }
service.OnActionTransition = func(_ string, record unified.ActionAuditRecord) { transitions = append(transitions, record) }
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
if _, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "wedge it"); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("Execute error=%v", err)
}
if current, _, err := service.Get("default", plan.ActionID); err != nil || current.State != unified.ActionStateExecuting {
t.Fatalf("precondition current=%#v err=%v", current, err)
}
failed, err := service.ForceFail("default", plan.ActionID, "operator", "verified by hand, container is running the new image")
if err != nil || failed.State != unified.ActionStateFailed || failed.Result == nil || failed.Result.ActionResultV2 == nil {
t.Fatalf("ForceFail failed=%#v err=%v", failed, err)
}
truth := failed.Result.ActionResultV2
if truth.Execution.Status != unified.ActionExecutionInconclusive || truth.Execution.ReasonCode != ForceFailReasonCode {
t.Fatalf("execution truth=%#v", truth.Execution)
}
if truth.Verification.Status != unified.ActionVerificationInconclusive || truth.Verification.EvidenceClass != unified.ActionEvidenceNone {
t.Fatalf("verification truth=%#v", truth.Verification)
}
if !strings.Contains(failed.Result.ErrorMessage, "verified by hand") {
t.Fatalf("operator reason missing from audit result: %q", failed.Result.ErrorMessage)
}
if executor.executeCalls != 1 {
t.Fatalf("ForceFail touched the transport: execute=%d", executor.executeCalls)
}
persisted, found, err := store.GetActionAudit(plan.ActionID)
if err != nil || !found || persisted.State != unified.ActionStateFailed {
t.Fatalf("persisted=%#v found=%v err=%v", persisted, found, err)
}
events, err := store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 100)
if err != nil || len(events) == 0 {
t.Fatalf("events=%#v err=%v", events, err)
}
terminal := events[0]
if terminal.State != unified.ActionStateFailed || terminal.Actor != "operator" || !strings.Contains(terminal.Message, "Operator force-failed") {
t.Fatalf("terminal audit event=%#v", terminal)
}
if len(completed) != 1 || completed[0].State != unified.ActionStateFailed {
t.Fatalf("completed=%#v", completed)
}
if len(transitions) == 0 || transitions[len(transitions)-1].State != unified.ActionStateFailed {
t.Fatalf("transitions=%#v", transitions)
}
// The row is settled, so the recovery loop stops reconciling it and the
// override cannot be replayed over the persisted outcome.
recovered, err := service.RecoverExecutingActions(context.Background(), "default", "system:restart-recovery", 100)
if err != nil || len(recovered) != 0 {
t.Fatalf("recovered=%#v err=%v", recovered, err)
}
if _, err := service.ForceFail("default", plan.ActionID, "operator", "again"); !errors.Is(err, unified.ErrActionExecutionFinal) {
t.Fatalf("replayed ForceFail error=%v", err)
}
}
func TestIssue1649ForceFailRefusesTerminalAndUndispatchedActions(t *testing.T) {
store := unified.NewMemoryStore()
service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), &stubExecutor{result: &unified.ExecutionResult{Success: true, Output: "restarted"}})
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
// Planned but never dispatched: expiry still owns this row, so the
// override must not manufacture an unknown-effect failure for it.
if _, err := service.ForceFail("default", plan.ActionID, "operator", ""); !errors.Is(err, unified.ErrActionNotExecuting) {
t.Fatalf("undispatched ForceFail error=%v", err)
}
current, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "run it")
if err != nil || current.State != unified.ActionStateCompleted {
t.Fatalf("current=%#v err=%v", current, err)
}
if _, err := service.ForceFail("default", plan.ActionID, "operator", ""); !errors.Is(err, unified.ErrActionExecutionFinal) {
t.Fatalf("completed ForceFail error=%v", err)
}
settled, found, err := store.GetActionAudit(plan.ActionID)
if err != nil || !found || settled.State != unified.ActionStateCompleted || !settled.Result.Success {
t.Fatalf("refused override rewrote settled truth: %#v found=%v err=%v", settled, found, err)
}
var notFound *ActionNotFoundError
if _, err := service.ForceFail("default", "act_missing", "operator", ""); !errors.As(err, ¬Found) {
t.Fatalf("missing action ForceFail error=%v", err)
}
}
// A refresh's replacement names the action it replaced, which is how a
// surface that projects actions tells the replaced plan's later transitions
// from its replacement's.
func TestRefreshedActionIDNamesTheReplacedAction(t *testing.T) {
now := time.Date(2026, 10, 23, 18, 0, 0, 0, time.UTC)
store := unified.NewMemoryStore()
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), &stubExecutor{})
service.Now = func() time.Time { return now }
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
service.Now = func() time.Time { return plan.ExpiresAt.Add(time.Second) }
replacement, err := service.Refresh(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"))
if err != nil {
t.Fatal(err)
}
if replaced, ok := RefreshedActionID(replacement); !ok || replaced != plan.ActionID {
t.Fatalf("RefreshedActionID(replacement) = %q, %v, want %q", replaced, ok, plan.ActionID)
}
original, found, err := store.GetActionAudit(plan.ActionID)
if err != nil || !found {
t.Fatalf("original: found=%v err=%v", found, err)
}
if replaced, ok := RefreshedActionID(original); ok {
t.Fatalf("a first plan names a replaced action: %q", replaced)
}
}
// The inbox and decision-queue sweeps publish each plan they expire, as a
// single action's expiry is published, so a surface that projects the action,
// such as a Patrol finding's queued fix, does not outlive the plan. Each expiry
// is published once.
func TestExpirySweepsPublishEachExpiredPlan(t *testing.T) {
sweeps := map[string]func(*Service) error{
"inbox": func(s *Service) error {
_, err := s.List("default", ActionListPending, 100)
return err
},
"decision queue": func(s *Service) error {
_, err := s.DecisionQueue("default", 100)
return err
},
}
for name, sweep := range sweeps {
t.Run(name, func(t *testing.T) {
now := time.Date(2026, 10, 23, 18, 0, 0, 0, time.UTC)
store := unified.NewMemoryStore()
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), &stubExecutor{})
service.Now = func() time.Time { return now }
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
if err != nil {
t.Fatal(err)
}
var published []unified.ActionAuditRecord
var orgs []string
service.OnActionTransition = func(orgID string, record unified.ActionAuditRecord) {
published = append(published, record)
orgs = append(orgs, orgID)
}
if err := sweep(service); err != nil || len(published) != 0 {
t.Fatalf("a sweep before expiry: err=%v published=%+v", err, published)
}
service.Now = func() time.Time { return plan.ExpiresAt.Add(time.Second) }
if err := sweep(service); err != nil {
t.Fatal(err)
}
if len(published) != 1 || published[0].ID != plan.ActionID || published[0].State != unified.ActionStateExpired || orgs[0] != "default" {
t.Fatalf("published = %+v for orgs %v, want one expiry of %s", published, orgs, plan.ActionID)
}
if err := sweep(service); err != nil || len(published) != 1 {
t.Fatalf("a later sweep published the expiry again: err=%v published=%+v", err, published)
}
})
}
}