Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

35.5 KB
Raw Blame
package agenttokens

import (
	"errors"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
)

type actionRunnerTestFS struct {
	blockOnce sync.Once
	entered   chan struct{}
	release   chan struct{}
	writeErr  error
}

type actionRunnerFailAfterWritesFS struct {
	mu          sync.Mutex
	writes      int
	allowWrites int
}

type actionRunnerPromotionTestTransaction struct {
	commitResult    bool
	commitCalls     int
	rollbackCalls   int
	failClosedCalls int
}

func (tx *actionRunnerPromotionTestTransaction) Commit() bool {
	tx.commitCalls++
	return tx.commitResult
}

func (tx *actionRunnerPromotionTestTransaction) Rollback() {
	tx.rollbackCalls++
}

func (tx *actionRunnerPromotionTestTransaction) FailClosed() {
	tx.failClosedCalls++
}

func (fs *actionRunnerFailAfterWritesFS) ReadFile(name string) ([]byte, error) {
	return os.ReadFile(name)
}
func (fs *actionRunnerFailAfterWritesFS) WriteFile(name string, data []byte, perm os.FileMode) error {
	if err := fs.beforeWrite(); err != nil {
		return err
	}
	return os.WriteFile(name, data, perm)
}
func (fs *actionRunnerFailAfterWritesFS) beforeWrite() error {
	fs.mu.Lock()
	fs.writes++
	writes := fs.writes
	fs.mu.Unlock()
	if writes > fs.allowWrites {
		return errors.New("injected rollback persistence failure")
	}
	return nil
}
func (fs *actionRunnerFailAfterWritesFS) CreateTemp(dir, pattern string) (config.TemporaryFile, error) {
	return newActionRunnerTempFile(dir, pattern, fs.beforeWrite)
}
func (fs *actionRunnerFailAfterWritesFS) Rename(oldPath, newPath string) error {
	return os.Rename(oldPath, newPath)
}
func (fs *actionRunnerFailAfterWritesFS) Remove(name string) error { return os.Remove(name) }
func (fs *actionRunnerFailAfterWritesFS) Stat(name string) (os.FileInfo, error) {
	return os.Stat(name)
}
func (fs *actionRunnerFailAfterWritesFS) MkdirAll(path string, perm os.FileMode) error {
	return os.MkdirAll(path, perm)
}

func (fs *actionRunnerTestFS) ReadFile(name string) ([]byte, error) { return os.ReadFile(name) }
func (fs *actionRunnerTestFS) WriteFile(name string, data []byte, perm os.FileMode) error {
	if err := fs.beforeWrite(); err != nil {
		return err
	}
	return os.WriteFile(name, data, perm)
}
func (fs *actionRunnerTestFS) beforeWrite() error {
	blocked := false
	fs.blockOnce.Do(func() {
		blocked = fs.entered != nil
		if blocked {
			close(fs.entered)
		}
	})
	if blocked {
		<-fs.release
	}
	return fs.writeErr
}
func (fs *actionRunnerTestFS) CreateTemp(dir, pattern string) (config.TemporaryFile, error) {
	return newActionRunnerTempFile(dir, pattern, fs.beforeWrite)
}

type actionRunnerTempFile struct {
	*os.File
	beforeWrite func() error
}

func (f *actionRunnerTempFile) Write(data []byte) (int, error) {
	if err := f.beforeWrite(); err != nil {
		return 0, err
	}
	return f.File.Write(data)
}

func newActionRunnerTempFile(dir, pattern string, beforeWrite func() error) (config.TemporaryFile, error) {
	f, err := os.CreateTemp(dir, pattern)
	if err != nil {
		return nil, err
	}
	return &actionRunnerTempFile{File: f, beforeWrite: beforeWrite}, nil
}
func (fs *actionRunnerTestFS) Rename(oldPath, newPath string) error {
	return os.Rename(oldPath, newPath)
}
func (fs *actionRunnerTestFS) Remove(name string) error { return os.Remove(name) }
func (fs *actionRunnerTestFS) Stat(name string) (os.FileInfo, error) {
	return os.Stat(name)
}
func (fs *actionRunnerTestFS) MkdirAll(path string, perm os.FileMode) error {
	return os.MkdirAll(path, perm)
}

func TestIssueAndPersistInstallToken(t *testing.T) {
	cfg := &config.Config{DataPath: t.TempDir()}
	raw, record, err := IssueAndPersist(cfg, nil, IssueOptions{
		TokenName:   "host-agent",
		OwnerUserID: "operator",
		Scopes:      HostScopes(true),
		Metadata:    map[string]string{"install_type": "host"},
	})
	if err != nil {
		t.Fatalf("IssueAndPersist: %v", err)
	}
	if raw == "" || record == nil || len(cfg.APITokens) != 1 {
		t.Fatalf("issued token = (%q, %#v), persisted=%d", raw, record, len(cfg.APITokens))
	}
	if OwnerUserID(*record) != "operator" || record.Metadata[IssuedAtMetadataKey] == "" {
		t.Fatalf("issued metadata = %#v", record.Metadata)
	}
	if !record.HasScope(config.ScopeAgentExec) {
		t.Fatalf("commands-enabled host scopes = %v", record.Scopes)
	}
	if got := record.Metadata[RuntimeRoleMetadataKey]; got != CredentialKindLegacyFullTrust {
		t.Fatalf("combined install runtime role = %q, want %q", got, CredentialKindLegacyFullTrust)
	}
}

func TestIssueActionRunnerAndPersistIsHostBoundAndExecOnly(t *testing.T) {
	cfg := &config.Config{DataPath: t.TempDir()}
	raw, record, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{
		OrgID: "org-a", AgentID: "machine-123", Hostname: " Node.EXAMPLE. ", OwnerUserID: "operator",
	})
	if err != nil {
		t.Fatalf("IssueActionRunnerAndPersist: %v", err)
	}
	if raw == "" || record == nil {
		t.Fatalf("issued action credential = (%q, %#v)", raw, record)
	}
	if len(record.Scopes) != 1 || record.Scopes[0] != config.ScopeAgentExec {
		t.Fatalf("action credential scopes = %v, want only %q", record.Scopes, config.ScopeAgentExec)
	}
	if record.HasScope(config.ScopeAgentReport) || record.HasScope(config.ScopeAgentConfigRead) || record.HasScope(config.ScopeAgentManage) {
		t.Fatalf("action credential inherited collector authority: %v", record.Scopes)
	}
	if record.OrgID != "org-a" || record.Metadata["bound_agent_id"] != "machine-123" || record.Metadata["bound_hostname"] != "node.example" {
		t.Fatalf("action credential binding = org=%q metadata=%#v", record.OrgID, record.Metadata)
	}
	if record.Metadata[RuntimeRoleMetadataKey] != CredentialKindActionRunner ||
		record.Metadata[ActionCapabilityMetadataKey] != ActionCapabilityTypedV1 ||
		record.Metadata[ActionBindingVersionMetadataKey] != ActionBindingVersion {
		t.Fatalf("action credential authority metadata = %#v", record.Metadata)
	}
}

func TestIssueActionRunnerAndPersistRejectsIncompleteBinding(t *testing.T) {
	for _, tc := range []ActionRunnerIssueOptions{
		{AgentID: "machine", Hostname: "node"},
		{OrgID: "org", Hostname: "node"},
		{OrgID: "org", AgentID: "machine"},
	} {
		if raw, record, err := IssueActionRunnerAndPersist(&config.Config{}, nil, tc); !errors.Is(err, ErrRecord) || raw != "" || record != nil {
			t.Fatalf("incomplete binding %#v = (%q, %#v, %v), want ErrRecord", tc, raw, record, err)
		}
	}
}

func TestIssueActionRunnerAndPersistRejectsAgentIDOutsideRunnerVocabulary(t *testing.T) {
	for _, agentID := range []string{"bad agent", "-agent", strings.Repeat("a", 129)} {
		t.Run(agentID, func(t *testing.T) {
			raw, record, err := IssueActionRunnerAndPersist(&config.Config{}, nil, ActionRunnerIssueOptions{
				OrgID: "org-a", AgentID: agentID, Hostname: "node.example",
			})
			if !errors.Is(err, ErrRecord) || raw != "" || record != nil {
				t.Fatalf("IssueActionRunnerAndPersist = (%q, %#v, %v), want ErrRecord", raw, record, err)
			}
		})
	}
}

func TestIssueActionRunnerAndPersistPreparesRotationWithoutRevokingActiveCredential(t *testing.T) {
	cfg := &config.Config{DataPath: t.TempDir()}
	firstToken, first, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{
		OrgID: "org-a", AgentID: "machine-123", Hostname: "Node.EXAMPLE",
	})
	if err != nil {
		t.Fatalf("first IssueActionRunnerAndPersist: %v", err)
	}
	if _, _, changed, err := ActivateActionRunnerAndPersist(cfg, config.NewConfigPersistence(cfg.DataPath), first.ID, "machine-123", "node.example"); err != nil || !changed {
		t.Fatalf("activate first credential = changed %v, error %v", changed, err)
	}
	_, otherHost, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{
		OrgID: "org-a", AgentID: "machine-456", Hostname: "other.example",
	})
	if err != nil {
		t.Fatalf("other-host IssueActionRunnerAndPersist: %v", err)
	}
	_, replacement, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{
		// Canonical agent identity survives a host rename. The old binding can no
		// longer admit that host and must not remain as an orphaned secret.
		OrgID: "org-a", AgentID: "machine-123", Hostname: "renamed.example",
	})
	if err != nil {
		t.Fatalf("replacement IssueActionRunnerAndPersist: %v", err)
	}
	if replacement.ID == first.ID {
		t.Fatalf("replacement reused token id %q", replacement.ID)
	}
	if len(cfg.APITokens) != 3 {
		t.Fatalf("token inventory = %#v, want pending replacement, active predecessor, and other host", cfg.APITokens)
	}
	foundReplacement, foundOther, foundFirst := false, false, false
	for _, record := range cfg.APITokens {
		foundReplacement = foundReplacement || record.ID == replacement.ID
		foundOther = foundOther || record.ID == otherHost.ID
		foundFirst = foundFirst || record.ID == first.ID
	}
	if !foundReplacement || !foundOther || !foundFirst {
		t.Fatalf("replacement inventory = %#v", cfg.APITokens)
	}
	if _, ok := cfg.ValidateAPIToken(firstToken); !ok {
		t.Fatal("prepare step revoked the active predecessor")
	}
	if replacement.ExpiresAt == nil || replacement.Metadata[ActionRunnerActivationPendingMetadataKey] != "true" || replacement.Metadata[ActionRunnerReplacesTokenIDsMetadataKey] != first.ID {
		t.Fatalf("prepared replacement = %#v", replacement)
	}
}

func TestIssueActionRunnerAndPersistDetailedReturnsOnlyDurablyReplacedRecords(t *testing.T) {
	cfg := &config.Config{DataPath: t.TempDir()}
	_, prior, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{
		OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example",
	})
	if err != nil {
		t.Fatal(err)
	}
	result, err := IssueActionRunnerAndPersistDetailed(cfg, config.NewConfigPersistence(cfg.DataPath), ActionRunnerIssueOptions{
		OrgID: "org-a", AgentID: "machine-123", Hostname: "renamed.example",
	})
	if err != nil {
		t.Fatal(err)
	}
	if result.Token == "" || result.Record == nil || len(result.Replaced) != 1 || result.Replaced[0].ID != prior.ID {
		t.Fatalf("detailed issue result = %#v", result)
	}
	if result.Record.ID == prior.ID || len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != result.Record.ID {
		t.Fatalf("persisted replacement = %#v", cfg.APITokens)
	}
}

func TestActivateActionRunnerAndPersistAtomicallyPromotesAndRevokesPredecessor(t *testing.T) {
	cfg := &config.Config{DataPath: t.TempDir()}
	firstToken, first, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
	if err != nil {
		t.Fatal(err)
	}
	if _, _, _, err := ActivateActionRunnerAndPersist(cfg, config.NewConfigPersistence(cfg.DataPath), first.ID, "machine-123", "node.example"); err != nil {
		t.Fatal(err)
	}
	secondToken, second, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
	if err != nil {
		t.Fatal(err)
	}
	activated, revoked, changed, err := ActivateActionRunnerAndPersist(cfg, config.NewConfigPersistence(cfg.DataPath), second.ID, "machine-123", "NODE")
	if err != nil || !changed {
		t.Fatalf("activation = (%#v, %#v, %v, %v)", activated, revoked, changed, err)
	}
	if activated.ExpiresAt != nil || activated.Metadata[ActionRunnerActivationPendingMetadataKey] != "" || len(revoked) != 1 || revoked[0].ID != first.ID {
		t.Fatalf("activation result = activated %#v, revoked %#v", activated, revoked)
	}
	if len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != second.ID {
		t.Fatalf("activated inventory = %#v", cfg.APITokens)
	}
	if _, ok := cfg.ValidateAPIToken(firstToken); ok {
		t.Fatal("activated rotation left predecessor valid")
	}
	if _, ok := cfg.ValidateAPIToken(secondToken); !ok {
		t.Fatal("activated replacement is not valid")
	}
	if _, revoked, changed, err := ActivateActionRunnerAndPersist(cfg, config.NewConfigPersistence(cfg.DataPath), second.ID, "machine-123", "node.example"); err != nil || changed || len(revoked) != 0 {
		t.Fatalf("idempotent activation = revoked %#v, changed %v, error %v", revoked, changed, err)
	}
}

func TestActivateActionRunnerAndPersistRequiresDurablePersistenceWithoutMutation(t *testing.T) {
	cfg := &config.Config{DataPath: t.TempDir()}
	_, pending, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{
		OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example",
	})
	if err != nil {
		t.Fatal(err)
	}
	before := cloneAPITokenRecords(cfg.APITokens)
	if activated, revoked, changed, err := ActivateActionRunnerAndPersist(
		cfg, nil, pending.ID, "machine-123", "node.example",
	); !errors.Is(err, ErrPersist) || activated != nil || revoked != nil || changed {
		t.Fatalf("nil persistence activation = activated %#v revoked %#v changed %v err %v", activated, revoked, changed, err)
	}
	if !reflect.DeepEqual(cfg.APITokens, before) {
		t.Fatalf("nil persistence mutated inventory: before %#v after %#v", before, cfg.APITokens)
	}
}

func TestActivateActionRunnerAndPersistFailureRestoresPendingAndActiveInventory(t *testing.T) {
	cfg := &config.Config{DataPath: t.TempDir()}
	_, first, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
	if err != nil {
		t.Fatal(err)
	}
	if _, _, _, err := ActivateActionRunnerAndPersist(cfg, config.NewConfigPersistence(cfg.DataPath), first.ID, "machine-123", "node.example"); err != nil {
		t.Fatal(err)
	}
	_, second, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
	if err != nil {
		t.Fatal(err)
	}
	statePath := filepath.Join(t.TempDir(), "blocked-state")
	persistence := config.NewConfigPersistence(statePath)
	if err := os.RemoveAll(statePath); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(statePath, []byte("not a directory"), 0o600); err != nil {
		t.Fatal(err)
	}
	if _, _, changed, err := ActivateActionRunnerAndPersist(cfg, persistence, second.ID, "machine-123", "node.example"); !errors.Is(err, ErrPersist) || changed {
		t.Fatalf("activation = changed %v, error %v", changed, err)
	}
	if len(cfg.APITokens) != 2 {
		t.Fatalf("restored inventory = %#v", cfg.APITokens)
	}
	for _, record := range cfg.APITokens {
		if record.ID == second.ID && (record.ExpiresAt == nil || record.Metadata[ActionRunnerActivationPendingMetadataKey] != "true" || record.Metadata[ActionRunnerReplacesTokenIDsMetadataKey] != first.ID) {
			t.Fatalf("pending replacement was not fully restored: %#v", record)
		}
	}
}

func TestActivateActionRunnerPromotionFencesBeforePersistenceAndRollsBackOnSaveFailure(t *testing.T) {
	cfg := &config.Config{DataPath: t.TempDir()}
	_, predecessor, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
	if err != nil {
		t.Fatal(err)
	}
	if _, _, _, err := ActivateActionRunnerAndPersist(cfg, config.NewConfigPersistence(cfg.DataPath), predecessor.ID, "machine-123", "node.example"); err != nil {
		t.Fatal(err)
	}
	_, pending, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
	if err != nil {
		t.Fatal(err)
	}

	entered := make(chan struct{})
	release := make(chan struct{})
	persistence := config.NewConfigPersistence(cfg.DataPath)
	persistence.SetFileSystem(&actionRunnerTestFS{entered: entered, release: release, writeErr: errors.New("injected activation save failure")})
	tx := &actionRunnerPromotionTestTransaction{commitResult: true}
	type activationOutcome struct {
		changed bool
		err     error
	}
	done := make(chan activationOutcome, 1)
	go func() {
		_, _, changed, err := ActivateActionRunnerAndPersistWithPromotion(cfg, persistence, pending.ID, "machine-123", "node.example", func() (ActionRunnerPromotionTransaction, bool) {
			return tx, true
		})
		done <- activationOutcome{changed: changed, err: err}
	}()

	select {
	case <-entered:
		if tx.commitCalls != 0 || tx.rollbackCalls != 0 || tx.failClosedCalls != 0 {
			t.Fatalf("promotion resolved before persistence outcome: commit %d rollback %d fail-closed %d", tx.commitCalls, tx.rollbackCalls, tx.failClosedCalls)
		}
	case <-time.After(2 * time.Second):
		t.Fatal("activation did not enter persistence after beginning promotion")
	}
	close(release)
	select {
	case outcome := <-done:
		if !errors.Is(outcome.err, ErrPersist) || outcome.changed {
			t.Fatalf("activation outcome = changed %v err %v", outcome.changed, outcome.err)
		}
	case <-time.After(2 * time.Second):
		t.Fatal("activation did not return after persistence failure")
	}
	if tx.commitCalls != 0 || tx.rollbackCalls != 1 || tx.failClosedCalls != 0 {
		t.Fatalf("failed-save promotion lifecycle = commit %d rollback %d fail-closed %d", tx.commitCalls, tx.rollbackCalls, tx.failClosedCalls)
	}
}

func TestActivateActionRunnerAndPersistWithPromotionRollsBackWhenExactTransportVanished(t *testing.T) {
	cfg := &config.Config{DataPath: t.TempDir()}
	_, predecessor, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
	if err != nil {
		t.Fatal(err)
	}
	if _, _, _, err := ActivateActionRunnerAndPersist(cfg, config.NewConfigPersistence(cfg.DataPath), predecessor.ID, "machine-123", "node.example"); err != nil {
		t.Fatal(err)
	}
	_, pending, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
	if err != nil {
		t.Fatal(err)
	}
	persistence := config.NewConfigPersistence(cfg.DataPath)
	tx := &actionRunnerPromotionTestTransaction{}
	beginCalls := 0
	activated, revoked, changed, err := ActivateActionRunnerAndPersistWithPromotion(cfg, persistence, pending.ID, "machine-123", "node.example", func() (ActionRunnerPromotionTransaction, bool) {
		beginCalls++
		return tx, true
	})
	if !errors.Is(err, ErrActionRunnerSessionUnavailable) || activated != nil || revoked != nil || changed {
		t.Fatalf("vanished transport activation = activated %#v revoked %#v changed %v err %v", activated, revoked, changed, err)
	}
	if beginCalls != 1 || tx.commitCalls != 1 || tx.rollbackCalls != 1 || tx.failClosedCalls != 0 {
		t.Fatalf("promotion lifecycle = begin %d commit %d rollback %d fail-closed %d", beginCalls, tx.commitCalls, tx.rollbackCalls, tx.failClosedCalls)
	}
	if len(cfg.APITokens) != 2 {
		t.Fatalf("rolled-back inventory = %#v", cfg.APITokens)
	}
	var foundPredecessor, foundPending bool
	for _, record := range cfg.APITokens {
		switch record.ID {
		case predecessor.ID:
			foundPredecessor = record.ExpiresAt == nil
		case pending.ID:
			foundPending = record.ExpiresAt != nil && record.Metadata[ActionRunnerActivationPendingMetadataKey] == "true"
		}
	}
	if !foundPredecessor || !foundPending {
		t.Fatalf("rollback did not restore predecessor and pending replacement: %#v", cfg.APITokens)
	}
	persisted, err := config.NewConfigPersistence(cfg.DataPath).LoadAPITokens()
	if err != nil {
		t.Fatal(err)
	}
	if len(persisted) != 2 {
		t.Fatalf("durable rollback inventory = %#v", persisted)
	}
}

func TestActivateActionRunnerPromotionRollbackPersistenceFailureKeepsLastDurableActivation(t *testing.T) {
	cfg := &config.Config{DataPath: t.TempDir()}
	_, predecessor, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
	if err != nil {
		t.Fatal(err)
	}
	if _, _, _, err := ActivateActionRunnerAndPersist(cfg, config.NewConfigPersistence(cfg.DataPath), predecessor.ID, "machine-123", "node.example"); err != nil {
		t.Fatal(err)
	}
	_, pending, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
	if err != nil {
		t.Fatal(err)
	}
	persistence := config.NewConfigPersistence(cfg.DataPath)
	// The pre-existing durable predecessor inventory causes activation to write
	// one backup plus the new token file. Permit both, then fail the compensating
	// rollback writes.
	persistence.SetFileSystem(&actionRunnerFailAfterWritesFS{allowWrites: 2})
	tx := &actionRunnerPromotionTestTransaction{}
	activated, revoked, changed, err := ActivateActionRunnerAndPersistWithPromotion(cfg, persistence, pending.ID, "machine-123", "node.example", func() (ActionRunnerPromotionTransaction, bool) {
		return tx, true
	})
	if !errors.Is(err, ErrActionRunnerActivationIndeterminate) || activated == nil || activated.ID != pending.ID || len(revoked) != 1 || revoked[0].ID != predecessor.ID || !changed {
		t.Fatalf("rollback persistence failure = activated %#v revoked %#v changed %v err %v", activated, revoked, changed, err)
	}
	if len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != pending.ID || cfg.APITokens[0].ExpiresAt != nil || cfg.APITokens[0].Metadata[ActionRunnerActivationPendingMetadataKey] != "" {
		t.Fatalf("memory diverged from last durable activation: %#v", cfg.APITokens)
	}
	if tx.commitCalls != 1 || tx.rollbackCalls != 0 || tx.failClosedCalls != 1 {
		t.Fatalf("indeterminate promotion lifecycle = commit %d rollback %d fail-closed %d", tx.commitCalls, tx.rollbackCalls, tx.failClosedCalls)
	}
	persisted, err := config.NewConfigPersistence(cfg.DataPath).LoadAPITokens()
	if err != nil {
		t.Fatal(err)
	}
	if len(persisted) != 1 || persisted[0].ID != pending.ID || persisted[0].ExpiresAt != nil || persisted[0].Metadata[ActionRunnerActivationPendingMetadataKey] != "" {
		t.Fatalf("last durable activation inventory = %#v", persisted)
	}
}

func TestCancelPendingActionRunnerAndPersistSerializesWithActivationInBothOrders(t *testing.T) {
	seed := func(t *testing.T) (*config.Config, *config.ConfigPersistence, *config.APITokenRecord, *config.APITokenRecord) {
		t.Helper()
		cfg := &config.Config{DataPath: t.TempDir()}
		_, first, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
		if err != nil {
			t.Fatal(err)
		}
		if _, _, _, err := ActivateActionRunnerAndPersist(cfg, config.NewConfigPersistence(cfg.DataPath), first.ID, "machine-123", "node.example"); err != nil {
			t.Fatal(err)
		}
		_, second, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
		if err != nil {
			t.Fatal(err)
		}
		return cfg, config.NewConfigPersistence(cfg.DataPath), first, second
	}

	t.Run("cancel wins", func(t *testing.T) {
		cfg, persistence, first, second := seed(t)
		removed, err := CancelPendingActionRunnerAndPersist(cfg, persistence, second.ID, "org-a", "machine-123", "NODE")
		if err != nil || removed == nil || removed.ID != second.ID {
			t.Fatalf("cancel = (%#v, %v)", removed, err)
		}
		if len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != first.ID {
			t.Fatalf("cancelled inventory = %#v", cfg.APITokens)
		}
		if _, _, _, err := ActivateActionRunnerAndPersist(cfg, persistence, second.ID, "machine-123", "node.example"); !errors.Is(err, ErrRecord) {
			t.Fatalf("late activation error = %v, want ErrRecord", err)
		}
	})

	t.Run("activation wins", func(t *testing.T) {
		cfg, persistence, _, second := seed(t)
		if _, _, changed, err := ActivateActionRunnerAndPersist(cfg, persistence, second.ID, "machine-123", "node.example"); err != nil || !changed {
			t.Fatalf("activation = changed %v, error %v", changed, err)
		}
		if removed, err := CancelPendingActionRunnerAndPersist(cfg, persistence, second.ID, "org-a", "machine-123", "node.example"); !errors.Is(err, ErrActionRunnerAlreadyActivated) || removed != nil {
			t.Fatalf("post-commit cancel = (%#v, %v)", removed, err)
		}
		if len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != second.ID || cfg.APITokens[0].ExpiresAt != nil {
			t.Fatalf("activated inventory = %#v", cfg.APITokens)
		}
	})
}

func TestCancelPendingActionRunnerAndPersistContendsUnderOneDurableTransaction(t *testing.T) {
	seed := func(t *testing.T) (*config.Config, *config.ConfigPersistence, *config.APITokenRecord, *config.APITokenRecord) {
		t.Helper()
		cfg := &config.Config{DataPath: t.TempDir()}
		_, first, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
		if err != nil {
			t.Fatal(err)
		}
		if _, _, _, err := ActivateActionRunnerAndPersist(cfg, config.NewConfigPersistence(cfg.DataPath), first.ID, "machine-123", "node.example"); err != nil {
			t.Fatal(err)
		}
		_, pending, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
		if err != nil {
			t.Fatal(err)
		}
		return cfg, config.NewConfigPersistence(cfg.DataPath), first, pending
	}
	waitSignal := func(t *testing.T, signal <-chan struct{}, label string) {
		t.Helper()
		select {
		case <-signal:
		case <-time.After(2 * time.Second):
			t.Fatalf("timed out waiting for %s", label)
		}
	}
	waitErr := func(t *testing.T, result <-chan error, label string) error {
		t.Helper()
		select {
		case err := <-result:
			return err
		case <-time.After(2 * time.Second):
			t.Fatalf("timed out waiting for %s", label)
			return nil
		}
	}

	t.Run("cancel owns lock before activation", func(t *testing.T) {
		cfg, persistence, predecessor, pending := seed(t)
		fs := &actionRunnerTestFS{entered: make(chan struct{}), release: make(chan struct{})}
		persistence.SetFileSystem(fs)
		cancelResult := make(chan error, 1)
		go func() {
			_, err := CancelPendingActionRunnerAndPersist(cfg, persistence, pending.ID, "org-a", "machine-123", "node.example")
			cancelResult <- err
		}()
		waitSignal(t, fs.entered, "cancel persistence")
		activateResult := make(chan error, 1)
		go func() {
			_, _, _, err := ActivateActionRunnerAndPersist(cfg, persistence, pending.ID, "machine-123", "node.example")
			activateResult <- err
		}()
		close(fs.release)
		if err := waitErr(t, cancelResult, "cancel result"); err != nil {
			t.Fatalf("cancel error = %v", err)
		}
		if err := waitErr(t, activateResult, "late activation result"); !errors.Is(err, ErrRecord) {
			t.Fatalf("late activation error = %v, want ErrRecord", err)
		}
		if len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != predecessor.ID {
			t.Fatalf("cancel-wins inventory = %#v", cfg.APITokens)
		}
	})

	t.Run("activation owns lock before cancel", func(t *testing.T) {
		cfg, persistence, predecessor, pending := seed(t)
		fs := &actionRunnerTestFS{entered: make(chan struct{}), release: make(chan struct{})}
		persistence.SetFileSystem(fs)
		activateResult := make(chan error, 1)
		go func() {
			_, _, _, err := ActivateActionRunnerAndPersist(cfg, persistence, pending.ID, "machine-123", "node.example")
			activateResult <- err
		}()
		waitSignal(t, fs.entered, "activation persistence")
		cancelResult := make(chan error, 1)
		go func() {
			_, err := CancelPendingActionRunnerAndPersist(cfg, persistence, pending.ID, "org-a", "machine-123", "node.example")
			cancelResult <- err
		}()
		close(fs.release)
		if err := waitErr(t, activateResult, "activation result"); err != nil {
			t.Fatalf("activation error = %v", err)
		}
		if err := waitErr(t, cancelResult, "post-commit cancel result"); !errors.Is(err, ErrActionRunnerAlreadyActivated) {
			t.Fatalf("post-commit cancel error = %v, want ErrActionRunnerAlreadyActivated", err)
		}
		if len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != pending.ID || cfg.APITokens[0].ExpiresAt != nil || cfg.APITokens[0].Metadata[ActionRunnerActivationPendingMetadataKey] != "" || cfg.APITokens[0].ID == predecessor.ID {
			t.Fatalf("activation-wins inventory = %#v", cfg.APITokens)
		}
	})
}

func TestCancelPendingActionRunnerAndPersistFailureNeverAuthorizesRollback(t *testing.T) {
	cfg := &config.Config{DataPath: t.TempDir()}
	_, pending, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"})
	if err != nil {
		t.Fatal(err)
	}
	persistence := config.NewConfigPersistence(t.TempDir())
	persistence.SetFileSystem(&actionRunnerTestFS{writeErr: errors.New("injected persistence failure")})
	if removed, err := CancelPendingActionRunnerAndPersist(cfg, persistence, pending.ID, "org-a", "machine-123", "node.example"); !errors.Is(err, ErrPersist) || removed != nil {
		t.Fatalf("persistence failure = (%#v, %v)", removed, err)
	}
	if len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != pending.ID || cfg.APITokens[0].Metadata[ActionRunnerActivationPendingMetadataKey] != "true" {
		t.Fatalf("failed cancel changed inventory = %#v", cfg.APITokens)
	}
	if removed, err := CancelPendingActionRunnerAndPersist(cfg, nil, pending.ID, "org-a", "machine-123", "node.example"); !errors.Is(err, ErrPersist) || removed != nil {
		t.Fatalf("nil persistence = (%#v, %v)", removed, err)
	}
}

func TestIssueActionRunnerAndPersistRestoresReplacedCredentialOnPersistenceFailure(t *testing.T) {
	cfg := &config.Config{DataPath: t.TempDir()}
	_, prior, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{
		OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example",
	})
	if err != nil {
		t.Fatalf("seed action credential: %v", err)
	}
	_, unrelated, err := IssueAndPersist(cfg, nil, IssueOptions{TokenName: "collector", OrgID: "org-a"})
	if err != nil {
		t.Fatalf("seed unrelated credential: %v", err)
	}

	statePath := filepath.Join(t.TempDir(), "blocked-state")
	persistence := config.NewConfigPersistence(statePath)
	if err := os.RemoveAll(statePath); err != nil {
		t.Fatalf("remove persistence directory: %v", err)
	}
	if err := os.WriteFile(statePath, []byte("not a directory"), 0o600); err != nil {
		t.Fatalf("create persistence blocker: %v", err)
	}
	raw, replacement, err := IssueActionRunnerAndPersist(cfg, persistence, ActionRunnerIssueOptions{
		OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example",
	})
	if !errors.Is(err, ErrPersist) {
		t.Fatalf("replacement error = %v, want ErrPersist", err)
	}
	if raw != "" || replacement != nil {
		t.Fatalf("failed replacement returned credential: raw=%q record=%#v", raw, replacement)
	}
	if len(cfg.APITokens) != 2 {
		t.Fatalf("restored inventory = %#v", cfg.APITokens)
	}
	foundPrior, foundUnrelated := false, false
	for _, record := range cfg.APITokens {
		foundPrior = foundPrior || record.ID == prior.ID
		foundUnrelated = foundUnrelated || record.ID == unrelated.ID
	}
	if !foundPrior || !foundUnrelated {
		t.Fatalf("full prior inventory was not restored: %#v", cfg.APITokens)
	}
}

func TestIssueAndPersistRejectsMonitoringRoleWithExecAuthority(t *testing.T) {
	_, _, err := IssueAndPersist(&config.Config{}, nil, IssueOptions{
		TokenName: "invalid",
		Scopes:    []string{config.ScopeAgentReport, config.ScopeAgentExec},
		Metadata:  map[string]string{RuntimeRoleMetadataKey: CredentialKindMonitoringCollector},
	})
	if !errors.Is(err, ErrRecord) {
		t.Fatalf("monitoring role with exec authority error = %v, want ErrRecord", err)
	}
}

func TestIssueAndPersistRejectsMonitoringRoleWithUnrelatedAuthority(t *testing.T) {
	for _, scopes := range [][]string{
		{config.ScopeAgentReport, config.ScopeAgentConfigRead, config.ScopeSettingsWrite},
		{config.ScopeAgentReport, config.ScopeAgentConfigRead, config.ScopeActionsExecute},
	} {
		_, _, err := IssueAndPersist(&config.Config{}, nil, IssueOptions{
			TokenName: "invalid",
			Scopes:    scopes,
			Metadata:  map[string]string{RuntimeRoleMetadataKey: CredentialKindMonitoringCollector},
		})
		if !errors.Is(err, ErrRecord) {
			t.Fatalf("monitoring role with scopes %v error = %v, want ErrRecord", scopes, err)
		}
	}
}

func TestIssueAndPersistRejectsInferredMonitoringRoleWithUnrelatedAuthority(t *testing.T) {
	_, _, err := IssueAndPersist(&config.Config{}, nil, IssueOptions{
		TokenName: "invalid inferred collector",
		Scopes:    []string{config.ScopeAgentReport, config.ScopeAgentConfigRead, config.ScopeSettingsWrite},
	})
	if !errors.Is(err, ErrRecord) {
		t.Fatalf("inferred monitoring role with unrelated authority error = %v, want ErrRecord", err)
	}
}

func TestProxmoxScopesRequireExplicitCommandAuthority(t *testing.T) {
	monitoringScopes := ProxmoxScopes(false)
	if (&config.APITokenRecord{Scopes: monitoringScopes}).HasScope(config.ScopeAgentExec) {
		t.Fatalf("monitoring-only Proxmox scopes include %s: %v", config.ScopeAgentExec, monitoringScopes)
	}
	if (&config.APITokenRecord{Scopes: monitoringScopes}).HasScope(config.ScopeAgentManage) {
		t.Fatalf("monitoring-only Proxmox scopes include cross-host management: %v", monitoringScopes)
	}

	commandScopes := ProxmoxScopes(true)
	if !(&config.APITokenRecord{Scopes: commandScopes}).HasScope(config.ScopeAgentExec) {
		t.Fatalf("explicit command profile is missing %s: %v", config.ScopeAgentExec, commandScopes)
	}
	if (&config.APITokenRecord{Scopes: commandScopes}).HasScope(config.ScopeAgentManage) {
		t.Fatalf("collector command profile includes cross-host management: %v", commandScopes)
	}
}

func TestIssueAndPersistDefaultsToMonitoringOnlyScopes(t *testing.T) {
	cfg := &config.Config{DataPath: t.TempDir()}
	_, record, err := IssueAndPersist(cfg, nil, IssueOptions{TokenName: "default-agent"})
	if err != nil {
		t.Fatalf("IssueAndPersist: %v", err)
	}
	if record.HasScope(config.ScopeAgentExec) {
		t.Fatalf("implicit install scopes include %s: %v", config.ScopeAgentExec, record.Scopes)
	}
}

func TestIssueAndPersistRollsBackCompleteInventoryWhenPersistenceFails(t *testing.T) {
	now := time.Now().UTC()
	tokens := []config.APITokenRecord{
		{ID: "newest", Name: "newest", Hash: "hash-newest", CreatedAt: now, Scopes: []string{config.ScopeWildcard}},
		{ID: "oldest", Name: "oldest", Hash: "hash-oldest", CreatedAt: now.Add(-time.Minute), Scopes: []string{config.ScopeWildcard}},
	}
	cfg := &config.Config{APITokens: append([]config.APITokenRecord(nil), tokens...)}
	cfg.SortAPITokens()

	stateDir := filepath.Join(t.TempDir(), "state")
	persistence := config.NewConfigPersistence(stateDir)
	if err := os.RemoveAll(stateDir); err != nil {
		t.Fatalf("remove persistence directory: %v", err)
	}
	if err := os.WriteFile(stateDir, []byte("not a directory"), 0o600); err != nil {
		t.Fatalf("create persistence blocker: %v", err)
	}

	raw, record, err := IssueAndPersist(cfg, persistence, IssueOptions{TokenName: "must-not-survive"})
	if !errors.Is(err, ErrPersist) {
		t.Fatalf("IssueAndPersist error = %v, want ErrPersist", err)
	}
	if raw != "" || record != nil {
		t.Fatalf("failed issue returned credential material: raw=%q record=%#v", raw, record)
	}
	if len(cfg.APITokens) != len(tokens) {
		t.Fatalf("token count = %d, want %d: %#v", len(cfg.APITokens), len(tokens), cfg.APITokens)
	}
	for index, want := range tokens {
		if cfg.APITokens[index].ID != want.ID {
			t.Fatalf("token[%d].ID = %q, want %q", index, cfg.APITokens[index].ID, want.ID)
		}
	}
	if cfg.APIToken != "hash-newest" {
		t.Fatalf("legacy primary token = %q, want rollback to %q", cfg.APIToken, "hash-newest")
	}
	for _, token := range cfg.APITokens {
		if token.Name == "must-not-survive" {
			t.Fatalf("failed issue left generated install token active: %#v", token)
		}
	}
}

func TestIssueAndPersistRejectsReservedOwnerMetadata(t *testing.T) {
	_, _, err := IssueAndPersist(&config.Config{}, nil, IssueOptions{
		TokenName: "invalid",
		Metadata:  map[string]string{OwnerUserIDMetadataKey: "forged"},
	})
	if !errors.Is(err, ErrRecord) {
		t.Fatalf("reserved owner metadata error = %v, want ErrRecord", err)
	}
}

func TestCommandPolicyIntentRoundTrip(t *testing.T) {
	for _, tc := range []struct {
		name    string
		value   string
		enabled bool
		valid   bool
	}{
		{name: "enabled", value: CommandPolicyIntent(true), enabled: true, valid: true},
		{name: "disabled", value: CommandPolicyIntent(false), enabled: false, valid: true},
		{name: "missing", value: "", enabled: false, valid: false},
		{name: "invalid", value: "yes", enabled: false, valid: false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			record := &config.APITokenRecord{Metadata: map[string]string{
				CommandPolicyIntentMetadataKey: tc.value,
			}}
			enabled, valid := ParseCommandPolicyIntent(record)
			if enabled != tc.enabled || valid != tc.valid {
				t.Fatalf("ParseCommandPolicyIntent() = (%v, %v), want (%v, %v)", enabled, valid, tc.enabled, tc.valid)
			}
		})
	}

	if _, valid := ParseCommandPolicyIntent(nil); valid {
		t.Fatal("nil record unexpectedly carried command-policy intent")
	}
}