Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Make cancellation receipt proof wait for an activated runner
pulse-triage[bot]1f5e45157818
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.mdindex ebfab787bc..f37b909f48 100644--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md@@ -8109,6 +8109,15 @@ kept running on the Proxmox host). Three coupled guarantees:healthy even when the original waiter has gone, and reconnect/replayreturns that exact receipt without a second mutation.+The cancellation/reconnect integration fixture must not equate published+registration with a ready runner. Each new client completes its real activated+health write and a read-only receipt-query round trip before the fixture sends+a mutation. A controlled failed-first activation proves this barrier across+reconnect; the existing three-second readiness/start limits are unchanged.+Runner and dispatch cleanup is installed before assertions and precedes test+server shutdown, including failed starts. These are proof-fixture requirements,+not changes to production admission, cancellation or replay authority.+Proofs: `internal/agentexec/server_websocket_test.go`(`TestExecuteCommand_ExpiredContextNeverDispatches`,`TestExecuteCommand_AbandonedCommandSendsCancel`),@@ -8117,7 +8126,8 @@ Proofs: `internal/agentexec/server_websocket_test.go``TestTypedOperation_TimeoutSendsCancelAndExpiredContextNeverDispatches`,`TestTypedOperation_SocketDropAfterSendUnblocksDispatch`),`internal/hostagent/operation_receipt_websocket_integration_test.go`-(`TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfterReconnect`),+(`TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfterReconnect`,+`TestReceiptRunnerReadinessWaitsForActivationReconnect`),`internal/hostagent/command_client_test.go`(`TestCommandClient_handleCancelCommand_CancelsRegisteredRequest`,`TestCommandClient_handleCancelCommand_UnknownRequestIsNoOp`,diff --git a/internal/hostagent/operation_receipt_websocket_integration_test.go b/internal/hostagent/operation_receipt_websocket_integration_test.goindex 1d6a421e16..8f30729e5d 100644--- a/internal/hostagent/operation_receipt_websocket_integration_test.go+++ b/internal/hostagent/operation_receipt_websocket_integration_test.go@@ -5,12 +5,14 @@ import ("context""encoding/json""errors"+ "fmt""net/http""net/http/httptest""os""path/filepath""strings""sync"+ "sync/atomic""testing""time"@@ -237,6 +239,132 @@ func TestRealServerAndUnifiedAgentWebSocketDockerDuplicateReplayMutatesOnce(t *t}}+// Registration publishes a server session before the runner has activated its+// credential, durably written health, and started reading command messages.+// Observe each new client's actual health write, not a file left by its predecessor.+func observeReceiptRunnerActivation(client *CommandClient) <-chan struct{} {+ activated := make(chan struct{}, 1)+ client.actionHealthWriter = func(ready bool) error {+ if err := client.writeActionRunnerHealth(ready); err != nil {+ return err+ }+ if ready {+ select {+ case activated <- struct{}{}:+ default:+ }+ }+ return nil+ }+ return activated+}++func waitForActivatedReceiptRunner(ctx context.Context, server *agentexec.Server, identity operationreceipt.Identity, activated <-chan struct{}, done <-chan error) error {+ select {+ case <-activated:+ case err := <-done:+ return fmt.Errorf("runner exited before activation: %v", err)+ case <-ctx.Done():+ return ctx.Err()+ }+ // An actual receipt-query round trip proves the activated client's reader is+ // serving this session. It admits no mutation and changes no receipt.+ if _, err := server.QueryAgentOperation(ctx, identity.AgentID, identity); err != nil {+ return fmt.Errorf("activated runner receipt query: %w", err)+ }+ return nil+}++func TestReceiptRunnerReadinessWaitsForActivationReconnect(t *testing.T) {+ admission := agentexec.AgentAdmission{+ TokenID: "readiness-token", AgentID: "agent-ready", Hostname: "ready.example.test",+ RuntimeRole: agentexec.RuntimeRoleActionRunner, ActionCapability: agentexec.ActionCapabilityTypedV1,+ }+ server := agentexec.NewServerWithAdmissionValidator(func(token, _, _ string) (agentexec.AgentAdmission, bool) {+ return admission, token == admission.TokenID+ }, func(agentexec.AgentAdmission) bool { return true })+ activationEntered := make(chan struct{})+ releaseActivation := make(chan struct{})+ var releaseOnce sync.Once+ var activationCalls atomic.Int32+ httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {+ if request.Method == http.MethodPatch && request.URL.Path == "/api/agents/action-runner/credential" {+ if activationCalls.Add(1) == 1 {+ close(activationEntered)+ select {+ case <-releaseActivation:+ w.WriteHeader(http.StatusServiceUnavailable)+ case <-request.Context().Done():+ }+ return+ }+ w.WriteHeader(http.StatusNoContent)+ return+ }+ server.HandleWebSocket(w, request)+ }))+ t.Cleanup(httpServer.Close)+ t.Cleanup(func() { releaseOnce.Do(func() { close(releaseActivation) }) })+ origReconnectDelay := reconnectDelay+ reconnectDelay = 50 * time.Millisecond+ t.Cleanup(func() { reconnectDelay = origReconnectDelay })+ dir := t.TempDir()+ logger := zerolog.Nop()+ client := NewActionRunnerClient(ActionRunnerClientConfig{+ PulseURL: httpServer.URL, APIToken: admission.TokenID, StateDir: dir,+ HealthPath: filepath.Join(dir, "health.json"), ActivationNonce: strings.Repeat("b", 32), Logger: &logger,+ }, admission.AgentID, admission.Hostname, "test")+ activated := observeReceiptRunnerActivation(client)+ ctx, cancel := context.WithCancel(context.Background())+ done := make(chan error, 1)+ go func() { done <- client.Run(ctx); close(done) }()+ t.Cleanup(func() {+ cancel()+ _ = client.Close()+ select {+ case <-done:+ case <-time.After(3 * time.Second):+ t.Error("readiness runner did not stop")+ }+ })+ select {+ case <-activationEntered:+ case <-time.After(3 * time.Second):+ t.Fatal("runner did not reach the controlled activation")+ }+ if !server.IsAgentConnected(admission.AgentID) {+ t.Fatal("control must expose the registered but unactivated session")+ }+ digest, err := operationreceipt.DigestCanonicalJSON(map[string]string{"readiness": "query-only"})+ if err != nil {+ t.Fatal(err)+ }+ identity := operationreceipt.Identity{AttemptID: "readiness.1", ActionID: "readiness", OperationKind: "fake.typed", OperationVersion: 1, RequestDigest: digest, AgentID: admission.AgentID}+ blockedCtx, cancelBlocked := context.WithTimeout(context.Background(), 100*time.Millisecond)+ err = waitForActivatedReceiptRunner(blockedCtx, server, identity, activated, done)+ cancelBlocked()+ if !errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("unactivated session passed readiness: %v", err)+ }+ releaseOnce.Do(func() { close(releaseActivation) })+ readyCtx, cancelReady := context.WithTimeout(context.Background(), 3*time.Second)+ defer cancelReady()+ if err := waitForActivatedReceiptRunner(readyCtx, server, identity, activated, done); err != nil {+ t.Fatalf("reconnected activated runner: %v", err)+ }+ if activationCalls.Load() != 2 {+ t.Fatalf("activation attempts = %d, want failed first session and activated replacement", activationCalls.Load())+ }+ data, err := os.ReadFile(filepath.Join(dir, "health.json"))+ if err != nil {+ t.Fatal(err)+ }+ var health actionRunnerHealth+ if err := json.Unmarshal(data, &health); err != nil || !health.Registered || !health.Activated {+ t.Fatalf("reconnected health = %+v, err=%v", health, err)+ }+}+func TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfterReconnect(t *testing.T) {admission := agentexec.AgentAdmission{TokenID: "runner-token", AgentID: "agent-pve", Hostname: "pve.example.test",@@ -252,7 +380,7 @@ func TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfter}server.HandleWebSocket(w, request)}))- defer httpServer.Close()+ t.Cleanup(httpServer.Close)// A runner that drops its session must come back well inside the replay// window rather than after the production reconnect backoff.@@ -294,27 +422,15 @@ func TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfterreturn nil, ctx.Err()}- startRunner := func(t *testing.T) (*CommandClient, context.CancelFunc, <-chan error) {- t.Helper()- client := NewActionRunnerClient(ActionRunnerClientConfig{- PulseURL: httpServer.URL, APIToken: admission.TokenID, StateDir: stateDir,- HealthPath: filepath.Join(stateDir, "health.json"), ActivationNonce: strings.Repeat("a", 32), Logger: &logger,- }, admission.AgentID, admission.Hostname, "test")- client.proxmoxGuestLifecycle = manager- ctx, cancel := context.WithCancel(context.Background())- done := make(chan error, 1)- go func() { done <- client.Run(ctx) }()- deadline := time.Now().Add(3 * time.Second)- for !server.IsAgentConnected(admission.AgentID) && time.Now().Before(deadline) {- time.Sleep(10 * time.Millisecond)- }- if !server.IsAgentConnected(admission.AgentID) {- cancel()- _ = client.Close()- t.Fatal("action runner did not connect")- }- return client, cancel, done+ request := agentexec.ProxmoxGuestLifecyclePayload{+ RequestID: "pve.cancel-replay.1", ActionID: "pve.cancel-replay", Operation: "shutdown",+ GuestKind: "vm", VMID: 101, ExpectedStatus: "running", Timeout: 30,+ }+ if err := agentexec.BindProxmoxGuestLifecyclePayload(&request); err != nil {+ t.Fatal(err)}+ identity := agentexec.ProxmoxGuestLifecycleOperationIdentity(admission.AgentID, request)+stopRunner := func(t *testing.T, client *CommandClient, cancel context.CancelFunc, done <-chan error) {t.Helper()cancel()@@ -322,7 +438,8 @@ func TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfterselect {case <-done:case <-time.After(3 * time.Second):- t.Fatal("action runner did not stop")+ t.Error("action runner did not stop")+ return}// The client has exited, but the server observes the socket close on// its own reader goroutine. Reconnecting before that lands would let@@ -333,21 +450,40 @@ func TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAftertime.Sleep(10 * time.Millisecond)}if server.IsAgentConnected(admission.AgentID) {- t.Fatal("server did not observe the action runner disconnect")+ t.Error("server did not observe the action runner disconnect")}}- request := agentexec.ProxmoxGuestLifecyclePayload{- RequestID: "pve.cancel-replay.1", ActionID: "pve.cancel-replay", Operation: "shutdown",- GuestKind: "vm", VMID: 101, ExpectedStatus: "running", Timeout: 30,- }- if err := agentexec.BindProxmoxGuestLifecyclePayload(&request); err != nil {- t.Fatal(err)+ startRunner := func(t *testing.T) func() {+ t.Helper()+ client := NewActionRunnerClient(ActionRunnerClientConfig{+ PulseURL: httpServer.URL, APIToken: admission.TokenID, StateDir: stateDir,+ HealthPath: filepath.Join(stateDir, "health.json"), ActivationNonce: strings.Repeat("a", 32), Logger: &logger,+ }, admission.AgentID, admission.Hostname, "test")+ client.proxmoxGuestLifecycle = manager+ activated := observeReceiptRunnerActivation(client)+ ctx, cancel := context.WithCancel(context.Background())+ done := make(chan error, 1)+ go func() {+ done <- client.Run(ctx)+ close(done)+ }()+ // Register cleanup before any readiness assertion. A failed start must+ // not leave the runner alive while httptest waits for its handlers.+ var stopOnce sync.Once+ stop := func() { stopOnce.Do(func() { stopRunner(t, client, cancel, done) }) }+ t.Cleanup(stop)+ readyCtx, cancelReady := context.WithTimeout(context.Background(), 3*time.Second)+ defer cancelReady()+ if err := waitForActivatedReceiptRunner(readyCtx, server, identity, activated, done); err != nil {+ t.Fatalf("action runner was not ready: %v", err)+ }+ return stop}- identity := agentexec.ProxmoxGuestLifecycleOperationIdentity(admission.AgentID, request)- first, cancelFirst, firstDone := startRunner(t)+ stopFirst := startRunner(t)dispatchCtx, cancelDispatch := context.WithCancel(context.Background())+ t.Cleanup(cancelDispatch)dispatchDone := make(chan error, 1)go func() {_, err := server.ExecuteProxmoxGuestLifecycle(dispatchCtx, admission.AgentID, request)@@ -355,6 +491,8 @@ func TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfter}()select {case <-mutationStarted:+ case err := <-dispatchDone:+ t.Fatalf("Proxmox dispatch returned before mutation: %v", err)case <-time.After(3 * time.Second):t.Fatal("Proxmox mutation did not start")}@@ -385,14 +523,13 @@ func TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfterif !canceled.MutationStarted || canceled.MutationCompleted || !strings.Contains(canceled.Error, "recovery inspection") {t.Fatalf("canceled durable receipt = %+v", canceled)}- stopRunner(t, first, cancelFirst, firstDone)+ stopFirst()- second, cancelSecond, secondDone := startRunner(t)- defer stopRunner(t, second, cancelSecond, secondDone)- // The server publishes the session before the runner finishes activating,- // so the replay can land on a session the runner then drops. That request- // can never be answered; send the same request again on the session the- // runner reconnects with. Every attempt must replay the durable receipt.+ stopSecond := startRunner(t)+ defer stopSecond()+ // Readiness above proves activation and a live reader. Retain the existing+ // bounded recovery for a genuinely dropped session after that proof: every+ // attempt must replay the durable receipt without a second mutation.var replayed *agentexec.ProxmoxGuestLifecycleResultPayloadvar err errorfor attempt := 1; ; attempt++ {