Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Make cancellation receipt proof wait for an activated runner

pulse-triage[bot]1f5e45157818
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index ebfab787bc..f37b909f48 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -8109,6 +8109,15 @@ kept running on the Proxmox host). Three coupled guarantees:
healthy even when the original waiter has gone, and reconnect/replay
returns that exact receipt without a second mutation.
+The cancellation/reconnect integration fixture must not equate published
+registration with a ready runner. Each new client completes its real activated
+health write and a read-only receipt-query round trip before the fixture sends
+a mutation. A controlled failed-first activation proves this barrier across
+reconnect; the existing three-second readiness/start limits are unchanged.
+Runner and dispatch cleanup is installed before assertions and precedes test
+server shutdown, including failed starts. These are proof-fixture requirements,
+not changes to production admission, cancellation or replay authority.
+
Proofs: `internal/agentexec/server_websocket_test.go`
(`TestExecuteCommand_ExpiredContextNeverDispatches`,
`TestExecuteCommand_AbandonedCommandSendsCancel`),
@@ -8117,7 +8126,8 @@ Proofs: `internal/agentexec/server_websocket_test.go`
`TestTypedOperation_TimeoutSendsCancelAndExpiredContextNeverDispatches`,
`TestTypedOperation_SocketDropAfterSendUnblocksDispatch`),
`internal/hostagent/operation_receipt_websocket_integration_test.go`
-(`TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfterReconnect`),
+(`TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfterReconnect`,
+`TestReceiptRunnerReadinessWaitsForActivationReconnect`),
`internal/hostagent/command_client_test.go`
(`TestCommandClient_handleCancelCommand_CancelsRegisteredRequest`,
`TestCommandClient_handleCancelCommand_UnknownRequestIsNoOp`,
diff --git a/internal/hostagent/operation_receipt_websocket_integration_test.go b/internal/hostagent/operation_receipt_websocket_integration_test.go
index 1d6a421e16..8f30729e5d 100644
--- a/internal/hostagent/operation_receipt_websocket_integration_test.go
+++ b/internal/hostagent/operation_receipt_websocket_integration_test.go
@@ -5,12 +5,14 @@ import (
"context"
"encoding/json"
"errors"
+ "fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
+ "sync/atomic"
"testing"
"time"
@@ -237,6 +239,132 @@ func TestRealServerAndUnifiedAgentWebSocketDockerDuplicateReplayMutatesOnce(t *t
}
}
+// Registration publishes a server session before the runner has activated its
+// credential, durably written health, and started reading command messages.
+// Observe each new client's actual health write, not a file left by its predecessor.
+func observeReceiptRunnerActivation(client *CommandClient) <-chan struct{} {
+ activated := make(chan struct{}, 1)
+ client.actionHealthWriter = func(ready bool) error {
+ if err := client.writeActionRunnerHealth(ready); err != nil {
+ return err
+ }
+ if ready {
+ select {
+ case activated <- struct{}{}:
+ default:
+ }
+ }
+ return nil
+ }
+ return activated
+}
+
+func waitForActivatedReceiptRunner(ctx context.Context, server *agentexec.Server, identity operationreceipt.Identity, activated <-chan struct{}, done <-chan error) error {
+ select {
+ case <-activated:
+ case err := <-done:
+ return fmt.Errorf("runner exited before activation: %v", err)
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ // An actual receipt-query round trip proves the activated client's reader is
+ // serving this session. It admits no mutation and changes no receipt.
+ if _, err := server.QueryAgentOperation(ctx, identity.AgentID, identity); err != nil {
+ return fmt.Errorf("activated runner receipt query: %w", err)
+ }
+ return nil
+}
+
+func TestReceiptRunnerReadinessWaitsForActivationReconnect(t *testing.T) {
+ admission := agentexec.AgentAdmission{
+ TokenID: "readiness-token", AgentID: "agent-ready", Hostname: "ready.example.test",
+ RuntimeRole: agentexec.RuntimeRoleActionRunner, ActionCapability: agentexec.ActionCapabilityTypedV1,
+ }
+ server := agentexec.NewServerWithAdmissionValidator(func(token, _, _ string) (agentexec.AgentAdmission, bool) {
+ return admission, token == admission.TokenID
+ }, func(agentexec.AgentAdmission) bool { return true })
+ activationEntered := make(chan struct{})
+ releaseActivation := make(chan struct{})
+ var releaseOnce sync.Once
+ var activationCalls atomic.Int32
+ httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+ if request.Method == http.MethodPatch && request.URL.Path == "/api/agents/action-runner/credential" {
+ if activationCalls.Add(1) == 1 {
+ close(activationEntered)
+ select {
+ case <-releaseActivation:
+ w.WriteHeader(http.StatusServiceUnavailable)
+ case <-request.Context().Done():
+ }
+ return
+ }
+ w.WriteHeader(http.StatusNoContent)
+ return
+ }
+ server.HandleWebSocket(w, request)
+ }))
+ t.Cleanup(httpServer.Close)
+ t.Cleanup(func() { releaseOnce.Do(func() { close(releaseActivation) }) })
+ origReconnectDelay := reconnectDelay
+ reconnectDelay = 50 * time.Millisecond
+ t.Cleanup(func() { reconnectDelay = origReconnectDelay })
+ dir := t.TempDir()
+ logger := zerolog.Nop()
+ client := NewActionRunnerClient(ActionRunnerClientConfig{
+ PulseURL: httpServer.URL, APIToken: admission.TokenID, StateDir: dir,
+ HealthPath: filepath.Join(dir, "health.json"), ActivationNonce: strings.Repeat("b", 32), Logger: &logger,
+ }, admission.AgentID, admission.Hostname, "test")
+ activated := observeReceiptRunnerActivation(client)
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan error, 1)
+ go func() { done <- client.Run(ctx); close(done) }()
+ t.Cleanup(func() {
+ cancel()
+ _ = client.Close()
+ select {
+ case <-done:
+ case <-time.After(3 * time.Second):
+ t.Error("readiness runner did not stop")
+ }
+ })
+ select {
+ case <-activationEntered:
+ case <-time.After(3 * time.Second):
+ t.Fatal("runner did not reach the controlled activation")
+ }
+ if !server.IsAgentConnected(admission.AgentID) {
+ t.Fatal("control must expose the registered but unactivated session")
+ }
+ digest, err := operationreceipt.DigestCanonicalJSON(map[string]string{"readiness": "query-only"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ identity := operationreceipt.Identity{AttemptID: "readiness.1", ActionID: "readiness", OperationKind: "fake.typed", OperationVersion: 1, RequestDigest: digest, AgentID: admission.AgentID}
+ blockedCtx, cancelBlocked := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ err = waitForActivatedReceiptRunner(blockedCtx, server, identity, activated, done)
+ cancelBlocked()
+ if !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("unactivated session passed readiness: %v", err)
+ }
+ releaseOnce.Do(func() { close(releaseActivation) })
+ readyCtx, cancelReady := context.WithTimeout(context.Background(), 3*time.Second)
+ defer cancelReady()
+ if err := waitForActivatedReceiptRunner(readyCtx, server, identity, activated, done); err != nil {
+ t.Fatalf("reconnected activated runner: %v", err)
+ }
+ if activationCalls.Load() != 2 {
+ t.Fatalf("activation attempts = %d, want failed first session and activated replacement", activationCalls.Load())
+ }
+ data, err := os.ReadFile(filepath.Join(dir, "health.json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var health actionRunnerHealth
+ if err := json.Unmarshal(data, &health); err != nil || !health.Registered || !health.Activated {
+ t.Fatalf("reconnected health = %+v, err=%v", health, err)
+ }
+}
+
func TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfterReconnect(t *testing.T) {
admission := agentexec.AgentAdmission{
TokenID: "runner-token", AgentID: "agent-pve", Hostname: "pve.example.test",
@@ -252,7 +380,7 @@ func TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfter
}
server.HandleWebSocket(w, request)
}))
- defer httpServer.Close()
+ t.Cleanup(httpServer.Close)
// A runner that drops its session must come back well inside the replay
// window rather than after the production reconnect backoff.
@@ -294,27 +422,15 @@ func TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfter
return nil, ctx.Err()
}
- startRunner := func(t *testing.T) (*CommandClient, context.CancelFunc, <-chan error) {
- t.Helper()
- client := NewActionRunnerClient(ActionRunnerClientConfig{
- PulseURL: httpServer.URL, APIToken: admission.TokenID, StateDir: stateDir,
- HealthPath: filepath.Join(stateDir, "health.json"), ActivationNonce: strings.Repeat("a", 32), Logger: &logger,
- }, admission.AgentID, admission.Hostname, "test")
- client.proxmoxGuestLifecycle = manager
- ctx, cancel := context.WithCancel(context.Background())
- done := make(chan error, 1)
- go func() { done <- client.Run(ctx) }()
- deadline := time.Now().Add(3 * time.Second)
- for !server.IsAgentConnected(admission.AgentID) && time.Now().Before(deadline) {
- time.Sleep(10 * time.Millisecond)
- }
- if !server.IsAgentConnected(admission.AgentID) {
- cancel()
- _ = client.Close()
- t.Fatal("action runner did not connect")
- }
- return client, cancel, done
+ request := agentexec.ProxmoxGuestLifecyclePayload{
+ RequestID: "pve.cancel-replay.1", ActionID: "pve.cancel-replay", Operation: "shutdown",
+ GuestKind: "vm", VMID: 101, ExpectedStatus: "running", Timeout: 30,
+ }
+ if err := agentexec.BindProxmoxGuestLifecyclePayload(&request); err != nil {
+ t.Fatal(err)
}
+ identity := agentexec.ProxmoxGuestLifecycleOperationIdentity(admission.AgentID, request)
+
stopRunner := func(t *testing.T, client *CommandClient, cancel context.CancelFunc, done <-chan error) {
t.Helper()
cancel()
@@ -322,7 +438,8 @@ func TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfter
select {
case <-done:
case <-time.After(3 * time.Second):
- t.Fatal("action runner did not stop")
+ t.Error("action runner did not stop")
+ return
}
// The client has exited, but the server observes the socket close on
// its own reader goroutine. Reconnecting before that lands would let
@@ -333,21 +450,40 @@ func TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfter
time.Sleep(10 * time.Millisecond)
}
if server.IsAgentConnected(admission.AgentID) {
- t.Fatal("server did not observe the action runner disconnect")
+ t.Error("server did not observe the action runner disconnect")
}
}
- request := agentexec.ProxmoxGuestLifecyclePayload{
- RequestID: "pve.cancel-replay.1", ActionID: "pve.cancel-replay", Operation: "shutdown",
- GuestKind: "vm", VMID: 101, ExpectedStatus: "running", Timeout: 30,
- }
- if err := agentexec.BindProxmoxGuestLifecyclePayload(&request); err != nil {
- t.Fatal(err)
+ startRunner := func(t *testing.T) func() {
+ t.Helper()
+ client := NewActionRunnerClient(ActionRunnerClientConfig{
+ PulseURL: httpServer.URL, APIToken: admission.TokenID, StateDir: stateDir,
+ HealthPath: filepath.Join(stateDir, "health.json"), ActivationNonce: strings.Repeat("a", 32), Logger: &logger,
+ }, admission.AgentID, admission.Hostname, "test")
+ client.proxmoxGuestLifecycle = manager
+ activated := observeReceiptRunnerActivation(client)
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan error, 1)
+ go func() {
+ done <- client.Run(ctx)
+ close(done)
+ }()
+ // Register cleanup before any readiness assertion. A failed start must
+ // not leave the runner alive while httptest waits for its handlers.
+ var stopOnce sync.Once
+ stop := func() { stopOnce.Do(func() { stopRunner(t, client, cancel, done) }) }
+ t.Cleanup(stop)
+ readyCtx, cancelReady := context.WithTimeout(context.Background(), 3*time.Second)
+ defer cancelReady()
+ if err := waitForActivatedReceiptRunner(readyCtx, server, identity, activated, done); err != nil {
+ t.Fatalf("action runner was not ready: %v", err)
+ }
+ return stop
}
- identity := agentexec.ProxmoxGuestLifecycleOperationIdentity(admission.AgentID, request)
- first, cancelFirst, firstDone := startRunner(t)
+ stopFirst := startRunner(t)
dispatchCtx, cancelDispatch := context.WithCancel(context.Background())
+ t.Cleanup(cancelDispatch)
dispatchDone := make(chan error, 1)
go func() {
_, err := server.ExecuteProxmoxGuestLifecycle(dispatchCtx, admission.AgentID, request)
@@ -355,6 +491,8 @@ func TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfter
}()
select {
case <-mutationStarted:
+ case err := <-dispatchDone:
+ t.Fatalf("Proxmox dispatch returned before mutation: %v", err)
case <-time.After(3 * time.Second):
t.Fatal("Proxmox mutation did not start")
}
@@ -385,14 +523,13 @@ func TestRealServerActionRunnerCancellationPersistsAndReplaysProxmoxReceiptAfter
if !canceled.MutationStarted || canceled.MutationCompleted || !strings.Contains(canceled.Error, "recovery inspection") {
t.Fatalf("canceled durable receipt = %+v", canceled)
}
- stopRunner(t, first, cancelFirst, firstDone)
+ stopFirst()
- second, cancelSecond, secondDone := startRunner(t)
- defer stopRunner(t, second, cancelSecond, secondDone)
- // The server publishes the session before the runner finishes activating,
- // so the replay can land on a session the runner then drops. That request
- // can never be answered; send the same request again on the session the
- // runner reconnects with. Every attempt must replay the durable receipt.
+ stopSecond := startRunner(t)
+ defer stopSecond()
+ // Readiness above proves activation and a live reader. Retain the existing
+ // bounded recovery for a genuinely dropped session after that proof: every
+ // attempt must replay the durable receipt without a second mutation.
var replayed *agentexec.ProxmoxGuestLifecycleResultPayload
var err error
for attempt := 1; ; attempt++ {