Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Fail agent installation when startup verification fails

pulse-triage[bot]25db08ec78d0
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index 653ee593a4..d0b5362490 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -15,6 +15,26 @@
## Purpose
+### Unix installer startup failure is not successful completion
+
+Fresh and upgrade callers of `complete_installation_flow` require successful
+startup verification before reporting success. A missing process or expired
+readiness window returns the existing service-start failure exit 13 and the
+matching `installed_unhealthy` or `updated_unhealthy` JSON event with exitCode
+13, rather than a successful completion. Authentication rejection remains exit
+18; a ready process whose registration is not yet confirmed keeps its existing
+warning and registration-retry behaviour.
+
+Both fresh and upgrade failures show the supplied platform-specific log command,
+or a service-status/log hint when none is available. They stop before success
+and uninstall guidance, without deleting saved state or automatically relaxing
+service isolation, changing credentials or retrying installation. This is not
+binary rollback, sustained native health or original-host recovery.
+
+`TestAgentInstallerStartupCompletion` exercises the production verification,
+JSON writer and completion chain with confined process/readiness/registration
+observations, including disabled health checks and retained synthetic state.
+
### Slow Assistant JSON reads do not own work cancellation
The synchronous execute and Kubernetes-analysis routes retain current API-token
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md
index 7fba2f67d9..a738dd40c6 100644
--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md
+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md
@@ -16,6 +16,24 @@
## Purpose
+### Agent installer completion retains failed startup
+
+The shared Unix agent completion path must return service-start failure exit
+13 when startup verification fails, including the reinstall/upgrade path.
+Its existing unhealthy JSON codes carry the same nonzero exitCode. The binary
+and service files having been written do not establish successful installation.
+Both paths retain saved state and display their platform's read-only log hint;
+neither suggests uninstalling as diagnosis or falls through to completion.
+Authentication rejection remains exit 18, and healthy/pending-registration
+behaviour is unchanged. No automatic rollback, privilege expansion or service
+mutation is added to the failure handler.
+
+`TestAgentInstallerStartupCompletion` in the ordinary installer test inventory
+checks the production verification/completion/JSON chain, caller exit, both
+health-check modes, existing authentication refusal and synthetic state
+preservation. Confined observation doubles are not native service acceptance
+or proof of the cause of a reported offline host.
+
### Served agent installers publish as a complete script bundle
The archive and source server installers share one script-publication path.
diff --git a/scripts/install.sh b/scripts/install.sh
index 0f74abd676..2fa889a47a 100755
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -3591,15 +3591,19 @@ complete_installation_flow() {
exit "$EXIT_AUTH_REJECTED"
else
if [[ "$UPGRADE_MODE" == "true" ]]; then
- log_warn "Upgrade complete, but the agent may not be running correctly."
- json_event "complete" "updated_unhealthy" "Agent updated but not responding"
+ log_error "Agent binary was updated, but startup verification failed; upgrade is not complete."
+ json_event "complete" "updated_unhealthy" "Agent updated but not responding" "$EXIT_SERVICE_START_FAILED"
else
- log_warn "Installation complete, but the agent may not be running correctly."
- if [[ -n "$unhealthy_log_hint" ]]; then
- log_warn "Check logs: $unhealthy_log_hint"
- fi
- json_event "complete" "installed_unhealthy" "Agent installed but not responding"
+ log_error "Agent files were installed, but startup verification failed; installation is not complete."
+ json_event "complete" "installed_unhealthy" "Agent installed but not responding" "$EXIT_SERVICE_START_FAILED"
+ fi
+ if [[ -n "$unhealthy_log_hint" ]]; then
+ log_warn "Check logs: $unhealthy_log_hint"
+ else
+ log_warn "Check the agent service status and logs before attempting another update."
fi
+ log_warn "Do not uninstall the agent or delete its saved state to diagnose this failure."
+ exit "$EXIT_SERVICE_START_FAILED"
fi
if [[ -n "$SAVED_INSTALL_SCRIPT" ]]; then
diff --git a/scripts/installtests/install_sh_test.go b/scripts/installtests/install_sh_test.go
index e65d7ada42..15fff466a7 100644
--- a/scripts/installtests/install_sh_test.go
+++ b/scripts/installtests/install_sh_test.go
@@ -9155,3 +9155,165 @@ func TestInstallSHNASRemovalTreatsLiveUnraidGoShellAsUnknownOwner(t *testing.T)
})
}
}
+
+// Exercise the production verification/completion chain, including its real
+// JSON writer and caller's unconditional exit 0. Only process/readiness and
+// server-registration observations are doubles: no service, installer download
+// or customer state is operated by these tests.
+func TestAgentInstallerStartupCompletion(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("Unix installer startup completion")
+ }
+ functions := ""
+ for _, name := range []string{"json_event", "resolve_agent_health_url", "verify_agent_started", "complete_installation_flow"} {
+ functions += extractInstallShellFunction(t, name) + "\n"
+ }
+ for _, tc := range []struct {
+ name, hint, code string
+ upgrade, running, ready, healthDisabled bool
+ persisted bool
+ registration, exit int
+ }{
+ {name: "fresh-stopped", code: "installed_unhealthy", hint: "journalctl -u pulse-agent --no-pager -n 20", exit: 13},
+ {name: "upgrade-stopped", upgrade: true, persisted: true, code: "updated_unhealthy", hint: "journalctl -u pulse-agent --no-pager -n 20", exit: 13},
+ {name: "upgrade-alive-but-unready", upgrade: true, running: true, code: "updated_unhealthy", hint: "tail -f /fixture/agent.log", exit: 13},
+ {name: "disabled-health-stopped", upgrade: true, healthDisabled: true, code: "updated_unhealthy", exit: 13},
+ {name: "fresh-ready", running: true, ready: true, code: "installed"},
+ {name: "upgrade-ready", upgrade: true, running: true, ready: true, persisted: true, code: "updated"},
+ {name: "registration-pending", upgrade: true, running: true, ready: true, registration: 1, code: "updated"},
+ {name: "disabled-health-running", upgrade: true, running: true, healthDisabled: true, code: "updated"},
+ {name: "credential-rejected", upgrade: true, running: true, ready: true, registration: 2, code: "auth_rejected", exit: 18},
+ {name: "disabled-health-credential-rejected", upgrade: true, running: true, healthDisabled: true, registration: 2, code: "auth_rejected", exit: 18},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ dir := t.TempDir()
+ // Failed completion must not propose deletion or mutate the already
+ // installed identity, credential, configuration or recovery script.
+ preserved := map[string]string{
+ "agent-id": "synthetic-existing-identity\n", "token": "synthetic-credential\n",
+ "connection.env": "synthetic-existing-configuration\n", "install.sh": "synthetic-retained-recovery-script\n",
+ }
+ for name, value := range preserved {
+ if err := os.WriteFile(filepath.Join(dir, name), []byte(value), 0600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ health := "127.0.0.1:9191"
+ if tc.healthDisabled {
+ health = ""
+ }
+ script := fmt.Sprintf(`set -euo pipefail
+OUTPUT_FORMAT=json
+EXIT_AUTH_REJECTED=18
+EXIT_SERVICE_START_FAILED=13
+UPGRADE_MODE=%t
+CONNECTION_INFO_PERSISTED=%t
+SAVED_INSTALL_SCRIPT=%q
+HEALTH_ADDR_SET=%t
+HEALTH_ADDR=%q
+LOG_FILE=%q
+save_connection_info() { printf 'SAVE-CONNECTION\n'; }
+sleep() { :; }
+curl() { return %d; }
+agent_process_running() { return %d; }
+verify_agent_server_registration_with_retry() { return %d; }
+warn_agent_token_rejected() { printf 'CREDENTIAL-REJECTED\n'; }
+report_proxmox_registration_outcome() { printf 'PROXMOX-OUTCOME\n'; }
+log_info() { printf 'INFO:%%s\n' "$*"; }
+log_warn() { printf 'WARN:%%s\n' "$*"; }
+log_error() { printf 'ERROR:%%s\n' "$*"; }
+`, tc.upgrade, tc.persisted, filepath.Join(dir, "install.sh"), tc.healthDisabled, health,
+ filepath.Join(dir, "absent.log"), agentStartupObservationExit(tc.ready), agentStartupObservationExit(tc.running), tc.registration) + functions + fmt.Sprintf(`
+complete_installation_flow %q 'INSTALL-SUCCESS' 'UPGRADE-SUCCESS' %q
+printf 'AFTER-COMPLETION\n'
+exit 0
+`, dir, tc.hint)
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ output, err := exec.CommandContext(ctx, "bash", "-c", script).CombinedOutput()
+ exit := 0
+ if err != nil {
+ var exitErr *exec.ExitError
+ if !errors.As(err, &exitErr) {
+ t.Fatalf("completion command: %v\n%s", err, output)
+ }
+ exit = exitErr.ExitCode()
+ }
+ if ctx.Err() != nil || exit != tc.exit {
+ t.Fatalf("completion exit = %d (%v), want %d\n%s", exit, ctx.Err(), tc.exit, output)
+ }
+ text := string(output)
+ var events []struct {
+ Phase, Code string
+ ExitCode int
+ }
+ for _, line := range strings.Split(text, "\n") {
+ if !strings.HasPrefix(line, "{") {
+ continue
+ }
+ var event struct {
+ Phase, Code string
+ ExitCode int
+ }
+ if err := json.Unmarshal([]byte(line), &event); err != nil {
+ t.Fatalf("invalid production JSON: %v\n%s", err, output)
+ }
+ events = append(events, event)
+ }
+ if len(events) != 1 || events[0].Phase != "complete" || events[0].Code != tc.code || events[0].ExitCode != tc.exit {
+ t.Fatalf("completion event = %+v, want complete/%s/%d\n%s", events, tc.code, tc.exit, output)
+ }
+ wantSaves := 1
+ if tc.persisted {
+ wantSaves = 0
+ }
+ if strings.Count(text, "SAVE-CONNECTION\n") != wantSaves {
+ t.Fatalf("saved connection persistence changed:\n%s", output)
+ }
+ if tc.exit != 0 {
+ for _, forbidden := range []string{"INSTALL-SUCCESS", "UPGRADE-SUCCESS", "AFTER-COMPLETION", "PROXMOX-OUTCOME", "To uninstall later:", "Upgrade complete", "Installation complete"} {
+ if strings.Contains(text, forbidden) {
+ t.Fatalf("failed completion emitted %q:\n%s", forbidden, output)
+ }
+ }
+ } else {
+ want := "INSTALL-SUCCESS"
+ if tc.upgrade {
+ want = "UPGRADE-SUCCESS"
+ }
+ for _, required := range []string{want, "AFTER-COMPLETION", "PROXMOX-OUTCOME", "To uninstall later:"} {
+ if !strings.Contains(text, required) {
+ t.Fatalf("successful completion missing %q:\n%s", required, output)
+ }
+ }
+ }
+ if tc.exit == 13 {
+ for _, required := range []string{"startup verification failed", "not complete", "Do not uninstall the agent or delete its saved state"} {
+ if !strings.Contains(text, required) {
+ t.Fatalf("failed startup missing %q:\n%s", required, output)
+ }
+ }
+ if tc.hint != "" && !strings.Contains(text, "Check logs: "+tc.hint) {
+ t.Fatalf("failed startup omitted its platform-specific log command:\n%s", output)
+ }
+ if tc.hint == "" && !strings.Contains(text, "Check the agent service status and logs") {
+ t.Fatalf("failed startup without a platform hint lost its diagnostic guidance:\n%s", output)
+ }
+ }
+ for name, value := range preserved {
+ got, err := os.ReadFile(filepath.Join(dir, name))
+ info, statErr := os.Stat(filepath.Join(dir, name))
+ if err != nil || statErr != nil || string(got) != value || info.Mode().Perm() != 0600 {
+ t.Fatalf("completion changed saved %s: read=%v stat=%v", name, err, statErr)
+ }
+ }
+ })
+ }
+}
+
+func agentStartupObservationExit(ok bool) int {
+ if ok {
+ return 0
+ }
+ return 1
+}