Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
fix(agent): retain update outcomes through recovery
rcourtmanbe6ef6cf4195
diff --git a/cmd/pulse-agent/main.go b/cmd/pulse-agent/main.goindex 20a7f5dc98..fb090b34a8 100644--- a/cmd/pulse-agent/main.go+++ b/cmd/pulse-agent/main.go@@ -128,11 +128,21 @@ func wireUpdaterHooks(hostCfg *hostagent.Config, updater *agentupdate.Updater) {hostCfg.OnServerVersion = updater.NudgeVersion}-func pendingUpdatePreviousVersion(pending *agentupdate.PendingPrivilegedUpdate) string {- if pending == nil {- return ""+// helperUpdatePreviousVersion supplies authoritative startup evidence for+// helper-backed installs. Only the activated executable may report its previous+// version. Other helper starts explicitly report no update, suppressing stale+// legacy markers even after a recovered handoff has been cleared. Nil retains+// the local updater's one-shot marker on installs without a typed helper.+func helperUpdatePreviousVersion(helperConfigured bool, pending *agentupdate.PendingPrivilegedUpdate, runningSHA256 string) *string {+ if !helperConfigured {+ return nil+ }+ previousVersion := ""+ runningSHA256 = strings.TrimSpace(runningSHA256)+ if pending != nil && runningSHA256 != "" && strings.EqualFold(runningSHA256, pending.Activation.ActiveSHA256) {+ previousVersion = pending.PreviousVersion}- return pending.PreviousVersion+ return &previousVersion}func currentCollectorExecutableSHA256() (string, error) {@@ -668,7 +678,7 @@ func run(ctx context.Context, args []string, getenv func(string) string) error {PrivilegeHelperStatus: privilegeHelperStatus,Observers: hostObserverTargets(cfg.Observers),PrivilegedTelemetry: privilegedTelemetry,- UpdatedFromVersion: pendingUpdatePreviousVersion(pendingUpdate),+ UpdatedFromVersion: helperUpdatePreviousVersion(privilegedUpdate != nil, pendingUpdate, pendingExecutableSHA256),DockerContainerUpdater: dockerUpdaterBridge,DockerContainerLifecycleOperator: dockerUpdaterBridge,diff --git a/cmd/pulse-agent/pending_update_evidence_test.go b/cmd/pulse-agent/pending_update_evidence_test.gonew file mode 100644index 0000000000..795d38588f--- /dev/null+++ b/cmd/pulse-agent/pending_update_evidence_test.go@@ -0,0 +1,52 @@+package main++import (+ "strings"+ "testing"++ "github.com/rcourtman/pulse-go-rewrite/internal/agenthelper"+ "github.com/rcourtman/pulse-go-rewrite/internal/agentupdate"+)++func TestHelperUpdatePreviousVersionRequiresActivatedBinary(t *testing.T) {+ activeSHA := strings.Repeat("a", 64)+ rollbackSHA := strings.Repeat("b", 64)+ pending := &agentupdate.PendingPrivilegedUpdate{+ Activation: agenthelper.UpdateResult{+ Action: "pending",+ ActivationID: "pulse-agent-0123456789abcdef0123456789abcdef:0123456789abcdef",+ ActiveSHA256: activeSHA,+ RollbackSHA256: rollbackSHA,+ },+ PreviousVersion: "1.0.0",+ }+ cases := []struct {+ name string+ helper bool+ pending *agentupdate.PendingPrivilegedUpdate+ running string+ want string+ }{+ {"LocalUpdateUsesLegacyEvidence", false, nil, "", ""},+ {"HelperWithoutHandoffOverridesLegacyEvidence", true, nil, activeSHA, ""},+ {"ActivatedBinaryRunning", true, pending, activeSHA, "1.0.0"},+ {"ActivatedBinaryRunningCaseAndSpace", true, pending, " " + strings.ToUpper(activeSHA) + "\n", "1.0.0"},+ {"RollbackBinaryRunning", true, pending, rollbackSHA, ""},+ {"UnknownBinaryRunning", true, pending, strings.Repeat("c", 64), ""},+ {"RunningIdentityUnavailable", true, pending, "", ""},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got := helperUpdatePreviousVersion(tc.helper, tc.pending, tc.running)+ if !tc.helper {+ if got != nil {+ t.Fatal("local update lost its legacy evidence source")+ }+ return+ }+ if got == nil || *got != tc.want {+ t.Fatalf("helper evidence = %v, want explicit %q", got, tc.want)+ }+ })+ }+}diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.mdindex df9628a04c..0a3932108e 100644--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md@@ -1282,12 +1282,38 @@ typed commit. Buffered or observer reports do not satisfy that floor. Anexpired, interrupted, invalid, or uncommitted activation rolls back duringhelper recovery or through the deadline watchdog; commit and rollback removetheir fixed staging/quarantine artifacts durably. The first accepted report-retains the previous-version update evidence from that handoff.+retains previous-version evidence only while the running executable matches+the activated candidate. Helper-backed startup owns an explicit update-evidence+override even when it is empty or no pending handoff remains. Recovered rollback+and subsequent helper starts therefore cannot consume a stale local update+marker as fresh success. Installs without the typed helper retain the local+updater's one-shot marker.Deadline recovery retains the exact pending identity and retries transientstate-read, last-known-good restore, cleanup, and persistence failures with anonzero bounded backoff until the durable state becomes terminal. Each failedattempt is emitted to the helper log with the activation identity and retrydelay; a failed attempt never silently disarms automatic rollback.+Agent-authored updater state and its accepted-report success history pass+through the canonical unified-resource agent facet and typed host view into+monitoring and fleet diagnostics. Snapshot conversion must not discard that+evidence or share mutable status/timestamp pointers with a reader. This is+canonical L13 read-state plumbing for the L16 update lifecycle, not a second+host cache or a frontend reconstruction of an update outcome.++Updater restart recovery follows typed post-replacement state. A failed+restart after successful helper rollback keeps the restored process running+with its error status, including any separate failure to clear the handoff.+A failed rollback or a failed local exec after replacement requires service+manager handover. Server-controlled error text cannot request process exit.+Local non-Windows replacement syncs staged bytes and final executable mode+before renaming over the service path, then syncs its directory. Staged-sync+failure preserves the old executable. Directory-sync failure reports uncertain+durability and retains any backup while recording current visible replacement+evidence. Persistent appliance copies use the same file/rename/directory order.+The existing Windows two-rename missing-path crash gap remains a separate+platform limitation. Barrier ordering and model tests do not establish physical+power-loss durability on every filesystem or storage device.+Before staging or activation, the helper must establish that the signed targetis the Pulse agent Go command, that its declared and operational version equalsthe requested target, and that the target advances the installed version.@@ -10622,8 +10648,13 @@ target name with the same status the row badge shows and a one-line verdict.Reasons, repair and re-enrol guidance, uninstall and safe-profile guidance andthe credential reveal render as shared `InlineNotice`s; agent and securityfacts as a `DetailSectionTable`; the connection id and identity evidence sit-in a closed Details disclosure. No diagnosis, repair or credential flow-changes: this is presentation only.+in a closed Details disclosure. Target rows keep their canonical connection+key across fleet reads and reordering. New target values update the existing+row instead of remounting its disclosure, so operators keep evidence open and+keyboard focus while updater status changes. Removing the target disposes its+local disclosure state. `InfrastructureAgentDoctorPage.test.tsx` covers updated+success evidence, reorder, focus continuity and removal. No diagnosis, repair+or credential authority changes.The agent profiles settings panel reads AI availability from`frontend-modern/src/stores/aiAvailability.ts` since 2026-10-22, when thediff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 6acce499e0..c0897f0364 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -5798,6 +5798,15 @@ an active process from an initialized monitoring source. The Kubernetes moduleuses the shared agent TLS constructor for custom CA and leaf-fingerprint trust;its Pulse transport must not regress to a Kubernetes-local insecure-only TLSconfiguration.+Updater evidence also survives canonical resource reads: host ingestion stores+the shared typed status on `AgentData`, resource clones and `HostView.AgentUpdate()`+own its optional timestamps, and `hostFromReadStateView` carries it into+`HostsSnapshot()`. Connection-ledger and Agent Doctor reads must preserve the+reported error and accepted one-shot success history, with no synthetic success+or timestamp refresh during projection. An absent status remains unknown.+`TestHostsSnapshotPreservesCanonicalUpdaterEvidence` covers failed-outcome+readback, reader isolation and replacement by newly accepted success; the+unified-resource adapter and clone tests cover the preceding ownership boundary.The same host report carries bounded OS package-update posture: supportedmanager, pending count, package/version identifiers, inspection time,reboot-required state, and an operator-safe inspection error. Monitoring ownsdiff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.jsonindex 009688e4c9..00863a1873 100644--- a/docs/release-control/v6/internal/subsystems/registry.json+++ b/docs/release-control/v6/internal/subsystems/registry.json@@ -1502,6 +1502,7 @@"cmd/pulse-agent/collector_lifecycle_test.go","cmd/pulse-agent/main_test.go","cmd/pulse-agent/observers_config_test.go",+ "cmd/pulse-agent/pending_update_evidence_test.go","scripts/installtests/agent_state_dir_lifecycle_test.go"]},@@ -1555,6 +1556,7 @@"internal/monitoring/monitor_host_agent_removal_lifecycle_test.go","internal/monitoring/monitor_host_agents_test.go","internal/monitoring/monitor_notification_startup_test.go",+ "internal/monitoring/monitor_snapshots_test.go","internal/monitoring/physical_disk_roundtrip_test.go","scripts/installtests/agent_state_dir_lifecycle_test.go","scripts/installtests/install_ps1_test.go",@@ -1573,7 +1575,8 @@"exact_files": ["internal/agentupdate/coverage_test.go","internal/agentupdate/update_hostagent_integration_test.go",- "internal/agentupdate/update_http_test.go"+ "internal/agentupdate/update_http_test.go",+ "internal/agentupdate/update_recovery_test.go"]},{@@ -7085,6 +7088,7 @@"internal/monitoring/monitor_notification_startup_test.go","internal/monitoring/monitor_pve_cluster_refresh_test.go","internal/monitoring/monitor_pve_guest_lxc_test.go",+ "internal/monitoring/monitor_snapshots_test.go","internal/monitoring/pbs_retryable_read_test.go","internal/monitoring/physical_disk_roundtrip_test.go","internal/monitoring/proxmox_large_cluster_poll_budget_test.go",diff --git a/docs/release-control/v6/internal/subsystems/security-privacy.md b/docs/release-control/v6/internal/subsystems/security-privacy.mdindex 51b2b30c9b..54d28c958d 100644--- a/docs/release-control/v6/internal/subsystems/security-privacy.md+++ b/docs/release-control/v6/internal/subsystems/security-privacy.md@@ -1302,7 +1302,12 @@ the `white_label` branding entitlement.toward accepted-report commit, the rollback identity may only acknowledgethe helper's idempotent rolled-back state and clear the stale handoff, andan unrelated executable identity must leave the handoff intact and fail- closed without mutation.+ closed without mutation. Helper-backed startup supplies an authoritative+ update-evidence override even when no update took effect or the recovered+ handoff was already cleared. It must not fall back to a stale local update+ marker and manufacture first-report success. Only typed post-replacement+ restart failure can trigger service-manager handover; upstream error text+ does not carry process-exit authority.Node-local LXC filesystem capacity is a distinct automatic host telemetryboundary, not inside-guest command authority. A PVE Unified Agent may usebounded `pct list` plus `pct df` only for guests reported running and maydiff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex db88246d4d..0f5f5a82b1 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -2504,6 +2504,17 @@ verification truth, or recovery evidence. Storage pressure and cleanup postureremain independently observed inputs, and the live authenticated agent-serverreadiness check remains authoritative before any action admission or dispatch.+Agent updater history on `AgentData` in `internal/unifiedresources/types.go`+is observational fleet metadata. Its shared typed status and optional update+check, attempt and success timestamps survive canonical resource cloning and+host read views as owned copies. This must not alter a coexisting storage or+recovery facet or become evidence of backup completeness, protection health,+restored data, restore readiness or successful recovery. A helper activation+commit or an accepted one-shot agent update report proves only that updater+outcome. `TestCloneResourceOwnsAgentUpdaterStatusAndTimes` and+`TestHostUpdateEvidenceSurvivesCanonicalAdapterViewAndClone` pin metadata+ownership and isolation, without qualifying a storage or recovery operation.+Per-member cluster endpoint connection-address overrides accepted by`internal/api/configapi/config_node_handlers.go` (`clusterEndpointOverrides` on the nodeupdate payload writing `ClusterEndpoints[n].IPOverride`), including thediff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.mdindex 3f8bf3dac4..3d4484d117 100644--- a/docs/release-control/v6/internal/subsystems/unified-resources.md+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md@@ -1497,6 +1497,20 @@ The compact By date layout reserves enough age space for the full cautionary UnkCompact Coverage age cells use the existing metadata-size text and reclaim numeric-age gutters so the full Unknown word stays visible even beside a Job column at 320px; text-range checks account for cell padding, not only the outer box.+### Agent updater evidence remains canonical through fleet reads++The agent facet carries the shared typed `models.AgentUpdateStatus` value from+host ingestion through resource storage, cloning and `HostView.AgentUpdate()`.+Adapters and views own that value and its three optional timestamps. Neither+source model mutation nor a snapshot reader may mutate retained evidence.+Monitoring reconstructs host updater status from that canonical facet, so fleet+connections and Agent Doctor retain real errors and last-success evidence+instead of silently dropping it. An absent status remains unknown. This read+projection creates no new update, enrollment or command authority.+`TestHostUpdateEvidenceSurvivesCanonicalAdapterViewAndClone` and+`TestHostsSnapshotPreservesCanonicalUpdaterEvidence` cover retention, mutation+isolation and replacement of a failed outcome by a newly accepted success.+## Canonical Files1. `internal/unifiedresources/types.go`diff --git a/frontend-modern/browser-verification/bb2e08ea736c35c9.json b/frontend-modern/browser-verification/bb2e08ea736c35c9.jsonnew file mode 100644index 0000000000..526c77096e--- /dev/null+++ b/frontend-modern/browser-verification/bb2e08ea736c35c9.json@@ -0,0 +1,45 @@+{+ "version": 1,+ "base_sha": "1655845baed66397c016e3e3cf077eb5365bbc40",+ "verified_at": "2026-11-01T11:53:42Z",+ "result": "passed",+ "changed_paths": [+ "frontend-modern/src/components/Settings/InfrastructureAgentDoctorPage.tsx"+ ],+ "content_sha256": {+ "frontend-modern/src/components/Settings/InfrastructureAgentDoctorPage.tsx": "5f04c49da9ed919080b16904c0bd1fd09ca013b18c79aba8efcad7c34db25fa5"+ },+ "routes": [+ "/settings/infrastructure/agent-doctor"+ ],+ "viewports": [+ {+ "width": 1440,+ "height": 1000+ },+ {+ "width": 768,+ "height": 1000+ },+ {+ "width": 390,+ "height": 844+ }+ ],+ "states": [+ "Empty fleet and structured-diagnostics loading.",+ "Real source-built collector after signed-helper activation, repeated rollback and later restart. No fabricated success or stale local marker.",+ "Public error-report fixture retains its failure in structured diagnostics and connection-ledger fallback.",+ "Real signed-helper candidate commit after its first accepted report. Last successful update is present only after that accepted update.",+ "Open evidence retains its DOM identity, contents and keyboard focus through actual fleet polls at all three widths.",+ "Desktop, intermediate and phone pixels inspected, including wrapped success timestamps and matching visible header and data columns."+ ],+ "interactions": [+ "Status filter selected and restored. A lone open target closed and reopened by keyboard.",+ "Nested native disclosure opened with Enter and closed with Space. Focus remains on its summary. Escape and outside click retain the open native disclosure.",+ "Actual connections and diagnostics poll responses update the open evidence without remounting it or losing focus.",+ "Page reload followed by mouse reopening of evidence preserves the accepted update history.",+ "Owned diagnostics 503 exercises ledger fallback. Retry by keyboard restores the real diagnostics response.",+ "Successful-update label scrolled into view and inspected at desktop, intermediate and phone widths."+ ]+}diff --git a/frontend-modern/src/components/Settings/InfrastructureAgentDoctorPage.tsx b/frontend-modern/src/components/Settings/InfrastructureAgentDoctorPage.tsxindex 7ba742dc43..8f94afb563 100644--- a/frontend-modern/src/components/Settings/InfrastructureAgentDoctorPage.tsx+++ b/frontend-modern/src/components/Settings/InfrastructureAgentDoctorPage.tsx@@ -231,6 +231,12 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorPreturn props.targets.filter((target) => target.status === filter);});+ // Fleet reads replace target objects. Keep rendered records keyed by+ // connection identity so fresh evidence updates an open drawer in place.+ const targetsByKey = createMemo(+ () => new Map(visibleTargets().map((target) => [target.key, target])),+ );+// One target open at a time, like every other drawer table. A lone target// (the common case when a platform page deep-links one stale agent) starts// open so its diagnosis and update command are immediately visible; larger@@ -547,58 +553,62 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorP</TableRow></TableHeader><TableBody>- <For each={visibleTargets()}>- {(target) => {- const status = () => STATUS_PRESENTATION[target.status];- const lastSeen = () => formatLastSeen(target.lastSeen);+ <For each={Array.from(targetsByKey().keys())}>+ {(key) => {+ const initialTarget = targetsByKey().get(key)!;+ const target = createMemo(() => targetsByKey().get(key) ?? initialTarget);+ const status = () => STATUS_PRESENTATION[target().status];+ const lastSeen = () => formatLastSeen(target().lastSeen);const lastSeenRelative = () =>- formatRelativeTime(target.lastSeen ?? undefined, {+ formatRelativeTime(target().lastSeen ?? undefined, {compact: true,emptyText: '—',now: now(),});- const command = () =>- target.connection && target.commandPlatform+ const command = () => {+ const current = target();+ return current.connection && current.commandPlatform? operations.getAgentConnectionUpgradeCommand(- target.connection,- target.installFlags,- target.commandPlatform,- target.needsCredentialRepair,+ current.connection,+ current.installFlags,+ current.commandPlatform,+ current.needsCredentialRepair,): '';+ };const otherRepairs = () =>- (target.diagnostic?.repairActions ?? []).filter(+ (target().diagnostic?.repairActions ?? []).filter((action) =>action.code !== 'copy_upgrade_command' &&action.code !== 'repair_authentication' &&action.code !== 'allow_reenroll',);const canAllowReenroll = () =>- (target.diagnostic?.repairActions ?? []).some(+ (target().diagnostic?.repairActions ?? []).some((action) => action.code === 'allow_reenroll' && action.supported,);const uninstallHandoff = () =>- getInfrastructureAgentDoctorUninstallHandoff(target);- const expanded = () => isExpanded(target);- const detailRowId = () => doctorDetailRowId(target);+ getInfrastructureAgentDoctorUninstallHandoff(target());+ const expanded = () => isExpanded(target());+ const detailRowId = () => doctorDetailRowId(target());const factSections = (): DetailSection[] =>compactDetailSections([{label: 'Agent',rows: compactDetailRows([- makeDetailRow('System', target.contextLabel, { wrap: true }),- makeDetailRow('Version', target.currentVersion),- makeDetailRow('Target version', target.expectedVersion),+ makeDetailRow('System', target().contextLabel, { wrap: true }),+ makeDetailRow('Version', target().currentVersion),+ makeDetailRow('Target version', target().expectedVersion),makeDetailRow('Last seen',lastSeen() ? `${lastSeen()} (${lastSeenRelative()})` : undefined,{ wrap: true },),- makeDetailRow('Updater', target.updaterLabel, { wrap: true }),+ makeDetailRow('Updater', target().updaterLabel, { wrap: true }),makeDetailRow('Profile',- target.profileLabel- ? [target.profileLabel, target.profileVersionLabel]+ target().profileLabel+ ? [target().profileLabel, target().profileVersionLabel].filter(Boolean).join(' '): undefined,@@ -609,22 +619,22 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorP{label: 'Security',rows: compactDetailRows([- makeDetailRow('Privilege', target.privilegeLabel, { wrap: true }),- target.status !== 'removed'+ makeDetailRow('Privilege', target().privilegeLabel, { wrap: true }),+ target().status !== 'removed'? makeDetailRow('Safe collector',- target.safeCollector ? 'Confirmed' : 'Not confirmed',- { tone: target.safeCollector ? 'success' : 'warning' },+ target().safeCollector ? 'Confirmed' : 'Not confirmed',+ { tone: target().safeCollector ? 'success' : 'warning' },): null,- makeDetailRow('Action runner', target.actionRunnerPosture.join(' · '), {+ makeDetailRow('Action runner', target().actionRunnerPosture.join(' · '), {wrap: true,}),makeDetailRow('Runner credential',- target.actionRunnerCredentialEligible+ target().actionRunnerCredentialEligible? undefined- : target.actionRunnerCredentialBlockReason,+ : target().actionRunnerCredentialBlockReason,{ wrap: true },),]),@@ -635,35 +645,35 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorP{label: 'Connection',rows: compactDetailRows([- makeDetailRow('Connection ID', target.connectionId, { wrap: true }),+ makeDetailRow('Connection ID', target().connectionId, { wrap: true }),]),},{label: 'Identity evidence',- rows: compactDetailRows(target.evidence.map(evidenceDetailRow)),+ rows: compactDetailRows(target().evidence.map(evidenceDetailRow)),},]);return (<>- <TableRow class="cursor-pointer" onClick={() => toggleExpanded(target)}>+ <TableRow class="cursor-pointer" onClick={() => toggleExpanded(target())}><TableCell class="py-1.5 pl-3 pr-3"><div class="flex items-center gap-1.5"><SummaryRowActionButtonkind="disclosure"expanded={expanded()}- subjectLabel={`details for ${target.displayName}`}+ subjectLabel={`details for ${target().displayName}`}controlsId={detailRowId()}hideWhenRowTappableOnMobile={false}- onAction={() => toggleExpanded(target)}+ onAction={() => toggleExpanded(target())}/><span class="truncate text-sm font-medium text-base-content">- {target.displayName}+ {target().displayName}</span></div></TableCell><TableCell class="hidden px-3 py-1.5 sm:table-cell">- <span class="truncate text-xs text-muted">{target.contextLabel}</span>+ <span class="truncate text-xs text-muted">{target().contextLabel}</span></TableCell><TableCell class="px-3 py-1.5"><span@@ -673,10 +683,10 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorP</span></TableCell><TableCell class="hidden px-3 py-1.5 text-xs text-base-content sm:table-cell">- {target.currentVersion || '—'}+ {target().currentVersion || '—'}</TableCell><TableCell class="agent-doctor-target-column px-3 py-1.5 text-xs text-base-content">- {target.expectedVersion || '—'}+ {target().expectedVersion || '—'}</TableCell><TableCellclass="px-2 py-1.5 text-xs text-muted sm:px-3"@@ -691,21 +701,21 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorP<InlineDetailTableRowcellId={detailRowId()}colspan={6}- data-agent-doctor-detail-for={target.connectionId}+ data-agent-doctor-detail-for={target().connectionId}><InlineDetailPaneltestId="agent-doctor-detail"- detailFor={target.key}- title={target.displayName}+ detailFor={target().key}+ title={target().displayName}status={{ variant: status().variant, label: status().label }}- summary={doctorTargetSummary(target)}+ summary={doctorTargetSummary(target())}sections={[]}- onClose={() => closeTarget(target)}+ onClose={() => closeTarget(target())}>- <For each={target.reasons}>+ <For each={target().reasons}>{(reason) => (<InlineNotice- tone={target.status === 'critical' ? 'danger' : 'warning'}+ tone={target().status === 'critical' ? 'danger' : 'warning'}><p class="font-medium">{reason.message}</p><Show when={reason.evidence?.length}>@@ -717,20 +727,20 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorP)}</For>- <Show when={target.needsUpdate || target.needsCredentialRepair}>+ <Show when={target().needsUpdate || target().needsCredentialRepair}><Show- when={!target.commandBlockedReason}+ when={!target().commandBlockedReason}fallback={<InlineNotice tone="info">- {target.commandBlockedReason}+ {target().commandBlockedReason}</InlineNotice>}><Show- when={commandReadyForTarget(target)}+ when={commandReadyForTarget(target())}fallback={<InlineNotice tone="info">- {commandRequiresToken(target) && tokenGatedTargetCount() > 1+ {commandRequiresToken(target()) && tokenGatedTargetCount() > 1? 'Select this installation above and generate a separate credential to unlock its host-local repair command.': 'Generate a token to unlock this host-local repair command.'}</InlineNotice>@@ -740,7 +750,7 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorP<CommandCopyButtononClick={() => void copyCommand(command())}title="Copy host-local agent repair command"- label={`Copy ${target.needsCredentialRepair ? 'authentication repair' : 'update'} command for ${target.displayName}`}+ label={`Copy ${target().needsCredentialRepair ? 'authentication repair' : 'update'} command for ${target().displayName}`}/><pretabIndex={0}@@ -750,8 +760,8 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorP</pre><Showwhen={- commandRequiresToken(target) &&- target.commandPlatform !== 'windows'+ commandRequiresToken(target()) &&+ target().commandPlatform !== 'windows'}><p class="mt-2 text-xs text-muted">@@ -777,10 +787,10 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorPvariant="outline"size="sm"class="mt-2"- disabled={reenrollPendingId() === target.connectionId}- onClick={() => void allowReenroll(target)}+ disabled={reenrollPendingId() === target().connectionId}+ onClick={() => void allowReenroll(target())}>- {reenrollPendingId() === target.connectionId+ {reenrollPendingId() === target().connectionId? 'Allowing…': 'Allow re-enrol'}</Button>@@ -826,7 +836,7 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorP)}title="Copy host-local agent uninstall command"- label={`Copy ${entry.label} uninstall command for ${target.displayName}`}+ label={`Copy ${entry.label} uninstall command for ${target().displayName}`}/><pretabIndex={0}@@ -853,16 +863,16 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorPwrapValues/>- <Show when={target.safeProfileGuidance}>+ <Show when={target().safeProfileGuidance}>{(guidance) => <InlineNotice tone="info">{guidance()}</InlineNotice>}</Show><Showwhen={- !target.safeCollector &&- target.status !== 'removed' &&- target.commandPlatform === 'linux' &&- target.actionRunnerAgentId &&- target.actionRunnerHostname+ !target().safeCollector &&+ target().status !== 'removed' &&+ target().commandPlatform === 'linux' &&+ target().actionRunnerAgentId &&+ target().actionRunnerHostname}><div class="space-y-2 text-xs">@@ -931,21 +941,21 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorP</div></Show>- <Show when={target.actionRunnerCredentialEligible}>+ <Show when={target().actionRunnerCredentialEligible}><Buttontype="button"variant="outline"size="sm"- isLoading={runnerCredentialPendingKey() === target.key}+ isLoading={runnerCredentialPendingKey() === target().key}disabled={Boolean(runnerCredentialReveal()) ||- issuedRunnerCredentialKeys().has(target.key)+ issuedRunnerCredentialKeys().has(target().key)}- onClick={() => void issueActionRunnerCredential(target)}+ onClick={() => void issueActionRunnerCredential(target())}>- {issuedRunnerCredentialKeys().has(target.key)+ {issuedRunnerCredentialKeys().has(target().key)? 'Credential issued for this page session'- : target.actionRunnerCredentialAction === 'rotate'+ : target().actionRunnerCredentialAction === 'rotate'? 'Rotate action-runner credential': 'Issue one-time action-runner credential'}</Button>@@ -953,7 +963,7 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorP<Showwhen={- runnerCredentialReveal()?.targetKey === target.key+ runnerCredentialReveal()?.targetKey === target().key? runnerCredentialReveal(): null}@@ -963,8 +973,8 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorPconst installCommand = () =>buildActionRunnerInstallCommand({pulseUrl: operations.selectedAgentUrl(),- agentId: target.actionRunnerAgentId!,- hostname: target.actionRunnerHostname!,+ agentId: target().actionRunnerAgentId!,+ hostname: target().actionRunnerHostname!,insecure: operations.insecureMode(),customCaPath: operations.customCaPath(),});@@ -1003,7 +1013,7 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorP<CommandCopyButtononClick={() => void copyCommand(reveal().token)}title="Copy one-time action-runner credential"- label={`Copy action-runner credential for ${target.displayName}`}+ label={`Copy action-runner credential for ${target().displayName}`}/><pretabIndex={0}@@ -1055,9 +1065,9 @@ export const InfrastructureAgentDoctorPage: Component<InfrastructureAgentDoctorPtype="button"variant="outline"size="sm"- onClick={() => void copyReport([target])}+ onClick={() => void copyReport([target()])}>- Copy diagnostic report for {target.displayName}+ Copy diagnostic report for {target().displayName}</Button></InlineDetailPanel></InlineDetailTableRow>diff --git a/frontend-modern/src/components/Settings/__tests__/InfrastructureAgentDoctorPage.test.tsx b/frontend-modern/src/components/Settings/__tests__/InfrastructureAgentDoctorPage.test.tsxindex 544fb03b40..845784228a 100644--- a/frontend-modern/src/components/Settings/__tests__/InfrastructureAgentDoctorPage.test.tsx+++ b/frontend-modern/src/components/Settings/__tests__/InfrastructureAgentDoctorPage.test.tsx@@ -1,3 +1,4 @@+import { createSignal } from 'solid-js';import { fireEvent, render, screen, waitFor, within } from '@solidjs/testing-library';import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';import { InfrastructureAgentDoctorPage } from '../InfrastructureAgentDoctorPage';@@ -54,6 +55,52 @@ const targetFixture = (...overrides,});+describe('InfrastructureAgentDoctorPage updater evidence identity', () => {+ it('keeps evidence open and focused as fleet reads replace and reorder targets', async () => {+ const other = targetFixture({+ key: 'agent:host-2',+ connectionId: 'agent:host-2',+ displayName: 'host-2',+ });+ const [targets, setTargets] = createSignal([+ targetFixture({ evidence: ['Last update attempt: before-refresh'] }),+ other,+ ]);+ render(() => <InfrastructureAgentDoctorPage targets={targets()} />);+ fireEvent.click(screen.getByRole('button', { name: 'Expand details for host-1' }));+ const evidence = screen.getByTestId('agent-doctor-evidence') as HTMLDetailsElement;+ const summary = evidence.querySelector('summary')!;+ evidence.open = true;+ fireEvent(evidence, new Event('toggle'));+ summary.focus();+ await waitFor(() => expect(within(evidence).getByText('before-refresh')).toBeInTheDocument());++ setTargets([+ { ...other },+ targetFixture({+ currentVersion: '6.7.0',+ evidence: ['Last successful update: accepted-refresh'],+ }),+ ]);+ expect(+ screen.getByTestId('agent-doctor-evidence'),+ 'UPDATER_EVIDENCE_DISCLOSURE_REMOUNTED',+ ).toBe(evidence);+ expect(evidence.open).toBe(true);+ expect(summary).toHaveFocus();+ await waitFor(() => expect(within(evidence).getByText('accepted-refresh')).toBeInTheDocument());+ expect(within(evidence).queryByText('before-refresh')).not.toBeInTheDocument();+ expect(within(screen.getByTestId('agent-doctor-facts')).getByText('6.7.0')).toBeInTheDocument();++ setTargets([{ ...other }]);+ expect(evidence.isConnected).toBe(false);+ setTargets([targetFixture(), { ...other }]);+ const replacement = screen.getByTestId('agent-doctor-evidence') as HTMLDetailsElement;+ expect(replacement).not.toBe(evidence);+ expect(replacement.open).toBe(false);+ });+});+describe('InfrastructureAgentDoctorPage action-runner enrollment', () => {beforeEach(() => {vi.clearAllMocks();diff --git a/internal/agentupdate/coverage_test.go b/internal/agentupdate/coverage_test.goindex 1fb5b8aacc..14cdd9d810 100644--- a/internal/agentupdate/coverage_test.go+++ b/internal/agentupdate/coverage_test.go@@ -1328,9 +1328,14 @@ func TestPerformUpdateErrors(t *testing.T) {})t.Run("BackupRenameError", func(t *testing.T) {+ // Only the Windows path moves the running binary aside; elsewhere the+ // staged binary is renamed over it (update_recovery_test.go).+ origGOOS := runtimeGOOS+ t.Cleanup(func() { runtimeGOOS = origGOOS })+ runtimeGOOS = goOSWindowsu := newUpdaterForTest("https://example")_, execPath := writeTempExec(t)- data := testBinary()+ data := []byte{'M', 'Z', 0x90, 0x00, 0x01, 0x02, 0x03, 0x04}origRename := renameFnorigRestart := restartProcessFn@@ -1363,9 +1368,13 @@ func TestPerformUpdateErrors(t *testing.T) {})t.Run("ReplaceRenameError", func(t *testing.T) {+ // The second rename exists only on the Windows path (see above).+ origGOOS := runtimeGOOS+ t.Cleanup(func() { runtimeGOOS = origGOOS })+ runtimeGOOS = goOSWindowsu := newUpdaterForTest("https://example")_, execPath := writeTempExec(t)- data := testBinary()+ data := []byte{'M', 'Z', 0x90, 0x00, 0x01, 0x02, 0x03, 0x04}origRename := renameFnorigRestart := restartProcessFndiff --git a/internal/agentupdate/pending_update_persistence_other.go b/internal/agentupdate/pending_update_persistence_other.goindex eb8f8fe1d2..f99c03fec6 100644--- a/internal/agentupdate/pending_update_persistence_other.go+++ b/internal/agentupdate/pending_update_persistence_other.go@@ -8,6 +8,18 @@ func replacePendingUpdateFile(from, to string) error {return os.Rename(from, to)}+// syncUpdateFile flushes a closed file's data and inode metadata, including+// a mode set after it was written; fsync applies to the file, not the+// descriptor, so a read-only one suffices.+func syncUpdateFile(path string) error {+ file, err := os.Open(path)+ if err != nil {+ return err+ }+ defer file.Close()+ return file.Sync()+}+func syncUpdateDirectory(path string) error {dir, err := os.Open(path)if err != nil {diff --git a/internal/agentupdate/pending_update_persistence_windows.go b/internal/agentupdate/pending_update_persistence_windows.goindex b364f112c0..e83bb64fb1 100644--- a/internal/agentupdate/pending_update_persistence_windows.go+++ b/internal/agentupdate/pending_update_persistence_windows.go@@ -4,6 +4,7 @@ package agentupdateimport ("fmt"+ "os""golang.org/x/sys/windows")@@ -27,6 +28,17 @@ func replacePendingUpdateFile(from, to string) error {return nil}+// syncUpdateFile flushes a closed file's data. FlushFileBuffers needs a+// handle with write access; Windows has no executable mode bit to persist.+func syncUpdateFile(path string) error {+ file, err := os.OpenFile(path, os.O_WRONLY, 0)+ if err != nil {+ return err+ }+ defer file.Close()+ return file.Sync()+}+func syncUpdateDirectory(string) error {// MoveFileEx with WRITE_THROUGH is the Windows replacement durability// barrier. Windows does not expose a portable directory fsync equivalent.diff --git a/internal/agentupdate/update.go b/internal/agentupdate/update.goindex 4a36d15e2e..e1ef4b7e3b 100644--- a/internal/agentupdate/update.go+++ b/internal/agentupdate/update.go@@ -110,6 +110,11 @@ var (chmodFn = os.ChmodrenameFn = os.RenamecloseFileFn = func(f *os.File) error { return f.Close() }+ syncFileFn = syncUpdateFile+ syncDirFn = syncUpdateDirectory+ removeFileFn = os.Remove+ clearPendingUpdateFn = ClearPendingPrivilegedUpdate+ exitFn = os.ExitreadFileFn = os.ReadFilewriteFileFn = os.WriteFileretrySleepFn = sleepWithContext@@ -503,11 +508,13 @@ func (u *Updater) CheckAndUpdate(ctx context.Context) {Msg("failed to self-update agent")// If the binary was replaced but exec failed, the old process is still// running with a stale binary. Exit so systemd restarts us with the- // new binary. The "failed to restart" error comes from restartProcess()- // which is called AFTER the binary is atomically replaced on disk.- if strings.Contains(err.Error(), "failed to restart") {+ // new binary. Only the replacement paths mark that condition: a helper+ // update that rolled back has already restored the running binary, and+ // error text (which can carry server-supplied status) proves nothing.+ var restartErr *restartAfterReplaceError+ if errors.As(err, &restartErr) {u.logger.Error().Msg("Binary replaced but exec failed - exiting so service manager restarts with new binary")- os.Exit(1)+ exitFn(1)}return}@@ -839,6 +846,17 @@ func (u *Updater) validatePulseURL() error {return nil}+// restartAfterReplaceError reports a failed in-place restart while the binary+// on disk is no longer the one this process is running, so the process must+// exit for the service manager to start the replacement.+type restartAfterReplaceError struct {+ err error+}++func (e *restartAfterReplaceError) Error() string { return e.err.Error() }++func (e *restartAfterReplaceError) Unwrap() error { return e.err }+// performUpdate downloads and installs the new agent binary.func (u *Updater) performUpdate(ctx context.Context) error {return u.performUpdateForVersion(ctx, "")@@ -1025,29 +1043,53 @@ func (u *Updater) performUpdateWithExecPathForVersion(ctx context.Context, execPreturn err}- // Atomic replacement with backup (use realExecPath for rename operations)- backupPath := realExecPath + ".backup"- if err := renameFn(realExecPath, backupPath); err != nil {- return fmt.Errorf("failed to backup current binary: %w", err)+ // Make the staged bytes and their final executable mode durable before a+ // rename can expose them at the service path.+ if err := syncFileFn(tmpPath); err != nil {+ return fmt.Errorf("failed to sync staged binary: %w", err)}- if err := renameFn(tmpPath, realExecPath); err != nil {- // Restore backup on failure- if restoreErr := renameFn(backupPath, realExecPath); restoreErr != nil {- return errors.Join(- fmt.Errorf("failed to replace binary: %w", err),- fmt.Errorf("failed to restore backup binary: %w", restoreErr),- )+ // Use realExecPath for rename operations+ backupPath := realExecPath + ".backup"+ switch runtimeGOOS {+ case goOSWindows:+ // Windows cannot rename over a running image, so the running binary is+ // moved aside first and the service path stays empty until the second+ // rename. A crash in that gap leaves only the backup to restore by hand.+ if err := renameFn(realExecPath, backupPath); err != nil {+ return fmt.Errorf("failed to backup current binary: %w", err)+ }+ if err := renameFn(tmpPath, realExecPath); err != nil {+ // Restore backup on failure+ if restoreErr := renameFn(backupPath, realExecPath); restoreErr != nil {+ return errors.Join(+ fmt.Errorf("failed to replace binary: %w", err),+ fmt.Errorf("failed to restore backup binary: %w", restoreErr),+ )+ }+ return fmt.Errorf("failed to replace binary: %w", err)+ }+ default:+ // Renaming over the service path keeps a complete executable there at+ // every instant, and a failed rename leaves the running binary in place.+ if err := renameFn(tmpPath, realExecPath); err != nil {+ return fmt.Errorf("failed to replace binary: %w", err)}- return fmt.Errorf("failed to replace binary: %w", err)}- // Remove backup on success- if err := os.Remove(backupPath); err != nil && !errors.Is(err, os.ErrNotExist) {+ // Any backup stays until the directory entry is durable. A failed sync+ // leaves the replacement, and the record written below, without any+ // crash-durability guarantee.+ if err := syncDirFn(targetDir); err != nil {+ u.logger.Warn().Err(err).Str("path", targetDir).Msg("agentupdate.performUpdateWithExecPath: failed to sync binary directory; replacement durability unconfirmed, keeping backup")+ } else if err := removeFileFn(backupPath); err != nil && !errors.Is(err, os.ErrNotExist) {u.logger.Warn().Err(err).Str("path", backupPath).Msg("agentupdate.performUpdateWithExecPath: failed to remove backup binary")}- // Write previous version to a file so the agent can report "updated from X" on next start+ // Write previous version to a file so the agent can report "updated from X"+ // on next start. It is written even after a failed sync: the visible+ // replacement is complete, and skipping it would leave any earlier,+ // unconsumed record to misattribute this update.updateInfoPath := filepath.Join(targetDir, ".pulse-update-info")if err := writeFileFn(updateInfoPath, []byte(u.cfg.CurrentVersion), 0644); err != nil {u.logger.Warn().Err(err).Str("path", updateInfoPath).Msg("agentupdate.performUpdateWithExecPath: failed to write update info")@@ -1072,7 +1114,10 @@ func (u *Updater) performUpdateWithExecPathForVersion(ctx context.Context, execP}// Restart the process using platform-specific implementation- return restartProcessFn(execPath)+ if err := restartProcessFn(execPath); err != nil {+ return &restartAfterReplaceError{err: err}+ }+ return nil}func (u *Updater) performPrivilegedUpdate(ctx context.Context, execPath string, body io.Reader, contentLength int64, checksumHeader, signature, targetVersion string) error {@@ -1154,6 +1199,7 @@ func (u *Updater) performPrivilegedUpdate(ctx context.Context, execPath string,if err != nil {return fmt.Errorf("activate signed update through typed helper: %w", err)}+ binaryRestored := falserollbackActivation := func(cause error) error {rollbackCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)defer cancel()@@ -1164,7 +1210,11 @@ func (u *Updater) performPrivilegedUpdate(ctx context.Context, execPath string,if rolledBack.Action != "rolled_back" || !strings.EqualFold(rolledBack.ActiveSHA256, activation.RollbackSHA256) {return errors.Join(cause, errors.New("typed helper returned an invalid rollback result"))}- if clearErr := ClearPendingPrivilegedUpdate(u.cfg.StateDir); clearErr != nil {+ // The helper has put the previous binary back on disk. Clearing the+ // local handoff is a separate step: if it fails, the handoff is stale+ // rather than settled, and the returned error names that failure.+ binaryRestored = true+ if clearErr := clearPendingUpdateFn(u.cfg.StateDir); clearErr != nil {return errors.Join(fmt.Errorf("%w; update rolled back", cause), fmt.Errorf("clear pending update handoff: %w", clearErr))}return fmt.Errorf("%w; update rolled back", cause)@@ -1181,7 +1231,15 @@ func (u *Updater) performPrivilegedUpdate(ctx context.Context, execPath string,}cleaned = trueif err := restartProcessFn(execPath); err != nil {- return rollbackActivation(fmt.Errorf("failed to restart after helper activation: %w", err))+ restartErr := rollbackActivation(fmt.Errorf("failed to restart after helper activation: %w", err))+ if !binaryRestored {+ // The activated binary stays on disk with its pending handoff+ // persisted while this process still runs the previous binary.+ return &restartAfterReplaceError{err: restartErr}+ }+ // The running binary is the one on disk again, so exiting would only+ // discard this error, including any stale handoff it names.+ return restartErr}return rollbackActivation(errors.New("restart returned without replacing the current process"))}@@ -1235,10 +1293,19 @@ func (u *Updater) syncPersistentBinaryCopy(execPath, persistPath, platform strinu.logger.Warn().Err(err).Str("path", tmpPersist).Str("platform", platform).Msg("failed to chmod persistent binary")return}+ // The persistent copy is what a reboot restores, so its bytes and final+ // mode must be durable before the rename replaces the previous copy.+ if err := syncFileFn(tmpPersist); err != nil {+ u.logger.Warn().Err(err).Str("path", tmpPersist).Str("platform", platform).Msg("failed to sync persistent binary")+ return+ }if err := renameFn(tmpPersist, persistPath); err != nil {u.logger.Warn().Err(err).Str("path", persistPath).Str("platform", platform).Msg("failed to replace persistent binary")return}+ if err := syncDirFn(filepath.Dir(persistPath)); err != nil {+ u.logger.Warn().Err(err).Str("path", persistPath).Str("platform", platform).Msg("failed to sync persistent binary directory")+ }u.logger.Info().Str("path", persistPath).Str("platform", platform).Msg("updated persistent binary")}diff --git a/internal/agentupdate/update_recovery_test.go b/internal/agentupdate/update_recovery_test.gonew file mode 100644index 0000000000..ad7b6887d1--- /dev/null+++ b/internal/agentupdate/update_recovery_test.go@@ -0,0 +1,742 @@+package agentupdate++import (+ "bytes"+ "context"+ "errors"+ "fmt"+ "io"+ "net/http"+ "os"+ "path/filepath"+ "reflect"+ "strings"+ "testing"++ "github.com/rcourtman/pulse-go-rewrite/internal/securityutil"+)++var (+ recoveryELFBinary = append([]byte{0x7f, 'E', 'L', 'F'}, bytes.Repeat([]byte("x"), 128)...)+ recoveryPEBinary = append([]byte{'M', 'Z'}, bytes.Repeat([]byte("x"), 130)...)+)++// recoveryUpdateClient answers the version check and binary download in+// memory, so CheckAndUpdate runs end to end without a socket.+func recoveryUpdateClient(version string, binary []byte, downloadStatus int, downloadStatusText string) *http.Client {+ return &http.Client{Transport: roundTripperFunc(func(r *http.Request) (*http.Response, error) {+ header := make(http.Header)+ status, statusText, body := http.StatusOK, "200 OK", []byte(nil)+ switch r.URL.Path {+ case "/api/agent/version":+ body = []byte(fmt.Sprintf(`{"version":%q}`, version))+ case "/download/pulse-agent":+ if downloadStatus != http.StatusOK {+ status, statusText = downloadStatus, downloadStatusText+ break+ }+ header.Set(checksumSHA256Header, checksum(binary))+ header.Set(signatureHeader, "signed-update")+ body = binary+ default:+ status, statusText = http.StatusNotFound, "404 Not Found"+ }+ return &http.Response{+ StatusCode: status,+ Status: statusText,+ Header: header,+ Body: io.NopCloser(bytes.NewReader(body)),+ ContentLength: int64(len(body)),+ Request: r,+ }, nil+ })}+}++// stubRecoveryProcess fakes the process boundary: the executable path, the+// in-place restart (which returns restartErr) and the exit call.+func stubRecoveryProcess(t *testing.T, execPath string, restartErr error) *[]int {+ t.Helper()+ origExecutable, origEval, origRestart, origExit := osExecutableFn, evalSymlinksFn, restartProcessFn, exitFn+ t.Cleanup(func() {+ osExecutableFn, evalSymlinksFn, restartProcessFn, exitFn = origExecutable, origEval, origRestart, origExit+ })+ exits := &[]int{}+ osExecutableFn = func() (string, error) { return execPath, nil }+ evalSymlinksFn = func(path string) (string, error) { return path, nil }+ restartProcessFn = func(string) error { return restartErr }+ exitFn = func(code int) { *exits = append(*exits, code) }+ return exits+}++// isolateRecoveryPlatformProbes keeps a host's own Unraid or QNAP marker files+// from adding persistent-copy steps to a recorded replacement.+func isolateRecoveryPlatformProbes(t *testing.T) {+ t.Helper()+ missing := filepath.Join(t.TempDir(), "absent")+ origUnraid, origGetcfg, origQpkg := unraidVersionPath, qnapGetcfgPath, qnapQpkgConfPath+ t.Cleanup(func() {+ unraidVersionPath, qnapGetcfgPath, qnapQpkgConfPath = origUnraid, origGetcfg, origQpkg+ })+ unraidVersionPath, qnapGetcfgPath, qnapQpkgConfPath = missing, missing, missing+}++// useRecoveryPlatform selects which replacement path runs, whatever the host,+// and returns a payload whose magic that platform accepts.+func useRecoveryPlatform(t *testing.T, goos goOS) []byte {+ t.Helper()+ original := runtimeGOOS+ t.Cleanup(func() { runtimeGOOS = original })+ runtimeGOOS = goos+ if goos == goOSWindows {+ return recoveryPEBinary+ }+ return recoveryELFBinary+}++// newHelperRecoveryUpdater wires CheckAndUpdate to the fake typed helper with+// an in-place restart that fails after activation, as restartProcess reports it.+func newHelperRecoveryUpdater(t *testing.T, rollbackErr error) (*fakePrivilegedUpdate, *Updater, *[]int) {+ t.Helper()+ binary := useRecoveryPlatform(t, goOSLinux)+ helper := &fakePrivilegedUpdate{root: t.TempDir(), rollbackErr: rollbackErr}+ if err := securityutil.HardenPrivatePath(helper.root, 0o700); err != nil {+ t.Fatal(err)+ }+ exits := stubRecoveryProcess(t, "/usr/local/bin/pulse-agent", fmt.Errorf("failed to restart: %w", errors.New("exec refused")))+ u := New(Config{+ PulseURL: "https://pulse.example.com",+ AgentName: "pulse-agent",+ CurrentVersion: "1.0.0",+ StateDir: helper.root,+ PrivilegedUpdate: helper,+ })+ u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")+ u.selfTestFn = func(context.Context, string) error { return nil }+ return helper, u, exits+}++var helperRestartRollbackEvents = []string{"quarantine", "signature:signed-update", "stage", "activate", "cleanup", "rollback"}++// A helper-backed update whose in-place restart fails and is rolled back has+// restored the running binary. Base matched "failed to restart" in that error+// and called os.Exit(1), so the failure was never reported and the restarted+// agent began with an idle, error-free status.+func TestCheckAndUpdateReportsRolledBackHelperRestartFailureWithoutExiting(t *testing.T) {+ helper, u, exits := newHelperRecoveryUpdater(t, nil)++ u.CheckAndUpdate(context.Background())++ if len(*exits) != 0 {+ t.Fatalf("exit codes = %v, want none after the helper restored the running binary", *exits)+ }+ status := u.Snapshot()+ if status.State != UpdateStateError || status.LastSuccessAt != nil || !strings.Contains(status.LastError, "update rolled back") {+ t.Fatalf("status after rolled-back helper update = %+v", status)+ }+ if !reflect.DeepEqual(helper.events, helperRestartRollbackEvents) {+ t.Fatalf("helper events = %#v, want %#v", helper.events, helperRestartRollbackEvents)+ }+ if pending, err := LoadPendingPrivilegedUpdate(helper.root); err != nil || pending != nil {+ t.Fatalf("pending handoff after rollback = %#v, %v", pending, err)+ }+}++// A binary restored on disk is not a settled transaction. When clearing the+// local handoff fails after the helper rolled back, nothing on disk differs+// from the running binary, so the process keeps running and reports both+// facts; the stale handoff stays visible instead of being treated as settled.+func TestCheckAndUpdateReportsStaleHandoffWhenClearFailsAfterRollback(t *testing.T) {+ helper, u, exits := newHelperRecoveryUpdater(t, nil)+ origClear := clearPendingUpdateFn+ t.Cleanup(func() { clearPendingUpdateFn = origClear })+ clearPendingUpdateFn = func(string) error { return errors.New("fixture clear refused") }++ u.CheckAndUpdate(context.Background())++ if len(*exits) != 0 {+ t.Fatalf("exit codes = %v, want none: the binary on disk is the running one again", *exits)+ }+ status := u.Snapshot()+ if status.State != UpdateStateError || !strings.Contains(status.LastError, "update rolled back") ||+ !strings.Contains(status.LastError, "clear pending update handoff: fixture clear refused") {+ t.Fatalf("status after rollback with a failed clear = %+v", status)+ }+ if !reflect.DeepEqual(helper.events, helperRestartRollbackEvents) {+ t.Fatalf("helper events = %#v, want %#v", helper.events, helperRestartRollbackEvents)+ }+ pending, err := LoadPendingPrivilegedUpdate(helper.root)+ if err != nil || pending == nil || pending.Activation.ActivationID != helper.activation.ActivationID {+ t.Fatalf("stale handoff after failed clear = %#v, %v; want the rolled-back activation still recorded", pending, err)+ }+}++// When the helper cannot roll back, the activated binary stays on disk under a+// persisted pending handoff while this process runs the old one, so the exit+// that hands over to the service manager is kept.+func TestCheckAndUpdateExitsWhenHelperCannotRollBackRestartFailure(t *testing.T) {+ helper, u, exits := newHelperRecoveryUpdater(t, errors.New("helper unavailable"))++ u.CheckAndUpdate(context.Background())++ if !reflect.DeepEqual(*exits, []int{1}) {+ t.Fatalf("exit codes = %v, want [1]", *exits)+ }+ if status := u.Snapshot(); status.State != UpdateStateError || !strings.Contains(status.LastError, "typed helper rollback failed") {+ t.Fatalf("status after failed rollback = %+v", status)+ }+ pending, err := LoadPendingPrivilegedUpdate(helper.root)+ if err != nil || pending == nil || pending.Activation.ActivationID != helper.activation.ActivationID {+ t.Fatalf("pending handoff after failed rollback = %#v, %v", pending, err)+ }+}++// A download refused before any replacement leaves the running binary in+// place. Base exited anyway when the server's status text contained "failed+// to restart", dropping the error it had just recorded.+func TestCheckAndUpdateKeepsRunningWhenPreReplacementErrorMentionsRestart(t *testing.T) {+ isolateRecoveryPlatformProbes(t)+ binary := useRecoveryPlatform(t, goOSLinux)+ _, execPath := writeTempExec(t)+ exits := stubRecoveryProcess(t, execPath, nil)+ u := newUpdaterForTest("https://pulse.example.com")+ u.client = recoveryUpdateClient("1.1.0", binary, http.StatusNotFound, "404 failed to restart")++ u.CheckAndUpdate(context.Background())++ if len(*exits) != 0 {+ t.Fatalf("exit codes = %v, want none before any replacement", *exits)+ }+ if status := u.Snapshot(); status.State != UpdateStateError || !strings.Contains(status.LastError, "404 failed to restart") {+ t.Fatalf("status after refused download = %+v", status)+ }+ if got, err := os.ReadFile(execPath); err != nil || string(got) != "old-binary" {+ t.Fatalf("executable after refused download = %q, %v", got, err)+ }+}++type fsStep struct {+ op string // create, close, chmod, syncfile, rename, remove, writefile, syncdir or restart+ path, to string+ mode os.FileMode+ servicePath bool // whether the service path existed after the step+}++type fsRecorder struct {+ servicePath string+ steps []fsStep+ syncFileErr error+ syncDirErr error+ restartErr error+ failRename func(from, to string) bool+}++func (r *fsRecorder) record(op, path, to string, mode os.FileMode) {+ _, err := os.Lstat(r.servicePath)+ r.steps = append(r.steps, fsStep{op: op, path: path, to: to, mode: mode, servicePath: err == nil})+}++func (r *fsRecorder) ops() []string {+ ops := make([]string, 0, len(r.steps))+ for _, step := range r.steps {+ ops = append(ops, step.op)+ }+ return ops+}++// recordUpdateFS routes the replacement's filesystem seams through a recorder.+// Creates, closes, chmods, renames, removals and writes really happen inside+// t.TempDir(); the durability barriers and the restart only record, or return+// the configured fixture error.+func recordUpdateFS(t *testing.T, servicePath string) *fsRecorder {+ t.Helper()+ rec := &fsRecorder{servicePath: servicePath}+ origCreate, origClose, origChmod, origRename := createTempFn, closeFileFn, chmodFn, renameFn+ origRemove, origWrite, origSyncFile, origSyncDir, origRestart := removeFileFn, writeFileFn, syncFileFn, syncDirFn, restartProcessFn+ t.Cleanup(func() {+ createTempFn, closeFileFn, chmodFn, renameFn = origCreate, origClose, origChmod, origRename+ removeFileFn, writeFileFn, syncFileFn, syncDirFn, restartProcessFn = origRemove, origWrite, origSyncFile, origSyncDir, origRestart+ })+ createTempFn = func(dir, pattern string) (*os.File, error) {+ file, err := origCreate(dir, pattern)+ if err == nil {+ rec.record("create", file.Name(), "", 0)+ }+ return file, err+ }+ closeFileFn = func(file *os.File) error {+ err := origClose(file)+ if err == nil {+ rec.record("close", file.Name(), "", 0)+ }+ return err+ }+ chmodFn = func(path string, mode os.FileMode) error {+ err := origChmod(path, mode)+ if err == nil {+ rec.record("chmod", path, "", mode)+ }+ return err+ }+ renameFn = func(from, to string) error {+ if rec.failRename != nil && rec.failRename(from, to) {+ return errors.New("fixture rename refused")+ }+ err := origRename(from, to)+ if err == nil {+ rec.record("rename", from, to, 0)+ }+ return err+ }+ removeFileFn = func(path string) error {+ err := origRemove(path)+ if err == nil {+ rec.record("remove", path, "", 0)+ }+ return err+ }+ writeFileFn = func(path string, data []byte, mode os.FileMode) error {+ err := origWrite(path, data, mode)+ if err == nil {+ rec.record("writefile", path, "", mode)+ }+ return err+ }+ syncFileFn = func(path string) error {+ if rec.syncFileErr != nil {+ return rec.syncFileErr+ }+ rec.record("syncfile", path, "", 0)+ return nil+ }+ syncDirFn = func(dir string) error {+ if rec.syncDirErr != nil {+ return rec.syncDirErr+ }+ rec.record("syncdir", dir, "", 0)+ return nil+ }+ restartProcessFn = func(string) error {+ rec.record("restart", "", "", 0)+ return rec.restartErr+ }+ return rec+}++type modelInode struct {+ complete, durableComplete bool+ mode, durableMode os.FileMode+ updateInfo bool+}++func describeServiceInode(inode *modelInode, durable bool) string {+ if inode == nil {+ return "missing"+ }+ complete, mode := inode.complete, inode.mode+ if durable {+ complete, mode = inode.durableComplete, inode.durableMode+ }+ if !complete {+ return "naming incomplete bytes"+ }+ if mode&0o111 == 0 {+ return fmt.Sprintf("not executable (%v)", mode)+ }+ return "valid"+}++// firstPowerLossViolation replays recorded steps against a deliberately+// pessimistic crash model. It is an assumption, not a measured filesystem: a+// file's bytes and mode reach disk only through a sync of that file after they+// were set, and each rename, removal or new file reaches disk independently,+// in any combination, unless a later sync of its directory forced it. After+// every step the visible service path, and in every combination the service+// path on disk, must name a complete executable; update evidence on disk must+// never outlive the replacement it describes. It returns the first violation.+func firstPowerLossViolation(steps []fsStep, servicePath string) string {+ type nsOp struct {+ step fsStep+ inode *modelInode+ forced bool+ }+ for crash := 0; crash <= len(steps); crash++ {+ previous := &modelInode{complete: true, durableComplete: true, mode: 0o755, durableMode: 0o755}+ visible := map[string]*modelInode{servicePath: previous}+ var ops []nsOp+ for _, step := range steps[:crash] {+ switch step.op {+ case "create":+ visible[step.path] = &modelInode{mode: 0o600}+ ops = append(ops, nsOp{step: step, inode: visible[step.path], forced: true})+ case "writefile":+ visible[step.path] = &modelInode{complete: true, mode: step.mode, updateInfo: true}+ ops = append(ops, nsOp{step: step, inode: visible[step.path]})+ case "close":+ if inode := visible[step.path]; inode != nil {+ inode.complete = true+ }+ case "chmod":+ if inode := visible[step.path]; inode != nil {+ inode.mode = step.mode+ }+ case "syncfile":+ if inode := visible[step.path]; inode != nil {+ inode.durableComplete, inode.durableMode = inode.complete, inode.mode+ }+ case "rename":+ visible[step.to] = visible[step.path]+ delete(visible, step.path)+ ops = append(ops, nsOp{step: step})+ case "remove":+ delete(visible, step.path)+ ops = append(ops, nsOp{step: step})+ case "syncdir":+ for i := range ops {+ if filepath.Dir(ops[i].step.path) == step.path {+ ops[i].forced = true+ }+ }+ }+ }+ if state := describeServiceInode(visible[servicePath], false); state != "valid" {+ return fmt.Sprintf("after step %d the visible service path is %s", crash, state)+ }+ var optional []int+ for i, op := range ops {+ if !op.forced {+ optional = append(optional, i)+ }+ }+ for mask := 0; mask < 1<<len(optional); mask++ {+ onDisk := make([]bool, len(ops))+ for i, op := range ops {+ onDisk[i] = op.forced+ }+ for bit, i := range optional {+ onDisk[i] = mask&(1<<bit) != 0+ }+ durable := map[string]*modelInode{servicePath: previous}+ var applied []string+ for i, op := range ops {+ if !onDisk[i] {+ continue+ }+ switch op.step.op {+ case "create", "writefile":+ durable[op.step.path] = op.inode+ case "rename":+ if inode, ok := durable[op.step.path]; ok {+ durable[op.step.to] = inode+ delete(durable, op.step.path)+ }+ case "remove":+ delete(durable, op.step.path)+ }+ if op.step.op != "create" {+ applied = append(applied, op.step.op+" "+filepath.Base(op.step.path))+ }+ }+ service := durable[servicePath]+ if state := describeServiceInode(service, true); state != "valid" {+ return fmt.Sprintf("power loss after step %d with %v on disk leaves the service path %s", crash, applied, state)+ }+ for _, inode := range durable {+ if inode.updateInfo && service == previous {+ return fmt.Sprintf("power loss after step %d with %v on disk keeps update evidence for a lost replacement", crash, applied)+ }+ }+ }+ }+ return ""+}++// Off Windows the staged binary is renamed over the service path only after+// its bytes and final executable mode were synced, the directory is synced+// before any cleanup or update evidence, and so after any step, or a power+// loss in the model above, the service path names a complete executable.+// Base moved the running binary aside first, leaving the path empty, and+// synced nothing.+func TestLocalReplacementKeepsExecutableServicePathThroughPowerLoss(t *testing.T) {+ isolateRecoveryPlatformProbes(t)+ binary := useRecoveryPlatform(t, goOSLinux)+ _, execPath := writeTempExec(t)+ stubRecoveryProcess(t, execPath, nil)+ rec := recordUpdateFS(t, execPath)+ u := newUpdaterForTest("https://pulse.example.com")+ u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")++ if err := u.performUpdateWithExecPath(context.Background(), execPath); err != nil {+ t.Fatalf("performUpdateWithExecPath: %v", err)+ }++ if violation := firstPowerLossViolation(rec.steps, execPath); violation != "" {+ t.Fatalf("%s; steps=%+v", violation, rec.steps)+ }+ for i, step := range rec.steps {+ if !step.servicePath {+ t.Fatalf("service path missing after step %d: %+v", i, step)+ }+ }+ want := []string{"create", "close", "chmod", "syncfile", "rename", "syncdir", "writefile", "restart"}+ if got := rec.ops(); !reflect.DeepEqual(got, want) {+ t.Fatalf("replacement steps = %v, want %v", got, want)+ }+ if got, err := os.ReadFile(execPath); err != nil || !bytes.Equal(got, binary) {+ t.Fatalf("executable after update = %q, %v", got, err)+ }+}++// A staged binary that cannot be made durable is a pre-replacement failure:+// the running binary stays untouched and nothing is renamed or restarted.+func TestLocalReplacementStagedSyncFailureKeepsRunningBinary(t *testing.T) {+ isolateRecoveryPlatformProbes(t)+ binary := useRecoveryPlatform(t, goOSLinux)+ dir, execPath := writeTempExec(t)+ stubRecoveryProcess(t, execPath, nil)+ rec := recordUpdateFS(t, execPath)+ rec.syncFileErr = errors.New("fixture fsync refused")+ u := newUpdaterForTest("https://pulse.example.com")+ u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")++ err := u.performUpdateWithExecPath(context.Background(), execPath)+ if err == nil || !strings.Contains(err.Error(), "failed to sync staged binary") {+ t.Fatalf("performUpdateWithExecPath error = %v", err)+ }+ if got := rec.ops(); !reflect.DeepEqual(got, []string{"create", "close", "chmod"}) {+ t.Fatalf("steps after sync failure = %v, want create, close, chmod only", got)+ }+ if got, readErr := os.ReadFile(execPath); readErr != nil || string(got) != "old-binary" {+ t.Fatalf("executable after sync failure = %q, %v", got, readErr)+ }+ if leftovers, _ := filepath.Glob(filepath.Join(dir, "pulse-agent-*.tmp")); len(leftovers) != 0 {+ t.Fatalf("staged temp files left behind: %v", leftovers)+ }+ if _, statErr := os.Stat(filepath.Join(dir, ".pulse-update-info")); !errors.Is(statErr, os.ErrNotExist) {+ t.Fatalf("update info written after a pre-replacement failure: %v", statErr)+ }+}++// Off Windows a refused rename leaves the service path exactly as it was: it+// is never moved aside, so there is nothing to restore. Base moved it aside+// first and depended on a second rename to put it back.+func TestLocalReplacementRenameFailureNeverEmptiesServicePath(t *testing.T) {+ isolateRecoveryPlatformProbes(t)+ binary := useRecoveryPlatform(t, goOSLinux)+ dir, execPath := writeTempExec(t)+ stubRecoveryProcess(t, execPath, nil)+ rec := recordUpdateFS(t, execPath)+ rec.failRename = func(from, to string) bool { return to == execPath && strings.HasSuffix(from, ".tmp") }+ u := newUpdaterForTest("https://pulse.example.com")+ u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")++ err := u.performUpdateWithExecPath(context.Background(), execPath)+ if err == nil || !strings.Contains(err.Error(), "failed to replace binary") {+ t.Fatalf("performUpdateWithExecPath error = %v", err)+ }+ for i, step := range rec.steps {+ if !step.servicePath || (step.op == "rename" && step.path == execPath) {+ t.Fatalf("service path moved or missing at step %d: %+v", i, step)+ }+ }+ if got := rec.ops(); !reflect.DeepEqual(got, []string{"create", "close", "chmod", "syncfile"}) {+ t.Fatalf("steps after refused rename = %v", got)+ }+ if got, readErr := os.ReadFile(execPath); readErr != nil || string(got) != "old-binary" {+ t.Fatalf("executable after refused rename = %q, %v", got, readErr)+ }+ if _, statErr := os.Stat(execPath + ".backup"); !errors.Is(statErr, os.ErrNotExist) {+ t.Fatalf("backup created although the service path was never moved: %v", statErr)+ }+ if leftovers, _ := filepath.Glob(filepath.Join(dir, "pulse-agent-*.tmp")); len(leftovers) != 0 {+ t.Fatalf("staged temp files left behind: %v", leftovers)+ }+}++// A failed directory sync leaves the replacement without a durability+// guarantee, so cleanup waits and any backup stays. The visible replacement is+// complete and the restart follows, so the "updated from" record is still+// written, which also replaces an earlier record nobody consumed. r2 skipped+// the write and left that earlier record to misattribute this update. No+// crash-durability claim is made on this path.+func TestLocalReplacementDirectorySyncFailureKeepsBackupAndReplacesStaleEvidence(t *testing.T) {+ isolateRecoveryPlatformProbes(t)+ binary := useRecoveryPlatform(t, goOSLinux)+ dir, execPath := writeTempExec(t)+ if err := os.WriteFile(execPath+".backup", []byte("stale-backup"), 0o755); err != nil {+ t.Fatal(err)+ }+ infoPath := filepath.Join(dir, ".pulse-update-info")+ if err := os.WriteFile(infoPath, []byte("0.9.0"), 0o644); err != nil {+ t.Fatal(err)+ }+ stubRecoveryProcess(t, execPath, nil)+ rec := recordUpdateFS(t, execPath)+ rec.syncDirErr = errors.New("fixture directory fsync refused")+ u := newUpdaterForTest("https://pulse.example.com")+ u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")++ if err := u.performUpdateWithExecPath(context.Background(), execPath); err != nil {+ t.Fatalf("performUpdateWithExecPath: %v", err)+ }+ if got := rec.ops(); !reflect.DeepEqual(got, []string{"create", "close", "chmod", "syncfile", "rename", "writefile", "restart"}) {+ t.Fatalf("steps with an unsynced directory = %v", got)+ }+ if got, err := os.ReadFile(execPath); err != nil || !bytes.Equal(got, binary) {+ t.Fatalf("executable after update = %q, %v", got, err)+ }+ if got, err := os.ReadFile(execPath + ".backup"); err != nil || string(got) != "stale-backup" {+ t.Fatalf("backup after unsynced directory = %q, %v", got, err)+ }+ if got, err := os.ReadFile(infoPath); err != nil || string(got) != "1.0.0" {+ t.Fatalf("update info = %q, %v; want this update's previous version, not the stale record", got, err)+ }+}++// Windows cannot rename over a running image, so its path still moves the+// running binary aside and the service path is empty until the second rename.+// This pins that platform gap so it is not mistaken for solved; the barriers+// around it are the same as elsewhere.+func TestWindowsReplacementPinsMissingServicePathGap(t *testing.T) {+ isolateRecoveryPlatformProbes(t)+ binary := useRecoveryPlatform(t, goOSWindows)+ _, execPath := writeTempExec(t)+ stubRecoveryProcess(t, execPath, nil)+ rec := recordUpdateFS(t, execPath)+ u := newUpdaterForTest("https://pulse.example.com")+ u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")++ if err := u.performUpdateWithExecPath(context.Background(), execPath); err != nil {+ t.Fatalf("performUpdateWithExecPath: %v", err)+ }+ want := []string{"create", "close", "chmod", "syncfile", "rename", "rename", "syncdir", "remove", "writefile", "restart"}+ if got := rec.ops(); !reflect.DeepEqual(got, want) {+ t.Fatalf("windows replacement steps = %v, want %v", got, want)+ }+ if aside := rec.steps[4]; aside.path != execPath || aside.servicePath {+ t.Fatalf("step 4 = %+v, want the running binary moved aside leaving the service path empty", aside)+ }+ if violation := firstPowerLossViolation(rec.steps, execPath); !strings.Contains(violation, "visible service path is missing") {+ t.Fatalf("power-loss model = %q, want the documented missing-path gap", violation)+ }+ if got, err := os.ReadFile(execPath); err != nil || !bytes.Equal(got, binary) {+ t.Fatalf("executable after update = %q, %v", got, err)+ }+}++// The Windows path keeps restoring the moved-aside binary when the second+// rename fails, as base did.+func TestWindowsReplacementRestoresBackupWhenSecondRenameFails(t *testing.T) {+ isolateRecoveryPlatformProbes(t)+ binary := useRecoveryPlatform(t, goOSWindows)+ _, execPath := writeTempExec(t)+ stubRecoveryProcess(t, execPath, nil)+ rec := recordUpdateFS(t, execPath)+ rec.failRename = func(from, to string) bool { return to == execPath && strings.HasSuffix(from, ".tmp") }+ u := newUpdaterForTest("https://pulse.example.com")+ u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")++ err := u.performUpdateWithExecPath(context.Background(), execPath)+ if err == nil || !strings.Contains(err.Error(), "failed to replace binary") {+ t.Fatalf("performUpdateWithExecPath error = %v", err)+ }+ var renames []string+ for _, step := range rec.steps {+ switch step.op {+ case "rename":+ renames = append(renames, filepath.Base(step.path)+"->"+filepath.Base(step.to))+ case "restart":+ t.Fatalf("restart after a failed replacement: %+v", rec.steps)+ }+ }+ base := filepath.Base(execPath)+ if want := []string{base + "->" + base + ".backup", base + ".backup->" + base}; !reflect.DeepEqual(renames, want) {+ t.Fatalf("renames = %v, want %v", renames, want)+ }+ if got, readErr := os.ReadFile(execPath); readErr != nil || string(got) != "old-binary" {+ t.Fatalf("executable after restore = %q, %v", got, readErr)+ }+ if _, statErr := os.Stat(execPath + ".backup"); !errors.Is(statErr, os.ErrNotExist) {+ t.Fatalf("backup left after restore: %v", statErr)+ }+}++// The local path replaces the executable before restarting, so a failed exec+// still exits for the service manager to start the new binary.+func TestCheckAndUpdateExitsWhenLocalReplacementCannotRestart(t *testing.T) {+ isolateRecoveryPlatformProbes(t)+ binary := useRecoveryPlatform(t, goOSLinux)+ _, execPath := writeTempExec(t)+ exits := stubRecoveryProcess(t, execPath, nil)+ rec := recordUpdateFS(t, execPath)+ rec.restartErr = fmt.Errorf("failed to restart: %w", errors.New("exec format error"))+ u := newUpdaterForTest("https://pulse.example.com")+ u.client = recoveryUpdateClient("1.1.0", binary, http.StatusOK, "")++ u.CheckAndUpdate(context.Background())++ if !reflect.DeepEqual(*exits, []int{1}) {+ t.Fatalf("exit codes = %v, want [1]", *exits)+ }+ if ops := rec.ops(); len(ops) == 0 || ops[len(ops)-1] != "restart" {+ t.Fatalf("steps = %v, want the replacement to end in the restart attempt", ops)+ }+ if got, err := os.ReadFile(execPath); err != nil || !bytes.Equal(got, binary) {+ t.Fatalf("executable after local replacement = %q, %v", got, err)+ }+}++// The Unraid flash and QNAP persistent copies are what a reboot restores, so+// their bytes and final mode are synced before the rename and their directory+// after it; a copy that cannot be synced never replaces the previous one.+func TestPersistentBinaryCopySyncsFinalModeAroundRename(t *testing.T) {+ setup := func(t *testing.T) (string, string, *fsRecorder) {+ t.Helper()+ dir := t.TempDir()+ execPath := filepath.Join(dir, "running-agent")+ persistDir := filepath.Join(dir, "flash")+ if err := os.Mkdir(persistDir, 0o755); err != nil {+ t.Fatal(err)+ }+ persistPath := filepath.Join(persistDir, "pulse-agent")+ if err := os.WriteFile(execPath, []byte("new-binary"), 0o755); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(persistPath, []byte("old-binary"), 0o755); err != nil {+ t.Fatal(err)+ }+ return execPath, persistPath, recordUpdateFS(t, persistPath)+ }++ t.Run("SyncedAroundRename", func(t *testing.T) {+ execPath, persistPath, rec := setup(t)+ New(Config{}).syncPersistentBinaryCopy(execPath, persistPath, "unraid")+ want := []string{"create", "close", "chmod", "syncfile", "rename", "syncdir"}+ if got := rec.ops(); !reflect.DeepEqual(got, want) {+ t.Fatalf("persistent copy steps = %v, want %v", got, want)+ }+ if rec.steps[2].path != rec.steps[3].path || rec.steps[2].mode != 0o755 {+ t.Fatalf("sync %+v does not follow the final chmod %+v of the same file", rec.steps[3], rec.steps[2])+ }+ if violation := firstPowerLossViolation(rec.steps, persistPath); violation != "" {+ t.Fatalf("%s; steps=%+v", violation, rec.steps)+ }+ if got, err := os.ReadFile(persistPath); err != nil || string(got) != "new-binary" {+ t.Fatalf("persistent copy = %q, %v", got, err)+ }+ })++ t.Run("SyncFailureKeepsPreviousCopy", func(t *testing.T) {+ execPath, persistPath, rec := setup(t)+ rec.syncFileErr = errors.New("fixture fsync refused")+ New(Config{}).syncPersistentBinaryCopy(execPath, persistPath, "unraid")+ if got := rec.ops(); !reflect.DeepEqual(got, []string{"create", "close", "chmod"}) {+ t.Fatalf("persistent copy steps after sync failure = %v", got)+ }+ if got, err := os.ReadFile(persistPath); err != nil || string(got) != "old-binary" {+ t.Fatalf("persistent copy after sync failure = %q, %v", got, err)+ }+ if leftovers, _ := filepath.Glob(persistPath + "-persist-*.tmp"); len(leftovers) != 0 {+ t.Fatalf("persistent temp files left behind: %v", leftovers)+ }+ })+}diff --git a/internal/hostagent/agent.go b/internal/hostagent/agent.goindex 4903d8eab1..a23e76b78b 100644--- a/internal/hostagent/agent.go+++ b/internal/hostagent/agent.go@@ -109,9 +109,11 @@ type Config struct {// collected authoritative report is accepted. Buffered reports and observer// acknowledgements do not satisfy this startup-health signal.OnPrimaryReportAccepted func()- // UpdatedFromVersion is supplied by the durable helper-update handoff and- // is carried by the first report built by the replacement process.- UpdatedFromVersion string+ // UpdatedFromVersion overrides legacy local update evidence. Nil reads the+ // local marker. A non-nil value belongs to the typed helper, including an+ // empty value when no candidate update took effect. Helper-backed installs+ // must not fall back to stale local evidence after rollback or on restart.+ UpdatedFromVersion *stringCollector SystemCollector // Optional: override default system information collector (for testing)// PrivilegedTelemetry routes the exceptional SMART and Proxmox LXC@@ -409,8 +411,10 @@ func New(cfg Config) (*Agent, error) {const bufferCapacity = 60// Check if agent was recently auto-updated (only reported once per restart)- updatedFrom := strings.TrimSpace(cfg.UpdatedFromVersion)- if updatedFrom == "" {+ updatedFrom := ""+ if cfg.UpdatedFromVersion != nil {+ updatedFrom = strings.TrimSpace(*cfg.UpdatedFromVersion)+ } else {updatedFrom = updatedFromVersionFn()}if updatedFrom != "" {diff --git a/internal/hostagent/agent_new_test.go b/internal/hostagent/agent_new_test.goindex 682ac9957f..6b693346f1 100644--- a/internal/hostagent/agent_new_test.go+++ b/internal/hostagent/agent_new_test.go@@ -1112,6 +1112,7 @@ func TestNew_CarriesUpdatedFromIntoFirstV6Report(t *testing.T) {}func TestNew_PrefersDurableHelperUpdatedFromHandoff(t *testing.T) {+ previousVersion := "6.0.0"mc := &mockCollector{hostInfoFn: func(context.Context) (*gohost.InfoStat, error) {return &gohost.InfoStat{Hostname: "upgraded-host", HostID: "machine-id-1", KernelArch: runtime.GOARCH}, nil@@ -1120,7 +1121,7 @@ func TestNew_PrefersDurableHelperUpdatedFromHandoff(t *testing.T) {agent, err := New(Config{APIToken: "token",Collector: mc,- UpdatedFromVersion: "6.0.0",+ UpdatedFromVersion: &previousVersion,updatedFromVersionFn: func() string { return "stale-local-marker" },})if err != nil {@@ -1487,3 +1488,35 @@ func TestGetReliableMachineIDNormalizesWindowsBraces(t *testing.T) {t.Fatalf("GetReliableMachineID() = %q, want %q", got, want)}}++// An explicit absence belongs to the helper just as a candidate version does.+// It must suppress legacy evidence on both recovered and later helper starts.+func TestNew_HelperNoUpdateEvidenceNeverReadsLegacyMarker(t *testing.T) {+ previousVersion := ""+ for start := 0; start < 2; start++ {+ mc := &mockCollector{+ hostInfoFn: func(context.Context) (*gohost.InfoStat, error) {+ return &gohost.InfoStat{Hostname: "rolled-back-host", HostID: "rollback-host", KernelArch: runtime.GOARCH}, nil+ },+ }+ agent, err := New(Config{+ APIToken: "token",+ Collector: mc,+ UpdatedFromVersion: &previousVersion,+ updatedFromVersionFn: func() string {+ t.Error("helper startup consulted stale legacy update evidence")+ return "stale-local-marker"+ },+ })+ if err != nil {+ t.Fatal(err)+ }+ report, err := agent.buildReport(context.Background())+ if err != nil {+ t.Fatal(err)+ }+ if report.Agent.UpdatedFrom != "" {+ t.Fatalf("helper start %d reported a successful update from %q after rollback", start, report.Agent.UpdatedFrom)+ }+ }+}diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.goindex 6590db2f3c..11e3d7b8ee 100644--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@ -3930,6 +3930,7 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host {IntervalSeconds: view.IntervalSeconds(),LastSeen: lastSeen,AgentVersion: view.AgentVersion(),+ AgentUpdate: view.AgentUpdate(),IntegrationSource: view.IntegrationSource(),MachineID: view.MachineID(),CommandsEnabled: view.CommandsEnabled(),diff --git a/internal/monitoring/monitor_snapshots_test.go b/internal/monitoring/monitor_snapshots_test.goindex 02fe012790..07ec1779ed 100644--- a/internal/monitoring/monitor_snapshots_test.go+++ b/internal/monitoring/monitor_snapshots_test.go@@ -2,7 +2,11 @@ package monitoringimport ("context"+ "reflect""testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources""github.com/rcourtman/pulse-go-rewrite/internal/models""github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"@@ -147,3 +151,32 @@ func TestCollectSnapshotSizes(t *testing.T) {t.Fatalf("unexpected size: got %d want %d", got, want)}}++func TestHostsSnapshotPreservesCanonicalUpdaterEvidence(t *testing.T) {+ now := time.Now().UTC()+ checked, attempted := now.Add(-time.Minute), now.Add(-time.Second)+ failed := &models.AgentUpdateStatus{State: "error", AutoUpdate: true, AvailableVersion: "6.7.0", LastCheckedAt: &checked, LastAttemptAt: &attempted, LastError: "restart failed; helper rollback restored running executable"}+ host := models.Host{ID: "updater-host", Hostname: "updater.example.test", AgentVersion: "6.6.0", Status: "online", LastSeen: now, AgentUpdate: failed}+ registry := unifiedresources.NewRegistry(nil)+ registry.IngestSnapshot(models.StateSnapshot{Hosts: []models.Host{host}, LastUpdate: now})+ m := &Monitor{state: models.NewState(), resourceStore: unifiedresources.NewMonitorAdapter(registry)}+ hosts := m.HostsSnapshot()+ if len(hosts) != 1 || !reflect.DeepEqual(hosts[0].AgentUpdate, failed) {+ t.Fatalf("UPDATER_EVIDENCE_DROPPED: host snapshot=%+v", hosts)+ }+ hosts[0].AgentUpdate.LastError = "snapshot reader mutation"+ *hosts[0].AgentUpdate.LastAttemptAt = now.Add(time.Hour)+ again := m.HostsSnapshot()+ if len(again) != 1 || !reflect.DeepEqual(again[0].AgentUpdate, failed) {+ t.Fatalf("snapshot mutation changed canonical updater evidence: %+v", again)+ }+ succeeded := now.Add(time.Second)+ host.AgentVersion = "6.7.0"+ host.LastSeen = succeeded+ host.AgentUpdate = &models.AgentUpdateStatus{State: "idle", AutoUpdate: true, UpdatedFrom: "6.6.0", LastSuccessAt: &succeeded}+ registry.IngestSnapshot(models.StateSnapshot{Hosts: []models.Host{host}, LastUpdate: succeeded})+ current := m.HostsSnapshot()+ if len(current) != 1 || !reflect.DeepEqual(current[0].AgentUpdate, host.AgentUpdate) {+ t.Fatalf("accepted update outcome did not replace error evidence: %+v", current)+ }+}diff --git a/internal/unifiedresources/adapters.go b/internal/unifiedresources/adapters.goindex e6a6d12b60..323d7e14bb 100644--- a/internal/unifiedresources/adapters.go+++ b/internal/unifiedresources/adapters.go@@ -178,6 +178,7 @@ func resourceFromHost(host models.Host) (Resource, ResourceIdentity) {agent := &AgentData{AgentID: host.ID,AgentVersion: host.AgentVersion,+ AgentUpdate: cloneAgentUpdateStatus(host.AgentUpdate),Hostname: host.Hostname,MachineID: host.MachineID,TokenID: host.TokenID,diff --git a/internal/unifiedresources/adapters_test.go b/internal/unifiedresources/adapters_test.goindex ce47394269..2c6b918794 100644--- a/internal/unifiedresources/adapters_test.go+++ b/internal/unifiedresources/adapters_test.go@@ -2160,3 +2160,32 @@ func TestProviderURLHostnamePreservesIPv6(t *testing.T) {}}}++func TestHostUpdateEvidenceSurvivesCanonicalAdapterViewAndClone(t *testing.T) {+ expected := func() *models.AgentUpdateStatus {+ checked, attempted, succeeded := time.Unix(100, 0).UTC(), time.Unix(200, 0).UTC(), time.Unix(300, 0).UTC()+ return &models.AgentUpdateStatus{State: "error", AutoUpdate: true, UpdatedFrom: "6.6.0", AvailableVersion: "6.7.0",+ LastCheckedAt: &checked, LastAttemptAt: &attempted, LastSuccessAt: &succeeded, LastError: "helper restored the previous executable"}+ }+ host := models.Host{ID: "updater-host", Hostname: "updater.example.test", AgentVersion: "6.6.0", Status: "online", AgentUpdate: expected()}+ record := HostIngestRecord(host)+ view := NewHostView(&record.Resource)+ if got := view.AgentUpdate(); !reflect.DeepEqual(got, expected()) {+ t.Fatalf("canonical updater evidence=%+v, want %+v", got, expected())+ }+ host.AgentUpdate.LastError = "source mutated"+ *host.AgentUpdate.LastCheckedAt = time.Unix(999, 0)+ if got := view.AgentUpdate(); !reflect.DeepEqual(got, expected()) {+ t.Fatalf("source mutation changed canonical updater evidence: %+v", got)+ }+ got := view.AgentUpdate()+ got.LastError = "reader mutated"+ *got.LastAttemptAt = time.Unix(999, 0)+ *got.LastSuccessAt = time.Unix(999, 0)+ if got := view.AgentUpdate(); !reflect.DeepEqual(got, expected()) {+ t.Fatalf("view mutation changed canonical updater evidence: %+v", got)+ }+ if NewHostView(&Resource{Agent: &AgentData{}}).AgentUpdate() != nil {+ t.Fatal("absent updater status became evidence")+ }+}diff --git a/internal/unifiedresources/clone.go b/internal/unifiedresources/clone.goindex 5a4941a1d9..18b2c495a7 100644--- a/internal/unifiedresources/clone.go+++ b/internal/unifiedresources/clone.go@@ -270,6 +270,7 @@ func cloneAgentData(in *AgentData) *AgentData {return nil}out := *in+ out.AgentUpdate = cloneAgentUpdateStatus(in.AgentUpdate)if in.Memory != nil {memory := *in.Memoryout.Memory = &memory@@ -291,6 +292,17 @@ func cloneAgentData(in *AgentData) *AgentData {return &out}+func cloneAgentUpdateStatus(in *models.AgentUpdateStatus) *models.AgentUpdateStatus {+ if in == nil {+ return nil+ }+ out := *in+ out.LastCheckedAt = cloneTimePtr(in.LastCheckedAt)+ out.LastAttemptAt = cloneTimePtr(in.LastAttemptAt)+ out.LastSuccessAt = cloneTimePtr(in.LastSuccessAt)+ return &out+}+func cloneHostIdentityConflict(in *models.HostIdentityConflict) *models.HostIdentityConflict {if in == nil {return nildiff --git a/internal/unifiedresources/clone_test.go b/internal/unifiedresources/clone_test.goindex 42f75da95f..2a2b0dd2a6 100644--- a/internal/unifiedresources/clone_test.go+++ b/internal/unifiedresources/clone_test.go@@ -1111,3 +1111,22 @@ func TestCloneNativeRateObservation(t *testing.T) {t.Fatal("rehydrated scalar invented a sample time")}}++func TestCloneResourceOwnsAgentUpdaterStatusAndTimes(t *testing.T) {+ checked, attempted, succeeded := time.Unix(100, 0).UTC(), time.Unix(200, 0).UTC(), time.Unix(300, 0).UTC()+ resource := &Resource{Agent: &AgentData{AgentUpdate: &models.AgentUpdateStatus{+ State: "error", AutoUpdate: true, UpdatedFrom: "6.6.0", AvailableVersion: "6.7.0",+ LastCheckedAt: &checked, LastAttemptAt: &attempted, LastSuccessAt: &succeeded, LastError: "helper restored previous binary",+ }}}+ cloned := cloneResource(resource)+ if cloned.Agent.AgentUpdate == nil || cloned.Agent.AgentUpdate.LastError != resource.Agent.AgentUpdate.LastError {+ t.Fatal("clone dropped updater status")+ }+ cloned.Agent.AgentUpdate.LastError = "clone changed"+ *cloned.Agent.AgentUpdate.LastCheckedAt = time.Unix(999, 0)+ *cloned.Agent.AgentUpdate.LastAttemptAt = time.Unix(999, 0)+ *cloned.Agent.AgentUpdate.LastSuccessAt = time.Unix(999, 0)+ if resource.Agent.AgentUpdate.LastError != "helper restored previous binary" || !resource.Agent.AgentUpdate.LastCheckedAt.Equal(time.Unix(100, 0)) || !resource.Agent.AgentUpdate.LastAttemptAt.Equal(time.Unix(200, 0)) || !resource.Agent.AgentUpdate.LastSuccessAt.Equal(time.Unix(300, 0)) {+ t.Fatal("clone shares updater status or timestamp pointers")+ }+}diff --git a/internal/unifiedresources/types.go b/internal/unifiedresources/types.goindex 0567966f58..21262cecc6 100644--- a/internal/unifiedresources/types.go+++ b/internal/unifiedresources/types.go@@ -962,6 +962,9 @@ type AgentStorageCleanupMeta struct {type AgentData struct {AgentID string `json:"agentId,omitempty"`AgentVersion string `json:"agentVersion,omitempty"`+ // AgentUpdate retains the collector's typed updater evidence in the+ // canonical facet, including error and one-shot successful-update history.+ AgentUpdate *models.AgentUpdateStatus `json:"agentUpdate,omitempty"`// Stale is set when the agent has stopped reporting (its host was marked// offline by the staleness evaluator) even though the row itself may stay// online via another source such as the Proxmox API poll. It lets the UIdiff --git a/internal/unifiedresources/views.go b/internal/unifiedresources/views.goindex a1c31267eb..8d27847edb 100644--- a/internal/unifiedresources/views.go+++ b/internal/unifiedresources/views.go@@ -1221,6 +1221,15 @@ func (v HostView) AgentVersion() string {return v.r.Agent.AgentVersion}+// AgentUpdate returns owned updater evidence, so a snapshot consumer cannot+// mutate the registry's status or timestamps.+func (v HostView) AgentUpdate() *models.AgentUpdateStatus {+ if v.r == nil || v.r.Agent == nil {+ return nil+ }+ return cloneAgentUpdateStatus(v.r.Agent.AgentUpdate)+}+func (v HostView) AgentID() string {if v.r == nil || v.r.Agent == nil {return ""diff --git a/scripts/release_control/canonical_completion_guard_test.py b/scripts/release_control/canonical_completion_guard_test.pyindex 3c50ad6629..040d162608 100644--- a/scripts/release_control/canonical_completion_guard_test.py+++ b/scripts/release_control/canonical_completion_guard_test.py@@ -265,6 +265,7 @@ class CanonicalCompletionGuardTest(unittest.TestCase):"internal/monitoring/monitor_host_agent_removal_lifecycle_test.go","internal/monitoring/monitor_host_agents_test.go","internal/monitoring/monitor_notification_startup_test.go",+ "internal/monitoring/monitor_snapshots_test.go","internal/monitoring/physical_disk_roundtrip_test.go","scripts/installtests/agent_state_dir_lifecycle_test.go","scripts/installtests/install_ps1_test.go",@@ -455,6 +456,7 @@ class CanonicalCompletionGuardTest(unittest.TestCase):"internal/monitoring/monitor_host_agent_removal_lifecycle_test.go","internal/monitoring/monitor_host_agents_test.go","internal/monitoring/monitor_notification_startup_test.go",+ "internal/monitoring/monitor_snapshots_test.go","internal/monitoring/physical_disk_roundtrip_test.go","scripts/installtests/agent_state_dir_lifecycle_test.go","scripts/installtests/install_ps1_test.go",@@ -855,6 +857,7 @@ class CanonicalCompletionGuardTest(unittest.TestCase):"internal/agentupdate/coverage_test.go","internal/agentupdate/update_hostagent_integration_test.go","internal/agentupdate/update_http_test.go",+ "internal/agentupdate/update_recovery_test.go",],}],diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.pyindex ef15af04fa..61b94eb0f1 100644--- a/scripts/release_control/subsystem_lookup_test.py+++ b/scripts/release_control/subsystem_lookup_test.py@@ -4653,6 +4653,7 @@ class SubsystemLookupTest(unittest.TestCase):"internal/monitoring/monitor_host_agent_removal_lifecycle_test.go","internal/monitoring/monitor_host_agents_test.go","internal/monitoring/monitor_notification_startup_test.go",+ "internal/monitoring/monitor_snapshots_test.go","internal/monitoring/physical_disk_roundtrip_test.go","scripts/installtests/agent_state_dir_lifecycle_test.go","scripts/installtests/install_ps1_test.go",@@ -5054,6 +5055,7 @@ class SubsystemLookupTest(unittest.TestCase):"internal/agentupdate/coverage_test.go","internal/agentupdate/update_hostagent_integration_test.go","internal/agentupdate/update_http_test.go",+ "internal/agentupdate/update_recovery_test.go",],)