Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

19.8 KB
Raw Blame
[CmdletBinding()]
param (
    [ValidateSet('Full', 'InstallUpdate', 'PostRebootUninstall')]
    [string]$Phase = 'Full',
    [Parameter(Mandatory = $true)]
    [string]$ServerBinary,
    [string]$AgentV1,
    [Parameter(Mandatory = $true)]
    [string]$AgentV2,
    [Parameter(Mandatory = $true)]
    [string]$InstallerPath,
    [int]$Port = 17655,
    [switch]$ConfirmLifecycleMutation
)

$ErrorActionPreference = 'Stop'
if ($PSVersionTable.PSEdition -ne 'Desktop' -or $PSVersionTable.PSVersion.Major -ne 5 -or $PSVersionTable.PSVersion.Minor -ne 1) {
    throw 'Native Windows lifecycle proof requires Windows PowerShell 5.1.'
}
Write-Host "Lifecycle engine: Windows PowerShell $($PSVersionTable.PSVersion)"
$serviceName = 'PulseAgent'
$stateDir = Join-Path $env:ProgramData 'Pulse'
$logFile = Join-Path $stateDir 'pulse-agent.log'
$proofStatePath = Join-Path $stateDir 'windows-lifecycle-proof.json'
$baseUrl = "http://127.0.0.1:$Port"
$serverProcess = $null
$previousDisableAutoUpdate = $null
$restoreAutoUpdateAtExit = $false

function Assert-Administrator {
    $identity = [Security.Principal.WindowsIdentity]::GetCurrent()
    $principal = [Security.Principal.WindowsPrincipal]$identity
    if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) {
        throw 'Windows lifecycle proof must run from an elevated PowerShell session.'
    }
}

function Resolve-RequiredPath {
    param([string]$Path, [string]$Label)
    if ([string]::IsNullOrWhiteSpace($Path) -or -not (Test-Path $Path -PathType Leaf)) {
        throw "$Label does not exist: $Path"
    }
    return (Resolve-Path $Path).Path
}

function Stop-LifecycleServer {
    if ($null -eq $script:serverProcess) { return }
    if (-not $script:serverProcess.HasExited) {
        Stop-Process -Id $script:serverProcess.Id -Force -ErrorAction Stop
        if (-not $script:serverProcess.WaitForExit(5000)) {
            throw 'Owned lifecycle server did not stop before the cleanup deadline.'
        }
    }
    $script:serverProcess.Dispose()
    $script:serverProcess = $null
}

function Start-LifecycleServer {
    param(
        [string]$AgentPath,
        [ValidateSet('confirm', 'http-503', 'wrong-id', 'string-success', 'missing-id')]
        [string]$UninstallMode = 'confirm'
    )
    Stop-LifecycleServer
    $existingListeners = @(Get-NetTCPConnection -ErrorAction Stop | Where-Object {
        $_.LocalPort -eq $Port -and $_.State -eq 'Listen'
    })
    if ($existingListeners.Count -ne 0) {
        throw 'Lifecycle peer port is already owned; no request or replacement is permitted.'
    }
    $version = ((& $AgentPath --version 2>$null) | Select-Object -First 1).Trim()
    if ($LASTEXITCODE -ne 0) { throw "Lifecycle input version check failed (exit $LASTEXITCODE)." }
    if ([string]::IsNullOrWhiteSpace($version)) {
        throw "Could not read agent version from $AgentPath"
    }
    $arguments = @(
        '--listen', "127.0.0.1:$Port",
        '--agent-binary', ('"{0}"' -f $AgentPath),
        '--version', $version,
        '--uninstall-mode', $UninstallMode
    )
    $script:serverProcess = Start-Process -FilePath $script:resolvedServerBinary -ArgumentList $arguments -PassThru -NoNewWindow
    for ($i = 0; $i -lt 30; $i++) {
        if ($script:serverProcess.HasExited) {
            throw "Lifecycle server exited with code $($script:serverProcess.ExitCode)."
        }
        try {
            $response = Invoke-RestMethod -Uri "$baseUrl/api/version" -TimeoutSec 2
            if ($response.version -eq $version) {
                Assert-UninstallObservation -Mode $UninstallMode -Requests 0
                return $version
            }
        } catch {
        }
        Start-Sleep -Milliseconds 500
    }
    throw "Lifecycle server did not become ready at $baseUrl."
}

function Invoke-Installer {
    param([string]$Arguments, [string]$Label)
    $escapedInstaller = $script:resolvedInstallerPath.Replace("'", "''")
    $command = "& '$escapedInstaller' $Arguments"
    & $script:powerShellPath -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command $command
    if ($LASTEXITCODE -ne 0) {
        throw "$Label failed with exit code $LASTEXITCODE."
    }
}

function Wait-AgentReady {
    param([int]$TimeoutSeconds = 45)
    $deadline = (Get-Date).AddSeconds($TimeoutSeconds)
    do {
        try {
            $service = Get-Service -Name $serviceName -ErrorAction Stop
            $ready = Invoke-RestMethod -Uri 'http://127.0.0.1:9191/readyz' -TimeoutSec 2
            if ($service.Status -eq 'Running' -and $ready.ready -eq $true) {
                return
            }
        } catch {
        }
        Start-Sleep -Seconds 1
    } while ((Get-Date) -lt $deadline)
    throw 'PulseAgent did not reach local readiness before the timeout.'
}

function Assert-AgentRuntime {
    param([string]$ExpectedVersion)
    Wait-AgentReady
    $service = Get-CimInstance Win32_Service | Where-Object Name -eq $serviceName
    if ($null -eq $service -or $service.State -ne 'Running' -or $service.StartMode -ne 'Auto') {
        throw "Unexpected service state: $($service | ConvertTo-Json -Compress)"
    }
    $installedVersion = ((& "$env:ProgramFiles\Pulse\pulse-agent.exe" --version 2>$null) | Select-Object -First 1).Trim()
    if ($LASTEXITCODE -ne 0) { throw "Installed agent version check failed (exit $LASTEXITCODE)." }
    if ($installedVersion -ne $ExpectedVersion) {
        throw "Installed version is $installedVersion; expected $ExpectedVersion."
    }
    if ($service.PathName -notlike '*--log-file*' -or $service.PathName -notlike '*pulse-agent.log*') {
        throw "Service command does not carry the canonical log-file argument: $($service.PathName)"
    }
    if (-not (Test-Path $logFile) -or (Get-Item $logFile).Length -le 0) {
        throw "Agent log file is missing or empty: $logFile"
    }
    $logText = Get-Content $logFile -Raw
    $hasStartupEvent = $logText -like '*Starting Pulse Unified Agent*' -or $logText -like '*Pulse Agent service is running*'
    if (-not $hasStartupEvent -or $logText -notlike "*$ExpectedVersion*") {
        throw "Agent log does not contain startup evidence for $ExpectedVersion."
    }
    $recovery = (& sc.exe qfailure $serviceName 2>&1 | Out-String)
    if ($LASTEXITCODE -ne 0) { throw "Service recovery query failed (exit $LASTEXITCODE)." }
    if ([regex]::Matches($recovery, 'RESTART').Count -lt 3) {
        throw "Service recovery actions are incomplete: $recovery"
    }
    $failureFlag = (& sc.exe qfailureflag $serviceName 2>&1 | Out-String)
    if ($LASTEXITCODE -ne 0) { throw "Service failure-flag query failed (exit $LASTEXITCODE)." }
    if ($failureFlag -notmatch 'TRUE|1') {
        throw "Service non-crash recovery flag is not enabled: $failureFlag"
    }
    return [uint32]$service.ProcessId
}

function Assert-CrashRecovery {
    param([string]$ExpectedVersion)
    $previousPid = Assert-AgentRuntime -ExpectedVersion $ExpectedVersion
    Stop-Process -Id $previousPid -Force
    $deadline = (Get-Date).AddSeconds(45)
    do {
        Start-Sleep -Seconds 1
        $service = Get-CimInstance Win32_Service | Where-Object Name -eq $serviceName
        if ($null -ne $service -and $service.State -eq 'Running' -and [uint32]$service.ProcessId -ne $previousPid) {
            try {
                Wait-AgentReady -TimeoutSeconds 5
                return
            } catch {
            }
        }
    } while ((Get-Date) -lt $deadline)
    throw 'PulseAgent did not recover after its process was terminated.'
}

# Absence is a specific SCM result, not an empty/suppressed failed query.
function Get-LifecycleService {
    try {
        return Get-Service -Name $serviceName -ErrorAction Stop
    } catch {
        if ($_.FullyQualifiedErrorId.Split(',')[0] -eq 'NoServiceFoundForGivenName' -and
            $_.CategoryInfo.Category -eq [System.Management.Automation.ErrorCategory]::ObjectNotFound) {
            return $null
        }
        throw
    }
}

function Assert-LifecycleAgentAbsent {
    $remaining = Get-LifecycleService
    if ($null -ne $remaining) {
        $remaining.Dispose()
        throw 'PulseAgent service still exists after uninstall.'
    }
    if (Test-Path "$env:ProgramFiles\Pulse\pulse-agent.exe") {
        throw 'Pulse Agent binary still exists after uninstall.'
    }
    if (Test-Path $stateDir) {
        throw 'Pulse Agent state directory still exists after uninstall.'
    }
    # Enumerate first: a missing selected port must not turn access/query
    # failure into apparently successful listener removal.
    $listeners = @(Get-NetTCPConnection -ErrorAction Stop | Where-Object {
        $_.LocalPort -eq 9191 -and $_.State -eq 'Listen'
    })
    if ($listeners.Count -ne 0) {
        throw 'Pulse Agent readiness listener still exists after uninstall.'
    }
}

function Invoke-UninstallAndAssertClean {
    foreach ($label in @('uninstall', 'repeated uninstall')) {
        Invoke-SavedUninstall -Label $label -ExpectFailure $false
        Assert-LifecycleAgentAbsent
        Write-Host "${label}: independent SCM, binary, state and listener absence confirmed."
    }
}

function Assert-UninstallObservation {
    param([string]$Mode, [int]$Requests)
    $observed = Invoke-RestMethod -Uri "$baseUrl/__pulse_lifecycle__/uninstall" -TimeoutSec 2 -ErrorAction Stop
    if ($observed.mode -cne $Mode -or $observed.requests -ne $Requests -or
        $observed.authenticated -ne $Requests -or $observed.exactIdentity -ne $Requests) {
        throw 'Lifecycle peer did not observe the expected single saved-credential/exact-ID request count.'
    }
}

function Invoke-SavedUninstall {
    param([string]$Label, [bool]$ExpectFailure)
    # No URL, identity, token or TLS arguments and no ambient connection input:
    # only the real installer's saved connection is available to this child.
    $start = New-Object System.Diagnostics.ProcessStartInfo
    $start.FileName = $script:powerShellPath
    $escapedInstaller = $script:resolvedInstallerPath.Replace("'", "''")
    $start.Arguments = "-NoLogo -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command `"& '$escapedInstaller' -Uninstall `$true -NonInteractive `$true`""
    $start.UseShellExecute = $false
    $start.CreateNoWindow = $true
    $start.RedirectStandardOutput = $true
    $start.RedirectStandardError = $true
    foreach ($name in @('PULSE_URL', 'PULSE_TOKEN', 'PULSE_TOKEN_FILE', 'PULSE_AGENT_ID',
        'PULSE_HOSTNAME', 'PULSE_CACERT', 'PULSE_SERVER_FINGERPRINT', 'PULSE_INSECURE_SKIP_VERIFY')) {
        $start.EnvironmentVariables.Remove($name)
    }
    $child = New-Object System.Diagnostics.Process
    $child.StartInfo = $start
    $started = $false
    try {
        $started = $child.Start()
        if (-not $started) { throw 'Saved removal child did not start.' }
        $stdout = $child.StandardOutput.ReadToEndAsync()
        $stderr = $child.StandardError.ReadToEndAsync()
        if (-not $child.WaitForExit(45000)) {
            throw 'Saved removal did not return within its bounded unattended deadline.'
        }
        $tasks = [System.Threading.Tasks.Task[]]@($stdout, $stderr)
        if (-not [System.Threading.Tasks.Task]::WaitAll($tasks, 5000)) {
            throw 'Saved removal output did not close; result remains unverified.'
        }
        $output = $stdout.Result + $stderr.Result
        if ($output.Contains('lifecycle-private-response-must-not-appear') -or
            $output.Contains('windows-lifecycle-fixture-not-a-secret')) {
            throw 'Saved removal exposed credential or private-response fixture content.'
        }
        $expectedExit = 0
        if ($ExpectFailure) { $expectedExit = 1 }
        if ($child.ExitCode -ne $expectedExit) {
            throw "$Label returned exit $($child.ExitCode); expected $expectedExit."
        }
        if ($ExpectFailure -and ($output.Contains('Uninstallation complete.') -or
            -not $output.Contains('binary, saved identity, credentials and logs have been retained'))) {
            throw 'Unconfirmed removal lost its truthful retained-state diagnostic.'
        }
        Write-Host $output.Trim()
    } finally {
        if ($started -and -not $child.HasExited) {
            # Only this owned child is terminated, never a selected agent PID.
            $child.Kill()
            if (-not $child.WaitForExit(5000)) {
                throw 'Owned removal child cleanup is unconfirmed.'
            }
        }
        $child.Dispose()
    }
}

function Get-LifecycleRecoverySnapshot {
    foreach ($path in @("$env:ProgramFiles\Pulse\pulse-agent.exe", "$stateDir\token", "$stateDir\connection.env", $logFile)) {
        if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
            throw 'Installed recovery fixture is incomplete.'
        }
    }
    $files = @(Get-ChildItem -LiteralPath $stateDir -File -Recurse -Force -ErrorAction Stop)
    $files += Get-Item -LiteralPath "$env:ProgramFiles\Pulse\pulse-agent.exe" -ErrorAction Stop
    $snapshot = @{}
    foreach ($file in $files) {
        $snapshot[$file.FullName] = @{
            Hash = (Get-FileHash -LiteralPath $file.FullName -Algorithm SHA256 -ErrorAction Stop).Hash
            Acl = (Get-Acl -LiteralPath $file.FullName -ErrorAction Stop).Sddl
        }
    }
    return @{ Files = $snapshot; StateAcl = (Get-Acl -LiteralPath $stateDir -ErrorAction Stop).Sddl }
}

function Assert-LifecycleRecoveryRetained {
    param($Before)
    $service = Get-LifecycleService
    if ($null -ne $service) {
        $service.Dispose()
        throw 'Unconfirmed removal did not independently establish SCM absence.'
    }
    $listeners = @(Get-NetTCPConnection -ErrorAction Stop | Where-Object {
        $_.LocalPort -eq 9191 -and $_.State -eq 'Listen'
    })
    if ($listeners.Count -ne 0) { throw 'Agent listener remains after unconfirmed removal.' }
    $after = Get-LifecycleRecoverySnapshot
    if ($after.StateAcl -cne $Before.StateAcl -or $after.Files.Count -ne $Before.Files.Count) {
        throw 'Unconfirmed removal changed recovery files or directory access.'
    }
    foreach ($path in $Before.Files.Keys) {
        if (-not $after.Files.ContainsKey($path) -or
            $after.Files[$path].Hash -cne $Before.Files[$path].Hash -or
            $after.Files[$path].Acl -cne $Before.Files[$path].Acl) {
            throw 'Unconfirmed removal erased or changed installed recovery bytes or file access.'
        }
    }
}

function Invoke-SavedRemovalAcceptance {
    # Quiesce the owned fixture before taking byte/ACL witnesses, so the
    # agent's legitimate final log writes cannot look like erased evidence.
    $service = Get-LifecycleService
    if ($null -eq $service) { throw 'Installed removal fixture service is missing.' }
    $agentProcess = $null
    try {
        $serviceInfo = Get-CimInstance Win32_Service -ErrorAction Stop | Where-Object Name -eq $serviceName
        if ($null -eq $serviceInfo -or [uint32]$serviceInfo.ProcessId -eq 0) { throw 'Fixture service process is unknown.' }
        $agentProcess = Get-Process -Id $serviceInfo.ProcessId -ErrorAction Stop
        if ($agentProcess.Path -ine "$env:ProgramFiles\Pulse\pulse-agent.exe") { throw 'Fixture service process identity differs.' }
        $service.Stop()
        $service.WaitForStatus([System.ServiceProcess.ServiceControllerStatus]::Stopped,
            [TimeSpan]::FromSeconds(30))
        $service.Refresh()
        if ($service.Status -ne 'Stopped') { throw 'Fixture service is not confirmed stopped.' }
        if (-not $agentProcess.WaitForExit(10000)) { throw 'Fixture process has not independently quiesced.' }
    } finally {
        $service.Dispose()
        if ($null -ne $agentProcess) { $agentProcess.Dispose() }
    }
    $before = Get-LifecycleRecoverySnapshot
    foreach ($mode in @('http-503', 'wrong-id', 'string-success', 'missing-id')) {
        Start-LifecycleServer -AgentPath $script:resolvedAgentV2 -UninstallMode $mode | Out-Null
        Invoke-SavedUninstall -Label $mode -ExpectFailure $true
        Assert-UninstallObservation -Mode $mode -Requests 1
        Assert-LifecycleRecoveryRetained -Before $before
        Write-Host "PASS saved removal ${mode}: one authenticated exact-ID POST; SCM/listener absent; every recovery byte and ACL retained."
    }
    Start-LifecycleServer -AgentPath $script:resolvedAgentV2 -UninstallMode confirm | Out-Null
    Invoke-UninstallAndAssertClean
    Assert-UninstallObservation -Mode confirm -Requests 1
    Write-Host 'PASS saved removal confirmation and repeat: one authenticated exact-ID POST total; independent cleanup after both attempts.'
}

Assert-Administrator
if (-not $ConfirmLifecycleMutation) {
    throw 'Pass -ConfirmLifecycleMutation to acknowledge service installation, process termination, and uninstall on this dedicated Windows runner.'
}

$resolvedServerBinary = Resolve-RequiredPath $ServerBinary 'Lifecycle server binary'
$resolvedAgentV2 = Resolve-RequiredPath $AgentV2 'Version-two agent binary'
$resolvedInstallerPath = Resolve-RequiredPath $InstallerPath 'Windows installer'
$resolvedAgentV1 = $null
if ($Phase -ne 'PostRebootUninstall') {
    $resolvedAgentV1 = Resolve-RequiredPath $AgentV1 'Version-one agent binary'
}
$powerShellPath = (Get-Process -Id $PID).Path

try {
    if ($Phase -eq 'PostRebootUninstall') {
        $proofState = Get-Content $proofStatePath -Raw | ConvertFrom-Json
        $previousDisableAutoUpdate = $proofState.previousDisableAutoUpdate
        $restoreAutoUpdateAtExit = $true
        $versionV2 = Start-LifecycleServer -AgentPath $resolvedAgentV2
        Assert-AgentRuntime -ExpectedVersion $versionV2 | Out-Null
        Invoke-SavedRemovalAcceptance
        Write-Host 'Post-reboot persistence and uninstall proof passed.' -ForegroundColor Green
        return
    }

    # Fail closed on pre-existing state and observation errors before changing
    # the machine environment or starting any test-owned process.
    Assert-LifecycleAgentAbsent
    $previousDisableAutoUpdate = [Environment]::GetEnvironmentVariable('PULSE_DISABLE_AUTO_UPDATE', 'Machine')
    $restoreAutoUpdateAtExit = $true
    [Environment]::SetEnvironmentVariable('PULSE_DISABLE_AUTO_UPDATE', 'true', 'Machine')

    $versionV1 = Start-LifecycleServer -AgentPath $resolvedAgentV1
    Invoke-Installer -Label 'preflight' -Arguments "-Url '$baseUrl' -PreflightOnly `$true -NonInteractive `$true"
    Invoke-Installer -Label 'install' -Arguments "-Url '$baseUrl' -Token 'windows-lifecycle-fixture-not-a-secret' -AgentId 'windows-lifecycle-agent' -Hostname 'windows-lifecycle-agent' -EnableHost `$true -EnableDocker `$false -EnableKubernetes `$false -EnableProxmox `$false -EnableCommands `$false -NonInteractive `$true"
    Assert-AgentRuntime -ExpectedVersion $versionV1 | Out-Null

    $versionV2 = Start-LifecycleServer -AgentPath $resolvedAgentV2
    Invoke-Installer -Label 'update' -Arguments "-Url '$baseUrl' -Token 'windows-lifecycle-fixture-not-a-secret' -AgentId 'windows-lifecycle-agent' -Hostname 'windows-lifecycle-agent' -EnableHost `$true -EnableDocker `$false -EnableKubernetes `$false -EnableProxmox `$false -EnableCommands `$false -NonInteractive `$true"
    Assert-AgentRuntime -ExpectedVersion $versionV2 | Out-Null
    Assert-CrashRecovery -ExpectedVersion $versionV2

    if ($Phase -eq 'InstallUpdate') {
        [ordered]@{
            expectedVersion = $versionV2
            previousDisableAutoUpdate = $previousDisableAutoUpdate
        } | ConvertTo-Json | Set-Content $proofStatePath -Encoding ascii
        $restoreAutoUpdateAtExit = $false
        Write-Host 'Install, update, logging, and crash-recovery proof passed; reboot the VM and run PostRebootUninstall.' -ForegroundColor Green
        return
    }

    Restart-Service $serviceName -Force
    Assert-AgentRuntime -ExpectedVersion $versionV2 | Out-Null
    Invoke-SavedRemovalAcceptance
    Write-Host 'Full Windows service lifecycle proof passed.' -ForegroundColor Green
} finally {
    try {
        Stop-LifecycleServer
    } finally {
        if ($restoreAutoUpdateAtExit) {
            [Environment]::SetEnvironmentVariable('PULSE_DISABLE_AUTO_UPDATE', $previousDisableAutoUpdate, 'Machine')
        }
    }
}