Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

windows: bound the MSI's beacon.exe calls and keep a log of each run (#1147)

Shukan Shah43472a11722b
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index dff66ee8..537641e7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -791,8 +791,9 @@ jobs:
windows-package-smoke:
runs-on: windows-2025
# Bounded for the same reason windows-test is: a wedged msiexec would otherwise hold a runner for
- # the GitHub default of six hours.
- timeout-minutes: 25
+ # the GitHub default of six hours. A healthy run takes about five minutes; the bound leaves room
+ # for one MSI step to run to its own limit (below) and for the diagnostics after it.
+ timeout-minutes: 40
defaults:
run:
shell: pwsh
@@ -866,7 +867,20 @@ jobs:
}
}
+ # Each MSI step has its own bound, inside the job's. A wedged msiexec then fails its step instead
+ # of holding the job until GitHub cancels it, and a cancelled job keeps no log at all -- which is
+ # how intermittent upgrade and uninstall hangs went undiagnosed. With the step bound, the
+ # diagnostics and the package logs below still run and upload.
+ #
+ # Each step bound is longer than the package scripts' own limits for the actions it runs
+ # (packaging/windows/install-endpoint.ps1 and uninstall-endpoint.ps1), so a hung beacon.exe call
+ # is stopped by the script -- which writes the TIMEOUT line and the goroutine stacks -- before
+ # GitHub stops the step. A step stopped first would leave the custom action running inside the
+ # Windows Installer service, outside the step's process tree, and record nothing. Install runs
+ # install (10 min) and user config (5 min): 17. An upgrade runs the old package's uninstall
+ # (3 min) before both: 20. Uninstall runs uninstall and its service fallback: 5.
- name: Install it
+ timeout-minutes: 17
run: |
# /qn is silent, and the verbose log is what makes a custom-action failure diagnosable --
# without it msiexec reports only an exit code.
@@ -915,6 +929,7 @@ jobs:
exit 0
- name: Upgrading must replace the payload and leave it running
+ timeout-minutes: 20
run: |
$configPath = Join-Path $env:ProgramData 'Beacon\Endpoint\config.json'
if (-not (Test-Path -LiteralPath $configPath)) { throw "no configuration before the upgrade" }
@@ -990,9 +1005,11 @@ jobs:
# hook reaches the log only through the collector's pipe, stamped as itself; it can neither read
# nor append to the log; it spools while the collector is stopped; and shared puts it all back.
- name: Restricted log access must hold against an ordinary account
+ timeout-minutes: 8
run: pwsh -NoProfile -File packaging/windows/smoke-restricted-log.ps1
- name: Uninstall must remove the service and keep the data
+ timeout-minutes: 5
run: |
$logDir = Join-Path $env:ProgramData 'Beacon\Endpoint\logs'
$configPath = Join-Path $env:ProgramData 'Beacon\Endpoint\config.json'
@@ -1058,3 +1075,74 @@ jobs:
Write-Output "marker survived: the operator's own configuration came back"
Write-Output "config retained: $(Test-Path -LiteralPath $configPath), logs retained: $(Test-Path -LiteralPath $logDir)"
exit 0
+
+ # The install and uninstall scripts bound every beacon.exe call and stop one that outlives its
+ # bound (packaging/windows/package-common.ps1), so a hang no longer wedges the MSI -- which also
+ # means it no longer fails a step on its own. Each script leaves a package log in SYSTEM's
+ # temporary directory, and a TIMEOUT line there is a hang this job must not pass over.
+ - name: Every package action must finish within its bound
+ if: always()
+ run: |
+ $logs = @(
+ foreach ($dir in @((Join-Path $env:SystemRoot 'Temp'), (Join-Path $env:SystemRoot 'SystemTemp'), $env:TEMP)) {
+ if ($dir -and (Test-Path -LiteralPath $dir)) {
+ Get-ChildItem -LiteralPath $dir -Filter 'beacon-package-*.log' -File -ErrorAction SilentlyContinue
+ }
+ }
+ ) | Sort-Object FullName -Unique | Sort-Object LastWriteTime
+ New-Item -ItemType Directory -Force -Path "$env:GITHUB_WORKSPACE\package-logs" | Out-Null
+ $timedOut = @()
+ foreach ($log in $logs) {
+ Write-Output "=== $($log.FullName) ==="
+ Get-Content -LiteralPath $log.FullName | Write-Output
+ Copy-Item -LiteralPath $log.FullName -Destination "$env:GITHUB_WORKSPACE\package-logs\" -Force
+ if (Select-String -LiteralPath $log.FullName -Pattern 'TIMEOUT:' -SimpleMatch -Quiet) { $timedOut += $log.Name }
+ }
+ Write-Output "package logs: $($logs.Count)"
+ if ($timedOut.Count -gt 0) {
+ throw "a package action outlived its bound and was stopped: $($timedOut -join ', ')"
+ }
+
+ - name: Diagnose a failed package step
+ if: failure()
+ run: |
+ Write-Output '--- processes ---'
+ Get-CimInstance Win32_Process |
+ Where-Object { $_.Name -match '^(beacon|msiexec|powershell|pwsh|cmd|sc|taskkill|net)' } |
+ Sort-Object ProcessId |
+ Format-Table -AutoSize ProcessId, ParentProcessId, Name, CreationDate, CommandLine |
+ Out-String -Width 500 | Write-Output
+ Write-Output '--- BeaconCollector ---'
+ & sc.exe queryex BeaconCollector 2>&1 | Write-Output
+ & sc.exe qc BeaconCollector 2>&1 | Write-Output
+ Write-Output '--- Windows Installer, collector and service control events ---'
+ foreach ($filter in @(
+ @{ LogName = 'Application'; ProviderName = 'MsiInstaller' },
+ @{ LogName = 'Application'; ProviderName = 'BeaconCollector' },
+ @{ LogName = 'System'; ProviderName = 'Service Control Manager' }
+ )) {
+ Get-WinEvent -FilterHashtable $filter -MaxEvents 25 -ErrorAction SilentlyContinue |
+ Sort-Object TimeCreated |
+ Format-List TimeCreated, ProviderName, Id, Message |
+ Out-String -Width 300 | Write-Output
+ }
+ foreach ($name in @('install.log', 'upgrade.log', 'uninstall.log')) {
+ $path = Join-Path $env:GITHUB_WORKSPACE $name
+ if (Test-Path -LiteralPath $path) {
+ Write-Output "--- $name (tail) ---"
+ Get-Content -LiteralPath $path -Tail 80 | Write-Output
+ }
+ }
+ $global:LASTEXITCODE = 0
+
+ - name: Upload the package logs
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: windows-package-logs
+ path: |
+ ${{ github.workspace }}/install.log
+ ${{ github.workspace }}/upgrade.log
+ ${{ github.workspace }}/uninstall.log
+ ${{ github.workspace }}/package-logs/
+ if-no-files-found: ignore
diff --git a/cli/beacon/cmd/debug_stacks.go b/cli/beacon/cmd/debug_stacks.go
new file mode 100644
index 00000000..db8ec8f5
--- /dev/null
+++ b/cli/beacon/cmd/debug_stacks.go
@@ -0,0 +1,45 @@
+package cmd
+
+import (
+ "fmt"
+ "io"
+ "os"
+ "runtime"
+ "time"
+)
+
+// debugStacksAfterEnv names a duration after which a beacon that is still running writes the stack
+// of every goroutine to stderr, then carries on.
+//
+// It exists for the one place a hang leaves nothing else behind: a Windows MSI custom action, which
+// has no console and whose output Windows Installer does not keep. The package scripts
+// (packaging/windows/package-common.ps1) run every beacon.exe call under a time bound, set this a
+// little inside it, and record stderr in their package log, so a call that wedges names the code it
+// is stuck in before it is stopped. Go on Windows has no SIGQUIT to ask for the same dump from
+// outside. Unset, or anything that is not a positive duration, does nothing.
+const debugStacksAfterEnv = "BEACON_DEBUG_STACKS_AFTER"
+
+// maxDebugStackBytes caps the dump. runtime.Stack truncates to the buffer it is given, so the
+// buffer grows until the dump fits or reaches this.
+const maxDebugStackBytes = 64 << 20
+
+func startDebugStackTimer(stderr io.Writer) *time.Timer {
+ after, err := time.ParseDuration(os.Getenv(debugStacksAfterEnv))
+ if err != nil || after <= 0 {
+ return nil
+ }
+ return time.AfterFunc(after, func() { writeGoroutineStacks(stderr, after) })
+}
+
+func writeGoroutineStacks(w io.Writer, after time.Duration) {
+ buf := make([]byte, 1<<20)
+ for {
+ n := runtime.Stack(buf, true)
+ if n < len(buf) || len(buf) >= maxDebugStackBytes {
+ buf = buf[:n]
+ break
+ }
+ buf = make([]byte, 2*len(buf))
+ }
+ fmt.Fprintf(w, "beacon: still running after %s (%s); the stack of every goroutine follows\n%s\n", after, debugStacksAfterEnv, buf)
+}
diff --git a/cli/beacon/cmd/debug_stacks_test.go b/cli/beacon/cmd/debug_stacks_test.go
new file mode 100644
index 00000000..d9830c06
--- /dev/null
+++ b/cli/beacon/cmd/debug_stacks_test.go
@@ -0,0 +1,70 @@
+package cmd
+
+import (
+ "bytes"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+)
+
+func TestDebugStackTimerIsOffUnlessAPositiveDurationIsSet(t *testing.T) {
+ for _, value := range []string{"", "soon", "0s", "-5s", "120"} {
+ t.Setenv(debugStacksAfterEnv, value)
+ if timer := startDebugStackTimer(&bytes.Buffer{}); timer != nil {
+ timer.Stop()
+ t.Fatalf("%s=%q started a timer; only a positive duration should", debugStacksAfterEnv, value)
+ }
+ }
+}
+
+// syncBuffer is written by the timer's goroutine and read by the test's.
+type syncBuffer struct {
+ mu sync.Mutex
+ buf bytes.Buffer
+}
+
+func (b *syncBuffer) Write(p []byte) (int, error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.buf.Write(p)
+}
+
+func (b *syncBuffer) String() string {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.buf.String()
+}
+
+func TestDebugStackTimerDumpsEveryGoroutineAndCarriesOn(t *testing.T) {
+ t.Setenv(debugStacksAfterEnv, "10ms")
+ release := make(chan struct{})
+ parked := make(chan struct{})
+ go func() {
+ close(parked)
+ parkedForDebugStackTest(release)
+ }()
+ <-parked
+ defer close(release)
+
+ out := &syncBuffer{}
+ timer := startDebugStackTimer(out)
+ if timer == nil {
+ t.Fatal("a positive duration started no timer")
+ }
+ defer timer.Stop()
+
+ deadline := time.Now().Add(5 * time.Second)
+ for !strings.Contains(out.String(), "parkedForDebugStackTest") {
+ if time.Now().After(deadline) {
+ t.Fatalf("no dump naming the parked goroutine within 5s; got:\n%s", out.String())
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ if !strings.Contains(out.String(), "still running after 10ms ("+debugStacksAfterEnv+")") {
+ t.Fatalf("the dump does not say why it was written:\n%s", out.String())
+ }
+}
+
+//go:noinline
+func parkedForDebugStackTest(release <-chan struct{}) { <-release }
diff --git a/cli/beacon/cmd/root.go b/cli/beacon/cmd/root.go
index c41be55b..3133df7a 100644
--- a/cli/beacon/cmd/root.go
+++ b/cli/beacon/cmd/root.go
@@ -236,6 +236,7 @@ var docsCmd = &cobra.Command{
// Execute runs the root command
func Execute() {
+ startDebugStackTimer(os.Stderr)
if err := rootCmd.Execute(); err != nil {
os.Exit(1)
}
diff --git a/docs/platforms/windows.mdx b/docs/platforms/windows.mdx
index 5aa265a3..dba4db48 100644
--- a/docs/platforms/windows.mdx
+++ b/docs/platforms/windows.mdx
@@ -23,6 +23,15 @@ msiexec /i BeaconEndpointAgent-<version>-x64.msi /qn /l*v install.log
The install needs administrator rights, because it writes under `%ProgramData%` and registers a service. Elevation is requested for you when you double-click the package. A silent install must already be running elevated, which is the normal case for Intune, SCCM, or a management agent.
+The `/l*v` log records what Windows Installer did, but not what Beacon's own install and uninstall steps printed. Those go to a separate log for each run, in the temporary directory of the account that ran the package (`C:\Windows\Temp` when a management agent installs as SYSTEM):
+
+```powershell title="Find the logs of Beacon's install and uninstall steps"
+Get-ChildItem "$env:SystemRoot\Temp", "$env:SystemRoot\SystemTemp" -Filter 'beacon-package-*.log' -ErrorAction SilentlyContinue |
+ Sort-Object LastWriteTime
+```
+
+Each step has a time limit: 10 minutes for the install, 5 for configuring the signed-in user's agent, and 3 for the uninstall. A step that runs past its limit is stopped, so the package finishes instead of hanging your deployment tool, and the log records the step and where it was stuck. An install stopped this way fails, so your deployment tool reports it. An uninstall stopped this way stops and removes the `BeaconCollector` service itself, so the removal or upgrade can finish.
+
<Warning>
**The installer is not code-signed yet.** Windows SmartScreen warns that the publisher is unknown, and you need to choose "More info" then "Run anyway". Verify what you downloaded against the published `.sha256` first:
diff --git a/packaging/windows/beacon.wxs b/packaging/windows/beacon.wxs
index f4d6978b..cc9e8326 100644
--- a/packaging/windows/beacon.wxs
+++ b/packaging/windows/beacon.wxs
@@ -73,6 +73,10 @@
<Component Id="UninstallScript" Directory="ScriptsFolder">
<File Id="UninstallEndpointScriptFile" Source="$(var.StageDir)\scripts\uninstall-endpoint.ps1" KeyPath="yes" />
</Component>
+ <!-- Dot-sourced by both scripts above: how they run beacon.exe, bounded and logged. -->
+ <Component Id="PackageCommonScript" Directory="ScriptsFolder">
+ <File Id="PackageCommonScriptFile" Source="$(var.StageDir)\scripts\package-common.ps1" KeyPath="yes" />
+ </Component>
</ComponentGroup>
<Feature Id="Main" Title="Beacon Endpoint Agent" Level="1">
diff --git a/packaging/windows/build-msi.ps1 b/packaging/windows/build-msi.ps1
index 90c2815c..2280f8dc 100644
--- a/packaging/windows/build-msi.ps1
+++ b/packaging/windows/build-msi.ps1
@@ -51,6 +51,7 @@ try {
Copy-Item -LiteralPath $CollectorExe -Destination (Join-Path $stage 'bin\beacon-otelcol.exe')
Copy-Item -LiteralPath (Join-Path $scriptDir 'install-endpoint.ps1') -Destination (Join-Path $stage 'scripts\install-endpoint.ps1')
Copy-Item -LiteralPath (Join-Path $scriptDir 'uninstall-endpoint.ps1') -Destination (Join-Path $stage 'scripts\uninstall-endpoint.ps1')
+ Copy-Item -LiteralPath (Join-Path $scriptDir 'package-common.ps1') -Destination (Join-Path $stage 'scripts\package-common.ps1')
# The dotnet tool rather than a preinstalled WiX: it pins a version, it does not depend on what a
# runner image happens to ship, and it works the same on a maintainer's machine.
diff --git a/packaging/windows/install-endpoint.ps1 b/packaging/windows/install-endpoint.ps1
index d8c54a85..47d1b2ab 100644
--- a/packaging/windows/install-endpoint.ps1
+++ b/packaging/windows/install-endpoint.ps1
@@ -13,6 +13,16 @@ param()
$ErrorActionPreference = 'Stop'
+. (Join-Path $PSScriptRoot 'package-common.ps1')
+Open-BeaconPackageLog 'install'
+
+# Bounds for the two calls below, generous because a slow machine must not fail an install: the first
+# install starts the service, waits for the collector to answer, configures runtimes and reads up to
+# 30 days of local session history. They exist so that a call that never returns fails the install
+# with a record of where, instead of holding msiexec and the fleet tool behind it open indefinitely.
+$installTimeoutSeconds = 600
+$userConfigTimeoutSeconds = 300
+
function Get-EnvOrDefault {
param([string]$Name, [string]$Default)
$value = [Environment]::GetEnvironmentVariable($Name)
@@ -26,6 +36,7 @@ $installRoot = Get-EnvOrDefault 'BEACON_INSTALL_ROOT' (Join-Path $env:ProgramFil
$beaconBin = Get-EnvOrDefault 'BEACON_BIN' (Join-Path $installRoot 'bin\beacon.exe')
if (-not (Test-Path -LiteralPath $beaconBin)) {
+ Write-BeaconPackageLog "beacon.exe was not found at $beaconBin"
throw "beacon.exe was not found at $beaconBin"
}
@@ -71,10 +82,12 @@ if (-not [string]::IsNullOrWhiteSpace($logAccess)) { $installArgs += @('--log-ac
# delete or rename anything in it. In the restricted log access mode the log directory grants
# ordinary users nothing at all. Do not add an icacls grant here: a second, broader grant is how
# the log directory once became writable enough to redirect the collector's writes.
-Write-Output "beacon: installing the system endpoint"
-& $beaconBin @installArgs
-if ($LASTEXITCODE -ne 0) {
- throw "beacon endpoint install failed with exit code $LASTEXITCODE"
+Write-BeaconPackageLog "beacon: installing the system endpoint"
+$code = Invoke-BeaconBounded -Beacon $beaconBin -Arguments $installArgs -TimeoutSeconds $installTimeoutSeconds
+if ($code -ne 0) {
+ Write-BeaconPackageLog "beacon endpoint install failed with exit code $code; log: $script:BeaconPackageLogPath"
+ Close-BeaconPackageLog
+ throw "beacon endpoint install failed with exit code $code"
}
# A system endpoint runs elevated, and the install above configured *this* account's Claude Code and
@@ -87,11 +100,12 @@ if ($LASTEXITCODE -ne 0) {
#
# Non-fatal, and it says why when it cannot. An unattended install may have no interactive user at
# all, which is a legitimate outcome rather than a failure.
-Write-Output "beacon: configuring the interactive user's agent runtime"
-& $beaconBin endpoint user-config repair-installed --system --harness $harnesses
-if ($LASTEXITCODE -ne 0) {
- Write-Warning ("beacon: could not configure the interactive user's agent runtime; " +
+Write-BeaconPackageLog "beacon: configuring the interactive user's agent runtime"
+$code = Invoke-BeaconBounded -Beacon $beaconBin -Arguments @('endpoint', 'user-config', 'repair-installed', '--system', '--harness', $harnesses) -TimeoutSeconds $userConfigTimeoutSeconds
+if ($code -ne 0) {
+ Write-BeaconPackageLog ("beacon: could not configure the interactive user's agent runtime (exit $code); " +
"run 'beacon endpoint user-config repair-installed --system' once logged in")
}
-Write-Output "beacon: endpoint installed. Check it with: beacon endpoint status --system"
+Write-BeaconPackageLog "beacon: endpoint installed. Check it with: beacon endpoint status --system"
+Close-BeaconPackageLog
diff --git a/packaging/windows/package-common.ps1 b/packaging/windows/package-common.ps1
new file mode 100644
index 00000000..d854924d
--- /dev/null
+++ b/packaging/windows/package-common.ps1
@@ -0,0 +1,153 @@
+# How the Windows package scripts run beacon.exe. Dot-sourced by install-endpoint.ps1 and
+# uninstall-endpoint.ps1, so the two cannot drift apart on it.
+#
+# An MSI custom action has no console and no time limit of its own. A beacon.exe call that never
+# returns therefore holds the whole installation open: msiexec waits on the action, a fleet tool
+# (Intune, Workspace ONE, SCCM) waits on msiexec, and nothing records which step it was. CI saw
+# exactly that, intermittently, in both an upgrade and an uninstall, and each time the job was
+# cancelled at its time limit with its log discarded. So every call made here is bounded, and every
+# call leaves a record.
+#
+# The record is a file of its own in the temporary directory of the account running the script --
+# SYSTEM, for the MSI -- because nothing else survives: Windows Installer keeps no output from an
+# executable custom action, and an uninstall removes the payload directory. The file name is new for
+# each run and the file is created rather than opened, so a file an ordinary user planted under that
+# name in advance can never receive SYSTEM's writes. The log is diagnostics only: if it cannot be
+# created, the script carries on without it.
+
+# ERROR_TIMEOUT. Returned by Invoke-BeaconBounded for a call it had to stop, so a caller can tell
+# "beacon.exe failed" from "beacon.exe never finished".
+$script:BeaconTimeoutExitCode = 1460
+
+# Returned by Invoke-BeaconBounded when beacon.exe could not be started at all (missing, locked,
+# blocked by application control, access denied). No process ran, so there is no exit code of its
+# own to report, and this is not a timeout.
+$script:BeaconLaunchFailedExitCode = -1
+
+$script:BeaconPackageLog = $null
+$script:BeaconPackageLogPath = $null
+
+function Open-BeaconPackageLog {
+ param([string]$Action)
+ $name = 'beacon-package-{0}-{1:yyyyMMddTHHmmss}-{2}.log' -f $Action, (Get-Date), ([Guid]::NewGuid().ToString('N').Substring(0, 8))
+ $path = Join-Path ([System.IO.Path]::GetTempPath()) $name
+ try {
+ $stream = New-Object System.IO.FileStream($path, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::Read)
+ $writer = New-Object System.IO.StreamWriter($stream, (New-Object System.Text.UTF8Encoding($false)))
+ $writer.AutoFlush = $true
+ $script:BeaconPackageLog = $writer
+ $script:BeaconPackageLogPath = $path
+ } catch {
+ $script:BeaconPackageLog = $null
+ $script:BeaconPackageLogPath = $null
+ }
+}
+
+# Written to the console as well as the log, through [Console] rather than Write-Output: Write-Output
+# inside a function becomes part of that function's return value, which would corrupt the exit code
+# Invoke-BeaconBounded returns.
+function Write-BeaconPackageLog {
+ param([string]$Message)
+ $line = '{0:o} {1}' -f (Get-Date), $Message
+ [Console]::Out.WriteLine($line)
+ if ($null -ne $script:BeaconPackageLog) {
+ try { $script:BeaconPackageLog.WriteLine($line) } catch { $script:BeaconPackageLog = $null }
+ }
+}
+
+function Close-BeaconPackageLog {
+ if ($null -ne $script:BeaconPackageLog) {
+ try { $script:BeaconPackageLog.Dispose() } catch { }
+ $script:BeaconPackageLog = $null
+ }
+}
+
+# Quotes one argument for the command line a Windows program parses back into argv, which is how
+# Go's runtime reads it. Windows PowerShell 5.1 runs on .NET Framework, whose ProcessStartInfo takes
+# one command-line string rather than an argument list, and the paths passed here contain spaces
+# (C:\Program Files\...). Backslashes are literal except in a run that ends at a quote, where each
+# one has to be doubled -- which includes a run at the very end, followed by the closing quote.
+function ConvertTo-BeaconArgument {
+ param([string]$Value)
+ if ($Value.Length -gt 0 -and $Value -notmatch '[\s"]') { return $Value }
+ $escaped = [regex]::Replace($Value, '(\\*)"', { param($m) $m.Groups[1].Value + $m.Groups[1].Value + '\"' })
+ $escaped = [regex]::Replace($escaped, '(\\+)$', { param($m) $m.Groups[1].Value + $m.Groups[1].Value })
+ return '"' + $escaped + '"'
+}
+
+function Write-BeaconStream {
+ param([string]$Name, $Task)
+ # The pipes close once every process holding them has exited, which the stop below guarantees
+ # for beacon's own process tree. A reader is still bounded, because a process outside that tree
+ # could have inherited one.
+ try {
+ if (-not $Task.Wait(5000)) {
+ Write-BeaconPackageLog " ($Name did not close)"
+ return
+ }
+ $text = $Task.Result
+ } catch {
+ Write-BeaconPackageLog " ($Name could not be read: $($_.Exception.Message))"
+ return
+ }
+ foreach ($line in ($text -split "`r?`n")) {
+ if ($line.Length -gt 0) { Write-BeaconPackageLog " $Name> $line" }
+ }
+}
+
+# Runs beacon.exe with Arguments, waits at most TimeoutSeconds, and returns its exit code,
+# $BeaconTimeoutExitCode once it had to be stopped, or $BeaconLaunchFailedExitCode when it could not
+# be started. Output is recorded line by line in the package log.
+#
+# A beacon.exe still running fifteen seconds before the bound writes every goroutine's stack to
+# stderr (BEACON_DEBUG_STACKS_AFTER), so a call that hangs names the code it is stuck in, not just
+# the command.
+function Invoke-BeaconBounded {
+ param([string]$Beacon, [string[]]$Arguments, [int]$TimeoutSeconds)
+
+ # Windows PowerShell turns a native command's stderr into a terminating error under the callers'
+ # 'Stop' preference once it is redirected, and taskkill reports a child that already exited on
+ # stderr. Stopping a hung call must never be what fails the install or uninstall.
+ $ErrorActionPreference = 'Continue'
+
+ $display = 'beacon ' + ($Arguments -join ' ')
+ Write-BeaconPackageLog "running: $display (bound ${TimeoutSeconds}s)"
+
+ $info = New-Object System.Diagnostics.ProcessStartInfo
+ $info.FileName = $Beacon
+ $info.Arguments = (@($Arguments | ForEach-Object { ConvertTo-BeaconArgument $_ })) -join ' '
+ $info.UseShellExecute = $false
+ $info.CreateNoWindow = $true
+ $info.RedirectStandardOutput = $true
+ $info.RedirectStandardError = $true
+ $info.StandardOutputEncoding = [System.Text.Encoding]::UTF8
+ $info.StandardErrorEncoding = [System.Text.Encoding]::UTF8
+ $info.EnvironmentVariables['BEACON_DEBUG_STACKS_AFTER'] = "$([Math]::Max(1, $TimeoutSeconds - 15))s"
+
+ # Caught here because the 'Continue' preference above would otherwise let a failed start carry on
+ # with no process, and be reported as a timeout.
+ try {
+ $process = [System.Diagnostics.Process]::Start($info)
+ } catch {
+ Write-BeaconPackageLog "could not start ${display}: $($_.Exception.Message)"
+ return $script:BeaconLaunchFailedExitCode
+ }
+ # Both streams are drained concurrently. Reading one to the end before the other deadlocks as
+ # soon as beacon fills the pipe nobody is reading.
+ $stdout = $process.StandardOutput.ReadToEndAsync()
+ $stderr = $process.StandardError.ReadToEndAsync()
+
+ $exited = $process.WaitForExit($TimeoutSeconds * 1000)
+ if (-not $exited) {
+ Write-BeaconPackageLog "TIMEOUT: $display did not finish within ${TimeoutSeconds}s; stopping it and its child processes"
+ & taskkill.exe /T /F /PID $process.Id 2>&1 | ForEach-Object { Write-BeaconPackageLog " $_" }
+ $global:LASTEXITCODE = 0
+ $null = $process.WaitForExit(10000)
+ }
+ Write-BeaconStream 'stdout' $stdout
+ Write-BeaconStream 'stderr' $stderr
+
+ if (-not $exited) { return $script:BeaconTimeoutExitCode }
+ Write-BeaconPackageLog "exit $($process.ExitCode): $display"
+ return $process.ExitCode
+}
diff --git a/packaging/windows/test-endpoint-scripts.ps1 b/packaging/windows/test-endpoint-scripts.ps1
index 76ad6f8a..36bb7f3d 100644
--- a/packaging/windows/test-endpoint-scripts.ps1
+++ b/packaging/windows/test-endpoint-scripts.ps1
@@ -20,7 +20,7 @@ function Fail {
$script:failures++
}
-$scripts = @('install-endpoint.ps1', 'uninstall-endpoint.ps1', 'build-msi.ps1', 'smoke-restricted-log.ps1')
+$scripts = @('install-endpoint.ps1', 'uninstall-endpoint.ps1', 'package-common.ps1', 'build-msi.ps1', 'smoke-restricted-log.ps1')
foreach ($name in $scripts) {
$path = Join-Path $scriptDir $name
if (-not (Test-Path -LiteralPath $path)) {
@@ -63,6 +63,75 @@ foreach ($needle in @('--keep-logs', '--keep-config', 'BEACON_PURGE')) {
}
}
+# Every beacon.exe call a package script makes is bounded and logged (package-common.ps1). A bare
+# `& $beaconBin` would bring back the failure that is for: a call that never returns holds msiexec,
+# and the fleet tool behind it, open with no record of which step it was.
+foreach ($name in @('install-endpoint.ps1', 'uninstall-endpoint.ps1')) {
+ $text = Get-Content -Raw -LiteralPath (Join-Path $scriptDir $name)
+ $code = ($text -split "`n" | Where-Object { $_ -notmatch '^\s*#' }) -join "`n"
+ if ($code -notmatch [regex]::Escape("package-common.ps1")) {
+ Fail "$name no longer dot-sources package-common.ps1"
+ }
+ if ($code -notmatch 'Invoke-BeaconBounded') {
+ Fail "$name no longer runs beacon.exe through Invoke-BeaconBounded"
+ }
+ if ($code -match '&\s*\$beaconBin') {
+ Fail "$name runs beacon.exe directly; use Invoke-BeaconBounded so the call is bounded and logged"
+ }
+}
+
+# The helpers themselves. Quoting decides whether a path with a space reaches beacon.exe as one
+# argument, so the cases are spelled out: Go parses the command line by the same rules as the
+# Microsoft C runtime.
+. (Join-Path $scriptDir 'package-common.ps1')
+$quoting = [ordered]@{
+ 'endpoint' = 'endpoint'
+ '' = '""'
+ 'C:\Program Files\Beacon\bin\beacon-otelcol.exe' = '"C:\Program Files\Beacon\bin\beacon-otelcol.exe"'
+ 'C:\dir with space\' = '"C:\dir with space\\"'
+ 'say "hi"' = '"say \"hi\""'
+ 'C:\a\"b' = '"C:\a\\\"b"'
+ 'C:\no\spaces\here' = 'C:\no\spaces\here'
+}
+foreach ($entry in $quoting.GetEnumerator()) {
+ $got = ConvertTo-BeaconArgument $entry.Key
+ if ($got -ne $entry.Value) {
+ Fail "ConvertTo-BeaconArgument '$($entry.Key)' gave '$got', want '$($entry.Value)'"
+ }
+}
+
+# A call that outlives its bound is stopped, reported as a timeout and recorded, and an ordinary
+# call returns its own exit code. Windows-only programs stand in for beacon.exe.
+if ($IsWindows -or $env:OS -eq 'Windows_NT') {
+ $cmd = Join-Path $env:SystemRoot 'System32\cmd.exe'
+ Open-BeaconPackageLog 'selftest'
+ $code = Invoke-BeaconBounded -Beacon $cmd -Arguments @('/d', '/c', 'exit 7') -TimeoutSeconds 30
+ if ($code -ne 7) { Fail "Invoke-BeaconBounded returned $code for a call that exited 7" }
+ $started = Get-Date
+ $code = Invoke-BeaconBounded -Beacon $cmd -Arguments @('/d', '/c', 'ping -n 60 127.0.0.1 >nul') -TimeoutSeconds 3
+ $elapsed = ((Get-Date) - $started).TotalSeconds
+ if ($code -ne $BeaconTimeoutExitCode) { Fail "Invoke-BeaconBounded returned $code for a call past its bound, not $BeaconTimeoutExitCode" }
+ if ($elapsed -gt 30) { Fail "Invoke-BeaconBounded took ${elapsed}s to give up on a 3s bound" }
+ if (Get-CimInstance Win32_Process -Filter "Name = 'PING.EXE'" | Where-Object { $_.CommandLine -match '-n 60 127\.0\.0\.1' }) {
+ Fail "Invoke-BeaconBounded left the timed-out call's child process running"
+ }
+ $code = Invoke-BeaconBounded -Beacon (Join-Path $env:SystemRoot 'System32\no-such-beacon.exe') -Arguments @('version') -TimeoutSeconds 30
+ if ($code -ne $BeaconLaunchFailedExitCode) { Fail "Invoke-BeaconBounded returned $code for a program that could not start, not $BeaconLaunchFailedExitCode" }
+ $logPath = $BeaconPackageLogPath
+ Close-BeaconPackageLog
+ if (-not $logPath -or -not (Test-Path -LiteralPath $logPath)) {
+ Fail "Open-BeaconPackageLog created no log"
+ } else {
+ $logText = Get-Content -Raw -LiteralPath $logPath
+ if ($logText -notmatch 'TIMEOUT:') { Fail "the package log does not record the timeout" }
+ if ($logText -notmatch 'exit 7:') { Fail "the package log does not record the exit code" }
+ if ($logText -notmatch 'could not start') { Fail "the package log does not record a failed start" }
+ if (([regex]::Matches($logText, 'TIMEOUT:')).Count -ne 1) { Fail "the package log records a timeout that did not happen" }
+ Remove-Item -LiteralPath $logPath -Force
+ Write-Output 'ok: Invoke-BeaconBounded returns exit codes, stops a call past its bound, reports a failed start, and logs all three'
+ }
+}
+
# The ACL on %ProgramData%\Beacon belongs to `beacon endpoint install --system`, which gives ordinary
# users read and execute plus append to the runtime log, and nothing that lets them delete, rename or
# plant links. A packaging script that grants Modify or Full Control on top would quietly undo that.
@@ -81,6 +150,11 @@ foreach ($needle in @('install-endpoint.ps1', 'uninstall-endpoint.ps1')) {
Fail "beacon.wxs no longer invokes $needle"
}
}
+# Both scripts dot-source package-common.ps1 from their own directory, so the package has to ship it.
+$build = Get-Content -Raw -LiteralPath (Join-Path $scriptDir 'build-msi.ps1')
+if ($wxs -notmatch [regex]::Escape('scripts\package-common.ps1') -or $build -notmatch [regex]::Escape('package-common.ps1')) {
+ Fail "the MSI no longer ships package-common.ps1 beside the scripts that dot-source it"
+}
if ($wxs -match 'ExeCommand="[^"]*beacon\.exe') {
Fail "beacon.wxs invokes beacon.exe directly; the install contract belongs in one place, not two"
}
diff --git a/packaging/windows/uninstall-endpoint.ps1 b/packaging/windows/uninstall-endpoint.ps1
index ed8d58eb..614643ea 100644
--- a/packaging/windows/uninstall-endpoint.ps1
+++ b/packaging/windows/uninstall-endpoint.ps1
@@ -17,6 +17,49 @@ param()
$ErrorActionPreference = 'Stop'
+. (Join-Path $PSScriptRoot 'package-common.ps1')
+Open-BeaconPackageLog 'uninstall'
+
+# Long enough for `endpoint uninstall` to stop the service (it waits up to 20 seconds for a graceful
+# stop) and remove what it installed. The bound exists because this action also runs at the start of
+# every upgrade, where a call that never returns holds the upgrade open indefinitely.
+$uninstallTimeoutSeconds = 180
+
+# Makes sure the collector is no longer running, for when `endpoint uninstall` could not finish.
+#
+# A stopped collector is the one thing the MSI cannot proceed without: it cannot replace or remove
+# beacon-otelcol.exe while the service holds it open, and an upgrade that carried on regardless would
+# defer the replacement to a reboot or keep running the old collector. The registration goes too, so
+# a removal leaves no service pointing at a deleted binary; an upgrade's install action registers it
+# again.
+function Stop-BeaconCollectorService {
+ param([string]$InstallRoot)
+ # Best effort throughout: sc.exe reports a service that is already gone on stderr, which the
+ # script's 'Stop' preference would otherwise turn into a failed uninstall.
+ $ErrorActionPreference = 'Continue'
+ $name = 'BeaconCollector'
+ $service = Get-Service -Name $name -ErrorAction SilentlyContinue
+ if ($null -eq $service) { return }
+ if ($service.Status -ne 'Stopped') {
+ Write-BeaconPackageLog "stopping the $name service directly"
+ & sc.exe stop $name 2>&1 | ForEach-Object { Write-BeaconPackageLog " $_" }
+ try {
+ $service.WaitForStatus('Stopped', [TimeSpan]::FromSeconds(30))
+ } catch {
+ Write-BeaconPackageLog "the $name service did not stop within 30s"
+ }
+ }
+ $collector = Join-Path $InstallRoot 'bin\beacon-otelcol.exe'
+ Get-Process -Name 'beacon-otelcol' -ErrorAction SilentlyContinue |
+ Where-Object { $_.Path -eq $collector } |
+ ForEach-Object {
+ Write-BeaconPackageLog "terminating collector process $($_.Id)"
+ Stop-Process -Id $_.Id -Force -ErrorAction SilentlyContinue
+ }
+ & sc.exe delete $name 2>&1 | ForEach-Object { Write-BeaconPackageLog " $_" }
+ $global:LASTEXITCODE = 0
+}
+
function Get-EnvOrDefault {
param([string]$Name, [string]$Default)
$value = [Environment]::GetEnvironmentVariable($Name)
@@ -32,7 +75,8 @@ if (-not (Test-Path -LiteralPath $beaconBin)) {
# Nothing to do, and not an error. An uninstall must not fail because the thing it removes is
# already gone -- a failed uninstall custom action leaves the MSI half-removed, which is a worse
# state than either finishing or never starting.
- Write-Output "beacon: $beaconBin is not present; nothing to uninstall"
+ Write-BeaconPackageLog "beacon: $beaconBin is not present; nothing to uninstall"
+ Close-BeaconPackageLog
exit 0
}
@@ -90,9 +134,9 @@ if ($purge -ne '1') {
Copy-Retained $source (Join-Path $stash $name) $protected[$name]
}
}
- Write-Output "beacon: removing the endpoint, keeping configuration and logs (set BEACON_PURGE=1 to remove them)"
+ Write-BeaconPackageLog "beacon: removing the endpoint, keeping configuration and logs (set BEACON_PURGE=1 to remove them)"
} else {
- Write-Output "beacon: removing the endpoint, configuration and logs"
+ Write-BeaconPackageLog "beacon: removing the endpoint, configuration and logs"
}
$uninstallArgs = @('endpoint', 'uninstall', '--system')
@@ -102,12 +146,16 @@ if ($purge -ne '1') {
$uninstallArgs += @('--keep-logs', '--keep-config')
}
-& $beaconBin @uninstallArgs
-if ($LASTEXITCODE -ne 0) {
+$code = Invoke-BeaconBounded -Beacon $beaconBin -Arguments $uninstallArgs -TimeoutSeconds $uninstallTimeoutSeconds
+if ($code -eq $script:BeaconTimeoutExitCode) {
+ # Stopped, not waited on. A removal that hangs is worse for the person running it than one that
+ # finishes with the collector forced down, and the package log records what happened.
+ Stop-BeaconCollectorService -InstallRoot $installRoot
+} elseif ($code -ne 0) {
# Reported, not thrown. The service and its registration are what matter here, and `endpoint
# uninstall` is idempotent -- so a non-zero exit is worth surfacing but not worth wedging an MSI
# removal over. A user who cannot uninstall the package has a worse problem than a leftover file.
- Write-Warning "beacon: endpoint uninstall exited $LASTEXITCODE; check 'beacon endpoint status --system'"
+ Write-BeaconPackageLog "beacon: endpoint uninstall exited $code; check 'beacon endpoint status --system'"
}
if ($null -ne $stash) {
@@ -122,7 +170,7 @@ if ($null -ne $stash) {
}
}
if ($restored -gt 0) {
- Write-Output "beacon: restored $restored configuration file(s) to $configDir"
+ Write-BeaconPackageLog "beacon: restored $restored configuration file(s) to $configDir"
}
} finally {
Remove-Item -LiteralPath $stash -Recurse -Force -ErrorAction SilentlyContinue
@@ -135,4 +183,5 @@ if ($null -ne $stash) {
Remove-Item -LiteralPath $configDir -Recurse -Force -ErrorAction SilentlyContinue
}
-Write-Output "beacon: endpoint removed"
+Write-BeaconPackageLog "beacon: endpoint removed"
+Close-BeaconPackageLog