Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

feat(sandbox): assert that uninstall actually removes the endpoint

shukieshah67a9ca0b9803
diff --git a/beacon-sandbox/check/uninstall.go b/beacon-sandbox/check/uninstall.go
new file mode 100644
index 00000000..20b07654
--- /dev/null
+++ b/beacon-sandbox/check/uninstall.go
@@ -0,0 +1,148 @@
+package check
+
+import (
+ "fmt"
+ "strings"
+)
+
+// Removal is what the runner observed after asking Beacon to uninstall itself.
+//
+// Every field is a string rather than a bool, and deliberately so: an empty value means the probe did
+// not make that observation, which is a third state distinct from true and false. Collapsing it into
+// false would let a probe that never ran report a removed service, and reporting a removal that was
+// never checked is the exact failure this whole check exists to prevent.
+type Removal struct {
+ // Ran is "true" when the scenario asked for an uninstall at all.
+ Ran string
+ // ExitCode is `endpoint uninstall`'s status, as text.
+ ExitCode string
+ // Output is what it printed.
+ Output string
+ // ServiceKind and ServiceLabel identify what was registered at install time.
+ ServiceKind string
+ ServiceLabel string
+ // ServiceGone is "true"/"false" when the service manager was asked, empty when it was not.
+ ServiceGone string
+ ServiceQuery string
+ // ConfigRetained and LogRetained are "true"/"false" when the path was checked, empty otherwise.
+ ConfigRetained string
+ LogRetained string
+ // Status is `endpoint status --json` after the removal, which is Beacon's own account of itself.
+ Status string
+}
+
+// Uninstall judges whether a removal actually removed anything.
+//
+// The failure this is aimed at is specific and has happened: an uninstall that reports success while
+// the service stays registered with automatic start, so the collector returns at the next reboot after
+// the operator was told it was gone. Nothing in the suite asked that question before -- every scenario
+// installed, captured, and stopped -- so it was caught by a reviewer reading a diff, which is not a
+// mechanism that scales.
+//
+// Two questions, and they are asked of two different authorities. Whether Beacon *thinks* it is
+// uninstalled comes from its own status output. Whether the service is still registered is a fact
+// about the machine, obtained from the service manager. Those are exactly the two that came apart in
+// the bug, so a check that only asked Beacon would have passed.
+func Uninstall(v *Verdict, r Removal) {
+ if r.Ran != "true" {
+ // The scenario did not ask. Silence is right: not every scenario should uninstall, and an
+ // absent check must not be reported as a passing one.
+ return
+ }
+
+ if r.ExitCode != "0" {
+ v.Add(Finding{
+ Check: "uninstall.command_succeeded",
+ Severity: SevFail,
+ Summary: fmt.Sprintf("`endpoint uninstall` exited %s: %s", r.ExitCode, oneLineOf(r.Output)),
+ Why: "An uninstall that cannot complete leaves the endpoint in a state no one asked for: " +
+ "binaries possibly gone, service possibly still registered.",
+ })
+ }
+
+ switch r.ServiceGone {
+ case "true":
+ v.Add(Finding{
+ Check: "uninstall.service_removed",
+ Severity: SevInfo,
+ Summary: fmt.Sprintf("the %s service %q is no longer registered", r.ServiceKind, r.ServiceLabel),
+ })
+ case "false":
+ v.Add(Finding{
+ Check: "uninstall.service_removed",
+ Severity: SevFail,
+ Summary: fmt.Sprintf("the %s service %q is still registered after uninstall: %s",
+ r.ServiceKind, r.ServiceLabel, oneLineOf(r.ServiceQuery)),
+ Why: "A service left registered with automatic start brings the collector back at the next " +
+ "reboot, after the operator was told it was removed. Uninstall reporting success while " +
+ "this is true is the failure this check exists for.",
+ })
+ default:
+ // Unverified, not clean. The supervised backend has no manager to ask, and a probe that could
+ // not identify what was registered has no question to put to one.
+ v.Add(Finding{
+ Check: "uninstall.service_removed",
+ Severity: SevWarn,
+ Summary: fmt.Sprintf("the service manager was not asked whether %q is gone (kind=%q)",
+ r.ServiceLabel, r.ServiceKind),
+ Why: "Removal of the service registration is unverified for this run. It is not evidence " +
+ "that anything was left behind, and it is not evidence that nothing was.",
+ })
+ }
+
+ // Retention is the other half of the contract, and it fails in the opposite direction: too much
+ // removed rather than too little. An uninstall is often the first half of a reinstall, and
+ // destroying an operator's configuration and collected telemetry is a separate, deliberate act.
+ for name, retained := range map[string]string{
+ "configuration": r.ConfigRetained,
+ "runtime log": r.LogRetained,
+ } {
+ switch retained {
+ case "true":
+ v.Add(Finding{
+ Check: "uninstall.data_retained",
+ Severity: SevInfo,
+ Summary: fmt.Sprintf("%s survived the uninstall, as --keep-logs and --keep-config ask", name),
+ })
+ case "false":
+ v.Add(Finding{
+ Check: "uninstall.data_retained",
+ Severity: SevFail,
+ Summary: fmt.Sprintf("%s was removed despite --keep-logs and --keep-config", name),
+ Why: "Removing the product must not destroy what it collected unless a purge asks for " +
+ "it. Note that --keep-config governs harness settings rather than Beacon's own " +
+ "config, which is why packaging stashes those files around a removal.",
+ })
+ default:
+ v.Add(Finding{
+ Check: "uninstall.data_retained",
+ Severity: SevWarn,
+ Summary: fmt.Sprintf("whether the %s survived was not checked", name),
+ Why: "Retention is unverified for this run rather than confirmed.",
+ })
+ }
+ }
+
+ // Beacon's own account, checked against the machine's. A status still describing a running
+ // collector after a successful removal means the two disagree, and the disagreement is worth a
+ // finding of its own even when the service really is gone.
+ if strings.Contains(r.Status, `"running":true`) {
+ v.Add(Finding{
+ Check: "uninstall.status_agrees",
+ Severity: SevFail,
+ Summary: "`endpoint status` still reports a running collector after uninstall",
+ Why: "Beacon's view of itself and the machine's have to agree, or one of them is lying to " +
+ "whoever reads it next.",
+ })
+ }
+}
+
+// oneLineOf keeps a finding's detail to one line. Multi-line output in a verdict makes the report
+// unreadable, which is how a real failure gets skimmed past.
+func oneLineOf(s string) string {
+ flat := strings.Join(strings.Fields(s), " ")
+ if len(flat) > 300 {
+ return flat[:300] + "…"
+ }
+ return flat
+}
diff --git a/beacon-sandbox/check/uninstall_test.go b/beacon-sandbox/check/uninstall_test.go
new file mode 100644
index 00000000..630ef88f
--- /dev/null
+++ b/beacon-sandbox/check/uninstall_test.go
@@ -0,0 +1,148 @@
+package check
+
+import (
+ "strings"
+ "testing"
+)
+
+// clean is the shape of a removal that did everything right: service gone, data kept, Beacon's own
+// status agreeing.
+func clean() Removal {
+ return Removal{
+ Ran: "true",
+ ExitCode: "0",
+ Output: "Endpoint uninstalled",
+ ServiceKind: "systemd",
+ ServiceLabel: "beacon-collector.service",
+ ServiceGone: "true",
+ ConfigRetained: "true",
+ LogRetained: "true",
+ Status: `{"service":{"running":false}}`,
+ }
+}
+
+func findingsFor(r Removal) []Finding {
+ var v Verdict
+ Uninstall(&v, r)
+ return v.Findings
+}
+
+func has(findings []Finding, name string, sev Severity) bool {
+ for _, f := range findings {
+ if f.Check == name && f.Severity == sev {
+ return true
+ }
+ }
+ return false
+}
+
+func TestACleanRemovalRaisesNoFailures(t *testing.T) {
+ for _, f := range findingsFor(clean()) {
+ if f.Severity == SevFail {
+ t.Fatalf("clean removal produced a failure: %s — %s", f.Check, f.Summary)
+ }
+ }
+}
+
+// TestAScenarioThatDidNotUninstallIsSilent is what keeps this check honest about its own absence.
+//
+// Most scenarios do not uninstall. Emitting a passing finding for them would report a check that
+// never ran as one that succeeded, which is the failure mode this package is built to avoid.
+func TestAScenarioThatDidNotUninstallIsSilent(t *testing.T) {
+ if findings := findingsFor(Removal{}); len(findings) != 0 {
+ t.Fatalf("a run with no uninstall produced %d finding(s): %#v", len(findings), findings)
+ }
+}
+
+// TestAServiceLeftRegisteredFails is the bug this check exists for.
+//
+// It shipped once: uninstall reported success while the Windows service stayed registered with
+// automatic start, so the collector returned at the next reboot after the operator was told it was
+// gone. A reviewer reading a diff caught it, because no run had ever asked.
+func TestAServiceLeftRegisteredFails(t *testing.T) {
+ r := clean()
+ r.ServiceGone = "false"
+ r.ServiceQuery = "SERVICE_NAME: BeaconCollector STATE: 4 RUNNING"
+
+ findings := findingsFor(r)
+ if !has(findings, "uninstall.service_removed", SevFail) {
+ t.Fatalf("a still-registered service did not fail: %#v", findings)
+ }
+ // The evidence has to travel with the finding, or whoever reads the report cannot tell a
+ // still-registered service from a probe that misfired.
+ for _, f := range findings {
+ if f.Check == "uninstall.service_removed" && !strings.Contains(f.Summary, "BeaconCollector") {
+ t.Fatalf("the finding does not quote what the service manager said: %q", f.Summary)
+ }
+ }
+}
+
+// TestAnUnaskedServiceQuestionIsWarnedNotPassed covers the third state.
+//
+// The supervised backend has no service manager to ask, and a probe that could not identify what was
+// registered has no question to put to one. Neither is evidence of a clean removal, and reporting
+// either as one would be exactly the silent pass this tool exists to prevent.
+func TestAnUnaskedServiceQuestionIsWarnedNotPassed(t *testing.T) {
+ r := clean()
+ r.ServiceGone = ""
+ r.ServiceKind = "none"
+
+ findings := findingsFor(r)
+ if has(findings, "uninstall.service_removed", SevInfo) {
+ t.Fatal("an unasked question was reported as a clean removal")
+ }
+ if !has(findings, "uninstall.service_removed", SevWarn) {
+ t.Fatalf("an unasked question produced no warning: %#v", findings)
+ }
+}
+
+// TestDestroyedDataFails covers the opposite direction from a leftover service: too much removed
+// rather than too little.
+func TestDestroyedDataFails(t *testing.T) {
+ r := clean()
+ r.ConfigRetained = "false"
+ if !has(findingsFor(r), "uninstall.data_retained", SevFail) {
+ t.Fatal("configuration destroyed by a non-purge uninstall did not fail")
+ }
+
+ r = clean()
+ r.LogRetained = "false"
+ if !has(findingsFor(r), "uninstall.data_retained", SevFail) {
+ t.Fatal("a runtime log destroyed by a non-purge uninstall did not fail")
+ }
+}
+
+func TestUncheckedRetentionIsWarnedNotPassed(t *testing.T) {
+ r := clean()
+ r.ConfigRetained = ""
+ r.LogRetained = ""
+ findings := findingsFor(r)
+ if has(findings, "uninstall.data_retained", SevInfo) {
+ t.Fatal("unchecked retention was reported as confirmed")
+ }
+ if !has(findings, "uninstall.data_retained", SevWarn) {
+ t.Fatalf("unchecked retention produced no warning: %#v", findings)
+ }
+}
+
+func TestANonZeroUninstallFails(t *testing.T) {
+ r := clean()
+ r.ExitCode = "1"
+ r.Output = "Error: could not stop the service"
+ if !has(findingsFor(r), "uninstall.command_succeeded", SevFail) {
+ t.Fatal("a failed uninstall command did not fail the run")
+ }
+}
+
+// TestStatusDisagreeingWithTheMachineFails covers the case where the two authorities disagree.
+//
+// Both are asked on purpose. Beacon's status is Beacon's opinion of itself; the service manager is a
+// fact about the machine. Those are the two that came apart in the original bug, so a check reading
+// only one of them would have passed it.
+func TestStatusDisagreeingWithTheMachineFails(t *testing.T) {
+ r := clean()
+ r.Status = `{"service":{"label":"beacon-collector.service","loaded":true,"running":true}}`
+ if !has(findingsFor(r), "uninstall.status_agrees", SevFail) {
+ t.Fatal("a status still reporting a running collector after uninstall did not fail")
+ }
+}
diff --git a/beacon-sandbox/cmd/beacon-sandbox/main.go b/beacon-sandbox/cmd/beacon-sandbox/main.go
index 97a5070c..ff30210d 100644
--- a/beacon-sandbox/cmd/beacon-sandbox/main.go
+++ b/beacon-sandbox/cmd/beacon-sandbox/main.go
@@ -398,6 +398,20 @@ func judge(sc scenario.Scenario, art runner.Artifacts, creds credentials.Resolve
Probed: art.SentinelProbed,
Detail: art.SentinelDetail,
}, check.Session{Known: art.SessionKnown, OK: art.SessionOK})
+ // Read from meta rather than from a live probe, so `verify` re-judges a saved run's removal for
+ // free exactly as it re-judges its capture.
+ check.Uninstall(&v, check.Removal{
+ Ran: art.Meta["uninstall_ran"],
+ ExitCode: art.Meta["uninstall_rc"],
+ Output: art.Meta["uninstall_output"],
+ ServiceKind: art.Meta["service_kind"],
+ ServiceLabel: art.Meta["service_label"],
+ ServiceGone: art.Meta["uninstall_service_gone"],
+ ServiceQuery: art.Meta["uninstall_service_query"],
+ ConfigRetained: art.Meta["uninstall_config_retained"],
+ LogRetained: art.Meta["uninstall_log_retained"],
+ Status: art.Meta["uninstall_status"],
+ })
v.Resolve()
return v, log
}
@@ -542,12 +556,20 @@ func judgeRunDir(root, dir, mutate string) (check.Verdict, error) {
logPath := filepath.Join(dir, "runtime.jsonl")
var planted string
if mutate != "" {
- logPath, planted, err = applyMutation(logPath, mutate)
- if err != nil {
- return check.Verdict{}, err
+ // The uninstall checks read what the runner observed about the machine, not the event log, so
+ // damaging the log could never exercise them. Their mutations rewrite the recorded observation
+ // instead -- which is the same idea applied to a different artifact, and the only way a check
+ // that reads meta can be shown to be capable of failing.
+ if applyMetaMutation(meta, mutate) {
+ fmt.Printf("(mutation %q applied to run metadata: a PASS becoming FAIL is the check working)\n", mutate)
+ } else {
+ logPath, planted, err = applyMutation(logPath, mutate)
+ if err != nil {
+ return check.Verdict{}, err
+ }
+ defer os.Remove(logPath)
+ fmt.Printf("(mutation %q applied: a PASS becoming FAIL is the check working)\n", mutate)
}
- defer os.Remove(logPath)
- fmt.Printf("(mutation %q applied: a PASS becoming FAIL is the check working)\n", mutate)
}
art := runner.Artifacts{RuntimeLog: logPath, Meta: meta}
@@ -702,7 +724,7 @@ func applyMutation(path, mode string) (mutated string, planted string, err error
}
default:
return "", "", fmt.Errorf("unknown mutation %q (want drop-commands, drop-action:<action>, "+
- "corrupt-line, or plant-secret)", mode)
+ "corrupt-line, plant-secret, leave-service, or drop-retained-config)", mode)
}
// The shared post-condition: the log must differ from what it was. Compared on the trimmed
@@ -812,3 +834,32 @@ func indent(s string) string {
}
return b.String()
}
+
+// applyMetaMutation damages a recorded observation rather than the event log, and reports whether it
+// recognised the mode.
+//
+// The uninstall checks judge what the runner saw the machine do — whether a service was still
+// registered, whether configuration survived. None of that is in runtime.jsonl, so the log mutations
+// cannot exercise them, and a check that cannot be made to fail is worse than no check at all.
+//
+// Each mode inverts exactly one observation, so the resulting failure names the check under test
+// rather than a cascade.
+func applyMetaMutation(meta map[string]string, mode string) bool {
+ if meta == nil || meta["uninstall_ran"] != "true" {
+ // Nothing to damage. A run that never uninstalled cannot demonstrate an uninstall check, and
+ // silently "succeeding" here would report a self-test as passed when it never ran.
+ return false
+ }
+ switch mode {
+ case "leave-service":
+ // The failure this simulates is the one that shipped: uninstall reports success while the
+ // service stays registered and starts at boot.
+ meta["uninstall_service_gone"] = "false"
+ return true
+ case "drop-retained-config":
+ // The opposite direction: a removal that took the operator's configuration with it.
+ meta["uninstall_config_retained"] = "false"
+ return true
+ }
+ return false
+}
diff --git a/beacon-sandbox/cmd/beacon-sandbox/mutation_meta_test.go b/beacon-sandbox/cmd/beacon-sandbox/mutation_meta_test.go
new file mode 100644
index 00000000..3d3c305a
--- /dev/null
+++ b/beacon-sandbox/cmd/beacon-sandbox/mutation_meta_test.go
@@ -0,0 +1,35 @@
+package main
+
+import "testing"
+
+// The meta mutations must turn a clean removal into a failure, or they prove nothing.
+func TestMetaMutationsInvertARecordedRemoval(t *testing.T) {
+ for _, mode := range []string{"leave-service", "drop-retained-config"} {
+ meta := map[string]string{
+ "uninstall_ran": "true",
+ "uninstall_service_gone": "true",
+ "uninstall_config_retained": "true",
+ }
+ if !applyMetaMutation(meta, mode) {
+ t.Fatalf("%s was not recognised as a meta mutation", mode)
+ }
+ if meta["uninstall_service_gone"] == "true" && meta["uninstall_config_retained"] == "true" {
+ t.Fatalf("%s changed nothing", mode)
+ }
+ }
+}
+
+// A run that never uninstalled cannot demonstrate an uninstall check, and claiming otherwise would
+// report a self-test as passed when it never ran.
+func TestMetaMutationRefusesARunThatDidNotUninstall(t *testing.T) {
+ if applyMetaMutation(map[string]string{}, "leave-service") {
+ t.Fatal("a run with no uninstall accepted an uninstall mutation")
+ }
+}
+
+func TestUnknownModesFallThroughToLogMutations(t *testing.T) {
+ meta := map[string]string{"uninstall_ran": "true"}
+ if applyMetaMutation(meta, "corrupt-line") {
+ t.Fatal("corrupt-line was claimed by the meta mutator; it damages the log")
+ }
+}
diff --git a/beacon-sandbox/runner/runner.go b/beacon-sandbox/runner/runner.go
index 84ca6f57..a0c16ddc 100644
--- a/beacon-sandbox/runner/runner.go
+++ b/beacon-sandbox/runner/runner.go
@@ -456,6 +456,13 @@ func Run(ctx context.Context, p sandbox.Provider, sc scenario.Scenario, opts Opt
art.RuntimeLog = localLog
logf("collected %s", localLog)
+ // Uninstall last, and only when the scenario asked. The runtime log is already on the host, so
+ // removing the endpoint cannot affect any capture assertion -- which is what makes it safe to do
+ // inside the same run rather than needing a scenario of its own.
+ if sc.Install != nil && sc.Install.VerifyUninstall {
+ probeUninstall(ctx, g, sc, privileged, agent, lay, sh, &art, logf)
+ }
+
// meta.json is written by the deferred host-state block above, so it always includes the
// host comparison.
return art, nil
@@ -802,8 +809,9 @@ func installedLogPath(ctx context.Context, g guest,
if err == nil {
art.Meta["install_status"] = oneLine(s.Stdout)
var status struct {
- LogPath string `json:"log_path"`
- Service struct {
+ LogPath string `json:"log_path"`
+ ConfigPath string `json:"config_path"`
+ Service struct {
Label string `json:"label"`
Loaded bool `json:"loaded"`
Running bool `json:"running"`
@@ -812,6 +820,10 @@ func installedLogPath(ctx context.Context, g guest,
} `json:"service"`
}
if json.Unmarshal([]byte(s.Stdout), &status) == nil {
+ // Recorded so a later uninstall probe checks the paths Beacon actually used rather than
+ // the ones a scenario assumed.
+ art.Meta["install_log_path"] = status.LogPath
+ art.Meta["install_config_path"] = status.ConfigPath
art.Meta["service_kind"] = status.Service.Kind
art.Meta["service_label"] = status.Service.Label
art.Meta["service_loaded"] = fmt.Sprintf("%v", status.Service.Loaded)
@@ -881,3 +893,85 @@ func teardownInstall(ctx context.Context, g guest, sc scenario.Scenario,
logf("teardown: uninstalled the %s endpoint", strings.TrimPrefix(scope, "--"))
}
}
+
+// probeUninstall removes the endpoint and records what survived.
+//
+// Deliberately records rather than judges. Every other verdict in this tool is produced by the pure
+// check package from collected artifacts, which is what makes `verify` able to re-judge a saved run
+// for free -- so this writes observations into meta and leaves the conclusions to check.Uninstall.
+//
+// The observations are gathered by asking the operating system, not by asking Beacon. `endpoint
+// status` reporting "not installed" is Beacon's opinion of its own state; whether the service is
+// still registered is a fact about the machine, and those are exactly the two that came apart in the
+// bug this exists to catch.
+func probeUninstall(ctx context.Context, g guest, sc scenario.Scenario,
+ privileged, agent sandbox.ExecOpts, lay image.Layout, sh shell,
+ art *Artifacts, logf func(string, ...any)) {
+
+ system := sc.Install.Mode == "system"
+ scope := "--user"
+ opts := agent
+ if system {
+ scope = "--system"
+ opts = privileged
+ }
+
+ // The service label and kind were recorded at install time. Without them there is nothing to ask
+ // the service manager about, so the probe says so rather than reporting a removal it could not
+ // verify.
+ label := art.Meta["service_label"]
+ kind := art.Meta["service_kind"]
+
+ // Config and log paths come from the install status, so the probe checks the paths Beacon
+ // actually used rather than the ones the scenario assumed.
+ configPath := art.Meta["install_config_path"]
+ logPath := art.Meta["install_log_path"]
+
+ // --keep-logs and --keep-config, because that is the contract a package removal keeps: an
+ // uninstall is often the first half of a reinstall, and destroying collected telemetry is a
+ // separate, deliberate act. Asserting the retention is half of what this probe is for.
+ r, err := g.Exec(ctx, "beacon endpoint uninstall "+scope+" --keep-logs --keep-config 2>&1", opts)
+ art.Meta["uninstall_ran"] = "true"
+ art.Meta["uninstall_output"] = oneLine(r.Stdout + r.Stderr)
+ art.Meta["uninstall_rc"] = fmt.Sprintf("%d", r.ExitCode)
+ if err != nil {
+ art.Meta["uninstall_exec_error"] = oneLine(err.Error())
+ }
+ logf("uninstall rc=%d: %s", r.ExitCode, art.Meta["uninstall_output"])
+
+ // Asked of the service manager directly. A supervised collector has no manager to ask, so its
+ // absence is established from the pidfile instead, and the probe records which question it asked.
+ if label != "" && kind != "" && kind != "none" {
+ query := sh.ServiceQuery(kind, label)
+ if query != "" {
+ q, _ := g.Exec(ctx, query, opts)
+ art.Meta["uninstall_service_query"] = oneLine(q.Stdout + q.Stderr)
+ art.Meta["uninstall_service_query_rc"] = fmt.Sprintf("%d", q.ExitCode)
+ art.Meta["uninstall_service_gone"] = fmt.Sprintf("%v", sh.ServiceAbsent(kind, q.ExitCode, q.Stdout+q.Stderr))
+ logf("service %s after uninstall: gone=%s", label, art.Meta["uninstall_service_gone"])
+ }
+ } else if kind == "none" {
+ // The supervised backend's registration *is* its pidfile, so that is the thing that has to
+ // be gone.
+ art.Meta["uninstall_service_kind_supervised"] = "true"
+ }
+
+ // Retention, asked of the filesystem. Absent paths are reported as unknown rather than as
+ // retained: a check that cannot see the file must not conclude it survived.
+ for key, path := range map[string]string{
+ "uninstall_config_retained": configPath,
+ "uninstall_log_retained": logPath,
+ } {
+ if path == "" {
+ continue
+ }
+ e, _ := g.Exec(ctx, sh.PathExists(path), opts)
+ art.Meta[key] = fmt.Sprintf("%v", e.ExitCode == 0)
+ }
+
+ // Beacon's own account of itself, recorded alongside the machine's. When the two disagree, that
+ // disagreement is the finding.
+ if s, err := g.Exec(ctx, "beacon endpoint status "+scope+" --json 2>&1", opts); err == nil {
+ art.Meta["uninstall_status"] = oneLine(s.Stdout)
+ }
+}
diff --git a/beacon-sandbox/runner/shell.go b/beacon-sandbox/runner/shell.go
index 5f499b80..fdf66f21 100644
--- a/beacon-sandbox/runner/shell.go
+++ b/beacon-sandbox/runner/shell.go
@@ -41,6 +41,18 @@ type shell interface {
ArgvSampler() string
// ArgvVerdictPath is where the sampler writes what it saw.
ArgvVerdictPath() string
+ // PathExists exits zero when the path is there, non-zero when it is not.
+ PathExists(path string) string
+ // ServiceQuery asks the platform's service manager about a service by name, or returns empty
+ // when this backend has no manager to ask.
+ ServiceQuery(kind, label string) string
+ // ServiceAbsent interprets a ServiceQuery result as "no such service".
+ //
+ // Split from ServiceQuery because absence is spelled differently on each platform, and getting it
+ // wrong is the dangerous direction: a query whose failure is misread as absence would report a
+ // still-registered service as removed, which is precisely the bug an uninstall check exists to
+ // catch.
+ ServiceAbsent(kind string, exitCode int, output string) bool
}
// shellFor returns the dialect for a platform.
@@ -356,3 +368,68 @@ if (-not $env:ANTHROPIC_API_KEY) {
Start-Process -FilePath 'pwsh' -ArgumentList '-NoProfile','-NonInteractive','-File',%[2]s,%[1]s -WindowStyle Hidden
'ARGV_SAMPLER_STARTED'`, s.Quote(verdict), s.Quote(scriptPath), enc)
}
+
+// PathExists uses test, which is a shell builtin and needs no external binary -- a minimal image may
+// have no coreutils.
+func (posixShell) PathExists(path string) string {
+ return "test -e " + (posixShell{}).Quote(path)
+}
+
+// ServiceQuery asks the manager that actually registered the service.
+//
+// Only systemd and launchd have anything to ask. The supervised backend's registration is its
+// pidfile, which the caller checks as a path instead, so this returns empty rather than inventing a
+// query that would always succeed.
+func (posixShell) ServiceQuery(kind, label string) string {
+ switch kind {
+ case "systemd":
+ // list-unit-files rather than status: status reports on a unit that no longer exists with the
+ // same exit code as one that is merely stopped, so it cannot distinguish removed from
+ // inactive. This prints nothing at all once the unit file is gone.
+ return "systemctl list-unit-files " + (posixShell{}).Quote(label) + " --no-legend 2>&1"
+ case "launchd":
+ return "launchctl print system/" + (posixShell{}).Quote(label) + " 2>&1"
+ }
+ return ""
+}
+
+// ServiceAbsent reads the query output for each manager's spelling of "not there".
+func (posixShell) ServiceAbsent(kind string, exitCode int, output string) bool {
+ trimmed := strings.TrimSpace(output)
+ switch kind {
+ case "systemd":
+ // A removed unit produces no rows. Non-zero with output is a different problem -- systemd
+ // unreachable, say -- and must not read as absence.
+ return trimmed == "" || strings.Contains(trimmed, "0 unit files listed")
+ case "launchd":
+ // launchctl prints this for a job it does not know about.
+ return exitCode != 0 && (strings.Contains(trimmed, "Could not find service") ||
+ strings.Contains(trimmed, "No such process"))
+ }
+ return false
+}
+
+// PathExists uses Test-Path and turns it into an exit code, because the caller compares exit codes
+// across platforms rather than parsing output.
+func (powerShell) PathExists(path string) string {
+ return "if (Test-Path -LiteralPath " + (powerShell{}).Quote(path) + ") { exit 0 } else { exit 1 }"
+}
+
+// ServiceQuery asks the Service Control Manager. The supervised backend has no manager, same as on
+// POSIX, so it gets no query.
+func (powerShell) ServiceQuery(kind, label string) string {
+ if kind == "windows-service" {
+ return "sc.exe query " + (powerShell{}).Quote(label) + " 2>&1"
+ }
+ return ""
+}
+
+// ServiceAbsent looks for error 1060, which is what sc.exe returns for a service that does not
+// exist. Matched on the number rather than the message text, because the message is localized and a
+// German or Japanese machine would otherwise report a removed service as still present.
+func (powerShell) ServiceAbsent(kind string, exitCode int, output string) bool {
+ if kind != "windows-service" {
+ return false
+ }
+ return strings.Contains(output, "1060")
+}
diff --git a/beacon-sandbox/scenario/scenario.go b/beacon-sandbox/scenario/scenario.go
index aecb3313..c93a5499 100644
--- a/beacon-sandbox/scenario/scenario.go
+++ b/beacon-sandbox/scenario/scenario.go
@@ -58,6 +58,16 @@ type Install struct {
// implied by "running": the supervised fallback also reports running, so a systemd scenario
// on a host without systemd would otherwise pass while testing the wrong thing.
ExpectServiceKind string `yaml:"expect_service_kind,omitempty"`
+ // VerifyUninstall removes the endpoint after the capture assertions and checks that it went.
+ //
+ // Worth asking separately from install because uninstall fails in a shape nothing else catches:
+ // it reports success while leaving the service registered, so the collector returns at the next
+ // reboot after the operator was told it was gone. That shipped once in this repository and was
+ // caught by a reviewer reading a diff, because no run had ever asked the question.
+ //
+ // The removal is real, not a rehearsal. It runs after the runtime log has been collected, so the
+ // capture assertions are already settled and cannot be affected by it.
+ VerifyUninstall bool `yaml:"verify_uninstall,omitempty"`
// NeedsRealSystemd runs the install inside a nested privileged container where systemd is
// genuinely PID 1.
//
diff --git a/beacon-sandbox/scenario/suite_shipped_test.go b/beacon-sandbox/scenario/suite_shipped_test.go
index 79b21ce5..2cecbccc 100644
--- a/beacon-sandbox/scenario/suite_shipped_test.go
+++ b/beacon-sandbox/scenario/suite_shipped_test.go
@@ -65,3 +65,46 @@ func TestWindowsScenariosDeclareTheirPlatform(t *testing.T) {
}
}
}
+
+// TestEveryPlatformVerifiesUninstallSomewhere encodes the gap this closed.
+//
+// The gap was not "one scenario is missing an assertion" — it was that *no* scenario on *any* platform
+// ever asked whether uninstall removed anything. Every one installed, captured, and stopped. So the
+// property worth pinning is per-platform coverage, which stays true as capture scenarios are added and
+// would go false again if the lifecycle ones lost the flag.
+//
+// Deliberately not "every install scenario verifies uninstall". w03 and w04 install in order to test
+// capture; making them assert removal too would report an uninstall regression against a capture
+// scenario, which sends whoever reads it to the wrong code.
+func TestEveryPlatformVerifiesUninstallSomewhere(t *testing.T) {
+ suite, err := LoadSuite("../scenarios")
+ if err != nil {
+ t.Fatalf("LoadSuite: %v", err)
+ }
+
+ installs := map[Platform]int{}
+ verifies := map[Platform]int{}
+ for _, scenario := range suite.Scenarios {
+ if scenario.Install == nil {
+ continue
+ }
+ platform := scenario.TargetPlatform()
+ installs[platform]++
+ if scenario.Install.VerifyUninstall {
+ verifies[platform]++
+ }
+ }
+
+ if len(installs) == 0 {
+ t.Fatal("no scenario installs an endpoint, so nothing covers the install path at all")
+ }
+ for platform, count := range installs {
+ if verifies[platform] == 0 {
+ t.Errorf("%s has %d install scenario(s) and none of them verifies uninstall; "+
+ "an uninstall that reports success while leaving the service registered would go "+
+ "unnoticed on this platform", platform, count)
+ } else {
+ t.Logf("%s: %d/%d install scenarios verify uninstall", platform, verifies[platform], count)
+ }
+ }
+}
diff --git a/beacon-sandbox/scenarios/i01-install-supervised.yaml b/beacon-sandbox/scenarios/i01-install-supervised.yaml
index 1b40cbb4..ca4b7660 100644
--- a/beacon-sandbox/scenarios/i01-install-supervised.yaml
+++ b/beacon-sandbox/scenarios/i01-install-supervised.yaml
@@ -21,6 +21,7 @@ install:
# Asserted rather than assumed: if systemd detection ever started returning true in a container,
# this scenario would silently stop covering the fallback backend it exists to cover.
expect_service_kind: none
+ verify_uninstall: true
prompt: >-
Run exactly this shell command and report its output: echo {{canary}} | tee {{sentinel}}
diff --git a/beacon-sandbox/scenarios/i02-install-systemd.yaml b/beacon-sandbox/scenarios/i02-install-systemd.yaml
index 8a835c50..574703f1 100644
--- a/beacon-sandbox/scenarios/i02-install-systemd.yaml
+++ b/beacon-sandbox/scenarios/i02-install-systemd.yaml
@@ -23,6 +23,7 @@ install:
# Not redundant with the above. The supervised backend also reports running, so without this a
# host where systemd detection failed would pass this scenario while testing the fallback.
expect_service_kind: systemd
+ verify_uninstall: true
prompt: >-
Run exactly this shell command and report its output: echo {{canary}} | tee {{sentinel}}
diff --git a/beacon-sandbox/scenarios/w01-install-service.yaml b/beacon-sandbox/scenarios/w01-install-service.yaml
index 9ddea89a..321db739 100644
--- a/beacon-sandbox/scenarios/w01-install-service.yaml
+++ b/beacon-sandbox/scenarios/w01-install-service.yaml
@@ -26,6 +26,7 @@ install:
service: windows-service
expect_status_running: true
expect_service_kind: windows-service
+ verify_uninstall: true
expect:
- action: prompt.submitted
diff --git a/beacon-sandbox/scenarios/w02-install-supervised.yaml b/beacon-sandbox/scenarios/w02-install-supervised.yaml
index 69a461ff..e0765c3f 100644
--- a/beacon-sandbox/scenarios/w02-install-supervised.yaml
+++ b/beacon-sandbox/scenarios/w02-install-supervised.yaml
@@ -21,6 +21,7 @@ install:
mode: user
expect_status_running: true
expect_service_kind: none
+ verify_uninstall: true
expect:
- action: prompt.submitted