The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
feat(sandbox): assert that uninstall actually removes the endpoint
shukieshah67a9ca0b9803
diff --git a/beacon-sandbox/check/uninstall.go b/beacon-sandbox/check/uninstall.gonew file mode 100644index 00000000..20b07654--- /dev/null+++ b/beacon-sandbox/check/uninstall.go@@ -0,0 +1,148 @@+package check++import (+ "fmt"+ "strings"+)++// Removal is what the runner observed after asking Beacon to uninstall itself.+//+// Every field is a string rather than a bool, and deliberately so: an empty value means the probe did+// not make that observation, which is a third state distinct from true and false. Collapsing it into+// false would let a probe that never ran report a removed service, and reporting a removal that was+// never checked is the exact failure this whole check exists to prevent.+type Removal struct {+ // Ran is "true" when the scenario asked for an uninstall at all.+ Ran string+ // ExitCode is `endpoint uninstall`'s status, as text.+ ExitCode string+ // Output is what it printed.+ Output string+ // ServiceKind and ServiceLabel identify what was registered at install time.+ ServiceKind string+ ServiceLabel string+ // ServiceGone is "true"/"false" when the service manager was asked, empty when it was not.+ ServiceGone string+ ServiceQuery string+ // ConfigRetained and LogRetained are "true"/"false" when the path was checked, empty otherwise.+ ConfigRetained string+ LogRetained string+ // Status is `endpoint status --json` after the removal, which is Beacon's own account of itself.+ Status string+}++// Uninstall judges whether a removal actually removed anything.+//+// The failure this is aimed at is specific and has happened: an uninstall that reports success while+// the service stays registered with automatic start, so the collector returns at the next reboot after+// the operator was told it was gone. Nothing in the suite asked that question before -- every scenario+// installed, captured, and stopped -- so it was caught by a reviewer reading a diff, which is not a+// mechanism that scales.+//+// Two questions, and they are asked of two different authorities. Whether Beacon *thinks* it is+// uninstalled comes from its own status output. Whether the service is still registered is a fact+// about the machine, obtained from the service manager. Those are exactly the two that came apart in+// the bug, so a check that only asked Beacon would have passed.+func Uninstall(v *Verdict, r Removal) {+ if r.Ran != "true" {+ // The scenario did not ask. Silence is right: not every scenario should uninstall, and an+ // absent check must not be reported as a passing one.+ return+ }++ if r.ExitCode != "0" {+ v.Add(Finding{+ Check: "uninstall.command_succeeded",+ Severity: SevFail,+ Summary: fmt.Sprintf("`endpoint uninstall` exited %s: %s", r.ExitCode, oneLineOf(r.Output)),+ Why: "An uninstall that cannot complete leaves the endpoint in a state no one asked for: " ++ "binaries possibly gone, service possibly still registered.",+ })+ }++ switch r.ServiceGone {+ case "true":+ v.Add(Finding{+ Check: "uninstall.service_removed",+ Severity: SevInfo,+ Summary: fmt.Sprintf("the %s service %q is no longer registered", r.ServiceKind, r.ServiceLabel),+ })+ case "false":+ v.Add(Finding{+ Check: "uninstall.service_removed",+ Severity: SevFail,+ Summary: fmt.Sprintf("the %s service %q is still registered after uninstall: %s",+ r.ServiceKind, r.ServiceLabel, oneLineOf(r.ServiceQuery)),+ Why: "A service left registered with automatic start brings the collector back at the next " ++ "reboot, after the operator was told it was removed. Uninstall reporting success while " ++ "this is true is the failure this check exists for.",+ })+ default:+ // Unverified, not clean. The supervised backend has no manager to ask, and a probe that could+ // not identify what was registered has no question to put to one.+ v.Add(Finding{+ Check: "uninstall.service_removed",+ Severity: SevWarn,+ Summary: fmt.Sprintf("the service manager was not asked whether %q is gone (kind=%q)",+ r.ServiceLabel, r.ServiceKind),+ Why: "Removal of the service registration is unverified for this run. It is not evidence " ++ "that anything was left behind, and it is not evidence that nothing was.",+ })+ }++ // Retention is the other half of the contract, and it fails in the opposite direction: too much+ // removed rather than too little. An uninstall is often the first half of a reinstall, and+ // destroying an operator's configuration and collected telemetry is a separate, deliberate act.+ for name, retained := range map[string]string{+ "configuration": r.ConfigRetained,+ "runtime log": r.LogRetained,+ } {+ switch retained {+ case "true":+ v.Add(Finding{+ Check: "uninstall.data_retained",+ Severity: SevInfo,+ Summary: fmt.Sprintf("%s survived the uninstall, as --keep-logs and --keep-config ask", name),+ })+ case "false":+ v.Add(Finding{+ Check: "uninstall.data_retained",+ Severity: SevFail,+ Summary: fmt.Sprintf("%s was removed despite --keep-logs and --keep-config", name),+ Why: "Removing the product must not destroy what it collected unless a purge asks for " ++ "it. Note that --keep-config governs harness settings rather than Beacon's own " ++ "config, which is why packaging stashes those files around a removal.",+ })+ default:+ v.Add(Finding{+ Check: "uninstall.data_retained",+ Severity: SevWarn,+ Summary: fmt.Sprintf("whether the %s survived was not checked", name),+ Why: "Retention is unverified for this run rather than confirmed.",+ })+ }+ }++ // Beacon's own account, checked against the machine's. A status still describing a running+ // collector after a successful removal means the two disagree, and the disagreement is worth a+ // finding of its own even when the service really is gone.+ if strings.Contains(r.Status, `"running":true`) {+ v.Add(Finding{+ Check: "uninstall.status_agrees",+ Severity: SevFail,+ Summary: "`endpoint status` still reports a running collector after uninstall",+ Why: "Beacon's view of itself and the machine's have to agree, or one of them is lying to " ++ "whoever reads it next.",+ })+ }+}++// oneLineOf keeps a finding's detail to one line. Multi-line output in a verdict makes the report+// unreadable, which is how a real failure gets skimmed past.+func oneLineOf(s string) string {+ flat := strings.Join(strings.Fields(s), " ")+ if len(flat) > 300 {+ return flat[:300] + "…"+ }+ return flat+}diff --git a/beacon-sandbox/check/uninstall_test.go b/beacon-sandbox/check/uninstall_test.gonew file mode 100644index 00000000..630ef88f--- /dev/null+++ b/beacon-sandbox/check/uninstall_test.go@@ -0,0 +1,148 @@+package check++import (+ "strings"+ "testing"+)++// clean is the shape of a removal that did everything right: service gone, data kept, Beacon's own+// status agreeing.+func clean() Removal {+ return Removal{+ Ran: "true",+ ExitCode: "0",+ Output: "Endpoint uninstalled",+ ServiceKind: "systemd",+ ServiceLabel: "beacon-collector.service",+ ServiceGone: "true",+ ConfigRetained: "true",+ LogRetained: "true",+ Status: `{"service":{"running":false}}`,+ }+}++func findingsFor(r Removal) []Finding {+ var v Verdict+ Uninstall(&v, r)+ return v.Findings+}++func has(findings []Finding, name string, sev Severity) bool {+ for _, f := range findings {+ if f.Check == name && f.Severity == sev {+ return true+ }+ }+ return false+}++func TestACleanRemovalRaisesNoFailures(t *testing.T) {+ for _, f := range findingsFor(clean()) {+ if f.Severity == SevFail {+ t.Fatalf("clean removal produced a failure: %s — %s", f.Check, f.Summary)+ }+ }+}++// TestAScenarioThatDidNotUninstallIsSilent is what keeps this check honest about its own absence.+//+// Most scenarios do not uninstall. Emitting a passing finding for them would report a check that+// never ran as one that succeeded, which is the failure mode this package is built to avoid.+func TestAScenarioThatDidNotUninstallIsSilent(t *testing.T) {+ if findings := findingsFor(Removal{}); len(findings) != 0 {+ t.Fatalf("a run with no uninstall produced %d finding(s): %#v", len(findings), findings)+ }+}++// TestAServiceLeftRegisteredFails is the bug this check exists for.+//+// It shipped once: uninstall reported success while the Windows service stayed registered with+// automatic start, so the collector returned at the next reboot after the operator was told it was+// gone. A reviewer reading a diff caught it, because no run had ever asked.+func TestAServiceLeftRegisteredFails(t *testing.T) {+ r := clean()+ r.ServiceGone = "false"+ r.ServiceQuery = "SERVICE_NAME: BeaconCollector STATE: 4 RUNNING"++ findings := findingsFor(r)+ if !has(findings, "uninstall.service_removed", SevFail) {+ t.Fatalf("a still-registered service did not fail: %#v", findings)+ }+ // The evidence has to travel with the finding, or whoever reads the report cannot tell a+ // still-registered service from a probe that misfired.+ for _, f := range findings {+ if f.Check == "uninstall.service_removed" && !strings.Contains(f.Summary, "BeaconCollector") {+ t.Fatalf("the finding does not quote what the service manager said: %q", f.Summary)+ }+ }+}++// TestAnUnaskedServiceQuestionIsWarnedNotPassed covers the third state.+//+// The supervised backend has no service manager to ask, and a probe that could not identify what was+// registered has no question to put to one. Neither is evidence of a clean removal, and reporting+// either as one would be exactly the silent pass this tool exists to prevent.+func TestAnUnaskedServiceQuestionIsWarnedNotPassed(t *testing.T) {+ r := clean()+ r.ServiceGone = ""+ r.ServiceKind = "none"++ findings := findingsFor(r)+ if has(findings, "uninstall.service_removed", SevInfo) {+ t.Fatal("an unasked question was reported as a clean removal")+ }+ if !has(findings, "uninstall.service_removed", SevWarn) {+ t.Fatalf("an unasked question produced no warning: %#v", findings)+ }+}++// TestDestroyedDataFails covers the opposite direction from a leftover service: too much removed+// rather than too little.+func TestDestroyedDataFails(t *testing.T) {+ r := clean()+ r.ConfigRetained = "false"+ if !has(findingsFor(r), "uninstall.data_retained", SevFail) {+ t.Fatal("configuration destroyed by a non-purge uninstall did not fail")+ }++ r = clean()+ r.LogRetained = "false"+ if !has(findingsFor(r), "uninstall.data_retained", SevFail) {+ t.Fatal("a runtime log destroyed by a non-purge uninstall did not fail")+ }+}++func TestUncheckedRetentionIsWarnedNotPassed(t *testing.T) {+ r := clean()+ r.ConfigRetained = ""+ r.LogRetained = ""+ findings := findingsFor(r)+ if has(findings, "uninstall.data_retained", SevInfo) {+ t.Fatal("unchecked retention was reported as confirmed")+ }+ if !has(findings, "uninstall.data_retained", SevWarn) {+ t.Fatalf("unchecked retention produced no warning: %#v", findings)+ }+}++func TestANonZeroUninstallFails(t *testing.T) {+ r := clean()+ r.ExitCode = "1"+ r.Output = "Error: could not stop the service"+ if !has(findingsFor(r), "uninstall.command_succeeded", SevFail) {+ t.Fatal("a failed uninstall command did not fail the run")+ }+}++// TestStatusDisagreeingWithTheMachineFails covers the case where the two authorities disagree.+//+// Both are asked on purpose. Beacon's status is Beacon's opinion of itself; the service manager is a+// fact about the machine. Those are the two that came apart in the original bug, so a check reading+// only one of them would have passed it.+func TestStatusDisagreeingWithTheMachineFails(t *testing.T) {+ r := clean()+ r.Status = `{"service":{"label":"beacon-collector.service","loaded":true,"running":true}}`+ if !has(findingsFor(r), "uninstall.status_agrees", SevFail) {+ t.Fatal("a status still reporting a running collector after uninstall did not fail")+ }+}diff --git a/beacon-sandbox/cmd/beacon-sandbox/main.go b/beacon-sandbox/cmd/beacon-sandbox/main.goindex 97a5070c..ff30210d 100644--- a/beacon-sandbox/cmd/beacon-sandbox/main.go+++ b/beacon-sandbox/cmd/beacon-sandbox/main.go@@ -398,6 +398,20 @@ func judge(sc scenario.Scenario, art runner.Artifacts, creds credentials.ResolveProbed: art.SentinelProbed,Detail: art.SentinelDetail,}, check.Session{Known: art.SessionKnown, OK: art.SessionOK})+ // Read from meta rather than from a live probe, so `verify` re-judges a saved run's removal for+ // free exactly as it re-judges its capture.+ check.Uninstall(&v, check.Removal{+ Ran: art.Meta["uninstall_ran"],+ ExitCode: art.Meta["uninstall_rc"],+ Output: art.Meta["uninstall_output"],+ ServiceKind: art.Meta["service_kind"],+ ServiceLabel: art.Meta["service_label"],+ ServiceGone: art.Meta["uninstall_service_gone"],+ ServiceQuery: art.Meta["uninstall_service_query"],+ ConfigRetained: art.Meta["uninstall_config_retained"],+ LogRetained: art.Meta["uninstall_log_retained"],+ Status: art.Meta["uninstall_status"],+ })v.Resolve()return v, log}@@ -542,12 +556,20 @@ func judgeRunDir(root, dir, mutate string) (check.Verdict, error) {logPath := filepath.Join(dir, "runtime.jsonl")var planted stringif mutate != "" {- logPath, planted, err = applyMutation(logPath, mutate)- if err != nil {- return check.Verdict{}, err+ // The uninstall checks read what the runner observed about the machine, not the event log, so+ // damaging the log could never exercise them. Their mutations rewrite the recorded observation+ // instead -- which is the same idea applied to a different artifact, and the only way a check+ // that reads meta can be shown to be capable of failing.+ if applyMetaMutation(meta, mutate) {+ fmt.Printf("(mutation %q applied to run metadata: a PASS becoming FAIL is the check working)\n", mutate)+ } else {+ logPath, planted, err = applyMutation(logPath, mutate)+ if err != nil {+ return check.Verdict{}, err+ }+ defer os.Remove(logPath)+ fmt.Printf("(mutation %q applied: a PASS becoming FAIL is the check working)\n", mutate)}- defer os.Remove(logPath)- fmt.Printf("(mutation %q applied: a PASS becoming FAIL is the check working)\n", mutate)}art := runner.Artifacts{RuntimeLog: logPath, Meta: meta}@@ -702,7 +724,7 @@ func applyMutation(path, mode string) (mutated string, planted string, err error}default:return "", "", fmt.Errorf("unknown mutation %q (want drop-commands, drop-action:<action>, "+- "corrupt-line, or plant-secret)", mode)+ "corrupt-line, plant-secret, leave-service, or drop-retained-config)", mode)}// The shared post-condition: the log must differ from what it was. Compared on the trimmed@@ -812,3 +834,32 @@ func indent(s string) string {}return b.String()}++// applyMetaMutation damages a recorded observation rather than the event log, and reports whether it+// recognised the mode.+//+// The uninstall checks judge what the runner saw the machine do — whether a service was still+// registered, whether configuration survived. None of that is in runtime.jsonl, so the log mutations+// cannot exercise them, and a check that cannot be made to fail is worse than no check at all.+//+// Each mode inverts exactly one observation, so the resulting failure names the check under test+// rather than a cascade.+func applyMetaMutation(meta map[string]string, mode string) bool {+ if meta == nil || meta["uninstall_ran"] != "true" {+ // Nothing to damage. A run that never uninstalled cannot demonstrate an uninstall check, and+ // silently "succeeding" here would report a self-test as passed when it never ran.+ return false+ }+ switch mode {+ case "leave-service":+ // The failure this simulates is the one that shipped: uninstall reports success while the+ // service stays registered and starts at boot.+ meta["uninstall_service_gone"] = "false"+ return true+ case "drop-retained-config":+ // The opposite direction: a removal that took the operator's configuration with it.+ meta["uninstall_config_retained"] = "false"+ return true+ }+ return false+}diff --git a/beacon-sandbox/cmd/beacon-sandbox/mutation_meta_test.go b/beacon-sandbox/cmd/beacon-sandbox/mutation_meta_test.gonew file mode 100644index 00000000..3d3c305a--- /dev/null+++ b/beacon-sandbox/cmd/beacon-sandbox/mutation_meta_test.go@@ -0,0 +1,35 @@+package main++import "testing"++// The meta mutations must turn a clean removal into a failure, or they prove nothing.+func TestMetaMutationsInvertARecordedRemoval(t *testing.T) {+ for _, mode := range []string{"leave-service", "drop-retained-config"} {+ meta := map[string]string{+ "uninstall_ran": "true",+ "uninstall_service_gone": "true",+ "uninstall_config_retained": "true",+ }+ if !applyMetaMutation(meta, mode) {+ t.Fatalf("%s was not recognised as a meta mutation", mode)+ }+ if meta["uninstall_service_gone"] == "true" && meta["uninstall_config_retained"] == "true" {+ t.Fatalf("%s changed nothing", mode)+ }+ }+}++// A run that never uninstalled cannot demonstrate an uninstall check, and claiming otherwise would+// report a self-test as passed when it never ran.+func TestMetaMutationRefusesARunThatDidNotUninstall(t *testing.T) {+ if applyMetaMutation(map[string]string{}, "leave-service") {+ t.Fatal("a run with no uninstall accepted an uninstall mutation")+ }+}++func TestUnknownModesFallThroughToLogMutations(t *testing.T) {+ meta := map[string]string{"uninstall_ran": "true"}+ if applyMetaMutation(meta, "corrupt-line") {+ t.Fatal("corrupt-line was claimed by the meta mutator; it damages the log")+ }+}diff --git a/beacon-sandbox/runner/runner.go b/beacon-sandbox/runner/runner.goindex 84ca6f57..a0c16ddc 100644--- a/beacon-sandbox/runner/runner.go+++ b/beacon-sandbox/runner/runner.go@@ -456,6 +456,13 @@ func Run(ctx context.Context, p sandbox.Provider, sc scenario.Scenario, opts Optart.RuntimeLog = localLoglogf("collected %s", localLog)+ // Uninstall last, and only when the scenario asked. The runtime log is already on the host, so+ // removing the endpoint cannot affect any capture assertion -- which is what makes it safe to do+ // inside the same run rather than needing a scenario of its own.+ if sc.Install != nil && sc.Install.VerifyUninstall {+ probeUninstall(ctx, g, sc, privileged, agent, lay, sh, &art, logf)+ }+// meta.json is written by the deferred host-state block above, so it always includes the// host comparison.return art, nil@@ -802,8 +809,9 @@ func installedLogPath(ctx context.Context, g guest,if err == nil {art.Meta["install_status"] = oneLine(s.Stdout)var status struct {- LogPath string `json:"log_path"`- Service struct {+ LogPath string `json:"log_path"`+ ConfigPath string `json:"config_path"`+ Service struct {Label string `json:"label"`Loaded bool `json:"loaded"`Running bool `json:"running"`@@ -812,6 +820,10 @@ func installedLogPath(ctx context.Context, g guest,} `json:"service"`}if json.Unmarshal([]byte(s.Stdout), &status) == nil {+ // Recorded so a later uninstall probe checks the paths Beacon actually used rather than+ // the ones a scenario assumed.+ art.Meta["install_log_path"] = status.LogPath+ art.Meta["install_config_path"] = status.ConfigPathart.Meta["service_kind"] = status.Service.Kindart.Meta["service_label"] = status.Service.Labelart.Meta["service_loaded"] = fmt.Sprintf("%v", status.Service.Loaded)@@ -881,3 +893,85 @@ func teardownInstall(ctx context.Context, g guest, sc scenario.Scenario,logf("teardown: uninstalled the %s endpoint", strings.TrimPrefix(scope, "--"))}}++// probeUninstall removes the endpoint and records what survived.+//+// Deliberately records rather than judges. Every other verdict in this tool is produced by the pure+// check package from collected artifacts, which is what makes `verify` able to re-judge a saved run+// for free -- so this writes observations into meta and leaves the conclusions to check.Uninstall.+//+// The observations are gathered by asking the operating system, not by asking Beacon. `endpoint+// status` reporting "not installed" is Beacon's opinion of its own state; whether the service is+// still registered is a fact about the machine, and those are exactly the two that came apart in the+// bug this exists to catch.+func probeUninstall(ctx context.Context, g guest, sc scenario.Scenario,+ privileged, agent sandbox.ExecOpts, lay image.Layout, sh shell,+ art *Artifacts, logf func(string, ...any)) {++ system := sc.Install.Mode == "system"+ scope := "--user"+ opts := agent+ if system {+ scope = "--system"+ opts = privileged+ }++ // The service label and kind were recorded at install time. Without them there is nothing to ask+ // the service manager about, so the probe says so rather than reporting a removal it could not+ // verify.+ label := art.Meta["service_label"]+ kind := art.Meta["service_kind"]++ // Config and log paths come from the install status, so the probe checks the paths Beacon+ // actually used rather than the ones the scenario assumed.+ configPath := art.Meta["install_config_path"]+ logPath := art.Meta["install_log_path"]++ // --keep-logs and --keep-config, because that is the contract a package removal keeps: an+ // uninstall is often the first half of a reinstall, and destroying collected telemetry is a+ // separate, deliberate act. Asserting the retention is half of what this probe is for.+ r, err := g.Exec(ctx, "beacon endpoint uninstall "+scope+" --keep-logs --keep-config 2>&1", opts)+ art.Meta["uninstall_ran"] = "true"+ art.Meta["uninstall_output"] = oneLine(r.Stdout + r.Stderr)+ art.Meta["uninstall_rc"] = fmt.Sprintf("%d", r.ExitCode)+ if err != nil {+ art.Meta["uninstall_exec_error"] = oneLine(err.Error())+ }+ logf("uninstall rc=%d: %s", r.ExitCode, art.Meta["uninstall_output"])++ // Asked of the service manager directly. A supervised collector has no manager to ask, so its+ // absence is established from the pidfile instead, and the probe records which question it asked.+ if label != "" && kind != "" && kind != "none" {+ query := sh.ServiceQuery(kind, label)+ if query != "" {+ q, _ := g.Exec(ctx, query, opts)+ art.Meta["uninstall_service_query"] = oneLine(q.Stdout + q.Stderr)+ art.Meta["uninstall_service_query_rc"] = fmt.Sprintf("%d", q.ExitCode)+ art.Meta["uninstall_service_gone"] = fmt.Sprintf("%v", sh.ServiceAbsent(kind, q.ExitCode, q.Stdout+q.Stderr))+ logf("service %s after uninstall: gone=%s", label, art.Meta["uninstall_service_gone"])+ }+ } else if kind == "none" {+ // The supervised backend's registration *is* its pidfile, so that is the thing that has to+ // be gone.+ art.Meta["uninstall_service_kind_supervised"] = "true"+ }++ // Retention, asked of the filesystem. Absent paths are reported as unknown rather than as+ // retained: a check that cannot see the file must not conclude it survived.+ for key, path := range map[string]string{+ "uninstall_config_retained": configPath,+ "uninstall_log_retained": logPath,+ } {+ if path == "" {+ continue+ }+ e, _ := g.Exec(ctx, sh.PathExists(path), opts)+ art.Meta[key] = fmt.Sprintf("%v", e.ExitCode == 0)+ }++ // Beacon's own account of itself, recorded alongside the machine's. When the two disagree, that+ // disagreement is the finding.+ if s, err := g.Exec(ctx, "beacon endpoint status "+scope+" --json 2>&1", opts); err == nil {+ art.Meta["uninstall_status"] = oneLine(s.Stdout)+ }+}diff --git a/beacon-sandbox/runner/shell.go b/beacon-sandbox/runner/shell.goindex 5f499b80..fdf66f21 100644--- a/beacon-sandbox/runner/shell.go+++ b/beacon-sandbox/runner/shell.go@@ -41,6 +41,18 @@ type shell interface {ArgvSampler() string// ArgvVerdictPath is where the sampler writes what it saw.ArgvVerdictPath() string+ // PathExists exits zero when the path is there, non-zero when it is not.+ PathExists(path string) string+ // ServiceQuery asks the platform's service manager about a service by name, or returns empty+ // when this backend has no manager to ask.+ ServiceQuery(kind, label string) string+ // ServiceAbsent interprets a ServiceQuery result as "no such service".+ //+ // Split from ServiceQuery because absence is spelled differently on each platform, and getting it+ // wrong is the dangerous direction: a query whose failure is misread as absence would report a+ // still-registered service as removed, which is precisely the bug an uninstall check exists to+ // catch.+ ServiceAbsent(kind string, exitCode int, output string) bool}// shellFor returns the dialect for a platform.@@ -356,3 +368,68 @@ if (-not $env:ANTHROPIC_API_KEY) {Start-Process -FilePath 'pwsh' -ArgumentList '-NoProfile','-NonInteractive','-File',%[2]s,%[1]s -WindowStyle Hidden'ARGV_SAMPLER_STARTED'`, s.Quote(verdict), s.Quote(scriptPath), enc)}++// PathExists uses test, which is a shell builtin and needs no external binary -- a minimal image may+// have no coreutils.+func (posixShell) PathExists(path string) string {+ return "test -e " + (posixShell{}).Quote(path)+}++// ServiceQuery asks the manager that actually registered the service.+//+// Only systemd and launchd have anything to ask. The supervised backend's registration is its+// pidfile, which the caller checks as a path instead, so this returns empty rather than inventing a+// query that would always succeed.+func (posixShell) ServiceQuery(kind, label string) string {+ switch kind {+ case "systemd":+ // list-unit-files rather than status: status reports on a unit that no longer exists with the+ // same exit code as one that is merely stopped, so it cannot distinguish removed from+ // inactive. This prints nothing at all once the unit file is gone.+ return "systemctl list-unit-files " + (posixShell{}).Quote(label) + " --no-legend 2>&1"+ case "launchd":+ return "launchctl print system/" + (posixShell{}).Quote(label) + " 2>&1"+ }+ return ""+}++// ServiceAbsent reads the query output for each manager's spelling of "not there".+func (posixShell) ServiceAbsent(kind string, exitCode int, output string) bool {+ trimmed := strings.TrimSpace(output)+ switch kind {+ case "systemd":+ // A removed unit produces no rows. Non-zero with output is a different problem -- systemd+ // unreachable, say -- and must not read as absence.+ return trimmed == "" || strings.Contains(trimmed, "0 unit files listed")+ case "launchd":+ // launchctl prints this for a job it does not know about.+ return exitCode != 0 && (strings.Contains(trimmed, "Could not find service") ||+ strings.Contains(trimmed, "No such process"))+ }+ return false+}++// PathExists uses Test-Path and turns it into an exit code, because the caller compares exit codes+// across platforms rather than parsing output.+func (powerShell) PathExists(path string) string {+ return "if (Test-Path -LiteralPath " + (powerShell{}).Quote(path) + ") { exit 0 } else { exit 1 }"+}++// ServiceQuery asks the Service Control Manager. The supervised backend has no manager, same as on+// POSIX, so it gets no query.+func (powerShell) ServiceQuery(kind, label string) string {+ if kind == "windows-service" {+ return "sc.exe query " + (powerShell{}).Quote(label) + " 2>&1"+ }+ return ""+}++// ServiceAbsent looks for error 1060, which is what sc.exe returns for a service that does not+// exist. Matched on the number rather than the message text, because the message is localized and a+// German or Japanese machine would otherwise report a removed service as still present.+func (powerShell) ServiceAbsent(kind string, exitCode int, output string) bool {+ if kind != "windows-service" {+ return false+ }+ return strings.Contains(output, "1060")+}diff --git a/beacon-sandbox/scenario/scenario.go b/beacon-sandbox/scenario/scenario.goindex aecb3313..c93a5499 100644--- a/beacon-sandbox/scenario/scenario.go+++ b/beacon-sandbox/scenario/scenario.go@@ -58,6 +58,16 @@ type Install struct {// implied by "running": the supervised fallback also reports running, so a systemd scenario// on a host without systemd would otherwise pass while testing the wrong thing.ExpectServiceKind string `yaml:"expect_service_kind,omitempty"`+ // VerifyUninstall removes the endpoint after the capture assertions and checks that it went.+ //+ // Worth asking separately from install because uninstall fails in a shape nothing else catches:+ // it reports success while leaving the service registered, so the collector returns at the next+ // reboot after the operator was told it was gone. That shipped once in this repository and was+ // caught by a reviewer reading a diff, because no run had ever asked the question.+ //+ // The removal is real, not a rehearsal. It runs after the runtime log has been collected, so the+ // capture assertions are already settled and cannot be affected by it.+ VerifyUninstall bool `yaml:"verify_uninstall,omitempty"`// NeedsRealSystemd runs the install inside a nested privileged container where systemd is// genuinely PID 1.//diff --git a/beacon-sandbox/scenario/suite_shipped_test.go b/beacon-sandbox/scenario/suite_shipped_test.goindex 79b21ce5..2cecbccc 100644--- a/beacon-sandbox/scenario/suite_shipped_test.go+++ b/beacon-sandbox/scenario/suite_shipped_test.go@@ -65,3 +65,46 @@ func TestWindowsScenariosDeclareTheirPlatform(t *testing.T) {}}}++// TestEveryPlatformVerifiesUninstallSomewhere encodes the gap this closed.+//+// The gap was not "one scenario is missing an assertion" — it was that *no* scenario on *any* platform+// ever asked whether uninstall removed anything. Every one installed, captured, and stopped. So the+// property worth pinning is per-platform coverage, which stays true as capture scenarios are added and+// would go false again if the lifecycle ones lost the flag.+//+// Deliberately not "every install scenario verifies uninstall". w03 and w04 install in order to test+// capture; making them assert removal too would report an uninstall regression against a capture+// scenario, which sends whoever reads it to the wrong code.+func TestEveryPlatformVerifiesUninstallSomewhere(t *testing.T) {+ suite, err := LoadSuite("../scenarios")+ if err != nil {+ t.Fatalf("LoadSuite: %v", err)+ }++ installs := map[Platform]int{}+ verifies := map[Platform]int{}+ for _, scenario := range suite.Scenarios {+ if scenario.Install == nil {+ continue+ }+ platform := scenario.TargetPlatform()+ installs[platform]+++ if scenario.Install.VerifyUninstall {+ verifies[platform]+++ }+ }++ if len(installs) == 0 {+ t.Fatal("no scenario installs an endpoint, so nothing covers the install path at all")+ }+ for platform, count := range installs {+ if verifies[platform] == 0 {+ t.Errorf("%s has %d install scenario(s) and none of them verifies uninstall; "++ "an uninstall that reports success while leaving the service registered would go "++ "unnoticed on this platform", platform, count)+ } else {+ t.Logf("%s: %d/%d install scenarios verify uninstall", platform, verifies[platform], count)+ }+ }+}diff --git a/beacon-sandbox/scenarios/i01-install-supervised.yaml b/beacon-sandbox/scenarios/i01-install-supervised.yamlindex 1b40cbb4..ca4b7660 100644--- a/beacon-sandbox/scenarios/i01-install-supervised.yaml+++ b/beacon-sandbox/scenarios/i01-install-supervised.yaml@@ -21,6 +21,7 @@ install:# Asserted rather than assumed: if systemd detection ever started returning true in a container,# this scenario would silently stop covering the fallback backend it exists to cover.expect_service_kind: none+ verify_uninstall: trueprompt: >-Run exactly this shell command and report its output: echo {{canary}} | tee {{sentinel}}diff --git a/beacon-sandbox/scenarios/i02-install-systemd.yaml b/beacon-sandbox/scenarios/i02-install-systemd.yamlindex 8a835c50..574703f1 100644--- a/beacon-sandbox/scenarios/i02-install-systemd.yaml+++ b/beacon-sandbox/scenarios/i02-install-systemd.yaml@@ -23,6 +23,7 @@ install:# Not redundant with the above. The supervised backend also reports running, so without this a# host where systemd detection failed would pass this scenario while testing the fallback.expect_service_kind: systemd+ verify_uninstall: trueprompt: >-Run exactly this shell command and report its output: echo {{canary}} | tee {{sentinel}}diff --git a/beacon-sandbox/scenarios/w01-install-service.yaml b/beacon-sandbox/scenarios/w01-install-service.yamlindex 9ddea89a..321db739 100644--- a/beacon-sandbox/scenarios/w01-install-service.yaml+++ b/beacon-sandbox/scenarios/w01-install-service.yaml@@ -26,6 +26,7 @@ install:service: windows-serviceexpect_status_running: trueexpect_service_kind: windows-service+ verify_uninstall: trueexpect:- action: prompt.submitteddiff --git a/beacon-sandbox/scenarios/w02-install-supervised.yaml b/beacon-sandbox/scenarios/w02-install-supervised.yamlindex 69a461ff..e0765c3f 100644--- a/beacon-sandbox/scenarios/w02-install-supervised.yaml+++ b/beacon-sandbox/scenarios/w02-install-supervised.yaml@@ -21,6 +21,7 @@ install:mode: userexpect_status_running: trueexpect_service_kind: none+ verify_uninstall: trueexpect:- action: prompt.submitted