The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
test(sandbox): prove rollback by making an install fail, instead of reasoning about it
shukieshahf99ab1ef9774
diff --git a/beacon-sandbox/check/lifecycle.go b/beacon-sandbox/check/lifecycle.goindex b5846962..67850f9c 100644--- a/beacon-sandbox/check/lifecycle.go+++ b/beacon-sandbox/check/lifecycle.go@@ -15,6 +15,12 @@ type Lifecycle struct {// UnprivilegedRefused is "true"/"false" when an unprivileged removal was attempted.UnprivilegedRefused stringUnprivilegedOutput string++ // RollbackInstallRC, RollbackCollectorCount and RollbackStatus describe a reinstall that was+ // made to fail on purpose. Empty when the scenario did not ask.+ RollbackInstallRC string+ RollbackCollectorCount string+ RollbackStatus string}// Reinstall judges whether installing over a running endpoint replaced the collector.@@ -59,6 +65,63 @@ func Reinstall(v *Verdict, l Lifecycle) {})}+// FailedReinstallRollback judges what a deliberately failed reinstall left behind.+//+// The contract is that a failed install returns the machine to the endpoint it had: one collector,+// running, on the configuration from before the attempt. Two ways of breaking it have shipped.+//+// Too few collectors means rollback took a healthy endpoint down -- and on systemd it also disables+// the unit, so the machine stays down across a reboot. Too many means rollback stopped the wrong+// process: the collector the failed install started keeps running on the new config, holding the+// ports, while another is started beside it. Asking whether a collector is running cannot tell any+// of these apart, because the answer is yes in every one.+func FailedReinstallRollback(v *Verdict, l Lifecycle) {+ if l.RollbackInstallRC == "" {+ return // the scenario did not ask+ }++ // A reinstall pointed at a program that never listens has to fail. If it reported success,+ // readiness is not being checked and the rest of this proves nothing.+ if l.RollbackInstallRC == "0" {+ v.Add(Finding{+ Check: "rollback.install_failed",+ Severity: SevFail,+ Summary: "a reinstall pointed at a collector that never listens reported success",+ Why: "Install is supposed to wait for the collector to become ready and roll back when " ++ "it does not. Reporting success here means an endpoint that cannot collect is " ++ "indistinguishable from one that can.",+ })+ return+ }++ switch l.RollbackCollectorCount {+ case "1":+ v.Add(Finding{+ Check: "rollback.left_one_collector",+ Severity: SevInfo,+ Summary: "a failed reinstall left exactly one collector running",+ })+ case "", "0":+ v.Add(Finding{+ Check: "rollback.left_one_collector",+ Severity: SevFail,+ Summary: "a failed reinstall left no collector running, so it took a working endpoint down with it",+ Why: "Rollback may undo what the failed install did; it may not remove what was working " ++ "beforehand. On systemd it also disables the unit, so the machine stays down across " ++ "a reboot -- an upgrade that fails and ends collection is worse than one that fails.",+ })+ default:+ v.Add(Finding{+ Check: "rollback.left_one_collector",+ Severity: SevFail,+ Summary: "a failed reinstall left " + l.RollbackCollectorCount + " collectors running",+ Why: "Rollback stopped the wrong process. The collector the failed install started is " ++ "still running on the configuration that failed, holding the OTLP ports, while " ++ "another was started beside it -- and the pidfile tracks only one of them.",+ })+ }+}+// UnprivilegedUninstall judges whether a system removal without privileges was refused.//// verify_uninstall cannot catch this, because it removes the endpoint elevated, which is the pathdiff --git a/beacon-sandbox/cmd/beacon-sandbox/main.go b/beacon-sandbox/cmd/beacon-sandbox/main.goindex 6aa96b0d..b782f2c2 100644--- a/beacon-sandbox/cmd/beacon-sandbox/main.go+++ b/beacon-sandbox/cmd/beacon-sandbox/main.go@@ -418,9 +418,14 @@ func judge(sc scenario.Scenario, art runner.Artifacts, creds credentials.ResolveReinstallErr: art.Meta["reinstall_error"],UnprivilegedRefused: art.Meta["unprivileged_uninstall_refused"],UnprivilegedOutput: art.Meta["unprivileged_uninstall_output"],++ RollbackInstallRC: art.Meta["rollback_install_rc"],+ RollbackCollectorCount: art.Meta["rollback_collector_count"],+ RollbackStatus: art.Meta["rollback_status"],}check.Reinstall(&v, lifecycle)check.UnprivilegedUninstall(&v, lifecycle)+ check.FailedReinstallRollback(&v, lifecycle)v.Resolve()return v, log}diff --git a/beacon-sandbox/runner/runner.go b/beacon-sandbox/runner/runner.goindex bd0ce29b..7f116227 100644--- a/beacon-sandbox/runner/runner.go+++ b/beacon-sandbox/runner/runner.go@@ -474,6 +474,12 @@ func Run(ctx context.Context, p sandbox.Provider, sc scenario.Scenario, opts OptprobeReinstall(ctx, g, sc, privileged, agent, &art, logf)}+ // After the reinstall probe, so a successful restart is established before a failing one is+ // induced, and before uninstall takes the endpoint away entirely.+ if sc.VerifiesRollbackOnFailedReinstall() {+ probeFailedReinstallRollback(ctx, g, sc, privileged, agent, &art, logf)+ }+// Unprivileged first, while there is still something installed for it to fail to remove.if sc.VerifiesUnprivilegedUninstallFails() {probeUnprivilegedUninstall(ctx, g, agent, &art, logf)@@ -1015,6 +1021,64 @@ func probeReinstall(ctx context.Context, g guest, sc scenario.Scenario,logf("reinstall: collector pid %s -> %s (restarted=%s)", pidBefore, pidAfter, art.Meta["reinstall_restarted"])}+// probeFailedReinstallRollback makes a reinstall fail and records what survived it.+//+// The failure is induced with a collector that starts and then exits without listening, so the+// service loads successfully and readiness is what fails. That ordering matters: it is the only way+// to reach rollback with a service it has to decide about, and every defect found here lived in+// that decision.+//+// Three things are recorded, and the count is the one that matters. A rollback that stops the wrong+// process leaves the collector the failed install started still running, and then starts another+// beside it -- so the machine ends with two collectors, the newer config live, and ports held by a+// process nothing is tracking. "Is a collector running?" is true in that state, which is why it is+// not the question asked.+func probeFailedReinstallRollback(ctx context.Context, g guest, sc scenario.Scenario,+ privileged, agent sandbox.ExecOpts, art *Artifacts, logf func(string, ...any)) {++ scope := "--user"+ opts := agent+ if sc.Install.Mode == "system" {+ scope = "--system"+ opts = privileged+ }++ // A stand-in collector that ignores its arguments and stays alive without listening.+ //+ // It has to keep running, which rules out the obvious `/bin/sleep`: given `--config <path>` that+ // exits immediately, and a process that exits on its own cannot be orphaned -- so the probe+ // would report a clean rollback whether or not rollback stopped anything. Staying alive is what+ // makes the collector count able to distinguish the two.+ fake := "/tmp/beacon-fake-collector"+ if _, err := g.Exec(ctx, "printf '#!/bin/sh\\nexec sleep 600\\n' > "+fake+" && chmod 0755 "+fake, opts); err != nil {+ art.Meta["rollback_error"] = "could not stage a stand-in collector: " + err.Error()+ return+ }++ args := "beacon endpoint install " + scope + " --harness claude --collector " + fake+ if svc := sc.Install.Service; svc != "" {+ args += " --service=" + svc+ }+ r, _ := g.Exec(ctx, args+" 2>&1", opts)+ art.Meta["rollback_install_rc"] = fmt.Sprintf("%d", r.ExitCode)+ art.Meta["rollback_install_output"] = oneLine(r.Stdout + r.Stderr)++ // Counted after the install has returned, so rollback has finished running. Both names are+ // counted: the stand-in is what a mishandled rollback leaves behind, and counting only the real+ // collector would miss exactly the process that should not still be there.+ c, _ := g.Exec(ctx, "pgrep -c 'beacon-otelcol|beacon-fake-collector' || true", opts)+ art.Meta["rollback_collector_count"] = strings.TrimSpace(c.Stdout)+ if leftovers, _ := g.Exec(ctx, "pgrep -a 'beacon-otelcol|beacon-fake-collector' || true", opts); leftovers.Stdout != "" {+ art.Meta["rollback_processes"] = oneLine(leftovers.Stdout)+ }++ s, _ := g.Exec(ctx, "beacon endpoint status "+scope+" --json 2>&1", opts)+ art.Meta["rollback_status"] = oneLine(s.Stdout)++ logf("failed-reinstall rollback: install rc=%d, collectors now=%s",+ r.ExitCode, art.Meta["rollback_collector_count"])+}+// probeUnprivilegedUninstall checks that a system removal without privileges reports failure.//// Run before the real uninstall, and deliberately as the unprivileged agent user. What makes thisdiff --git a/beacon-sandbox/scenario/scenario.go b/beacon-sandbox/scenario/scenario.goindex 2908f00e..818f33b4 100644--- a/beacon-sandbox/scenario/scenario.go+++ b/beacon-sandbox/scenario/scenario.go@@ -84,6 +84,20 @@ type Install struct {// the endpoint is a version behind until reboot. Nothing short of a real service manager and a// real second install exercises that.VerifyRestartOnReinstall bool `yaml:"verify_restart_on_reinstall,omitempty"`+ // VerifyRollbackOnFailedReinstall makes a reinstall fail on purpose and checks what the machine+ // is left with.+ //+ // Rollback is the least exercised path in install and the easiest to get wrong, because it runs+ // only when something else has already gone wrong. Two defects lived here: it disabled a+ // collector that was healthy before the attempt, and -- since the supervised backend's unit+ // file is its pidfile, which rollback restores -- it stopped a pid that was already dead while+ // the collector the failed install started kept running, with a second one started beside it.+ //+ // Neither is visible from a successful install, and both leave a machine that looks installed.+ // The failure is induced by pointing the reinstall at a program that starts and never listens,+ // so the service loads and readiness is what fails, which is the ordering that reaches rollback+ // with a service to undo.+ VerifyRollbackOnFailedReinstall bool `yaml:"verify_rollback_on_failed_reinstall,omitempty"`// VerifyUnprivilegedUninstallFails checks that a system uninstall run without privileges// reports failure instead of pretending it removed anything.//@@ -428,6 +442,12 @@ func (s Scenario) VerifiesRestartOnReinstall() bool {return s.Install != nil && s.Install.VerifyRestartOnReinstall}+// VerifiesRollbackOnFailedReinstall reports whether a deliberately failed reinstall must leave the+// machine with exactly the endpoint it had beforehand.+func (s Scenario) VerifiesRollbackOnFailedReinstall() bool {+ return s.Install != nil && s.Install.VerifyRollbackOnFailedReinstall+}+// VerifiesUnprivilegedUninstallFails reports whether an unprivileged removal must be refused.func (s Scenario) VerifiesUnprivilegedUninstallFails() bool {return s.Install != nil && s.Install.VerifyUnprivilegedUninstallFailsdiff --git a/beacon-sandbox/scenarios/i01-install-supervised.yaml b/beacon-sandbox/scenarios/i01-install-supervised.yamlindex ca4b7660..75d76e1d 100644--- a/beacon-sandbox/scenarios/i01-install-supervised.yaml+++ b/beacon-sandbox/scenarios/i01-install-supervised.yaml@@ -21,6 +21,12 @@ install:# Asserted rather than assumed: if systemd detection ever started returning true in a container,# this scenario would silently stop covering the fallback backend it exists to cover.expect_service_kind: none+ # This backend is where rollback is hardest to get right, because its unit file is its pidfile --+ # a file rollback restores. Restoring it puts back a pid that has already been terminated, so a+ # rollback that stops the service afterwards stops nothing and leaves the failed install's+ # collector running with another started beside it. Cheap to check here: no root, no systemd.+ verify_restart_on_reinstall: true+ verify_rollback_on_failed_reinstall: trueverify_uninstall: trueprompt: >-