Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

test(sandbox): prove rollback by making an install fail, instead of reasoning about it

shukieshahf99ab1ef9774
diff --git a/beacon-sandbox/check/lifecycle.go b/beacon-sandbox/check/lifecycle.go
index b5846962..67850f9c 100644
--- a/beacon-sandbox/check/lifecycle.go
+++ b/beacon-sandbox/check/lifecycle.go
@@ -15,6 +15,12 @@ type Lifecycle struct {
// UnprivilegedRefused is "true"/"false" when an unprivileged removal was attempted.
UnprivilegedRefused string
UnprivilegedOutput string
+
+ // RollbackInstallRC, RollbackCollectorCount and RollbackStatus describe a reinstall that was
+ // made to fail on purpose. Empty when the scenario did not ask.
+ RollbackInstallRC string
+ RollbackCollectorCount string
+ RollbackStatus string
}
// Reinstall judges whether installing over a running endpoint replaced the collector.
@@ -59,6 +65,63 @@ func Reinstall(v *Verdict, l Lifecycle) {
})
}
+// FailedReinstallRollback judges what a deliberately failed reinstall left behind.
+//
+// The contract is that a failed install returns the machine to the endpoint it had: one collector,
+// running, on the configuration from before the attempt. Two ways of breaking it have shipped.
+//
+// Too few collectors means rollback took a healthy endpoint down -- and on systemd it also disables
+// the unit, so the machine stays down across a reboot. Too many means rollback stopped the wrong
+// process: the collector the failed install started keeps running on the new config, holding the
+// ports, while another is started beside it. Asking whether a collector is running cannot tell any
+// of these apart, because the answer is yes in every one.
+func FailedReinstallRollback(v *Verdict, l Lifecycle) {
+ if l.RollbackInstallRC == "" {
+ return // the scenario did not ask
+ }
+
+ // A reinstall pointed at a program that never listens has to fail. If it reported success,
+ // readiness is not being checked and the rest of this proves nothing.
+ if l.RollbackInstallRC == "0" {
+ v.Add(Finding{
+ Check: "rollback.install_failed",
+ Severity: SevFail,
+ Summary: "a reinstall pointed at a collector that never listens reported success",
+ Why: "Install is supposed to wait for the collector to become ready and roll back when " +
+ "it does not. Reporting success here means an endpoint that cannot collect is " +
+ "indistinguishable from one that can.",
+ })
+ return
+ }
+
+ switch l.RollbackCollectorCount {
+ case "1":
+ v.Add(Finding{
+ Check: "rollback.left_one_collector",
+ Severity: SevInfo,
+ Summary: "a failed reinstall left exactly one collector running",
+ })
+ case "", "0":
+ v.Add(Finding{
+ Check: "rollback.left_one_collector",
+ Severity: SevFail,
+ Summary: "a failed reinstall left no collector running, so it took a working endpoint down with it",
+ Why: "Rollback may undo what the failed install did; it may not remove what was working " +
+ "beforehand. On systemd it also disables the unit, so the machine stays down across " +
+ "a reboot -- an upgrade that fails and ends collection is worse than one that fails.",
+ })
+ default:
+ v.Add(Finding{
+ Check: "rollback.left_one_collector",
+ Severity: SevFail,
+ Summary: "a failed reinstall left " + l.RollbackCollectorCount + " collectors running",
+ Why: "Rollback stopped the wrong process. The collector the failed install started is " +
+ "still running on the configuration that failed, holding the OTLP ports, while " +
+ "another was started beside it -- and the pidfile tracks only one of them.",
+ })
+ }
+}
+
// UnprivilegedUninstall judges whether a system removal without privileges was refused.
//
// verify_uninstall cannot catch this, because it removes the endpoint elevated, which is the path
diff --git a/beacon-sandbox/cmd/beacon-sandbox/main.go b/beacon-sandbox/cmd/beacon-sandbox/main.go
index 6aa96b0d..b782f2c2 100644
--- a/beacon-sandbox/cmd/beacon-sandbox/main.go
+++ b/beacon-sandbox/cmd/beacon-sandbox/main.go
@@ -418,9 +418,14 @@ func judge(sc scenario.Scenario, art runner.Artifacts, creds credentials.Resolve
ReinstallErr: art.Meta["reinstall_error"],
UnprivilegedRefused: art.Meta["unprivileged_uninstall_refused"],
UnprivilegedOutput: art.Meta["unprivileged_uninstall_output"],
+
+ RollbackInstallRC: art.Meta["rollback_install_rc"],
+ RollbackCollectorCount: art.Meta["rollback_collector_count"],
+ RollbackStatus: art.Meta["rollback_status"],
}
check.Reinstall(&v, lifecycle)
check.UnprivilegedUninstall(&v, lifecycle)
+ check.FailedReinstallRollback(&v, lifecycle)
v.Resolve()
return v, log
}
diff --git a/beacon-sandbox/runner/runner.go b/beacon-sandbox/runner/runner.go
index bd0ce29b..7f116227 100644
--- a/beacon-sandbox/runner/runner.go
+++ b/beacon-sandbox/runner/runner.go
@@ -474,6 +474,12 @@ func Run(ctx context.Context, p sandbox.Provider, sc scenario.Scenario, opts Opt
probeReinstall(ctx, g, sc, privileged, agent, &art, logf)
}
+ // After the reinstall probe, so a successful restart is established before a failing one is
+ // induced, and before uninstall takes the endpoint away entirely.
+ if sc.VerifiesRollbackOnFailedReinstall() {
+ probeFailedReinstallRollback(ctx, g, sc, privileged, agent, &art, logf)
+ }
+
// Unprivileged first, while there is still something installed for it to fail to remove.
if sc.VerifiesUnprivilegedUninstallFails() {
probeUnprivilegedUninstall(ctx, g, agent, &art, logf)
@@ -1015,6 +1021,64 @@ func probeReinstall(ctx context.Context, g guest, sc scenario.Scenario,
logf("reinstall: collector pid %s -> %s (restarted=%s)", pidBefore, pidAfter, art.Meta["reinstall_restarted"])
}
+// probeFailedReinstallRollback makes a reinstall fail and records what survived it.
+//
+// The failure is induced with a collector that starts and then exits without listening, so the
+// service loads successfully and readiness is what fails. That ordering matters: it is the only way
+// to reach rollback with a service it has to decide about, and every defect found here lived in
+// that decision.
+//
+// Three things are recorded, and the count is the one that matters. A rollback that stops the wrong
+// process leaves the collector the failed install started still running, and then starts another
+// beside it -- so the machine ends with two collectors, the newer config live, and ports held by a
+// process nothing is tracking. "Is a collector running?" is true in that state, which is why it is
+// not the question asked.
+func probeFailedReinstallRollback(ctx context.Context, g guest, sc scenario.Scenario,
+ privileged, agent sandbox.ExecOpts, art *Artifacts, logf func(string, ...any)) {
+
+ scope := "--user"
+ opts := agent
+ if sc.Install.Mode == "system" {
+ scope = "--system"
+ opts = privileged
+ }
+
+ // A stand-in collector that ignores its arguments and stays alive without listening.
+ //
+ // It has to keep running, which rules out the obvious `/bin/sleep`: given `--config <path>` that
+ // exits immediately, and a process that exits on its own cannot be orphaned -- so the probe
+ // would report a clean rollback whether or not rollback stopped anything. Staying alive is what
+ // makes the collector count able to distinguish the two.
+ fake := "/tmp/beacon-fake-collector"
+ if _, err := g.Exec(ctx, "printf '#!/bin/sh\\nexec sleep 600\\n' > "+fake+" && chmod 0755 "+fake, opts); err != nil {
+ art.Meta["rollback_error"] = "could not stage a stand-in collector: " + err.Error()
+ return
+ }
+
+ args := "beacon endpoint install " + scope + " --harness claude --collector " + fake
+ if svc := sc.Install.Service; svc != "" {
+ args += " --service=" + svc
+ }
+ r, _ := g.Exec(ctx, args+" 2>&1", opts)
+ art.Meta["rollback_install_rc"] = fmt.Sprintf("%d", r.ExitCode)
+ art.Meta["rollback_install_output"] = oneLine(r.Stdout + r.Stderr)
+
+ // Counted after the install has returned, so rollback has finished running. Both names are
+ // counted: the stand-in is what a mishandled rollback leaves behind, and counting only the real
+ // collector would miss exactly the process that should not still be there.
+ c, _ := g.Exec(ctx, "pgrep -c 'beacon-otelcol|beacon-fake-collector' || true", opts)
+ art.Meta["rollback_collector_count"] = strings.TrimSpace(c.Stdout)
+ if leftovers, _ := g.Exec(ctx, "pgrep -a 'beacon-otelcol|beacon-fake-collector' || true", opts); leftovers.Stdout != "" {
+ art.Meta["rollback_processes"] = oneLine(leftovers.Stdout)
+ }
+
+ s, _ := g.Exec(ctx, "beacon endpoint status "+scope+" --json 2>&1", opts)
+ art.Meta["rollback_status"] = oneLine(s.Stdout)
+
+ logf("failed-reinstall rollback: install rc=%d, collectors now=%s",
+ r.ExitCode, art.Meta["rollback_collector_count"])
+}
+
// probeUnprivilegedUninstall checks that a system removal without privileges reports failure.
//
// Run before the real uninstall, and deliberately as the unprivileged agent user. What makes this
diff --git a/beacon-sandbox/scenario/scenario.go b/beacon-sandbox/scenario/scenario.go
index 2908f00e..818f33b4 100644
--- a/beacon-sandbox/scenario/scenario.go
+++ b/beacon-sandbox/scenario/scenario.go
@@ -84,6 +84,20 @@ type Install struct {
// the endpoint is a version behind until reboot. Nothing short of a real service manager and a
// real second install exercises that.
VerifyRestartOnReinstall bool `yaml:"verify_restart_on_reinstall,omitempty"`
+ // VerifyRollbackOnFailedReinstall makes a reinstall fail on purpose and checks what the machine
+ // is left with.
+ //
+ // Rollback is the least exercised path in install and the easiest to get wrong, because it runs
+ // only when something else has already gone wrong. Two defects lived here: it disabled a
+ // collector that was healthy before the attempt, and -- since the supervised backend's unit
+ // file is its pidfile, which rollback restores -- it stopped a pid that was already dead while
+ // the collector the failed install started kept running, with a second one started beside it.
+ //
+ // Neither is visible from a successful install, and both leave a machine that looks installed.
+ // The failure is induced by pointing the reinstall at a program that starts and never listens,
+ // so the service loads and readiness is what fails, which is the ordering that reaches rollback
+ // with a service to undo.
+ VerifyRollbackOnFailedReinstall bool `yaml:"verify_rollback_on_failed_reinstall,omitempty"`
// VerifyUnprivilegedUninstallFails checks that a system uninstall run without privileges
// reports failure instead of pretending it removed anything.
//
@@ -428,6 +442,12 @@ func (s Scenario) VerifiesRestartOnReinstall() bool {
return s.Install != nil && s.Install.VerifyRestartOnReinstall
}
+// VerifiesRollbackOnFailedReinstall reports whether a deliberately failed reinstall must leave the
+// machine with exactly the endpoint it had beforehand.
+func (s Scenario) VerifiesRollbackOnFailedReinstall() bool {
+ return s.Install != nil && s.Install.VerifyRollbackOnFailedReinstall
+}
+
// VerifiesUnprivilegedUninstallFails reports whether an unprivileged removal must be refused.
func (s Scenario) VerifiesUnprivilegedUninstallFails() bool {
return s.Install != nil && s.Install.VerifyUnprivilegedUninstallFails
diff --git a/beacon-sandbox/scenarios/i01-install-supervised.yaml b/beacon-sandbox/scenarios/i01-install-supervised.yaml
index ca4b7660..75d76e1d 100644
--- a/beacon-sandbox/scenarios/i01-install-supervised.yaml
+++ b/beacon-sandbox/scenarios/i01-install-supervised.yaml
@@ -21,6 +21,12 @@ install:
# Asserted rather than assumed: if systemd detection ever started returning true in a container,
# this scenario would silently stop covering the fallback backend it exists to cover.
expect_service_kind: none
+ # This backend is where rollback is hardest to get right, because its unit file is its pidfile --
+ # a file rollback restores. Restoring it puts back a pid that has already been terminated, so a
+ # rollback that stops the service afterwards stops nothing and leaves the failed install's
+ # collector running with another started beside it. Cheap to check here: no root, no systemd.
+ verify_restart_on_reinstall: true
+ verify_rollback_on_failed_reinstall: true
verify_uninstall: true
prompt: >-