Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix: preserve machine evidence during uninstall verification (#989)

Divyam Talwar1714f62cdb4d
diff --git a/beacon-sandbox/check/uninstall.go b/beacon-sandbox/check/uninstall.go
index 6e57e38f..bedf3729 100644
--- a/beacon-sandbox/check/uninstall.go
+++ b/beacon-sandbox/check/uninstall.go
@@ -1,6 +1,7 @@
package check
import (
+ "encoding/json"
"fmt"
"strings"
)
@@ -16,6 +17,8 @@ type Removal struct {
Ran string
// ExitCode is `endpoint uninstall`'s status, as text.
ExitCode string
+ // ExecError is a transport/operation failure, even when no exit status was returned.
+ ExecError string
// Output is what it printed.
Output string
// ServiceKind and ServiceLabel identify what was registered at install time.
@@ -55,11 +58,15 @@ func Uninstall(v *Verdict, r Removal) {
return
}
- if r.ExitCode != "0" {
+ if r.ExitCode != "0" || r.ExecError != "" {
+ detail := r.Output
+ if r.ExecError != "" {
+ detail = r.ExecError + ": " + detail
+ }
v.Add(Finding{
Check: "uninstall.command_succeeded",
Severity: SevFail,
- Summary: fmt.Sprintf("`endpoint uninstall` exited %s: %s", r.ExitCode, oneLineOf(r.Output)),
+ Summary: fmt.Sprintf("`endpoint uninstall` did not complete (exit=%s): %s", r.ExitCode, oneLineOf(detail)),
Why: "An uninstall that cannot complete leaves the endpoint in a state no one asked for: " +
"binaries possibly gone, service possibly still registered.",
})
@@ -88,7 +95,7 @@ func Uninstall(v *Verdict, r Removal) {
v.Add(Finding{
Check: "uninstall.service_removed",
Severity: SevWarn,
- Summary: fmt.Sprintf("the service manager was not asked whether %q is gone (kind=%q)",
+ Summary: fmt.Sprintf("the service manager did not establish whether %q is gone (kind=%q)",
r.ServiceLabel, r.ServiceKind),
Why: "Removal of the service registration is unverified for this run. It is not evidence " +
"that anything was left behind, and it is not evidence that nothing was.",
@@ -128,13 +135,28 @@ func Uninstall(v *Verdict, r Removal) {
// Beacon's own account, checked against the machine's. A status still describing a running
// collector after a successful removal means the two disagree, and the disagreement is worth a
// finding of its own even when the service really is gone.
- if strings.Contains(r.Status, `"running":true`) {
+ var status struct {
+ Service struct {
+ Running *bool `json:"running"`
+ } `json:"service"`
+ }
+ if json.Unmarshal([]byte(r.Status), &status) != nil || status.Service.Running == nil {
+ v.Add(Finding{
+ Check: "uninstall.status_agrees",
+ Severity: SevWarn,
+ Summary: "`endpoint status --json` after uninstall did not report service.running",
+ Why: "A missing or unreadable status cannot establish that the collector stopped.",
+ })
+ return
+ }
+ if *status.Service.Running {
v.Add(Finding{
Check: "uninstall.status_agrees",
Severity: SevFail,
Summary: "`endpoint status` still reports a running collector after uninstall",
Why: "Beacon's view of itself and the machine's have to agree, or one of them is lying to " +
"whoever reads it next.",
+ Evidence: []string{"meta.json:uninstall_status"},
})
}
}
diff --git a/beacon-sandbox/check/uninstall_status_test.go b/beacon-sandbox/check/uninstall_status_test.go
new file mode 100644
index 00000000..76fd4d9f
--- /dev/null
+++ b/beacon-sandbox/check/uninstall_status_test.go
@@ -0,0 +1,33 @@
+package check
+
+import "testing"
+
+func TestUninstallStatusRequiresStructuralEvidence(t *testing.T) {
+ for _, tc := range []struct {
+ name, status string
+ want Severity
+ }{
+ {"pretty running", "{\n \"service\": {\n \"running\": true\n }\n}", SevFail},
+ {"stopped despite other running fields", `{"collector":{"running":true},"service":{"running":false},"script_forwarders":[{"running":true}]}`, ""},
+ {"missing status", "", SevWarn},
+ {"missing service", `{}`, SevWarn},
+ {"missing running", `{"service":{}}`, SevWarn},
+ {"null service", `{"service":null}`, SevWarn},
+ {"null running", `{"service":{"running":null}}`, SevWarn},
+ {"wrong type", `{"service":{"running":"false"}}`, SevWarn},
+ {"malformed", `{"service":`, SevWarn},
+ {"clipped legacy", `{"version":"1.3.91","service":{"running":...`, SevWarn},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ r := clean()
+ r.Status = tc.status
+ findings := findingsFor(r)
+ if tc.want != "" && !has(findings, "uninstall.status_agrees", tc.want) {
+ t.Fatalf("status observation needs %s finding: %#v", tc.want, findings)
+ }
+ if tc.want == "" && (has(findings, "uninstall.status_agrees", SevFail) || has(findings, "uninstall.status_agrees", SevWarn)) {
+ t.Fatalf("explicitly stopped service was not verified: %#v", findings)
+ }
+ })
+ }
+}
diff --git a/beacon-sandbox/cmd/beacon-sandbox/main.go b/beacon-sandbox/cmd/beacon-sandbox/main.go
index 52c20fa0..d397c920 100644
--- a/beacon-sandbox/cmd/beacon-sandbox/main.go
+++ b/beacon-sandbox/cmd/beacon-sandbox/main.go
@@ -404,6 +404,7 @@ func judge(sc scenario.Scenario, art runner.Artifacts, creds credentials.Resolve
check.Uninstall(&v, check.Removal{
Ran: art.Meta["uninstall_ran"],
ExitCode: art.Meta["uninstall_rc"],
+ ExecError: art.Meta["uninstall_exec_error"],
Output: art.Meta["uninstall_output"],
ServiceKind: art.Meta["service_kind"],
ServiceLabel: art.Meta["service_label"],
diff --git a/beacon-sandbox/runner/runner.go b/beacon-sandbox/runner/runner.go
index 00d09ac4..f862125e 100644
--- a/beacon-sandbox/runner/runner.go
+++ b/beacon-sandbox/runner/runner.go
@@ -1178,10 +1178,12 @@ func probeUninstall(ctx context.Context, g guest, sc scenario.Scenario,
if label != "" && kind != "" && kind != "none" {
query := sh.ServiceQuery(kind, label)
if query != "" {
- q, _ := g.Exec(ctx, query, opts)
+ q, err := g.Exec(ctx, query, opts)
art.Meta["uninstall_service_query"] = oneLine(q.Stdout + q.Stderr)
art.Meta["uninstall_service_query_rc"] = fmt.Sprintf("%d", q.ExitCode)
- art.Meta["uninstall_service_gone"] = fmt.Sprintf("%v", sh.ServiceAbsent(kind, q.ExitCode, q.Stdout+q.Stderr))
+ if err == nil {
+ art.Meta["uninstall_service_gone"] = fmt.Sprintf("%v", sh.ServiceAbsent(kind, q.ExitCode, q.Stdout+q.Stderr))
+ }
logf("service %s after uninstall: gone=%s", label, art.Meta["uninstall_service_gone"])
}
} else if kind == "none" {
@@ -1193,13 +1195,15 @@ func probeUninstall(ctx context.Context, g guest, sc scenario.Scenario,
// Retention, asked of the filesystem. An absent path is reported as unknown rather than as
// retained: a check that could not look must not conclude the file survived.
if logPath != "" {
- e, _ := g.Exec(ctx, sh.PathExists(logPath), opts)
- art.Meta["uninstall_log_retained"] = fmt.Sprintf("%v", e.ExitCode == 0)
+ if e, err := g.Exec(ctx, sh.PathExists(logPath), opts); err == nil {
+ art.Meta["uninstall_log_retained"] = fmt.Sprintf("%v", e.ExitCode == 0)
+ }
}
// Beacon's own account of itself, recorded alongside the machine's. When the two disagree, that
// disagreement is the finding.
+ // Saved whole: the checker decodes it, and clipping hides service.running.
if s, err := g.Exec(ctx, "beacon endpoint status "+scope+" --json 2>&1", opts); err == nil {
- art.Meta["uninstall_status"] = oneLine(s.Stdout)
+ art.Meta["uninstall_status"] = s.Stdout
}
}
diff --git a/beacon-sandbox/runner/uninstall_test.go b/beacon-sandbox/runner/uninstall_test.go
new file mode 100644
index 00000000..88342959
--- /dev/null
+++ b/beacon-sandbox/runner/uninstall_test.go
@@ -0,0 +1,55 @@
+package runner
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "github.com/asymptote-labs/agent-beacon/beacon-sandbox/check"
+ "github.com/asymptote-labs/agent-beacon/beacon-sandbox/image"
+ "github.com/asymptote-labs/agent-beacon/beacon-sandbox/sandbox"
+ "github.com/asymptote-labs/agent-beacon/beacon-sandbox/scenario"
+)
+
+// Exec answers `endpoint status` with the supplied result and every other script with success.
+type uninstallGuest struct{ status sandbox.Result }
+
+func (g uninstallGuest) Exec(_ context.Context, script string, _ sandbox.ExecOpts) (sandbox.Result, error) {
+ if strings.Contains(script, "endpoint status") {
+ return g.status, nil
+ }
+ return sandbox.Result{}, nil
+}
+
+func (uninstallGuest) Get(context.Context, string, string) error { return nil }
+func (uninstallGuest) Describe() string { return "synthetic uninstall observations" }
+
+func TestUninstallProbePreservesStatusForSavedJudgment(t *testing.T) {
+ // The long diagnostic places service.running well past the runner's display limit, as the
+ // real status output does.
+ status := `{"version":"1.3.91","config_path":"/etc/beacon/endpoint/config.json","log_path":"/var/log/beacon-agent/runtime.jsonl","runtime_log":{"warning":` +
+ `"` + strings.Repeat("synthetic diagnostic; ", 250) + `"},"service":{"running":true,"kind":"systemd"}}`
+ for _, running := range []bool{true, false} {
+ t.Run(map[bool]string{true: "running", false: "stopped"}[running], func(t *testing.T) {
+ stdout := status
+ if !running {
+ stdout = strings.Replace(stdout, `"running":true`, `"running":false`, 1)
+ }
+ art := Artifacts{Meta: map[string]string{"service_kind": "systemd", "service_label": "beacon.service", "install_log_path": "/synthetic/runtime.jsonl"}}
+ probeUninstall(context.Background(), uninstallGuest{status: sandbox.Result{Stdout: stdout}}, scenario.Scenario{Install: &scenario.Install{Mode: "user"}}, sandbox.ExecOpts{}, sandbox.ExecOpts{}, image.Layout{}, posixShell{}, &art, func(string, ...any) {})
+ if art.Meta["uninstall_status"] != stdout {
+ t.Errorf("status changed: got %d bytes, want %d", len(art.Meta["uninstall_status"]), len(stdout))
+ }
+ var v check.Verdict
+ check.Uninstall(&v, check.Removal{Ran: art.Meta["uninstall_ran"], ExitCode: art.Meta["uninstall_rc"], ServiceGone: art.Meta["uninstall_service_gone"], LogRetained: art.Meta["uninstall_log_retained"], Status: art.Meta["uninstall_status"]})
+ v.Resolve()
+ want := check.Pass
+ if running {
+ want = check.Fail
+ }
+ if v.Outcome != want {
+ t.Errorf("verdict=%s, want %s:\n%s", v.Outcome, want, v.Report())
+ }
+ })
+ }
+}