The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix: preserve machine evidence during uninstall verification (#989)
Divyam Talwar1714f62cdb4d
diff --git a/beacon-sandbox/check/uninstall.go b/beacon-sandbox/check/uninstall.goindex 6e57e38f..bedf3729 100644--- a/beacon-sandbox/check/uninstall.go+++ b/beacon-sandbox/check/uninstall.go@@ -1,6 +1,7 @@package checkimport (+ "encoding/json""fmt""strings")@@ -16,6 +17,8 @@ type Removal struct {Ran string// ExitCode is `endpoint uninstall`'s status, as text.ExitCode string+ // ExecError is a transport/operation failure, even when no exit status was returned.+ ExecError string// Output is what it printed.Output string// ServiceKind and ServiceLabel identify what was registered at install time.@@ -55,11 +58,15 @@ func Uninstall(v *Verdict, r Removal) {return}- if r.ExitCode != "0" {+ if r.ExitCode != "0" || r.ExecError != "" {+ detail := r.Output+ if r.ExecError != "" {+ detail = r.ExecError + ": " + detail+ }v.Add(Finding{Check: "uninstall.command_succeeded",Severity: SevFail,- Summary: fmt.Sprintf("`endpoint uninstall` exited %s: %s", r.ExitCode, oneLineOf(r.Output)),+ Summary: fmt.Sprintf("`endpoint uninstall` did not complete (exit=%s): %s", r.ExitCode, oneLineOf(detail)),Why: "An uninstall that cannot complete leaves the endpoint in a state no one asked for: " +"binaries possibly gone, service possibly still registered.",})@@ -88,7 +95,7 @@ func Uninstall(v *Verdict, r Removal) {v.Add(Finding{Check: "uninstall.service_removed",Severity: SevWarn,- Summary: fmt.Sprintf("the service manager was not asked whether %q is gone (kind=%q)",+ Summary: fmt.Sprintf("the service manager did not establish whether %q is gone (kind=%q)",r.ServiceLabel, r.ServiceKind),Why: "Removal of the service registration is unverified for this run. It is not evidence " +"that anything was left behind, and it is not evidence that nothing was.",@@ -128,13 +135,28 @@ func Uninstall(v *Verdict, r Removal) {// Beacon's own account, checked against the machine's. A status still describing a running// collector after a successful removal means the two disagree, and the disagreement is worth a// finding of its own even when the service really is gone.- if strings.Contains(r.Status, `"running":true`) {+ var status struct {+ Service struct {+ Running *bool `json:"running"`+ } `json:"service"`+ }+ if json.Unmarshal([]byte(r.Status), &status) != nil || status.Service.Running == nil {+ v.Add(Finding{+ Check: "uninstall.status_agrees",+ Severity: SevWarn,+ Summary: "`endpoint status --json` after uninstall did not report service.running",+ Why: "A missing or unreadable status cannot establish that the collector stopped.",+ })+ return+ }+ if *status.Service.Running {v.Add(Finding{Check: "uninstall.status_agrees",Severity: SevFail,Summary: "`endpoint status` still reports a running collector after uninstall",Why: "Beacon's view of itself and the machine's have to agree, or one of them is lying to " +"whoever reads it next.",+ Evidence: []string{"meta.json:uninstall_status"},})}}diff --git a/beacon-sandbox/check/uninstall_status_test.go b/beacon-sandbox/check/uninstall_status_test.gonew file mode 100644index 00000000..76fd4d9f--- /dev/null+++ b/beacon-sandbox/check/uninstall_status_test.go@@ -0,0 +1,33 @@+package check++import "testing"++func TestUninstallStatusRequiresStructuralEvidence(t *testing.T) {+ for _, tc := range []struct {+ name, status string+ want Severity+ }{+ {"pretty running", "{\n \"service\": {\n \"running\": true\n }\n}", SevFail},+ {"stopped despite other running fields", `{"collector":{"running":true},"service":{"running":false},"script_forwarders":[{"running":true}]}`, ""},+ {"missing status", "", SevWarn},+ {"missing service", `{}`, SevWarn},+ {"missing running", `{"service":{}}`, SevWarn},+ {"null service", `{"service":null}`, SevWarn},+ {"null running", `{"service":{"running":null}}`, SevWarn},+ {"wrong type", `{"service":{"running":"false"}}`, SevWarn},+ {"malformed", `{"service":`, SevWarn},+ {"clipped legacy", `{"version":"1.3.91","service":{"running":...`, SevWarn},+ } {+ t.Run(tc.name, func(t *testing.T) {+ r := clean()+ r.Status = tc.status+ findings := findingsFor(r)+ if tc.want != "" && !has(findings, "uninstall.status_agrees", tc.want) {+ t.Fatalf("status observation needs %s finding: %#v", tc.want, findings)+ }+ if tc.want == "" && (has(findings, "uninstall.status_agrees", SevFail) || has(findings, "uninstall.status_agrees", SevWarn)) {+ t.Fatalf("explicitly stopped service was not verified: %#v", findings)+ }+ })+ }+}diff --git a/beacon-sandbox/cmd/beacon-sandbox/main.go b/beacon-sandbox/cmd/beacon-sandbox/main.goindex 52c20fa0..d397c920 100644--- a/beacon-sandbox/cmd/beacon-sandbox/main.go+++ b/beacon-sandbox/cmd/beacon-sandbox/main.go@@ -404,6 +404,7 @@ func judge(sc scenario.Scenario, art runner.Artifacts, creds credentials.Resolvecheck.Uninstall(&v, check.Removal{Ran: art.Meta["uninstall_ran"],ExitCode: art.Meta["uninstall_rc"],+ ExecError: art.Meta["uninstall_exec_error"],Output: art.Meta["uninstall_output"],ServiceKind: art.Meta["service_kind"],ServiceLabel: art.Meta["service_label"],diff --git a/beacon-sandbox/runner/runner.go b/beacon-sandbox/runner/runner.goindex 00d09ac4..f862125e 100644--- a/beacon-sandbox/runner/runner.go+++ b/beacon-sandbox/runner/runner.go@@ -1178,10 +1178,12 @@ func probeUninstall(ctx context.Context, g guest, sc scenario.Scenario,if label != "" && kind != "" && kind != "none" {query := sh.ServiceQuery(kind, label)if query != "" {- q, _ := g.Exec(ctx, query, opts)+ q, err := g.Exec(ctx, query, opts)art.Meta["uninstall_service_query"] = oneLine(q.Stdout + q.Stderr)art.Meta["uninstall_service_query_rc"] = fmt.Sprintf("%d", q.ExitCode)- art.Meta["uninstall_service_gone"] = fmt.Sprintf("%v", sh.ServiceAbsent(kind, q.ExitCode, q.Stdout+q.Stderr))+ if err == nil {+ art.Meta["uninstall_service_gone"] = fmt.Sprintf("%v", sh.ServiceAbsent(kind, q.ExitCode, q.Stdout+q.Stderr))+ }logf("service %s after uninstall: gone=%s", label, art.Meta["uninstall_service_gone"])}} else if kind == "none" {@@ -1193,13 +1195,15 @@ func probeUninstall(ctx context.Context, g guest, sc scenario.Scenario,// Retention, asked of the filesystem. An absent path is reported as unknown rather than as// retained: a check that could not look must not conclude the file survived.if logPath != "" {- e, _ := g.Exec(ctx, sh.PathExists(logPath), opts)- art.Meta["uninstall_log_retained"] = fmt.Sprintf("%v", e.ExitCode == 0)+ if e, err := g.Exec(ctx, sh.PathExists(logPath), opts); err == nil {+ art.Meta["uninstall_log_retained"] = fmt.Sprintf("%v", e.ExitCode == 0)+ }}// Beacon's own account of itself, recorded alongside the machine's. When the two disagree, that// disagreement is the finding.+ // Saved whole: the checker decodes it, and clipping hides service.running.if s, err := g.Exec(ctx, "beacon endpoint status "+scope+" --json 2>&1", opts); err == nil {- art.Meta["uninstall_status"] = oneLine(s.Stdout)+ art.Meta["uninstall_status"] = s.Stdout}}diff --git a/beacon-sandbox/runner/uninstall_test.go b/beacon-sandbox/runner/uninstall_test.gonew file mode 100644index 00000000..88342959--- /dev/null+++ b/beacon-sandbox/runner/uninstall_test.go@@ -0,0 +1,55 @@+package runner++import (+ "context"+ "strings"+ "testing"++ "github.com/asymptote-labs/agent-beacon/beacon-sandbox/check"+ "github.com/asymptote-labs/agent-beacon/beacon-sandbox/image"+ "github.com/asymptote-labs/agent-beacon/beacon-sandbox/sandbox"+ "github.com/asymptote-labs/agent-beacon/beacon-sandbox/scenario"+)++// Exec answers `endpoint status` with the supplied result and every other script with success.+type uninstallGuest struct{ status sandbox.Result }++func (g uninstallGuest) Exec(_ context.Context, script string, _ sandbox.ExecOpts) (sandbox.Result, error) {+ if strings.Contains(script, "endpoint status") {+ return g.status, nil+ }+ return sandbox.Result{}, nil+}++func (uninstallGuest) Get(context.Context, string, string) error { return nil }+func (uninstallGuest) Describe() string { return "synthetic uninstall observations" }++func TestUninstallProbePreservesStatusForSavedJudgment(t *testing.T) {+ // The long diagnostic places service.running well past the runner's display limit, as the+ // real status output does.+ status := `{"version":"1.3.91","config_path":"/etc/beacon/endpoint/config.json","log_path":"/var/log/beacon-agent/runtime.jsonl","runtime_log":{"warning":` ++ `"` + strings.Repeat("synthetic diagnostic; ", 250) + `"},"service":{"running":true,"kind":"systemd"}}`+ for _, running := range []bool{true, false} {+ t.Run(map[bool]string{true: "running", false: "stopped"}[running], func(t *testing.T) {+ stdout := status+ if !running {+ stdout = strings.Replace(stdout, `"running":true`, `"running":false`, 1)+ }+ art := Artifacts{Meta: map[string]string{"service_kind": "systemd", "service_label": "beacon.service", "install_log_path": "/synthetic/runtime.jsonl"}}+ probeUninstall(context.Background(), uninstallGuest{status: sandbox.Result{Stdout: stdout}}, scenario.Scenario{Install: &scenario.Install{Mode: "user"}}, sandbox.ExecOpts{}, sandbox.ExecOpts{}, image.Layout{}, posixShell{}, &art, func(string, ...any) {})+ if art.Meta["uninstall_status"] != stdout {+ t.Errorf("status changed: got %d bytes, want %d", len(art.Meta["uninstall_status"]), len(stdout))+ }+ var v check.Verdict+ check.Uninstall(&v, check.Removal{Ran: art.Meta["uninstall_ran"], ExitCode: art.Meta["uninstall_rc"], ServiceGone: art.Meta["uninstall_service_gone"], LogRetained: art.Meta["uninstall_log_retained"], Status: art.Meta["uninstall_status"]})+ v.Resolve()+ want := check.Pass+ if running {+ want = check.Fail+ }+ if v.Outcome != want {+ t.Errorf("verdict=%s, want %s:\n%s", v.Outcome, want, v.Report())+ }+ })+ }+}