Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(sandbox): correct four issues Cursor Bugbot found on #316

shukieshahb25035ab4908
diff --git a/.github/workflows/windows-sandbox.yml b/.github/workflows/windows-sandbox.yml
index 753a9253..4b67ca47 100644
--- a/.github/workflows/windows-sandbox.yml
+++ b/.github/workflows/windows-sandbox.yml
@@ -119,11 +119,18 @@ jobs:
# The native installer, which is the only Windows install path that self-updates. Pinned so
# a verdict is attributable to a specific agent build.
- name: Install Claude Code
+ env:
+ # Through the environment for the same reason as SCENARIO below. The whitelist that
+ # follows guards what the value is *used for*; it cannot guard the interpolation itself,
+ # because `${{ }}` expands before PowerShell parses the line -- so a value containing a
+ # quote would already have closed the literal and started executing by the time the regex
+ # ran. Reading from $env: means no expansion happens at all.
+ CLAUDE_VERSION: ${{ inputs.claude_version }}
run: |
- $version = '${{ inputs.claude_version }}'
+ $version = $env:CLAUDE_VERSION
if ([string]::IsNullOrWhiteSpace($version)) { $version = '2.1.220' }
- # Whitelist rather than escape: the value reaches a shell, and Claude Code versions are
- # dotted numerals, so anything else is a mistake worth rejecting outright.
+ # Whitelist rather than escape: the value reaches an installer, and Claude Code versions
+ # are dotted numerals, so anything else is a mistake worth rejecting outright.
if ($version -notmatch '^[0-9]+(\.[0-9]+)*$') {
throw "invalid claude_version '$version': expected a dotted numeric version"
}
@@ -147,12 +154,23 @@ jobs:
working-directory: beacon-sandbox
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+ # Passed through the environment, never interpolated into the script. `${{ }}` expansion
+ # happens before PowerShell parses anything, so a value containing a quote would close the
+ # literal and the rest would execute as code -- in this step, with the API key in scope.
+ # claude_version is whitelisted above for the same reason; this makes the two consistent.
+ # Reported by Cursor Bugbot.
+ SCENARIO: ${{ inputs.scenario }}
run: |
# Not $args: that name is a PowerShell automatic variable holding the script's own
# arguments, and appending to it does not behave the way it reads.
$sandboxArgs = @('run', '--provider', 'local')
- if (-not [string]::IsNullOrWhiteSpace('${{ inputs.scenario }}')) {
- $sandboxArgs += @('--scenario', '${{ inputs.scenario }}')
+ if (-not [string]::IsNullOrWhiteSpace($env:SCENARIO)) {
+ # Validated as well as passed safely. A scenario id is a lowercase slug, so anything
+ # else is a mistake worth rejecting outright rather than forwarding to a flag.
+ if ($env:SCENARIO -notmatch '^[a-z0-9][a-z0-9-]*$') {
+ throw "invalid scenario id '$env:SCENARIO': expected a lowercase slug such as w00-probe"
+ }
+ $sandboxArgs += @('--scenario', $env:SCENARIO)
}
go run ./cmd/beacon-sandbox @sandboxArgs
@@ -171,8 +189,14 @@ jobs:
# Without this the job would go green on a failing scenario because of continue-on-error.
- name: Reflect the scenario outcome
if: always()
+ env:
+ # A step outcome is a GitHub-controlled enum rather than user input, so interpolating it
+ # would be safe. It goes through the environment anyway so that "no ${{ }} inside a run
+ # block" holds without exception -- a rule with one blessed exception is a rule nobody can
+ # check mechanically.
+ RUN_OUTCOME: ${{ steps.run.outcome }}
run: |
- Write-Output "scenario outcome: ${{ steps.run.outcome }}"
- if ('${{ steps.run.outcome }}' -ne 'success') {
+ Write-Output "scenario outcome: $env:RUN_OUTCOME"
+ if ($env:RUN_OUTCOME -ne 'success') {
throw "the scenario did not pass; read the uploaded runs/ artifacts and the verdict.json inside them"
}
diff --git a/beacon-sandbox/cmd/beacon-sandbox/doctor_github.go b/beacon-sandbox/cmd/beacon-sandbox/doctor_github.go
index c6576cfb..03a0c95b 100644
--- a/beacon-sandbox/cmd/beacon-sandbox/doctor_github.go
+++ b/beacon-sandbox/cmd/beacon-sandbox/doctor_github.go
@@ -102,6 +102,11 @@ func workflowDispatchableCheck(repo string) doctorCheck {
// the same class of wasted investigation the stale-collector warning exists to prevent, so it gets
// the same treatment: a warning naming exactly what is not covered, satisfiable by doing what it
// asks.
+//
+// The ref in question is the current branch, because dispatch.Run resolves it that way rather than
+// letting `gh workflow run` fall back to the default branch. Both halves are load-bearing: this
+// check compares against the current branch's upstream, so if the dispatcher ran main instead, a
+// clean feature branch would pass the check while the run exercised entirely different code.
func dispatchRefCheck(root string) doctorCheck {
// Only the trees a Windows run actually exercises. Warning on unrelated edits would make the
// check fire constantly and be ignored.
@@ -128,16 +133,24 @@ func dispatchRefCheck(root string) doctorCheck {
// Committed but unpushed is the subtler half: `git status` is clean, so everything looks fine,
// and the dispatch quietly runs the previous commit.
- if b, err := exec.Command("git", "-C", root, "rev-list", "--count", "@{upstream}..HEAD").Output(); err == nil {
- if n := strings.TrimSpace(string(b)); n != "" && n != "0" {
- return doctorCheck{Name: "dispatch_ref", Status: statusWarn,
- Detail: n + " commit(s) ahead of the upstream branch; a dispatch would run the " +
- "pushed ref, which does not include them",
- Fix: "git push, then dispatch with --ref <branch>"}
- }
+ b, err = exec.Command("git", "-C", root, "rev-list", "--count", "@{upstream}..HEAD").Output()
+ if err != nil {
+ // No upstream at all, which is worse than being ahead of one rather than better: a branch
+ // that was never pushed has no remote ref, so the dispatch cannot run it and fails outright.
+ // The previous version let this error fall through to the OK below, which reported a branch
+ // that cannot be dispatched as ready to dispatch.
+ return doctorCheck{Name: "dispatch_ref", Status: statusWarn,
+ Detail: "this branch has no upstream, so there is no pushed ref for a dispatch to run",
+ Fix: "git push -u origin HEAD"}
+ }
+ if n := strings.TrimSpace(string(b)); n != "" && n != "0" {
+ return doctorCheck{Name: "dispatch_ref", Status: statusWarn,
+ Detail: n + " commit(s) ahead of the upstream branch; a dispatch runs the pushed ref, " +
+ "which does not include them",
+ Fix: "git push"}
}
return doctorCheck{Name: "dispatch_ref", Status: statusOK,
- Detail: "the working tree matches the pushed ref, so a dispatch tests what you are looking at"}
+ Detail: "the working tree matches the pushed current branch, which is the ref a dispatch runs"}
}
// windowsSecretCheck reports whether the agent credential the workflow needs is configured.
diff --git a/beacon-sandbox/cmd/beacon-sandbox/doctor_github_test.go b/beacon-sandbox/cmd/beacon-sandbox/doctor_github_test.go
index 4d9be606..b9898425 100644
--- a/beacon-sandbox/cmd/beacon-sandbox/doctor_github_test.go
+++ b/beacon-sandbox/cmd/beacon-sandbox/doctor_github_test.go
@@ -87,12 +87,89 @@ func TestDispatchRefCheckIgnoresUnrelatedEdits(t *testing.T) {
}
}
-// newTestRepo builds a committed git repo with the directory shape the check watches.
-func newTestRepo(t *testing.T) string {
+// A branch that was never pushed has no remote ref, so a dispatch cannot run it and fails outright.
+// That is worse than being merely ahead of an upstream, not better -- and the first version of the
+// check let the `rev-list @{upstream}..HEAD` error fall through to OK, reporting a branch that
+// cannot be dispatched at all as ready to dispatch.
+func TestDispatchRefCheckWarnsWhenTheBranchWasNeverPushed(t *testing.T) {
+ root := newTestRepo(t, withoutUpstream)
+
+ got := dispatchRefCheck(root)
+ if got.Status == statusOK {
+ t.Error("a branch with no upstream has no ref to dispatch and must not pass")
+ }
+ if !strings.Contains(got.Detail, "upstream") {
+ t.Errorf("the warning should name the missing upstream, got %q", got.Detail)
+ }
+ if got.Fix == "" {
+ t.Error("the warning must be satisfiable by doing what it asks")
+ }
+}
+
+// A branch that is pushed but has local commits on top would dispatch the pushed commit, silently
+// running code the contributor is not looking at while `git status` reads clean.
+func TestDispatchRefCheckWarnsWhenAheadOfUpstream(t *testing.T) {
+ root := newTestRepo(t)
+ writeFile(t, root, filepath.Join("cli", "beacon", "extra.go"), "package main\n")
+ gitIn(t, root, "add", "-A")
+ gitIn(t, root, "commit", "-qm", "local only")
+
+ got := dispatchRefCheck(root)
+ if got.Status != statusWarn {
+ t.Errorf("being ahead of upstream must warn, got %q: %s", got.Status, got.Detail)
+ }
+ if !strings.Contains(got.Detail, "ahead") {
+ t.Errorf("the warning should say the branch is ahead, got %q", got.Detail)
+ }
+}
+
+// repoOption tunes the fixture.
+type repoOption func(*repoConfig)
+
+type repoConfig struct{ upstream bool }
+
+// withoutUpstream leaves the branch unpushed.
+func withoutUpstream(c *repoConfig) { c.upstream = false }
+
+func writeFile(t *testing.T, root, rel, body string) {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Dir(filepath.Join(root, rel)), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(root, rel), []byte(body), 0o600); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// gitIn runs git with a hermetic configuration.
+//
+// The global and system configs are neutralized because a contributor's own settings -- commit
+// signing, a hooks path, a default branch name -- would otherwise fail these commits for reasons
+// unrelated to what is being tested.
+func gitIn(t *testing.T, root string, args ...string) {
+ t.Helper()
+ cmd := exec.Command("git", append([]string{"-C", root}, args...)...)
+ cmd.Env = append(os.Environ(),
+ "GIT_CONFIG_GLOBAL="+os.DevNull, "GIT_CONFIG_SYSTEM="+os.DevNull,
+ "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@e",
+ "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@e")
+ if out, err := cmd.CombinedOutput(); err != nil {
+ t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out)
+ }
+}
+
+// newTestRepo builds a committed git repo with the directory shape the check watches, pushed to a
+// local bare remote so the branch has an upstream -- which is the normal state for a branch someone
+// is about to dispatch.
+func newTestRepo(t *testing.T, opts ...repoOption) string {
t.Helper()
if _, err := exec.LookPath("git"); err != nil {
t.Skip("no git on PATH")
}
+ cfg := repoConfig{upstream: true}
+ for _, o := range opts {
+ o(&cfg)
+ }
root := t.TempDir()
for _, dir := range []string{
filepath.Join("cli", "beacon"),
@@ -104,27 +181,20 @@ func newTestRepo(t *testing.T) string {
t.Fatal(err)
}
}
- write := func(rel, body string) {
- if err := os.WriteFile(filepath.Join(root, rel), []byte(body), 0o600); err != nil {
- t.Fatal(err)
- }
- }
- write(filepath.Join("cli", "beacon", "main.go"), "package main\n")
- write("README.md", "# fixture\n")
-
- run := func(args ...string) {
- cmd := exec.Command("git", append([]string{"-C", root}, args...)...)
- // A contributor's global git config can set commit.gpgsign or a hooks path, either of
- // which would fail the commit here for reasons unrelated to what is being tested.
- cmd.Env = append(os.Environ(),
- "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null",
- "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@e", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@e")
- if out, err := cmd.CombinedOutput(); err != nil {
- t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out)
- }
+ writeFile(t, root, filepath.Join("cli", "beacon", "main.go"), "package main\n")
+ writeFile(t, root, "README.md", "# fixture\n")
+
+ gitIn(t, root, "init", "-q", "-b", "work")
+ gitIn(t, root, "add", "-A")
+ gitIn(t, root, "commit", "-qm", "initial")
+
+ if cfg.upstream {
+ // A local bare repo stands in for the remote. Real enough for the check, which only asks
+ // git whether the branch has an upstream and how far ahead of it HEAD is.
+ remote := t.TempDir()
+ gitIn(t, remote, "init", "-q", "--bare")
+ gitIn(t, root, "remote", "add", "origin", remote)
+ gitIn(t, root, "push", "-q", "-u", "origin", "work")
}
- run("init", "-q")
- run("add", "-A")
- run("commit", "-qm", "initial")
return root
}
diff --git a/beacon-sandbox/dispatch/dispatch.go b/beacon-sandbox/dispatch/dispatch.go
index 79be3d52..362bf86f 100644
--- a/beacon-sandbox/dispatch/dispatch.go
+++ b/beacon-sandbox/dispatch/dispatch.go
@@ -31,7 +31,13 @@ const Workflow = "windows-sandbox.yml"
type Options struct {
// Repo is owner/name. Empty lets gh infer it from the checkout.
Repo string
- // Ref is the branch or tag to run. Empty uses the current branch.
+ // Ref is the branch or tag to run.
+ //
+ // Empty is resolved to the current branch by Run, not left to gh -- `gh workflow run` with no
+ // --ref uses the repository's *default* branch, so an unset value would silently run main while
+ // the caller believed they were testing the branch they are on. That is not a hypothetical: the
+ // doctor check that exists to catch exactly this class of mistake asserted the opposite.
+ // Reported by Cursor Bugbot.
Ref string
// Scenario is the scenario id, or empty for every Windows scenario.
Scenario string
@@ -73,6 +79,18 @@ func Run(opts Options) (Result, error) {
if err := requireGH(); err != nil {
return Result{}, err
}
+ // Resolve the ref before dispatching so the run tests the branch the caller is on rather than
+ // the default branch. Reported rather than silent, because which ref ran decides what the
+ // verdict is about.
+ if strings.TrimSpace(opts.Ref) == "" {
+ branch, err := currentBranch()
+ if err != nil {
+ return Result{}, fmt.Errorf("could not determine the current branch to dispatch, and "+
+ "leaving it unset would silently run the default branch instead: %w", err)
+ }
+ opts.Ref = branch
+ }
+ logf("dispatching against ref %s", opts.Ref)
// The newest run id *before* dispatching, so the new one can be identified without guessing.
// `gh workflow run` prints nothing machine-readable and returns no id, which is the whole
@@ -130,6 +148,22 @@ func Run(opts Options) (Result, error) {
return res, nil
}
+// currentBranch reports the checked-out branch name.
+//
+// A detached HEAD has no branch to dispatch, and is reported rather than guessed at: workflow_dispatch
+// takes a ref by name, so there is nothing sensible to substitute.
+func currentBranch() (string, error) {
+ out, err := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output()
+ if err != nil {
+ return "", err
+ }
+ branch := strings.TrimSpace(string(out))
+ if branch == "" || branch == "HEAD" {
+ return "", fmt.Errorf("HEAD is detached, so there is no branch to dispatch; pass --ref explicitly")
+ }
+ return branch, nil
+}
+
// requireGH fails early with the fix rather than letting exec produce "file not found".
func requireGH() error {
if _, err := exec.LookPath("gh"); err != nil {
@@ -177,35 +211,45 @@ func listRuns(opts Options, limit int) ([]ghRun, error) {
return runs, nil
}
-func latestRunID(opts Options) (string, error) {
- runs, err := listRuns(opts, 1)
+// latestRunID returns the highest existing run id, or 0 when there is no history.
+//
+// The maximum rather than the first listed: the ordering is documented as newest-first but this
+// value is the floor every later comparison rests on, and taking the max is correct even if the
+// ordering ever changes.
+func latestRunID(opts Options) (int64, error) {
+ runs, err := listRuns(opts, 10)
if err != nil {
- return "", err
- }
- if len(runs) == 0 {
- // No history yet, which is normal on the first dispatch.
- return "", nil
+ return 0, err
}
- return fmt.Sprint(runs[0].DatabaseID), nil
+ // Zero is the correct floor when there is no history, which is normal on a first dispatch:
+ // every real run id is greater than it.
+ return maxRunID(runs), nil
}
-// awaitNewRun waits for a run id that differs from the one seen before dispatching.
+// awaitNewRun waits for a run created after the dispatch.
+//
+// Compared numerically, and that is the whole correctness of this function. GitHub run ids
+// increase monotonically, so "created after ours" is exactly "id greater than the newest id that
+// existed before we dispatched".
//
-// Identifying the run by "newest that is not the one already there" rather than by "newest" is
-// what stops a still-running earlier dispatch being mistaken for this one -- which would report
-// somebody else's verdict as yours.
-func awaitNewRun(opts Options, before string, logf func(string, ...any)) (string, error) {
+// An earlier version accepted the first listed run whose id merely *differed* from that one, which
+// is wrong in a way that produces a confident wrong answer rather than an error: the listing is
+// newest-first, the freshly dispatched run usually does not exist yet on the first poll, so the
+// second-newest *historical* run satisfied "differs" and was returned. The dispatch then waited on,
+// downloaded and judged a previous run's artifacts and reported that stale verdict as this run's
+// result -- the precise confusion this function exists to prevent. Reported by Cursor Bugbot.
+//
+// The smallest qualifying id is taken rather than the newest, so that if another dispatch lands
+// while we poll, we follow the earlier one -- ours.
+func awaitNewRun(opts Options, before int64, logf func(string, ...any)) (string, error) {
deadline := time.Now().Add(5 * time.Minute)
for time.Now().Before(deadline) {
- runs, err := listRuns(opts, 5)
+ runs, err := listRuns(opts, 10)
if err != nil {
return "", err
}
- for _, r := range runs {
- id := fmt.Sprint(r.DatabaseID)
- if id != before && runIDPattern.MatchString(id) {
- return id, nil
- }
+ if found := pickNewRun(runs, before); found > 0 {
+ return fmt.Sprint(found), nil
}
time.Sleep(5 * time.Second)
}
@@ -213,6 +257,34 @@ func awaitNewRun(opts Options, before string, logf func(string, ...any)) (string
"`gh run list --workflow %s`", Workflow, Workflow)
}
+// pickNewRun returns the earliest run created after the floor, or 0 if none has appeared.
+//
+// Separated from the polling loop so the selection rule -- the part that was wrong -- is testable
+// without a GitHub account or a live workflow.
+func pickNewRun(runs []ghRun, before int64) int64 {
+ var found int64
+ for _, r := range runs {
+ if r.DatabaseID <= before {
+ continue
+ }
+ if found == 0 || r.DatabaseID < found {
+ found = r.DatabaseID
+ }
+ }
+ return found
+}
+
+// maxRunID is the floor "created after our dispatch" is measured against.
+func maxRunID(runs []ghRun) int64 {
+ var newest int64
+ for _, r := range runs {
+ if r.DatabaseID > newest {
+ newest = r.DatabaseID
+ }
+ }
+ return newest
+}
+
// awaitCompletion polls until the run finishes, returning GitHub's conclusion.
//
// A non-success conclusion is returned rather than raised: the job uploads artifacts even when
diff --git a/beacon-sandbox/dispatch/dispatch_test.go b/beacon-sandbox/dispatch/dispatch_test.go
index e3cffe54..d1932a12 100644
--- a/beacon-sandbox/dispatch/dispatch_test.go
+++ b/beacon-sandbox/dispatch/dispatch_test.go
@@ -68,6 +68,52 @@ func TestFindRunDirsToleratesNothingCollected(t *testing.T) {
}
}
+// The bug this pins is one that produced a confident wrong answer rather than an error: the run
+// listing is newest-first, and the freshly dispatched run usually does not exist on the first poll,
+// so accepting "the first id that differs from the pre-dispatch id" returned the second-newest
+// *historical* run. The dispatch then judged a previous run's artifacts as this run's result.
+//
+// Selection is pure given a listing, so it is tested directly rather than through gh.
+func TestNewRunSelectionRequiresAnIDGreaterThanTheFloor(t *testing.T) {
+ // A realistic listing at the moment of dispatch: history only, newest first, none of it ours.
+ history := []ghRun{{DatabaseID: 500}, {DatabaseID: 499}, {DatabaseID: 498}}
+ if got := pickNewRun(history, 500); got != 0 {
+ t.Errorf("with only pre-existing runs the selection must find nothing, got %d", got)
+ }
+ // The specific regression: 499 differs from 500 and is listed, but predates the dispatch.
+ if got := pickNewRun(history, 500); got == 499 {
+ t.Error("an older run must never be selected; that is the stale-verdict bug")
+ }
+
+ // Once ours appears it is selected.
+ withOurs := append([]ghRun{{DatabaseID: 501}}, history...)
+ if got := pickNewRun(withOurs, 500); got != 501 {
+ t.Errorf("the run created after the floor must be selected, got %d", got)
+ }
+
+ // Two dispatches racing: follow the earlier new run, which is ours.
+ withRace := append([]ghRun{{DatabaseID: 502}, {DatabaseID: 501}}, history...)
+ if got := pickNewRun(withRace, 500); got != 501 {
+ t.Errorf("with two new runs the earlier one is ours, got %d", got)
+ }
+
+ // A first-ever dispatch has no history, so the floor is zero and any real run qualifies.
+ if got := pickNewRun([]ghRun{{DatabaseID: 7}}, 0); got != 7 {
+ t.Errorf("a zero floor must accept the first run, got %d", got)
+ }
+}
+
+// The floor is the maximum, not the first element: it is what every later comparison rests on, so
+// it stays correct even if the listing order ever changes.
+func TestRunFloorIsTheMaximumID(t *testing.T) {
+ if got := maxRunID([]ghRun{{DatabaseID: 3}, {DatabaseID: 9}, {DatabaseID: 5}}); got != 9 {
+ t.Errorf("floor = %d, want 9", got)
+ }
+ if got := maxRunID(nil); got != 0 {
+ t.Errorf("no history must yield a zero floor, got %d", got)
+ }
+}
+
// gh returns numeric run ids. Anything else means the output shape changed, and interpolating it
// into a URL or a filesystem path would be worse than reporting it.
func TestRunIDPatternRejectsAnythingButDigits(t *testing.T) {
diff --git a/beacon-sandbox/sandbox/localexec.go b/beacon-sandbox/sandbox/localexec.go
index 71405545..e12acf48 100644
--- a/beacon-sandbox/sandbox/localexec.go
+++ b/beacon-sandbox/sandbox/localexec.go
@@ -204,10 +204,19 @@ func (l *LocalExec) Exec(ctx context.Context, _ Instance, script string, opts Ex
err := cmd.Run()
res := Result{Stdout: stdout.String(), Stderr: stderr.String()}
+ // The deadline is checked before the error is classified, and the order matters. When the
+ // context expires, CommandContext kills the child and Run returns an *ExitError like any other
+ // non-zero exit -- so classifying first would record a killed session as an exit status, and
+ // the verdict would read it as Beacon failing to capture rather than as the harness never
+ // having finished the run. "The command failed" and "we never ran it to completion" must not
+ // collapse into the same result. Reported by Cursor Bugbot.
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ return res, fmt.Errorf("exec %s did not complete (%w): the command was killed after %s",
+ l.shell, ctxErr, opts.Timeout)
+ }
// An ExitError carries the status and is not a failure of Exec itself; every caller reads
- // ExitCode and decides what it means. Anything else -- the shell missing, a timeout -- is a
- // harness failure and must surface as one, or "the command failed" and "we never ran it"
- // become the same result.
+ // ExitCode and decides what it means. Anything else -- the shell missing, for instance -- is a
+ // harness failure and must surface as one.
if err != nil {
var ee *exec.ExitError
if errors.As(err, &ee) {
diff --git a/beacon-sandbox/sandbox/localexec_test.go b/beacon-sandbox/sandbox/localexec_test.go
new file mode 100644
index 00000000..40029b0f
--- /dev/null
+++ b/beacon-sandbox/sandbox/localexec_test.go
@@ -0,0 +1,181 @@
+package sandbox
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "testing"
+ "time"
+)
+
+// newTestLocalExec builds a provider without the disposability gate, which is the point of a
+// separate constructor path in tests: the gate exists to stop a scenario installing Beacon on a
+// workstation, and these tests run trivial shell commands rather than scenarios.
+func newTestLocalExec(t *testing.T) *LocalExec {
+ t.Helper()
+ l, err := NewLocalExec(LocalExecOptions{AllowHostMutation: true})
+ if err != nil {
+ t.Skipf("no usable shell on this host: %v", err)
+ }
+ return l
+}
+
+// The gate is the only thing standing between `--provider local` and a scenario installing Beacon
+// and registering services on somebody's workstation, so its outcomes are pinned.
+func TestRequireDisposableRefusesAnUnprovableHost(t *testing.T) {
+ cases := []struct {
+ name string
+ actions string
+ runnerEnv string
+ allow bool
+ wantErr bool
+ }{
+ {"workstation", "", "", false, true},
+ {"workstation with opt-in", "", "", true, false},
+ // Self-hosted runners persist between jobs, so an install there outlives the run -- the same
+ // escape the host guard exists to catch, relocated.
+ {"self-hosted runner", "true", "self-hosted", false, true},
+ {"self-hosted with opt-in", "true", "self-hosted", true, false},
+ {"github-hosted runner", "true", "github-hosted", false, false},
+ // Older Actions runners did not report the environment; GITHUB_ACTIONS is already the
+ // stronger signal, so an unreported value is accepted.
+ {"unreported environment", "true", "", false, false},
+ }
+ for _, c := range cases {
+ t.Setenv("GITHUB_ACTIONS", c.actions)
+ t.Setenv("RUNNER_ENVIRONMENT", c.runnerEnv)
+
+ err := requireDisposable(c.allow)
+ if c.wantErr && err == nil {
+ t.Errorf("%s: expected a refusal", c.name)
+ }
+ if !c.wantErr && err != nil {
+ t.Errorf("%s: expected acceptance, got %v", c.name, err)
+ }
+ }
+}
+
+// The evidence a verdict reports must distinguish "GitHub hands out a fresh VM per job" from "the
+// operator asserted it". They are materially different claims, and a reader of a verdict that
+// cannot compare host state is entitled to know which one it rested on.
+func TestDisposabilityEvidenceNamesItsBasis(t *testing.T) {
+ t.Setenv("GITHUB_ACTIONS", "true")
+ t.Setenv("RUNNER_ENVIRONMENT", "github-hosted")
+ if got := DisposabilityEvidence(); !strings.Contains(got, "ephemeral") {
+ t.Errorf("a github-hosted runner should say so, got %q", got)
+ }
+
+ t.Setenv("GITHUB_ACTIONS", "")
+ t.Setenv("RUNNER_ENVIRONMENT", "")
+ got := DisposabilityEvidence()
+ if !strings.Contains(got, "asserted") {
+ t.Errorf("an operator opt-in must be named as such, got %q", got)
+ }
+ if strings.Contains(got, "ephemeral") {
+ t.Error("an asserted host must not borrow the wording of a provably ephemeral one")
+ }
+}
+
+// A non-zero exit is a finding about the thing under test; a timeout is the harness failing to
+// finish. They must not collapse into one result -- CommandContext kills the child and Run returns
+// an *ExitError either way, so classifying the error without consulting the deadline recorded a
+// killed session as an exit status and the verdict read it as a capture failure.
+func TestExecDistinguishesATimeoutFromANonZeroExit(t *testing.T) {
+ l := newTestLocalExec(t)
+ inst := localInstance{id: "test"}
+
+ // A command that exits non-zero promptly: a result, not an error.
+ script := "exit 3"
+ if l.platform.IsWindows() {
+ script = "exit 3"
+ }
+ res, err := l.Exec(context.Background(), inst, script, ExecOpts{Timeout: 30 * time.Second})
+ if err != nil {
+ t.Fatalf("a non-zero exit must not be an Exec error: %v", err)
+ }
+ if res.ExitCode != 3 {
+ t.Errorf("exit code = %d, want 3", res.ExitCode)
+ }
+
+ // A command that outlives its deadline: an error, not a result.
+ sleep := "sleep 10"
+ if l.platform.IsWindows() {
+ sleep = "Start-Sleep -Seconds 10"
+ }
+ res, err = l.Exec(context.Background(), inst, sleep, ExecOpts{Timeout: 300 * time.Millisecond})
+ if err == nil {
+ t.Fatalf("a timeout must surface as an error, got a result with exit code %d", res.ExitCode)
+ }
+ if !strings.Contains(err.Error(), "did not complete") {
+ t.Errorf("the error should say the command never finished, got %v", err)
+ }
+}
+
+// Stdout and stderr are captured separately, because every verdict is built from them and the
+// runner reads each one for different signals.
+func TestExecCapturesBothStreams(t *testing.T) {
+ l := newTestLocalExec(t)
+ inst := localInstance{id: "test"}
+
+ script := "echo out; echo err 1>&2"
+ if l.platform.IsWindows() {
+ script = "Write-Output 'out'; [Console]::Error.WriteLine('err')"
+ }
+ res, err := l.Exec(context.Background(), inst, script, ExecOpts{Timeout: 30 * time.Second})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(res.Stdout, "out") {
+ t.Errorf("stdout = %q, want it to contain \"out\"", res.Stdout)
+ }
+ if !strings.Contains(res.Stderr, "err") {
+ t.Errorf("stderr = %q, want it to contain \"err\"", res.Stderr)
+ }
+}
+
+// Snapshot must fail rather than quietly succeed. A caller that believes it snapshotted, and did
+// not, would reuse a stale environment and attribute the result to the wrong build.
+func TestSnapshotIsRefusedRatherThanFaked(t *testing.T) {
+ l := newTestLocalExec(t)
+ if _, err := l.Snapshot(context.Background(), localInstance{id: "test"}); err == nil {
+ t.Error("Snapshot must report that it cannot snapshot this backend")
+ }
+}
+
+// Put and Get carry collected artifacts into the run directory, and those retain prompt text and
+// command output -- so the copies must not be readable by every local user.
+func TestPutAndGetCopyWithOwnerOnlyMode(t *testing.T) {
+ l := newTestLocalExec(t)
+ inst := localInstance{id: "test"}
+ dir := t.TempDir()
+
+ src := filepath.Join(dir, "src.jsonl")
+ if err := os.WriteFile(src, []byte(`{"event":"one"}`+"\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ dst := filepath.Join(dir, "nested", "dst.jsonl")
+ if err := l.Put(context.Background(), inst, src, dst); err != nil {
+ t.Fatalf("Put: %v", err)
+ }
+ b, err := os.ReadFile(dst)
+ if err != nil || !strings.Contains(string(b), "one") {
+ t.Fatalf("Put did not copy the content: %v %q", err, b)
+ }
+
+ back := filepath.Join(dir, "back.jsonl")
+ if err := l.Get(context.Background(), inst, dst, back); err != nil {
+ t.Fatalf("Get: %v", err)
+ }
+ info, err := os.Stat(back)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Windows does not implement Unix permission bits, so the mode assertion is meaningful only
+ // where it is enforced. Asserting it there anyway would fail for a reason unrelated to the
+ // property being protected.
+ if runtime.GOOS != "windows" && info.Mode().Perm() != 0o600 {
+ t.Errorf("collected artifacts must be owner-only, got %v", info.Mode().Perm())
+ }
+}