Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(beacon-sandbox): remove plant-secret's precondition, close the provenance false green

shukieshah32a79e8a1c76
diff --git a/beacon-sandbox/AGENTS.md b/beacon-sandbox/AGENTS.md
index f971c3c4..c6ee9e70 100644
--- a/beacon-sandbox/AGENTS.md
+++ b/beacon-sandbox/AGENTS.md
@@ -168,11 +168,9 @@ Other mutations: `drop-action:<action>`, `drop-commands`, `plant-secret`.
Every mutation refuses to be a no-op: an empty log, or an action the log does not contain, errors
rather than reporting a self-test that proved nothing.
-**`plant-secret` only works on a run captured with the `ANTHROPIC_API_KEY` currently set.** It
-plants that key and expects the leak check to find it, but the leak check is withheld when the value
-is not the one the run used — a provider secret, a rotated key, or a run recorded before credential
-fingerprints existed. It refuses in that case and names the modes that do work, so if you hit that
-error pick `corrupt-line` instead rather than treating it as a bug.
+`plant-secret` needs no credential of its own. It plants a unique synthetic value and the leak check
+is told to search for that, so it works on any run directory regardless of which credential the run
+used, or whether one is set at all.
## Writing or editing a scenario
diff --git a/beacon-sandbox/README.md b/beacon-sandbox/README.md
index 643f9382..b1bfeec6 100644
--- a/beacon-sandbox/README.md
+++ b/beacon-sandbox/README.md
@@ -104,9 +104,8 @@ go run ./cmd/beacon-sandbox verify --mutate corrupt-line runs/<id>/ # a PASS mu
```
Every mutation refuses to be a no-op, so a self-test can never pass without having damaged
-anything. `plant-secret` additionally needs a run captured with the `ANTHROPIC_API_KEY` currently
-set, since otherwise the leak check is withheld and the planted key would never be searched for;
-it refuses and names the modes that do work.
+anything. `plant-secret` plants a unique synthetic credential and the leak check is told to search
+for it, so it works on any run directory with any credential arrangement, including none.
Tests need no Modal account, no API key, and no sandbox, so they cost nothing. (They still
resolve Go modules, like every other test suite in this repo.)
diff --git a/beacon-sandbox/cmd/beacon-sandbox/doctor_test.go b/beacon-sandbox/cmd/beacon-sandbox/doctor_test.go
index e5990319..fb1ae7e8 100644
--- a/beacon-sandbox/cmd/beacon-sandbox/doctor_test.go
+++ b/beacon-sandbox/cmd/beacon-sandbox/doctor_test.go
@@ -407,7 +407,7 @@ func TestMutationsRefuseToNoOp(t *testing.T) {
t.Fatal(err)
}
for _, mode := range []string{"corrupt-line", "plant-secret", "drop-commands"} {
- if _, err := applyMutation(empty, mode); err == nil {
+ if _, _, err := applyMutation(empty, mode); err == nil {
t.Errorf("%s on an empty log must error rather than report a passing self-test", mode)
}
}
@@ -425,7 +425,7 @@ func TestPlantSecretWorksWithoutTheMessageAnchor(t *testing.T) {
t.Fatal(err)
}
- mutatedPath, err := applyMutation(p, "plant-secret")
+ mutatedPath, planted, err := applyMutation(p, "plant-secret")
if err != nil {
t.Fatalf("plant-secret should handle any object, got: %v", err)
}
@@ -433,8 +433,11 @@ func TestPlantSecretWorksWithoutTheMessageAnchor(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if !strings.Contains(string(body), key) {
- t.Fatalf("the credential was not planted:\n%s", body)
+ if planted == "" {
+ t.Fatal("plant-secret must report what it planted so the judge can search for it")
+ }
+ if !strings.Contains(string(body), planted) {
+ t.Fatalf("the planted credential is not in the mutated log:\n%s", body)
}
// It should still be parseable, so the planted secret trips the leak check rather than the
// parse invariant -- otherwise the self-test would pass for the wrong reason.
@@ -455,7 +458,7 @@ func TestPlantSecretWithTheMessageAnchorStaysValidJSON(t *testing.T) {
t.Fatal(err)
}
- mutatedPath, err := applyMutation(p, "plant-secret")
+ mutatedPath, planted, err := applyMutation(p, "plant-secret")
if err != nil {
t.Fatal(err)
}
@@ -464,8 +467,8 @@ func TestPlantSecretWithTheMessageAnchorStaysValidJSON(t *testing.T) {
if err := json.Unmarshal([]byte(strings.TrimSpace(string(body))), &m); err != nil {
t.Fatalf("planting broke the JSON: %v\n%s", err, body)
}
- if m["leaked"] != key {
- t.Errorf("expected the key under \"leaked\", got %v", m["leaked"])
+ if m["leaked"] != planted {
+ t.Errorf("expected the planted value under \"leaked\", got %v", m["leaked"])
}
}
@@ -539,3 +542,49 @@ func TestOfflineSentinelDecodingNeverInventsAnIdleAgent(t *testing.T) {
t.Error("absent metadata must never decode as a probed-and-absent sentinel")
}
}
+
+// plant-secret must exercise the leak check on any run directory, with any credential arrangement
+// including none. Planting the ambient ANTHROPIC_API_KEY only worked when it happened to be the key
+// the run was captured with -- the leak check is withheld for a provider secret, a rotated key, or a
+// run predating credential fingerprints -- so the planted value went unsearched and the self-test
+// passed having exercised nothing. Refusing in those cases was honest but left the mutation
+// unavailable on most existing runs. Planting a value the judge is told to search for removes the
+// precondition entirely.
+func TestPlantSecretNeedsNoAmbientCredential(t *testing.T) {
+ // Deliberately no ANTHROPIC_API_KEY, the case that used to be refused outright.
+ t.Setenv("ANTHROPIC_API_KEY", "")
+
+ p := filepath.Join(t.TempDir(), "runtime.jsonl")
+ line := `{"timestamp":"2026-08-02T18:00:00Z","vendor":"beacon","message":"m"}`
+ if err := os.WriteFile(p, []byte(line+"\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+
+ mutatedPath, planted, err := applyMutation(p, "plant-secret")
+ if err != nil {
+ t.Fatalf("plant-secret must work without an ambient credential: %v", err)
+ }
+ if planted == "" {
+ t.Fatal("the planted value must be reported so the judge can search for it")
+ }
+ body, err := os.ReadFile(mutatedPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(body), planted) {
+ t.Errorf("the planted value is missing from the mutated log:\n%s", body)
+ }
+ // It must not be a fixed string, or two concurrent self-tests could collide, and a real log
+ // containing the constant would produce a false leak.
+ _, second, err := applyMutation(p, "plant-secret")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if second == planted {
+ t.Error("the planted value must be unique per invocation")
+ }
+ // And it must be recognisable as a synthetic self-test value rather than a plausible real key.
+ if !strings.Contains(planted, "selftest") {
+ t.Errorf("the planted value should identify itself as a self-test artifact, got %q", planted)
+ }
+}
diff --git a/beacon-sandbox/cmd/beacon-sandbox/main.go b/beacon-sandbox/cmd/beacon-sandbox/main.go
index 29f302d4..c1babdb8 100644
--- a/beacon-sandbox/cmd/beacon-sandbox/main.go
+++ b/beacon-sandbox/cmd/beacon-sandbox/main.go
@@ -10,6 +10,8 @@ package main
import (
"context"
+ "crypto/rand"
+ "encoding/hex"
"encoding/json"
"flag"
"fmt"
@@ -26,6 +28,16 @@ import (
const appName = "beacon-sandbox"
+// randomHex makes the planted self-test credential unique per invocation, so it cannot collide
+// with anything already in a collected log.
+func randomHex() string {
+ b := make([]byte, 8)
+ if _, err := rand.Read(b); err != nil {
+ return "fallback0000000"
+ }
+ return hex.EncodeToString(b)
+}
+
func main() {
if len(os.Args) < 2 {
usage()
@@ -179,7 +191,7 @@ func cmdRun(args []string) error {
verdicts = append(verdicts, v)
continue
}
- v, log := judge(sc, art, creds)
+ v, log := judge(sc, art, creds, "")
fmt.Print(indent(v.Report()))
if art.Dir != "" {
_ = v.WriteJSON(filepath.Join(art.Dir, "verdict.json"))
@@ -193,7 +205,7 @@ func cmdRun(args []string) error {
// judge is the pure step: artifacts in, verdict out. Returns the parsed log too so callers
// can build a fingerprint without re-reading it.
-func judge(sc scenario.Scenario, art runner.Artifacts, creds credentials.Resolved) (check.Verdict, check.Log) {
+func judge(sc scenario.Scenario, art runner.Artifacts, creds credentials.Resolved, planted string) (check.Verdict, check.Log) {
v := check.Verdict{Scenario: sc.ID, Meta: art.Meta}
// Host escape and credential-in-argv come from run metadata, not from the log, so they must
@@ -219,6 +231,15 @@ func judge(sc scenario.Scenario, art runner.Artifacts, creds credentials.Resolve
} else {
secrets.Withheld = map[string]string{credentials.EnvVar: creds.WithheldReason()}
}
+ // A self-test plants a synthetic credential, and the leak check has to be searching for it or
+ // the mutation proves nothing. Searched in addition to the real credential, never instead of
+ // it, so a mutated run still reports on the genuine one.
+ if planted != "" {
+ if secrets.Values == nil {
+ secrets.Values = map[string]string{}
+ }
+ secrets.Values["planted self-test credential"] = planted
+ }
check.Invariants(&v, log, secrets)
check.Expectations(&v, log, sc, art.Canary, check.Sentinel{
Declared: sc.Sentinel != "",
@@ -349,24 +370,9 @@ func cmdVerify(args []string) error {
}
logPath := filepath.Join(dir, "runtime.jsonl")
- if *mutate == "plant-secret" {
- // The mutation plants the local key, but offline judging withholds the leak search
- // for provider-secret runs and for fingerprint mismatches. In those cases the
- // planted credential could never trip invariant.no_secret_leak, so the self-test
- // would report PASS and prove nothing -- a self-test that cannot fail is exactly
- // what this tool refuses to ship. Reported by Cursor Bugbot.
- if c := offlineCredential(meta); !c.LeakCheckPossible() {
- return fmt.Errorf("plant-secret cannot self-test %s.\n"+
- " The leak check is withheld for this run because %s.\n"+
- " A planted credential would therefore never be searched for, and the "+
- "mutation would report PASS while proving nothing.\n"+
- " Use a run captured with the ANTHROPIC_API_KEY currently set, or try a "+
- "different mutation: corrupt-line, drop-commands, drop-action:<action>.",
- dir, c.WithheldReason())
- }
- }
+ var planted string
if *mutate != "" {
- logPath, err = applyMutation(logPath, *mutate)
+ logPath, planted, err = applyMutation(logPath, *mutate)
if err != nil {
return err
}
@@ -383,7 +389,8 @@ func cmdVerify(args []string) error {
art.SentinelDetail = meta["sentinel_detail"]
art.SentinelProbed, art.SentinelPresent = decodeSentinel(meta)
art.SessionKnown, art.SessionOK = decodeSession(meta)
- v, log := judge(sc, art, offlineCredential(meta))
+ creds := offlineCredential(meta)
+ v, log := judge(sc, art, creds, planted)
fmt.Print(v.Report())
// Refresh both artifacts, or neither. Only the fingerprint was rewritten before, so a
// re-judge with changed expectations left verdict.json holding the old outcome while
@@ -448,10 +455,10 @@ func findScenario(root, id string) (scenario.Scenario, error) {
// applyMutation deliberately damages a copy of the log. If the checks still pass afterwards
// they are not actually checking anything, so this is a self-test of the oracle.
-func applyMutation(path, mode string) (string, error) {
+func applyMutation(path, mode string) (mutated string, planted string, err error) {
b, err := os.ReadFile(path)
if err != nil {
- return "", err
+ return "", "", err
}
// strings.Split("", "\n") yields one empty element, so an empty log would otherwise be
// treated as having a phantom line -- corrupt-line would "damage" a line that never existed
@@ -480,14 +487,14 @@ func applyMutation(path, mode string) (string, error) {
out = append(out, l)
}
if dropped == 0 {
- return "", fmt.Errorf("mutation %q removed nothing: no %q events in this log, "+
+ return "", "", fmt.Errorf("mutation %q removed nothing: no %q events in this log, "+
"so the self-test would prove nothing", mode, action)
}
tmp := path + ".mutated"
if err := os.WriteFile(tmp, []byte(strings.Join(out, "\n")+"\n"), 0o644); err != nil {
- return "", err
+ return "", "", err
}
- return tmp, nil
+ return tmp, planted, nil
}
// A mutation that changes nothing proves nothing, so every branch below has to actually
@@ -503,21 +510,31 @@ func applyMutation(path, mode string) (string, error) {
out[len(out)/2] = `{"vendor":"beacon","this is not json`
}
case "plant-secret":
- key := os.Getenv("ANTHROPIC_API_KEY")
- if key == "" {
- return "", fmt.Errorf("plant-secret needs ANTHROPIC_API_KEY to plant")
- }
+ // A synthetic credential, not the run's own.
+ //
+ // Planting the ambient ANTHROPIC_API_KEY only worked when it happened to be the key the
+ // run was captured with: the leak check is withheld for a provider secret, a rotated
+ // key, or a run predating credential fingerprints, so the planted value would never be
+ // searched for and the self-test reported PASS having exercised nothing. The previous
+ // fix refused in those cases, which was honest but meant the mutation was unavailable on
+ // most existing run directories.
+ //
+ // Planting a value the judge is then told to search for removes the precondition
+ // entirely: the leak check is exercised on any run, with any credential arrangement,
+ // including none at all. The synthetic value is unique per invocation so it cannot
+ // collide with real log content.
+ planted = "sk-ant-beacon-sandbox-selftest-" + randomHex()
out = append(out, lines...)
if len(out) > 0 {
- out[0] = plantSecret(out[0], key)
+ out[0] = plantSecret(out[0], planted)
}
- if len(out) == 0 || !strings.Contains(out[0], key) {
- return "", fmt.Errorf("plant-secret could not insert the credential into %s: "+
+ if len(out) == 0 || !strings.Contains(out[0], planted) {
+ return "", "", fmt.Errorf("plant-secret could not insert a credential into %s: "+
"the first line is empty or not an object, so the leak check would never see "+
"it and the self-test would prove nothing", path)
}
default:
- return "", fmt.Errorf("unknown mutation %q (want drop-commands, drop-action:<action>, "+
+ return "", "", fmt.Errorf("unknown mutation %q (want drop-commands, drop-action:<action>, "+
"corrupt-line, or plant-secret)", mode)
}
@@ -525,16 +542,16 @@ func applyMutation(path, mode string) (string, error) {
// bodies, because appending the trailing newline made an empty log look like it had changed
// from "" to "\n" -- a difference in the file that is no difference in the events, which is
// the only thing a self-test cares about.
- mutated := strings.Join(out, "\n") + "\n"
- if len(out) == 0 || strings.TrimRight(mutated, "\n") == trimmed {
- return "", fmt.Errorf("mutation %q left %s unchanged, so the self-test would prove "+
+ body := strings.Join(out, "\n") + "\n"
+ if len(out) == 0 || strings.TrimRight(body, "\n") == trimmed {
+ return "", "", fmt.Errorf("mutation %q left %s unchanged, so the self-test would prove "+
"nothing (is the log empty?)", mode, path)
}
tmp := path + ".mutated"
- if err := os.WriteFile(tmp, []byte(mutated), 0o644); err != nil {
- return "", err
+ if err := os.WriteFile(tmp, []byte(body), 0o644); err != nil {
+ return "", "", err
}
- return tmp, nil
+ return tmp, planted, nil
}
// plantSecret inserts a credential into a JSONL event line.
diff --git a/beacon-sandbox/image/artifacts.go b/beacon-sandbox/image/artifacts.go
index d901491f..1714ebf0 100644
--- a/beacon-sandbox/image/artifacts.go
+++ b/beacon-sandbox/image/artifacts.go
@@ -124,7 +124,8 @@ func CollectorIsStale(repoRoot string) (bool, string) {
return false, ""
}
- if tag := recordedRelease(repoRoot); tag != "" {
+ tag, state := recordedRelease(repoRoot)
+ if state == provenanceTrusted {
changed, resolved := collectorChangesSince(repoRoot, tag)
switch {
case resolved && changed != "":
@@ -141,10 +142,35 @@ func CollectorIsStale(repoRoot string) (bool, string) {
}
}
- // Locally built: a source file newer than the binary means the binary predates it.
+ // A source file newer than the binary means the binary predates it, whatever its origin.
if newer := sourcesNewerThan(repoRoot, fi.ModTime()); newer != "" {
return true, "collector-builder/ sources are newer than the built binary: " + newer
}
+
+ // Sources are older than the binary, which for a local rebuild means fresh -- but with no
+ // trustworthy provenance there is no way to tell a local build from a release download, and a
+ // downloaded binary is essentially always newer than local sources. So this signal cannot rule
+ // out release drift, and saying "fresh" here is exactly the false green this check exists to
+ // prevent.
+ if state == provenanceAbsent {
+ // Warning on every unprovenanced binary would penalise the common legitimate case -- a
+ // local build -- to guard a case that is now closed for fresh downloads, since
+ // EnsureCollector always records provenance. So the warning is narrowed to the situation
+ // where the ambiguity can actually bite: the tree carries exporter changes since the most
+ // recent release it knows about, and the binary might predate them.
+ //
+ // A contributor who has not touched the exporter sees nothing. One who has, with a binary
+ // of unknown origin, is told the truth: this cannot be ruled out either way.
+ if tag, ok := newestLocalReleaseTag(repoRoot); ok {
+ if changed, resolved := collectorChangesSince(repoRoot, tag); resolved && changed != "" {
+ return true, fmt.Sprintf("collector-builder/ has changed since release %s (%s) and "+
+ "this binary's origin is unrecorded, so it cannot be confirmed to contain "+
+ "those changes (rebuild it, or delete it and rerun `doctor --fix`)", tag, changed)
+ }
+ }
+ }
+ // provenanceStale: the binary was replaced after a recorded download, which is the documented
+ // rebuild. The mtime comparison above is then meaningful, and it passed.
return false, ""
}
@@ -171,16 +197,33 @@ func uncommittedCollectorChanges(repoRoot string) string {
return summarize3(files)
}
-// recordedRelease returns the release tag a downloaded binary came from, empty when the binary was
-// built locally, predates provenance recording, or has been replaced since the marker was written.
+// provenanceState describes how much the recorded provenance can be trusted.
+type provenanceState int
+
+const (
+ // provenanceAbsent means no usable marker: never recorded, or recorded before the binary's
+ // identity was captured. The binary's origin is unknown.
+ provenanceAbsent provenanceState = iota
+ // provenanceStale means a marker exists but describes a different binary, so the binary was
+ // replaced after it was written -- almost always a local rebuild.
+ provenanceStale
+ // provenanceTrusted means the marker still describes the binary on disk.
+ provenanceTrusted
+)
+
+// recordedRelease returns the release a binary was downloaded from, and how much that can be
+// trusted.
//
-// That last case is the important one: a marker that outlives the binary it described would keep a
-// freshness warning permanently lit after a local rebuild, so the size and mtime recorded at
-// download time are checked against the binary that is actually there now.
-func recordedRelease(repoRoot string) string {
+// Collapsing all three states into an empty tag was a false-green generator. When the tag was
+// empty, CollectorIsStale skipped release-drift detection entirely and fell back to comparing
+// mtimes -- and a downloaded release binary is essentially always newer than local sources, so
+// committed exporter changes since that release went unreported and the check went green while the
+// run verified the wrong binary. Reported by Cursor Bugbot, and the same shape as the
+// unresolvable-tag case: unknowable rendered as clean.
+func recordedRelease(repoRoot string) (tag string, state provenanceState) {
b, err := os.ReadFile(provenancePath(repoRoot))
if err != nil {
- return ""
+ return "", provenanceAbsent
}
fields := map[string]string{}
for _, line := range strings.Split(string(b), "\n") {
@@ -188,25 +231,25 @@ func recordedRelease(repoRoot string) string {
fields[k] = v
}
}
- tag := strings.TrimSpace(fields["release"])
+ tag = strings.TrimSpace(fields["release"])
if tag == "" {
- return ""
+ return "", provenanceAbsent
}
fi, err := os.Stat(CollectorPath(repoRoot))
if err != nil {
- return ""
+ return "", provenanceAbsent
}
- // A marker written before identity was recorded cannot be validated, so it is not trusted.
+ // A marker written before identity was recorded cannot be validated either way.
if fields["size"] == "" || fields["mtime"] == "" {
- return ""
+ return tag, provenanceAbsent
}
if fields["size"] != strconv.FormatInt(fi.Size(), 10) ||
fields["mtime"] != strconv.FormatInt(fi.ModTime().UnixNano(), 10) {
- // The binary was replaced after the marker was written -- almost certainly a local
- // rebuild, which is what we asked for. Fall through to the mtime comparison.
- return ""
+ // Replaced after the marker was written, so the marker's release says nothing about what
+ // is on disk now. Treated as a local rebuild, which the mtime comparison can judge.
+ return tag, provenanceStale
}
- return tag
+ return tag, provenanceTrusted
}
// collectorChangesSince lists collector-builder/ files that changed between a release tag and HEAD,
@@ -443,3 +486,18 @@ func downloadCollector(rel releaseAsset, dst string, log func(string, ...any)) e
return os.Rename(tmp, dst)
}
}
+
+// newestLocalReleaseTag returns the most recent v* tag reachable from HEAD.
+//
+// Used as a conservative reference point when a binary's origin is unrecorded: if the exporter has
+// not changed since the last release this clone knows about, then a binary of unknown origin cannot
+// be carrying stale exporter code, whatever its provenance. That keeps the unprovenanced warning
+// off the common local-build path while still catching the case that matters.
+func newestLocalReleaseTag(repoRoot string) (string, bool) {
+ out, err := exec.Command("git", "-C", repoRoot, "describe", "--tags", "--abbrev=0", "--match", "v*").Output()
+ if err != nil {
+ return "", false
+ }
+ tag := strings.TrimSpace(string(out))
+ return tag, tag != ""
+}
diff --git a/beacon-sandbox/image/image_test.go b/beacon-sandbox/image/image_test.go
index 19898d41..e67061f1 100644
--- a/beacon-sandbox/image/image_test.go
+++ b/beacon-sandbox/image/image_test.go
@@ -351,14 +351,14 @@ func TestProvenanceIsTrustedWhileItMatchesTheBinary(t *testing.T) {
if err := writeProvenance(root, "v1.0.6"); err != nil {
t.Fatal(err)
}
- if got := recordedRelease(root); got != "v1.0.6" {
- t.Errorf("an untouched binary must keep its provenance, got %q", got)
+ if got, state := recordedRelease(root); got != "v1.0.6" || state != provenanceTrusted {
+ t.Errorf("an untouched binary must keep its provenance, got %q state=%v", got, state)
}
// Replace the binary: the marker no longer describes it.
writeFile(t, CollectorPath(root), "something-else-entirely")
- if got := recordedRelease(root); got != "" {
- t.Errorf("a replaced binary must invalidate the marker, got %q", got)
+ if _, state := recordedRelease(root); state == provenanceTrusted {
+ t.Error("a replaced binary must not keep a trusted marker")
}
}
@@ -368,8 +368,8 @@ func TestLegacyProvenanceWithoutIdentityIsNotTrusted(t *testing.T) {
writeFile(t, CollectorPath(root), "binary")
writeFile(t, provenancePath(root), "release:v1.0.5\n")
- if got := recordedRelease(root); got != "" {
- t.Errorf("an unverifiable marker must not be trusted, got %q", got)
+ if _, state := recordedRelease(root); state == provenanceTrusted {
+ t.Error("an unverifiable marker must not be trusted")
}
}
@@ -510,3 +510,88 @@ func TestResolvableTagWithNoDriftStaysFresh(t *testing.T) {
t.Errorf("a resolvable tag at HEAD has no drift, got stale: %s", why)
}
}
+
+// The false green Bugbot reported: a downloaded release binary whose provenance is absent or no
+// longer describes it. CollectorIsStale then skipped release-drift detection and fell back to
+// comparing mtimes -- and a downloaded binary is essentially always newer than local sources, so
+// committed exporter changes since that release went unreported and the check went green while the
+// run verified the wrong beacon-otelcol.
+func TestUnprovenancedBinaryWithExporterDriftIsNotFresh(t *testing.T) {
+ root := gitRepo(t)
+ src := filepath.Join(root, "collector-builder", "exporter", "exp.go")
+ writeFile(t, src, "package exp\n")
+ commitAll(t, root, "base")
+ if out, err := exec.Command("git", "-C", root, "tag", "v1.0.6").CombinedOutput(); err != nil {
+ t.Fatalf("tag: %v: %s", err, out)
+ }
+ // Exporter changes land after that release.
+ writeFile(t, src, "package exp\n// the change under test\n")
+ commitAll(t, root, "change the exporter")
+
+ // A binary with no provenance at all, newer than the sources -- which is what a download
+ // looks like, and also what a local build looks like. Origin is genuinely unknown.
+ writeFile(t, CollectorPath(root), "binary-of-unknown-origin")
+ later := time.Now().Add(time.Minute)
+ if err := os.Chtimes(CollectorPath(root), later, later); err != nil {
+ t.Fatal(err)
+ }
+
+ stale, why := CollectorIsStale(root)
+ if !stale {
+ t.Fatal("exporter drift since the last release plus an unknown origin cannot be called fresh")
+ }
+ for _, want := range []string{"v1.0.6", "origin is unrecorded", "exp.go"} {
+ if !strings.Contains(why, want) {
+ t.Errorf("the reason should name the release, the gap, and the file; got %q", why)
+ }
+ }
+}
+
+// The narrowing that keeps this off the common path: no exporter changes since the last release
+// means an unprovenanced binary cannot be carrying stale exporter code, so no warning.
+func TestUnprovenancedBinaryWithNoDriftStaysQuiet(t *testing.T) {
+ root := gitRepo(t)
+ writeFile(t, filepath.Join(root, "collector-builder", "exporter", "exp.go"), "package exp\n")
+ commitAll(t, root, "base")
+ if out, err := exec.Command("git", "-C", root, "tag", "v1.0.6").CombinedOutput(); err != nil {
+ t.Fatalf("tag: %v: %s", err, out)
+ }
+ writeFile(t, CollectorPath(root), "binary-of-unknown-origin")
+ later := time.Now().Add(time.Minute)
+ if err := os.Chtimes(CollectorPath(root), later, later); err != nil {
+ t.Fatal(err)
+ }
+
+ if stale, why := CollectorIsStale(root); stale {
+ t.Errorf("no exporter drift since the release means nothing to warn about, got: %s", why)
+ }
+}
+
+// A provenance marker whose identity no longer matches is the documented local rebuild, so the
+// mtime comparison governs and a rebuilt-from-current-sources binary is fresh.
+func TestStaleProvenanceFallsBackToTheMtimeComparison(t *testing.T) {
+ root := gitRepo(t)
+ src := filepath.Join(root, "collector-builder", "exporter", "exp.go")
+ writeFile(t, src, "package exp\n")
+ commitAll(t, root, "base")
+ if out, err := exec.Command("git", "-C", root, "tag", "v1.0.6").CombinedOutput(); err != nil {
+ t.Fatalf("tag: %v: %s", err, out)
+ }
+ writeFile(t, src, "package exp\n// changed\n")
+ commitAll(t, root, "change the exporter")
+
+ writeFile(t, CollectorPath(root), "downloaded")
+ if err := writeProvenance(root, "v1.0.6"); err != nil {
+ t.Fatal(err)
+ }
+ // The documented rebuild replaces the binary, invalidating the marker's identity.
+ writeFile(t, CollectorPath(root), "rebuilt-locally-with-a-different-size")
+ later := time.Now().Add(time.Minute)
+ if err := os.Chtimes(CollectorPath(root), later, later); err != nil {
+ t.Fatal(err)
+ }
+
+ if stale, why := CollectorIsStale(root); stale {
+ t.Errorf("a local rebuild from current sources is fresh, got: %s", why)
+ }
+}
diff --git a/docs/contributing/beacon-sandbox.mdx b/docs/contributing/beacon-sandbox.mdx
index d3cf7212..a4f345f5 100644
--- a/docs/contributing/beacon-sandbox.mdx
+++ b/docs/contributing/beacon-sandbox.mdx
@@ -218,9 +218,7 @@ The mutation must turn a `PASS` into a `FAIL`. Available modes: `corrupt-line`,
Every mutation refuses to be a no-op — an empty log, or an action the log does not contain, is an error rather than a passing self-test that proved nothing.
-<Warning>
- `plant-secret` only works on a run captured with the `ANTHROPIC_API_KEY` currently set. It plants that key and expects the leak check to find it, but the leak check is withheld when the value is not the one the run used — a provider secret, a rotated key, or a run recorded before credential fingerprints existed. Rather than report a passing self-test that exercised nothing, it refuses and names the modes that do work on that run.
-</Warning>
+`plant-secret` needs no credential of its own: it plants a unique synthetic value and the leak check is told to search for that, so the check is exercised on any run directory whatever credential the run used — including none.
Two Beacon behaviors the matcher models deliberately, because ignoring either produces false failures: identical events written within a **two-second window** are collapsed by duplicate suppression, so a repeated command yields one event; and token totals must come from `beacon token-usage --json` rather than summing the raw JSONL, because overlapping-channel deduplication zeroes metric-channel usage when the log channel already reported it.