The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(sandbox): keep inconclusive comparisons neutral (#996)
Divyam Talwar1c1937d5fe48
diff --git a/beacon-sandbox/check/fingerprint.go b/beacon-sandbox/check/fingerprint.goindex 10ad9221..35e1dac4 100644--- a/beacon-sandbox/check/fingerprint.go+++ b/beacon-sandbox/check/fingerprint.go@@ -108,6 +108,8 @@ func LoadFingerprint(path string) (Fingerprint, error) {// FingerprintDiff is the comparison between two runs of the same scenario.type FingerprintDiff struct {Scenario string+ // NotComparable distinguishes an unmeasured run from observed capability equivalence.+ NotComparable bool// Regressions are strictly-worse changes: a field or action that used to be captured// and now is not. These are the reason this exists.Regressions []string@@ -128,6 +130,13 @@ func (d FingerprintDiff) HasRegression() bool { return len(d.Regressions) > 0 }// explained by model nondeterminism.func CompareFingerprints(before, after Fingerprint) FingerprintDiff {d := FingerprintDiff{Scenario: after.Scenario}+ // Partial artifacts from an inconclusive run are not a capability measurement, even when+ // both runs are inconclusive. Decide this before classifying any presence or count changes.+ if before.Outcome == Inconclusive || after.Outcome == Inconclusive {+ d.NotComparable = true+ d.Neutral = append(d.Neutral, fmt.Sprintf("outcome %s -> %s (not comparable: an inconclusive run measured nothing)", before.Outcome, after.Outcome))+ return d+ }for _, f := range TrackedFields {b, a := before.PopulatedFields[f], after.PopulatedFields[f]@@ -174,12 +183,7 @@ func CompareFingerprints(before, after Fingerprint) FingerprintDiff {case before.Outcome == Fail && after.Outcome == Pass:d.Improvements = append(d.Improvements, msg)default:- // One side is INCONCLUSIVE, which means the agent never did the work, so that- // run measured nothing. A measurement cannot be compared against a- // non-measurement in either direction: treating the transition as an- // improvement previously made INCONCLUSIVE -> FAIL read as progress, and- // PASS -> INCONCLUSIVE hide a lost signal.- d.Neutral = append(d.Neutral, msg+" (not comparable: one run was inconclusive)")+ d.Neutral = append(d.Neutral, msg)}}@@ -193,7 +197,9 @@ func CompareFingerprints(before, after Fingerprint) FingerprintDiff {func (d FingerprintDiff) Describe() string {var b strings.Builderfmt.Fprintf(&b, "%s\n", d.Scenario)- if len(d.Regressions) == 0 && len(d.Improvements) == 0 {+ if d.NotComparable {+ fmt.Fprintf(&b, " capabilities not compared\n")+ } else if len(d.Regressions) == 0 && len(d.Improvements) == 0 {fmt.Fprintf(&b, " no capability change\n")}for _, r := range d.Regressions {diff --git a/beacon-sandbox/check/fingerprint_inconclusive_test.go b/beacon-sandbox/check/fingerprint_inconclusive_test.gonew file mode 100644index 00000000..a2dfedd7--- /dev/null+++ b/beacon-sandbox/check/fingerprint_inconclusive_test.go@@ -0,0 +1,53 @@+package check++import (+ "strings"+ "testing"+)++func measuredFingerprint(outcome Outcome, count int) Fingerprint {+ return Fingerprint{+ Scenario: "same-scenario",+ Outcome: outcome,+ Actions: map[string]int{"command.executed": count},+ Harnesses: map[string]int{"claude_code": count},+ PopulatedFields: map[string]int{"command.command": count},+ EventCount: count,+ }+}++func TestInconclusiveFingerprintsNeverClaimDirectionalChanges(t *testing.T) {+ for _, tc := range []struct {+ before Outcome+ after Outcome+ }{+ {Pass, Inconclusive},+ {Fail, Inconclusive},+ {Inconclusive, Pass},+ {Inconclusive, Fail},+ {Inconclusive, Inconclusive},+ } {+ t.Run(string(tc.before)+" to "+string(tc.after), func(t *testing.T) {+ before := measuredFingerprint(tc.before, 3)+ after := Fingerprint{+ Scenario: before.Scenario,+ Outcome: tc.after,+ Actions: map[string]int{"file.read": 1},+ Harnesses: map[string]int{"copilot": 1},+ PopulatedFields: map[string]int{"file.path": 1},+ EventCount: 1,+ }+ d := CompareFingerprints(before, after)+ if d.HasRegression() || len(d.Regressions) != 0 || len(d.Improvements) != 0 {+ t.Errorf("an unmeasured run created directional evidence:\n%s", d.Describe())+ }+ if !strings.Contains(strings.Join(d.Neutral, "\n"), "not comparable") {+ t.Errorf("comparison needs an explicit neutral explanation: %+v", d)+ }+ report := d.Describe()+ if strings.Contains(report, "no capability change") || !strings.Contains(report, "capabilities not compared") {+ t.Errorf("report must distinguish missing measurement from equivalence:\n%s", report)+ }+ })+ }+}