Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(sandbox): keep inconclusive comparisons neutral (#996)

Divyam Talwar1c1937d5fe48
diff --git a/beacon-sandbox/check/fingerprint.go b/beacon-sandbox/check/fingerprint.go
index 10ad9221..35e1dac4 100644
--- a/beacon-sandbox/check/fingerprint.go
+++ b/beacon-sandbox/check/fingerprint.go
@@ -108,6 +108,8 @@ func LoadFingerprint(path string) (Fingerprint, error) {
// FingerprintDiff is the comparison between two runs of the same scenario.
type FingerprintDiff struct {
Scenario string
+ // NotComparable distinguishes an unmeasured run from observed capability equivalence.
+ NotComparable bool
// Regressions are strictly-worse changes: a field or action that used to be captured
// and now is not. These are the reason this exists.
Regressions []string
@@ -128,6 +130,13 @@ func (d FingerprintDiff) HasRegression() bool { return len(d.Regressions) > 0 }
// explained by model nondeterminism.
func CompareFingerprints(before, after Fingerprint) FingerprintDiff {
d := FingerprintDiff{Scenario: after.Scenario}
+ // Partial artifacts from an inconclusive run are not a capability measurement, even when
+ // both runs are inconclusive. Decide this before classifying any presence or count changes.
+ if before.Outcome == Inconclusive || after.Outcome == Inconclusive {
+ d.NotComparable = true
+ d.Neutral = append(d.Neutral, fmt.Sprintf("outcome %s -> %s (not comparable: an inconclusive run measured nothing)", before.Outcome, after.Outcome))
+ return d
+ }
for _, f := range TrackedFields {
b, a := before.PopulatedFields[f], after.PopulatedFields[f]
@@ -174,12 +183,7 @@ func CompareFingerprints(before, after Fingerprint) FingerprintDiff {
case before.Outcome == Fail && after.Outcome == Pass:
d.Improvements = append(d.Improvements, msg)
default:
- // One side is INCONCLUSIVE, which means the agent never did the work, so that
- // run measured nothing. A measurement cannot be compared against a
- // non-measurement in either direction: treating the transition as an
- // improvement previously made INCONCLUSIVE -> FAIL read as progress, and
- // PASS -> INCONCLUSIVE hide a lost signal.
- d.Neutral = append(d.Neutral, msg+" (not comparable: one run was inconclusive)")
+ d.Neutral = append(d.Neutral, msg)
}
}
@@ -193,7 +197,9 @@ func CompareFingerprints(before, after Fingerprint) FingerprintDiff {
func (d FingerprintDiff) Describe() string {
var b strings.Builder
fmt.Fprintf(&b, "%s\n", d.Scenario)
- if len(d.Regressions) == 0 && len(d.Improvements) == 0 {
+ if d.NotComparable {
+ fmt.Fprintf(&b, " capabilities not compared\n")
+ } else if len(d.Regressions) == 0 && len(d.Improvements) == 0 {
fmt.Fprintf(&b, " no capability change\n")
}
for _, r := range d.Regressions {
diff --git a/beacon-sandbox/check/fingerprint_inconclusive_test.go b/beacon-sandbox/check/fingerprint_inconclusive_test.go
new file mode 100644
index 00000000..a2dfedd7
--- /dev/null
+++ b/beacon-sandbox/check/fingerprint_inconclusive_test.go
@@ -0,0 +1,53 @@
+package check
+
+import (
+ "strings"
+ "testing"
+)
+
+func measuredFingerprint(outcome Outcome, count int) Fingerprint {
+ return Fingerprint{
+ Scenario: "same-scenario",
+ Outcome: outcome,
+ Actions: map[string]int{"command.executed": count},
+ Harnesses: map[string]int{"claude_code": count},
+ PopulatedFields: map[string]int{"command.command": count},
+ EventCount: count,
+ }
+}
+
+func TestInconclusiveFingerprintsNeverClaimDirectionalChanges(t *testing.T) {
+ for _, tc := range []struct {
+ before Outcome
+ after Outcome
+ }{
+ {Pass, Inconclusive},
+ {Fail, Inconclusive},
+ {Inconclusive, Pass},
+ {Inconclusive, Fail},
+ {Inconclusive, Inconclusive},
+ } {
+ t.Run(string(tc.before)+" to "+string(tc.after), func(t *testing.T) {
+ before := measuredFingerprint(tc.before, 3)
+ after := Fingerprint{
+ Scenario: before.Scenario,
+ Outcome: tc.after,
+ Actions: map[string]int{"file.read": 1},
+ Harnesses: map[string]int{"copilot": 1},
+ PopulatedFields: map[string]int{"file.path": 1},
+ EventCount: 1,
+ }
+ d := CompareFingerprints(before, after)
+ if d.HasRegression() || len(d.Regressions) != 0 || len(d.Improvements) != 0 {
+ t.Errorf("an unmeasured run created directional evidence:\n%s", d.Describe())
+ }
+ if !strings.Contains(strings.Join(d.Neutral, "\n"), "not comparable") {
+ t.Errorf("comparison needs an explicit neutral explanation: %+v", d)
+ }
+ report := d.Describe()
+ if strings.Contains(report, "no capability change") || !strings.Contains(report, "capabilities not compared") {
+ t.Errorf("report must distinguish missing measurement from equivalence:\n%s", report)
+ }
+ })
+ }
+}