Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Have the sandbox verifier ask the writer about duplicates instead of modelling it

Claude15a43ae2199d
diff --git a/beacon-sandbox/check/check_test.go b/beacon-sandbox/check/check_test.go
index 8c287a50..4024c1bf 100644
--- a/beacon-sandbox/check/check_test.go
+++ b/beacon-sandbox/check/check_test.go
@@ -312,6 +312,35 @@ func TestMatchingCallIDsCollapseAtAnyDistance(t *testing.T) {
}
}
+// The hook and the collector describe one call in different words. The hook writes
+// "Shell command executed"; the collector writes "Tool execution observed". The writer's key does
+// not include the message, so it collapses them -- and the verifier has to agree, or it counts two
+// events where the log will only ever hold one. It did not agree while it kept its own copy of the
+// rule and keyed that copy on the message. Reported by Cursor Bugbot.
+func TestOneCallDescribedInDifferentWordsCollapses(t *testing.T) {
+ base := goodLog()
+ withIDAndMessage := func(line, ts, message string) string {
+ line = strings.Replace(line, "18:00:01Z", ts, 1)
+ line = strings.Replace(line, `"message":"claude_code.tool_result"`, `"message":"`+message+`"`, 1)
+ return strings.Replace(line, `"tool":{"name":"Bash"}`,
+ `"tool":{"name":"Bash"},"gen_ai":{"tool":{"call":{"id":"toolu_1"}}}`, 1)
+ }
+ hook := withIDAndMessage(base[1], "18:00:01Z", "Shell command executed")
+ collector := withIDAndMessage(base[1], "18:00:07Z", "Tool execution observed")
+
+ sc := demoScenario()
+ sc.Expect = []scenario.Expect{{
+ Action: "command.executed", Fields: []string{"command.command"}, MinCount: 2,
+ Why: "one call cannot be counted twice just because the two paths word it differently",
+ }}
+
+ v := run(t, []string{base[0], hook, collector}, sc, presentSentinel(), nil)
+ if v.Outcome != Fail {
+ t.Errorf("the two reports of one call share a call ID and must collapse to one, so "+
+ "min_count 2 has to fail:\n%s", v.Report())
+ }
+}
+
// Only a fixed set of actions is eligible for suppression at all. Treating others as
// collapsible would invent suppression the writer never performs.
func TestNonCandidateActionsAreNeverCollapsed(t *testing.T) {
diff --git a/beacon-sandbox/check/expect.go b/beacon-sandbox/check/expect.go
index 999577d5..92dd1003 100644
--- a/beacon-sandbox/check/expect.go
+++ b/beacon-sandbox/check/expect.go
@@ -4,17 +4,17 @@ import (
"fmt"
"sort"
"strings"
- "time"
"github.com/asymptote-labs/agent-beacon/beacon-sandbox/scenario"
+ obs "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)
-// DuplicateWindow mirrors IsDuplicateEndpointEvent's suppression window in the writer.
+// DuplicateWindow is the writer's own base suppression window, not a copy of it.
//
// Identical events written within this window are collapsed to one, so a scenario that runs
// the same command twice in quick succession legitimately yields a single event. Without
// modelling this, such a scenario would be a guaranteed false failure.
-const DuplicateWindow = 2 * time.Second
+const DuplicateWindow = obs.EndpointDuplicateWindow
// Sentinel describes whether the agent demonstrably performed the scenario's work.
type Sentinel struct {
@@ -217,63 +217,28 @@ func evaluate(v *Verdict, log Log, exp scenario.Expect, idx int, canary string)
})
}
-// collapseDuplicates removes matches the writer would have suppressed, so counting reflects what
-// Beacon can actually be expected to have recorded.
+// collapseDuplicates removes matches the writer would have suppressed, so counting reflects
+// what Beacon can actually be expected to have recorded.
//
-// This mirrors asymptoteobserve.IsDuplicateEndpointEvent rather than approximating it. An earlier
-// version collapsed any same-identity pair inside the window regardless of harness or call ID,
-// which over-collapsed badly: every event in a sandbox run carries harness claude_code, and the
-// real writer preserves same-harness repeats unless both events share an equal, non-empty tool
-// call ID -- because two adjacent calls can legitimately touch the same file or command. The
-// crude model therefore under-counted legitimate repeats and could fail a min_count the writer
-// would never have collapsed. Reported by Cursor Bugbot.
+// It asks the writer rather than modelling it. This used to be a hand-written mirror of
+// asymptoteobserve.IsDuplicateEndpointEvent, and it drifted twice: once by collapsing any
+// same-identity pair regardless of harness or call ID, which over-collapsed and could fail a
+// min_count the writer would never have suppressed (reported by Cursor Bugbot); and once by
+// keying on event.message, which the writer's key does not include -- so the hook and collector
+// reports of one call, whose messages differ, stayed two events here and one there. Both were
+// the same bug: a second copy of a rule, drifting.
//
-// Three rules matter, all taken from pkg/asymptoteobserve/dedupe.go:
-//
-// - only dedupeActions are candidates at all; every other action is preserved verbatim
-// - equal non-empty call IDs collapse on their own, with no window and regardless of harness;
-// otherwise a different harness collapses on the window alone and the same harness never does
-// - the window is 2s, except tool.completed which uses 10s
+// obs.DuplicateEndpointLines takes the raw lines, which is exactly what a verifier has, and
+// applies the writer's own comparison. There is nothing left here to drift.
func collapseDuplicates(events []Event) []Event {
if len(events) < 2 {
return events
}
var out, kept []Event
for _, e := range events {
- if !dedupeCandidate(e) {
- out = append(out, e)
- continue
- }
- ts, err := time.Parse(time.RFC3339, e.Typed.Timestamp)
- if err != nil {
- out = append(out, e)
- continue
- }
suppressed := false
for _, prev := range kept {
- if prev.Action() != e.Action() || identity(prev) != identity(e) {
- continue
- }
- // An equal, non-empty call ID settles it on its own: the writer applies no window
- // there, because the hook and the collector report one call seconds apart.
- if a, b := callIDOf(prev), callIDOf(e); a != "" && a == b {
- suppressed = true
- break
- }
- // Without that, the same harness never collapses: two adjacent calls can
- // legitimately touch the same file or command.
- if harnessOf(prev) == harnessOf(e) {
- continue
- }
- pts, err := time.Parse(time.RFC3339, prev.Typed.Timestamp)
- if err != nil {
- continue
- }
- diff := ts.Sub(pts)
- if diff < 0 {
- diff = -diff
- }
- if diff <= dedupeWindowFor(e.Action()) {
+ if obs.DuplicateEndpointLines([]byte(prev.Raw), []byte(e.Raw), DuplicateWindow) {
suppressed = true
break
}
@@ -287,65 +252,6 @@ func collapseDuplicates(events []Event) []Event {
return out
}
-// dedupeActions is the exact set the writer considers for suppression. Anything outside it is
-// never collapsed, so treating it as collapsible would invent suppression that does not happen.
-var dedupeActions = map[string]bool{
- "mcp.tool_invoked": true, "command.executed": true,
- "file.read": true, "file.modified": true,
- "tool.invoked": true, "tool.completed": true, "tool.failed": true,
-}
-
-// dedupeCandidate mirrors the writer's preconditions: a candidate action plus a session id, since
-// an event without one is never suppressed.
-func dedupeCandidate(e Event) bool {
- if !dedupeActions[e.Action()] {
- return false
- }
- return e.Typed.Session != nil && e.Typed.Session.ID != ""
-}
-
-// dedupeWindowFor mirrors endpointDedupeWindow: tool.completed gets a longer window because the
-// same completion can legitimately be reported twice further apart.
-func dedupeWindowFor(action string) time.Duration {
- if strings.EqualFold(action, "tool.completed") {
- return 10 * time.Second
- }
- return DuplicateWindow
-}
-
-func harnessOf(e Event) string {
- if v, ok := e.Field("harness.name"); ok {
- return strings.ToLower(fmt.Sprint(v))
- }
- return ""
-}
-
-func callIDOf(e Event) string {
- if v, ok := e.Field("gen_ai.tool.call.id"); ok {
- return fmt.Sprint(v)
- }
- return ""
-}
-
-// identity approximates what the writer considers "the same event", using the fields that
-// carry the payload rather than the whole line (which includes varying timestamps).
-func identity(e Event) string {
- parts := []string{e.Typed.Message}
- if e.Typed.Command != nil {
- parts = append(parts, e.Typed.Command.Command)
- }
- if e.Typed.Tool != nil {
- parts = append(parts, e.Typed.Tool.Name, e.Typed.Tool.Command)
- }
- if e.Typed.File != nil {
- parts = append(parts, e.Typed.File.Path)
- }
- if e.Typed.Prompt != nil {
- parts = append(parts, e.Typed.Prompt.Text)
- }
- return strings.Join(parts, "\x00")
-}
-
// sortedKeys keeps findings stable across runs, so a diff between two verdicts reflects
// real change rather than map iteration order.
func sortedKeys(m map[string]int) []string {
diff --git a/pkg/asymptoteobserve/dedupe.go b/pkg/asymptoteobserve/dedupe.go
index 3761ee4d..b08cebf2 100644
--- a/pkg/asymptoteobserve/dedupe.go
+++ b/pkg/asymptoteobserve/dedupe.go
@@ -39,44 +39,77 @@ func IsDuplicateEndpointEvent(path string, candidateLine []byte, window time.Dur
effectiveWindow := endpointDedupeWindow(candidate.action, window)
for _, line := range bytes.Split(data, []byte{'\n'}) {
existing, ok := endpointDedupeCandidate(line)
- if !ok || existing.key != candidate.key {
+ if !ok {
continue
}
- // An equal, non-empty tool call ID is the runtime's own statement that
- // these two lines describe one call, and it settles the question on its
- // own: no window, and no interest in which capture path reported it.
- //
- // The window is deliberately not applied here. A hook writes when the
- // tool runs and the collector writes when its batch flushes, so the two
- // reports of one call routinely land five seconds apart -- past the
- // two-second window that is the only thing that ever collapsed them.
- if existing.callID != "" && existing.callID == candidate.callID {
- return true
- }
- // Without matching IDs, only the timing heuristic is left, and only
- // across capture paths: two adjacent calls from one path can
- // legitimately touch the same file or command, so same-harness
- // lookalikes are preserved.
- //
- // Unequal IDs are not evidence of two calls unless one path assigned
- // both. Beacon's paths do not share an ID space -- OpenCode's own call
- // ID and the span ID the collector sees for the same call are simply
- // different names -- so across harnesses the IDs are ignored rather
- // than trusted to disagree.
- if existing.harness == candidate.harness {
- continue
- }
- diff := candidate.ts.Sub(existing.ts)
- if diff < 0 {
- diff = -diff
- }
- if diff <= effectiveWindow {
+ if duplicateEndpointEvents(existing, candidate, effectiveWindow) {
return true
}
}
return false
}
+// DuplicateEndpointLines reports whether candidate duplicates existing under the
+// same rules the writer applies when appending to a runtime log.
+//
+// Exported for readers that must agree with the writer about what one action is
+// -- the sandbox verifier, which counts captured events against a scenario's
+// expectations, and anything else that would otherwise keep a second copy of
+// these rules. A second copy is the thing this change exists to stop: two
+// implementations of one rule have already drifted apart here once, silently,
+// and a duplicated suppression rule is no safer than a duplicated harness name
+// was.
+//
+// window is the caller's base window; the action-specific floor still applies.
+func DuplicateEndpointLines(existing, candidate []byte, window time.Duration) bool {
+ parsedCandidate, ok := endpointDedupeCandidate(candidate)
+ if !ok {
+ return false
+ }
+ parsedExisting, ok := endpointDedupeCandidate(existing)
+ if !ok {
+ return false
+ }
+ return duplicateEndpointEvents(parsedExisting, parsedCandidate,
+ endpointDedupeWindow(parsedCandidate.action, window))
+}
+
+// duplicateEndpointEvents is the one pairwise decision both entry points share,
+// kept on parsed events so the tail scan does not reparse the candidate for
+// every line it compares it against.
+func duplicateEndpointEvents(existing, candidate endpointDedupeEvent, effectiveWindow time.Duration) bool {
+ if existing.key != candidate.key {
+ return false
+ }
+ // An equal, non-empty tool call ID is the runtime's own statement that these
+ // two lines describe one call, and it settles the question on its own: no
+ // window, and no interest in which capture path reported it.
+ //
+ // The window is deliberately not applied here. A hook writes when the tool
+ // runs and the collector writes when its batch flushes, so the two reports
+ // of one call routinely land five seconds apart -- past the two-second
+ // window that is the only thing that ever collapsed them.
+ if existing.callID != "" && existing.callID == candidate.callID {
+ return true
+ }
+ // Without matching IDs, only the timing heuristic is left, and only across
+ // capture paths: two adjacent calls from one path can legitimately touch the
+ // same file or command, so same-harness lookalikes are preserved.
+ //
+ // Unequal IDs are not evidence of two calls unless one path assigned both.
+ // Beacon's paths do not share an ID space -- OpenCode's own call ID and the
+ // span ID the collector sees for the same call are simply different names --
+ // so across harnesses the IDs are ignored rather than trusted to disagree.
+ if existing.harness == candidate.harness {
+ return false
+ }
+ diff := candidate.ts.Sub(existing.ts)
+ if diff < 0 {
+ diff = -diff
+ }
+ return diff <= effectiveWindow
+}
+
func readEndpointDedupeTail(path string) ([]byte, error) {
f, err := os.Open(path)
if err != nil {