The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Have the sandbox verifier ask the writer about duplicates instead of modelling it
Claude15a43ae2199d
diff --git a/beacon-sandbox/check/check_test.go b/beacon-sandbox/check/check_test.goindex 8c287a50..4024c1bf 100644--- a/beacon-sandbox/check/check_test.go+++ b/beacon-sandbox/check/check_test.go@@ -312,6 +312,35 @@ func TestMatchingCallIDsCollapseAtAnyDistance(t *testing.T) {}}+// The hook and the collector describe one call in different words. The hook writes+// "Shell command executed"; the collector writes "Tool execution observed". The writer's key does+// not include the message, so it collapses them -- and the verifier has to agree, or it counts two+// events where the log will only ever hold one. It did not agree while it kept its own copy of the+// rule and keyed that copy on the message. Reported by Cursor Bugbot.+func TestOneCallDescribedInDifferentWordsCollapses(t *testing.T) {+ base := goodLog()+ withIDAndMessage := func(line, ts, message string) string {+ line = strings.Replace(line, "18:00:01Z", ts, 1)+ line = strings.Replace(line, `"message":"claude_code.tool_result"`, `"message":"`+message+`"`, 1)+ return strings.Replace(line, `"tool":{"name":"Bash"}`,+ `"tool":{"name":"Bash"},"gen_ai":{"tool":{"call":{"id":"toolu_1"}}}`, 1)+ }+ hook := withIDAndMessage(base[1], "18:00:01Z", "Shell command executed")+ collector := withIDAndMessage(base[1], "18:00:07Z", "Tool execution observed")++ sc := demoScenario()+ sc.Expect = []scenario.Expect{{+ Action: "command.executed", Fields: []string{"command.command"}, MinCount: 2,+ Why: "one call cannot be counted twice just because the two paths word it differently",+ }}++ v := run(t, []string{base[0], hook, collector}, sc, presentSentinel(), nil)+ if v.Outcome != Fail {+ t.Errorf("the two reports of one call share a call ID and must collapse to one, so "++ "min_count 2 has to fail:\n%s", v.Report())+ }+}+// Only a fixed set of actions is eligible for suppression at all. Treating others as// collapsible would invent suppression the writer never performs.func TestNonCandidateActionsAreNeverCollapsed(t *testing.T) {diff --git a/beacon-sandbox/check/expect.go b/beacon-sandbox/check/expect.goindex 999577d5..92dd1003 100644--- a/beacon-sandbox/check/expect.go+++ b/beacon-sandbox/check/expect.go@@ -4,17 +4,17 @@ import ("fmt""sort""strings"- "time""github.com/asymptote-labs/agent-beacon/beacon-sandbox/scenario"+ obs "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve")-// DuplicateWindow mirrors IsDuplicateEndpointEvent's suppression window in the writer.+// DuplicateWindow is the writer's own base suppression window, not a copy of it.//// Identical events written within this window are collapsed to one, so a scenario that runs// the same command twice in quick succession legitimately yields a single event. Without// modelling this, such a scenario would be a guaranteed false failure.-const DuplicateWindow = 2 * time.Second+const DuplicateWindow = obs.EndpointDuplicateWindow// Sentinel describes whether the agent demonstrably performed the scenario's work.type Sentinel struct {@@ -217,63 +217,28 @@ func evaluate(v *Verdict, log Log, exp scenario.Expect, idx int, canary string)})}-// collapseDuplicates removes matches the writer would have suppressed, so counting reflects what-// Beacon can actually be expected to have recorded.+// collapseDuplicates removes matches the writer would have suppressed, so counting reflects+// what Beacon can actually be expected to have recorded.//-// This mirrors asymptoteobserve.IsDuplicateEndpointEvent rather than approximating it. An earlier-// version collapsed any same-identity pair inside the window regardless of harness or call ID,-// which over-collapsed badly: every event in a sandbox run carries harness claude_code, and the-// real writer preserves same-harness repeats unless both events share an equal, non-empty tool-// call ID -- because two adjacent calls can legitimately touch the same file or command. The-// crude model therefore under-counted legitimate repeats and could fail a min_count the writer-// would never have collapsed. Reported by Cursor Bugbot.+// It asks the writer rather than modelling it. This used to be a hand-written mirror of+// asymptoteobserve.IsDuplicateEndpointEvent, and it drifted twice: once by collapsing any+// same-identity pair regardless of harness or call ID, which over-collapsed and could fail a+// min_count the writer would never have suppressed (reported by Cursor Bugbot); and once by+// keying on event.message, which the writer's key does not include -- so the hook and collector+// reports of one call, whose messages differ, stayed two events here and one there. Both were+// the same bug: a second copy of a rule, drifting.//-// Three rules matter, all taken from pkg/asymptoteobserve/dedupe.go:-//-// - only dedupeActions are candidates at all; every other action is preserved verbatim-// - equal non-empty call IDs collapse on their own, with no window and regardless of harness;-// otherwise a different harness collapses on the window alone and the same harness never does-// - the window is 2s, except tool.completed which uses 10s+// obs.DuplicateEndpointLines takes the raw lines, which is exactly what a verifier has, and+// applies the writer's own comparison. There is nothing left here to drift.func collapseDuplicates(events []Event) []Event {if len(events) < 2 {return events}var out, kept []Eventfor _, e := range events {- if !dedupeCandidate(e) {- out = append(out, e)- continue- }- ts, err := time.Parse(time.RFC3339, e.Typed.Timestamp)- if err != nil {- out = append(out, e)- continue- }suppressed := falsefor _, prev := range kept {- if prev.Action() != e.Action() || identity(prev) != identity(e) {- continue- }- // An equal, non-empty call ID settles it on its own: the writer applies no window- // there, because the hook and the collector report one call seconds apart.- if a, b := callIDOf(prev), callIDOf(e); a != "" && a == b {- suppressed = true- break- }- // Without that, the same harness never collapses: two adjacent calls can- // legitimately touch the same file or command.- if harnessOf(prev) == harnessOf(e) {- continue- }- pts, err := time.Parse(time.RFC3339, prev.Typed.Timestamp)- if err != nil {- continue- }- diff := ts.Sub(pts)- if diff < 0 {- diff = -diff- }- if diff <= dedupeWindowFor(e.Action()) {+ if obs.DuplicateEndpointLines([]byte(prev.Raw), []byte(e.Raw), DuplicateWindow) {suppressed = truebreak}@@ -287,65 +252,6 @@ func collapseDuplicates(events []Event) []Event {return out}-// dedupeActions is the exact set the writer considers for suppression. Anything outside it is-// never collapsed, so treating it as collapsible would invent suppression that does not happen.-var dedupeActions = map[string]bool{- "mcp.tool_invoked": true, "command.executed": true,- "file.read": true, "file.modified": true,- "tool.invoked": true, "tool.completed": true, "tool.failed": true,-}--// dedupeCandidate mirrors the writer's preconditions: a candidate action plus a session id, since-// an event without one is never suppressed.-func dedupeCandidate(e Event) bool {- if !dedupeActions[e.Action()] {- return false- }- return e.Typed.Session != nil && e.Typed.Session.ID != ""-}--// dedupeWindowFor mirrors endpointDedupeWindow: tool.completed gets a longer window because the-// same completion can legitimately be reported twice further apart.-func dedupeWindowFor(action string) time.Duration {- if strings.EqualFold(action, "tool.completed") {- return 10 * time.Second- }- return DuplicateWindow-}--func harnessOf(e Event) string {- if v, ok := e.Field("harness.name"); ok {- return strings.ToLower(fmt.Sprint(v))- }- return ""-}--func callIDOf(e Event) string {- if v, ok := e.Field("gen_ai.tool.call.id"); ok {- return fmt.Sprint(v)- }- return ""-}--// identity approximates what the writer considers "the same event", using the fields that-// carry the payload rather than the whole line (which includes varying timestamps).-func identity(e Event) string {- parts := []string{e.Typed.Message}- if e.Typed.Command != nil {- parts = append(parts, e.Typed.Command.Command)- }- if e.Typed.Tool != nil {- parts = append(parts, e.Typed.Tool.Name, e.Typed.Tool.Command)- }- if e.Typed.File != nil {- parts = append(parts, e.Typed.File.Path)- }- if e.Typed.Prompt != nil {- parts = append(parts, e.Typed.Prompt.Text)- }- return strings.Join(parts, "\x00")-}-// sortedKeys keeps findings stable across runs, so a diff between two verdicts reflects// real change rather than map iteration order.func sortedKeys(m map[string]int) []string {diff --git a/pkg/asymptoteobserve/dedupe.go b/pkg/asymptoteobserve/dedupe.goindex 3761ee4d..b08cebf2 100644--- a/pkg/asymptoteobserve/dedupe.go+++ b/pkg/asymptoteobserve/dedupe.go@@ -39,44 +39,77 @@ func IsDuplicateEndpointEvent(path string, candidateLine []byte, window time.DureffectiveWindow := endpointDedupeWindow(candidate.action, window)for _, line := range bytes.Split(data, []byte{'\n'}) {existing, ok := endpointDedupeCandidate(line)- if !ok || existing.key != candidate.key {+ if !ok {continue}- // An equal, non-empty tool call ID is the runtime's own statement that- // these two lines describe one call, and it settles the question on its- // own: no window, and no interest in which capture path reported it.- //- // The window is deliberately not applied here. A hook writes when the- // tool runs and the collector writes when its batch flushes, so the two- // reports of one call routinely land five seconds apart -- past the- // two-second window that is the only thing that ever collapsed them.- if existing.callID != "" && existing.callID == candidate.callID {- return true- }- // Without matching IDs, only the timing heuristic is left, and only- // across capture paths: two adjacent calls from one path can- // legitimately touch the same file or command, so same-harness- // lookalikes are preserved.- //- // Unequal IDs are not evidence of two calls unless one path assigned- // both. Beacon's paths do not share an ID space -- OpenCode's own call- // ID and the span ID the collector sees for the same call are simply- // different names -- so across harnesses the IDs are ignored rather- // than trusted to disagree.- if existing.harness == candidate.harness {- continue- }- diff := candidate.ts.Sub(existing.ts)- if diff < 0 {- diff = -diff- }- if diff <= effectiveWindow {+ if duplicateEndpointEvents(existing, candidate, effectiveWindow) {return true}}return false}+// DuplicateEndpointLines reports whether candidate duplicates existing under the+// same rules the writer applies when appending to a runtime log.+//+// Exported for readers that must agree with the writer about what one action is+// -- the sandbox verifier, which counts captured events against a scenario's+// expectations, and anything else that would otherwise keep a second copy of+// these rules. A second copy is the thing this change exists to stop: two+// implementations of one rule have already drifted apart here once, silently,+// and a duplicated suppression rule is no safer than a duplicated harness name+// was.+//+// window is the caller's base window; the action-specific floor still applies.+func DuplicateEndpointLines(existing, candidate []byte, window time.Duration) bool {+ parsedCandidate, ok := endpointDedupeCandidate(candidate)+ if !ok {+ return false+ }+ parsedExisting, ok := endpointDedupeCandidate(existing)+ if !ok {+ return false+ }+ return duplicateEndpointEvents(parsedExisting, parsedCandidate,+ endpointDedupeWindow(parsedCandidate.action, window))+}++// duplicateEndpointEvents is the one pairwise decision both entry points share,+// kept on parsed events so the tail scan does not reparse the candidate for+// every line it compares it against.+func duplicateEndpointEvents(existing, candidate endpointDedupeEvent, effectiveWindow time.Duration) bool {+ if existing.key != candidate.key {+ return false+ }+ // An equal, non-empty tool call ID is the runtime's own statement that these+ // two lines describe one call, and it settles the question on its own: no+ // window, and no interest in which capture path reported it.+ //+ // The window is deliberately not applied here. A hook writes when the tool+ // runs and the collector writes when its batch flushes, so the two reports+ // of one call routinely land five seconds apart -- past the two-second+ // window that is the only thing that ever collapsed them.+ if existing.callID != "" && existing.callID == candidate.callID {+ return true+ }+ // Without matching IDs, only the timing heuristic is left, and only across+ // capture paths: two adjacent calls from one path can legitimately touch the+ // same file or command, so same-harness lookalikes are preserved.+ //+ // Unequal IDs are not evidence of two calls unless one path assigned both.+ // Beacon's paths do not share an ID space -- OpenCode's own call ID and the+ // span ID the collector sees for the same call are simply different names --+ // so across harnesses the IDs are ignored rather than trusted to disagree.+ if existing.harness == candidate.harness {+ return false+ }+ diff := candidate.ts.Sub(existing.ts)+ if diff < 0 {+ diff = -diff+ }+ return diff <= effectiveWindow+}+func readEndpointDedupeTail(path string) ([]byte, error) {f, err := os.Open(path)if err != nil {