The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Harden Devin connector dedup: partial-failure retry and message-id collisions
shukieshah61c56bf42595
diff --git a/cli/beacon/internal/devincloud/mapper.go b/cli/beacon/internal/devincloud/mapper.goindex a4e74fa4..497490cc 100644--- a/cli/beacon/internal/devincloud/mapper.go+++ b/cli/beacon/internal/devincloud/mapper.go@@ -1,6 +1,7 @@package devincloudimport (+ "fmt""strings""time"@@ -67,24 +68,40 @@ func MapSession(s Session, msgs []Message) []MappedEvent {Event: sessionLifecycleEvent(s, "session.started", "Devin Cloud session started", s.CreatedAt),}}- for _, m := range msgs {+ seen := map[string]bool{}+ for i, m := range msgs {+ dedupID := messageDedupID(s.SessionID, m, i, seen)switch strings.ToLower(m.Source) {case "user":ev := baseEvent(s, "prompt.submitted", "prompt", schema.SeverityInfo, m.CreatedAt)ev.Prompt = &schema.PromptInfo{Text: m.Message}ev.Content = &schema.ContentInfo{Retention: schema.ContentRetentionFull, Included: true}ev.Message = "Devin Cloud user prompt"- out = append(out, MappedEvent{DedupID: m.EventID, Event: ev})+ out = append(out, MappedEvent{DedupID: dedupID, Event: ev})default: // "devin" (assistant) and any future agent-side sourceev := baseEvent(s, "agent.message", "session", schema.SeverityInfo, m.CreatedAt)ev.Content = &schema.ContentInfo{Retention: schema.ContentRetentionFull, Included: true}ev.Message = m.Message- out = append(out, MappedEvent{DedupID: m.EventID, Event: ev})+ out = append(out, MappedEvent{DedupID: dedupID, Event: ev})}}return out}+// messageDedupID returns a stable, unique dedup id for a message. It prefers the+// Devin event_id, but synthesizes one from the session, index, and created_at+// when event_id is empty or duplicated — otherwise messages sharing (or missing)+// an event_id would be silently dropped by the dedup set. Messages are+// append-only and ordered, so the synthesized index is stable across re-polls.+func messageDedupID(sessionID string, m Message, index int, seen map[string]bool) string {+ id := strings.TrimSpace(m.EventID)+ if id == "" || seen[id] {+ id = fmt.Sprintf("%s:msg:%d:%d", sessionID, index, m.CreatedAt)+ }+ seen[id] = true+ return id+}+// EndedEvent builds the session.ended event for a terminal session, carrying// final status/PR/ACU/duration metadata.func EndedEvent(s Session) schema.Event {diff --git a/cli/beacon/internal/devincloud/mapper_test.go b/cli/beacon/internal/devincloud/mapper_test.goindex d829931c..8ca77ebf 100644--- a/cli/beacon/internal/devincloud/mapper_test.go+++ b/cli/beacon/internal/devincloud/mapper_test.go@@ -92,6 +92,28 @@ func TestEndedEventCarriesMetadata(t *testing.T) {}}+func TestMessageDedupIDHandlesEmptyAndDuplicate(t *testing.T) {+ // Empty and duplicate event_ids must still yield unique dedup ids, or the+ // dedup set would silently drop messages.+ msgs := []Message{+ {EventID: "", Source: "user", Message: "a", CreatedAt: 10},+ {EventID: "", Source: "devin", Message: "b", CreatedAt: 11},+ {EventID: "dup", Source: "user", Message: "c", CreatedAt: 12},+ {EventID: "dup", Source: "devin", Message: "d", CreatedAt: 13},+ }+ mapped := MapSession(Session{SessionID: "s", Status: "working", CreatedAt: 5}, msgs)+ if len(mapped) != 5 { // started + 4 messages+ t.Fatalf("got %d events, want 5", len(mapped))+ }+ ids := map[string]bool{}+ for _, me := range mapped {+ if ids[me.DedupID] {+ t.Fatalf("duplicate dedup id %q would drop a message", me.DedupID)+ }+ ids[me.DedupID] = true+ }+}+func TestMapSessionHasNoEndedRegardlessOfStatus(t *testing.T) {for _, st := range []string{"working", "suspended", "finished"} {mapped := MapSession(Session{SessionID: "s", Status: st, CreatedAt: 10}, nil)diff --git a/cli/beacon/internal/devincloud/pull.go b/cli/beacon/internal/devincloud/pull.goindex c8f509e3..6dbf8dfc 100644--- a/cli/beacon/internal/devincloud/pull.go+++ b/cli/beacon/internal/devincloud/pull.go@@ -82,10 +82,14 @@ func PullOnce(ctx context.Context, client *Client, opts PullOptions) (sum Summar// earlier run and is now enabled) is never skipped on the upload account.if !opts.ForceRefresh {uploadSatisfied := opts.Upload == nil || ss.UploadedAt == s.UpdatedAt+ // SyncedAt == updated_at means a prior sweep FULLY processed this+ // exact cursor; a partial failure leaves SyncedAt unadvanced so the+ // session is retried rather than treated as complete.+ fullySynced := ss.SyncedAt == s.UpdatedAt && ss.Status == s.Statusif ss.Done && uploadSatisfied {continue}- if _, seen := state.Sessions[s.SessionID]; seen && ss.UpdatedAt == s.UpdatedAt && ss.Status == s.Status && len(ss.Emitted) > 0 && uploadSatisfied {+ if fullySynced && uploadSatisfied {continue}}@@ -109,6 +113,10 @@ func PullOnce(ctx context.Context, client *Client, opts PullOptions) (sum Summar// Status / Done) is advanced only on full success, so a partial failure causes// a retry on the next sweep.func processSession(ctx context.Context, client *Client, s Session, ss *SessionState, opts PullOptions, sum *Summary) error {+ // Mark this session incomplete until the whole sweep succeeds, so any early+ // return below leaves it eligible for retry on the next sweep.+ ss.SyncedAt = 0+msgs, err := client.SessionMessages(ctx, s.SessionID)if err != nil {return fmt.Errorf("messages: %w", err)@@ -169,6 +177,7 @@ func processSession(ctx context.Context, client *Client, s Session, ss *SessionSss.UpdatedAt = s.UpdatedAtss.Status = s.Status+ ss.SyncedAt = s.UpdatedAt // fully processed this cursor// A final session (finished/expired) can never resume, so stop polling it.// Suspended stays pollable in case it resumes.if IsFinal(s.Status) {diff --git a/cli/beacon/internal/devincloud/pull_test.go b/cli/beacon/internal/devincloud/pull_test.goindex f9e2adcd..1257dca5 100644--- a/cli/beacon/internal/devincloud/pull_test.go+++ b/cli/beacon/internal/devincloud/pull_test.go@@ -59,6 +59,56 @@ func TestUploadEnabledLaterStillUploads(t *testing.T) {}}+type flakyUploader struct {+ failFirst int+ calls int+}++func (f *flakyUploader) Upload(_ context.Context, _ string, _ []byte) error {+ f.calls+++ if f.calls <= f.failFirst {+ return fmt.Errorf("transient upload failure")+ }+ return nil+}++func TestPartialFailureRetriedThenSkipped(t *testing.T) {+ dir := t.TempDir()+ logPath := filepath.Join(dir, "runtime.jsonl")+ statePath := filepath.Join(dir, "state.json")+ client := pullTestServer(t) // one suspended session, one user message+ up := &flakyUploader{failFirst: 1}+ opts := PullOptions{Write: true, LogPath: logPath, StatePath: statePath, Upload: up, UploadPrefix: "agent-traces"}++ // Sweep 1: local emit succeeds, upload fails -> session left unsynced.+ if _, err := PullOnce(context.Background(), client, opts); err == nil {+ t.Fatal("expected upload error on first sweep")+ }+ lines1 := len(nonEmptyLines(t, logPath))++ // Sweep 2: not skipped (SyncedAt never advanced); reprocess + upload succeeds,+ // with no duplicate local lines.+ sum2, err := PullOnce(context.Background(), client, opts)+ if err != nil {+ t.Fatalf("second sweep: %v", err)+ }+ if sum2.SessionsChanged != 1 || sum2.Uploaded != 1 {+ t.Fatalf("partial-failed session should retry+upload, got %+v", sum2)+ }+ if got := len(nonEmptyLines(t, logPath)); got != lines1 {+ t.Fatalf("retry duplicated local lines: %d -> %d", lines1, got)+ }++ // Sweep 3: now fully synced and uploaded -> skipped.+ sum3, err := PullOnce(context.Background(), client, opts)+ if err != nil {+ t.Fatalf("third sweep: %v", err)+ }+ if sum3.SessionsChanged != 0 {+ t.Fatalf("fully synced session should be skipped, got %+v", sum3)+ }+}+func pullTestServer(t *testing.T) *Client {srv := fakeDevinAPI(t, `{"items":[{"session_id":"s1","status":"suspended","user_id":"user-1","created_at":1000,"updated_at":1600,"acus_consumed":1}diff --git a/cli/beacon/internal/devincloud/state.go b/cli/beacon/internal/devincloud/state.goindex 52c473a4..5e9d713d 100644--- a/cli/beacon/internal/devincloud/state.go+++ b/cli/beacon/internal/devincloud/state.go@@ -16,6 +16,7 @@ type State struct {type SessionState struct {UpdatedAt int64 `json:"updated_at"`Status string `json:"status"`+ SyncedAt int64 `json:"synced_at"` // updated_at at which a sweep last FULLY processed this session; 0 while incompleteDone bool `json:"done"` // final status; skip henceforthEndedEmitted bool `json:"ended_emitted"` // session.ended emitted for the current terminal episodeUploadedAt int64 `json:"uploaded_at"` // updated_at at which this session's snapshot was last uploaded to GCS