AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(copilot): deduplicate SSE-replayed messages by content fingerprint (#12759)
Zamil Majdyb2b6f754200f
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/helpers.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/helpers.test.tsindex a7919430d2..91e09efde3 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/helpers.test.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/helpers.test.ts@@ -2,6 +2,7 @@ import type { UIMessage } from "ai";import { describe, expect, it } from "vitest";import {ORIGINAL_TITLE,+ deduplicateMessages,extractSendMessageText,formatNotificationTitle,getSendSuppressionReason,@@ -291,3 +292,177 @@ describe("getSendSuppressionReason", () => {).toBeNull();});});++// Helper that creates messages with explicit IDs for dedup tests+function makeMsgWithId(+ id: string,+ role: "user" | "assistant",+ text: string,+): UIMessage {+ return { id, role, parts: [{ type: "text", text }] };+}++describe("deduplicateMessages", () => {+ it("removes messages with duplicate IDs", () => {+ const msgs = [+ makeMsgWithId("1", "user", "hello"),+ makeMsgWithId("1", "user", "hello"),+ ];+ expect(deduplicateMessages(msgs)).toHaveLength(1);+ });++ it("removes non-adjacent assistant duplicates with different IDs (SSE replay)", () => {+ const msgs = [+ makeMsgWithId("u1", "user", "hello"),+ makeMsgWithId("a1", "assistant", "Plan of Attack"),+ makeMsgWithId("a2", "assistant", "Next step"),+ // SSE replay appends the same content with new IDs+ makeMsgWithId("a3", "assistant", "Plan of Attack"),+ makeMsgWithId("a4", "assistant", "Next step"),+ ];+ const result = deduplicateMessages(msgs);+ expect(result).toHaveLength(3); // user + 2 unique assistant msgs+ expect(result.map((m) => m.id)).toEqual(["u1", "a1", "a2"]);+ });++ it("keeps identical assistant replies to different user prompts", () => {+ const msgs = [+ makeMsgWithId("u1", "user", "What is 2+2?"),+ makeMsgWithId("a1", "assistant", "4"),+ makeMsgWithId("u2", "user", "What is 1+3?"),+ makeMsgWithId("a2", "assistant", "4"),+ ];+ const result = deduplicateMessages(msgs);+ expect(result).toHaveLength(4);+ });++ it("keeps second answer when same question is asked twice in one session", () => {+ // Regression: scoping by user message TEXT instead of ID would treat both+ // turns as the same context and drop the second identical assistant reply.+ const msgs = [+ makeMsgWithId("u1", "user", "What is 2+2?"),+ makeMsgWithId("a1", "assistant", "4"),+ makeMsgWithId("u2", "user", "What is 2+2?"), // same question, different ID+ makeMsgWithId("a2", "assistant", "4"), // same answer — must be kept+ ];+ const result = deduplicateMessages(msgs);+ expect(result).toHaveLength(4);+ expect(result.map((m) => m.id)).toEqual(["u1", "a1", "u2", "a2"]);+ });++ it("removes adjacent assistant duplicates", () => {+ const msgs = [+ makeMsgWithId("u1", "user", "hello"),+ makeMsgWithId("a1", "assistant", "hi there"),+ makeMsgWithId("a2", "assistant", "hi there"),+ ];+ const result = deduplicateMessages(msgs);+ expect(result).toHaveLength(2);+ });++ it("handles empty message list", () => {+ expect(deduplicateMessages([])).toEqual([]);+ });++ it("passes through unique messages unchanged", () => {+ const msgs = [+ makeMsgWithId("u1", "user", "question 1"),+ makeMsgWithId("a1", "assistant", "answer 1"),+ makeMsgWithId("u2", "user", "question 2"),+ makeMsgWithId("a2", "assistant", "answer 2"),+ ];+ expect(deduplicateMessages(msgs)).toHaveLength(4);+ });++ it("does not create false positives for text parts that contain the separator", () => {+ // "a|b" + "c" and "a" + "b|c" previously collided when joined with "|"+ const msgs: UIMessage[] = [+ makeMsgWithId("u1", "user", "hello"),+ {+ id: "a1",+ role: "assistant",+ parts: [+ { type: "text", text: "a|b" },+ { type: "text", text: "c" },+ ],+ },+ {+ id: "a2",+ role: "assistant",+ parts: [+ { type: "text", text: "a" },+ { type: "text", text: "b|c" },+ ],+ },+ ];+ const result = deduplicateMessages(msgs);+ expect(result).toHaveLength(3); // both assistant messages should be kept+ });++ it("deduplicates by toolCallId for tool-call parts", () => {+ const msgs: UIMessage[] = [+ makeMsgWithId("u1", "user", "run tool"),+ {+ id: "a1",+ role: "assistant",+ parts: [+ {+ type: "dynamic-tool",+ toolCallId: "tc-1",+ toolName: "test",+ state: "input-available",+ input: {},+ },+ ],+ },+ {+ id: "a2",+ role: "assistant",+ parts: [+ {+ type: "dynamic-tool",+ toolCallId: "tc-1",+ toolName: "test",+ state: "input-available",+ input: {},+ },+ ],+ },+ ];+ const result = deduplicateMessages(msgs);+ expect(result).toHaveLength(2); // user + first tool call+ });++ it("passes through assistant messages with empty parts without deduplicating them", () => {+ // contentFingerprint === "[]" when parts is empty; the guard skips fingerprint+ // tracking so these messages are never incorrectly deduplicated against each other.+ const msgs: UIMessage[] = [+ makeMsgWithId("u1", "user", "hello"),+ { id: "a1", role: "assistant", parts: [] },+ { id: "a2", role: "assistant", parts: [] },+ ];+ const result = deduplicateMessages(msgs);+ expect(result).toHaveLength(3); // both empty-parts messages are kept+ });++ it("does not collapse structurally different no-text parts to the same fingerprint", () => {+ // Parts lacking both 'text' and 'toolCallId' (e.g. step-start) previously+ // all mapped to "" causing false-positive deduplication. Now JSON.stringify(p)+ // is used as the fallback so distinct part shapes produce distinct fingerprints.+ const msgs: UIMessage[] = [+ makeMsgWithId("u1", "user", "hello"),+ {+ id: "a1",+ role: "assistant",+ parts: [{ type: "step-start" }],+ },+ {+ id: "a2",+ role: "assistant",+ parts: [{ type: "step-start" }],+ },+ ];+ const result = deduplicateMessages(msgs);+ expect(result).toHaveLength(2); // duplicate step-start messages are deduped+ });+});diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/helpers.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/helpers.tsindex 6462b72d27..66c437eb86 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/helpers.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/helpers.ts@@ -154,39 +154,63 @@ export function shouldSuppressDuplicateSend(}/**- * Deduplicate messages by ID and by consecutive content fingerprint.+ * Deduplicate messages by ID and by content fingerprint.** ID dedup catches exact duplicates within the same source.- * Content dedup only compares each assistant message to its **immediate- * predecessor** — this catches hydration/stream boundary duplicates (where- * the same content appears under different IDs) without accidentally- * removing legitimately repeated assistant responses that are far apart.+ * Content dedup uses a composite key of `role + preceding-user-message-id ++ * content-fingerprint` to detect replayed messages that arrive with new+ * IDs after an SSE reconnection replays from the beginning of the Redis+ * stream. Scoping by user message ID (not text) preserves the second+ * assistant reply when the user asks the same question twice and gets the+ * same answer — two different user messages produce two different IDs even+ * when their text is identical.*/export function deduplicateMessages(messages: UIMessage[]): UIMessage[] {const seenIds = new Set<string>();- let lastAssistantFingerprint = "";+ const seenFingerprints = new Set<string>();+ let lastUserMsgID = "";return messages.filter((msg) => {if (seenIds.has(msg.id)) return false;seenIds.add(msg.id);+ if (msg.role === "user") {+ // Track the ID (not text) of the latest user message so we can scope+ // assistant fingerprints to their conversational turn. Using the ID+ // means two user messages with identical text are still treated as+ // distinct turns, preventing false-positive deduplication.+ lastUserMsgID = msg.id;+ }+if (msg.role === "assistant") {- const fingerprint = msg.parts- .map(+ // JSON.stringify the parts array to avoid separator-collision false+ // positives: a plain join("|") on ["a|b", "c"] and ["a", "b|c"]+ // produces the same string. JSON encoding each element is unambiguous.+ // Fall back to JSON.stringify(p) for parts that carry neither a text nor+ // a toolCallId (e.g. step-start) so structurally different parts never+ // collapse to the same empty-string fingerprint element.+ const contentFingerprint = JSON.stringify(+ msg.parts.map((p) =>("text" in p && p.text) ||("toolCallId" in p && p.toolCallId) ||- "",- )- .join("|");-- // Only dedup if this assistant message is identical to the previous one- if (fingerprint && fingerprint === lastAssistantFingerprint) return false;- if (fingerprint) lastAssistantFingerprint = fingerprint;- } else {- // Reset on non-assistant messages so that identical assistant responses- // separated by a user message (e.g. "Done!" → user → "Done!") are kept.- lastAssistantFingerprint = "";+ JSON.stringify(p),+ ),+ );++ if (contentFingerprint !== "[]") {+ // Scope to the preceding user message turn so that identical assistant+ // replies to *different* user prompts are preserved.+ // NOTE: A streaming (in-progress) assistant message has a partial+ // fingerprint that differs from its final form, so it would not be+ // caught by this dedup. This is safe because every caller that invokes+ // resumeStream() first strips the in-progress assistant message —+ // handleReconnect, the wake-resync path, and the hydration-effect path+ // all do this. See useCopilotStream.ts.+ const contextKey = `assistant:${lastUserMsgID}:${contentFingerprint}`;+ if (seenFingerprints.has(contextKey)) return false;+ seenFingerprints.add(contextKey);+ }}return true;diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/useCopilotStream.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/useCopilotStream.tsindex 92f04d1e54..918047d3d8 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/useCopilotStream.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/useCopilotStream.ts@@ -147,6 +147,15 @@ export function useCopilotStream({reconnectTimerRef.current = setTimeout(() => {isReconnectScheduledRef.current = false;setIsReconnectScheduled(false);+ // Strip any stale in-progress assistant message before resuming.+ // The backend replays from "0-0", so the partial message would+ // otherwise sit alongside the fully-replayed version.+ setMessages((prev) => {+ if (prev.length > 0 && prev[prev.length - 1].role === "assistant") {+ return prev.slice(0, -1);+ }+ return prev;+ });resumeStream();}, delay);}