Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/copilot): reject intra-pass near-duplicate dream writes (#13392)

Nicholas Tindle83a6f9eb74c8
diff --git a/autogpt_platform/backend/backend/copilot/dream/orchestrator.py b/autogpt_platform/backend/backend/copilot/dream/orchestrator.py
index ec39f171e3..2f53c875aa 100644
--- a/autogpt_platform/backend/backend/copilot/dream/orchestrator.py
+++ b/autogpt_platform/backend/backend/copilot/dream/orchestrator.py
@@ -18,6 +18,7 @@ from __future__ import annotations
import asyncio
import logging
+import re
import uuid as uuidlib
from datetime import datetime, timezone
@@ -52,6 +53,7 @@ from .prompts import (
)
from .routing import ExecutionPath, resolve_dream_execution_path
from .schemas import (
+ ConsolidatedFact,
ConsolidationOutput,
DreamOperations,
DreamOperationsSnapshot,
@@ -96,6 +98,216 @@ def _resolve_lock_ttl(transport_is_local: bool) -> int:
return LOCAL_LOCK_TTL_SECONDS if transport_is_local else DEFAULT_LOCK_TTL_SECONDS
+# Intra-pass near-duplicate write rejection (#13387). The consolidate
+# prompt asks the model to merge near-duplicates, but a single pass still
+# emits the same fact phrased 2-3 ways ("Nick uses Terminus on iPhone for
+# CLI work" / "Nick wants Terminus on iPhone to show more ASCII"). This is
+# a CONSERVATIVE, deterministic backstop: it only collapses writes that are
+# near-identical by content-word overlap, keeping the longest (most
+# specific) of each cluster. It deliberately does NOT do semantic
+# (embedding) merging or compare against the user's EXISTING active facts —
+# both need real-data threshold tuning + edge-merge design and are tracked
+# as the follow-up P2 dedup pass. High threshold so distinct facts about
+# the same entity ("prefers Python" vs "prefers Rust") are never merged.
+_DEDUP_JACCARD_THRESHOLD = 0.7
+# A short fact whose content words are (nearly) all contained in a longer
+# fact is a duplicate the longer one subsumes — even when the longer fact's
+# extra detail drags Jaccard below the threshold ("Nick uses Terminus on
+# iPhone for CLI" ⊂ "…for CLI and wants more ASCII"). Set high (0.9): for
+# facts under ~10 words it effectively requires the shorter to be a strict
+# SUBSET of the longer, so a single distinguishing word keeps them apart
+# ("deployed the AUTH service" vs "deployed the BILLING service" → 4/5=0.8,
+# NOT merged). Only applied when the smaller fact has enough words to be
+# specific, so 1-2 word fragments don't spuriously match.
+_DEDUP_CONTAINMENT_THRESHOLD = 0.9
+_DEDUP_CONTAINMENT_MIN_TOKENS = 3
+# Facts with IDENTICAL content-word sets can still describe different
+# events when word order differs ("Alice introduced Bob to Carol" vs
+# "Alice introduced Carol to Bob"). Equal token sets therefore also
+# require agreement on adjacent-word bigrams before merging.
+_DEDUP_BIGRAM_THRESHOLD = 0.6
+# A negation marker present in one fact but not the other flips the
+# meaning ("uses vim" vs "never uses vim") — never merge across it.
+# The tokenizer splits contractions ("doesn't" → "doesn", "t"), so the
+# bare stems cover the apostrophe forms; a false veto only ever KEEPS
+# both facts, which is the conservative direction.
+_DEDUP_NEGATION_MARKERS = frozenset(
+ {
+ "no",
+ "not",
+ "never",
+ "none",
+ "nor",
+ "cannot",
+ "nt",
+ "dont",
+ "don",
+ "doesnt",
+ "doesn",
+ "didnt",
+ "didn",
+ "isnt",
+ "isn",
+ "wasnt",
+ "wasn",
+ "werent",
+ "weren",
+ "wont",
+ "cant",
+ "hasnt",
+ "hasn",
+ "havent",
+ "haven",
+ "hadnt",
+ "hadn",
+ "wouldnt",
+ "wouldn",
+ "shouldnt",
+ "shouldn",
+ "couldnt",
+ "couldn",
+ "arent",
+ "aren",
+ }
+)
+_DEDUP_STOPWORDS = frozenset(
+ {
+ "a",
+ "an",
+ "the",
+ "to",
+ "of",
+ "and",
+ "or",
+ "is",
+ "are",
+ "was",
+ "were",
+ "for",
+ "on",
+ "in",
+ "at",
+ "by",
+ "with",
+ "that",
+ "this",
+ "it",
+ "their",
+ "they",
+ "them",
+ "his",
+ "her",
+ "its",
+ "as",
+ "be",
+ "has",
+ "have",
+ "had",
+ "s",
+ "so",
+ "more",
+ "user",
+ "users",
+ }
+)
+
+
+def _content_tokens(content: str) -> frozenset[str]:
+ """Lowercased alphanumeric content words, minus a small stopword set.
+ ``user`` is a stopword because nearly every fact begins with it."""
+ return frozenset(
+ t
+ for t in re.findall(r"[a-z0-9]+", content.lower())
+ if t not in _DEDUP_STOPWORDS
+ )
+
+
+def _word_bigrams(content: str) -> frozenset[tuple[str, str]]:
+ """Adjacent word pairs over the full (unfiltered) word sequence — the
+ order-sensitive signal used when two facts' token sets are equal."""
+ words = re.findall(r"[a-z0-9]+", content.lower())
+ return frozenset(zip(words, words[1:]))
+
+
+def _near_duplicate(
+ a: frozenset[str],
+ b: frozenset[str],
+ a_bigrams: frozenset[tuple[str, str]],
+ b_bigrams: frozenset[tuple[str, str]],
+) -> bool:
+ """Two content-word sets are near-duplicate when their Jaccard overlap
+ is high, OR the smaller (sufficiently specific) set is nearly contained
+ in the larger one. A negation marker on only one side always vetoes;
+ identical sets additionally need word-order (bigram) agreement."""
+ if not a or not b:
+ return False
+ if _DEDUP_NEGATION_MARKERS & (a ^ b):
+ return False
+ if a == b:
+ bigram_union = len(a_bigrams | b_bigrams)
+ if not bigram_union:
+ return True
+ return len(a_bigrams & b_bigrams) / bigram_union >= _DEDUP_BIGRAM_THRESHOLD
+ inter = len(a & b)
+ union = len(a | b)
+ if union and inter / union >= _DEDUP_JACCARD_THRESHOLD:
+ return True
+ smaller = min(len(a), len(b))
+ if smaller < _DEDUP_CONTAINMENT_MIN_TOKENS:
+ return False
+ return inter / smaller >= _DEDUP_CONTAINMENT_THRESHOLD
+
+
+def _dedupe_near_duplicate_writes(
+ writes: list[ConsolidatedFact],
+) -> tuple[list[ConsolidatedFact], int]:
+ """Collapse near-duplicate writes, keeping the longest of each cluster.
+
+ Greedy by content length (desc) so the most specific phrasing survives;
+ output preserves the writes' original order. Writes are only compared
+ within the same ``scope`` (facts must never merge across scopes), and a
+ dropped write's ``source_episode_uuids`` are unioned into its cluster's
+ survivor so provenance is never lost. Returns (kept, dropped)."""
+ by_len_desc = sorted(
+ range(len(writes)), key=lambda i: len(writes[i].content), reverse=True
+ )
+ tokens = [_content_tokens(w.content) for w in writes]
+ bigrams = [_word_bigrams(w.content) for w in writes]
+ kept_idx: list[int] = []
+ absorbed: dict[int, list[str]] = {}
+ for i in by_len_desc:
+ survivor = next(
+ (
+ k
+ for k in kept_idx
+ if writes[k].scope == writes[i].scope
+ and _near_duplicate(tokens[i], tokens[k], bigrams[i], bigrams[k])
+ ),
+ None,
+ )
+ if survivor is None:
+ kept_idx.append(i)
+ absorbed[i] = []
+ else:
+ absorbed[survivor].extend(writes[i].source_episode_uuids)
+ kept = [_absorb_provenance(writes[i], absorbed[i]) for i in sorted(kept_idx)]
+ return kept, len(writes) - len(kept)
+
+
+def _absorb_provenance(
+ write: ConsolidatedFact, absorbed_uuids: list[str]
+) -> ConsolidatedFact:
+ """Union dropped near-duplicates' episode uuids into the survivor."""
+ extra = [
+ u for u in dict.fromkeys(absorbed_uuids) if u not in write.source_episode_uuids
+ ]
+ if not extra:
+ return write
+ return write.model_copy(
+ update={"source_episode_uuids": [*write.source_episode_uuids, *extra]}
+ )
+
+
def _clamp_operations(
ops: DreamOperations,
active_fact_count: int,
@@ -145,8 +357,18 @@ def _clamp_operations(
demotion_cap = 0
elif active_fact_count > 0:
demotion_cap = min(MAX_DEMOTIONS_PER_PASS, max(1, active_fact_count * 5 // 100))
+
+ # Collapse intra-pass near-duplicate writes before the cap slice so the
+ # cap counts distinct facts, not paraphrases of one (#13387).
+ writes, w_dropped = _dedupe_near_duplicate_writes(ops.writes)
+ if w_dropped:
+ logger.info(
+ "Dream clamp: collapsed %d near-duplicate write(s) into their "
+ "canonical (longest) phrasing",
+ w_dropped,
+ )
return DreamOperations(
- writes=ops.writes[:MAX_WRITES_PER_PASS],
+ writes=writes[:MAX_WRITES_PER_PASS],
proposals=ops.proposals[:MAX_PROPOSALS_PER_PASS],
demotions=demotions[:demotion_cap],
entity_invalidations=ops.entity_invalidations[
diff --git a/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py b/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py
index 6c38a9ac09..9e4539bbe6 100644
--- a/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/orchestrator_test.py
@@ -698,3 +698,150 @@ async def test_partial_failure_still_charges_completed_phases(mocker):
_ = MagicMock # keep import for editor convenience; not directly used
+
+
+class TestNearDuplicateWriteDedup:
+ """#13387: a single pass that emits the same fact phrased multiple ways
+ is collapsed to the longest (canonical) phrasing; genuinely distinct
+ facts — even about the same entity — are preserved."""
+
+ def test_collapses_near_identical_phrasings_keeping_longest(self):
+ writes = [
+ ConsolidatedFact(
+ content="Nick uses Terminus on his iPhone for CLI work",
+ confidence=0.6,
+ ),
+ ConsolidatedFact(
+ content=(
+ "Nick uses Terminus on his iPhone for CLI work and wants "
+ "it to display more ASCII characters"
+ ),
+ confidence=0.7,
+ ),
+ ]
+ kept, dropped = orchestrator_mod._dedupe_near_duplicate_writes(writes)
+ assert dropped == 1
+ assert len(kept) == 1
+ # The longer, more specific phrasing survives.
+ assert "more ASCII characters" in kept[0].content
+
+ def test_keeps_distinct_facts_about_same_entity(self):
+ writes = [
+ ConsolidatedFact(content="Nick prefers Python for backend", confidence=0.8),
+ ConsolidatedFact(
+ content="Nick prefers Rust for systems work", confidence=0.8
+ ),
+ ]
+ kept, dropped = orchestrator_mod._dedupe_near_duplicate_writes(writes)
+ assert dropped == 0
+ assert len(kept) == 2
+
+ def test_one_distinguishing_word_is_not_a_duplicate(self):
+ """Containment guard: two facts differing by a single key word
+ (auth vs billing) must NOT be merged despite high overlap."""
+ writes = [
+ ConsolidatedFact(
+ content="Nick deployed the auth service to prod", confidence=0.7
+ ),
+ ConsolidatedFact(
+ content="Nick deployed the billing service to prod", confidence=0.7
+ ),
+ ]
+ kept, dropped = orchestrator_mod._dedupe_near_duplicate_writes(writes)
+ assert dropped == 0
+ assert len(kept) == 2
+
+ def test_preserves_original_order_of_survivors(self):
+ writes = [
+ ConsolidatedFact(content="Alpha fact about onboarding", confidence=0.5),
+ ConsolidatedFact(content="Beta fact about billing", confidence=0.5),
+ ConsolidatedFact(content="Gamma fact about deploys", confidence=0.5),
+ ]
+ kept, dropped = orchestrator_mod._dedupe_near_duplicate_writes(writes)
+ assert dropped == 0
+ assert [w.content for w in kept] == [
+ "Alpha fact about onboarding",
+ "Beta fact about billing",
+ "Gamma fact about deploys",
+ ]
+
+ def test_identical_content_in_different_scopes_is_kept(self):
+ writes = [
+ ConsolidatedFact(
+ content="Nick uses Terminus on his iPhone for CLI work",
+ scope="real:global",
+ confidence=0.7,
+ ),
+ ConsolidatedFact(
+ content="Nick uses Terminus on his iPhone for CLI work",
+ scope="project:foo",
+ confidence=0.7,
+ ),
+ ]
+ kept, dropped = orchestrator_mod._dedupe_near_duplicate_writes(writes)
+ assert dropped == 0
+ assert len(kept) == 2
+
+ def test_survivor_absorbs_dropped_writes_provenance(self):
+ writes = [
+ ConsolidatedFact(
+ content="Nick uses Terminus on his iPhone for CLI work",
+ confidence=0.6,
+ source_episode_uuids=["ep-1", "ep-2"],
+ ),
+ ConsolidatedFact(
+ content=(
+ "Nick uses Terminus on his iPhone for CLI work and wants "
+ "it to display more ASCII characters"
+ ),
+ confidence=0.7,
+ source_episode_uuids=["ep-2", "ep-3"],
+ ),
+ ]
+ kept, dropped = orchestrator_mod._dedupe_near_duplicate_writes(writes)
+ assert dropped == 1
+ assert len(kept) == 1
+ # Survivor keeps its own uuids first, then the absorbed extras.
+ assert kept[0].source_episode_uuids == ["ep-2", "ep-3", "ep-1"]
+
+ def test_word_order_permutation_is_not_merged(self):
+ writes = [
+ ConsolidatedFact(content="Alice introduced Bob to Carol", confidence=0.7),
+ ConsolidatedFact(content="Alice introduced Carol to Bob", confidence=0.7),
+ ]
+ kept, dropped = orchestrator_mod._dedupe_near_duplicate_writes(writes)
+ assert dropped == 0
+ assert len(kept) == 2
+
+ def test_negated_fact_is_not_merged(self):
+ writes = [
+ ConsolidatedFact(content="Nick uses vim", confidence=0.7),
+ ConsolidatedFact(content="Nick never uses vim", confidence=0.7),
+ ]
+ kept, dropped = orchestrator_mod._dedupe_near_duplicate_writes(writes)
+ assert dropped == 0
+ assert len(kept) == 2
+
+ def test_clamp_collapses_duplicate_writes(self):
+ ops = DreamOperations(
+ writes=[
+ ConsolidatedFact(
+ content="Churn rate rose sharply in Q2", confidence=0.6
+ ),
+ ConsolidatedFact(
+ content="The churn rate rose sharply during Q2 this year",
+ confidence=0.7,
+ ),
+ ConsolidatedFact(
+ content="Revenue grew 12 percent in Q2", confidence=0.8
+ ),
+ ],
+ proposals=[],
+ summary_for_user="ok",
+ )
+ clamped = orchestrator_mod._clamp_operations(ops, active_fact_count=50)
+ contents = [w.content for w in clamped.writes]
+ # The two churn paraphrases collapse to one; revenue fact untouched.
+ assert len(contents) == 2
+ assert any("Revenue grew" in c for c in contents)
+ assert sum("churn rate rose" in c.lower() for c in contents) == 1
diff --git a/autogpt_platform/backend/backend/copilot/dream/prompts.py b/autogpt_platform/backend/backend/copilot/dream/prompts.py
index e13d13f21e..e442639fdf 100644
--- a/autogpt_platform/backend/backend/copilot/dream/prompts.py
+++ b/autogpt_platform/backend/backend/copilot/dream/prompts.py
@@ -90,7 +90,15 @@ CONSOLIDATE_SYSTEM = (
"that merge near-duplicates into a canonical form.\n\n"
"RULES:\n"
" * Do not invent new information that isn't present in the inputs.\n"
+ " * Emit ONE statement per distinct fact. If two recent episodes "
+ "express the same fact with different words, MERGE them into a single "
+ "canonical statement — never emit both phrasings. Paraphrases, "
+ "restatements, and partial fragments of the same fact are duplicates.\n"
" * Prefer the longer, more specific phrasing when merging duplicates.\n"
+ " * Do NOT restate a fact that already appears in the active-facts "
+ "list unchanged — consolidation is for merging RECENT episodes into "
+ "canonical form, not for re-emitting memory that already exists. Emit "
+ "a fact only if it is new or meaningfully refines an existing one.\n"
" * Group by scope — never merge a fact in 'project:foo' with one in "
"'real:global'.\n"
" * Carry forward the source episode uuids exactly. Never make uuids up.\n"
diff --git a/autogpt_platform/backend/backend/copilot/dream/prompts_test.py b/autogpt_platform/backend/backend/copilot/dream/prompts_test.py
index c756e39157..3f1dc8881c 100644
--- a/autogpt_platform/backend/backend/copilot/dream/prompts_test.py
+++ b/autogpt_platform/backend/backend/copilot/dream/prompts_test.py
@@ -87,6 +87,16 @@ def test_consolidate_prompt_has_system_and_user_messages():
assert "[scope=project:x]" in user_body
+def test_consolidate_prompt_instructs_merge_and_no_restate():
+ """#13387: the consolidate prompt must tell the model to merge
+ paraphrases into one statement and not re-emit existing active facts."""
+ sys = build_consolidate_prompt(_build_bundle())[0]["content"].lower()
+ assert "one statement per distinct fact" in sys
+ assert "paraphrases" in sys
+ # No-restate-existing rule present.
+ assert "already appears in the active-facts" in sys
+
+
def test_recombine_prompt_includes_consolidated_output_and_active_facts():
bundle = _build_bundle()
consolidated_json = '{"facts": [{"content": "consolidated"}]}'