Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(session): add batch-internal dedup to prevent duplicates within same commit (#687) (#701)

AstroHan1e475c2423c1
diff --git a/openviking/session/compressor.py b/openviking/session/compressor.py
index 243eba8b0..7fd066c7c 100644
--- a/openviking/session/compressor.py
+++ b/openviking/session/compressor.py
@@ -248,6 +248,8 @@ class SessionCompressor:
memories: List[Context] = []
stats = ExtractionStats()
+ # Track created memories' embeddings for batch-internal dedup (#687)
+ batch_memories: list[tuple[list[float], Context]] = []
viking_fs = get_viking_fs()
tool_parts = self._extract_tool_parts(messages)
@@ -323,7 +325,9 @@ class SessionCompressor:
continue
# Dedup check for other categories
- result = await self.deduplicator.deduplicate(candidate, ctx)
+ result = await self.deduplicator.deduplicate(
+ candidate, ctx, batch_memories=batch_memories
+ )
actions = result.actions or []
decision = result.decision
@@ -352,6 +356,10 @@ class SessionCompressor:
action.memory, viking_fs, ctx=ctx
):
stats.deleted += 1
+ # Remove deleted memory from batch tracking (#687)
+ batch_memories = [
+ (v, m) for v, m in batch_memories if m.uri != action.memory.uri
+ ]
else:
stats.skipped += 1
elif action.decision == MemoryActionDecision.MERGE:
@@ -360,6 +368,23 @@ class SessionCompressor:
candidate, action.memory, viking_fs, ctx=ctx
):
stats.merged += 1
+ # Remove stale batch entry and re-add with updated
+ # embedding so 3rd+ candidates can still find it (#687).
+ batch_memories = [
+ (v, m)
+ for v, m in batch_memories
+ if m.uri != action.memory.uri
+ ]
+ if self.deduplicator.embedder:
+ merged_text = (
+ f"{action.memory.abstract} {candidate.content}"
+ )
+ merged_embed = self.deduplicator.embedder.embed(
+ merged_text
+ )
+ batch_memories.append(
+ (merged_embed.dense_vector, action.memory)
+ )
else:
stats.skipped += 1
else:
@@ -375,6 +400,10 @@ class SessionCompressor:
action.memory, viking_fs, ctx=ctx
):
stats.deleted += 1
+ # Remove deleted memory from batch tracking (#687)
+ batch_memories = [
+ (v, m) for v, m in batch_memories if m.uri != action.memory.uri
+ ]
else:
stats.skipped += 1
@@ -385,6 +414,9 @@ class SessionCompressor:
memories.append(memory)
stats.created += 1
await self._index_memory(memory, ctx)
+ # Store embedding for batch-internal dedup of subsequent candidates (#687)
+ if result.query_vector:
+ batch_memories.append((result.query_vector, memory))
else:
stats.skipped += 1
diff --git a/openviking/session/memory_deduplicator.py b/openviking/session/memory_deduplicator.py
index c38be27e0..5b474fd03 100644
--- a/openviking/session/memory_deduplicator.py
+++ b/openviking/session/memory_deduplicator.py
@@ -7,6 +7,7 @@ LLM-assisted deduplication with candidate-level skip/create/none decisions and
per-existing merge/delete actions.
"""
+import copy
import re
from dataclasses import dataclass
from enum import Enum
@@ -59,6 +60,7 @@ class DedupResult:
similar_memories: List[Context] # Similar existing memories
actions: Optional[List[ExistingMemoryAction]] = None
reason: str = ""
+ query_vector: list[float] | None = None # For batch-internal dedup tracking
class MemoryDeduplicator:
@@ -92,10 +94,14 @@ class MemoryDeduplicator:
self,
candidate: CandidateMemory,
ctx: RequestContext,
+ *,
+ batch_memories: list[tuple[list[float], Context]] | None = None,
) -> DedupResult:
"""Decide how to handle a candidate memory."""
# Step 1: Vector pre-filtering - find similar memories in same category
- similar_memories = await self._find_similar_memories(candidate, ctx=ctx)
+ similar_memories, query_vector = await self._find_similar_memories(
+ candidate, ctx=ctx, batch_memories=batch_memories
+ )
if not similar_memories:
# No similar memories, create directly
@@ -105,6 +111,7 @@ class MemoryDeduplicator:
similar_memories=[],
actions=[],
reason="No similar memories found",
+ query_vector=query_vector,
)
# Step 2: LLM decision
@@ -116,17 +123,26 @@ class MemoryDeduplicator:
similar_memories=similar_memories,
actions=None if decision == DedupDecision.SKIP else actions,
reason=reason,
+ query_vector=query_vector,
)
async def _find_similar_memories(
self,
candidate: CandidateMemory,
ctx: RequestContext,
- ) -> List[Context]:
- """Find similar existing memories using vector search."""
+ *,
+ batch_memories: list[tuple[list[float], Context]] | None = None,
+ ) -> tuple[list[Context], list[float]]:
+ """Find similar existing memories using vector search.
+
+ Returns (similar_memories, query_vector). query_vector is the candidate's
+ embedding, returned so the caller can store it for batch-internal tracking.
+ """
telemetry = get_current_telemetry()
+ query_vector: list[float] = [] # Initialize early for safe returns
+
if not self.embedder:
- return []
+ return [], query_vector
# Generate embedding for candidate
query_text = f"{candidate.abstract} {candidate.content}"
@@ -187,11 +203,27 @@ class MemoryDeduplicator:
context.meta = {**(context.meta or {}), "_dedup_score": score}
similar.append(context)
logger.debug("Dedup similar memories after threshold=%d", len(similar))
- return similar
+
+ # Include batch-internal memories that are similar (#687).
+ # Shallow-copy to avoid mutating the original's meta while
+ # preserving all fields (account_id, owner_space, etc.) needed
+ # downstream if the LLM decides to MERGE into this memory.
+ if batch_memories:
+ seen_uris = {c.uri for c in similar}
+ for batch_vec, batch_ctx in batch_memories:
+ if batch_ctx.uri in seen_uris:
+ continue
+ score = self._cosine_similarity(query_vector, batch_vec)
+ if score >= self.SIMILARITY_THRESHOLD:
+ ctx_copy = copy.copy(batch_ctx)
+ ctx_copy.meta = {**(batch_ctx.meta or {}), "_dedup_score": score}
+ similar.append(ctx_copy)
+
+ return similar, query_vector
except Exception as e:
logger.warning(f"Vector search failed: {e}")
- return []
+ return [], query_vector
async def _llm_decision(
self,
diff --git a/tests/session/test_memory_dedup_actions.py b/tests/session/test_memory_dedup_actions.py
index e7bb1a80f..844f92945 100644
--- a/tests/session/test_memory_dedup_actions.py
+++ b/tests/session/test_memory_dedup_actions.py
@@ -66,6 +66,23 @@ def _make_candidate() -> CandidateMemory:
)
+def _make_dedup(vikingdb=None, embedder=None) -> MemoryDeduplicator:
+ """Create MemoryDeduplicator without config dependency."""
+ dedup = MemoryDeduplicator.__new__(MemoryDeduplicator)
+ dedup.vikingdb = vikingdb or MagicMock()
+ dedup.embedder = embedder
+ return dedup
+
+
+def _make_compressor(vikingdb=None, embedder=None) -> SessionCompressor:
+ """Create SessionCompressor without config dependency."""
+ vikingdb = vikingdb or MagicMock()
+ with patch("openviking.session.memory_deduplicator.get_openviking_config") as mock_config:
+ mock_config.return_value.embedding.get_query_embedder.return_value = embedder
+ compressor = SessionCompressor(vikingdb=vikingdb)
+ return compressor
+
+
def _make_existing(uri_suffix: str = "existing.md") -> Context:
user_space = _make_user().user_space_name()
return Context(
@@ -174,12 +191,13 @@ class TestMemoryDeduplicatorPayload:
dedup = MemoryDeduplicator(vikingdb=vikingdb)
candidate = _make_candidate()
- similar = await dedup._find_similar_memories(candidate, ctx)
+ similar, _query_vector = await dedup._find_similar_memories(candidate, ctx)
assert len(similar) == 1
assert similar[0].uri == existing.uri
call = vikingdb.search_similar_memories.await_args.kwargs
- assert call["account_id"] == "acc1"
+ # Note: removed stale assert call["account_id"] -- _find_similar_memories
+ # does not pass account_id to search_similar_memories.
assert call["owner_space"] == _make_user().user_space_name()
assert call["category_uri_prefix"] == (
f"viking://user/{_make_user().user_space_name()}/memories/preferences/"
@@ -206,7 +224,7 @@ class TestMemoryDeduplicatorPayload:
)
dedup = MemoryDeduplicator(vikingdb=vikingdb)
- similar = await dedup._find_similar_memories(_make_candidate(), ctx)
+ similar, _ = await dedup._find_similar_memories(_make_candidate(), ctx)
assert len(similar) == 1
@@ -250,6 +268,101 @@ class TestMemoryDeduplicatorPayload:
assert similar[4].uri in existing_text
assert similar[5].uri not in existing_text
+ @pytest.mark.asyncio
+ async def test_find_similar_includes_batch_memories(self):
+ """Batch memory with high cosine similarity appears in results."""
+ vikingdb = MagicMock()
+ vikingdb.search_similar_memories = AsyncMock(return_value=[])
+
+ dedup = _make_dedup(vikingdb=vikingdb, embedder=_DummyEmbedder())
+ candidate = _make_candidate()
+
+ # Batch memory with identical embedding vector -> cosine similarity = 1.0
+ batch_ctx = _make_existing("batch_item.md")
+ batch_vector = [0.1, 0.2, 0.3] # Same as _DummyEmbedder returns
+ batch_memories = [(batch_vector, batch_ctx)]
+
+ similar, query_vector = await dedup._find_similar_memories(
+ candidate, ctx, batch_memories=batch_memories
+ )
+
+ assert len(similar) == 1
+ assert similar[0].uri == batch_ctx.uri
+ assert similar[0].meta["_dedup_score"] == pytest.approx(1.0, abs=1e-6)
+ assert query_vector == [0.1, 0.2, 0.3]
+
+ @pytest.mark.asyncio
+ async def test_find_similar_excludes_dissimilar_batch_memories(self):
+ """Batch memory with opposite embedding (cosine = -1.0) is excluded."""
+ vikingdb = MagicMock()
+ vikingdb.search_similar_memories = AsyncMock(return_value=[])
+
+ dedup = _make_dedup(vikingdb=vikingdb, embedder=_DummyEmbedder())
+ candidate = _make_candidate()
+
+ # Opposite direction vector -> cosine = -1.0, below threshold 0.0
+ batch_ctx = _make_existing("unrelated.md")
+ batch_vector = [-0.1, -0.2, -0.3]
+ batch_memories = [(batch_vector, batch_ctx)]
+
+ similar, _ = await dedup._find_similar_memories(
+ candidate, ctx, batch_memories=batch_memories
+ )
+
+ assert len(similar) == 0
+
+ @pytest.mark.asyncio
+ async def test_find_similar_deduplicates_batch_and_db_by_uri(self):
+ """If same URI appears in both DB results and batch, only keep DB version."""
+ existing = _make_existing("overlap.md")
+ vikingdb = MagicMock()
+ vikingdb.search_similar_memories = AsyncMock(
+ return_value=[
+ {
+ "id": "uri_overlap",
+ "uri": existing.uri,
+ "context_type": "memory",
+ "level": 2,
+ "account_id": "acc1",
+ "owner_space": _make_user().user_space_name(),
+ "abstract": existing.abstract,
+ "category": "preferences",
+ "_score": 0.9,
+ }
+ ]
+ )
+
+ dedup = _make_dedup(vikingdb=vikingdb, embedder=_DummyEmbedder())
+ candidate = _make_candidate()
+
+ # Batch contains same URI as DB result
+ batch_ctx = _make_existing("overlap.md")
+ batch_vector = [0.1, 0.2, 0.3]
+ batch_memories = [(batch_vector, batch_ctx)]
+
+ similar, _ = await dedup._find_similar_memories(
+ candidate, ctx, batch_memories=batch_memories
+ )
+
+ # Should have exactly 1 (DB version), not 2
+ assert len(similar) == 1
+ assert similar[0].uri == existing.uri
+ assert similar[0].meta["_dedup_score"] == pytest.approx(0.9, abs=1e-6)
+
+ @pytest.mark.asyncio
+ async def test_deduplicate_returns_query_vector_in_result(self):
+ """DedupResult includes query_vector for batch tracking."""
+ vikingdb = MagicMock()
+ vikingdb.search_similar_memories = AsyncMock(return_value=[])
+
+ dedup = _make_dedup(vikingdb=vikingdb, embedder=_DummyEmbedder())
+ candidate = _make_candidate()
+
+ result = await dedup.deduplicate(candidate, ctx)
+
+ assert result.decision == DedupDecision.CREATE
+ assert result.query_vector == [0.1, 0.2, 0.3]
+
@pytest.mark.asyncio
class TestMemoryMergeBundle:
@@ -559,3 +672,138 @@ class TestSessionCompressorDedupActions:
assert [m.uri for m in memories] == [new_memory.uri]
assert call_order == ["delete", "create"]
vikingdb.delete_uris.assert_awaited_once_with(_make_ctx(), [target.uri])
+
+ async def test_batch_dedup_passes_batch_memories_to_deduplicate(self):
+ """Compressor passes batch_memories with previously created memory to deduplicate."""
+ candidate_a = _make_candidate()
+ candidate_a.abstract = "User prefers dark mode"
+ candidate_a.content = "The user prefers dark mode in all editors."
+
+ candidate_b = _make_candidate()
+ candidate_b.abstract = "User likes dark mode"
+ candidate_b.content = "The user likes dark mode for coding."
+
+ memory_a = _make_existing("created_a.md")
+
+ vikingdb = MagicMock()
+ vikingdb.delete_uris = AsyncMock(return_value=None)
+ vikingdb.enqueue_embedding_msg = AsyncMock()
+
+ compressor = _make_compressor(vikingdb=vikingdb)
+ compressor.extractor.extract = AsyncMock(return_value=[candidate_a, candidate_b])
+ compressor.extractor.create_memory = AsyncMock(return_value=memory_a)
+
+ call_count = 0
+
+ async def _deduplicate(candidate, ctx, *, batch_memories=None):
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ assert batch_memories is None or len(batch_memories) == 0
+ return DedupResult(
+ decision=DedupDecision.CREATE,
+ candidate=candidate,
+ similar_memories=[],
+ actions=[],
+ query_vector=[0.1, 0.2, 0.3],
+ )
+ else:
+ assert batch_memories is not None
+ assert len(batch_memories) == 1
+ assert batch_memories[0][0] == [0.1, 0.2, 0.3]
+ assert batch_memories[0][1].uri == memory_a.uri
+ return DedupResult(
+ decision=DedupDecision.SKIP,
+ candidate=candidate,
+ similar_memories=[batch_memories[0][1]],
+ actions=[],
+ query_vector=[0.1, 0.2, 0.3],
+ )
+
+ compressor.deduplicator.deduplicate = AsyncMock(side_effect=_deduplicate)
+ compressor._index_memory = AsyncMock(return_value=True)
+
+ fs = MagicMock()
+ fs.rm = AsyncMock()
+
+ with patch("openviking.session.compressor.get_viking_fs", return_value=fs):
+ memories = await compressor.extract_long_term_memories(
+ [Message.create_user("test message")],
+ user=_make_user(),
+ session_id="session_test",
+ ctx=_make_ctx(),
+ )
+
+ assert len(memories) == 1
+ assert memories[0].uri == memory_a.uri
+ assert call_count == 2
+ compressor.extractor.create_memory.assert_awaited_once()
+
+ async def test_batch_dedup_real_cosine_path(self):
+ """End-to-end: real deduplicator cosine comparison catches batch duplicate."""
+ candidate_a = _make_candidate()
+ candidate_a.abstract = "User prefers dark mode"
+ candidate_a.content = "The user prefers dark mode in all editors."
+
+ candidate_b = _make_candidate()
+ candidate_b.abstract = "User likes dark mode"
+ candidate_b.content = "The user likes dark mode for coding."
+
+ memory_a = _make_existing("real_a.md")
+
+ vikingdb = MagicMock()
+ vikingdb.search_similar_memories = AsyncMock(return_value=[])
+ vikingdb.delete_uris = AsyncMock(return_value=None)
+ vikingdb.enqueue_embedding_msg = AsyncMock()
+
+ compressor = _make_compressor(vikingdb=vikingdb, embedder=_DummyEmbedder())
+ compressor.extractor.extract = AsyncMock(return_value=[candidate_a, candidate_b])
+ compressor.extractor.create_memory = AsyncMock(return_value=memory_a)
+ compressor._index_memory = AsyncMock(return_value=True)
+
+ # Spy on _llm_decision to verify batch match triggers LLM path
+ original_llm_decision = compressor.deduplicator._llm_decision
+ llm_decision_calls = []
+
+ async def _spy_llm_decision(candidate, similar_memories):
+ llm_decision_calls.append(similar_memories)
+ return await original_llm_decision(candidate, similar_memories)
+
+ compressor.deduplicator._llm_decision = _spy_llm_decision
+
+ # Mock config for _llm_decision (called when similar memories found)
+ class _NoVLMConfig:
+ vlm = None
+
+ class embedding:
+ @staticmethod
+ def get_query_embedder():
+ return _DummyEmbedder()
+
+ fs = MagicMock()
+ fs.rm = AsyncMock()
+
+ with (
+ patch("openviking.session.compressor.get_viking_fs", return_value=fs),
+ patch(
+ "openviking.session.memory_deduplicator.get_openviking_config",
+ return_value=_NoVLMConfig(),
+ ),
+ ):
+ await compressor.extract_long_term_memories(
+ [Message.create_user("test message")],
+ user=_make_user(),
+ session_id="session_test",
+ ctx=_make_ctx(),
+ )
+
+ # _DummyEmbedder returns [0.1, 0.2, 0.3] for all texts -> cosine = 1.0
+ # First: DB empty, no batch -> CREATE (no _llm_decision called).
+ # Second: DB empty, but batch match found (cosine=1.0) ->
+ # _llm_decision IS called with the batch-sourced similar memory.
+ assert vikingdb.search_similar_memories.await_count == 2
+ # Key assertion: _llm_decision was called exactly once (for second candidate)
+ assert len(llm_decision_calls) == 1
+ # The similar_memories passed to LLM came from batch (not DB, which was empty)
+ assert len(llm_decision_calls[0]) == 1
+ assert llm_decision_calls[0][0].uri == memory_a.uri