Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(session): add batch-internal dedup to prevent duplicates within same commit (#687) (#701)
AstroHan1e475c2423c1
diff --git a/openviking/session/compressor.py b/openviking/session/compressor.pyindex 243eba8b0..7fd066c7c 100644--- a/openviking/session/compressor.py+++ b/openviking/session/compressor.py@@ -248,6 +248,8 @@ class SessionCompressor:memories: List[Context] = []stats = ExtractionStats()+ # Track created memories' embeddings for batch-internal dedup (#687)+ batch_memories: list[tuple[list[float], Context]] = []viking_fs = get_viking_fs()tool_parts = self._extract_tool_parts(messages)@@ -323,7 +325,9 @@ class SessionCompressor:continue# Dedup check for other categories- result = await self.deduplicator.deduplicate(candidate, ctx)+ result = await self.deduplicator.deduplicate(+ candidate, ctx, batch_memories=batch_memories+ )actions = result.actions or []decision = result.decision@@ -352,6 +356,10 @@ class SessionCompressor:action.memory, viking_fs, ctx=ctx):stats.deleted += 1+ # Remove deleted memory from batch tracking (#687)+ batch_memories = [+ (v, m) for v, m in batch_memories if m.uri != action.memory.uri+ ]else:stats.skipped += 1elif action.decision == MemoryActionDecision.MERGE:@@ -360,6 +368,23 @@ class SessionCompressor:candidate, action.memory, viking_fs, ctx=ctx):stats.merged += 1+ # Remove stale batch entry and re-add with updated+ # embedding so 3rd+ candidates can still find it (#687).+ batch_memories = [+ (v, m)+ for v, m in batch_memories+ if m.uri != action.memory.uri+ ]+ if self.deduplicator.embedder:+ merged_text = (+ f"{action.memory.abstract} {candidate.content}"+ )+ merged_embed = self.deduplicator.embedder.embed(+ merged_text+ )+ batch_memories.append(+ (merged_embed.dense_vector, action.memory)+ )else:stats.skipped += 1else:@@ -375,6 +400,10 @@ class SessionCompressor:action.memory, viking_fs, ctx=ctx):stats.deleted += 1+ # Remove deleted memory from batch tracking (#687)+ batch_memories = [+ (v, m) for v, m in batch_memories if m.uri != action.memory.uri+ ]else:stats.skipped += 1@@ -385,6 +414,9 @@ class SessionCompressor:memories.append(memory)stats.created += 1await self._index_memory(memory, ctx)+ # Store embedding for batch-internal dedup of subsequent candidates (#687)+ if result.query_vector:+ batch_memories.append((result.query_vector, memory))else:stats.skipped += 1diff --git a/openviking/session/memory_deduplicator.py b/openviking/session/memory_deduplicator.pyindex c38be27e0..5b474fd03 100644--- a/openviking/session/memory_deduplicator.py+++ b/openviking/session/memory_deduplicator.py@@ -7,6 +7,7 @@ LLM-assisted deduplication with candidate-level skip/create/none decisions andper-existing merge/delete actions."""+import copyimport refrom dataclasses import dataclassfrom enum import Enum@@ -59,6 +60,7 @@ class DedupResult:similar_memories: List[Context] # Similar existing memoriesactions: Optional[List[ExistingMemoryAction]] = Nonereason: str = ""+ query_vector: list[float] | None = None # For batch-internal dedup trackingclass MemoryDeduplicator:@@ -92,10 +94,14 @@ class MemoryDeduplicator:self,candidate: CandidateMemory,ctx: RequestContext,+ *,+ batch_memories: list[tuple[list[float], Context]] | None = None,) -> DedupResult:"""Decide how to handle a candidate memory."""# Step 1: Vector pre-filtering - find similar memories in same category- similar_memories = await self._find_similar_memories(candidate, ctx=ctx)+ similar_memories, query_vector = await self._find_similar_memories(+ candidate, ctx=ctx, batch_memories=batch_memories+ )if not similar_memories:# No similar memories, create directly@@ -105,6 +111,7 @@ class MemoryDeduplicator:similar_memories=[],actions=[],reason="No similar memories found",+ query_vector=query_vector,)# Step 2: LLM decision@@ -116,17 +123,26 @@ class MemoryDeduplicator:similar_memories=similar_memories,actions=None if decision == DedupDecision.SKIP else actions,reason=reason,+ query_vector=query_vector,)async def _find_similar_memories(self,candidate: CandidateMemory,ctx: RequestContext,- ) -> List[Context]:- """Find similar existing memories using vector search."""+ *,+ batch_memories: list[tuple[list[float], Context]] | None = None,+ ) -> tuple[list[Context], list[float]]:+ """Find similar existing memories using vector search.++ Returns (similar_memories, query_vector). query_vector is the candidate's+ embedding, returned so the caller can store it for batch-internal tracking.+ """telemetry = get_current_telemetry()+ query_vector: list[float] = [] # Initialize early for safe returns+if not self.embedder:- return []+ return [], query_vector# Generate embedding for candidatequery_text = f"{candidate.abstract} {candidate.content}"@@ -187,11 +203,27 @@ class MemoryDeduplicator:context.meta = {**(context.meta or {}), "_dedup_score": score}similar.append(context)logger.debug("Dedup similar memories after threshold=%d", len(similar))- return similar++ # Include batch-internal memories that are similar (#687).+ # Shallow-copy to avoid mutating the original's meta while+ # preserving all fields (account_id, owner_space, etc.) needed+ # downstream if the LLM decides to MERGE into this memory.+ if batch_memories:+ seen_uris = {c.uri for c in similar}+ for batch_vec, batch_ctx in batch_memories:+ if batch_ctx.uri in seen_uris:+ continue+ score = self._cosine_similarity(query_vector, batch_vec)+ if score >= self.SIMILARITY_THRESHOLD:+ ctx_copy = copy.copy(batch_ctx)+ ctx_copy.meta = {**(batch_ctx.meta or {}), "_dedup_score": score}+ similar.append(ctx_copy)++ return similar, query_vectorexcept Exception as e:logger.warning(f"Vector search failed: {e}")- return []+ return [], query_vectorasync def _llm_decision(self,diff --git a/tests/session/test_memory_dedup_actions.py b/tests/session/test_memory_dedup_actions.pyindex e7bb1a80f..844f92945 100644--- a/tests/session/test_memory_dedup_actions.py+++ b/tests/session/test_memory_dedup_actions.py@@ -66,6 +66,23 @@ def _make_candidate() -> CandidateMemory:)+def _make_dedup(vikingdb=None, embedder=None) -> MemoryDeduplicator:+ """Create MemoryDeduplicator without config dependency."""+ dedup = MemoryDeduplicator.__new__(MemoryDeduplicator)+ dedup.vikingdb = vikingdb or MagicMock()+ dedup.embedder = embedder+ return dedup+++def _make_compressor(vikingdb=None, embedder=None) -> SessionCompressor:+ """Create SessionCompressor without config dependency."""+ vikingdb = vikingdb or MagicMock()+ with patch("openviking.session.memory_deduplicator.get_openviking_config") as mock_config:+ mock_config.return_value.embedding.get_query_embedder.return_value = embedder+ compressor = SessionCompressor(vikingdb=vikingdb)+ return compressor++def _make_existing(uri_suffix: str = "existing.md") -> Context:user_space = _make_user().user_space_name()return Context(@@ -174,12 +191,13 @@ class TestMemoryDeduplicatorPayload:dedup = MemoryDeduplicator(vikingdb=vikingdb)candidate = _make_candidate()- similar = await dedup._find_similar_memories(candidate, ctx)+ similar, _query_vector = await dedup._find_similar_memories(candidate, ctx)assert len(similar) == 1assert similar[0].uri == existing.uricall = vikingdb.search_similar_memories.await_args.kwargs- assert call["account_id"] == "acc1"+ # Note: removed stale assert call["account_id"] -- _find_similar_memories+ # does not pass account_id to search_similar_memories.assert call["owner_space"] == _make_user().user_space_name()assert call["category_uri_prefix"] == (f"viking://user/{_make_user().user_space_name()}/memories/preferences/"@@ -206,7 +224,7 @@ class TestMemoryDeduplicatorPayload:)dedup = MemoryDeduplicator(vikingdb=vikingdb)- similar = await dedup._find_similar_memories(_make_candidate(), ctx)+ similar, _ = await dedup._find_similar_memories(_make_candidate(), ctx)assert len(similar) == 1@@ -250,6 +268,101 @@ class TestMemoryDeduplicatorPayload:assert similar[4].uri in existing_textassert similar[5].uri not in existing_text+ @pytest.mark.asyncio+ async def test_find_similar_includes_batch_memories(self):+ """Batch memory with high cosine similarity appears in results."""+ vikingdb = MagicMock()+ vikingdb.search_similar_memories = AsyncMock(return_value=[])++ dedup = _make_dedup(vikingdb=vikingdb, embedder=_DummyEmbedder())+ candidate = _make_candidate()++ # Batch memory with identical embedding vector -> cosine similarity = 1.0+ batch_ctx = _make_existing("batch_item.md")+ batch_vector = [0.1, 0.2, 0.3] # Same as _DummyEmbedder returns+ batch_memories = [(batch_vector, batch_ctx)]++ similar, query_vector = await dedup._find_similar_memories(+ candidate, ctx, batch_memories=batch_memories+ )++ assert len(similar) == 1+ assert similar[0].uri == batch_ctx.uri+ assert similar[0].meta["_dedup_score"] == pytest.approx(1.0, abs=1e-6)+ assert query_vector == [0.1, 0.2, 0.3]++ @pytest.mark.asyncio+ async def test_find_similar_excludes_dissimilar_batch_memories(self):+ """Batch memory with opposite embedding (cosine = -1.0) is excluded."""+ vikingdb = MagicMock()+ vikingdb.search_similar_memories = AsyncMock(return_value=[])++ dedup = _make_dedup(vikingdb=vikingdb, embedder=_DummyEmbedder())+ candidate = _make_candidate()++ # Opposite direction vector -> cosine = -1.0, below threshold 0.0+ batch_ctx = _make_existing("unrelated.md")+ batch_vector = [-0.1, -0.2, -0.3]+ batch_memories = [(batch_vector, batch_ctx)]++ similar, _ = await dedup._find_similar_memories(+ candidate, ctx, batch_memories=batch_memories+ )++ assert len(similar) == 0++ @pytest.mark.asyncio+ async def test_find_similar_deduplicates_batch_and_db_by_uri(self):+ """If same URI appears in both DB results and batch, only keep DB version."""+ existing = _make_existing("overlap.md")+ vikingdb = MagicMock()+ vikingdb.search_similar_memories = AsyncMock(+ return_value=[+ {+ "id": "uri_overlap",+ "uri": existing.uri,+ "context_type": "memory",+ "level": 2,+ "account_id": "acc1",+ "owner_space": _make_user().user_space_name(),+ "abstract": existing.abstract,+ "category": "preferences",+ "_score": 0.9,+ }+ ]+ )++ dedup = _make_dedup(vikingdb=vikingdb, embedder=_DummyEmbedder())+ candidate = _make_candidate()++ # Batch contains same URI as DB result+ batch_ctx = _make_existing("overlap.md")+ batch_vector = [0.1, 0.2, 0.3]+ batch_memories = [(batch_vector, batch_ctx)]++ similar, _ = await dedup._find_similar_memories(+ candidate, ctx, batch_memories=batch_memories+ )++ # Should have exactly 1 (DB version), not 2+ assert len(similar) == 1+ assert similar[0].uri == existing.uri+ assert similar[0].meta["_dedup_score"] == pytest.approx(0.9, abs=1e-6)++ @pytest.mark.asyncio+ async def test_deduplicate_returns_query_vector_in_result(self):+ """DedupResult includes query_vector for batch tracking."""+ vikingdb = MagicMock()+ vikingdb.search_similar_memories = AsyncMock(return_value=[])++ dedup = _make_dedup(vikingdb=vikingdb, embedder=_DummyEmbedder())+ candidate = _make_candidate()++ result = await dedup.deduplicate(candidate, ctx)++ assert result.decision == DedupDecision.CREATE+ assert result.query_vector == [0.1, 0.2, 0.3]+@pytest.mark.asyncioclass TestMemoryMergeBundle:@@ -559,3 +672,138 @@ class TestSessionCompressorDedupActions:assert [m.uri for m in memories] == [new_memory.uri]assert call_order == ["delete", "create"]vikingdb.delete_uris.assert_awaited_once_with(_make_ctx(), [target.uri])++ async def test_batch_dedup_passes_batch_memories_to_deduplicate(self):+ """Compressor passes batch_memories with previously created memory to deduplicate."""+ candidate_a = _make_candidate()+ candidate_a.abstract = "User prefers dark mode"+ candidate_a.content = "The user prefers dark mode in all editors."++ candidate_b = _make_candidate()+ candidate_b.abstract = "User likes dark mode"+ candidate_b.content = "The user likes dark mode for coding."++ memory_a = _make_existing("created_a.md")++ vikingdb = MagicMock()+ vikingdb.delete_uris = AsyncMock(return_value=None)+ vikingdb.enqueue_embedding_msg = AsyncMock()++ compressor = _make_compressor(vikingdb=vikingdb)+ compressor.extractor.extract = AsyncMock(return_value=[candidate_a, candidate_b])+ compressor.extractor.create_memory = AsyncMock(return_value=memory_a)++ call_count = 0++ async def _deduplicate(candidate, ctx, *, batch_memories=None):+ nonlocal call_count+ call_count += 1+ if call_count == 1:+ assert batch_memories is None or len(batch_memories) == 0+ return DedupResult(+ decision=DedupDecision.CREATE,+ candidate=candidate,+ similar_memories=[],+ actions=[],+ query_vector=[0.1, 0.2, 0.3],+ )+ else:+ assert batch_memories is not None+ assert len(batch_memories) == 1+ assert batch_memories[0][0] == [0.1, 0.2, 0.3]+ assert batch_memories[0][1].uri == memory_a.uri+ return DedupResult(+ decision=DedupDecision.SKIP,+ candidate=candidate,+ similar_memories=[batch_memories[0][1]],+ actions=[],+ query_vector=[0.1, 0.2, 0.3],+ )++ compressor.deduplicator.deduplicate = AsyncMock(side_effect=_deduplicate)+ compressor._index_memory = AsyncMock(return_value=True)++ fs = MagicMock()+ fs.rm = AsyncMock()++ with patch("openviking.session.compressor.get_viking_fs", return_value=fs):+ memories = await compressor.extract_long_term_memories(+ [Message.create_user("test message")],+ user=_make_user(),+ session_id="session_test",+ ctx=_make_ctx(),+ )++ assert len(memories) == 1+ assert memories[0].uri == memory_a.uri+ assert call_count == 2+ compressor.extractor.create_memory.assert_awaited_once()++ async def test_batch_dedup_real_cosine_path(self):+ """End-to-end: real deduplicator cosine comparison catches batch duplicate."""+ candidate_a = _make_candidate()+ candidate_a.abstract = "User prefers dark mode"+ candidate_a.content = "The user prefers dark mode in all editors."++ candidate_b = _make_candidate()+ candidate_b.abstract = "User likes dark mode"+ candidate_b.content = "The user likes dark mode for coding."++ memory_a = _make_existing("real_a.md")++ vikingdb = MagicMock()+ vikingdb.search_similar_memories = AsyncMock(return_value=[])+ vikingdb.delete_uris = AsyncMock(return_value=None)+ vikingdb.enqueue_embedding_msg = AsyncMock()++ compressor = _make_compressor(vikingdb=vikingdb, embedder=_DummyEmbedder())+ compressor.extractor.extract = AsyncMock(return_value=[candidate_a, candidate_b])+ compressor.extractor.create_memory = AsyncMock(return_value=memory_a)+ compressor._index_memory = AsyncMock(return_value=True)++ # Spy on _llm_decision to verify batch match triggers LLM path+ original_llm_decision = compressor.deduplicator._llm_decision+ llm_decision_calls = []++ async def _spy_llm_decision(candidate, similar_memories):+ llm_decision_calls.append(similar_memories)+ return await original_llm_decision(candidate, similar_memories)++ compressor.deduplicator._llm_decision = _spy_llm_decision++ # Mock config for _llm_decision (called when similar memories found)+ class _NoVLMConfig:+ vlm = None++ class embedding:+ @staticmethod+ def get_query_embedder():+ return _DummyEmbedder()++ fs = MagicMock()+ fs.rm = AsyncMock()++ with (+ patch("openviking.session.compressor.get_viking_fs", return_value=fs),+ patch(+ "openviking.session.memory_deduplicator.get_openviking_config",+ return_value=_NoVLMConfig(),+ ),+ ):+ await compressor.extract_long_term_memories(+ [Message.create_user("test message")],+ user=_make_user(),+ session_id="session_test",+ ctx=_make_ctx(),+ )++ # _DummyEmbedder returns [0.1, 0.2, 0.3] for all texts -> cosine = 1.0+ # First: DB empty, no batch -> CREATE (no _llm_decision called).+ # Second: DB empty, but batch match found (cosine=1.0) ->+ # _llm_decision IS called with the batch-sourced similar memory.+ assert vikingdb.search_similar_memories.await_count == 2+ # Key assertion: _llm_decision was called exactly once (for second candidate)+ assert len(llm_decision_calls) == 1+ # The similar_memories passed to LLM came from batch (not DB, which was empty)+ assert len(llm_decision_calls[0]) == 1+ assert llm_decision_calls[0][0].uri == memory_a.uri