Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix: simplify memory dedup decisions and fix retrieval recursion bug (#53)
qin-ctx7c4dbf27a2a3
diff --git a/openviking/core/context.py b/openviking/core/context.pyindex 6d37fc187..64255917b 100644--- a/openviking/core/context.py+++ b/openviking/core/context.py@@ -55,11 +55,12 @@ class Context:meta: Optional[Dict[str, Any]] = None,session_id: Optional[str] = None,user: Optional[str] = None,+ id: Optional[str] = None,):"""Initialize a Context object."""- self.id = str(uuid4())+ self.id = id or str(uuid4())self.uri = uriself.parent_uri = parent_uriself.is_leaf = is_leafdiff --git a/openviking/prompts/templates/compression/dedup_decision.yaml b/openviking/prompts/templates/compression/dedup_decision.yamlindex 23b8ab637..c41fee019 100644--- a/openviking/prompts/templates/compression/dedup_decision.yaml+++ b/openviking/prompts/templates/compression/dedup_decision.yaml@@ -1,8 +1,8 @@metadata:id: "compression.dedup_decision"name: "Memory Deduplication Decision"- description: "Decide whether new memory should CREATE/UPDATE/MERGE/SKIP"- version: "1.1.0"+ description: "Decide whether new memory should CREATE/MERGE/SKIP"+ version: "2.0.0"language: "en"category: "compression"@@ -35,24 +35,16 @@ template: |**Existing Similar Memories**:{{ existing_memories }}- Please decide the operation type:- - CREATE: This is a completely new memory, should be created- - UPDATE: Candidate memory is a supplement/update to existing memory, should update the most relevant one- - MERGE: Candidate memory is related to multiple existing memories, should be merged+ Please decide:- SKIP: Candidate memory duplicates existing memories, no need to save+ - CREATE: This is completely new information, should be created+ - MERGE: Candidate memory should be merged with existing memoriesReturn JSON format:{- "decision": "create|update|merge|skip",- "reason": "Decision reason",- "merged_content": "If UPDATE or MERGE, provide merged L2 content"+ "decision": "skip|create|merge",+ "reason": "Decision reason"}- Notes:- - If candidate memory provides new information, choose UPDATE or MERGE- - If candidate memory is completely duplicate, choose SKIP- - If candidate memory is a new topic, choose CREATE- - merged_content is only needed for UPDATE/MERGE, and should be complete L2 content-llm_config:temperature: 0.0diff --git a/openviking/prompts/templates/compression/memory_merge.yaml b/openviking/prompts/templates/compression/memory_merge.yamlnew file mode 100644index 000000000..c1a133c24--- /dev/null+++ b/openviking/prompts/templates/compression/memory_merge.yaml@@ -0,0 +1,41 @@+metadata:+ id: "compression.memory_merge"+ name: "Memory Merge"+ description: "Merge existing memory with new information"+ version: "1.0.0"+ language: "en"+ category: "compression"++variables:+ - name: "existing_content"+ type: "string"+ description: "Existing memory content"+ required: true+ - name: "new_content"+ type: "string"+ description: "New memory content to merge"+ required: true+ - name: "category"+ type: "string"+ description: "Memory category (profile, preferences, etc.)"+ required: true++template: |+ Merge the following memory information into a single, coherent content.++ **Category**: {{ category }}++ **Existing Content:**+ {{ existing_content }}++ **New Information:**+ {{ new_content }}++ Requirements:+ - Remove duplicate information+ - Keep the most up-to-date details+ - Maintain a coherent narrative+ - Output ONLY the merged content, no explanation++llm_config:+ temperature: 0.0diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.pyindex 6881e5691..a0a54b2f1 100644--- a/openviking/retrieve/hierarchical_retriever.py+++ b/openviking/retrieve/hierarchical_retriever.py@@ -237,6 +237,8 @@ class HierarchicalRetriever:def passes_threshold(score: float) -> bool:"""Check if score passes threshold."""+ if not self._rerank_client or mode != RetrieverMode.THINKING:+ return Trueif score_gte:return score >= effective_thresholdreturn score > effective_threshold@@ -277,7 +279,7 @@ class HierarchicalRetriever:current_score = -temp_scoreif current_uri in visited:continue-+ visited.add(current_uri)logger.info(f"[RecursiveSearch] Entering URI: {current_uri}")pre_filter_limit = max(limit * 2, 20)@@ -318,11 +320,11 @@ class HierarchicalRetriever:if passes_threshold(final_score) and uri not in visited:r["_final_score"] = final_scorecollected.append(r)- visited.add(uri)logger.info(f"[RecursiveSearch] Added URI: {uri} to candidates with score: {final_score}")- if not r.get("is_leaf"):+ if r.get("is_leaf"):+ visited.add(uri)continueheapq.heappush(dir_queue, (-final_score, uri))else:diff --git a/openviking/session/compressor.py b/openviking/session/compressor.pyindex 8c4981dc8..b6d9aae51 100644--- a/openviking/session/compressor.py+++ b/openviking/session/compressor.py@@ -10,24 +10,33 @@ Uses MemoryExtractor for 6-category extraction and MemoryDeduplicator for LLM-bafrom dataclasses import dataclassfrom typing import Dict, List-from openviking.core.context import Context+from openviking.core.context import Context, Vectorizefrom openviking.message import Messagefrom openviking.storage import VikingDBManagerfrom openviking.storage.viking_fs import get_viking_fsfrom openviking.utils import get_loggerfrom .memory_deduplicator import DedupDecision, MemoryDeduplicator-from .memory_extractor import MemoryExtractor+from .memory_extractor import MemoryCategory, MemoryExtractorlogger = get_logger(__name__)+# Categories that always merge (skip dedup)+ALWAYS_MERGE_CATEGORIES = {MemoryCategory.PROFILE}++# Categories that support MERGE decision+MERGE_SUPPORTED_CATEGORIES = {+ MemoryCategory.PREFERENCES,+ MemoryCategory.ENTITIES,+ MemoryCategory.PATTERNS,+}+@dataclassclass ExtractionStats:"""Statistics for memory extraction."""created: int = 0- updated: int = 0merged: int = 0skipped: int = 0@@ -71,8 +80,19 @@ class SessionCompressor:memories: List[Context] = []stats = ExtractionStats()+ viking_fs = get_viking_fs()for candidate in candidates:+ # Profile: skip dedup, always merge+ if candidate.category in ALWAYS_MERGE_CATEGORIES:+ memory = await self.extractor.create_memory(candidate, user, session_id)+ if memory:+ memories.append(memory)+ stats.created += 1+ await self._index_memory(memory)+ continue++ # Dedup check for other categoriesresult = await self.deduplicator.deduplicate(candidate)if result.decision == DedupDecision.SKIP:@@ -86,23 +106,28 @@ class SessionCompressor:stats.created += 1await self._index_memory(memory)- elif result.decision == DedupDecision.UPDATE:- if result.similar_memories and result.merged_content:- candidate.content = result.merged_content- memory = await self.extractor.create_memory(candidate, user, session_id)- if memory:- memories.append(memory)- stats.updated += 1- await self._index_memory(memory)-elif result.decision == DedupDecision.MERGE:- if result.merged_content:- candidate.content = result.merged_content- memory = await self.extractor.create_memory(candidate, user, session_id)- if memory:- memories.append(memory)- stats.merged += 1- await self._index_memory(memory)+ # Only merge for supported categories+ if candidate.category in MERGE_SUPPORTED_CATEGORIES and result.similar_memories and viking_fs:+ target_memory = result.similar_memories[0]+ try:+ existing_content = await viking_fs.read_file(target_memory.uri)+ merged = await self.extractor._merge_memory(+ existing_content, candidate.content, candidate.category.value+ )+ if merged:+ await viking_fs.write_file(target_memory.uri, merged)+ target_memory.set_vectorize(Vectorize(text=merged))+ await self._index_memory(target_memory)+ stats.merged += 1+ else:+ stats.skipped += 1+ except Exception as e:+ logger.error(f"Failed to merge memory: {e}")+ stats.skipped += 1+ else:+ # events/cases don't support MERGE, treat as SKIP+ stats.skipped += 1# Extract URIs used in messages, create relationsused_uris = self._extract_used_uris(messages)@@ -110,7 +135,7 @@ class SessionCompressor:await self._create_relations(memories, used_uris)logger.info(- f"Memory extraction: created={stats.created}, updated={stats.updated}, "+ f"Memory extraction: created={stats.created}, "f"merged={stats.merged}, skipped={stats.skipped}")return memoriesdiff --git a/openviking/session/memory_deduplicator.py b/openviking/session/memory_deduplicator.pyindex 266efda68..1ecbdd809 100644--- a/openviking/session/memory_deduplicator.py+++ b/openviking/session/memory_deduplicator.py@@ -3,12 +3,12 @@"""Memory Deduplicator for OpenViking.-LLM-assisted deduplication with CREATE/UPDATE/MERGE/SKIP decisions (Mem0 inspired).+LLM-assisted deduplication with CREATE/MERGE/SKIP decisions"""from dataclasses import dataclassfrom enum import Enum-from typing import List, Optional+from typing import Listfrom openviking.core.context import Contextfrom openviking.models.embedder.base import EmbedResult@@ -26,7 +26,6 @@ class DedupDecision(str, Enum):"""Deduplication decision types."""CREATE = "create" # New memory, create directly- UPDATE = "update" # Update existing memoryMERGE = "merge" # Merge with existing memoriesSKIP = "skip" # Duplicate, skip@@ -38,7 +37,6 @@ class DedupResult:decision: DedupDecisioncandidate: CandidateMemorysimilar_memories: List[Context] # Similar existing memories- merged_content: Optional[str] = None # For UPDATE/MERGEreason: str = ""@@ -73,13 +71,12 @@ class MemoryDeduplicator:)# Step 2: LLM decision- decision, merged_content, reason = await self._llm_decision(candidate, similar_memories)+ decision, reason = await self._llm_decision(candidate, similar_memories)return DedupResult(decision=decision,candidate=candidate,similar_memories=similar_memories,- merged_content=merged_content,reason=reason,)@@ -108,7 +105,7 @@ class MemoryDeduplicator:collection=collection,query_vector=query_vector,limit=5,- filter={"category": category_value},+ filter= {"op": "and", "conds": [{"field": "category", "op": "must", "conds": [category_value]}, {"field": "is_leaf", "op": "must", "conds": [True]}]},)# Filter by similarity threshold@@ -129,12 +126,12 @@ class MemoryDeduplicator:self,candidate: CandidateMemory,similar_memories: List[Context],- ) -> tuple[DedupDecision, Optional[str], str]:+ ) -> tuple[DedupDecision, str]:"""Use LLM to decide deduplication action."""vlm = get_openviking_config().vlmif not vlm or not vlm.is_available():# Without LLM, default to CREATE (conservative)- return DedupDecision.CREATE, None, "LLM not available, defaulting to CREATE"+ return DedupDecision.CREATE, "LLM not available, defaulting to CREATE"# Format existing memories for promptexisting_formatted = []@@ -160,22 +157,20 @@ class MemoryDeduplicator:decision_str = data.get("decision", "create").lower()reason = data.get("reason", "")- merged_content = data.get("merged_content")# Map to enumdecision_map = {"create": DedupDecision.CREATE,- "update": DedupDecision.UPDATE,"merge": DedupDecision.MERGE,"skip": DedupDecision.SKIP,}decision = decision_map.get(decision_str, DedupDecision.CREATE)- return decision, merged_content, reason+ return decision, reasonexcept Exception as e:logger.warning(f"LLM dedup decision failed: {e}")- return DedupDecision.CREATE, None, f"LLM failed: {e}"+ return DedupDecision.CREATE, f"LLM failed: {e}"@staticmethoddef _cosine_similarity(vec_a: List[float], vec_b: List[float]) -> float:diff --git a/openviking/session/memory_extractor.py b/openviking/session/memory_extractor.pyindex 230acb40c..14fc7a21b 100644--- a/openviking/session/memory_extractor.py+++ b/openviking/session/memory_extractor.py@@ -154,9 +154,7 @@ class MemoryExtractor:session_id=session_id,user=user,)- logger.info(- f"uri {memory_uri} abstract: {candidate.abstract} content: {candidate.content}"- )+ logger.info(f"uri {memory_uri} abstract: {candidate.abstract} content: {candidate.content}")memory.set_vectorize(Vectorize(text=candidate.content))return memory@@ -198,20 +196,37 @@ class MemoryExtractor:return memoryasync def _append_to_profile(self, candidate: CandidateMemory, viking_fs) -> None:- """Append user profile information to profile.md."""+ """Update user profile - always merge with existing content."""uri = "viking://user/memories/profile.md"+ existing = ""+ try:+ existing = await viking_fs.read_file(uri) or ""+ except Exception:+ pass++ if not existing.strip():+ await viking_fs.write_file(uri=uri, content=candidate.content)+ logger.info(f"Created profile at {uri}")+ else:+ merged = await self._merge_memory(existing, candidate.content, "profile")+ content = merged if merged else candidate.content+ await viking_fs.write_file(uri=uri, content=content)+ logger.info(f"Merged profile info to {uri}")++ async def _merge_memory(self, existing: str, new: str, category: str) -> Optional[str]:+ """Use LLM to merge existing and new memory content."""+ vlm = get_openviking_config().vlm+ if not vlm or not vlm.is_available():+ return None++ prompt = render_prompt(+ "compression.memory_merge",+ {"existing_content": existing, "new_content": new, "category": category},+ )+try:- existing_content = ""- try:- existing_content = await viking_fs.read_file(uri) or ""- except Exception:- pass-- # Append new content- new_section = f"\n{candidate.content}\n"- updated_content = existing_content + new_section-- await viking_fs.write_file(uri=uri, content=updated_content)- logger.info(f"Appended profile info to {uri}")+ merged = await vlm.get_completion_async(prompt)+ return merged.strip() if merged else Noneexcept Exception as e:- logger.error(f"Failed to append profile: {e}")+ logger.error(f"Memory merge failed: {e}")+ return Nonediff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.pyindex 87b300bc0..ff656f0bc 100644--- a/openviking/storage/viking_vector_index_backend.py+++ b/openviking/storage/viking_vector_index_backend.py@@ -655,7 +655,6 @@ class VikingVectorIndexBackend(VikingDBInterface):records.append(record)- logger.info(f"Vector search returned {len(records)} records: {records}")return recordselse:# Pure filtering without vector search