Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix: simplify memory dedup decisions and fix retrieval recursion bug (#53)

qin-ctx7c4dbf27a2a3
diff --git a/openviking/core/context.py b/openviking/core/context.py
index 6d37fc187..64255917b 100644
--- a/openviking/core/context.py
+++ b/openviking/core/context.py
@@ -55,11 +55,12 @@ class Context:
meta: Optional[Dict[str, Any]] = None,
session_id: Optional[str] = None,
user: Optional[str] = None,
+ id: Optional[str] = None,
):
"""
Initialize a Context object.
"""
- self.id = str(uuid4())
+ self.id = id or str(uuid4())
self.uri = uri
self.parent_uri = parent_uri
self.is_leaf = is_leaf
diff --git a/openviking/prompts/templates/compression/dedup_decision.yaml b/openviking/prompts/templates/compression/dedup_decision.yaml
index 23b8ab637..c41fee019 100644
--- a/openviking/prompts/templates/compression/dedup_decision.yaml
+++ b/openviking/prompts/templates/compression/dedup_decision.yaml
@@ -1,8 +1,8 @@
metadata:
id: "compression.dedup_decision"
name: "Memory Deduplication Decision"
- description: "Decide whether new memory should CREATE/UPDATE/MERGE/SKIP"
- version: "1.1.0"
+ description: "Decide whether new memory should CREATE/MERGE/SKIP"
+ version: "2.0.0"
language: "en"
category: "compression"
@@ -35,24 +35,16 @@ template: |
**Existing Similar Memories**:
{{ existing_memories }}
- Please decide the operation type:
- - CREATE: This is a completely new memory, should be created
- - UPDATE: Candidate memory is a supplement/update to existing memory, should update the most relevant one
- - MERGE: Candidate memory is related to multiple existing memories, should be merged
+ Please decide:
- SKIP: Candidate memory duplicates existing memories, no need to save
+ - CREATE: This is completely new information, should be created
+ - MERGE: Candidate memory should be merged with existing memories
Return JSON format:
{
- "decision": "create|update|merge|skip",
- "reason": "Decision reason",
- "merged_content": "If UPDATE or MERGE, provide merged L2 content"
+ "decision": "skip|create|merge",
+ "reason": "Decision reason"
}
- Notes:
- - If candidate memory provides new information, choose UPDATE or MERGE
- - If candidate memory is completely duplicate, choose SKIP
- - If candidate memory is a new topic, choose CREATE
- - merged_content is only needed for UPDATE/MERGE, and should be complete L2 content
-
llm_config:
temperature: 0.0
diff --git a/openviking/prompts/templates/compression/memory_merge.yaml b/openviking/prompts/templates/compression/memory_merge.yaml
new file mode 100644
index 000000000..c1a133c24
--- /dev/null
+++ b/openviking/prompts/templates/compression/memory_merge.yaml
@@ -0,0 +1,41 @@
+metadata:
+ id: "compression.memory_merge"
+ name: "Memory Merge"
+ description: "Merge existing memory with new information"
+ version: "1.0.0"
+ language: "en"
+ category: "compression"
+
+variables:
+ - name: "existing_content"
+ type: "string"
+ description: "Existing memory content"
+ required: true
+ - name: "new_content"
+ type: "string"
+ description: "New memory content to merge"
+ required: true
+ - name: "category"
+ type: "string"
+ description: "Memory category (profile, preferences, etc.)"
+ required: true
+
+template: |
+ Merge the following memory information into a single, coherent content.
+
+ **Category**: {{ category }}
+
+ **Existing Content:**
+ {{ existing_content }}
+
+ **New Information:**
+ {{ new_content }}
+
+ Requirements:
+ - Remove duplicate information
+ - Keep the most up-to-date details
+ - Maintain a coherent narrative
+ - Output ONLY the merged content, no explanation
+
+llm_config:
+ temperature: 0.0
diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.py
index 6881e5691..a0a54b2f1 100644
--- a/openviking/retrieve/hierarchical_retriever.py
+++ b/openviking/retrieve/hierarchical_retriever.py
@@ -237,6 +237,8 @@ class HierarchicalRetriever:
def passes_threshold(score: float) -> bool:
"""Check if score passes threshold."""
+ if not self._rerank_client or mode != RetrieverMode.THINKING:
+ return True
if score_gte:
return score >= effective_threshold
return score > effective_threshold
@@ -277,7 +279,7 @@ class HierarchicalRetriever:
current_score = -temp_score
if current_uri in visited:
continue
-
+ visited.add(current_uri)
logger.info(f"[RecursiveSearch] Entering URI: {current_uri}")
pre_filter_limit = max(limit * 2, 20)
@@ -318,11 +320,11 @@ class HierarchicalRetriever:
if passes_threshold(final_score) and uri not in visited:
r["_final_score"] = final_score
collected.append(r)
- visited.add(uri)
logger.info(
f"[RecursiveSearch] Added URI: {uri} to candidates with score: {final_score}"
)
- if not r.get("is_leaf"):
+ if r.get("is_leaf"):
+ visited.add(uri)
continue
heapq.heappush(dir_queue, (-final_score, uri))
else:
diff --git a/openviking/session/compressor.py b/openviking/session/compressor.py
index 8c4981dc8..b6d9aae51 100644
--- a/openviking/session/compressor.py
+++ b/openviking/session/compressor.py
@@ -10,24 +10,33 @@ Uses MemoryExtractor for 6-category extraction and MemoryDeduplicator for LLM-ba
from dataclasses import dataclass
from typing import Dict, List
-from openviking.core.context import Context
+from openviking.core.context import Context, Vectorize
from openviking.message import Message
from openviking.storage import VikingDBManager
from openviking.storage.viking_fs import get_viking_fs
from openviking.utils import get_logger
from .memory_deduplicator import DedupDecision, MemoryDeduplicator
-from .memory_extractor import MemoryExtractor
+from .memory_extractor import MemoryCategory, MemoryExtractor
logger = get_logger(__name__)
+# Categories that always merge (skip dedup)
+ALWAYS_MERGE_CATEGORIES = {MemoryCategory.PROFILE}
+
+# Categories that support MERGE decision
+MERGE_SUPPORTED_CATEGORIES = {
+ MemoryCategory.PREFERENCES,
+ MemoryCategory.ENTITIES,
+ MemoryCategory.PATTERNS,
+}
+
@dataclass
class ExtractionStats:
"""Statistics for memory extraction."""
created: int = 0
- updated: int = 0
merged: int = 0
skipped: int = 0
@@ -71,8 +80,19 @@ class SessionCompressor:
memories: List[Context] = []
stats = ExtractionStats()
+ viking_fs = get_viking_fs()
for candidate in candidates:
+ # Profile: skip dedup, always merge
+ if candidate.category in ALWAYS_MERGE_CATEGORIES:
+ memory = await self.extractor.create_memory(candidate, user, session_id)
+ if memory:
+ memories.append(memory)
+ stats.created += 1
+ await self._index_memory(memory)
+ continue
+
+ # Dedup check for other categories
result = await self.deduplicator.deduplicate(candidate)
if result.decision == DedupDecision.SKIP:
@@ -86,23 +106,28 @@ class SessionCompressor:
stats.created += 1
await self._index_memory(memory)
- elif result.decision == DedupDecision.UPDATE:
- if result.similar_memories and result.merged_content:
- candidate.content = result.merged_content
- memory = await self.extractor.create_memory(candidate, user, session_id)
- if memory:
- memories.append(memory)
- stats.updated += 1
- await self._index_memory(memory)
-
elif result.decision == DedupDecision.MERGE:
- if result.merged_content:
- candidate.content = result.merged_content
- memory = await self.extractor.create_memory(candidate, user, session_id)
- if memory:
- memories.append(memory)
- stats.merged += 1
- await self._index_memory(memory)
+ # Only merge for supported categories
+ if candidate.category in MERGE_SUPPORTED_CATEGORIES and result.similar_memories and viking_fs:
+ target_memory = result.similar_memories[0]
+ try:
+ existing_content = await viking_fs.read_file(target_memory.uri)
+ merged = await self.extractor._merge_memory(
+ existing_content, candidate.content, candidate.category.value
+ )
+ if merged:
+ await viking_fs.write_file(target_memory.uri, merged)
+ target_memory.set_vectorize(Vectorize(text=merged))
+ await self._index_memory(target_memory)
+ stats.merged += 1
+ else:
+ stats.skipped += 1
+ except Exception as e:
+ logger.error(f"Failed to merge memory: {e}")
+ stats.skipped += 1
+ else:
+ # events/cases don't support MERGE, treat as SKIP
+ stats.skipped += 1
# Extract URIs used in messages, create relations
used_uris = self._extract_used_uris(messages)
@@ -110,7 +135,7 @@ class SessionCompressor:
await self._create_relations(memories, used_uris)
logger.info(
- f"Memory extraction: created={stats.created}, updated={stats.updated}, "
+ f"Memory extraction: created={stats.created}, "
f"merged={stats.merged}, skipped={stats.skipped}"
)
return memories
diff --git a/openviking/session/memory_deduplicator.py b/openviking/session/memory_deduplicator.py
index 266efda68..1ecbdd809 100644
--- a/openviking/session/memory_deduplicator.py
+++ b/openviking/session/memory_deduplicator.py
@@ -3,12 +3,12 @@
"""
Memory Deduplicator for OpenViking.
-LLM-assisted deduplication with CREATE/UPDATE/MERGE/SKIP decisions (Mem0 inspired).
+LLM-assisted deduplication with CREATE/MERGE/SKIP decisions
"""
from dataclasses import dataclass
from enum import Enum
-from typing import List, Optional
+from typing import List
from openviking.core.context import Context
from openviking.models.embedder.base import EmbedResult
@@ -26,7 +26,6 @@ class DedupDecision(str, Enum):
"""Deduplication decision types."""
CREATE = "create" # New memory, create directly
- UPDATE = "update" # Update existing memory
MERGE = "merge" # Merge with existing memories
SKIP = "skip" # Duplicate, skip
@@ -38,7 +37,6 @@ class DedupResult:
decision: DedupDecision
candidate: CandidateMemory
similar_memories: List[Context] # Similar existing memories
- merged_content: Optional[str] = None # For UPDATE/MERGE
reason: str = ""
@@ -73,13 +71,12 @@ class MemoryDeduplicator:
)
# Step 2: LLM decision
- decision, merged_content, reason = await self._llm_decision(candidate, similar_memories)
+ decision, reason = await self._llm_decision(candidate, similar_memories)
return DedupResult(
decision=decision,
candidate=candidate,
similar_memories=similar_memories,
- merged_content=merged_content,
reason=reason,
)
@@ -108,7 +105,7 @@ class MemoryDeduplicator:
collection=collection,
query_vector=query_vector,
limit=5,
- filter={"category": category_value},
+ filter= {"op": "and", "conds": [{"field": "category", "op": "must", "conds": [category_value]}, {"field": "is_leaf", "op": "must", "conds": [True]}]},
)
# Filter by similarity threshold
@@ -129,12 +126,12 @@ class MemoryDeduplicator:
self,
candidate: CandidateMemory,
similar_memories: List[Context],
- ) -> tuple[DedupDecision, Optional[str], str]:
+ ) -> tuple[DedupDecision, str]:
"""Use LLM to decide deduplication action."""
vlm = get_openviking_config().vlm
if not vlm or not vlm.is_available():
# Without LLM, default to CREATE (conservative)
- return DedupDecision.CREATE, None, "LLM not available, defaulting to CREATE"
+ return DedupDecision.CREATE, "LLM not available, defaulting to CREATE"
# Format existing memories for prompt
existing_formatted = []
@@ -160,22 +157,20 @@ class MemoryDeduplicator:
decision_str = data.get("decision", "create").lower()
reason = data.get("reason", "")
- merged_content = data.get("merged_content")
# Map to enum
decision_map = {
"create": DedupDecision.CREATE,
- "update": DedupDecision.UPDATE,
"merge": DedupDecision.MERGE,
"skip": DedupDecision.SKIP,
}
decision = decision_map.get(decision_str, DedupDecision.CREATE)
- return decision, merged_content, reason
+ return decision, reason
except Exception as e:
logger.warning(f"LLM dedup decision failed: {e}")
- return DedupDecision.CREATE, None, f"LLM failed: {e}"
+ return DedupDecision.CREATE, f"LLM failed: {e}"
@staticmethod
def _cosine_similarity(vec_a: List[float], vec_b: List[float]) -> float:
diff --git a/openviking/session/memory_extractor.py b/openviking/session/memory_extractor.py
index 230acb40c..14fc7a21b 100644
--- a/openviking/session/memory_extractor.py
+++ b/openviking/session/memory_extractor.py
@@ -154,9 +154,7 @@ class MemoryExtractor:
session_id=session_id,
user=user,
)
- logger.info(
- f"uri {memory_uri} abstract: {candidate.abstract} content: {candidate.content}"
- )
+ logger.info(f"uri {memory_uri} abstract: {candidate.abstract} content: {candidate.content}")
memory.set_vectorize(Vectorize(text=candidate.content))
return memory
@@ -198,20 +196,37 @@ class MemoryExtractor:
return memory
async def _append_to_profile(self, candidate: CandidateMemory, viking_fs) -> None:
- """Append user profile information to profile.md."""
+ """Update user profile - always merge with existing content."""
uri = "viking://user/memories/profile.md"
+ existing = ""
+ try:
+ existing = await viking_fs.read_file(uri) or ""
+ except Exception:
+ pass
+
+ if not existing.strip():
+ await viking_fs.write_file(uri=uri, content=candidate.content)
+ logger.info(f"Created profile at {uri}")
+ else:
+ merged = await self._merge_memory(existing, candidate.content, "profile")
+ content = merged if merged else candidate.content
+ await viking_fs.write_file(uri=uri, content=content)
+ logger.info(f"Merged profile info to {uri}")
+
+ async def _merge_memory(self, existing: str, new: str, category: str) -> Optional[str]:
+ """Use LLM to merge existing and new memory content."""
+ vlm = get_openviking_config().vlm
+ if not vlm or not vlm.is_available():
+ return None
+
+ prompt = render_prompt(
+ "compression.memory_merge",
+ {"existing_content": existing, "new_content": new, "category": category},
+ )
+
try:
- existing_content = ""
- try:
- existing_content = await viking_fs.read_file(uri) or ""
- except Exception:
- pass
-
- # Append new content
- new_section = f"\n{candidate.content}\n"
- updated_content = existing_content + new_section
-
- await viking_fs.write_file(uri=uri, content=updated_content)
- logger.info(f"Appended profile info to {uri}")
+ merged = await vlm.get_completion_async(prompt)
+ return merged.strip() if merged else None
except Exception as e:
- logger.error(f"Failed to append profile: {e}")
+ logger.error(f"Memory merge failed: {e}")
+ return None
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index 87b300bc0..ff656f0bc 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -655,7 +655,6 @@ class VikingVectorIndexBackend(VikingDBInterface):
records.append(record)
- logger.info(f"Vector search returned {len(records)} records: {records}")
return records
else:
# Pure filtering without vector search