Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
refactor(memory): redesign extraction/dedup flow and add conflict-aware delete handling (#225)
kkkwjxa06219968b2c
diff --git a/docs/en/concepts/08-session.md b/docs/en/concepts/08-session.mdindex 0ebb6282c..38737edfc 100644--- a/docs/en/concepts/08-session.md+++ b/docs/en/concepts/08-session.md@@ -136,19 +136,20 @@ Messages → LLM Extract → Candidate Memories↓Vector Pre-filter → Find Similar Memories↓-LLM Dedup Decision → CREATE/UPDATE/MERGE/SKIP+LLM Dedup Decision → candidate(skip/create/none) + item(merge/delete)↓Write to AGFS → Vectorize```### Dedup Decisions-| Decision | Description |-|----------|-------------|-| `CREATE` | New memory, create directly |-| `UPDATE` | Update existing memory |-| `MERGE` | Merge multiple memories |-| `SKIP` | Duplicate, skip |+| Level | Decision | Description |+|------|----------|-------------|+| Candidate | `skip` | Candidate is duplicate, skip and do nothing |+| Candidate | `create` | Create candidate memory (optionally delete conflicting existing memories first) |+| Candidate | `none` | Do not create candidate; resolve existing memories by item decisions |+| Per-existing item | `merge` | Merge candidate content into specified existing memory |+| Per-existing item | `delete` | Delete specified conflicting existing memory |## Storage Structurediff --git a/examples/memory_demo.py b/examples/memory_demo.pynew file mode 100644index 000000000..de9320cf2--- /dev/null+++ b/examples/memory_demo.py@@ -0,0 +1,179 @@+"""OpenViking memory dedup demo with REAL LLM decisions (no fake/mock).++This demo focuses on the user preference scenario:+1) "I like apples." -> expected create+2) "I like strawberries." -> expected create+3) "I like Fuji apples." -> expected merge/none with apple memory+4) "I do not like fruits anymore." -> expected delete old positive fruit preferences+5) repeat negative preference -> expected skip/none++The script prints commit and find results after each round so you can inspect+whether memory handling is reasonable.++Usage:+ export OPENVIKING_CONFIG_FILE=ov.conf+ python examples/memory_dedup_cases_demo.py+"""++from __future__ import annotations++import argparse+import os+import shutil+from dataclasses import dataclass+from pathlib import Path+from typing import Any, Iterable, List++from openviking.message.part import TextPart+from openviking.sync_client import SyncOpenViking+++@dataclass+class RoundCase:+ title: str+ user_text: str+ expected: str+++def _print_section(title: str, body: str = "") -> None:+ print("\n" + "=" * 80)+ print(title)+ if body:+ print("-" * 80)+ print(body)+++def _safe_list(items: Iterable[Any]) -> list:+ try:+ return list(items)+ except Exception:+ return []+++def _format_find_result(result: Any, max_items: int = 6) -> str:+ memories = _safe_list(getattr(result, "memories", []))+ if not memories:+ return "(no memory hit)"++ lines: List[str] = []+ for i, mem in enumerate(memories[:max_items], 1):+ score = getattr(mem, "score", None)+ score_s = "n/a" if score is None else f"{float(score):.4f}"+ abstract = getattr(mem, "abstract", "") or ""+ uri = getattr(mem, "uri", "")+ lines.append(f"{i}. score={score_s} | {abstract} | {uri}")+ return "\n".join(lines)+++def main() -> int:+ parser = argparse.ArgumentParser(description="OpenViking real-LLM dedup cases demo")+ parser.add_argument(+ "--path",+ default="./ov_data_dedup_cases_demo",+ help="Fixed demo storage path. This script clears it at startup.",+ )+ parser.add_argument(+ "--wait-timeout",+ type=float,+ default=60.0,+ help="Queue wait timeout in seconds.",+ )+ args = parser.parse_args()++ if not os.environ.get("OPENVIKING_CONFIG_FILE"):+ repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))+ cfg = os.path.join(repo_root, "ov.conf")+ if os.path.exists(cfg):+ os.environ["OPENVIKING_CONFIG_FILE"] = cfg++ data_path = Path(args.path)+ if data_path.exists():+ shutil.rmtree(data_path)+ data_path.mkdir(parents=True, exist_ok=True)++ client = SyncOpenViking(path=str(data_path))+ client.initialize()++ try:+ client.is_healthy()+ sess_info = client.create_session()+ session_id = sess_info["session_id"]+ sess = client.session(session_id)++ rounds = [+ RoundCase(+ title="Round 1",+ user_text=(+ "我是一名程序员。"+ "我爱吃苹果。"+ "我爱吃草莓。"+ "我每天早上7点起床。"+ "我通勤主要骑共享单车。"+ "我习惯在周末整理书桌。"+ "我最常用的云盘是Dropbox。"+ "我对坚果过敏,尤其是腰果。"+ "我最近在学西班牙语。"+ "我喜欢在雨天听爵士乐。"+ "我的常用笔记软件是Obsidian。"+ "我每周三晚上会去游泳。"+ "我偏好27英寸的外接显示器。"+ ),+ expected="Expected dedup: create multiple unrelated memories",+ ),+ RoundCase(+ title="Round 2",+ user_text="我爱吃红富士苹果。我是外卖员。",+ expected="Expected dedup: none+merge (细化苹果偏好)",+ ),+ RoundCase(+ title="Round 3",+ user_text="我不爱吃水果了,把之前关于喜欢水果的偏好作废。",+ expected="Expected dedup: delete old positive fruit preferences",+ ),+ ]++ queries = [+ "我喜欢吃什么?",+ "我是做什么工作的?",+ ]++ for r in rounds:+ sess.add_message("user", parts=[TextPart(text=r.user_text)])+ sess.add_message("assistant", parts=[TextPart(text="收到。")])+ commit_result = sess.commit()++ _print_section(+ f"{r.title} commit",+ body=(f"user={r.user_text}\n{r.expected}\ncommit={commit_result}"),+ )++ try:+ client.wait_processed(timeout=args.wait_timeout)+ except Exception:+ pass++ for q in queries:+ try:+ result = client.find(q, target_uri="viking://user/memories", limit=8)+ _print_section(f"{r.title} find: {q}", body=_format_find_result(result))+ except Exception as e:+ _print_section(f"{r.title} find: {q} (failed)", body=str(e))++ _print_section(+ "Done",+ body=(+ "Check whether the later rounds are dominated by negative fruit preference.\n"+ "If old positive fruit preferences disappear or stop ranking high, delete likely worked."+ ),+ )+ finally:+ try:+ client.close()+ except Exception:+ pass++ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/openviking/prompts/templates/compression/dedup_decision.yaml b/openviking/prompts/templates/compression/dedup_decision.yamlindex c41fee019..a89d61e98 100644--- a/openviking/prompts/templates/compression/dedup_decision.yaml+++ b/openviking/prompts/templates/compression/dedup_decision.yaml@@ -1,8 +1,8 @@metadata:id: "compression.dedup_decision"name: "Memory Deduplication Decision"- description: "Decide whether new memory should CREATE/MERGE/SKIP"- version: "2.0.0"+ description: "Decide candidate action (skip/create/none) and per-memory actions (merge/delete)"+ version: "3.3.1"language: "en"category: "compression"@@ -25,25 +25,79 @@ variables:required: truetemplate: |- Determine how to handle this candidate memory.+ You are deciding how to update long-term memory with:+ 1) one candidate memory (new fact)+ 2) existing similar memories (retrieved from store)- **Candidate Memory**:- Abstract: {{ candidate_abstract }}- Overview: {{ candidate_overview }}- Content: {{ candidate_content }}+ Candidate memory:+ - Abstract: {{ candidate_abstract }}+ - Overview: {{ candidate_overview }}+ - Content: {{ candidate_content }}- **Existing Similar Memories**:+ Existing similar memories:{{ existing_memories }}- Please decide:- - SKIP: Candidate memory duplicates existing memories, no need to save- - CREATE: This is completely new information, should be created- - MERGE: Candidate memory should be merged with existing memories+ Goal:+ Keep memory consistent and useful while minimizing destructive edits.- Return JSON format:+ Candidate-level decision:+ - skip:+ Use only when candidate adds no useful new information (duplicate, paraphrase,+ or too weak/uncertain). No memory should change.+ - create:+ Use when candidate is a valid new memory that should be stored as a separate item.+ It may optionally delete fully-invalidated existing memories.+ - none:+ Use when candidate itself should not be stored, but existing memories should be+ reconciled with per-item actions.++ Existing-memory per-item action:+ - merge:+ Existing memory and candidate are about the same subject and should be unified.+ Use for refinement, correction, partial conflict, or complementary details.+ - delete:+ Existing memory must be removed only if candidate fully invalidates the entire+ existing memory (not just one sub-part).++ Critical delete boundary:+ - If conflict is partial (some statements conflict, others remain valid), DO NOT delete.+ Use merge instead so non-conflicting information is preserved.+ - Delete only when the whole existing memory is obsolete/invalidated.+ - Topic/facet mismatch must never be deleted. If candidate is about one facet+ (for example any single preference facet), existing memories from other facets+ must be omitted from list (treated as unchanged).++ Decision guidance:+ - Prefer skip when candidate is redundant.+ - Prefer none+merge for same-subject updates and partial contradictions.+ - Prefer create for clearly new independent memory.+ - If uncertain, choose non-destructive behavior (skip or merge), not delete.++ Practical checklist before emitting each list item:+ 1) Is existing memory about the same topic/facet as candidate?+ 2) If no, do not include it in list.+ 3) If yes and candidate only updates part of it, use merge.+ 4) Use delete only when candidate explicitly invalidates the whole existing memory.++ Hard constraints:+ - If decision is "skip", do not return "list".+ - If any list item uses "merge", decision must be "none".+ - If decision is "create", list can be empty or contain delete items only.+ - Use uri exactly from existing memories list.+ - Omit unchanged existing memories from list.+ - Return JSON only, no prose.++ Return JSON in this exact structure:{- "decision": "skip|create|merge",- "reason": "Decision reason"+ "decision": "skip|create|none",+ "reason": "short reason",+ "list": [+ {+ "uri": "<existing memory uri>",+ "decide": "merge|delete",+ "reason": "short reason (for delete, explain full invalidation)"+ }+ ]}llm_config:diff --git a/openviking/prompts/templates/compression/memory_extraction.yaml b/openviking/prompts/templates/compression/memory_extraction.yamlindex 849cb99c3..a8bd1f1bc 100644--- a/openviking/prompts/templates/compression/memory_extraction.yaml+++ b/openviking/prompts/templates/compression/memory_extraction.yaml@@ -2,7 +2,7 @@ metadata:id: "compression.memory_extraction"name: "Memory Extraction (Three-Level)"description: "Extract memories from session context using L0/L1/L2 three-level structure"- version: "5.0.0"+ version: "5.1.0"language: "en"category: "compression"@@ -90,6 +90,15 @@ template: |- Characteristics: Changeable choices, styles- Test: Can it be described as "User prefers/likes..."+ ### Preference Granularity (Important)+ - For category `preferences`, each memory item should represent one independently+ updatable preference unit (single facet).+ - Do NOT mix unrelated preference facets in one memory item.+ Examples of different facets: food, commute, schedule, tools, music, study habits.+ - If the conversation contains multiple facets, output multiple `preferences` items.+ - This granularity is required so future updates/conflicts can affect only the+ relevant memory without damaging unrelated preferences.+**entities** - Entities (continuously existing nouns)- Core: Describes "current state of something"- Characteristics: Entities with lifecycle (person/project/organization)@@ -166,6 +175,43 @@ template: |```❌ **Bad**: abstract says "Code preferences" (too general) or "No type hints" (too specific, cannot merge other style preferences)+ ## preferences Granularity Example+ ❌ **Bad (mixed facets in one memory)**:+ ```json+ {+ "category": "preferences",+ "abstract": "User preferences: likes apples, commutes by bike, uses Obsidian",+ "overview": "Mixed food/commute/tool preferences",+ "content": "User likes apples, usually commutes by bike, and prefers Obsidian."+ }+ ```++ ✅ **Good (split by independently updatable facets)**:+ ```json+ {+ "memories": [+ {+ "category": "preferences",+ "abstract": "Food preference: Likes apples",+ "overview": "## Preference Domain\\n- **Domain**: Food\\n\\n## Specific Preference\\n- Likes apples",+ "content": "User shows a food preference for apples."+ },+ {+ "category": "preferences",+ "abstract": "Commute preference: Usually rides a bike",+ "overview": "## Preference Domain\\n- **Domain**: Commute\\n\\n## Specific Preference\\n- Usually rides a bike",+ "content": "User usually commutes by bike."+ },+ {+ "category": "preferences",+ "abstract": "Tool preference: Uses Obsidian for notes",+ "overview": "## Preference Domain\\n- **Domain**: Tools\\n\\n## Specific Preference\\n- Uses Obsidian for notes",+ "content": "User prefers Obsidian as note-taking software."+ }+ ]+ }+ ```+## entities Example (Merge type)✅ **Good**:```json@@ -228,8 +274,7 @@ template: |- The values of "abstract", "overview", and "content" MUST be written in {{ output_language }} (if output_language is "auto", use the dominant language in recent_messages).- Only extract truly valuable personalized information- If nothing worth recording, return {"memories": []}- - Preferences should be aggregated by topic, similar preferences should be merged into one memory+ - For preferences, keep each memory as one independently updatable facet; do not combine unrelated facets in one memoryllm_config:temperature: 0.0-diff --git a/openviking/prompts/templates/compression/memory_merge.yaml b/openviking/prompts/templates/compression/memory_merge.yamlindex 868a6faac..6fef1ec01 100644--- a/openviking/prompts/templates/compression/memory_merge.yaml+++ b/openviking/prompts/templates/compression/memory_merge.yaml@@ -2,7 +2,7 @@ metadata:id: "compression.memory_merge"name: "Memory Merge"description: "Merge existing memory with new information"- version: "1.0.0"+ version: "1.1.0"language: "en"category: "compression"@@ -40,6 +40,8 @@ template: |Requirements:- Remove duplicate information- Keep the most up-to-date details+ - If there is a conflict, update only the conflicting statement with the newer fact+ - Preserve non-conflicting details from existing content; do not drop unrelated information- Maintain a coherent narrative- Output ONLY the merged content, no explanation- Output MUST be written in **{{ output_language }}** (if output_language is "auto", infer dominant language from inputs)diff --git a/openviking/session/__init__.py b/openviking/session/__init__.pyindex d9ee9efbb..7aa033d17 100644--- a/openviking/session/__init__.py+++ b/openviking/session/__init__.py@@ -3,7 +3,13 @@"""Session management module."""from openviking.session.compressor import ExtractionStats, SessionCompressor-from openviking.session.memory_deduplicator import DedupDecision, DedupResult, MemoryDeduplicator+from openviking.session.memory_deduplicator import (+ DedupDecision,+ DedupResult,+ ExistingMemoryAction,+ MemoryActionDecision,+ MemoryDeduplicator,+)from openviking.session.memory_extractor import CandidateMemory, MemoryCategory, MemoryExtractorfrom openviking.session.session import Session, SessionCompression, SessionStats@@ -22,5 +28,7 @@ __all__ = [# Memory Deduplicator"MemoryDeduplicator","DedupDecision",+ "MemoryActionDecision",+ "ExistingMemoryAction","DedupResult",]diff --git a/openviking/session/compressor.py b/openviking/session/compressor.pyindex a4b498b95..befa49f4f 100644--- a/openviking/session/compressor.py+++ b/openviking/session/compressor.py@@ -17,8 +17,8 @@ from openviking.storage.viking_fs import get_viking_fsfrom openviking_cli.session.user_id import UserIdentifierfrom openviking_cli.utils import get_logger-from .memory_deduplicator import DedupDecision, MemoryDeduplicator-from .memory_extractor import MemoryCategory, MemoryExtractor+from .memory_deduplicator import DedupDecision, MemoryActionDecision, MemoryDeduplicator+from .memory_extractor import CandidateMemory, MemoryCategory, MemoryExtractorlogger = get_logger(__name__)@@ -39,6 +39,7 @@ class ExtractionStats:created: int = 0merged: int = 0+ deleted: int = 0skipped: int = 0@@ -63,6 +64,47 @@ class SessionCompressor:logger.info(f"Enqueued memory for vectorization: {memory.uri}")return True+ async def _merge_into_existing(+ self,+ candidate: CandidateMemory,+ target_memory: Context,+ viking_fs,+ ) -> bool:+ """Merge candidate content into an existing memory file."""+ try:+ existing_content = await viking_fs.read_file(target_memory.uri)+ merged = await self.extractor._merge_memory(+ existing_content,+ candidate.content,+ candidate.category.value,+ output_language=candidate.language,+ )+ if not merged:+ return False++ await viking_fs.write_file(target_memory.uri, merged)+ target_memory.set_vectorize(Vectorize(text=merged))+ await self._index_memory(target_memory)+ return True+ except Exception as e:+ logger.error(f"Failed to merge memory {target_memory.uri}: {e}")+ return False++ async def _delete_existing_memory(self, memory: Context, viking_fs) -> bool:+ """Hard delete an existing memory file and clean up its vector record."""+ try:+ await viking_fs.rm(memory.uri, recursive=False)+ except Exception as e:+ logger.error(f"Failed to delete memory file {memory.uri}: {e}")+ return False++ try:+ # rm() already syncs vector deletion in most cases; keep this as a safe fallback.+ await self.vikingdb.remove_by_uri("context", memory.uri)+ except Exception as e:+ logger.warning(f"Failed to remove vector record for {memory.uri}: {e}")+ return True+async def extract_long_term_memories(self,messages: List[Message],@@ -95,46 +137,64 @@ class SessionCompressor:# Dedup check for other categoriesresult = await self.deduplicator.deduplicate(candidate)+ actions = result.actions or []+ decision = result.decision++ # Safety net: create+merge should be treated as none.+ if decision == DedupDecision.CREATE and any(+ a.decision == MemoryActionDecision.MERGE for a in actions+ ):+ logger.warning(+ f"Dedup returned create with merge action, normalizing to none: "+ f"{candidate.abstract}"+ )+ decision = DedupDecision.NONE- if result.decision == DedupDecision.SKIP:+ if decision == DedupDecision.SKIP:stats.skipped += 1continue- if result.decision == DedupDecision.CREATE:+ if decision == DedupDecision.NONE:+ if not actions:+ stats.skipped += 1+ continue++ for action in actions:+ if action.decision == MemoryActionDecision.DELETE:+ if viking_fs and await self._delete_existing_memory(+ action.memory, viking_fs+ ):+ stats.deleted += 1+ else:+ stats.skipped += 1+ elif action.decision == MemoryActionDecision.MERGE:+ if candidate.category in MERGE_SUPPORTED_CATEGORIES and viking_fs:+ if await self._merge_into_existing(candidate, action.memory, viking_fs):+ stats.merged += 1+ else:+ stats.skipped += 1+ else:+ # events/cases don't support MERGE, treat as SKIP+ stats.skipped += 1+ continue++ if decision == DedupDecision.CREATE:+ # create can optionally include delete actions (delete first, then create)+ for action in actions:+ if action.decision == MemoryActionDecision.DELETE:+ if viking_fs and await self._delete_existing_memory(+ action.memory, viking_fs+ ):+ stats.deleted += 1+ else:+ stats.skipped += 1+memory = await self.extractor.create_memory(candidate, user, session_id)if memory:memories.append(memory)stats.created += 1await self._index_memory(memory)-- elif result.decision == DedupDecision.MERGE:- # Only merge for supported categories- if (- candidate.category in MERGE_SUPPORTED_CATEGORIES- and result.similar_memories- and viking_fs- ):- target_memory = result.similar_memories[0]- try:- existing_content = await viking_fs.read_file(target_memory.uri)- merged = await self.extractor._merge_memory(- existing_content,- candidate.content,- candidate.category.value,- output_language=candidate.language,- )- if merged:- await viking_fs.write_file(target_memory.uri, merged)- target_memory.set_vectorize(Vectorize(text=merged))- await self._index_memory(target_memory)- stats.merged += 1- else:- stats.skipped += 1- except Exception as e:- logger.error(f"Failed to merge memory: {e}")- stats.skipped += 1else:- # events/cases don't support MERGE, treat as SKIPstats.skipped += 1# Extract URIs used in messages, create relations@@ -144,7 +204,7 @@ class SessionCompressor:logger.info(f"Memory extraction: created={stats.created}, "- f"merged={stats.merged}, skipped={stats.skipped}"+ f"merged={stats.merged}, deleted={stats.deleted}, skipped={stats.skipped}")return memoriesdiff --git a/openviking/session/memory_deduplicator.py b/openviking/session/memory_deduplicator.pyindex 83121de25..9bbc0e830 100644--- a/openviking/session/memory_deduplicator.py+++ b/openviking/session/memory_deduplicator.py@@ -3,12 +3,14 @@"""Memory Deduplicator for OpenViking.-LLM-assisted deduplication with CREATE/MERGE/SKIP decisions+LLM-assisted deduplication with candidate-level skip/create/none decisions and+per-existing merge/delete actions."""+import refrom dataclasses import dataclassfrom enum import Enum-from typing import List+from typing import Dict, List, Optionalfrom openviking.core.context import Contextfrom openviking.models.embedder.base import EmbedResult@@ -25,9 +27,25 @@ logger = get_logger(__name__)class DedupDecision(str, Enum):"""Deduplication decision types."""- CREATE = "create" # New memory, create directly- MERGE = "merge" # Merge with existing memoriesSKIP = "skip" # Duplicate, skip+ CREATE = "create" # Create candidate memory+ NONE = "none" # No candidate creation; resolve existing memories only+++class MemoryActionDecision(str, Enum):+ """Decision for each existing memory candidate."""++ MERGE = "merge" # Merge candidate into existing memory+ DELETE = "delete" # Delete conflicting existing memory+++@dataclass+class ExistingMemoryAction:+ """Decision for one existing memory."""++ memory: Context+ decision: MemoryActionDecision+ reason: str = ""@dataclass@@ -37,13 +55,22 @@ class DedupResult:decision: DedupDecisioncandidate: CandidateMemorysimilar_memories: List[Context] # Similar existing memories+ actions: Optional[List[ExistingMemoryAction]] = Nonereason: str = ""class MemoryDeduplicator:"""Handles memory deduplication with LLM decision making."""- SIMILARITY_THRESHOLD = 0.7 # Vector similarity threshold for pre-filtering+ SIMILARITY_THRESHOLD = 0.0 # Vector similarity threshold for pre-filtering+ MAX_PROMPT_SIMILAR_MEMORIES = 5 # Number of similar memories sent to LLM+ CATEGORY_URI_PREFIX = {+ "preferences": "viking://user/memories/preferences/",+ "entities": "viking://user/memories/entities/",+ "events": "viking://user/memories/events/",+ "cases": "viking://agent/memories/cases/",+ "patterns": "viking://agent/memories/patterns/",+ }def __init__(self,@@ -67,16 +94,18 @@ class MemoryDeduplicator:decision=DedupDecision.CREATE,candidate=candidate,similar_memories=[],+ actions=[],reason="No similar memories found",)# Step 2: LLM decision- decision, reason = await self._llm_decision(candidate, similar_memories)+ decision, reason, actions = await self._llm_decision(candidate, similar_memories)return DedupResult(decision=decision,candidate=candidate,similar_memories=similar_memories,+ actions=None if decision == DedupDecision.SKIP else actions,reason=reason,)@@ -96,32 +125,54 @@ class MemoryDeduplicator:# Determine collection and filter based on categorycollection = "context"- # Build category filter- category_value = candidate.category.value+ category_uri_prefix = self.CATEGORY_URI_PREFIX.get(candidate.category.value, "")++ # Build filter by memory scope + uri prefix (schema does not have category field yet).+ filter_conds = [+ {"field": "context_type", "op": "must", "conds": ["memory"]},+ {"field": "is_leaf", "op": "must", "conds": [True]},+ ]+ if category_uri_prefix:+ filter_conds.append({"field": "uri", "op": "prefix", "prefix": category_uri_prefix})+ dedup_filter = {"op": "and", "conds": filter_conds}+ logger.debug(+ "Dedup prefilter candidate category=%s filter=%s",+ candidate.category.value,+ dedup_filter,+ )try:- # Search with category filter+ # Search with memory-scope filter.results = await self.vikingdb.search(collection=collection,query_vector=query_vector,limit=5,- filter={- "op": "and",- "conds": [- {"field": "category", "op": "must", "conds": [category_value]},- {"field": "is_leaf", "op": "must", "conds": [True]},- ],- },+ filter=dedup_filter,)# Filter by similarity thresholdsimilar = []+ logger.debug(+ "Dedup prefilter raw hits=%d threshold=%.2f",+ len(results),+ self.SIMILARITY_THRESHOLD,+ )for result in results:- if result.get("score", 0) >= self.SIMILARITY_THRESHOLD:+ score = float(result.get("_score", result.get("score", 0)) or 0)+ logger.debug(+ "Dedup hit score=%.4f uri=%s abstract=%s",+ score,+ result.get("uri", ""),+ result.get("abstract", ""),+ )+ if score >= self.SIMILARITY_THRESHOLD:# Reconstruct Context objectcontext = Context.from_dict(result)if context:+ # Keep retrieval score for later destructive-action guardrails.+ context.meta = {**(context.meta or {}), "_dedup_score": score}similar.append(context)+ logger.debug("Dedup similar memories after threshold=%d", len(similar))return similarexcept Exception as e:@@ -132,18 +183,23 @@ class MemoryDeduplicator:self,candidate: CandidateMemory,similar_memories: List[Context],- ) -> tuple[DedupDecision, str]:+ ) -> tuple[DedupDecision, str, List[ExistingMemoryAction]]:"""Use LLM to decide deduplication action."""vlm = get_openviking_config().vlmif not vlm or not vlm.is_available():# Without LLM, default to CREATE (conservative)- return DedupDecision.CREATE, "LLM not available, defaulting to CREATE"+ return DedupDecision.CREATE, "LLM not available, defaulting to CREATE", []# Format existing memories for promptexisting_formatted = []- for i, mem in enumerate(similar_memories[:3]): # Max 3 for context- abstract = mem._abstract_cache or mem.meta.get("abstract", "")- existing_formatted.append(f"{i + 1}. {abstract}")+ for i, mem in enumerate(similar_memories[: self.MAX_PROMPT_SIMILAR_MEMORIES]):+ abstract = getattr(mem, "_abstract_cache", "") or mem.meta.get("abstract", "")+ facet = self._extract_facet_key(abstract)+ score = mem.meta.get("_dedup_score")+ score_text = "n/a" if score is None else f"{float(score):.4f}"+ existing_formatted.append(+ f"{i + 1}. uri={mem.uri}\n score={score_text}\n facet={facet}\n abstract={abstract}"+ )prompt = render_prompt("compression.dedup_decision",@@ -160,23 +216,136 @@ class MemoryDeduplicator:response = await vlm.get_completion_async(prompt)data = parse_json_from_response(response) or {}+ return self._parse_decision_payload(data, similar_memories, candidate)- decision_str = data.get("decision", "create").lower()- reason = data.get("reason", "")+ except Exception as e:+ logger.warning(f"LLM dedup decision failed: {e}")+ return DedupDecision.CREATE, f"LLM failed: {e}", []- # Map to enum- decision_map = {- "create": DedupDecision.CREATE,- "merge": DedupDecision.MERGE,- "skip": DedupDecision.SKIP,- }- decision = decision_map.get(decision_str, DedupDecision.CREATE)+ def _parse_decision_payload(+ self,+ data: dict,+ similar_memories: List[Context],+ candidate: Optional[CandidateMemory] = None,+ ) -> tuple[DedupDecision, str, List[ExistingMemoryAction]]:+ """Parse/normalize dedup payload from LLM."""+ decision_str = str(data.get("decision", "create")).lower().strip()+ reason = str(data.get("reason", "") or "")++ decision_map = {+ "skip": DedupDecision.SKIP,+ "create": DedupDecision.CREATE,+ "none": DedupDecision.NONE,+ # Backward compatibility: legacy candidate-level merge maps to none.+ "merge": DedupDecision.NONE,+ }+ decision = decision_map.get(decision_str, DedupDecision.CREATE)++ raw_actions = data.get("list", [])+ if not isinstance(raw_actions, list):+ raw_actions = []++ # Legacy response compatibility: {"decision":"merge"}.+ if decision_str == "merge" and not raw_actions and similar_memories:+ raw_actions = [+ {+ "uri": similar_memories[0].uri,+ "decide": "merge",+ "reason": "Legacy candidate merge mapped to none",+ }+ ]+ if not reason:+ reason = "Legacy candidate merge mapped to none"++ action_map = {+ "merge": MemoryActionDecision.MERGE,+ "delete": MemoryActionDecision.DELETE,+ }+ similar_by_uri: Dict[str, Context] = {m.uri: m for m in similar_memories}+ actions: List[ExistingMemoryAction] = []+ seen: Dict[str, MemoryActionDecision] = {}++ for item in raw_actions:+ if not isinstance(item, dict):+ continue++ action_str = str(item.get("decide", "")).lower().strip()+ action = action_map.get(action_str)+ if not action:+ continue++ memory = None+ uri = item.get("uri")+ if isinstance(uri, str):+ memory = similar_by_uri.get(uri)++ # Tolerate index-based responses (1-based preferred, 0-based fallback).+ if memory is None:+ index = item.get("index")+ if isinstance(index, int):+ if 1 <= index <= len(similar_memories):+ memory = similar_memories[index - 1]+ elif 0 <= index < len(similar_memories):+ memory = similar_memories[index]++ if memory is None:+ continue++ previous = seen.get(memory.uri)+ if previous and previous != action:+ actions = [a for a in actions if a.memory.uri != memory.uri]+ seen.pop(memory.uri, None)+ logger.warning(f"Conflicting actions for memory {memory.uri}, dropping both")+ continue+ if previous == action:+ continue++ seen[memory.uri] = action+ actions.append(+ ExistingMemoryAction(+ memory=memory,+ decision=action,+ reason=str(item.get("reason", "") or ""),+ )+ )- return decision, reason+ # Rule: skip should never carry per-memory actions.+ if decision == DedupDecision.SKIP:+ return decision, reason, []- except Exception as e:- logger.warning(f"LLM dedup decision failed: {e}")- return DedupDecision.CREATE, f"LLM failed: {e}"+ has_merge_action = any(a.decision == MemoryActionDecision.MERGE for a in actions)++ # Rule: if any merge exists, ignore create and execute as none.+ if decision == DedupDecision.CREATE and has_merge_action:+ decision = DedupDecision.NONE+ reason = f"{reason} | normalized:create+merge->none".strip(" |")+ return decision, reason, actions++ # Rule: create can only carry delete actions (or empty list).+ if decision == DedupDecision.CREATE:+ actions = [a for a in actions if a.decision == MemoryActionDecision.DELETE]++ return decision, reason, actions++ @staticmethod+ def _extract_facet_key(text: str) -> str:+ """Extract normalized facet key from memory abstract (before separator)."""+ if not text:+ return ""++ normalized = " ".join(str(text).strip().split())+ # Prefer common separators used by extraction templates.+ for sep in (":", ":", "-", "—"):+ if sep in normalized:+ left = normalized.split(sep, 1)[0].strip().lower()+ if left:+ return left++ # Fallback: short leading phrase.+ m = re.match(r"^(.{1,24})\s", normalized.lower())+ if m:+ return m.group(1).strip()+ return normalized[:24].lower().strip()@staticmethoddef _cosine_similarity(vec_a: List[float], vec_b: List[float]) -> float:diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex 83f55d5d1..584cc42b1 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -7,6 +7,7 @@ Provides centralized schema definitions and factory functions for creating collesimilar to how init_viking_fs encapsulates VikingFS initialization."""+import hashlibimport jsonfrom typing import Any, Dict, Optional@@ -172,6 +173,11 @@ class TextEmbeddingHandler(DequeueHandlerBase):# Write to vector databasetry:+ # Ensure vector DB has at most one record per URI.+ uri = inserted_data.get("uri")+ if uri:+ inserted_data["id"] = hashlib.md5(uri.encode("utf-8")).hexdigest()+record_id = await self._vikingdb.insert(self._collection_name, inserted_data)if record_id:logger.debug(diff --git a/openviking/storage/vectordb/utils/validation.py b/openviking/storage/vectordb/utils/validation.pyindex 150d3638a..62ff3fff8 100644--- a/openviking/storage/vectordb/utils/validation.py+++ b/openviking/storage/vectordb/utils/validation.py@@ -61,7 +61,7 @@ class FieldTypeEnum(str, Enum):class DenseVectorize(BaseModel):- model_config = ConfigDict(extra="forbid")+ model_config = ConfigDict()ModelName: strModelVersion: Optional[str] = None@@ -94,7 +94,7 @@ class DenseVectorize(BaseModel):class SparseVectorize(BaseModel):- model_config = ConfigDict(extra="forbid")+ model_config = ConfigDict()ModelName: strModelVersion: Optional[str] = None@@ -102,14 +102,14 @@ class SparseVectorize(BaseModel):class VectorizeConfig(BaseModel):- model_config = ConfigDict(extra="forbid")+ model_config = ConfigDict()Dense: Optional[DenseVectorize] = NoneSparse: Optional[SparseVectorize] = Noneclass CollectionField(BaseModel):- model_config = ConfigDict(extra="forbid")+ model_config = ConfigDict()FieldName: strFieldType: FieldTypeEnum@@ -145,7 +145,7 @@ class CollectionField(BaseModel):class CollectionMetaConfig(BaseModel):- model_config = ConfigDict(extra="forbid")+ model_config = ConfigDict()CollectionName: strFields: List[CollectionField]@@ -180,7 +180,7 @@ class CollectionMetaConfig(BaseModel):class VectorIndexConfig(BaseModel):- model_config = ConfigDict(extra="forbid")+ model_config = ConfigDict()IndexType: Literal["flat", "flat_hybrid", "FLAT", "FLAT_HYBRID"]Distance: Optional[Literal["l2", "ip", "cosine", "L2", "IP", "COSINE"]] = None@@ -227,7 +227,7 @@ class VectorIndexConfig(BaseModel):class IndexMetaConfig(BaseModel):- model_config = ConfigDict(extra="forbid")+ model_config = ConfigDict()IndexName: strVectorIndex: VectorIndexConfig@@ -245,7 +245,7 @@ class IndexMetaConfig(BaseModel):class IndexMetaUpdateConfig(BaseModel):- model_config = ConfigDict(extra="forbid")+ model_config = ConfigDict()IndexName: Optional[str] = NoneVectorIndex: Optional[VectorIndexConfig] = None@@ -263,7 +263,7 @@ class IndexMetaUpdateConfig(BaseModel):class CollectionMetaUpdateConfig(BaseModel):- model_config = ConfigDict(extra="forbid")+ model_config = ConfigDict()CollectionName: Optional[str] = NoneFields: Optional[List[CollectionField]] = Nonediff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.pyindex 4f1b7efd4..57eecb833 100644--- a/openviking/storage/viking_vector_index_backend.py+++ b/openviking/storage/viking_vector_index_backend.py@@ -475,7 +475,7 @@ class VikingVectorIndexBackend(VikingDBInterface):record = dict(item.fields) if item.fields else {}record["id"] = item.idrecords.append(record)- if len(records) > 0:+ if len(records) > 1:raise ValueError(f"Duplicate records found for URI: {uri}")if len(records) == 0:raise ValueError(f"Record not found for URI: {uri}")@@ -560,29 +560,25 @@ class VikingVectorIndexBackend(VikingDBInterface):async def remove_by_uri(self, collection: str, uri: str) -> int:"""Remove resource(s) by URI."""try:- # Find records with matching URItarget_records = await self.filter(collection=collection,filter={"op": "must", "field": "uri", "conds": [uri]},- limit=1,+ limit=10,)if not target_records:return 0total_deleted = 0- target = target_records[0]- is_leaf = target.get("is_leaf", False)- # If not leaf (i.e., intermediate directory), find and delete all descendants recursively- if not is_leaf:+ # If any record indicates this URI is a directory node, remove descendants first.+ if any(not r.get("is_leaf", False) for r in target_records):descendant_count = await self._remove_descendants(collection, uri)total_deleted += descendant_count- # Delete the target itself- if "id" in target:- await self.delete(collection, [target["id"]])- total_deleted += 1+ ids = [r.get("id") for r in target_records if r.get("id")]+ if ids:+ total_deleted += await self.delete(collection, ids)logger.info(f"Removed {total_deleted} record(s) for URI: {uri}")return total_deleted