Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

refactor(memory): redesign extraction/dedup flow and add conflict-aware delete handling (#225)

kkkwjxa06219968b2c
diff --git a/docs/en/concepts/08-session.md b/docs/en/concepts/08-session.md
index 0ebb6282c..38737edfc 100644
--- a/docs/en/concepts/08-session.md
+++ b/docs/en/concepts/08-session.md
@@ -136,19 +136,20 @@ Messages → LLM Extract → Candidate Memories
↓
Vector Pre-filter → Find Similar Memories
↓
-LLM Dedup Decision → CREATE/UPDATE/MERGE/SKIP
+LLM Dedup Decision → candidate(skip/create/none) + item(merge/delete)
↓
Write to AGFS → Vectorize
```
### Dedup Decisions
-| Decision | Description |
-|----------|-------------|
-| `CREATE` | New memory, create directly |
-| `UPDATE` | Update existing memory |
-| `MERGE` | Merge multiple memories |
-| `SKIP` | Duplicate, skip |
+| Level | Decision | Description |
+|------|----------|-------------|
+| Candidate | `skip` | Candidate is duplicate, skip and do nothing |
+| Candidate | `create` | Create candidate memory (optionally delete conflicting existing memories first) |
+| Candidate | `none` | Do not create candidate; resolve existing memories by item decisions |
+| Per-existing item | `merge` | Merge candidate content into specified existing memory |
+| Per-existing item | `delete` | Delete specified conflicting existing memory |
## Storage Structure
diff --git a/examples/memory_demo.py b/examples/memory_demo.py
new file mode 100644
index 000000000..de9320cf2
--- /dev/null
+++ b/examples/memory_demo.py
@@ -0,0 +1,179 @@
+"""OpenViking memory dedup demo with REAL LLM decisions (no fake/mock).
+
+This demo focuses on the user preference scenario:
+1) "I like apples." -> expected create
+2) "I like strawberries." -> expected create
+3) "I like Fuji apples." -> expected merge/none with apple memory
+4) "I do not like fruits anymore." -> expected delete old positive fruit preferences
+5) repeat negative preference -> expected skip/none
+
+The script prints commit and find results after each round so you can inspect
+whether memory handling is reasonable.
+
+Usage:
+ export OPENVIKING_CONFIG_FILE=ov.conf
+ python examples/memory_dedup_cases_demo.py
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable, List
+
+from openviking.message.part import TextPart
+from openviking.sync_client import SyncOpenViking
+
+
+@dataclass
+class RoundCase:
+ title: str
+ user_text: str
+ expected: str
+
+
+def _print_section(title: str, body: str = "") -> None:
+ print("\n" + "=" * 80)
+ print(title)
+ if body:
+ print("-" * 80)
+ print(body)
+
+
+def _safe_list(items: Iterable[Any]) -> list:
+ try:
+ return list(items)
+ except Exception:
+ return []
+
+
+def _format_find_result(result: Any, max_items: int = 6) -> str:
+ memories = _safe_list(getattr(result, "memories", []))
+ if not memories:
+ return "(no memory hit)"
+
+ lines: List[str] = []
+ for i, mem in enumerate(memories[:max_items], 1):
+ score = getattr(mem, "score", None)
+ score_s = "n/a" if score is None else f"{float(score):.4f}"
+ abstract = getattr(mem, "abstract", "") or ""
+ uri = getattr(mem, "uri", "")
+ lines.append(f"{i}. score={score_s} | {abstract} | {uri}")
+ return "\n".join(lines)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="OpenViking real-LLM dedup cases demo")
+ parser.add_argument(
+ "--path",
+ default="./ov_data_dedup_cases_demo",
+ help="Fixed demo storage path. This script clears it at startup.",
+ )
+ parser.add_argument(
+ "--wait-timeout",
+ type=float,
+ default=60.0,
+ help="Queue wait timeout in seconds.",
+ )
+ args = parser.parse_args()
+
+ if not os.environ.get("OPENVIKING_CONFIG_FILE"):
+ repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+ cfg = os.path.join(repo_root, "ov.conf")
+ if os.path.exists(cfg):
+ os.environ["OPENVIKING_CONFIG_FILE"] = cfg
+
+ data_path = Path(args.path)
+ if data_path.exists():
+ shutil.rmtree(data_path)
+ data_path.mkdir(parents=True, exist_ok=True)
+
+ client = SyncOpenViking(path=str(data_path))
+ client.initialize()
+
+ try:
+ client.is_healthy()
+ sess_info = client.create_session()
+ session_id = sess_info["session_id"]
+ sess = client.session(session_id)
+
+ rounds = [
+ RoundCase(
+ title="Round 1",
+ user_text=(
+ "我是一名程序员。"
+ "我爱吃苹果。"
+ "我爱吃草莓。"
+ "我每天早上7点起床。"
+ "我通勤主要骑共享单车。"
+ "我习惯在周末整理书桌。"
+ "我最常用的云盘是Dropbox。"
+ "我对坚果过敏,尤其是腰果。"
+ "我最近在学西班牙语。"
+ "我喜欢在雨天听爵士乐。"
+ "我的常用笔记软件是Obsidian。"
+ "我每周三晚上会去游泳。"
+ "我偏好27英寸的外接显示器。"
+ ),
+ expected="Expected dedup: create multiple unrelated memories",
+ ),
+ RoundCase(
+ title="Round 2",
+ user_text="我爱吃红富士苹果。我是外卖员。",
+ expected="Expected dedup: none+merge (细化苹果偏好)",
+ ),
+ RoundCase(
+ title="Round 3",
+ user_text="我不爱吃水果了,把之前关于喜欢水果的偏好作废。",
+ expected="Expected dedup: delete old positive fruit preferences",
+ ),
+ ]
+
+ queries = [
+ "我喜欢吃什么?",
+ "我是做什么工作的?",
+ ]
+
+ for r in rounds:
+ sess.add_message("user", parts=[TextPart(text=r.user_text)])
+ sess.add_message("assistant", parts=[TextPart(text="收到。")])
+ commit_result = sess.commit()
+
+ _print_section(
+ f"{r.title} commit",
+ body=(f"user={r.user_text}\n{r.expected}\ncommit={commit_result}"),
+ )
+
+ try:
+ client.wait_processed(timeout=args.wait_timeout)
+ except Exception:
+ pass
+
+ for q in queries:
+ try:
+ result = client.find(q, target_uri="viking://user/memories", limit=8)
+ _print_section(f"{r.title} find: {q}", body=_format_find_result(result))
+ except Exception as e:
+ _print_section(f"{r.title} find: {q} (failed)", body=str(e))
+
+ _print_section(
+ "Done",
+ body=(
+ "Check whether the later rounds are dominated by negative fruit preference.\n"
+ "If old positive fruit preferences disappear or stop ranking high, delete likely worked."
+ ),
+ )
+ finally:
+ try:
+ client.close()
+ except Exception:
+ pass
+
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/openviking/prompts/templates/compression/dedup_decision.yaml b/openviking/prompts/templates/compression/dedup_decision.yaml
index c41fee019..a89d61e98 100644
--- a/openviking/prompts/templates/compression/dedup_decision.yaml
+++ b/openviking/prompts/templates/compression/dedup_decision.yaml
@@ -1,8 +1,8 @@
metadata:
id: "compression.dedup_decision"
name: "Memory Deduplication Decision"
- description: "Decide whether new memory should CREATE/MERGE/SKIP"
- version: "2.0.0"
+ description: "Decide candidate action (skip/create/none) and per-memory actions (merge/delete)"
+ version: "3.3.1"
language: "en"
category: "compression"
@@ -25,25 +25,79 @@ variables:
required: true
template: |
- Determine how to handle this candidate memory.
+ You are deciding how to update long-term memory with:
+ 1) one candidate memory (new fact)
+ 2) existing similar memories (retrieved from store)
- **Candidate Memory**:
- Abstract: {{ candidate_abstract }}
- Overview: {{ candidate_overview }}
- Content: {{ candidate_content }}
+ Candidate memory:
+ - Abstract: {{ candidate_abstract }}
+ - Overview: {{ candidate_overview }}
+ - Content: {{ candidate_content }}
- **Existing Similar Memories**:
+ Existing similar memories:
{{ existing_memories }}
- Please decide:
- - SKIP: Candidate memory duplicates existing memories, no need to save
- - CREATE: This is completely new information, should be created
- - MERGE: Candidate memory should be merged with existing memories
+ Goal:
+ Keep memory consistent and useful while minimizing destructive edits.
- Return JSON format:
+ Candidate-level decision:
+ - skip:
+ Use only when candidate adds no useful new information (duplicate, paraphrase,
+ or too weak/uncertain). No memory should change.
+ - create:
+ Use when candidate is a valid new memory that should be stored as a separate item.
+ It may optionally delete fully-invalidated existing memories.
+ - none:
+ Use when candidate itself should not be stored, but existing memories should be
+ reconciled with per-item actions.
+
+ Existing-memory per-item action:
+ - merge:
+ Existing memory and candidate are about the same subject and should be unified.
+ Use for refinement, correction, partial conflict, or complementary details.
+ - delete:
+ Existing memory must be removed only if candidate fully invalidates the entire
+ existing memory (not just one sub-part).
+
+ Critical delete boundary:
+ - If conflict is partial (some statements conflict, others remain valid), DO NOT delete.
+ Use merge instead so non-conflicting information is preserved.
+ - Delete only when the whole existing memory is obsolete/invalidated.
+ - Topic/facet mismatch must never be deleted. If candidate is about one facet
+ (for example any single preference facet), existing memories from other facets
+ must be omitted from list (treated as unchanged).
+
+ Decision guidance:
+ - Prefer skip when candidate is redundant.
+ - Prefer none+merge for same-subject updates and partial contradictions.
+ - Prefer create for clearly new independent memory.
+ - If uncertain, choose non-destructive behavior (skip or merge), not delete.
+
+ Practical checklist before emitting each list item:
+ 1) Is existing memory about the same topic/facet as candidate?
+ 2) If no, do not include it in list.
+ 3) If yes and candidate only updates part of it, use merge.
+ 4) Use delete only when candidate explicitly invalidates the whole existing memory.
+
+ Hard constraints:
+ - If decision is "skip", do not return "list".
+ - If any list item uses "merge", decision must be "none".
+ - If decision is "create", list can be empty or contain delete items only.
+ - Use uri exactly from existing memories list.
+ - Omit unchanged existing memories from list.
+ - Return JSON only, no prose.
+
+ Return JSON in this exact structure:
{
- "decision": "skip|create|merge",
- "reason": "Decision reason"
+ "decision": "skip|create|none",
+ "reason": "short reason",
+ "list": [
+ {
+ "uri": "<existing memory uri>",
+ "decide": "merge|delete",
+ "reason": "short reason (for delete, explain full invalidation)"
+ }
+ ]
}
llm_config:
diff --git a/openviking/prompts/templates/compression/memory_extraction.yaml b/openviking/prompts/templates/compression/memory_extraction.yaml
index 849cb99c3..a8bd1f1bc 100644
--- a/openviking/prompts/templates/compression/memory_extraction.yaml
+++ b/openviking/prompts/templates/compression/memory_extraction.yaml
@@ -2,7 +2,7 @@ metadata:
id: "compression.memory_extraction"
name: "Memory Extraction (Three-Level)"
description: "Extract memories from session context using L0/L1/L2 three-level structure"
- version: "5.0.0"
+ version: "5.1.0"
language: "en"
category: "compression"
@@ -90,6 +90,15 @@ template: |
- Characteristics: Changeable choices, styles
- Test: Can it be described as "User prefers/likes..."
+ ### Preference Granularity (Important)
+ - For category `preferences`, each memory item should represent one independently
+ updatable preference unit (single facet).
+ - Do NOT mix unrelated preference facets in one memory item.
+ Examples of different facets: food, commute, schedule, tools, music, study habits.
+ - If the conversation contains multiple facets, output multiple `preferences` items.
+ - This granularity is required so future updates/conflicts can affect only the
+ relevant memory without damaging unrelated preferences.
+
**entities** - Entities (continuously existing nouns)
- Core: Describes "current state of something"
- Characteristics: Entities with lifecycle (person/project/organization)
@@ -166,6 +175,43 @@ template: |
```
❌ **Bad**: abstract says "Code preferences" (too general) or "No type hints" (too specific, cannot merge other style preferences)
+ ## preferences Granularity Example
+ ❌ **Bad (mixed facets in one memory)**:
+ ```json
+ {
+ "category": "preferences",
+ "abstract": "User preferences: likes apples, commutes by bike, uses Obsidian",
+ "overview": "Mixed food/commute/tool preferences",
+ "content": "User likes apples, usually commutes by bike, and prefers Obsidian."
+ }
+ ```
+
+ ✅ **Good (split by independently updatable facets)**:
+ ```json
+ {
+ "memories": [
+ {
+ "category": "preferences",
+ "abstract": "Food preference: Likes apples",
+ "overview": "## Preference Domain\\n- **Domain**: Food\\n\\n## Specific Preference\\n- Likes apples",
+ "content": "User shows a food preference for apples."
+ },
+ {
+ "category": "preferences",
+ "abstract": "Commute preference: Usually rides a bike",
+ "overview": "## Preference Domain\\n- **Domain**: Commute\\n\\n## Specific Preference\\n- Usually rides a bike",
+ "content": "User usually commutes by bike."
+ },
+ {
+ "category": "preferences",
+ "abstract": "Tool preference: Uses Obsidian for notes",
+ "overview": "## Preference Domain\\n- **Domain**: Tools\\n\\n## Specific Preference\\n- Uses Obsidian for notes",
+ "content": "User prefers Obsidian as note-taking software."
+ }
+ ]
+ }
+ ```
+
## entities Example (Merge type)
✅ **Good**:
```json
@@ -228,8 +274,7 @@ template: |
- The values of "abstract", "overview", and "content" MUST be written in {{ output_language }} (if output_language is "auto", use the dominant language in recent_messages).
- Only extract truly valuable personalized information
- If nothing worth recording, return {"memories": []}
- - Preferences should be aggregated by topic, similar preferences should be merged into one memory
+ - For preferences, keep each memory as one independently updatable facet; do not combine unrelated facets in one memory
llm_config:
temperature: 0.0
-
diff --git a/openviking/prompts/templates/compression/memory_merge.yaml b/openviking/prompts/templates/compression/memory_merge.yaml
index 868a6faac..6fef1ec01 100644
--- a/openviking/prompts/templates/compression/memory_merge.yaml
+++ b/openviking/prompts/templates/compression/memory_merge.yaml
@@ -2,7 +2,7 @@ metadata:
id: "compression.memory_merge"
name: "Memory Merge"
description: "Merge existing memory with new information"
- version: "1.0.0"
+ version: "1.1.0"
language: "en"
category: "compression"
@@ -40,6 +40,8 @@ template: |
Requirements:
- Remove duplicate information
- Keep the most up-to-date details
+ - If there is a conflict, update only the conflicting statement with the newer fact
+ - Preserve non-conflicting details from existing content; do not drop unrelated information
- Maintain a coherent narrative
- Output ONLY the merged content, no explanation
- Output MUST be written in **{{ output_language }}** (if output_language is "auto", infer dominant language from inputs)
diff --git a/openviking/session/__init__.py b/openviking/session/__init__.py
index d9ee9efbb..7aa033d17 100644
--- a/openviking/session/__init__.py
+++ b/openviking/session/__init__.py
@@ -3,7 +3,13 @@
"""Session management module."""
from openviking.session.compressor import ExtractionStats, SessionCompressor
-from openviking.session.memory_deduplicator import DedupDecision, DedupResult, MemoryDeduplicator
+from openviking.session.memory_deduplicator import (
+ DedupDecision,
+ DedupResult,
+ ExistingMemoryAction,
+ MemoryActionDecision,
+ MemoryDeduplicator,
+)
from openviking.session.memory_extractor import CandidateMemory, MemoryCategory, MemoryExtractor
from openviking.session.session import Session, SessionCompression, SessionStats
@@ -22,5 +28,7 @@ __all__ = [
# Memory Deduplicator
"MemoryDeduplicator",
"DedupDecision",
+ "MemoryActionDecision",
+ "ExistingMemoryAction",
"DedupResult",
]
diff --git a/openviking/session/compressor.py b/openviking/session/compressor.py
index a4b498b95..befa49f4f 100644
--- a/openviking/session/compressor.py
+++ b/openviking/session/compressor.py
@@ -17,8 +17,8 @@ from openviking.storage.viking_fs import get_viking_fs
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils import get_logger
-from .memory_deduplicator import DedupDecision, MemoryDeduplicator
-from .memory_extractor import MemoryCategory, MemoryExtractor
+from .memory_deduplicator import DedupDecision, MemoryActionDecision, MemoryDeduplicator
+from .memory_extractor import CandidateMemory, MemoryCategory, MemoryExtractor
logger = get_logger(__name__)
@@ -39,6 +39,7 @@ class ExtractionStats:
created: int = 0
merged: int = 0
+ deleted: int = 0
skipped: int = 0
@@ -63,6 +64,47 @@ class SessionCompressor:
logger.info(f"Enqueued memory for vectorization: {memory.uri}")
return True
+ async def _merge_into_existing(
+ self,
+ candidate: CandidateMemory,
+ target_memory: Context,
+ viking_fs,
+ ) -> bool:
+ """Merge candidate content into an existing memory file."""
+ try:
+ existing_content = await viking_fs.read_file(target_memory.uri)
+ merged = await self.extractor._merge_memory(
+ existing_content,
+ candidate.content,
+ candidate.category.value,
+ output_language=candidate.language,
+ )
+ if not merged:
+ return False
+
+ await viking_fs.write_file(target_memory.uri, merged)
+ target_memory.set_vectorize(Vectorize(text=merged))
+ await self._index_memory(target_memory)
+ return True
+ except Exception as e:
+ logger.error(f"Failed to merge memory {target_memory.uri}: {e}")
+ return False
+
+ async def _delete_existing_memory(self, memory: Context, viking_fs) -> bool:
+ """Hard delete an existing memory file and clean up its vector record."""
+ try:
+ await viking_fs.rm(memory.uri, recursive=False)
+ except Exception as e:
+ logger.error(f"Failed to delete memory file {memory.uri}: {e}")
+ return False
+
+ try:
+ # rm() already syncs vector deletion in most cases; keep this as a safe fallback.
+ await self.vikingdb.remove_by_uri("context", memory.uri)
+ except Exception as e:
+ logger.warning(f"Failed to remove vector record for {memory.uri}: {e}")
+ return True
+
async def extract_long_term_memories(
self,
messages: List[Message],
@@ -95,46 +137,64 @@ class SessionCompressor:
# Dedup check for other categories
result = await self.deduplicator.deduplicate(candidate)
+ actions = result.actions or []
+ decision = result.decision
+
+ # Safety net: create+merge should be treated as none.
+ if decision == DedupDecision.CREATE and any(
+ a.decision == MemoryActionDecision.MERGE for a in actions
+ ):
+ logger.warning(
+ f"Dedup returned create with merge action, normalizing to none: "
+ f"{candidate.abstract}"
+ )
+ decision = DedupDecision.NONE
- if result.decision == DedupDecision.SKIP:
+ if decision == DedupDecision.SKIP:
stats.skipped += 1
continue
- if result.decision == DedupDecision.CREATE:
+ if decision == DedupDecision.NONE:
+ if not actions:
+ stats.skipped += 1
+ continue
+
+ for action in actions:
+ if action.decision == MemoryActionDecision.DELETE:
+ if viking_fs and await self._delete_existing_memory(
+ action.memory, viking_fs
+ ):
+ stats.deleted += 1
+ else:
+ stats.skipped += 1
+ elif action.decision == MemoryActionDecision.MERGE:
+ if candidate.category in MERGE_SUPPORTED_CATEGORIES and viking_fs:
+ if await self._merge_into_existing(candidate, action.memory, viking_fs):
+ stats.merged += 1
+ else:
+ stats.skipped += 1
+ else:
+ # events/cases don't support MERGE, treat as SKIP
+ stats.skipped += 1
+ continue
+
+ if decision == DedupDecision.CREATE:
+ # create can optionally include delete actions (delete first, then create)
+ for action in actions:
+ if action.decision == MemoryActionDecision.DELETE:
+ if viking_fs and await self._delete_existing_memory(
+ action.memory, viking_fs
+ ):
+ stats.deleted += 1
+ else:
+ stats.skipped += 1
+
memory = await self.extractor.create_memory(candidate, user, session_id)
if memory:
memories.append(memory)
stats.created += 1
await self._index_memory(memory)
-
- elif result.decision == DedupDecision.MERGE:
- # Only merge for supported categories
- if (
- candidate.category in MERGE_SUPPORTED_CATEGORIES
- and result.similar_memories
- and viking_fs
- ):
- target_memory = result.similar_memories[0]
- try:
- existing_content = await viking_fs.read_file(target_memory.uri)
- merged = await self.extractor._merge_memory(
- existing_content,
- candidate.content,
- candidate.category.value,
- output_language=candidate.language,
- )
- if merged:
- await viking_fs.write_file(target_memory.uri, merged)
- target_memory.set_vectorize(Vectorize(text=merged))
- await self._index_memory(target_memory)
- stats.merged += 1
- else:
- stats.skipped += 1
- except Exception as e:
- logger.error(f"Failed to merge memory: {e}")
- stats.skipped += 1
else:
- # events/cases don't support MERGE, treat as SKIP
stats.skipped += 1
# Extract URIs used in messages, create relations
@@ -144,7 +204,7 @@ class SessionCompressor:
logger.info(
f"Memory extraction: created={stats.created}, "
- f"merged={stats.merged}, skipped={stats.skipped}"
+ f"merged={stats.merged}, deleted={stats.deleted}, skipped={stats.skipped}"
)
return memories
diff --git a/openviking/session/memory_deduplicator.py b/openviking/session/memory_deduplicator.py
index 83121de25..9bbc0e830 100644
--- a/openviking/session/memory_deduplicator.py
+++ b/openviking/session/memory_deduplicator.py
@@ -3,12 +3,14 @@
"""
Memory Deduplicator for OpenViking.
-LLM-assisted deduplication with CREATE/MERGE/SKIP decisions
+LLM-assisted deduplication with candidate-level skip/create/none decisions and
+per-existing merge/delete actions.
"""
+import re
from dataclasses import dataclass
from enum import Enum
-from typing import List
+from typing import Dict, List, Optional
from openviking.core.context import Context
from openviking.models.embedder.base import EmbedResult
@@ -25,9 +27,25 @@ logger = get_logger(__name__)
class DedupDecision(str, Enum):
"""Deduplication decision types."""
- CREATE = "create" # New memory, create directly
- MERGE = "merge" # Merge with existing memories
SKIP = "skip" # Duplicate, skip
+ CREATE = "create" # Create candidate memory
+ NONE = "none" # No candidate creation; resolve existing memories only
+
+
+class MemoryActionDecision(str, Enum):
+ """Decision for each existing memory candidate."""
+
+ MERGE = "merge" # Merge candidate into existing memory
+ DELETE = "delete" # Delete conflicting existing memory
+
+
+@dataclass
+class ExistingMemoryAction:
+ """Decision for one existing memory."""
+
+ memory: Context
+ decision: MemoryActionDecision
+ reason: str = ""
@dataclass
@@ -37,13 +55,22 @@ class DedupResult:
decision: DedupDecision
candidate: CandidateMemory
similar_memories: List[Context] # Similar existing memories
+ actions: Optional[List[ExistingMemoryAction]] = None
reason: str = ""
class MemoryDeduplicator:
"""Handles memory deduplication with LLM decision making."""
- SIMILARITY_THRESHOLD = 0.7 # Vector similarity threshold for pre-filtering
+ SIMILARITY_THRESHOLD = 0.0 # Vector similarity threshold for pre-filtering
+ MAX_PROMPT_SIMILAR_MEMORIES = 5 # Number of similar memories sent to LLM
+ CATEGORY_URI_PREFIX = {
+ "preferences": "viking://user/memories/preferences/",
+ "entities": "viking://user/memories/entities/",
+ "events": "viking://user/memories/events/",
+ "cases": "viking://agent/memories/cases/",
+ "patterns": "viking://agent/memories/patterns/",
+ }
def __init__(
self,
@@ -67,16 +94,18 @@ class MemoryDeduplicator:
decision=DedupDecision.CREATE,
candidate=candidate,
similar_memories=[],
+ actions=[],
reason="No similar memories found",
)
# Step 2: LLM decision
- decision, reason = await self._llm_decision(candidate, similar_memories)
+ decision, reason, actions = await self._llm_decision(candidate, similar_memories)
return DedupResult(
decision=decision,
candidate=candidate,
similar_memories=similar_memories,
+ actions=None if decision == DedupDecision.SKIP else actions,
reason=reason,
)
@@ -96,32 +125,54 @@ class MemoryDeduplicator:
# Determine collection and filter based on category
collection = "context"
- # Build category filter
- category_value = candidate.category.value
+ category_uri_prefix = self.CATEGORY_URI_PREFIX.get(candidate.category.value, "")
+
+ # Build filter by memory scope + uri prefix (schema does not have category field yet).
+ filter_conds = [
+ {"field": "context_type", "op": "must", "conds": ["memory"]},
+ {"field": "is_leaf", "op": "must", "conds": [True]},
+ ]
+ if category_uri_prefix:
+ filter_conds.append({"field": "uri", "op": "prefix", "prefix": category_uri_prefix})
+ dedup_filter = {"op": "and", "conds": filter_conds}
+ logger.debug(
+ "Dedup prefilter candidate category=%s filter=%s",
+ candidate.category.value,
+ dedup_filter,
+ )
try:
- # Search with category filter
+ # Search with memory-scope filter.
results = await self.vikingdb.search(
collection=collection,
query_vector=query_vector,
limit=5,
- filter={
- "op": "and",
- "conds": [
- {"field": "category", "op": "must", "conds": [category_value]},
- {"field": "is_leaf", "op": "must", "conds": [True]},
- ],
- },
+ filter=dedup_filter,
)
# Filter by similarity threshold
similar = []
+ logger.debug(
+ "Dedup prefilter raw hits=%d threshold=%.2f",
+ len(results),
+ self.SIMILARITY_THRESHOLD,
+ )
for result in results:
- if result.get("score", 0) >= self.SIMILARITY_THRESHOLD:
+ score = float(result.get("_score", result.get("score", 0)) or 0)
+ logger.debug(
+ "Dedup hit score=%.4f uri=%s abstract=%s",
+ score,
+ result.get("uri", ""),
+ result.get("abstract", ""),
+ )
+ if score >= self.SIMILARITY_THRESHOLD:
# Reconstruct Context object
context = Context.from_dict(result)
if context:
+ # Keep retrieval score for later destructive-action guardrails.
+ context.meta = {**(context.meta or {}), "_dedup_score": score}
similar.append(context)
+ logger.debug("Dedup similar memories after threshold=%d", len(similar))
return similar
except Exception as e:
@@ -132,18 +183,23 @@ class MemoryDeduplicator:
self,
candidate: CandidateMemory,
similar_memories: List[Context],
- ) -> tuple[DedupDecision, str]:
+ ) -> tuple[DedupDecision, str, List[ExistingMemoryAction]]:
"""Use LLM to decide deduplication action."""
vlm = get_openviking_config().vlm
if not vlm or not vlm.is_available():
# Without LLM, default to CREATE (conservative)
- return DedupDecision.CREATE, "LLM not available, defaulting to CREATE"
+ return DedupDecision.CREATE, "LLM not available, defaulting to CREATE", []
# Format existing memories for prompt
existing_formatted = []
- for i, mem in enumerate(similar_memories[:3]): # Max 3 for context
- abstract = mem._abstract_cache or mem.meta.get("abstract", "")
- existing_formatted.append(f"{i + 1}. {abstract}")
+ for i, mem in enumerate(similar_memories[: self.MAX_PROMPT_SIMILAR_MEMORIES]):
+ abstract = getattr(mem, "_abstract_cache", "") or mem.meta.get("abstract", "")
+ facet = self._extract_facet_key(abstract)
+ score = mem.meta.get("_dedup_score")
+ score_text = "n/a" if score is None else f"{float(score):.4f}"
+ existing_formatted.append(
+ f"{i + 1}. uri={mem.uri}\n score={score_text}\n facet={facet}\n abstract={abstract}"
+ )
prompt = render_prompt(
"compression.dedup_decision",
@@ -160,23 +216,136 @@ class MemoryDeduplicator:
response = await vlm.get_completion_async(prompt)
data = parse_json_from_response(response) or {}
+ return self._parse_decision_payload(data, similar_memories, candidate)
- decision_str = data.get("decision", "create").lower()
- reason = data.get("reason", "")
+ except Exception as e:
+ logger.warning(f"LLM dedup decision failed: {e}")
+ return DedupDecision.CREATE, f"LLM failed: {e}", []
- # Map to enum
- decision_map = {
- "create": DedupDecision.CREATE,
- "merge": DedupDecision.MERGE,
- "skip": DedupDecision.SKIP,
- }
- decision = decision_map.get(decision_str, DedupDecision.CREATE)
+ def _parse_decision_payload(
+ self,
+ data: dict,
+ similar_memories: List[Context],
+ candidate: Optional[CandidateMemory] = None,
+ ) -> tuple[DedupDecision, str, List[ExistingMemoryAction]]:
+ """Parse/normalize dedup payload from LLM."""
+ decision_str = str(data.get("decision", "create")).lower().strip()
+ reason = str(data.get("reason", "") or "")
+
+ decision_map = {
+ "skip": DedupDecision.SKIP,
+ "create": DedupDecision.CREATE,
+ "none": DedupDecision.NONE,
+ # Backward compatibility: legacy candidate-level merge maps to none.
+ "merge": DedupDecision.NONE,
+ }
+ decision = decision_map.get(decision_str, DedupDecision.CREATE)
+
+ raw_actions = data.get("list", [])
+ if not isinstance(raw_actions, list):
+ raw_actions = []
+
+ # Legacy response compatibility: {"decision":"merge"}.
+ if decision_str == "merge" and not raw_actions and similar_memories:
+ raw_actions = [
+ {
+ "uri": similar_memories[0].uri,
+ "decide": "merge",
+ "reason": "Legacy candidate merge mapped to none",
+ }
+ ]
+ if not reason:
+ reason = "Legacy candidate merge mapped to none"
+
+ action_map = {
+ "merge": MemoryActionDecision.MERGE,
+ "delete": MemoryActionDecision.DELETE,
+ }
+ similar_by_uri: Dict[str, Context] = {m.uri: m for m in similar_memories}
+ actions: List[ExistingMemoryAction] = []
+ seen: Dict[str, MemoryActionDecision] = {}
+
+ for item in raw_actions:
+ if not isinstance(item, dict):
+ continue
+
+ action_str = str(item.get("decide", "")).lower().strip()
+ action = action_map.get(action_str)
+ if not action:
+ continue
+
+ memory = None
+ uri = item.get("uri")
+ if isinstance(uri, str):
+ memory = similar_by_uri.get(uri)
+
+ # Tolerate index-based responses (1-based preferred, 0-based fallback).
+ if memory is None:
+ index = item.get("index")
+ if isinstance(index, int):
+ if 1 <= index <= len(similar_memories):
+ memory = similar_memories[index - 1]
+ elif 0 <= index < len(similar_memories):
+ memory = similar_memories[index]
+
+ if memory is None:
+ continue
+
+ previous = seen.get(memory.uri)
+ if previous and previous != action:
+ actions = [a for a in actions if a.memory.uri != memory.uri]
+ seen.pop(memory.uri, None)
+ logger.warning(f"Conflicting actions for memory {memory.uri}, dropping both")
+ continue
+ if previous == action:
+ continue
+
+ seen[memory.uri] = action
+ actions.append(
+ ExistingMemoryAction(
+ memory=memory,
+ decision=action,
+ reason=str(item.get("reason", "") or ""),
+ )
+ )
- return decision, reason
+ # Rule: skip should never carry per-memory actions.
+ if decision == DedupDecision.SKIP:
+ return decision, reason, []
- except Exception as e:
- logger.warning(f"LLM dedup decision failed: {e}")
- return DedupDecision.CREATE, f"LLM failed: {e}"
+ has_merge_action = any(a.decision == MemoryActionDecision.MERGE for a in actions)
+
+ # Rule: if any merge exists, ignore create and execute as none.
+ if decision == DedupDecision.CREATE and has_merge_action:
+ decision = DedupDecision.NONE
+ reason = f"{reason} | normalized:create+merge->none".strip(" |")
+ return decision, reason, actions
+
+ # Rule: create can only carry delete actions (or empty list).
+ if decision == DedupDecision.CREATE:
+ actions = [a for a in actions if a.decision == MemoryActionDecision.DELETE]
+
+ return decision, reason, actions
+
+ @staticmethod
+ def _extract_facet_key(text: str) -> str:
+ """Extract normalized facet key from memory abstract (before separator)."""
+ if not text:
+ return ""
+
+ normalized = " ".join(str(text).strip().split())
+ # Prefer common separators used by extraction templates.
+ for sep in (":", ":", "-", "—"):
+ if sep in normalized:
+ left = normalized.split(sep, 1)[0].strip().lower()
+ if left:
+ return left
+
+ # Fallback: short leading phrase.
+ m = re.match(r"^(.{1,24})\s", normalized.lower())
+ if m:
+ return m.group(1).strip()
+ return normalized[:24].lower().strip()
@staticmethod
def _cosine_similarity(vec_a: List[float], vec_b: List[float]) -> float:
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index 83f55d5d1..584cc42b1 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -7,6 +7,7 @@ Provides centralized schema definitions and factory functions for creating colle
similar to how init_viking_fs encapsulates VikingFS initialization.
"""
+import hashlib
import json
from typing import Any, Dict, Optional
@@ -172,6 +173,11 @@ class TextEmbeddingHandler(DequeueHandlerBase):
# Write to vector database
try:
+ # Ensure vector DB has at most one record per URI.
+ uri = inserted_data.get("uri")
+ if uri:
+ inserted_data["id"] = hashlib.md5(uri.encode("utf-8")).hexdigest()
+
record_id = await self._vikingdb.insert(self._collection_name, inserted_data)
if record_id:
logger.debug(
diff --git a/openviking/storage/vectordb/utils/validation.py b/openviking/storage/vectordb/utils/validation.py
index 150d3638a..62ff3fff8 100644
--- a/openviking/storage/vectordb/utils/validation.py
+++ b/openviking/storage/vectordb/utils/validation.py
@@ -61,7 +61,7 @@ class FieldTypeEnum(str, Enum):
class DenseVectorize(BaseModel):
- model_config = ConfigDict(extra="forbid")
+ model_config = ConfigDict()
ModelName: str
ModelVersion: Optional[str] = None
@@ -94,7 +94,7 @@ class DenseVectorize(BaseModel):
class SparseVectorize(BaseModel):
- model_config = ConfigDict(extra="forbid")
+ model_config = ConfigDict()
ModelName: str
ModelVersion: Optional[str] = None
@@ -102,14 +102,14 @@ class SparseVectorize(BaseModel):
class VectorizeConfig(BaseModel):
- model_config = ConfigDict(extra="forbid")
+ model_config = ConfigDict()
Dense: Optional[DenseVectorize] = None
Sparse: Optional[SparseVectorize] = None
class CollectionField(BaseModel):
- model_config = ConfigDict(extra="forbid")
+ model_config = ConfigDict()
FieldName: str
FieldType: FieldTypeEnum
@@ -145,7 +145,7 @@ class CollectionField(BaseModel):
class CollectionMetaConfig(BaseModel):
- model_config = ConfigDict(extra="forbid")
+ model_config = ConfigDict()
CollectionName: str
Fields: List[CollectionField]
@@ -180,7 +180,7 @@ class CollectionMetaConfig(BaseModel):
class VectorIndexConfig(BaseModel):
- model_config = ConfigDict(extra="forbid")
+ model_config = ConfigDict()
IndexType: Literal["flat", "flat_hybrid", "FLAT", "FLAT_HYBRID"]
Distance: Optional[Literal["l2", "ip", "cosine", "L2", "IP", "COSINE"]] = None
@@ -227,7 +227,7 @@ class VectorIndexConfig(BaseModel):
class IndexMetaConfig(BaseModel):
- model_config = ConfigDict(extra="forbid")
+ model_config = ConfigDict()
IndexName: str
VectorIndex: VectorIndexConfig
@@ -245,7 +245,7 @@ class IndexMetaConfig(BaseModel):
class IndexMetaUpdateConfig(BaseModel):
- model_config = ConfigDict(extra="forbid")
+ model_config = ConfigDict()
IndexName: Optional[str] = None
VectorIndex: Optional[VectorIndexConfig] = None
@@ -263,7 +263,7 @@ class IndexMetaUpdateConfig(BaseModel):
class CollectionMetaUpdateConfig(BaseModel):
- model_config = ConfigDict(extra="forbid")
+ model_config = ConfigDict()
CollectionName: Optional[str] = None
Fields: Optional[List[CollectionField]] = None
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index 4f1b7efd4..57eecb833 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -475,7 +475,7 @@ class VikingVectorIndexBackend(VikingDBInterface):
record = dict(item.fields) if item.fields else {}
record["id"] = item.id
records.append(record)
- if len(records) > 0:
+ if len(records) > 1:
raise ValueError(f"Duplicate records found for URI: {uri}")
if len(records) == 0:
raise ValueError(f"Record not found for URI: {uri}")
@@ -560,29 +560,25 @@ class VikingVectorIndexBackend(VikingDBInterface):
async def remove_by_uri(self, collection: str, uri: str) -> int:
"""Remove resource(s) by URI."""
try:
- # Find records with matching URI
target_records = await self.filter(
collection=collection,
filter={"op": "must", "field": "uri", "conds": [uri]},
- limit=1,
+ limit=10,
)
if not target_records:
return 0
total_deleted = 0
- target = target_records[0]
- is_leaf = target.get("is_leaf", False)
- # If not leaf (i.e., intermediate directory), find and delete all descendants recursively
- if not is_leaf:
+ # If any record indicates this URI is a directory node, remove descendants first.
+ if any(not r.get("is_leaf", False) for r in target_records):
descendant_count = await self._remove_descendants(collection, uri)
total_deleted += descendant_count
- # Delete the target itself
- if "id" in target:
- await self.delete(collection, [target["id"]])
- total_deleted += 1
+ ids = [r.get("id") for r in target_records if r.get("id")]
+ if ids:
+ total_deleted += await self.delete(collection, ids)
logger.info(f"Removed {total_deleted} record(s) for URI: {uri}")
return total_deleted