Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
Feat/memory diff (#1710)
chenjw9bcbaad0ed67
diff --git a/openviking/session/compressor.py b/openviking/session/compressor.pyindex c8edd24ab..0c04d7825 100644--- a/openviking/session/compressor.py+++ b/openviking/session/compressor.py@@ -306,6 +306,7 @@ class SessionCompressor:ctx: Optional[RequestContext] = None,strict_extract_errors: bool = False,latest_archive_overview: str = "",+ archive_uri: str = "",) -> List[Context]:"""Extract long-term memories from messages."""if not messages:diff --git a/openviking/session/compressor_v2.py b/openviking/session/compressor_v2.pyindex 0afda749d..730f4cde9 100644--- a/openviking/session/compressor_v2.py+++ b/openviking/session/compressor_v2.py@@ -8,13 +8,16 @@ Maintains the same interface as compressor.py for backward compatibility."""import asyncio-from typing import List, Optional+import json+from datetime import datetime+from typing import Any, Dict, List, Optionalfrom openviking.core.context import Contextfrom openviking.core.namespace import agent_space_fragment, user_space_fragmentfrom openviking.message import Messagefrom openviking.server.identity import RequestContextfrom openviking.session.memory import ExtractLoop, MemoryUpdater+from openviking.session.memory.memory_updater import MemoryUpdateResultfrom openviking.session.memory.utils.json_parser import JsonUtilsfrom openviking.storage import VikingDBManagerfrom openviking.storage.viking_fs import get_viking_fs@@ -89,11 +92,15 @@ class SessionCompressorV2:ctx: Optional[RequestContext] = None,strict_extract_errors: bool = False,latest_archive_overview: str = "",+ archive_uri: str = "",) -> List[Context]:"""Extract long-term memories from messages using v2 templating system.Note: Returns empty List[Context] because v2 directly writes to storage.The list length is used for stats in session.py.++ Args:+ archive_uri: The archive URI for this extraction, used to write memory_diff.json."""if not messages:@@ -246,6 +253,10 @@ class SessionCompressorV2:telemetry.set("memory.extract.deleted", len(result.deleted_uris))telemetry.set("memory.extract.skipped", len(result.errors))+ # Build and write memory_diff.json to archive+ if archive_uri and result.has_changes():+ await self._write_memory_diff(result, viking_fs, ctx, archive_uri)+# Build Context objects for stats in session.pycontexts: List[Context] = []@@ -293,3 +304,65 @@ class SessionCompressorV2:await lock_manager.release(transaction_handle)except Exception as e:logger.warning(f"Failed to release transaction lock: {e}")++ def _build_memory_diff(self, result: MemoryUpdateResult, archive_uri: str) -> Dict[str, Any]:+ """Build memory_diff.json structure from MemoryUpdateResult.++ Uses before/after content already captured during _apply_edit/_apply_delete.+ """+ adds = []+ for entry in result.adds:+ adds.append({+ "uri": entry.uri,+ "after": entry.after,+ })++ updates = []+ for entry in result.updates:+ updates.append({+ "uri": entry.uri,+ "before": entry.before,+ "after": entry.after,+ })++ deletes = []+ for entry in result.deletes:+ deletes.append({+ "uri": entry.uri,+ "deleted_content": entry.deleted_content,+ })++ return {+ "archive_uri": archive_uri,+ "extracted_at": datetime.utcnow().isoformat() + "Z",+ "operations": {+ "adds": adds,+ "updates": updates,+ "deletes": deletes,+ },+ "summary": {+ "total_adds": len(adds),+ "total_updates": len(updates),+ "total_deletes": len(deletes),+ },+ }++ async def _write_memory_diff(+ self,+ result: MemoryUpdateResult,+ viking_fs: Any,+ ctx: RequestContext,+ archive_uri: str,+ ) -> None:+ """Build and write memory_diff.json to the archive directory."""+ try:+ diff = self._build_memory_diff(result, archive_uri)+ diff_json = json.dumps(diff, ensure_ascii=False, indent=2)+ diff_uri = f"{archive_uri}/memory_diff.json"+ await viking_fs.write_file(diff_uri, diff_json, ctx=ctx)+ logger.info(f"Wrote memory_diff.json to {diff_uri}: "+ f"adds={diff['summary']['total_adds']}, "+ f"updates={diff['summary']['total_updates']}, "+ f"deletes={diff['summary']['total_deletes']}")+ except Exception as e:+ logger.warning(f"Failed to write memory_diff.json: {e}")diff --git a/openviking/session/memory/memory_updater.py b/openviking/session/memory/memory_updater.pyindex 4a3ec986d..d497e0c92 100644--- a/openviking/session/memory/memory_updater.py+++ b/openviking/session/memory/memory_updater.py@@ -180,6 +180,23 @@ class MessageRange:return None+class MemoryDiffEntry:+ """A single memory diff entry tracking before/after content."""++ def __init__(self, uri: str, before: str = "", after: str = ""):+ self.uri = uri+ self.before = before+ self.after = after+++class MemoryDeleteEntry:+ """A single memory delete entry tracking deleted content."""++ def __init__(self, uri: str, deleted_content: str = ""):+ self.uri = uri+ self.deleted_content = deleted_content++class MemoryUpdateResult:"""Result of memory update operation."""@@ -188,15 +205,21 @@ class MemoryUpdateResult:self.edited_uris: List[str] = []self.deleted_uris: List[str] = []self.errors: List[Tuple[str, Exception]] = []+ self.adds: List[MemoryDiffEntry] = []+ self.updates: List[MemoryDiffEntry] = []+ self.deletes: List[MemoryDeleteEntry] = []- def add_written(self, uri: str) -> None:+ def add_written(self, uri: str, before: str = "", after: str = "") -> None:self.written_uris.append(uri)+ self.adds.append(MemoryDiffEntry(uri, before=before, after=after))- def add_edited(self, uri: str) -> None:+ def add_edited(self, uri: str, before: str = "", after: str = "") -> None:self.edited_uris.append(uri)+ self.updates.append(MemoryDiffEntry(uri, before=before, after=after))- def add_deleted(self, uri: str) -> None:+ def add_deleted(self, uri: str, deleted_content: str = "") -> None:self.deleted_uris.append(uri)+ self.deletes.append(MemoryDeleteEntry(uri, deleted_content=deleted_content))def add_error(self, uri: str, error: Exception) -> None:self.errors.append((uri, error))@@ -287,10 +310,10 @@ class MemoryUpdater:result.add_error("unknown", ValueError(error))return result- # Apply unified operations - _apply_edit returns True if edited, False if written+ # Apply unified operations - _apply_edit returns (is_edited, before_content, after_content)for resolved_op in resolved_ops.operations:try:- is_edited = await self._apply_edit(+ is_edited, before_content, after_content = await self._apply_edit(resolved_op.model,resolved_op.uri,ctx,@@ -298,9 +321,16 @@ class MemoryUpdater:memory_type=resolved_op.memory_type,)if is_edited:- result.add_edited(resolved_op.uri)+ result.add_edited(+ resolved_op.uri,+ before=before_content,+ after=after_content,+ )else:- result.add_written(resolved_op.uri)+ result.add_written(+ resolved_op.uri,+ after=after_content,+ )except Exception as e:tracer.error(f"Failed to apply operation: {e}, op={resolved_op.model}, op type={type(resolved_op.model)}",@@ -313,8 +343,8 @@ class MemoryUpdater:# Apply delete operationsfor _uri_str, uri in resolved_ops.delete_operations:try:- await self._apply_delete(uri, ctx)- result.add_deleted(uri)+ deleted_content = await self._apply_delete(uri, ctx)+ result.add_deleted(uri, deleted_content=deleted_content)except Exception as e:tracer.error(f"Failed to delete memory {uri}", e)result.add_error(uri, e)@@ -364,11 +394,13 @@ class MemoryUpdater:ctx: RequestContext,extract_context: Any = None,memory_type: str = None,- ) -> bool:+ ) -> Tuple[bool, str, str]:"""Apply edit operation from a flat model.Returns:- True if file was edited (existed), False if file was written (new)+ Tuple of (file_existed, before_content, after_content).+ before_content is the previous plain content (empty for new files).+ after_content is the new plain content."""viking_fs = self._get_viking_fs()@@ -380,9 +412,11 @@ class MemoryUpdater:# Read current memory (or use empty if not found)current_full_content = ""+ before_plain_content = ""file_existed = Truetry:current_full_content = await viking_fs.read_file(uri, ctx=ctx) or ""+ before_plain_content, _ = deserialize_full(current_full_content)except NotFoundError:file_existed = False@@ -431,12 +465,29 @@ class MemoryUpdater:self._print_diff(uri, current_plain_content, new_full_content)await viking_fs.write_file(uri, new_full_content, ctx=ctx)- return file_existed- async def _apply_delete(self, uri: str, ctx: RequestContext) -> None:- """Apply delete operation (uri is already a string)."""+ # Extract after plain content for memory_diff+ after_plain_content, _ = deserialize_full(new_full_content)+ return file_existed, before_plain_content, after_plain_content++ async def _apply_delete(self, uri: str, ctx: RequestContext) -> str:+ """Apply delete operation (uri is already a string).++ Returns:+ The plain content that was deleted (empty string if not found).+ """viking_fs = self._get_viking_fs()+ # Read content before deletion for memory_diff+ deleted_plain_content = ""+ try:+ raw_content = await viking_fs.read_file(uri, ctx=ctx) or ""+ deleted_plain_content, _ = deserialize_full(raw_content)+ except NotFoundError:+ pass+ except Exception:+ pass+# Delete from VikingFS# VikingFS automatically handles vector index cleanuptry:@@ -445,6 +496,8 @@ class MemoryUpdater:tracer.error(f"Memory not found for delete: {uri}")# Idempotent - deleting non-existent file succeeds+ return deleted_plain_content+def _print_diff(self, uri: str, old_content: str, new_content: str) -> None:"""Print a diff of the memory edit using diff_match_patch."""try:diff --git a/openviking/session/session.py b/openviking/session/session.pyindex 6f303794b..17131f0cb 100644--- a/openviking/session/session.py+++ b/openviking/session/session.py@@ -629,6 +629,7 @@ class Session:session_id=self.session_id,ctx=self.ctx,latest_archive_overview=latest_archive_overview,+ archive_uri=archive_uri,)logger.info(f"Extracted {len(extracted)} memories")for ctx_item in extracted:diff --git a/tests/session/memory/test_memory_updater.py b/tests/session/memory/test_memory_updater.pyindex 0d198d73f..8e9074790 100644--- a/tests/session/memory/test_memory_updater.py+++ b/tests/session/memory/test_memory_updater.py@@ -156,7 +156,7 @@ class TestMemoryUpdater:updater = MemoryUpdater(registry=registry)updater._get_viking_fs = MagicMock(return_value=MagicMock())- updater._apply_edit = AsyncMock(return_value=False)+ updater._apply_edit = AsyncMock(return_value=(False, "", ""))updater._vectorize_memories = AsyncMock()updater.generate_overview = AsyncMock()diff --git a/uv.lock b/uv.lockindex 11e8ad6b3..508813016 100644--- a/uv.lock+++ b/uv.lock@@ -3731,7 +3731,7 @@ requires-dist = [{ name = "langfuse", marker = "extra == 'bot-langfuse'", specifier = ">=3.0.0" },{ name = "lark-oapi", specifier = ">=1.5.3" },{ name = "lark-oapi", marker = "extra == 'bot-feishu'", specifier = ">=1.0.0" },- { name = "litellm", specifier = ">=1.0.0,<1.83.1" },+ { name = "litellm", specifier = ">=1.0.0,<1.83.13" },{ name = "llama-cpp-python", marker = "extra == 'local-embed'", specifier = ">=0.3.0" },{ name = "loguru", specifier = ">=0.7.3" },{ name = "markdownify", specifier = ">=0.11.0" },