Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat: chunk long messages for memory extraction (#2543)
agent11022412dd3d
diff --git a/openviking/session/compressor_v2.py b/openviking/session/compressor_v2.pyindex 596f99b05..fb55d20fb 100644--- a/openviking/session/compressor_v2.py+++ b/openviking/session/compressor_v2.py@@ -20,7 +20,6 @@ from openviking.session.memory.constants import EXECUTION_MEMORY_TYPESfrom openviking.session.memory.dataclass import ResolvedOperations, StoredLinkfrom openviking.session.memory.memory_isolation_handler import MemoryIsolationHandlerfrom openviking.session.memory.memory_updater import (- ExtractContext,MemoryUpdateResult,write_stored_links,)@@ -140,6 +139,7 @@ class SessionCompressorV2:latest_archive_overview: str = "",isolation_handler: Optional[MemoryIsolationHandler] = None,transaction_handle=None,+ context_provider: Optional[Any] = None,) -> ExtractLoop:"""Create new ExtractLoop instance with current ctx.@@ -150,19 +150,20 @@ class SessionCompressorV2:vlm = config.vlm.get_vlm_instance()viking_fs = get_viking_fs()- # Create context provider with messages (provider 负责加载 schema)- from openviking.session.memory.session_extract_context_provider import (- SessionExtractContextProvider,- )+ if context_provider is None:+ # Create context provider with messages (provider 负责加载 schema)+ from openviking.session.memory.session_extract_context_provider import (+ SessionExtractContextProvider,+ )- context_provider = SessionExtractContextProvider(- messages=messages,- latest_archive_overview=latest_archive_overview,- isolation_handler=isolation_handler,- ctx=ctx,- viking_fs=viking_fs,- transaction_handle=transaction_handle,- )+ context_provider = SessionExtractContextProvider(+ messages=messages,+ latest_archive_overview=latest_archive_overview,+ isolation_handler=isolation_handler,+ ctx=ctx,+ viking_fs=viking_fs,+ transaction_handle=transaction_handle,+ )return ExtractLoop(vlm=vlm,@@ -296,12 +297,19 @@ class SessionCompressorV2:logger.debug("AGFS unavailable, running memory extraction without locks")try:- # Create extract context from messages- from openviking.session.memory.memory_updater import ExtractContext-- extract_context = ExtractContext(messages)+ from openviking.session.memory.session_extract_context_provider import (+ SessionExtractContextProvider,+ )- # Create MemoryIsolationHandler+ context_provider = SessionExtractContextProvider(+ messages=messages,+ latest_archive_overview=latest_archive_overview,+ isolation_handler=None,+ ctx=ctx,+ viking_fs=viking_fs,+ transaction_handle=transaction_handle,+ )+ extract_context = context_provider.get_extract_context()isolation_handler = MemoryIsolationHandler(ctx,extract_context,@@ -310,6 +318,8 @@ class SessionCompressorV2:allowed_peer_ids=allowed_peer_ids,)isolation_handler.prepare_messages()+ context_provider._isolation_handler = isolation_handler+# 获取所有记忆 schema 目录并加锁(仅在有锁管理器时)orchestrator = self._get_or_create_react(ctx=ctx,@@ -317,6 +327,7 @@ class SessionCompressorV2:latest_archive_overview=latest_archive_overview,isolation_handler=isolation_handler,transaction_handle=transaction_handle,+ context_provider=context_provider,)read_scope = isolation_handler.get_read_scope()if lock_manager:@@ -704,9 +715,9 @@ class SessionCompressorV2:vlm = config.vlm.get_vlm_instance()viking_fs = get_viking_fs()- # Build isolation_handler BEFORE creating the orchestrator so that- # ExtractLoop.resolve_operations() can fill identity fields correctly.- extract_context = ExtractContext(messages)+ # Use the provider's extraction context so prompt ranges and memory+ # rendering resolve against the same message list.+ extract_context = provider.get_extract_context()isolation_handler = MemoryIsolationHandler(ctx,extract_context,diff --git a/openviking/session/memory/memory_updater.py b/openviking/session/memory/memory_updater.pyindex cedacd706..b7c0eac2c 100644--- a/openviking/session/memory/memory_updater.py+++ b/openviking/session/memory/memory_updater.py@@ -9,12 +9,15 @@ to the storage system.from __future__ import annotations+import re+from dataclasses import dataclassfrom typing import TYPE_CHECKING, Any, Dict, List, Optional, Tupleif TYPE_CHECKING:from openviking.session.memory.memory_isolation_handler import MemoryIsolationHandlerfrom openviking.message import Message+from openviking.message.part import TextPartfrom openviking.server.identity import RequestContextfrom openviking.session.memory.dataclass import (MemoryFile,@@ -37,6 +40,18 @@ from openviking_cli.utils import get_loggerlogger = get_logger(__name__)+_EXTRACTION_CHUNK_MIN_CHARS = 100+_EXTRACTION_CHUNK_BOUNDARY_RE = re.compile(r"(\n+|[。!?;!?;]+|(?<!\d)\.(?!\d))")+++@dataclass(frozen=True)+class ChunkMeta:+ """Metadata for a derived extraction chunk message."""++ source_message_id: str+ chunk_index: int+ chunk_count: int+async def write_stored_links(links: List[StoredLink],@@ -82,10 +97,105 @@ async def write_stored_links(class ExtractContext:"""Extract context for template rendering."""- def __init__(self, messages: List[Message]):- self.messages = messages+ def __init__(self, messages: List[Message], chunk_meta: Optional[Dict[int, ChunkMeta]] = None):+ if chunk_meta is None:+ self.messages, self.chunk_meta = self._build_extraction_messages(messages)+ else:+ self.messages = messages+ self.chunk_meta = chunk_metaself.page_id_map = PageIdMap()+ @classmethod+ def _build_extraction_messages(+ cls, messages: List[Message]+ ) -> Tuple[List[Message], Dict[int, ChunkMeta]]:+ """Build messages used by memory extraction.++ Long text-only messages are split into derived chunks so event `ranges`+ can point to a narrower source span without relying on brittle text+ matching. The original session messages are not modified.+ """+ extraction_messages: List[Message] = []+ chunk_meta: Dict[int, ChunkMeta] = {}+ for message in messages:+ for extraction_message, meta in cls._split_message_for_extraction(message):+ extraction_messages.append(extraction_message)+ if meta is not None:+ chunk_meta[id(extraction_message)] = meta+ return extraction_messages, chunk_meta++ @classmethod+ def _split_message_for_extraction(+ cls, message: Message+ ) -> List[Tuple[Message, Optional[ChunkMeta]]]:+ parts = getattr(message, "parts", [])+ if not parts or not all(isinstance(part, TextPart) for part in parts):+ return [(message, None)]++ text = "".join(part.text for part in parts)+ chunks = cls._split_text_for_extraction(text)+ if len(chunks) <= 1:+ return [(message, None)]++ chunk_messages = []+ for idx, chunk in enumerate(chunks):+ chunk_message = Message(+ id=f"{message.id}#chunk_{idx}",+ role=message.role,+ peer_id=getattr(message, "peer_id", None),+ parts=[TextPart(chunk)],+ created_at=message.created_at,+ )+ chunk_messages.append(+ (+ chunk_message,+ ChunkMeta(+ source_message_id=message.id,+ chunk_index=idx,+ chunk_count=len(chunks),+ ),+ )+ )+ return chunk_messages++ @classmethod+ def _split_text_for_extraction(cls, text: str) -> List[str]:+ return cls._pack_text_units(cls._split_text_units(text)) or [text]++ @staticmethod+ def _pack_text_units(units: List[str]) -> List[str]:+ chunks: List[str] = []+ current = ""+ for unit in units:+ current += unit+ if len(current) < _EXTRACTION_CHUNK_MIN_CHARS:+ continue+ chunks.append(current)+ current = ""++ if current:+ if chunks:+ chunks[-1] += current+ else:+ chunks.append(current)+ return chunks++ @staticmethod+ def _split_text_units(text: str) -> List[str]:+ pieces = _EXTRACTION_CHUNK_BOUNDARY_RE.split(text)+ units: List[str] = []+ current = ""+ for piece in pieces:+ if not piece:+ continue+ current += piece+ if _EXTRACTION_CHUNK_BOUNDARY_RE.fullmatch(piece):+ units.append(current)+ current = ""+ if current:+ units.append(current)+ return units or [text]+def get_first_message_time_from_ranges(self, ranges_str: str) -> str | None:"""根据 ranges 字符串获取第一条消息的时间(YAML 日期格式)"""if not ranges_str:@@ -226,27 +336,87 @@ class ExtractContext:range_msgs = self.messages[start : end + 1]elements.append(range_msgs)- return MessageRange(elements)+ return MessageRange(elements, chunk_meta=self.chunk_meta)class MessageRange:"""Represents a range of messages for formatting."""- def __init__(self, elements: List[List[Message]]):+ def __init__(+ self,+ elements: List[List[Message]],+ chunk_meta: Optional[Dict[int, ChunkMeta]] = None,+ ):self.elements = elements+ self.chunk_meta = chunk_meta or {}def pretty_print(self) -> str:"""Pretty print the message range with '...' separator between non-contiguous ranges."""result = []for i, msg_group in enumerate(self.elements):- for msg in msg_group:- speaker = msg.peer_id or msg.role- result.append(f"[{speaker}]: {msg.content}")+ result.extend(self._format_contiguous_group(msg_group))# Add "..." separator between non-contiguous message groupsif i < len(self.elements) - 1:result.append("...")return "\n".join(result)+ def _format_contiguous_group(self, msg_group: List[Message]) -> List[str]:+ formatted = []+ current_messages: List[Message] = []++ def flush_current() -> None:+ nonlocal current_messages+ if not current_messages:+ return+ content = self._format_merged_content(current_messages)+ formatted.append(f"[{self._speaker_for(current_messages[0])}]: {content}")+ current_messages = []++ for msg in msg_group:+ if current_messages and not self._can_merge_messages(current_messages[-1], msg):+ flush_current()+ current_messages.append(msg)++ flush_current()+ return formatted++ @staticmethod+ def _speaker_for(message: Message) -> str:+ return (+ getattr(message, "peer_id", None) or getattr(message, "role_id", None) or message.role+ )++ def _can_merge_messages(self, previous: Message, current: Message) -> bool:+ previous_meta = self._chunk_meta_for(previous)+ current_meta = self._chunk_meta_for(current)+ if previous_meta is None or current_meta is None:+ return False+ if self._speaker_for(previous) != self._speaker_for(current):+ return False+ return (+ previous_meta.source_message_id == current_meta.source_message_id+ and current_meta.chunk_index == previous_meta.chunk_index + 1+ )++ def _format_merged_content(self, messages: List[Message]) -> str:+ content = "".join((msg.content or "") for msg in messages)+ if not messages or not self._contains_chunk_message(messages):+ return content++ first_chunk = self._chunk_meta_for(messages[0])+ if first_chunk is not None and first_chunk.chunk_index > 0:+ content = "..." + content.lstrip()+ last_chunk = self._chunk_meta_for(messages[-1])+ if last_chunk is not None and last_chunk.chunk_index < last_chunk.chunk_count - 1:+ content = content.rstrip() + "..."+ return content++ def _contains_chunk_message(self, messages: List[Message]) -> bool:+ return any(self._chunk_meta_for(msg) is not None for msg in messages)++ def _chunk_meta_for(self, message: Message) -> Optional[ChunkMeta]:+ return self.chunk_meta.get(id(message))+def _first_message_time(self) -> str | None:"""获取第一条消息的时间(内部方法)"""for msg_group in self.elements:diff --git a/openviking/session/memory/session_extract_context_provider.py b/openviking/session/memory/session_extract_context_provider.pyindex cce533c2e..3afed47fe 100644--- a/openviking/session/memory/session_extract_context_provider.py+++ b/openviking/session/memory/session_extract_context_provider.py@@ -99,7 +99,9 @@ class SessionExtractContextProvider(ExtractContextProvider):from openviking.session.memory.memory_updater import ExtractContextif self._extract_context is None:- self._extract_context = ExtractContext(self.messages or [])+ self._extract_context = ExtractContext(+ self.messages if isinstance(self.messages, list) else []+ )return self._extract_contextdef _detect_language(self) -> str:@@ -180,7 +182,8 @@ from neighboring messages.else:time_display = session_time_str- conversation = self._assemble_conversation(self.messages)+ extract_context = self.get_extract_context()+ conversation = self._assemble_conversation(extract_context.messages)return {"role": "user",diff --git a/tests/session/memory/test_compressor_v2.py b/tests/session/memory/test_compressor_v2.pyindex d1dda6366..8dec9e1f6 100644--- a/tests/session/memory/test_compressor_v2.py+++ b/tests/session/memory/test_compressor_v2.py@@ -328,6 +328,9 @@ class TestCompressorV2:def get_memory_schemas(self, ctx):return []+ def get_extract_context(self):+ return ExtractContext(messages)+return DummyProvider()async def run(self):@@ -552,6 +555,9 @@ class TestCompressorV2:def get_memory_schemas(self, _ctx):return [FixedSchema(), VariableSchema()]+ def get_extract_context(self):+ return ExtractContext(messages)+def _get_registry(self):return object()@@ -619,6 +625,9 @@ class TestCompressorV2:def get_memory_schemas(self, _ctx):return []+ def get_extract_context(self):+ return ExtractContext(messages)+def _get_registry(self):return object()diff --git a/tests/session/memory/test_memory_timestamp_parsing.py b/tests/session/memory/test_memory_timestamp_parsing.pyindex 86e121253..2f05c26f6 100644--- a/tests/session/memory/test_memory_timestamp_parsing.py+++ b/tests/session/memory/test_memory_timestamp_parsing.py@@ -92,6 +92,170 @@ def test_message_range_uses_peer_id_when_present():assert "[default]: invoice follow-up" not in msg_range.pretty_print()+def test_extraction_context_chunks_long_text_and_preserves_decimal_numbers(stub_provider_config):+ text = (+ "周一 "+ + "需求沟通。" * 20+ + "上下文用量out:21.9K、in:110.2k、缓存读取501.0k,先简单记录下来。 "+ + "周二 面试三个新员工:王军优势是agent算法,田伟优势是分布式后端架构,"+ + "金然优势是Agent架构全栈,缺点是缺乏CPU服务器底层调优经验 "+ + "周三 query-builder的输出直接进行判断会触发错误,原因不明。"+ )+ extract_context = ExtractContext(+ [+ Message(+ id="msg-long",+ role="user",+ parts=[TextPart(text=text)],+ created_at="2026-06-01T08:56:37.382Z",+ )+ ]+ )++ extraction_messages = extract_context.messages+ merged = "".join(message.content for message in extraction_messages)++ assert len(extraction_messages) > 1+ assert merged == text+ assert all(message.id.startswith("msg-long#chunk_") for message in extraction_messages)+ assert all(id(message) in extract_context.chunk_meta for message in extraction_messages)+ assert "21.9K" in merged+ assert "110.2k" in merged+ assert "501.0k" in merged+ assert not any(message.content.startswith(("9K", "2k")) for message in extraction_messages)+++def test_message_range_merges_adjacent_chunks_for_same_speaker():+ first = Message(+ id="msg-long#chunk_0",+ role="user",+ parts=[TextPart(text="first chunk. ")],+ peer_id="web:visitor:alice",+ )+ second = Message(+ id="msg-long#chunk_1",+ role="user",+ parts=[TextPart(text="second chunk.")],+ peer_id="web:visitor:alice",+ )+ msg_range = MessageRange(+ [[first, second]],+ chunk_meta={+ id(first): SimpleNamespace(+ source_message_id="msg-long",+ chunk_index=0,+ chunk_count=2,+ ),+ id(second): SimpleNamespace(+ source_message_id="msg-long",+ chunk_index=1,+ chunk_count=2,+ ),+ },+ )++ assert msg_range.pretty_print() == "[web:visitor:alice]: first chunk. second chunk."+++def test_message_range_keeps_regular_same_speaker_messages_separate():+ msg_range = MessageRange(+ [+ [+ Message(+ id="msg-a",+ role="user",+ parts=[TextPart(text="first message")],+ peer_id="web:visitor:alice",+ ),+ Message(+ id="msg-b",+ role="user",+ parts=[TextPart(text="second message")],+ peer_id="web:visitor:alice",+ ),+ ]+ ]+ )++ assert msg_range.pretty_print() == (+ "[web:visitor:alice]: first message\n[web:visitor:alice]: second message"+ )+++def test_message_range_keeps_different_source_chunks_separate():+ first = Message(+ id="msg-a#chunk_0",+ role="user",+ parts=[TextPart(text="first source chunk")],+ peer_id="web:visitor:alice",+ )+ second = Message(+ id="msg-b#chunk_0",+ role="user",+ parts=[TextPart(text="second source chunk")],+ peer_id="web:visitor:alice",+ )+ msg_range = MessageRange(+ [[first, second]],+ chunk_meta={+ id(first): SimpleNamespace(+ source_message_id="msg-a",+ chunk_index=0,+ chunk_count=2,+ ),+ id(second): SimpleNamespace(+ source_message_id="msg-b",+ chunk_index=0,+ chunk_count=2,+ ),+ },+ )++ assert msg_range.pretty_print() == (+ "[web:visitor:alice]: first source chunk...\n[web:visitor:alice]: second source chunk..."+ )+++def test_message_range_does_not_treat_original_chunk_like_id_as_chunk():+ msg_range = MessageRange(+ [+ [+ Message(+ id="msg-long#chunk_0",+ role="user",+ parts=[TextPart(text="original message id contains chunk marker")],+ peer_id="web:visitor:alice",+ ),+ ]+ ]+ )++ assert msg_range.pretty_print() == (+ "[web:visitor:alice]: original message id contains chunk marker"+ )+++def test_message_range_marks_middle_chunk_as_abbreviated():+ message = Message(+ id="msg-long#chunk_1",+ role="user",+ parts=[TextPart(text="middle chunk")],+ peer_id="web:visitor:alice",+ )+ msg_range = MessageRange(+ [[message]],+ chunk_meta={+ id(message): SimpleNamespace(+ source_message_id="msg-long",+ chunk_index=1,+ chunk_count=3,+ )+ },+ )++ assert msg_range.pretty_print() == "[web:visitor:alice]: ...middle chunk..."++def test_peer_id_routes_peer_memory_for_all_role_selected_types(stub_provider_config):messages = [Message(