Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat: chunk long messages for memory extraction (#2543)

agent11022412dd3d
diff --git a/openviking/session/compressor_v2.py b/openviking/session/compressor_v2.py
index 596f99b05..fb55d20fb 100644
--- a/openviking/session/compressor_v2.py
+++ b/openviking/session/compressor_v2.py
@@ -20,7 +20,6 @@ from openviking.session.memory.constants import EXECUTION_MEMORY_TYPES
from openviking.session.memory.dataclass import ResolvedOperations, StoredLink
from openviking.session.memory.memory_isolation_handler import MemoryIsolationHandler
from openviking.session.memory.memory_updater import (
- ExtractContext,
MemoryUpdateResult,
write_stored_links,
)
@@ -140,6 +139,7 @@ class SessionCompressorV2:
latest_archive_overview: str = "",
isolation_handler: Optional[MemoryIsolationHandler] = None,
transaction_handle=None,
+ context_provider: Optional[Any] = None,
) -> ExtractLoop:
"""Create new ExtractLoop instance with current ctx.
@@ -150,19 +150,20 @@ class SessionCompressorV2:
vlm = config.vlm.get_vlm_instance()
viking_fs = get_viking_fs()
- # Create context provider with messages (provider 负责加载 schema)
- from openviking.session.memory.session_extract_context_provider import (
- SessionExtractContextProvider,
- )
+ if context_provider is None:
+ # Create context provider with messages (provider 负责加载 schema)
+ from openviking.session.memory.session_extract_context_provider import (
+ SessionExtractContextProvider,
+ )
- context_provider = SessionExtractContextProvider(
- messages=messages,
- latest_archive_overview=latest_archive_overview,
- isolation_handler=isolation_handler,
- ctx=ctx,
- viking_fs=viking_fs,
- transaction_handle=transaction_handle,
- )
+ context_provider = SessionExtractContextProvider(
+ messages=messages,
+ latest_archive_overview=latest_archive_overview,
+ isolation_handler=isolation_handler,
+ ctx=ctx,
+ viking_fs=viking_fs,
+ transaction_handle=transaction_handle,
+ )
return ExtractLoop(
vlm=vlm,
@@ -296,12 +297,19 @@ class SessionCompressorV2:
logger.debug("AGFS unavailable, running memory extraction without locks")
try:
- # Create extract context from messages
- from openviking.session.memory.memory_updater import ExtractContext
-
- extract_context = ExtractContext(messages)
+ from openviking.session.memory.session_extract_context_provider import (
+ SessionExtractContextProvider,
+ )
- # Create MemoryIsolationHandler
+ context_provider = SessionExtractContextProvider(
+ messages=messages,
+ latest_archive_overview=latest_archive_overview,
+ isolation_handler=None,
+ ctx=ctx,
+ viking_fs=viking_fs,
+ transaction_handle=transaction_handle,
+ )
+ extract_context = context_provider.get_extract_context()
isolation_handler = MemoryIsolationHandler(
ctx,
extract_context,
@@ -310,6 +318,8 @@ class SessionCompressorV2:
allowed_peer_ids=allowed_peer_ids,
)
isolation_handler.prepare_messages()
+ context_provider._isolation_handler = isolation_handler
+
# 获取所有记忆 schema 目录并加锁(仅在有锁管理器时)
orchestrator = self._get_or_create_react(
ctx=ctx,
@@ -317,6 +327,7 @@ class SessionCompressorV2:
latest_archive_overview=latest_archive_overview,
isolation_handler=isolation_handler,
transaction_handle=transaction_handle,
+ context_provider=context_provider,
)
read_scope = isolation_handler.get_read_scope()
if lock_manager:
@@ -704,9 +715,9 @@ class SessionCompressorV2:
vlm = config.vlm.get_vlm_instance()
viking_fs = get_viking_fs()
- # Build isolation_handler BEFORE creating the orchestrator so that
- # ExtractLoop.resolve_operations() can fill identity fields correctly.
- extract_context = ExtractContext(messages)
+ # Use the provider's extraction context so prompt ranges and memory
+ # rendering resolve against the same message list.
+ extract_context = provider.get_extract_context()
isolation_handler = MemoryIsolationHandler(
ctx,
extract_context,
diff --git a/openviking/session/memory/memory_updater.py b/openviking/session/memory/memory_updater.py
index cedacd706..b7c0eac2c 100644
--- a/openviking/session/memory/memory_updater.py
+++ b/openviking/session/memory/memory_updater.py
@@ -9,12 +9,15 @@ to the storage system.
from __future__ import annotations
+import re
+from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
if TYPE_CHECKING:
from openviking.session.memory.memory_isolation_handler import MemoryIsolationHandler
from openviking.message import Message
+from openviking.message.part import TextPart
from openviking.server.identity import RequestContext
from openviking.session.memory.dataclass import (
MemoryFile,
@@ -37,6 +40,18 @@ from openviking_cli.utils import get_logger
logger = get_logger(__name__)
+_EXTRACTION_CHUNK_MIN_CHARS = 100
+_EXTRACTION_CHUNK_BOUNDARY_RE = re.compile(r"(\n+|[。!?;!?;]+|(?<!\d)\.(?!\d))")
+
+
+@dataclass(frozen=True)
+class ChunkMeta:
+ """Metadata for a derived extraction chunk message."""
+
+ source_message_id: str
+ chunk_index: int
+ chunk_count: int
+
async def write_stored_links(
links: List[StoredLink],
@@ -82,10 +97,105 @@ async def write_stored_links(
class ExtractContext:
"""Extract context for template rendering."""
- def __init__(self, messages: List[Message]):
- self.messages = messages
+ def __init__(self, messages: List[Message], chunk_meta: Optional[Dict[int, ChunkMeta]] = None):
+ if chunk_meta is None:
+ self.messages, self.chunk_meta = self._build_extraction_messages(messages)
+ else:
+ self.messages = messages
+ self.chunk_meta = chunk_meta
self.page_id_map = PageIdMap()
+ @classmethod
+ def _build_extraction_messages(
+ cls, messages: List[Message]
+ ) -> Tuple[List[Message], Dict[int, ChunkMeta]]:
+ """Build messages used by memory extraction.
+
+ Long text-only messages are split into derived chunks so event `ranges`
+ can point to a narrower source span without relying on brittle text
+ matching. The original session messages are not modified.
+ """
+ extraction_messages: List[Message] = []
+ chunk_meta: Dict[int, ChunkMeta] = {}
+ for message in messages:
+ for extraction_message, meta in cls._split_message_for_extraction(message):
+ extraction_messages.append(extraction_message)
+ if meta is not None:
+ chunk_meta[id(extraction_message)] = meta
+ return extraction_messages, chunk_meta
+
+ @classmethod
+ def _split_message_for_extraction(
+ cls, message: Message
+ ) -> List[Tuple[Message, Optional[ChunkMeta]]]:
+ parts = getattr(message, "parts", [])
+ if not parts or not all(isinstance(part, TextPart) for part in parts):
+ return [(message, None)]
+
+ text = "".join(part.text for part in parts)
+ chunks = cls._split_text_for_extraction(text)
+ if len(chunks) <= 1:
+ return [(message, None)]
+
+ chunk_messages = []
+ for idx, chunk in enumerate(chunks):
+ chunk_message = Message(
+ id=f"{message.id}#chunk_{idx}",
+ role=message.role,
+ peer_id=getattr(message, "peer_id", None),
+ parts=[TextPart(chunk)],
+ created_at=message.created_at,
+ )
+ chunk_messages.append(
+ (
+ chunk_message,
+ ChunkMeta(
+ source_message_id=message.id,
+ chunk_index=idx,
+ chunk_count=len(chunks),
+ ),
+ )
+ )
+ return chunk_messages
+
+ @classmethod
+ def _split_text_for_extraction(cls, text: str) -> List[str]:
+ return cls._pack_text_units(cls._split_text_units(text)) or [text]
+
+ @staticmethod
+ def _pack_text_units(units: List[str]) -> List[str]:
+ chunks: List[str] = []
+ current = ""
+ for unit in units:
+ current += unit
+ if len(current) < _EXTRACTION_CHUNK_MIN_CHARS:
+ continue
+ chunks.append(current)
+ current = ""
+
+ if current:
+ if chunks:
+ chunks[-1] += current
+ else:
+ chunks.append(current)
+ return chunks
+
+ @staticmethod
+ def _split_text_units(text: str) -> List[str]:
+ pieces = _EXTRACTION_CHUNK_BOUNDARY_RE.split(text)
+ units: List[str] = []
+ current = ""
+ for piece in pieces:
+ if not piece:
+ continue
+ current += piece
+ if _EXTRACTION_CHUNK_BOUNDARY_RE.fullmatch(piece):
+ units.append(current)
+ current = ""
+ if current:
+ units.append(current)
+ return units or [text]
+
def get_first_message_time_from_ranges(self, ranges_str: str) -> str | None:
"""根据 ranges 字符串获取第一条消息的时间(YAML 日期格式)"""
if not ranges_str:
@@ -226,27 +336,87 @@ class ExtractContext:
range_msgs = self.messages[start : end + 1]
elements.append(range_msgs)
- return MessageRange(elements)
+ return MessageRange(elements, chunk_meta=self.chunk_meta)
class MessageRange:
"""Represents a range of messages for formatting."""
- def __init__(self, elements: List[List[Message]]):
+ def __init__(
+ self,
+ elements: List[List[Message]],
+ chunk_meta: Optional[Dict[int, ChunkMeta]] = None,
+ ):
self.elements = elements
+ self.chunk_meta = chunk_meta or {}
def pretty_print(self) -> str:
"""Pretty print the message range with '...' separator between non-contiguous ranges."""
result = []
for i, msg_group in enumerate(self.elements):
- for msg in msg_group:
- speaker = msg.peer_id or msg.role
- result.append(f"[{speaker}]: {msg.content}")
+ result.extend(self._format_contiguous_group(msg_group))
# Add "..." separator between non-contiguous message groups
if i < len(self.elements) - 1:
result.append("...")
return "\n".join(result)
+ def _format_contiguous_group(self, msg_group: List[Message]) -> List[str]:
+ formatted = []
+ current_messages: List[Message] = []
+
+ def flush_current() -> None:
+ nonlocal current_messages
+ if not current_messages:
+ return
+ content = self._format_merged_content(current_messages)
+ formatted.append(f"[{self._speaker_for(current_messages[0])}]: {content}")
+ current_messages = []
+
+ for msg in msg_group:
+ if current_messages and not self._can_merge_messages(current_messages[-1], msg):
+ flush_current()
+ current_messages.append(msg)
+
+ flush_current()
+ return formatted
+
+ @staticmethod
+ def _speaker_for(message: Message) -> str:
+ return (
+ getattr(message, "peer_id", None) or getattr(message, "role_id", None) or message.role
+ )
+
+ def _can_merge_messages(self, previous: Message, current: Message) -> bool:
+ previous_meta = self._chunk_meta_for(previous)
+ current_meta = self._chunk_meta_for(current)
+ if previous_meta is None or current_meta is None:
+ return False
+ if self._speaker_for(previous) != self._speaker_for(current):
+ return False
+ return (
+ previous_meta.source_message_id == current_meta.source_message_id
+ and current_meta.chunk_index == previous_meta.chunk_index + 1
+ )
+
+ def _format_merged_content(self, messages: List[Message]) -> str:
+ content = "".join((msg.content or "") for msg in messages)
+ if not messages or not self._contains_chunk_message(messages):
+ return content
+
+ first_chunk = self._chunk_meta_for(messages[0])
+ if first_chunk is not None and first_chunk.chunk_index > 0:
+ content = "..." + content.lstrip()
+ last_chunk = self._chunk_meta_for(messages[-1])
+ if last_chunk is not None and last_chunk.chunk_index < last_chunk.chunk_count - 1:
+ content = content.rstrip() + "..."
+ return content
+
+ def _contains_chunk_message(self, messages: List[Message]) -> bool:
+ return any(self._chunk_meta_for(msg) is not None for msg in messages)
+
+ def _chunk_meta_for(self, message: Message) -> Optional[ChunkMeta]:
+ return self.chunk_meta.get(id(message))
+
def _first_message_time(self) -> str | None:
"""获取第一条消息的时间(内部方法)"""
for msg_group in self.elements:
diff --git a/openviking/session/memory/session_extract_context_provider.py b/openviking/session/memory/session_extract_context_provider.py
index cce533c2e..3afed47fe 100644
--- a/openviking/session/memory/session_extract_context_provider.py
+++ b/openviking/session/memory/session_extract_context_provider.py
@@ -99,7 +99,9 @@ class SessionExtractContextProvider(ExtractContextProvider):
from openviking.session.memory.memory_updater import ExtractContext
if self._extract_context is None:
- self._extract_context = ExtractContext(self.messages or [])
+ self._extract_context = ExtractContext(
+ self.messages if isinstance(self.messages, list) else []
+ )
return self._extract_context
def _detect_language(self) -> str:
@@ -180,7 +182,8 @@ from neighboring messages.
else:
time_display = session_time_str
- conversation = self._assemble_conversation(self.messages)
+ extract_context = self.get_extract_context()
+ conversation = self._assemble_conversation(extract_context.messages)
return {
"role": "user",
diff --git a/tests/session/memory/test_compressor_v2.py b/tests/session/memory/test_compressor_v2.py
index d1dda6366..8dec9e1f6 100644
--- a/tests/session/memory/test_compressor_v2.py
+++ b/tests/session/memory/test_compressor_v2.py
@@ -328,6 +328,9 @@ class TestCompressorV2:
def get_memory_schemas(self, ctx):
return []
+ def get_extract_context(self):
+ return ExtractContext(messages)
+
return DummyProvider()
async def run(self):
@@ -552,6 +555,9 @@ class TestCompressorV2:
def get_memory_schemas(self, _ctx):
return [FixedSchema(), VariableSchema()]
+ def get_extract_context(self):
+ return ExtractContext(messages)
+
def _get_registry(self):
return object()
@@ -619,6 +625,9 @@ class TestCompressorV2:
def get_memory_schemas(self, _ctx):
return []
+ def get_extract_context(self):
+ return ExtractContext(messages)
+
def _get_registry(self):
return object()
diff --git a/tests/session/memory/test_memory_timestamp_parsing.py b/tests/session/memory/test_memory_timestamp_parsing.py
index 86e121253..2f05c26f6 100644
--- a/tests/session/memory/test_memory_timestamp_parsing.py
+++ b/tests/session/memory/test_memory_timestamp_parsing.py
@@ -92,6 +92,170 @@ def test_message_range_uses_peer_id_when_present():
assert "[default]: invoice follow-up" not in msg_range.pretty_print()
+def test_extraction_context_chunks_long_text_and_preserves_decimal_numbers(stub_provider_config):
+ text = (
+ "周一 "
+ + "需求沟通。" * 20
+ + "上下文用量out:21.9K、in:110.2k、缓存读取501.0k,先简单记录下来。 "
+ + "周二 面试三个新员工:王军优势是agent算法,田伟优势是分布式后端架构,"
+ + "金然优势是Agent架构全栈,缺点是缺乏CPU服务器底层调优经验 "
+ + "周三 query-builder的输出直接进行判断会触发错误,原因不明。"
+ )
+ extract_context = ExtractContext(
+ [
+ Message(
+ id="msg-long",
+ role="user",
+ parts=[TextPart(text=text)],
+ created_at="2026-06-01T08:56:37.382Z",
+ )
+ ]
+ )
+
+ extraction_messages = extract_context.messages
+ merged = "".join(message.content for message in extraction_messages)
+
+ assert len(extraction_messages) > 1
+ assert merged == text
+ assert all(message.id.startswith("msg-long#chunk_") for message in extraction_messages)
+ assert all(id(message) in extract_context.chunk_meta for message in extraction_messages)
+ assert "21.9K" in merged
+ assert "110.2k" in merged
+ assert "501.0k" in merged
+ assert not any(message.content.startswith(("9K", "2k")) for message in extraction_messages)
+
+
+def test_message_range_merges_adjacent_chunks_for_same_speaker():
+ first = Message(
+ id="msg-long#chunk_0",
+ role="user",
+ parts=[TextPart(text="first chunk. ")],
+ peer_id="web:visitor:alice",
+ )
+ second = Message(
+ id="msg-long#chunk_1",
+ role="user",
+ parts=[TextPart(text="second chunk.")],
+ peer_id="web:visitor:alice",
+ )
+ msg_range = MessageRange(
+ [[first, second]],
+ chunk_meta={
+ id(first): SimpleNamespace(
+ source_message_id="msg-long",
+ chunk_index=0,
+ chunk_count=2,
+ ),
+ id(second): SimpleNamespace(
+ source_message_id="msg-long",
+ chunk_index=1,
+ chunk_count=2,
+ ),
+ },
+ )
+
+ assert msg_range.pretty_print() == "[web:visitor:alice]: first chunk. second chunk."
+
+
+def test_message_range_keeps_regular_same_speaker_messages_separate():
+ msg_range = MessageRange(
+ [
+ [
+ Message(
+ id="msg-a",
+ role="user",
+ parts=[TextPart(text="first message")],
+ peer_id="web:visitor:alice",
+ ),
+ Message(
+ id="msg-b",
+ role="user",
+ parts=[TextPart(text="second message")],
+ peer_id="web:visitor:alice",
+ ),
+ ]
+ ]
+ )
+
+ assert msg_range.pretty_print() == (
+ "[web:visitor:alice]: first message\n[web:visitor:alice]: second message"
+ )
+
+
+def test_message_range_keeps_different_source_chunks_separate():
+ first = Message(
+ id="msg-a#chunk_0",
+ role="user",
+ parts=[TextPart(text="first source chunk")],
+ peer_id="web:visitor:alice",
+ )
+ second = Message(
+ id="msg-b#chunk_0",
+ role="user",
+ parts=[TextPart(text="second source chunk")],
+ peer_id="web:visitor:alice",
+ )
+ msg_range = MessageRange(
+ [[first, second]],
+ chunk_meta={
+ id(first): SimpleNamespace(
+ source_message_id="msg-a",
+ chunk_index=0,
+ chunk_count=2,
+ ),
+ id(second): SimpleNamespace(
+ source_message_id="msg-b",
+ chunk_index=0,
+ chunk_count=2,
+ ),
+ },
+ )
+
+ assert msg_range.pretty_print() == (
+ "[web:visitor:alice]: first source chunk...\n[web:visitor:alice]: second source chunk..."
+ )
+
+
+def test_message_range_does_not_treat_original_chunk_like_id_as_chunk():
+ msg_range = MessageRange(
+ [
+ [
+ Message(
+ id="msg-long#chunk_0",
+ role="user",
+ parts=[TextPart(text="original message id contains chunk marker")],
+ peer_id="web:visitor:alice",
+ ),
+ ]
+ ]
+ )
+
+ assert msg_range.pretty_print() == (
+ "[web:visitor:alice]: original message id contains chunk marker"
+ )
+
+
+def test_message_range_marks_middle_chunk_as_abbreviated():
+ message = Message(
+ id="msg-long#chunk_1",
+ role="user",
+ parts=[TextPart(text="middle chunk")],
+ peer_id="web:visitor:alice",
+ )
+ msg_range = MessageRange(
+ [[message]],
+ chunk_meta={
+ id(message): SimpleNamespace(
+ source_message_id="msg-long",
+ chunk_index=1,
+ chunk_count=3,
+ )
+ },
+ )
+
+ assert msg_range.pretty_print() == "[web:visitor:alice]: ...middle chunk..."
+
+
def test_peer_id_routes_peer_memory_for_all_role_selected_types(stub_provider_config):
messages = [
Message(