Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(session): add chunked vectorization for long memories (#734)
Deepak Dev Panwar1efb9fb015a3
diff --git a/openviking/session/compressor.py b/openviking/session/compressor.pyindex 7fd066c7c..f6c294f97 100644--- a/openviking/session/compressor.py+++ b/openviking/session/compressor.py@@ -148,13 +148,43 @@ class SessionCompressor:) -> bool:"""Add memory to vectorization queue and record semantic change.+ For long memories, splits content into chunks and enqueues each chunk+ as a separate vector record for better retrieval precision.+Args:memory: The memory context to indexctx: Request contextchange_type: One of "added" or "modified""""from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter+ from openviking_cli.utils.config import get_openviking_config++ semantic = get_openviking_config().semantic+ vectorize_text = memory.get_vectorization_text()+ if vectorize_text and len(vectorize_text) > semantic.memory_chunk_chars:+ # Chunk long memory into multiple vector records+ chunks = self._chunk_text(+ vectorize_text,+ semantic.memory_chunk_chars,+ semantic.memory_chunk_overlap,+ )+ logger.info(+ f"Chunking memory {memory.uri} into {len(chunks)} chunks "+ f"({len(vectorize_text)} chars)"+ )+ import copy++ for i, chunk in enumerate(chunks):+ chunk_memory = copy.deepcopy(memory)+ chunk_memory.uri = f"{memory.uri}#chunk_{i:04d}"+ chunk_memory.parent_uri = memory.uri+ chunk_memory.set_vectorize(Vectorize(text=chunk))+ chunk_msg = EmbeddingMsgConverter.from_context(chunk_memory)+ if chunk_msg:+ await self.vikingdb.enqueue_embedding_msg(chunk_msg)++ # Always enqueue the base record (uses abstract as vector text)embedding_msg = EmbeddingMsgConverter.from_context(memory)await self.vikingdb.enqueue_embedding_msg(embedding_msg)logger.info(f"Enqueued memory for vectorization: {memory.uri}")@@ -162,6 +192,30 @@ class SessionCompressor:self._record_semantic_change(memory.uri, change_type, parent_uri=memory.parent_uri)return True+ @staticmethod+ def _chunk_text(text: str, chunk_size: int, overlap: int) -> list:+ """Split text into overlapping chunks, preferring paragraph boundaries."""+ if len(text) <= chunk_size:+ return [text]++ chunks = []+ start = 0+ while start < len(text):+ end = start + chunk_size++ # Try to break at paragraph boundary+ if end < len(text):+ boundary = text.rfind("\n\n", start, end)+ if boundary > start + chunk_size // 2:+ end = boundary + 2 # Include the double newline++ chunks.append(text[start:end].strip())+ start = end - overlap+ if start >= len(text):+ break++ return [c for c in chunks if c]+async def _merge_into_existing(self,candidate: CandidateMemory,@@ -379,9 +433,7 @@ class SessionCompressor:merged_text = (f"{action.memory.abstract} {candidate.content}")- merged_embed = self.deduplicator.embedder.embed(- merged_text- )+ merged_embed = self.deduplicator.embedder.embed(merged_text)batch_memories.append((merged_embed.dense_vector, action.memory))diff --git a/openviking_cli/utils/config/parser_config.py b/openviking_cli/utils/config/parser_config.pyindex c8ff46aab..5cf3860f2 100644--- a/openviking_cli/utils/config/parser_config.py+++ b/openviking_cli/utils/config/parser_config.py@@ -505,6 +505,13 @@ class SemanticConfig:overview_max_chars: int = 4000"""Maximum characters for generated overviews."""+ memory_chunk_chars: int = 2000+ """Maximum characters per chunk when splitting long memories for vectorization.+ Memories shorter than this are vectorized as a single record."""++ memory_chunk_overlap: int = 200+ """Character overlap between adjacent memory chunks for context continuity."""+# Configuration registry for dynamic loadingPARSER_CONFIG_REGISTRY = {diff --git a/tests/misc/test_semantic_config.py b/tests/misc/test_semantic_config.pyindex 0a974dc87..e68b768d0 100644--- a/tests/misc/test_semantic_config.py+++ b/tests/misc/test_semantic_config.py@@ -1,8 +1,9 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0-"""Tests for SemanticConfig and overview budget estimation."""+"""Tests for SemanticConfig, overview budget estimation, and memory chunking."""+from openviking.session.compressor import SessionCompressorfrom openviking_cli.utils.config.parser_config import SemanticConfig@@ -14,6 +15,8 @@ def test_semantic_config_defaults():assert config.overview_batch_size == 50assert config.abstract_max_chars == 256assert config.overview_max_chars == 4000+ assert config.memory_chunk_chars == 2000+ assert config.memory_chunk_overlap == 200def test_semantic_config_custom_values():@@ -80,3 +83,54 @@ def test_batch_splitting():assert len(batches[0]) == 50assert len(batches[1]) == 50assert len(batches[2]) == 20+++# --- Memory chunking tests ---+++def test_chunk_text_short_text_no_split():+ """Short text below chunk_size returns single chunk."""+ text = "Short memory content."+ chunks = SessionCompressor._chunk_text(text, chunk_size=2000, overlap=200)+ assert len(chunks) == 1+ assert chunks[0] == text+++def test_chunk_text_long_text_splits():+ """Long text is split into multiple chunks."""+ text = "A" * 5000+ chunks = SessionCompressor._chunk_text(text, chunk_size=2000, overlap=200)+ assert len(chunks) >= 3+ # Each chunk should be at most chunk_size+ for chunk in chunks:+ assert len(chunk) <= 2000+++def test_chunk_text_overlap():+ """Chunks should overlap by the specified amount."""+ # Create text with clear markers every 500 chars+ text = "".join(f"[BLOCK{i:03d}]" + "x" * 490 for i in range(10))+ chunks = SessionCompressor._chunk_text(text, chunk_size=2000, overlap=200)+ assert len(chunks) >= 2+ # The end of chunk N should overlap with the start of chunk N+1+ for i in range(len(chunks) - 1):+ tail = chunks[i][-200:]+ assert tail in chunks[i + 1] or chunks[i + 1].startswith(tail[:50])+++def test_chunk_text_prefers_paragraph_boundaries():+ """Chunking should prefer splitting at paragraph boundaries."""+ paragraphs = ["Paragraph about topic " + str(i) + ". " * 50 for i in range(10)]+ text = "\n\n".join(paragraphs)+ chunks = SessionCompressor._chunk_text(text, chunk_size=500, overlap=50)+ # Chunks should tend to start at paragraph beginnings+ assert len(chunks) >= 2+ for chunk in chunks:+ assert len(chunk) > 0+++def test_memory_chunk_config_custom():+ """Custom memory chunk config values work."""+ config = SemanticConfig(memory_chunk_chars=500, memory_chunk_overlap=50)+ assert config.memory_chunk_chars == 500+ assert config.memory_chunk_overlap == 50