Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix: apply embedding input limits in embedder (#2266)
Qin Haojiec72c8a87e8d0
diff --git a/openviking/models/embedder/base.py b/openviking/models/embedder/base.pyindex 618ae3ece..30f42d12b 100644--- a/openviking/models/embedder/base.py+++ b/openviking/models/embedder/base.py@@ -11,6 +11,10 @@ from threading import Lockfrom typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVarfrom openviking.telemetry import get_current_telemetry+from openviking.utils.embedding_input import (+ resolve_embedding_max_input_tokens,+ truncate_embedding_input,+)from openviking.utils.model_retry import retry_async, retry_syncfrom openviking_cli.utils import get_logger@@ -45,30 +49,28 @@ def _get_token_tracker():return _token_tracker_instance-async def embed_compat(embedder: Any, text: str, *, is_query: bool = False) -> "EmbedResult":- """Call async embedding when available, otherwise fall back to sync embed()."""+async def embed_compat(+ embedder: "EmbedderBase", text: str, *, is_query: bool = False+) -> "EmbedResult":+ """Prepare input, then call the embedder's async-compatible entrypoint."""from openviking.telemetry import bind_telemetry_stagestage = "embed_query" if is_query else "embed_resource"- embed_async = getattr(embedder, "embed_async", None)+ embedding_input = embedder.prepare_embedding_input(text)with bind_telemetry_stage(stage):- if callable(embed_async):- return await embed_async(text, is_query=is_query)- return embedder.embed(text, is_query=is_query)+ return await embedder.embed_async(embedding_input, is_query=is_query)async def embed_batch_compat(- embedder: Any, texts: List[str], *, is_query: bool = False+ embedder: "EmbedderBase", texts: List[str], *, is_query: bool = False) -> List["EmbedResult"]:- """Call async batch embedding when available, otherwise fall back to sync embed_batch()."""+ """Prepare inputs, then call the embedder's async-compatible batch entrypoint."""from openviking.telemetry import bind_telemetry_stagestage = "embed_query" if is_query else "embed_resource"- embed_batch_async = getattr(embedder, "embed_batch_async", None)+ embedding_inputs = embedder.prepare_embedding_inputs(texts)with bind_telemetry_stage(stage):- if callable(embed_batch_async):- return await embed_batch_async(texts, is_query=is_query)- return embedder.embed_batch(texts, is_query=is_query)+ return await embedder.embed_batch_async(embedding_inputs, is_query=is_query)def truncate_and_normalize(embedding: List[float], dimension: Optional[int]) -> List[float]:@@ -136,6 +138,7 @@ class EmbedderBase(ABC):"""self.model_name = model_nameself.config = config or {}+ self.max_input_tokens = resolve_embedding_max_input_tokens(self.config)self.max_retries = int(self.config.get("max_retries", 3))self.max_concurrent = int(self.config.get("max_concurrent", 10))self.provider = self.config.get("provider", "unknown")@@ -144,6 +147,18 @@ class EmbedderBase(ABC):self._token_tracker = _get_token_tracker()self._active_call_started_at: float | None = None+ def prepare_embedding_input(self, text: str) -> str:+ """Apply this embedder's input guard before provider calls."""+ if self.max_input_tokens is None:+ return text+ return truncate_embedding_input(text, self.max_input_tokens)++ def prepare_embedding_inputs(self, texts: List[str]) -> List[str]:+ """Apply this embedder's input guard to a batch."""+ if self.max_input_tokens is None:+ return texts+ return [self.prepare_embedding_input(text) for text in texts]+@abstractmethoddef embed(self, text: str, is_query: bool = False) -> EmbedResult:"""Embed single text@@ -498,14 +513,19 @@ class CompositeHybridEmbedder(HybridEmbedderBase):def __init__(self, dense_embedder: DenseEmbedderBase, sparse_embedder: SparseEmbedderBase):"""Initialize with two separate embedders"""- super().__init__(model_name=f"{dense_embedder.model_name}+{sparse_embedder.model_name}")+ super().__init__(+ model_name=f"{dense_embedder.model_name}+{sparse_embedder.model_name}",+ config={},+ )self.dense_embedder = dense_embedderself.sparse_embedder = sparse_embedderdef embed(self, text: str, is_query: bool = False) -> EmbedResult:"""Combine results from both embedders"""- dense_res = self.dense_embedder.embed(text, is_query=is_query)- sparse_res = self.sparse_embedder.embed(text, is_query=is_query)+ dense_input = self.dense_embedder.prepare_embedding_input(text)+ sparse_input = self.sparse_embedder.prepare_embedding_input(text)+ dense_res = self.dense_embedder.embed(dense_input, is_query=is_query)+ sparse_res = self.sparse_embedder.embed(sparse_input, is_query=is_query)return EmbedResult(dense_vector=dense_res.dense_vector, sparse_vector=sparse_res.sparse_vector@@ -513,8 +533,10 @@ class CompositeHybridEmbedder(HybridEmbedderBase):def embed_batch(self, texts: List[str], is_query: bool = False) -> List[EmbedResult]:"""Combine batch results"""- dense_results = self.dense_embedder.embed_batch(texts, is_query=is_query)- sparse_results = self.sparse_embedder.embed_batch(texts, is_query=is_query)+ dense_inputs = self.dense_embedder.prepare_embedding_inputs(texts)+ sparse_inputs = self.sparse_embedder.prepare_embedding_inputs(texts)+ dense_results = self.dense_embedder.embed_batch(dense_inputs, is_query=is_query)+ sparse_results = self.sparse_embedder.embed_batch(sparse_inputs, is_query=is_query)return [EmbedResult(dense_vector=d.dense_vector, sparse_vector=s.sparse_vector)@@ -522,9 +544,11 @@ class CompositeHybridEmbedder(HybridEmbedderBase):]async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:+ dense_input = self.dense_embedder.prepare_embedding_input(text)+ sparse_input = self.sparse_embedder.prepare_embedding_input(text)dense_res, sparse_res = await asyncio.gather(- self.dense_embedder.embed_async(text, is_query=is_query),- self.sparse_embedder.embed_async(text, is_query=is_query),+ self.dense_embedder.embed_async(dense_input, is_query=is_query),+ self.sparse_embedder.embed_async(sparse_input, is_query=is_query),)return EmbedResult(dense_vector=dense_res.dense_vector, sparse_vector=sparse_res.sparse_vector@@ -533,9 +557,11 @@ class CompositeHybridEmbedder(HybridEmbedderBase):async def embed_batch_async(self, texts: List[str], is_query: bool = False) -> List[EmbedResult]:+ dense_inputs = self.dense_embedder.prepare_embedding_inputs(texts)+ sparse_inputs = self.sparse_embedder.prepare_embedding_inputs(texts)dense_results, sparse_results = await asyncio.gather(- self.dense_embedder.embed_batch_async(texts, is_query=is_query),- self.sparse_embedder.embed_batch_async(texts, is_query=is_query),+ self.dense_embedder.embed_batch_async(dense_inputs, is_query=is_query),+ self.sparse_embedder.embed_batch_async(sparse_inputs, is_query=is_query),)return [EmbedResult(dense_vector=d.dense_vector, sparse_vector=s.sparse_vector)diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex 15c260362..00e1767f3 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -33,7 +33,6 @@ from openviking.utils.circuit_breaker import (CircuitBreakerOpen,classify_api_error,)-from openviking.utils.embedding_input import truncate_embedding_inputfrom openviking.utils.model_retry import ERROR_CLASS_INPUT_TOO_LARGE, ERROR_CLASS_PERMANENTfrom openviking_cli.session.user_id import UserIdentifierfrom openviking_cli.utils import get_logger@@ -323,7 +322,6 @@ class TextEmbeddingHandler(DequeueHandlerBase):config = get_openviking_config()self._collection_name = config.storage.vectordb.nameself._vector_dim = config.embedding.dimension- self._max_input_tokens = int(getattr(config.embedding, "max_input_tokens", 4096) or 4096)self._initialize_embedder(config)breaker_cfg = config.embedding.circuit_breakerself._circuit_breaker = CircuitBreaker(@@ -500,12 +498,8 @@ class TextEmbeddingHandler(DequeueHandlerBase):import time as _time_embed_t0 = _time.monotonic()- embedding_input = truncate_embedding_input(- embedding_msg.message,- self._max_input_tokens,- )result: EmbedResult = await embed_compat(- self._embedder, embedding_input, is_query=False+ self._embedder, embedding_msg.message, is_query=False)_embed_elapsed = _time.monotonic() - _embed_t0try:diff --git a/openviking/utils/embedding_input.py b/openviking/utils/embedding_input.pyindex 1eb35bd8b..07e5bddcd 100644--- a/openviking/utils/embedding_input.py+++ b/openviking/utils/embedding_input.py@@ -46,3 +46,22 @@ def truncate_embedding_input(else:high = mid - 1return text[:low].rstrip() + suffix+++def resolve_embedding_max_input_tokens(+ config: dict[str, object] | None,+ default: int | None = None,+) -> int | None:+ """Read and normalize max_input_tokens from an embedder config dict."""+ raw_value = (config or {}).get("max_input_tokens", default)+ if raw_value is None:+ return default++ try:+ max_tokens = int(raw_value)+ except (TypeError, ValueError):+ return default++ if max_tokens <= 0:+ return default+ return max_tokensdiff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.pyindex af28cdfda..44c808cbb 100644--- a/openviking/utils/embedding_utils.py+++ b/openviking/utils/embedding_utils.py@@ -16,7 +16,6 @@ from openviking.server.identity import RequestContextfrom openviking.storage.queuefs import get_queue_managerfrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverterfrom openviking.storage.viking_fs import get_viking_fs-from openviking.utils.embedding_input import truncate_embedding_inputfrom openviking.utils.time_utils import parse_iso_datetimefrom openviking_cli.utils import VikingURI, get_loggerfrom openviking_cli.utils.config import get_openviking_config@@ -354,7 +353,6 @@ async def vectorize_file(embedding_cfg = get_openviking_config().embeddingconfigured_text_source = getattr(embedding_cfg, "text_source", "content_only")effective_text_source = "summary_only" if use_summary else configured_text_source- max_input_tokens = int(getattr(embedding_cfg, "max_input_tokens", 4096) or 4096)if content_type is None:# Unsupported file type: fall back to summary if available@@ -372,12 +370,11 @@ async def vectorize_file(if summary and effective_text_source in {"summary_first", "summary_only"}:context.set_vectorize(Vectorize(text=summary))else:- # Read raw file content and apply configured truncation guard.+ # Read raw file content; embedders apply their own input guard.try:content = await viking_fs.read_file(file_path, ctx=ctx)if isinstance(content, bytes):content = content.decode("utf-8", errors="replace")- content = truncate_embedding_input(content, max_input_tokens)context.set_vectorize(Vectorize(text=content))except Exception as e:logger.warning(diff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.pyindex 3774aeca1..afb531927 100644--- a/openviking_cli/utils/config/embedding_config.py+++ b/openviking_cli/utils/config/embedding_config.py@@ -462,6 +462,7 @@ class EmbeddingConfig(BaseModel):runtime_config = {"max_retries": self.max_retries,"max_concurrent": self.max_concurrent,+ "max_input_tokens": self.max_input_tokens,}factory_registry = {diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.pyindex f87f89817..8dbe0d48b 100644--- a/tests/storage/test_collection_schemas.py+++ b/tests/storage/test_collection_schemas.py@@ -10,7 +10,7 @@ from types import SimpleNamespaceimport pytest-from openviking.models.embedder.base import EmbedResult+from openviking.models.embedder.base import DenseEmbedderBase, EmbedResultfrom openviking.storage.collection_schemas import (CollectionSchemas,TextEmbeddingHandler,@@ -404,8 +404,9 @@ async def test_embedding_handler_truncates_queue_input_before_embed(monkeypatch)async def upsert(self, _data, *, ctx):return "rec-1"- class _CapturingEmbedder:+ class _CapturingEmbedder(DenseEmbedderBase):def __init__(self):+ super().__init__("capturing-test", config={"max_input_tokens": 10})self.text = Nonedef embed(self, text: str, is_query: bool = False) -> EmbedResult:@@ -413,6 +414,9 @@ async def test_embedding_handler_truncates_queue_input_before_embed(monkeypatch)self.text = textreturn EmbedResult(dense_vector=[0.1, 0.2])+ def get_dimension(self) -> int:+ return 2+embedder = _CapturingEmbedder()monkeypatch.setattr("openviking_cli.utils.config.get_openviking_config",diff --git a/tests/test_telemetry_runtime.py b/tests/test_telemetry_runtime.pyindex 793bae181..52c5aa507 100644--- a/tests/test_telemetry_runtime.py+++ b/tests/test_telemetry_runtime.py@@ -9,7 +9,7 @@ from types import SimpleNamespaceimport pytest-from openviking.models.embedder.base import EmbedResult, embed_compat+from openviking.models.embedder.base import DenseEmbedderBase, EmbedResult, embed_compatfrom openviking.models.vlm.base import VLMBasefrom openviking.observability.context import (bind_operation_observability_context,@@ -150,16 +150,27 @@ async def test_bind_telemetry_stage_propagates_across_async_tasks():@pytest.mark.asyncioasync def test_embed_compat_binds_query_stage_for_embedding_tokens():telemetry = MemoryOperationTelemetry(operation="search.find", enabled=True)+ query = " ".join(f"token-{idx}" for idx in range(200))++ class _TelemetryAwareAsyncEmbedder(DenseEmbedderBase):+ def __init__(self):+ super().__init__("telemetry-test", config={"max_input_tokens": 20})++ def embed(self, text: str, is_query: bool = False) -> EmbedResult:+ raise AssertionError("embed_async should be used")- class _TelemetryAwareAsyncEmbedder:async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:- assert text == "hello"assert is_query is True+ assert text.endswith("...(truncated for embedding)")+ assert "token-199" not in textget_current_telemetry().record_token_usage("embedding", 9, 0)return EmbedResult(dense_vector=[0.1, 0.2])+ def get_dimension(self) -> int:+ return 2+with bind_telemetry(telemetry):- await embed_compat(_TelemetryAwareAsyncEmbedder(), "hello", is_query=True)+ await embed_compat(_TelemetryAwareAsyncEmbedder(), query, is_query=True)summary = telemetry.finish().summaryassert summary["tokens"]["stages"]["embed_query"]["embedding"] == {"total": 9}diff --git a/tests/unit/test_embedding_vectorize_strategy.py b/tests/unit/test_embedding_vectorize_strategy.pyindex bfd6a8c1c..1f3743a1e 100644--- a/tests/unit/test_embedding_vectorize_strategy.py+++ b/tests/unit/test_embedding_vectorize_strategy.py@@ -42,6 +42,13 @@ def test_embedding_max_input_tokens_validation_accepts_reasonable_value():assert cfg.max_input_tokens == 1000+def test_embedding_runtime_config_includes_max_input_tokens():+ cfg = _cfg(max_input_tokens=1000)+ embedder = cfg.get_embedder()++ assert embedder.config["max_input_tokens"] == 1000++def test_embedding_max_input_tokens_validation_rejects_too_small_value():with pytest.raises(ValueError):_cfg(max_input_tokens=10)diff --git a/tests/unit/test_model_retry.py b/tests/unit/test_model_retry.pyindex 996d1e8fe..5e0f7a844 100644--- a/tests/unit/test_model_retry.py+++ b/tests/unit/test_model_retry.py@@ -132,6 +132,11 @@ def test_quota_exceeded_case_insensitive():["BadRequestError: 400 maximum context length is 8192 tokens","Error code: 413 - Payload Too Large",+ (+ "Error code: 500 - {'error': {'code': 500, 'message': "+ "'input (8525 tokens) is too large to process. increase the physical batch size "+ "(current batch size: 2048)', 'type': 'server_error'}}"+ ),],)def test_classify_input_too_large_errors(message):diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.pyindex 1635e6164..89b5af47f 100644--- a/tests/unit/test_vectorize_file_strategy.py+++ b/tests/unit/test_vectorize_file_strategy.py@@ -91,7 +91,7 @@ async def test_vectorize_file_uses_summary_first(monkeypatch):@pytest.mark.asyncio-async def test_vectorize_file_truncates_content_when_content_only(monkeypatch):+async def test_vectorize_file_preserves_content_until_embedder_input_guard(monkeypatch):queue = DummyQueue()content = " ".join(f"token-{i}" for i in range(200))monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))@@ -118,8 +118,7 @@ async def test_vectorize_file_truncates_content_when_content_only(monkeypatch):assert len(queue.items) == 1text = queue.items[0].get_vectorization_text()- assert text.endswith("...(truncated for embedding)")- assert "token-199" not in text+ assert text == content@pytest.mark.asyncio