Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix: apply embedding input limits in embedder (#2266)

Qin Haojiec72c8a87e8d0
diff --git a/openviking/models/embedder/base.py b/openviking/models/embedder/base.py
index 618ae3ece..30f42d12b 100644
--- a/openviking/models/embedder/base.py
+++ b/openviking/models/embedder/base.py
@@ -11,6 +11,10 @@ from threading import Lock
from typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVar
from openviking.telemetry import get_current_telemetry
+from openviking.utils.embedding_input import (
+ resolve_embedding_max_input_tokens,
+ truncate_embedding_input,
+)
from openviking.utils.model_retry import retry_async, retry_sync
from openviking_cli.utils import get_logger
@@ -45,30 +49,28 @@ def _get_token_tracker():
return _token_tracker_instance
-async def embed_compat(embedder: Any, text: str, *, is_query: bool = False) -> "EmbedResult":
- """Call async embedding when available, otherwise fall back to sync embed()."""
+async def embed_compat(
+ embedder: "EmbedderBase", text: str, *, is_query: bool = False
+) -> "EmbedResult":
+ """Prepare input, then call the embedder's async-compatible entrypoint."""
from openviking.telemetry import bind_telemetry_stage
stage = "embed_query" if is_query else "embed_resource"
- embed_async = getattr(embedder, "embed_async", None)
+ embedding_input = embedder.prepare_embedding_input(text)
with bind_telemetry_stage(stage):
- if callable(embed_async):
- return await embed_async(text, is_query=is_query)
- return embedder.embed(text, is_query=is_query)
+ return await embedder.embed_async(embedding_input, is_query=is_query)
async def embed_batch_compat(
- embedder: Any, texts: List[str], *, is_query: bool = False
+ embedder: "EmbedderBase", texts: List[str], *, is_query: bool = False
) -> List["EmbedResult"]:
- """Call async batch embedding when available, otherwise fall back to sync embed_batch()."""
+ """Prepare inputs, then call the embedder's async-compatible batch entrypoint."""
from openviking.telemetry import bind_telemetry_stage
stage = "embed_query" if is_query else "embed_resource"
- embed_batch_async = getattr(embedder, "embed_batch_async", None)
+ embedding_inputs = embedder.prepare_embedding_inputs(texts)
with bind_telemetry_stage(stage):
- if callable(embed_batch_async):
- return await embed_batch_async(texts, is_query=is_query)
- return embedder.embed_batch(texts, is_query=is_query)
+ return await embedder.embed_batch_async(embedding_inputs, is_query=is_query)
def truncate_and_normalize(embedding: List[float], dimension: Optional[int]) -> List[float]:
@@ -136,6 +138,7 @@ class EmbedderBase(ABC):
"""
self.model_name = model_name
self.config = config or {}
+ self.max_input_tokens = resolve_embedding_max_input_tokens(self.config)
self.max_retries = int(self.config.get("max_retries", 3))
self.max_concurrent = int(self.config.get("max_concurrent", 10))
self.provider = self.config.get("provider", "unknown")
@@ -144,6 +147,18 @@ class EmbedderBase(ABC):
self._token_tracker = _get_token_tracker()
self._active_call_started_at: float | None = None
+ def prepare_embedding_input(self, text: str) -> str:
+ """Apply this embedder's input guard before provider calls."""
+ if self.max_input_tokens is None:
+ return text
+ return truncate_embedding_input(text, self.max_input_tokens)
+
+ def prepare_embedding_inputs(self, texts: List[str]) -> List[str]:
+ """Apply this embedder's input guard to a batch."""
+ if self.max_input_tokens is None:
+ return texts
+ return [self.prepare_embedding_input(text) for text in texts]
+
@abstractmethod
def embed(self, text: str, is_query: bool = False) -> EmbedResult:
"""Embed single text
@@ -498,14 +513,19 @@ class CompositeHybridEmbedder(HybridEmbedderBase):
def __init__(self, dense_embedder: DenseEmbedderBase, sparse_embedder: SparseEmbedderBase):
"""Initialize with two separate embedders"""
- super().__init__(model_name=f"{dense_embedder.model_name}+{sparse_embedder.model_name}")
+ super().__init__(
+ model_name=f"{dense_embedder.model_name}+{sparse_embedder.model_name}",
+ config={},
+ )
self.dense_embedder = dense_embedder
self.sparse_embedder = sparse_embedder
def embed(self, text: str, is_query: bool = False) -> EmbedResult:
"""Combine results from both embedders"""
- dense_res = self.dense_embedder.embed(text, is_query=is_query)
- sparse_res = self.sparse_embedder.embed(text, is_query=is_query)
+ dense_input = self.dense_embedder.prepare_embedding_input(text)
+ sparse_input = self.sparse_embedder.prepare_embedding_input(text)
+ dense_res = self.dense_embedder.embed(dense_input, is_query=is_query)
+ sparse_res = self.sparse_embedder.embed(sparse_input, is_query=is_query)
return EmbedResult(
dense_vector=dense_res.dense_vector, sparse_vector=sparse_res.sparse_vector
@@ -513,8 +533,10 @@ class CompositeHybridEmbedder(HybridEmbedderBase):
def embed_batch(self, texts: List[str], is_query: bool = False) -> List[EmbedResult]:
"""Combine batch results"""
- dense_results = self.dense_embedder.embed_batch(texts, is_query=is_query)
- sparse_results = self.sparse_embedder.embed_batch(texts, is_query=is_query)
+ dense_inputs = self.dense_embedder.prepare_embedding_inputs(texts)
+ sparse_inputs = self.sparse_embedder.prepare_embedding_inputs(texts)
+ dense_results = self.dense_embedder.embed_batch(dense_inputs, is_query=is_query)
+ sparse_results = self.sparse_embedder.embed_batch(sparse_inputs, is_query=is_query)
return [
EmbedResult(dense_vector=d.dense_vector, sparse_vector=s.sparse_vector)
@@ -522,9 +544,11 @@ class CompositeHybridEmbedder(HybridEmbedderBase):
]
async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:
+ dense_input = self.dense_embedder.prepare_embedding_input(text)
+ sparse_input = self.sparse_embedder.prepare_embedding_input(text)
dense_res, sparse_res = await asyncio.gather(
- self.dense_embedder.embed_async(text, is_query=is_query),
- self.sparse_embedder.embed_async(text, is_query=is_query),
+ self.dense_embedder.embed_async(dense_input, is_query=is_query),
+ self.sparse_embedder.embed_async(sparse_input, is_query=is_query),
)
return EmbedResult(
dense_vector=dense_res.dense_vector, sparse_vector=sparse_res.sparse_vector
@@ -533,9 +557,11 @@ class CompositeHybridEmbedder(HybridEmbedderBase):
async def embed_batch_async(
self, texts: List[str], is_query: bool = False
) -> List[EmbedResult]:
+ dense_inputs = self.dense_embedder.prepare_embedding_inputs(texts)
+ sparse_inputs = self.sparse_embedder.prepare_embedding_inputs(texts)
dense_results, sparse_results = await asyncio.gather(
- self.dense_embedder.embed_batch_async(texts, is_query=is_query),
- self.sparse_embedder.embed_batch_async(texts, is_query=is_query),
+ self.dense_embedder.embed_batch_async(dense_inputs, is_query=is_query),
+ self.sparse_embedder.embed_batch_async(sparse_inputs, is_query=is_query),
)
return [
EmbedResult(dense_vector=d.dense_vector, sparse_vector=s.sparse_vector)
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index 15c260362..00e1767f3 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -33,7 +33,6 @@ from openviking.utils.circuit_breaker import (
CircuitBreakerOpen,
classify_api_error,
)
-from openviking.utils.embedding_input import truncate_embedding_input
from openviking.utils.model_retry import ERROR_CLASS_INPUT_TOO_LARGE, ERROR_CLASS_PERMANENT
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils import get_logger
@@ -323,7 +322,6 @@ class TextEmbeddingHandler(DequeueHandlerBase):
config = get_openviking_config()
self._collection_name = config.storage.vectordb.name
self._vector_dim = config.embedding.dimension
- self._max_input_tokens = int(getattr(config.embedding, "max_input_tokens", 4096) or 4096)
self._initialize_embedder(config)
breaker_cfg = config.embedding.circuit_breaker
self._circuit_breaker = CircuitBreaker(
@@ -500,12 +498,8 @@ class TextEmbeddingHandler(DequeueHandlerBase):
import time as _time
_embed_t0 = _time.monotonic()
- embedding_input = truncate_embedding_input(
- embedding_msg.message,
- self._max_input_tokens,
- )
result: EmbedResult = await embed_compat(
- self._embedder, embedding_input, is_query=False
+ self._embedder, embedding_msg.message, is_query=False
)
_embed_elapsed = _time.monotonic() - _embed_t0
try:
diff --git a/openviking/utils/embedding_input.py b/openviking/utils/embedding_input.py
index 1eb35bd8b..07e5bddcd 100644
--- a/openviking/utils/embedding_input.py
+++ b/openviking/utils/embedding_input.py
@@ -46,3 +46,22 @@ def truncate_embedding_input(
else:
high = mid - 1
return text[:low].rstrip() + suffix
+
+
+def resolve_embedding_max_input_tokens(
+ config: dict[str, object] | None,
+ default: int | None = None,
+) -> int | None:
+ """Read and normalize max_input_tokens from an embedder config dict."""
+ raw_value = (config or {}).get("max_input_tokens", default)
+ if raw_value is None:
+ return default
+
+ try:
+ max_tokens = int(raw_value)
+ except (TypeError, ValueError):
+ return default
+
+ if max_tokens <= 0:
+ return default
+ return max_tokens
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index af28cdfda..44c808cbb 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -16,7 +16,6 @@ from openviking.server.identity import RequestContext
from openviking.storage.queuefs import get_queue_manager
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
from openviking.storage.viking_fs import get_viking_fs
-from openviking.utils.embedding_input import truncate_embedding_input
from openviking.utils.time_utils import parse_iso_datetime
from openviking_cli.utils import VikingURI, get_logger
from openviking_cli.utils.config import get_openviking_config
@@ -354,7 +353,6 @@ async def vectorize_file(
embedding_cfg = get_openviking_config().embedding
configured_text_source = getattr(embedding_cfg, "text_source", "content_only")
effective_text_source = "summary_only" if use_summary else configured_text_source
- max_input_tokens = int(getattr(embedding_cfg, "max_input_tokens", 4096) or 4096)
if content_type is None:
# Unsupported file type: fall back to summary if available
@@ -372,12 +370,11 @@ async def vectorize_file(
if summary and effective_text_source in {"summary_first", "summary_only"}:
context.set_vectorize(Vectorize(text=summary))
else:
- # Read raw file content and apply configured truncation guard.
+ # Read raw file content; embedders apply their own input guard.
try:
content = await viking_fs.read_file(file_path, ctx=ctx)
if isinstance(content, bytes):
content = content.decode("utf-8", errors="replace")
- content = truncate_embedding_input(content, max_input_tokens)
context.set_vectorize(Vectorize(text=content))
except Exception as e:
logger.warning(
diff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.py
index 3774aeca1..afb531927 100644
--- a/openviking_cli/utils/config/embedding_config.py
+++ b/openviking_cli/utils/config/embedding_config.py
@@ -462,6 +462,7 @@ class EmbeddingConfig(BaseModel):
runtime_config = {
"max_retries": self.max_retries,
"max_concurrent": self.max_concurrent,
+ "max_input_tokens": self.max_input_tokens,
}
factory_registry = {
diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.py
index f87f89817..8dbe0d48b 100644
--- a/tests/storage/test_collection_schemas.py
+++ b/tests/storage/test_collection_schemas.py
@@ -10,7 +10,7 @@ from types import SimpleNamespace
import pytest
-from openviking.models.embedder.base import EmbedResult
+from openviking.models.embedder.base import DenseEmbedderBase, EmbedResult
from openviking.storage.collection_schemas import (
CollectionSchemas,
TextEmbeddingHandler,
@@ -404,8 +404,9 @@ async def test_embedding_handler_truncates_queue_input_before_embed(monkeypatch)
async def upsert(self, _data, *, ctx):
return "rec-1"
- class _CapturingEmbedder:
+ class _CapturingEmbedder(DenseEmbedderBase):
def __init__(self):
+ super().__init__("capturing-test", config={"max_input_tokens": 10})
self.text = None
def embed(self, text: str, is_query: bool = False) -> EmbedResult:
@@ -413,6 +414,9 @@ async def test_embedding_handler_truncates_queue_input_before_embed(monkeypatch)
self.text = text
return EmbedResult(dense_vector=[0.1, 0.2])
+ def get_dimension(self) -> int:
+ return 2
+
embedder = _CapturingEmbedder()
monkeypatch.setattr(
"openviking_cli.utils.config.get_openviking_config",
diff --git a/tests/test_telemetry_runtime.py b/tests/test_telemetry_runtime.py
index 793bae181..52c5aa507 100644
--- a/tests/test_telemetry_runtime.py
+++ b/tests/test_telemetry_runtime.py
@@ -9,7 +9,7 @@ from types import SimpleNamespace
import pytest
-from openviking.models.embedder.base import EmbedResult, embed_compat
+from openviking.models.embedder.base import DenseEmbedderBase, EmbedResult, embed_compat
from openviking.models.vlm.base import VLMBase
from openviking.observability.context import (
bind_operation_observability_context,
@@ -150,16 +150,27 @@ async def test_bind_telemetry_stage_propagates_across_async_tasks():
@pytest.mark.asyncio
async def test_embed_compat_binds_query_stage_for_embedding_tokens():
telemetry = MemoryOperationTelemetry(operation="search.find", enabled=True)
+ query = " ".join(f"token-{idx}" for idx in range(200))
+
+ class _TelemetryAwareAsyncEmbedder(DenseEmbedderBase):
+ def __init__(self):
+ super().__init__("telemetry-test", config={"max_input_tokens": 20})
+
+ def embed(self, text: str, is_query: bool = False) -> EmbedResult:
+ raise AssertionError("embed_async should be used")
- class _TelemetryAwareAsyncEmbedder:
async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:
- assert text == "hello"
assert is_query is True
+ assert text.endswith("...(truncated for embedding)")
+ assert "token-199" not in text
get_current_telemetry().record_token_usage("embedding", 9, 0)
return EmbedResult(dense_vector=[0.1, 0.2])
+ def get_dimension(self) -> int:
+ return 2
+
with bind_telemetry(telemetry):
- await embed_compat(_TelemetryAwareAsyncEmbedder(), "hello", is_query=True)
+ await embed_compat(_TelemetryAwareAsyncEmbedder(), query, is_query=True)
summary = telemetry.finish().summary
assert summary["tokens"]["stages"]["embed_query"]["embedding"] == {"total": 9}
diff --git a/tests/unit/test_embedding_vectorize_strategy.py b/tests/unit/test_embedding_vectorize_strategy.py
index bfd6a8c1c..1f3743a1e 100644
--- a/tests/unit/test_embedding_vectorize_strategy.py
+++ b/tests/unit/test_embedding_vectorize_strategy.py
@@ -42,6 +42,13 @@ def test_embedding_max_input_tokens_validation_accepts_reasonable_value():
assert cfg.max_input_tokens == 1000
+def test_embedding_runtime_config_includes_max_input_tokens():
+ cfg = _cfg(max_input_tokens=1000)
+ embedder = cfg.get_embedder()
+
+ assert embedder.config["max_input_tokens"] == 1000
+
+
def test_embedding_max_input_tokens_validation_rejects_too_small_value():
with pytest.raises(ValueError):
_cfg(max_input_tokens=10)
diff --git a/tests/unit/test_model_retry.py b/tests/unit/test_model_retry.py
index 996d1e8fe..5e0f7a844 100644
--- a/tests/unit/test_model_retry.py
+++ b/tests/unit/test_model_retry.py
@@ -132,6 +132,11 @@ def test_quota_exceeded_case_insensitive():
[
"BadRequestError: 400 maximum context length is 8192 tokens",
"Error code: 413 - Payload Too Large",
+ (
+ "Error code: 500 - {'error': {'code': 500, 'message': "
+ "'input (8525 tokens) is too large to process. increase the physical batch size "
+ "(current batch size: 2048)', 'type': 'server_error'}}"
+ ),
],
)
def test_classify_input_too_large_errors(message):
diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.py
index 1635e6164..89b5af47f 100644
--- a/tests/unit/test_vectorize_file_strategy.py
+++ b/tests/unit/test_vectorize_file_strategy.py
@@ -91,7 +91,7 @@ async def test_vectorize_file_uses_summary_first(monkeypatch):
@pytest.mark.asyncio
-async def test_vectorize_file_truncates_content_when_content_only(monkeypatch):
+async def test_vectorize_file_preserves_content_until_embedder_input_guard(monkeypatch):
queue = DummyQueue()
content = " ".join(f"token-{i}" for i in range(200))
monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue))
@@ -118,8 +118,7 @@ async def test_vectorize_file_truncates_content_when_content_only(monkeypatch):
assert len(queue.items) == 1
text = queue.items[0].get_vectorization_text()
- assert text.endswith("...(truncated for embedding)")
- assert "token-199" not in text
+ assert text == content
@pytest.mark.asyncio