Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(embed): add text chunking for oversized embedding inputs (#616) (#642)
Li_Tang_Kecbf8befcea57
diff --git a/openviking/models/embedder/base.py b/openviking/models/embedder/base.pyindex 3d2144c28..583cf8194 100644--- a/openviking/models/embedder/base.py+++ b/openviking/models/embedder/base.py@@ -1,11 +1,16 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0+import logging+import mathimport random+import reimport timefrom abc import ABC, abstractmethodfrom dataclasses import dataclassfrom typing import Any, Callable, Dict, List, Optional, TypeVar+logger = logging.getLogger(__name__)+T = TypeVar("T")@@ -22,8 +27,6 @@ def truncate_and_normalize(embedding: List[float], dimension: Optional[int]) ->if not dimension or len(embedding) <= dimension:return embedding- import math-embedding = embedding[:dimension]norm = math.sqrt(sum(x**2 for x in embedding))if norm > 0:@@ -65,15 +68,181 @@ class EmbedderBase(ABC):Provides unified embedding interface supporting dense, sparse, and hybrid modes."""- def __init__(self, model_name: str, config: Optional[Dict[str, Any]] = None):+ def __init__(+ self,+ model_name: str,+ config: Optional[Dict[str, Any]] = None,+ max_tokens: Optional[int] = None,+ ):"""Initialize embedderArgs:model_name: Model nameconfig: Configuration dict containing api_key, api_base, etc.+ max_tokens: Maximum token count per embedding request, None to use default (8000)"""self.model_name = model_nameself.config = config or {}+ self._max_tokens = max_tokens++ @property+ def max_tokens(self) -> int:+ """Maximum token count per embedding request. Subclasses can override."""+ if self._max_tokens is not None:+ return self._max_tokens+ return 8000++ def _estimate_tokens(self, text: str) -> int:+ """Estimate token count for the given text.++ Tries tiktoken (for OpenAI models) first, falls back to len(text) // 3.++ Args:+ text: Input text++ Returns:+ Estimated token count+ """+ try:+ import tiktoken++ enc = tiktoken.encoding_for_model(self.model_name)+ return len(enc.encode(text))+ except Exception:+ logger.info(+ "tiktoken unavailable for model '%s', using character-based estimation",+ self.model_name,+ )+ return len(text) // 3++ def _chunk_text(self, text: str) -> List[str]:+ """Split text into chunks, each within max_tokens.++ Splitting priority: paragraphs (\\n\\n) > sentences (。.!?\\n) > fixed length.++ Args:+ text: Input text++ Returns:+ List of text chunks+ """+ max_tok = self.max_tokens+ if self._estimate_tokens(text) <= max_tok:+ return [text]++ # Try paragraph split first+ paragraphs = text.split("\n\n")+ if len(paragraphs) > 1:+ chunks = self._merge_segments(paragraphs, max_tok, "\n\n")+ if all(self._estimate_tokens(c) <= max_tok for c in chunks):+ return chunks++ # Try sentence split+ sentences = re.split(r"(?<=[。.!?\n])", text)+ sentences = [s for s in sentences if s]+ if len(sentences) > 1:+ chunks = self._merge_segments(sentences, max_tok, "")+ if all(self._estimate_tokens(c) <= max_tok for c in chunks):+ return chunks++ # Fixed-length split as last resort+ return self._fixed_length_split(text, max_tok)++ def _merge_segments(self, segments: List[str], max_tok: int, separator: str) -> List[str]:+ """Merge small segments into chunks that fit within max_tokens."""+ chunks: List[str] = []+ current = ""+ for seg in segments:+ candidate = (current + separator + seg) if current else seg+ if self._estimate_tokens(candidate) <= max_tok:+ current = candidate+ else:+ if current:+ chunks.append(current)+ # If a single segment exceeds max_tok, it will be handled later+ current = seg+ if current:+ chunks.append(current)+ return chunks++ def _fixed_length_split(self, text: str, max_tok: int) -> List[str]:+ """Split text into fixed-length chunks based on estimated char-per-token ratio."""+ # Estimate chars per token+ total_tokens = self._estimate_tokens(text)+ chars_per_token = len(text) / max(total_tokens, 1)+ chunk_size = max(int(max_tok * chars_per_token * 0.9), 100)++ chunks: List[str] = []+ start = 0+ while start < len(text):+ end = start + chunk_size+ if end < len(text):+ # Try to break at a whitespace boundary+ boundary = text.rfind(" ", start, end)+ if boundary > start:+ end = boundary + 1+ chunks.append(text[start:end])+ start = end+ return chunks++ def _chunk_and_embed(self, text: str) -> EmbedResult:+ """Chunk text if it exceeds max_tokens, embed each chunk, and merge results.++ For text within limits, delegates to _embed_single directly.+ For oversized text, splits into chunks, embeds each, then computes a+ token-weighted average of dense vectors with L2 normalization.++ Args:+ text: Input text++ Returns:+ EmbedResult with merged embedding+ """+ estimated = self._estimate_tokens(text)+ if estimated <= self.max_tokens:+ return self._embed_single(text)++ chunks = self._chunk_text(text)+ logger.debug(+ "Chunking text: original ~%d tokens -> %d chunks",+ estimated,+ len(chunks),+ )++ results: List[EmbedResult] = []+ weights: List[int] = []+ for chunk in chunks:+ result = self._embed_single(chunk)+ results.append(result)+ weights.append(self._estimate_tokens(chunk))++ # Merge dense vectors via weighted average + L2 normalization+ merged_dense = None+ if results and results[0].dense_vector is not None:+ dim = len(results[0].dense_vector)+ total_weight = sum(weights)+ merged = [0.0] * dim+ for result, w in zip(results, weights):+ if result.dense_vector:+ for i in range(dim):+ merged[i] += result.dense_vector[i] * w+ if total_weight > 0:+ merged = [v / total_weight for v in merged]+ # L2 normalization+ norm = math.sqrt(sum(v * v for v in merged))+ if norm > 0:+ merged = [v / norm for v in merged]+ merged_dense = merged++ return EmbedResult(dense_vector=merged_dense)++ def _embed_single(self, text: str) -> EmbedResult:+ """Embed a single text without chunking logic. Defaults to self.embed().++ Subclasses that override embed() with chunking should also override+ this method to provide the raw embedding call.+ """+ return self.embed(text)@abstractmethoddef embed(self, text: str) -> EmbedResult:diff --git a/openviking/models/embedder/openai_embedders.py b/openviking/models/embedder/openai_embedders.pyindex 032d766d0..b474215ed 100644--- a/openviking/models/embedder/openai_embedders.py+++ b/openviking/models/embedder/openai_embedders.py@@ -2,6 +2,7 @@# SPDX-License-Identifier: Apache-2.0"""OpenAI Embedder Implementation"""+import loggingfrom typing import Any, Dict, List, Optionalimport openai@@ -14,6 +15,8 @@ from openviking.models.embedder.base import ()from openviking.telemetry import get_current_telemetry+logger = logging.getLogger(__name__)+class OpenAIDenseEmbedder(DenseEmbedderBase):"""OpenAI Dense Embedder Implementation@@ -38,6 +41,7 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):api_base: Optional[str] = None,dimension: Optional[int] = None,config: Optional[Dict[str, Any]] = None,+ max_tokens: Optional[int] = None,):"""Initialize OpenAI Dense Embedder@@ -47,11 +51,12 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):api_base: API base URL, optionaldimension: Dimension (if model supports), optionalconfig: Additional configuration dict+ max_tokens: Maximum token count per embedding request, None to use default (8000)Raises:ValueError: If api_key is not provided and env vars are not set"""- super().__init__(model_name, config)+ super().__init__(model_name, config, max_tokens=max_tokens)self.api_key = api_keyself.api_base = api_base@@ -66,15 +71,43 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):client_kwargs["base_url"] = self.api_baseself.client = openai.OpenAI(**client_kwargs)+ # Initialize tiktoken encoder+ self._tiktoken_enc = None+ try:+ import tiktoken++ self._tiktoken_enc = tiktoken.encoding_for_model(model_name)+ except Exception:+ logger.info(+ "tiktoken unavailable for model '%s', will use character-based estimation",+ model_name,+ )+# Auto-detect dimensionself._dimension = dimensionif self._dimension is None:self._dimension = self._detect_dimension()+ @property+ def max_tokens(self) -> int:+ """OpenAI embedding models have 8192 token limit; use 8000 for safety buffer.++ Can be overridden via the max_tokens constructor parameter.+ """+ if self._max_tokens is not None:+ return self._max_tokens+ return 8000++ def _estimate_tokens(self, text: str) -> int:+ """Estimate tokens using tiktoken if available, fallback to len(text) // 3."""+ if self._tiktoken_enc is not None:+ return len(self._tiktoken_enc.encode(text))+ return len(text) // 3+def _detect_dimension(self) -> int:"""Detect dimension by making an actual API call"""try:- result = self.embed("test")+ result = self._embed_single("test")return len(result.dense_vector) if result.dense_vector else 1536except Exception:# Use default value, text-embedding-3-small defaults to 1536@@ -99,8 +132,8 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):output_tokens,)- def embed(self, text: str) -> EmbedResult:- """Perform dense embedding on text+ def _embed_single(self, text: str) -> EmbedResult:+ """Perform raw embedding without chunking logic.Args:text: Input text@@ -126,8 +159,30 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):except Exception as e:raise RuntimeError(f"Embedding failed: {str(e)}") from e+ def embed(self, text: str) -> EmbedResult:+ """Embed single text, with automatic chunking for oversized input.++ Args:+ text: Input text++ Returns:+ EmbedResult: Result containing only dense_vector++ Raises:+ RuntimeError: When API call fails+ """+ if not text:+ return self._embed_single(text)++ if self._estimate_tokens(text) > self.max_tokens:+ return self._chunk_and_embed(text)+ return self._embed_single(text)+def embed_batch(self, texts: List[str]) -> List[EmbedResult]:- """Batch embedding (OpenAI native support)+ """Batch embedding with automatic chunking for oversized inputs.++ Short texts are batched together via the OpenAI API for efficiency.+ Oversized texts are individually chunked and embedded.Args:texts: List of texts@@ -141,19 +196,33 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):if not texts:return []- try:- kwargs = {"input": texts, "model": self.model_name}- if self.dimension:- kwargs["dimensions"] = self.dimension-- response = self.client.embeddings.create(**kwargs)- self._update_telemetry_token_usage(response)-- return [EmbedResult(dense_vector=item.embedding) for item in response.data]- except openai.APIError as e:- raise RuntimeError(f"OpenAI API error: {e.message}") from e- except Exception as e:- raise RuntimeError(f"Batch embedding failed: {str(e)}") from e+ results: List[Optional[EmbedResult]] = [None] * len(texts)+ short_indices: List[int] = []+ short_texts: List[str] = []++ for i, text in enumerate(texts):+ if text and self._estimate_tokens(text) > self.max_tokens:+ results[i] = self._chunk_and_embed(text)+ else:+ short_indices.append(i)+ short_texts.append(text)++ if short_texts:+ try:+ kwargs = {"input": short_texts, "model": self.model_name}+ if self.dimension:+ kwargs["dimensions"] = self.dimension++ response = self.client.embeddings.create(**kwargs)+ self._update_telemetry_token_usage(response)+ for idx, item in zip(short_indices, response.data):+ results[idx] = EmbedResult(dense_vector=item.embedding)+ except openai.APIError as e:+ raise RuntimeError(f"OpenAI API error: {e.message}") from e+ except Exception as e:+ raise RuntimeError(f"Batch embedding failed: {str(e)}") from e++ return results # type: ignore[return-value]def get_dimension(self) -> int:"""Get embedding dimensiondiff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.pyindex 2d4f1164b..17e77266a 100644--- a/openviking_cli/utils/config/embedding_config.py+++ b/openviking_cli/utils/config/embedding_config.py@@ -27,6 +27,10 @@ class EmbeddingModelConfig(BaseModel):sk: Optional[str] = Field(default=None, description="Access Key Secretfor VikingDB API")region: Optional[str] = Field(default=None, description="Region for VikingDB API")host: Optional[str] = Field(default=None, description="Host for VikingDB API")+ max_tokens: Optional[int] = Field(+ default=None,+ description="Maximum token count per embedding request. If None, uses model default (e.g., 8000 for OpenAI).",+ )model_config = {"extra": "forbid"}@@ -153,6 +157,7 @@ class EmbeddingConfig(BaseModel):"api_key": cfg.api_key,"api_base": cfg.api_base,"dimension": cfg.dimension,+ "max_tokens": cfg.max_tokens,},),("volcengine", "dense"): (diff --git a/tests/unit/test_openai_embedder_chunking.py b/tests/unit/test_openai_embedder_chunking.pynew file mode 100644index 000000000..fd6d72f5d--- /dev/null+++ b/tests/unit/test_openai_embedder_chunking.py@@ -0,0 +1,449 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0+"""Tests for OpenAI Embedder text chunking functionality (Issue #616)++Uses importlib to load modules directly, avoiding the deep import chain+from openviking/__init__.py that requires the full build (C++ engine, litellm, etc.).+"""++import importlib.util+import math+import os+import sys+from unittest.mock import MagicMock, patch++import pytest++# -- Direct module loading to avoid openviking/__init__.py import chain ---+_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))+++def _load_module_from_file(module_name: str, file_path: str):+ """Load a Python module directly from file path."""+ spec = importlib.util.spec_from_file_location(module_name, file_path)+ mod = importlib.util.module_from_spec(spec)+ sys.modules[module_name] = mod+ spec.loader.exec_module(mod)+ return mod+++# Load base module first+_base_mod = _load_module_from_file(+ "openviking.models.embedder.base",+ os.path.join(_PROJECT_ROOT, "openviking", "models", "embedder", "base.py"),+)+EmbedResult = _base_mod.EmbedResult+EmbedderBase = _base_mod.EmbedderBase+DenseEmbedderBase = _base_mod.DenseEmbedderBase+++DIMENSION = 8+++def _make_mock_openai_client(dimension=DIMENSION):+ """Create a mock OpenAI client that returns deterministic embeddings."""+ mock_client = MagicMock()++ def fake_create(**kwargs):+ inp = kwargs["input"]+ if isinstance(inp, list):+ items = []+ for text in inp:+ vec = _deterministic_vector(text, dimension)+ item = MagicMock()+ item.embedding = vec+ items.append(item)+ resp = MagicMock()+ resp.data = items+ return resp+ else:+ vec = _deterministic_vector(inp, dimension)+ item = MagicMock()+ item.embedding = vec+ resp = MagicMock()+ resp.data = [item]+ return resp++ mock_client.embeddings.create.side_effect = fake_create+ return mock_client+++def _deterministic_vector(text, dim):+ """Generate a simple deterministic vector from text for testing."""+ seed = sum(ord(c) for c in text) % 1000+ vec = [(seed + i) * 0.001 for i in range(dim)]+ # Normalize+ norm = math.sqrt(sum(v * v for v in vec))+ if norm > 0:+ vec = [v / norm for v in vec]+ return vec+++def _load_openai_embedders():+ """Load the openai_embedders module with mocked dependencies."""+ return _load_module_from_file(+ "openviking.models.embedder.openai_embedders",+ os.path.join(_PROJECT_ROOT, "openviking", "models", "embedder", "openai_embedders.py"),+ )+++@patch("openai.OpenAI")+class TestOpenAIEmbedderChunking:+ """Test cases for OpenAI embedder text chunking (Issue #616)"""++ def _make_embedder(self, mock_openai_class, max_tokens=100):+ """Helper to create a test embedder with a mocked client."""+ mock_client = _make_mock_openai_client()+ mock_openai_class.return_value = mock_client++ mod = _load_openai_embedders()+ OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder++ embedder = OpenAIDenseEmbedder(+ model_name="text-embedding-3-small",+ api_key="test-api-key",+ dimension=DIMENSION,+ )++ # Override max_tokens for testing (lower limit to trigger chunking easily)+ type(embedder).max_tokens = property(lambda self: max_tokens)++ return embedder, mock_client++ def test_short_text_no_chunking(self, mock_openai_class):+ """Short text (< max_tokens) should embed normally without chunking."""+ embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=8000)+ result = embedder.embed("Hello world")++ assert result.dense_vector is not None+ assert len(result.dense_vector) == DIMENSION+ assert mock_client.embeddings.create.call_count >= 1++ def test_oversized_text_triggers_chunking(self, mock_openai_class):+ """Text exceeding max_tokens should trigger chunking and return correct dimension."""+ embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=10)++ # Create text that will exceed 10 tokens (using fallback: len//3 > 10 means len > 30)+ long_text = "This is a sentence. " * 20 # ~400 chars -> ~133 estimated tokens++ result = embedder.embed(long_text)++ assert result.dense_vector is not None+ assert len(result.dense_vector) == DIMENSION+ # Multiple API calls should have been made (one per chunk + dimension detection)+ assert mock_client.embeddings.create.call_count > 2++ def test_very_long_text_chunking(self, mock_openai_class):+ """Text 10x over the limit should still chunk and return normally."""+ embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=10)++ # Very long text: ~3000 chars -> ~1000 estimated tokens (100x over limit of 10)+ very_long_text = "A" * 3000++ result = embedder.embed(very_long_text)++ assert result.dense_vector is not None+ assert len(result.dense_vector) == DIMENSION++ def test_empty_text_no_error(self, mock_openai_class):+ """Empty text should not raise an error."""+ embedder, _ = self._make_embedder(mock_openai_class, max_tokens=8000)+ result = embedder.embed("")++ assert result.dense_vector is not None+ assert len(result.dense_vector) == DIMENSION++ def test_embed_batch_mixed_lengths(self, mock_openai_class):+ """embed_batch with mixed short and long texts should handle each correctly."""+ embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=10)++ short_text = "Hi"+ long_text = "This is a very long sentence. " * 20 # exceeds 10 tokens++ results = embedder.embed_batch([short_text, long_text])++ assert len(results) == 2+ for result in results:+ assert result.dense_vector is not None+ assert len(result.dense_vector) == DIMENSION++ def test_embed_batch_empty_list(self, mock_openai_class):+ """embed_batch with empty list should return empty list."""+ embedder, _ = self._make_embedder(mock_openai_class, max_tokens=8000)+ results = embedder.embed_batch([])+ assert results == []++ def test_merged_vector_is_normalized(self, mock_openai_class):+ """Merged vector from chunking should be L2 normalized."""+ embedder, _ = self._make_embedder(mock_openai_class, max_tokens=10)++ long_text = "Word " * 100 # definitely exceeds 10 tokens++ result = embedder.embed(long_text)+ assert result.dense_vector is not None++ # Check L2 norm is approximately 1.0+ norm = math.sqrt(sum(v * v for v in result.dense_vector))+ assert abs(norm - 1.0) < 1e-6++ def test_paragraph_splitting(self, mock_openai_class):+ """Text with paragraph breaks should be split by paragraphs first."""+ embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=10)++ # Create text with clear paragraph separators+ paragraphs = ["Paragraph one content."] * 5+ text_with_paragraphs = "\n\n".join(paragraphs)++ result = embedder.embed(text_with_paragraphs)+ assert result.dense_vector is not None+ assert len(result.dense_vector) == DIMENSION+++@patch("openai.OpenAI")+class TestTiktokenFallback:+ """Test tiktoken fallback behavior."""++ def test_fallback_when_tiktoken_unavailable(self, mock_openai_class):+ """When tiktoken is not available, should fall back to character-based estimation."""+ mock_client = _make_mock_openai_client()+ mock_openai_class.return_value = mock_client++ mod = _load_openai_embedders()+ OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder++ embedder = OpenAIDenseEmbedder(+ model_name="text-embedding-3-small",+ api_key="test-api-key",+ dimension=DIMENSION,+ )++ # Force tiktoken encoder to None to simulate unavailability+ embedder._tiktoken_enc = None++ # Character-based estimation: len("hello world") // 3 = 3+ estimated = embedder._estimate_tokens("hello world")+ assert estimated == len("hello world") // 3++ def test_estimate_tokens_with_tiktoken_encoder(self, mock_openai_class):+ """When tiktoken encoder is available, should use it for estimation."""+ mock_client = _make_mock_openai_client()+ mock_openai_class.return_value = mock_client++ mod = _load_openai_embedders()+ OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder++ embedder = OpenAIDenseEmbedder(+ model_name="text-embedding-3-small",+ api_key="test-api-key",+ dimension=DIMENSION,+ )++ # If tiktoken is available, _tiktoken_enc should not be None+ # and estimation should use it+ if embedder._tiktoken_enc is not None:+ tokens = embedder._estimate_tokens("hello world")+ assert isinstance(tokens, int)+ assert tokens > 0+++class TestBaseChunkText:+ """Test _chunk_text and helper methods on a concrete subclass."""++ def _make_simple_embedder(self, max_tokens=10):+ """Create a minimal concrete embedder for testing chunking logic."""++ class SimpleEmbedder(DenseEmbedderBase):+ @property+ def max_tokens(self_inner):+ return max_tokens++ def _estimate_tokens(self_inner, text):+ # Simple: 1 token per word+ return len(text.split())++ def embed(self_inner, text):+ dim = 4+ vec = [1.0 / dim] * dim+ return EmbedResult(dense_vector=vec)++ def _embed_single(self_inner, text):+ return self_inner.embed(text)++ def get_dimension(self_inner):+ return 4++ return SimpleEmbedder(model_name="test")++ def test_chunk_text_short(self):+ """Text within limit should return single chunk."""+ embedder = self._make_simple_embedder(max_tokens=100)+ chunks = embedder._chunk_text("hello world")+ assert chunks == ["hello world"]++ def test_chunk_text_paragraph_split(self):+ """Text with paragraphs should split by paragraph boundaries."""+ embedder = self._make_simple_embedder(max_tokens=5)+ text = "one two three\n\nfour five six"+ chunks = embedder._chunk_text(text)+ assert len(chunks) >= 2+ for chunk in chunks:+ assert embedder._estimate_tokens(chunk) <= 5++ def test_chunk_text_sentence_split(self):+ """Text with sentences but no paragraphs should split by sentences."""+ embedder = self._make_simple_embedder(max_tokens=5)+ text = "One two three four. Five six seven eight."+ chunks = embedder._chunk_text(text)+ assert len(chunks) >= 2++ def test_chunk_text_fixed_length_fallback(self):+ """Text without good natural boundaries should fall back to fixed-length split."""+ embedder = self._make_simple_embedder(max_tokens=3)+ # Use a long text with no sentence/paragraph boundaries, >100 chars to exceed min chunk_size+ text = "word " * 200 # 200 words = 200 tokens, 1000 chars+ chunks = embedder._chunk_text(text)+ assert len(chunks) >= 2++ def test_chunk_and_embed_returns_normalized(self):+ """_chunk_and_embed should return a normalized vector."""+ embedder = self._make_simple_embedder(max_tokens=3)+ text = "one two three four five six seven eight"+ result = embedder._chunk_and_embed(text)++ assert result.dense_vector is not None+ norm = math.sqrt(sum(v * v for v in result.dense_vector))+ assert abs(norm - 1.0) < 1e-6++ def test_chunk_and_embed_short_text_no_split(self):+ """_chunk_and_embed on short text should just call _embed_single."""+ embedder = self._make_simple_embedder(max_tokens=100)+ result = embedder._chunk_and_embed("hello")++ assert result.dense_vector is not None+ assert len(result.dense_vector) == 4+++@patch("openai.OpenAI")+class TestMaxTokensConfigurable:+ """Test cases for configurable max_tokens parameter."""++ def test_custom_max_tokens_via_constructor(self, mock_openai_class):+ """Custom max_tokens passed to OpenAIDenseEmbedder should be used."""+ mock_client = _make_mock_openai_client()+ mock_openai_class.return_value = mock_client++ mod = _load_openai_embedders()+ OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder++ embedder = OpenAIDenseEmbedder(+ model_name="text-embedding-3-small",+ api_key="test-api-key",+ dimension=DIMENSION,+ max_tokens=32000,+ )++ assert embedder.max_tokens == 32000++ def test_default_max_tokens_when_not_provided(self, mock_openai_class):+ """When max_tokens is not provided, should default to 8000."""+ mock_client = _make_mock_openai_client()+ mock_openai_class.return_value = mock_client++ mod = _load_openai_embedders()+ OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder++ embedder = OpenAIDenseEmbedder(+ model_name="text-embedding-3-small",+ api_key="test-api-key",+ dimension=DIMENSION,+ )++ assert embedder.max_tokens == 8000++ def test_custom_max_tokens_affects_chunking(self, mock_openai_class):+ """A higher max_tokens should allow longer texts without chunking."""+ mock_client = _make_mock_openai_client()+ mock_openai_class.return_value = mock_client++ mod = _load_openai_embedders()+ OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder++ # Create embedder with high max_tokens+ embedder = OpenAIDenseEmbedder(+ model_name="text-embedding-3-small",+ api_key="test-api-key",+ dimension=DIMENSION,+ max_tokens=100000,+ )+ # Force fallback estimation (len//3)+ embedder._tiktoken_enc = None++ # This text would exceed 8000 tokens with fallback estimation (30000 chars -> 10000 tokens)+ # but should NOT trigger chunking with max_tokens=100000+ long_text = "word " * 6000 # 30000 chars -> 10000 estimated tokens++ mock_client.embeddings.create.reset_mock()+ result = embedder.embed(long_text)++ assert result.dense_vector is not None+ # Should be a single API call (plus dimension detection), NOT chunked+ assert mock_client.embeddings.create.call_count == 1++ def test_low_custom_max_tokens_triggers_chunking(self, mock_openai_class):+ """A lower max_tokens should trigger chunking for shorter texts."""+ mock_client = _make_mock_openai_client()+ mock_openai_class.return_value = mock_client++ mod = _load_openai_embedders()+ OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder++ embedder = OpenAIDenseEmbedder(+ model_name="text-embedding-3-small",+ api_key="test-api-key",+ dimension=DIMENSION,+ max_tokens=5,+ )+ # Force fallback estimation+ embedder._tiktoken_enc = None++ # Need text long enough to produce multiple chunks.+ # Fallback estimation: len(text)//3. With max_tokens=5, need >5 tokens.+ # Fixed-length split has min chunk_size=100, so text must be >100 chars to split.+ text = "Hello world test. " * 30 # 540 chars -> 180 estimated tokens, well over max_tokens=5++ mock_client.embeddings.create.reset_mock()+ result = embedder.embed(text)++ assert result.dense_vector is not None+ # Should have chunked (multiple API calls)+ assert mock_client.embeddings.create.call_count > 1+++class TestBaseMaxTokensConfigurable:+ """Test configurable max_tokens on EmbedderBase directly."""++ def test_base_default_max_tokens(self):+ """EmbedderBase without max_tokens should return 8000."""++ class ConcreteEmbedder(DenseEmbedderBase):+ def embed(self, text):+ return EmbedResult(dense_vector=[0.0])++ def get_dimension(self):+ return 1++ embedder = ConcreteEmbedder(model_name="test")+ assert embedder.max_tokens == 8000++ def test_base_custom_max_tokens(self):+ """EmbedderBase with max_tokens should return the custom value."""++ class ConcreteEmbedder(DenseEmbedderBase):+ def embed(self, text):+ return EmbedResult(dense_vector=[0.0])++ def get_dimension(self):+ return 1++ embedder = ConcreteEmbedder(model_name="test", max_tokens=32768)+ assert embedder.max_tokens == 32768