Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(embed): add text chunking for oversized embedding inputs (#616) (#642)

Li_Tang_Kecbf8befcea57
diff --git a/openviking/models/embedder/base.py b/openviking/models/embedder/base.py
index 3d2144c28..583cf8194 100644
--- a/openviking/models/embedder/base.py
+++ b/openviking/models/embedder/base.py
@@ -1,11 +1,16 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
+import logging
+import math
import random
+import re
import time
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, TypeVar
+logger = logging.getLogger(__name__)
+
T = TypeVar("T")
@@ -22,8 +27,6 @@ def truncate_and_normalize(embedding: List[float], dimension: Optional[int]) ->
if not dimension or len(embedding) <= dimension:
return embedding
- import math
-
embedding = embedding[:dimension]
norm = math.sqrt(sum(x**2 for x in embedding))
if norm > 0:
@@ -65,15 +68,181 @@ class EmbedderBase(ABC):
Provides unified embedding interface supporting dense, sparse, and hybrid modes.
"""
- def __init__(self, model_name: str, config: Optional[Dict[str, Any]] = None):
+ def __init__(
+ self,
+ model_name: str,
+ config: Optional[Dict[str, Any]] = None,
+ max_tokens: Optional[int] = None,
+ ):
"""Initialize embedder
Args:
model_name: Model name
config: Configuration dict containing api_key, api_base, etc.
+ max_tokens: Maximum token count per embedding request, None to use default (8000)
"""
self.model_name = model_name
self.config = config or {}
+ self._max_tokens = max_tokens
+
+ @property
+ def max_tokens(self) -> int:
+ """Maximum token count per embedding request. Subclasses can override."""
+ if self._max_tokens is not None:
+ return self._max_tokens
+ return 8000
+
+ def _estimate_tokens(self, text: str) -> int:
+ """Estimate token count for the given text.
+
+ Tries tiktoken (for OpenAI models) first, falls back to len(text) // 3.
+
+ Args:
+ text: Input text
+
+ Returns:
+ Estimated token count
+ """
+ try:
+ import tiktoken
+
+ enc = tiktoken.encoding_for_model(self.model_name)
+ return len(enc.encode(text))
+ except Exception:
+ logger.info(
+ "tiktoken unavailable for model '%s', using character-based estimation",
+ self.model_name,
+ )
+ return len(text) // 3
+
+ def _chunk_text(self, text: str) -> List[str]:
+ """Split text into chunks, each within max_tokens.
+
+ Splitting priority: paragraphs (\\n\\n) > sentences (。.!?\\n) > fixed length.
+
+ Args:
+ text: Input text
+
+ Returns:
+ List of text chunks
+ """
+ max_tok = self.max_tokens
+ if self._estimate_tokens(text) <= max_tok:
+ return [text]
+
+ # Try paragraph split first
+ paragraphs = text.split("\n\n")
+ if len(paragraphs) > 1:
+ chunks = self._merge_segments(paragraphs, max_tok, "\n\n")
+ if all(self._estimate_tokens(c) <= max_tok for c in chunks):
+ return chunks
+
+ # Try sentence split
+ sentences = re.split(r"(?<=[。.!?\n])", text)
+ sentences = [s for s in sentences if s]
+ if len(sentences) > 1:
+ chunks = self._merge_segments(sentences, max_tok, "")
+ if all(self._estimate_tokens(c) <= max_tok for c in chunks):
+ return chunks
+
+ # Fixed-length split as last resort
+ return self._fixed_length_split(text, max_tok)
+
+ def _merge_segments(self, segments: List[str], max_tok: int, separator: str) -> List[str]:
+ """Merge small segments into chunks that fit within max_tokens."""
+ chunks: List[str] = []
+ current = ""
+ for seg in segments:
+ candidate = (current + separator + seg) if current else seg
+ if self._estimate_tokens(candidate) <= max_tok:
+ current = candidate
+ else:
+ if current:
+ chunks.append(current)
+ # If a single segment exceeds max_tok, it will be handled later
+ current = seg
+ if current:
+ chunks.append(current)
+ return chunks
+
+ def _fixed_length_split(self, text: str, max_tok: int) -> List[str]:
+ """Split text into fixed-length chunks based on estimated char-per-token ratio."""
+ # Estimate chars per token
+ total_tokens = self._estimate_tokens(text)
+ chars_per_token = len(text) / max(total_tokens, 1)
+ chunk_size = max(int(max_tok * chars_per_token * 0.9), 100)
+
+ chunks: List[str] = []
+ start = 0
+ while start < len(text):
+ end = start + chunk_size
+ if end < len(text):
+ # Try to break at a whitespace boundary
+ boundary = text.rfind(" ", start, end)
+ if boundary > start:
+ end = boundary + 1
+ chunks.append(text[start:end])
+ start = end
+ return chunks
+
+ def _chunk_and_embed(self, text: str) -> EmbedResult:
+ """Chunk text if it exceeds max_tokens, embed each chunk, and merge results.
+
+ For text within limits, delegates to _embed_single directly.
+ For oversized text, splits into chunks, embeds each, then computes a
+ token-weighted average of dense vectors with L2 normalization.
+
+ Args:
+ text: Input text
+
+ Returns:
+ EmbedResult with merged embedding
+ """
+ estimated = self._estimate_tokens(text)
+ if estimated <= self.max_tokens:
+ return self._embed_single(text)
+
+ chunks = self._chunk_text(text)
+ logger.debug(
+ "Chunking text: original ~%d tokens -> %d chunks",
+ estimated,
+ len(chunks),
+ )
+
+ results: List[EmbedResult] = []
+ weights: List[int] = []
+ for chunk in chunks:
+ result = self._embed_single(chunk)
+ results.append(result)
+ weights.append(self._estimate_tokens(chunk))
+
+ # Merge dense vectors via weighted average + L2 normalization
+ merged_dense = None
+ if results and results[0].dense_vector is not None:
+ dim = len(results[0].dense_vector)
+ total_weight = sum(weights)
+ merged = [0.0] * dim
+ for result, w in zip(results, weights):
+ if result.dense_vector:
+ for i in range(dim):
+ merged[i] += result.dense_vector[i] * w
+ if total_weight > 0:
+ merged = [v / total_weight for v in merged]
+ # L2 normalization
+ norm = math.sqrt(sum(v * v for v in merged))
+ if norm > 0:
+ merged = [v / norm for v in merged]
+ merged_dense = merged
+
+ return EmbedResult(dense_vector=merged_dense)
+
+ def _embed_single(self, text: str) -> EmbedResult:
+ """Embed a single text without chunking logic. Defaults to self.embed().
+
+ Subclasses that override embed() with chunking should also override
+ this method to provide the raw embedding call.
+ """
+ return self.embed(text)
@abstractmethod
def embed(self, text: str) -> EmbedResult:
diff --git a/openviking/models/embedder/openai_embedders.py b/openviking/models/embedder/openai_embedders.py
index 032d766d0..b474215ed 100644
--- a/openviking/models/embedder/openai_embedders.py
+++ b/openviking/models/embedder/openai_embedders.py
@@ -2,6 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
"""OpenAI Embedder Implementation"""
+import logging
from typing import Any, Dict, List, Optional
import openai
@@ -14,6 +15,8 @@ from openviking.models.embedder.base import (
)
from openviking.telemetry import get_current_telemetry
+logger = logging.getLogger(__name__)
+
class OpenAIDenseEmbedder(DenseEmbedderBase):
"""OpenAI Dense Embedder Implementation
@@ -38,6 +41,7 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
api_base: Optional[str] = None,
dimension: Optional[int] = None,
config: Optional[Dict[str, Any]] = None,
+ max_tokens: Optional[int] = None,
):
"""Initialize OpenAI Dense Embedder
@@ -47,11 +51,12 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
api_base: API base URL, optional
dimension: Dimension (if model supports), optional
config: Additional configuration dict
+ max_tokens: Maximum token count per embedding request, None to use default (8000)
Raises:
ValueError: If api_key is not provided and env vars are not set
"""
- super().__init__(model_name, config)
+ super().__init__(model_name, config, max_tokens=max_tokens)
self.api_key = api_key
self.api_base = api_base
@@ -66,15 +71,43 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
client_kwargs["base_url"] = self.api_base
self.client = openai.OpenAI(**client_kwargs)
+ # Initialize tiktoken encoder
+ self._tiktoken_enc = None
+ try:
+ import tiktoken
+
+ self._tiktoken_enc = tiktoken.encoding_for_model(model_name)
+ except Exception:
+ logger.info(
+ "tiktoken unavailable for model '%s', will use character-based estimation",
+ model_name,
+ )
+
# Auto-detect dimension
self._dimension = dimension
if self._dimension is None:
self._dimension = self._detect_dimension()
+ @property
+ def max_tokens(self) -> int:
+ """OpenAI embedding models have 8192 token limit; use 8000 for safety buffer.
+
+ Can be overridden via the max_tokens constructor parameter.
+ """
+ if self._max_tokens is not None:
+ return self._max_tokens
+ return 8000
+
+ def _estimate_tokens(self, text: str) -> int:
+ """Estimate tokens using tiktoken if available, fallback to len(text) // 3."""
+ if self._tiktoken_enc is not None:
+ return len(self._tiktoken_enc.encode(text))
+ return len(text) // 3
+
def _detect_dimension(self) -> int:
"""Detect dimension by making an actual API call"""
try:
- result = self.embed("test")
+ result = self._embed_single("test")
return len(result.dense_vector) if result.dense_vector else 1536
except Exception:
# Use default value, text-embedding-3-small defaults to 1536
@@ -99,8 +132,8 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
output_tokens,
)
- def embed(self, text: str) -> EmbedResult:
- """Perform dense embedding on text
+ def _embed_single(self, text: str) -> EmbedResult:
+ """Perform raw embedding without chunking logic.
Args:
text: Input text
@@ -126,8 +159,30 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
except Exception as e:
raise RuntimeError(f"Embedding failed: {str(e)}") from e
+ def embed(self, text: str) -> EmbedResult:
+ """Embed single text, with automatic chunking for oversized input.
+
+ Args:
+ text: Input text
+
+ Returns:
+ EmbedResult: Result containing only dense_vector
+
+ Raises:
+ RuntimeError: When API call fails
+ """
+ if not text:
+ return self._embed_single(text)
+
+ if self._estimate_tokens(text) > self.max_tokens:
+ return self._chunk_and_embed(text)
+ return self._embed_single(text)
+
def embed_batch(self, texts: List[str]) -> List[EmbedResult]:
- """Batch embedding (OpenAI native support)
+ """Batch embedding with automatic chunking for oversized inputs.
+
+ Short texts are batched together via the OpenAI API for efficiency.
+ Oversized texts are individually chunked and embedded.
Args:
texts: List of texts
@@ -141,19 +196,33 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
if not texts:
return []
- try:
- kwargs = {"input": texts, "model": self.model_name}
- if self.dimension:
- kwargs["dimensions"] = self.dimension
-
- response = self.client.embeddings.create(**kwargs)
- self._update_telemetry_token_usage(response)
-
- return [EmbedResult(dense_vector=item.embedding) for item in response.data]
- except openai.APIError as e:
- raise RuntimeError(f"OpenAI API error: {e.message}") from e
- except Exception as e:
- raise RuntimeError(f"Batch embedding failed: {str(e)}") from e
+ results: List[Optional[EmbedResult]] = [None] * len(texts)
+ short_indices: List[int] = []
+ short_texts: List[str] = []
+
+ for i, text in enumerate(texts):
+ if text and self._estimate_tokens(text) > self.max_tokens:
+ results[i] = self._chunk_and_embed(text)
+ else:
+ short_indices.append(i)
+ short_texts.append(text)
+
+ if short_texts:
+ try:
+ kwargs = {"input": short_texts, "model": self.model_name}
+ if self.dimension:
+ kwargs["dimensions"] = self.dimension
+
+ response = self.client.embeddings.create(**kwargs)
+ self._update_telemetry_token_usage(response)
+ for idx, item in zip(short_indices, response.data):
+ results[idx] = EmbedResult(dense_vector=item.embedding)
+ except openai.APIError as e:
+ raise RuntimeError(f"OpenAI API error: {e.message}") from e
+ except Exception as e:
+ raise RuntimeError(f"Batch embedding failed: {str(e)}") from e
+
+ return results # type: ignore[return-value]
def get_dimension(self) -> int:
"""Get embedding dimension
diff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.py
index 2d4f1164b..17e77266a 100644
--- a/openviking_cli/utils/config/embedding_config.py
+++ b/openviking_cli/utils/config/embedding_config.py
@@ -27,6 +27,10 @@ class EmbeddingModelConfig(BaseModel):
sk: Optional[str] = Field(default=None, description="Access Key Secretfor VikingDB API")
region: Optional[str] = Field(default=None, description="Region for VikingDB API")
host: Optional[str] = Field(default=None, description="Host for VikingDB API")
+ max_tokens: Optional[int] = Field(
+ default=None,
+ description="Maximum token count per embedding request. If None, uses model default (e.g., 8000 for OpenAI).",
+ )
model_config = {"extra": "forbid"}
@@ -153,6 +157,7 @@ class EmbeddingConfig(BaseModel):
"api_key": cfg.api_key,
"api_base": cfg.api_base,
"dimension": cfg.dimension,
+ "max_tokens": cfg.max_tokens,
},
),
("volcengine", "dense"): (
diff --git a/tests/unit/test_openai_embedder_chunking.py b/tests/unit/test_openai_embedder_chunking.py
new file mode 100644
index 000000000..fd6d72f5d
--- /dev/null
+++ b/tests/unit/test_openai_embedder_chunking.py
@@ -0,0 +1,449 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""Tests for OpenAI Embedder text chunking functionality (Issue #616)
+
+Uses importlib to load modules directly, avoiding the deep import chain
+from openviking/__init__.py that requires the full build (C++ engine, litellm, etc.).
+"""
+
+import importlib.util
+import math
+import os
+import sys
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+# -- Direct module loading to avoid openviking/__init__.py import chain ---
+_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+def _load_module_from_file(module_name: str, file_path: str):
+ """Load a Python module directly from file path."""
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
+ mod = importlib.util.module_from_spec(spec)
+ sys.modules[module_name] = mod
+ spec.loader.exec_module(mod)
+ return mod
+
+
+# Load base module first
+_base_mod = _load_module_from_file(
+ "openviking.models.embedder.base",
+ os.path.join(_PROJECT_ROOT, "openviking", "models", "embedder", "base.py"),
+)
+EmbedResult = _base_mod.EmbedResult
+EmbedderBase = _base_mod.EmbedderBase
+DenseEmbedderBase = _base_mod.DenseEmbedderBase
+
+
+DIMENSION = 8
+
+
+def _make_mock_openai_client(dimension=DIMENSION):
+ """Create a mock OpenAI client that returns deterministic embeddings."""
+ mock_client = MagicMock()
+
+ def fake_create(**kwargs):
+ inp = kwargs["input"]
+ if isinstance(inp, list):
+ items = []
+ for text in inp:
+ vec = _deterministic_vector(text, dimension)
+ item = MagicMock()
+ item.embedding = vec
+ items.append(item)
+ resp = MagicMock()
+ resp.data = items
+ return resp
+ else:
+ vec = _deterministic_vector(inp, dimension)
+ item = MagicMock()
+ item.embedding = vec
+ resp = MagicMock()
+ resp.data = [item]
+ return resp
+
+ mock_client.embeddings.create.side_effect = fake_create
+ return mock_client
+
+
+def _deterministic_vector(text, dim):
+ """Generate a simple deterministic vector from text for testing."""
+ seed = sum(ord(c) for c in text) % 1000
+ vec = [(seed + i) * 0.001 for i in range(dim)]
+ # Normalize
+ norm = math.sqrt(sum(v * v for v in vec))
+ if norm > 0:
+ vec = [v / norm for v in vec]
+ return vec
+
+
+def _load_openai_embedders():
+ """Load the openai_embedders module with mocked dependencies."""
+ return _load_module_from_file(
+ "openviking.models.embedder.openai_embedders",
+ os.path.join(_PROJECT_ROOT, "openviking", "models", "embedder", "openai_embedders.py"),
+ )
+
+
+@patch("openai.OpenAI")
+class TestOpenAIEmbedderChunking:
+ """Test cases for OpenAI embedder text chunking (Issue #616)"""
+
+ def _make_embedder(self, mock_openai_class, max_tokens=100):
+ """Helper to create a test embedder with a mocked client."""
+ mock_client = _make_mock_openai_client()
+ mock_openai_class.return_value = mock_client
+
+ mod = _load_openai_embedders()
+ OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder
+
+ embedder = OpenAIDenseEmbedder(
+ model_name="text-embedding-3-small",
+ api_key="test-api-key",
+ dimension=DIMENSION,
+ )
+
+ # Override max_tokens for testing (lower limit to trigger chunking easily)
+ type(embedder).max_tokens = property(lambda self: max_tokens)
+
+ return embedder, mock_client
+
+ def test_short_text_no_chunking(self, mock_openai_class):
+ """Short text (< max_tokens) should embed normally without chunking."""
+ embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=8000)
+ result = embedder.embed("Hello world")
+
+ assert result.dense_vector is not None
+ assert len(result.dense_vector) == DIMENSION
+ assert mock_client.embeddings.create.call_count >= 1
+
+ def test_oversized_text_triggers_chunking(self, mock_openai_class):
+ """Text exceeding max_tokens should trigger chunking and return correct dimension."""
+ embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=10)
+
+ # Create text that will exceed 10 tokens (using fallback: len//3 > 10 means len > 30)
+ long_text = "This is a sentence. " * 20 # ~400 chars -> ~133 estimated tokens
+
+ result = embedder.embed(long_text)
+
+ assert result.dense_vector is not None
+ assert len(result.dense_vector) == DIMENSION
+ # Multiple API calls should have been made (one per chunk + dimension detection)
+ assert mock_client.embeddings.create.call_count > 2
+
+ def test_very_long_text_chunking(self, mock_openai_class):
+ """Text 10x over the limit should still chunk and return normally."""
+ embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=10)
+
+ # Very long text: ~3000 chars -> ~1000 estimated tokens (100x over limit of 10)
+ very_long_text = "A" * 3000
+
+ result = embedder.embed(very_long_text)
+
+ assert result.dense_vector is not None
+ assert len(result.dense_vector) == DIMENSION
+
+ def test_empty_text_no_error(self, mock_openai_class):
+ """Empty text should not raise an error."""
+ embedder, _ = self._make_embedder(mock_openai_class, max_tokens=8000)
+ result = embedder.embed("")
+
+ assert result.dense_vector is not None
+ assert len(result.dense_vector) == DIMENSION
+
+ def test_embed_batch_mixed_lengths(self, mock_openai_class):
+ """embed_batch with mixed short and long texts should handle each correctly."""
+ embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=10)
+
+ short_text = "Hi"
+ long_text = "This is a very long sentence. " * 20 # exceeds 10 tokens
+
+ results = embedder.embed_batch([short_text, long_text])
+
+ assert len(results) == 2
+ for result in results:
+ assert result.dense_vector is not None
+ assert len(result.dense_vector) == DIMENSION
+
+ def test_embed_batch_empty_list(self, mock_openai_class):
+ """embed_batch with empty list should return empty list."""
+ embedder, _ = self._make_embedder(mock_openai_class, max_tokens=8000)
+ results = embedder.embed_batch([])
+ assert results == []
+
+ def test_merged_vector_is_normalized(self, mock_openai_class):
+ """Merged vector from chunking should be L2 normalized."""
+ embedder, _ = self._make_embedder(mock_openai_class, max_tokens=10)
+
+ long_text = "Word " * 100 # definitely exceeds 10 tokens
+
+ result = embedder.embed(long_text)
+ assert result.dense_vector is not None
+
+ # Check L2 norm is approximately 1.0
+ norm = math.sqrt(sum(v * v for v in result.dense_vector))
+ assert abs(norm - 1.0) < 1e-6
+
+ def test_paragraph_splitting(self, mock_openai_class):
+ """Text with paragraph breaks should be split by paragraphs first."""
+ embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=10)
+
+ # Create text with clear paragraph separators
+ paragraphs = ["Paragraph one content."] * 5
+ text_with_paragraphs = "\n\n".join(paragraphs)
+
+ result = embedder.embed(text_with_paragraphs)
+ assert result.dense_vector is not None
+ assert len(result.dense_vector) == DIMENSION
+
+
+@patch("openai.OpenAI")
+class TestTiktokenFallback:
+ """Test tiktoken fallback behavior."""
+
+ def test_fallback_when_tiktoken_unavailable(self, mock_openai_class):
+ """When tiktoken is not available, should fall back to character-based estimation."""
+ mock_client = _make_mock_openai_client()
+ mock_openai_class.return_value = mock_client
+
+ mod = _load_openai_embedders()
+ OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder
+
+ embedder = OpenAIDenseEmbedder(
+ model_name="text-embedding-3-small",
+ api_key="test-api-key",
+ dimension=DIMENSION,
+ )
+
+ # Force tiktoken encoder to None to simulate unavailability
+ embedder._tiktoken_enc = None
+
+ # Character-based estimation: len("hello world") // 3 = 3
+ estimated = embedder._estimate_tokens("hello world")
+ assert estimated == len("hello world") // 3
+
+ def test_estimate_tokens_with_tiktoken_encoder(self, mock_openai_class):
+ """When tiktoken encoder is available, should use it for estimation."""
+ mock_client = _make_mock_openai_client()
+ mock_openai_class.return_value = mock_client
+
+ mod = _load_openai_embedders()
+ OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder
+
+ embedder = OpenAIDenseEmbedder(
+ model_name="text-embedding-3-small",
+ api_key="test-api-key",
+ dimension=DIMENSION,
+ )
+
+ # If tiktoken is available, _tiktoken_enc should not be None
+ # and estimation should use it
+ if embedder._tiktoken_enc is not None:
+ tokens = embedder._estimate_tokens("hello world")
+ assert isinstance(tokens, int)
+ assert tokens > 0
+
+
+class TestBaseChunkText:
+ """Test _chunk_text and helper methods on a concrete subclass."""
+
+ def _make_simple_embedder(self, max_tokens=10):
+ """Create a minimal concrete embedder for testing chunking logic."""
+
+ class SimpleEmbedder(DenseEmbedderBase):
+ @property
+ def max_tokens(self_inner):
+ return max_tokens
+
+ def _estimate_tokens(self_inner, text):
+ # Simple: 1 token per word
+ return len(text.split())
+
+ def embed(self_inner, text):
+ dim = 4
+ vec = [1.0 / dim] * dim
+ return EmbedResult(dense_vector=vec)
+
+ def _embed_single(self_inner, text):
+ return self_inner.embed(text)
+
+ def get_dimension(self_inner):
+ return 4
+
+ return SimpleEmbedder(model_name="test")
+
+ def test_chunk_text_short(self):
+ """Text within limit should return single chunk."""
+ embedder = self._make_simple_embedder(max_tokens=100)
+ chunks = embedder._chunk_text("hello world")
+ assert chunks == ["hello world"]
+
+ def test_chunk_text_paragraph_split(self):
+ """Text with paragraphs should split by paragraph boundaries."""
+ embedder = self._make_simple_embedder(max_tokens=5)
+ text = "one two three\n\nfour five six"
+ chunks = embedder._chunk_text(text)
+ assert len(chunks) >= 2
+ for chunk in chunks:
+ assert embedder._estimate_tokens(chunk) <= 5
+
+ def test_chunk_text_sentence_split(self):
+ """Text with sentences but no paragraphs should split by sentences."""
+ embedder = self._make_simple_embedder(max_tokens=5)
+ text = "One two three four. Five six seven eight."
+ chunks = embedder._chunk_text(text)
+ assert len(chunks) >= 2
+
+ def test_chunk_text_fixed_length_fallback(self):
+ """Text without good natural boundaries should fall back to fixed-length split."""
+ embedder = self._make_simple_embedder(max_tokens=3)
+ # Use a long text with no sentence/paragraph boundaries, >100 chars to exceed min chunk_size
+ text = "word " * 200 # 200 words = 200 tokens, 1000 chars
+ chunks = embedder._chunk_text(text)
+ assert len(chunks) >= 2
+
+ def test_chunk_and_embed_returns_normalized(self):
+ """_chunk_and_embed should return a normalized vector."""
+ embedder = self._make_simple_embedder(max_tokens=3)
+ text = "one two three four five six seven eight"
+ result = embedder._chunk_and_embed(text)
+
+ assert result.dense_vector is not None
+ norm = math.sqrt(sum(v * v for v in result.dense_vector))
+ assert abs(norm - 1.0) < 1e-6
+
+ def test_chunk_and_embed_short_text_no_split(self):
+ """_chunk_and_embed on short text should just call _embed_single."""
+ embedder = self._make_simple_embedder(max_tokens=100)
+ result = embedder._chunk_and_embed("hello")
+
+ assert result.dense_vector is not None
+ assert len(result.dense_vector) == 4
+
+
+@patch("openai.OpenAI")
+class TestMaxTokensConfigurable:
+ """Test cases for configurable max_tokens parameter."""
+
+ def test_custom_max_tokens_via_constructor(self, mock_openai_class):
+ """Custom max_tokens passed to OpenAIDenseEmbedder should be used."""
+ mock_client = _make_mock_openai_client()
+ mock_openai_class.return_value = mock_client
+
+ mod = _load_openai_embedders()
+ OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder
+
+ embedder = OpenAIDenseEmbedder(
+ model_name="text-embedding-3-small",
+ api_key="test-api-key",
+ dimension=DIMENSION,
+ max_tokens=32000,
+ )
+
+ assert embedder.max_tokens == 32000
+
+ def test_default_max_tokens_when_not_provided(self, mock_openai_class):
+ """When max_tokens is not provided, should default to 8000."""
+ mock_client = _make_mock_openai_client()
+ mock_openai_class.return_value = mock_client
+
+ mod = _load_openai_embedders()
+ OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder
+
+ embedder = OpenAIDenseEmbedder(
+ model_name="text-embedding-3-small",
+ api_key="test-api-key",
+ dimension=DIMENSION,
+ )
+
+ assert embedder.max_tokens == 8000
+
+ def test_custom_max_tokens_affects_chunking(self, mock_openai_class):
+ """A higher max_tokens should allow longer texts without chunking."""
+ mock_client = _make_mock_openai_client()
+ mock_openai_class.return_value = mock_client
+
+ mod = _load_openai_embedders()
+ OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder
+
+ # Create embedder with high max_tokens
+ embedder = OpenAIDenseEmbedder(
+ model_name="text-embedding-3-small",
+ api_key="test-api-key",
+ dimension=DIMENSION,
+ max_tokens=100000,
+ )
+ # Force fallback estimation (len//3)
+ embedder._tiktoken_enc = None
+
+ # This text would exceed 8000 tokens with fallback estimation (30000 chars -> 10000 tokens)
+ # but should NOT trigger chunking with max_tokens=100000
+ long_text = "word " * 6000 # 30000 chars -> 10000 estimated tokens
+
+ mock_client.embeddings.create.reset_mock()
+ result = embedder.embed(long_text)
+
+ assert result.dense_vector is not None
+ # Should be a single API call (plus dimension detection), NOT chunked
+ assert mock_client.embeddings.create.call_count == 1
+
+ def test_low_custom_max_tokens_triggers_chunking(self, mock_openai_class):
+ """A lower max_tokens should trigger chunking for shorter texts."""
+ mock_client = _make_mock_openai_client()
+ mock_openai_class.return_value = mock_client
+
+ mod = _load_openai_embedders()
+ OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder
+
+ embedder = OpenAIDenseEmbedder(
+ model_name="text-embedding-3-small",
+ api_key="test-api-key",
+ dimension=DIMENSION,
+ max_tokens=5,
+ )
+ # Force fallback estimation
+ embedder._tiktoken_enc = None
+
+ # Need text long enough to produce multiple chunks.
+ # Fallback estimation: len(text)//3. With max_tokens=5, need >5 tokens.
+ # Fixed-length split has min chunk_size=100, so text must be >100 chars to split.
+ text = "Hello world test. " * 30 # 540 chars -> 180 estimated tokens, well over max_tokens=5
+
+ mock_client.embeddings.create.reset_mock()
+ result = embedder.embed(text)
+
+ assert result.dense_vector is not None
+ # Should have chunked (multiple API calls)
+ assert mock_client.embeddings.create.call_count > 1
+
+
+class TestBaseMaxTokensConfigurable:
+ """Test configurable max_tokens on EmbedderBase directly."""
+
+ def test_base_default_max_tokens(self):
+ """EmbedderBase without max_tokens should return 8000."""
+
+ class ConcreteEmbedder(DenseEmbedderBase):
+ def embed(self, text):
+ return EmbedResult(dense_vector=[0.0])
+
+ def get_dimension(self):
+ return 1
+
+ embedder = ConcreteEmbedder(model_name="test")
+ assert embedder.max_tokens == 8000
+
+ def test_base_custom_max_tokens(self):
+ """EmbedderBase with max_tokens should return the custom value."""
+
+ class ConcreteEmbedder(DenseEmbedderBase):
+ def embed(self, text):
+ return EmbedResult(dense_vector=[0.0])
+
+ def get_dimension(self):
+ return 1
+
+ embedder = ConcreteEmbedder(model_name="test", max_tokens=32768)
+ assert embedder.max_tokens == 32768