Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
revert-embedding-chunking (#741)
Qin Haojied30d96cadfce
diff --git a/openviking/models/embedder/base.py b/openviking/models/embedder/base.pyindex bd178df88..a8c23a5f7 100644--- a/openviking/models/embedder/base.py+++ b/openviking/models/embedder/base.py@@ -1,16 +1,11 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0-import logging-import mathimport random-import reimport timefrom abc import ABC, abstractmethodfrom dataclasses import dataclassfrom typing import Any, Callable, Dict, List, Optional, TypeVar-logger = logging.getLogger(__name__)-T = TypeVar("T")@@ -27,6 +22,8 @@ def truncate_and_normalize(embedding: List[float], dimension: Optional[int]) ->if not dimension or len(embedding) <= dimension:return embedding+ import math+embedding = embedding[:dimension]norm = math.sqrt(sum(x**2 for x in embedding))if norm > 0:@@ -68,185 +65,15 @@ class EmbedderBase(ABC):Provides unified embedding interface supporting dense, sparse, and hybrid modes."""- def __init__(- self,- model_name: str,- config: Optional[Dict[str, Any]] = None,- max_tokens: Optional[int] = None,- ):+ def __init__(self, model_name: str, config: Optional[Dict[str, Any]] = None):"""Initialize embedderArgs:model_name: Model nameconfig: Configuration dict containing api_key, api_base, etc.- max_tokens: Maximum token count per embedding request, None to use default (8000)"""self.model_name = model_nameself.config = config or {}- self._max_tokens = max_tokens-- @property- def max_tokens(self) -> int:- """Maximum token count per embedding request. Subclasses can override."""- if self._max_tokens is not None:- return self._max_tokens- return 8000-- def _estimate_tokens(self, text: str) -> int:- """Estimate token count for the given text.-- Tries tiktoken (for OpenAI models) first, falls back to a character-based- heuristic that accounts for multi-byte (CJK) characters.-- Args:- text: Input text-- Returns:- Estimated token count- """- try:- import tiktoken-- enc = tiktoken.encoding_for_model(self.model_name)- return len(enc.encode(text))- except Exception:- logger.info(- "tiktoken unavailable for model '%s', using character-based estimation",- self.model_name,- )- # Use the higher of char-based and byte-based estimates so that- # CJK text (3 bytes per char in UTF-8) is not underestimated.- return max(len(text) // 3, len(text.encode("utf-8")) // 4)-- def _chunk_text(self, text: str) -> List[str]:- """Split text into chunks, each within max_tokens.-- Splitting priority: paragraphs (\\n\\n) > sentences (。.!?\\n) > fixed length.-- Args:- text: Input text-- Returns:- List of text chunks- """- max_tok = self.max_tokens- if self._estimate_tokens(text) <= max_tok:- return [text]-- # Try paragraph split first- paragraphs = text.split("\n\n")- if len(paragraphs) > 1:- chunks = self._merge_segments(paragraphs, max_tok, "\n\n")- if all(self._estimate_tokens(c) <= max_tok for c in chunks):- return chunks-- # Try sentence split- sentences = re.split(r"(?<=[。.!?\n])", text)- sentences = [s for s in sentences if s]- if len(sentences) > 1:- chunks = self._merge_segments(sentences, max_tok, "")- if all(self._estimate_tokens(c) <= max_tok for c in chunks):- return chunks-- # Fixed-length split as last resort- return self._fixed_length_split(text, max_tok)-- def _merge_segments(self, segments: List[str], max_tok: int, separator: str) -> List[str]:- """Merge small segments into chunks that fit within max_tokens."""- chunks: List[str] = []- current = ""- for seg in segments:- candidate = (current + separator + seg) if current else seg- if self._estimate_tokens(candidate) <= max_tok:- current = candidate- else:- if current:- chunks.append(current)- # If a single segment exceeds max_tok, it will be handled later- current = seg- if current:- chunks.append(current)- return chunks-- def _fixed_length_split(self, text: str, max_tok: int) -> List[str]:- """Split text into fixed-length chunks based on estimated char-per-token ratio."""- # Estimate chars per token- total_tokens = self._estimate_tokens(text)- chars_per_token = len(text) / max(total_tokens, 1)- chunk_size = max(int(max_tok * chars_per_token * 0.9), 100)-- chunks: List[str] = []- start = 0- while start < len(text):- end = start + chunk_size- if end < len(text):- # Try to break at a whitespace boundary- boundary = text.rfind(" ", start, end)- if boundary > start:- end = boundary + 1- chunks.append(text[start:end])- start = end- return chunks-- def _chunk_and_embed(self, text: str, is_query: bool = False) -> EmbedResult:- """Chunk text if it exceeds max_tokens, embed each chunk, and merge results.-- For text within limits, delegates to _embed_single directly.- For oversized text, splits into chunks, embeds each, then computes a- token-weighted average of dense vectors with L2 normalization.-- Args:- text: Input text- is_query: Flag to indicate if this is a query embedding-- Returns:- EmbedResult with merged embedding- """- estimated = self._estimate_tokens(text)- if estimated <= self.max_tokens:- return self._embed_single(text, is_query=is_query)-- chunks = self._chunk_text(text)- logger.debug(- "Chunking text: original ~%d tokens -> %d chunks",- estimated,- len(chunks),- )-- results: List[EmbedResult] = []- weights: List[int] = []- for chunk in chunks:- result = self._embed_single(chunk, is_query=is_query)- results.append(result)- weights.append(self._estimate_tokens(chunk))-- # Merge dense vectors via weighted average + L2 normalization- merged_dense = None- if results and results[0].dense_vector is not None:- dim = len(results[0].dense_vector)- total_weight = sum(weights)- merged = [0.0] * dim- for result, w in zip(results, weights):- if result.dense_vector:- for i in range(dim):- merged[i] += result.dense_vector[i] * w- if total_weight > 0:- merged = [v / total_weight for v in merged]- # L2 normalization- norm = math.sqrt(sum(v * v for v in merged))- if norm > 0:- merged = [v / norm for v in merged]- merged_dense = merged-- return EmbedResult(dense_vector=merged_dense)-- def _embed_single(self, text: str, is_query: bool = False) -> EmbedResult:- """Embed a single text without chunking logic. Defaults to self.embed().-- Subclasses that override embed() with chunking should also override- this method to provide the raw embedding call.- """- return self.embed(text, is_query=is_query)@abstractmethoddef embed(self, text: str, is_query: bool = False) -> EmbedResult:diff --git a/openviking/models/embedder/openai_embedders.py b/openviking/models/embedder/openai_embedders.pyindex 2924b98d8..709c14647 100644--- a/openviking/models/embedder/openai_embedders.py+++ b/openviking/models/embedder/openai_embedders.py@@ -2,7 +2,6 @@# SPDX-License-Identifier: Apache-2.0"""OpenAI Embedder Implementation"""-import loggingfrom typing import Any, Dict, List, Optionalimport openai@@ -15,8 +14,6 @@ from openviking.models.embedder.base import ()from openviking.telemetry import get_current_telemetry-logger = logging.getLogger(__name__)-class OpenAIDenseEmbedder(DenseEmbedderBase):"""OpenAI-Compatible Dense Embedder Implementation@@ -70,7 +67,6 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):query_param: Optional[str] = None,document_param: Optional[str] = None,config: Optional[Dict[str, Any]] = None,- max_tokens: Optional[int] = None,extra_headers: Optional[Dict[str, str]] = None,input_type: Optional[str] = None,):@@ -93,7 +89,6 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):Setting this (or query_param) activates non-symmetric mode.Only supported by OpenAI-compatible third-party models.config: Additional configuration dict- max_tokens: Maximum token count per embedding request, None to use default (8000)extra_headers: Extra HTTP headers to include in API requests (e.g., for OpenRouter:{'HTTP-Referer': 'https://your-site.com', 'X-Title': 'Your App'})@@ -106,7 +101,7 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):supported by OpenAI-compatible third-party models (e.g., BGE-M3, Jina, Cohere) thatimplement the input_type parameter."""- super().__init__(model_name, config, max_tokens=max_tokens)+ super().__init__(model_name, config)self.api_key = api_keyself.api_base = api_base@@ -128,43 +123,15 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):client_kwargs["default_headers"] = extra_headersself.client = openai.OpenAI(**client_kwargs)- # Initialize tiktoken encoder- self._tiktoken_enc = None- try:- import tiktoken-- self._tiktoken_enc = tiktoken.encoding_for_model(model_name)- except Exception:- logger.info(- "tiktoken unavailable for model '%s', will use character-based estimation",- model_name,- )-# Auto-detect dimensionself._dimension = dimensionif self._dimension is None:self._dimension = self._detect_dimension()- @property- def max_tokens(self) -> int:- """OpenAI embedding models have 8192 token limit; use 8000 for safety buffer.-- Can be overridden via the max_tokens constructor parameter.- """- if self._max_tokens is not None:- return self._max_tokens- return 8000-- def _estimate_tokens(self, text: str) -> int:- """Estimate tokens using tiktoken if available, fallback to len(text) // 3."""- if self._tiktoken_enc is not None:- return len(self._tiktoken_enc.encode(text))- return len(text) // 3-def _detect_dimension(self) -> int:"""Detect dimension by making an actual API call"""try:- result = self._embed_single("test", is_query=False)+ result = self.embed("test")return len(result.dense_vector) if result.dense_vector else 1536except Exception:# Use default value, text-embedding-3-small defaults to 1536@@ -244,8 +211,8 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):return extra_body if extra_body else None- def _embed_single(self, text: str, is_query: bool = False) -> EmbedResult:- """Perform raw embedding without chunking logic.+ def embed(self, text: str, is_query: bool = False) -> EmbedResult:+ """Perform dense embedding on textArgs:text: Input text@@ -274,31 +241,8 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):except Exception as e:raise RuntimeError(f"Embedding failed: {str(e)}") from e- def embed(self, text: str, is_query: bool = False) -> EmbedResult:- """Embed single text, with automatic chunking for oversized input.-- Args:- text: Input text- is_query: Flag to indicate if this is a query embedding-- Returns:- EmbedResult: Result containing only dense_vector-- Raises:- RuntimeError: When API call fails- """- if not text:- return self._embed_single(text, is_query=is_query)-- if self._estimate_tokens(text) > self.max_tokens:- return self._chunk_and_embed(text, is_query=is_query)- return self._embed_single(text, is_query=is_query)-def embed_batch(self, texts: List[str], is_query: bool = False) -> List[EmbedResult]:- """Batch embedding with automatic chunking for oversized inputs.-- Short texts are batched together via the OpenAI API for efficiency.- Oversized texts are individually chunked and embedded.+ """Batch embedding (OpenAI native support)Args:texts: List of texts@@ -313,35 +257,23 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):if not texts:return []- results: List[Optional[EmbedResult]] = [None] * len(texts)- short_indices: List[int] = []- short_texts: List[str] = []+ try:+ kwargs: Dict[str, Any] = {"input": texts, "model": self.model_name}+ if self.dimension:+ kwargs["dimensions"] = self.dimension- for i, text in enumerate(texts):- if text and self._estimate_tokens(text) > self.max_tokens:- results[i] = self._chunk_and_embed(text, is_query=is_query)- else:- short_indices.append(i)- short_texts.append(text)-- if short_texts:- try:- kwargs: Dict[str, Any] = {"input": short_texts, "model": self.model_name}-- extra_body = self._build_extra_body(is_query=is_query)- if extra_body:- kwargs["extra_body"] = extra_body-- response = self.client.embeddings.create(**kwargs)- self._update_telemetry_token_usage(response)- for idx, item in zip(short_indices, response.data):- results[idx] = EmbedResult(dense_vector=item.embedding)- except openai.APIError as e:- raise RuntimeError(f"OpenAI API error: {e.message}") from e- except Exception as e:- raise RuntimeError(f"Batch embedding failed: {str(e)}") from e-- return results # type: ignore[return-value]+ extra_body = self._build_extra_body(is_query=is_query)+ if extra_body:+ kwargs["extra_body"] = extra_body++ response = self.client.embeddings.create(**kwargs)+ self._update_telemetry_token_usage(response)++ return [EmbedResult(dense_vector=item.embedding) for item in response.data]+ except openai.APIError as e:+ raise RuntimeError(f"OpenAI API error: {e.message}") from e+ except Exception as e:+ raise RuntimeError(f"Batch embedding failed: {str(e)}") from edef get_dimension(self) -> int:"""Get embedding dimensiondiff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.pyindex 3f504d402..a0a1b9640 100644--- a/openviking_cli/utils/config/embedding_config.py+++ b/openviking_cli/utils/config/embedding_config.py@@ -51,10 +51,6 @@ class EmbeddingModelConfig(BaseModel):sk: Optional[str] = Field(default=None, description="Access Key Secretfor VikingDB API")region: Optional[str] = Field(default=None, description="Region for VikingDB API")host: Optional[str] = Field(default=None, description="Host for VikingDB API")- max_tokens: Optional[int] = Field(- default=None,- description="Maximum token count per embedding request. If None, uses model default (e.g., 8000 for OpenAI).",- )extra_headers: Optional[dict[str, str]] = Field(default=None,description=(@@ -227,7 +223,6 @@ class EmbeddingConfig(BaseModel):"dimension": cfg.dimension,**({"query_param": cfg.query_param} if cfg.query_param else {}),**({"document_param": cfg.document_param} if cfg.document_param else {}),- "max_tokens": cfg.max_tokens,**({"extra_headers": cfg.extra_headers} if cfg.extra_headers else {}),},),@@ -316,7 +311,6 @@ class EmbeddingConfig(BaseModel):or "no-key", # Ollama ignores the key, but client requires non-empty"api_base": cfg.api_base or "http://localhost:11434/v1","dimension": cfg.dimension,- "max_tokens": cfg.max_tokens,},),("voyage", "dense"): (diff --git a/tests/unit/test_openai_embedder_chunking.py b/tests/unit/test_openai_embedder_chunking.pydeleted file mode 100644index 6c6c71afa..000000000--- a/tests/unit/test_openai_embedder_chunking.py+++ /dev/null@@ -1,532 +0,0 @@-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.-# SPDX-License-Identifier: Apache-2.0-"""Tests for OpenAI Embedder text chunking functionality (Issue #616)--Uses importlib to load modules directly, avoiding the deep import chain-from openviking/__init__.py that requires the full build (C++ engine, litellm, etc.).-"""--import importlib.util-import math-import os-import sys-from unittest.mock import MagicMock, patch--import pytest--# -- Direct module loading to avoid openviking/__init__.py import chain ----_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))---def _load_module_from_file(module_name: str, file_path: str):- """Load a Python module directly from file path."""- spec = importlib.util.spec_from_file_location(module_name, file_path)- mod = importlib.util.module_from_spec(spec)- sys.modules[module_name] = mod- spec.loader.exec_module(mod)- return mod---# Load base module first-_base_mod = _load_module_from_file(- "openviking.models.embedder.base",- os.path.join(_PROJECT_ROOT, "openviking", "models", "embedder", "base.py"),-)-EmbedResult = _base_mod.EmbedResult-EmbedderBase = _base_mod.EmbedderBase-DenseEmbedderBase = _base_mod.DenseEmbedderBase---DIMENSION = 8---def _make_mock_openai_client(dimension=DIMENSION):- """Create a mock OpenAI client that returns deterministic embeddings."""- mock_client = MagicMock()-- def fake_create(**kwargs):- inp = kwargs["input"]- if isinstance(inp, list):- items = []- for text in inp:- vec = _deterministic_vector(text, dimension)- item = MagicMock()- item.embedding = vec- items.append(item)- resp = MagicMock()- resp.data = items- return resp- else:- vec = _deterministic_vector(inp, dimension)- item = MagicMock()- item.embedding = vec- resp = MagicMock()- resp.data = [item]- return resp-- mock_client.embeddings.create.side_effect = fake_create- return mock_client---def _deterministic_vector(text, dim):- """Generate a simple deterministic vector from text for testing."""- seed = sum(ord(c) for c in text) % 1000- vec = [(seed + i) * 0.001 for i in range(dim)]- # Normalize- norm = math.sqrt(sum(v * v for v in vec))- if norm > 0:- vec = [v / norm for v in vec]- return vec---def _load_openai_embedders():- """Load the openai_embedders module with mocked dependencies."""- return _load_module_from_file(- "openviking.models.embedder.openai_embedders",- os.path.join(_PROJECT_ROOT, "openviking", "models", "embedder", "openai_embedders.py"),- )---@patch("openai.OpenAI")-class TestOpenAIEmbedderChunking:- """Test cases for OpenAI embedder text chunking (Issue #616)"""-- def _make_embedder(self, mock_openai_class, max_tokens=100):- """Helper to create a test embedder with a mocked client."""- mock_client = _make_mock_openai_client()- mock_openai_class.return_value = mock_client-- mod = _load_openai_embedders()- OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder-- embedder = OpenAIDenseEmbedder(- model_name="text-embedding-3-small",- api_key="test-api-key",- dimension=DIMENSION,- )-- # Override max_tokens for testing (lower limit to trigger chunking easily)- type(embedder).max_tokens = property(lambda self: max_tokens)-- return embedder, mock_client-- def test_short_text_no_chunking(self, mock_openai_class):- """Short text (< max_tokens) should embed normally without chunking."""- embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=8000)- result = embedder.embed("Hello world")-- assert result.dense_vector is not None- assert len(result.dense_vector) == DIMENSION- assert mock_client.embeddings.create.call_count >= 1-- def test_oversized_text_triggers_chunking(self, mock_openai_class):- """Text exceeding max_tokens should trigger chunking and return correct dimension."""- embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=10)-- # Create text that will exceed 10 tokens (using fallback: len//3 > 10 means len > 30)- long_text = "This is a sentence. " * 20 # ~400 chars -> ~133 estimated tokens-- result = embedder.embed(long_text)-- assert result.dense_vector is not None- assert len(result.dense_vector) == DIMENSION- # Multiple API calls should have been made (one per chunk + dimension detection)- assert mock_client.embeddings.create.call_count > 2-- def test_very_long_text_chunking(self, mock_openai_class):- """Text 10x over the limit should still chunk and return normally."""- embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=10)-- # Very long text: ~3000 chars -> ~1000 estimated tokens (100x over limit of 10)- very_long_text = "A" * 3000-- result = embedder.embed(very_long_text)-- assert result.dense_vector is not None- assert len(result.dense_vector) == DIMENSION-- def test_empty_text_no_error(self, mock_openai_class):- """Empty text should not raise an error."""- embedder, _ = self._make_embedder(mock_openai_class, max_tokens=8000)- result = embedder.embed("")-- assert result.dense_vector is not None- assert len(result.dense_vector) == DIMENSION-- def test_embed_batch_mixed_lengths(self, mock_openai_class):- """embed_batch with mixed short and long texts should handle each correctly."""- embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=10)-- short_text = "Hi"- long_text = "This is a very long sentence. " * 20 # exceeds 10 tokens-- results = embedder.embed_batch([short_text, long_text])-- assert len(results) == 2- for result in results:- assert result.dense_vector is not None- assert len(result.dense_vector) == DIMENSION-- def test_embed_batch_empty_list(self, mock_openai_class):- """embed_batch with empty list should return empty list."""- embedder, _ = self._make_embedder(mock_openai_class, max_tokens=8000)- results = embedder.embed_batch([])- assert results == []-- def test_merged_vector_is_normalized(self, mock_openai_class):- """Merged vector from chunking should be L2 normalized."""- embedder, _ = self._make_embedder(mock_openai_class, max_tokens=10)-- long_text = "Word " * 100 # definitely exceeds 10 tokens-- result = embedder.embed(long_text)- assert result.dense_vector is not None-- # Check L2 norm is approximately 1.0- norm = math.sqrt(sum(v * v for v in result.dense_vector))- assert abs(norm - 1.0) < 1e-6-- def test_paragraph_splitting(self, mock_openai_class):- """Text with paragraph breaks should be split by paragraphs first."""- embedder, mock_client = self._make_embedder(mock_openai_class, max_tokens=10)-- # Create text with clear paragraph separators- paragraphs = ["Paragraph one content."] * 5- text_with_paragraphs = "\n\n".join(paragraphs)-- result = embedder.embed(text_with_paragraphs)- assert result.dense_vector is not None- assert len(result.dense_vector) == DIMENSION---@patch("openai.OpenAI")-class TestTiktokenFallback:- """Test tiktoken fallback behavior."""-- def test_fallback_when_tiktoken_unavailable(self, mock_openai_class):- """When tiktoken is not available, should fall back to character-based estimation."""- mock_client = _make_mock_openai_client()- mock_openai_class.return_value = mock_client-- mod = _load_openai_embedders()- OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder-- embedder = OpenAIDenseEmbedder(- model_name="text-embedding-3-small",- api_key="test-api-key",- dimension=DIMENSION,- )-- # Force tiktoken encoder to None to simulate unavailability- embedder._tiktoken_enc = None-- # Character-based estimation: len("hello world") // 3 = 3- estimated = embedder._estimate_tokens("hello world")- assert estimated == len("hello world") // 3-- def test_estimate_tokens_with_tiktoken_encoder(self, mock_openai_class):- """When tiktoken encoder is available, should use it for estimation."""- mock_client = _make_mock_openai_client()- mock_openai_class.return_value = mock_client-- mod = _load_openai_embedders()- OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder-- embedder = OpenAIDenseEmbedder(- model_name="text-embedding-3-small",- api_key="test-api-key",- dimension=DIMENSION,- )-- # If tiktoken is available, _tiktoken_enc should not be None- # and estimation should use it- if embedder._tiktoken_enc is not None:- tokens = embedder._estimate_tokens("hello world")- assert isinstance(tokens, int)- assert tokens > 0---class TestBaseChunkText:- """Test _chunk_text and helper methods on a concrete subclass."""-- def _make_simple_embedder(self, max_tokens=10):- """Create a minimal concrete embedder for testing chunking logic."""-- class SimpleEmbedder(DenseEmbedderBase):- @property- def max_tokens(self_inner):- return max_tokens-- def _estimate_tokens(self_inner, text):- # Simple: 1 token per word- return len(text.split())-- def embed(self_inner, text, is_query=False):- dim = 4- vec = [1.0 / dim] * dim- return EmbedResult(dense_vector=vec)-- def _embed_single(self_inner, text, is_query=False):- return self_inner.embed(text, is_query=is_query)-- def get_dimension(self_inner):- return 4-- return SimpleEmbedder(model_name="test")-- def test_chunk_text_short(self):- """Text within limit should return single chunk."""- embedder = self._make_simple_embedder(max_tokens=100)- chunks = embedder._chunk_text("hello world")- assert chunks == ["hello world"]-- def test_chunk_text_paragraph_split(self):- """Text with paragraphs should split by paragraph boundaries."""- embedder = self._make_simple_embedder(max_tokens=5)- text = "one two three\n\nfour five six"- chunks = embedder._chunk_text(text)- assert len(chunks) >= 2- for chunk in chunks:- assert embedder._estimate_tokens(chunk) <= 5-- def test_chunk_text_sentence_split(self):- """Text with sentences but no paragraphs should split by sentences."""- embedder = self._make_simple_embedder(max_tokens=5)- text = "One two three four. Five six seven eight."- chunks = embedder._chunk_text(text)- assert len(chunks) >= 2-- def test_chunk_text_fixed_length_fallback(self):- """Text without good natural boundaries should fall back to fixed-length split."""- embedder = self._make_simple_embedder(max_tokens=3)- # Use a long text with no sentence/paragraph boundaries, >100 chars to exceed min chunk_size- text = "word " * 200 # 200 words = 200 tokens, 1000 chars- chunks = embedder._chunk_text(text)- assert len(chunks) >= 2-- def test_chunk_and_embed_returns_normalized(self):- """_chunk_and_embed should return a normalized vector."""- embedder = self._make_simple_embedder(max_tokens=3)- text = "one two three four five six seven eight"- result = embedder._chunk_and_embed(text)-- assert result.dense_vector is not None- norm = math.sqrt(sum(v * v for v in result.dense_vector))- assert abs(norm - 1.0) < 1e-6-- def test_chunk_and_embed_short_text_no_split(self):- """_chunk_and_embed on short text should just call _embed_single."""- embedder = self._make_simple_embedder(max_tokens=100)- result = embedder._chunk_and_embed("hello")-- assert result.dense_vector is not None- assert len(result.dense_vector) == 4---@patch("openai.OpenAI")-class TestMaxTokensConfigurable:- """Test cases for configurable max_tokens parameter."""-- def test_custom_max_tokens_via_constructor(self, mock_openai_class):- """Custom max_tokens passed to OpenAIDenseEmbedder should be used."""- mock_client = _make_mock_openai_client()- mock_openai_class.return_value = mock_client-- mod = _load_openai_embedders()- OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder-- embedder = OpenAIDenseEmbedder(- model_name="text-embedding-3-small",- api_key="test-api-key",- dimension=DIMENSION,- max_tokens=32000,- )-- assert embedder.max_tokens == 32000-- def test_default_max_tokens_when_not_provided(self, mock_openai_class):- """When max_tokens is not provided, should default to 8000."""- mock_client = _make_mock_openai_client()- mock_openai_class.return_value = mock_client-- mod = _load_openai_embedders()- OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder-- embedder = OpenAIDenseEmbedder(- model_name="text-embedding-3-small",- api_key="test-api-key",- dimension=DIMENSION,- )-- assert embedder.max_tokens == 8000-- def test_custom_max_tokens_affects_chunking(self, mock_openai_class):- """A higher max_tokens should allow longer texts without chunking."""- mock_client = _make_mock_openai_client()- mock_openai_class.return_value = mock_client-- mod = _load_openai_embedders()- OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder-- # Create embedder with high max_tokens- embedder = OpenAIDenseEmbedder(- model_name="text-embedding-3-small",- api_key="test-api-key",- dimension=DIMENSION,- max_tokens=100000,- )- # Force fallback estimation (len//3)- embedder._tiktoken_enc = None-- # This text would exceed 8000 tokens with fallback estimation (30000 chars -> 10000 tokens)- # but should NOT trigger chunking with max_tokens=100000- long_text = "word " * 6000 # 30000 chars -> 10000 estimated tokens-- mock_client.embeddings.create.reset_mock()- result = embedder.embed(long_text)-- assert result.dense_vector is not None- # Should be a single API call (plus dimension detection), NOT chunked- assert mock_client.embeddings.create.call_count == 1-- def test_low_custom_max_tokens_triggers_chunking(self, mock_openai_class):- """A lower max_tokens should trigger chunking for shorter texts."""- mock_client = _make_mock_openai_client()- mock_openai_class.return_value = mock_client-- mod = _load_openai_embedders()- OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder-- embedder = OpenAIDenseEmbedder(- model_name="text-embedding-3-small",- api_key="test-api-key",- dimension=DIMENSION,- max_tokens=5,- )- # Force fallback estimation- embedder._tiktoken_enc = None-- # Need text long enough to produce multiple chunks.- # Fallback estimation: len(text)//3. With max_tokens=5, need >5 tokens.- # Fixed-length split has min chunk_size=100, so text must be >100 chars to split.- text = (- "Hello world test. " * 30- ) # 540 chars -> 180 estimated tokens, well over max_tokens=5-- mock_client.embeddings.create.reset_mock()- result = embedder.embed(text)-- assert result.dense_vector is not None- # Should have chunked (multiple API calls)- assert mock_client.embeddings.create.call_count > 1---class TestBaseMaxTokensConfigurable:- """Test configurable max_tokens on EmbedderBase directly."""-- def test_base_default_max_tokens(self):- """EmbedderBase without max_tokens should return 8000."""-- class ConcreteEmbedder(DenseEmbedderBase):- def embed(self, text, is_query=False):- return EmbedResult(dense_vector=[0.0])-- def get_dimension(self):- return 1-- embedder = ConcreteEmbedder(model_name="test")- assert embedder.max_tokens == 8000-- def test_base_custom_max_tokens(self):- """EmbedderBase with max_tokens should return the custom value."""-- class ConcreteEmbedder(DenseEmbedderBase):- def embed(self, text, is_query=False):- return EmbedResult(dense_vector=[0.0])-- def get_dimension(self):- return 1-- embedder = ConcreteEmbedder(model_name="test", max_tokens=32768)- assert embedder.max_tokens == 32768---@patch("openai.OpenAI")-class TestOpenAIEmbedderWithoutApiKey:- """Test cases for OpenAI embedder without api_key (local servers)."""-- def test_init_with_api_base_no_api_key(self, mock_openai_class):- """OpenAIDenseEmbedder should initialize with api_base but no api_key."""- mock_client = _make_mock_openai_client()- mock_openai_class.return_value = mock_client-- mod = _load_openai_embedders()- OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder-- # Should NOT raise when api_base is set but api_key is not- embedder = OpenAIDenseEmbedder(- model_name="text-embedding-3-small",- api_base="http://localhost:11434/v1",- dimension=DIMENSION,- )-- assert embedder is not None- # Verify OpenAI client was initialized with placeholder key- mock_openai_class.assert_called_once()- call_kwargs = mock_openai_class.call_args[1]- assert call_kwargs["api_key"] == "no-key"- assert call_kwargs["base_url"] == "http://localhost:11434/v1"-- def test_init_without_api_key_or_api_base_raises(self, mock_openai_class):- """OpenAIDenseEmbedder should raise when neither api_key nor api_base is set."""- mock_client = _make_mock_openai_client()- mock_openai_class.return_value = mock_client-- mod = _load_openai_embedders()- OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder-- # Should raise when neither is provided- with pytest.raises(ValueError, match="api_key is required"):- OpenAIDenseEmbedder(- model_name="text-embedding-3-small",- dimension=DIMENSION,- )-- def test_init_with_api_key_works_normally(self, mock_openai_class):- """OpenAIDenseEmbedder should work normally with api_key provided."""- mock_client = _make_mock_openai_client()- mock_openai_class.return_value = mock_client-- mod = _load_openai_embedders()- OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder-- embedder = OpenAIDenseEmbedder(- model_name="text-embedding-3-small",- api_key="test-api-key",- dimension=DIMENSION,- )-- assert embedder is not None- mock_openai_class.assert_called_once()- call_kwargs = mock_openai_class.call_args[1]- assert call_kwargs["api_key"] == "test-api-key"-- def test_embed_with_api_base_no_api_key(self, mock_openai_class):- """Embedding should work with api_base but no api_key (local server)."""- mock_client = _make_mock_openai_client()- mock_openai_class.return_value = mock_client-- mod = _load_openai_embedders()- OpenAIDenseEmbedder = mod.OpenAIDenseEmbedder-- embedder = OpenAIDenseEmbedder(- model_name="text-embedding-3-small",- api_base="http://localhost:11434/v1",- dimension=DIMENSION,- )-- result = embedder.embed("Hello world")-- assert result.dense_vector is not None- assert len(result.dense_vector) == DIMENSION- mock_client.embeddings.create.assert_called()