Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix: enable explicit openai multimodal embeddings (#4433)
zgy1ee1219ab8aa
diff --git a/openviking/models/embedder/openai_embedders.py b/openviking/models/embedder/openai_embedders.pyindex c0dab6efc..26defb990 100644--- a/openviking/models/embedder/openai_embedders.py+++ b/openviking/models/embedder/openai_embedders.py@@ -9,6 +9,7 @@ import openaifrom openviking.models.embedder.base import (DenseEmbedderBase,+ EmbeddingInput,EmbedResult,HybridEmbedderBase,SparseEmbedderBase,@@ -104,6 +105,9 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):config: Additional configuration dictextra_headers: Extra HTTP headers to include in API requests (e.g., for OpenRouter:{'HTTP-Referer': 'https://your-site.com', 'X-Title': 'Your App'})+ input_type: OpenViking embedding input mode ('text' or 'multimodal'). This controls+ whether content parts are passed through, and is distinct from+ query_param/document_param values sent as extra_body.input_type.encoding_format: Wire format for embedding values. ``None`` (default) lets theOpenAI Python SDK pick its default (currently ``base64``). Set to``"float"`` to send/receive plain JSON arrays — the recommended@@ -131,6 +135,7 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):self.dimension = dimensionself.query_param = query_paramself.document_param = document_param+ self._supports_multimodal = input_type == "multimodal"self.encoding_format = encoding_formatself.extra_body = extra_bodyself._provider = provider.lower()@@ -291,7 +296,18 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):return extra_body if extra_body else None- def _build_kwargs(self, text_input: str | List[str], is_query: bool = False) -> Dict[str, Any]:+ @property+ def supports_multimodal(self) -> bool:+ return self._supports_multimodal++ def _prepare_embedding_input(self, text: EmbeddingInput | List[str]) -> EmbeddingInput | List[str]:+ if isinstance(text, list) and all(isinstance(item, str) for item in text):+ return text+ return self.prepare_embedding_input(text)++ def _build_kwargs(+ self, text_input: EmbeddingInput | List[str], is_query: bool = False+ ) -> Dict[str, Any]:kwargs: Dict[str, Any] = {"input": text_input, "model": self.model_name}if self.dimension and self._should_send_dimensions():kwargs["dimensions"] = self.dimension@@ -325,11 +341,11 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):return self._async_client_cache.get(_build_async_client)- def embed(self, text: str, is_query: bool = False) -> EmbedResult:- """Perform dense embedding on text+ def embed(self, text: EmbeddingInput | List[str], is_query: bool = False) -> EmbedResult:+ """Perform dense embedding on text, text batches, or multimodal contentArgs:- text: Input text+ text: Input text, OpenAI text batch, or multimodal content partsis_query: Flag to indicate if this is a query embeddingReturns:@@ -338,9 +354,12 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):Raises:RuntimeError: When API call fails"""+ embedding_input = self._prepare_embedding_input(text)def _call() -> EmbedResult:- response = self.client.embeddings.create(**self._build_kwargs(text, is_query=is_query))+ response = self.client.embeddings.create(+ **self._build_kwargs(embedding_input, is_query=is_query)+ )self._update_telemetry_token_usage(response)vector = response.data[0].embedding@@ -360,11 +379,14 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):except Exception as e:raise RuntimeError(f"Embedding failed: {str(e)}") from e- async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:+ async def embed_async(self, text: EmbeddingInput | List[str], is_query: bool = False) -> EmbedResult:client = self._get_async_client()+ embedding_input = self._prepare_embedding_input(text)async def _call() -> EmbedResult:- response = await client.embeddings.create(**self._build_kwargs(text, is_query=is_query))+ response = await client.embeddings.create(+ **self._build_kwargs(embedding_input, is_query=is_query)+ )self._update_telemetry_token_usage(response)return EmbedResult(dense_vector=self._truncate_vector(response.data[0].embedding))diff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.pyindex b3205dd9a..bb0c6b0b2 100644--- a/openviking_cli/utils/config/embedding_config.py+++ b/openviking_cli/utils/config/embedding_config.py@@ -769,6 +769,11 @@ class EmbeddingConfig(BaseModel):if cfg.encoding_format is not Noneelse {}),+ **(+ {"input_type": "multimodal"}+ if "input" in cfg.model_fields_set and cfg.input == "multimodal"+ else {}+ ),**({"extra_body": cfg.extra_body} if cfg.extra_body else {}),},),@@ -1052,12 +1057,12 @@ class EmbeddingConfig(BaseModel):credential_ids = []for cred in config.credentials:- # Create a temporary config merged from the model config and credential+ # Preserve whether `input` was explicitly set; the default multimodal+ # value must not opt OpenAI-compatible failover configs into image input.merged_config = EmbeddingModelConfig(model=cred.model or config.model,dimension=config.dimension,batch_size=config.batch_size,- input=config.input,query_param=config.query_param,document_param=config.document_param,provider=cred.provider or config.provider,@@ -1079,6 +1084,7 @@ class EmbeddingConfig(BaseModel):enable_fusion=config.enable_fusion,res_level=config.res_level,max_video_frames=config.max_video_frames,+ **({"input": config.input} if "input" in config.model_fields_set else {}),)provider = self._require_provider(merged_config.provider)embedders.append(self._create_embedder(provider, embedder_type, merged_config))diff --git a/tests/misc/test_embedding_input_type.py b/tests/misc/test_embedding_input_type.pyindex b07e92d62..84fed058d 100644--- a/tests/misc/test_embedding_input_type.py+++ b/tests/misc/test_embedding_input_type.py@@ -137,6 +137,74 @@ class TestEmbeddingConfigContextualEmbedders:call_kwargs = mock_embedder_class.call_args[1]assert "query_param" not in call_kwargsassert "document_param" not in call_kwargs+ assert "input_type" not in call_kwargs++ @patch("openviking.models.embedder.OpenAIDenseEmbedder")+ def test_get_embedder_openai_passes_explicit_multimodal_input(+ self, mock_embedder_class+ ):+ """Explicit input=multimodal should opt OpenAI-compatible embedders into multimodal."""+ mock_embedder_class.return_value = MagicMock()+ config = EmbeddingConfig(+ dense=EmbeddingModelConfig(+ model="custom-multimodal-embedding",+ provider="openai",+ api_key="sk-test",+ input="multimodal",+ )+ )++ config.get_embedder()++ mock_embedder_class.assert_called_once()+ call_kwargs = mock_embedder_class.call_args[1]+ assert call_kwargs.get("input_type") == "multimodal"++ @patch("openviking.models.embedder.OpenAIDenseEmbedder")+ def test_get_embedder_openai_failover_omits_default_input(self, mock_embedder_class):+ """Failover merging must not turn the default input value into explicit opt-in."""+ mock_embedder_class.return_value = MagicMock()+ config = EmbeddingConfig(+ dense=EmbeddingModelConfig(+ model="text-embedding-3-small",+ provider="openai",+ credentials=[+ {"id": "primary", "api_key": "sk-primary"},+ {"id": "backup", "api_key": "sk-backup"},+ ],+ )+ )++ config.get_embedder()++ assert mock_embedder_class.call_count == 2+ for call in mock_embedder_class.call_args_list:+ assert "input_type" not in call.kwargs++ @patch("openviking.models.embedder.OpenAIDenseEmbedder")+ def test_get_embedder_openai_failover_passes_explicit_multimodal_input(+ self, mock_embedder_class+ ):+ """Failover credentials should inherit explicit input=multimodal from parent config."""+ mock_embedder_class.return_value = MagicMock()+ config = EmbeddingConfig(+ dense=EmbeddingModelConfig(+ model="custom-multimodal-embedding",+ provider="openai",+ input="multimodal",+ dimension=1536,+ credentials=[+ {"id": "primary", "api_key": "sk-primary"},+ {"id": "backup", "api_key": "sk-backup"},+ ],+ )+ )++ config.get_embedder()++ assert mock_embedder_class.call_count == 2+ for call in mock_embedder_class.call_args_list:+ assert call.kwargs.get("input_type") == "multimodal"@patch("openviking.models.embedder.JinaDenseEmbedder")def test_get_embedder_jina_no_params_when_not_set(self, mock_embedder_class):diff --git a/tests/unit/test_openai_embedder.py b/tests/unit/test_openai_embedder.pyindex 89fb9db98..5ca3090ef 100644--- a/tests/unit/test_openai_embedder.py+++ b/tests/unit/test_openai_embedder.py@@ -57,6 +57,91 @@ class TestOpenAIDenseEmbedder:call_kwargs = mock_client.embeddings.create.call_args[1]assert "extra_body" not in call_kwargs+ @patch("openviking.models.embedder.openai_embedders.openai.OpenAI")+ def test_default_input_type_embed_downgrades_multimodal_parts(self, mock_openai_class):+ """Direct embed(parts) should also apply the text-only input guard."""+ mock_client = MagicMock()+ mock_openai_class.return_value = mock_client++ mock_embedding = MagicMock()+ mock_embedding.embedding = [0.1] * 1536++ mock_response = MagicMock()+ mock_response.data = [mock_embedding]+ mock_client.embeddings.create.return_value = mock_response++ embedder = OpenAIDenseEmbedder(+ model_name="text-embedding-3-small",+ api_key="test-api-key",+ dimension=1536,+ )+ parts = [+ {"type": "text", "text": "find this"},+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,xxx"}},+ ]++ embedder.embed(parts)++ call_kwargs = mock_client.embeddings.create.call_args[1]+ assert call_kwargs["input"] == "find this"++ @patch("openviking.models.embedder.openai_embedders.openai.OpenAI")+ def test_embed_preserves_openai_batch_text_input(self, mock_openai_class):+ """OpenAI batch text input should keep the historical list[str] passthrough."""+ mock_client = MagicMock()+ mock_openai_class.return_value = mock_client++ mock_embedding = MagicMock()+ mock_embedding.embedding = [0.1] * 1536++ mock_response = MagicMock()+ mock_response.data = [mock_embedding]+ mock_client.embeddings.create.return_value = mock_response++ embedder = OpenAIDenseEmbedder(+ model_name="text-embedding-3-small",+ api_key="test-api-key",+ dimension=1536,+ )++ embedder.embed(["first text", "second text"])++ call_kwargs = mock_client.embeddings.create.call_args[1]+ assert call_kwargs["input"] == ["first text", "second text"]++ @patch("openviking.models.embedder.openai_embedders.openai.OpenAI")+ def test_multimodal_input_type_passes_content_parts(self, mock_openai_class):+ """OpenAI-compatible multimodal mode should pass content parts to embeddings.create."""+ mock_client = MagicMock()+ mock_openai_class.return_value = mock_client++ mock_embedding = MagicMock()+ mock_embedding.embedding = [0.1] * 1536++ mock_response = MagicMock()+ mock_response.data = [mock_embedding]+ mock_client.embeddings.create.return_value = mock_response++ embedder = OpenAIDenseEmbedder(+ model_name="custom-multimodal-embedding",+ api_key="test-api-key",+ api_base="https://example.com/v1",+ dimension=1536,+ input_type="multimodal",+ )+ parts = [+ {"type": "text", "text": "find this"},+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,xxx"}},+ ]++ assert embedder.supports_multimodal is True+ assert embedder.prepare_embedding_input(parts) == parts++ embedder.embed(parts)++ call_kwargs = mock_client.embeddings.create.call_args[1]+ assert call_kwargs["input"] == parts+@patch("openviking.models.embedder.openai_embedders.openai.OpenAI")def test_embed_with_context_query(self, mock_openai_class):"""OpenAI embed should include extra_body with input_type='query' when is_query=True"""