Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix: enable explicit openai multimodal embeddings (#4433)

zgy1ee1219ab8aa
diff --git a/openviking/models/embedder/openai_embedders.py b/openviking/models/embedder/openai_embedders.py
index c0dab6efc..26defb990 100644
--- a/openviking/models/embedder/openai_embedders.py
+++ b/openviking/models/embedder/openai_embedders.py
@@ -9,6 +9,7 @@ import openai
from openviking.models.embedder.base import (
DenseEmbedderBase,
+ EmbeddingInput,
EmbedResult,
HybridEmbedderBase,
SparseEmbedderBase,
@@ -104,6 +105,9 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
config: Additional configuration dict
extra_headers: Extra HTTP headers to include in API requests (e.g., for OpenRouter:
{'HTTP-Referer': 'https://your-site.com', 'X-Title': 'Your App'})
+ input_type: OpenViking embedding input mode ('text' or 'multimodal'). This controls
+ whether content parts are passed through, and is distinct from
+ query_param/document_param values sent as extra_body.input_type.
encoding_format: Wire format for embedding values. ``None`` (default) lets the
OpenAI Python SDK pick its default (currently ``base64``). Set to
``"float"`` to send/receive plain JSON arrays — the recommended
@@ -131,6 +135,7 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
self.dimension = dimension
self.query_param = query_param
self.document_param = document_param
+ self._supports_multimodal = input_type == "multimodal"
self.encoding_format = encoding_format
self.extra_body = extra_body
self._provider = provider.lower()
@@ -291,7 +296,18 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
return extra_body if extra_body else None
- def _build_kwargs(self, text_input: str | List[str], is_query: bool = False) -> Dict[str, Any]:
+ @property
+ def supports_multimodal(self) -> bool:
+ return self._supports_multimodal
+
+ def _prepare_embedding_input(self, text: EmbeddingInput | List[str]) -> EmbeddingInput | List[str]:
+ if isinstance(text, list) and all(isinstance(item, str) for item in text):
+ return text
+ return self.prepare_embedding_input(text)
+
+ def _build_kwargs(
+ self, text_input: EmbeddingInput | List[str], is_query: bool = False
+ ) -> Dict[str, Any]:
kwargs: Dict[str, Any] = {"input": text_input, "model": self.model_name}
if self.dimension and self._should_send_dimensions():
kwargs["dimensions"] = self.dimension
@@ -325,11 +341,11 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
return self._async_client_cache.get(_build_async_client)
- def embed(self, text: str, is_query: bool = False) -> EmbedResult:
- """Perform dense embedding on text
+ def embed(self, text: EmbeddingInput | List[str], is_query: bool = False) -> EmbedResult:
+ """Perform dense embedding on text, text batches, or multimodal content
Args:
- text: Input text
+ text: Input text, OpenAI text batch, or multimodal content parts
is_query: Flag to indicate if this is a query embedding
Returns:
@@ -338,9 +354,12 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
Raises:
RuntimeError: When API call fails
"""
+ embedding_input = self._prepare_embedding_input(text)
def _call() -> EmbedResult:
- response = self.client.embeddings.create(**self._build_kwargs(text, is_query=is_query))
+ response = self.client.embeddings.create(
+ **self._build_kwargs(embedding_input, is_query=is_query)
+ )
self._update_telemetry_token_usage(response)
vector = response.data[0].embedding
@@ -360,11 +379,14 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
except Exception as e:
raise RuntimeError(f"Embedding failed: {str(e)}") from e
- async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:
+ async def embed_async(self, text: EmbeddingInput | List[str], is_query: bool = False) -> EmbedResult:
client = self._get_async_client()
+ embedding_input = self._prepare_embedding_input(text)
async def _call() -> EmbedResult:
- response = await client.embeddings.create(**self._build_kwargs(text, is_query=is_query))
+ response = await client.embeddings.create(
+ **self._build_kwargs(embedding_input, is_query=is_query)
+ )
self._update_telemetry_token_usage(response)
return EmbedResult(dense_vector=self._truncate_vector(response.data[0].embedding))
diff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.py
index b3205dd9a..bb0c6b0b2 100644
--- a/openviking_cli/utils/config/embedding_config.py
+++ b/openviking_cli/utils/config/embedding_config.py
@@ -769,6 +769,11 @@ class EmbeddingConfig(BaseModel):
if cfg.encoding_format is not None
else {}
),
+ **(
+ {"input_type": "multimodal"}
+ if "input" in cfg.model_fields_set and cfg.input == "multimodal"
+ else {}
+ ),
**({"extra_body": cfg.extra_body} if cfg.extra_body else {}),
},
),
@@ -1052,12 +1057,12 @@ class EmbeddingConfig(BaseModel):
credential_ids = []
for cred in config.credentials:
- # Create a temporary config merged from the model config and credential
+ # Preserve whether `input` was explicitly set; the default multimodal
+ # value must not opt OpenAI-compatible failover configs into image input.
merged_config = EmbeddingModelConfig(
model=cred.model or config.model,
dimension=config.dimension,
batch_size=config.batch_size,
- input=config.input,
query_param=config.query_param,
document_param=config.document_param,
provider=cred.provider or config.provider,
@@ -1079,6 +1084,7 @@ class EmbeddingConfig(BaseModel):
enable_fusion=config.enable_fusion,
res_level=config.res_level,
max_video_frames=config.max_video_frames,
+ **({"input": config.input} if "input" in config.model_fields_set else {}),
)
provider = self._require_provider(merged_config.provider)
embedders.append(self._create_embedder(provider, embedder_type, merged_config))
diff --git a/tests/misc/test_embedding_input_type.py b/tests/misc/test_embedding_input_type.py
index b07e92d62..84fed058d 100644
--- a/tests/misc/test_embedding_input_type.py
+++ b/tests/misc/test_embedding_input_type.py
@@ -137,6 +137,74 @@ class TestEmbeddingConfigContextualEmbedders:
call_kwargs = mock_embedder_class.call_args[1]
assert "query_param" not in call_kwargs
assert "document_param" not in call_kwargs
+ assert "input_type" not in call_kwargs
+
+ @patch("openviking.models.embedder.OpenAIDenseEmbedder")
+ def test_get_embedder_openai_passes_explicit_multimodal_input(
+ self, mock_embedder_class
+ ):
+ """Explicit input=multimodal should opt OpenAI-compatible embedders into multimodal."""
+ mock_embedder_class.return_value = MagicMock()
+ config = EmbeddingConfig(
+ dense=EmbeddingModelConfig(
+ model="custom-multimodal-embedding",
+ provider="openai",
+ api_key="sk-test",
+ input="multimodal",
+ )
+ )
+
+ config.get_embedder()
+
+ mock_embedder_class.assert_called_once()
+ call_kwargs = mock_embedder_class.call_args[1]
+ assert call_kwargs.get("input_type") == "multimodal"
+
+ @patch("openviking.models.embedder.OpenAIDenseEmbedder")
+ def test_get_embedder_openai_failover_omits_default_input(self, mock_embedder_class):
+ """Failover merging must not turn the default input value into explicit opt-in."""
+ mock_embedder_class.return_value = MagicMock()
+ config = EmbeddingConfig(
+ dense=EmbeddingModelConfig(
+ model="text-embedding-3-small",
+ provider="openai",
+ credentials=[
+ {"id": "primary", "api_key": "sk-primary"},
+ {"id": "backup", "api_key": "sk-backup"},
+ ],
+ )
+ )
+
+ config.get_embedder()
+
+ assert mock_embedder_class.call_count == 2
+ for call in mock_embedder_class.call_args_list:
+ assert "input_type" not in call.kwargs
+
+ @patch("openviking.models.embedder.OpenAIDenseEmbedder")
+ def test_get_embedder_openai_failover_passes_explicit_multimodal_input(
+ self, mock_embedder_class
+ ):
+ """Failover credentials should inherit explicit input=multimodal from parent config."""
+ mock_embedder_class.return_value = MagicMock()
+ config = EmbeddingConfig(
+ dense=EmbeddingModelConfig(
+ model="custom-multimodal-embedding",
+ provider="openai",
+ input="multimodal",
+ dimension=1536,
+ credentials=[
+ {"id": "primary", "api_key": "sk-primary"},
+ {"id": "backup", "api_key": "sk-backup"},
+ ],
+ )
+ )
+
+ config.get_embedder()
+
+ assert mock_embedder_class.call_count == 2
+ for call in mock_embedder_class.call_args_list:
+ assert call.kwargs.get("input_type") == "multimodal"
@patch("openviking.models.embedder.JinaDenseEmbedder")
def test_get_embedder_jina_no_params_when_not_set(self, mock_embedder_class):
diff --git a/tests/unit/test_openai_embedder.py b/tests/unit/test_openai_embedder.py
index 89fb9db98..5ca3090ef 100644
--- a/tests/unit/test_openai_embedder.py
+++ b/tests/unit/test_openai_embedder.py
@@ -57,6 +57,91 @@ class TestOpenAIDenseEmbedder:
call_kwargs = mock_client.embeddings.create.call_args[1]
assert "extra_body" not in call_kwargs
+ @patch("openviking.models.embedder.openai_embedders.openai.OpenAI")
+ def test_default_input_type_embed_downgrades_multimodal_parts(self, mock_openai_class):
+ """Direct embed(parts) should also apply the text-only input guard."""
+ mock_client = MagicMock()
+ mock_openai_class.return_value = mock_client
+
+ mock_embedding = MagicMock()
+ mock_embedding.embedding = [0.1] * 1536
+
+ mock_response = MagicMock()
+ mock_response.data = [mock_embedding]
+ mock_client.embeddings.create.return_value = mock_response
+
+ embedder = OpenAIDenseEmbedder(
+ model_name="text-embedding-3-small",
+ api_key="test-api-key",
+ dimension=1536,
+ )
+ parts = [
+ {"type": "text", "text": "find this"},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,xxx"}},
+ ]
+
+ embedder.embed(parts)
+
+ call_kwargs = mock_client.embeddings.create.call_args[1]
+ assert call_kwargs["input"] == "find this"
+
+ @patch("openviking.models.embedder.openai_embedders.openai.OpenAI")
+ def test_embed_preserves_openai_batch_text_input(self, mock_openai_class):
+ """OpenAI batch text input should keep the historical list[str] passthrough."""
+ mock_client = MagicMock()
+ mock_openai_class.return_value = mock_client
+
+ mock_embedding = MagicMock()
+ mock_embedding.embedding = [0.1] * 1536
+
+ mock_response = MagicMock()
+ mock_response.data = [mock_embedding]
+ mock_client.embeddings.create.return_value = mock_response
+
+ embedder = OpenAIDenseEmbedder(
+ model_name="text-embedding-3-small",
+ api_key="test-api-key",
+ dimension=1536,
+ )
+
+ embedder.embed(["first text", "second text"])
+
+ call_kwargs = mock_client.embeddings.create.call_args[1]
+ assert call_kwargs["input"] == ["first text", "second text"]
+
+ @patch("openviking.models.embedder.openai_embedders.openai.OpenAI")
+ def test_multimodal_input_type_passes_content_parts(self, mock_openai_class):
+ """OpenAI-compatible multimodal mode should pass content parts to embeddings.create."""
+ mock_client = MagicMock()
+ mock_openai_class.return_value = mock_client
+
+ mock_embedding = MagicMock()
+ mock_embedding.embedding = [0.1] * 1536
+
+ mock_response = MagicMock()
+ mock_response.data = [mock_embedding]
+ mock_client.embeddings.create.return_value = mock_response
+
+ embedder = OpenAIDenseEmbedder(
+ model_name="custom-multimodal-embedding",
+ api_key="test-api-key",
+ api_base="https://example.com/v1",
+ dimension=1536,
+ input_type="multimodal",
+ )
+ parts = [
+ {"type": "text", "text": "find this"},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,xxx"}},
+ ]
+
+ assert embedder.supports_multimodal is True
+ assert embedder.prepare_embedding_input(parts) == parts
+
+ embedder.embed(parts)
+
+ call_kwargs = mock_client.embeddings.create.call_args[1]
+ assert call_kwargs["input"] == parts
+
@patch("openviking.models.embedder.openai_embedders.openai.OpenAI")
def test_embed_with_context_query(self, mock_openai_class):
"""OpenAI embed should include extra_body with input_type='query' when is_query=True"""