Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat: 计算图片embed向量时支持图文embed,通过配置参数image_vectorization决定embed模式(summary_only,image_only,image_and_summary) (#2460)

Haoyu Zhangc675b1732ebb
diff --git a/openviking/core/context.py b/openviking/core/context.py
index ae59b8243..8fb92e246 100644
--- a/openviking/core/context.py
+++ b/openviking/core/context.py
@@ -41,12 +41,14 @@ class ContextLevel(int, Enum):
class Vectorize:
text: str = ""
- # image: str = ""
+ # images: list of image references (data URIs or URLs) for multimodal embedding
+ images: List[str] = []
# video: str = ""
# audio: str = ""
- def __init__(self, text: str = ""):
+ def __init__(self, text: str = "", images: Optional[List[str]] = None):
self.text = text
+ self.images = list(images) if images else []
class Context:
@@ -144,9 +146,12 @@ class Context:
def get_vectorization_text(self) -> str:
"""Get text for vectorization."""
- # todo: multi-modal support
return self.vectorize.text
+ def get_vectorization_images(self) -> List[str]:
+ """Get image references (data URIs or URLs) for multimodal vectorization."""
+ return self.vectorize.images
+
def update_activity(self):
"""Update activity statistics."""
self.active_count += 1
diff --git a/openviking/models/embedder/base.py b/openviking/models/embedder/base.py
index 30f42d12b..7fa429e89 100644
--- a/openviking/models/embedder/base.py
+++ b/openviking/models/embedder/base.py
@@ -8,7 +8,7 @@ import weakref
from abc import ABC, abstractmethod
from dataclasses import dataclass
from threading import Lock
-from typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVar
+from typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVar, Union
from openviking.telemetry import get_current_telemetry
from openviking.utils.embedding_input import (
@@ -21,6 +21,10 @@ from openviking_cli.utils import get_logger
T = TypeVar("T")
logger = get_logger(__name__)
+# A multimodal embedding input is a list of content parts, e.g.
+# [{"type": "text", "text": "..."}, {"type": "image_url", "image_url": {"url": "..."}}]
+EmbeddingInput = Union[str, List[Dict[str, Any]]]
+
_token_tracker_instance = None
_ASYNC_EMBED_SEMAPHORES: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, Dict[int, asyncio.Semaphore]]" = weakref.WeakKeyDictionary()
@@ -49,14 +53,37 @@ def _get_token_tracker():
return _token_tracker_instance
+def extract_text_from_content(content: "EmbeddingInput") -> str:
+ """Extract and join the text parts from a multimodal input.
+
+ ``content`` may be a plain string (returned as-is) or a list of content
+ parts such as ``[{"type": "text", "text": "..."}, {"type": "image_url", ...}]``.
+ Non-text parts (e.g. images) are ignored.
+ """
+ if isinstance(content, str):
+ return content
+ text_parts = [
+ part.get("text", "")
+ for part in content
+ if isinstance(part, dict) and part.get("type") == "text"
+ ]
+ return "\n".join(p for p in text_parts if p)
+
+
async def embed_compat(
- embedder: "EmbedderBase", text: str, *, is_query: bool = False
+ embedder: "EmbedderBase", content: "EmbeddingInput", *, is_query: bool = False
) -> "EmbedResult":
- """Prepare input, then call the embedder's async-compatible entrypoint."""
+ """Prepare input, then call the embedder's async-compatible entrypoint.
+
+ Accepts either a plain text string or a multimodal input (a list of content
+ parts). ``prepare_embedding_input`` downgrades multimodal inputs to text for
+ embedders that do not support images, so all embedders can be called the same
+ way.
+ """
from openviking.telemetry import bind_telemetry_stage
stage = "embed_query" if is_query else "embed_resource"
- embedding_input = embedder.prepare_embedding_input(text)
+ embedding_input = embedder.prepare_embedding_input(content)
with bind_telemetry_stage(stage):
return await embedder.embed_async(embedding_input, is_query=is_query)
@@ -147,24 +174,56 @@ class EmbedderBase(ABC):
self._token_tracker = _get_token_tracker()
self._active_call_started_at: float | None = None
- def prepare_embedding_input(self, text: str) -> str:
- """Apply this embedder's input guard before provider calls."""
- if self.max_input_tokens is None:
- return text
- return truncate_embedding_input(text, self.max_input_tokens)
+ def prepare_embedding_input(self, content: "EmbeddingInput") -> "EmbeddingInput":
+ """Apply this embedder's input guard before provider calls.
- def prepare_embedding_inputs(self, texts: List[str]) -> List[str]:
+ Plain text is truncated to ``max_input_tokens``. For embedders that
+ support images, multimodal inputs (a list of content parts) are kept as a
+ list. Embedders that do not support multimodal input get the
+ text parts extracted, so image parts are safely dropped.
+ """
+ if isinstance(content, list) and self.supports_multimodal:
+ if self.max_input_tokens is None:
+ return content
+ truncated_parts = []
+ for part in content:
+ if isinstance(part, dict) and part.get("type") == "text":
+ part = {
+ **part,
+ "text": truncate_embedding_input(
+ part.get("text", ""), self.max_input_tokens
+ ),
+ }
+ truncated_parts.append(part)
+ return truncated_parts
+ else:
+ content = extract_text_from_content(content)
+ if self.max_input_tokens is None:
+ return content
+ return truncate_embedding_input(content, self.max_input_tokens)
+
+ def prepare_embedding_inputs(
+ self, contents: List["EmbeddingInput"]
+ ) -> List["EmbeddingInput"]:
"""Apply this embedder's input guard to a batch."""
- if self.max_input_tokens is None:
- return texts
- return [self.prepare_embedding_input(text) for text in texts]
+ return [self.prepare_embedding_input(content) for content in contents]
+
+ @property
+ def supports_multimodal(self) -> bool:
+ """Whether this embedder can consume multimodal (image) inputs directly.
+
+ Text-only embedders return False so that multimodal inputs are downgraded
+ to their text parts before being embedded.
+ """
+ return False
@abstractmethod
- def embed(self, text: str, is_query: bool = False) -> EmbedResult:
- """Embed single text
+ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:
+ """Embed text or multimodal content.
Args:
- text: Input text
+ content: Input text, or a list of multimodal content parts such as
+ ``[{"type": "text", "text": "..."}, {"type": "image_url", "image_url": {"url": "..."}}]``
is_query: Flag to indicate if this is a query embedding
Returns:
@@ -172,17 +231,19 @@ class EmbedderBase(ABC):
"""
pass
- def embed_batch(self, texts: List[str], is_query: bool = False) -> List[EmbedResult]:
+ def embed_batch(
+ self, contents: List["EmbeddingInput"], is_query: bool = False
+ ) -> List[EmbedResult]:
"""Batch embedding (default implementation loops, subclasses can override for optimization)
Args:
- texts: List of texts
+ contents: List of texts or multimodal content parts
is_query: Flag to indicate if these are query embeddings
Returns:
List[EmbedResult]: List of embedding results
"""
- return [self.embed(text, is_query=is_query) for text in texts]
+ return [self.embed(content, is_query=is_query) for content in contents]
def embed_query(self, text: str) -> EmbedResult:
"""Embed query text with explicit retrieval-side semantics."""
@@ -200,22 +261,24 @@ class EmbedderBase(ABC):
"""Batch embed document texts."""
return self.embed_batch(texts, is_query=False)
- async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:
- """Async embed single text.
+ async def embed_async(
+ self, content: "EmbeddingInput", is_query: bool = False
+ ) -> EmbedResult:
+ """Async embed text or multimodal content.
Subclasses should override this with a non-blocking implementation.
The default implementation preserves compatibility for test doubles and
third-party embedders that only implement the sync interface.
"""
- return self.embed(text, is_query=is_query)
+ return self.embed(content, is_query=is_query)
async def embed_batch_async(
- self, texts: List[str], is_query: bool = False
+ self, contents: List["EmbeddingInput"], is_query: bool = False
) -> List[EmbedResult]:
"""Async batch embedding."""
results: List[EmbedResult] = []
- for text in texts:
- results.append(await self.embed_async(text, is_query=is_query))
+ for content in contents:
+ results.append(await self.embed_async(content, is_query=is_query))
return results
async def embed_query_async(self, text: str) -> EmbedResult:
@@ -407,11 +470,11 @@ class DenseEmbedderBase(EmbedderBase):
"""
@abstractmethod
- def embed(self, text: str, is_query: bool = False) -> EmbedResult:
- """Perform dense embedding on text
+ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:
+ """Perform dense embedding on text or multimodal content
Args:
- text: Input text
+ content: Input text, or a list of multimodal content parts
is_query: Flag to indicate if this is a query embedding
Returns:
@@ -440,11 +503,11 @@ class SparseEmbedderBase(EmbedderBase):
"""
@abstractmethod
- def embed(self, text: str, is_query: bool = False) -> EmbedResult:
- """Perform sparse embedding on text
+ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:
+ """Perform sparse embedding on text or multimodal content
Args:
- text: Input text
+ content: Input text, or a list of multimodal content parts
is_query: Flag to indicate if this is a query embedding
Returns:
@@ -469,11 +532,11 @@ class HybridEmbedderBase(EmbedderBase):
"""
@abstractmethod
- def embed(self, text: str, is_query: bool = False) -> EmbedResult:
- """Perform hybrid embedding on text
+ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:
+ """Perform hybrid embedding on text or multimodal content
Args:
- text: Input text
+ content: Input text, or a list of multimodal content parts
is_query: Flag to indicate if this is a query embedding
Returns:
@@ -520,10 +583,18 @@ class CompositeHybridEmbedder(HybridEmbedderBase):
self.dense_embedder = dense_embedder
self.sparse_embedder = sparse_embedder
- def embed(self, text: str, is_query: bool = False) -> EmbedResult:
+ @property
+ def supports_multimodal(self) -> bool:
+ """Supports multimodal input only if both sub-embedders do."""
+ return (
+ self.dense_embedder.supports_multimodal
+ and self.sparse_embedder.supports_multimodal
+ )
+
+ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:
"""Combine results from both embedders"""
- dense_input = self.dense_embedder.prepare_embedding_input(text)
- sparse_input = self.sparse_embedder.prepare_embedding_input(text)
+ dense_input = self.dense_embedder.prepare_embedding_input(content)
+ sparse_input = self.sparse_embedder.prepare_embedding_input(content)
dense_res = self.dense_embedder.embed(dense_input, is_query=is_query)
sparse_res = self.sparse_embedder.embed(sparse_input, is_query=is_query)
@@ -531,10 +602,12 @@ class CompositeHybridEmbedder(HybridEmbedderBase):
dense_vector=dense_res.dense_vector, sparse_vector=sparse_res.sparse_vector
)
- def embed_batch(self, texts: List[str], is_query: bool = False) -> List[EmbedResult]:
+ def embed_batch(
+ self, contents: List["EmbeddingInput"], is_query: bool = False
+ ) -> List[EmbedResult]:
"""Combine batch results"""
- dense_inputs = self.dense_embedder.prepare_embedding_inputs(texts)
- sparse_inputs = self.sparse_embedder.prepare_embedding_inputs(texts)
+ dense_inputs = self.dense_embedder.prepare_embedding_inputs(contents)
+ sparse_inputs = self.sparse_embedder.prepare_embedding_inputs(contents)
dense_results = self.dense_embedder.embed_batch(dense_inputs, is_query=is_query)
sparse_results = self.sparse_embedder.embed_batch(sparse_inputs, is_query=is_query)
@@ -543,9 +616,11 @@ class CompositeHybridEmbedder(HybridEmbedderBase):
for d, s in zip(dense_results, sparse_results, strict=True)
]
- async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:
- dense_input = self.dense_embedder.prepare_embedding_input(text)
- sparse_input = self.sparse_embedder.prepare_embedding_input(text)
+ async def embed_async(
+ self, content: "EmbeddingInput", is_query: bool = False
+ ) -> EmbedResult:
+ dense_input = self.dense_embedder.prepare_embedding_input(content)
+ sparse_input = self.sparse_embedder.prepare_embedding_input(content)
dense_res, sparse_res = await asyncio.gather(
self.dense_embedder.embed_async(dense_input, is_query=is_query),
self.sparse_embedder.embed_async(sparse_input, is_query=is_query),
@@ -555,10 +630,10 @@ class CompositeHybridEmbedder(HybridEmbedderBase):
)
async def embed_batch_async(
- self, texts: List[str], is_query: bool = False
+ self, contents: List["EmbeddingInput"], is_query: bool = False
) -> List[EmbedResult]:
- dense_inputs = self.dense_embedder.prepare_embedding_inputs(texts)
- sparse_inputs = self.sparse_embedder.prepare_embedding_inputs(texts)
+ dense_inputs = self.dense_embedder.prepare_embedding_inputs(contents)
+ sparse_inputs = self.sparse_embedder.prepare_embedding_inputs(contents)
dense_results, sparse_results = await asyncio.gather(
self.dense_embedder.embed_batch_async(dense_inputs, is_query=is_query),
self.sparse_embedder.embed_batch_async(sparse_inputs, is_query=is_query),
diff --git a/openviking/models/embedder/volcengine_embedders.py b/openviking/models/embedder/volcengine_embedders.py
index cd4ce28bd..6d609856a 100644
--- a/openviking/models/embedder/volcengine_embedders.py
+++ b/openviking/models/embedder/volcengine_embedders.py
@@ -2,15 +2,18 @@
# SPDX-License-Identifier: AGPL-3.0
"""Volcengine Embedder Implementation"""
+import asyncio
from typing import Any, Dict, List, Optional
import volcenginesdkarkruntime
from openviking.models.embedder.base import (
DenseEmbedderBase,
+ EmbeddingInput,
EmbedResult,
HybridEmbedderBase,
SparseEmbedderBase,
+ extract_text_from_content,
truncate_and_normalize,
)
from openviking.telemetry import get_current_telemetry
@@ -18,6 +21,18 @@ from openviking.utils.async_client_cache import LoopScopedAsyncClientCache
from openviking_cli.utils.logger import default_logger as logger
+def to_multimodal_input(content: "EmbeddingInput") -> List[Dict[str, Any]]:
+ """Normalize an embedding input into the content-parts payload the Volcengine
+ multimodal embeddings API expects.
+
+ A plain string is wrapped as a single text part. A list of content parts
+ (text and/or ``image_url``) is passed through unchanged.
+ """
+ if isinstance(content, str):
+ return [{"type": "text", "text": content}]
+ return list(content)
+
+
def process_sparse_embedding(sparse_data: Any) -> Dict[str, float]:
"""Process sparse embedding data from SDK response"""
if not sparse_data:
@@ -113,6 +128,11 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
except Exception:
return 2048 # Default dimension
+ @property
+ def supports_multimodal(self) -> bool:
+ """Multimodal inputs are supported when using the multimodal endpoint."""
+ return self.input_type == "multimodal"
+
def _update_telemetry_token_usage(self, response) -> None:
usage = getattr(response, "usage", None)
if not usage:
@@ -142,11 +162,12 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
completion_tokens=completion_tokens,
)
- def embed(self, text: str, is_query: bool = False) -> EmbedResult:
- """Perform dense embedding on text
+ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:
+ """Perform dense embedding on text or multimodal content
Args:
- text: Input text
+ content: Input text, or a list of multimodal content parts such as
+ ``[{"type": "text", "text": "..."}, {"type": "image_url", "image_url": {"url": "..."}}]``
is_query: Flag to indicate if this is a query embedding
Returns:
@@ -160,12 +181,13 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
if self.input_type == "multimodal":
# Use multimodal embeddings API
response = self.client.multimodal_embeddings.create(
- input=[{"type": "text", "text": text}], model=self.model_name
+ input=to_multimodal_input(content), model=self.model_name
)
self._update_telemetry_token_usage(response)
vector = response.data.embedding
else:
- # Use text embeddings API
+ # Use text embeddings API (text-only)
+ text = extract_text_from_content(content)
response = self.client.embeddings.create(input=text, model=self.model_name)
self._update_telemetry_token_usage(response)
vector = response.data[0].embedding
@@ -187,17 +209,20 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
lambda: volcenginesdkarkruntime.AsyncArk(**self._ark_kwargs)
)
- async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:
+ async def embed_async(
+ self, content: "EmbeddingInput", is_query: bool = False
+ ) -> EmbedResult:
client = self._get_async_client()
async def _embed_call() -> EmbedResult:
if self.input_type == "multimodal":
response = await client.multimodal_embeddings.create(
- input=[{"type": "text", "text": text}], model=self.model_name
+ input=to_multimodal_input(content), model=self.model_name
)
self._update_telemetry_token_usage(response)
vector = response.data.embedding
else:
+ text = extract_text_from_content(content)
response = await client.embeddings.create(input=text, model=self.model_name)
self._update_telemetry_token_usage(response)
vector = response.data[0].embedding
@@ -213,11 +238,13 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
except Exception as e:
raise RuntimeError(f"Volcengine embedding failed: {str(e)}") from e
- def embed_batch(self, texts: List[str], is_query: bool = False) -> List[EmbedResult]:
+ def embed_batch(
+ self, contents: List["EmbeddingInput"], is_query: bool = False
+ ) -> List[EmbedResult]:
"""Batch embedding
Args:
- texts: List of texts
+ contents: List of texts or multimodal content parts
is_query: Flag to indicate if these are query embeddings
Returns:
@@ -226,76 +253,22 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):
Raises:
RuntimeError: When API call fails
"""
- if not texts:
+ if not contents:
return []
-
- def _call() -> List[EmbedResult]:
- if self.input_type == "multimodal":
- multimodal_inputs = [{"type": "text", "text": text} for text in texts]
- response = self.client.multimodal_embeddings.create(
- input=multimodal_inputs, model=self.model_name
- )
- self._update_telemetry_token_usage(response)
- data = response.data
- else:
- response = self.client.embeddings.create(input=texts, model=self.model_name)
- self._update_telemetry_token_usage(response)
- data = response.data
-
- return [
- EmbedResult(dense_vector=truncate_and_normalize(item.embedding, self.dimension))
- for item in data
- ]
-
- try:
- return self._run_with_retry(
- _call,
- logger=logger,
- operation_name="Volcengine batch embedding",
- )
- except Exception as e:
- logger.error(
- f"Volcengine batch embedding failed, texts length: {len(texts)}, input_type: {self.input_type}, model_name: {self.model_name}"
- )
- raise RuntimeError(f"Volcengine batch embedding failed: {str(e)}") from e
+ return [self.embed(content, is_query=is_query) for content in contents]
async def embed_batch_async(
- self, texts: List[str], is_query: bool = False
+ self, contents: List["EmbeddingInput"], is_query: bool = False
) -> List[EmbedResult]:
- if not texts:
+ if not contents:
return []
- client = self._get_async_client()
-
- async def _call() -> List[EmbedResult]:
- if self.input_type == "multimodal":
- multimodal_inputs = [{"type": "text", "text": text} for text in texts]
- response = await client.multimodal_embeddings.create(
- input=multimodal_inputs, model=self.model_name
- )
- self._update_telemetry_token_usage(response)
- data = response.data
- else:
- response = await client.embeddings.create(input=texts, model=self.model_name)
- self._update_telemetry_token_usage(response)
- data = response.data
-
- return [
- EmbedResult(dense_vector=truncate_and_normalize(item.embedding, self.dimension))
- for item in data
- ]
-
- try:
- return await self._run_with_async_retry(
- _call,
- logger=logger,
- operation_name="Volcengine async batch embedding",
- )
- except Exception as e:
- logger.error(
- f"Volcengine async batch embedding failed, texts length: {len(texts)}, input_type: {self.input_type}, model_name: {self.model_name}"
+ # Embed each content individually and run them concurrently.
+ return list(
+ await asyncio.gather(
+ *(self.embed_async(content, is_query=is_query) for content in contents)
)
- raise RuntimeError(f"Volcengine batch embedding failed: {str(e)}") from e
+ )
def get_dimension(self) -> int:
return self._dimension
@@ -312,6 +285,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
model_name: str,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
+ input_type: str = "multimodal",
config: Optional[Dict[str, Any]] = None,
):
"""Initialize Volcengine Sparse Embedder
@@ -320,6 +294,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
model_name: Volcengine model name
api_key: API key for authentication
api_base: API base URL
+ input_type: Input type - "text" or "multimodal" (default: "multimodal")
config: Additional configuration dict
Raises:
@@ -330,6 +305,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
self.api_key = api_key
self.api_base = api_base
+ self.input_type = input_type
if not self.api_key:
raise ValueError("api_key is required")
@@ -370,11 +346,12 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
completion_tokens=completion_tokens,
)
- def embed(self, text: str, is_query: bool = False) -> EmbedResult:
- """Perform sparse embedding on text
+ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:
+ """Perform sparse embedding on text or multimodal content
Args:
- text: Input text
+ content: Input text, or a list of multimodal content parts such as
+ ``[{"type": "text", "text": "..."}, {"type": "image_url", "image_url": {"url": "..."}}]``
is_query: Flag to indicate if this is a query embedding
Returns:
@@ -387,7 +364,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
def _embed_call():
# Must use multimodal endpoint for sparse
response = self.client.multimodal_embeddings.create(
- input=[{"type": "text", "text": text}],
+ input=to_multimodal_input(content),
model=self.model_name,
sparse_embedding={"type": "enabled"},
)
@@ -410,12 +387,14 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
lambda: volcenginesdkarkruntime.AsyncArk(**self._ark_kwargs)
)
- async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:
+ async def embed_async(
+ self, content: "EmbeddingInput", is_query: bool = False
+ ) -> EmbedResult:
client = self._get_async_client()
async def _embed_call() -> EmbedResult:
response = await client.multimodal_embeddings.create(
- input=[{"type": "text", "text": text}],
+ input=to_multimodal_input(content),
model=self.model_name,
sparse_embedding={"type": "enabled"},
)
@@ -433,11 +412,13 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
except Exception as e:
raise RuntimeError(f"Volcengine sparse embedding failed: {str(e)}") from e
- def embed_batch(self, texts: List[str], is_query: bool = False) -> List[EmbedResult]:
+ def embed_batch(
+ self, contents: List["EmbeddingInput"], is_query: bool = False
+ ) -> List[EmbedResult]:
"""Batch sparse embedding
Args:
- texts: List of texts
+ contents: List of texts or multimodal content parts
is_query: Flag to indicate if these are query embeddings
Returns:
@@ -446,41 +427,28 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):
Raises:
RuntimeError: When API call fails
"""
- if not texts:
+ if not contents:
return []
- return [self.embed(text) for text in texts]
+ return [self.embed(content, is_query=is_query) for content in contents]
async def embed_batch_async(
- self, texts: List[str], is_query: bool = False
+ self, contents: List["EmbeddingInput"], is_query: bool = False
) -> List[EmbedResult]:
- if not texts:
+ if not contents:
return []
- client = self._get_async_client()
-
- async def _call() -> List[EmbedResult]:
- response = await client.multimodal_embeddings.create(
- input=[{"type": "text", "text": text} for text in texts],
- model=self.model_name,
- sparse_embedding={"type": "enabled"},
+ # The multimodal endpoint embeds a single content per call; run the items
+ # concurrently while sharing the async client.
+ return list(
+ await asyncio.gather(
+ *(self.embed_async(content, is_query=is_query) for content in contents)
)
- self._update_telemetry_token_usage(response)
- data = response.data
- return [
- EmbedResult(
- sparse_vector=process_sparse_embedding(getattr(item, "sparse_embedding", None))
- )
- for item in data
- ]
+ )
- try:
- return await self._run_with_async_retry(
- _call,
- logger=logger,
- operation_name="Volcengine async sparse batch embedding",
- )
- except Exception as e:
- raise RuntimeError(f"Volcengine sparse embedding failed: {str(e)}") from e
+ @property
+ def supports_multimodal(self) -> bool:
+ """Multimodal inputs are supported when using the multimodal endpoint."""
+ return self.input_type == "multimodal"
class VolcengineHybridEmbedder(HybridEmbedderBase):
@@ -559,11 +527,17 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):
completion_tokens=completion_tokens,
)
- def embed(self, text: str, is_query: bool = False) -> EmbedResult:
- """Perform hybrid embedding on text
+ @property
+ def supports_multimodal(self) -> bool:
+ """Hybrid embeddings always use the multimodal endpoint."""
+ return True
+
+ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:
+ """Perform hybrid embedding on text or multimodal content
Args:
- text: Input text
+ content: Input text, or a list of multimodal content parts such as
+ ``[{"type": "text", "text": "..."}, {"type": "image_url", "image_url": {"url": "..."}}]``
is_query: Flag to indicate if this is a query embedding
Returns:
@@ -577,7 +551,7 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):
# Always use multimodal for hybrid to get both
response = self.client.multimodal_embeddings.create(
- input=[{"type": "text", "text": text}],
+ input=to_multimodal_input(content),
model=self.model_name,
sparse_embedding={"type": "enabled"},
)
@@ -604,12 +578,14 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):
lambda: volcenginesdkarkruntime.AsyncArk(**self._ark_kwargs)
)
- async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:
+ async def embed_async(
+ self, content: "EmbeddingInput", is_query: bool = False
+ ) -> EmbedResult:
client = self._get_async_client()
async def _embed_call() -> EmbedResult:
response = await client.multimodal_embeddings.create(
- input=[{"type": "text", "text": text}],
+ input=to_multimodal_input(content),
model=self.model_name,
sparse_embedding={"type": "enabled"},
)
@@ -631,11 +607,13 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):
except Exception as e:
raise RuntimeError(f"Volcengine hybrid embedding failed: {str(e)}") from e
- def embed_batch(self, texts: List[str], is_query: bool = False) -> List[EmbedResult]:
+ def embed_batch(
+ self, contents: List["EmbeddingInput"], is_query: bool = False
+ ) -> List[EmbedResult]:
"""Batch hybrid embedding
Args:
- texts: List of texts
+ contents: List of texts or multimodal content parts
is_query: Flag to indicate if these are query embeddings
Returns:
@@ -644,42 +622,23 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):
Raises:
RuntimeError: When API call fails
"""
- if not texts:
+ if not contents:
return []
- return [self.embed(text, is_query=is_query) for text in texts]
+ return [self.embed(content, is_query=is_query) for content in contents]
async def embed_batch_async(
- self, texts: List[str], is_query: bool = False
+ self, contents: List["EmbeddingInput"], is_query: bool = False
) -> List[EmbedResult]:
- if not texts:
+ if not contents:
return []
- client = self._get_async_client()
-
- async def _call() -> List[EmbedResult]:
- response = await client.multimodal_embeddings.create(
- input=[{"type": "text", "text": text} for text in texts],
- model=self.model_name,
- sparse_embedding={"type": "enabled"},
- )
- self._update_telemetry_token_usage(response)
- data = response.data
- return [
- EmbedResult(
- dense_vector=truncate_and_normalize(item.embedding, self.dimension),
- sparse_vector=process_sparse_embedding(getattr(item, "sparse_embedding", None)),
- )
- for item in data
- ]
-
- try:
- return await self._run_with_async_retry(
- _call,
- logger=logger,
- operation_name="Volcengine async hybrid batch embedding",
+ # The multimodal endpoint embeds a single content per call; run the items
+ # concurrently while sharing the async client.
+ return list(
+ await asyncio.gather(
+ *(self.embed_async(content, is_query=is_query) for content in contents)
)
- except Exception as e:
- raise RuntimeError(f"Volcengine hybrid embedding failed: {str(e)}") from e
+ )
def get_dimension(self) -> int:
return self._dimension
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index 5edd6cc54..a322b9223 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -444,9 +444,9 @@ class TextEmbeddingHandler(DequeueHandlerBase):
report_success = True
return None
- # Only process string messages
- if not isinstance(embedding_msg.message, str):
- logger.debug(f"Skipping non-string message type: {type(embedding_msg.message)}")
+ # Process string (text) or list (multimodal) messages
+ if not isinstance(embedding_msg.message, (str, list)):
+ logger.debug(f"Skipping unsupported message type: {type(embedding_msg.message)}")
self._merge_request_stats(embedding_msg.telemetry_id, processed=1)
self._record_request_success(embedding_msg)
report_success = True
diff --git a/openviking/storage/queuefs/embedding_msg_converter.py b/openviking/storage/queuefs/embedding_msg_converter.py
index fc4523cb7..9cac93025 100644
--- a/openviking/storage/queuefs/embedding_msg_converter.py
+++ b/openviking/storage/queuefs/embedding_msg_converter.py
@@ -25,7 +25,8 @@ class EmbeddingMsgConverter:
Convert a Context object to EmbeddingMsg.
"""
vectorization_text = context.get_vectorization_text()
- if not vectorization_text:
+ vectorization_images = context.get_vectorization_images()
+ if not vectorization_text and not vectorization_images:
return None
context_data = context.to_dict()
@@ -67,8 +68,21 @@ class EmbeddingMsgConverter:
resolved_level = int(resolved_level.value)
context_data["level"] = int(resolved_level)
+ if vectorization_images:
+ # Multimodal message: combine text (if any) and image references into the
+ # multimodal embedding input format. Image-aware embedders consume this list;
+ # text-only embedders fall back to the text part.
+ parts = []
+ if vectorization_text:
+ parts.append({"type": "text", "text": vectorization_text})
+ for image_ref in vectorization_images:
+ parts.append({"type": "image_url", "image_url": {"url": image_ref}})
+ message = parts
+ else:
+ message = vectorization_text
+
embedding_msg = EmbeddingMsg(
- message=vectorization_text,
+ message=message,
context_data=context_data,
telemetry_id=get_current_telemetry().telemetry_id,
)
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index 44c808cbb..54b1f46ee 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -6,6 +6,7 @@ Embedding utilities for OpenViking.
Common logic for creating Context objects and enqueuing them to EmbeddingQueue.
"""
+import base64
import os
from datetime import datetime, timezone
from typing import Any, Dict, Optional
@@ -194,6 +195,42 @@ def get_resource_content_type(file_name: str) -> Optional[ResourceContentType]:
return None
+_IMAGE_MIME_TYPES = {
+ ".png": "image/png",
+ ".jpg": "image/jpeg",
+ ".jpeg": "image/jpeg",
+ ".gif": "image/gif",
+ ".bmp": "image/bmp",
+ ".svg": "image/svg+xml",
+ ".webp": "image/webp",
+}
+
+
+def _image_mime_type(file_name: str) -> str:
+ """Resolve the MIME type for an image file based on its extension."""
+ _, ext = os.path.splitext(file_name.lower())
+ return _IMAGE_MIME_TYPES.get(ext, "image/png")
+
+
+async def _build_image_data_uri(
+ file_path: str,
+ file_name: str,
+ viking_fs,
+ ctx: Optional[RequestContext],
+) -> Optional[str]:
+ """Read an image file and encode it as a base64 ``data:`` URI.
+
+ Returns None if the image cannot be read.
+ """
+ try:
+ content = await viking_fs.read_file_bytes(file_path, ctx=ctx)
+ encoded = base64.b64encode(content).decode("ascii")
+ return f"data:{_image_mime_type(file_name)};base64,{encoded}"
+ except Exception as e:
+ logger.warning(f"Failed to read image for multimodal vectorization {file_path}: {e}")
+ return None
+
+
async def vectorize_directory_meta(
uri: str,
abstract: str,
@@ -353,6 +390,7 @@ async def vectorize_file(
embedding_cfg = get_openviking_config().embedding
configured_text_source = getattr(embedding_cfg, "text_source", "content_only")
effective_text_source = "summary_only" if use_summary else configured_text_source
+ image_vectorization = getattr(embedding_cfg, "image_vectorization", "summary_only")
if content_type is None:
# Unsupported file type: fall back to summary if available
@@ -387,6 +425,23 @@ async def vectorize_file(
f"No summary available for {file_path}, skipping vectorization"
)
return
+ elif content_type == ResourceContentType.IMAGE and image_vectorization in {
+ "image_only",
+ "image_and_summary",
+ }:
+ # Multimodal: embed the image itself (optionally with its text summary).
+ image_uri = await _build_image_data_uri(file_path, file_name, viking_fs, ctx)
+ if image_uri:
+ text = summary if image_vectorization == "image_and_summary" else ""
+ context.set_vectorize(Vectorize(text=text, images=[image_uri]))
+ elif summary:
+ # Could not load image; fall back to summary text.
+ context.set_vectorize(Vectorize(text=summary))
+ else:
+ logger.debug(
+ f"Skipping image {file_path} (image unreadable and no summary available)"
+ )
+ return
elif summary:
# For non-text files, use summary
context.set_vectorize(Vectorize(text=summary))
diff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.py
index afb531927..5f06e196d 100644
--- a/openviking_cli/utils/config/embedding_config.py
+++ b/openviking_cli/utils/config/embedding_config.py
@@ -381,6 +381,17 @@ class EmbeddingConfig(BaseModel):
default="content_only",
description="Text source for file vectorization: summary_first|summary_only|content_only",
)
+ image_vectorization: str = Field(
+ default="summary_only",
+ description=(
+ "How IMAGE files are vectorized: "
+ "'summary_only' embeds only the generated text summary (text embedding); "
+ "'image_only' sends the image itself to a multimodal embedder; "
+ "'image_and_summary' sends both the image and its text summary together. "
+ "'image_only' and 'image_and_summary' require a multimodal-capable embedder "
+ "(e.g. Volcengine with input='multimodal')."
+ ),
+ )
max_input_tokens: int = Field(
default=4096,
ge=100,
@@ -416,6 +427,11 @@ class EmbeddingConfig(BaseModel):
raise ValueError(
"embedding.text_source must be one of: summary_first, summary_only, content_only"
)
+ if self.image_vectorization not in {"summary_only", "image_only", "image_and_summary"}:
+ raise ValueError(
+ "embedding.image_vectorization must be one of: "
+ "summary_only, image_only, image_and_summary"
+ )
return self
def _create_embedder(