Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat: 计算图片embed向量时支持图文embed,通过配置参数image_vectorization决定embed模式(summary_only,image_only,image_and_summary) (#2460)
Haoyu Zhangc675b1732ebb
diff --git a/openviking/core/context.py b/openviking/core/context.pyindex ae59b8243..8fb92e246 100644--- a/openviking/core/context.py+++ b/openviking/core/context.py@@ -41,12 +41,14 @@ class ContextLevel(int, Enum):class Vectorize:text: str = ""- # image: str = ""+ # images: list of image references (data URIs or URLs) for multimodal embedding+ images: List[str] = []# video: str = ""# audio: str = ""- def __init__(self, text: str = ""):+ def __init__(self, text: str = "", images: Optional[List[str]] = None):self.text = text+ self.images = list(images) if images else []class Context:@@ -144,9 +146,12 @@ class Context:def get_vectorization_text(self) -> str:"""Get text for vectorization."""- # todo: multi-modal supportreturn self.vectorize.text+ def get_vectorization_images(self) -> List[str]:+ """Get image references (data URIs or URLs) for multimodal vectorization."""+ return self.vectorize.images+def update_activity(self):"""Update activity statistics."""self.active_count += 1diff --git a/openviking/models/embedder/base.py b/openviking/models/embedder/base.pyindex 30f42d12b..7fa429e89 100644--- a/openviking/models/embedder/base.py+++ b/openviking/models/embedder/base.py@@ -8,7 +8,7 @@ import weakreffrom abc import ABC, abstractmethodfrom dataclasses import dataclassfrom threading import Lock-from typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVar+from typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVar, Unionfrom openviking.telemetry import get_current_telemetryfrom openviking.utils.embedding_input import (@@ -21,6 +21,10 @@ from openviking_cli.utils import get_loggerT = TypeVar("T")logger = get_logger(__name__)+# A multimodal embedding input is a list of content parts, e.g.+# [{"type": "text", "text": "..."}, {"type": "image_url", "image_url": {"url": "..."}}]+EmbeddingInput = Union[str, List[Dict[str, Any]]]+_token_tracker_instance = None_ASYNC_EMBED_SEMAPHORES: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, Dict[int, asyncio.Semaphore]]" = weakref.WeakKeyDictionary()@@ -49,14 +53,37 @@ def _get_token_tracker():return _token_tracker_instance+def extract_text_from_content(content: "EmbeddingInput") -> str:+ """Extract and join the text parts from a multimodal input.++ ``content`` may be a plain string (returned as-is) or a list of content+ parts such as ``[{"type": "text", "text": "..."}, {"type": "image_url", ...}]``.+ Non-text parts (e.g. images) are ignored.+ """+ if isinstance(content, str):+ return content+ text_parts = [+ part.get("text", "")+ for part in content+ if isinstance(part, dict) and part.get("type") == "text"+ ]+ return "\n".join(p for p in text_parts if p)++async def embed_compat(- embedder: "EmbedderBase", text: str, *, is_query: bool = False+ embedder: "EmbedderBase", content: "EmbeddingInput", *, is_query: bool = False) -> "EmbedResult":- """Prepare input, then call the embedder's async-compatible entrypoint."""+ """Prepare input, then call the embedder's async-compatible entrypoint.++ Accepts either a plain text string or a multimodal input (a list of content+ parts). ``prepare_embedding_input`` downgrades multimodal inputs to text for+ embedders that do not support images, so all embedders can be called the same+ way.+ """from openviking.telemetry import bind_telemetry_stagestage = "embed_query" if is_query else "embed_resource"- embedding_input = embedder.prepare_embedding_input(text)+ embedding_input = embedder.prepare_embedding_input(content)with bind_telemetry_stage(stage):return await embedder.embed_async(embedding_input, is_query=is_query)@@ -147,24 +174,56 @@ class EmbedderBase(ABC):self._token_tracker = _get_token_tracker()self._active_call_started_at: float | None = None- def prepare_embedding_input(self, text: str) -> str:- """Apply this embedder's input guard before provider calls."""- if self.max_input_tokens is None:- return text- return truncate_embedding_input(text, self.max_input_tokens)+ def prepare_embedding_input(self, content: "EmbeddingInput") -> "EmbeddingInput":+ """Apply this embedder's input guard before provider calls.- def prepare_embedding_inputs(self, texts: List[str]) -> List[str]:+ Plain text is truncated to ``max_input_tokens``. For embedders that+ support images, multimodal inputs (a list of content parts) are kept as a+ list. Embedders that do not support multimodal input get the+ text parts extracted, so image parts are safely dropped.+ """+ if isinstance(content, list) and self.supports_multimodal:+ if self.max_input_tokens is None:+ return content+ truncated_parts = []+ for part in content:+ if isinstance(part, dict) and part.get("type") == "text":+ part = {+ **part,+ "text": truncate_embedding_input(+ part.get("text", ""), self.max_input_tokens+ ),+ }+ truncated_parts.append(part)+ return truncated_parts+ else:+ content = extract_text_from_content(content)+ if self.max_input_tokens is None:+ return content+ return truncate_embedding_input(content, self.max_input_tokens)++ def prepare_embedding_inputs(+ self, contents: List["EmbeddingInput"]+ ) -> List["EmbeddingInput"]:"""Apply this embedder's input guard to a batch."""- if self.max_input_tokens is None:- return texts- return [self.prepare_embedding_input(text) for text in texts]+ return [self.prepare_embedding_input(content) for content in contents]++ @property+ def supports_multimodal(self) -> bool:+ """Whether this embedder can consume multimodal (image) inputs directly.++ Text-only embedders return False so that multimodal inputs are downgraded+ to their text parts before being embedded.+ """+ return False@abstractmethod- def embed(self, text: str, is_query: bool = False) -> EmbedResult:- """Embed single text+ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:+ """Embed text or multimodal content.Args:- text: Input text+ content: Input text, or a list of multimodal content parts such as+ ``[{"type": "text", "text": "..."}, {"type": "image_url", "image_url": {"url": "..."}}]``is_query: Flag to indicate if this is a query embeddingReturns:@@ -172,17 +231,19 @@ class EmbedderBase(ABC):"""pass- def embed_batch(self, texts: List[str], is_query: bool = False) -> List[EmbedResult]:+ def embed_batch(+ self, contents: List["EmbeddingInput"], is_query: bool = False+ ) -> List[EmbedResult]:"""Batch embedding (default implementation loops, subclasses can override for optimization)Args:- texts: List of texts+ contents: List of texts or multimodal content partsis_query: Flag to indicate if these are query embeddingsReturns:List[EmbedResult]: List of embedding results"""- return [self.embed(text, is_query=is_query) for text in texts]+ return [self.embed(content, is_query=is_query) for content in contents]def embed_query(self, text: str) -> EmbedResult:"""Embed query text with explicit retrieval-side semantics."""@@ -200,22 +261,24 @@ class EmbedderBase(ABC):"""Batch embed document texts."""return self.embed_batch(texts, is_query=False)- async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:- """Async embed single text.+ async def embed_async(+ self, content: "EmbeddingInput", is_query: bool = False+ ) -> EmbedResult:+ """Async embed text or multimodal content.Subclasses should override this with a non-blocking implementation.The default implementation preserves compatibility for test doubles andthird-party embedders that only implement the sync interface."""- return self.embed(text, is_query=is_query)+ return self.embed(content, is_query=is_query)async def embed_batch_async(- self, texts: List[str], is_query: bool = False+ self, contents: List["EmbeddingInput"], is_query: bool = False) -> List[EmbedResult]:"""Async batch embedding."""results: List[EmbedResult] = []- for text in texts:- results.append(await self.embed_async(text, is_query=is_query))+ for content in contents:+ results.append(await self.embed_async(content, is_query=is_query))return resultsasync def embed_query_async(self, text: str) -> EmbedResult:@@ -407,11 +470,11 @@ class DenseEmbedderBase(EmbedderBase):"""@abstractmethod- def embed(self, text: str, is_query: bool = False) -> EmbedResult:- """Perform dense embedding on text+ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:+ """Perform dense embedding on text or multimodal contentArgs:- text: Input text+ content: Input text, or a list of multimodal content partsis_query: Flag to indicate if this is a query embeddingReturns:@@ -440,11 +503,11 @@ class SparseEmbedderBase(EmbedderBase):"""@abstractmethod- def embed(self, text: str, is_query: bool = False) -> EmbedResult:- """Perform sparse embedding on text+ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:+ """Perform sparse embedding on text or multimodal contentArgs:- text: Input text+ content: Input text, or a list of multimodal content partsis_query: Flag to indicate if this is a query embeddingReturns:@@ -469,11 +532,11 @@ class HybridEmbedderBase(EmbedderBase):"""@abstractmethod- def embed(self, text: str, is_query: bool = False) -> EmbedResult:- """Perform hybrid embedding on text+ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:+ """Perform hybrid embedding on text or multimodal contentArgs:- text: Input text+ content: Input text, or a list of multimodal content partsis_query: Flag to indicate if this is a query embeddingReturns:@@ -520,10 +583,18 @@ class CompositeHybridEmbedder(HybridEmbedderBase):self.dense_embedder = dense_embedderself.sparse_embedder = sparse_embedder- def embed(self, text: str, is_query: bool = False) -> EmbedResult:+ @property+ def supports_multimodal(self) -> bool:+ """Supports multimodal input only if both sub-embedders do."""+ return (+ self.dense_embedder.supports_multimodal+ and self.sparse_embedder.supports_multimodal+ )++ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:"""Combine results from both embedders"""- dense_input = self.dense_embedder.prepare_embedding_input(text)- sparse_input = self.sparse_embedder.prepare_embedding_input(text)+ dense_input = self.dense_embedder.prepare_embedding_input(content)+ sparse_input = self.sparse_embedder.prepare_embedding_input(content)dense_res = self.dense_embedder.embed(dense_input, is_query=is_query)sparse_res = self.sparse_embedder.embed(sparse_input, is_query=is_query)@@ -531,10 +602,12 @@ class CompositeHybridEmbedder(HybridEmbedderBase):dense_vector=dense_res.dense_vector, sparse_vector=sparse_res.sparse_vector)- def embed_batch(self, texts: List[str], is_query: bool = False) -> List[EmbedResult]:+ def embed_batch(+ self, contents: List["EmbeddingInput"], is_query: bool = False+ ) -> List[EmbedResult]:"""Combine batch results"""- dense_inputs = self.dense_embedder.prepare_embedding_inputs(texts)- sparse_inputs = self.sparse_embedder.prepare_embedding_inputs(texts)+ dense_inputs = self.dense_embedder.prepare_embedding_inputs(contents)+ sparse_inputs = self.sparse_embedder.prepare_embedding_inputs(contents)dense_results = self.dense_embedder.embed_batch(dense_inputs, is_query=is_query)sparse_results = self.sparse_embedder.embed_batch(sparse_inputs, is_query=is_query)@@ -543,9 +616,11 @@ class CompositeHybridEmbedder(HybridEmbedderBase):for d, s in zip(dense_results, sparse_results, strict=True)]- async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:- dense_input = self.dense_embedder.prepare_embedding_input(text)- sparse_input = self.sparse_embedder.prepare_embedding_input(text)+ async def embed_async(+ self, content: "EmbeddingInput", is_query: bool = False+ ) -> EmbedResult:+ dense_input = self.dense_embedder.prepare_embedding_input(content)+ sparse_input = self.sparse_embedder.prepare_embedding_input(content)dense_res, sparse_res = await asyncio.gather(self.dense_embedder.embed_async(dense_input, is_query=is_query),self.sparse_embedder.embed_async(sparse_input, is_query=is_query),@@ -555,10 +630,10 @@ class CompositeHybridEmbedder(HybridEmbedderBase):)async def embed_batch_async(- self, texts: List[str], is_query: bool = False+ self, contents: List["EmbeddingInput"], is_query: bool = False) -> List[EmbedResult]:- dense_inputs = self.dense_embedder.prepare_embedding_inputs(texts)- sparse_inputs = self.sparse_embedder.prepare_embedding_inputs(texts)+ dense_inputs = self.dense_embedder.prepare_embedding_inputs(contents)+ sparse_inputs = self.sparse_embedder.prepare_embedding_inputs(contents)dense_results, sparse_results = await asyncio.gather(self.dense_embedder.embed_batch_async(dense_inputs, is_query=is_query),self.sparse_embedder.embed_batch_async(sparse_inputs, is_query=is_query),diff --git a/openviking/models/embedder/volcengine_embedders.py b/openviking/models/embedder/volcengine_embedders.pyindex cd4ce28bd..6d609856a 100644--- a/openviking/models/embedder/volcengine_embedders.py+++ b/openviking/models/embedder/volcengine_embedders.py@@ -2,15 +2,18 @@# SPDX-License-Identifier: AGPL-3.0"""Volcengine Embedder Implementation"""+import asynciofrom typing import Any, Dict, List, Optionalimport volcenginesdkarkruntimefrom openviking.models.embedder.base import (DenseEmbedderBase,+ EmbeddingInput,EmbedResult,HybridEmbedderBase,SparseEmbedderBase,+ extract_text_from_content,truncate_and_normalize,)from openviking.telemetry import get_current_telemetry@@ -18,6 +21,18 @@ from openviking.utils.async_client_cache import LoopScopedAsyncClientCachefrom openviking_cli.utils.logger import default_logger as logger+def to_multimodal_input(content: "EmbeddingInput") -> List[Dict[str, Any]]:+ """Normalize an embedding input into the content-parts payload the Volcengine+ multimodal embeddings API expects.++ A plain string is wrapped as a single text part. A list of content parts+ (text and/or ``image_url``) is passed through unchanged.+ """+ if isinstance(content, str):+ return [{"type": "text", "text": content}]+ return list(content)++def process_sparse_embedding(sparse_data: Any) -> Dict[str, float]:"""Process sparse embedding data from SDK response"""if not sparse_data:@@ -113,6 +128,11 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):except Exception:return 2048 # Default dimension+ @property+ def supports_multimodal(self) -> bool:+ """Multimodal inputs are supported when using the multimodal endpoint."""+ return self.input_type == "multimodal"+def _update_telemetry_token_usage(self, response) -> None:usage = getattr(response, "usage", None)if not usage:@@ -142,11 +162,12 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):completion_tokens=completion_tokens,)- def embed(self, text: str, is_query: bool = False) -> EmbedResult:- """Perform dense embedding on text+ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:+ """Perform dense embedding on text or multimodal contentArgs:- text: Input text+ content: Input text, or a list of multimodal content parts such as+ ``[{"type": "text", "text": "..."}, {"type": "image_url", "image_url": {"url": "..."}}]``is_query: Flag to indicate if this is a query embeddingReturns:@@ -160,12 +181,13 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):if self.input_type == "multimodal":# Use multimodal embeddings APIresponse = self.client.multimodal_embeddings.create(- input=[{"type": "text", "text": text}], model=self.model_name+ input=to_multimodal_input(content), model=self.model_name)self._update_telemetry_token_usage(response)vector = response.data.embeddingelse:- # Use text embeddings API+ # Use text embeddings API (text-only)+ text = extract_text_from_content(content)response = self.client.embeddings.create(input=text, model=self.model_name)self._update_telemetry_token_usage(response)vector = response.data[0].embedding@@ -187,17 +209,20 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):lambda: volcenginesdkarkruntime.AsyncArk(**self._ark_kwargs))- async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:+ async def embed_async(+ self, content: "EmbeddingInput", is_query: bool = False+ ) -> EmbedResult:client = self._get_async_client()async def _embed_call() -> EmbedResult:if self.input_type == "multimodal":response = await client.multimodal_embeddings.create(- input=[{"type": "text", "text": text}], model=self.model_name+ input=to_multimodal_input(content), model=self.model_name)self._update_telemetry_token_usage(response)vector = response.data.embeddingelse:+ text = extract_text_from_content(content)response = await client.embeddings.create(input=text, model=self.model_name)self._update_telemetry_token_usage(response)vector = response.data[0].embedding@@ -213,11 +238,13 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):except Exception as e:raise RuntimeError(f"Volcengine embedding failed: {str(e)}") from e- def embed_batch(self, texts: List[str], is_query: bool = False) -> List[EmbedResult]:+ def embed_batch(+ self, contents: List["EmbeddingInput"], is_query: bool = False+ ) -> List[EmbedResult]:"""Batch embeddingArgs:- texts: List of texts+ contents: List of texts or multimodal content partsis_query: Flag to indicate if these are query embeddingsReturns:@@ -226,76 +253,22 @@ class VolcengineDenseEmbedder(DenseEmbedderBase):Raises:RuntimeError: When API call fails"""- if not texts:+ if not contents:return []-- def _call() -> List[EmbedResult]:- if self.input_type == "multimodal":- multimodal_inputs = [{"type": "text", "text": text} for text in texts]- response = self.client.multimodal_embeddings.create(- input=multimodal_inputs, model=self.model_name- )- self._update_telemetry_token_usage(response)- data = response.data- else:- response = self.client.embeddings.create(input=texts, model=self.model_name)- self._update_telemetry_token_usage(response)- data = response.data-- return [- EmbedResult(dense_vector=truncate_and_normalize(item.embedding, self.dimension))- for item in data- ]-- try:- return self._run_with_retry(- _call,- logger=logger,- operation_name="Volcengine batch embedding",- )- except Exception as e:- logger.error(- f"Volcengine batch embedding failed, texts length: {len(texts)}, input_type: {self.input_type}, model_name: {self.model_name}"- )- raise RuntimeError(f"Volcengine batch embedding failed: {str(e)}") from e+ return [self.embed(content, is_query=is_query) for content in contents]async def embed_batch_async(- self, texts: List[str], is_query: bool = False+ self, contents: List["EmbeddingInput"], is_query: bool = False) -> List[EmbedResult]:- if not texts:+ if not contents:return []- client = self._get_async_client()-- async def _call() -> List[EmbedResult]:- if self.input_type == "multimodal":- multimodal_inputs = [{"type": "text", "text": text} for text in texts]- response = await client.multimodal_embeddings.create(- input=multimodal_inputs, model=self.model_name- )- self._update_telemetry_token_usage(response)- data = response.data- else:- response = await client.embeddings.create(input=texts, model=self.model_name)- self._update_telemetry_token_usage(response)- data = response.data-- return [- EmbedResult(dense_vector=truncate_and_normalize(item.embedding, self.dimension))- for item in data- ]-- try:- return await self._run_with_async_retry(- _call,- logger=logger,- operation_name="Volcengine async batch embedding",- )- except Exception as e:- logger.error(- f"Volcengine async batch embedding failed, texts length: {len(texts)}, input_type: {self.input_type}, model_name: {self.model_name}"+ # Embed each content individually and run them concurrently.+ return list(+ await asyncio.gather(+ *(self.embed_async(content, is_query=is_query) for content in contents))- raise RuntimeError(f"Volcengine batch embedding failed: {str(e)}") from e+ )def get_dimension(self) -> int:return self._dimension@@ -312,6 +285,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):model_name: str,api_key: Optional[str] = None,api_base: Optional[str] = None,+ input_type: str = "multimodal",config: Optional[Dict[str, Any]] = None,):"""Initialize Volcengine Sparse Embedder@@ -320,6 +294,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):model_name: Volcengine model nameapi_key: API key for authenticationapi_base: API base URL+ input_type: Input type - "text" or "multimodal" (default: "multimodal")config: Additional configuration dictRaises:@@ -330,6 +305,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):self.api_key = api_keyself.api_base = api_base+ self.input_type = input_typeif not self.api_key:raise ValueError("api_key is required")@@ -370,11 +346,12 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):completion_tokens=completion_tokens,)- def embed(self, text: str, is_query: bool = False) -> EmbedResult:- """Perform sparse embedding on text+ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:+ """Perform sparse embedding on text or multimodal contentArgs:- text: Input text+ content: Input text, or a list of multimodal content parts such as+ ``[{"type": "text", "text": "..."}, {"type": "image_url", "image_url": {"url": "..."}}]``is_query: Flag to indicate if this is a query embeddingReturns:@@ -387,7 +364,7 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):def _embed_call():# Must use multimodal endpoint for sparseresponse = self.client.multimodal_embeddings.create(- input=[{"type": "text", "text": text}],+ input=to_multimodal_input(content),model=self.model_name,sparse_embedding={"type": "enabled"},)@@ -410,12 +387,14 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):lambda: volcenginesdkarkruntime.AsyncArk(**self._ark_kwargs))- async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:+ async def embed_async(+ self, content: "EmbeddingInput", is_query: bool = False+ ) -> EmbedResult:client = self._get_async_client()async def _embed_call() -> EmbedResult:response = await client.multimodal_embeddings.create(- input=[{"type": "text", "text": text}],+ input=to_multimodal_input(content),model=self.model_name,sparse_embedding={"type": "enabled"},)@@ -433,11 +412,13 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):except Exception as e:raise RuntimeError(f"Volcengine sparse embedding failed: {str(e)}") from e- def embed_batch(self, texts: List[str], is_query: bool = False) -> List[EmbedResult]:+ def embed_batch(+ self, contents: List["EmbeddingInput"], is_query: bool = False+ ) -> List[EmbedResult]:"""Batch sparse embeddingArgs:- texts: List of texts+ contents: List of texts or multimodal content partsis_query: Flag to indicate if these are query embeddingsReturns:@@ -446,41 +427,28 @@ class VolcengineSparseEmbedder(SparseEmbedderBase):Raises:RuntimeError: When API call fails"""- if not texts:+ if not contents:return []- return [self.embed(text) for text in texts]+ return [self.embed(content, is_query=is_query) for content in contents]async def embed_batch_async(- self, texts: List[str], is_query: bool = False+ self, contents: List["EmbeddingInput"], is_query: bool = False) -> List[EmbedResult]:- if not texts:+ if not contents:return []- client = self._get_async_client()-- async def _call() -> List[EmbedResult]:- response = await client.multimodal_embeddings.create(- input=[{"type": "text", "text": text} for text in texts],- model=self.model_name,- sparse_embedding={"type": "enabled"},+ # The multimodal endpoint embeds a single content per call; run the items+ # concurrently while sharing the async client.+ return list(+ await asyncio.gather(+ *(self.embed_async(content, is_query=is_query) for content in contents))- self._update_telemetry_token_usage(response)- data = response.data- return [- EmbedResult(- sparse_vector=process_sparse_embedding(getattr(item, "sparse_embedding", None))- )- for item in data- ]+ )- try:- return await self._run_with_async_retry(- _call,- logger=logger,- operation_name="Volcengine async sparse batch embedding",- )- except Exception as e:- raise RuntimeError(f"Volcengine sparse embedding failed: {str(e)}") from e+ @property+ def supports_multimodal(self) -> bool:+ """Multimodal inputs are supported when using the multimodal endpoint."""+ return self.input_type == "multimodal"class VolcengineHybridEmbedder(HybridEmbedderBase):@@ -559,11 +527,17 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):completion_tokens=completion_tokens,)- def embed(self, text: str, is_query: bool = False) -> EmbedResult:- """Perform hybrid embedding on text+ @property+ def supports_multimodal(self) -> bool:+ """Hybrid embeddings always use the multimodal endpoint."""+ return True++ def embed(self, content: "EmbeddingInput", is_query: bool = False) -> EmbedResult:+ """Perform hybrid embedding on text or multimodal contentArgs:- text: Input text+ content: Input text, or a list of multimodal content parts such as+ ``[{"type": "text", "text": "..."}, {"type": "image_url", "image_url": {"url": "..."}}]``is_query: Flag to indicate if this is a query embeddingReturns:@@ -577,7 +551,7 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):# Always use multimodal for hybrid to get bothresponse = self.client.multimodal_embeddings.create(- input=[{"type": "text", "text": text}],+ input=to_multimodal_input(content),model=self.model_name,sparse_embedding={"type": "enabled"},)@@ -604,12 +578,14 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):lambda: volcenginesdkarkruntime.AsyncArk(**self._ark_kwargs))- async def embed_async(self, text: str, is_query: bool = False) -> EmbedResult:+ async def embed_async(+ self, content: "EmbeddingInput", is_query: bool = False+ ) -> EmbedResult:client = self._get_async_client()async def _embed_call() -> EmbedResult:response = await client.multimodal_embeddings.create(- input=[{"type": "text", "text": text}],+ input=to_multimodal_input(content),model=self.model_name,sparse_embedding={"type": "enabled"},)@@ -631,11 +607,13 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):except Exception as e:raise RuntimeError(f"Volcengine hybrid embedding failed: {str(e)}") from e- def embed_batch(self, texts: List[str], is_query: bool = False) -> List[EmbedResult]:+ def embed_batch(+ self, contents: List["EmbeddingInput"], is_query: bool = False+ ) -> List[EmbedResult]:"""Batch hybrid embeddingArgs:- texts: List of texts+ contents: List of texts or multimodal content partsis_query: Flag to indicate if these are query embeddingsReturns:@@ -644,42 +622,23 @@ class VolcengineHybridEmbedder(HybridEmbedderBase):Raises:RuntimeError: When API call fails"""- if not texts:+ if not contents:return []- return [self.embed(text, is_query=is_query) for text in texts]+ return [self.embed(content, is_query=is_query) for content in contents]async def embed_batch_async(- self, texts: List[str], is_query: bool = False+ self, contents: List["EmbeddingInput"], is_query: bool = False) -> List[EmbedResult]:- if not texts:+ if not contents:return []- client = self._get_async_client()-- async def _call() -> List[EmbedResult]:- response = await client.multimodal_embeddings.create(- input=[{"type": "text", "text": text} for text in texts],- model=self.model_name,- sparse_embedding={"type": "enabled"},- )- self._update_telemetry_token_usage(response)- data = response.data- return [- EmbedResult(- dense_vector=truncate_and_normalize(item.embedding, self.dimension),- sparse_vector=process_sparse_embedding(getattr(item, "sparse_embedding", None)),- )- for item in data- ]-- try:- return await self._run_with_async_retry(- _call,- logger=logger,- operation_name="Volcengine async hybrid batch embedding",+ # The multimodal endpoint embeds a single content per call; run the items+ # concurrently while sharing the async client.+ return list(+ await asyncio.gather(+ *(self.embed_async(content, is_query=is_query) for content in contents))- except Exception as e:- raise RuntimeError(f"Volcengine hybrid embedding failed: {str(e)}") from e+ )def get_dimension(self) -> int:return self._dimensiondiff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex 5edd6cc54..a322b9223 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -444,9 +444,9 @@ class TextEmbeddingHandler(DequeueHandlerBase):report_success = Truereturn None- # Only process string messages- if not isinstance(embedding_msg.message, str):- logger.debug(f"Skipping non-string message type: {type(embedding_msg.message)}")+ # Process string (text) or list (multimodal) messages+ if not isinstance(embedding_msg.message, (str, list)):+ logger.debug(f"Skipping unsupported message type: {type(embedding_msg.message)}")self._merge_request_stats(embedding_msg.telemetry_id, processed=1)self._record_request_success(embedding_msg)report_success = Truediff --git a/openviking/storage/queuefs/embedding_msg_converter.py b/openviking/storage/queuefs/embedding_msg_converter.pyindex fc4523cb7..9cac93025 100644--- a/openviking/storage/queuefs/embedding_msg_converter.py+++ b/openviking/storage/queuefs/embedding_msg_converter.py@@ -25,7 +25,8 @@ class EmbeddingMsgConverter:Convert a Context object to EmbeddingMsg."""vectorization_text = context.get_vectorization_text()- if not vectorization_text:+ vectorization_images = context.get_vectorization_images()+ if not vectorization_text and not vectorization_images:return Nonecontext_data = context.to_dict()@@ -67,8 +68,21 @@ class EmbeddingMsgConverter:resolved_level = int(resolved_level.value)context_data["level"] = int(resolved_level)+ if vectorization_images:+ # Multimodal message: combine text (if any) and image references into the+ # multimodal embedding input format. Image-aware embedders consume this list;+ # text-only embedders fall back to the text part.+ parts = []+ if vectorization_text:+ parts.append({"type": "text", "text": vectorization_text})+ for image_ref in vectorization_images:+ parts.append({"type": "image_url", "image_url": {"url": image_ref}})+ message = parts+ else:+ message = vectorization_text+embedding_msg = EmbeddingMsg(- message=vectorization_text,+ message=message,context_data=context_data,telemetry_id=get_current_telemetry().telemetry_id,)diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.pyindex 44c808cbb..54b1f46ee 100644--- a/openviking/utils/embedding_utils.py+++ b/openviking/utils/embedding_utils.py@@ -6,6 +6,7 @@ Embedding utilities for OpenViking.Common logic for creating Context objects and enqueuing them to EmbeddingQueue."""+import base64import osfrom datetime import datetime, timezonefrom typing import Any, Dict, Optional@@ -194,6 +195,42 @@ def get_resource_content_type(file_name: str) -> Optional[ResourceContentType]:return None+_IMAGE_MIME_TYPES = {+ ".png": "image/png",+ ".jpg": "image/jpeg",+ ".jpeg": "image/jpeg",+ ".gif": "image/gif",+ ".bmp": "image/bmp",+ ".svg": "image/svg+xml",+ ".webp": "image/webp",+}+++def _image_mime_type(file_name: str) -> str:+ """Resolve the MIME type for an image file based on its extension."""+ _, ext = os.path.splitext(file_name.lower())+ return _IMAGE_MIME_TYPES.get(ext, "image/png")+++async def _build_image_data_uri(+ file_path: str,+ file_name: str,+ viking_fs,+ ctx: Optional[RequestContext],+) -> Optional[str]:+ """Read an image file and encode it as a base64 ``data:`` URI.++ Returns None if the image cannot be read.+ """+ try:+ content = await viking_fs.read_file_bytes(file_path, ctx=ctx)+ encoded = base64.b64encode(content).decode("ascii")+ return f"data:{_image_mime_type(file_name)};base64,{encoded}"+ except Exception as e:+ logger.warning(f"Failed to read image for multimodal vectorization {file_path}: {e}")+ return None++async def vectorize_directory_meta(uri: str,abstract: str,@@ -353,6 +390,7 @@ async def vectorize_file(embedding_cfg = get_openviking_config().embeddingconfigured_text_source = getattr(embedding_cfg, "text_source", "content_only")effective_text_source = "summary_only" if use_summary else configured_text_source+ image_vectorization = getattr(embedding_cfg, "image_vectorization", "summary_only")if content_type is None:# Unsupported file type: fall back to summary if available@@ -387,6 +425,23 @@ async def vectorize_file(f"No summary available for {file_path}, skipping vectorization")return+ elif content_type == ResourceContentType.IMAGE and image_vectorization in {+ "image_only",+ "image_and_summary",+ }:+ # Multimodal: embed the image itself (optionally with its text summary).+ image_uri = await _build_image_data_uri(file_path, file_name, viking_fs, ctx)+ if image_uri:+ text = summary if image_vectorization == "image_and_summary" else ""+ context.set_vectorize(Vectorize(text=text, images=[image_uri]))+ elif summary:+ # Could not load image; fall back to summary text.+ context.set_vectorize(Vectorize(text=summary))+ else:+ logger.debug(+ f"Skipping image {file_path} (image unreadable and no summary available)"+ )+ returnelif summary:# For non-text files, use summarycontext.set_vectorize(Vectorize(text=summary))diff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.pyindex afb531927..5f06e196d 100644--- a/openviking_cli/utils/config/embedding_config.py+++ b/openviking_cli/utils/config/embedding_config.py@@ -381,6 +381,17 @@ class EmbeddingConfig(BaseModel):default="content_only",description="Text source for file vectorization: summary_first|summary_only|content_only",)+ image_vectorization: str = Field(+ default="summary_only",+ description=(+ "How IMAGE files are vectorized: "+ "'summary_only' embeds only the generated text summary (text embedding); "+ "'image_only' sends the image itself to a multimodal embedder; "+ "'image_and_summary' sends both the image and its text summary together. "+ "'image_only' and 'image_and_summary' require a multimodal-capable embedder "+ "(e.g. Volcengine with input='multimodal')."+ ),+ )max_input_tokens: int = Field(default=4096,ge=100,@@ -416,6 +427,11 @@ class EmbeddingConfig(BaseModel):raise ValueError("embedding.text_source must be one of: summary_first, summary_only, content_only")+ if self.image_vectorization not in {"summary_only", "image_only", "image_and_summary"}:+ raise ValueError(+ "embedding.image_vectorization must be one of: "+ "summary_only, image_only, image_and_summary"+ )return selfdef _create_embedder(