Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

14.3 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""DashScope Embedder Implementation

Supports both text (via OpenAI-compatible endpoint) and multimodal (via native
DashScope REST API) embedding modes.
"""

from typing import Any, Dict, List, Optional

import httpx
import openai

from openviking.models.embedder.base import (
    DenseEmbedderBase,
    EmbeddingInput,
    EmbedResult,
    truncate_and_normalize,
)
from openviking.models.model_error import model_error_boundary
from openviking.telemetry import get_current_telemetry
from openviking.utils.async_client_cache import LoopScopedAsyncClientCache
from openviking_cli.utils.logger import default_logger as logger

_DASHSCOPE_DIMENSIONS: Dict[str, int] = {
    "text-embedding-v3": 1024,
    "text-embedding-v4": 1024,
    "qwen3-vl-embedding": 2560,
    "qwen2.5-vl-embedding": 1024,
}

_DASHSCOPE_PREFIX_DIMENSIONS: Dict[str, int] = {
    "tongyi-embedding-vision-plus": 1152,
    "tongyi-embedding-vision-flash": 768,
}


def get_dashscope_model_default_dimension(model_name: str) -> int:
    """Return default embedding dimension for a DashScope model.

    Lookup order:
    1. Exact match in _DASHSCOPE_DIMENSIONS
    2. Prefix match in _DASHSCOPE_PREFIX_DIMENSIONS
    3. Fallback: 1024
    """
    if model_name in _DASHSCOPE_DIMENSIONS:
        return _DASHSCOPE_DIMENSIONS[model_name]
    for prefix, dim in _DASHSCOPE_PREFIX_DIMENSIONS.items():
        if model_name.startswith(prefix):
            return dim
    return 1024


class DashScopeDenseEmbedder(DenseEmbedderBase):
    """DashScope Dense Embedder — dual-mode (text / multimodal).

    Text mode uses the OpenAI-compatible endpoint; multimodal mode uses the
    native DashScope REST API.
    """

    def __init__(
        self,
        model_name: str,
        api_key: Optional[str] = None,
        api_base: Optional[str] = None,
        dimension: Optional[int] = None,
        input_type: str = "multimodal",
        enable_fusion: Optional[bool] = None,
        res_level: Optional[int] = None,
        max_video_frames: Optional[int] = None,
        config: Optional[Dict[str, Any]] = None,
    ):
        super().__init__(model_name, config)
        self.provider = "dashscope"

        self.api_key = api_key
        self.api_base = (api_base or "https://dashscope.aliyuncs.com").rstrip("/")
        self._input_type = input_type
        self.enable_fusion = enable_fusion
        self.res_level = res_level
        self.max_video_frames = max_video_frames

        if not self.api_key:
            raise ValueError("api_key is required")

        self._dimension = dimension or get_dashscope_model_default_dimension(model_name)

        # --- sync clients ---
        # Text mode: OpenAI-compatible
        self._openai_client = openai.OpenAI(
            api_key=self.api_key,
            base_url=f"{self.api_base}/compatible-mode/v1",
            max_retries=0,
        )
        # Multimodal mode: httpx
        self._httpx_client = httpx.Client(
            headers={"Authorization": f"Bearer {self.api_key}"},
            timeout=60.0,
        )
        self._multimodal_url = (
            f"{self.api_base}/api/v1/services/embeddings/multimodal-embedding/multimodal-embedding"
        )

        # --- async clients (lazy, event-loop scoped) ---
        self._async_openai_client_cache = LoopScopedAsyncClientCache()
        self._async_httpx_client_cache = LoopScopedAsyncClientCache()

    # ------------------------------------------------------------------
    # Token telemetry
    # ------------------------------------------------------------------

    def _update_telemetry_token_usage(self, response) -> None:
        """Track token usage from either an OpenAI SDK response or an httpx response."""
        usage = None

        # OpenAI SDK response object
        if hasattr(response, "usage"):
            usage = response.usage
        # httpx response — parse JSON body
        elif hasattr(response, "json"):
            try:
                body: Dict[str, Any] = response.json() if callable(response.json) else response.json
                usage = body.get("usage")
            except Exception:
                return

        if not usage:
            return

        def _val(key: str, default: int = 0) -> int:
            if isinstance(usage, dict):
                return int(usage.get(key, default) or default)
            return int(getattr(usage, key, default) or default)

        prompt_tokens = _val("prompt_tokens") or _val("input_tokens", 0)
        total_tokens = _val("total_tokens", prompt_tokens)
        completion_tokens = max(total_tokens - prompt_tokens, 0)

        get_current_telemetry().add_token_usage_by_source(
            "embedding", prompt_tokens, completion_tokens
        )
        self.update_token_usage(
            model_name=self.model_name,
            provider="dashscope",
            prompt_tokens=prompt_tokens,
            completion_tokens=completion_tokens,
        )

    # ------------------------------------------------------------------
    # Multimodal helpers
    # ------------------------------------------------------------------

    @property
    def supports_multimodal(self) -> bool:
        return self._input_type == "multimodal" and self.model_name.startswith(
            ("qwen3-vl-embedding", "qwen2.5-vl-embedding")
        )

    @staticmethod
    def _to_dashscope_contents(content: List[Dict[str, Any]]) -> List[Dict[str, str]]:
        contents = []
        for part in content:
            if part.get("type") == "text":
                contents.append({"text": str(part.get("text", ""))})
            elif part.get("type") == "image_url":
                image_url = part.get("image_url")
                url = image_url.get("url") if isinstance(image_url, dict) else image_url
                if url:
                    contents.append({"image": str(url)})
        return contents

    def _standard_input_fusion(self, contents: List[Dict[str, str]]) -> Optional[bool]:
        if len(contents) <= 1:
            return None
        if self.model_name.startswith("qwen3-vl-embedding"):
            return True
        if self.model_name.startswith("qwen2.5-vl-embedding"):
            return None
        raise ValueError(f"{self.model_name} cannot fuse multiple multimodal input parts")

    def _multimodal_params(self) -> Dict[str, Any]:
        """Build parameters dict for multimodal requests, excluding None values."""
        return {
            k: v
            for k, v in {
                "dimension": self._dimension,
                "enable_fusion": self.enable_fusion,
                "res_level": self.res_level,
                "max_video_frames": self.max_video_frames,
            }.items()
            if v is not None
        }

    def _multimodal_body(self, text: str) -> Dict[str, Any]:
        body: Dict[str, Any] = {
            "model": self.model_name,
            "input": {"contents": [{"text": text}]},
        }
        params = self._multimodal_params()
        if params:
            body["parameters"] = params
        return body

    # ------------------------------------------------------------------
    # Lazy async clients
    # ------------------------------------------------------------------

    def _get_async_openai_client(self) -> openai.AsyncOpenAI:
        return self._async_openai_client_cache.get(
            lambda: openai.AsyncOpenAI(
                api_key=self.api_key,
                base_url=f"{self.api_base}/compatible-mode/v1",
                max_retries=0,
            )
        )

    def _get_async_httpx_client(self) -> httpx.AsyncClient:
        return self._async_httpx_client_cache.get(
            lambda: httpx.AsyncClient(
                headers={"Authorization": f"Bearer {self.api_key}"},
                timeout=60.0,
            )
        )

    # ------------------------------------------------------------------
    # embed
    # ------------------------------------------------------------------

    @model_error_boundary
    def embed(self, text: EmbeddingInput, is_query: bool = False) -> EmbedResult:
        if isinstance(text, list):
            contents = self._to_dashscope_contents(text)
            return self.embed_content(contents, enable_fusion=self._standard_input_fusion(contents))

        def _call() -> EmbedResult:
            if self._input_type == "text":
                response = self._openai_client.embeddings.create(
                    input=text, model=self.model_name, dimensions=self._dimension
                )
                self._update_telemetry_token_usage(response)
                vector = response.data[0].embedding
            else:
                resp = self._httpx_client.post(
                    self._multimodal_url, json=self._multimodal_body(text)
                )
                resp.raise_for_status()
                self._update_telemetry_token_usage(resp)
                vector = resp.json()["output"]["embeddings"][0]["embedding"]

            return EmbedResult(dense_vector=truncate_and_normalize(vector, self._dimension))

        try:
            return self._run_once(_call, logger=logger, operation_name="DashScope embedding")
        except Exception as e:
            raise RuntimeError(f"DashScope embedding failed: {e}") from e

    # ------------------------------------------------------------------
    # embed_async
    # ------------------------------------------------------------------

    @model_error_boundary
    async def embed_async(self, text: EmbeddingInput, is_query: bool = False) -> EmbedResult:
        if isinstance(text, list):
            contents = self._to_dashscope_contents(text)
            return await self.embed_content_async(
                contents,
                enable_fusion=self._standard_input_fusion(contents),
                is_query=is_query,
            )

        async def _call() -> EmbedResult:
            if self._input_type == "text":
                client = self._get_async_openai_client()
                response = await client.embeddings.create(
                    input=text, model=self.model_name, dimensions=self._dimension
                )
                self._update_telemetry_token_usage(response)
                vector = response.data[0].embedding
            else:
                client = self._get_async_httpx_client()
                resp = await client.post(self._multimodal_url, json=self._multimodal_body(text))
                resp.raise_for_status()
                self._update_telemetry_token_usage(resp)
                vector = resp.json()["output"]["embeddings"][0]["embedding"]

            return EmbedResult(dense_vector=truncate_and_normalize(vector, self._dimension))

        try:
            return await self._run_once_async(
                _call, is_query=is_query, logger=logger, operation_name="DashScope async embedding"
            )
        except Exception as e:
            raise RuntimeError(f"DashScope embedding failed: {e}") from e

    # ------------------------------------------------------------------
    # embed_content — multimodal content (text + image URLs)
    # ------------------------------------------------------------------

    @model_error_boundary
    def embed_content(
        self, contents: List[Dict[str, str]], *, enable_fusion: Optional[bool] = None
    ) -> EmbedResult:
        """Embed multimodal content (text + image URLs) via native DashScope API.

        Args:
            contents: List of content dicts, e.g. [{"text": "描述"}, {"image": "https://..."}]

        Returns:
            EmbedResult with dense_vector
        """

        def _call() -> EmbedResult:
            body: Dict[str, Any] = {
                "model": self.model_name,
                "input": {"contents": contents},
            }
            params = self._multimodal_params()
            if enable_fusion is not None:
                params["enable_fusion"] = enable_fusion
            if params:
                body["parameters"] = params
            resp = self._httpx_client.post(self._multimodal_url, json=body)
            resp.raise_for_status()
            self._update_telemetry_token_usage(resp)
            vector = resp.json()["output"]["embeddings"][0]["embedding"]
            return EmbedResult(dense_vector=truncate_and_normalize(vector, self._dimension))

        try:
            return self._run_once(
                _call, logger=logger, operation_name="DashScope content embedding"
            )
        except Exception as e:
            raise RuntimeError(f"DashScope content embedding failed: {e}") from e

    @model_error_boundary
    async def embed_content_async(
        self,
        contents: List[Dict[str, str]],
        *,
        enable_fusion: Optional[bool] = None,
        is_query: bool = False,
    ) -> EmbedResult:
        """Async version of embed_content."""

        client = self._get_async_httpx_client()

        async def _call() -> EmbedResult:
            body: Dict[str, Any] = {
                "model": self.model_name,
                "input": {"contents": contents},
            }
            params = self._multimodal_params()
            if enable_fusion is not None:
                params["enable_fusion"] = enable_fusion
            if params:
                body["parameters"] = params
            resp = await client.post(self._multimodal_url, json=body)
            resp.raise_for_status()
            self._update_telemetry_token_usage(resp)
            vector = resp.json()["output"]["embeddings"][0]["embedding"]
            return EmbedResult(dense_vector=truncate_and_normalize(vector, self._dimension))

        try:
            return await self._run_once_async(
                _call,
                is_query=is_query,
                logger=logger,
                operation_name="DashScope async content embedding",
            )
        except Exception as e:
            raise RuntimeError(f"DashScope content embedding failed: {e}") from e

    # ------------------------------------------------------------------
    # Lifecycle
    # ------------------------------------------------------------------

    def get_dimension(self) -> int:
        return self._dimension

    def close(self):
        # --- sync clients ---
        if self._openai_client is not None:
            self._openai_client.close()
        if self._httpx_client is not None:
            self._httpx_client.close()
        self._async_openai_client_cache.close_all_with_close()
        self._async_httpx_client_cache.close_all_with_aclose()