Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat: add Azure OpenAI support for embedding and VLM (#808)

zeng12183a1cc44b0cb
diff --git a/README_CN.md b/README_CN.md
index 43fb804d8..8814e0ef9 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -101,6 +101,7 @@ OpenViking 支持三种 VLM 提供商:
|----------|-------------|-------------|
| `volcengine` | 火山引擎豆包模型 | [Volcengine 控制台](https://console.volcengine.com/ark/region:ark+cn-beijing/overview?briefPage=0&briefType=introduce&type=new&utm_content=OpenViking&utm_medium=devrel&utm_source=OWO&utm_term=OpenViking) |
| `openai` | OpenAI 官方 API | [OpenAI 平台](https://platform.openai.com) |
+| `azure` | Azure OpenAI 服务 | [Azure OpenAI 服务](https://portal.azure.com) |
| `litellm` | 统一调用多种第三方模型 (Anthropic, DeepSeek, Gemini, vLLM, Ollama 等) | 参见 [LiteLLM 提供商](https://docs.litellm.ai/docs/providers) |
> 💡 **提示**:
@@ -171,6 +172,30 @@ Volcengine 支持模型名称和端点 ID。为简单起见,建议使用模型
</details>
+<details>
+<summary><b>Azure OpenAI</b></summary>
+
+使用 Azure OpenAI 服务。`model` 字段需要填写 Azure 上的**部署名称(deployment name)**,而非模型官方名字:
+
+```json
+{
+ "vlm": {
+ "provider": "azure",
+ "model": "your-deployment-name",
+ "api_key": "your-azure-api-key",
+ "api_base": "https://your-resource-name.openai.azure.com",
+ "api_version": "2025-01-01-preview"
+ }
+}
+```
+
+> 💡 **提示**:
+> - `api_base` 填写你的 Azure OpenAI 资源端点,支持 `*.openai.azure.com` 和 `*.cognitiveservices.azure.com` 两种格式
+> - `api_version` 可选,默认值为 `2025-01-01-preview`
+> - `model` 必须与 Azure Portal 中创建的部署名称一致
+
+</details>
+
<details>
<summary><b>LiteLLM (Anthropic, DeepSeek, Gemini, Qwen, vLLM, Ollama 等)</b></summary>
@@ -259,23 +284,25 @@ ollama serve
"dense": {
"api_base" : "<api-endpoint>", // API 端点地址
"api_key" : "<your-api-key>", // 模型服务 API Key
- "provider" : "<provider-type>", // 提供商类型:"volcengine" 或 "openai"(当前支持)
+ "provider" : "<provider-type>", // 提供商类型:"volcengine"、"openai"、"azure" 等
+ "api_version": "2025-01-01-preview", // (仅 azure)API 版本,可选,默认 "2025-01-01-preview"
"dimension": 1024, // 向量维度
- "model" : "<model-name>" // Embedding 模型名称(如 doubao-embedding-vision-250615 或 text-embedding-3-large)
+ "model" : "<model-name>" // Embedding 模型名称或 Azure 部署名(如 doubao-embedding-vision-250615 或 text-embedding-3-large)
},
"max_concurrent": 10 // 最大并发 embedding 请求(默认:10)
},
"vlm": {
"api_base" : "<api-endpoint>", // API 端点地址
"api_key" : "<your-api-key>", // 模型服务 API Key
- "provider" : "<provider-type>", // 提供商类型 (volcengine, openai, deepseek, anthropic 等)
- "model" : "<model-name>", // VLM 模型名称(如 doubao-seed-2-0-pro-260215 或 gpt-4-vision-preview)
+ "provider" : "<provider-type>", // 提供商类型 (volcengine, openai, azure, litellm 等)
+ "api_version": "2025-01-01-preview", // (仅 azure)API 版本,可选,默认 "2025-01-01-preview"
+ "model" : "<model-name>", // VLM 模型名称或 Azure 部署名(如 doubao-seed-2-0-pro-260215 或 gpt-4-vision-preview)
"max_concurrent": 100 // 语义处理的最大并发 LLM 调用(默认:100)
}
}
```
-> **注意**:对于 embedding 模型,目前支持 `volcengine`(豆包)、`openai` 和 `jina` 提供商。对于 VLM 模型,我们支持三种提供商:`volcengine`、`openai` 和 `litellm`。``litellm` 提供商支持各种模型,包括 Anthropic (Claude)、DeepSeek、Gemini、Moonshot、Zhipu、DashScope、MiniMax、vLLM、Ollama 等。
+> **注意**:对于 embedding 模型,目前支持 `volcengine`(豆包)、`openai`、`azure`、`jina` 等提供商。对于 VLM 模型,我们支持 `volcengine`、`openai`、`azure` 和 `litellm` 提供商。`litellm` 提供商支持各种模型,包括 Anthropic (Claude)、DeepSeek、Gemini、Moonshot、Zhipu、DashScope、MiniMax、vLLM、Ollama 等。
#### 服务器配置示例
@@ -349,6 +376,47 @@ ollama serve
</details>
+<details>
+<summary><b>示例 3:使用 Azure OpenAI 模型</b></summary>
+
+```json
+{
+ "storage": {
+ "workspace": "/home/your-name/openviking_workspace"
+ },
+ "log": {
+ "level": "INFO",
+ "output": "stdout"
+ },
+ "embedding": {
+ "dense": {
+ "api_base" : "https://your-resource-name.openai.azure.com",
+ "api_key" : "your-azure-api-key",
+ "provider" : "azure",
+ "api_version": "2025-01-01-preview",
+ "dimension": 1024,
+ "model" : "text-embedding-3-large"
+ },
+ "max_concurrent": 10
+ },
+ "vlm": {
+ "api_base" : "https://your-resource-name.openai.azure.com",
+ "api_key" : "your-azure-api-key",
+ "provider" : "azure",
+ "api_version": "2025-01-01-preview",
+ "model" : "gpt-4o",
+ "max_concurrent": 100
+ }
+}
+```
+
+> 💡 **提示**:
+> - `model` 必须填写 Azure Portal 中创建的**部署名称**,而非模型官方名字
+> - `api_base` 支持 `*.openai.azure.com` 和 `*.cognitiveservices.azure.com` 两种端点格式
+> - Embedding 和 VLM 可以使用不同的 Azure 资源和 API Key
+
+</details>
+
#### 设置服务器配置环境变量
创建配置文件后,设置环境变量指向它(Linux/macOS):
diff --git a/openviking/models/embedder/openai_embedders.py b/openviking/models/embedder/openai_embedders.py
index 709c14647..c57ec9ff3 100644
--- a/openviking/models/embedder/openai_embedders.py
+++ b/openviking/models/embedder/openai_embedders.py
@@ -6,6 +6,7 @@ from typing import Any, Dict, List, Optional
import openai
+from openviking.models.vlm.registry import DEFAULT_AZURE_API_VERSION
from openviking.models.embedder.base import (
DenseEmbedderBase,
EmbedResult,
@@ -63,12 +64,14 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
model_name: str = "text-embedding-3-small",
api_key: Optional[str] = None,
api_base: Optional[str] = None,
+ api_version: Optional[str] = None,
dimension: Optional[int] = None,
query_param: Optional[str] = None,
document_param: Optional[str] = None,
config: Optional[Dict[str, Any]] = None,
extra_headers: Optional[Dict[str, str]] = None,
input_type: Optional[str] = None,
+ provider: str = "openai",
):
"""Initialize OpenAI-Compatible Dense Embedder
@@ -105,23 +108,31 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):
self.api_key = api_key
self.api_base = api_base
+ self.api_version = api_version
self.dimension = dimension
self.query_param = query_param
self.document_param = document_param
+ self._provider = provider.lower()
# Allow missing api_key when api_base is set (e.g. local OpenAI-compatible servers)
if not self.api_key and not self.api_base:
raise ValueError("api_key is required")
- # Initialize OpenAI client
- # Use a placeholder api_key when not provided (for local OpenAI-compatible servers)
- client_kwargs = {"api_key": self.api_key or "no-key"}
- if self.api_base:
- client_kwargs["base_url"] = self.api_base
- # 透传自定义请求头(如 OpenRouter 要求的 HTTP-Referer / X-Title)
- if extra_headers:
- client_kwargs["default_headers"] = extra_headers
- self.client = openai.OpenAI(**client_kwargs)
+ client_kwargs: Dict[str, Any] = {"api_key": self.api_key or "no-key"}
+ if self._provider == "azure":
+ if not self.api_base:
+ raise ValueError("api_base (Azure endpoint) is required for Azure provider")
+ client_kwargs["azure_endpoint"] = self.api_base
+ client_kwargs["api_version"] = self.api_version or DEFAULT_AZURE_API_VERSION
+ if extra_headers:
+ client_kwargs["default_headers"] = extra_headers
+ self.client = openai.AzureOpenAI(**client_kwargs)
+ else:
+ if self.api_base:
+ client_kwargs["base_url"] = self.api_base
+ if extra_headers:
+ client_kwargs["default_headers"] = extra_headers
+ self.client = openai.OpenAI(**client_kwargs)
# Auto-detect dimension
self._dimension = dimension
diff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.py
index 633c9816d..7a45f727d 100644
--- a/openviking/models/vlm/backends/openai_vlm.py
+++ b/openviking/models/vlm/backends/openai_vlm.py
@@ -9,18 +9,42 @@ from pathlib import Path
from typing import Any, Dict, List, Union
from ..base import VLMBase
+from ..registry import DEFAULT_AZURE_API_VERSION
logger = logging.getLogger(__name__)
+def _build_openai_client_kwargs(
+ provider: str,
+ api_key: str,
+ api_base: str,
+ api_version: str | None,
+ extra_headers: Dict[str, str] | None,
+) -> Dict[str, Any]:
+ """Build kwargs dict shared by sync and async OpenAI/Azure client constructors."""
+ if provider == "azure":
+ if not api_base:
+ raise ValueError("api_base (Azure endpoint) is required for Azure provider")
+ kwargs: Dict[str, Any] = {
+ "api_key": api_key,
+ "azure_endpoint": api_base,
+ "api_version": api_version or DEFAULT_AZURE_API_VERSION,
+ }
+ else:
+ kwargs = {"api_key": api_key, "base_url": api_base}
+ if extra_headers:
+ kwargs["default_headers"] = extra_headers
+ return kwargs
+
+
class OpenAIVLM(VLMBase):
- """OpenAI VLM backend"""
+ """OpenAI / Azure OpenAI VLM backend"""
def __init__(self, config: Dict[str, Any]):
super().__init__(config)
self._sync_client = None
self._async_client = None
- self.provider = "openai"
+ self.api_version = config.get("api_version")
def get_client(self):
"""Get sync client"""
@@ -29,10 +53,14 @@ class OpenAIVLM(VLMBase):
import openai
except ImportError:
raise ImportError("Please install openai: pip install openai")
- client_kwargs = {"api_key": self.api_key, "base_url": self.api_base}
- if self.extra_headers:
- client_kwargs["default_headers"] = self.extra_headers
- self._sync_client = openai.OpenAI(**client_kwargs)
+ kwargs = _build_openai_client_kwargs(
+ self.provider, self.api_key, self.api_base,
+ self.api_version, self.extra_headers,
+ )
+ if self.provider == "azure":
+ self._sync_client = openai.AzureOpenAI(**kwargs)
+ else:
+ self._sync_client = openai.OpenAI(**kwargs)
return self._sync_client
def get_async_client(self):
@@ -42,10 +70,14 @@ class OpenAIVLM(VLMBase):
import openai
except ImportError:
raise ImportError("Please install openai: pip install openai")
- client_kwargs = {"api_key": self.api_key, "base_url": self.api_base}
- if self.extra_headers:
- client_kwargs["default_headers"] = self.extra_headers
- self._async_client = openai.AsyncOpenAI(**client_kwargs)
+ kwargs = _build_openai_client_kwargs(
+ self.provider, self.api_key, self.api_base,
+ self.api_version, self.extra_headers,
+ )
+ if self.provider == "azure":
+ self._async_client = openai.AsyncAzureOpenAI(**kwargs)
+ else:
+ self._async_client = openai.AsyncOpenAI(**kwargs)
return self._async_client
def _update_token_usage_from_response(self, response):
diff --git a/openviking/models/vlm/base.py b/openviking/models/vlm/base.py
index 079cd3a9f..76cb3095a 100644
--- a/openviking/models/vlm/base.py
+++ b/openviking/models/vlm/base.py
@@ -150,7 +150,7 @@ class VLMFactory:
return VolcEngineVLM(config)
- elif provider == "openai":
+ elif provider in ("openai", "azure"):
from .backends.openai_vlm import OpenAIVLM
return OpenAIVLM(config)
diff --git a/openviking/models/vlm/registry.py b/openviking/models/vlm/registry.py
index ff3ddb3bf..c650dfe58 100644
--- a/openviking/models/vlm/registry.py
+++ b/openviking/models/vlm/registry.py
@@ -8,7 +8,9 @@ Supported providers: volcengine, openai, litellm
from __future__ import annotations
-VALID_PROVIDERS: tuple[str, ...] = ("volcengine", "openai", "litellm")
+VALID_PROVIDERS: tuple[str, ...] = ("volcengine", "openai", "azure", "litellm")
+
+DEFAULT_AZURE_API_VERSION: str = "2025-01-01-preview"
def get_all_provider_names() -> list[str]:
diff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.py
index 20cb06c98..95c140f18 100644
--- a/openviking_cli/utils/config/embedding_config.py
+++ b/openviking_cli/utils/config/embedding_config.py
@@ -59,6 +59,10 @@ class EmbeddingModelConfig(BaseModel):
"or other OpenAI-compatible providers that require custom headers."
),
)
+ api_version: Optional[str] = Field(
+ default=None,
+ description="API version for Azure OpenAI (e.g., '2025-01-01-preview').",
+ )
model_config = {"extra": "forbid"}
@@ -91,6 +95,7 @@ class EmbeddingModelConfig(BaseModel):
if self.provider not in [
"openai",
+ "azure",
"volcengine",
"vikingdb",
"jina",
@@ -100,7 +105,7 @@ class EmbeddingModelConfig(BaseModel):
]:
raise ValueError(
f"Invalid embedding provider: '{self.provider}'. Must be one of: "
- "'openai', 'volcengine', 'vikingdb', 'jina', 'ollama', 'voyage', 'minimax'"
+ "'openai', 'azure', 'volcengine', 'vikingdb', 'jina', 'ollama', 'voyage', 'minimax'"
)
# Provider-specific validation
@@ -109,6 +114,12 @@ class EmbeddingModelConfig(BaseModel):
if not self.api_key and not self.api_base:
raise ValueError("OpenAI provider requires 'api_key' to be set")
+ elif self.provider == "azure":
+ if not self.api_key:
+ raise ValueError("Azure provider requires 'api_key' to be set")
+ if not self.api_base:
+ raise ValueError("Azure provider requires 'api_base' (Azure endpoint) to be set")
+
elif self.provider == "ollama":
# Ollama runs locally, no API key required
pass
@@ -233,7 +244,23 @@ class EmbeddingConfig(BaseModel):
"api_key": cfg.api_key
or "no-key", # Placeholder for local OpenAI-compatible servers
"api_base": cfg.api_base,
+ "api_version": cfg.api_version,
+ "dimension": cfg.dimension,
+ "provider": "openai",
+ **({"query_param": cfg.query_param} if cfg.query_param else {}),
+ **({"document_param": cfg.document_param} if cfg.document_param else {}),
+ **({"extra_headers": cfg.extra_headers} if cfg.extra_headers else {}),
+ },
+ ),
+ ("azure", "dense"): (
+ OpenAIDenseEmbedder,
+ lambda cfg: {
+ "model_name": cfg.model,
+ "api_key": cfg.api_key,
+ "api_base": cfg.api_base,
+ "api_version": cfg.api_version,
"dimension": cfg.dimension,
+ "provider": "azure",
**({"query_param": cfg.query_param} if cfg.query_param else {}),
**({"document_param": cfg.document_param} if cfg.document_param else {}),
**({"extra_headers": cfg.extra_headers} if cfg.extra_headers else {}),
diff --git a/openviking_cli/utils/config/vlm_config.py b/openviking_cli/utils/config/vlm_config.py
index 2a33711bd..a8c7da078 100644
--- a/openviking_cli/utils/config/vlm_config.py
+++ b/openviking_cli/utils/config/vlm_config.py
@@ -37,6 +37,11 @@ class VLMConfig(BaseModel):
default=100, description="Maximum number of concurrent LLM calls for semantic processing"
)
+ api_version: Optional[str] = Field(
+ default=None,
+ description="API version for Azure OpenAI (e.g., '2025-01-01-preview').",
+ )
+
extra_headers: Optional[Dict[str, str]] = Field(
default=None, description="Extra HTTP headers for OpenAI-compatible providers"
)
@@ -158,6 +163,7 @@ class VLMConfig(BaseModel):
"thinking": self.thinking,
"max_tokens": self.max_tokens,
"stream": stream,
+ "api_version": self.api_version,
}
if config: