Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(vlm): support extra request body passthrough (#1973)
Asish Kumar25e7615c516c
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.mdindex 2c6bd0028..48938e7f8 100644--- a/docs/en/guides/01-configuration.md+++ b/docs/en/guides/01-configuration.md@@ -542,6 +542,7 @@ Vision Language Model for semantic extraction (L0/L1 generation).| `max_retries` | int | Maximum retry attempts for transient VLM provider errors (default: `3`; `0` disables retry) || `timeout` | float | Per-request HTTP timeout in seconds passed to the underlying OpenAI/LiteLLM client. Increase for slow endpoints (e.g., DashScope, local inference). Must be `> 0` (default: `60.0`) || `extra_headers` | object | Custom HTTP headers for compatible HTTP providers. `kimi` also accepts header overrides, but already injects the required subscription headers by default |+| `extra_request_body` | object | Extra JSON body fields for OpenAI-compatible completion requests, useful for provider-specific options such as Ollama `{"think": false}` || `stream` | bool | Enable streaming mode (for OpenAI-compatible providers, default: `false`) |`vlm.max_retries` only applies to transient errors such as `429`, `5xx`, timeouts, and connection failures. Permanent authentication, authorization, and billing errors are not retried automatically. The backoff strategy is exponential backoff with jitter, starting at `0.5s` and capped at `8s`.@@ -594,6 +595,24 @@ Common use cases:- **Custom proxies**: Add authentication or tracing headers- **API gateways**: Add version or routing identifiers+**Custom Request Body**++For OpenAI-compatible providers that accept provider-specific JSON body fields, add them via `extra_request_body`. OpenViking merges these fields into the `extra_body` sent by the OpenAI SDK or LiteLLM:++```json+{+ "vlm": {+ "provider": "litellm",+ "api_key": "ollama",+ "model": "ollama/llama3.1",+ "api_base": "http://127.0.0.1:11434",+ "extra_request_body": {+ "think": false+ }+ }+}+```+**Streaming Mode**For OpenAI-compatible providers that return SSE (Server-Sent Events) format responses, enable `stream` mode:@@ -1215,6 +1234,7 @@ For detailed encryption explanations, see [Data Encryption](../concepts/10-encry"max_concurrent": 100,"max_retries": 3,"extra_headers": {},+ "extra_request_body": {},"stream": false},"rerank": {diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.mdindex 1a211b464..b77202ae8 100644--- a/docs/zh/guides/01-configuration.md+++ b/docs/zh/guides/01-configuration.md@@ -514,6 +514,7 @@ openviking-server doctor| `max_retries` | int | VLM provider 瞬时错误的最大重试次数(默认:`3`;`0` 表示禁用重试) || `timeout` | float | 单次 VLM API 请求的 HTTP 超时时间(秒),传递给底层 OpenAI/LiteLLM 客户端。慢端点(如 DashScope、本地推理)可调大。必须 `> 0`(默认:`60.0`) || `extra_headers` | object | 兼容 HTTP provider 的自定义请求头。`kimi` 默认已注入所需订阅请求头,也支持在这里覆盖或扩展 |+| `extra_request_body` | object | 传给 OpenAI 兼容 completion 请求的额外 JSON body 字段,可用于 Ollama `{"think": false}` 等 provider 专有参数 || `stream` | bool | 启用流式模式(OpenAI 兼容 provider 可用,默认:`false`) |`vlm.max_retries` 仅对瞬时错误生效,例如 `429`、`5xx`、超时和连接错误;认证、鉴权、欠费等永久错误不会自动重试。退避策略为指数退避,初始延迟 `0.5s`,上限 `8s`,并带随机抖动。@@ -566,6 +567,24 @@ openviking-server doctor- **自定义代理**: 添加认证头或追踪头- **API 网关**: 添加版本或路由标识+**自定义请求 Body**++对于接受 provider 专有 JSON body 字段的 OpenAI 兼容 provider,可以通过 `extra_request_body` 配置。OpenViking 会把这些字段合并到 OpenAI SDK 或 LiteLLM 发送的 `extra_body` 中:++```json+{+ "vlm": {+ "provider": "litellm",+ "api_key": "ollama",+ "model": "ollama/llama3.1",+ "api_base": "http://127.0.0.1:11434",+ "extra_request_body": {+ "think": false+ }+ }+}+```+**流式模式**对于返回 SSE(Server-Sent Events)格式响应的 OpenAI 兼容 provider,启用 `stream` 模式:@@ -1189,6 +1208,7 @@ openviking add-resource ./docs --exclude "*.tmp""max_concurrent": 100,"max_retries": 3,"extra_headers": {},+ "extra_request_body": {},"stream": false},"rerank": {diff --git a/openviking/models/vlm/backends/litellm_vlm.py b/openviking/models/vlm/backends/litellm_vlm.pyindex 04d019558..d328274a6 100644--- a/openviking/models/vlm/backends/litellm_vlm.py+++ b/openviking/models/vlm/backends/litellm_vlm.py@@ -234,6 +234,8 @@ class LiteLLMVLMProvider(VLMBase):if tools:kwargs["tools"] = toolskwargs["tool_choice"] = tool_choice or "auto"+ if self.extra_request_body:+ kwargs["extra_body"] = dict(self.extra_request_body)# Only send enable_thinking to DashScope-compatible providersprovider = self._detected_provider or detect_provider_by_model(model)diff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.pyindex 25547d6a7..47fe8f58e 100644--- a/openviking/models/vlm/backends/openai_vlm.py+++ b/openviking/models/vlm/backends/openai_vlm.py@@ -141,8 +141,11 @@ class OpenAIVLM(VLMBase):def _apply_provider_specific_extra_body(self, kwargs: Dict[str, Any], thinking: bool) -> None:"""Attach provider-specific raw body parameters understood by compatible APIs."""+ extra_body = dict(self.extra_request_body)if self._supports_enable_thinking():- kwargs["extra_body"] = {"enable_thinking": bool(thinking)}+ extra_body["enable_thinking"] = bool(thinking)+ if extra_body:+ kwargs["extra_body"] = extra_bodydef _update_token_usage_from_response(self,diff --git a/openviking/models/vlm/base.py b/openviking/models/vlm/base.pyindex 0ac0b1b30..c77318dc4 100644--- a/openviking/models/vlm/base.py+++ b/openviking/models/vlm/base.py@@ -65,6 +65,7 @@ class VLMBase(ABC):self.timeout = config.get("timeout", 60.0)self.max_tokens = config.get("max_tokens")self.extra_headers = config.get("extra_headers")+ self.extra_request_body = dict(config.get("extra_request_body") or {})self.stream = config.get("stream", False)# Token usage trackingdiff --git a/openviking_cli/utils/config/vlm_config.py b/openviking_cli/utils/config/vlm_config.pyindex f941809da..3d154c7f7 100644--- a/openviking_cli/utils/config/vlm_config.py+++ b/openviking_cli/utils/config/vlm_config.py@@ -68,6 +68,14 @@ class VLMConfig(BaseModel):default=None, description="Extra HTTP headers for OpenAI-compatible providers")+ extra_request_body: Optional[Dict[str, Any]] = Field(+ default=None,+ description=(+ "Extra JSON body fields passed to OpenAI-compatible VLM completion requests. "+ "Useful for provider-specific options such as Ollama's {'think': false}."+ ),+ )+stream: bool = Field(default=False, description="Enable streaming mode for OpenAI-compatible providers")@@ -136,6 +144,11 @@ class VLMConfig(BaseModel):self.providers[self.provider]["api_base"] = self.api_baseif self.extra_headers and "extra_headers" not in self.providers[self.provider]:self.providers[self.provider]["extra_headers"] = self.extra_headers+ if (+ self.extra_request_body+ and "extra_request_body" not in self.providers[self.provider]+ ):+ self.providers[self.provider]["extra_request_body"] = self.extra_request_bodyif self.stream and "stream" not in self.providers[self.provider]:self.providers[self.provider]["stream"] = self.stream@@ -164,6 +177,8 @@ class VLMConfig(BaseModel):config["api_base"] = self.api_baseif self.extra_headers and "extra_headers" not in config:config["extra_headers"] = self.extra_headers+ if self.extra_request_body and "extra_request_body" not in config:+ config["extra_request_body"] = self.extra_request_bodyif self.stream and "stream" not in config:config["stream"] = self.streamreturn config@@ -255,6 +270,8 @@ class VLMConfig(BaseModel):result["api_base"] = config.get("api_base")if config.get("extra_headers"):result["extra_headers"] = config.get("extra_headers")+ if config.get("extra_request_body"):+ result["extra_request_body"] = config.get("extra_request_body")return resultdiff --git a/tests/unit/test_extra_headers_vlm.py b/tests/unit/test_extra_headers_vlm.pyindex fc1f51e27..db676bb4e 100644--- a/tests/unit/test_extra_headers_vlm.py+++ b/tests/unit/test_extra_headers_vlm.py@@ -330,6 +330,83 @@ class TestVLMBaseExtraHeaders:assert vlm.extra_headers is None+class TestVLMExtraRequestBody:+ """Test provider-specific VLM request body passthrough."""++ @patch("openviking.models.vlm.backends.openai_vlm.openai.OpenAI")+ def test_openai_text_completion_passes_extra_request_body(self, mock_openai_class):+ mock_client = MagicMock()+ mock_openai_class.return_value = mock_client+ mock_response = MagicMock()+ mock_response.choices = [MagicMock(message=MagicMock(content="ok"), finish_reason="stop")]+ mock_response.usage = None+ mock_client.chat.completions.create.return_value = mock_response++ vlm = OpenAIVLM(+ {+ "api_key": "sk-test",+ "api_base": "https://api.openai.com/v1",+ "model": "gpt-4o-mini",+ "extra_request_body": {"think": False, "keep_alive": "5m"},+ }+ )++ vlm.get_completion("hello")++ call_kwargs = mock_client.chat.completions.create.call_args.kwargs+ assert call_kwargs["extra_body"] == {"think": False, "keep_alive": "5m"}++ @patch("openviking.models.vlm.backends.openai_vlm.openai.OpenAI")+ def test_dashscope_thinking_merges_with_extra_request_body(self, mock_openai_class):+ mock_client = MagicMock()+ mock_openai_class.return_value = mock_client+ mock_response = MagicMock()+ mock_response.choices = [MagicMock(message=MagicMock(content="ok"), finish_reason="stop")]+ mock_response.usage = None+ mock_client.chat.completions.create.return_value = mock_response++ vlm = OpenAIVLM(+ {+ "api_key": "sk-test",+ "api_base": "https://dashscope.aliyuncs.com/compatible-mode/v1",+ "model": "qwen3.5-plus",+ "extra_request_body": {"seed": 7},+ }+ )++ vlm.get_completion("hello", thinking=True)++ call_kwargs = mock_client.chat.completions.create.call_args.kwargs+ assert call_kwargs["extra_body"] == {"seed": 7, "enable_thinking": True}++ def test_litellm_build_kwargs_passes_extra_request_body(self):+ vlm = LiteLLMVLMProvider(+ {+ "model": "ollama/llama3",+ "provider": "litellm",+ "api_base": "http://127.0.0.1:11434",+ "extra_request_body": {"think": False},+ }+ )++ kwargs = vlm._build_text_kwargs(prompt="hello")++ assert kwargs["extra_body"] == {"think": False}++ def test_litellm_dashscope_merges_thinking_with_extra_request_body(self):+ vlm = LiteLLMVLMProvider(+ {+ "model": "qwen-plus",+ "provider": "litellm",+ "extra_request_body": {"seed": 7},+ }+ )++ kwargs = vlm._build_text_kwargs(prompt="hello", thinking=False)++ assert kwargs["extra_body"] == {"seed": 7, "enable_thinking": False}++class TestVLMConfigExtraHeaders:"""Test VLMConfig passes extra_headers to VLM instance."""@@ -400,6 +477,45 @@ class TestVLMConfigExtraHeaders:}+class TestVLMConfigExtraRequestBody:+ """Test VLMConfig passes extra_request_body to VLM instance config."""++ def test_vlm_config_accepts_extra_request_body_in_providers(self):+ from openviking_cli.utils.config.vlm_config import VLMConfig++ config = VLMConfig(+ model="llama3",+ provider="litellm",+ providers={+ "litellm": {+ "api_key": "sk-test",+ "api_base": "http://127.0.0.1:11434",+ "extra_request_body": {"think": False},+ }+ },+ )++ result = config._build_vlm_config_dict()+ assert result["extra_request_body"] == {"think": False}++ def test_vlm_config_accepts_flat_extra_request_body(self):+ from openviking_cli.utils.config.vlm_config import VLMConfig++ config = VLMConfig(+ model="llama3",+ provider="litellm",+ api_key="sk-test",+ api_base="http://127.0.0.1:11434",+ extra_request_body={"think": False},+ )++ config._migrate_legacy_config()+ assert config.providers["litellm"]["extra_request_body"] == {"think": False}++ result = config._build_vlm_config_dict()+ assert result["extra_request_body"] == {"think": False}++class TestLiteLLMVLMModelResolution:"""Regression tests for LiteLLM model prefix resolution."""